爬取豆瓣电影 top 前100
# ============================================================================= # 采集猫眼看电影中top前100的电影名称 # # # 1. 电影名称 # 2. 电影主演 # 3. 上映时间 # 4. 电影分数 # 5. 电影封面图片 # 6. 电影主页网址 # ============================================================================= import requests import re import os from bs4 import BeautifulSoup import pandas as pd url = 'https://maoyan.com/board/4' headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64)\ AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 \ Safari/537.36'} r = requests.get(url,headers = headers) r.encoding = 'utf-8' r.text bs = BeautifulSoup(r.text,'lxml') bs # 将得到的数据类型转化为需要的格式 bs.find_all('p','name')[0].text # 首先提取出电影的名称 # ============================================================================= # name = [] # for i in bs.find_all('p','name'): # name.append(i.text) # name # ============================================================================= # 利用列表生成式 [i.text for i in bs.find_all('p','name')] # 提取出电影的主演 # ============================================================================= # bs.find_all('p','star')[0].text.strip() # 其中的strip是去除得到数据的前后空格 # # user = [] # for i in bs.find_all('p','star'): # user.append(i.text.strip()) # user # ============================================================================= # 列表生成式 [i.text.strip() for i in bs.find_all('p','star')] # 提取出上映时间 a = bs.find_all('p','releasetime')[0].text # ============================================================================= # time = [] # for i in bs.find_all('p','releasetime'): # time.append(i.text.replace('上映时间:','')) # time # ============================================================================= [i.text.replace('上映时间:','') for i in bs.find_all('p','releasetime')] # 电影的评分 bs.find_all('p','score')[0].text [i.text for i in bs.find_all('p','score')] # 电影封面的网址 bs.find_all('img','board-img')[0]['data-src'] [i['data-src'] for i in bs.find_all('img','board-img')] # 电影主页的网址 bs.find_all('a','image-link')[0]['href'] [i['href'] for i in bs.find_all('a','image-link')] # In[] =======================================================================================
# 进行整合 一次爬取电影一页的信息 def fun1(url): name,score,star,time,img,href = [],[],[],[],[],[] url = 'https://maoyan.com/board/4' headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64)\ AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 \ Safari/537.36'} r = requests.get(url,headers = headers) r.encoding = 'utf-8' bs = BeautifulSoup(r.text,'lxml') # 将得到的数据类型转化为需要的格式 bs.find_all('p','name')[0].text name = [i.text for i in bs.find_all('p','name')] score = [i.text for i in bs.find_all('p','score')] star = [i.text.strip() for i in bs.find_all('p','star')] time = [i.text.replace('上映时间:','') for i in bs.find_all('p','releasetime')] img = [i['data-src'] for i in bs.find_all('img','board-img')] href = [i['href'] for i in bs.find_all('a','image-link')] df = pd.DataFrame() df['电影名'] = name df['评分'] = score df['主演'] = star df['上映时间'] = time df['电影画面网址'] = img df['电影主页网址'] = href return df # =================================================== 调用 url = 'https://maoyan.com/board/4' data = fun1(url) data['完整主页网址'] = 'https://maoyan.com'+ data.电影主页网址 data.完整主页网址 # In[]========================================================================== # 将上面的函数用到 想提取几页就提取几页 想提取那个网址 直接设置地址就好 # 首先提取网页源代码 url = 'https://maoyan.com/board/4?offset=10' import time empty_df = pd.DataFrame() # 为了节省时间 此处只截取前5页 如果想多采集更该循环次数即可 for i in range(0,5): url = 'https://maoyan.com/board/4?offset={}'.format(i*10) print(url) # 每采集完一夜的信息打印一次网址 df = fun1(url) empty_df = empty_df.append(df,ignore_index= True ) # 将得到的每一页的信息拼接到这张大表中 ignore_index = True 表示对得到的列表序列重新排序 time.sleep(3) print(i+1) # ============================================================================= # # 在爬取信息时 为了防止封ip 设置休眠 隔几秒采取一次 # import time # time.sleep(3) # =============================================================================