Day04
课堂笔记
爬取梨视频：
''' 爬取梨视频 请求url： https://www.pearvideo.com/ 请求方式： User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36 ''' import requests import re#正则模块 #uuid.uuid4() 可以根据实际戳生成一段世界上唯一的随机字符串 import uuid #爬虫三部曲 #1.发送请求 def get_page(url): response=requests.get(url) return response #2.解析数据 #解析主页获取视频详情页ID def parse_index(text): res = re.findall('<a href="video_(.*?)"',text, re.S) # print(res) detail_url_list = [] for m_id in res: detail_url = 'https://www.pearvideo.com/video_' + m_id # print(detail_url) detail_url_list.append(detail_url) # print(detail_url_list) return detail_url_list #解析详情页获取视频URl def parse_detail(text): # re.findall（'正则匹配规则'，'解析文本'，'正则模式） # re.S:全局模块（对整个文本进行匹配) # .指的是当前位置 # *指的是查找所有 movie_url=re.findall('srcUrl="(.*?)"',text,re.S)[0] return movie_url #3.保存数据 def save_movie(movie_url): response=requests.get(movie_url) #把视频写到本地 with open(f'{uuid.uuid4()}.mp4','wb')as f: f.write(response.content) f.flush() if __name__=='__main__' : #1.对主页发送请求 index_res=get_page(url='https://www.pearvideo.com/') #2.对主页进行解析，获取详情页id detail_url_list=parse_index(index_res.text) # print(detail_url_list) #3.对每个详情页url发送请求 for detail_url in detail_url_list: detail_res=get_page(url=detail_url) #print(detail_res.text) #4.解析详情页获取视频url movie_url=parse_detail(detail_res.text) print(movie_url) #5.保存视频 # save_movie(movie_url)
requests详细使用：
import requests headers={ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36' } # #在get请求内添加user-agent # response=requests.get(url='https://www.zhihu.com/explore',headers=headers) # # print(response.status_code) # # with open('zhihu.html','w',encoding='utf-8')as f: # f.write(response.text) # ''' params参数 访问百度搜查王嘉尔url ''' # # from urllib.parse import urlencode # # url='https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&rsv_idx=1&tn=baidu&wd=%E7%8E%8B%E5%98%89%E5%B0%94&oq=%25E7%259F%25A5%25E4%25B9%258E&rsv_pq=847b40850001732b&rsv_t=4a75X49edBkcU8OcSUPfEvxeRVv0RD12dGNsJ7AmeAQVqP6ZFqOzLS1gjXs&rqlang=cn&rsv_enter=1&rsv_sug3=9&rsv_sug1=7&rsv_sug7=101&bs=%E7%9F%A5%E4%B9%8E/' # url='https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&rsv_idx=1&tn=baidu&sv_sug1=7&'+urlencode({"wd":"王嘉尔"}) # print(url) # response=requests.get(url) # # with open('z','w',encoding='utf-8')as f: # f.write(response.text) response=requests.get(url,headers=headers,params={'百度':'安徽工程大学'}) with open('gonngcheng.html','w',encoding='utf-8')as f: f.write(response.text)
爬取豆瓣top250电影信息：
'''''' ''' 主页: https://movie.douban.com/top250 GET User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36 re正则: # 电影详情页url、图片链接、电影名称、电影评分、评价人数 <div class="item">.*?href="(.*?)">.*?src="(.*?)".*?<span class="title">(.*?)</span>.*?<span class="rating_num".*?>(.*?)</span>.*?<span>(.*?)人评价 ''' import requests import re url = 'https://movie.douban.com/top250' headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36' } # 1、往豆瓣TOP250发送请求获取响应数据 response = requests.get(url, headers=headers) # print(response.text) # 2、通过正则解析提取数据 # 电影详情页url、图片链接、电影名称、电影评分、评价人数 movie_content_list = re.findall( # 正则规则 '<div class="item">.*?href="(.*?)">.*?src="(.*?)".*?<span class="title">(.*?)</span>.*?<span class="rating_num".*?>(.*?)</span>.*?<span>(.*?)人评价', # 解析文本 response.text, # 匹配模式 re.S) for movie_content in movie_content_list: # 解压赋值每一部电影 detail_url, movie_jpg, name, point, num = movie_content data = f'电影名称:{name}, 详情页url:{detail_url}, 图片url:{movie_jpg}, 评分: {point}, 评价人数: {num} \n' print(data) # 3、保存数据，把电影信息写入文件中 with open('douban.txt', 'a', encoding='utf-8') as f: f.write(data)
高性能爬虫：
import requests import re # 正则模块 # uuid.uuid4() 可以根据时间戳生成一段世界上唯一的随机字符串 import uuid # 导入线程池模块 from concurrent.futures import ThreadPoolExecutor # 线程池限制50个线程 pool = ThreadPoolExecutor(50) # 爬虫三部曲 # 1、发送请求 def get_page(url): print(f'开始异步任务: {url}') response = requests.get(url) return response # 2、解析数据 # 解析主页获取视频详情页ID def parse_index(res): response = res.result() # 提取出主页所有ID id_list = re.findall('<a href="video_(.*?)"', response.text, re.S) # print(res) # 循环id列表 for m_id in id_list: # 拼接详情页url detail_url = 'https://www.pearvideo.com/video_' + m_id # print(detail_url) # 把详情页url提交给get_page函数 pool.submit(get_page, detail_url).add_done_callback(parse_detail) # 解析详情页获取视频url def parse_detail(res): response = res.result() movie_url = re.findall('srcUrl="(.*?)"', response.text, re.S)[0] # 异步提交把视频url传给get_page函数,把返回的结果传给save_movie pool.submit(get_page, movie_url).add_done_callback(save_movie) # 3、保存数据 def save_movie(res): movie_res = res.result() # 把视频写到本地 with open(f'{uuid.uuid4()}.mp4', 'wb') as f: f.write(movie_res.content) print(f'视频下载结束: {movie_res.url}') f.flush() if __name__ == '__main__': # main + 回车键 # 一 往get_page发送异步请求，把结果交给parse_index函数 url = 'https://www.pearvideo.com/' pool.submit(get_page, url).add_done_callback(parse_index)
