Day 03
今天主要学习了爬虫的应用。
今日学习内容总结
''''''
'''
视频选项:
1.梨视频
'''
# import requests
#
# # 往视频源地址发送请求
# response = requests.get(
# 'https://video.pearvideo.com/mp4/adshort/20190625/cont-1570302-14057031_adpkg-ad_hd.mp4')
#
# # 打印二进制流，比如图片、视频等数据
# print(response.content)
#
# # 保存视频到本地
# with open('视频.mp4', 'wb') as f:
# f.write(response.content)
'''
1、先往梨视频主页发送请求
https://www.pearvideo.com/
解析获取所有视频的id:
video_1570302
re.findall()
2、获取视频详情页url:
惊险!男子抢上地铁滑倒,就脚进去了
https://www.pearvideo.com/video_1570302
揭秘坎儿井
https://www.pearvideo.com/video_1570107
'''
import requests
import re # 正则，用于解析文本数据
# 1、先往梨视频主页发送请求
response = requests.get('https://www.pearvideo.com/')
# print(response.text)
# re正则匹配获取所有视频id
# 参数1: 正则匹配规则
# 参数2: 解析文本
# 参数3: 匹配模式
res_list = re.findall('<a href="video_(.*?)"', response.text, re.S)
# print(res_list)
# 拼接每一个视频详情页url
for v_id in res_list:
detail_url = 'https://www.pearvideo.com/video_' + v_id
# print(detail_url)
# 对每一个视频详情页发送请求获取视 频源url
response = requests.get(url=detail_url)
# print(response.text)
# 解析并提取详情页视频url
# 视频url
video_url = re.findall('srcUrl="(.*?)"', response.text, re.S)[0]
print(video_url)
# 视频名称
video_name = re.findall(
'<h1 class="video-tt">(.*?)</h1>', response.text, re.S)[0]
print(video_name)
# 往视频url发送请求获取视频二进制流
v_response = requests.get(video_url)
with open('%s.mp4' % video_name, 'wb') as f:
f.write(v_response.content)
print(video_name, '视频爬取完成')
import requests # 导入requests请求库
# 往百度主页发送请求，获取响应对象
response = requests.get(url='https://www.baidu.com/')
# 设置字符编码为 utf-8
response.encoding = 'utf-8'
# 打印响应文本
print(response.text)
# 把响应文本写入本地
with open('baidu.html', 'w', encoding='utf-8') as f:
f.write(response.text)
'''''' ''' https://movie.douban.com/top250?start=0&filter= https://movie.douban.com/top250?start=25&filter= https://movie.douban.com/top250?start=50&filter= 1.发送请求 2.解析数据 3.保存数据 ''' import requests import re # 爬虫三部曲 # 1.发送请求 def get_page(base_url): response = requests.get(base_url) return response # 2.解析文本 def parse_index(text): res = re.findall('
.*?(.*?).*?.*?(.*?).*?导演:(.*?)
.*?(.*?).*?(.*?)人评价.*?(.*?)', text, re.S) # print(res) return res # 3.保存数据 def save_data(data): with open('douban.txt', 'a', encoding='utf-8') as f: f.write(data) # main + 回车键 if __name__ == '__main__': # num = 10 # base_url = 'https://movie.douban.com/top250?start={}&filter='.format(num) num = 0 for line in range(10): base_url = f'https://movie.douban.com/top250?start={num}&filter=' num += 25 print(base_url) # 1.发送请求，调用函数 response = get_page(base_url) # 2.解析文本 movie_list = parse_index(response.text) # 3.保存数据 # 数据的格式化 for movie in movie_list: # print(movie) # 解压赋值 # 电影排名、电影url、电影名称、导演 - 主演 - 类型、电影评分、评价人数、电影简介 v_top, v_url, v_name, v_daoyan, v_point, v_num, v_desc = movie # v_top = movie[0] # v_url = movie[1] moive_content = f''' 电影排名: {v_top} 电影url: {v_url} 电影名称: {v_name} 导演主演: {v_daoyan} 电影评分: {v_point} 评价人数: {v_num} 电影简介: {v_desc} \n ''' print(moive_content) # 保存数据 save_data(moive_content)
'''''' ''' 视频选项: 1.梨视频 ''' # import requests # # # 往视频源地址发送请求 # response = requests.get( # 'https://video.pearvideo.com/mp4/adshort/20190625/cont-1570302-14057031_adpkg-ad_hd.mp4') # # # 打印二进制流，比如图片、视频等数据 # print(response.content) # # # 保存视频到本地 # with open('视频.mp4', 'wb') as f: # f.write(response.content) ''' 1、先往梨视频主页发送请求 https://www.pearvideo.com/ 解析获取所有视频的id: video_1570302 re.findall() 2、获取视频详情页url: 惊险!男子抢上地铁滑倒,就脚进去了 https://www.pearvideo.com/video_1570302 揭秘坎儿井 https://www.pearvideo.com/video_1570107 ''' import requests import re # 正则，用于解析文本数据 # 1、先往梨视频主页发送请求 response = requests.get('https://www.pearvideo.com/') # print(response.text) # re正则匹配获取所有视频id # 参数1: 正则匹配规则 # 参数2: 解析文本 # 参数3: 匹配模式 res_list = re.findall('<a href="video_(.*?)"', response.text, re.S) # print(res_list) # 拼接每一个视频详情页url for v_id in res_list: detail_url = 'https://www.pearvideo.com/video_' + v_id # print(detail_url) # 对每一个视频详情页发送请求获取视 频源url response = requests.get(url=detail_url) # print(response.text) # 解析并提取详情页视频url # 视频url video_url = re.findall('srcUrl="(.*?)"', response.text, re.S)[0] print(video_url) # 视频名称 video_name = re.findall( '<h1 class="video-tt">(.*?)</h1>', response.text, re.S)[0] print(video_name) # 往视频url发送请求获取视频二进制流 v_response = requests.get(video_url) with open('%s.mp4' % video_name, 'wb') as f: f.write(v_response.content) print(video_name, '视频爬取完成') import requests # 导入requests请求库 # 往百度主页发送请求，获取响应对象 response = requests.get(url='https://www.baidu.com/') # 设置字符编码为 utf-8 response.encoding = 'utf-8' # 打印响应文本 print(response.text) # 把响应文本写入本地 with open('baidu.html', 'w', encoding='utf-8') as f: f.write(response.text) '''''' ''' https://movie.douban.com/top250?start=0&filter= https://movie.douban.com/top250?start=25&filter= https://movie.douban.com/top250?start=50&filter= 1.发送请求 2.解析数据 3.保存数据 ''' import requests import re # 爬虫三部曲 # 1.发送请求 def get_page(base_url): response = requests.get(base_url) return response # 2.解析文本 def parse_index(text): res = re.findall(' .*?(.*?).*?.*?(.*?).*?导演:(.*?) .*?(.*?).*?(.*?)人评价.*?(.*?)', text, re.S) # print(res) return res # 3.保存数据 def save_data(data): with open('douban.txt', 'a', encoding='utf-8') as f: f.write(data) # main + 回车键 if __name__ == '__main__': # num = 10 # base_url = 'https://movie.douban.com/top250?start={}&filter='.format(num) num = 0 for line in range(10): base_url = f'https://movie.douban.com/top250?start={num}&filter=' num += 25 print(base_url) # 1.发送请求，调用函数 response = get_page(base_url) # 2.解析文本 movie_list = parse_index(response.text) # 3.保存数据 # 数据的格式化 for movie in movie_list: # print(movie) # 解压赋值 # 电影排名、电影url、电影名称、导演 - 主演 - 类型、电影评分、评价人数、电影简介 v_top, v_url, v_name, v_daoyan, v_point, v_num, v_desc = movie # v_top = movie[0] # v_url = movie[1] moive_content = f''' 电影排名: {v_top} 电影url: {v_url} 电影名称: {v_name} 导演主演: {v_daoyan} 电影评分: {v_point} 评价人数: {v_num} 电影简介: {v_desc} \n ''' print(moive_content) # 保存数据 save_data(moive_content)
