Python 爬虫 (五)
# 头条街拍图片爬取
1 import re 2 import requests 3 from urllib import request 4 import json 5 import os 6 i = 0 7 headers = { 8 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36' 9 } 10 while True: 11 pag_all_url = 'https://www.toutiao.com/search_content/?offset={}&format=json&keyword=%E8%A1%97%E6%8B%8D&autoload=true&count=20&cur_tab=1&from=search_tab' 12 i += 20 13 full_pag_url = pag_all_url.format(i) 14 # print(full_pag_url) # 街拍的首页 根据不同的i来请求ajax,从而获得所有的街拍网址 像这样https://www.toutiao.com/search/?keyword=%E8%A1%97%E6%8B%8D 15 pag_html = requests.get(full_pag_url,headers = headers).text 16 pag_html_str = str(json.loads(pag_html)) 17 # print(pag_html_str) #把网页转化为字符串 进行正则匹配 18 img_pag_id = re.findall(r'\'item_source_url\': \'\/group\/(\d*)\/\',',pag_html_str) 19 # print(img_pag_id) #获得每个街拍的url like this--->https://www.toutiao.com/a6590127156037157379/ 20 for l in img_pag_id: #图片下载 21 img_all_url = 'https://www.toutiao.com/a{}' 22 full_url = img_all_url.format(l) 23 # print(full_url)#图片的url print(full_pag_url)#图片所在的url 24 html = requests.get(full_url,headers=headers).text 25 pattern = r'gallery: JSON\.parse\((.*)\),' 26 ans1 = re.search(pattern,html) 27 try: 28 ans1_str = json.loads(ans1[1]) 29 ans1_dic = json.loads(ans1_str) 30 # print(ans1_dic) 31 # if not os.path.exists('1'): 32 # os.mkdir('1') 33 for q in ans1_dic['sub_images']: 34 img_url = q['url'] 35 print(img_url) 36 filename = '1/' + img_url.split('/')[-1] + '.jpg' 37 request.urlretrieve(img_url, filename) 38 except:continue
- -疏影横斜水清浅,暗香浮动月黄昏。- -