1 import requests
2 import json
3 from requests.exceptions import ConnectionError
4 from json.decoder import JSONDecodeError
5 from lxml import etree
6 import re
7 #from config import *
8 import pymongo
9 import os
10 from hashlib import md5
11 from multiprocessing import Pool
12
13 client=pymongo.MongoClient('lacalhost')
14 db=client['toutiao1']
15 def get_page_parse(offset,keyword):
16 data={
17 'offset': offset,
18 'format': 'json',
19 'keyword':keyword,
20 'autoload':'true',
21 'count': 20,
22 'cur_tab': 3
23 }
24 url="https://www.toutiao.com/search_content/"
25 try:
26 response = requests.get(url,params=data)
27 if response.status_code == 200:
28 #json_data=response.json()
29 json_data=response.text
30 return json_data
31 return None
32 except ConnectionError:
33 print('Error occurred')
34 return None
35
36 def parse_page_index(text):
37 try:
38 data = json.loads(text)
39 for item in data.get('data'):
40 #print(item["article_url"])
41 if "article_url" in item:
42 yield item.get("article_url")
43 #yield item.get('article_url','default_value')
44 except JSONDecodeError:
45 pass
46 #html=get_page_parse(0,'街拍')
47 #for url in parse_page_index(html):
48 #if len(url)>9:
49 #print(url)
50 def get_page_detail(url):
51 try:
52 response = requests.get(url)
53 if response.status_code == 200:
54 return response.text
55 return None
56 except ConnectionError:
57 print('Error occurred')
58 return None
59
60 def parse_page_details(html,url):
61 html1 = etree.HTML(html)
62 title = html1.xpath('//head/title/text()')
63 pattern = re.compile('gallery: JSON.parse\("(.*?)"\),\s+siblingList', re.S)
64 urls = re.findall(pattern, html)
65 # print(urls)
66 #print(title)
67 d = ",".join(urls)
68 s = d.replace('\\', "")
69 j = json.loads(s)
70 images_urls = [item.get('url') for item in j["sub_images"]]
71 for images_url in images_urls:download_image(images_url)
72 return {
73 'title':title,
74 'url':url,
75 'images_urls': images_urls
76
77 }
78
79 def save_to_mongo(resuit):
80 if db['toutiao1'].insert(resuit):
81 print("yes")
82 return True
83 else:
84 return False
85
86 def download_image(url):
87 print('brgain',url)
88 try:
89 response = requests.get(url)
90 if response.status_code == 200:
91 save_image(response.content)
92 return None
93 except ConnectionError:
94 print('Error occurred')
95 return None
96
97 def save_image(content):
98 file_path='{0}/{1}/{2}.{3}'.format(os.getcwd(),'pictyre',md5(content).hexdigest(),'jpg')
99 if not os.path.exists(file_path):
100 with open(file_path,'wb') as f:
101 f.write(content)
102 f.close()
103
104
105 def main(offset):
106 text=get_page_parse(offset,'街拍')
107 urls=parse_page_index(text)
108 for url in urls:
109 html=get_page_detail(url)
110 parse_page_details(html,url)
111 #print(result)
112 #save_to_mongo(result)
113
114 if __name__=='__main__':
115 for i in range(1,2):
116 pool=Pool()
117 pool.map(main,[offset*20 for offset in range(1,2)])
