1 import json
2 import os
3 from urllib.parse import urlencode
4 import pymongo
5 import requests
6 from bs4 import BeautifulSoup
7 from requests.exceptions import ConnectionError
8 import re
9 from multiprocessing import Pool
10 from hashlib import md5
11 from json.decoder import JSONDecodeError
12 from config import *
13
14 client = pymongo.MongoClient(MONGO_URL, connect=False)
15 db = client[MONGO_DB]
16
17
18 def get_page_index(offset, keyword):
19 data = {
20 'autoload': 'true',
21 'count': 20,
22 'cur_tab': 3,
23 'format': 'json',
24 'keyword': keyword,
25 'offset': offset,
26 }
27 params = urlencode(data)
28 base = 'http://www.toutiao.com/search_content/'
29 url = base + '?' + params
30 try:
31 response = requests.get(url)
32 if response.status_code == 200:
33 return response.text
34 return None
35 except ConnectionError:
36 print('Error occurred')
37 return None
38
39
40 def download_image(url):
41 print('Downloading', url)
42 try:
43 response = requests.get(url)
44 if response.status_code == 200:
45 save_image(response.content)
46 return None
47 except ConnectionError:
48 return None
49
50
51 def save_image(content):
52 file_path = '{0}/{1}.{2}'.format(os.getcwd(), md5(content).hexdigest(), 'jpg')
53 print(file_path)
54 if not os.path.exists(file_path):
55 with open(file_path, 'wb') as f:
56 f.write(content)
57 f.close()
58
59
60 def parse_page_index(text):
61 try:
62 data = json.loads(text)
63 if data and 'data' in data.keys():
64 for item in data.get('data'):
65 yield item.get('article_url')
66 except JSONDecodeError:
67 pass
68
69
70 def get_page_detail(url):
71 try:
72 response = requests.get(url)
73 if response.status_code == 200:
74 return response.text
75 return None
76 except ConnectionError:
77 print('Error occurred')
78 return None
79
80
81 def parse_page_detail(html, url):
82 soup = BeautifulSoup(html, 'lxml')
83 result = soup.select('title')
84 title = result[0].get_text() if result else ''
85 images_pattern = re.compile('gallery: JSON.parse\("(.*)"\)', re.S)
86 result = re.search(images_pattern, html)
87 if result:
88 data = json.loads(result.group(1).replace('\\', ''))
89 if data and 'sub_images' in data.keys():
90 sub_images = data.get('sub_images')
91 images = [item.get('url') for item in sub_images]
92 for image in images: download_image(image)
93 return {
94 'title': title,
95 'url': url,
96 'images': images
97 }
98
99
100 def save_to_mongo(result):
101 if db[MONGO_TABLE].insert(result):
102 print('Successfully Saved to Mongo', result)
103 return True
104 return False
105
106
107 def main(offset):
108 text = get_page_index(offset, KEYWORD)
109 urls = parse_page_index(text)
110 for url in urls:
111 html = get_page_detail(url)
112 result = parse_page_detail(html, url)
113 if result: save_to_mongo(result)
114
115
116 if __name__ == '__main__':
117 pool = Pool()
118 groups = ([x * 20 for x in range(GROUP_START, GROUP_END + 1)])
119 pool.map(main, groups)
120 pool.close()
121 pool.join()
