1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3 # @Time : 2018/7/12 21:10
4 # @Author : chenxiaowei
5 # @Email : chen1020xiaowei@163.com
6 # @File : vip.py
7
8 from pymongo.errors import ConfigurationError
9 from selenium import webdriver
10 from selenium.common.exceptions import TimeoutException, WebDriverException
11 from selenium.webdriver.common.by import By
12 from selenium.webdriver.support.ui import WebDriverWait
13 from selenium.webdriver.support import expected_conditions as EC
14 from pyquery import PyQuery
15 from urllib3.exceptions import NewConnectionError, MaxRetryError
16 from config_vip import *
17 from multiprocessing import Pool
18 from selenium.webdriver.chrome.options import Options
19 import os
20 import pymongo
21 import requests
22 import hashlib
23 import time
24
25 if browser_method == 0:
26 browser = webdriver.Chrome()
27 print('你选择使用Chrome()方法...')
28 elif browser_method == 1:
29 browser = webdriver.PhantomJS(service_args=['--load-images=false', '--disk-cache=false'])
30 print('你选择使用PhantomJS()方法...')
31 else:
32 chrome_option = Options()
33 chrome_option.add_argument('--headless')
34 browser = webdriver.Chrome(options=chrome_option)
35 print('你选择使用Headless()方法...')
36
37 browser.set_window_size(1920, 1080)
38 wait = WebDriverWait(browser, 10)
39
40 try:
41 client = pymongo.MongoClient(mongo_url)
42 database = client[mongo_database]
43 except TypeError:
44 print('数据库创建失败'.center(130, '*'))
45 except ConfigurationError:
46 print('数据库创建失败'.center(130, '*'))
47
48
49 # 实现数据库对象
50
51 def drop_down_scrollbar():
52 # 定义下拉滚动条方法
53 times = 1
54 while times < total_times:
55 js = "var q=document.documentElement.scrollTop={}".format(times * size)
56 browser.execute_script(js)
57 time.sleep(1)
58 times += 1
59
60
61 def get_search(search_word):
62 # 定义get_()search方法
63 url = main_url
64 browser.get(url)
65 # 打开url，获得内容
66 time.sleep(3)
67 try:
68 search_bar = wait.until(
69 EC.presence_of_element_located((By.CSS_SELECTOR, '#J-search > div.c-search-form > input')))
70 enter_button = wait.until(
71 EC.element_to_be_clickable((By.CSS_SELECTOR, '#J-search > div.c-search-form > a > span')))
72 # 确定输入框和搜索按钮可用
73 search_bar.send_keys(search_word)
74 time.sleep(1)
75 enter_button.click()
76 # 输入关键字并点击搜索
77 time.sleep(5)
78 drop_down_scrollbar()
79 pages = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '#J_pagingCt > a:nth-child(6)')))
80 # 获得总页数，main()中作为for循环参数
81 print('搜索到{}共{}页的内容'.format(search_word, pages.text))
82 time.sleep(3)
83 print('开始获取{}第{}页的内容...'.format(search_word, str(1)))
84 get_page_detail(search_word)
85 print('完成获取{}第{}页的内容...'.format(search_word, str(1)))
86 return pages.text
87 except TimeoutException:
88 print('网页未加载完成，无法搜索信息！', TimeoutException.args)
89 pass
90 except WebDriverException:
91 print(WebDriverException.args)
92 pass
93
94
95 def get_next_page(search_word, page):
96 # 定义get_next_page()方法进行跳转
97 try:
98 url1 = url_search.format(search_word, str(page))
99 # 找出网页规律，定个模板
100 print('开始获取{}第{}页的内容...\n'.format(search_word, page))
101 browser.get(url1)
102 drop_down_scrollbar()
103 get_page_detail(search_word)
104 print('完成获取{}第{}页的内容...\n'.format(search_word, page))
105 except TimeoutException:
106 print('跳转网页超时！', TimeoutException.args)
107 pass
108 except WebDriverException:
109 print(WebDriverException.args)
110 pass
111
112
113 def get_page_detail(search_word):
114 # 定义get_page_detail()方法获取网页详细信息
115 try:
116 source = browser.page_source
117 html = PyQuery(source)
118 print('解析数据成功'.center(130, '*'))
119 # PyQuery解析源代码
120 good_items = html('.goods-list .goods-list-item').items()
121 # 调用items()方法获得数据
122 for item in good_items:
123 goods = {
124 'good-title': item.find('.goods-title-info ').text().split('\n')[1],
125 'good-sells-price': item.find('.goods-info .goods-price-wrapper .goods-sells-price .price').text(),
126 'good-market-price': item.find('.goods-info .goods-market-price').text()[2:],
127 'good-discount': item.find('.goods-info .goods-discount').text(),
128 'good-brand': item.find('.goods-info .goods-brand').text(),
129 'image': 'http:{}'.format(item.find('.goods-slide .goods-image-link .J_img').attr('src')),
130 'detail': 'http:{}'.format(item.find(' .goods-slide .goods-image-link').attr('href'))
131 }
132 image_url = goods['image']
133 content = get_image_content(image_url)
134 if content:
135 # 确定图片网页是否可以打开
136 download_image(content, search_word, image_url)
137 save_to_mongodb(goods, search_word)
138 # 调用find方法和CSS取得数据
139 except TimeoutException:
140 print('爬取网页超时！', TimeoutException.args)
141 pass
142
143
144 def save_to_mongodb(goods, database_table):
145 # 定义save_to_mongoDB(goods)方法将数据存储到mongoDB数据
146 try:
147 if database[database_table].insert(goods):
148 # 插入数据成功
149 print('存储数据成功'.center(130, '*'))
150 print(goods, '\n')
151 except Exception:
152 print('写入数据出错！', Exception.args)
153 pass
154
155
156 def get_image_content(url):
157 try:
158 response = requests.get(url)
159 if response.status_code == 200:
160 return response.content
161 else:
162 print('请求图片链接失败！')
163 except ConnectionError:
164 print(ConnectionError.args)
165 return False
166 except NewConnectionError:
167 print(NewConnectionError.args)
168 return False
169 except MaxRetryError:
170 print(MaxRetryError.args)
171 return False
172
173
174 def download_image(content, folder, image_url):
175 # 定义download_image(content)保存图片
176 time_stamp = time.strftime("%Y%m%d", time.localtime())
177 path = file_path.format(mongo_database, time_stamp, folder)
178 if os.path.exists(path):
179 pass
180 else:
181 os.makedirs(path)
182 # 利用hash算法获得content MD5值以16进制显示
183 filename = hashlib.md5(content).hexdigest()
184 with open(file_type.format(path, filename), 'wb')as f:
185 f.write(content)
186 f.close()
187 # 打开文件保存路径，文件名，格式，wb写入形式
188 print(' {} 下载图片成功'.format(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())).center(125, '*'))
189 print(filename, image_url)
190
191
192 def main(search_word):
193 pages = int(get_search(search_word))
194 page = 2
195 if pages >= end:
196 pages = end
197 try:
198 while page <= pages:
199 get_next_page(search_word, page)
200 page += 1
201 except TimeoutException:
202 print(TimeoutException.args)
203 pass
204
205
206 if __name__ == '__main__':
207 pool = Pool(processes=2)
208 pool.map(main, [keyword for keyword in keywords])
209 pool.close()
210 #锁定进程池
211 pool.join()
212 os.system('taskkill /im chromedriver.exe /F')
213 os.system('taskkill /im chrome.exe /F')
214 #杀死多余的chromedriver进程以及chrome进程
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3 # @Time : 2018/7/12 23:48
4 # @Author : chenxiaowei
5 # @Email : chen1020xiaowei@163.com
6 # @File : config_vip.py
7 mongo_url = 'localhost'
8 mongo_database = 'vip'
9 #数据库地址以及名称
10 main_url = 'https://www.vip.com/'
11
12 total_times =16
13 size =500
14 #设定下来滚动条的次数和大小
15
16 browser_method = 2
17 #驱动浏览器的方法
18
19 start=1
20 end = 45
21 #设定结束网页，有些网页没有内容，容易引起一场
22 url_search = 'https://category.vip.com/suggest.php?keyword={}&page={}&count=100&suggestType=brand#catPerPos'
23 #定义模板
24 file_path = 'H:/Python_download/{}/{}/image/{}/'
25 file_type = '{}{}.jpg'
26 # 文件类型以及文件夹
27
28
29 keywords=['苹果','雪梨','香蕉']
浙公网安备 33010602011771号