多线程爬虫
应用场景
1、多进程 ：CPU密集程序
2、多线程 ：爬虫(网络I/O)、本地磁盘I/O
队列
# 导入模块 from queue import Queue # 使用 q = Queue() q.put(url) q.get() # 当队列为空时，阻塞 q.empty() # 判断队列是否为空，True/False
线程模块
# 导入模块 from threading import Thread # 使用流程 t = Thread(target=函数名) # 创建线程对象 t.start() # 创建并启动线程 t.join() # 阻塞等待回收线程 #如何创建多线程 t_list=[] for i in range(5): t = Thread(target=函数名) t_list.append(t) t.start() for t in t_list: t.join()
小米应用商店抓取(多线程)
目标
1、网址 ：百度搜 - 小米应用商店，进入官网 2、目标 ：应用分类 - 聊天社交 应用名称 应用链接
实现步骤
1.确认是否为动态加载
1、页面局部刷新 2、右键查看网页源代码，搜索关键字未搜到 # 此网站为动态加载网站，需要抓取网络数据包分析
2.F12抓取网络数据包
1、抓取返回json数据的URL地址（Headers中的Request URL） http://app.mi.com/categotyAllListApi?page={}&categoryId=2&pageSize=30 2、查看并分析查询参数（headers中的Query String Parameters） page: 1 categoryId: 2 pageSize: 30 # 只有page再变，0 1 2 3 ... ... ，这样我们就可以通过控制page的直拼接多个返回json数据的URL地址
3.代码实现
import requests from threading import Thread from queue import Queue import time from fake_useragent import UserAgent from lxml import etree class XiaomiSpider(object): def __init__(self): self.url = 'http://app.mi.com/categotyAllListApi?page={}&categoryId={}&pageSize=30' # 存放所有URL地址的队列 self.q = Queue() self.ua = UserAgent() self.i = 0 # 存放所有类型id的空列表 self.id_list = [] def get_cateid(self): # 请求 url = 'http://app.mi.com/' headers = { 'User-Agent':self.ua.random } html = requests.get(url=url,headers=headers).text # 解析 parse_html = etree.HTML(html) xpath_bds = '//ul[@class="category-list"]/li' li_list = parse_html.xpath(xpath_bds) for li in li_list: typ_name = li.xpath('./a/text()')[0] typ_id = li.xpath('./a/@href')[0].split('/')[-1] # 计算每个类型的页数 pages = self.get_pages(typ_id) self.id_list.append( (typ_id,pages) ) # 入队列 self.url_in() # 获取counts的值并计算页数 def get_pages(self,typ_id): # 每页返回的json数据中,都有count这个key url = self.url.format(0,typ_id) html = requests.get( url=url, headers={'User-Agent':self.ua.random} ).json() count = html['count'] pages = int(count) // 30 + 1 return pages # url入队列 def url_in(self): for id in self.id_list: # id为元组,('2',pages) for page in range(id[1]): url = self.url.format(page,id[0]) print(url) # 把URL地址入队列 self.q.put(url) # 线程事件函数: get() - 请求 - 解析 - 处理数据 def get_data(self): while True: if not self.q.empty(): url = self.q.get() headers = {'User-Agent':self.ua.random} html = requests.get(url=url,headers=headers).json() self.parse_html(html) else: break # 解析函数 def parse_html(self,html): for app in html['data']: # 应用名称 name = app['displayName'] link = 'http://app.mi.com/details?id=' + app['packageName'] print(name,link) self.i += 1 # 主函数 def main(self): # URL入队列 self.get_cateid() t_list = [] # 创建多个线程 for i in range(1): t = Thread(target=self.get_data) t_list.append(t) t.start() # 回收线程 for t in t_list: t.join() print('数量:',self.i) if __name__ == '__main__': start = time.time() spider = XiaomiSpider() spider.main() end = time.time() print('执行时间:%.2f' % (end-start))
腾讯招聘数据抓取
1.确定url地址及目标
1、URL: 百度搜索腾讯招聘 - 查看工作岗位
2、目标: 职位名称、工作职责、岗位要求
2.要求与分析
1、通过查看网页源码,得知所需数据均为 Ajax 动态加载 2、通过F12抓取网络数据包,进行分析 3、一级页面抓取数据: 职位名称 4、二级页面抓取数据: 工作职责、岗位要求
3.一级页面json地址(index在变,timestamp未检查)
https://careers.tencent.com/tencentcareer/api/post/Query?timestamp=1563912271089&countryId=&cityId=&bgIds=&productId=&categoryId=&parentCategoryId=&attrId=&keyword=&pageIndex={}&pageSize=10&language=zh-cn&area=cn
4.二级页面地址(postId在变,在一级页面中可拿到)
https://careers.tencent.com/tencentcareer/api/post/ByPostId?timestamp=1563912374645&postId={}&language=zh-cn
5.代码实现
import requests import json import time import random from fake_useragent import UserAgent class TencentSpider(object): def __init__(self): self.one_url = 'https://careers.tencent.com/tencentcareer/api/post/Query?timestamp=1563912271089&countryId=&cityId=&bgIds=&productId=&categoryId=&parentCategoryId=&attrId=&keyword=&pageIndex={}&pageSize=10&language=zh-cn&area=cn' self.two_url = 'https://careers.tencent.com/tencentcareer/api/post/ByPostId?timestamp=1563912374645&postId={}&language=zh-cn' # 获取User-Agent def get_headers(self): ua = UserAgent() headers = { 'User-Agent': ua.random } return headers # 获取响应内容函数 def get_page(self,url): html = requests.get(url=url,headers=self.get_headers()).content.decode('utf-8','ignore') # json.loads()把json格式的字符串转为python数据类型 html = json.loads(html) return html # 主线函数: 获取所有数据 def parse_page(self,one_url): html = self.get_page(one_url) item = {} for job in html['Data']['Posts']: item['name'] = job['RecruitPostName'] item['address'] = job['LocationName'] # 拿postid为了拼接二级页面地址 post_id = job['PostId'] # 职责和要求(二级页面) two_url = self.two_url.format(post_id) item['duty'],item['requirement'] = self.parse_two_page(two_url) print(item) def parse_two_page(self,two_url): html = self.get_page(two_url) # 职责 + 要求 duty = html['Data']['Responsibility'] requirement = html['Data']['Requirement'] return duty,requirement # 获取总页数 def get_pages(self): url = self.one_url.format(1) html = self.get_page(url) pages = int(html['Data']['Count']) // 10 + 1 return pages def main(self): # 总页数 pages = self.get_pages() for index in range(1,pages): one_url = self.one_url.format(index) self.parse_page(one_url) time.sleep(random.uniform(0.5,1.5)) if __name__ == '__main__': start = time.time() spider = TencentSpider() spider.main() end = time.time() print('执行时间:%.2f' % (end-start))
6.多线程思路:
把所有一级页面链接提交到队列,进行多线程数据抓取
7.多线程代码实现
import requests import json import time import random from fake_useragent import UserAgent from threading import Thread from queue import Queue class TencentSpider(object): def __init__(self): self.one_url = 'https://careers.tencent.com/tencentcareer/api/post/Query?timestamp=1563912271089&countryId=&cityId=&bgIds=&productId=&categoryId=&parentCategoryId=&attrId=&keyword=&pageIndex={}&pageSize=10&language=zh-cn&area=cn' self.two_url = 'https://careers.tencent.com/tencentcareer/api/post/ByPostId?timestamp=1563912374645&postId={}&language=zh-cn' self.q = Queue() # 计数 self.i = 0 # 获取User-Agent def get_headers(self): ua = UserAgent() headers = { 'User-Agent': ua.random } return headers # 获取响应内容函数 def get_page(self,url): html = requests.get(url=url,headers=self.get_headers()).content.decode('utf-8','ignore') # json.loads()把json格式的字符串转为python数据类型 html = json.loads(html) return html # 主线函数: 获取所有数据 def parse_page(self): while True: if not self.q.empty(): one_url = self.q.get() html = self.get_page(one_url) item = {} for job in html['Data']['Posts']: item['name'] = job['RecruitPostName'] item['address'] = job['LocationName'] # 拿postid为了拼接二级页面地址 post_id = job['PostId'] # 职责和要求(二级页面) two_url = self.two_url.format(post_id) item['duty'],item['requirement'] = self.parse_two_page(two_url) print(item) self.i += 1 # 每爬取按完成1页随机休眠 time.sleep(random.uniform(0,1)) else: break def parse_two_page(self,two_url): html = self.get_page(two_url) # 职责 + 要求 duty = html['Data']['Responsibility'] requirement = html['Data']['Requirement'] return duty,requirement # 获取总页数 def get_pages(self): url = self.one_url.format(1) html = self.get_page(url) pages = int(html['Data']['Count']) // 10 + 1 return pages def main(self): # one_url入队列 pages = self.get_pages() for index in range(1,pages): one_url = self.one_url.format(index) self.q.put(one_url) t_list = [] for i in range(5): t = Thread(target=self.parse_page) t_list.append(t) t.start() for t in t_list: t.join() print('数量:',self.i) if __name__ == '__main__': start = time.time() spider = TencentSpider() spider.main() end = time.time() print('执行时间:%.2f' % (end-start))
