web爬虫02-多线程爬虫
多线程爬虫
原理
利用CPU和IO可以同时执行的原理,让CPU不会干巴巴的等待IO的完成
#网站:还是豆瓣250 https://movie.douban.com/top250
import requests
from lxml import etree
import time
from threading import Thread
import codecs
# 时间装饰器
def timer(func):
def inner(*args, **kw):
t1 = time.time()
func(*args, **kw)
t2 = time.time()
print("-------一共花费时间:{}秒".format(t2-t1))
return t2 - t1
return inner
def get_content(url):
res = requests.get(url, headers=headers)
deal_content(res.text)
def deal_content(content):
res = etree.HTML(content)
items = res.xpath('//div[@id="wrapper"]//ol//li/div[@class="item"]/div[@class="info"]')
infos = []
for item in items:
name = item.xpath('./div/a/span[1]/text()')[0]
description = item.xpath('./div/p/text()')[0].strip()
evaluate = item.xpath('./div[2]/div[@class="star"]/span[4]/text()')[0]
infos.append(name)
infos.append(description)
infos.append(evaluate)
f.write(u'{movies}\n'.format(movies='\n'.join(infos)))
@timer
def multi_thread(urls):
threads = []
for url in urls:
threads.append(
Thread(target=get_content, args=(url,))
)
for thread in threads:
thread.start()
for thread in threads:
thread.join()
if __name__ == '__main__':
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36'
}
urls = ['https://movie.douban.com/top250?start={}&filter='.format(i*25) for i in range(10)]
with codecs.open('moviess', 'wb', encoding='utf-8') as f:
multi_thread(urls)
花费时间
单线程为:3.102s
多线程为:0.348s
大致为10倍速度