python学习笔记-scrapy源码流程和自定义爬虫框架
一、scrapy源码流程
- 流程要点:
1、执行CrawlerProcess构造方法
2、CrawlerProcess对象(含有配置文件)的spiders
2.1、为每个爬虫创建一个Crawler
2.2、执行d=Crawler.crawl(...)
d.addBoth(_done)
2.3、CrawlerProcess对象._active={d,}
3、dd=defer.DeferredList(self._active)
dd.addBoth(self._stop_reactor) #self._stop_reactor ==>reactor.stop()
reactor.run()
实现原理同twisted
- Twisted示例
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
from twisted.internet import reactor from twisted.web.client import Agent,readBody from twisted.internet import defer from twisted.web._newclient import ResponseFailed import warnings warnings.filterwarnings("ignore",category=DeprecationWarning) @defer.inlineCallbacks def task(url): agent = Agent(reactor) v = agent.request(b'GET', url.encode("utf-8")) v.addBoth(download) yield v def download(response): print(response.code) r=readBody(response) r.addCallbacks(handle_response,handle_error) def stop(*args,**kwargs): reactor.stop() def handle_response(body): #处理响应 print(body) def handle_error(failure): #错误处理 if isinstance(failure.value, ResponseFailed): print("HTTP请求失败:", failure.value.response) else: print("发生未知错误:", failure.value) if __name__=='__main__': url_list=[ 'http://www.sogou.com', 'http://www.baidu.com', 'http://www.cnblogs.com', ] _active=set() for url in url_list: d=task(url) _active.add(d) dd=defer.DeferredList(_active) dd.addBoth(stop) reactor.run()
*原案例用的getPage,在twisted的版本16.7.0后弃用,示例用的Agent
二、Twisted使用
1、调用模块的功能原理如下:
from twisted.internet import reactor #事件循环(终止条件,所有的socket都已经移除)
from twisted.web.client import Agent #socket对象(如果下载完成,自动从事件循环中移除)
from twisted.internet import defer #defer.Deferred 特殊的socket对象(不会发请求,手动移除)
示例1:
实现基本流程,此时事件循环没有终止
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
from twisted.internet import reactor #事件循环(终止条件,所有的socket都已经移除) from twisted.web.client import Agent,readBody #socket对象(如果下载完成,自动从事件循环中移除) from twisted.internet import defer #defer.Deferred 特殊的socket对象(不会发请求,手动移除) #1、利用Agent创建socket #2、将socket添加到事件循环中 #3、开始事件循环(内部发送请求,并接收响应;当所有的socket请求完成后,终止事件循环) def hand_response(body): print(body) def response(response): print(response.code) # r=readBody(response) # r.addCallback(hand_response) @defer.inlineCallbacks def task(): url='http://www.baidu.com' agent = Agent(reactor) d=agent.request(b'GET',url.encode('utf-8')) print(type(d)) d.addCallbacks(response) yield d task() reactor.run()
2、让事件循环终止
示例2:
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
def hand_response(body): print(body) def response(response): print(response.code) # r=readBody(response) # r.addCallback(hand_response) @defer.inlineCallbacks def task(): url='http://www.baidu.com' agent = Agent(reactor) d=agent.request(b'GET',url.encode('utf-8')) d.addCallbacks(response) yield d def done(*args,**kwargs): reactor.stop() d=task() dd=defer.DeferredList([d,]) dd.addBoth(done) reactor.run()
3、特色的socket对象defer,使用defer.Deferred()进行handle住,再使用计数器终止
示例3:
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
from twisted.internet import reactor from twisted.web.client import Agent from twisted.internet import defer _close=None count=0 def response(response): print(response.code) global count count+=1 if count==3: _close.callback(None) @defer.inlineCallbacks def task(): ''' #相当于每个爬虫的 start_request :return: ''' url="http://www.baidu.com" agent = Agent(reactor) d=agent.request(b'GET',url.encode('utf-8')) d.addCallback(response) url = "http://www.cnblogs.com" agent = Agent(reactor) d2 = agent.request(b'GET',url.encode('utf-8')) d2.addCallback(response) url = "http://www.bing.com" agent = Agent(reactor) d3 = agent.request(b'GET',url.encode('utf-8')) d3.addCallback(response) global _close _close=defer.Deferred() yield _close def done(*args,**kwargs): reactor.stop() #每一个爬虫 spider1=task() spider2=task() dd=defer.DeferredList([spider1,spider2]) dd.addBoth(done) reactor.run()
三、自定义框架
疑问,defer.DeferredList()作用:
defer.DeferredList()是Twisted中的一个函数,用于创建一个Deferred对象列表,并在所有Deferred对象都触发时返回一个Deferred对象。它的作用是允许你同时处理多个Deferred对象,并在它们全部完成时获得通知
具体来说,defer.DeferredList()接受一个Deferred对象的列表作为参数,并返回一个新的Deferred对象。当传入的所有Deferred对象都出发时,返回的Deferred对象会触发,其回调函数会接收一个列表作为参数,列表
中包含了原始Deferred对象触发时传递的结果或Failure对象。
例子:
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
from twisted.internet import defer, reactor def task1(): d = defer.Deferred() reactor.callLater(2, d.callback, "Task 1 done,") return d def task2(): d = defer.Deferred() reactor.callLater(3, d.callback, "Task 2 done") return d def task3(): d = defer.Deferred() reactor.callLater(1, d.callback, "Task 3 done") return d def callback(results): print("All tasks completed") for result in results: print(result) def errback(failure): print("An error occurred:", failure) if __name__ == "__main__": deferred_list = defer.DeferredList([task1(), task2(), task3()]) deferred_list.addCallback(callback) deferred_list.addErrback(errback) reactor.run()
结果:
1、自定义的框架
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
from twisted.internet import reactor from twisted.web.client import Agent,readBody from twisted.internet import defer class Request(object): def __init__(self,url,callback): self.url=url self.callback=callback class ChoutiSpider(object): name="chouti" def start_requests(self): start_url=['http://www.baidu.com','http://www.bing.com'] for url in start_url: yield Request(url,self.parse) def parse(self,response): print('response',response) # yield Request('http://www.cnblogs.com') import queue Q=queue.Queue() class Engine(object): def __init__(self): self._close=None self.spider=None @defer.inlineCallbacks def crawl(self,spider): #pass 进行下载操作 self.spider=spider start_requests=iter(spider.start_requests()) while True: try: request=next(start_requests) Q.put(request) except StopIteration as e: break self._close=defer.Deferred() yield self._close spider=ChoutiSpider() _active=set() engine=Engine() d=engine.crawl(spider) _active.add(d) dd=defer.DeferredList(_active) dd.addBoth(lambda a:reactor.stop()) reactor.run()
*还未实现从队列取数据发请求
2、框架实现发送请求
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
from twisted.internet import reactor from twisted.web.client import Agent,readBody from twisted.internet import defer class Request(object): def __init__(self,url,callback): self.url=url self.callback=callback def hand_response(body): print(body) class HttpResponse(object): def __init__(self,content,request): try: self.content=content self.request=request self.url=request.url self.d_content=readBody(content) #self.text=str(content,encoding='utf-8') except Exception as e: print(e) class ChoutiSpider(object): name="chouti" def start_requests(self): start_url=['http://www.baidu.com','http://www.bing.com'] for url in start_url: yield Request(url,self.parse) def parse(self,response): response.d_content.addCallback(hand_response) #在回调函数中实现打印response yield Request('http://www.cnblogs.com',callback=self.parse) #1、crawlling移除 #2、获取paese yield值 #3、再次去队列中获取 import queue Q=queue.Queue() class Engine(object): def __init__(self): self._close=None self.max=5 self.crawlling=[] def get_response_callback(self,content,request): self.crawlling.remove(request) print('response&req:',content,request) rep=HttpResponse(content,request) result=request.callback(rep) #parse import types if isinstance(result,types.GeneratorType):#判断是否是生成器 for req in result: Q.put(req) def _next_request(self): ''' 去取request对象,并发送请求 最大并发数限制 :return: ''' if Q.qsize==0 and len(self.crawlling)==0: self._close.callback(None) return if len(self.crawlling)>=self.max: return while len(self.crawlling)<self.max: try: req=Q.get(block=False) #false表示队列中无数据时不等候 self.crawlling.append(req) agent=Agent(reactor) d=agent.request(b'GET',req.url.encode('utf-8')) #页面下载完成,get_response_callback,调用用户spider中定义的parse方法,并将新请求添加到调度器 d.addCallback(self.get_response_callback,req) #未达到最大并发数,可以再去调度器中获取request d.addCallback(lambda _:reactor.callLater(0,self._next_request)) except Exception as e: return @defer.inlineCallbacks def crawl(self,spider): #将初始request对象添加到调度器 start_requests=iter(spider.start_requests()) while True: try: request=next(start_requests) Q.put(request) except StopIteration as e: break #去调度器中取request,并发请求 #self._next_request() reactor.callLater(0, self._next_request) self._close=defer.Deferred() yield self._close spider=ChoutiSpider() _active=set() engine=Engine() d=engine.crawl(spider) _active.add(d) dd=defer.DeferredList(_active) dd.addBoth(lambda a:reactor.stop()) reactor.run()
3、tiny爬虫框架
封装到类,模仿scrapy的流程
结构:
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
from engine import Request class ChoutiSpider(object): name='chouti' def start_requests(self): start_url=['http://www.baidu.com','http://www.bing.com'] for url in start_url: yield Request(url,self.parse) def parse(self,response): print('chouti',response) #response.d_content.addCallback(response.hand_response) #在回调函数中实现打印response #yield Request('http://www.cnblogs.com',callback=self.parse)
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
from engine import Request class CnblogsSpider(object): name='cnblogs' def start_requests(self): start_url=['http://www.cnblogs.com'] for url in start_url: yield Request(url,self.parse) def parse(self,response): print('cnblogs',response) #yield Request('http://www.cnblogs.com',callback=self.parse)
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
from twisted.internet import reactor from twisted.web.client import Agent,readBody from twisted.internet import defer import queue class Request(object): ''' 用于封装用户请求相关信息 ''' def __init__(self,url,callback): self.url=url self.callback=callback class HttpResponse(object): def __init__(self,content,request): try: self.content=content self.request=request self.url=request.url self.d_content=readBody(content) except Exception as e: print(e) def hand_response(self,body): print(body) class CrawlerProcess(object): ''' 开启时间循环 ''' def __init__(self): self._active=set() def crawl(self,spider_cls_path): crawler=Crawler() d=crawler.crawl(spider_cls_path) self._active.add(d) def start(self): dd=defer.DeferredList(self._active) dd.addBoth(lambda _:reactor.stop()) reactor.run() class Crawler(object): ''' 用于封装调度器和引擎的 ''' def _create_engine(self): return ExecutionEngine() def _create_spider(self,spider_cls_path): module_path,cls_name=spider_cls_path.rsplit('.',maxsplit=1) import importlib m=importlib.import_module(module_path) cls=getattr(m,cls_name) return cls() @defer.inlineCallbacks def crawl(self,spider_cls_path): engine=self._create_engine() spider=self._create_spider(spider_cls_path) start_requests=iter(spider.start_requests()) yield engine.open_spider(start_requests) yield engine.start() class Scheduler(object): ''' 调度器 ''' def __init__(self): self.q=queue.Queue() def enqueue_request(self,req): self.q.put(req) def open(self): pass def next_request(self): try: req=self.q.get(block=False) except Exception as e: req=None return req def size(self): return self.q.qsize() class ExecutionEngine(object): ''' 引擎 ''' def __init__(self): self._close=None self.scheduler=None self.max=5 self.crawlling=[] def get_response_callback(self,content,request): self.crawlling.remove(request) response=HttpResponse(content,request) result=request.callback(response) import types if isinstance(result,types.GeneratorType):#判断是否是生成器 for req in result: self.scheduler.enqueue_request(req) def _next_request(self): if self.scheduler.size==0 and len(self.crawlling)==0: self._close.callback(None) return while len(self.crawlling)<self.max: req=self.scheduler.next_request() if not req: return self.crawlling.append(req) agent=Agent(reactor) d=agent.request(b'GET',req.url.encode('utf-8')) d.addCallback(self.get_response_callback,req) d.addCallback(lambda _:reactor.callLater(0,self._next_request)) @defer.inlineCallbacks def open_spider(self,start_requests): self.scheduler=Scheduler() yield self.scheduler.open() while True: try: req=next(start_requests) except StopIteration as e: break self.scheduler.enqueue_request(req) reactor.callLater(0,self._next_request) @defer.inlineCallbacks def start(self): self._close=defer.Deferred() yield self._close class Commond(object): def run(self): crawl_process=CrawlerProcess() spider_clss_path_list=[ 'spider.chouti.ChoutiSpider', 'spider.cnblogs.CnblogsSpider' ] for spider_clss_path in spider_clss_path_list: crawl_process.crawl(spider_clss_path) crawl_process.start() if __name__=="__main__": cmd=Commond() cmd.run()
运行结果: