scrapy 单文件启动单个spider

 

 

 

import scrapy
from scrapy.crawler import CrawlerRunner
from scrapy.settings import Settings
from twisted.internet import reactor


class ExampleSpider(scrapy.Spider):
    name = 'example'
    allowed_domains = ['example.com']
    start_urls = ['']

    def parse(self, response):
        print(response.text)


settings = Settings({'USER_AGENT': 'PYTHON'})
runner = CrawlerRunner(settings)
d = runner.crawl(ExampleSpider)
d.addBoth(lambda _: reactor.stop())
reactor.run()