爬虫框架scrapy

- 框架：具有很多功能且具有很强通用性的一个项目模板。

环境换装：

        Linux：

        pip3 install scrapy

 

Windows：

      a. pip3 install wheel

      b. 下载twisted http://www.lfd.uci.edu/~gohlke/pythonlibs/#twisted

      c. 进入下载目录，执行 pip3 install Twisted‑17.1.0‑cp35‑cp35m‑win_amd64.whl

      d. pip3 install pywin32

    e. pip3 install scrapy

在命令行：

创建一个工程：scrapy startproject proName

      切换到创建的工程下：cd proName

      创建爬虫文件：scrapy genspider spiderName www.xxx.com

      执行工程：scrapy crawl spiderName

 

爬取数据（__init__.py文件同一级的文件中（wangyi.py））：

# -*- coding: utf-8 -*-
from wangyiPro.items import WangyiproItem
import scrapy
import re


class WangyiSpider(scrapy.Spider):
    name = 'wangyi'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['https://temp.163.com/special/00804KVA/cm_guonei.js?callback=data_callback'] #发送请求的url
    
    def layer_parse(self,response):
        item = response.meta['item']
        p = response.xpath('//*[@id="endText"]/p/text() | //*[@id="endText"]/p/b/b/text() | //*[@id="endText"]/p/b/text()')
        item['content'] = p
        yield item    #将item发送给管道进行存储


    def parse(self, response):
        ret = response.text
        ex = '"title":"(.*?)",.*?"docurl":"(.*?)",' #正则表达式
        img_src_list = re.findall(ex, ret, re.S)    #正则匹配
        for i in img_src_list:
            item = WangyiproItem()    #实例化item对象
            name = i[0].encode('iso-8859-1').decode('gbk')    #解决中文乱码
            name_url = i[1]
            item['name'] = name
            item['name_url'] = name_url
            yield scrapy.Request(url=name_url,callback=self.layer_parse,meta={'item':item})    #callback是回调函数，  meta是给回调函数传参

创建item对象（在items.py文件中）

# -*- coding: utf-8 -*-


# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html


import scrapy
class WangyiproItem(scrapy.Item):
    # define the fields for your item here like:
    name = scrapy.Field()    #格式必须是这样的
    content = scrapy.Field()
    name_url = scrapy.Field()

配置 settings文件

# -*- coding: utf-8 -*-


# Scrapy settings for xioahuaPro project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html


BOT_NAME = 'xioahuaPro'


SPIDER_MODULES = ['xioahuaPro.spiders']
NEWSPIDER_MODULE = 'xioahuaPro.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'xioahuaPro (+http://www.yourdomain.com)'
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'


# Obey robots.txt rules
ROBOTSTXT_OBEY = False   #反爬机制


# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32


# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16


# Disable cookies (enabled by default)
#COOKIES_ENABLED = False


# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False


# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
#}


# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'xioahuaPro.middlewares.XioahuaproSpiderMiddleware': 543,
#}


# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
#    'xioahuaPro.middlewares.XioahuaproDownloaderMiddleware': 543,
#}


# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}


# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
#300表示的是优先级，数值越小优先级越高
ITEM_PIPELINES = {
    'xioahuaPro.pipelines.XioahuaproPipeline': 300, #数据库（权重越小越先执行）
    # 'xioahuaPro.pipelines.MysqlPipeline': 301,
    # 'xioahuaPro.pipelines.RedisPipeline': 302,
}


# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False


# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'


LOG_LEVEL = 'ERROR'      #只打印错误日志
LOG_FILE = './log.txt'   #

管道存储item（在pipelines.py文件中）

# -*- coding: utf-8 -*-
# Define your item pipelines here
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
#作用：将解析到的数据存储到某一个平台中。
import pymysql
from redis import Redis
#存在本地磁盘中
class XioahuaproPipeline(object):
    fp = None
    def open_spider(self,spider):
        print('开始爬虫！')
        self.fp = open('./xiaohua.txt','w',encoding='utf-8')
    #作用：实现持久化存储的操作
    #该方法的item参数就可以接收爬虫文件提交过来的item对象
    #该方法每接收一个item就会被调用一次（调用多次）
    def process_item(self, item, spider):
        name = item['name']
        img_url = item['img_url']
        self.fp.write(name+':'+img_url+'\n')
        #返回值的作用：就是将item传递给下一个即将被执行的管道类
        return item




    def close_spider(self,spider):
        print('结束爬虫！')
        self.fp.close()


#存在mysql数据库中
class MysqlPipeline(object):
    conn = None
    cursor = None
    def open_spider(self, spider):
        #解决数据库字段无法存储中文处理：alter table tableName convert to charset utf8;
        self.conn = pymysql.Connect(host='127.0.0.1',port=3306,user='root',password='123',db='test')
        print(self.conn)
    def process_item(self, item, spider):
        self.cursor = self.conn.cursor()
        try:
            self.cursor.execute('insert into xiahua values ("%s","%s")'%(item['name'],item['img_url']))
            self.conn.commit()
        except Exception as e:
            print(e)
            self.conn.rollback()
        return item
    def close_spider(self, spider):
        self.cursor.close()
        self.conn.close()


#存在redis数据库中
class RedisPipeline(object):
    conn = None
    def open_spider(self, spider):
        self.conn = Redis(host='127.0.0.1',port=6379)
        print(self.conn)
    def process_item(self, item, spider):
        dic = {
            'name':item['name'],
            'img_url':item['img_url']
        }
        print(dic)
        self.conn.lpush('xiaohua',dic)#保存到redis数据库
        return item
    def close_spider(self, spider):
        p