<4> pipeline

"""scrapy 保存管道数据"""
from scrapy.exporters import CsvItemExporter


class CsvPipeline:

    def __init__(self):
        # 文件存储初始化操作
        self.file = open('filename.csv', 'wb')
        self.exporter = CsvItemExporter(file=self.file, include_headers_line=True)
        self.exporter.start_exporting()

    def process_item(self, item, spider):
        self.exporter.export_item(item)
        return item

    def close_spider(self, spider):
        self.exporter.finish_exporting()
        self.file.close()


"""scrapy 管道去重保存"""
import json
import hashlib
from scrapy.exceptions import DropItem
from scrapy.exporters import CsvItemExporter


class DupefiltePipeline:

    def __init__(self):
        # 去重集合
        self.fingerprints = set()
        # 文件存储初始化操作
        self.file = open('filename.csv', 'wb')
        self.exporter = CsvItemExporter(file=self.file, include_headers_line=True)
        self.exporter.start_exporting()

    def process_item(self, item, spider):
        fp = self.item_fingerprint(item)
        if fp in self.fingerprints:
            raise DropItem('duperfilters')
        self.fingerprints.add(fp)
        self.exporter.export_item(item)
        return item

    def item_fingerprint(self, item):
        dump_item = json.dumps(item)
        return hashlib.md5(bytes(dump_item, encoding='utf-8')).digest().hex()

    def close_spider(self, spider):
        self.exporter.finish_exporting()
        self.file.close()


"""scrapy 管道保存到MongoDB"""
class BookPipeline:
    # 开始爬虫
    def open_spider( self， spider ):
        print( " open. . . ")
        # 定义一个变盘，记录爬取的数量
        self.count = 0
        try:
            # 1. 获取MongoD8客户端
            self.client - Mongoclient( " 1ocalhost" , 27817)
            # 2. 获取数据库
            self.db = self.client.dangdang
            # 3.获取数据集合
            self.collection = self.db.collection_dangdang
            # 标识
            self.opend = True
        except BaseException as e :
            print(e)
            self.opend = Flase
            
    # 执行数据棱型的处理
    def process_item( se1f，item,spider ):
        if self.opened:
            self.collection.insert_one(dict(item))
            self.count += 1
        return item
    
    # 结束爬虫
    def close_spider( self, spider) :
        print(f"爬取了{self.count}条数据")
        print('closed...')
        if self.opend:
            # 说明: 如果数据库打开成立了，爬虫结束后，就掠数据库连接关闭棹
            self.client.close()