Scrapy尝试爬取微博热搜（初稿）

  首先自己想要的item：

import scrapy


class WeiboItem(scrapy.Item):

    rank = scrapy.Field()
    title = scrapy.Field()
    hot_totle = scrapy.Field()
    tag_pic = scrapy.Field()
    watch = scrapy.Field()
    talk = scrapy.Field()
    weibo_detail = scrapy.Field()
    bozhu = scrapy.Field()
    biaoqian = scrapy.Field()
    time = scrapy.Field()

 

 

主要spider：记住referer！！！搜索详情页面会判断来源

import scrapy
import logging
import json
logger = logging.getLogger(__name__)
from weibo.items import WeiboItem
# from weibo.settings import MYSQL_HOST
import datetime
"""
class=icon-top     是置顶 判断其是否是买的  看后面的 icon-txt 如果是icon-txt-recommend推荐
"""
class WeiboResouSpider(scrapy.Spider):
    name = 'weibo-resou'    # 爬虫的名字
    allowed_domains = ['weibo.com'] # 允许爬取的范围
    start_urls = ['http://s.weibo.com/top/summary/']  # 热搜排行榜 
    detail_url = 'https://s.weibo.com/weibo?q=%23{}%23&Refer=top'    # 搜索页面
    tag_url = 'https://s.weibo.com/top/summary/summary?cate=socialevent'  # tag

    headers = {
        "User-Agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36",
        "Referer" : "https://s.weibo.com/"　　
        }

    Cookie = {
        'SCF' : 'XXXX',
        'SUB' : 'XXXX',
        'SUBP' : 'XXXX',
        'SUHB' : 'XXXX',
        'ALF' : 'XXXX',
        'SSOLoginState' : 'XXXXX'}



    
    def parse(self, response):
        # .extract  返回包含字符串数据的列表  .extract_first() 返回第一个
        # 这里yield 返回可以是Request , BaseItem , 字典 或 None
        # 此处yield 即可在pipeline里操作 生成器返回的值，或数据
        # yield 返回的化就可 每次 循环都返回一个
        # log 日志获取错误（主要为了定位
        # 注意看如果图片链接前有 网站的域名 需要在获取到img 后进行拼接！列表推导式
        # 如果要 循环调用 ，记得yield scrapu.Request 这个自己的parse url记得更新
        # self.settings['MYSQL_HOST']
        # self.settings.get('MYSQL_HOST')
        logging.warning('this is spider warning')
        tr_list = response.xpath('//*[@id="pl_top_realtimehot"]/table//tr')
        for tr in tr_list[1:]:
            item = WeiboItem()
            # item = {}
            # 排名
            if tr.xpath('./td[1]/text()').extract_first() is None:
                item['rank'] = '置顶向'
            else:
                item['rank'] = tr.xpath('./td[1]/text()').extract_first()
            # 获取热搜标题
            if tr.xpath('./td[2]/a//text()').extract_first() is None:
                item['title'] = '找不到热搜哦'
            else:
                item['title'] = tr.xpath('./td[2]/a//text()').extract_first()


            # 判断这里是否有热度

            if tr.xpath('./td[2]/span/text()').extract_first() is None:
                item['hot_totle'] = '找不到热度哟'
            else:
                item['hot_totle'] = tr.xpath('./td[2]/span/text()').extract_first()
            # 获取热搜tag
            if not tr.xpath('./td[3]//i/text()').extract_first():
                item['tag_pic'] = '没有热度标签'
            else:
                item['tag_pic'] = tr.xpath('./td[3]//i/text()').extract_first()
            # print(item)
            title_search = item['title']
            # print(item)
            yield scrapy.Request(
                url=self.detail_url.format(title_search),
                meta={'item':item},
                headers=self.headers,   # 非常关键！一定要referer是微博的来访！
                cookies=self.Cookie,
                callback=self.search_detail
            )
    def search_detail(self,response):
        item = response.meta['item']

        if response.xpath('//div[@class="total"]/span[1]/text()').extract_first() is None:
            item['watch'] = '阅读量爆炸'
        else:
            item['watch'] = response.xpath('//div[@class="total"]/span[1]/text()').extract_first()
        # print(item)
        if response.xpath('//div[@class="total"]/span[2]/text()').extract_first() is None:
            item['talk'] = '讨论量爆炸'
        else:
            item['talk'] = response.xpath('//div[@class="total"]/span[2]/text()').extract_first()
        # print(item)
        # 获取具体消息 会有置顶 当事方 热门 这里读取

        page_row = response.xpath('//div[@class="content"]')
        # print(page_row)
        for detail in page_row:
            item['weibo_detail'] = detail.xpath('//p[@class="txt"]/text()').extract()
            item['bozhu'] = detail.xpath('//div[@class="content"]//a[@class="name"]/text()').extract()
        # print(item['bozhu'])
            # 置顶标签
            if detail.xpath('//div[@class="card-wrap"]//h4[@class="title"]/a/text()') is None:
                item['biaoqian'] = '普通微博哟'
            else:
                item['biaoqian'] = detail.xpath('//div[@class="card-wrap"]//h4[@class="title"]/a/text()').extract_first()
            # print(item['biaoqian'])

            item['time'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M')


        yield item

 

管道：

这里很粗糙，博文被js修饰了分隔没法一条一条，有待改进

这里先保存成csv 做数据分析，也可入库

import csv

class WeiboPipeline:
    # 可以通过spider 来判断 我这里是哪个爬虫  如果是哪个爬虫再执行 可用if 判断是从那个爬虫来的
    # 这样区分后可以分别存到 Mongodb 或者 Mysql里面
    # process_item 名不能改
    # 可以在这里 被传给这里后进行处理 用 content = [i.replace('替换',"") for i in content
    # 取空内容 content = [i.strip() for i in content]  去空格
    #        content = [i for i in content if len(i)>0]
    # 上面的处理内容 也可以：
    # content = [re.sub(r'\xa0|\s|\r\n', "", i) for i in content]
    # content = [i for i in content if len(i)>0]
    def __init__(self):

        self.file = open('resou.csv','w',encoding='utf-8-sig',newline='')
        self.writer = csv.writer(self.file)
        self.head = ['热搜名称','热度','讨论','阅读量','记录时间','热搜标签','排名']
        self.writer.writerow(self.head)

    def process_item(self, item, spider):
        # spider.settings.get('MYSQL_HOST')
        item['weibo_detail'] = self.parse_content(item['weibo_detail'])
        print(item)
        self.writer.writerow([item['title'],item['hot_totle'],item['talk'],item['watch'],item['time'],item['tag_pic'],item['rank']])
        return item

    def parse_content(self,content):
        content = [i.replace('\u200b',"") for i in content]
        content = [i.replace('\n',"") for i in content]
        content = [i.replace('【', "") for i in content]
        content = [i.replace('】', "") for i in content]
        content = [i.strip() for i in content]
        content = [i for i in content if len(i) > 0]
        # print(content)
        return content

    def close_spider(self, spider):
        self.file.close()

 

ps：！！可以使用进程  循环调取！ emm 发现问题 咋这里 应该再主调用外面取写 csv 头，要不然重复 写入自己的 标签头了！！

 

 

 

这里以及可以完成定时关闭和永远循环执行，如果服务器想要执行爬虫文件最好还是执行这个文件。此处循环  定300s 

from multiprocessing import Process
from scrapy import cmdline
import time
import logging
import datetime
# 配置参数即可, 爬虫名称，运行频率
confs = [
    {
        "spider_name": "weibo-resou",
        "frequency": 120,
    },
]

def start_spider(spider_name, frequency,):
    args = ["scrapy", "crawl", spider_name]
    start_time = time.time()


    while True:

        start = time.time()
        p = Process(target=cmdline.execute, args=(args,))
        p.start()
        p.join()
        logging.debug("### use time: %s" % (time.time() - start))
        during_time = (time.time() - start_time)
        print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), "正在爬取微博热搜哟")
        if during_time > 300:
            print("爬取结束哟")
            break
        else:
            time.sleep(frequency)
            print(during_time)



if __name__ == '__main__':
    for conf in confs:

        process = Process(target=start_spider, args=(conf["spider_name"], conf["frequency"]))
        process.start()

        start_time = time.time()
        during_time = (time.time() - start_time)

        if during_time > 300:
            process.terminate()
            break
        # time.sleep(86400)

 

 

 

 

停止时间：settings里面

CLOSESPIDER_TIMEOUT = 秒数

哈哈接下来就是处理数据咯