scrapy+selenium爬取网易云音乐评论

废话不多说，先卡主代码

import scrapy
from selenium.webdriver import Chrome
from selenium.webdriver.common.by import By
import time
from CodeNav.items import CodenavItem


class CodeSpider(scrapy.Spider):
    name = 'code'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['https://music.163.com/#/playlist?id=2329680016']
    bro = Chrome()

    # bro.page_source

    def parse(self, response, **kwargs):
        div_list = response.xpath('/html/body/div[3]/div[1]/div/div/div[3]/div/div[2]/div[2]/div')
        for div in div_list:
            title = div.xpath('./div[2]/div[1]/div/a/text()').extract_first()
            cnt = div.xpath('./div[2]/div[1]/div/text()').extract_first()
            item = CodenavItem()
            item['title'] = title
            item['cnt'] = cnt

            # print(title + '   ' + cnt)
            yield item

    def close(self, spider):
        self.bro.quit()

 

```
再看中间件的代码

```

```

# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals

# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
from scrapy.http import HtmlResponse
from selenium.webdriver.common.by import By
from time import sleep


class CodenavDownloaderMiddleware:

    def process_request(self, request, spider):
        web = spider.bro
        web.get(request.url)
        sleep(2)
        web.execute_script('window.scrollTo(0,document.body.scrollHeight)')
        sleep(2)
        web.switch_to.frame('g_iframe')  # 评论嵌套在了iframe里，括号里是iframe的id
        sleep(2)
        page_count = web.page_source
        # print(page_count)
        return HtmlResponse(url=request.url, body=page_count, encoding='utf-8', request=request)
        # return None

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

***Note***:网易云音乐的评论都嵌套在一个iframe中，需要switch to到对应的iframe框架中，改好以后可以打印出来看下
剩下的就是scrapy的基本操作了，这里我只为了练习selenium在scrapy的应用，所以只是爬了第一页评论，后续页的评论要做适当修改