直播平台源代码，Scrapy抓取滚动加载的页面

编写spider

打开qqmovie.py<br>import scrapy<br>from movies.items import MoviesItem<br> <br>class QqmovieSpider(scrapy.Spider):<br>    name = 'qqmovie'<br>    allowed_domains = ['v.qq.com']<br>    # 抓取开始的URL<br>    start_urls = ['https://v.qq.com/channel/movie?listpage=1&channel=movie&itype=100004']<br>    # 偏移标志位<br>    offset = 0<br>    <br>    def parse(self, response):<br>        # 通过class=list_item的div标签提取数据<br>        movies = response.xpath('//div[@class="list_item"]')<br>        for i in movies:<br>            item = MoviesItem()<br>            # 提取<a>标签的title作为名称<br>            item['name'] = i.xpath('./a/@title').get()<br>            # 提取<div>标签的title作为主演<br>            item['starring'] = i.xpath('./div/div/@title').get()<br>            yield item<br>        <br>        mod_page = response.xpath('//div[@class="mod_pages"]')<br>        # 按下一页按钮的正常class提取，提取不到说明翻页到头了<br>        next_page = mod_page.xpath('//button[@class="page_next "]').extract_first('not found')<br>        if next_page != 'not found':<br>            # 原有网页的逻辑是一页30条数据<br>            self.offset += 30<br>            url = 'https://v.qq.com/x/bu/pagesheet/list?append=1&channel=movie&itype=100004&listpage=2&offset={}&pagesize=30'.format(str(self.offset))<br>            yield scrapy.Request(url=url, callback=self.parse)<br>        else:<br>            print('数据抓取完毕')