scrapy+splash 爬取京东动态商品

作业来源:https://edu.cnblogs.com/campus/gzcc/GZCC-16SE1/homework/3159

splash是容器安装的，从docker官网上下载windows下的docker进行安装。

下载完成之后直接点击安装，安装成功后，桌边会出现三个图标：

点击 Docker QuickStart 图标来启动 Docker Toolbox 终端。

使用docker启动服务命令启动Splash服务

docker run -p 8050:8050 scrapinghub/splash

这里我已经开启服务了

打开cmd，在当前目录下开始scrapy爬虫:
scrapy startproject scrapy_examples
在spider文件夹中新建python文件jd_book.py用于编写爬虫
在项目下新建pybook.py用于对数据文件csv处理

京东上的商品是动态加载的，爬取python书籍的前20页，获取每个商品的评论数、书名、简介。

SplashRequest(url,endpoint='execute',args={'lua_source':lua_script},cache_args=['lua_source'])请求页面并执行JS函数渲染页面
endpoint='execute':在页面中执行一些用户自定义的JavaScript代码
args={'lua_source':lua_script}:用户自定义的lua脚本
cache_args=['lua_source']:让Splash服务器缓存该函数
用户自定义的lua脚本中必须包含一个main函数作为程序入口，main函数被调用时会传入一个splash对象（lua中的对象），用户可以调用该对象上的方法操纵Splash。
splash.args属性：用户传入参数的表，通过该属性可以访问用户传入的参数
splash:go方法：类似于在浏览器中打开某url地址的页面，页面所需资源会被加载，并进行JavaScript渲染
splash:wait方法：等待页面渲染，time参数为等待的秒数
splash:runjs方法：在当前页面下，执行一段JavaScript代码
splash:html方法：splash:html()获取当前页面的HTML文本。
middlewares.py随机产生User-Agent添加到每个请求头中
pipelines.py处理爬取的数据并存入数据库
settings.py配置splash服务信息、设置请求延迟反爬虫、添加数据库信息

# -*- coding:utf-8 -*-
import scrapy
from scrapy import Request
from scrapy_splash import SplashRequest
from splash_examples.items import PyBooksItem

lua_script ='''
function main(splash)
    splash:go(splash.args.url)
    splash:wait(2)
    splash:runjs("document.getElementsByClassName('pn-next')[0].scrollIntoView(true)")
    splash:wait(2)
    return splash.html()
end
'''
class JDBookSpider(scrapy.Spider):
    name = "jd_book"
    allowed_domains = ['search.jd.com']
    base_url = 'https://search.jd.com/Search?keyword=python&enc=utf-8&wq=python'
    def start_requests(self):
        yield Request(self.base_url,callback=self.parse_urls,dont_filter=True)
    def parse_urls(self,response):
        for i in range(20):
            url = '%s&page=%s' % (self.base_url,2*i+1)
            yield SplashRequest(url,
                                endpoint='execute',
                                args={'lua_source':lua_script},
                                cache_args=['lua_source'])
    def parse(self, response):
        for sel in response.css('ul.gl-warp.clearfix>li.gl-item'):
            pyjdbooks = PyBooksItem()
            pyjdbooks['name'] = sel.css('div.p-name').xpath('string(.//em)').extract_first()
            pyjdbooks['comment']=sel.css('div.p-commit').xpath('string(.//a)').extract_first()
            pyjdbooks['promo_words']=sel.css('div.p-name').xpath('string(.//i)').extract_first()
            yield pyjdbooks

import scrapy

class PyBooksItem(scrapy.Item):
    name=scrapy.Field()
    comment=scrapy.Field()
    promo_words=scrapy.Field()

from fake_useragent import UserAgent
# 随机的User-Agent
class RandomUserAgent(object):
    def process_request(self, request, spider):
        request.headers.setdefault("User-Agent", UserAgent().random)

class SplashExamplesPipeline(object):
    def __init__(self):
        self.book_set = set()

    def process_item(self, item, spider):
        if not(item['promo_words']):
            item['promo_words'] = item['name']
        comment = item['comment']
        if comment[-2:] == "万+":
            item['comment'] = str(int(float(comment[:-2])*10000))
        elif comment[-1] == '+':
            item['comment'] = comment[:-1]
        return item


import pymysql

class MysqlPipeline(object):
    def __init__(self, host, database, user, password, port):
        self.host = host
        self.database = database
        self.user = user
        self.password = password
        self.port = port

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            host=crawler.settings.get('MYSQL_HOST'),
            database=crawler.settings.get('MYSQL_DATABASE'),
            user=crawler.settings.get('MYSQL_USER'),
            password=crawler.settings.get('MYSQL_PASSWORD'),
            port=crawler.settings.get('MYSQL_PORT'),
        )

    def open_spider(self, spider):
        self.db = pymysql.connect(self.host, self.user, self.password, self.database, charset='utf8', port=self.port)
        self.cursor = self.db.cursor()

    def close_spider(self, spider):
        self.db.close()

    def process_item(self, item, spider):
        data = dict(item)
        keys = ', '.join(data.keys())
        values = ', '.join(['%s'] * len(data))
        sql = 'insert into books (%s) values (%s)' % (keys, values)
        self.cursor.execute(sql, tuple(data.values()))
        self.db.commit()
        return item

BOT_NAME = 'splash_examples'

SPIDER_MODULES = ['splash_examples.spiders']
NEWSPIDER_MODULE = 'splash_examples.spiders'

#Splash服务器地址
SPLASH_URL = 'http://192.168.99.100:8050'

#开启Splash的两个下载中间件并调整HttpCompressionMiddleware的次序
DOWNLOADER_MIDDLEWARES = {
    'splash_examples.middlewares.RandomUserAgent':345,
    'scrapy_splash.SplashCookiesMiddleware': 723,
    'scrapy_splash.SplashMiddleware': 725,
    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
}
#设置去重过滤器
DUPEFILTER_CLASS='scrapy_splash.SplashAwareDupeFilter'

#用来支持cache_args
SPIDER_MIDDLEWARES ={
    'scrapy_splash.SplashDeduplicateArgsMiddleware':100,
}
# 使用Splash的Http缓存
HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

COOKIES_ENABLED = False
DOWNLOAD_DELAY = 3

ITEM_PIPELINES = {
    'splash_examples.pipelines.SplashExamplesPipeline':400,
    'splash_examples.pipelines.MysqlPipeline':543,
}

MYSQL_HOST = 'localhost'
MYSQL_DATABASE = 'pybooks'
MYSQL_PORT = 3306
MYSQL_USER = 'root'
MYSQL_PASSWORD = 'root'

在Terminal中执行爬虫:scrapy crawl jd_book -o pybooks.csv
将数据存储到数据库并生成csv文件用于分析可视化

在数据库中查看有1187条信息

做数据分析可视化

import pandas as pd
import jieba
from wordcloud import WordCloud
import matplotlib.pyplot as plt
obj = pd.read_csv('pybooks.csv')
Books = obj.sort_values('comment',ascending=False)[:200]
promoWords = []
for promo in Books['promo_words']:
    promoWords.append(promo)
promoWordsStr = ''.join(promoWords)
bookTxt = jieba.lcut(promoWordsStr)
stopwords = ['学习','入门','掌握','教程','图书','使用','全面','推荐','读者','专家']
bookTxt = [token for token in bookTxt if token not in stopwords]
bookTxtSet = set(bookTxt)
txtCount = {}
for i in bookTxtSet:
    if len(i) == 1:
        continue
    txtCount[i] = bookTxt.count(i)
txtCount = sorted(txtCount.items(),key=lambda key:key[1],reverse=True)
TxtStr = ' '.join(bookTxt)
ciyun = WordCloud(background_color = '#122',width=400,height=300,margin = 1).generate(TxtStr)
plt.imshow(ciyun)
plt.axis("off")
plt.show()
print(txtCount)