爬虫 - 异步统计博客

'''
对崔庆才的个人博客上的文章基本信息的爬取 (共41页）存入mongo
https://cuiqingcai.com/page/1
标题、链接、浏览的数目、评论的数目以及喜欢的人数
分别将浏览数、评论数以及喜欢数排前十的文章统计出来并绘制出图表。
'''
import logging
import re
import aiohttp
import asyncio
from lxml import etree
import pymongo
import pyecharts as pye

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

'''将所有的方法用协程的方式实现'''
info_list = []
db = pymongo.MongoClient(host='127.0.0.1',port=27017)
col = db['abc']['blogs']
# 获取网页内容
async def parse_url(url):
    # 异步http请求
    session = aiohttp.ClientSession()
    response = await session.get(url)
    text = await response.text()
    await session.close()
    return text

# 解析网页，得到所需要数据
async def parse_content(url):
    text = await parse_url(url)
    tree = etree.HTML(text)
    articles = tree.xpath("//div[@class='content']/article")

    for article in articles:
        title = article.xpath("./header/h2/a/text()")[0] if article.xpath("./header/h2/a/text()") else None
        author = article.xpath('./p/span[1]/a/text()')[0]
        publish_time = article.xpath("./p/span[2]/text()")[0]
        page_view = article.xpath("./p/span[3]/text()")[0]
        page_view = int(re.findall('\d+', page_view)[0])
        comment = article.xpath("./p/span[4]/a/text()")[0]
        comment = int(re.findall('\d+', comment)[0])
        data_dic = {'title': title, 'author': author, 'publish_time': publish_time,
                    'page_view': page_view, 'comment': comment}
        info_list.append(data_dic)

# 根据数据画出柱状图
def plot(title,name,index_list,value_list):
    index_list = index_list[::-1]
    value_list = value_list[::-1]
    bar = pye.Bar(title=title)
    bar.add('数量',index_list,value_list,is_convert=True,is_label_show=True,label_pos='right')
    bar.render('{}.html'.format(name))

# 分析数据，画图
def analy():
    data = [(i['title'],i['page_view'],i['comment']) for i in col.find()]
    # 按照page_view 进行排序
    list1 = sorted(data,key=lambda x:int(x[1]),reverse=True)
    # print(11111111,list1[:10])
    # 按照评论进行排序
    list2 = sorted(data,key=lambda x:int(x[2]),reverse=True)
    # print(2222222,list2[:10])
    #浏览量前十的文章，名字列表和浏览数列表
    plot('浏览量前十的文章','page_view',[i[0] for i in list1[:10]], [i[1] for i in list1[:10]])
    plot('评论量前十的文章','comment',[i[0] for i in list2[:10]],[i[1] for i in list2[:10]])


def main():
    urls = ['https://cuiqingcai.com/page/{}'.format(i) for i in range(1,41)]
    tasks = [asyncio.ensure_future(parse_content(url)) for url in urls]
    loop = asyncio.get_event_loop()
    loop.run_until_complete(asyncio.wait(tasks))
    col.insert(info_list)

    # 进行分析，画图
    analy()


if __name__ == '__main__':
    main()