6.5
完成python作业
【题目描述】豆瓣图书评论数据爬取。以《平凡的世界》、《都挺好》等为分析对象，编写程序爬取豆瓣读书上针对该图书的短评信息，要求：
（1）对前3页短评信息进行跨页连续爬取；
（2）爬取的数据包含用户名、短评内容、评论时间、评分和点赞数（有用数）；
（3）能够根据选择的排序方式（热门或最新）进行爬取，并分别针对热门和最新排序，输出前10位短评信息（包括用户名、短评内容、评论时间、评分和点赞数）。
（4）根据点赞数的多少，按照从多到少的顺序将排名前10位的短评信息输出；
（5附加）结合中文分词和词云生成，对前3页的短评内容进行文本分析：按照词语出现的次数从高到低排序，输出前10位排序结果；并生成一个属于自己的词云图形。
【练习要求】请给出源代码程序和运行测试结果，源代码程序要求添加必要的注释。
import re
from collections import Counter
import requests
from lxml import etree
import pandas as pd
import jieba
import matplotlib.pyplot as plt
from wordcloud import WordCloud
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36 Edg/101.0.1210.39"
}
comments_list = []
words_list = []
def clean_text(text):
text = re.sub(r"[\n”“|,，；;''/?! 。的了是]", "", text)
text = re.sub(r"^\d+::", "", text)
text = re.sub(
r"(https?://)?([a-zA-Z0-9]+)(\.[a-zA-Z0-9]+)(\.[a-zA-Z0-9]+)*(/[a-zA-Z0-9]+)*", "", text, re.IGNORECASE)
text = re.sub(u"年|月|日|周一|周二|周三|周四|周五|周六", "", text)
text = re.sub(r"[^a-zA-Z]\d+", "", text)
text = re.sub(r"\s+", "", text)
return text
def extract_comments(url):
resp = requests.get(url, headers=headers).text
html = etree.HTML(resp)
comment_list = html.xpath(".//div[@class='comment']")
for comment in comment_list:
username = comment.xpath(".//span[@class='comment-info']/a/text()")[0]
content = comment.xpath(".//p[@class='comment-content']/span[@class='short']/text()")[0].strip()
words_list.extend([clean_text(word) for word in jieba.cut(content, cut_all=False, HMM=False) if len(clean_text(word)) >= 2])
time = comment.xpath(".//span[@class='comment-info']/a/text()")[1]
mark = comment.xpath(".//span[@class='comment-info']/span/@title")
score = 0 if len(mark) == 0 else 5 if mark[0] == "力荐" else 4 if mark[0] == "推荐" else 3 if mark[0] == "还行" else 2 if mark[0] == "较差" else 1
likes = int(comment.xpath(".//span[@class='comment-vote']/span[@class='vote-count']/text()")[0])
comments_list.append([username, content, time, score, likes])
def generate_wordcloud(words):
text = ' '.join(words)
wordcloud = WordCloud(background_color='white', width=1000, height=700, font_path='simhei.ttf', margin=10).generate(text)
plt.imshow(wordcloud)
plt.axis("off")
plt.show()
wordcloud.to_file('wordcloud.png')
print("请选择以下选项:")
print(" 1.热门评论")
print(" 2.最新评论")
selected_option = int(input())
if selected_option == 1 or selected_option == 2:
comments_list.clear()
words_list.clear()
for i in range(0, 60, 20):
sort_type = "new_score" if selected_option == 1 else "time"
url = f"https://book.douban.com/subject/10517238/comments/?start={i}&limit=20&status=P&sort={sort_type}"
extract_comments(url)
columns = ['用户名', '短评内容', '评论时间', '评分', '点赞数']
df = pd.DataFrame(comments_list, columns=columns)
print("前10位短评信息：")
print(df.head(10))
print("点赞数前10位的短评信息：")
df_sorted = df.sort_values(by='点赞数', ascending=False)
print(df_sorted.head(10))
generate_wordcloud(words_list)
else:
print("无效选项，请重新运行程序并选择1或2。")
