2023年5月5日(软件工程日报)
python数据处理
（一）、中国大学排名数据分析与可视化；（写到实验报告中）
【源代码程序】
import requests
from bs4 import BeautifulSoup as bs
import pandas as pd
from matplotlib import pyplot as plt
def get_rank(url):
count = 0
rank = []
headers = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36 Edg/101.0.1210.3"
}
resp = requests.get(url, headers=headers).content.decode()
soup = bs(resp, "lxml")
univname = soup.find_all('a', class_="name-cn")
for i in univname:
if count != 10:
university = i.text.replace(" ", "")
score = soup.select("#content-box > div.rk-table-box > table > tbody > tr:nth-child({}) > td:nth-child(5)"
.format(count + 1))[0].text.strip()
rank.append([university, score])
else:
break
count += 1
return rank
total = []
u_year = 2015
for i in range(15, 20):
url = "https://www.shanghairanking.cn/rankings/bcur/20{}11".format(i)
print(url)
title = ['学校名称', '总分']
df = pd.DataFrame(get_rank(url), columns=title)
total.append(df)
for i in total:
plt.rcParams['font.sans-serif'] = ['SimHei'] # 用来正常显示中文标签
x = list(i["学校名称"])[::-1]
y = list(i["总分"])[::-1]
# 1.创建画布
plt.figure(figsize=(20, 8), dpi=100)
# 2.绘制图像
plt.plot(x, y, label="大学排名")
# 2.2 添加网格显示
plt.grid(True, linestyle="--", alpha=0.5)
# 2.3 添加描述信息
plt.xlabel("大学名称")
plt.ylabel("总分")
plt.title(str(u_year) + "年软科中国最好大学排名Top10", fontsize=20)
# 2.5 添加图例
plt.legend(loc="best")
# 3.图像显示
plt.savefig(str(u_year)+".png")
plt.show()
u_year += 1
while True:
info = input("请输入要查询的大学名称和年份：")
count = 0
university, year = info.split()
year = int(year)
judge = 2019 - year
tmp = total[::-1]
if 4 >= judge >= 0:
name = list(total[judge - 1]["学校名称"])
for j in name:
if university == j:
print(university + "在{0}年排名第{1}".format(year, count + 1))
break
count += 1
if count ==10:
print("很抱歉，没有该学校的排名记录！！！")
print("请选择以下选项:")
print(" 1.继续查询")
print(" 2.结束查询")
select = int(input(""))
if select == 1:
continue
elif select == 2:
break
else:
break
else:
print("很抱歉，没有该年份的排名记录！！！")
print("请选择以下选项:")
print(" 1.继续查询")
print(" 2.结束查询")
select = int(input(""))
if select == 1:
continue
elif select == 2:
break
import re
from collections import Counter
import requests
from lxml import etree
import pandas as pd
import jieba
import matplotlib.pyplot as plt
from wordcloud import WordCloud
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36 Edg/101.0.1210.39"
}
comments = []
words = []
def regex_change(line):
# 前缀的正则
username_regex = re.compile(r"^\d+::")
# URL，为了防止对中文的过滤，所以使用[a-zA-Z0-9]而不是\w
url_regex = re.compile(r"""
(https?://)?
([a-zA-Z0-9]+)
(\.[a-zA-Z0-9]+)
(\.[a-zA-Z0-9]+)*
(/[a-zA-Z0-9]+)*
""", re.VERBOSE | re.IGNORECASE)
# 剔除日期
data_regex = re.compile(u""" #utf-8编码
年 |
月 |
日 |
(周一) |
(周二) |
(周三) |
(周四) |
(周五) |
(周六)
""", re.VERBOSE)
# 剔除所有数字
decimal_regex = re.compile(r"[^a-zA-Z]\d+")
# 剔除空格
space_regex = re.compile(r"\s+")
regEx = "[\n”“|,，；;''/?! 。的了是]" # 去除字符串中的换行符、中文冒号、|，需要去除什么字符就在里面写什么字符
line = re.sub(regEx, "", line)
line = username_regex.sub(r"", line)
line = url_regex.sub(r"", line)
line = data_regex.sub(r"", line)
line = decimal_regex.sub(r"", line)
line = space_regex.sub(r"", line)
return line
def getComments(url):
score = 0
resp = requests.get(url, headers=headers).text
html = etree.HTML(resp)
comment_list = html.xpath(".//div[@class='comment']")
for comment in comment_list:
status = ""
name = comment.xpath(".//span[@class='comment-info']/a/text()")[0] # 用户名
content = comment.xpath(".//p[@class='comment-content']/span[@class='short']/text()")[0] # 短评内容
content = str(content).strip()
word = jieba.cut(content, cut_all=False, HMM=False)
time = comment.xpath(".//span[@class='comment-info']/a/text()")[1] # 评论时间
mark = comment.xpath(".//span[@class='comment-info']/span/@title") # 评分
if len(mark) == 0:
score = 0
else:
for i in mark:
status = str(i)
if status == "力荐":
score = 5
elif status == "推荐":
score = 4
elif status == "还行":
score = 3
elif status == "较差":
score = 2
elif status == "很差":
score = 1
good = comment.xpath(".//span[@class='comment-vote']/span[@class='vote-count']/text()")[0] # 点赞数（有用数）
comments.append([str(name), content, str(time), score, int(good)])
for i in word:
if len(regex_change(i)) >= 2:
words.append(regex_change(i))
def getWordCloud(words):
# 生成词云
all_words = []
all_words += [word for word in words]
dict_words = dict(Counter(all_words))
bow_words = sorted(dict_words.items(), key=lambda d: d[1], reverse=True)
print("热词前10位：")
for i in range(10):
print(bow_words[i])
text = ' '.join(words)
w = WordCloud(background_color='white',
width=1000,
height=700,
font_path='simhei.ttf',
margin=10).generate(text)
plt.show()
plt.imshow(w)
w.to_file('wordcloud.png')
print("请选择以下选项:")
print(" 1.热门评论")
print(" 2.最新评论")
info = int(input())
print("前10位短评信息：")
title = ['用户名', '短评内容', '评论时间', '评分', '点赞数']
if info == 1:
comments = []
words = []
for i in range(0, 60, 20):
url = "https://book.douban.com/subject/10517238/comments/?start={}&limit=20&status=P&sort=new_score".format(
i) # 前3页短评信息（热门）
getComments(url)
df = pd.DataFrame(comments, columns=title)
print(df.head(10))
print("点赞数前10位的短评信息：")
df = df.sort_values(by='点赞数', ascending=False)
print(df.head(10))
getWordCloud(words)
elif info == 2:
comments = []
words=[]
for i in range(0, 60, 20):
url = "https://book.douban.com/subject/10517238/comments/?start={}&limit=20&status=P&sort=time".format(
i) # 前3页短评信息（最新）
getComments(url)
df = pd.DataFrame(comments, columns=title)
print(df.head(10))
print("点赞数前10位的短评信息：")
df = df.sort_values(by='点赞数', ascending=False)
print(df.head(10))
getWordCloud(words)
