CSIC_716_20191028【爬小破站】

 

1、爬取小破站的弹幕

2、展示爬取内容

 

打开网页，用教的方法找到cid 和header

import requests

from bs4 import BeautufulSoup

import pandas as pd

import re

header={

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'

}#小破站的header可以通过谷歌浏览器的network中的request header获取

url=' http://comment.破站.com/125507930.xml'  #以爬取小破站cid=125507930为例

response=request.get(headers=header,url=url) #获取响应

response.encoding=response.apparent_encoding

data=response.text

soup=BeautifulSoup(data,'lxml' )#解析

dlist=soup.find_all('d') #将内容按照规律向需要获取的内容筛选靠拢,存入dlist

d_list=[ ]

for i in d_list

danmu{ }

danmu['弹幕']=i.text

d_list.append(danmu)  #将取到的值放到d_list中

df=pd.dateframe(d_list)     #df应该还是列表，pd是pandas

f.open('sign.txt','encoding='utf-8')

for p in danmu['danmu'].value:

pat=re.compile(r'[一-龥]+')

filter_data=re.findall(pattern=pat,string=p )#筛选

f.write( " ".join(filter_data))  #保存数据进入sign文件

f.close()

------------------------------------------------------------------------------------------------------------------------------------------------------

利用上面获得的文件 sign.txt进行展示

import jieba

from imageio import imread

from wordcloud import WordCloud

f.open('sign.txt ',encoding='utf-8' )

data=f.read()

result=" ".join(jieba.lcut(data))

f.close( )

mask_color=imread('XXXX.jpg')

wc=WordCloud(

font_path='font的路径',

mask=mask_color,

width=xx,

height=xx,

)

wc.generate(result)

wc.to_file('xxxx.png')

plt.imshow( wc)

plt.show()

 

 

----------------------------------------------------以下为正式代码将两者合并--------------------------------------------------------------------------------------------------

 

如果不生成中间文件，爬完网页直接生成图片，代码合并，如下所示

#coding:utf-8
import requests
import jieba
import pandas as pd
import re
import matplotlib.pyplot as plt
from bs4 import BeautifulSoup
from imageio import imread
from wordcloud import WordCloud
header={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'
}
url='http://comment.bilibili.com/122593266.xml'  #换一下数字的cid即可
response = requests.get(url=url,headers=header)
response.encoding=response.apparent_encoding
data=response.text
suop=BeautifulSoup(data,'lxml')
list_filter=suop.find_all('d')
plist=[]
for i in list_filter:
    danmu={}
    danmu['弹幕']= i.text
    plist.append(danmu)
df=pd.DataFrame(plist)
reslist=[]
for p in df['弹幕'].values:
    txtfilter= re.compile(r'[一-龥]+')
    res=re.findall(pattern=txtfilter,string=p)
    result = " ".join(res)
    reslist.append(result)
result= " ".join(reslist)
finalResult=" ".join(jieba.lcut(result))
mask_color=imread('五角星.jpg')  #图片可以随意替换，放在project中即可
wc=WordCloud(
    font_path=r'C:\Windows\Fonts\STLITI.TTF',
    mask=mask_color,
    width=1920,
    height=1080,
    background_color='white'
)
wc.generate(finalResult)
wc.to_file('hunt.png')
plt.imshow(wc)
plt.show( )