Python爬虫：微博热搜榜前200条评论

整体思路：

　1、从热搜榜获取详情页的链接

 

 　　

2、排除掉广告以及置顶 广告热力值为空 置定为第一个tr

 

 

3、点击加载更多评论页面会跳转

 

 

 

 

4、跳转页面后抓包获得评论所在真实地址

 

 

5、观察真实地址所需要的参数 从详情页获取 完成构造

 

 

6、对构造地址进行请求将count=20改为count=200 可以显示二百条

7、使用re对数据进行提取并保存

8、仅供学习使用

import re
import csv
import requests
from lxml import etree
 
url = 'https://s.weibo.com/top/summary?sudaref=www.baidu.com'
headers = {
    '你的': 'cookie',
    'referer': 'https://login.sina.com.cn/',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
 
}
with open('微博热搜评论.csv', 'a', encoding='utf-8-sig', newline='') as f:
    csv_writer = csv.writer(f)
    csv_writer.writerow(['标题', '评论名称', '所在地区', '评论内容'])
 
resp = requests.get(url=url, headers=headers)
tree = etree.HTML(resp.text)
# 第一个不在热搜范围内
tr_list = tree.xpath('//*[@id="pl_top_realtimehot"]/table/tbody/tr')[1:]
# print(tr_list)
for tr in tr_list:
    # 判断是不是广告 广告后面没有热力值 如果为空就为广告
    if tr.xpath('./td[2]/span') != '':
        href = tr.xpath('./td[2]/a/@href')[0]
        # void 我也不知道是啥 反正不是正常热搜链接 直接排除掉
        if 'void' not in href:
            # 在此或得到详情页链接
            title = tr.xpath('./td[2]/a/text()')[0]
            wanzheng = 'https://s.weibo.com/'+href
            xiangqing_resp = requests.get(url=wanzheng, headers=headers).text
            uid_obj = re.compile(r'uid=(?P<uid>.*?)&')
            uid = uid_obj.search(xiangqing_resp).group('uid')
            # print(uid)
            mid_obj = re.compile(r'mid="(?P<mid>.*?)" >')
            mid = mid_obj.search(xiangqing_resp).group('mid')
            # print(mid)
            pinglun_200url = f'https://weibo.com/ajax/statuses/buildComments?is_reload=1&id={mid}&is_show_bulletin=2&is_mix=0&count=200&uid={uid}'
            # print(pinglun_200url)
            neirong = requests.get(url=pinglun_200url)
            neirong_obj = re.compile(r'"source":"(.*?)",.*?"screen_name":"(.*?)","profile_image_url":.*?"text_raw":"(.*?)"}', re.S)
            tiqu = neirong_obj.findall(neirong.text)
            for i in tiqu:
                diqu = i[0]
                name = i[1]
                neirong = i[2]
                with open('微博热搜评论.csv', 'a', encoding='utf-8-sig', newline='') as f:
                    csv_writer = csv.writer(f)
                    csv_writer.writerow([title,name, diqu, neirong])
                print( title+'保存完成')