微博h5端爬取
#!/usr/bin/env python
# encoding: utf-8
from requests_html import HTMLSession
from json import JSONDecodeError
from glom import glom
from retrying import retry
import datetime
import time
from lxml import html
from dbs.db import SaveData
from functools import reduce
from requests.packages.urllib3.exceptions import InsecureRequestWarning
import requests
import time
import traceback
from decorators.decorators import decorator
from logger.log import get_logger
from fake_useragent import UserAgent
from config.config import *
# 禁用安全请求警告
class WeiboCN():
def __init__(self,i,_loop=None):
self.uid = i.get("user_id")
self.client_id = i.get("client_id")
self.get_container_url = 'https://m.weibo.cn/api/container/getIndex?type=uid&value=' + self.uid
self.page_url=f"https://m.weibo.cn/api/container/getIndex?containerid={{}}&page={{}}"
self.headers={"User-Agent":UserAgent(use_cache_server=False).chrome}#可以禁用服务器缓存
self.session= HTMLSession()
self.wbinfodatas = {}
self.wb_contentdatas = {}
self.first_comment_dict={}
self.second_comment_dict = {}
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
# 同一个账号的log_date相同
self.LOG_DATE = datetime.datetime.now()
self.need_days=31
self.proxy = chrome_args.get("proxy")
self.proxies = {"http": self.proxy, "https": self.proxy}
@property
@retry(stop_max_attempt_number =3,stop_max_delay =10000,retry_on_exception=lambda x: isinstance(x, JSONDecodeError))
def get_containerid(self):
json=self.session.get(self.get_container_url,headers=self.headers,timeout=60,verify=False).json()
content = glom(json,'data')
for data in glom(content,'tabsInfo.tabs'):
if (data.get('tab_type') == 'weibo'):
containerid = data.get('containerid')
return containerid
@retry(stop_max_attempt_number=3, stop_max_delay=60000,wait_fixed=6000)
def get(self,url):
try:
response=self.session.get(url,headers=self.headers,timeout=60,verify=False)
if response.status_code == 200: # 判断请求码
return response
except ConnectionError:
print("出现错误了休眠1分钟")
raise
@decorator
def get_feed_info(self,result):
stop_flag=False
json=result.json()
Updatedate = datetime.datetime.now()
for i in glom(json, 'data.cards'):
url = glom(i, 'scheme')
created_time = self.get_format_datetime(glom(i, 'mblog.created_at'))
html_info = glom(i, 'mblog.text')
infostr = html_info
mid = glom(i, 'mblog.mid')
if infostr.strip().endswith("全文"):
longtextwithemojiurl =self.getlongtext(mid,self.session)
if longtextwithemojiurl:
infostr = longtextwithemojiurl
source = glom(i, 'mblog.source')
reposts_count = glom(i, 'mblog.reposts_count')
comments_count = glom(i, 'mblog.comments_count')
attitudes_count = glom(i, 'mblog.attitudes_count')
self.wbinfodatas.setdefault("uid", []).append(self.uid)
self.wbinfodatas.setdefault("read_num", []).append("0")
self.wbinfodatas.setdefault("share_num", []).append(str(reposts_count))
self.wbinfodatas.setdefault("comment_num", []).append(str(comments_count))
self.wbinfodatas.setdefault("like_num", []).append(str(attitudes_count))
self.wbinfodatas.setdefault("wb_id", []).append(mid)
# 重新构造数据插入sns_object,list作为键值
self.wb_contentdatas.setdefault("MEDIA_TYPE", []).append("WB")
self.wb_contentdatas.setdefault("OBJ_TYPE", []).append("FEED")
self.wb_contentdatas.setdefault("BIZ_ID", []).append("{}_{}".format(self.uid, mid))
self.wb_contentdatas.setdefault("STATUS", []).append("0")
self.wb_contentdatas.setdefault("SOURCE", []).append(source)
self.wb_contentdatas.setdefault("BIZ_PARENT_ID", []).append(self.uid)
self.wb_contentdatas.setdefault("BIZ_ID_PATH", []).append( "{}/{}_{}".format(self.uid, self.uid, mid))
self.wb_contentdatas.setdefault("BIZ_CREATE_TIME", []).append(created_time)
self.wb_contentdatas.setdefault("URL", []).append(url)
self.wb_contentdatas.setdefault("FORM_ID", []).append("")
self.wb_contentdatas.setdefault("FORM_NAME", []).append("")
self.wb_contentdatas.setdefault("CONTENT", []).append(infostr)
#清洗出文本内容
text_info =""
try:
text_info=self.getinfo_emoji(html.fromstring(infostr))
except:
get_logger().error(f"get_feed_info is error,here is detail {traceback.format_exc()}")
self.wb_contentdatas.setdefault("CONTENT_TEXT", []).append(text_info)
self.wb_contentdatas.setdefault("CLIENT_ID", []).append(self.client_id)
self.wb_contentdatas.setdefault("UPD_DATE", []).append(Updatedate)
#一条内容取完了以后,开始获取当前的评论内容
self.get_first_comment(mid)
last_date=self.wb_contentdatas.get("BIZ_CREATE_TIME")[-1]
if self.wb_send_times(last_date):
stop_flag=True
SaveData().save_content_data(self.wbinfodatas, self.wb_contentdatas, self.LOG_DATE)
self.wbinfodatas.clear()
self.wb_contentdatas.clear()
return stop_flag
@decorator
def wb_send_times(self,last_date):
before_day_datetime = datetime.datetime.fromtimestamp(time.time()) - datetime.timedelta(days=self.need_days)
wb_datetime = last_date
# 默认31天之前的不取
if (before_day_datetime > wb_datetime):
return True
else:
return False
@decorator
def get_format_datetime(self,datestr):
now = datetime.datetime.now()
ymd = now.strftime("%Y-%m-%d")
y = now.strftime("%Y")
newstr = datestr
newdate = now
if u"楼" in datestr:
newstr = datestr.split(u"楼")[-1].strip()
if (u"今天" in newstr):
mdate = time.mktime(time.strptime(ymd + newstr, '%Y-%m-%d今天 %H:%M'))
newdate = datetime.datetime.fromtimestamp(mdate)
elif (u"月" in newstr):
mdate = time.mktime(time.strptime(y + newstr, '%Y%m月%d日 %H:%M'))
newdate = datetime.datetime.fromtimestamp(mdate)
elif (u"分钟前" in newstr):
newdate = now - datetime.timedelta(minutes=int(newstr[:-3]))
elif (u"秒前" in newstr):
newdate = now - datetime.timedelta(minutes=int(newstr[:-2]))
#以下为手机端
elif "-" in newstr :
try:
mdate = time.mktime(time.strptime(y+"-"+ newstr, '%Y-%m-%d'))
newdate = datetime.datetime.fromtimestamp(mdate)
except:
mdate = time.mktime(time.strptime(newstr, '%Y-%m-%d'))
newdate = datetime.datetime.fromtimestamp(mdate)
elif (u"昨天" in newstr):
mdate = time.mktime(time.strptime(ymd + newstr, '%Y-%m-%d昨天 %H:%M'))
newdate = datetime.datetime.fromtimestamp(mdate)-datetime.timedelta(days=1)
elif (u"小时前" in newstr):
newdate = now - datetime.timedelta(hours=int(newstr[:-3]))
else:
newdate = datetime.datetime.strptime(newstr, "%Y-%m-%d %H:%M")
return newdate
@decorator
def getinfo_emoji(self, node):
try:
info = node.xpath(".//*")
infolist= [[i.attrib.get("alt") ,i.text,i.tail] if i.tag=="a" else [i.attrib.get("alt"),i.tail ,i.text] for i in info]
newstr1 = ""
if infolist:
newstr1 = reduce(self.listlisttostr, infolist)
if isinstance(newstr1, list):
newstr1 = "".join([i.replace(" ", "").replace("\u200b", "") for i in newstr1 if i not in ['', ' ', None]])
newstr2 = node.text.strip().replace(" ", "").replace("\u200b", "")
newstr = newstr2 + newstr1
return newstr
except:
return None
@decorator
def listlisttostr(self, x, y):
xstr = "".join([i.replace(" ", "").replace("\u200b", "") for i in x if i not in ['', ' ', None]])
ystr = "".join([j.replace(" ", "").replace("\u200b", "") for j in y if j not in ['', ' ', None]])
return xstr + ystr
@decorator
def run(self):
id=self.get_containerid
page_index=1
stop_flag=False
while not stop_flag:
url=self.page_url.format(id, page_index)
response=self.get(url)
stop_flag =self.get_feed_info(response) #存储内容,返回是否一个月
page_index+=1
time.sleep(0.5)
print(self.uid,"取完了")
@decorator
def getlongtext(self,mid:str,req:requests)->"获取全文逻辑":
allinfo=None
getlongtexturl=f"https://m.weibo.cn/statuses/extend?id={mid}"
resp=self.get(getlongtexturl).json()
okcode=resp["ok"]
if okcode==1:
allinfo=resp["data"]["longTextContent"]
else:
get_logger().error("get log text is error,here are detail:{}",traceback.format_exc())
return allinfo
@decorator
def get_first_comment(self,mid):
'''
#获取一级评论
:return:
'''
max_id=0
try:
url=f"https://m.weibo.cn/comments/hotflow?id={mid}&mid={mid}&max_id_type=0"
json=self.get(url).json()
isok = glom(json, 'ok')
if isok:
max_id = glom(json, 'data.max_id')
self.save_first_comment(json,mid,url)
#开始获取第二页评论
index=0
while max_id:
next_url=f"https://m.weibo.cn/comments/hotflow?id={mid}&mid={mid}&max_id={max_id}&max_id_type=0"
json = self.get(next_url).json()
isok = glom(json, 'ok')
if isok:
max_id = glom(json, 'data.max_id')
self.save_first_comment(json, mid, next_url)
time.sleep(0.5)
except:
get_logger().error(f"get_first_comment is error,here is detail {traceback.format_exc()}")
@decorator
def save_first_comment(self,json,mid,url):
Updatedate=datetime.datetime.now()
for item in glom(json, 'data.data'):
create_date = glom(item, 'created_at')
create_time=datetime.datetime.strptime(create_date, "%a %b %d %H:%M:%S +0800 %Y")
first_cid = glom(item, 'id')
text = glom(item, 'text')
user_name = glom(item, 'user.screen_name')
user_id = glom(item, 'user.id')
total_number =glom(item, 'total_number')
self.first_comment_dict.setdefault("MEDIA_TYPE", []).append("WB")
self.first_comment_dict.setdefault("OBJ_TYPE", []).append("COMMENT")
self.first_comment_dict.setdefault("BIZ_ID", []).append("{}_{}".format(mid, first_cid))
self.first_comment_dict.setdefault("STATUS", []).append("0")
self.first_comment_dict.setdefault("SOURCE", []).append("")
self.first_comment_dict.setdefault("BIZ_PARENT_ID", []).append("{}_{}".format(self.uid, mid))
self.first_comment_dict.setdefault("BIZ_ID_PATH", []).append(
"{}/{}_{}/{}_{}".format(self.uid, self.uid, mid, mid, first_cid))
self.first_comment_dict.setdefault("BIZ_CREATE_TIME", []).append(create_time)
self.first_comment_dict.setdefault("FORM_ID", []).append(user_id)
self.first_comment_dict.setdefault("FORM_NAME", []).append(user_name)
self.first_comment_dict.setdefault("URL", []).append(url)
self.first_comment_dict.setdefault("CONTENT", []).append(text)
self.first_comment_dict.setdefault("CONTENT_TEXT", []).append(text)
self.first_comment_dict.setdefault("CLIENT_ID", []).append(self.client_id)
self.first_comment_dict.setdefault("UPD_DATE", []).append(Updatedate)
#开始获取二级评论
#如果total小于3则可以直接获取
if total_number:
if total_number<3:
self.save_second_comment(item, mid,first_cid, url,key= 'comments')
else:
self.get_second_comment(mid,first_cid)
if self.first_comment_dict:
SaveData().save_object_data(self.first_comment_dict)
self.first_comment_dict.clear()
# print("一级评论",cDatetime,UserName_nodes[i].strip(),get_wb_url)
@decorator
def get_second_comment(self,mid,first_cid):
'''
#获取二级评论
:return:
'''
url = f"https://m.weibo.cn/comments/hotFlowChild?cid={first_cid}&max_id=0&max_id_type=0"
json = self.get(url).json()
isok = glom(json, 'ok')
if isok:
max_id = glom(json, 'max_id')
self.save_second_comment(json, mid, first_cid, url)
# 开始获取第二页评论
index = 0
while max_id:
next_url = f"https://m.weibo.cn/comments/hotFlowChild?cid={first_cid}&max_id={max_id}&max_id_type=0"
json = self.get(next_url).json()
isok = glom(json, 'ok')
if isok:
max_id = glom(json, 'max_id')
print("json",json)
self.save_second_comment(json, mid,first_cid, next_url)
time.sleep(0.5)
@decorator
def save_second_comment(self, json, mid,first_cid, url,key='data'):
Updatedate = datetime.datetime.now()
if not glom(json,key):
return None
for item in glom(json,key):
create_date = glom(item, 'created_at')
create_time = datetime.datetime.strptime(create_date, "%a %b %d %H:%M:%S +0800 %Y")
second_cid = glom(item, 'id')
text = glom(item, 'text')
user_name = glom(item, 'user.screen_name')
user_id = glom(item, 'user.id')
self.second_comment_dict.setdefault("MEDIA_TYPE", []).append("WB")
self.second_comment_dict.setdefault("OBJ_TYPE", []).append("COMMENT")
self.second_comment_dict.setdefault("BIZ_ID", []).append("{}_{}".format(first_cid,second_cid ))
self.second_comment_dict.setdefault("STATUS", []).append("0")
self.second_comment_dict.setdefault("SOURCE", []).append("")
self.second_comment_dict.setdefault("BIZ_PARENT_ID", []).append("{}_{}".format(mid, first_cid))
self.second_comment_dict.setdefault("BIZ_ID_PATH", []).append(
"{}/{}_{}/{}_{}/{}_{}".format(self.uid, self.uid, mid, mid, first_cid, first_cid,
second_cid))
self.second_comment_dict.setdefault("BIZ_CREATE_TIME", []).append(create_time)
self.second_comment_dict.setdefault("FORM_ID", []).append(user_id)
self.second_comment_dict.setdefault("FORM_NAME", []).append(user_name)
self.second_comment_dict.setdefault("CONTENT", []).append(text)
self.second_comment_dict.setdefault("CONTENT_TEXT", []).append(text)
self.second_comment_dict.setdefault("URL", []).append(url)
self.second_comment_dict.setdefault("CLIENT_ID", []).append(self.client_id)
self.second_comment_dict.setdefault("UPD_DATE", []).append(Updatedate)
if self.second_comment_dict:
SaveData().save_object_data(self.second_comment_dict)
self.second_comment_dict.clear()
本文作者:一起来学python
本文链接:https://www.cnblogs.com/c-x-a/p/9343626.html
版权声明:本作品采用知识共享署名-非商业性使用-禁止演绎 2.5 中国大陆许可协议进行许可。