爬微博

# -*- coding:utf-8 -*-
import time
import json
import uuid
import traceback
import requests
import datetime
import os
import random
from utils.qiniu_util import QiNiu
from bs4 import BeautifulSoup
__version__ = '1.0.0.0'
"""
@brief  :   简介
@details:   详细信息
@author :   zhphuang
@date   :   2018-11-29
"""


class SpiderWeibo(object):

    def __init__(self, window=None):
        """
        "关注": 25, "美食": 33, "读书": 37, "设计": 39, "时尚": 41, "动漫": 43, "萌宠": 45,
        "综艺": 10, "电影": 12, "运动健身": 16, "旅游": 30, "星座": 36, "校园": 32, "艺术": 40
        """
        self.qiniu = QiNiu()
        self.url = [
            {"sub_type": "热门", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803&openApp=0"},
            {"sub_type": "新鲜事", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_7978_-_ctg1_7978&openApp=0"},
            {"sub_type": "搞笑", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_4388_-_ctg1_4388&openApp=0"},
            {"sub_type": "情感", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_1988_-_ctg1_1988&openApp=0"},
            {"sub_type": "明星", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_4288_-_ctg1_4288&openApp=0"},
            {"sub_type": "社会", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_4188_-_ctg1_4188&openApp=0"},
            {"sub_type": "数码", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_5088_-_ctg1_5088&openApp=0"},
            {"sub_type": "体育", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_1388_-_ctg1_1388&openApp=0"},
            {"sub_type": "汽车", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_5188_-_ctg1_5188&openApp=0"},
            {"sub_type": "电影", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_5188_-_ctg1_5188&openApp=0"},
            {"sub_type": "游戏", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_4888_-_ctg1_4888&openApp=0"},
            {"sub_type": "美食", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_2688_-_ctg1_2688&openApp=0"},
            {"sub_type": "读书", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_4588_-_ctg1_4588&openApp=0"},
            {"sub_type": "设计", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_5388_-_ctg1_5388&openApp=0"},
            {"sub_type": "时尚", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_4488_-_ctg1_4488&openApp=0"},
            {"sub_type": "动漫", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_2388_-_ctg1_2388&openApp=0"},
            {"sub_type": "萌宠", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_2788_-_ctg1_2788&openApp=0"},
            {"sub_type": "综艺", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_4688_-_ctg1_4688&openApp=0"},
            {"sub_type": "旅游", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_2588_-_ctg1_2588&openApp=0"},
            {"sub_type": "星座", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_1688_-_ctg1_1688&openApp=0"},
            {"sub_type": "校园",  "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_1488_-_ctg1_1488&openApp=0"},
            {"sub_type": "艺术", "url": "https://m.weibo.cn/api/container/getIndex?containerid=102803_ctg1_5488_-_ctg1_5488&openApp=0"}
        ]
        self.cookies = "_T_WM=6e170ca7910c8a0400cc34f8812ee08a; " \
                       "ALF=1548469366; SCF=AgozafmBO6saBbFys4DjAOQFlYFxRK6CuW_YwqYMgRKRkIh2Or_PEsE7BKZwQNDSRBoJu9EbU9DiOGNoTiRp7As.; " \
                       "SUB=_2A25xIEcnDeRhGeNM7FMW8ybEzT6IHXVS62lvrDV6PUNbktAKLRDWkW1NSetNTi5Tt63QYZkk2y4kc76CgKb4VoPf; " \
                       "SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9W5MoCM8JudP_s60h7KX4dpy5JpX5KMhUgL.Fo-ES02Ne0nRSoz2dJLoI7_VIPHVIPHoPN9DM5tt; " \
                       "SUHB=0z1wYnUQHMLUAy; SSOLoginState=1545877367; MLOGIN=1; WEIBOCN_FROM=1110006030; " \
                       "M_WEIBOCN_PARAMS=luicode%3D10000011%26lfid%3D102803_ctg1_3288_-_ctg1_3288%26uicode%3D20000174%26fid%3D102803"
        self.records = None
        self.agents = [
            "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
            "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
            "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
            "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
            "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
            "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
            "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
            "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
            "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
            "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
            "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
            "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
            "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
            "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
            "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
            "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
            "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
            "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
            "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
        ]
        self.window = window

    def log(self, msg):
        if self.window:
            self.window.write_log_to_text(msg)

    def save_to_csv(self, key):
        dir = os.path.join(os.path.dirname(__file__), "data", "weibo", "%s" % datetime.date.today())
        if not os.path.exists(dir):
            os.mkdir(dir)
        with open('data/weibo/%s/result_%s_%s.json' % (datetime.date.today(), key, datetime.date.today()), "w", encoding='utf-8') as json_file:
            json.dump(self.records, json_file, ensure_ascii=False)

    def get_random_header(self, path):
        cookies = "_T_WM=6e170ca7910c8a0400cc34f8812ee08a; " \
                  "ALF=1548469366; SCF=AgozafmBO6saBbFys4DjAOQFlYFxRK6CuW_YwqYMgRKRkIh2Or_PEsE7BKZwQNDSRBoJu9EbU9DiOGNoTiRp7As.; " \
                  "SUB=_2A25xIEcnDeRhGeNM7FMW8ybEzT6IHXVS62lvrDV6PUNbktAKLRDWkW1NSetNTi5Tt63QYZkk2y4kc76CgKb4VoPf; " \
                  "SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9W5MoCM8JudP_s60h7KX4dpy5JpX5KMhUgL.Fo-ES02Ne0nRSoz2dJLoI7_VIPHVIPHoPN9DM5tt; " \
                  "SUHB=0z1wYnUQHMLUAy; SSOLoginState=1545877367; MLOGIN=1; WEIBOCN_FROM=1110006030; " \
                  "M_WEIBOCN_PARAMS=luicode%3D10000011%26lfid%3D102803_ctg1_3288_-_ctg1_3288%26uicode%3D20000174%26fid%3D102803"
        return {
            "authority": "m.weibo.cn",
            "method": "GET",
            "path": path,
            "scheme": "https",
            "accept": "application/json, text/plain, */*",
            "accept-encoding": "gzip, deflate, br",
            "accept-language": "zh-CN,zh;q=0.9",
            "cookie": cookies,
            "mweibo-pwa": "1",
            "referer": "https://m.weibo.cn/",
            "x-requested-with": "XMLHttpRequest",
            "user-agent": random.choice(self.agents)
        }

    def get_comments(self, mid):
        comment_list = []
        commments_url = "https://m.weibo.cn/api/comments/show?id=%s&page=%s" % (mid, 1)
        header = {
            'User-agent': random.choice(self.agents),
            'Host': 'm.weibo.cn',
            'Accept': 'application/json, text/plain, */*',
            'Accept-Language': 'zh-CN,zh;q=0.8',
            'Accept-Encoding': 'gzip, deflate, sdch, br',
            'Referer': "https://m.weibo.cn/u/%s" % mid,
            'Connection': 'keep-alive',
            "cookie": self.cookies,

        }
        try:
            time.sleep(random.randint(1, 3))
            res = requests.get(commments_url, headers=header)
            if res.json().get("ok") == 1:
                data = res.json().get("data").get("data")
                for item in data:
                    try:
                        comment_content = BeautifulSoup(item["text"], "html.parser")
                        [s.extract() for s in comment_content('a')]  # 去掉<a>标签
                        comment_content = comment_content.text.replace("回复:", "")
                        if comment_content.strip():
                            comment_time = int(time.time()) + random.randint(0, 3600 * 24)
                            comment_list.append({"nickname": item["user"]["screen_name"],
                                                 "publish_time": comment_time,
                                                 "content": comment_content})
                    except Exception as e:
                        print(traceback.format_exc())
                        continue

        except Exception as e:
            print(traceback.format_exc())
        return comment_list

    def get_data(self):
        for url in self.url:
            self.records = []
            header = self.get_random_header(url.get("url").replace("https://m.weibo.cn", ""))
            time.sleep(random.randint(5, 20))
            res = requests.get(url.get("url"), headers=header)
            res = res.json()
            if res.get("ok") == 1:
                datas = res.get("data").get("cards")
                for data in datas:
                    if not data.__contains__('mblog'):
                        continue
                    try:
                        record = dict()
                        record['_type'] = 0  # 代表微博
                        record['sub_type'] = url["sub_type"]  # 分类
                        record['uid'] = data["mblog"]["id"]
                        record['username'] = data["mblog"]["user"]["screen_name"]
                        record["signature"] = data["mblog"]["user"]["description"]
                        record['user_photo'] = data["mblog"]["user"]["avatar_hd"]
                        record['user_photo'] = self.qiniu.fetch(record['user_photo'], record['uid'] + "_avatar.jpg")
                        record['sex'] = 1 if data["mblog"]["user"]["gender"] == "m" else 0
                        record["publish_time"] = int(time.time())
                        bs = BeautifulSoup(data["mblog"]["text"], "html.parser")
                        [s.extract() for s in bs('a')]  # 去掉<a>标签
                        record["content"] = bs.text
                        record["face_list"] = []
                        record["src_list"] = [image["url"] for image in data["mblog"].get("pics", [])]
                        record["src_list"] = [self.qiniu.fetch(i, str(uuid.uuid1()) + "weibo_image_%s.jpg" % index)
                                              for index, i in enumerate(record["src_list"])]
                        span_list = bs.select("span.surl-text")
                        record['label_list'] = [label_div.text for label_div in span_list]
                        record["zhuanfa_count"] = data["mblog"]["reposts_count"]
                        record["comment_count"] = data["mblog"]["comments_count"]
                        record["dianzan_count"] = data["mblog"]["attitudes_count"]
                        record["comment_list"] = self.get_comments(record['uid'])
                        print(len(record["comment_list"]))
                        record['link'] = data["scheme"]
                        record["video_url"] = {}
                        if data["mblog"].__contains__("page_info"):
                            if data["mblog"]["page_info"].__contains__("media_info") and data["mblog"]["page_info"]["type"] == "video":
                                page_pic = data["mblog"]["page_info"]["page_pic"]
                                video_url = data["mblog"]["page_info"]["media_info"].get("mp4_sd_url")
                                if not video_url:
                                    video_url = data["mblog"]["page_info"]["media_info"].get("mp4_hd_url")
                                if video_url:
                                    video_url = self.qiniu.fetch(video_url, str(uuid.uuid1()) + "_weibo_video.mp4")
                                    page_pic = self.qiniu.fetch(page_pic["url"], str(uuid.uuid1()) + "_weibo_pre_video_image.jpg")
                                    record["video_url"] = {"pre_image": page_pic, "video_url": video_url}
                        print(record)
                        self.records.append(record)
                    except Exception as e:
                        print(traceback.format_exc())
            self.save_to_csv(url["sub_type"])


if __name__ == '__main__':
    obj = SpiderWeibo()
    obj.get_data()