腾讯微博用户关注与听众的爬取
按广度的方式爬取用户的关注和听众，腾讯微博已经停运了，网上找的登入代码已经都过时了，自己分析不出来，就直接把cookie复制下了，这样就能获取要登录的内容了。
由于停运，只能获取40页的内容，文件格式为[source,target] 表示source 关注 target。由于从source爬取的话可以从关注里找到target，而从target爬取的话会从听众找到source，所以就需要写个去重了。
一小时大概能获取2万条消息。就一路写下去，没用线程。
辣鸡代码如下：
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
@auther: Starry
@file: Tencentweibo.py
@time: 2018/7/15 9:50
'''
import requests
from bs4 import BeautifulSoup
from queue import Queue
import time
import datetime
import json
import csv
import os
cookies = {
}
headers = {
"Accept": "*/*",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "zh-CN,zh;q=0.9",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"Host": "api.t.qq.com",
"Pragma": "no-cache",
"Referer": "http://api.t.qq.com/proxy.html",
"rf": "http://t.qq.com/anjianbin1979/following?t=1#u=anjianbin1979&t=1&st=1&p=2",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"
}
class TencentWeibo:
COUNT = 0
def __init__(self, start_name, start_title):
self.start_name = start_name
self.start_titile = start_title
self.que = Queue()
self.nameToId = {}
self.current_num = 1
self.visName = []
self.unique = {}
self.init_exe()
def init_exe(self):
if not os.path.exists('information.csv'):
self.csv_information = csv.writer(open('information.csv','a',newline='',encoding='utf-8'),dialect='excel')
self.csv_information.writerow(['id','user','name'])
self.csv_information.writerow([1,self.start_name,self.start_titile])
self.que.put(self.start_name)
self.nameToId[self.start_name] = self.current_num
self.unique[self.nameToId[self.start_name]] = []
else:
with open('information.csv','r',encoding='utf-8') as f:
csvFile = csv.reader(f,dialect='excel')
for index, item in enumerate(csvFile):
if index == 0:continue
self.que.put(item[1])
self.nameToId[item[1]] = int(item[0])
self.current_num = int(item[0])
self.csv_information = csv.writer(open('information.csv', 'a', newline='', encoding='utf-8'),
dialect='excel')
if not os.path.exists('data.csv'):
self.csv_data = csv.writer(open('data.csv', 'a', newline='', encoding='utf-8'), dialect='excel')
self.csv_data.writerow(['Source', 'Target'])
else:
FLAG = 0
with open('data.csv', 'r', encoding='utf-8') as f:
csvFile = csv.reader(f, dialect='excel')
for index, item in enumerate(csvFile):
if index==0:continue
id1, id2 = int(item[0]),int(item[1])
if id1 not in self.unique.keys():
self.unique[id1] = []
if id2 not in self.unique.keys():
self.unique[id2] = []
self.unique[id1].append(id2)
FLAG = min(id1,id2)
while not self.que.empty():
name = self.que.get()
id = self.nameToId[name]
if id == FLAG:
break
else:
self.visName.append(name)
self.csv_data = csv.writer(open('data.csv', 'a', newline='', encoding='utf-8'), dialect='excel')
print('开始爬取啦！！！')
def DealHtml(self,html, Flag, name):
soup = BeautifulSoup(html, 'html.parser')
li = soup.find_all('div', attrs={"class": "userName"})
for chlid in li:
try:
id = chlid.find('a').get('href')[1:]
title = chlid.find('a').string
if id not in self.nameToId.keys():
self.current_num += 1
self.nameToId[id] = self.current_num
self.que.put(id)
self.csv_information.writerow([self.current_num, id, title])
if self.nameToId[id] not in self.unique.keys():
self.unique[self.nameToId[id]] = []
# if self.COUNT == 1000:
# print('已经爬取了%s条消息了'%self.current_num)
# self.COUNT = 0
# self.COUNT += 1
if Flag == 1:
# print("关注",id,title)
if self.nameToId[id] not in self.unique[self.nameToId[name]]:
self.unique[self.nameToId[name]].append(self.nameToId[id])
# print([self.nameToId[name],self.nameToId[id]])
self.csv_data.writerow([self.nameToId[name], self.nameToId[id]])
elif Flag == 2:
# print("粉丝",id, title)
if self.nameToId[name] not in self.unique[self.nameToId[id]]:
self.unique[self.nameToId[id]].append(self.nameToId[name])
# print([self.nameToId[id], self.nameToId[name]])
self.csv_data.writerow([self.nameToId[id], self.nameToId[name]])
except Exception as e:
print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),e)
def getFans(self, name):
for i in range(1, 41):
try:
ctime = str(int(time.time() * 1000))
url = "http://api.t.qq.com/relations/follow_apollo.php?u={0}&t=2&st=1&p={1}&apiType=14&apiHost=http://api.t.qq.com&_r={2}&g_tk=325301840".format(
name, str(i), ctime)
ret = requests.get(url=url, headers=headers, cookies=cookies,timeout=10)
ret_json = json.loads(ret.text)
if "info" in ret_json.keys():
self.DealHtml(ret_json['info'], 2, name)
else:
break
except Exception as e:
print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),e)
def getIdol(self, name):
for i in range(1, 41):
try:
ctime = str(int(time.time() * 1000))
url = "http://api.t.qq.com/relations/follow_apollo.php?u={0}&t=1&st=1&p={1}&apiType=14&apiHost=http://api.t.qq.com&_r={2}&g_tk=325301840".format(
name, str(i), ctime)
ret = requests.get(url=url, headers=headers, cookies=cookies,timeout=10)
ret_json = json.loads(ret.text)
if "info" in ret_json.keys():
self.DealHtml(ret_json['info'], 1, name)
else:
break
except Exception as e:
print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),e)
def start(self):
while not self.que.empty():
visiter = self.que.get()
if visiter not in self.visName:
self.visName.append(visiter)
self.getIdol(visiter)
self.getFans(visiter)
class TencentWeiboArticles:
def __init__(self):
self.que = Queue()
self.IdToInformation = {}
# self.
def start(self):
pass
weibo = TencentWeibo('xie_na','谢娜')
weibo.start()
