selenium爬取qq空间,requests爬取雪球网数据
一、爬取qq空间好友动态数据
# 爬取qq空间好友状态信息(说说,好友名称),并屏蔽广告 from selenium import webdriver from time import sleep from lxml import etree # 自动操作浏览器 bro = webdriver.Chrome(executable_path=r'D:\爬虫+数据分析\tools\chromedriver.exe') bro.get('https://qzone.qq.com/') sleep(3) #注意:如果想要通过find系列函数去定位某一个iframe标签下的子标签的话,一定要使用如下操作: bro.switch_to.frame('login_frame')#参数表示的是iframe标签的id属性值,,ifram是子标签 bro.find_element_by_id('switcher_plogin').click() # 单击id为switcher_plogin的页面标签,即点击账号密码登录 sleep(3) # 等待数据加载 # 自动输入用户名,密码登录空间 bro.find_element_by_id('u').send_keys('1156145880') bro.find_element_by_id('p').send_keys('qq密码') sleep(3) bro.find_element_by_id('login_button').click() sleep(3) # 点击‘个人中心’,进到好友动态 bro.find_element_by_id('aIcenter').click() sleep(3) # 拖动滚轮,一次一屏;加载动态数据 bro.execute_script('window.scrollTo(0,document.body.scrollHeight)') sleep(3) bro.execute_script('window.scrollTo(0,document.body.scrollHeight)') sleep(3) bro.execute_script('window.scrollTo(0,document.body.scrollHeight)') sleep(3) # 获取浏览器当前的页面源码数据 page_text = bro.page_source # 数据解析 tree = etree.HTML(page_text) li_list = tree.xpath('//ul[@id="feed_friend_list"]/li') for li in li_list: user_name_list = li.xpath(".//div[@class='user-info']/div[@class='f-nick']/a/text()") text_list = li.xpath('.//div[@class="f-info"]/text()|.//div[@class="f-info qz_info_cut"]//text()') # 需要展开的说说类名不同 for tu in zip(user_name_list,text_list): text = '\n'.join(tu) print(text+'\n\n') bro.close() # 关闭浏览器
二、爬取雪球网的新闻的标题、作者、来源等
import requests import json headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.12 Safari/537.36', } url_index = 'https://xueqiu.com' url = 'https://xueqiu.com/v4/statuses/public_timeline_by_category.json?since_id=-1&max_id=-1&count=10&category=-1' # 创建一个session对象 session = requests.Session() #使用session进行请求的发送:获取cookie,且将cookie保存到session中 session.get(url_index,headers=headers) # 获取json响应数据 json_dic = session.get(url=url,headers=headers).json() for dic in json_dic["list"]: data = dic["data"] data_dic = json.loads(data) title = data_dic["title"] # description = data_dic["description"] column = dic["column"] author = data_dic["user"]["screen_name"] print(f"标题:{title}\n来源:{column}\n作者:{author}\n")