(十)selenium实现微博高级搜索信息爬取
1.selenium模拟登陆
2.定位进入高级搜索页面
3.对高级搜索进行定位，设置。
4.代码实现
import time from selenium import webdriver from lxml import etree from selenium.webdriver import ChromeOptions import requests from PIL import Image from hashlib import md5 from selenium.webdriver.support.select import Select headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36', } # 超级鹰 class Chaojiying_Client(object): """超级鹰源代码""" def __init__(self, username, password, soft_id): self.username = username password = password.encode('utf8') self.password = md5(password).hexdigest() self.soft_id = soft_id self.base_params = { 'user': self.username, 'pass2': self.password, 'softid': self.soft_id, } self.headers = { 'Connection': 'Keep-Alive', 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)', } def PostPic(self, im, codetype): """ im: 图片字节 codetype: 题目类型 参考 http://www.chaojiying.com/price.html """ params = { 'codetype': codetype, } params.update(self.base_params) files = {'userfile': ('ccc.jpg', im)} r = requests.post('http://upload.chaojiying.net/Upload/Processing.php', data=params, files=files, headers=self.headers) return r.json() def ReportError(self, im_id): """ im_id:报错题目的图片ID """ params = { 'id': im_id, } params.update(self.base_params) r = requests.post('http://upload.chaojiying.net/Upload/ReportError.php', data=params, headers=self.headers) return r.json() # 输入用户名 密码 option = ChromeOptions() option.add_experimental_option('excludeSwitches', ['enable-automation']) url = 'https://www.weibo.com/' bro = webdriver.Chrome(executable_path='./chromedriver.exe',options=option) bro.maximize_window() bro.get(url=url) time.sleep(10) # 视网速而定 bro.find_element_by_id('loginname').send_keys('你的账号') time.sleep(2) bro.find_element_by_css_selector(".info_list.password input[node-type='password']").send_keys( "你的密码") time.sleep(1) #识别验证码 def recognize(bro): bro.save_screenshot('weibo.png') pic = bro.find_element_by_xpath('//div[@class="login_innerwrap"]/div[3]/div[3]/a/img') location = pic.location size = pic.size rangle = (location['x'] * 1.25, location['y'] * 1.25, (location['x'] + size['width']) * 1.25, (location['y'] + size['height']) * 1.25) i = Image.open('./weibo.png') code_img_name = 'code.png' # 裁剪文件的文件名称 frame = i.crop(rangle) # 根据指定区域进行裁剪 frame.save(code_img_name) chaojiying = Chaojiying_Client('超级鹰账号', '密码', ' 905993') im = open('./code.png', 'rb').read() result = chaojiying.PostPic(im, 3005)['pic_str'] return result # 输入验证码 # 微博第一次点击登陆可能不成功 确保成功登陆 for i in range(5): try: bro.find_element_by_xpath('//div[@class="login_innerwrap"]/div[3]/div[3]/div/input').send_keys(recognize(bro)) bro.find_element_by_xpath('//div[@class="login_innerwrap"]/div[3]/div[6]/a').click() except Exception: continue time.sleep(5) # 进入高级搜索页面 bro.find_element_by_xpath('//div[@class="gn_header clearfix"]/div[2]/a').click() bro.find_element_by_xpath('//div[@class="m-search"]/div[3]/a').click() # 填入关键词 key_word = bro.find_element_by_xpath('//div[@class="m-layer"]/div[2]/div/div[1]/dl//input') key_word.clear() key_word.send_keys('雾霾') # 填入地点 province = bro.find_element_by_xpath('//div[@class="m-adv-search"]/div/dl[5]//select[1]') city = bro.find_element_by_xpath('//div[@class="m-adv-search"]/div/dl[5]//select[2]') Select(province).select_by_visible_text('陕西') Select(city).select_by_visible_text('西安') # 填入时间 # 起始 bro.find_element_by_xpath('//div[@class="m-adv-search"]/div[1]/dl[4]//input[1]').click() # 点击input输入框 sec_1 = bro.find_element_by_xpath('//div[@class="m-caldr"]/div/select[1]') Select(sec_1).select_by_visible_text('一月') sec_2 = bro.find_element_by_xpath('//div[@class="m-caldr"]/div/select[2]') Select(sec_2).select_by_visible_text('2019') bro.find_element_by_xpath('//div[@class="m-caldr"]/ul[2]/li[3]').click() # 起始日期 # 终止 bro.find_element_by_xpath('//div[@class="m-adv-search"]/div[1]/dl[4]//input[2]').click() # 点击input输入框 sec_1 = bro.find_element_by_xpath('//div[@class="m-caldr"]/div/select[1]') Select(sec_1).select_by_visible_text('一月') #月份 sec_2 = bro.find_element_by_xpath('//div[@class="m-caldr"]/div/select[2]') Select(sec_2).select_by_visible_text('2019') # 年份 bro.find_element_by_xpath('//div[@class="m-caldr"]/ul[2]/li[6]').click() # 日期 sec_3 = bro.find_element_by_xpath('//div[@class="m-adv-search"]/div[1]/dl[4]//select[2]') # 点击input输入框 Select(sec_3).select_by_visible_text('8时') # 小时 bro.find_element_by_xpath('//div[@class="btn-box"]/a[1]').click() # 爬取用户ID 发帖内容 时间 客户端 评论数 转发量 点赞数 并持久化存储 page_num = 1 with open('2019.txt','a',encoding='utf-8') as f: while page_num<=50: page_text = bro.page_source tree = etree.HTML(page_text) div_list = tree.xpath('//div[@id="pl_feedlist_index"]/div[2]/div') for div in div_list: try: user_id = div.xpath('./div/div[1]/div[2]/div[1]/div[2]/a[1]/text()')[0] # 用户id flag = div.xpath('./div/div[1]/div[2]/p[3]') if not flag: content = div.xpath('./div/div[1]/div[2]/p[1]')[0].xpath('string(.)') # 内容 time = div.xpath('./div/div[1]/div[2]/p[2]/a/text()')[0] # 发布时间 client = div.xpath('./div/div[1]/div[2]/p[2]/a[2]/text()')[0] else: content = div.xpath('./div/div[1]/div[2]/p[2]')[0].xpath('string(.)') # 内容 time = div.xpath('./div/div[1]/div[2]/p[3]/a/text()')[0] # 发布时间 client = div.xpath('./div/div[1]/div[2]/p[3]/a[2]/text()')[0] #客户端 up=div.xpath('./div/div[2]/ul/li[4]/a')[0].xpath('string(.)') #点赞数 transfer = div.xpath('./div/div[2]/ul/li[2]/a/text()')[0] # 转发量 comment = div.xpath('./div/div[2]/ul/li[3]/a/text()')[0] # 评论数 f.write('\n'+user_id+'\n'+content+'\n'+time+client+' '+transfer+' '+comment+' '+'赞'+up+'\n'+'\n') except IndexError: continue if page_num ==1: # 第一页 元素位置不同 进行判断 bro.find_element_by_xpath('//div[@class="m-page"]/div/a').click() else: bro.find_element_by_xpath('//div[@class="m-page"]/div/a[@class="next"]').click() page_num+=1
