selenium
基本操作
from selenium import webdriver
from time import sleep
#实例化一个浏览器对象
bro = webdriver.Chrome(executable_path=r'C:\pycahrm文件\chromedriver.exe')
url = 'https://www.jd.com/'
#用户发起请求
bro.get(url)
#定位标签
search_input = bro.find_element_by_id('key')
#对指定标签进行数据交互
search_input.send_keys('macPro')
btn = bro.find_element_by_xpath('//*[@id="search"]/div/div[2]/button')
btn.click()
sleep(2)
#执行js代码
jsCode = 'window.scrollTo(0,document.body.scrollHeight)'
bro.execute_script(jsCode)
sleep(3)
bro.quit()
selenium
- 概念：基于浏览器自动化的一个模块。
- 环境的安装：
- pip install selenium
- selenium和爬虫之间的关联：
- 模拟登录
- 便捷的捕获到动态加载的数据（重点）
获取的页码源码数据 : page_source
- 特点：可见及可得
- 缺点：效率低
- selenium的具体使用
- 准备浏览器的驱动程序：http://chromedriver.storage.googleapis.com/index.html
- 动作链：ActionChains，一系列的行为动作
- 使用流程：
- 实例化一个动作连对象，需要将指定的浏览器和动作连对象进行绑定
- 执行相关的连续的动作
- perform()立即执行动作连制定好的动作
滑动操作
方式一:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver import ActionChains #动作链
import time
bro = webdriver.Chrome(executable_path=r'C:\pycahrm文件\chromedriver.exe')
bro.get('http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable')
bro.implicitly_wait(3)
bro.switch_to.frame('iframeResult') #切换到frame标签下
frame_tag = bro.find_element_by_id('draggable') #如果不切换到iframe下,就找不到该标签
begin_tag = bro.find_element_by_id('draggable') #滑动块的起始位置
end_tag = bro.find_element_by_id('droppable')#滑动块的终止位置
actions = ActionChains(bro) #拿到动作链对象
actions.drag_and_drop(begin_tag,end_tag) #把动作放到动作链中，准备串行执行
actions.perform() #开始执行
time.sleep(2)
bro.quit()
方式二:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver import ActionChains
import time
bro = webdriver.Chrome(executable_path=r'C:\pycahrm文件\chromedriver.exe')
bro.get('http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable')
bro.implicitly_wait(3)
bro.switch_to.frame('iframeResult')
frame_tag = bro.find_element_by_id('draggable')
begin_tag = bro.find_element_by_id('draggable')
end_tag = bro.find_element_by_id('droppable')
ActionChains(bro).click_and_hold(begin_tag).perform() #起始位置的滑动块,点击并按住
distance = end_tag.location['x']-begin_tag.location['x']
#滑动的距离
trsck = 0
while trsck < distance: ActionChains(bro).move_by_offset(xoffset=50,yoffset=0).perform() #xoffset横向滑动距离为50像素
trsck += 50
ActionChains(bro).release().perform() #滑动结束,释放滑动块
time.sleep(2)
bro.quit()
基于selenium模拟登陆12306
import requests
from hashlib import md5
class Chaojiying_Client(object):
def __init__(self, username, password, soft_id):
self.username = username
password = password.encode('utf8')
self.password = md5(password).hexdigest()
self.soft_id = soft_id
self.base_params = {
'user': self.username,
'pass2': self.password,
'softid': self.soft_id,
}
self.headers = {
'Connection': 'Keep-Alive',
'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)',
}
def PostPic(self, im, codetype):
"""
im: 图片字节
codetype: 题目类型 参考 http://www.chaojiying.com/price.html
"""
params = {
'codetype': codetype,
}
params.update(self.base_params)
files = {'userfile': ('ccc.jpg', im)}
r = requests.post('http://upload.chaojiying.net/Upload/Processing.php', data=params, files=files, headers=self.headers)
return r.json()
def ReportError(self, im_id):
"""
im_id:报错题目的图片ID
"""
params = {
'id': im_id,
}
params.update(self.base_params)
r = requests.post('http://upload.chaojiying.net/Upload/ReportError.php', data=params, headers=self.headers)
return r.json()
from selenium import webdriver
from selenium.webdriver import ActionChains
from time import sleep
from PIL import Image #安装PIL或者是Pillow
from CJY import Chaojiying_Client
#封装一个识别验证码的函数
def transformCode(imgPath,imgType):
chaojiying = Chaojiying_Client('username', 'password', '902590')
im = open(imgPath, 'rb').read()
return chaojiying.PostPic(im, imgType)['pic_str']
bro = webdriver.Chrome(executable_path=r'C:\pycahrm文件\chromedriver.exe')
bro.get('https://kyfw.12306.cn/otn/login/init')
sleep(2)
#将当前浏览器页面进行图片保存
bro.save_screenshot('./main.png')
#将验证码的局部区域进行裁剪
#捕获标签在页面中的位置信息
img_tag = bro.find_element_by_xpath('//*[@id="loginForm"]/div/ul[2]/li[4]/div/div/div[3]/img')
location = img_tag.location#标签的起始位置坐标（左下角坐标）
size = img_tag.size#标签的尺寸
#裁剪范围对应的矩形区域
rangle = (int(location['x']),int(location['y']),int(location['x']+size['width']),int(location['y']+size['height']))
#使用Image工具进行指定区域的裁剪
i = Image.open('./main.png')
frame = i.crop(rangle)#crop就是根据指定的裁剪范围进行图片的截取
frame.save('code.png')
#调用打码平台进行验证码的识别
result = transformCode('./code.png',9004)
print(result) #x1,y1|x2,y2|x3,y3
#x1,y1|x2,y2|x3,y3 ==>[[x1,y1],[x2,y2],[x3,y3]]
all_list = []#[[x1,y1],[x2,y2],[x3,y3]]
if '|' in result:
list_1 = result.split('|')
count_1 = len(list_1)
for i in range(count_1):
xy_list = []
x = int(list_1[i].split(',')[0])
y = int(list_1[i].split(',')[1])
xy_list.append(x)
xy_list.append(y)
all_list.append(xy_list)
else:
x = int(result.split(',')[0])
y = int(result.split(',')[1])
xy_list = []
xy_list.append(x)
xy_list.append(y)
all_list.append(xy_list)
for point in all_list:
x = point[0]
y = point[1]
ActionChains(bro).move_to_element_with_offset(img_tag,x,y).click().perform()
sleep(1)
bro.find_element_by_id('username').send_keys('xxxxxx')
sleep(1)
bro.find_element_by_id('password').send_keys('xxxx')
sleep(1)
bro.find_element_by_id('loginSub').click()
sleep(10)
print(bro.page_source)
bro.quit()
selenium规避风险
规避检测
from selenium import webdriver
from selenium.webdriver import ChromeOptions
option = ChromeOptions()
option.add_experimental_option('excludeSwitches', ['enable-automation'])
bro = webdriver.Chrome(executable_path='./chromedriver.exe',options=option)
url = 'https://www.taobao.com/'
bro.get(url)
# 当用爬虫程序发起的请求时,在后台进行window.navigator.webdriver的js的注入,返回值为true
# 正常访问一个页面是注入js会返回一个undefind
无头浏览器
#无头浏览器
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from time import sleep
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
bro = webdriver.Chrome(executable_path='./chromedriver.exe',chrome_options=chrome_options) #看不见页面,不显示可视化页面
url = 'https://www.taobao.com/'
bro.get(url)
sleep(2)
bro.save_screenshot('123.png')
print(bro.page_source)
