【Python爬虫】selenium爬虫入门大全
初识selenium
selenium最初是一个自动化测试工具,而爬虫中使用它主要是为了解决requests无法直接执行JavaScript代码的问题
selenium本质是通过驱动浏览器，完全模拟浏览器的操作，比如跳转、输入、点击、下拉等，来拿到网页渲染之后的结果，可支持多种浏览器
声明浏览器对象：
from selenium import webdriver browser = webdriver.Chrome() browser = webdriver.Firefox() browser = webdriver.Edge() browser = webdriver.PhantomJS() #无界面浏览器，目前已经停止维护后期使用者会越来越少，推荐使用chrome无头liu浏览器 browser = webdriver.Safari()
安装
#安装：selenium+chromedriver pip3 install selenium
（注意：selenium需要的驱动chromdriver.exe要与Chrome浏览器版本对应）
下载chromdriver.exe 国内镜像网站地址：http://npm.taobao.org/mirrors/chromedriver/2.29/ 最新的版本去官网找:https://sites.google.com/a/chromium.org/chromedriver/downloads#注意： selenium3默认支持的webdriver是Firfox，而Firefox需要安装geckodriver 下载链接：https://github.com/mozilla/geckodriver/releases
基本用法
from selenium import webdriver from time import sleep broswer = webdriver.Chrome(executable_path = r'C:\Users\Administrator\chromedriver.exe') broswer.get(url='https://www.baidu.com/') #获取网页，chromedriver.exe会打开浏览器模仿人的操作 sleep(2) text_input = broswer.find_element_by_id('kw') #通过id属性查找到标签 text_input.send_keys('人民币') #向id=kw的输入框中输入 sleep(2) broswer.find_element_by_id('su').click() #点击按钮 sleep(3) print(broswer.page_source) #获取当前的页面源码数据
broswer.quit() #浏览器退出
查找元素
单个元素
from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.taobao.com')
input_first = browser.find_element_by_id('q')
input_second = browser.find_element_by_css_selector('#q')
input_third = browser.find_element_by_xpath('//*[@id="q"]')
print(input_first, input_second, input_third)
browser.close()
<selenium.webdriver.remote.webelement.WebElement (session="5e53d9e1c8646e44c14c1c2880d424af", element="0.5649563096161541-1")> <selenium.webdriver.remote.webelement.WebElement (session="5e53d9e1c8646e44c14c1c2880d424af", element="0.5649563096161541-1")> <selenium.webdriver.remote.webelement.WebElement (session="5e53d9e1c8646e44c14c1c2880d424af", element="0.5649563096161541-1")>
selenium一些常用的命令释义：
browser.page_sourse #打印出页面源代码
browser.title #打印页面标题
browser.quit() #退出浏览器
browser.close() #关闭这个页面
browser.maximize_window() #将浏览器最大化显示
browser.set_window_size(480, 800) #设置浏览器宽480、高800显示
browser.back() #浏览器后退
browser.forward() #浏览器前进
webdriver提供了一系列的对象定位方法，常用的有以下几种
我们可以看到，一个百度的输入框，可以用这么用种方式去定位
browser.get("http://www.baidu.com") time.sleep(2)
#########百度输入框的定位方式###########通过id方式定位browser.find_element_by_id("kw").send_keys("selenium") #通过name方式定位 browser.find_element_by_name("wd").send_keys("selenium")
#通过tag name方式定位 browser.find_element_by_tag_name("input").send_keys("selenium")
#通过class name 方式定位 browser.find_element_by_class_name("s_ipt").send_keys("selenium")
#通过CSS方式定位 browser.find_element_by_css_selector("#kw").send_keys("selenium")
#通过xpatn方式定位 browser.find_element_by_xpath("//input[@id='kw']").send_keys("selenium")
#通过link 文字超链接定位 broswer.find_element_by_link_text("贴吧").click()
#通过部分文字超链接定位 broswer.find_element_by_partial_link_text("贴").click() browser.find_element_by_id("su").click()
time.sleep(3) browser.quit()
多个元素
查找class=service-bd 标签下的所有 li 标签
from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.taobao.com')
lis = browser.find_elements_by_css_selector('.service-bd li')
#lis = browser.find_elements(By.CSS_SELECTOR, '.service-bd li')
print(lis)
browser.close()
[<selenium.webdriver.remote.webelement.WebElement (session="c26290835d4457ebf7d96bfab3740d19", element="0.09221044033125603-1")>, <selenium.webdriver.remote.webelement.WebElement (session="c26290835d4457ebf7d96bfab3740d19", element="0.09221044033125603-2")>, <selenium.webdriver.remote.webelement.WebElement (session="c26290835d4457ebf7d96bfab3740d19", element="0.09221044033125603-3")>, <selenium.webdriver.remote.webelement.WebElement (session="c26290835d4457ebf7d96bfab3740d19", element="0.09221044033125603-4")>, <selenium.webdriver.remote.webelement.WebElement (session="c26290835d4457ebf7d96bfab3740d19", element="0.09221044033125603-5")>, <selenium.webdriver.remote.webelement.WebElement (session="c26290835d4457ebf7d96bfab3740d19", element="0.09221044033125603-6")>, <selenium.webdriver.remote.webelement.WebElement (session="c26290835d4457ebf7d96bfab3740d19", element="0.09221044033125603-7")>, <selenium.webdriver.remote.webelement.WebElement (session="c26290835d4457ebf7d96bfab3740d19", element="0.09221044033125603-8")>, <selenium.webdriver.remote.webelement.WebElement (session="c26290835d4457ebf7d96bfab3740d19", element="0.09221044033125603-9")>, <selenium.webdriver.remote.webelement.WebElement (session="c26290835d4457ebf7d96bfab3740d19", element="0.09221044033125603-10")>, <selenium.webdriver.remote.webelement.WebElement (session="c26290835d4457ebf7d96bfab3740d19", element="0.09221044033125603-11")>, <selenium.webdriver.remote.webelement.WebElement (session="c26290835d4457ebf7d96bfab3740d19", element="0.09221044033125603-12")>, <selenium.webdriver.remote.webelement.WebElement (session="c26290835d4457ebf7d96bfab3740d19", element="0.09221044033125603-13")>, <selenium.webdriver.remote.webelement.WebElement (session="c26290835d4457ebf7d96bfab3740d19", element="0.09221044033125603-14")>, <selenium.webdriver.remote.webelement.WebElement (session="c26290835d4457ebf7d96bfab3740d19", element="0.09221044033125603-15")>, <selenium.webdriver.remote.webelement.WebElement (session="c26290835d4457ebf7d96bfab3740d19", element="0.09221044033125603-16")>]
对元素进行操作
对获取的元素进行操作，如input标签进行输入、对按钮进行click点击
from selenium import webdriver import time browser = webdriver.Chrome() browser.get('https://www.taobao.com') input = browser.find_element_by_id('q') input.send_keys('iPhone') time.sleep(1) input.clear() input.send_keys('iPad') button = browser.find_element_by_class_name('btn-search') button.click()
更多操作: http://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.remote.webelement
交互动作
将动作附加到动作链中串行执行
from selenium import webdriver from selenium.webdriver import ActionChains browser = webdriver.Chrome() url = 'http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable' browser.get(url) browser.switch_to.frame('iframeResult') #切换到iframe标签里
source = browser.find_element_by_css_selector('#draggable') target = browser.find_element_by_css_selector('#droppable')
actions = ActionChains(browser) actions.drag_and_drop(source, target) actions.perform()
更多操作: http://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.common.action_chains
执行JavaScript脚本
一般用于拖动网页滑动条ScrollBar 从上到下拉动界面渲染，或重新刷新网页等操作
from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.zhihu.com/explore')
browser.execute_script('window.scrollTo(0, document.body.scrollHeight)') browser.execute_script('alert("To Bottom")')
一直拖动ScrollBar到网页底部
1 import time 2 3 def scroll_to_bottom(driver): 4 js = "return action=document.body.scrollHeight" 5 # 初始化现在滚动条所在高度为0 6 height = 0 7 # 当前窗口总高度 8 new_height = driver.execute_script(js) 9 10 while height < new_height: 11 # 将滚动条调整至页面底部 12 for i in range(height, new_height, 100): 13 driver.execute_script('window.scrollTo(0, {})'.format(i)) 14 time.sleep(0.5) 15 height = new_height 16 time.sleep(2) 17 new_height = driver.execute_script(js)
获取元素信息
获取属性
from selenium import webdriver from selenium.webdriver import ActionChains browser = webdriver.Chrome() url = 'https://www.zhihu.com/explore' browser.get(url) logo = browser.find_element_by_id('zh-top-link-logo')
print(logo) print(logo.get_attribute('class'))
<selenium.webdriver.remote.webelement.WebElement (session="5a9e00352fbb1bb3df8b81a9f666cba9", element="0.7456057855126614-1")> zu-top-link-logo
获取文本
from selenium import webdriver browser = webdriver.Chrome() url = 'https://www.zhihu.com/explore' browser.get(url) input = browser.find_element_by_class_name('zu-top-add-question') print(input.text) #提问
获取ID、位置、标签名、大小
from selenium import webdriver browser = webdriver.Chrome() url = 'https://www.zhihu.com/explore' browser.get(url) input = browser.find_element_by_class_name('zu-top-add-question') print(input.id) print(input.location) print(input.tag_name) print(input.size)
0.6822924344980397-1 {'y': 7, 'x': 774} button {'height': 32, 'width': 66}
Frame
import time from selenium import webdriver from selenium.common.exceptions import NoSuchElementException browser = webdriver.Chrome() url = 'http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable' browser.get(url)
browser.switch_to.frame('iframeResult') source = browser.find_element_by_css_selector('#draggable') print(source)
try: logo = browser.find_element_by_class_name('logo') except NoSuchElementException: print('NO LOGO')
browser.switch_to.parent_frame() logo = browser.find_element_by_class_name('logo') print(logo) print(logo.text)
<selenium.webdriver.remote.webelement.WebElement (session="4bb8ac03ced4ecbdefef03ffdc0e4ccd", element="0.44746093888932004-1")> NO LOGO <selenium.webdriver.remote.webelement.WebElement (session="4bb8ac03ced4ecbdefef03ffdc0e4ccd", element="0.13792611320464965-2")> RUNOOB.COM
等待
隐式等待
当使用了隐式等待执行测试的时候，如果 WebDriver没有在 DOM中找到元素，将继续等待，超出设定时间后则抛出找不到元素的异常, 换句话说，当查找元素或元素并没有立即出现的时候，隐式等待将等待一段时间再查找 DOM，默认的时间是0
from selenium import webdriver browser = webdriver.Chrome() browser.implicitly_wait(10) browser.get('https://www.zhihu.com/explore') input = browser.find_element_by_class_name('zu-top-add-question') print(input)
<selenium.webdriver.remote.webelement.WebElement (session="b29214772d59e912f1ac52e96ed29abe", element="0.12886805191194894-1")>
显示等待
在10s内等待网页加载，直到出现某个元素（如下：直到出现 id='q' 的元素出现）
from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC browser = webdriver.Chrome() browser.get('https://www.taobao.com/') wait = WebDriverWait(browser, 10) input = wait.until(EC.presence_of_element_located((By.ID, 'q'))) button = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, '.btn-search'))) print(input, button)
<selenium.webdriver.remote.webelement.WebElement (session="07dd2fbc2d5b1ce40e82b9754aba8fa8", element="0.5642646294074107-1")> <selenium.webdriver.remote.webelement.WebElement (session="07dd2fbc2d5b1ce40e82b9754aba8fa8", element="0.5642646294074107-2")>
- title_is 标题是某内容
- title_contains 标题包含某内容
- presence_of_element_located 元素加载出，传入定位元组，如(By.ID, 'p')
- visibility_of_element_located 元素可见，传入定位元组
- visibility_of 可见，传入元素对象
- presence_of_all_elements_located 所有元素加载出
- text_to_be_present_in_element 某个元素文本包含某文字
- text_to_be_present_in_element_value 某个元素值包含某文字
- frame_to_be_available_and_switch_to_it frame加载并切换
- invisibility_of_element_located 元素不可见
- element_to_be_clickable 元素可点击
- staleness_of 判断一个元素是否仍在DOM，可判断页面是否已经刷新
- element_to_be_selected 元素可选择，传元素对象
- element_located_to_be_selected 元素可选择，传入定位元组
- element_selection_state_to_be 传入元素对象以及状态，相等返回True，否则返回False
- element_located_selection_state_to_be 传入定位元组以及状态，相等返回True，否则返回False
- alert_is_present 是否出现Alert
详细内容：http://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.support.expected_conditions
前进后退
import time from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.baidu.com/') browser.get('https://www.taobao.com/') browser.get('https://www.python.org/') browser.back() time.sleep(1) browser.forward() browser.close()
Cookies
from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.zhihu.com/explore') print(browser.get_cookies()) browser.add_cookie({'name': 'name', 'domain': 'www.zhihu.com', 'value': 'germey'}) print(browser.get_cookies()) browser.delete_all_cookies() print(browser.get_cookies())
[{'secure': False, 'value': '"NGM0ZTM5NDAwMWEyNDQwNDk5ODlkZWY3OTkxY2I0NDY=|1491604091|236e34290a6f407bfbb517888849ea509ac366d0"', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': 'l_cap_id', 'expiry': 1494196091.403418}, {'secure': False, 'value': '"YWEyOGY4MmI1MzQ2NGY5MmFiMjgzZGUzZWJjYTgwYjY=|1491604091|ff946847ddb5881245bdb7a5e6401b70dc61013f"', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': 'cap_id', 'expiry': 1494196091.402855}, {'secure': False, 'value': '1', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': 'l_n_c'}, {'secure': False, 'value': '"MjcxMDE3YzU1YjI4NDljZjljNTQ4ZDIyOWJjZTBhNmY=|1491604091|8da4722b56a1545c2020dba97394a220c0eca8d9"', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': 'r_cap_id', 'expiry': 1494196091.402525}, {'secure': False, 'value': '51854390', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': '__utmc'}, {'secure': False, 'value': '"AADCo7e1kguPTqvEOMieRUzwkA7ZUBhV-VY=|1491604091"', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': 'd_c0', 'expiry': 1586212091.344773}, {'secure': False, 'value': '51854390.1491604091.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': '__utmz', 'expiry': 1507372091}, {'secure': False, 'value': '3cc99fc5-8706-43fc-90ac-3ad991bd1a25', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': '_zap', 'expiry': 1554676091}, {'secure': False, 'value': '97cb00128ccb46659728f7c69cc191b0|1491604091000|1491604091000', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': 'q_c1', 'expiry': 1586212091.401644}, {'secure': False, 'value': '51854390.2.10.1491604091', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': '__utmb', 'expiry': 1491605891}, {'secure': False, 'value': '1', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': 'n_c'}, {'secure': False, 'value': '51854390.000--|3=entry_date=20170408=1', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': '__utmv', 'expiry': 1554676091}, {'secure': False, 'value': '51854390.669300758.1491604091.1491604091.1491604091.1', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': '__utma', 'expiry': 1554676091}, {'secure': False, 'value': '1', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': '__utmt', 'expiry': 1491604691}, {'secure': False, 'value': 'AQAAALCZuwh+dgIAeu3PPHA+csDPnXvT', 'domain': 'www.zhihu.com', 'path': '/', 'httpOnly': True, 'name': 'aliyungf_tc'}]
[{'secure': False, 'value': 'germey', 'domain': '.www.zhihu.com', 'path': '/', 'httpOnly': False, 'name': 'name'}, {'secure': False, 'value': '"NGM0ZTM5NDAwMWEyNDQwNDk5ODlkZWY3OTkxY2I0NDY=|1491604091|236e34290a6f407bfbb517888849ea509ac366d0"', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': 'l_cap_id', 'expiry': 1494196091.403418}, {'secure': False, 'value': '"YWEyOGY4MmI1MzQ2NGY5MmFiMjgzZGUzZWJjYTgwYjY=|1491604091|ff946847ddb5881245bdb7a5e6401b70dc61013f"', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': 'cap_id', 'expiry': 1494196091.402855}, {'secure': False, 'value': '1', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': 'l_n_c'}, {'secure': False, 'value': '"MjcxMDE3YzU1YjI4NDljZjljNTQ4ZDIyOWJjZTBhNmY=|1491604091|8da4722b56a1545c2020dba97394a220c0eca8d9"', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': 'r_cap_id', 'expiry': 1494196091.402525}, {'secure': False, 'value': '51854390', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': '__utmc'}, {'secure': False, 'value': '"AADCo7e1kguPTqvEOMieRUzwkA7ZUBhV-VY=|1491604091"', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': 'd_c0', 'expiry': 1586212091.344773}, {'secure': False, 'value': '51854390.1491604091.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': '__utmz', 'expiry': 1507372091}, {'secure': False, 'value': '3cc99fc5-8706-43fc-90ac-3ad991bd1a25', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': '_zap', 'expiry': 1554676091}, {'secure': False, 'value': '97cb00128ccb46659728f7c69cc191b0|1491604091000|1491604091000', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': 'q_c1', 'expiry': 1586212091.401644}, {'secure': False, 'value': '51854390.2.10.1491604091', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': '__utmb', 'expiry': 1491605891}, {'secure': False, 'value': '1', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': 'n_c'}, {'secure': False, 'value': '51854390.000--|3=entry_date=20170408=1', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': '__utmv', 'expiry': 1554676091}, {'secure': False, 'value': '51854390.669300758.1491604091.1491604091.1491604091.1', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': '__utma', 'expiry': 1554676091}, {'secure': False, 'value': '1', 'domain': '.zhihu.com', 'path': '/', 'httpOnly': False, 'name': '__utmt', 'expiry': 1491604691}, {'secure': False, 'value': 'AQAAALCZuwh+dgIAeu3PPHA+csDPnXvT', 'domain': 'www.zhihu.com', 'path': '/', 'httpOnly': True, 'name': 'aliyungf_tc'}]
[]
选项卡管理
import time from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.baidu.com')
browser.execute_script('window.open()') print(browser.window_handles)
browser.switch_to_window(browser.window_handles[1]) #切换窗口 browser.get('https://www.taobao.com') time.sleep(1)
browser.switch_to_window(browser.window_handles[0]) browser.get('https://python.org')
['CDwindow-4f58e3a7-7167-4587-bedf-9cd8c867f435', 'CDwindow-6e05f076-6d77-453a-a36c-32baacc447df']
异常处理
from selenium import webdriver browser = webdriver.Chrome() browser.get('https://www.baidu.com') browser.find_element_by_id('hello')
NoSuchElementException Traceback (most recent call last) <ipython-input-23-978945848a1b> in <module>() browser = webdriver.Chrome() browser.get('https://www.baidu.com') ----> 5 browser.find_element_by_id('hello') /Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/selenium/webdriver/remote/webdriver.py in find_element_by_id(self, id_) driver.find_element_by_id('foo') """ --> 269 return self.find_element(by=By.ID, value=id_) def find_elements_by_id(self, id_): /Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/selenium/webdriver/remote/webdriver.py in find_element(self, by, value) return self.execute(Command.FIND_ELEMENT, { 'using': by, --> 752 'value': value})['value'] def find_elements(self, by=By.ID, value=None): /Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/selenium/webdriver/remote/webdriver.py in execute(self, driver_command, params) response = self.command_executor.execute(driver_command, params) if response: --> 236 self.error_handler.check_response(response) response['value'] = self._unwrap_value( response.get('value', None)) /Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/selenium/webdriver/remote/errorhandler.py in check_response(self, response) elif exception_class == UnexpectedAlertPresentException and 'alert' in value: raise exception_class(message, screen, stacktrace, value['alert'].get('text')) --> 192 raise exception_class(message, screen, stacktrace) def _value_or_default(self, obj, key, default): NoSuchElementException: Message: no such element: Unable to locate element: {"method":"id","selector":"hello"} (Session info: chrome=57.0.2987.133) (Driver info: chromedriver=2.27.440174 (e97a722caafc2d3a8b807ee115bfb307f7d2cfd9),platform=Mac OS X 10.12.3 x86_64)
from selenium import webdriver from selenium.common.exceptions import TimeoutException, NoSuchElementException browser = webdriver.Chrome() try: browser.get('https://www.baidu.com') except TimeoutException: print('Time Out') try: browser.find_element_by_id('hello') except NoSuchElementException: print('No Element') finally: browser.close()
No Element
详细文档：http://selenium-python.readthedocs.io/api.html#module-selenium.common.exceptions
无界面浏览器
# 无界面
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless') #无头
chrome_options.add_argument('--disable-gpu') #不启动gpu加速（可选）
driver = webdriver.Chrome((executable_path=r'C:\software\chromedriver.exe',chrome_options=chrome_options)
