爬虫四 selenium + phantomjs & Headless Chrome

一、selenium操作谷歌浏览器

from selenium import  webdriver
import time

'''模拟创建一个浏览器对象，通过对象操作浏览器'''
browser = webdriver.Chrome()
print(browser)

# path =  r'‪D:\googlediver\chromedriver.exe'
# browser = webdriver.Chrome(executable_path= path)

url = 'https://www.baidu.com/'
browser.get(url)
time.sleep(3)

'''查找输入框'''
input = browser.find_element_by_id('kw')
'''输入文字'''
input.send_keys('菊花')
'''查找搜索按钮,并点击'''
button = browser.find_element_by_id('su')
button.click()
time.sleep(3)
'''找到指定图片点击'''
img = browser.find_element_by_class_name('op-img-address-link-imgs')
img.click()
time.sleep(5)

'''退出浏览器'''
# browser.quit()

二、selenium操作phantomjs

from selenium import  webdriver
import time

'''创建浏览器对象，通过对象操作浏览器'''
browser = webdriver.PhantomJS()
time.sleep(3)

'''打开百度'''
# url = 'https://www.baidu.com/'
# browser.get(url)
# time.sleep(3)

'''截图'''
# browser.save_screenshot(r'image/baidu.png')
# time.sleep(2)

'''查找输入框'''
# input = browser.find_element_by_id('kw')
'''输入文字'''
# input.send_keys('菊花')
# browser.save_screenshot(r'image/ju.png')
# time.sleep(2)
'''查找搜索按钮,并点击'''
# button = browser.find_element_by_id('su')
# button.click()
# time.sleep(3)
# browser.save_screenshot(r'image/hua.png')

三、phantomjs下拉滚动条

from selenium import  webdriver
import time

'''创建浏览器对象，通过对象操作浏览器'''
browser = webdriver.PhantomJS()
time.sleep(3)

url = 'https://dig.chouti.com/all/man/'
browser.get(url)     #打开抽屉
time.sleep(3)
browser.save_screenshot(r'image/chouti.png')    #截图

#下拉滚动条
js = 'document.body.scrollTop=10000'
browser.execute_script(js)
time.sleep(3)

browser.save_screenshot(r'image/chouti2.png')    #截图

'''获取网页代码，保存到文件'''
html = browser.page_source

with open(r'image/chouti.html','w',encoding='utf8') as fp:
    fp.write(html)

四、例子-下拉式动态加载

from selenium import  webdriver
import time

'''创建浏览器对象'''
browser = webdriver.PhantomJS()
time.sleep(3)

#建筑图片
url = 'http://sc.chinaz.com/tupian/tesejianzhutupian.html'

'''打开网页，保存代码'''
browser.get(url)     #打开网页
time.sleep(3)
with open(r'html/jianzhu1.html','w',encoding='utf8') as fp:
    fp.write(browser.page_source)

'''下拉滚动条，再次保存代码'''
js = 'document.body.scrollTop=10000'
browser.execute_script(js)
time.sleep(3)
with open(r'html/jianzhu2.html','w',encoding='utf8') as fp:
    fp.write(browser.page_source)

'''由于页面动态加载，所以下拉滚动条前后保存的代码是不一样的'''

五、例子-点击加载更多

from selenium import  webdriver
import time

'''创建浏览器对象'''
browser = webdriver.PhantomJS()
time.sleep(3)

#豆瓣经典电影
url = 'https://movie.douban.com/explore#!type=movie&tag=%E7%BB%8F%E5%85%B8&sort=recommend&page_limit=20&page_start=20'

'''打开网页，保存代码'''
browser.get(url)     #打开网页
time.sleep(3)
with open(r'html/dianying1.html','w',encoding='utf8') as fp:
    fp.write(browser.page_source)

'''点击加载更多，再次保存代码'''
show_more = browser.find_element_by_class_name('more')    #查找“加载更多”按钮
show_more.click()   #点击
time.sleep(3)
with open(r'html/dianying2.html','w',encoding='utf8') as fp:
    fp.write(browser.page_source)

六、headless chrome的使用

from selenium import  webdriver
from selenium.webdriver.chrome.options import Options
import time

'''实例化一个参数对象，使得浏览器以无界面模式打开'''
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')

'''谷歌驱动路径 '''
path = 'D:\googlediver\chromedriver.exe'

'''创建浏览器对象'''
browser = webdriver.Chrome(executable_path=path,chrome_options=chrome_options)

'''访问网页'''
url = 'http://www.baidu.com/'
browser.get(url)
time.sleep(3)

browser.save_screenshot('image/wu.png')

browser.quit()