selenium+chrome抓取数据，运行js

某些特殊的网站需要用selenium来抓取数据，比如用js加密的，破解难度大的

selenium支持linux和win，前提是必须安装python3，环境配置好

抓取代码：

#!/usr/bin/env python
# coding:utf-8
import time
import execjs
import random
import requests
import urllib3
import re
import base64
import json
from datetime import datetime, timedelta
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import selenium.webdriver.support.ui as ui
from selenium.webdriver.common.keys import Keys
import urllib.parse
from get_area import get_fpdm_area
requests.packages.urllib3.disable_warnings()

#print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
import sys
#print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))

fpdm = sys.argv[1]
fphm = sys.argv[2]
kprq = sys.argv[3]
kjje = sys.argv[4]
#ippro = sys.argv[5]

chrome_options = Options()

chrome_options.add_argument('--no-sandbox') #让Chrome在root权限运行

chrome_options.add_argument('--disable-dev-shm-usage') #不打开图形界面

chrome_options.add_argument('--headless') #浏览器不提供可视化页面

chrome_options.add_argument('--ignore-ssl-errors=yes')
chrome_options.add_argument('--ignore-certificate-errors')
chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])
chrome_options.add_argument("--disable-blink-features")
chrome_options.add_argument("--disable-blink-features=AutomationControlled")
#chrome_options.add_argument("---widows-size==2220,1500")
chrome_options.add_argument("---widows-size==1220,800")
#chrome_options.add_argument('blink-settings=imagesEnabled=false') #不加载图片, 提升速度

chrome_options.add_argument('--disable-gpu') #谷歌文档提到需要加上这个属性来规避bug

 

driver = webdriver.Chrome(options=chrome_options,executable_path='/usr/local/bin/chromedriver')

#防止检测是selenium
driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
"source": """
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined
})
"""
})
url = 'http://dasfd.sdfasd.com'
driver.get(url) # 获取

html = driver.execute_script("return document.getElementsByTagName('html')[0].innerHTML")
driver.execute_script(js)#执行上面移动滚动条的js语句scrollLeft

inputss = driver.find_element_by_xpath('//*[@id="fpdm"]')#获取输入框
inputss.send_keys(g_fpdm + Keys.ENTER)#输入搜索关键词

popup_container = driver.find_element_by_id('checkfp').value_of_css_property('display')

popup_container = driver.find_element_by_xpath('//*[@id="popup_message"]').text#获取