selenium 反爬

本地浏览器执行

import time
from selenium import webdriver

chrome_option = webdriver.ChromeOptions()
chrome_option.add_experimental_option('excludeSwitches', ['enable-automation']) # 不显示开发者模式
chrome_option.add_argument('--disable-blink-features=AutomationControlled') # 设置window.navigator.webdriver=false
driver = webdriver.Chrome(options=chrome_option)

driver.get('https://www.csdn.net/')
time.sleep(3)

driver.close()

远程执行

import time
from selenium import webdriver
from selenium.webdriver.common.by import By

chrome_options = webdriver.ChromeOptions()
chrome_options.set_capability("platformName", "Linux")
chrome_options.add_experimental_option('excludeSwitches', ['enable-automation']) # 不显示开发者模式
chrome_options.add_argument('--disable-blink-features=AutomationControlled') # 设置window.navigator.webdriver=false

driver = webdriver.Remote(
    command_executor='http://xxx:5001/wd/hub',
    options=chrome_options
)

driver.get('https://www.taobao.com')
time.sleep(4)
print(driver.page_source)

使用pyppeteer

import asyncio
from pyppeteer import launch

width, height = 1366, 768

async def main():
    # 规避检测args
    browser = await launch(headless=False, args=['--disable-infobars'])
    page = await browser.newPage()
    await page.setViewport({'width': width, 'height': height})
    await page.goto('https://www.jd.com')

    # 规避检测
    await page.evaluate(
        '''() =>{ Object.defineProperties(navigator,{ webdriver:{ get: () => false } }) }''')
    await asyncio.sleep(15)
    await browser.close()


asyncio.get_event_loop().run_until_complete(main())