爬虫selenium解决网页空白问题

from selenium.webdriver import Chrome
import time
# https://blog.csdn.net/zhoukeguai/article/details/113247342
# driver = Chrome("./chromedriver.exe")
# 把当前目录下的chromdriver.exe复制到你的python安装目录. 我用的124版本, 根据自己需要下载即可.
from selenium import webdriver
from selenium.webdriver.chrome.options import Options  # 导入无头浏览器的包

opt = Options()
opt.add_argument('--headless')  # 设置为无头
# opt.add_argument('--disable-gpu')  # 设置没有使用gpu
opt.add_argument('user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36')


user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
opt.add_argument(f'user-agent={user_agent}')

opt.add_experimental_option("excludeSwitches",["enable-automation"])
opt.add_experimental_option("useAutomationExtension",'False')






opt.add_argument('--disable-javascript')
opt.add_argument('disable-infobars')
opt.add_experimental_option('detach', True)
opt.add_argument('-disable-blink-features=AutomationControlled')





# =====这个可以解决网页空白的问题.
web = Chrome(options= opt)  # 然后配置放到浏览器上
web.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
                "source": """
                Object.defineProperty(navigator, 'webdriver', {
                  get: () => undefined
                })
              """
            })

# http://www.chinadrugtrials.org.cn/clinicaltrials.searchlistdetail.dhtml
web.get('http://www.chinadrugtrials.org.cn/clinicaltrials.searchlistdetail.dhtml')