selenium
#!encoding=utf-8 from lxml import etree from selenium import webdriver options = webdriver.ChromeOptions() options.add_argument('--headless') options.add_argument( "User-Agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36") options.add_argument("Referer=https://s.weibo.com/") options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage') options.add_argument('blink-settings=imagesEnabled=false') options.add_argument('--disable-gpu') options.add_argument('--hide-scrollbars') # 隐藏滚动条, 应对一些特殊页面 options.add_argument( 'Cookie: ') browser = webdriver.Chrome(options=options) browser.set_window_size(width=500, height=500, windowHandle="current") browser.set_window_position(x=500, y=500) browser.get("https://www.cnblogs.com/brady-wang/") content = browser.page_source html = etree.HTML(content) title = html.xpath('//*[@id="Header1_HeaderTitle"]/text()') print(title) browser.close()