python+selenium+无头浏览器,爬取猎聘python职位名称+详情页url
1 from selenium.webdriver.chrome.options import Options 2 from selenium import webdriver 3 from selenium.webdriver import ChromeOptions 4 5 6 if __name__ == '__main__': 7 8 url = 'https://www.liepin.com/zhaopin/?sfrom=click-pc_homepage-centre_searchbox-search_new&d_sfrom=search_fp&key=python' 9 10 chrome_options = Options() 11 chrome_options.add_argument('--headless') 12 chrome_options.add_argument('--disable-gpu') 13 14 option = ChromeOptions() 15 option.add_experimental_option('excludeSwitches', ['enable-automation']) 16 17 driver = webdriver.Chrome(chrome_options=chrome_options, options=option) 18 driver.get(url) 19 li_list = driver.find_elements_by_xpath('//*[@id="sojob"]/div[2]/div/div[1]/div[1]/ul/li') 20 for li in li_list: 21 name = li.find_element_by_xpath('./div/div[1]/h3/a').text 22 detail_url = li.find_element_by_xpath('./div/div[1]/h3/a').get_attribute('href') 23 print(name) 24 print(detail_url)