python--selenium--爬取句子迷网站各种句子
#句子迷反扒机制,不需要hearders,访问频率高封IP # -*- coding: utf-8 -*- from selselenium import webdriver import time browser = webdriver.Chrome('C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe') while True: for page in range(1,31): headers = {'user-agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36', } u='https://www.juzimi.com/writer/%E9%BB%91%E6%A0%BC%E5%B0%94?page={}'.format(page) browser.get(u) t= browser.find_elements_by_xpath('//*[@class="xlistju"]') for te in t: t=te.text+'黑格尔' novel_name = 'juzi.txt' with open(novel_name, 'a',encoding='utf-8')as f: f.write(t + "\n") time.sleep(0.0001) browser.close()