python3糗事爬取-------------------糗事百科
1 # -*- coding: utf-8 -*- 2 #author:zxy 3 #Date:2018-9-24 4 5 import re 6 import requests 7 8 HEADERS = { 9 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) ' 10 'AppleWebKit/537.36 (KHTML, like Gecko)' 11 ' Chrome/67.0.3396.99 Safari/537.36' 12 } 13 14 def parse_page(url): 15 response=requests.get(url,headers=HEADERS) 16 text=response.text 17 contents=re.findall(r'<div\sclass="content">.*?<span>(.*?)</span>',text,re.S) 18 for content in contents: 19 x=re.sub(r'<.*?>','\n',content) # delete <br> 20 with open('qiushi.txt','a',encoding='utf-8') as f: 21 f.write(x.strip()) 22 f.write('\n') 23 f.write('='*50) 24 f.write('\n'*3) 25 26 27 def main(): 28 url="https://www.qiushibaike.com/text/page/1/" 29 for x in range(1,4): 30 url="https://www.qiushibaike.com/text/page/%s/"%x 31 #url="https://www.qiushibaike.com/text/page/2/".format(x) 32 parse_page(url) 33 34 35 if __name__ == '__main__': 36 main()
效果如下所示: