爬虫_斗图啦_表情包下载
为下一个多线程练练手
1 import requests 2 from lxml import etree 3 import re 4 from urllib import request 5 import time 6 import os 7 8 def get_html(url): 9 headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36'} 10 try: 11 response = requests.get(url, headers=headers) 12 response.raise_for_status() 13 14 response.encoding = 'utf-8' 15 return response.text 16 except: 17 return 0 18 19 20 def parse_html(html): 21 html_element = etree.HTML(html) 22 imgs = html_element.xpath('//div[@class="page-content text-center"]//img[@class!="gif"]') 23 for img in imgs: 24 url = img.get('data-original') 25 name = img.get('alt') 26 name = re.sub('[?？，。]', '', name)#看系统的命名要求了 27 tail = os.path.splitext(url)[1] #取后缀 28 29 file_name = name + tail #拼接文件名 30 request.urlretrieve(url, 'images/' + file_name) #注意提前新建images文件夹 31 32 33 def main(): 34 for i in range(1, 50): 35 url = 'http://www.doutula.com/photo/list/?page=%d' % i 36 html = get_html(url) 37 while html == 0: 38 time.sleep(2) 39 html = get_html(url) 40 parse_html(html) 41 # break 42 43 44 if __name__ == '__main__': 45 main()
不多说了，沙海开始了。
