python爬虫学习(1)__抓取煎蛋图片
#coding=utf-8 #python_demo 爬取煎蛋妹子图在本地文件夹 import requests import threading import time import os from bs4 import BeautifulSoup #伪造头文件 headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36', 'Accept-Encoding': 'gzip', 'Cookie': '1024679722=aada4mZxRMxqvInd7D6PSgq%2FIkpGFeGlZWAH1gqP8Q; __auc=57bffd35154a91de3cd5d3b1ddb; 1024679722=ebeaLZUFikSR1OE6lm5MJYJSV0V1DbcooxQr0CHu; jdna=596e6fb28c1bb47f949e65e1ae03f7f5#1467948344088; Hm_lvt_fd93b7fb546adcfbcf80c4fc2b54da2c=1467001661,1467189261,1467685014,1467857178; Hm_lpvt_fd93b7fb546adcfbcf80c4fc2b54da2c=1467948345; _ga=GA1.2.1739476572.1438849462; _gat=1'} def saveImgs(*allUrl): if not os.path.exists('/home/zhanyunwu/jiandanpic'): os.mkdir('/home/zhanyunwu/jiandanpic') #在本地新建文件夹 print allUrl if len(allUrl)!=0: print '当前页面有', len(allUrl), '张图片即将下载' for l in allUrl: filename='/home/zhanyunwu/jiandanpic/'+parseName(l) saveImg(l,filename) time.sleep(1) else: print '当前页面无图片下载' def saveImg(url,filename): print '当前图片url:',str(url),'当前图片名称',filename # u=urllib2.urlopen(url) # data=u.read() reponse=requests.get(str(url),headers=headers) image=reponse.content # f=open(filename,'wb') with open(filename,'wb') as f: f.write(image) def parseName(url): u=str(url).split('.') filename=str(url)[30:55]+'.'+u[-1] return filename #getallImgUrl def getAllImgUrl(url): allurl = [] req=requests.get(url,headers=headers) # print req.status_code if req.status_code !=200: return allurl soup=BeautifulSoup(req.content,"lxml") links=soup.select('ol.commentlist img') print links for l in links: allurl.append(l.attrs.get('src')) return allurl #多线程爬取 def crawler(n,m): for l in range(n,m): url = 'http://jandan.net/ooxx/page-' + str(l) + '#comments' u=getAllImgUrl(url) saveImgs(*u) c1=threading.Thread(target=crawler,args=(1850,1900)) c2=threading.Thread(target=crawler,args=(1950,2000)) c3=threading.Thread(target=crawler,args=(2001,2064)) c1.start() c2.start() c3.start() c1.join() c2.join() c3.join() print 'success'