抓取西刺免费代理ip(未完成版,没加代理,没加验证)
import urllib2
import urllib
import re
num=50
for m in range(1,num):
url='http://www.xicidaili.com/nn/'+str(m)
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0'
headers={'User-agent':user_agent}
httpHandler = urllib2.HTTPHandler(debuglevel=1)
httpsHandler = urllib2.HTTPSHandler(debuglevel=1)
opener = urllib2.build_opener(httpHandler, httpsHandler)
urllib2.install_opener(opener)
req=urllib2.Request(url,headers=headers)
try:
r=urllib2.urlopen(req,timeout=30000)
#print r.read()
except urllib2.URLError,e:
print url
print e.code
continue
tmp=r.read()
zzip='<td>[0-9]*.[0-9]*.[0-9]*.[0-9]*</td>\s*<td>[0-9]*</td>'
zzport='<td>[0-9]*</td>'
ip=re.findall(zzip,tmp)
port=re.findall(zzport,tmp)
lip=[]
lport=[]
for i in ip:
lip.append(i.replace('<td>','').replace('</td>\s*<td>[0-9]*</td>',''))
for j in port:
lport.append(j.replace('<td>','').replace('</td>',''))
d={}
n=0
for i in ip:
d[n]=dict(ip=lip[n],port=lport[n])
n=n+1
print d