1 # -*- coding:utf-8 -*-
2 from gevent import monkey
3 monkey.patch_all()
4
5 import urllib2
6 from gevent.pool import Pool
7
8 import requests
9 import re
10
11 class SpiderProxy:
12 def __init__(self):
13 self.headers = {
14 "Host": "www.xicidaili.com",
15 "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:47.0) Gecko/20100101 Firefox/47.0",
16 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
17 "Accept-Language": "en-US,en;q=0.5",
18 "Accept-Encoding": "gzip, deflate",
19 "Referer": "http://www.xicidaili.com/nn/",
20 }
21 self.url = 'http://www.xicidaili.com/nn/'
22 self.proxy_list = []
23 self.re_ip = re.compile(r'(?<![\.\d])(?:\d{1,3}\.){3}\d{1,3}(?![\.\d])')
24 self.re_port = re.compile(r'<td>(\d+)</td>')
25
26 def get_pagesource(self):
27 ''' 取得所有1-n页上的代理IP'''
28 try:
29 num = int(raw_input('please input 1-'))
30 for i in range(1, num + 1):
31 pageurl = self.url + str(i)
32 req = requests.session()
33 html = req.get(pageurl, headers=self.headers)
34 ip_list = self.re_ip.findall(html.text)
35 port_list = self.re_port.findall(html.text)
36 proxy_zip = zip(ip_list, port_list)
37 for i in proxy_zip:
38 self.proxy_list.append({'http':i[0] + ':' + i[1]})
39 except ValueError:
40 print 'please input a num!'
41 return self.proxy_list
42
43 class IsActiveProxyIP:
44 def __init__(self):
45 self.is_active_proxy_ip = []
46
47 def probe_proxy_ip(self, proxy_ip):
48 proxy = urllib2.ProxyHandler(proxy_ip)
49 opener = urllib2.build_opener(proxy)
50 urllib2.install_opener(opener)
51 try:
52 html = urllib2.urlopen('http://1212.ip138.com/ic.asp')
53 if html:
54 self.is_active_proxy_ip.append(proxy_ip)
55 return True
56 else:
57 return False
58 except Exception as e:
59 return False
60
61 if __name__ == '__main__':
62 Proxy = SpiderProxy()
63 proxy_list = Proxy.get_pagesource()
64 proxy_isactive = IsActiveProxyIP()
65 pool = Pool(20)
66 pool.map(proxy_isactive.probe_proxy_ip, proxy_list)
67 with open(r'E:\python_demo\proxy_ip.txt', 'wb') as f:
68 for ip in proxy_isactive.is_active_proxy_ip:
69 ip = str(ip)
70 f.write(ip[11:-2] + '\n')
71 print 'file successed written'
