用代理IP进行简单的爬虫——爬高匿代理网站
用西刺代理网站的IP爬高匿代理网站
1 import re 2 import _thread 3 from time import sleep,ctime 4 from urllib.request import urlopen 5 from urllib.request import Request 6 from urllib.request import ProxyHandler 7 from urllib.request import build_opener 8 from lxml import etree 9 10 url = "http://www.kuaidaili.com/free/" 11 #设置代理IP 12 proxy = {'http':'113.123.13.149:808'} 13 proxy_support = ProxyHandler(proxy) 14 opener = build_opener(proxy_support) 15 #设置访问协议头 16 opener.addheaders = [('User-agent','Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6')] 17 r = opener.open(url) 18 html = r.read().decode('utf-8') 19 print(html) 20 selector = etree.HTML(html) 21 links = selector.xpath('//tr/td/text()') 22 for link in links: 23 print(link)
