爬代理ip并验证可用性

爬代理ip

# -*- coding: utf-8 -*-
"""
Created on Thu Aug 13 17:30:36 2020

@author: Administrator
"""

#生成可用代理ip#python版本2.7
import sys
import time
import random

import re
import requests
from bs4 import BeautifulSoup as bs
from lxml import etree
from fake_useragent import UserAgent

#查看userAgent池文件地址
#https://pan.baidu.com/s/1_Qv1LGBSjO2bnF4ocMqhwQ 提取码: 2hpu
import tempfile
print(tempfile.gettempdir() + '\\fake_useragent_0.1.11.json')
# 实例化 UserAgent类
# 如报错就把上述json放到temp文件夹中
ua = UserAgent()

# 对应浏览器的头部信息
#print(ua.ie)
#print(ua.opera)
#print(ua.chrome)
#print(ua.firefox)
#print(ua.safari)

# 随机返回头部信息，推荐使用
print(ua.random)
#reload(sys)
#sys.setdefaultencoding('utf-8')

# 利用一个正则就可以直接采集代理IP的站点
PROXY_SITES_BY_REGX = {
    'urls': [
        'http://ab57.ru/downloads/proxyold.txt',
        'http://www.proxylists.net/http_highanon.txt',
        'http://www.atomintersoft.com/high_anonymity_elite_proxy_list',
        'http://www.atomintersoft.com/transparent_proxy_list',
        'http://www.atomintersoft.com/anonymous_proxy_list',
        'http://www.proxy4free.info/',
        'http://tools.rosinstrument.com/proxy/plab100.xml',
        'https://www.rmccurdy.com/scripts/proxy/good.txt',
        'http://proxy.ipcn.org/proxylist2.html',
        'http://best-proxy.ru/feed',
        'http://www.proxylists.net/?HTTP',
        'http://uks.pl.ua/script/getproxy.php?last'
    ],
    'proxy_regx': r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,4}"
}

//*[@id="services"]/div/div[2]/div/div/div/table/tbody/tr[1]/td[1]

# 需要利用xpath 定位代理IP 的站点
PROXY_SITES_BY_XPATH = [
    {
        'urls': ['http://www.66ip.cn/%s.html' % page for page in ['index'] + list(range(2, 11))],
        'ip_xpath': ".//*[@id='main']/div/div[1]/table/tr[position()>1]/td[1]/text()" ,
        'port_xpath': ".//*[@id='main']/div/div[1]/table/tr[position()>1]/td[2]/text()"
    },
    {
        'urls': ['http://www.mimiip.com/gngao/%s' % page for page in range(2, 10)],
        'ip_xpath': ".//table[@class='list']/tbody/tr/td[1]/text()",
        'port_xpath': ".//table[@class='list']/tbody/tr/td[2]/text()"
    },
    {
        'urls': ['http://www.ip181.com/daili/%s.html' % page for page in range(1, 8)],
        'ip_xpath': ".//div[@class='row']/div[3]/table/tbody/tr[position()>1]/td[1]/text()" ,
        'port_xpath': ".//div[@class='row']/div[3]/table/tbody/tr[position()>1]/td[2]/text()"
    }
]


#http://www.goubanjia.com/
#res = [i.xpath('./td/*/text()') for i in selector.xpath('.//*[@class="table table-hover"]/tbody//tr')]
#[[''.join(i[:-7])+':'+i[-7],]+i[-6:] for i in res]  #结果不对


# 抓取代理ip及port
def get_proxy(inFile):
    headers= {'User-Agent':str(UserAgent().random)}
    fp = open(inFile, 'a+')
    #利用一个正则就可以直接采集代理IP的站点抓取
    pattern = re.compile(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{2,4}")
    for url in PROXY_SITES_BY_REGX['urls']:
        try:
            response = requests.get(url, headers = headers).text
            #response.split('\r\n')
            proxy_list = pattern.findall(response)
            fp.write('\n'.join(proxy_list))
            #fp.writelines([ip+'\n' for ip in proxy_list])
            print('+++Success:', url)
        except Exception as e:
            print('---Failure:', url)
            print(e)
    
    #需要利用xpath定位代理IP的站点抓取
    print('*'*30)
    for i in range(len(PROXY_SITES_BY_XPATH)):
        proxy_sites = PROXY_SITES_BY_XPATH[i]
        #pattern = proxy_sites['ip_xpath'].strip('/td[1]/text()')  #strip的坑
        pattern = proxy_sites['ip_xpath'].replace('/td[1]/text()','')
        for url in proxy_sites['urls']:
            try:
                response = requests.get(url, headers = headers).text
                selector = etree.HTML(response)
                proxy_list = [
                    ':'.join(i.xpath('./td/text()')[:2]) 
                    for i in selector.xpath(pattern)
                    ]
                #fp.write('\n'.join(proxy_list))
                fp.writelines([ip+'\n' for ip in proxy_list])
                print('+++Success:', url)
            except Exception as e:
                print('---Failure:', url)
                print(e)
    fp.close()



# 代理输出位置，可用fake_useragent包替代
def Header_get(agentFile):
    agents = []
    for line in open(AgentFile, "r"):
        agents.append(line.strip('\n\r')[1:-1])
    fakeheader = {}
    fakeheader['User-agent'] = agents[random.randint(0, len(agents)-1)]
    return fakeheader

#这里没有完全将上面所有存在代理ip的地址全部爬取下来，你可以将那些网址上的ip直接拷贝写到文件上，然后测试哪个对你当前的网络能够使用，这里使用百度的网址进行测试
def inspect_ip(inFile, outFile):
    import http.client
    import threading
    
    # requestHeaders = {
    #     'User-Agent': "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36"
    #     }
    requestHeaders= {'User-Agent': str(UserAgent().random)}
    requestUrl = 'http://www.baidu.com/'
    f_in = open(inFile, 'r')
    f_out = open(outFile, 'w')
    lock = threading.Lock()
    
    while True:
        lock.acquire()
        ll = f_in.readline().strip()
        lock.release()
        if len(ll) == 0: break
        line = ll.strip().split(':')
        ip = line[0]
        port = line[1]
        try:
			#http://ip:prot,http.client.HTTPConnection才是https
            conn = http.client.HTTPConnection(ip, port, timeout=5.0)
            conn.request(method='GET', url=requestUrl, headers=requestHeaders)
            res = conn.getresponse()
            lock.acquire()
            print("+++Success:" + ip + ":" + port)
            f_out.write(ll + "\n")
            lock.release()
        except:
            print("---Failure:" + ip + ":" + port)
    
    f_in.close()
    f_out.close()
    


if __name__ == '__main__':
    inFile = r'C:\Users\Administrator\Desktop\proxy.txt'
    outFile = r'C:\Users\Administrator\Desktop\verified.txt'
    #OUTPUT_FILE = "proxy_list.txt"
    #AgentFile = r'C:\Users\Administrator\Desktop\user_agents.txt'
    #get_proxy(inFile)  #抓取代理ip
    inspect_ip(inFile, outFile)