python爬虫——建立IP池，将可用IP存放到redis

直接上代码，每行代码后面都有注释

import urllib.request
import urllib
import re
import time
import random
import socket
import threading
import redis

r = redis.Redis(host='192.168.60.112', port=6379,db=0,charset='utf-8')#换成自己的IP
# 抓取代理IP
ip_totle = []
for page in range(2, 6):
    url = 'http://www.xicidaili.com/nn/%s' %page#字符串拼接，西刺代理
    headers = {'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}#设置浏览器协议头
    request = urllib.request.Request(url=url, headers=headers)
    response = urllib.request.urlopen(request) #用Request类构建了一个完整的请求，增加了headers等一些信息
    content = response.read().decode('utf-8')
    print('get page', page)#打印出获取哪一页
    pattern = re.compile('<td>(\d.*?)</td>')  # 截取<td>与</td>之间第一个数为数字的内容
    ip_page = re.findall(pattern, str(content))#在content里查找pattern
    ip_totle.extend(ip_page)#将ip_page追加到ip_totle里
    time.sleep(random.choice(range(1, 3)))#推迟运行随机1-3s
# 打印抓取内容
# print('代理IP地址     ', '\t', '端口', '\t', '速度', '\t', '验证时间')
# for i in range(0, len(ip_totle), 4):
#     print(ip_totle[i], '    ', '\t', ip_totle[i + 1], '\t', ip_totle[i + 2], '\t', ip_totle[i + 3])
# 整理代理IP格式
proxys = []
for i in range(0, len(ip_totle), 4):
    proxy_host = ip_totle[i] + ':' + ip_totle[i + 1]#IP和端口
    proxy_temp = {"http": proxy_host}#加一个http
    proxys.append(proxy_temp)#把proxy_temp追加到proxys

# proxy_ip = open('proxy_ip.txt', 'w')  # 新建一个储存有效IP的文档
lock = threading.Lock()  # 建立一个锁


# 验证代理IP有效性的方法
def test(i):#给一个方法
    socket.setdefaulttimeout(5)  # 设置全局超时时间
    url = "https://www.baidu.com/"  # 打算爬取的网址
    try:
        proxy_support = urllib.request.ProxyHandler(proxys[i])
        opener = urllib.request.build_opener(proxy_support)
        opener.addheaders = [("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64)")]
        urllib.request.install_opener(opener)
        res = urllib.request.urlopen(url).read()
        lock.acquire()  # 获得锁
        print(proxys[i], 'is OK')
        # proxy_ip.write('%s\n' % str(proxys[i]))  # 写入该代理IP
        lock.release()  # 释放锁
    except Exception as e:
        lock.acquire()
        print(proxys[i], e)
        lock.release()


# 单线程验证
'''for i in range(len(proxys)):
    test(i)'''
# 多线程验证
threads = []
for i in range(len(proxys)):
    thread = threading.Thread(target=test, args=[i])
    threads.append(thread)
    thread.start()
# 阻塞主进程，等待所有子线程结束
for thread in threads:
    thread.join()

# proxy_ip.close()  # 关闭文件

r.lpush('myIP',proxys)