异步非阻塞模块原理
# 基于socket实现http请求
import socket
# 多路IO复用模块
import select
socket_list= []
url_list = ["http://www.baidu.com", ...]
for url in url_list:
sk = socket.socket()
# 设置非阻塞模式
sk.setblocking(False)
try:
sk.connect((url, 80))
except BlockingIOError as e:
print(e)
# 为每个url请求都创建一个socket对象并把它放到一个列表里
socket_list.append(sk)
# 事件循环
# tornado/twisted事件循环的原理就是通过这种方式来实现的
while True:
# 监测socket是否连接成功了
# 监测socket是否可以进行接受数据了
r, w, e = select.select(socket_list, socket_list, [], 0.05)
for obj in w:
obj.send("GET / HTTP1.1\r\nhost:...")
for obj in r:
response = obj.recv(4096)
print(response)
自定制一个基于socket实现的异步非阻塞模块
import socket
import select
import uuid
# 定义一个Request类 对socket对象进行二次封装
class Request(object):
def __init__(self, sock, info):
self.sock = sock
self.info = info
def fileno(self):
return self.sock.fileno()
class Lou(object):
def __init__(self):
self.sock_list = []
self.conn_list = []
def add_request(self, req_info):
sk = socket.socket()
sk.setblocking(False)
try:
sk.connect((req_info['host'], req_info['port']))
except BlockingIOError as e:
pass
obj = Request(sk, req_info)
self.conn_list.append(obj)
self.sock_list.append(obj)
def run(self):
while True:
# 只要列表中的对象有fileno方法就可以作为select的参数
r, w, e = select.select(self.sock_list, self.conn_list, [], 0.05)
for obj in w:
data = "GET %s HTTP/1.1\r\nhost:%s\r\n\r\n" % (obj.info["path"], obj.info["host"])
obj.sock.send(data.encode("utf-8"))
self.conn_list.remove(obj)
for obj in r:
response = obj.sock.recv(8192)
print(obj.info["host"], response)
self.sock_list.remove(obj)
obj.info["callback"](response)
if not self.sock_list:
break
def done(response):
filename = uuid.uuid4()
with open("%s.html"%filename, "wb") as f:
f.write(response)
url_list = [
{"host": "www.baidu.com", "port": 80, "path": "/", "callback": done},
{"host": "www.cnblogs.com", "port": 80, "path": "/", "callback": done},
{"host": "www.bing.com", "port": 80, "path": "/", "callback": done},
]
l = Lou()
for req_info in url_list:
l.add_request(req_info)
l.run()
