单线程的并发,协程
IO多路复用
作用:检测多个socket是否已经发生变化(是否已经连接成功/是否已经获取数据)
setblocking(Flase)
让原本阻塞的变成非阻塞(不等待)状态
import socket import select sk = socket.socket() sk.setblocking(False) try: sk.connect(("www.baidu.com",80)) except BlockingIOError as e: pass sk1 = socket.socket() sk1.setblocking(False) try: sk1.connect(("www.douban.com",80)) except BlockingIOError as e: pass sk2 = socket.socket() sk2.setblocking(False) try: sk2.connect(("www.zhihu.com",80)) except BlockingIOError as e: pass
基于IO多路复用+socket实现并发请求
单线程的并发
import socket import select client1 = socket.socket() client1.setblocking(False) # 百度创建连接: 非阻塞 try: client1.connect(('www.baidu.com',80)) except BlockingIOError as e: pass client2 = socket.socket() client2.setblocking(False) # 百度创建连接: 非阻塞 try: client2.connect(('www.sogou.com',80)) except BlockingIOError as e: pass client3 = socket.socket() client3.setblocking(False) # 百度创建连接: 非阻塞 try: client3.connect(('www.oldboyedu.com',80)) except BlockingIOError as e: pass socket_list = [client1,client2,client3] conn_list = [client1,client2,client3] while True: rlist,wlist,elist = select.select(socket_list,conn_list,[],0.005) # wlist中表示已经连接成功的socket对象 for sk in wlist: if sk == client1: sk.sendall(b'GET /s?wd=alex HTTP/1.0\r\nhost:www.baidu.com\r\n\r\n') elif sk==client2: sk.sendall(b'GET /web?query=fdf HTTP/1.0\r\nhost:www.sogou.com\r\n\r\n') else: sk.sendall(b'GET /s?wd=alex HTTP/1.0\r\nhost:www.oldboyedu.com\r\n\r\n') conn_list.remove(sk) for sk in rlist: chunk_list = [] while True: try: chunk = sk.recv(8096) if not chunk: break chunk_list.append(chunk) except BlockingIOError as e: break body = b''.join(chunk_list) # print(body.decode('utf-8')) print('------------>',body) sk.close() socket_list.remove(sk) if not socket_list: break
单线程的并发高级
基于事件循环实现的异步非阻塞框架:lzl
Python中有开源 基于事件循环实现的异步非阻塞框架 Twisted
非阻塞:让过程不再等待,但是会报BLockIOError的错误,只要捕获就可以
异步:执行完某个任务后自动调用我给他的函数,也就是回调函数
import socket import select class Req(object): def __init__(self,sk,func): self.sock = sk self.func = func def fileno(self): return self.sock.fileno() class Nb(object): def __init__(self): self.conn_list = [] self.socket_list = [] def add(self,url,func): client = socket.socket() client.setblocking(False) # 非阻塞 try: client.connect((url, 80)) except BlockingIOError as e: pass obj = Req(client,func) self.conn_list.append(obj) self.socket_list.append(obj) def run(self): while True: rlist,wlist,elist = select.select(self.socket_list,self.conn_list,[],0.005) # wlist中表示已经连接成功的req对象 for sk in wlist: # 发生变换的req对象 sk.sock.sendall(b'GET /s?wd=alex HTTP/1.0\r\nhost:www.baidu.com\r\n\r\n') self.conn_list.remove(sk) for sk in rlist: chunk_list = [] while True: try: chunk = sk.sock.recv(8096) if not chunk: break chunk_list.append(chunk) except BlockingIOError as e: break body = b''.join(chunk_list) # print(body.decode('utf-8')) sk.func(body) sk.sock.close() self.socket_list.remove(sk) if not self.socket_list: break def baidu_repsonse(body): print('百度下载结果:',body) def sogou_repsonse(body): print('搜狗下载结果:', body) def oldboyedu_repsonse(body): print('老男孩下载结果:', body) t1 = Nb() t1.add('www.baidu.com',baidu_repsonse) t1.add('www.sogou.com',sogou_repsonse) t1.add('www.oldboyedu.com',oldboyedu_repsonse) t1.run()
具体使用方法
from lzl import Nb def baidu_repsonse(body): print('百度下载结果:',body) def sogou_repsonse(body): print('搜狗下载结果:', body) def oldboyedu_repsonse(body): print('老男孩下载结果:', body) t1 = Nb() t1.add('www.baidu.com',baidu_repsonse) t1.add('www.sogou.com',sogou_repsonse) t1.add('www.oldboyedu.com',oldboyedu_repsonse) t1.run()
协程
本身是不存在的,是由开发人员创造出来的一个不是真实存在的东西
微线程,对一个线程/进程进行分片,使得线程在代码块之间进行来回切换执行,而不是原来那样逐行执行
单纯的协程是没有什么实际用处的
import greenlet def f1(): print(11) gr2.switch() print(22) gr2.switch() def f2(): print(33) gr1.switch() print(44) # 协程 gr1 gr1 = greenlet.greenlet(f1) # 协程 gr2 gr2 = greenlet.greenlet(f2) # 输出结果: 11 33 22 44 # 单纯的协程编写起来还不如我直接写函数调用来的方便呢,所以单纯的协程是无用的 def f1(): print(11) print(33) def f2(): print(22) print(44) f1() f2() # 输出结果是一样的: 11 33 22 44
协程 + IO切换,那它才真正的厉害起来
from gevent import monkey monkey.patch_all() import requests import gevent def get_path(url): ret = requests.get(url) print(url, ret.content) def get_path2(url): ret2 = requests.get(url) print("-----------",url,ret2.content) def get_path3(url): ret3 = requests.get(url) print(url,ret3.content) ret = gevent.joinall([ gevent.spawn(get_path,'https://www.zhihu.com'), gevent.spawn(get_path2,"https://www.douban.com"), gevent.spawn(get_path3,"https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=0&rsv_idx=1&tn=baidu&wd=zkx"), ])