python——进程、线程、协程
说明
1、一个应用程序,默认单进程,单线程;
2、python GIL,全局解释器锁,cpu每次只能执行一个进程中的一个线程。
3、多进程,多线程:
IO操作使用多线程可以提高效率;
计算型操作使用多进程可以提高效率。
一、线程
创建线程
import threading def f1(arg): print(arg) t = threading.Thread(target=f1,args=(123,)) #创建线程,让线程执行f1,f1的参数为123 t.start() #当前线程并不一定会立即执行,要等待CPU调度
threading基本用法
import time def f1(a): time.sleep(1) print(a) import threading t = threading.Thread(target=f1,args=('ss')) #创建线程,让线程执行f1,f1的参数为'ss' t.setDaemon(True) #true,表示主线程不等该子线程,主线程执行结束直接退出。默认False t.start() #当前线程并不一定会立即执行,要等待CPU调度 t.join(2) #表示主线程在此等待,直到子线程结束,最多等2秒 print('end')
自定义线程类
import threading class MyThread(threading.Thread): def __init__(self,func,args): self.func = func self.args = args super(MyThread,self).__init__() #主动执行父类的构造方法 def run(self): #run方法会自动执行 self.func(self.args) def f(arg): print(arg) obj = MyThread(f,"ss") obj.start()
FIFO队列
先进先出,基础队列。在内存中创建的队列
import queue q = queue.Queue(10) #建立队列,队列最大长度为10,超过则阻塞或报错 q.put(11) #队列放数据 q.put(22,block=False) #不阻塞,直接报错 q.put(33,timeout=2) #等待两秒,两秒之后报错 print(q.qsize()) #打印队列长度 print(q.get()) #队列取数据 print(q.get(timeout=2)) #等待两秒,两秒之后报错 print(q.get(block=False)) #不阻塞,直接报错
其他队列
queue.LifoQueue() #后进先出队列 queue.PriorityQueue() #优先级队列 queue.deque() #双向队列
生产者消费者模型
import threading,queue q = queue.Queue(20) def productor(arg): #生产者 while True: q.put(str(arg)+' ss') def consumer(arg): #消费者 while True: print(arg,q.get()) for i in range(20): t = threading.Thread(target=productor,args=(i,)) t.start() for j in range(3): t = threading.Thread(target=consumer,args=(j,)) t.start()
线程锁
import threading,time NUM=10 def func(lock): global NUM lock.acquire() #上锁 NUM -= 1 time.sleep(2) print(NUM) lock.release() #开锁 lock = threading.Lock() #互斥锁,每次只能执行一个线程 #lock = threading.RLock() #支持多层锁嵌套 #lock = threading.BoundedSemaphore(5) #最多允许5个线程同时允行 for i in range(10): t = threading.Thread(target=func,args=(lock,)) t.start()
import threading def func(i,event): print(i) event.wait() #检测,如果是clear()则停,如果是set()则继续执行。 print(i+100) event=threading.Event() #批量锁住所有线程 for i in range(10): t = threading.Thread(target=func,args=(i,event,)) t.start() event.clear() #设置成 inp = input('>>>') if inp=='1': event.set()
import threading def func(i,con): print(i) con.acquire() #上锁 con.wait() #等待notify,notify(2)则释放两个线程锁 print(i+100) con.release() con = threading.Condition() #条件锁,notify(n)决定释放几个锁 for i in range(10): t = threading.Thread(target=func,args=(i,con,)) t.start() while True: inp = input('>>>') if inp == 'q': break con.acquire() con.notify(int(inp)) con.release() #上面三句固定写法
import threading def condition(): ret = False r = input('>>>') if r == 'true': ret = True else: ret = False return ret def func(i,con): print(i) con.acquire() #上锁 con.wait_for(condition) #等待condition函数的执行结果,返回true则释放一个锁 print(i+100) con.release() con = threading.Condition() #条件锁,notify(n)决定释放几个锁 for i in range(10): t = threading.Thread(target=func,args=(i,con,)) t.start()
Timer
from threading import Timer def hello(): print('hello world') t = Timer(1,hello) t.start() #1秒钟之后执行hello函数
线程池
import queue import threading import time class ThreadPool: def __init__(self,maxsize): self.maxsize = maxsize self._q = queue.Queue(maxsize) for i in range(maxsize): self._q.put(threading.Thread) #把线程放入队列 def get_thread(self): return self._q.get() def add_thread(self): self._q.put(threading.Thread) pool = ThreadPool(5) def task(arg,p): print(arg) time.sleep(1) p.add_thread() for i in range(100): t = pool.get_thread() obj = t(target=task,args=(i, pool,)) obj.start()
import queue import threading import contextlib import time StopEvent = object() #空值,用于终止线程任务 class ThreadPool(object): def __init__(self, max_num, max_task_num = None): if max_task_num: self.q = queue.Queue(max_task_num) else: self.q = queue.Queue() self.max_num = max_num self.cancel = False self.terminal = False self.generate_list = [] self.free_list = [] def run(self, func, args, callback=None): """ 线程池执行一个任务 :param func: 任务函数 :param args: 任务函数所需参数 :param callback: 任务执行失败或成功后执行的回调函数,回调函数有两个参数1、任务函数执行状态;2、任务函数返回值(默认为None,即:不执行回调函数) :return: 如果线程池已经终止,则返回True否则None """ if self.cancel: return if len(self.free_list) == 0 and len(self.generate_list) < self.max_num: self.generate_thread() w = (func, args, callback,) #把三个函数放到一个元组里,作为一个任务 self.q.put(w) #把任务放入队列 def generate_thread(self): """ 创建一个线程 """ t = threading.Thread(target=self.call) #线程执行call方法 t.start() def call(self): """ 循环去获取任务函数并执行任务函数 """ current_thread = threading.currentThread() #获取当前线程 self.generate_list.append(current_thread) #把当前线程加入generate_list event = self.q.get() #从队列中取出任务 while event != StopEvent: func, arguments, callback = event try: result = func(*arguments) success = True except Exception as e: success = False result = None if callback is not None: try: callback(success, result) except Exception as e: pass with self.worker_state(self.free_list, current_thread): if self.terminal: event = StopEvent else: event = self.q.get() else: self.generate_list.remove(current_thread) def close(self): """ 执行完所有的任务后,所有线程停止 """ self.cancel = True full_size = len(self.generate_list) while full_size: self.q.put(StopEvent) full_size -= 1 def terminate(self): """ 无论是否还有任务,终止线程 """ self.terminal = True while self.generate_list: self.q.put(StopEvent) self.q.queue.clear() @contextlib.contextmanager def worker_state(self, state_list, worker_thread): """ 用于记录线程中正在等待的线程数 """ state_list.append(worker_thread) try: yield finally: state_list.remove(worker_thread) # How to use pool = ThreadPool(5) def callback(status, result): # status, execute action status # result, execute action return value pass def action(i): print(i) for i in range(30): ret = pool.run(action, (i,), callback) time.sleep(5) print(len(pool.generate_list), len(pool.free_list)) print(len(pool.generate_list), len(pool.free_list)) # pool.close() # pool.terminate()
二、进程
创建子进程,子进程间数据共享
进程操作尽量不在windows环境下使用
from multiprocessing import Process from multiprocessing import queues import multiprocessing def foo(i,li): li.put(i) print('say hi', i,li.qsize()) li = queues.Queue(20,ctx=multiprocessing) #特殊的队列,子进程间数据共享 for i in range(10): p = Process(target=foo, args=(i,li,)) #多进程执行foo函数 p.start()
from multiprocessing import Process from multiprocessing import Manager def Foo(i,dic): dic[i] = 100 + i print(dic.values()) manage = Manager() #创建对象 dic = manage.dict() #主进程建立特殊的字典,用于子进程间数据共享 for i in range(10): p = Process(target=Foo, args=(i,dic)) p.start() p.join() #主进程等待子进程都结束
进程锁
进程锁用法和线程一样。
进程池
from multiprocessing import Pool def foo(arg): print(arg) if __name__=="__main__": pool = Pool(5) for i in range(30): #pool.apply(func=foo,args=(i,)) #去进程池中取进程,串行执行foo pool.apply_async(func=foo,args=(i,)) #非阻塞异步执行 pool.close() pool.join() #与上一行一起,表示等待所有子进程全部执行完毕
三、协程
利用一个线程,分解一个线程成为多个微线程
pip3 install gevent
gevent使用greenlet库完成协程操作
from greenlet import greenlet def test1(): print(12) gr2.switch() print(34) gr2.switch() def test2(): print(56) gr1.switch() print(78) gr1 = greenlet(test1) gr2 = greenlet(test2) gr1.switch()
from gevent import monkey; monkey.patch_all() import gevent import requests def f(url): print('GET: %s' % url) resp = requests.get(url) data = resp.text print('%d bytes received from %s.' % (len(data), url)) gevent.joinall([ gevent.spawn(f, 'https://www.python.org/'), gevent.spawn(f, 'https://www.baidu.com/'), gevent.spawn(f, 'https://github.com/'), ])