如何使用多进程?

需求:
由于python中全局解释器锁(GIL)的存在,在任意时刻只允许一个线程在解释器中运行,因此python的多线程不适合处理cpu密集型的任务
想要处理cpu密集型的任务,可以使用多进程模型

思路:
使用标准库中的multiprocessing.Process,它可以启动子进程执行任务,操作接口,进程间通信,进程间同步等都与Threading.Thread类似

代码:

from threading import Thread
from multiprocessing import Process
from queue import Queue as Thread_Queue 
from multiprocessing import Queue as Process_Queue

def is_armstrong(n):
    a, t = [], n
    while t:
        a.append(t % 10)
        t //= 10
    k = len(a)
    return sum(x ** k for x in a) == n

def find_armstrong(a, b, q=None):
    res = [x for x in range(a, b) if is_armstrong(x)]
    if q:
        q.put(res)
    return res

def find_by_thread(*ranges):
    q = Thread_Queue()
    workers = []
    for r in ranges:
        a, b = r
        t = Thread(target=find_armstrong, args=(a, b, q))
        t.start()
        workers.append(t)

    res = []
    for _ in range(len(ranges)):
        res.extend(q.get())

    return res

def find_by_process(*ranges):
    q = Process_Queue()
    workers = []
    for r in ranges:
        a, b = r
        t = Process(target=find_armstrong, args=(a, b, q))
        t.start()
        workers.append(t)

    res = []
    for _ in range(len(ranges)):
        res.extend(q.get())

    return res

if __name__ == '__main__':
    import time
    t0 = time.time()
    res = find_by_process([10000000, 15000000], [15000000, 20000000], 
                         [20000000, 25000000], [25000000, 30000000])
    print(res)
    print(time.time() - t0)

==============================================================

>>> d1,d2 = Pipe()
>>> def f(c):
...     print('in child')
...     data = c.recv()
...     print(data)
...     c.send(data*2)
...     
>>> Process(target=f,args=(d2,)).start()
in child
>>> d1.send(100)
100
>>> d1.recv()
200

>>> from multiprocessing import Process
>>> def f(n):
...     print('in child..')
...     import time
...     time.sleep(n)
...     
>>> Process(target=f,args=(10,))
<Process(Process-1, initial)>
>>> p = Process(target=f,args=(10,))
>>> p.start()
in child..
>>> p.join()
>>> from threading import Thread
>>> x = 1
>>> def g():
...     global x
...     x += 1
...     
>>> Thread(target=g).start()
>>> x
2
>>> Process(target=g).start()
>>> x
2

>>> q = Queue()
>>> Process(target=f,args=(q,)).start()
in child
>>> q.put('abc')
>>> 
>>> from multiprocessing import Queue,Pipe
>>> def f(q):
...     print('in child')
...     print(q.get())
...     
>>> Process(target=f,args(q,)).start()
>>> Process(target=f,args=(q,)).start()
in child
abc

posted @ 2020-11-10 23:06  Richardo-M-Lu  阅读(133)  评论(0编辑  收藏  举报