Python多进程

Python多进程

由于GIL的存在，Python多线程变得有些鸡肋，为了充分利用多核，可以使用multiprocessing来实现多进程

示例

# coding=utf-8
from multiprocessing import Process
import os

# 子进程
def run_proc(name):
    print 'Hello', name
    print os.getpid()

# 创建进程
p = Process(target=run_proc, args=('World',))
# 执行进程
p.start()
p.join()
print 'main process end'
print os.getpid()

使用进程池

# coding=utf-8
from multiprocessing import Pool
import os, time, random

def long_time_task(name):
    print 'task %d(%s) start...' % (os.getpid(), name)
    t1 = time.time()
    time.sleep(random.random() * 3)
    t2 = time.time()
    print 'task %d(%s) runs %0.2fs...' % (os.getpid(), name, t2-t1)

# 创建进程池
p = Pool()
for i in range(5):
    p.apply_async(long_time_task, args=(i,))
p.close()
p.join()
print 'All subprocess done'

# 运行结果
task 17752(2) start...
task 17752(2) runs 0.79s...
task 17750(0) start...
task 17750(0) runs 1.93s...
task 17753(3) start...
task 17753(3) runs 2.45s...
task 17751(1) start...
task 17751(1) runs 0.79s...
task 17751(4) start...
task 17751(4) runs 1.92s...
All subprocess done

使用队列做消息通信

# coding=utf-8
from multiprocessing import Queue
from multiprocessing import Process
import os, time, random

# 写入数据
def write(q):
    for v in ['A', 'B', 'C']:
        print 'puts %s to queue...' % v
        q.put(v)
        time.sleep(random.random())

# 读入数据
def read(q):
    while True:
        v = q.get(True)
        print 'Get %s from queue' % v

# 创建Queue，并传给子进程
q = Queue()
pw = Process(target=write, args=(q,))
pr = Process(target=read, args=(q,))
# 启动子进程pw
pw.start()
# 启动子进程pr
pr.start()
# 等待pw结束
pw.join()
# pr进程里是死循环，强行终止:
pr.terminate()