python--再看并行之协程线程进程
1、gevent协程适合I/O密集，不适合CPU密集。
3、gevent协程无法发挥多核优势，事实上，协程只是以单线程的方式在运行。
3、子程序就是协程的一种特例
项目实际应用
from gevent import monkey
from gevent.monkey import patch_all
from gevent.pool import Pool
import time
def lr_classify():
print('lr_classify')
def rf_classify():
print('rf_classify')
def svm_classify():
print('svm_classify')
def decisionTree_classify():
print('decisionTree_classify')
def gbt_classify():
print('gbt_classify')
def naive_bayes_classify():
print('naive_bayes_classify')
model_dict = {'logistic':lr_classify
,'random_forest':rf_classify
,'linear_svm':svm_classify
,'decision_tree':decisionTree_classify
,'gbt':gbt_classify
,'naive_bayes':naive_bayes_classify}
def get_task(names):
return [task for name,task in model_dict.items() if name in names]
# return model_dict.get(name,'no such model')
start = time.time()
model_list = ['logistic','gbt', 'naive_bayes', 'linear_svm', 'decision_tree', 'random_forest']
names = model_dict.keys()&(set(model_list))
tasks = get_task(names)
p = Pool(6)
# jobs = [i for i in range(10)]
for task in tasks:
p.spawn(task)
# for task in tasks:
# p.apply_async(task,(None,))
# p.spawn
p.join()
end = time.time()
print('cost {} seconds in total...'.format((end-start)))
#输出结果如下：
lr_classify
svm_classify
gbt_classify
naive_bayes_classify
decisionTree_classify
rf_classify
cost 0.0018892288208007812 seconds in total...
线程进程协程比较
# 多线程
import threading
import time
def loop_5(interval):
for i in range(5):
print('loop_5: ',i)
time.sleep(interval)
def loop_10(interval):
for i in range(10):
print('loop_10: ',i)
time.sleep(interval)
if __name__ == '__main__':
print('start...:')
start = time.time()
threads = []
tasks = [loop_5,loop_10]
for task in tasks:
t = threading.Thread(target=task,args=(1,))
threads.append(t)
t.start()
for t in threads:
t.join()
end = time.time()
print('end...and cost {} seconds in total...'.format((end-start)))
# 输出如下
start...:
loop_5: loop_10: 00
loop_5: loop_10: 1
1
loop_10: 2
loop_5: 2
loop_10: 3
loop_5: 3
loop_10: 4
loop_5: 4
loop_10: 5
loop_10: 6
loop_10: 7
loop_10: 8
loop_10: 9
end...and cost 10.02077603340149 seconds in total...
#多进程
from multiprocessing import Pool
import time
def loop_5(interval):
for i in range(5):
print('loop_5: ',i)
time.sleep(interval)
def loop_10(interval):
for i in range(10):
print('loop_10: ',i)
time.sleep(interval)
if __name__ == '__main__':
print('start ...')
start = time.time()
p = Pool(2)
tasks = [loop_5,loop_10]
for task in tasks:
p.apply_async(task,args=(1,))
p.close()
p.join()
end = time.time()
print('end...cost {} seconds in total...'.format((end-start)))
# 然而，发现多进程仍然耗费 10 秒左右,不难理解啊，因为loop_5虽然跟10交替跑，但是还是要等待10跑完才会主进程结束啊。。。
start ...
loop_5: 0
loop_10: 0
loop_5: 1
loop_10: 1
loop_5: 2
loop_10: 2
loop_5: 3
loop_10: 3
loop_5: 4
loop_10: 4
loop_10: 5
loop_10: 6
loop_10: 7
loop_10: 8
loop_10: 9
end...cost 10.231749534606934 seconds in total...
### 不使用进程池
from multiprocessing import Process
import time
def loop_5(interval):
for i in range(5):
print('loop_5: ',i)
time.sleep(interval)
def loop_10(interval):
for i in range(10):
print('loop_10: ',i)
time.sleep(interval)
if __name__ == '__main__':
print('start ...')
start = time.time()
tasks = [loop_5,loop_10]
processes = []
for task in tasks:
p = Process(target=task,args=(1,))
processes.append(p)
p.start()
for p in processes:
p.join()
end = time.time()
print('end...cost {} seconds in total...'.format((end-start)))
#输出结果如下：
start ...
loop_5: 0
loop_10: 0
loop_5: 1
loop_10: 1
loop_5: 2
loop_10: 2
loop_5: 3
loop_10: 3
loop_5: 4
loop_10: 4
loop_10: 5
loop_10: 6
loop_10: 7
loop_10: 8
loop_10: 9
end...cost 10.139782667160034 seconds in total...
协程进行文件复制
from gevent.pool import Pool
import time
def copy_file(src,target):
with open(src,'r') as fr:
with open(target,'w') as fw:
for line in fr:
fw.write(line)
if __name__ == '__main__':
print('start...')
start = time.time()
p = Pool(6)
args = [('./test.py','./test2.py'),('./test.py','./test3.py')]
for arg in args:
p.spawn(copy_file,*arg)
p.join()
end = time.time()
print('cost {} seconds in total...'.format((end-start)))
多线程进行文件复制
import threading
import asyncio
import time
def copy(src,tar):
print('{} start...'.format(threading.current_thread().name))
with open(src,'rb') as binFileInputStream:
with open(tar,'wb') as binFileOutputStream:
binFileOutputStream.write(binFileInputStream.read())
time.sleep(10)
print('{} end...'.format(threading.current_thread().name))
threads = []
args = [('./test.py','./hella.py'),('./diabetes.csv','./diabetes.py')]
for i,arg in enumerate(args):
t = threading.Thread(target=copy,args=arg,name='thread-{}'.format(i))
threads.append(t)
for thread in threads:
thread.start()
for thread in threads:
thread.join()
多进程
from multiprocessing import Pool
import time
import random
import os
def copy_file_multiprocess(src,target):
with open(src,'rb') as fr:
with open(target,'wb') as fw:
print('{} start to copy file...'.format(os.getpid()))
fw.write(fr.read())
time.sleep(random.random()*3)
print('pid {} finished...'.format(os.getpid()))
if __name__ == '__main__':
p = Pool(3)
args = [('./config.txt','./babao/config.txt'),('./config.txt','./babao/config2.txt'),('./config.txt','./babao/config3.txt')]
for i in range(3):
p.apply_async(copy_file_multiprocess,args[i])
print('Waiting for all subprocesses done...')
p.close()
p.join()
print('All suprocesses finished!')
扩展知识
Python多线程编程时经常会用到join()和setDaemon()方法，基本用法如下：
join([time]): 等待至线程中止。这阻塞调用线程直至线程的join() 方法被调用中止-正常退出或者抛出未处理的异常-或者是可选的超时发生。
setDaemon，将该线程标记为守护线程或用户线程
1、join ()方法：主线程A中，创建了子线程B，并且在主线程A中调用了B.join()，那么，主线程A会在调用的地方等待，直到子线程B完成操作后，才可以接着往下执行，那么在调用这个线程时可以使用被调用线程的join方法。
原型：join([timeout])，里面的参数时可选的，代表线程运行的最大时间，即如果超过这个时间，不管这个此线程有没有执行完毕都会被回收，然后主线程或函数都会接着执行的。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
import threading
import time
class MyThread(threading.Thread):
def __init__(self, id):
threading.Thread.__init__(self)
self.id = id
def run(self):
x = 0
time.sleep(10)
print(self.id)
print('线程结束：'+str(time.time()))
if __name__ == "__main__":
t1 = MyThread(999)
print('线程开始：'+str(time.time()))
t1.start()
print('主线程打印开始：'+str(time.time()))
for i in range(5):
print(i)
time.sleep(2)
print('主线程打印结束：' + str(time.time()))
线程开始：1497534590.2784667
主线程打印开始：1497534590.2794669
0
1
2
3
4
主线程打印结束：1497534592.279581
999
线程结束：1497534600.2800388
从打印结果可知，线程t1 start后，主线程并没有等线程t1运行结束后再执行，而是在线程执行的同时，执行了后面的语句。
现在，把join()方法加到启动线程后面(其他代码不变)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
import threading
import time
class MyThread(threading.Thread):
def __init__(self, id):
threading.Thread.__init__(self)
self.id = id
def run(self):
x = 0
time.sleep(10)
print(self.id)
print('线程结束：'+str(time.time()))
if __name__ == "__main__":
t1 = MyThread(999)
print('线程开始：'+str(time.time()))
t1.start()
t1.join()
print('主线程打印开始：'+str(time.time()))
for i in range(5):
print(i)
time.sleep(2)
print('主线程打印结束：' + str(time.time()))
线程开始：1497535176.5019968
999
线程结束：1497535186.5025687
主线程打印开始：1497535186.5025687
0
1
2
3
4
主线程打印结束：1497535188.5026832
线程t1 start后，主线程停在了join()方法处，等子线程t1结束后，主线程继续执行join后面的语句。
2、setDaemon()方法。主线程A中，创建了子线程B，并且在主线程A中调用了B.setDaemon()，这个的意思是，把主线程A设置为守护线程，这时候，要是主线程A执行结束了，就不管子线程B是否完成，一并和主线程A退出.这就是setDaemon方法的含义，这基本和join是相反的。此外，还有个要特别注意的：必须在start() 方法调用之前设置。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
import threading
import time
class MyThread(threading.Thread):
def __init__(self, id):
threading.Thread.__init__(self)
self.id = id
def run(self):
x = 0
time.sleep(10)
print(self.id)
print("This is：" + self.getName()) # 获取线程名称
print('线程结束：' + str(time.time()))
if __name__ == "__main__":
t1 = MyThread(999)
print('线程开始：'+str(time.time()))
t1.setDaemon(True)
t1.start()
print('主线程打印开始：'+str(time.time()))
for i in range(5):
print(i)
time.sleep(2)
print('主线程打印结束：' + str(time.time()))
线程开始：1497536678.8509264
主线程打印开始：1497536678.8509264
0
1
2
3
4
主线程打印结束：1497536680.8510408
t1.setDaemon(True)的操作，将子线程设置为了守护线程。根据setDaemon()方法的含义，父线程打印内容后便结束了，不管子线程是否执行完毕了。
如果在线程启动前没有加t1.setDaemon(True)，输出结果为：
线程开始：1497536865.3215919
主线程打印开始：1497536865.3215919
0
1
2
3
4
主线程打印结束：1497536867.3217063
999
This is：Thread-1
线程结束：1497536875.3221638
程序运行中，执行一个主线程，如果主线程又创建一个子线程，主线程和子线程就分兵两路，分别运行，那么当主线程完成想退出时，会检验子线程是否完成，如果子线程未完成，则主线程会等待子线程完成后再退出；
有时我们需要的是，子线程运行完，才继续运行主线程，这时就可以用join方法（在线程启动后面）；
但是有时候我们需要的是，只要主线程完成了，不管子线程是否完成，都要和主线程一起退出，这时就可以用setDaemon方法（在线程启动前面）。
这里一个坑 (pool.map 中不能使用匿名函数)， 就是 lambda 无法被 pickle，可见匿名函数并不是到处通用 ，正确的 下面再贴出
# -*- coding: utf-8 -*-
__author__ = 'Frank Li'
from functools import wraps,reduce
import threading as td
import multiprocessing as mp
from multiprocessing import Queue
from multiprocessing import Pool
import time
def time_count(func):
@wraps(func)
def inner(*args,**kw):
start = time.time()
result = func(*args,**kw)
end = time.time()
num = kw.get('num','default')
print('func: {}-{} cost {:.2f}s\n'.format(func.__name__,num,end-start))# 这里可以插入日志
return result
return inner
@time_count
def multicore(data,q,**kw):
q.put(reduce(lambda x,y:x+y,data))
@time_count
def multi_thread(data,q,**kw):
q.put(reduce(lambda x, y: x + y, data))
@time_count
def main_process():
q = Queue()
datas = [range(10**5),range(10**6),range(10**7),range(10**8)]
processes = []
for i,data in enumerate(datas):
p = mp.Process(target=multicore,name='process-{}'.format(i),args=(data,q),kwargs={'num':i})
p.start()
processes.append(p)
for p in processes:
p.join()
sum = 0
for n in range(len(datas)):
sum += q.get()
print('main_process sum result: {}\n'.format(sum))
@time_count
def main_thread():
q = Queue()
datas = [range(10 ** 5), range(10 ** 6), range(10 ** 7), range(10 ** 8)]
threads = []
for i,data in enumerate(datas):
t = td.Thread(target=multi_thread,name='thread-{}'.format(i),args=(data,q),kwargs={'num':i})
t.start()
threads.append(t)
for t in threads:
t.join()
sum = 0
for n in range(len(datas)):
sum += q.get()
print('main_thread sum result: {}\n'.format(sum))
def another_multi_process_sub(data):
return reduce(lambda x, y: x + y, data)
@time_count
def another_multi_process():
pool = Pool(processes=2)
datas = [range(10 ** 5), range(10 ** 6), range(10 ** 7), range(10 ** 8)]
results = []
for data in datas:
res = pool.map(lambda data:lambda x,y:x+y,(data,))
results.extend(res)
sum = reduce(lambda x,y:x+y,results)
print('another_multi_process sum result: {}'.format(sum))
@time_count
def multi_process_pool():
pool = Pool(processes=2)
datas = [range(10 ** 5), range(10 ** 6), range(10 ** 7), range(10 ** 8)]
dataset= []
sum = reduce(lambda x,y:x+y,[ pool.apply_async(another_multi_process_sub,(data,)).get() for data in datas])
print('multi_process_pool sum result: {}'.format(sum))
def main():
# main_process()
# main_thread()
another_multi_process()
# multi_process_pool()
@time_count
def single_process():
datas = [range(10 ** 5), range(10 ** 6), range(10 ** 7), range(10 ** 8)]
sum = 0
for data in datas:
sum += reduce(lambda x,y:x+y,data)
print('\n single process sum result: {}'.format(sum))
if __name__ == '__main__':
main()
single_process()
对比多进程 多线程 效率 ， 总结 优先使用顺序 (多进程+协程>多进程 >= 单进程多线程 >=单进程单线程)
# -*- coding: utf-8 -*-
__author__ = 'Frank Li'
from functools import wraps,reduce
import threading as td
import multiprocessing as mp
from multiprocessing import Queue
from multiprocessing import Pool
import time
def time_count(func):
@wraps(func)
def inner(*args,**kw):
start = time.time()
result = func(*args,**kw)
end = time.time()
num = kw.get('num','default')
print('func: {}-{} cost {:.2f}s\n'.format(func.__name__,num,end-start))# 这里可以插入日志
return result
return inner
@time_count
def multicore(data,q,**kw):
q.put(reduce(lambda x,y:x+y,data))
@time_count
def multi_thread(data,q,**kw):
q.put(reduce(lambda x, y: x + y, data))
@time_count
def main_process():
q = Queue()
datas = [range(10**5),range(10**6),range(10**7),range(10**8)]
processes = []
for i,data in enumerate(datas):
p = mp.Process(target=multicore,name='process-{}'.format(i),args=(data,q),kwargs={'num':i})
p.start()
processes.append(p)
for p in processes:
p.join()
sum = 0
for n in range(len(datas)):
sum += q.get()
print('main_process sum result: {}\n'.format(sum))
@time_count
def main_thread():
q = Queue()
datas = [range(10 ** 5), range(10 ** 6), range(10 ** 7), range(10 ** 8)]
threads = []
for i,data in enumerate(datas):
t = td.Thread(target=multi_thread,name='thread-{}'.format(i),args=(data,q),kwargs={'num':i})
t.start()
threads.append(t)
for t in threads:
t.join()
sum = 0
for n in range(len(datas)):
sum += q.get()
print('main_thread sum result: {}\n'.format(sum))
def another_multi_process_sub(data):
return reduce(lambda x, y: x + y, data)
@time_count
def another_multi_process():
pool = Pool(processes=2)
datas = [range(10 ** 5), range(10 ** 6), range(10 ** 7), range(10 ** 8)]
results = []
for data in datas:
res = pool.map(another_multi_process_sub,(data,))
results.extend(res)
sum = reduce(lambda x,y:x+y,results)
print('another_multi_process sum result: {}'.format(sum))
@time_count
def multi_process_pool():
pool = Pool(processes=2)
datas = [range(10 ** 5), range(10 ** 6), range(10 ** 7), range(10 ** 8)]
dataset= []
sum = reduce(lambda x,y:x+y,[ pool.apply_async(another_multi_process_sub,(data,)).get() for data in datas])
print('multi_process_pool sum result: {}'.format(sum))
def main():
main_process()
main_thread()
another_multi_process()
multi_process_pool()
@time_count
def single_process():
datas = [range(10 ** 5), range(10 ** 6), range(10 ** 7), range(10 ** 8)]
sum = 0
for data in datas:
sum += reduce(lambda x,y:x+y,data)
print('\n single process sum result: {}'.format(sum))
if __name__ == '__main__':
main()
single_process()
下面结果 可能跟我电脑有其他进程没有关闭有关系，只是贴出一下，这个结果仁者见仁智者见智
# 结果如下 ：
func: multicore-0 cost 0.06s
func: multicore-1 cost 0.50s
func: multicore-2 cost 3.51s
func: multicore-3 cost 27.56s
main_process sum result: 5050504944450000
func: main_process-default cost 28.36s
func: multi_thread-0 cost 0.08s
func: multi_thread-1 cost 0.67s
func: multi_thread-2 cost 5.00s
func: multi_thread-3 cost 25.31s
main_thread sum result: 5050504944450000
func: main_thread-default cost 25.36s
another_multi_process sum result: 5050504944450000
func: another_multi_process-default cost 24.31s
multi_process_pool sum result: 5050504944450000
func: multi_process_pool-default cost 25.84s
single process sum result: 5050504944450000
func: single_process-default cost 25.58s
多核cpu 共享内存 ，进程通信安全问题， p1, p2 都有可能 抢到资源，一个抢到 一顿执行完事儿，第二个才能接着执行
# -*- coding: utf-8 -*-
__author__ = 'Frank Li'
import multiprocessing as mp
import time
'''
进程间共享内存，加锁
'''
def add_num(v,num,num_lock):
with num_lock:
for _ in range(10):
v.value+=num
time.sleep(0.1)
print(v.value)
if __name__ == '__main__':
v = mp.Value('i',0) ### 还有一个 mp.Array
num_lock = mp.Lock()
p1 = mp.Process(target=add_num,args=(v,1,num_lock))
p2 = mp.Process(target=add_num,args=(v,3,num_lock))
p1.start()
p2.start()
p1.join()
p2.join()
线程 操作共享变量 安全 加锁
# -*- coding: utf-8 -*-
__author__ = 'Frank Li'
import threading
def thread_job1():
print('current threading... {}'.format(threading.current_thread()))
global A,lock
lock.acquire()
for i in range(10):
A+=1
print('jb1 A value: {}'.format(A))
lock.release()
def thread_job2():
print('current threading... {}'.format(threading.current_thread()))
global A,lock
lock.acquire()
for j in range(20):
A+=5
print('job2 value of A： {}'.format(A))
lock.release()
def main():
global A,lock
A = 0
lock = threading.Lock()
t1 = threading.Thread(target=thread_job1,name='thread-1')
t2 = threading.Thread(target=thread_job2,name='thread-2')
t1.start()
t2.start()
t1.join()
t2.join()
if __name__ == '__main__':
main()
多线程
# -*- coding: utf-8 -*-
__author__ = 'Frank Li'
import threading
from threading import Thread
from queue import Queue
def thread_job(q,l=None):
print(threading.current_thread())
for i in range(len(l)):
l[i] = l[i]**2
q.put(l)
def multithreading(datas=None):
q = Queue()
threads = [] # 线程 列表
# print(datas)
for i,data in enumerate(datas):
print(data)
thread = Thread(target=thread_job,name='thread{}'.format(i),args=(q,data))
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
result = []
for _ in range(len(datas)):
result.append(q.get())
print(result)
if __name__ == '__main__':
datas = [[1,2],[3,4],[5,6],[7,8]]
multithreading(datas)
如果有来生，一个人去远行，看不同的风景，感受生命的活力。。。
