python多进程详解和协程

1、由于python多线程适合于多IO操作，但不适合于cpu计算型工作，这时候可以通过多进程实现。python多进程简单实用

# 多进程，可以cpu保持一致，python多线程适合多io.对于高cpu的可以通过多进程实现。
import  multiprocessing
import time

def run(name):
    print(" %s process is running "%(name))
    time.sleep(2)



if  __name__ == '__main__':
    for i in range(1, 10):
        process = multiprocessing.Process(target=run, args=(i,))
        process.start()

2、python多进程。在操作系统中所有的进程都是有根进程（进程号0来创建的）。python获取主进程号和子进程号

# 多进程，可以cpu保持一致，python多线程适合多io.对于高cpu的可以通过多进程实现。
import  multiprocessing
import time
import os

def run(name):
    print(" %s process is running "%(name))
    time.sleep(2)

def info(name):
    print("%s"%(name))
    print("当前进程号：%s"%(os.getpid()))
    print("父进程号：%s"%(os.getppid()))





if  __name__ == '__main__':
    info("parent process")
    process = multiprocessing.Process(target=info,args=("child process",))
    process.start()

E:\Users\xiajinqi\PycharmProjects\Atm\venv\Scripts\python.exe E:/Users/xiajinqi/PycharmProjects/Atm/333.py
parent process
当前进程号：23420
父进程号：2372
child process
当前进程号：23468
父进程号：23420

Process finished with exit code 0

 

3、queue 实现进程通讯，进程Q要实现通讯。必须指明为进程Q。原理：在创建子进程时候，传递一个Q，实际相当于复制了一个新的Q给子进程。子进程向新的Q输入数据时候。进程Q通过一个反系列化实现将新的数据同步到旧的Q(即父进程Q)。两个程序之间内存是隔离的。Q也是相互独立的。

# 多进程，可以cpu保持一致，python多线程适合多io.对于高cpu的可以通过多进程实现。
from   multiprocessing import Process,Queue # 只名为 进程Q
import time
import os

qu = Queue()


def test(qu):
    qu.put(["克隆Q"])




if  __name__ == '__main__':

    process = Process(target=test,args=(qu,))
    process.start()
    print(qu.get())

E:\Users\xiajinqi\PycharmProjects\Atm\venv\Scripts\python.exe E:/Users/xiajinqi/PycharmProjects/Atm/333.py
['克隆Q']

Process finished with exit code 0

 4、Pipe()实现管实现进程通讯。

# Pipe()
from   multiprocessing import Process,Queue,Pipe # 只名为 进程Q
import time
import os


def test(child_conn):
    child_conn.send(['hello world'])
    child_conn.send(['hello world'])
    child_conn.close()



if  __name__ == '__main__':
    parent_conn,child_conn = Pipe()
    process = Process(target=test,args=(child_conn,))
    process.start()
    message1 = parent_conn.recv()
    message2 = parent_conn.recv()
    print(message1)
    print(message2)

E:\Users\xiajinqi\PycharmProjects\Atm\venv\Scripts\python.exe E:/Users/xiajinqi/PycharmProjects/Atm/333.py
['hello world']
['hello world']

Process finished with exit code 0

　5、Manager 实现进程通讯，实现修改同一个数据。不需要锁

# Mangager
from   multiprocessing import Process,Queue,Pipe,Manager # 只名为 进程Q
import time
import os


def test(d,l):
    d[os.getpid()] = os.getpid()
    l.append( os.getpid())
    print(d)
    print(l)



if  __name__ == '__main__':
    with Manager()  as  manager:
        d = manager.dict()
        l = manager.list(range(10))
        p_list = []
        for i in range(0, 10):
            process = Process(target=test, args=(d, l))
            process.start()
            p_list.append(process)

        for process in p_list:
            process.join()



E:\Users\xiajinqi\PycharmProjects\Atm\venv\Scripts\python.exe E:/Users/xiajinqi/PycharmProjects/Atm/333.py
{17876: 17876}
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 17876]
{11504: 11504, 17876: 17876}
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 17876, 11504]
{11504: 11504, 26212: 26212, 17876: 17876}
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 17876, 11504, 26212]
{11504: 11504, 26212: 26212, 24244: 24244, 17876: 17876, 19900: 19900}
{11504: 11504, 26212: 26212, 24244: 24244, 17876: 17876, 19900: 19900}
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 17876, 11504, 26212, 24244, 19900]
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 17876, 11504, 26212, 24244, 19900]
{11504: 11504, 11824: 11824, 26212: 26212, 24244: 24244, 19900: 19900, 17876: 17876}
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 17876, 11504, 26212, 24244, 19900, 11824]
{11504: 11504, 11824: 11824, 26212: 26212, 27592: 27592, 24244: 24244, 19900: 19900, 17876: 17876}
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 17876, 11504, 26212, 24244, 19900, 11824, 27592]
{11504: 11504, 11824: 11824, 26212: 26212, 27592: 27592, 24244: 24244, 19900: 19900, 17876: 17876, 3248: 3248}
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 17876, 11504, 26212, 24244, 19900, 11824, 27592, 3248]
{11504: 11504, 11824: 11824, 26212: 26212, 26312: 26312, 27592: 27592, 24244: 24244, 19900: 19900, 17876: 17876, 3248: 3248}
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 17876, 11504, 26212, 24244, 19900, 11824, 27592, 3248, 26312]
{11504: 11504, 11824: 11824, 26212: 26212, 14320: 14320, 26312: 26312, 27592: 27592, 24244: 24244, 19900: 19900, 17876: 17876, 3248: 3248}
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 17876, 11504, 26212, 24244, 19900, 11824, 27592, 3248, 26312, 14320]

Process finished with exit code 0

6、进程池pool的使用，每启动一个进程就会消耗一定资源。使用pool可以避免机器崩溃发生。

pool = Pool(5) # 进程池5个，启动100个进程，只有在进程池中的5个才会执行。而不是同时执行100个，其他挂起状态。下面每次会打印5个

# Mangager
from   multiprocessing import Process,Queue,Pipe,Manager,Pool,freeze_support # 只名为 进程Q
import time
import os


def test(test):
    time.sleep(1)
    print("test")

def bar(arg):
    print("子进程执行完以后，回调后开始执行",arg)


if  __name__ == '__main__':
    freeze_support()
    pool = Pool(5) # 进程池5个，启动100个进程，只有在进程池中的5个才会执行。而不是同时执行100个，其他挂起状态，避免机器崩溃
    for i in range(0, 10):
        # pool.apply()串行
        pool.apply_async(func=test,args=(test,),callback=bar)  #并行

    pool.close()
    pool.join()





E:\Users\xiajinqi\PycharmProjects\Atm\venv\Scripts\python.exe E:/Users/xiajinqi/PycharmProjects/Atm/333.py
test
子进程执行完以后，回调后开始执行 None
test
子进程执行完以后，回调后开始执行 None
test
子进程执行完以后，回调后开始执行 None
test
子进程执行完以后，回调后开始执行 None
test
子进程执行完以后，回调后开始执行 None
test
子进程执行完以后，回调后开始执行 None
test
子进程执行完以后，回调后开始执行 None
test
子进程执行完以后，回调后开始执行 None
test
子进程执行完以后，回调后开始执行 None
test
子进程执行完以后，回调后开始执行 None

Process finished with exit code 0

 

7、__name__ = "__main__" :手工执行时候。模块名为main。其他调用时候则是模块名。

8、协程本质上是单线程，是一种轻量级线程，协程有自己的寄存器上下文和堆栈。切换时候，将寄存器上下文和堆栈保存到其他地方。切换回来时候。恢复之前保存的寄存器上下文内容。协程能保留上一次执行时候的状态。

协程的好处：

  无需线程上下文切换的开销（cpu）

  无需原子锁定以及同步的开销

 方便切换控制流，简化编程模型

 高并发+高拓展性+低成本。一个CPU支持上万的携程不是问题。

缺点：

   无法利用多核资源。因为协程本质就是一个单线程。需要和进程配合才能运行在多CPU上。

  进行阻塞操作时候会阻塞整个程序。。

 

#yiled实现的协程
# Mangager
from   multiprocessing import Process,Queue,Pipe,Manager,Pool,freeze_support # 只名为 进程Q
import time
import os

def  consumer(name) :
    print("test")
    while True:
        new_baozi =yield
        print("%s is eating %s"%(name,new_baozi))


def producer():
    # 函数中如果有yiled，系统判定为生成器，需要用next才会执行
    r = con1.__next__()
    r = con2.__next__()

    n= 0
    while n < 5:
        n+1
        time.sleep(2)
        #send 唤醒生成器。并且给他传递一个值
        con1.send(n)
        con2.send(n)


if __name__ =="__main__":
    con1 = consumer("test1")
    con2 = consumer("test1")
    p = producer()

8、手动切换协greenlet

# Mangager
from   multiprocessing import Process,Queue,Pipe,Manager,Pool,freeze_support # 只名为 进程Q
from greenlet import greenlet
import time
import os

def test1():
    print("test1")
    t2.switch()
    time.sleep(1) #为iO占用时间
    print("test11")
    t2.switch()



def test2():
    print("tets2")
    t1.switch()
    time.sleep(1)
    print("test22")



t1 = greenlet(test1)
t2 = greenlet(test2)
t1.switch()

9、gevent 自动切换（前面的封装）gevent 每个1秒切换

# Mangager
from   multiprocessing import Process,Queue,Pipe,Manager,Pool,freeze_support # 只名为 进程Q
import gevent
import time
import os

def test1():
    print("test1")
    gevent.sleep(1)
    print("test11")




def test2():
    print("tets2")
    gevent.sleep(1)
    print("test22")

#启动携程加入队列
gevent.joinall([gevent.spawn(test1),agevent.spawn(test2)])

# Mangager
from   multiprocessing import Process,Queue,Pipe,Manager,Pool,freeze_support # 只名为 进程Q
import gevent
import time
import os

def test1():
    print("test1")
    gevent.sleep(3)
    print("test11")




def test2():
    print("tets2")
    gevent.sleep(1)
    print("test22")

#启动携程加入队列
gevent.joinall([gevent.spawn(test1),gevent.spawn(test2)])

E:\Users\xiajinqi\PycharmProjects\Atm\venv\Scripts\python.exe E:/Users/xiajinqi/PycharmProjects/Atm/333.py
test1
tets2
test22
test11

Process finished with exit code 0

10 、  gevent 实际应用时间测试。爬多个网页

# Mangager
from   multiprocessing import Process,Queue,Pipe,Manager,Pool,freeze_support # 只名为 进程Q
import gevent
import time
import os
from urllib  import request #
from gevent  import monkey # 这个补丁才可以让gevent捕获IO

def f(url):
    res = request.urlopen(url)

url = [
    "https://www.baidu.com",
    "https://www.yylending.com",
    "https://www.yyfax.com"
]

start_time = time.time()
gevent.joinall([gevent.spawn(f,"https://www.baidu.com"),gevent.spawn(f,"https://www.yylending.com"),gevent.spawn(f,"https://www.yyfax.com")])
print("并行",(time.time()-start_time))


start_time = time.time()
for i in url:
    f(i)
print("串行",(time.time()-start_time))

E:\Users\xiajinqi\PycharmProjects\Atm\venv\Scripts\python.exe E:/Users/xiajinqi/PycharmProjects/Atm/333.py
并行 1.3772048950195312
串行 0.9304625988006592

Process finished with exit code 0