python多进程

python多进程学习笔记

---

1.队列

队列数据的存入与取出

import multiprocessing
import random
import time

class producer(multiprocessing.Process):
    def __init__(self,queue):
        multiprocessing.Process.__init__(self)
        self.queue=queue

    def run(self):
        for i in range(10):
            item=random.randint(0,256)
            self.queue.put(item)
            print('Process producer:item %d appended queue %s'%(item,self.queue))
            time.sleep(1)
            print('The size of queue is %s'%self.queue.qsize())


class consumer ( multiprocessing.Process ):
    def __init__(self, queue):
        multiprocessing.Process.__init__ ( self )
        self.queue = queue

    def run(self):
        while True:
            if (self.queue.empty()):
                print('queue is empty')
                break
            else:
                time.sleep(2)
                #queue get method is popped from queue
                item=self.queue.get()
                print('Process consumer: item %d popped from by %s '%(item,self.name))
                time.sleep(1)


if __name__=='__main__':
    queue=multiprocessing.Queue()
    process_producer=producer(queue)
    process_consumer=consumer(queue)
    process_producer.start()
    process_consumer.start()
    process_producer.join()
    process_consumer.join()

2.管道

从一个管道的output段输出到另一个队列的input端

import multiprocessing

def create_item(pipe):
    output_pipe,_=pipe
    for item in range(10):
        output_pipe.send(item)
    output_pipe.close()


def multiply_items(pipe_1,pipe_2):
    output,input_pipe=pipe_1
    output.close()

    output_pipe,_=pipe_2
    try:
        while True:
            item=input_pipe.recv()
            output_pipe.send(item**2)
    except EOFError:
        output_pipe.close()


if __name__=='__main__':

    #first pipe with numbers from 0 to 9
    pipe_1=multiprocessing.Pipe(True)
    process_pipe_1=multiprocessing.Process(target=create_item,args=(pipe_1,))
    process_pipe_1.start()

    #second pipe
    pipe_2=multiprocessing.Pipe(True)
    process_pipe_2=multiprocessing.Process(target=multiply_items,args=(pipe_1,pipe_2,))
    process_pipe_2.start()

    pipe_1[0].close()
    pipe_2[0].close()

    try:
        while True:
            #print(pipe_1[1].recv())
            print('-----------------------')
            print(pipe_2[1].recv())
    except EOFError:
        print('End')

3.端到端数据发送与接收

from mpi4py import MPI

#the task can communicate with each other
comm=MPI.COMM_WORLD
#to identify a task or processes inside the group ,we use the rank value
rank=comm.Get_rank()

print('my rabk is :',rank)

if rank==0:
    data=10000
    destiantion_process=4
    comm.send(data,dest=destiantion_process)
    print('sending data %s :'%data+'to process %d'%destiantion_process)

if rank==1:
    data='hello'
    destiantion_process=8
    comm.send(data,dest=destiantion_process)
    print('sending data %s :'%data+'to process %d'%destiantion_process)

if rank==4:
    data=comm.recv(source=0)
    print('data received is= %s :'%data)

if rank==8:
    data=comm.recv(source=1)
    print ( 'data received is= %s :' % data )

输出

C:\Users\杨景\Desktop\multiprocess>mpiexec -n 9 python mpi.py
my rabk is : 6
my rabk is : 5
my rabk is : 3
my rabk is : 2
my rabk is : 7
my rabk is : 0
sending data 10000 :to process 4
my rabk is : 1
sending data hello :to process 8
my rabk is : 4
data received is= 10000 :
my rabk is : 8
data received is= hello :