Python 多生产者+多消费者模式

参考

* https://www.jianshu.com/p/8a9af2e7e1b4
* https://www.kancloud.cn/noahs/python/956687
* https://www.jianshu.com/p/a16da95a679d
* https://deepinout.com/python/python-qa/t_how-to-implement-multithreaded-queue-with-python.html

程序1

import threading
import time
import queue


def producer(name, q, data):  # 生产者，从data里面取数据，塞入队列q，如果q已满，则等待
    for i in data:
        q.put(i)
        print('Producer {} put {}; '.format(name, i))
        time.sleep(1)
    q.put(None)  # 将None放入queue作为标记给生产者用


def consumer(name, q):  # 消费者，从q里拿出数据并处理
    while True:  # 通过while的方式推动生产者不断尝试从queue中取数据
        get_result = q.get()  # 从队列里挨个取出数据
        q.task_done()  # 发送当前q中对应元素已被取出的消息，与后面的q.join()配合形成阻塞，保证q结束再执行其他程序，否则后面的q.join()将永远阻塞（后面程序无法执行）

        if get_result is None:  # 判断生产者是否已经结束，即将data的所有数据都加入queue中
            q.put(None)  # 该步很重要，当producer()放入的None被某个consumer()抽取后，其他consumer()就没有结束标志了。缺点是最后queue中始终留有结束标志
            print("All data have been tooken out!")
            break

        print('consumer {} get {}; '.format(name, get_result))  # 输出当前从queue中得到的数据
        time.sleep(0.1)


def main():
    data = list(range(10))  # 待处理的原始数据

    consumer_names = ['甲', '乙', '丙']  # 3个消费者

    q = queue.Queue(maxsize=5)  # 生成一个最大容量为5的queue

    threads = []  # 线程列表
    p = threading.Thread(target=producer, args=('生产者', q, data))  # 生成一个生产者线程对象，该生产者名为A
    threads.append(p)  # 将该线程加入线程列表

    for consumer_name in consumer_names:  # 为每一个消费者分配一个线程
        t = threading.Thread(target=consumer, args=(consumer_name, q))  # 消费者都从q里面拿数据
        threads.append(t)  # 将五个消费者线程加入线程列表

    for i in threads:  # 并列启动所有线程
        i.setDaemon(True)  # 保证子线程在主线程退出时，无论出于什么状况都强制退出
        i.start()

    for i in threads:  # 将所有线程阻塞（即，不执行完，就不执行后面程序）
        i.join()

    # 判断q里面是否还有剩余的对象没有处理（包括None），有则挨个拿出，否则q不为空后面的q.join()将一直阻塞（后面程序无法执行）
    if not q.empty():
        for i in range(q.qsize()):
            q.get()
            q.task_done()  # 不能删除，作用于前一个q.task_done()相同，

    q.join()  # 保证q阻塞，接受前面所有的q.task_done()发来的信息，否则程序一直停在该处不往下执行。（必须保证前面任何一处出现q.get()后都有q.task_done()）

    print("Program is over!")


if __name__ == '__main__':
    main()

程序2

# -*- coding:utf-8 -*-

import queue
import threading
from loguru import logger
from collections import deque

producer_count = 3
consumer_count = 9
queue = queue.Queue()
sentinels = [object() for _ in range(0, producer_count)]


class Producer(threading.Thread):
    def __init__(self, name, sentinel):
        super().__init__(name=name)
        self.queue = queue
        self.sentinel = sentinel
        logger.debug('{} 已创建'.format(name))

    def run(self):
        for item in range(1, 201):
            self.queue.put(item)
            logger.info("{} 已生产 {} 到队列".format(self.getName(), item))
        self.queue.put(self.sentinel)
        logger.debug("{} 完成生产并销毁".format(self.getName()))


class Consumer(threading.Thread):
    def __init__(self, name):
        super().__init__(name=name)
        self.queue = queue
        self.deque = deque()
        logger.debug('{} 已创建'.format(name))

    def run(self):
        while True:
            item = self.queue.get()
            if item in sentinels:
                if item not in self.deque:
                    self.deque.append(item)
                self.queue.put(item)
                if len(self.deque) == len(sentinels):
                    logger.debug('{} 已销毁'.format(self.getName()))
                    break
            else:
                logger.info("{} 已经被 {} 消费".format(item, self.getName()))


def main():
    producer_threads = []
    for i in range(0, producer_count):
        producer = Producer('生产者' + str(i + 1), sentinels[i])
        producer_threads.append(producer)
        producer.start()

    consumer_threads = []
    for i in range(0, consumer_count):
        consumer = Consumer('消费者' + str(i + 1))
        consumer_threads.append(consumer)
        consumer.start()

    for consumer in consumer_threads:
        consumer.join()

    for producer in producer_threads:
        producer.join()

    logger.debug(queue.qsize())
    while not queue.empty():
        logger.debug(queue.get())
    logger.debug(queue.empty())


if __name__ == '__main__':
    main()

程序3

from multiprocessing import Process, Queue


def square_producer(inputqueue, resultqueue):
    """
    A producer that pops numbers off the inputqueue, squares them and puts the result on resultqueue
    """
    while True:
        num = inputqueue.get()
        if num is None:
            return

        resultqueue.put(num*num)
        print("Produced", num*num)


def consumer(resultqueue):
    """
    A consumer that pops results off the resultqueue and prints them to screen
    """
    while True:
        numsq = resultqueue.get()
        if numsq is None:
            return

        print("Consumed", numsq)


num_producers = 3

# Generate input
inputqueue = Queue()
for i in range(100):
    inputqueue.put(i % 10)
for _ in range(num_producers):
    inputqueue.put(None)  # Ensures that producers terminate

resultqueue = Queue()  # For transfer of data from producer to consumer

# Set up and start producer processes
producers = [Process(target=square_producer, args=(inputqueue, resultqueue)) for _ in range(num_producers)]
for p in producers:
    p.start()

# Set up and start consumer process
consumer = Process(target=consumer, args=(resultqueue,))
consumer.start()

# Wait for producers to finish
for p in producers:
    p.join()

# Wait for consumer to finish
resultqueue.put(None)
consumer.join()

print("All done")