python实现RabbitMQ同步跟异步消费模型

1，消息推送类

 1 import pika
 2 
 3 
 4 # 同步消息推送类
 5 class RabbitPublisher(object):
 6 
 7     # 传入RabbitMQ的ip，用户名，密码，实例化一个管道
 8     def __init__(self, host, user, password):
 9         self.host = host
10         self.user = user
11         self.password = password
12         self.connection = pika.BlockingConnection(pika.ConnectionParameters(host=self.host, credentials=pika.PlainCredentials(self.user, self.password)))
13         self.channel = self.connection.channel()
14 
15     # 发送消息在队列中
16     def send(self, queue_name, body):
17         self.channel.queue_declare(queue=queue_name, durable=True)  # 声明一个持久化队列
18         self.channel.basic_publish(exchange='',
19                                    routing_key=queue_name,  # 队列名字
20                                    body=body,  # 消息内容
21                                    properties=pika.BasicProperties(
22                                        delivery_mode=2,  # 消息持久化
23                                    ))
24 
25     # 清除指定队列的所有的消息
26     def purge(self, queue_name):
27         self.channel.queue_purge(queue_name)
28 
29     # 删除指定队列
30     def delete(self, queue_name, if_unused=False, if_empty=False):
31         self.channel.queue_delete(queue_name, if_unused=if_unused, if_empty=if_empty)
32 
33     # 断开连接
34     def stop(self):
35         self.connection.close()
View Code

2.消息消费类

（1）同步消息消费

 在同步消息消费的时候可能会出现pika库断开的情况，原因是因为pika客户端没有及时发送心跳，连接就被server端断开了。解决方案就是做一个心跳线程来维护连接。

心跳线程类

 1 class Heartbeat(threading.Thread):
 2 
 3     def __init__(self, connection):
 4         super(Heartbeat, self).__init__()
 5         self.lock = threading.Lock()  # 线程锁
 6         self.connection = connection  # rabbit连接
 7         self.quitflag = False  # 退出标志
 8         self.stopflag = True  # 暂停标志
 9         self.setDaemon(True)  # 设置为守护线程，当消息处理完，自动清除
10 
11     # 间隔10s发送心跳
12     def run(self):
13         while not self.quitflag:
14             time.sleep(10)  # 睡10s发一次心跳
15             self.lock.acquire()  # 加线程锁
16             if self.stopflag:
17                 self.lock.release()
18                 continue
19             try:
20                 self.connection.process_data_events()  # 一直等待服务段发来的消息
21             except Exception as e:
22                 print "Error format: %s" % (str(e))
23                 self.lock.release()
24                 return
25             self.lock.release()
26 
27     # 开启心跳保护
28     def startheartbeat(self):
29         self.lock.acquire()
30         if self.quitflag:
31             self.lock.release()
32             return
33         self.stopflag = False
34         self.lock.release()
View Code

消息消费类

 1 # 同步消息消费类
 2 class RabbitConsumer(object):
 3 
 4     # 传入RabbitMQ的ip，用户名，密码，实例化一个管道
 5     def __init__(self, host, user, password):
 6         self.host = host
 7         self.user = user
 8         self.password = password
 9         self.connection = pika.BlockingConnection(pika.ConnectionParameters(host=self.host, credentials=pika.PlainCredentials(self.user, self.password)))
10         self.channel = self.connection.channel()
11 
12     # 进行消费
13     def receive(self, queue_name, callback_worker, prefetch_count=1):  # callback_worker为消费的回调函数
14         self.channel.queue_declare(queue=queue_name, durable=True)
15         self.channel.basic_qos(prefetch_count=prefetch_count)  # 设置预取的数量，如果为0则不预取，消费者处理越快，可以将这个这设置的越高
16         self.channel.basic_consume(callback_worker, queue=queue_name)  # callback_worker为消费的回调函数
17         heartbeat = Heartbeat(self.connection)  # 实例化一个心跳类
18         heartbeat.start()  # 开启一个心跳线程，不传target的值默认运行run函数
19         heartbeat.startheartbeat()  # 开启心跳保护
20         self.channel.start_consuming()  # 开始消费
View Code

调用方法

# 消费回调函数
def callback(ch, method, properties, body):
    print(" [x] Received %r" % body)
    ch.basic_ack(delivery_tag=method.delivery_tag)  # 告诉生产者处理完成

consumer = RabbitConsumer(host="12.12.12.12", user="test", password="123")
consumer.receive(queue_name="queue1", callback_worker=callback)

（2）异步消息消费（推荐）

pika提供了支持异步发送模式的selectconnection方法支持异步发送接收（通过回调的方式）

在连接的时候stop_ioloop_on_close=False需要低版本的pika，比如0.13.1，安装方式 pip install pika==0.13.1

connectioon建立时回调建立channel， channel建立时一次回调各种declare方法，declare建立时依次回调publish。

同使用blockconnection方法相比，通过wireshark抓包来看，使用 异步的方式会对发包进行一些优化，会将几个包合并成一个大包，然后做一次ack应答从而提高效率，与之相反使用blockconnection时将会做至少两次ack，head一次content一次等

因此再试用异步的方式时会获得一定的优化 

异步消息消费类

  1 # 异步消息消费类
  2 class RabbitConsumerAsync(object):
  3     EXCHANGE = 'amq.direct'
  4     EXCHANGE_TYPE = 'direct'
  5 
  6     def __init__(self, host, user, password, queue_name="fish_test", callback_worker=None, prefetch_count=1):
  7         self.host = host
  8         self.user = user
  9         self.password = password
 10         self._connection = None
 11         self._channel = None
 12         self._closing = False
 13         self._consumer_tag = None
 14         self.QUEUE = queue_name
 15         self.callbackworker = callback_worker
 16         self.prefetch_count = prefetch_count
 17 
 18     def connect(self):
 19         return pika.SelectConnection(pika.ConnectionParameters(host=self.host, credentials=pika.PlainCredentials(self.user, self.password)), self.on_connection_open,
 20                                      stop_ioloop_on_close=False)
 21 
 22     def on_connection_open(self, unused_connection):
 23         self.add_on_connection_close_callback()
 24         self.open_channel()
 25 
 26     def add_on_connection_close_callback(self):
 27         self._connection.add_on_close_callback(self.on_connection_closed)
 28 
 29     def on_connection_closed(self, connection, reply_code, reply_text):
 30         self._channel = None
 31         if self._closing:
 32             self._connection.ioloop.stop()
 33         else:
 34             self._connection.add_timeout(5, self.reconnect)
 35 
 36     def reconnect(self):
 37         self._connection.ioloop.stop()
 38         if not self._closing:
 39             self._connection = self.connect()
 40             self._connection.ioloop.start()
 41 
 42     def open_channel(self):
 43         self._connection.channel(on_open_callback=self.on_channel_open)
 44 
 45     def on_channel_open(self, channel):
 46         self._channel = channel
 47         self._channel.basic_qos(prefetch_count=self.prefetch_count)
 48         self.add_on_channel_close_callback()
 49         self.setup_exchange(self.EXCHANGE)
 50 
 51     def add_on_channel_close_callback(self):
 52         self._channel.add_on_close_callback(self.on_channel_closed)
 53 
 54     def on_channel_closed(self, channel, reply_code, reply_text):
 55         print reply_text
 56         self._connection.close()
 57 
 58     def setup_exchange(self, exchange_name):
 59         self._channel.exchange_declare(self.on_exchange_declareok, exchange_name, self.EXCHANGE_TYPE, durable=True)
 60 
 61     def on_exchange_declareok(self, unused_frame):
 62         self.setup_queue()
 63 
 64     def setup_queue(self):
 65         self._channel.queue_declare(self.on_queue_declareok, self.QUEUE, durable=True)
 66 
 67     def on_queue_declareok(self, method_frame):
 68         self._channel.queue_bind(self.on_bindok, self.QUEUE, self.EXCHANGE, self.QUEUE)
 69 
 70     def on_bindok(self, unused_frame):
 71         self.start_consuming()
 72 
 73     def start_consuming(self):
 74         self.add_on_cancel_callback()
 75         self._consumer_tag = self._channel.basic_consume(self.on_message, self.QUEUE)
 76 
 77     def add_on_cancel_callback(self):
 78         self._channel.add_on_cancel_callback(self.on_consumer_cancelled)
 79 
 80     def on_consumer_cancelled(self, method_frame):
 81         if self._channel:
 82             self._channel.close()
 83 
 84     def on_message(self, unused_channel, basic_deliver, properties, body):
 85         self.callbackworker(body)
 86         self.acknowledge_message(basic_deliver.delivery_tag)
 87 
 88     def acknowledge_message(self, delivery_tag):
 89         self._channel.basic_ack(delivery_tag)
 90 
 91     def stop_consuming(self):
 92         if self._channel:
 93             self._channel.basic_cancel(self.on_cancelok, self._consumer_tag)
 94 
 95     def on_cancelok(self, unused_frame):
 96         self.close_channel()
 97 
 98     def close_channel(self):
 99         self._channel.close()
100 
101     def run(self):
102         self._connection = self.connect()
103         self._connection.ioloop.start()
104 
105     def stop(self):
106         self._closing = True
107         self.stop_consuming()
108         self._connection.ioloop.start()
109 
110     def close_connection(self):
111         self._connection.close()
View Code

调用方法

# 消费回调函数
def callback(body):
    print(" [x] Received %r" % body)

consumer = RabbitConsumerAsync(host="12.12.12.12", user="test", password="123", queue_name="fish_test", callback_worker=callback, prefetch_count=2)
consumer.run()

 （后面这两个可不加入）守护进程类（保证消费运行）

class CDaemon(object):
    """
    a generic daemon class.
    usage: subclass the CDaemon class and override the run() method
    stderr  表示错误日志文件绝对路径, 收集启动过程中的错误日志
    verbose 表示将启动运行过程中的异常错误信息打印到终端,便于调试,建议非调试模式下关闭, 默认为1, 表示开启
    save_path 表示守护进程pid文件的绝对路径
    """

    def __init__(self, save_path, stdin=os.devnull, stdout=os.devnull, stderr=os.devnull, home_dir='.', umask=022, verbose=1):
        self.stdin = stdin
        self.stdout = stdout
        self.stderr = stderr
        self.pidfile = save_path  # pid文件绝对路径
        self.home_dir = home_dir
        self.verbose = verbose  # 调试开关
        self.umask = umask
        self.daemon_alive = True

    def daemonize(self):
        try:
            pid = os.fork()
            if pid > 0:
                sys.exit(0)
        except OSError, e:
            sys.stderr.write('fork #1 failed: %d (%s)\n' % (e.errno, e.strerror))
            sys.exit(1)

        os.chdir(self.home_dir)
        os.setsid()
        os.umask(self.umask)

        try:
            pid = os.fork()
            if pid > 0:
                sys.exit(0)
        except OSError, e:
            sys.stderr.write('fork #2 failed: %d (%s)\n' % (e.errno, e.strerror))
            sys.exit(1)

        sys.stdout.flush()
        sys.stderr.flush()

        si = file(self.stdin, 'r')
        so = file(self.stdout, 'a+')
        if self.stderr:
            se = file(self.stderr, 'a+', 0)
        else:
            se = so

        os.dup2(si.fileno(), sys.stdin.fileno())
        os.dup2(so.fileno(), sys.stdout.fileno())
        os.dup2(se.fileno(), sys.stderr.fileno())

        def sig_handler(signum, frame):
            self.daemon_alive = False

        signal.signal(signal.SIGTERM, sig_handler)
        signal.signal(signal.SIGINT, sig_handler)

        if self.verbose >= 1:
            print 'daemon process started ...'

        atexit.register(self.del_pid)
        pid = str(os.getpid())
        file(self.pidfile, 'w+').write('%s\n' % pid)

    def get_pid(self):
        try:
            pf = file(self.pidfile, 'r')
            pid = int(pf.read().strip())
            pf.close()
        except IOError:
            pid = None
        except SystemExit:
            pid = None
        return pid

    def del_pid(self):
        if os.path.exists(self.pidfile):
            os.remove(self.pidfile)

    def start(self, *args, **kwargs):
        if self.verbose >= 1:
            print 'ready to starting ......'
        # check for a pid file to see if the daemon already runs
        pid = self.get_pid()
        if pid:
            msg = 'pid file %s already exists, is it already running?\n'
            sys.stderr.write(msg % self.pidfile)
            sys.exit(0)
        # start the daemon
        self.daemonize()
        self.run(*args, **kwargs)

    def stop(self):
        if self.verbose >= 1:
            print 'stopping ...'
        pid = self.get_pid()
        if not pid:
            msg = 'pid file [%s] does not exist. Not running?\n' % self.pidfile
            sys.stderr.write(msg)
            if os.path.exists(self.pidfile):
                os.remove(self.pidfile)
            return
        # try to kill the daemon process
        try:
            i = 0
            while 1:
                os.kill(pid, signal.SIGTERM)
                time.sleep(0.1)
                i = i + 1
                if i % 10 == 0:
                    os.kill(pid, signal.SIGHUP)
        except OSError, err:
            err = str(err)
            if err.find('No such process') > 0:
                if os.path.exists(self.pidfile):
                    os.remove(self.pidfile)
            else:
                print str(err)
                sys.exit(1)
            if self.verbose >= 1:
                print 'Stopped!'

    def restart(self, *args, **kwargs):
        self.stop()
        self.start(*args, **kwargs)

    def is_running(self):
        pid = self.get_pid()
        # print(pid)
        return pid and os.path.exists('/proc/%d' % pid)

    def run(self, *args, **kwargs):
        # NOTE: override the method in subclass
        print 'base class run()'
View Code

调用

class RabbitDaemon(CDaemon):
    def __init__(self, name, save_path, stdin=os.devnull, stdout=os.devnull, stderr=os.devnull, home_dir='.', umask=022, verbose=1):
        CDaemon.__init__(self, save_path, stdin, stdout, stderr, home_dir, umask, verbose)
        self.name = name  # 派生守护进程类的名称

    def run(self, **kwargs):
        # 新建一个队列链接
        rab_con = RabbitConsumerAysnc(queuename="test", callbackworker=liando_sf_consumer, prefetch_count=2)
        # 开启消费者进程
        rab_con.run()


p_name = 'test'  # 守护进程名称
pid_fn = '/www/rabbit/test.pid'  # 守护进程pid文件的绝对路径
err_fn = '/www/rabbit/test_err.log'  # 守护进程启动过程中的错误日志,内部出错能从这里看到
cD = RabbitDaemon(p_name, pid_fn, stderr=err_fn, verbose=1)
View Code

参考链接

https://pika.readthedocs.io/en/0.10.0/examples.html

https://www.jianshu.com/p/a4671c59351a

源码下载地址：https://github.com/sy159/RabbiyMQ.git

posted @ 2018-12-24 15:25  如何好听  阅读(4599)  评论(0)  编辑  收藏  举报