python多进程管理DEMO

在使用python的时候，为了提高效率，也为了避开python的GIL限制充分发挥多核的优势，经常会使用multiprocessing模块来实现多进程处理。效率会有很大的提升，但这样一来就引入的多进程管理的问题，比如：假定父进程设定的是启动5个子进程，但在实际运行中因为种种原因有3个子进程异常退出，此时该怎么办？运行过程中想退出整个程序的运行，怎么退出更合理，更优雅？
下面的python代码简单展示了如何使用signal来处理multiprocessing协作。

# -*- coding: utf-8 -*-
'''
Created on 2012-10-14 19:50
@summary:  multiprocessing coordinate demo
@author: JerryKwan
'''

from multiprocessing import Event, Process
import signal
import time

SUBPROCESSES = []
WORKERS = []
EXIT = Event()

def exit_handler(signum, frame):
    print "signal handler called with signal ", signum, " ", WORKERS, " ", frame
    EXIT.set()
    time.sleep(1)
    for w in WORKERS:
        print "begin to invoke stop_main_loop"
        w.stop_main_loop()

signal.signal(signal.SIGTERM, exit_handler)
signal.signal(signal.SIGINT, exit_handler)

class Worker(object):
    """docstring for Worker"""
    def __init__(self, arg = "default arg"):
        super(Worker, self).__init__()
        self.arg = arg
        self.be_exit = Event()
        self.be_exit.clear()

    def stop_main_loop(self):
        print "stop_main_loop invoked"
        self.be_exit.set()

    def main_loop(self):
        print "main_loop invoked"
        while not self.be_exit.is_set():
            print "do something"
            try:
                time.sleep(1)
            except Exception, e:
                print "exception occured, ", e

        print "exit from main_loop"


def do_work(worker):
    print "do_work invoked"
    worker.main_loop()

def create_subprocess():
    '''
    @summary: create subprocess
    @return: (worker, process)
    '''
    w = Worker()
    p = Process(target = do_work, args = (w,))
    p.daemon = True
    return (w, p)



def main():
    process_num = 2
    for i in xrange(process_num):
        w, p = create_subprocess()
        WORKERS.append(w)
        SUBPROCESSES.append(p)

    # signal.signal(signal.SIGTERM, exit_handler)
    # signal.signal(signal.SIGINT, exit_handler)
    # start
    for p in SUBPROCESSES:
        p.start()

    # for p in SUBPROCESSES:
    #     p.join()

    print "monitor subprocess"
    while not EXIT.is_set():
        for i in xrange(len(SUBPROCESSES)):
            p = SUBPROCESSES[i]
            if not p.is_alive():
                # create new process to replace the dead one
                print "%d process is dead, so i will create a new one to relace it." %i
                w, p = create_subprocess()
                WORKERS[i] = w
                SUBPROCESSES[i] = p
                p.start()
        time.sleep(1)

    print "main process exit"
    # wait subprocess exit
    # for i in xrange(process_num):
    #     while
if __name__ == '__main__':
    main()