Python之多进程

 1.最近比较喜欢的用法：

from concurrent.futures import ProcessPoolExecutor, as_completed

ex = ProcessPoolExecutor(max_workers=50)


tasks = [ex.submit(process_function, file) for file in file_list]

results = []
finish_num = 0
start_time = time.time()
for future in as_completed(tasks):
    results.append(future.result())
    used_time = time.time() - start_time
    finish_num += 1
    if finish_num % 100 == 0:
        print('{}/{}, used {:.3}h, {:.3}s/line, ETS:{:.3}h'.format(finish_num, len(file_list),
                                                                   used_time / 3600, used_time / finish_num,
                                                                   used_time / finish_num * (
                                                                           len(file_list) - finish_num) / 3600))

 

 

2.Pool的用法

#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
@author: Shiyu Huang
@contact: huangsy13@gmail.com
@file: asy_test.py
'''


import numpy as np
import multiprocessing
from multiprocessing import Process, Pool
import time

def test_func(x):
    x = x+1
    print x
    time.sleep(3)
    print 'end'
    return x

if __name__ == '__main__':
    test_size = 20
    number = np.zeros(test_size).tolist()
    for i in range(len(number)):
        number[i] = i

    pool = Pool(processes=multiprocessing.cpu_count())

    # for i in range(test_size):
    #     pool.apply_async(test_func, (number[i]))
    resultList = pool.map(test_func,number)
    pool.close()
    pool.join()

    print resultList