分组聚合使用多进程

1.创建数据。

import pandas as pd
import numpy as np
import uuid
import random

def get_id():
    return uuid.uuid1()

all_data = []
for _ in range(1000000):
    now_id = get_id()
    all_data.append([now_id, now_id, 3, 4])
    if random.randint(0,1):
        all_data.append([now_id, now_id, None, None])
    



data = pd.DataFrame(all_data)
data.columns = ['name','age','high','breadth']
print 'done'

2. 分组聚合

import time
import bottleneck as bk
import multiprocessing

# def do_pool(func, args):
#     pool = multiprocessing.Pool(2)
#     pool_res = pool.map(func, args)
#     pool.close()
#     pool.join()
#     return pool_res

# def agg_t(df):
#     return group.agg(['max'])
start = time.time()

data_grouped = data.groupby(['name','age']).agg([bk.nanmin])
print 'Start aggregation!'

# tobe_agg = [group for name, group in data_grouped if len(group) > 1]

# print len(tobe_agg)
print time.time() -start
# do_pool(agg_t,tobe_agg)