python性能优化

1。去除不必要的显式for循环，使用向量化计算。

import time
import numpy as np


def for_time():
    """Make a array, len = 1000000, use for loop add one."""
    start = time.time()
    list_data = np.arange(0, 10000000, 1)
    for i in range(1000000):
        list_data[i] += 1
    print 'for loop used time: ', time.time() - start


def vector_time():
    """make a array, use vector calculation add one."""
    start = time.time()
    list_data = np.arange(0, 10000000, 1)
    list_data += 1
    print 'vector calculation used time: ', time.time() - start


if __name__ == '__main__':
    for_time()
    vector_time()

for loop used time:  0.359999895096
vector calculation used time:  0.0160000324249

 

2. 使用多进程，开核。

import multiprocessing


def use_pool(func, args):
    pool = multiprocessing.Pool(processes=2)
    res = pool.map(func, args)
    pool.close()
    pool.join()
    return res

3.使用sklearn.extenals.joblib 扩展库

from sklearn.externals.joblib import Parallel, delayed


def parallel(func, arg):
    Parallel(-1)(delayed(func)(i) for i in arg)

4. 使用bottleneck库。

该库基于Cpython实现，着眼于高性能。