python引入线程池进行多线程操作
import sys
from glob import glob
from os import path
import random
from redis import StrictRedis
from BICDetection import biccheck
from concurrent.futures import ProcessPoolExecutor # 引入进程池 然后将进程处理过的结果存入大Redis中,这样可以在多台服务器上直接跑数据。
def get_direct_fps(fp):
with open(fp) as fd:
fps = [path.join(root_3421, i[:8], i.split()[0]) for i in fd]
fps.sort()
return fps
def get_fps(fp):
with open(fp) as fd:
return [i.strip() for i in fd]
folder = 'PWD'
fps = glob(path.join(folder, '*.wav'))
redis_conn = StrictRedis(host="IP", db=0)
def bic_check_wrapper(args):
fp, idx, label = args
if not path.exists(fp):
print(fp)
return
filename = path.basename(fp)
if redis_conn.hget(label, filename) is not None:
return
r = biccheck([fp])
flag = r['flag'].iloc[0]
redis_conn.hset(label, filename, flag)
return flag
def cu_flag_detect(suffix, nproc=30, select_num=None):
passed_fps = get_direct_fps('cu.log')
new_args = [(fp, idx, 'passed_{}'.format(suffix)) for idx, fp in enumerate(passed_fps)]
unpassed_fps = get_fps('cu_71.log')
new_args += [(fp, idx, 'unpassed_{}'.format(suffix)) for idx, fp in enumerate(unpassed_fps)]
random.seed(1)
random.shuffle(new_args)
if select_num is None:
selected = new_args
with ProcessPoolExecutor(int(nproc)) as executor:
executor.map(bic_check_wrapper, selected)
关注公众号 海量干货等你