python pandas使用chunksize异步拆分固定行数的文件
import pandas as pd import asyncio from collections import defaultdict collect = defaultdict(list) #######创建处理一个对象的方法,并返回期物 async def dealone(chunk,sas): path='/data/chaifen/testphone_%d.txt'%sas chunk.to_csv(path,index=False) return path async def submitor(chunk,sas): future = await dealone(chunk,sas) collect['res'].append(future) #print(future) chunks = pd.read_csv('/data/testphone.txt',dtype='object',chunksize=3) loop = asyncio.get_event_loop() tasks = asyncio.wait([submitor(chunk,index) for index,chunk in enumerate(chunks,1)]) loop.run_until_complete(tasks) print(collect['res'])
每天扫一扫,到店付款省钱!
动动小手支持一下作者,谢谢🙏