# 将详细页的内容分成三组,三线程处理
def content_split(**arg_list):
# 目的将list顺序分组,分成3-4个组把数据写入文件
try:
all_list = arg_list.get('all_list_content')
meta1 = arg_list.get('meta')
title = arg_list.get('title')
print(f"meta {arg_list['meta']}")
print(f'contents size is {len(all_list)}')
if len(all_list) <= 3:
write_list(meta1=meta1, title=title, dt=all_list)
return
res = []
step = len(all_list) / 3
last_end = 0
for i in range(3):
end_step = step * (i + 1)
if i == 0:
# i start end last
# 0 0 8 8
# 1 8 16 16
# 2 16 24 24
start_step = 0
res.append(all_list[int(start_step):int(end_step):1])
last_end = end_step
else:
start_step = last_end
res.append(all_list[int(start_step):int(end_step):1])
last_end = end_step
print(f'start_step: {start_step} , end_step:{end_step}, last_end:{last_end} ')
# 处理落单情况
if len(all_list) > last_end:
res.append(all_list[int(last_end)::1])
# 发起多线程
print(' res 的len is ', len(res))
arg_list.pop('all_list_content')
for dt in res:
#是为了解决meta问题
t = threading.Thread(target=write_list, kwargs={'meta1': meta1, 'title': title, 'dt': dt})
t.start()
print(f'当前运行的线程是{threading.currentThread().name} is running ')
except Exception as e:
print('content split exception-------------')