python :控制线程上限,防止无限开启线程
import threading sem = threading.Semaphore(10)#设置最大线程数 #在线程调用的函数开始设置sem.acquire() #在线程调用的函数最后设置sem.release() def xiazai(iiv,chapter_url_list): sem.acquire() file_name = bt1+str(iiv)+'.txt' with open(file_name, 'w', encoding='utf-8') as f: print("the time is : %s" % datetime.now()) for index, chapter_url in enumerate(chapter_url_list,start=1): while 1: try: item = get_chapter_detail(chapter_url) break except: time.sleep(2) f.write(''+item['title']+'\n') #f.write('原文链接: '+item['url']+'\n') neirong1=item['content'].replace(' ','').replace("ps:投票了兄弟们!","").replace(' ','') neirong2=neirong1.replace("亲,点击进去,给个好评呗,分数越高更新越快,据说给香书小说打满分的最后都找到了漂亮的老婆哦!手机站全新改版升级地址:https://wap.ibiquge.la,数据和书签与电脑站同步,无广告清新阅读!","") try: neirong3=neirong2.split('ps')[0] except: neirong3=neirong2 try: neirong3=neirong3.split('PS')[0] except: neirong3=neirong3 f.write(''+neirong3+'\n') #sys.stdout.write(f'进度:{index}/{len(chapter_url_list)}\r') print('生成文件:',file_name) sem.release() #如果各线程处理同名的文件,为了防止冲突 #在开始修改指定文件时设置:lock.acquire(),修改完成后设置:lock.release lock = threading.Lock() def handler(fid, fpath, is_file): if is_file: lock.acquire() complete_files.append(fpath) with open('complete.log', mode='r', encoding='UTF-8') as (log): log_lst = log.readlines() log_lst_fmt = [] for c in log_lst: log_lst_fmt.append(c.rstrip('\n')) with open('complete.log', mode='a', encoding='UTF-8') as (log): if fpath not in log_lst_fmt: log.write('{}\n'.format(fpath)) lock.release()