python :控制线程上限,防止无限开启线程

import threading
sem = threading.Semaphore(10)#设置最大线程数
#在线程调用的函数开始设置sem.acquire()
#在线程调用的函数最后设置sem.release()
def xiazai(iiv,chapter_url_list):
    sem.acquire()
    file_name = bt1+str(iiv)+'.txt'
    with open(file_name, 'w', encoding='utf-8') as f:
        print("the time is : %s" % datetime.now())
        for index, chapter_url in enumerate(chapter_url_list,start=1):
            while 1:
                try:
                    item = get_chapter_detail(chapter_url)
                    break
                except:
                    time.sleep(2)
            f.write(''+item['title']+'\n')
            #f.write('原文链接: '+item['url']+'\n')
            neirong1=item['content'].replace(' ','').replace("ps:投票了兄弟们!","").replace('    ','')
            neirong2=neirong1.replace("亲,点击进去,给个好评呗,分数越高更新越快,据说给香书小说打满分的最后都找到了漂亮的老婆哦!手机站全新改版升级地址:https://wap.ibiquge.la,数据和书签与电脑站同步,无广告清新阅读!","")
            try:
                neirong3=neirong2.split('ps')[0]
            except:
                neirong3=neirong2
            try:
                neirong3=neirong3.split('PS')[0]
            except:
                neirong3=neirong3
            f.write(''+neirong3+'\n')
            #sys.stdout.write(f'进度:{index}/{len(chapter_url_list)}\r')
        print('生成文件:',file_name)
    sem.release()

#如果各线程处理同名的文件,为了防止冲突
#在开始修改指定文件时设置:lock.acquire(),修改完成后设置:lock.release
lock = threading.Lock()
def handler(fid, fpath, is_file):
    if is_file:
        lock.acquire()
        complete_files.append(fpath)
        with open('complete.log', mode='r', encoding='UTF-8') as (log):
            log_lst = log.readlines()
            log_lst_fmt = []
            for c in log_lst:
                log_lst_fmt.append(c.rstrip('\n'))

        with open('complete.log', mode='a', encoding='UTF-8') as (log):
            if fpath not in log_lst_fmt:
                log.write('{}\n'.format(fpath))
        lock.release()

 

posted @ 2022-12-09 21:51  myrj  阅读(411)  评论(0编辑  收藏  举报