Python 多线程下载大文件

*仅实现，可优化
*因为第三方下载请求次数限制，个人设备网络限制，多线程实现后效果一般。
已知：
1、网络环境较差时，下载线程没有全部下载完成就走了后面的逻辑；（bug）
2、实现线程进度条、线程断点续传；（优化）
3、下载文件处理；（优化）

 

拆分url，total_size作为self.变量

def get_url(self):
    '''
    获取下载url
    '''
    down_token = self.download_token()
    releaseid = self.findApk()
    print("%s : %s(%s) %s" % (self.which, self.version, self.build, self.changelog))
    url = "http://download.bq04.com/apps/"+self.id+"/install?download_token="+down_token+"&release_id="+releaseid
    return url

 

def get_total_size(self):
    '''
    获取下载文件大小
    '''
    Response = requests.get(self.url, stream=True, verify=False)
    # 总大小
    total_size = int(Response.headers.get('content-length', 0))
    return total_size

 

获取每个线程下载大小。逻辑：按照线程数拆分计算每个线程下载的字节，有余数的情况下，最后几个字节放进最后一个线程

def get_thread_download(self):
    '''
    获取每个线程下载大小
    '''
    thread_size = {}
    cut_size = self.total_size // self.thread_num
    start_bytes = -1
    for i in range(1, self.thread_num + 1):
        thread_size[i] = [start_bytes + 1, cut_size * i]
        start_bytes = cut_size * i
    if start_bytes != self.total_size:
        thread_size[self.thread_num][1] = self.total_size
    return thread_size

 

def create_thread(self):
    """
    开启多线程下载
    """
    start = time()
    self.url = self.get_url()
    self.total_size = self.get_total_size()
    thread_size = self.get_thread_download()
    t = {}
    for i in range(1, self.thread_num + 1):
        t[i] = Thread(target=self.loadAPKs, args=(thread_size[i][0], thread_size[i][1], i))
        t[i].start()
    for i in range(1, self.thread_num + 1):
        t[i].join()
    end = time()
    print('总共耗费了%.2f秒.' % (end - start))

 

多线程下载函数。逻辑：根据线程的传参下载对应字节

def loadAPKs(self, start_bytes, end_bytes, tread_count):
    '''
    多线程下载ing
    '''
    paths = str(downloadPath) + "/" + str(tread_count) + ".tmp"
    header = {"Range": f"bytes={start_bytes}-{end_bytes}"}
    Req = requests.get(self.url, headers=header, stream=True, verify=False)
    file = open(paths, "wb")
    for chunk in Req.iter_content(chunk_size=512):
        if chunk:
            file.write(chunk)

 

下载完成后，以字节合并所有文件

def combine_file(self):
    '''
    合并文件
    '''
    self.create_thread()
    paths = str(downloadPath) + "/Hello_" + self.version + "_" + self.build + "_" + self.which + ".apk"
    print(paths)
    with open(paths, "ab") as f:
        for i in range(1, self.thread_num + 1):  # 根据id查找文件
            print(i)
            with open(str(downloadPath) + "/" + str(i) + ".tmp", "rb") as bytes_f:
                f.write(bytes_f.read())
    return paths