具体实现
import math
from concurrent.futures import ProcessPoolExecutor
import requests
DEFAULT_MIN_PART = 2048 * 1024
GROUP_PART = 0
MAX_THREAD = 10
def make_file_buff(name, length):
"""
生成整个文件大小的空文件
:param name: 文件名
:param length: 总字节数
"""
with open(name, 'wb') as f:
f.seek(length - 1)
f.write(b'\x00')
def write_part(name, index, part):
"""
写入片
:param name: 文件名
:param index: 第几个片
:param part: 二进制片
"""
with open(name, 'rb+') as f:
f.seek(index * GROUP_PART + DEFAULT_MIN_PART)
f.write(part)
def down_part(url, name, index):
"""
下载片
:param url: url链接
:param name: 文件名
:param index: 下载第几个片
:return: 片的索引
"""
while True:
try:
write_part(name, index, requests.get(url, headers={'Range': 'bytes=%d-%d' % (
index * GROUP_PART + DEFAULT_MIN_PART, (index + 1) * GROUP_PART + DEFAULT_MIN_PART)}).content)
return index
except Exception as e:
print(e)
if __name__ == '__main__':
url = 'https://download.dbeaver.com/community/21.2.2/dbeaver-ce_21.2.2_amd64.deb'
file_name = url[url.rindex('/') + 1:]
res = requests.get(url, headers={'Range': 'bytes=0-%d' % DEFAULT_MIN_PART})
content_length = res.headers['Content-Range']
size = int(content_length[content_length.index('/') + 1:])
make_file_buff(file_name, size)
with open(file_name, 'rb+') as f:
f.write(res.content)
if size <= DEFAULT_MIN_PART:
exit(0)
workers = math.ceil(size / DEFAULT_MIN_PART) - 1
if workers >= 10:
GROUP_PART = math.ceil((size - DEFAULT_MIN_PART) / 10)
else:
GROUP_PART = math.ceil((size - DEFAULT_MIN_PART) / DEFAULT_MIN_PART)
workers = math.ceil((size - DEFAULT_MIN_PART) / GROUP_PART)
processPool = ProcessPoolExecutor(max_workers=MAX_THREAD)
futures = {}
for i in range(0, workers):
job = processPool.submit(down_part, url, file_name, i)
futures[job] = i
for job in futures:
re = job.result()
if re != -1:
print('片{0}下载结束'.format(re))