python urllib2 实现大文件下载

 

使用urllib2下载并分块copy：

# from urllib2 import urlopen # Python 2

from urllib.request import urlopen # Python 3



response = urlopen(url)

CHUNK = 16 * 1024

with open(file, 'wb') as f:

    while True:

        chunk = response.read(CHUNK)

        if not chunk:

            break

        f.write(chunk)

 

 

另一种大文件copy方式， shutil：

import shutil

try:

    from urllib.request import urlopen # Python 3

except ImportError:

    from urllib2 import urlopen # Python 2



def get_large_file(url, file, length=16*1024):

    req = urlopen(url)

    with open(file, 'wb') as fp:

        shutil.copyfileobj(req, fp, length)

 

关于shutil的一些介绍：https://www.cnblogs.com/zhangboblogs/p/7821702.html

 

使用urlib2并显示下载进度，以视频为例：

#coding:utf-8
import urllib 
import urllib2 
import requests
import random 
import uuid
import time
import sys
from threading import Thread

#img_url = "https://p.ssl.qhimg.com/dm/48_48_100/t017aee03b28107657b.jpg"


img="http://vip.zuiku8.com/1810/妖精的尾巴最终季-01.mp4"

my_headers={
    "User-Agent":"Mozilla/5.0 (Windows NT 6.3; Win64; x64) \
    AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36",
   'Referer':'https://www.bilibili.com/bangumi/play/ep250436',
}


def chunk_report(bytes_so_far, chunk_size, total_size):
   percent = float(bytes_so_far) / total_size
   percent = round(percent*100, 2)
   if percent %1==0:
      sys.stdout.write("Downloaded %d of %d bytes (%0.2f%%)\n" % 
           (bytes_so_far, total_size, percent))

   if bytes_so_far >= total_size:
      sys.stdout.write('\n')

def chunk_read(response, url,chunk_size=8192, report_hook=None):
   total_size = response.info().getheader('Content-Length').strip()
   total_size = int(total_size)
   bytes_so_far = 0
   path_name=url.split("/")[-1]
   path_name=path_name.replace("\n","")
   path_name=path_name.decode("utf-8")
   print path_name
   with open("%s" % path_name, "wb") as f:
      while 1:
         chunk = response.read(chunk_size)
         f.write(chunk)
         f.flush() 
         bytes_so_far += len(chunk)
         if not chunk:
            break
         if report_hook:
            report_hook(bytes_so_far, chunk_size, total_size)
   return bytes_so_far

def down_load(img):
        print img
        request =  urllib2.Request(url=img, headers=my_headers)
        response = urllib2.urlopen(request);
        chunk_read(response,img, report_hook=chunk_report)
        print "downloading with urllib  --->"

if __name__ == '__main__':
     down_load(img)

 

结果：

 

 

 如果想在一行显示，打印时加\r，end为空 ：

print ('\r downloading...{:.1f}'.format(percent), end="")

\r 是移至本行行首
\b 是退一个字符

 

 

此外，提一下urllib，之后没有用它，是因为不支持https：

#!/usr/bin/python
#encoding:utf-8
import urllib
import os

img="http://vip.zuiku8.com/1810/妖精的尾巴最终季-01.mp4"

def Schedule(a,b,c):
   '''
   a:已经下载的数据块
   b:数据块的大小
   c:远程文件的大小
   '''
   per = 100.0*a*b/c
   if per > 100:
      per = 100
   print '%.2f%%' % per


def main():
   path=img.split(".")[-1]
   urllib.urlretrieve(img,path,Schedule)

if __name__ == '__main__':
   main()

 

 

使用urllib2时，发现https下载不成功，添加了如下代码：

import ssl
ssl._create_default_https_context = ssl._create_unverified_context