简单Http多线程下载实现

闲着没事试着写写，本来想应该挺简单的，但一写就折腾大半天。

Http要实现多线程现在需要WebHost对HttpHeader中Range支持，有些资源不支持Range头就必须顺序下载。

协议参考 rfc2616：http://www.ietf.org/rfc/rfc2616.txt 

 

大概步骤：

 1.检测Range支持，同时获取长度

 2. 通过长度创建一个相当大小的文件

 3. 创建线程组

 4. 分隔文件

 5. 各个线程获取一个文件块任务（比较小），读取后放在内存中，完成块后写入文件，再领取下一个文件块任务

 6. 直到全部块都完成

*如果将完成进度实时持久化，启动的时候加载进度，那么就能断的续传了。

  线程组用异步IO 代替，为了简便任务用Stack管理块得分配，失败后再次push回，按照大概的顺序关系下载，直到栈空结束下载。实现比较简单，实际网络情况复杂，还有很多功能需要完善，不多说直接贴代码吧：

 1.

  class ConcurrentDownLoader
    {
        public Uri ToUri { get; set; }

        public string SavePath { get; set; }

        private int _contentLength;
        public int ContentLength { get{return _contentLength;} }

        private bool _acceptRanges;
        public bool AcceptRanges { get { return _acceptRanges; } }

        private int _maxThreadCount;
        public int MaxThreadCount { get{return _maxThreadCount;} }

        public event Action OnDownLoaded;

        public event Action<Exception>  OnError;

        private FileStream saveFileStream;
        private object _syncRoot = new object();

        public ConcurrentDownLoader(Uri uri, string savePath)
            : this(uri, savePath, 5)
        { 
        
        }

        public ConcurrentDownLoader(Uri uri,string savePath,int maxThreadCount)
        {
            ToUri = uri;
            SavePath = savePath;
            _maxThreadCount = maxThreadCount;

            ServicePointManager.DefaultConnectionLimit = 100;
        }

     
        public void DownLoadAsync()
        {
            _acceptRanges = CheckAcceptRange();
            
            if (!_acceptRanges) //可以使用顺序下载
                throw new Exception("resource cannot allow seek");
            
            //create a File the same as ContentLength
            if (File.Exists(SavePath))
                throw new Exception(string.Format("SavePath:{0} already exists",SavePath));
            saveFileStream = File.Create(SavePath);
            saveFileStream.SetLength(ContentLength);
            saveFileStream.Flush();

            PartManager pm = new PartManager(this);
            pm.OnDownLoaded += () => 
            {
                saveFileStream.Close();
                if (OnDownLoaded != null) ;
                OnDownLoaded();
            };

            pm.OnError+=(ex)=>
                {
                    saveFileStream.Close();
                if(OnError!=null)
                    OnError(ex);
                };

            pm.Proc();
        }

         public void WriteFile(DPart part, MemoryStream mm)
        {
            lock (_syncRoot)
            {
                try
                {
                    mm.Seek(0, SeekOrigin.Begin);
                    saveFileStream.Seek(part.BeginIndex, SeekOrigin.Begin);
                    byte[] buffer = new byte[4096];
                    int count = 0;
                    while ((count = mm.Read(buffer, 0, buffer.Length)) > 0)
                        saveFileStream.Write(buffer, 0, count);

                    saveFileStream.Flush();

                    Console.WriteLine("写入：{0}~{1} 成功",part.BeginIndex,part.EndIndex);   
                }
                catch (Exception ex)
                {
                    Console.WriteLine("{0},Write Error 这该咋办呢？？ fu*K",ex.Message);   
                }
            }
        }

        //检测资源是否支持断点续传
        public bool CheckAcceptRange()
        {
            bool isRange = false;

            //同步方式，取到应答头即可
            HttpWebRequest req = (HttpWebRequest)WebRequest.Create(ToUri);
            WebResponse rsp = req.GetResponse();
            if(rsp.Headers["Accept-Ranges"]=="bytes")
                isRange=true;

            _contentLength = (int)rsp.ContentLength;
            rsp.Close();

            return isRange;
        }

    

    }

 

2.

 class PartManager

    {
        private int partSize = 128 * 1024; //128K每份
        private ConcurrentDownLoader _loader;
        private ConcurrentStack<DPart> _dParts;
        private AsyncWebRequest[] _reqs;
        public event Action OnDownLoaded;
        public event Action<Exception> OnError;
        private int _aliveThreadCount;
        private Thread _checkComplete;

        public PartManager(ConcurrentDownLoader loader)
        {
            _loader = loader;
            _dParts = new ConcurrentStack<DPart>();
            _checkComplete = new Thread(new ThreadStart(() =>
            {
                Thread.Sleep(3 * 1000);
                while (true)
                {
                    int count = 0;
                    foreach (var req in _reqs)
                    {
                        if (req.IsComplete)
                            count++;
                    }
                    if (_reqs.Length == count)
                    {
                        if (OnDownLoaded != null)
                        {
                            OnDownLoaded();
                            _checkComplete.Abort();
                        }
                    }
                    else
                        Thread.Sleep(1000);
                }
            }));
            _checkComplete.IsBackground = true;
            _checkComplete.Start();

            //parts Data 
            if (loader.ContentLength < partSize)
                _dParts.Push(new DPart() { BeginIndex = 1, EndIndex = loader.ContentLength });
            else
            {
                int count = loader.ContentLength % partSize == 0 ? loader.ContentLength / partSize : loader.ContentLength / partSize + 1;
                for (int i = 0; i < count; i++)
                {
                    DPart p = new DPart();
                    p.BeginIndex = i * partSize;
                    p.EndIndex = (i + 1) * partSize - 1;

                    if (count - 1 == i)
                        p.EndIndex = loader.ContentLength - 1;

                    _dParts.Push(p);
                }
            }

            //建立工作线程
            _aliveThreadCount = loader.MaxThreadCount;
            _reqs = new AsyncWebRequest[loader.MaxThreadCount];
            for (int i = 0; i < loader.MaxThreadCount; i++)
            {
                _reqs[i] = new AsyncWebRequest(i, loader);
            }
        }

        public void Proc()
        {
            foreach (AsyncWebRequest req in _reqs)
            {
                if (_dParts.Count > 0)
                {
                    DPart d;
                    if (_dParts.TryPop(out d))
                    {
                        req.IsComplete = false;
                        req.BeginGetStream(Callback, d);
                    }
                    else
                        req.IsComplete = true;
                }
                else
                    req.IsComplete = true;
            }
        }

        public void Callback(AsyncWebRequest req, MemoryStream mm, Exception ex)
        {
            //一个线程如果3次都失败，就不再使用了，可能是线程数有限制，
            if (ex == null && mm != null)
            {
                //check mm size
                if ((int)mm.Length == req.EndIndex - req.BeginIndex + 1)
                {
                    _loader.WriteFile(req.Part, mm);
                    //重新分配 Part
                    if (_dParts.Count > 0)
                    {
                        DPart d;
                        if (_dParts.TryPop(out d))
                        {
                            req.BeginGetStream(Callback, d);
                        }
                    }
                    else
                    {
                        //所有Part都已经完成鸟，ok success
                        req.IsComplete = true;
                    }
                }
                else
                {
                    req.IsComplete = true;
                    Console.WriteLine("mm Length:{0}, Part Length:{1} ,Why not the same ~~~shit", mm.Length, req.EndIndex - req.BeginIndex);
                    //回收分区
                    _dParts.Push(req.Part);
                    Interlocked.Decrement(ref _aliveThreadCount);
                    if (_aliveThreadCount == 0)
                    {
                        if (OnError != null)
                            OnError(null);
                    }
                }
            }
            else
            {
                req.IsComplete = true;
                //回收分区
                _dParts.Push(req.Part);
                Interlocked.Decrement(ref _aliveThreadCount);
                if (_aliveThreadCount == 0)
                {
                    _checkComplete.Abort();
                    if (OnError != null)
                        OnError(null);
                }
            }
        }
    }

 

 3. 文件分片

    class DPart
    {
        public int BeginIndex;
        public int EndIndex;
        public bool IsComlete;
        public int tryTimes;

    }

 

4.异步WebRequst简单封装

    class AsyncWebRequest

    {
        //http://www.ietf.org/rfc/rfc2616.txt
        // suffix-byte-range-spec = "-" suffix-length
        //suffix-length = 1*DIGIT
        // Range = "Range" ":" ranges-specifier
        //Range: bytes=100-300

        //  Content-Range = "Content-Range" ":" content-range-spec
        //content-range-spec      = byte-content-range-spec
        //byte-content-range-spec = bytes-unit SP
        //                          byte-range-resp-spec "/"
        //                          ( instance-length | "*" )
        //byte-range-resp-spec = (first-byte-pos "-" last-byte-pos)
        //                               | "*"
        //instance-length           = 1*DIGIT

        //      HTTP/1.1 206 Partial content
        //Date: Wed, 15 Nov 1995 06:25:24 GMT
        //Last-Modified: Wed, 15 Nov 1995 04:58:08 GMT
        //Content-Range: bytes 21010-47021/47022
        //Content-Length: 26012
        //Content-Type: image/gif

        private int _threadId;
        public int ThreadId { get { return _threadId; } }

        public bool IsComplete { get; set; }

        private ConcurrentDownLoader _loader;
        public ConcurrentDownLoader Loader { get { return _loader; } }

        private DPart _part;
        public DPart Part { get { return _part; } }
        private int _beginIndex;
        public int BeginIndex { get { return _beginIndex; } }

        private int _endIndex;
        public int EndIndex { get { return _endIndex; } }

        private int _tryTimeLeft;

        private Action<AsyncWebRequest, MemoryStream, Exception> _onResponse;
        private HttpWebRequest _webRequest;
        private MemoryStream _mmStream;
        private Stream _rspStream;
        public AsyncWebRequest(int threadId, ConcurrentDownLoader loader)
        {
            _threadId = threadId;
            _loader = loader;
        }

        public void BeginGetStream(Action<AsyncWebRequest, MemoryStream, Exception> rep, DPart part)
        {
            IsComplete = false;
            _beginIndex = part.BeginIndex;
            _endIndex = part.EndIndex;
            _part = part;
            _onResponse = rep;
            _tryTimeLeft = 3;
            DoRequest();
        }

        private void DoRequest()
        {
            _webRequest = (HttpWebRequest)WebRequest.Create(Loader.ToUri);
            _webRequest.AddRange(BeginIndex, EndIndex);
            //  _webRequest.Headers.Add(string.Format("Range: bytes={0}-{1}", BeginIndex, EndIndex));
            _mmStream = new MemoryStream(EndIndex - BeginIndex);

            Console.WriteLine("开始获取{0}-{1}段", BeginIndex, EndIndex);

            _webRequest.BeginGetResponse((result) =>
            {
                try
                {
                    HttpWebRequest req = result.AsyncState as HttpWebRequest;
                    WebResponse rsp = req.EndGetResponse(result);
                    //验证Content-Range: bytes的正确性
                    string contentRange = rsp.Headers["Content-Range"];
                    // (\d+)\-(\d+)\/(\d+) Match Content-Range: bytes 21010-47021/47022
                    Regex reg = new Regex(@"(\d+)\-(\d+)\/(\d+)");
                    Match mc = reg.Match(contentRange);
                    if (mc.Groups.Count == 4)
                    {
                        int bid = Convert.ToInt32(mc.Groups[1].Value);
                        int eid = Convert.ToInt32(mc.Groups[2].Value);
                        int len = Convert.ToInt32(mc.Groups[3].Value);
                        if (bid == BeginIndex && eid == EndIndex && Loader.ContentLength == len)
                            Console.WriteLine("开始获取{0}-{1}段时返回成功，Content-Range:{2}", BeginIndex, EndIndex, contentRange);
                        else
                            throw new Exception(string.Format("开始获取{0}-{1}段时返回失败，Content-Range 验证错误:{2}", BeginIndex, EndIndex, contentRange));
                    }
                    else
                    {
                        throw new Exception("return Content-Range Error :" + contentRange);
                    }

                    Console.WriteLine("开始获取{0}-{1}段时返回成功，开始读取数据", BeginIndex, EndIndex);

                    _rspStream = rsp.GetResponseStream();
                    byte[] buffer = new byte[4096];
                    _rspStream.BeginRead(buffer, 0, 4096, EndReadStream, buffer);
                }
                catch (Exception ex)
                {
                    if (_tryTimeLeft > 0)
                    {
                        Console.WriteLine("获取{0}-{1}失败，ex:{2},重试", BeginIndex, EndIndex, ex.Message);
                        _tryTimeLeft--;
                        _rspStream.Close();
                        _rspStream = null;
                        _webRequest = null;
                        DoRequest();
                    }
                    else
                    {
                        Console.WriteLine("获取{0}-{1}失败，ex:{2},已经重试3次放弃~~", BeginIndex, EndIndex, ex.Message);
                        if (_onResponse != null)
                            _onResponse(this, null, ex);
                    }
                }
            }, _webRequest);
        }

        private void EndReadStream(IAsyncResult result)
        {
            try
            {
                byte[] buffer = result.AsyncState as byte[];
                int count = _rspStream.EndRead(result);
                if (count > 0)
                {
                    Console.WriteLine("读取{0}-{1}段数据中，读取到：{2}字节，continue···", BeginIndex, EndIndex, count);
                    _mmStream.Write(buffer, 0, count);
                    _rspStream.BeginRead(buffer, 0, 4096, EndReadStream, buffer);
                }
                else
                {
                    //OK now all is back
                    if (_onResponse != null)
                        _onResponse(this, _mmStream, null);
                }
            }
            catch (Exception ex)
            {
                if (_tryTimeLeft > 0)
                {
                    Console.WriteLine("获取{0}-{1}失败，ex:{2},重试", BeginIndex, EndIndex, ex.Message);
                    _tryTimeLeft--;
                    if (_rspStream != null)
                        _rspStream.Close();
                    _rspStream = null;
                    _webRequest = null;
                    DoRequest();
                }
                else
                {
                    Console.WriteLine("获取{0}-{1}失败，ex:{2},已经重试3次放弃~~", BeginIndex, EndIndex, ex.Message);
                    if (_onResponse != null)
                        _onResponse(this, _mmStream, ex);
                }
            }
        }
    }

 

自己测试100多M的文件开10个线程可以正常下载，因为仓促写的，没经过大量测试可能还有很多没考虑的地方，仅供参考。