多线程下载图片

TransferWebFileHelper1

  1 public static class TransferWebFileHelper1
  2     {
  3         /// <summary>
  4         /// 异步网络请求
  5         /// </summary>
  6         public static async Task CommunicatingWithWebServerAsync(string url,string saveFilePath, bool isGet = true, string postData = "", string contentType = "application/x-www-form-urlencoded;charset=utf-8")
  7         {
  8             if (string.IsNullOrWhiteSpace(url))
  9                 throw new ArgumentNullException("url");
 10             if (string.IsNullOrWhiteSpace(contentType))
 11                 throw new ArgumentNullException("contentType");
 12             HttpWebResponse response = null;
 13             Stream stream = Stream.Null;
 14             FileStream outStream = null;
 15             try
 16             {
 17                 if (url.ToUpper().Contains("HTTPS:")) //解决：“访问https请求被中止: 未能创建 SSL/TLS 安全通道”
 18                 {
 19                     ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls |
 20                                                            SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;
 21                 }
 22                 HttpWebRequest request = isGet
 23                     ? GenerateHttpWebRequest(new Uri(url))
 24                     : GenerateHttpWebRequest(new Uri(url), postData, contentType);
 25                 response = await request.GetResponseAsync() as HttpWebResponse;
 26                 if (CategorizeResponse(response) == ResponseCategories.Success)
 27                 {
 28                     stream = response.GetResponseStream();
 29 
 30                     outStream = new FileStream(saveFilePath, FileMode.Create);
 31                     int readByteCount = 0;
 32                     byte[] buffers = new byte[1024];
 33                     while (true)
 34                     {
 35                         readByteCount = stream.Read(buffers, 0, buffers.Length);
 36                         if (readByteCount <= 0)
 37                         {
 38                             break;
 39                         }
 40                         outStream.Write(buffers, 0, readByteCount);
 41                     }
 42 
 43                     stream.Close();
 44                     stream.Dispose();
 45                     outStream.Close();
 46                     outStream.Dispose();
 47                     response.Close();
 48                     response.Dispose();
 49                 }
 50 
 51 
 52             }
 53             catch (Exception ex)
 54             {
 55 
 56             }
 57             finally
 58             {
 59                 System.GC.Collect();//解决程序提示线程用尽崩溃的问题
 60             }
 61         }
 62 
 63         public static HttpWebRequest GenerateHttpWebRequest(Uri uri)
 64         {
 65             HttpWebRequest httpRequest = (HttpWebRequest)WebRequest.Create(uri);
 66             httpRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36";//部分网站会检查此项
 67             return httpRequest;
 68         }
 69 
 70         public static HttpWebRequest GenerateHttpWebRequest(Uri uri,
 71             string postData,
 72             string contentType)
 73         {
 74             HttpWebRequest httpRequest = GenerateHttpWebRequest(uri);
 75 
 76             byte[] bytes = Encoding.UTF8.GetBytes(postData);
 77 
 78             httpRequest.ContentType = contentType;
 79 
 80             httpRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022)";           
 81 
 82             httpRequest.ContentLength = postData.Length;
 83 
 84             using (Stream requestStream = httpRequest.GetRequestStream())
 85             {
 86                 requestStream.Write(bytes, 0, bytes.Length);
 87             }
 88             return httpRequest;
 89         }
 90 
 91         public static ResponseCategories CategorizeResponse(HttpWebResponse httpResponse)
 92         {
 93             int statusCode = (int)httpResponse.StatusCode;
 94             if ((statusCode >= 100) && (statusCode <= 199))
 95             {
 96                 return ResponseCategories.Informational;
 97             }
 98             else if ((statusCode >= 200) && (statusCode <= 299))
 99             {
100                 return ResponseCategories.Success;
101             }
102             else if ((statusCode >= 300) && (statusCode <= 399))
103             {
104                 return ResponseCategories.Redirected;
105             }
106             else if ((statusCode >= 400) && (statusCode <= 499))
107             {
108                 return ResponseCategories.ClientError;
109             }
110             else if ((statusCode >= 500) && (statusCode <= 599))
111             {
112                 return ResponseCategories.ServerError;
113             }
114             return ResponseCategories.Unknown;
115         }
116     }

Enumerator

    public class Enumerator
    {
        public enum HttpMethodEnum
        {
            Post,
            Get,
            Put,
            Delete,
            Patch
        }
    }

ResponseCategories

    public enum ResponseCategories
    {
        Unknown,        
        Informational, 
        Success,       
        Redirected,     
        ClientError,    
        ServerError
    }

Program

class Program
    {
        private static string[] randomArray = new[]
        {
            "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V",
            "W", "X", "Y", "Z"
        };
        private static string GetRandomStr()
        {
            Random ran = new Random();
            int i = ran.Next(0, 26);
            Thread.Sleep(1);//尽量让生成的字符每一个都不相同
            return randomArray[i];
        }

        static List<DownloadFile> ReadFileUrl(string folderName)
        {
            List<DownloadFile> list = new List<DownloadFile>();
            DirectoryInfo dir;
            List<string> contents = new List<string>();
            string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName);
            if (Directory.Exists(path))
            {
                dir = new DirectoryInfo(path);
                FileInfo[] files = dir.GetFiles();
                contents = File.ReadAllLines(files[0].FullName, Encoding.Default).ToList();

                string saveFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName + "_download");

                contents.ForEach(fileName =>
                {
                    //TODO 完全限定文件名必须少于 260 个字符，并且目录名必须少于 248 个字符。
                    string tempFileName = fileName.Split('?')[0].Replace("\\", "").Replace(":", "").Replace("*", "").Replace("\"", "").Replace("<", "").Replace(">", "").Replace("|", "");
                    int index = tempFileName.LastIndexOf("/");
                    tempFileName = tempFileName.Substring(index + 1, tempFileName.Length - index - 1);
                    int tempFileNameLength = tempFileName.Length;
                    if ((260 - saveFolder.Length - 2) < tempFileNameLength)
                    {
                        int difference = tempFileNameLength - (260 - saveFolder.Length - 2);//差
                        tempFileName = tempFileName.Substring(difference, tempFileName.Length - difference);
                    }
                    tempFileName = GetRandomStr() + "_" + tempFileName;


                    string saveFileName = Path.Combine(saveFolder, tempFileName);
                    var model = new DownloadFile(fileName, folderName, saveFileName);
                    list.Add(model);
                });
            }
            return list;
        }

        private static void CreateFolder(string folderName)
        {
            string saveFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName + "_download");
            if (!Directory.Exists(saveFolder))
            {
                Directory.CreateDirectory(saveFolder);
            }
        }

        private static void Main(string[] args)
        {
            List<string> folders = new List<string>()
            {
                "demo"
            };
            Parallel.ForEach(folders, folder => CreateFolder(folder));

            List<DownloadFile> downloadFiles = new List<DownloadFile>();

            Parallel.ForEach(folders, folder =>
            {
                downloadFiles.AddRange(ReadFileUrl(folder));
            });

            int i = 0;
            Task[] taskList = new Task[downloadFiles.Count];
            downloadFiles.ForEach(downloadFile =>
            {
                try
                {
                    Task t = TransferWebFileHelper1.CommunicatingWithWebServerAsync(downloadFile.FileName,downloadFile.SaveFileName);
                    taskList[i] = t;

                }
                catch (Exception ex)
                {

                }
                i++;
            });
            Task.WaitAll(taskList);
        }
    }

 

测试

程序根目录下，新建一个文件夹，名demo。内新建一个txt文件，文件名随意。测试如下图片链接[随意]：

http://www.banktunnel.eu/tumblr.com/tabea-lara.2_files/tumblr_n4kxopXkJ41sq93cpo10_1280.jpg
http://www.pptok.com/wp-content/uploads/2012/08/xunguang-9.jpg
http://www.xiangshu.com/UploadFiles/Day_190324/52_387785_3163ee0990499b7.jpg
http://www.xiangshu.com/UploadFiles/Day_190324/52_387785_25d8b5e402d0391.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WVPNFNWIA9vCAAmNhS2C8eAAAPyvQApsb4ACY2d098.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WlPNFOGIIc0VAAom_BUlna4AAPyvQCLDjYACicU104.png
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WlPNFOqIWH2cAAXWIl1DvA4AAPyvQDbhzkABdY6485.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WVPNFPGIfiurAAKxaF37_vcAAPyvQGII2YAArGA075.jpg
http://www.hr.com.cn/attachments/2015/06/b16af76b691f47609f3397e8cdd9c1e2.jpg
http://www.hr.com.cn/attachments/2015/06/226fc8acee0d478c9588ded1d04616b1.jpg
http://www.hr.com.cn/attachments/2015/06/c36e39b67d4c410cb794f553a271377b.jpg

下载完成的图片存放在根目录下名demo_download的文件夹下

测试过支持10W左右的图片下载，不崩溃。

 

posted @ 2019-03-25 15:38  jeff151013  阅读(582)  评论(0)  编辑  收藏  举报