WebClient下载代理的IP+Port获取方法

在做下载的时候通常会用到代理！

如下获取下载代理的代码，其中通过cnproxy，heibai还有proxy360得到的代理IP，里面也有些获取的限制，具体怎么解决的见代码吧！

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Diagnostics;
using System.Net;
using System.IO;
using System.Threading.Tasks;
using System.Threading;
using System.Data.SqlClient;
using System.Data;
using HtmlAgilityPack;
using System.Web;
using System.Text.RegularExpressions;

namespace ProxyTools
{
    public class ProxyTest
    {
        public static bool Error = false;

        public int GetProxyTotal = 0;
        public int ProxyHashCount = 0;
        public List<string> proxyResultList;

        private int RepeateTimes = 5;       //下载测试次数
        private int MaxThread = 300;        //最大线程数
        private int FileLength;             //下载内容大小
        private readonly ProxyDetail[] Proxys;      //代理类数组
        private Uri TestUri = new Uri("http://www.baidu.com/");         //下载baidu.com来测试代理

        Regex ipPattern = new Regex(@"^\d+\.\d+\.\d+\.\d+$", RegexOptions.Compiled);
        Regex numPattern = new Regex(@"^\d+$", RegexOptions.Compiled);
        Regex numCnPattern = new Regex(@"(?<word>[a-z])=""(?<num>\d)""", RegexOptions.Compiled);
        Regex ipPortPattern = new Regex(@"(?is)(?<ip>\d+\.\d+\.\d+\.\d+)[^+]*?(?<word>(\+[a-z])+)", RegexOptions.Compiled);
        Regex dailiipPattern = new Regex(@"(?is)<td>(?<ip>(\d{1,3}\.){3}\d{1,3})</td>.*?<td>(?<port>\d+)</td>", RegexOptions.Compiled);
        Regex xkerPattern = new Regex(@"(?is)(?<ip>(\d{1,3}\.){3}\d{1,3})([:\s]|(</?div>.*?(port\d*"">)))(?<port>\d+)", RegexOptions.Compiled);
        Regex daili18Pattern = new Regex(@"(?is)<td>(?<ip>(\d{1,3}\.){3}\d{1,3})</td><td>(?<port>\d+)", RegexOptions.Compiled);
        Regex wl35Pattern = new Regex(@"(?is)<td[^>]*?>(?<ip>(\d{1,3}\.){3}\d{1,3})\s*</td>[^>]*>(?<port>\d+)", RegexOptions.Compiled);
        Regex proxiedPattern = new Regex(@"<td>(?<ip>(\d{1,3}\.){3}\d{1,3})</td>\s*?<td[^>]*>(?<port>\d+)</td>", RegexOptions.Compiled);
        Regex numnntimePattern = new Regex(@"(?<word>[a-z])=(?<num>\d)", RegexOptions.Compiled);
        Regex nntimePattern = new Regex(@"<td>(?<ip>(\d{1,3}\.){3}\d{1,3}).*?(?<word>(\+[a-z])*)\)", RegexOptions.Compiled);
        Regex realboosterPattern = new Regex(@"(?<ip>(\d{1,3}\.){3}\d{1,3}):(?<port>\d+)</td>", RegexOptions.Compiled);
        Regex simpleproxylistPattern = new Regex(@"(?<ip>(\d{1,3}\.){3}\d{1,3}).*?(?<word>(&#\d{2})+)</td>", RegexOptions.Compiled);

        public ProxyTest(int maxThread,int repeateTimes,string downUrl,CheckProxyWeb checkWeb)
        {
            this.MaxThread = maxThread;
            this.RepeateTimes = repeateTimes;
            this.TestUri = new Uri(downUrl);

            FileLength = GetFileLength();           //得到下载测试内容的大小

            if (FileLength == 0)
            {
                Error = true;
                return;
            }

            string[] pArray = GetProxyFromWeb(checkWeb).ToArray();   //读取需要测试的代理列表

            List<ProxyDetail> pList = new List<ProxyDetail>();

            foreach (var item in pArray)
            {
                WebProxy proxy;
                try { proxy = new WebProxy(item); }
                catch { continue; }

                ProxyDetail pDetail = new ProxyDetail();
                pDetail.Proxy = proxy;
                pDetail.ProxyString = item;
                pList.Add(pDetail);
            }

            proxyResultList = pList.Select(a => a.ProxyString).ToList();
            ProxyHashCount = pList.Count;
            Proxys = pList.ToArray();           //得到需要测试的代理列表
        }

        //得到下载测试内容的大小
        private int GetFileLength()
        {
            try
            {

                WebClient client = new WebClient();
                byte[] datas = client.DownloadData(TestUri);
                return datas.Length;
            }
            catch { return 0; }
        }

        //从cnProxy得到代理地址
        private HashSet<string> GetFromCnproxy()
        {
            HashSet<string> proxyHash = new HashSet<string>();

            WebClient client = new WebClient();
            string DefaultUserAgent = @"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1";
            string DefaultPostContentType = "application/x-www-form-urlencoded";
            client.Credentials = CredentialCache.DefaultCredentials;
            client.Headers.Add("User-Agent", DefaultUserAgent);
            client.Headers.Add("Content-Type", DefaultPostContentType);

            for (int pageIndex = 1; pageIndex < 11; ++pageIndex)
            {
                Uri uri = new Uri(string.Format("http://www.cnproxy.com/proxy{0}.html", pageIndex));

                client.Headers.Add("Referer", uri.AbsoluteUri);
                string content;
                try
                {
                    content = Encoding.GetEncoding("gb2312").GetString(client.DownloadData(uri));
                }
                catch { continue; }
                //该网站的代理端口是通过js得到的
                Dictionary<string, string> wordToNum = new Dictionary<string, string>();
                foreach (Match m in numCnPattern.Matches(content))
                    if (!wordToNum.ContainsKey(m.Groups["word"].Value))
                        wordToNum.Add(m.Groups["word"].Value, m.Groups["num"].Value);

                //正则匹配到代理地址等
                foreach (Match m in ipPortPattern.Matches(content))
                {
                    try
                    {
                        string ip = m.Groups["ip"].Value + ":";
                        string[] words = m.Groups["word"].Value.Split(new char[] { '+' }, StringSplitOptions.RemoveEmptyEntries);
                        foreach (var word in words)
                            ip += wordToNum[word];
                        proxyHash.Add(ip);
                    }
                    catch { }
                }
            }
            return proxyHash;
        }

        //通过heibai网站得到代理地址
        private HashSet<string> GetFromHeibai()
        {
            HashSet<string> ipHash = new HashSet<string>();
            HtmlDocument htmlDoc = new HtmlDocument();
            WebClient client = new WebClient();
            string DefaultUserAgent = @"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1";
            string DefaultPostContentType = "application/x-www-form-urlencoded";

            client.Credentials = CredentialCache.DefaultCredentials;
            client.Headers.Add("User-Agent", DefaultUserAgent);
            client.Headers.Add("Content-Type", DefaultPostContentType);

            //得到十页的代理内容
            for (int pageIndex = 1; pageIndex < 11; ++pageIndex)
            {
                Uri uri = new Uri(string.Format("http://www.heibai.net/proxy/index.php?act=list&port=&type=&country=&page={0}", pageIndex));

                client.Headers.Add("Referer", uri.AbsoluteUri);
                byte[] bytes ;string content ;
                try
                {
                    bytes = client.DownloadData(uri);
                    content = WebUtility.HtmlDecode(Encoding.GetEncoding("utf-8").GetString(bytes));
                }
                catch { continue; }

                htmlDoc.LoadHtml(content);

                //get paraNodes part
                var paraNodes = htmlDoc.DocumentNode.SelectNodes(@"//tr[@class='cells']");

                if (paraNodes == null)
                    return ipHash;

                HtmlDocument htmlD = new HtmlDocument();

                foreach (var node in paraNodes)
                {
                    htmlD.LoadHtml(node.OuterHtml);
                    var ipNode = htmlD.DocumentNode.SelectSingleNode(@"//td[2]");
                    var portNode = htmlD.DocumentNode.SelectSingleNode(@"//td[3]");

                    if ((ipNode != null && ipPattern.IsMatch(ipNode.InnerText.Trim()) &&
                    portNode != null) && numPattern.IsMatch(portNode.InnerText.Trim()))
                        ipHash.Add(ipNode.InnerText.Trim() + ":" + portNode.InnerText.Trim());
                }
            }
            return ipHash;
        }

        //通过proxy360得到代理地址
        private HashSet<string> GetFromProxy360()
        {
            HashSet<string> ipHash = new HashSet<string>();
            HtmlDocument htmlDoc = new HtmlDocument();
            WebClient client = new WebClient();
            string DefaultUserAgent = @"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1";
            string DefaultPostContentType = "application/x-www-form-urlencoded";

            client.Credentials = CredentialCache.DefaultCredentials;
            client.Headers.Add("User-Agent", DefaultUserAgent);
            client.Headers.Add("Content-Type", DefaultPostContentType);

            Uri uri = new Uri("http://www.proxy360.cn/default.aspx");

            client.Headers.Add("Referer", uri.AbsoluteUri);

            byte[] bytes;
            try { bytes = client.DownloadData(uri); }
            catch { return ipHash; }
            string content = WebUtility.HtmlDecode(Encoding.GetEncoding("utf-8").GetString(bytes));

            htmlDoc.LoadHtml(content);

            //get paraNodes part
            var paraNodes = htmlDoc.DocumentNode.SelectNodes(@"//div[@class='proxylistitem']");

            if (paraNodes == null)
                return ipHash;

            HtmlDocument htmlD = new HtmlDocument();

            foreach (var node in paraNodes)
            {
                htmlD.LoadHtml(node.OuterHtml);
                var ipNode = htmlD.DocumentNode.SelectSingleNode(@"//span[@class='tbBottomLine'][1]");
                var portNode = htmlD.DocumentNode.SelectSingleNode(@"//span[@class='tbBottomLine'][2]");

                if ((ipNode != null && ipPattern.IsMatch(ipNode.InnerText.Trim()) &&
                    portNode != null) && numPattern.IsMatch(portNode.InnerText.Trim()))
                    ipHash.Add(ipNode.InnerText.Trim() + ":" + portNode.InnerText.Trim());
            }

            return ipHash;
        }

        //通过dailiip网站得到代理地址
        private HashSet<string> GetFromDailiip()
        {
            HashSet<string> ipHash = new HashSet<string>();
            WebClient client = new WebClient();
            string DefaultUserAgent = @"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1";
            string DefaultPostContentType = "application/x-www-form-urlencoded";

            client.Credentials = CredentialCache.DefaultCredentials;
            client.Headers.Add("User-Agent", DefaultUserAgent);
            client.Headers.Add("Content-Type", DefaultPostContentType);

            Uri uri = new Uri("http://www.dailiip.com/");

            client.Headers.Add("Referer", uri.AbsoluteUri);

            byte[] bytes;
            try { bytes = client.DownloadData(uri); }
            catch { return ipHash; }
            string content = WebUtility.HtmlDecode(Encoding.GetEncoding("gb2312").GetString(bytes));
            foreach (Match m in dailiipPattern.Matches(content))
            {
                ipHash.Add(m.Groups["ip"].Value + ":" + m.Groups["port"].Value);
            }
            return ipHash;
        }

        //通过xker网站得到代理地址
        private HashSet<string> GetFromXker()
        {
            HashSet<string> ipHash = new HashSet<string>();
            WebClient client = new WebClient();
            string DefaultUserAgent = @"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1";
            string DefaultPostContentType = "application/x-www-form-urlencoded";

            client.Credentials = CredentialCache.DefaultCredentials;
            client.Headers.Add("User-Agent", DefaultUserAgent);
            client.Headers.Add("Content-Type", DefaultPostContentType);

            Uri uri = new Uri("http://www.xker.com/ip/");

            client.Headers.Add("Referer", uri.AbsoluteUri);

            byte[] bytes;
            try { bytes = client.DownloadData(uri); }
            catch { return ipHash; }
            string content = WebUtility.HtmlDecode(Encoding.GetEncoding("gb2312").GetString(bytes));
            foreach (Match m in xkerPattern.Matches(content))
            {
                ipHash.Add(m.Groups["ip"].Value + ":" + m.Groups["port"].Value);
            }
            return ipHash;
        }

        //通过18daili网站得到代理地址
        private HashSet<string> GetFrom18daili()
        {
            HashSet<string> ipHash = new HashSet<string>();
            WebClient client = new WebClient();
            string DefaultUserAgent = @"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1";
            string DefaultPostContentType = "application/x-www-form-urlencoded";

            client.Credentials = CredentialCache.DefaultCredentials;
            client.Headers.Add("User-Agent", DefaultUserAgent);
            client.Headers.Add("Content-Type", DefaultPostContentType);

            Uri uri = new Uri("http://www.18daili.com/SearchLocationForAjax.php");

            client.Headers.Add("Referer", uri.AbsoluteUri);

            byte[] bytes;
            try { bytes = client.DownloadData(uri); }
            catch { return ipHash; }
            string content = WebUtility.HtmlDecode(Encoding.GetEncoding("UTF-8").GetString(bytes));
            foreach (Match m in daili18Pattern.Matches(content))
            {
                ipHash.Add(m.Groups["ip"].Value + ":" + m.Groups["port"].Value);
            }
            return ipHash;
        }

        //通过35wl网站得到代理地址
        private HashSet<string> GetFrom35wl()
        {
            HashSet<string> ipHash = new HashSet<string>();
            WebClient client = new WebClient();
            string DefaultUserAgent = @"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1";
            string DefaultPostContentType = "application/x-www-form-urlencoded";

            client.Credentials = CredentialCache.DefaultCredentials;
            client.Headers.Add("User-Agent", DefaultUserAgent);
            client.Headers.Add("Content-Type", DefaultPostContentType);

            Uri uri = new Uri("http://www.35wl.com/tools/dlfwq.htm");

            client.Headers.Add("Referer", uri.AbsoluteUri);

            byte[] bytes;
            try { bytes = client.DownloadData(uri); }
            catch { return ipHash; }
            string content = WebUtility.HtmlDecode(Encoding.GetEncoding("gb2312").GetString(bytes));
            foreach (Match m in wl35Pattern.Matches(content))
            {
                ipHash.Add(m.Groups["ip"].Value + ":" + m.Groups["port"].Value);
            }
            return ipHash;
        }

        //通过51proxied网站得到代理地址
        private HashSet<string> GetFrom51proxied()
        {
            HashSet<string> ipHash = new HashSet<string>();
            WebClient client = new WebClient();

            string DefaultUserAgent = @"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1";
            string DefaultPostContentType = "application/x-www-form-urlencoded";

            client.Credentials = CredentialCache.DefaultCredentials;
            client.Headers.Add("User-Agent", DefaultUserAgent);
            client.Headers.Add("Content-Type", DefaultPostContentType);

            List<Uri> uriList = new List<Uri>();
            uriList.Add(new Uri("http://www.51proxied.com/http_fast.html"));
            uriList.Add(new Uri("http://www.51proxied.com/http_anonymous.html"));
            uriList.Add(new Uri("http://www.51proxied.com/http_non_anonymous.html"));
            uriList.Add(new Uri("http://www.51proxied.com/socks5.html"));
            uriList.ForEach(uri =>
                {
                    client.Headers.Add("Referer", uri.AbsoluteUri);

                    byte[] bytes;
                    try { bytes = client.DownloadData(uri); }
                    catch { return; }
                    string content = WebUtility.HtmlDecode(Encoding.GetEncoding("UTF-8").GetString(bytes));
                    foreach (Match m in proxiedPattern.Matches(content))
                    {
                        ipHash.Add(m.Groups["ip"].Value + ":" + m.Groups["port"].Value);
                    }
                });
            return ipHash;
        }

        //通过nntime网站得到代理地址
        private HashSet<string> GetFromnntime()
        {

            HashSet<string> ipHash = new HashSet<string>();
            WebClient client = new WebClient();
            string DefaultUserAgent = @"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1";
            string DefaultPostContentType = "application/x-www-form-urlencoded";

            client.Credentials = CredentialCache.DefaultCredentials;
            client.Headers.Add("User-Agent", DefaultUserAgent);
            client.Headers.Add("Content-Type", DefaultPostContentType);

            Uri uri = new Uri("http://nntime.com/");

            client.Headers.Add("Referer", uri.AbsoluteUri);
            byte[] bytes;
            try
            {
                bytes = client.DownloadData(uri);
            }
            catch { return ipHash; }
            string content = WebUtility.HtmlDecode(Encoding.GetEncoding("iso-8859-1").GetString(bytes));
            Dictionary<string, string> charDic = new Dictionary<string, string>();
            foreach (Match m in numnntimePattern.Matches(content))
                if (!charDic.ContainsKey(m.Groups["word"].Value))
                    charDic.Add(m.Groups["word"].Value, m.Groups["num"].Value);

            foreach (Match m in nntimePattern.Matches(content))
            {
                try
                {
                    string ip = m.Groups["ip"].Value + ":";
                    string[] words = m.Groups["word"].Value.Split(new char[] { '+' }, StringSplitOptions.RemoveEmptyEntries);
                    foreach (var word in words)
                        ip += charDic[word];
                    ipHash.Add(ip);
                }
                catch { }
            }
            return ipHash;
        }

        //通过realbooster网站得到代理地址
        private HashSet<string> GetFromrealbooster()
        {
            HashSet<string> ipHash = new HashSet<string>();
            WebClient client = new WebClient();
            string DefaultUserAgent = @"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1";
            string DefaultPostContentType = "application/x-www-form-urlencoded";

            client.Credentials = CredentialCache.DefaultCredentials;
            client.Headers.Add("User-Agent", DefaultUserAgent);
            client.Headers.Add("Content-Type", DefaultPostContentType);

            Uri uri = new Uri("http://realbooster.com/seo-services/free-proxy-list-tool/");

            client.Headers.Add("Referer", uri.AbsoluteUri);

            byte[] bytes;
            try { bytes = client.DownloadData(uri); }
            catch { return ipHash; }
            string content = WebUtility.HtmlDecode(Encoding.GetEncoding("UTF-8").GetString(bytes));

            foreach (Match m in realboosterPattern.Matches(content))
            {
                try
                {
                    ipHash.Add(m.Groups["ip"].Value + ":" + m.Groups["port"].Value);
                }
                catch { }
            }
            return ipHash;
        }

        //通过simpleproxylist网站得到代理地址
        private HashSet<string> GetFromsimpleproxylist()
        {

            HashSet<string> ipHash = new HashSet<string>();
            WebClient client = new WebClient();
            string DefaultUserAgent = @"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1";
            string DefaultPostContentType = "application/x-www-form-urlencoded";

            client.Credentials = CredentialCache.DefaultCredentials;
            client.Headers.Add("User-Agent", DefaultUserAgent);
            client.Headers.Add("Content-Type", DefaultPostContentType);
            for (int i = 1; i < 20; ++i)
            {
                Uri uri = new Uri("http://simpleproxylist.com/search.php?p=" + i.ToString() + "&country=CN");

                client.Headers.Add("Referer", uri.AbsoluteUri);

                byte[] bytes;
                try { bytes = client.DownloadData(uri); }
                catch { continue; }
                string content = HttpUtility.HtmlDecode(Encoding.GetEncoding("UTF-8").GetString(bytes));

                foreach (Match m in simpleproxylistPattern.Matches(content))
                {
                    try
                    {
                        string ip = m.Groups["ip"].Value + ":";
                        string[] word = m.Groups["word"].Value.Split(new string[] { "&#" }, StringSplitOptions.RemoveEmptyEntries);
                        foreach (var w in word)
                            ip += (char)(int.Parse(w));
                        ipHash.Add(ip);
                    }
                    catch { }
                }
            }
            return ipHash;
        }

        //通过proxy-ip-list网站得到代理地址
        private static HashSet<string> GetFromproxyiplist()
        {

            HashSet<string> ipHash = new HashSet<string>();
            WebClient client = new WebClient();
            string DefaultUserAgent = @"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1";
            string DefaultPostContentType = "application/x-www-form-urlencoded";

            client.Credentials = CredentialCache.DefaultCredentials;
            client.Headers.Add("User-Agent", DefaultUserAgent);
            client.Headers.Add("Content-Type", DefaultPostContentType);
            Uri uri = new Uri("http://proxy-ip-list.com/");

            client.Headers.Add("Referer", uri.AbsoluteUri);

            byte[] bytes;
            try { bytes = client.DownloadData(uri); }
            catch { return ipHash; }
            string content = HttpUtility.HtmlDecode(HttpUtility.HtmlDecode(Encoding.GetEncoding("UTF-8").GetString(bytes)));


            Regex proxyiplistPattern = new Regex(@"(?<ip>(\d{1,3}\.){3}\d{1,3}):(?<port>\d+)</td>", RegexOptions.Compiled);

            foreach (Match m in proxyiplistPattern.Matches(content))
            {
                try
                {
                    ipHash.Add(m.Groups["ip"].Value + ":" + m.Groups["port"].Value);

                }
                catch { }
            }

            return ipHash;
        }

        //通过Webs
        private static HashSet<string> GetFromWebs()
        {
            HashSet<string> ipHash = new HashSet<string>();
            WebClient client = new WebClient();
            string DefaultUserAgent = @"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1";
            string DefaultPostContentType = "application/x-www-form-urlencoded";

            client.Credentials = CredentialCache.DefaultCredentials;
            client.Headers.Add("User-Agent", DefaultUserAgent);
            client.Headers.Add("Content-Type", DefaultPostContentType);
            Uri uri = new Uri("http://anonymous-proxy-list.webs.com/");

            client.Headers.Add("Referer", uri.AbsoluteUri);

            byte[] bytes;
            try { bytes = client.DownloadData(uri); }
            catch { return ipHash; }
            string content = HttpUtility.HtmlDecode(HttpUtility.HtmlDecode(Encoding.GetEncoding("ISO-8859-1").GetString(bytes)));


            Regex proxyiplistPattern = new Regex(@"(?<ip>(\d{1,3}\.){3}\d{1,3}):(?<port>\d+)", RegexOptions.Compiled);

            foreach (Match m in proxyiplistPattern.Matches(content))
            {
                try
                {
                    ipHash.Add(m.Groups["ip"].Value + ":" + m.Groups["port"].Value);

                }
                catch { }
            }

            return ipHash;
        }

        //在网站上得到新代理地址
        private HashSet<string> GetProxyFromWeb(CheckProxyWeb checkWeb)
        {
            HashSet<string> proxyHash = new HashSet<string>();
            HashSet<string> hash;

            ///cnproxy
            if (checkWeb.CnProxy)
            {
                hash = GetFromCnproxy();
                GetProxyTotal += hash.Count;
                foreach (var proxy in hash)
                    proxyHash.Add(proxy);
            }
            ///heibai
            if (checkWeb.Heibai)
            {
                hash = GetFromHeibai();
                GetProxyTotal += hash.Count;
                foreach(var proxy in hash)
                    proxyHash.Add(proxy);
            }
            ///proxy360
            if (checkWeb.Proxy360)
            {
                hash = GetFromProxy360();
                GetProxyTotal += hash.Count;
                foreach (var proxy in hash)
                    proxyHash.Add(proxy);
            }
            ///dailiip
            if (checkWeb.Dailiip)
            {
                hash = GetFromDailiip();
                GetProxyTotal += hash.Count;
                foreach (var proxy in hash)
                    proxyHash.Add(proxy);
            }
            ///xker
            if (checkWeb.Xker)
            {
                hash =  GetFromXker();
                GetProxyTotal += hash.Count;
                foreach (var proxy in hash)
                    proxyHash.Add(proxy);
            }
            ///18daili
            if (checkWeb.Daili18)
            {
                hash = GetFrom18daili();
                GetProxyTotal += hash.Count;
                foreach (var proxy in hash)
                    proxyHash.Add(proxy);
            }
            ///35wl
            if (checkWeb.Wl35)
            {
                hash = GetFrom35wl();
                GetProxyTotal += hash.Count;
                foreach (var proxy in hash)
                    proxyHash.Add(proxy);
            }
            ///51proxied
            if (checkWeb.Proxied51)
            {
                hash = GetFrom51proxied();
                GetProxyTotal += hash.Count;
                foreach (var proxy in hash)
                    proxyHash.Add(proxy);
            }
            ///nntime
            if (checkWeb.Nntime)
            {
                hash = GetFromnntime();
                GetProxyTotal += hash.Count;
                foreach (var proxy in hash)
                    proxyHash.Add(proxy);
            }
            ///realbooster
            if (checkWeb.Realbooster)
            {
                hash =  GetFromrealbooster();
                GetProxyTotal += hash.Count;
                foreach (var proxy in hash)
                    proxyHash.Add(proxy);
            }
            ///simpleproxylist
            if (checkWeb.Simpleproxylist)
            {
                hash = GetFromsimpleproxylist();
                GetProxyTotal += hash.Count;
                foreach (var proxy in hash)
                    proxyHash.Add(proxy);
            }
            ///proxyiplist
            if (checkWeb.Proxyiplist)
            {
                hash =  GetFromproxyiplist();
                GetProxyTotal += hash.Count;
                foreach (var proxy in hash)
                    proxyHash.Add(proxy);
            }
            ///webs
            if (checkWeb.Webs)
            {
                hash = GetFromWebs();
                GetProxyTotal += hash.Count;
                foreach (var proxy in hash)
                    proxyHash.Add(proxy);
            }

            return proxyHash;
        }

        //测试代理是否可用
        public void Start(ProxyHelp help)
        {

            ParallelOptions taskParallet = new ParallelOptions();
            taskParallet.MaxDegreeOfParallelism = MaxThread;
            int count = 0;

            for (int i = 0; i < RepeateTimes; i++)
            {
                Parallel.ForEach(Proxys, taskParallet, current =>
                {
                    if (current.Error != null)
                        return;

                    Stopwatch sw = new Stopwatch();
                    WebClient wc = new WebClient();
                    InitWebClient(wc, TestUri, TestUri);
                    wc.Proxy = current.Proxy;
                    sw.Restart();
                    try
                    {
                        byte[] data = wc.DownloadData(TestUri);
                        sw.Stop();
                        current.Milliseconds = sw.ElapsedMilliseconds;
                        current.ContentLength = data.Length;

                        if (current.ContentLength != FileLength)
                            current.Error = new Exception("下载错误");

                        string log = string.Format("{0} - {1} - {2} - {3}", current.ProxyString, current.ContentLength, current.Milliseconds, count);
                        help.AddText(log);
                        //Console.WriteLine(log);
                    }
                    catch (Exception e)
                    {
                        current.Error = e;
                        string log = string.Format("{0} - {1} - {2}", current.ProxyString, e.Message, count);
                        help.AddText(log);
                        //Console.WriteLine(log);
                    }

                    Interlocked.Increment(ref count);
                });

                Array.Sort(Proxys, new Comparison<ProxyDetail>((a, b) => b.Milliseconds.CompareTo(a.Milliseconds)));
            }

            SaveProxy();
            help.AddText("测试代理地址结束！");
            Console.WriteLine("结束");
        }

        //保存代理地址
        private void SaveProxy()
        {
            //在这里可以排序取前多少可以使用的
            //Array.Sort(Proxys, new Comparison<ProxyDetail>((a, b) => a.Milliseconds.CompareTo(b.Milliseconds)));

            //这里取了下载时间小于10000毫秒的代理
            //for (int i = 0; i < Proxys.Length; ++i)
            //    if (Proxys[i].Error == null && Proxys[i].Milliseconds < 10000)
            //        proxyList.Add(new ProxyTable(Proxys[i].ProxyString));

            proxyResultList = Proxys.Where(a => a.Error == null).Select(b=>b.ProxyString).ToList();
        }

        private const string DefaultUserAgent = @"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; CIBA; .NET4.0C; .NET4.0E)";
        private const string DefaultPostContentType = "application/x-www-form-urlencoded";

        //初始化WebClient
        private static void InitWebClient(WebClient webClient, Uri uri, Uri pUri)
        {
            webClient.Credentials = CredentialCache.DefaultCredentials;
            webClient.Headers.Add("User-Agent", DefaultUserAgent);
            webClient.Headers.Add("Content-Type", DefaultPostContentType);
            webClient.Headers.Add("Referer", pUri.AbsoluteUri);
            ServicePoint servicePoint = ServicePointManager.FindServicePoint(uri);
            if (servicePoint.Expect100Continue == true)
                servicePoint.Expect100Continue = false;
        }

    }

    /// <summary>
    /// 测试Proxy时需要用到的一些参数
    /// </summary>
    public class ProxyDetail
    {
        public WebProxy Proxy;
        public string ProxyString;
        public int ContentLength;
        public long Milliseconds;
        public Exception Error;
    }

    public class CheckProxyWeb
    {
        public bool CnProxy;
        public bool Heibai;
        public bool Proxy360;
        public bool Dailiip;
        public bool Xker;
        public bool Daili18;
        public bool Wl35;
        public bool Proxied51;
        public bool Nntime;
        public bool Realbooster;
        public bool Simpleproxylist;
        public bool Proxyiplist;
        public bool Webs;
    }
}

 

其中调用的代码就很简单了

由于我是隔段时间获取一次，所以用到了Timer的内容，也贴在这里了

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Runtime.InteropServices;
using System.Threading;
using System.IO;

namespace ProxyTest
{
    class Program
    {

        static System.Timers.Timer proxyTimer = new System.Timers.Timer();

        static void Main(string[] args)
        {
            
            proxyTimer = new System.Timers.Timer();
            proxyTimer.Interval = 1000 * 60;
            proxyTimer.Elapsed += new System.Timers.ElapsedEventHandler(DoProxyTest);
            proxyTimer.Enabled = true;
            proxyTimer.Start();

            Thread.Sleep(-1);
        }

        static bool proxyIsRunning = false;

        static void DoProxyTest(object sender, System.Timers.ElapsedEventArgs e)
        {
            Write("Start to running ProxyTest function!");
            if (proxyIsRunning)
            {
                Write("End ProxyTest function! by : ProxyTest is running now!" + Environment.NewLine);
                return;
            }

            proxyTimer.Interval = 1000 * 60 * 60 * 24; //每天执行一次

            int rowCount = Pmars.DataBaseHelper.GetHelper().GetRowCount("ProxyTable");
            Write("ProxyTable Count:" + rowCount);
            Pmars.ProxyTest proxy = new Pmars.ProxyTest();
            proxy.Start();
            rowCount = Pmars.DataBaseHelper.GetHelper().GetRowCount("ProxyTable");
            Write("ProxyTable Count:" + rowCount);

            Write("End ProxyTest function!" + Environment.NewLine);
        }

        static void Write(string contents)
        {
            File.AppendAllText(@"log/proxyLog.txt", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + " : " + contents + Environment.NewLine);
        }
    }
}

 

第一部分的代码后来改了很多，也用不到timer部分的代码了，但是下面的timer部分的代码就不删掉了，稍微改下其实就可以的！

贴在这里，学习一下！