支持Cookie并开放了一些特殊设置项的HttpWebClient

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using System.Collections.Specialized;
using System.Web;

namespace Common.Helpers
{
    /// <summary>
    /// 网络访问辅助类
    /// </summary>
    public class HttpWebClient : WebClient
    {
        #region 公共属性
        /// <summary>
        /// 浏览器用户标识,默认采用Chrome的标识
        /// </summary>
        public string UserAgent { get; set; }
        /// <summary>
        /// Cookie容器
        /// </summary>
        public CookieContainer CookieContainer { get; set; }
        /// <summary>
        /// 如果 POST 请求需要 100-Continue 响应，则为 true；否则为 false。
        /// </summary>
        public bool Expect100Continue { get; set; }

        private WebResponse m_LastWebResponse = null;
        /// <summary>
        /// 最后一次的响应对象
        /// </summary>
        public WebResponse LastWebResponse { get { return this.m_LastWebResponse; } }

        private int m_Timeout = 120000;
        /// <summary>
        /// 超时时间，默认120000毫秒(120秒)
        /// </summary>
        public int Timeout
        {
            get { return m_Timeout; }
            set { m_Timeout = value; }
        }

        private HttpWebClientSetting m_HttpWebClientSetting = null;
        /// <summary>
        /// WebClient设置项,该属性始终不会为null
        /// </summary>
        public HttpWebClientSetting HttpWebClientSetting
        {
            get
            {
                if (m_HttpWebClientSetting == null)
                {
                    m_HttpWebClientSetting = new HttpWebClientSetting();
                }
                return m_HttpWebClientSetting;
            }
            set
            {
                m_HttpWebClientSetting = value ?? new HttpWebClientSetting();
            }
        }
        

        /// <summary>
        /// 预处理Web请求对象的委托方法(会在每次获取WebRequest对象后调用)，默认值为null
        /// </summary>
        public Action<HttpWebRequest> PrepareProcessWebRequest { get; set; }
        #endregion

        #region 构造方法
        public HttpWebClient()
            : this(new CookieContainer())
        {
        }

        public HttpWebClient(CookieContainer cookieContainer)
        {
            this.CookieContainer = cookieContainer;
            this.UserAgent = UserAgentValues.FireFox;
            this.Expect100Continue = false;
        }
        #endregion

        #region 重写方法，增加对CookieContainer的支持
        protected override WebRequest GetWebRequest(Uri address)
        {
            if (!string.IsNullOrEmpty(this.UserAgent))
            {
                this.Headers.Add(HttpRequestHeader.UserAgent, this.UserAgent);
            }

            WebRequest request = base.GetWebRequest(address);
            request.Timeout = this.Timeout;
            
            if (request is HttpWebRequest)
            {
                HttpWebRequest httpRequest = request as HttpWebRequest;
                httpRequest.CookieContainer = this.CookieContainer;
                httpRequest.ServicePoint.Expect100Continue = this.Expect100Continue; // 取消100-continue

                //读取自定义设置项
                if (this.HttpWebClientSetting != null)
                {
                    httpRequest.AllowAutoRedirect = this.HttpWebClientSetting.AllowAutoRedirect;
                }

                //使用外部委托属性处理Request对象
                if (this.PrepareProcessWebRequest != null)
                {
                    this.PrepareProcessWebRequest(httpRequest);
                }
            }
            
            return request;
        }
        #endregion

        #region 重写方法，增加对响应对象的访问
        protected override WebResponse GetWebResponse(WebRequest request)
        {
            WebResponse response = base.GetWebResponse(request);
            this.m_LastWebResponse = response;
            return response;
        }
        #endregion

        #region (public) 向一个URL用POST提交数据，并返回其响应内容 PostData
        /// <summary>
        /// 向一个URL用POST提交数据，并返回其响应内容
        /// ZhangQingFeng    2014-12-14    Add
        ///    EditLog：
        ///        ZhangQingFeng    2015-05-12    Edit        因WebClient的UpdateValues方法中固定为UTF-8格式进行UrlEncode，因此此处需用UploadString方式来间接实现    --见微软WebClient类源码UploadValuesInternal方法中
        /// </summary>
        /// <param name="url">请求的URL</param>
        /// <param name="data">要提交的数据</param>
        /// <param name="encoding">请求所使用的编码</param>
        /// <param name="responseEncoding">响应内容所使用的编码，为null时使用请求的编码</param>
        /// <returns>响应的内容</returns>
        public string PostData(string url, NameValueCollection data, Encoding encoding, Encoding responseEncoding)
        {
            WebClient client = this;

            /*
            client.Encoding = encoding ?? Encoding.UTF8;

            byte[] response = client.UploadValues(url, "POST", data ?? new NameValueCollection());

            string html = string.Empty;

            if (responseEncoding == null)
            {
                html = client.Encoding.GetString(response);
            }
            else
            {
                html = responseEncoding.GetString(response);
            }
             */

            client.Encoding = encoding ?? Encoding.UTF8;
            client.Headers.Add(HttpRequestHeader.ContentType, "application/x-www-form-urlencoded");

            string delimiter = String.Empty;
            StringBuilder values = new StringBuilder();
            foreach (string name in data.AllKeys)
            {
                values.Append(delimiter);
                values.Append(HttpUtility.UrlEncode(name, encoding));
                values.Append("=");
                values.Append(HttpUtility.UrlEncode(data[name], encoding));
                delimiter = "&";
            }

            byte[] arrData = client.UploadData(url, "POST", Encoding.ASCII.GetBytes(values.ToString()));
            string html = (responseEncoding ?? client.Encoding).GetString(arrData);

            return html;
        }

        /// <summary>
        /// 向一个URL用POST提交数据，并返回其响应内容
        /// ZhangQingFeng    2014-12-14    Add
        /// </summary>
        /// <param name="url">请求的URL</param>
        /// <param name="data">要提交的数据</param>
        /// <param name="encoding">请求和响应所使用的编码</param>
        /// <returns>响应的内容</returns>
        public string PostData(string url, NameValueCollection data, Encoding encoding)
        {
            return PostData(url, data, encoding, null);
        }

        /// <summary>
        /// 向一个URL用POST提交数据，并返回其响应内容(使用this.Encoding来作请求编码和响应编码)
        /// ZhangQingFeng    2014-12-14    Add
        /// </summary>
        /// <param name="url">请求的URL</param>
        /// <param name="data">要提交的数据</param>
        /// <returns>响应的内容</returns>
        public string PostData(string url, NameValueCollection data)
        {
            return PostData(url, data, this.Encoding);
        }
        #endregion

        #region (public) 向一个URL用POST提交数据，并返回其响应内容 PostData
        /// <summary>
        /// 向一个URL用POST提交数据，并返回其响应内容
        /// ZhangQingFeng    2014-12-14    Add
        /// </summary>
        /// <param name="url">请求的URL</param>
        /// <param name="data">要提交的数据</param>
        /// <param name="encoding">请求和响应内容所使用的编码</param>
        /// <returns>响应的内容</returns>
        public string PostData(string url, Dictionary<string, string> data, Encoding encoding, Encoding responseEncoding)
        {
            NameValueCollection postData = new NameValueCollection();
            if (data != null)
            {
                foreach (var item in data)
                {
                    postData.Add(item.Key, item.Value);
                }
            }
            return PostData(url, postData, encoding, responseEncoding);
        }


        /// <summary>
        /// 向一个URL用POST提交数据，并返回其响应内容
        /// ZhangQingFeng    2014-12-14    Add
        /// </summary>
        /// <param name="url">请求的URL</param>
        /// <param name="data">要提交的数据</param>
        /// <param name="encoding">请求和响应所使用的编码</param>
        /// <returns>响应的内容</returns>
        public string PostData(string url, Dictionary<string, string> data, Encoding encoding)
        {
            return PostData(url, data, encoding, null);
        }

        /// <summary>
        /// 向一个URL用POST提交数据，并返回其响应内容(使用this.Encoding来作请求编码和响应编码)
        /// ZhangQingFeng    2014-12-14    Add
        /// </summary>
        /// <param name="url">请求的URL</param>
        /// <param name="data">要提交的数据</param>
        /// <returns>响应的内容</returns>
        public string PostData(string url, Dictionary<string, string> data)
        {
            return PostData(url, data, this.Encoding);
        }
        #endregion

        #region 辅助类
        /// <summary>
        /// 浏览器用户标识类
        /// </summary>
        public class UserAgentValues
        {
            public static readonly string FireFox = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:37.0) Gecko/20100101 Firefox/37.0";
            public static readonly string Chrome = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36";
            public static readonly string IE8 = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2;)";
        }
        #endregion
    }

    /// <summary>
    /// HttpWebClient对象设置类
    /// </summary>
    public class HttpWebClientSetting
    {
        private bool m_AllowAutoRedirect = true;
        /// <summary>
        /// 当响应内容为重定向时客户端是否自动重定向(如果该属性为true，则取到的响应则为重定向后的内容，否则则为响应原文)，默认值为true
        /// </summary>
        public bool AllowAutoRedirect
        {
            get { return m_AllowAutoRedirect; }
            set { m_AllowAutoRedirect = value; }
        }
    }
}

在做页面抓取的过程中，发现自带的WebClient不够灵活，因此做了一个实现。

 

关于在PostData方法中不使用UploadValues()方法的原因：

1.查看微软的源代码实现时发现，无论设置请求时的Encoding是否为GB2312，在使用WebClient的UploadValues()上传内容时，其内在都是使用UTF-8编码进行UrlEncode，因此传到服务端中的数据中若包含有中文时则一定会乱码，因此重写PostData以规避此问题。

 

关于HttpWebClientSetting中的AllowAutoRedirect属性：

在WebClient发起请求时，若响应内容为重定向，则WebClient会自动做重定向，因此该类提供此设置项以控制在访问时是否自动做重定向(第二次访问Refer后的网站时会将请求中的Refer头置空，将该AllowAutoRedirect设置为false，然后手动从Response.Header中取出Location对象地址，设置Refer后再访问，则可真实模拟浏览器访问，从而避开一些网站的防抓取设置)

 

关于HttpWebClient中的LastWebResponse属性：

当存在多次重定向时，系统记录了最后一次返回的内容，从此内容的Header中取出ResponseUri，则可以取到最后返回响应的页面真实地址，从而为下一次的设置请求Refer头作准备。

 

大约就是如此，后期如有Bug会继续更新。