[转]HttpWebRequest提高效率之连接数,代理,自动跳转,gzip请求等设置有关问题
[转]HttpWebRequest提高效率之连接数,代理,自动跳转,gzip请求等设置有关问题
www.MyException.Cn 发布于:2012-09-15 19:09:29 浏览:7次
[记录]HttpWebRequest提高效率之连接数,代理,自动跳转,gzip请求等设置问题
先设置4个:
webrequest.ServicePoint.Expect100Continue = false; //是否使用 Nagle 不使用 提高效率 webrequest.ServicePoint.UseNagleAlgorithm = false; //最大连接数 webrequest.ServicePoint.ConnectionLimit = 65500; //数据是否缓冲 false 提高效率 webrequest.AllowWriteStreamBuffering = false;
代理的使用:我用的代理都是不要账号和密码的,因为要不是免费的,要是是我们自己的,对来访IP 有限制
if (proxy!=null) { model.proxy = proxy; WebProxy myProxy = new WebProxy(proxy.ProxyInfo, false); myProxy.Credentials = new NetworkCredential("", ""); webrequest.Proxy = myProxy; } else { webrequest.Proxy = GlobalProxySelection.GetEmptyWebProxy(); }
webrequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
带上gzip,如果对方支持gzip,流量上可以节省五分之四,原来250KB,用gzip压缩有 大概在50KB,减少带宽压力,增加速度
值得注意的是,如果是gzip,返回值必须要gzip解压的。
if (response.ContentEncoding.ToLower().Contains("gzip")) { using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress)) { using (StreamReader reader = new StreamReader(stream, Encoding.UTF8)) { model.html = reader.ReadToEnd(); } } }
还有一点:gzip解压的时候回耗费CPU,所以这个需要注意下
附完成方法
/// <summary> /// Get 方式 获取数据 /// </summary> /// <param name="webUrl"></param> /// <returns></returns> public Model.WebRequestReturnModel WebRequestGetHtmlByGet(string webUrl, CookieContainer cookieContainer, Encoding encoding, string refer,Model.Proxy proxy) { Model.WebRequestReturnModel model = new Model.WebRequestReturnModel(); HttpWebRequest webrequest = null; try { webrequest = WebRequest.Create(webUrl) as HttpWebRequest; webrequest.ServicePoint.Expect100Continue = false; webrequest.ServicePoint.UseNagleAlgorithm = false; webrequest.ServicePoint.ConnectionLimit = 65500; webrequest.AllowWriteStreamBuffering = false; if (proxy!=null) { model.proxy = proxy; WebProxy myProxy = new WebProxy(proxy.ProxyInfo, false); myProxy.Credentials = new NetworkCredential("", ""); webrequest.Proxy = myProxy; } else { webrequest.Proxy = GlobalProxySelection.GetEmptyWebProxy(); } //设置其他的herader webrequest =WebRequestSetHeaders(webrequest); if (!String.IsNullOrEmpty(refer)) { webrequest.Referer = refer; } else { webrequest.Referer = webUrl; } webrequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate"); webrequest.CookieContainer = cookieContainer; webrequest.Timeout = 5000; webrequest.UserAgent = "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)"; webrequest.Accept = "*/*"; webrequest.KeepAlive = true; webrequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5"); using (HttpWebResponse response = (HttpWebResponse)webrequest.GetResponse()) { if (response.ContentEncoding.ToLower().Contains("gzip")) { using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress)) { using (StreamReader reader = new StreamReader(stream, Encoding.UTF8)) { model.html = reader.ReadToEnd(); } } } else if (response.ContentEncoding.ToLower().Contains("deflate")) { using (DeflateStream stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress)) { using (StreamReader reader = new StreamReader(stream, Encoding.UTF8)) { model.html = reader.ReadToEnd(); } } } else { using (Stream stream = response.GetResponseStream()) { using (StreamReader reader = new StreamReader(stream, Encoding.UTF8)) { model.html = reader.ReadToEnd(); } } } } } catch (Exception ex) { model.exception = ex; model.html = string.Empty; } model.cookieContainer = cookieContainer; return model; }