asp.net 利用HttpWebRequest自动获取网页编码并获取网页源代码
/// <summary> /// 获取源代码 /// </summary> /// <param name="url"></param> /// <returns></returns> public static string GetHtml(string url, Encoding encoding) { HttpWebRequest request = null; HttpWebResponse response = null; StreamReader reader = null; try { request = (HttpWebRequest)WebRequest.Create(url); request.Timeout = 20000; request.AllowAutoRedirect = false; response = (HttpWebResponse)request.GetResponse(); if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024) { if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase)) reader = new StreamReader(new GZipStream(response.GetResponseStream(), CompressionMode.Decompress), encoding); else reader = new StreamReader(response.GetResponseStream(), encoding); string html = reader.ReadToEnd(); return html; } } catch { } finally { if (response != null) { response.Close(); response = null; } if (reader != null) reader.Close(); if (request != null) request = null; } return string.Empty; }
public static string GetEncoding(string url) { HttpWebRequest request = null; HttpWebResponse response = null; StreamReader reader = null; try { request = (HttpWebRequest)WebRequest.Create(url); request.Timeout = 20000; request.AllowAutoRedirect = false; response = (HttpWebResponse)request.GetResponse(); if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024) { if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase)) reader = new StreamReader(new GZipStream(response.GetResponseStream(), CompressionMode.Decompress)); else reader = new StreamReader(response.GetResponseStream(), Encoding.ASCII); string html = reader.ReadToEnd(); Regex reg_charset = new Regex(@"charset\b\s*=\s*(?<charset>[^""]*)"); if (reg_charset.IsMatch(html)) { return reg_charset.Match(html).Groups["charset"].Value; } else if (response.CharacterSet != string.Empty) { return response.CharacterSet; } else return Encoding.Default.BodyName; } } catch { } finally { if (response != null) { response.Close(); response = null; } if (reader != null) reader.Close(); if (request != null) request = null; } }
using System; using System.Net; using System.Text; using System.Text.RegularExpressions; class Program { // 获取网页的HTML内容,根据网页的charset自动判断Encoding static string GetHtml(string url) { return GetHtml(url, null); } // 获取网页的HTML内容,指定Encoding static string GetHtml(string url, Encoding encoding) { byte[] buf = new WebClient().DownloadData(url); if (encoding != null) return encoding.GetString(buf); string html = Encoding.UTF8.GetString(buf); encoding = GetEncoding(html); if (encoding == null || encoding == Encoding.UTF8) return html; return encoding.GetString(buf); } // 根据网页的HTML内容提取网页的Encoding static Encoding GetEncoding(string html) { string pattern = @"(?i)\bcharset=(? <charset>[-a-zA-Z_0-9]+)"; string charset = Regex.Match(html, pattern).Groups["charset"].Value; try { return Encoding.GetEncoding(charset); } catch (ArgumentException) { return null; } } // 根据网页的HTML内容提取网页的Title static string GetTitle(string html) { string pattern = @"(?si) <title(?:\s+(?:""[^""]*""|'[^']*'|[^""'>])*)?>(? <title>.*?) </title>"; return Regex.Match(html, pattern).Groups["title"].Value.Trim(); } // 打印网页的Encoding和Title static void PrintEncodingAndTitle(string url) { string html = GetHtml(url); Console.WriteLine("[{0}] [{1}]", GetEncoding(html), GetTitle(html)); } // 程序入口 static void Main() { PrintEncodingAndTitle("http://www.msdn.net/"); PrintEncodingAndTitle("http://www.cnblogs.com/"); PrintEncodingAndTitle("http://www.cnblogs.com/skyiv/"); PrintEncodingAndTitle("http://www.csdn.net/"); PrintEncodingAndTitle("http://news.163.com/"); } } /* 程序输出: [] [MSDN: Microsoft Developer Network] [System.Text.UTF8Encoding] [博客园 - 程序员的网上家园] [System.Text.UTF8Encoding] [空间/IV - 博客园] [System.Text.UTF8Encoding] [CSDN.NET - 中国最大的IT技术社区,为IT专业技术人员提供最全面的信息传播和服务平台] [System.Text.DBCSCodePageEncoding] [新闻中心_网易新闻] */