关于抓取百度搜索内容
在一次模拟抓取百度搜索内容的实战中，记录了该文章。。。。以记录几个问题
(嘿嘿，没什么技术含量，只是记录遇到的麻烦和思路)
在C#中，为了实现网页内容的抓取，我们一般使用 System.Net 下的一些常用类,如
HttpWebRequest、HttpWebResponse等等。。。。。
为了抓取网页内容，首先我们要根据我们的目录来分析这个网站。。。
目的：根据关键字来抓取百度搜索的前50条记录(排除推广)
抓取内容：标题、连接、显示内容
分析结果：
搜索连接 ： http://www.baidu.com/s?wd=关键字&rn=pagesize
显示结果 ：采用每条记录一个table来显示内容，table的 id从1开始递增。。
根据我们的目的和分析结果，我们就知道这次抓取任务很简单啦，由于我们只有抓取50条记录，那么就可以把连接
修改成 http://www.baidu.com/s?wd=关键字&rn=50 ,然后直接带上关键字访问这个连接就可以了。。
此时第一个问题出现：
中文编码：
刚开始我以为使用 UrlEncode把中文编码一下就可以了，结果发现返回的结果出现乱码，又用firebug看了下返回的编码是utf8的，于是修改我使用的编码，采用utf8，结果发现除了我的关键字其他都是正确的，晕啦，还有这种事情哇？
终于在一次无意中查看到原来正常请求都是按照gb2312来传的，所以我们的把 “关键字”改成gb2312的格式再去请求就可以了
方法一：
string keyWord = System.Web.HttpUtility.UrlEncode(_keyWord, System.Text.UnicodeEncoding.GetEncoding("GB2312")).ToUpper();
方法二：(一个朋友的方法)
public string ConvertText(string _str, string enconding)
{
byte[] buffer = Encoding.GetEncoding(enconding).GetBytes(_str);
string tmpCity = "";
foreach (byte b in buffer) tmpCity += string.Format("%{0:X}", b);
return tmpCity;
}
当我们正确请求拿到html字符串的时候就可以去分析处理了，原先我采用的Httpwebrequest来请求并返回html的字符串方式，发现取里面的内容挺麻烦，尤其正则表达式写的还挺烦的呢。
此时出现第二个问题：(也不算问题啦，只能算思路)
改用WebBrowser代替Httpwebrequest来请求
由于我们知道我们要根据结果table的 id来取得里面的内容，所以我放弃了原先的请求方式，采用了webbrowser，原因很简单，webbrowser里可以直接根据id来获取对象来直接操作，嘿嘿，岂不是比我们手写正则来匹配方便？而且正则还不太好些，还真有点麻烦。。。
使用 webbrowser 遇到的问题，就是 DocumentCompleted事件会调用多次
解决办法：
判断ReadyState == WebBrowserReadyState.Complete
if (myBrowser.ReadyState != WebBrowserReadyState.Complete)
{
return;
}
一般都可以解决，如果还是有问题，就像我的一样，那就定义一个变量判断变量的值好了。。。嘿嘿
到此为止 这次实战存在的2个问题全部得以解决，顺利抓取到想要的数据。。。
附：封装的httphelper
代码
1 using System;
2 using System.Data;
3 using System.Configuration;
4 using System.Web;
5 using System.Net;
6 using System.IO;
7 using System.Threading;
8 using System.Text;
9
10 namespace End.AdWords
11 {
12 /// <summary>
13 /// HttpHelper 类
14 /// </summary>
15 public class HttpHelper
16 {
17 #region 私有变量
18 private CookieContainer cc;
19 private string contentType = "application/x-www-form-urlencoded";
20 private string accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/x-silverlight, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/x-silverlight-2-b1, */*";
21 private string userAgent = "Mozilla/4.0 (compatible; MSIE 7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729)";//"Googlebot/2.1 (+http://www.google.com/bot.html)";
22 private Encoding encoding = Encoding.GetEncoding("UTF-8");
23 //private Encoding encoding = Encoding.GetEncoding("GB2312");
24 private int delay = 500;
25 private int maxTry = 3;
26 private int currentTry = 0;
27 #endregion
28
29 #region 属性
30 /// <summary>
31 /// Cookie容器
32 /// </summary>
33 public CookieContainer CookieContainer
34 {
35 get
36 {
37 return cc;
38 }
39 }
40
41 /// <summary>
42 /// 获取网页源码时使用的编码
43 /// </summary>
44 /// <value></value>
45 public Encoding Encoding
46 {
47 get
48 {
49 return encoding;
50 }
51 set
52 {
53 encoding = value;
54 }
55 }
56
57 public int NetworkDelay
58 {
59 get
60 {
61 return delay;
62 }
63 set
64 {
65 delay = value;
66 }
67 }
68
69 public int MaxTry
70 {
71 get
72 {
73 return maxTry;
74 }
75 set
76 {
77 maxTry = value;
78 }
79 }
80 #endregion
81
82 #region 构造函数
83 /// <summary>
84 /// Initializes a new instance of the <see cref="HttpHelper"/> class.
85 /// </summary>
86 public HttpHelper()
87 {
88 cc = new CookieContainer();
89 }
90
91 /// <summary>
92 /// Initializes a new instance of the <see cref="HttpHelper"/> class.
93 /// </summary>
94 /// <param name="cc">The cc.</param>
95 public HttpHelper(CookieContainer cc)
96 {
97 this.cc = cc;
98 }
99
100 /// <summary>
101 /// Initializes a new instance of the <see cref="HttpHelper"/> class.
102 /// </summary>
103 /// <param name="contentType">Type of the content.</param>
104 /// <param name="accept">The accept.</param>
105 /// <param name="userAgent">The user agent.</param>
106 public HttpHelper(string contentType, string accept, string userAgent)
107 {
108 this.contentType = contentType;
109 this.accept = accept;
110 this.userAgent = userAgent;
111 }
112
113 /// <summary>
114 /// Initializes a new instance of the <see cref="HttpHelper"/> class.
115 /// </summary>
116 /// <param name="cc">The cc.</param>
117 /// <param name="contentType">Type of the content.</param>
118 /// <param name="accept">The accept.</param>
119 /// <param name="userAgent">The user agent.</param>
120 public HttpHelper(CookieContainer cc, string contentType, string accept, string userAgent)
121 {
122 this.cc = cc;
123 this.contentType = contentType;
124 this.accept = accept;
125 this.userAgent = userAgent;
126 }
127 #endregion
128
129 #region 公共方法
130 /// <summary>
131 /// 获取指定页面的HTML代码
132 /// </summary>
133 /// <param name="url">指定页面的路径</param>
134 /// <param name="postData">回发的数据</param>
135 /// <param name="isPost">是否以post方式发送请求</param>
136 /// <param name="cookieCollection">Cookie集合</param>
137 /// <returns></returns>
138 public string GetHtml(string url, string postData, bool isPost, CookieContainer cookieContainer)
139 {
140
141 string html = "";
142
143
144 if (string.IsNullOrEmpty(postData))
145 {
146 return GetHtml(url, cookieContainer);
147 }
148
149 try
150 {
151 byte[] byteRequest = Encoding.Default.GetBytes(postData);
152 HttpWebRequest httpWebRequest;
153 httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(url);
154 httpWebRequest.CookieContainer = cookieContainer;
155 httpWebRequest.ContentType = contentType;
156 httpWebRequest.Referer = url;
157 httpWebRequest.Accept = accept;
158 httpWebRequest.UserAgent = userAgent;
159 httpWebRequest.Method = isPost ? "POST" : "GET";
160 httpWebRequest.ContentLength = byteRequest.Length;
161 //Stream stream = httpWebRequest.GetRequestStream();
162 httpWebRequest.Timeout = 50000;
163 Stream stream = null;
164 try
165 {
166
167 stream = httpWebRequest.GetRequestStream();
168
169 }
170 catch (Exception ex)
171 {
172 string ss = ex.Message;
173
174
175 }
176
177
178
179 stream.Write(byteRequest, 0, byteRequest.Length);
180 stream.Close();
181 HttpWebResponse httpWebResponse;
182 httpWebRequest.Timeout = 20000;
183
184 try
185 {
186 httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();
187 //判断HTTP响应状态
188 if (httpWebResponse.StatusCode != HttpStatusCode.OK)
189 {
190 httpWebResponse.Close();
191 html = "访问失败！";
192 return html;
193 }
194
195 Stream responseStream = null;
196 try
197 {
198 responseStream = httpWebResponse.GetResponseStream();
199 }
200 catch (Exception ex)
201 {
202 string aaa = ex.Message.ToString();
203 }
204 StreamReader streamReader = new StreamReader(responseStream, encoding);
205 html = streamReader.ReadToEnd();
206 streamReader.Close();
207 responseStream.Close();
208
209 currentTry = 0;
210 return html;
211 }
212 catch
213 {
214 html = "操作超时";
215 return html;
216 }
217
218
219 }
220 catch (Exception e)
221 {
222 Console.ForegroundColor = ConsoleColor.Red;
223 Console.WriteLine(DateTime.Now.ToString("HH:mm:ss ") + e.Message);
224 Console.ForegroundColor = ConsoleColor.White;
225
226 if (currentTry <= maxTry)
227 {
228 GetHtml(url, postData, isPost, cookieContainer);
229 }
230
231 currentTry = 0;
232 return string.Empty;
233 }
234 }
235
236 #region 暂时不用
237 ///// <summary>
238 ///// 获取指定页面的HTML代码
239 ///// </summary>
240 ///// <param name="url">指定页面的路径</param>
241 ///// <param name="postData">回发的数据</param>
242 ///// <param name="isPost">是否以post方式发送请求</param>
243 ///// <param name="cookieCollection">Cookie集合</param>
244 ///// <returns></returns>
245 //public string GetHtml(string url, string postData, bool isPost, CookieContainer cookieContainer)
246 //{
247 // if (string.IsNullOrEmpty(postData))
248 // {
249 // return GetHtml(url, cookieContainer);
250 // }
251
252 // //Thread.Sleep(NetworkDelay);
253
254 // currentTry++;
255
256 // try
257 // {
258 // byte[] byteRequest = Encoding.Default.GetBytes(postData);
259
260 // HttpWebRequest httpWebRequest;
261 // httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(url);
262 // httpWebRequest.CookieContainer = cookieContainer;
263 // httpWebRequest.ContentType = contentType;
264 // httpWebRequest.Referer = url;
265 // httpWebRequest.Accept = accept;
266 // httpWebRequest.UserAgent = userAgent;
267 // httpWebRequest.Method = isPost ? "POST" : "GET";
268 // httpWebRequest.ContentLength = byteRequest.Length;
269
270 // Stream stream = httpWebRequest.GetRequestStream();
271 // stream.Write(byteRequest, 0, byteRequest.Length);
272 // stream.Close();
273
274 // HttpWebResponse httpWebResponse;
275 // httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();
276 // Stream responseStream = httpWebResponse.GetResponseStream();
277 // StreamReader streamReader = new StreamReader(responseStream, encoding);
278 // string html = streamReader.ReadToEnd();
279 // streamReader.Close();
280 // responseStream.Close();
281
282 // currentTry = 0;
283 // return html;
284 // }
285 // catch (Exception e)
286 // {
287 // Console.ForegroundColor = ConsoleColor.Red;
288 // Console.WriteLine(DateTime.Now.ToString("HH:mm:ss ") + e.Message);
289 // Console.ForegroundColor = ConsoleColor.White;
290
291 // if (currentTry <= maxTry)
292 // {
293 // GetHtml(url, postData, isPost, cookieContainer);
294 // }
295
296 // currentTry = 0;
297 // return string.Empty;
298 // }
299 //}
300 #endregion
301
302 /// <summary>
303 /// 获取指定页面的HTML代码
304 /// </summary>
305 /// <param name="url">指定页面的路径</param>
306 /// <param name="cookieCollection">Cookie集合</param>
307 /// <returns></returns>
308 public string GetHtml(string url, CookieContainer cookieContainer)
309 {
310 currentTry++;
311
312 try
313 {
314 HttpWebRequest httpWebRequest;
315 httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(url);
316 httpWebRequest.CookieContainer = cookieContainer;
317 httpWebRequest.ContentType = contentType;
318 // httpWebRequest.Referer = url;
319 //httpWebRequest.Referer
320 httpWebRequest.Accept = accept;
321 httpWebRequest.UserAgent = userAgent;
322 httpWebRequest.Method = "GET";
323 //httpWebRequest.Headers.Add("Accept-Charset", "gb2312");
324
325 HttpWebResponse httpWebResponse;
326 httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();
327 Stream responseStream = httpWebResponse.GetResponseStream();
328 StreamReader streamReader = new StreamReader(responseStream, encoding);
329
330 //string html = streamReader.ReadToEnd();
331
332 StringBuilder sb = new StringBuilder();
333 while (streamReader.Peek() > 0)
334 {
335 sb.Append(streamReader.ReadLine());
336 }
337 //string html = sb.ToString();
338 streamReader.Close();
339 responseStream.Close();
340
341 currentTry = 0;
342 //return html;
343 return sb.ToString();
344 }
345 catch (Exception e)
346 {
347 Console.ForegroundColor = ConsoleColor.Red;
348 Console.WriteLine(DateTime.Now.ToString("HH:mm:ss ") + e.Message);
349 Console.ForegroundColor = ConsoleColor.White;
350
351 if (currentTry <= maxTry)
352 {
353 GetHtml(url, cookieContainer);
354 }
355
356 currentTry = 0;
357 return string.Empty;
358 }
359 }
360
361 /// <summary>
362 /// 获取指定页面的HTML代码
363 /// </summary>
364 /// <param name="url">指定页面的路径</param>
365 /// <returns></returns>
366 public string GetHtml(string url)
367 {
368 return GetHtml(url, cc);
369 }
370
371 /// <summary>
372 /// 获取指定页面的HTML代码
373 /// </summary>
374 /// <param name="url">指定页面的路径</param>
375 /// <returns></returns>
376 public string GetHtmlByXiAHang(string url)
377 {
378 return GetHtmlByXiAHang(url, cc);
379 }
380
381 /// <summary>
382 /// 获取指定页面的HTML代码
383 /// </summary>
384 /// <param name="url">指定页面的路径</param>
385 /// <param name="cookieCollection">Cookie集合</param>
386 /// <returns></returns>
387 public string GetHtmlByXiAHang(string url, CookieContainer cookieContainer)
388 {
389 currentTry++;
390
391 try
392 {
393 HttpWebRequest httpWebRequest;
394 httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(url);
395 httpWebRequest.Headers.Add("content", "text/html; charset=GB2312");
396 httpWebRequest.CookieContainer = cookieContainer;
397 httpWebRequest.ContentType = contentType;
398 httpWebRequest.Accept = accept;
399 httpWebRequest.UserAgent = userAgent;
400 httpWebRequest.Method = "GET";
401
402
403
404
405 HttpWebResponse httpWebResponse;
406 httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();
407 Stream responseStream = httpWebResponse.GetResponseStream();
408 StreamReader streamReader = new StreamReader(responseStream, encoding);
409 string html = streamReader.ReadToEnd();
410 streamReader.Close();
411 responseStream.Close();
412
413 currentTry = 0;
414 return html;
415 }
416 catch (Exception e)
417 {
418 Console.ForegroundColor = ConsoleColor.Red;
419 Console.WriteLine(DateTime.Now.ToString("HH:mm:ss ") + e.Message);
420 Console.ForegroundColor = ConsoleColor.White;
421
422 if (currentTry <= maxTry)
423 {
424 GetHtml(url, cookieContainer);
425 }
426
427 currentTry = 0;
428 return string.Empty;
429 }
430 }
431
432
433 /// <summary>
434 /// 获取指定页面的HTML代码
435 /// </summary>
436 /// <param name="url">指定页面的路径</param>
437 /// <param name="postData">回发的数据</param>
438 /// <param name="isPost">是否以post方式发送请求</param>
439 /// <returns></returns>
440 public string GetHtml(string url, string postData, bool isPost)
441 {
442 return GetHtml(url, postData, isPost, cc);
443 }
444
445 /// <summary>
446 /// 获取指定页面的Stream
447 /// </summary>
448 /// <param name="url">指定页面的路径</param>
449 /// <param name="postData">回发的数据</param>
450 /// <param name="isPost">是否以post方式发送请求</param>
451 /// <param name="cookieCollection">Cookie集合</param>
452 /// <returns></returns>
453 public Stream GetStream(string url, CookieContainer cookieContainer)
454 {
455 //Thread.Sleep(delay);
456
457 currentTry++;
458
459 try
460 {
461 HttpWebRequest httpWebRequest;
462 httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(url);
463 httpWebRequest.CookieContainer = cookieContainer;
464 httpWebRequest.ContentType = contentType;
465 httpWebRequest.Referer = url;
466 httpWebRequest.Accept = accept;
467 httpWebRequest.UserAgent = userAgent;
468 httpWebRequest.Method = "GET";
469
470
471 HttpWebResponse httpWebResponse;
472 httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();
473 Stream responseStream = httpWebResponse.GetResponseStream();
474 currentTry = 0;
475 return responseStream;
476 }
477 catch (Exception e)
478 {
479 Console.ForegroundColor = ConsoleColor.Red;
480 Console.WriteLine(DateTime.Now.ToString("HH:mm:ss ") + e.Message);
481 Console.ForegroundColor = ConsoleColor.White;
482
483 if (currentTry <= maxTry)
484 {
485 GetHtml(url, cookieContainer);
486 }
487
488 currentTry = 0;
489 return null;
490 }
491 }
492
493 #endregion
494 }
495 }
