C#写一个简单爬虫

最近研究C#的爬虫写法，搞了半天，才在网上很多的写法中整理出了一个简单的demo（本人菜鸟，大神勿喷）。一是为了自己记录一下以免日后用到，二是为了供需要朋友参考。

废话不多说，上代码

using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;

namespace Crawler
{
    class Program
    {
        static void Main(string[] args)
        {

            //WebProxy proxyObject = new WebProxy(IP, PORT);//这里我是用的代理。

            //向指定地址发送请求
            HttpWebRequest HttpWReq = (HttpWebRequest)WebRequest.Create("http://news.baidu.com/");
            //HttpWReq.Proxy = proxyObject;
            HttpWReq.Timeout = 10000;
            HttpWebResponse HttpWResp = (HttpWebResponse)HttpWReq.GetResponse();
            StreamReader sr = new StreamReader(HttpWResp.GetResponseStream(), Encoding.GetEncoding("UTF-8"));
            HtmlDocument doc = new HtmlDocument();
            doc.Load(sr);
            HtmlNodeCollection ulNodes = doc.DocumentNode.SelectSingleNode("//div[@id='pane-news']").SelectNodes("ul");
            if (ulNodes != null && ulNodes.Count > 0)
            {
                for (int i = 0; i < ulNodes.Count; i++)
                {
                    HtmlNodeCollection liNodes = ulNodes[i].SelectNodes("li");
                    for (int j = 0; j < liNodes.Count; j++)
                    {
                        string title = liNodes[j].SelectSingleNode("a").InnerHtml.Trim();
                        string href = liNodes[j].SelectSingleNode("a").GetAttributeValue("href", "").Trim();
                        Console.WriteLine("新闻标题：" + title + ",链接：" + href);
                    }
                }
            }
            Console.ReadLine();
            sr.Close();
            HttpWResp.Close();
            HttpWReq.Abort();
        }
    }
}

其中解析html的写法用到了XPath的语法，大家可以自行百度下，比较简单。