菜鸟练习C#htmlparser----C#正则加htmlDOM进行网页解析腾讯新闻帖子列表相关信息提取

http://blog.csdn.net/finallyliuyu/archive/2009/05/06/4156071.aspx

 

适用网站有

7.奇闻异事：http://news.qq.com/newssh/qwqs/qwqs3j.htm（简单无其他连接）

<tr><td>·<a target="_blank" class="rlk1" href="/a/20090504/000868.htm">组图：地球上最像外星的九大地方</a><span class="text06">(2009年05月04日11:17)</span>

</td></tr>

时间

8.社会观察：http://news.qq.com/newssh/sh-shgc3jy.htm（同上）
tr><td>·<a target="_blank" class="rlk1" href="/a/20090427/000359.htm">齐鲁晚报：“三鹿”之后为何仍有“晨园”</a><span class="text06">(2009年04月27日08:01)</span>

9.新闻云南http://news.qq.com/yn/ywy.htm（同上）
<tr><td>· <a target="_blank" href="/a/20090407/000619.htm">云铜腐案追踪：国资领导机制存巨大腐败漏洞</a>　<span class="time">(04/07&#160;09:22)</span>

</td></tr>

10.冷暖人间http://news.qq.com/newssh/sh-lnrj3jy.htm(同上)

<tr><td>·<a target="_blank" class="rlk1" href="/a/20090504/000525.htm">打工女子患肾衰竭 希望捐眼角膜送他人光明</a><span class="text06">(2009年05月04日09:09)</span>

</td></tr>

11.滚动新闻 http://news.qq.com/scroll/now.htm（同上）

14.港澳台http://news.qq.com/newsgn/x_gat3.htm(同上)

<tr><td>·<a target="_blank" class="rlk1" href="/a/20090502/001016.htm">香港万余人参加佛典 佛教联合会长谈流感(图)</a><span class="text06">(2009年05月02日21:17)</span>

15. 国内评论http://news.qq.com/newsgn/gnplgdy.htm（同上）
16. 社会万象http://news.qq.com/newssh/shwx_3jy.htm(同上)

 

 

<tr><td class="text01">·<a target="_blank" href="/a/20090504/001612.htm">一个白领的“五四”青年节</a><span class="text06">(04日

    19:41)</span>

</td></tr>

 

代码说明：

1.提上面得网页信息完全可以用一个正则搞定。我是为了练习C#htmlparser的使用才搞的这么麻烦的。(项目需要涉及htmlDOM)

2.代码注释不祥很抱歉。这个代码要装C#hmlparser库才能运行（见我博客的上一篇文章）

3.我会把可执行代码放到我的csdn资源空间大家可以下载（绝对能够正常运行）但是再次为我不规范的代码给大家阅读上造成的困难道歉。

4.代码中调用htmlparser之前有一些用正则对字符串进行预处理的环节。这是因为<TD><A></A></TD>。htmlparser能够识别<A>是<TD>的孩子。但是要<TD>(任意字符，空白或回车)<A></A></TD>则htmlparser就识别不出来了。它的智能实在太差啦！！！

using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.IO;
using System.Diagnostics;
using System.Text.RegularExpressions;
using Winista.Text.HtmlParser;
using Winista.Text.HtmlParser.Visitors;
using Winista.Text.HtmlParser.Util;
using Winista.Text.HtmlParser.Tags;
using Winista.Text.HtmlParser.Filters;
using Winista.Text.HtmlParser.Lex;

namespace ListRetrival
{
    class Program
    {
        static void Main(string[] args)
        {
            string url = "http://news.qq.com/newsgn/x_gat3.htm";
           string rawtext = GetDataFromUrl(url);
            //StreamReader sr=new StreamReader("C:\\temp.txt",Encoding.GetEncoding("gb2312"));
            //string   rawtext=sr.ReadToEnd();
            //sr.Close();
            //文本预处理
            Regex regex = new Regex(@"<tr>\s+<td>",RegexOptions.Multiline|RegexOptions.IgnoreCase);
            rawtext=regex.Replace(rawtext,@"<tr><td>");
            rawtext = new Regex(@"<td(.*?)>\s*·\s*<a", RegexOptions.Multiline | RegexOptions.IgnoreCase).Replace(rawtext, "<td${1}><a");
            rawtext = new Regex(@"<br>", RegexOptions.Multiline | RegexOptions.IgnoreCase).Replace(rawtext, "");
            rawtext = new Regex(@"<(?<first>.*?)>\s+<(?<second>.*?)>", RegexOptions.IgnoreCase | RegexOptions.Multiline).Replace(rawtext, "<${first}><${second}>");

           // Console.Write(rawtext);
            //StreamWriter sw = new StreamWriter("C:\\finally.txt", true, Encoding.GetEncoding("gb2312"));
           // sw.Write(rawtext);
            //sw.Close();
            //Console.Write(rawtext);
            Lexer lexer = new Lexer(rawtext);
            Parser parser = new Parser(lexer);
            NodeFilter filter = new TagNameFilter("TR");
            NodeList htmlNodes = parser.Parse(filter);
            NodeList pure = new NodeList();
            for (int j = 0; j < htmlNodes.Count; j++)
            {
                if (htmlNodes[j].FirstChild.FirstChild != null)
                    if (htmlNodes[j].FirstChild.FirstChild.GetType().ToString() == "Winista.Text.HtmlParser.Tags.ATag")
                        pure.Add(htmlNodes[j]);
            }
            string pureafter = pure.ToHtml();
            //再一步去纯
            Lexer lexer2 = new Lexer(pureafter);
            Parser parser2 = new Parser(lexer2);
            NodeFilter filter2 = new TagNameFilter("TR");
            NodeList htmlNodes2 = parser2.Parse(filter2);
            NodeList final = new NodeList();   
            for (int i = 0; i<htmlNodes2.Count; i++)
           {
               if (htmlNodes2[i].FirstChild.FirstChild.NextSibling != null)
                   if (htmlNodes2[i].FirstChild.FirstChild.NextSibling.GetType().ToString() == "Winista.Text.HtmlParser.Tags.Span")
                       final.Add(htmlNodes2[i]);
                       
            }
           // Console.Write(final.ToHtml());
            if (final.Count == 0)
            {
                Console.Write("时间不是<span>格式");
            }
            else
            {   List<String> infolist=new List<string>();
                string finalstr = final.ToHtml();
                //StreamWriter sw = new StreamWriter("C:\\finally.txt", true, Encoding.GetEncoding("gb2312"));
                //sw.Write(finalstr);
                //sw.Close();
                Regex keyregex = new Regex(@"<a.*?href=""(.*?)"">(.*?)</a><span.*?>(.*?)</span>");
                MatchCollection mycollection = keyregex.Matches(finalstr);
                foreach (Match mymatch in mycollection)
                {
                    infolist.Add(mymatch.Groups[1].ToString() + mymatch.Groups[2].ToString() + mymatch.Groups[3].ToString());
                }
                string keyinfo = "";
                for (int k = 0; k < infolist.Count; k++)
                {
                    Console.WriteLine(infolist[k]);
                    keyinfo += infolist[k];
                }
                StreamWriter sw = new StreamWriter("C:\\finally.txt", true, Encoding.GetEncoding("gb2312"));
                sw.Write(keyinfo);
                sw.Close();
               
               
            }

           
           
           
        
           //Console.WriteLine("finish!");

           
            Console.ReadLine();

        }

        public static string GetDataFromUrl(string url)
        {
            string str = string.Empty;
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
            //设置Http头；
            request.AllowAutoRedirect = true;
            request.AllowWriteStreamBuffering = true;
            request.Referer = "";
            request.Timeout = 10 * 1000;
            request.UserAgent = "";
            HttpWebResponse response = null;
            try
            {
                response = (HttpWebResponse)request.GetResponse();
                if (response.StatusCode == HttpStatusCode.OK)
                {
                    //根据http应答头来判别编码
                    string Characterset = response.CharacterSet;
                    Encoding encode;
                    if (Characterset != "")
                    {
                        if (Characterset == "ISO-8859-1")
                        {
                            Characterset = "gb2312";
                        }
                        encode = Encoding.GetEncoding(Characterset);
                    }
                    else
                    {
                        encode = Encoding.Default;
                    }
                    //声明一个内存流来贮存http应答流
                    Stream Receivestream = response.GetResponseStream();
                    MemoryStream mstream = new MemoryStream();
                    byte[] bf = new byte[255];
                    int count = Receivestream.Read(bf, 0, 255);
                    while (count > 0)
                    {
                        mstream.Write(bf, 0, count);
                        count = Receivestream.Read(bf, 0, 255);
                    }
                    Receivestream.Close();
                    mstream.Seek(0, SeekOrigin.Begin);
                    //从内存流里读取字符串这里涉及到了编码方案
                    StreamReader reader = new StreamReader(mstream, encode);
                    char[] buf = new char[1024];
                    count = reader.Read(buf, 0, 1024);
                    while (count > 0)
                    {
                        str += new string(buf, 0, 1024);
                        count = reader.Read(buf, 0, 1024);

                    }
                    reader.Close();
                    mstream.Close();

                }

            }
            catch (Exception ex)
            {
                Trace.TraceError(ex.ToString());

            }
            finally
            {
                if (response != null)
                    response.Close();
            }

            return str;
        }

      

    }
}

 

本文来自CSDN博客，转载请标明出处：http://blog.csdn.net/finallyliuyu/archive/2009/05/06/4156071.aspx