一个C#分析html的包，从老外的一个Spider程序里提取出来的

一个C#分析html的包，以前思考如何分析html，总是毫无头绪，情况太多，太复杂。后来在一个老外写的Spider程序里发现了这个。非常好用，屡试不爽，现在才知道自己写程序还在一个很低的层次，尤其是面向对象思想的理解。利用继承的思想来分解问题，利用多个类来实现多种情况的变化，这个才是OO思想，而自己很多时候只是为了OO而去OO,其实水平还是停留在面向过程里。好好分析下这些代码，受益颇多。

/// <summary> /// Attribute holds one attribute, as is normally stored in /// an HTML or XML file. This includes a name, value and delimiter. /// /// This spider is copyright 2003 by Jeff Heaton. However, it is /// released under a Limited GNU Public License (LGPL). You may /// use it freely in your own programs. For the latest version visit /// http://www.jeffheaton.com. /// /// </summary> public class Attribute: ICloneable { /// <summary> /// The name of this attribute /// </summary> private string m_name; /// <summary> /// The value of this attribute /// </summary> private string m_value; /// <summary> /// The delimiter for the value of this /// attribute(i.e. " or '). /// </summary> private char m_delim; /// <summary> /// Construct a new Attribute. The name, delim and value /// properties can be specified here. /// </summary> /// <param name="name">The name of this attribute.</param> /// <param name="value">The value of this attribute.</param> /// <param name="delim">The delimiter character for the value.</param> public Attribute(string name,string value,char delim) { m_name = name; m_value = value; m_delim = delim; } /// <summary> /// The default constructor. Construct a blank attribute. /// </summary> public Attribute():this("","",(char)0) { } /// <summary> /// Construct an attribute without a delimiter. /// </summary> /// <param name="name">The name of this attribute.</param> /// <param name="value">The value of this attribute.</param> public Attribute(String name,String value):this(name,value,(char)0) { } /// <summary> /// The delimiter for this attribute. /// </summary> public char Delim { get { return m_delim; } set { m_delim = value; } } /// <summary> /// The name for this attribute. /// </summary> public string Name { get { return m_name; } set { m_name = value; } } /// <summary> /// The value for this attribute. /// </summary> public string Value { get { return m_value; } set { m_value = value; } } #region ICloneable Members public virtual object Clone() { return new Attribute(m_name,m_value,m_delim); } #endregion }
以上是一个基础类，专门用来分析html属性的。

public class AttributeList:Attribute { /// <summary> /// An internally used Vector. This vector contains /// the entire list of attributes. /// </summary> protected ArrayList m_list; /// <summary> /// Make an exact copy of this object using the cloneable interface. /// </summary> /// <returns>A new object that is a clone of the specified object.</returns> public override Object Clone() { AttributeList rtn = new AttributeList(); for ( int i=0;i<m_list.Count;i++ ) rtn.Add( (Attribute)this[i].Clone() ); return rtn; } /// <summary> /// Create a new, empty, attribute list. /// </summary> public AttributeList():base("","") { m_list = new ArrayList(); } /// <summary> /// Add the specified attribute to the list of attributes. /// </summary> /// <param name="a">An attribute to add to this AttributeList.</param> public void Add(Attribute a) { m_list.Add(a); } /// <summary> /// Clear all attributes from this AttributeList and return it /// to a empty state. /// </summary> public void Clear() { m_list.Clear(); } /// <summary> /// Returns true of this AttributeList is empty, with no attributes. /// </summary> /// <returns>True if this AttributeList is empty, false otherwise.</returns> public bool IsEmpty() { return( m_list.Count<=0); } /// <summary> /// If there is already an attribute with the specified name, /// then it will have its value changed to match the specified value. /// If there is no Attribute with the specified name, then one will /// be created. This method is case-insensitive. /// </summary> /// <param name="name">The name of the Attribute to edit or create. Case-insensitive.</param> /// <param name="value">The value to be held in this attribute.</param> public void Set(string name,string value) { if ( name==null ) return; if ( value==null ) value=""; Attribute a = this[name]; if ( a==null ) { a = new Attribute(name,value); Add(a); } else a.Value = value; } /// <summary> /// How many attributes are in this AttributeList /// </summary> public int Count { get { return m_list.Count; } } /// <summary> /// A list of the attributes in this AttributeList /// </summary> public ArrayList List { get { return m_list; } } /// <summary> /// Access the individual attributes /// </summary> public Attribute this[int index] { get { if ( index<m_list.Count ) return(Attribute)m_list[index]; else return null; } } /// <summary> /// Access the individual attributes by name. /// </summary> public Attribute this[string index] { get { int i=0; while ( this[i]!=null ) { if ( this[i].Name.ToLower().Equals( (index.ToLower()) )) return this[i]; i++; } return null; } } }
以上是一个属性列表。

public class Parse:AttributeList { /// <summary> /// The source text that is being parsed. /// </summary> private string m_source; /// <summary> /// The current position inside of the text that /// is being parsed. /// </summary> private int m_idx; /// <summary> /// The most reciently parsed attribute delimiter. /// </summary> private char m_parseDelim; /// <summary> /// This most receintly parsed attribute name. /// </summary> private string m_parseName; /// <summary> /// The most reciently parsed attribute value. /// </summary> private string m_parseValue; /// <summary> /// The most reciently parsed tag. /// </summary> public string m_tag; /// <summary> /// Determine if the specified character is whitespace or not. /// </summary> /// <param name="ch">A character to check</param> /// <returns>true if the character is whitespace</returns> public static bool IsWhiteSpace(char ch) { return( "\t\n\r ".IndexOf(ch) != -1 ); } /// <summary> /// Advance the index until past any whitespace. /// </summary> public void EatWhiteSpace() { while ( !Eof() ) { if ( !IsWhiteSpace(GetCurrentChar()) ) return; m_idx++; } } /// <summary> /// Determine if the end of the source text has been /// reached. /// </summary> /// <returns>True if the end of the source text has been /// reached.</returns> public bool Eof() { return(m_idx>=m_source.Length ); } /// <summary> /// Parse the attribute name. /// </summary> public void ParseAttributeName() { EatWhiteSpace(); // get attribute name while ( !Eof() ) { if ( IsWhiteSpace(GetCurrentChar()) || (GetCurrentChar()=='=') || (GetCurrentChar()=='>') ) break; m_parseName+=GetCurrentChar(); m_idx++; } EatWhiteSpace(); } /// <summary> /// Parse the attribute value /// </summary> public void ParseAttributeValue() { if ( m_parseDelim!=0 ) return; if ( GetCurrentChar()=='=' ) { m_idx++; EatWhiteSpace(); if ( (GetCurrentChar()=='\'') || (GetCurrentChar()=='\"') ) { m_parseDelim = GetCurrentChar(); m_idx++; while ( GetCurrentChar()!=m_parseDelim ) { m_parseValue+=GetCurrentChar(); m_idx++; } m_idx++; } else { while ( !Eof() && !IsWhiteSpace(GetCurrentChar()) && (GetCurrentChar()!='>') ) { m_parseValue+=GetCurrentChar(); m_idx++; } } EatWhiteSpace(); } } /// <summary> /// Add a parsed attribute to the collection. /// </summary> public void AddAttribute() { Attribute a = new Attribute(m_parseName, m_parseValue,m_parseDelim); Add(a); } /// <summary> /// Get the current character that is being parsed. /// </summary> /// <returns></returns> public char GetCurrentChar() { return GetCurrentChar(0); } /// <summary> /// Get a few characters ahead of the current character. /// </summary> /// <param name="peek">How many characters to peek ahead for.</param> /// <returns>The character that was retrieved.</returns> public char GetCurrentChar(int peek) { if( (m_idx+peek)<m_source.Length ) return m_source[m_idx+peek]; else return (char)0; } /// <summary> /// Obtain the next character and advance the index by one. /// </summary> /// <returns>The next character</returns> public char AdvanceCurrentChar() { return m_source[m_idx++]; } /// <summary> /// Move the index forward by one. /// </summary> public void Advance() { m_idx++; } /// <summary> /// The last attribute name that was encountered. /// </summary> public string ParseName { get { return m_parseName; } set { m_parseName = value; } } /// <summary> /// The last attribute value that was encountered. /// </summary> public string ParseValue { get { return m_parseValue; } set { m_parseValue = value; } } /// <summary> /// The last attribute delimeter that was encountered. /// </summary> public char ParseDelim { get { return m_parseDelim; } set { m_parseDelim = value; } } /// <summary> /// The text that is to be parsed. /// </summary> public string Source { get { return m_source; } set { m_source = value; } } }
上面的是分析标签的，并且将标签的属性Parse进属性列表里。

public class ParseHTML:Parse { public AttributeList GetTag() { AttributeList tag = new AttributeList(); tag.Name = m_tag; foreach(Attribute x in List) { tag.Add((Attribute)x.Clone()); } return tag; } public String BuildTag() { String buffer="<"; buffer+=m_tag; int i=0; while ( this[i]!=null ) {// has attributes buffer+=" "; if ( this[i].Value == null ) { if ( this[i].Delim!=0 ) buffer+=this[i].Delim; buffer+=this[i].Name; if ( this[i].Delim!=0 ) buffer+=this[i].Delim; } else { buffer+=this[i].Name; if ( this[i].Value!=null ) { buffer+="="; if ( this[i].Delim!=0 ) buffer+=this[i].Delim; buffer+=this[i].Value; if ( this[i].Delim!=0 ) buffer+=this[i].Delim; } } i++; } buffer+=">"; return buffer; } protected void ParseTag() { m_tag=""; Clear(); // Is it a comment? if ( (GetCurrentChar()=='!') && (GetCurrentChar(1)=='-')&& (GetCurrentChar(2)=='-') ) { while ( !Eof() ) { if ( (GetCurrentChar()=='-') && (GetCurrentChar(1)=='-')&& (GetCurrentChar(2)=='>') ) break; if ( GetCurrentChar()!='\r' ) m_tag+=GetCurrentChar(); Advance(); } m_tag+="--"; Advance(); Advance(); Advance(); ParseDelim = (char)0; return; } // Find the tag name while ( !Eof() ) { if ( IsWhiteSpace(GetCurrentChar()) || (GetCurrentChar()=='>') ) break; m_tag+=GetCurrentChar(); Advance(); } EatWhiteSpace(); // Get the attributes while ( GetCurrentChar()!='>' ) { ParseName = ""; ParseValue = ""; ParseDelim = (char)0; ParseAttributeName(); if ( GetCurrentChar()=='>' ) { AddAttribute(); break; } // Get the value(if any) ParseAttributeValue(); AddAttribute(); } Advance(); } public char Parse() { if( GetCurrentChar()=='<' ) { Advance(); char ch=char.ToUpper(GetCurrentChar()); if ( (ch>='A') && (ch<='Z') || (ch=='!') || (ch=='/') ) { ParseTag(); return (char)0; } else return(AdvanceCurrentChar()); } else return(AdvanceCurrentChar()); } }
最后一个类是用来从HTML里分离出Tag，进而适合分析属性时使用。

通过以上4个类，将一个原本很难分析问题的变得很轻松的解决了。其实这里就透露出一种分析问题解决问题的思考过程。我想程序员应该养成的习惯就是将一个复杂的问题步步分解，分解成很多个很细小的问题，小问题解决了，这个复杂的问题也就解决了。而这些问题之间的逻辑联系如何组织，如何做到偶合性最低，是很值得思考的一个问题。现在想想以前在大学里学的软件工程，似乎可以渐渐明白了什么是健壮，什么是可复用。