词法分析器Demo

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace LexerDemo
{

    public enum TokenType
    {
        Word,
        String,
        Number,
        SplitChar,
        EscChar,

        YearFlag,
        MonthFlag,
        DayFlag,
        HourFlag,
        MinutFlag,
        SecondFlag
    }
    public class Token
    {
        public string Text;
        public TokenType Kind;
        public int Len;
        public Token(TokenType type)
        {
            Text = string.Empty;
            Len = 0;
            Kind = type;
        }
    }

    public class Lexer
    {
        public Token ReadNumber(string text, int pos)
        {

            Token t = new Token(TokenType.Number);

            for (int i = pos; i < text.Length; i++)
            {
                char c;
                c = text[i];

                if (char.IsNumber(c) != true)
                {
                    break;
                }
                t.Text += c;
                t.Len++;
            }
            return t;
        }

        public Token ReadToken(string text, int pos, char prefix)
        {
            Token t = new Token(TokenType.Word);

            for (int i = pos; i < text.Length; i++)
            {
                char c;
                c = text[i];

                if (c != prefix)
                {
                    break;
                }
                t.Text += c;
                t.Len++;
            }
            return t;
        }

        public Token ReadSplitOperater(string text, int pos, char prefix)
        {
            Token t = new Token(TokenType.Word);

            for (int i = pos; i < pos+1; i++)
            {
                char c;
                c = text[i];

                if (c != prefix)
                {
                    break;
                }
                t.Text += c;
                t.Len++;
            }
            return t;
        }

        public Token ReadSplitString(string text, int pos)
        {
            Token t = new Token(TokenType.SplitChar);

            for (int i = pos; i < text.Length; i++)
            {
                char c;
                c = text[i];

                if (c != '#')
                {
                    break;
                }
                t.Text += c;
                t.Len++;
            }
            return t;
        }


        public IEnumerable<Token> Parse(string line)
        {
            for (int i = 0; i < line.Length; i++)
            {
                char c = line[i];

                Token token = null;
                if (Char.IsNumber(c) == true)
                {
                    token = ReadNumber(line, i);
                }
                else if (c == 'y')
                {
                    token = ReadToken(line, i, c);
                    token.Kind = TokenType.YearFlag;
                }
                else if (c == 'm')
                {
                    token = ReadToken(line, i, c);
                    token.Kind = TokenType.MonthFlag;
                }
                else if (c == 'd')
                {
                    token = ReadToken(line, i, c);
                    token.Kind = TokenType.DayFlag;
                }
                else //记录分隔符，也就是说除了以上的字符，其他均看做分隔符
                {
                    token = ReadToken(line, i, c);
                    token.Kind = TokenType.SplitChar;
//                    continue;
                }
                i += token.Len - 1;
                yield return token;
            }
            yield break;
        }
    }
}

 

 

 

 

 

/*
 * 由SharpDevelop创建。
 * 用户： Administrator
 * 日期: 2013/9/8
 * 时间: 17:27
 * 
 * 要改变这种模板请点击 工具|选项|代码编写|编辑标准头文件
 */
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Text;
using System.Windows.Forms;


namespace LexerDemo
{
    /// <summary>
    /// Description of MainForm.
    /// </summary>
    public partial class MainForm : Form
    {
        public MainForm()
        {
            InitializeComponent();
        }
        
        void Button1Click(object sender, EventArgs e)
        {
                string pbmask = textBox1.Text;
         
                string mskstr=  GetPBMask(pbmask);
           
            
        }
        
        private string GetPBMask(string pbmask)
        {
            Lexer o = new Lexer();
            
            IEnumerable<Token> strlist = o.Parse(pbmask);
            
            StringBuilder sb=new StringBuilder();
            List<string> tokenlist=new List<string>();
            foreach (Token item in strlist)
            {
               
                tokenlist.Add(item.Text);
             
            }
          
          
            return sb.ToString() ;
        }
    }
}