lucene下的一个自定义分词

public class ICTCLASAnalyzer : Analyzer
    {
        //定义要过滤的词
        public static readonly System.String[] CHINESE_ENGLISH_STOP_WORDS = new string[428];
        public string NoisePath = Environment.CurrentDirectory + "\\data\\stopwords.txt";
        public ICTCLASAnalyzer()
        {
            StreamReader reader = new StreamReader(NoisePath, System.Text.Encoding.Default);
            string noise = reader.ReadLine();
            int i = 0;
            while (!string.IsNullOrEmpty(noise))
            {
                CHINESE_ENGLISH_STOP_WORDS[i] = noise;
                noise = reader.ReadLine();
                i++;
                if (i >= 428)
                    break;
            }
        }

        public override TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader)
        {
            TokenStream result = new ICTCLASTokenizer(reader);
            result = new StandardFilter(result);
            result = new LowerCaseFilter(result);
          
            result = new StopFilter(result, CHINESE_ENGLISH_STOP_WORDS);
            return result;
        }


    }

posted @ 2010-04-27 21:51  searchDM  阅读(235)  评论(0编辑  收藏  举报