Lucene 自定义分词器 

 1 package com.lucene.util;
 2 
 3 import java.io.Reader;
 4 import java.util.Set;
 5 
 6 import org.apache.lucene.analysis.Analyzer;
 7 import org.apache.lucene.analysis.LetterTokenizer;
 8 import org.apache.lucene.analysis.LowerCaseFilter;
 9 import org.apache.lucene.analysis.StopAnalyzer;
10 import org.apache.lucene.analysis.StopFilter;
11 import org.apache.lucene.analysis.TokenStream;
12 import org.apache.lucene.util.Version;
13 
14 //定义禁用词分词器
15 public class UserDefinedAnalyzer extends Analyzer {
16 
17     //定义禁用词集合
18     private Set stops;
19     
20     //无参构造器使用默认的禁用词分词器
21     public UserDefinedAnalyzer(){
22         stops=StopAnalyzer.ENGLISH_STOP_WORDS_SET;
23     }
24     
25     /**
26      * 传一个禁用词数组
27      * @param sws
28      */
29     public UserDefinedAnalyzer(String[] sws){
30         //使用stopFilter创建禁用词集合
31         stops=StopFilter.makeStopSet(Version.LUCENE_35,sws,true);
32         //将默认的禁用词添加进集合
33         stops.addAll(StopAnalyzer.ENGLISH_STOP_WORDS_SET);
34     }
35     
36     /**
37      * 自定义分词器
38      */
39     @Override
40     public TokenStream tokenStream(String str, Reader reader) {
41         
42         return new StopFilter    
43                 (Version.LUCENE_35, 
44                         new LowerCaseFilter    
45                 (Version.LUCENE_35, 
46                         new LetterTokenizer(
47                 Version.LUCENE_35, reader)), stops);
48     }
49 
50 }
    @Test
    public void test04(){
        Analyzer a1=new UserDefinedAnalyzer(new String[]{"my","name"});
        //Analyzer a1=new UserDefinedAnalyzer();
        String str="my name is paul";
        AnalyzerUtil.displayToken(str, a1);
    }

 

posted @ 2012-04-22 11:49  Paul.Lau  阅读(3291)  评论(0)  编辑  收藏  举报