Lucene:如何写一个自定义的分词器
实现一个自定义分词器
实现一个简单的英文分词器，主要分为以下几个步骤：
1.建立自己的Attribute接口MyCharAttribute
1 /** 2 * MyCharAttribute 3 * 4 * @author limingcheng 5 * @Date 2019/11/28 6 */ 7 public interface MyCharAttribute extends Attribute { 8 void setChars(char[] buffer, int length); 9 10 char[] getChars(); 11 12 int getLength(); 13 14 String getString(); 15 }
2.建立自定义attribute接口MyCharAttribute的实现类MyCharAttributeImpl
1 /** 2 * MyCharAttributeImpl 3 * 2.建立自定义attribute接口MyCharAttribute的实现类MyCharAttributeImpl 4 * 注意：MyCharAttributeImpl一定要和MyCharAttribute放在一个包下，否则会出现没有MyCharAttribute的实现类， 5 * 这是由org.apache.lucene.util.AttributeFactory.DefaultAttributeFactory.findImplClass(Class<? extends Attribute>)这个方法决定的 6 * @author limingcheng 7 * @Date 2019/11/28 8 */ 9 public class MyCharAttributeImpl extends AttributeImpl implements MyCharAttribute { 10 11 private char[] chatTerm = new char[255]; 12 private int length = 0; 13 14 @Override 15 public void setChars(char[] buffer, int length) { 16 this.length = length; 17 if (length > 0) { 18 System.arraycopy(buffer, 0, this.chatTerm, 0, length); 19 } 20 } 21 22 @Override 23 public char[] getChars() { 24 return this.chatTerm; 25 } 26 27 @Override 28 public int getLength() { 29 return this.length; 30 } 31 32 @Override 33 public String getString() { 34 if (this.length > 0) { 35 return new String(this.chatTerm, 0, length); 36 } 37 return null; 38 } 39 40 @Override 41 public void clear() { 42 this.length = 0; 43 } 44 45 @Override 46 public void reflectWith(AttributeReflector reflector) { 47 48 } 49 50 @Override 51 public void copyTo(AttributeImpl target) { 52 53 } 54 } 55 3.建立分词器MyWhitespaceTokenizer：实现对英文按空白字符进行分词 56 /** 57 * MyWhitespaceTokenizer 58 * 59 * 3. 建立分词器MyWhitespaceTokenizer：实现对英文按空白字符进行分词 60 * @author limingcheng 61 * @Date 2019/11/28 62 */ 63 public class MyWhitespaceTokenizer extends Tokenizer { 64 65 // 需要记录的属性 66 // 词 67 MyCharAttribute charAttr = this.addAttribute(MyCharAttribute.class); 68 69 // 存词的出现位置 70 71 // 存放词的偏移 72 73 // 74 char[] buffer = new char[255]; 75 int length = 0; 76 int c; 77 78 @Override 79 public boolean incrementToken() throws IOException { 80 // 清除所有的词项属性 81 clearAttributes(); 82 length = 0; 83 while (true) { 84 c = this.input.read(); 85 86 if (c == -1) { 87 if (length > 0) { 88 // 复制到charAttr 89 this.charAttr.setChars(buffer, length); 90 return true; 91 } else { 92 return false; 93 } 94 } 95 96 if (Character.isWhitespace(c)) { 97 if (length > 0) { 98 // 复制到charAttr 99 this.charAttr.setChars(buffer, length); 100 return true; 101 } 102 } 103 104 buffer[length++] = (char) c; 105 } 106 } 107 108 }
4.建立分项过滤器：把大写字母转换为小写字母
1 /** 2 * MyLowerCaseTokenFilter 3 * 4 * 4.建立分项过滤器：把大写字母转换为小写字母 5 * @author limingcheng 6 * @Date 2019/11/28 7 */ 8 public class MyLowerCaseTokenFilter extends TokenFilter { 9 public MyLowerCaseTokenFilter(TokenStream input) { 10 super(input); 11 } 12 13 MyCharAttribute charAttr = this.addAttribute(MyCharAttribute.class); 14 15 @Override 16 public boolean incrementToken() throws IOException { 17 boolean res = this.input.incrementToken(); 18 if (res) { 19 char[] chars = charAttr.getChars(); 20 int length = charAttr.getLength(); 21 if (length > 0) { 22 for (int i = 0; i < length; i++) { 23 chars[i] = Character.toLowerCase(chars[i]); 24 } 25 } 26 } 27 return res; 28 } 29 }
5.建立分析器
1 /** 2 * MyWhitespaceAnalyzer 3 * 4 * 5. 建立分析器 5 * @author limingcheng 6 * @Date 2019/11/28 7 */ 8 public class MyWhitespaceAnalyzer extends Analyzer { 9 @Override 10 protected TokenStreamComponents createComponents(String fieldName) { 11 Tokenizer source = new MyWhitespaceTokenizer(); 12 TokenStream filter = new MyLowerCaseTokenFilter(source); 13 return new TokenStreamComponents(source, filter); 14 } 15 16 public static void main(String[] args) { 17 18 String text = "广州华为有限公司 An AttributeSource contains a list of different AttributeImpls, and methods to add and get them. "; 19 20 try { 21 Analyzer ana = new MyWhitespaceAnalyzer(); 22 TokenStream ts = ana.tokenStream("aa", text); 23 MyCharAttribute ca = ts.getAttribute(MyCharAttribute.class); 24 ts.reset(); 25 while (ts.incrementToken()) { 26 System.out.print(ca.getString() + "|"); 27 } 28 ts.end(); 29 ana.close(); 30 System.out.println(); 31 } catch (IOException e) { 32 e.printStackTrace(); 33 } 34 35 } 36 }
一个简单的分词器可以这样实现，但是要实现一个可以对中文分词的分词器就需要算法方面的知识了。
本文参考：https://www.cnblogs.com/leeSmall/p/8993185.html
