lucenc代码阅读指南、测试范例
2013-03-18 14:08 zhenjing 阅读(2101) 评论(1) 编辑 收藏 举报阅读指南
Lucene 原理与代码分析完整版 -- 力荐
Lucene介绍及源码剖析: http://javenstudio.org/blog/annotated-lucene -- 核心IndexWriter
下载:Annotated+Lucene+.pdf: http://ishare.iask.sina.com.cn/f/24103589.html
阅读步骤:
1、了解检索的基本原理和概念
2、了解lucene的基本概念
3、熟悉lucene的索引文件格式 -- 关键
4、熟悉lucene的索引流程:具体代码的类层次较多,且引入不必要的设计模式致使代码阅读相对困难。基本思路:controler + model 封装索引链,实现多线程并发处理(数据不共享)。
5、熟悉lucene的搜索流程
6、了解lucene搜索语法解析器 和 熟悉分词
推荐资料深入剖析lucene的源码,非常有价值。光看文档,不够形象,大体看过文档后,建议结合源码理解文档内容。代码能让读者有大体的基本概念,但文档对源码细节的解释容易让读者"只见枝叶不见森林”,理解困难。根据文档作者提供的大体思路,结合实际源码,读起来更容易。
测试
测试对于了解lucene的工作原理、代码执行流程极有帮助,是阅读代码的重要辅助手段。
IndexerExample.java
/* * Compiler: javac -classpath .:../lucene-core-2.9.1.jar:http://www.cnblogs.com/ChineseSegmenter/chineseSegmenter.jar IndexerExample.java * Exec : java -classpath .:../lucene-core-2.9.1.jar:http://www.cnblogs.com/ChineseSegmenter/chineseSegmenter.jar IndexerExample * */ import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.StringReader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.PerFieldAnalyzerWrapper; import org.apache.lucene.analysis.WhitespaceAnalyzer; import org.apache.lucene.analysis.cn.ChineseAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.DateTools; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; public class IndexerExample { private static void EnExample() throws Exception { // Store the index on disk Directory directory = FSDirectory.getDirectory("/tmp/testindex"); // Use standard analyzer Analyzer analyzer = new StandardAnalyzer(); // Create IndexWriter object IndexWriter iwriter = new IndexWriter(directory, analyzer, true); iwriter.setMaxFieldLength(25000); // make a new, empty document Document doc = new Document(); File f = new File("/tmp/test.txt"); // Add the path of the file as a field named "path". Use a field that is // indexed (i.e. searchable), but don't tokenize the field into words. doc.add(new Field("path", f.getPath(), Field.Store.YES, Field.Index.UN_TOKENIZED)); String text = "This is the text to be indexed."; doc.add(new Field("fieldname", text, Field.Store.YES, Field.Index.TOKENIZED)); doc.add(new Field("name", text, Field.Store.YES, Field.Index.TOKENIZED)); // Add the last modified date of the file a field named "modified". Use // a field that is indexed (i.e. searchable), but don't tokenize the field // into words. doc.add(new Field("modified", DateTools.timeToString(f.lastModified(), DateTools.Resolution.MINUTE), Field.Store.YES, Field.Index.UN_TOKENIZED)); // Add the contents of the file to a field named "contents". Specify a Reader, // so that the text of the file is tokenized and indexed, but not stored. // Note that FileReader expects the file to be in the system's default encoding. // If that's not the case searching for special characters will fail. doc.add(new Field("contents", new FileReader(f))); iwriter.addDocument(doc); iwriter.optimize(); iwriter.close(); } private static void CnExample() throws Exception { // Store the index on disk Directory directory = FSDirectory.getDirectory("/tmp/testindex"); // Use chinese analyzer Analyzer analyzer = new ChineseAnalyzer(); PerFieldAnalyzerWrapper wrapper = new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer()); wrapper.addAnalyzer("name", analyzer); // Create IndexWriter object IndexWriter iwriter = new IndexWriter(directory, wrapper, true); iwriter.setMaxFieldLength(25000); // make a new, empty document Document doc = new Document(); File f = new File("/tmp/test.txt"); // Add the path of the file as a field named "path". Use a field that is // indexed (i.e. searchable), but don't tokenize the field into words. doc.add(new Field("path", f.getPath(), Field.Store.YES, Field.Index.UN_TOKENIZED)); String text = "This is the text to be indexed."; doc.add(new Field("fieldname", text, Field.Store.YES, Field.Index.TOKENIZED)); String name = "2013春装新款女气质修身风衣大翻领双层大摆长款外套 系腰带"; doc.add(new Field("name", name, Field.Store.YES, Field.Index.TOKENIZED)); // Add the last modified date of the file a field named "modified". Use // a field that is indexed (i.e. searchable), but don't tokenize the field // into words. doc.add(new Field("modified", DateTools.timeToString(f.lastModified(), DateTools.Resolution.MINUTE), Field.Store.YES, Field.Index.UN_TOKENIZED)); // Add the contents of the file to a field named "contents". Specify a Reader, // so that the text of the file is tokenized and indexed, but not stored. // Note that FileReader expects the file to be in the system's default encoding. // If that's not the case searching for special characters will fail. doc.add(new Field("contents", new FileReader(f))); iwriter.addDocument(doc); iwriter.optimize(); iwriter.close(); } public static void main(String[] args) throws Exception { System.out.println("Start test: "); if( args.length > 0){ CnExample(); } else{ EnExample(); } System.out.println("Index dir: /tmp/testindex"); } }
SearcherExample.java
/* * Compiler: javac -classpath .:../lucene-core-2.9.1.jar:http://www.cnblogs.com/ChineseSegmenter/chineseSegmenter.jar SearcherExample.java * Exec : java -classpath .:../lucene-core-2.9.1.jar:http://www.cnblogs.com/ChineseSegmenter/chineseSegmenter.jar SearcherExample * */ import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.StringReader; import java.util.Date; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.cn.ChineseAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.DateTools; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Searcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.Hits; import org.apache.lucene.queryParser.QueryParser; public class SearcherExample { public static void main(String[] args) throws Exception { if (args.length < 2) { throw new Exception("Usage: java " + Searcher.class.getName() + "<index dir> <query> [cn]"); } File indexDir = new File(args[0]); String q = args[1]; boolean bCn = args.length > 2? true : false; if (!indexDir.exists() || !indexDir.isDirectory()) { throw new Exception(indexDir + " does not exist or is not a directory."); } search(indexDir, q, bCn); } public static void search(File indexDir, String q, boolean bCn) throws Exception { Directory fsDir = FSDirectory.getDirectory(indexDir, false); IndexSearcher is = new IndexSearcher(fsDir); Analyzer analyzer = new StandardAnalyzer(); if( bCn ){ analyzer = new ChineseAnalyzer(); } QueryParser parser = new QueryParser( "name", analyzer); Query query = parser.parse(q); System.out.println("Query: " + query.toString()); long start = new Date().getTime(); Hits hits = is.search(query); long end = new Date().getTime(); System.err.println("Found " + hits.length() + " document(s) (in " + (end - start) + " milliseconds) that matched query '" + q + "'"); for (int i = 0; i < hits.length(); i++) { Document doc = hits.doc(i); System.out.println( "HIT " + i + " :" + doc.get("name")); } } }
中文分词可采用lucene自带的库,效果不好,或者自行封装,核心就是封装分词Tokenizer。
package org.apache.lucene.analysis.cn; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; import java.util.List; import org.apache.commons.lang.StringUtils; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.Tokenizer; public class SnippetTermTokenizer extends Tokenizer { private StringBuffer buffer = new StringBuffer(); private BufferedReader inputBuffer; private JNISelecter selecter; // 中文分词核心类 private List<Token> tokenList = null; private List<String> phraseTokenList = null; private Iterator<Token> tokenIter = null; public SnippetTermTokenizer(Reader reader, JNISelecter s) { inputBuffer = new BufferedReader(reader, 2048); selecter = s; } public Token next() throws IOException { if (tokenIter != null) { if (tokenIter.hasNext()) { return tokenIter.next(); } else { // finish read input return null; } } // need to read content readContent(); if (segment()) { // segment succeed, create iterator return tokenIter.next(); } return null; } public void close() throws IOException { inputBuffer.close(); } // 分词相关略 }
作者:zhenjing.chen
出处:http://www.cnblogs.com/zhenjing/
未注明转载的文章,版权归作者所有,欢迎转载,但未经作者同意必须保留此段声明,且在文章页面明显位置给出原文连接,否则保留追究法律责任的权利。