Lucene.net 2.0 中文分词后语法高亮问题

Lucene.net 2.0  src包里自带了Highlighter.Net-2.0.0,可以用来实现语法高亮。

    //定义多条件搜索分析器
  BooleanQuery bquery = new BooleanQuery();
   //定义分词器
   Analyzer analyzer  = new StandardAnalyzer();

   //多条件搜索拆分器
            MultiFieldQueryParser parser = new MultiFieldQueryParser(new string[] { "title", "content" }, analyzer);
            query = parser.Parse(key);
            bquery.Add(query, BooleanClause.Occur.MUST);
                
            DateTime now = DateTime.Now;
   //初始化搜索器
   //实现分布式搜索
   List<IndexReader> indexreaders = new List<IndexReader>();
   string[] dirs = Directory.GetDirectories(dir);
   if (searchtype == SearchType.None)
   {
    foreach (string item in dirs)
    {
     //System.Web.HttpContext.Current.Response.Write(item);
     indexreaders.Add(IndexReader.Open(Path.Combine(Path.Combine(dir, item), "Save")));
    }
   }
   else
   {
    //System.Web.HttpContext.Current.Response.Write(searchtype.ToString());
    indexreaders.Add(IndexReader.Open(Path.Combine(Path.Combine(dir, searchtype.ToString()), "Save")));
   }

   MultiReader reader = new MultiReader(indexreaders.ToArray());
   indexSearcher = new IndexSearcher(reader);

   Hits hits = null;
   hits = indexSearcher.Search(bquery);
   timer = (DateTime.Now - now).TotalMilliseconds;

 int count = hits.Length();
   
            /* 计算显示的条目 */
      int       start = (pageNo - 1) * 10;
      int       end = pageNo * 10 > count ? count : pageNo * 10;
            //Response.Write(readerhelper.MyQuery.ToString());
            /* 语法高亮显示设置 */
         Highlighter        highlighter = new Highlighter(new QueryScorer(query ));
       highlighter.SetTextFragmenter(new SimpleFragmenter(100));

for (int i = start; i < end; i++)
{
     Lucene.Net.Documents.Document doc = hits.Doc(i);
              System.String text = doc.Get("content");
     //添加结尾，保证结尾特殊符号不被过滤
              string title = doc.Get("title") + "+aaaaaaaaa";
     Lucene.Net.Analysis.TokenStream tokenStream = analyzer  .TokenStream("content", new System.IO.StringReader(text));
     Lucene.Net.Analysis.TokenStream titkeStream = analyzer .TokenStream("title", new System.IO.StringReader(title));
     System.String result = highlighter.GetBestFragments(tokenStream, text, 2, "...");
     string tresult = highlighter.GetBestFragments(titkeStream, title, 0, "..");
     //祛除标题结尾标记
     if (tresult.Length > 10)
      tresult = tresult.Remove(tresult.Length - 10, 10);
     if (string.IsNullOrEmpty(tresult))
      tresult = title.Remove(title.Length - 10, 10);
     //未标注内容读取
     if (string.IsNullOrEmpty(result))
     {
      if (text.Length > 100)
       result = text.Substring(0, 100);
      else
       result = text;
     }
     if (result.Length < text.Length)
      result = result + "...";
}

这是使用lucene.net的自带分词器StandardAnalyzer，有个弊端是一个字就认为是一个词。现在我们要用自己的中文词义分词器——MyAnalyzer的话，问题来了。Highlighter一直报错。为什么会出现这种情况呢？那是因为，中文分词器，分词完成一般有分割符号。比如，对于词“沪江英语”，分割完成后变成"沪江 英语"。也就是说，返回的Token是(沪江,0,2)(英语,3,5)。而如果用Highlighter，它想要取得词是(沪江,0,2)(英语,2,4)，这个就是因为空格而产生的偏差，引起了Highlighter的报错。修改Highlighter比较麻烦，要对分词器做特殊处理也比较难。可以使用字典解释器，分割结果后再高亮。比如，取得"沪江英语"的分词结果"沪江 英语",然后，把分词结果传递给Highlighter。弊端是，搜索结果会凭空出现很多空格。俺们懒人自然有懒人的解决办法。那就是用MyAnalyzer实现索引和搜索，用StandardAnalyzer实现高亮：

    //定义多条件搜索分析器
  BooleanQuery bquery = new BooleanQuery();
   //定义分词器
   Analyzer analyzer  = new MyAnalyzer();
  Analyzer highanalyzer  = new StandardAnalyzer();
MultiFieldQueryParser parser = new MultiFieldQueryParser(new string[] { "title", "content" }, highanalyzer );
Query highquery = parser .Parse(key);

   //多条件搜索拆分器
            MultiFieldQueryParser parser = new MultiFieldQueryParser(new string[] { "title", "content" }, analyzer);
            query = parser.Parse(key);
            bquery.Add(query, BooleanClause.Occur.MUST);
                
            DateTime now = DateTime.Now;
   //初始化搜索器
   //实现分布式搜索
   List<IndexReader> indexreaders = new List<IndexReader>();
   string[] dirs = Directory.GetDirectories(dir);
   if (searchtype == SearchType.None)
   {
    foreach (string item in dirs)
    {
     //System.Web.HttpContext.Current.Response.Write(item);
     indexreaders.Add(IndexReader.Open(Path.Combine(Path.Combine(dir, item), "Save")));
    }
   }
   else
   {
    //System.Web.HttpContext.Current.Response.Write(searchtype.ToString());
    indexreaders.Add(IndexReader.Open(Path.Combine(Path.Combine(dir, searchtype.ToString()), "Save")));
   }

   MultiReader reader = new MultiReader(indexreaders.ToArray());
   indexSearcher = new IndexSearcher(reader);

   Hits hits = null;
   hits = indexSearcher.Search(bquery);
   timer = (DateTime.Now - now).TotalMilliseconds;

 int count = hits.Length();
   
            /* 计算显示的条目 */
      int       start = (pageNo - 1) * 10;
      int       end = pageNo * 10 > count ? count : pageNo * 10;
            //Response.Write(readerhelper.MyQuery.ToString());
            /* 语法高亮显示设置 */
         Highlighter        highlighter = new Highlighter(new QueryScorer(highquery));
       highlighter.SetTextFragmenter(new SimpleFragmenter(100));

for (int i = start; i < end; i++)
{
     Lucene.Net.Documents.Document doc = hits.Doc(i);
              System.String text = doc.Get("content");
     //添加结尾，保证结尾特殊符号不被过滤
              string title = doc.Get("title") + "+aaaaaaaaa";
     Lucene.Net.Analysis.TokenStream tokenStream = highanalyzer  .TokenStream("content", new System.IO.StringReader(text));
     Lucene.Net.Analysis.TokenStream titkeStream = highanalyzer  .TokenStream("title", new System.IO.StringReader(title));
     System.String result = highlighter.GetBestFragments(tokenStream, text, 2, "...");
     string tresult = highlighter.GetBestFragments(titkeStream, title, 0, "..");
     //祛除标题结尾标记
     if (tresult.Length > 10)
      tresult = tresult.Remove(tresult.Length - 10, 10);
     if (string.IsNullOrEmpty(tresult))
      tresult = title.Remove(title.Length - 10, 10);
     //未标注内容读取
     if (string.IsNullOrEmpty(result))
     {
      if (text.Length > 100)
       result = text.Substring(0, 100);
      else
       result = text;
     }
     if (result.Length < text.Length)
      result = result + "...";
}
这样的结果还是很友好的全文完。