solr4.0.0学习（二） 数据库导入clob与blob为索引

导入clob很简单。但是blob好像没有提供方法，所以改了一下源码，重新编译替换class文件，竟然成功了。

先把配置文件贴上

SCHEMA.XML

 

<?xml version="1.0" ?>
<schema name="test" version="1.1">
  <types>
   <fieldtype name="string"  class="solr.StrField" sortMissingLast="true" omitNorms="true"/>

   <fieldType name="standard" class="solr.TextField" positionIncrementGap="100">
	  <analyzer type="index">
		<tokenizer class="solr.StandardTokenizerFactory"/>
		<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
		<filter class="solr.LowerCaseFilterFactory"/>
	  </analyzer>
	  <analyzer type="query">
		<tokenizer class="solr.StandardTokenizerFactory"/>
		<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
		<filter class="solr.LowerCaseFilterFactory"/>
	  </analyzer>
	</fieldType>

	<fieldType name="ik" class="solr.TextField">   
       <analyzer class="org.wltea.analyzer.lucene.IKAnalyzer"/>   
	</fieldType>

  </types>

 <fields>   
  <field name="blogId"      type="string"   indexed="false"  stored="true"  multiValued="false"/>
  <field name="blogTitle"   type="ik"   indexed="true"  stored="true"  multiValued="false" /> 
  <field name="blogAuthorName"   type="ik"   indexed="true"  stored="true"  multiValued="false" /> 
  <field name="blogContent"   type="ik"   indexed="true"  stored="true"  multiValued="false" /> 
  <field name="TITLE"   type="ik"   indexed="true"  stored="true"  /> 
  <field name="TEXT"   type="ik"   indexed="true"  stored="true"  /> 
 </fields>
 <defaultSearchField>blogTitle</defaultSearchField>
 <solrQueryParser defaultOperator="OR"/>

</schema>

这里的field只用到了blogContent一个。

 

SOLRCONFIG.XML

 

<?xml version="1.0" encoding="UTF-8" ?>
<config>
  <luceneMatchVersion>LUCENE_34</luceneMatchVersion>
  <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.StandardDirectoryFactory}"/>
  <updateHandler class="solr.DirectUpdateHandler2" />

  <requestDispatcher handleSelect="true" >
    <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
  </requestDispatcher>
  
  <requestHandler name="standard" class="solr.StandardRequestHandler" default="true" />
  <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" />
  <requestHandler name="/admin/" class="org.apache.solr.handler.admin.AdminHandlers" />
      
 <!-- the dataimport requestHandler --> 
       <requestHandler name="/dataimport" class="org.apache.solr.handler.dataimport.DataImportHandler"> 
               <lst name="defaults"> 
              <str name="config">db-data-config.xml</str> 
             </lst> 
       </requestHandler> 

  <admin>
    <defaultQuery>solr</defaultQuery>
  </admin>
  <unlockOnStartup>true</unlockOnStartup>
	<lockType>simple</lockType>
  <requestHandler name="/analysis/field" 
                  startup="lazy"
                  class="solr.FieldAnalysisRequestHandler" />

</config>

db-data-config.xml

 

 

<dataConfig> 
<dataSource name="f1" type="FieldStreamDataSource"/>
 <dataSource driver="oracle.jdbc.driver.OracleDriver"  
 url="jdbc:oracle:thin:@127.0.0.1:1521:orcl" user="HT" password="HT"/> 
 <document> 
		<entity name="blog" query="SELECT BLOG_CONTENT from  TB_ENT_BLOG" transformer="ClobTransformer"> 
				 <field column="BLOG_CONTENT" name="blogContent" clob="true"/> 
		</entity>
 </document> 
</dataConfig> 

然后修改了ClobTransformer.java。使其同时支持BLOG格式。

 

package org.apache.solr.handler.dataimport;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.sql.Blob;
import java.sql.Clob;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

public class ClobTransformer extends Transformer
{
  public static final String CLOB = "clob";

  public Object transformRow(Map<String, Object> aRow, Context context)
  {
    for (Map map : context.getAllEntityFields()) {
      if ("true".equals(map.get("clob"))) {
        String column = (String)map.get("column");
        String srcCol = (String)map.get("sourceColName");
        if (srcCol == null)
          srcCol = column;
        Object o = aRow.get(srcCol);
        if ((o instanceof List)) {
          List inputs = (List)o;
          List results = new ArrayList();
          for (Object input : inputs) {
            if ((input instanceof Clob)) {
              Clob clob = (Clob)input;
              results.add(readFromClob(clob));
            }else if(input instanceof Blob){
            	Blob blob = (Blob)input;
            	results.add(readFromBlob(blob));
            }
          }
          aRow.put(column, results);
        }
        else if ((o instanceof Clob)) {
          Clob clob = (Clob)o;
          aRow.put(column, readFromClob(clob));
        }else if(o instanceof Blob){
        	Blob blob = (Blob)o;
        	aRow.put(column, readFromBlob(blob));
        }
      }
    }
    return aRow;
  }

  private String readFromBlob(Blob blob) {
	  try{
		  	InputStream is = blob.getBinaryStream();
			BufferedReader br = new BufferedReader(new InputStreamReader(is));
			String str = "";
			String res = "";
			while((str=br.readLine())!=null){
				res += str;
			}
			return res;
	  }catch (Exception e) {
		  e.printStackTrace();
		  return "";
	}
}

private String readFromClob(Clob clob) {
    Reader reader = null;
	try {
		reader = clob.getCharacterStream();
	} catch (SQLException e1) {
		e1.printStackTrace();
	}
    StringBuilder sb = new StringBuilder();
    char[] buf = new char[1024];
    try
    {
      int len;
      while ((len = reader.read(buf)) != -1)
        sb.append(buf, 0, len);
    }
    catch (IOException e) {
      DataImportHandlerException.wrapAndThrow(500, e);
    }
    return sb.toString();
  }
}

 

这里加了一个readFromBlob方法，加了两个else if。异常的处理很粗糙。

这样替换class文件，导入索引就正常了。在query ":" 页面的response会出现所有blob内容。

如果response没有blob字段或者显示为对象地址，都是错了。