hbase数据导入

hbase数据导入：

参考http://blog.csdn.net/hua840812/article/details/7414875，在把代码copy下来后，发现运行总是报错：

java.io.IOException: Type mismatch in key from map: expected org.apache.hadoop.hbase.io.ImmutableBytesWritable, recieved org.apache.hadoop.io.LongWritable；

原因是map的输出必须按照现有的版本来写，也就是extends Mapper<LongWritable, Text, ImmutableBytesWritable, KeyValue>

要这样写，不能简单的写成extends Mapper,

代码还是贴出来：

生成hfile的代码：

package com.bonc.db;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat;
import org.apache.hadoop.hbase.mapreduce.KeyValueSortReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class TestHFileToHBase {

    public static class TestHFileToHBaseMapper  extends Mapper<LongWritable, Text, ImmutableBytesWritable, KeyValue> {

        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String[] values = value.toString().split("\\|");
            ImmutableBytesWritable rowkey = new ImmutableBytesWritable(
                    values[0].toString().trim().getBytes());
            KeyValue kvProtocol;
            if (values.length>1){
             kvProtocol = new KeyValue(values[0].toString().trim().getBytes(), "url_type".getBytes(), "type".getBytes(),System.currentTimeMillis(), values[1].toString().trim()
                    .getBytes());
            }else{
                kvProtocol=new KeyValue(values[0].toString().trim().getBytes(), "url_type".getBytes(), "type".getBytes(),System.currentTimeMillis(), "NULL".getBytes());
            }
            context.write(rowkey, kvProtocol);
            // KeyValue kvSrcip = new KeyValue(row, "SRCIP".getBytes(),
            // "SRCIP".getBytes(), values[1].getBytes());
            // context.write(k, kvSrcip);
//             HFileOutputFormat.getRecordWriter 
        }

    }

    public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
        Configuration conf = HBaseConfiguration.create();
        Job job = new Job(conf, "TestHFileToHBase");
        job.setJarByClass(TestHFileToHBase.class);

        job.setOutputKeyClass(ImmutableBytesWritable.class);
        job.setOutputValueClass(KeyValue.class);

        job.setMapperClass(TestHFileToHBaseMapper.class);
        job.setReducerClass(KeyValueSortReducer.class);
//        job.setOutputFormatClass(org.apache.hadoop.hbase.mapreduce.HFileOutputFormat.class);
        job.setOutputFormatClass(HFileOutputFormat.class);
        // job.setNumReduceTasks(4);
        // job.setPartitionerClass(org.apache.hadoop.hbase.mapreduce.SimpleTotalOrderPartitioner.class);

        // HBaseAdmin admin = new HBaseAdmin(conf);
        HTable table = new HTable(conf, "url_rule");

        HFileOutputFormat.configureIncrementalLoad(job, table);

        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }

}

hfile导入到表的代码：

package com.bonc.db;
import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;  
import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;  
import org.apache.hadoop.hbase.util.Bytes;  
  
public class TestLoadIncrementalHFileToHBase {  
  
    // private static final byte[] TABLE = Bytes.toBytes("hua");  
    // private static final byte[] QUALIFIER = Bytes.toBytes("PROTOCOLID");  
    // private static final byte[] FAMILY = Bytes.toBytes("PROTOCOLID");  
  
    public static void main(String[] args) throws IOException {  
        Configuration conf = HBaseConfiguration.create();  
//      byte[] TABLE = Bytes.toBytes("hua");  
        byte[] TABLE = Bytes.toBytes(args[0]);  
        HTable table = new HTable(TABLE);  
        LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);  
        loader.doBulkLoad(new Path(args[1]), table);  
//      loader.doBulkLoad(new Path("/hua/testHFileResult/"), table);  
    }  
  
}  

 悲剧的是在从hfile导入到表的时候报错：

java.io.IOException: java.io.IOException: Failed rename of maprfs://133.0.76.41:7222/user/hbasetest/url_type/4447706551787973235 to maprfs://133.0.76.41:7222/hbase/url_rule/732e6d3d150caa8bd3d8d228e3d9c9a0/url_type/914168143008836217
        at org.apache.hadoop.hbase.regionserver.StoreFile.rename(StoreFile.java:512)

虽然解决办法在这里：

http://answers.mapr.com/questions/685/hbase-and-completebulkload

但是我实在是没看懂。so，我采用了最原始的方法：

split将文件分割成小文件，然后：

package com.bonc.db;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;

import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.HTablePool;
import org.apache.hadoop.hbase.client.Put;

import com.bonc.URLMatch.HBaseMain;
public class URL_HBase {

        public static void main(String[] args) throws IOException{
            //文件绝对路径改成你自己的文件路径
            FileReader fr=new FileReader(args[0]);
            //可以换成工程目录下的其他文本文件
            HTablePool pool = new HTablePool(HBaseMain.conf, 1000);
            HTable table = (HTable) pool.getTable("url_rule");
            BufferedReader br=new BufferedReader(fr);
            while(br.readLine()!=null){
                String[] s=br.readLine().toString().split("\\|");
                if(s.length>1){
                Put put = new Put(s[0].trim().getBytes());
                put.add("url_type".getBytes(), "type".getBytes(), s[1].trim().getBytes());
                table.put(put);
                }else{
                    System.out.println(s);
                }
            }
            br.close();
        }
    }

终于成功了，希望有人能够帮我翻译一下，怎么解决是个什么意思。。唉。