MapReduce-从HBase读取数据处理后再写入HBase

MapReduce-从HBase读取处理后再写入HBase

代码如下

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

package com.hbase.mapreduce;
 
import java.io.IOException;
 
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
 
/**
* @author:FengZhen
* @create:2018年9月17日
* 从HBase读写入HBase
* zip -d HBaseToHBase.jar 'META-INF/.SF' 'META-INF/.RSA' 'META-INF/*SF'
*/
public class HBaseToHBase extends Configured implements Tool{
 
    private static String addr="HDP233,HDP232,HDP231";
    private static String port="2181";
     
    public enum Counters { ROWS, COLS, VALID, ERROR, EMPTY, NOT_EMPTY}
     
    static class ParseMapper extends TableMapper<ImmutableBytesWritable, Put>{
        private byte[] columnFamily = null;
        @Override
        protected void setup(Mapper<ImmutableBytesWritable, Result, ImmutableBytesWritable, Put>.Context context)
                throws IOException, InterruptedException {
            columnFamily = Bytes.toBytes(context.getConfiguration().get("conf.columnfamily"));
        }
        @Override
        protected void map(ImmutableBytesWritable key, Result value,
                Mapper<ImmutableBytesWritable, Result, ImmutableBytesWritable, Put>.Context context)
                throws IOException, InterruptedException {
            context.getCounter(Counters.ROWS).increment(1);
            String hbaseValue = null;
             
            Put put = new Put(key.get());
            for (Cell cell : value.listCells()) {
                context.getCounter(Counters.COLS).increment(1);
                hbaseValue = Bytes.toString(CellUtil.cloneValue(cell));
                if (hbaseValue.length() > 0) {
                    String top = hbaseValue.substring(0, hbaseValue.length()/2);
                    String detail = hbaseValue.substring(hbaseValue.length()/2, hbaseValue.length() - 1);
                    put.addColumn(columnFamily, Bytes.toBytes("top"), Bytes.toBytes(top));
                    put.addColumn(columnFamily, Bytes.toBytes("detail"), Bytes.toBytes(detail));
                    context.getCounter(Counters.NOT_EMPTY).increment(1);
                }else {
                    put.addColumn(columnFamily, Bytes.toBytes("empty"), Bytes.toBytes(hbaseValue));
                    context.getCounter(Counters.EMPTY).increment(1);
                }
            }
            try {
                context.write(key, put);
                context.getCounter(Counters.VALID).increment(1);
            } catch (Exception e) {
                e.printStackTrace();
                context.getCounter(Counters.ERROR).increment(1);
            }
        }
    }
     
    static class ParseTableReducer extends TableReducer<ImmutableBytesWritable, Put, ImmutableBytesWritable>{
        @Override
        protected void reduce(ImmutableBytesWritable key, Iterable<Put> values,
                Reducer<ImmutableBytesWritable, Put, ImmutableBytesWritable, Mutation>.Context context)
                throws IOException, InterruptedException {
            for (Put put : values) {
                context.write(key, put);
            }
        }
    }
     
    public int run(String[] arg0) throws Exception {
        String table = arg0[0];
        String column = arg0[1];
        String destTable = arg0[2];
         
        Configuration configuration = HBaseConfiguration.create();
        configuration.set("hbase.zookeeper.quorum",addr);
        configuration.set("hbase.zookeeper.property.clientPort", port);
         
        Scan scan = new Scan();
        if (null != column) {
            byte[][] colkey = KeyValue.parseColumn(Bytes.toBytes(column));
            if (colkey.length > 1) {
                scan.addColumn(colkey[0], colkey[1]);
                configuration.set("conf.columnfamily", Bytes.toString(colkey[0]));
                configuration.set("conf.columnqualifier", Bytes.toString(colkey[1]));
            }else {
                scan.addFamily(colkey[0]);
                configuration.set("conf.columnfamily", Bytes.toString(colkey[0]));
            }
        }
         
        Job job = Job.getInstance(configuration);
        job.setJobName("HBaseToHBase2");
        job.setJarByClass(HBaseToHBase2.class);
         
        job.getConfiguration().set(TableInputFormat.INPUT_TABLE, table);
        job.getConfiguration().set(TableOutputFormat.OUTPUT_TABLE, destTable);
         
        job.setMapperClass(ParseMapper.class);
        job.setMapOutputKeyClass(ImmutableBytesWritable.class);
        job.setMapOutputValueClass(Put.class);
         
//      job.setReducerClass(ParseTableReducer.class);
        job.setOutputKeyClass(ImmutableBytesWritable.class);
        job.setOutputValueClass(Put.class);
         
        job.setInputFormatClass(TableInputFormat.class);
        TableInputFormat.addColumns(scan, KeyValue.parseColumn(Bytes.toBytes(column)));
        job.setOutputFormatClass(TableOutputFormat.class);
         
        job.setNumReduceTasks(0);
         
        //使用TableMapReduceUtil会报类找不到错误
        //Caused by: java.lang.ClassNotFoundException: com.yammer.metrics.core.MetricsRegistry
//      TableMapReduceUtil.initTableMapperJob(table, scan, ParseMapper.class, ImmutableBytesWritable.class, Put.class, job);
//      TableMapReduceUtil.initTableReducerJob(table, IdentityTableReducer.class, job);
         
        return job.waitForCompletion(true) ? 0 : 1;
    }
    public static void main(String[] args) throws Exception {
        String[] params = new String[] {"test_table_mr", "data:info", "test_table_dest"};
        int exitCode = ToolRunner.run(new HBaseToHBase2(), params);
        System.exit(exitCode);
    }
}

　打包测试

1 2	`zip -d HBaseToHBase.jar` `'META-INF/.SF'` `'META-INF/.RSA'` `'META-INF/*SF'` `hadoop jar HBaseToHBase.jar com.hbase.mapreduce.HBaseToHBase`

出现的问题

一开始使用额TableMapReduceUtil,但是报下面这个错

Exception in thread "main" java.lang.NoClassDefFoundError: com/yammer/metrics/core/MetricsRegistry
    at org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addHBaseDependencyJars(TableMapReduceUtil.java:732)
    at org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(TableMapReduceUtil.java:777)
    at org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.initTableMapperJob(TableMapReduceUtil.java:212)
    at org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.initTableMapperJob(TableMapReduceUtil.java:168)
    at org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.initTableMapperJob(TableMapReduceUtil.java:291)
    at org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.initTableMapperJob(TableMapReduceUtil.java:92)
    at com.hbase.mapreduce.HBaseToHBase.run(HBaseToHBase.java:108)
    at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
    at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:90)
    at com.hbase.mapreduce.HBaseToHBase.main(HBaseToHBase.java:115)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.apache.hadoop.util.RunJar.run(RunJar.java:233)
    at org.apache.hadoop.util.RunJar.main(RunJar.java:148)
Caused by: java.lang.ClassNotFoundException: com.yammer.metrics.core.MetricsRegistry
    at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:338)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
    ... 16 more