mapreduce往Hbase里插入数据(有map无reduce)
package com.asp; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; public class HFileGenerator { public static Configuration conf = HBaseConfiguration.create(); static { Configuration HBASE_CONFIG = new Configuration(); HBASE_CONFIG.set("hbase.zookeeper.quorum", "BJ-YZ-103R-63-38,BJ-YZ-103R-63-39,BJ-YZ-103R-63-40"); HBASE_CONFIG.set("hbase.zookeeper.property.clientPort", "2181"); conf = HBaseConfiguration.create(HBASE_CONFIG); } public static class HourlyImporter extends Mapper<LongWritable, Text, ImmutableBytesWritable, Put> { protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { String line = value.toString(); String[] items = line.split("@_@"); if (items.length < 52) { return; } String[] cols = { "name", "BRAND", "PROVINCE", "CITY", "ADDR", "SEX", "AGE", "GROUP_NAME", "CHILD", "CAR", "CAR_ID", "SVIP", "FINANACE", "BOSS", "ABROAD", "HIGH_VALUE", "VIP", "UA_BRAND", "UA_TYPE", "UA_OS", "UA_DATE", "BROWSER", "SMS_USER", "MMS_USER", "WAP_USER", "APP_USER", "CURR_ARPU", "TOTAL_ARPU", "WAP_LAST_DATE", "WAP_LAST_SUB_DATE", "WAP_CURR_DAYS", "WAP_TOTAL_ARPU", "CXP_DURATION", "CXP_IS_OFF", "CXP_IS_ACTIVE", "CXP_CURR_ARPU", "TV_SG_NUM", "TV_KEY_NUM", "TV_SUB_DATE", "TV_SUB_CHANNEL", "TV_SERVICE_ID", "TY_EXP_NUM", "TY_ORDER_NUM", "TY_TOTAL_EXP_NUM", "TY_TOTAL_ORDER_NUM", "SQ_USER", "SQ_RECV_NUM", "SQ_AREA", "WAP_FEATURE_STR", "CXP_FEATURE_STR", "KEYWORD_STR" }; byte[] bRowKey = Bytes.toBytes(items[0]); ImmutableBytesWritable rowkey = new ImmutableBytesWritable(bRowKey); Put put = new Put(bRowKey); for (int i = 0; i < cols.length; i++) { if (items[i+1].equals(null) || items[i+1].equals("")) { continue; } put.add("c1".getBytes(), cols[i].getBytes(), items[i+1].getBytes()); } context.write(rowkey, put); } } public static Job createSubmittableJob(Configuration conf, String[] args) throws IOException { String tableName = args[0]; Path inputDir = new Path(args[1]); Job job = new Job(conf, "hello"); job.setJarByClass(HFileGenerator.class); FileInputFormat.setInputPaths(job, inputDir); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(HourlyImporter.class); TableMapReduceUtil.initTableReducerJob(tableName, null, job); job.setNumReduceTasks(0); TableMapReduceUtil.addDependencyJars(job); return job; } public static void main(String[] args) throws Exception { Job job = createSubmittableJob(conf, args); System.exit(job.waitForCompletion(true) ? 0 : 1); } }