mapreduce往Hbase里插入数据(有map无reduce)

package com.asp;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
public class HFileGenerator {
    public static Configuration conf = HBaseConfiguration.create();
    static {
        Configuration HBASE_CONFIG = new Configuration();
        HBASE_CONFIG.set("hbase.zookeeper.quorum",
                "BJ-YZ-103R-63-38,BJ-YZ-103R-63-39,BJ-YZ-103R-63-40");
        HBASE_CONFIG.set("hbase.zookeeper.property.clientPort", "2181");
        conf = HBaseConfiguration.create(HBASE_CONFIG);
    }
    public static class HourlyImporter extends
            Mapper<LongWritable, Text, ImmutableBytesWritable, Put> {
        protected void map(LongWritable key, Text value, Context context)
                throws IOException, InterruptedException {
            String line = value.toString();
            String[] items = line.split("@_@");
            if (items.length < 52) {
                return;
            }
            String[] cols = { "name", "BRAND", "PROVINCE", "CITY", "ADDR",
                    "SEX", "AGE", "GROUP_NAME", "CHILD", "CAR", "CAR_ID",

                    "SVIP", "FINANACE", "BOSS", "ABROAD", "HIGH_VALUE", "VIP",
                    "UA_BRAND", "UA_TYPE", "UA_OS", "UA_DATE", "BROWSER",
                    "SMS_USER", "MMS_USER", "WAP_USER", "APP_USER",
                    "CURR_ARPU", "TOTAL_ARPU", "WAP_LAST_DATE",
                    "WAP_LAST_SUB_DATE", "WAP_CURR_DAYS", "WAP_TOTAL_ARPU",
                    "CXP_DURATION", "CXP_IS_OFF", "CXP_IS_ACTIVE",
                    "CXP_CURR_ARPU", "TV_SG_NUM", "TV_KEY_NUM", "TV_SUB_DATE",
                    "TV_SUB_CHANNEL", "TV_SERVICE_ID", "TY_EXP_NUM",
                    "TY_ORDER_NUM", "TY_TOTAL_EXP_NUM", "TY_TOTAL_ORDER_NUM",
                    "SQ_USER", "SQ_RECV_NUM", "SQ_AREA", "WAP_FEATURE_STR",
                    "CXP_FEATURE_STR", "KEYWORD_STR" };
            byte[] bRowKey = Bytes.toBytes(items[0]);
            ImmutableBytesWritable rowkey = new ImmutableBytesWritable(bRowKey);
            Put put = new Put(bRowKey);
            for (int i = 0; i < cols.length; i++) {
                if (items[i+1].equals(null) || items[i+1].equals("")) {
                    continue;
                }
                put.add("c1".getBytes(), cols[i].getBytes(), items[i+1].getBytes());
            }
            context.write(rowkey, put);
        }
    }
    public static Job createSubmittableJob(Configuration conf, String[] args)
            throws IOException {
        String tableName = args[0];
        Path inputDir = new Path(args[1]);
        Job job = new Job(conf, "hello");
        job.setJarByClass(HFileGenerator.class);
        FileInputFormat.setInputPaths(job, inputDir);
        job.setInputFormatClass(TextInputFormat.class);
        job.setMapperClass(HourlyImporter.class);
        TableMapReduceUtil.initTableReducerJob(tableName, null, job);
        job.setNumReduceTasks(0);
        TableMapReduceUtil.addDependencyJars(job);
        return job;
    }
    public static void main(String[] args) throws Exception {
        Job job = createSubmittableJob(conf, args);
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}

 

posted @ 2014-04-02 09:01  沙漠里的小鱼  阅读(1070)  评论(0编辑  收藏  举报