Hbase 表的Rowkey设计避免数据热点

一、案例分析

常见避免数据热点问题的处理方式有：加盐、哈希、反转等方法结合预分区使用。

由于目前原数据第一字段为时间戳形式，第二字段为电话号码，直接存储容易引起热点问题，通过加随机列、组合时间戳、字段反转的方式来设计Rowkey，来实现既能高效查询又能避免热点问题。

二、代码部分

package beifeng.hadoop.hbase;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.io.Text;

/**
 * 遵循rowkey的设计原则
 *  1.rowkey不能过长
 *  2.唯一性，加随机列  md5
 *  3.注意避免产生数据热点
 *  4.满足更多的查询场景
 * @author Administrator
 *
 */
public class LoadData extends Configured implements Tool {

    /**
     * 综合考虑 使用时间和手机 做组合key，能更好的满足应用场景
     * @author Administrator
     *
     */
    public static class LoadDataMapper extends Mapper<LongWritable, Text, LongWritable, Text> {
        //专门处理时间戳 =》标准时间格式
        SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHsss");
        private Text mapOutputValue = new Text();
        @Override
        protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, LongWritable, Text>.Context context)
                throws IOException, InterruptedException {
            String line = value.toString();
            String[] splited = line.split("\t");
            
            //将切分的第一个字段转成标准时间
            String formatDate = sdf.format(new Date(Long.parseLong(splited[0].trim())));
            //将手机号码反转
            String phoneNumber = splited[1].toString();
                        String reversePhoneNumber = new StringBuffer(phoneNumber).reverse().toString();
            
            String rowKeyString = reversePhoneNumber +"|"+formatDate;
            //反转手机号+“|”+时间 +正行内容拼接
            mapOutputValue.set(rowKeyString+"\t"+ line);
            context.write(key, mapOutputValue);
        }
    }
    
    public static class LoadDataReuducer extends TableReducer<LongWritable, Text, NullWritable>{

        //设置HBase的列簇
            private static final String COLUMN_FAMAILY = "info";            
        @Override
        protected void reduce(LongWritable key, Iterable<Text> values,
                Reducer<LongWritable, Text, NullWritable, Mutation>.Context context)
                throws IOException, InterruptedException {
            for (Text value:values) {
                String[] splited = value.toString().split("\t");
                String rowKey = splited[0];
            //    System.err.println(rowKey);
                Put put = new Put(rowKey.getBytes());
                //put.addColumn(COLUMN_FAMAILY.getBytes(),"row".getBytes(),value.getBytes());
                put.add(COLUMN_FAMAILY.getBytes(), "reportTime".getBytes(), splited[1].getBytes());
                put.add(COLUMN_FAMAILY.getBytes(), "apmac".getBytes(), splited[3].getBytes());
                put.add(COLUMN_FAMAILY.getBytes(), "acmac".getBytes(), splited[4].getBytes());
                put.add(COLUMN_FAMAILY.getBytes(), "host".getBytes(), splited[5].getBytes());
                put.add(COLUMN_FAMAILY.getBytes(), "siteType".getBytes(), splited[6].getBytes());
                put.add(COLUMN_FAMAILY.getBytes(), "upPackNum".getBytes(), splited[7].getBytes());
                put.add(COLUMN_FAMAILY.getBytes(), "downPackNum".getBytes(), splited[8].getBytes());
                put.add(COLUMN_FAMAILY.getBytes(), "unPayLoad".getBytes(), splited[9].getBytes());
                put.add(COLUMN_FAMAILY.getBytes(), "downPayLoad".getBytes(), splited[10].getBytes());
                put.add(COLUMN_FAMAILY.getBytes(),"httpStatus".getBytes(),splited[11].getBytes());
                context.write(NullWritable.get(), put);
                
            }
        }
    }

    public static void createTable(String tableName) throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
        Configuration  conf = HBaseConfiguration.create();
        conf.set("hbase.zookeeper.quorum", "beifeng01");
        
        HBaseAdmin admin = new HBaseAdmin(conf);
        
        TableName tName = TableName.valueOf(tableName);
        
        HTableDescriptor htd = new HTableDescriptor(tName);
        HColumnDescriptor hcd = new HColumnDescriptor("info");
        htd.addFamily(hcd);
        
        if(admin.tableExists(tName)) {
            System.out.println(tableName+"is exist,trying to recrate the table");
            admin.disableTable(tName);
            admin.deleteTable(tName);
        } 
        admin.createTable(htd);
        System.out.println("create new table"+ " " + tableName);
        
    }
    
    public int run(String[] args) throws Exception {

    Configuration conf = this.getConf();
        conf.set("hbase.zookeeper.quorum", "beifeng01"); 
        conf.set(TableOutputFormat.OUTPUT_TABLE, "phoneLog"); 
        
        createTable("phoneLog"); 
        
        Job job = Job.getInstance(conf, this.getClass().getSimpleName());  
        job.setJarByClass(this.getClass());  
        job.setNumReduceTasks(1); 
        
        // map class  
        job.setMapperClass(LoadDataMapper.class);  
        job.setMapOutputKeyClass(LongWritable.class);  
        job.setMapOutputValueClass(Text.class);  
   
        // reduce class  
        job.setReducerClass(LoadDataReuducer.class);  
        job.setOutputFormatClass(TableOutputFormat.class); 
        
        Path inPath = new Path(args[0]);  
        FileInputFormat.addInputPath(job, inPath);
   
       boolean isSucced = job.waitForCompletion(true);
       
       return isSucced ? 0 : 1; 
    }

    public static void main(String[] args) throws Exception {
        Configuration conf = HBaseConfiguration.create(); 
        
        //指定HDFS数据地址
        args = new String[] {"hdfs://hbase/data/input/HTTP_20130313143750.data"};  
           int status = ToolRunner.run(
                   conf, 
                   new LoadData(), 
                   args);
             
    System.exit(status);
    }
}

运行完程序后scan 查看效果

hbase(main):004:0> scan 'phoneLog', {LIMIT => 2}
ROW                                COLUMN+CELL                                                                                         
 01787706731|2013031314048         column=info:acmac, timestamp=1544022103345, value=120.196.100.82                                    
 01787706731|2013031314048         column=info:apmac, timestamp=1544022103345, value=00-FD-07-A4-7B-08:CMCC                            
 01787706731|2013031314048         column=info:downPackNum, timestamp=1544022103345, value=2                                           
 01787706731|2013031314048         column=info:downPayLoad, timestamp=1544022103345, value=120                                         
 01787706731|2013031314048         column=info:host, timestamp=1544022103345, value=                                                   
 01787706731|2013031314048         column=info:httpStatus, timestamp=1544022103345, value=200                                          
 01787706731|2013031314048         column=info:reportTime, timestamp=1544022103345, value=1363157988072                                
 01787706731|2013031314048         column=info:siteType, timestamp=1544022103345, value=                                               
 01787706731|2013031314048         column=info:unPayLoad, timestamp=1544022103345, value=120                                           
 01787706731|2013031314048         column=info:upPackNum, timestamp=1544022103345, value=2                                             
 10007032831|2013031314045         column=info:acmac, timestamp=1544022103345, value=120.196.100.99                                    
 10007032831|2013031314045         column=info:apmac, timestamp=1544022103345, value=20-7C-8F-70-68-1F:CMCC                            
 10007032831|2013031314045         column=info:downPackNum, timestamp=1544022103345, value=3                                           
 10007032831|2013031314045         column=info:downPayLoad, timestamp=1544022103345, value=180                                         
 10007032831|2013031314045         column=info:host, timestamp=1544022103345, value=                                                   
 10007032831|2013031314045         column=info:httpStatus, timestamp=1544022103345, value=200                                          
 10007032831|2013031314045         column=info:reportTime, timestamp=1544022103345, value=1363157985079                                
 10007032831|2013031314045         column=info:siteType, timestamp=1544022103345, value=                                               
 10007032831|2013031314045         column=info:unPayLoad, timestamp=1544022103345, value=360                                           
 10007032831|2013031314045         column=info:upPackNum, timestamp=1544022103345, value=6