Hbase开发-hbase连接-查询-插入-表数据统计-orm

Hbase开发-hbase连接-查询-插入-表数据统计-orm

封装了一个丐版的基于HBASE的ORM框架，需要的加我QQ 184377367

springboot依赖

   <!-- hbase 客户端 -->
        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-client</artifactId>
            <version>2.1.0</version>
        </dependency>

        <!-- hbase协处理器 -->
        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-endpoint</artifactId>
            <version>2.1.0</version>
        </dependency>

package com.ag.dtools.utils;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.client.coprocessor.AggregationClient;
import org.apache.hadoop.hbase.client.coprocessor.DoubleColumnInterpreter;
import org.apache.hadoop.hbase.client.coprocessor.LongColumnInterpreter;
import org.apache.hadoop.hbase.coprocessor.AggregateImplementation;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.filter.SubstringComparator;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import java.io.IOException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

@Component
public class HbaseUtil {

    /**
     * The Logger.
     */
    Logger logger = LoggerFactory.getLogger(HbaseUtil.class);

    /**
     * hbase连接对象
     */
    private Connection conn;

    /**
     * hbase zookeeper地址
     */
    @Value("${hbase.zookeeper.quorum}")
    private String zookeeper;

    /**
     * hbase自带聚和协处理器
     */
    private String gatherCoprocessor = AggregateImplementation.class.getName();

    /**
     * 初始化连接
     */
    @PostConstruct
    private void initConnection() {
        Configuration config = getConfig();
        try {
            //获取连接
            conn = ConnectionFactory.createConnection(config);
            logger.info("初始化hbase连接");
        } catch (Exception e) {
            logger.error("初始化失败", e);
        }
    }

    /**
     * 获取配置对象
     *
     * @return
     */
    private Configuration getConfig() {
        Configuration config = HBaseConfiguration.create();
        config.set("hbase.zookeeper.quorum", zookeeper);
        return config;
    }

    /**
     * 获取连接
     */
    public Connection getConnection() {
        if (conn.isClosed()) {
            synchronized (this) {
                if (conn.isClosed()) {
                    initConnection();
                }
            }
        }
        return conn;
    }

    /**
     * 获取表连接
     *
     * @param tableName
     * @return
     * @throws IOException
     */
    private Table getTable(String tableName)
            throws IOException {
        return getConnection().getTable(TableName.valueOf(tableName));
    }

    /**
     * 获取admin连接
     *
     * @return
     * @throws IOException
     */
    private Admin getAdmin()
            throws IOException {
        return getConnection().getAdmin();
    }

    /**
     * Creat table boolean.创建表
     *
     * @param tableName the table name表名
     * @param columnFamily the column family列族名的集合
     * @return the boolean
     */
    public boolean creatTable(String tableName, List<String> columnFamily) {
        TableName table = TableName.valueOf(tableName);
        try (Admin admin = getAdmin();) {
            if (!admin.tableExists(table)) {
                TableDescriptorBuilder tableDescriptor = TableDescriptorBuilder.newBuilder(table);
                for (String s : columnFamily) {
                    tableDescriptor.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(s)).build());
                }
                admin.createTable(tableDescriptor.build());
            }
        } catch (Exception e) {
            logger.error("创建表失败", e);
        }
        return true;
    }

    /**
     * Gets all table names.获取所有表名
     *
     * @return the all table names
     */
    public List<String> getAllTableNames() {
        List<String> result = new ArrayList<>();
        try (Admin admin = getAdmin();) {
            TableName[] tableNames = admin.listTableNames();
            for (TableName tableName : tableNames) {
                result.add(tableName.getNameAsString());
            }
        } catch (Exception e) {
            logger.error("获取所有表的表名失败", e);
        }
        return result;
    }

    /**
     * Delete table boolean.删除表
     *
     * @param tableName the table name要删除的表名
     * @return the boolean
     */
    public boolean deleteTable(String tableName) {
        try (Admin admin = getAdmin();) {
            if (admin.tableExists(TableName.valueOf(tableName))) {
                admin.disableTable(TableName.valueOf(tableName));
                admin.deleteTable(TableName.valueOf(tableName));
                logger.debug("{} 已删除!", tableName);
            }
        } catch (Exception e) {
            logger.error(MessageFormat.format("删除表失败,tableName:{0}", tableName), e);
            return false;
        }
        return true;
    }

    /**
     * Save data.新增或者更新数据
     *
     * @param tableName the table name表名
     * @param rowKey the row key行key
     * @param familyName the family name列族名
     * @param columns the columns需要插入数据的列名
     * @param values the values需要插入的数据，与前面的列名一一对应
     */
    public boolean saveData(String tableName, String rowKey, String familyName, String[] columns, String[] values) {
        // 获取表
        try (Table table = getTable(tableName);) {
            saveData(table, rowKey, tableName, familyName, columns, values);
        } catch (Exception e) {
            logger.error(
                    MessageFormat
                            .format("为表添加 or 更新数据失败,tableName:{0},rowKey:{1},familyName:{2}", tableName, rowKey, familyName),
                    e);
            return false;
        }
        return true;
    }

    private boolean saveData(Table table, String rowKey, String tableName, String familyName, String[] columns,
                             String[] values) {
        try {
            //设置rowkey
            Put put = new Put(Bytes.toBytes(rowKey));
            if (columns != null && values != null && columns.length == values.length) {
                for (int i = 0; i < columns.length; i++) {
                    if (columns[i] != null && values[i] != null) {
                        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes(columns[i]), Bytes.toBytes(values[i]));
                    } else {
                        throw new NullPointerException(MessageFormat.format("列名和列数据都不能为空,column:{0},value:{1}", columns[i], values[i]));
                    }
                }
            }
            table.put(put);
            logger.debug("为表添加 or 更新数据成功,rowKey:{}", rowKey);
        } catch (Exception e) {
            logger.error(MessageFormat.format("为表添加 or 更新数据失败,tableName:{0},rowKey:{1},familyName:{2}", tableName, rowKey, familyName), e);
            return false;
        }
        return true;
    }

    /**
     * Sets column value.为表的某个单元格赋值
     *
     * @param tableName the table name 表名
     * @param rowKey the row key rowKey
     * @param familyName the family name 列族
     * @param column the column 需要赋值的列名
     * @param value the value 值
     */
    public boolean setColumnValue(String tableName, String rowKey, String familyName, String column, String value) {
        return saveData(tableName, rowKey, familyName, new String[] {column}, new String[] {value});
    }

    /**
     * Delete by row boolean.删除指定的行
     *
     * @param tableName the table name 表名
     * @param rowKey the row key rowKey
     * @return the boolean
     */
    public boolean deleteByRow(String tableName, String rowKey) {
        try (Table table = getTable(tableName); Admin admin = getAdmin();) {
            if (admin.tableExists(TableName.valueOf(tableName))) {
                Delete delete = new Delete(Bytes.toBytes(rowKey));
                table.delete(delete);
                logger.debug("row {} 已删除!", rowKey);
            }
        } catch (Exception e) {
            logger.error(MessageFormat.format("删除指定的行失败,tableName:{0},rowKey:{1}", tableName, rowKey), e);
            return false;
        }
        return true;
    }

    /**
     * Gets scanner result.遍历获取表所有数据
     *
     * @param tableName the table name表名
     * @return the scanner result
     */
    public Map<String, Map<String, String>> getScannerResult(String tableName) {
        Scan scan = new Scan();
        return queryData(tableName, scan);
    }

    /**
     * Gets scanner range row key.根据startRowKey和stopRowKey遍历查询指定表中的所有数据
     *
     * @param tableName the table name
     * @param startRowKey the start row key
     * @param stopRowKey the stop row key
     * @return the scanner range row key
     */
    public Map<String, Map<String, String>> getScannerRangeRowKey(String tableName, String startRowKey,
                                                                  String stopRowKey) {
        Scan scan = new Scan();
        if (StringUtils.isNotEmpty(startRowKey) && StringUtils.isNotEmpty(stopRowKey)) {
            scan.withStartRow(Bytes.toBytes(startRowKey));
            scan.withStopRow(Bytes.toBytes(stopRowKey));
        }
        return queryData(tableName, scan);
    }

    /**
     * Gets scanner by prefix filter.通过行前缀过滤器查询数据
     *
     * @param tableName the table name
     * @param prefix the prefix 以prefix开始的行键
     * @return the scanner by prefix filter
     */
    public Map<String, Map<String, String>> getScannerByPrefixFilter(String tableName, String prefix) {
        Scan scan = new Scan();
        if (StringUtils.isNotEmpty(prefix)) {
            Filter filter = new PrefixFilter(Bytes.toBytes(prefix));
            scan.setFilter(filter);
        }
        return queryData(tableName, scan);
    }

    /**
     * Gets scanner by row filter.查询行键中包含特定字符的数据
     *
     * @param tableName the table name
     * @param keyword the keyword包含指定关键词的行键
     * @return the scanner by row filter
     */
    public Map<String, Map<String, String>> getScannerByRowFilter(String tableName, String keyword) {
        Scan scan = new Scan();
        if (StringUtils.isNotEmpty(keyword)) {
            Filter filter = new RowFilter(CompareOperator.GREATER_OR_EQUAL, new SubstringComparator(keyword));
            scan.setFilter(filter);
        }
        return queryData(tableName, scan);
    }

    /**
     * Gets row data.根据tableName和rowKey精确查询一行的数据
     *
     * @param tableName the table name
     * @param rowKey the row key
     * @return the row data
     */
    public Map<String, String> getRowData(String tableName, String rowKey) {
        //返回的键值对
        Map<String, String> result = new HashMap<>();
        Get get = new Get(Bytes.toBytes(rowKey));
        // 获取表
        try (Table table = getTable(tableName);) {
            Result hTableResult = table.get(get);
            if (hTableResult != null && !hTableResult.isEmpty()) {
                for (Cell cell : hTableResult.listCells()) {
                    result.put(Bytes.toString(CellUtil.cloneQualifier(cell)),
                            Bytes.toString(CellUtil.cloneValue(cell)));
                }
            }
        } catch (Exception e) {
            logger.error(MessageFormat.format("查询一行的数据失败,tableName:{0},rowKey:{1}", tableName, rowKey), e);
        }

        return result;
    }

    /**
     * Gets row data by list.根据多个rowkey查询数据
     *
     * @param tableName the table name
     * @param rowKeys the row keys
     * @return the row data by list
     */
    public Map<String, Map<String, String>> getRowDataByList(String tableName, List<String> rowKeys) {
        //返回的键值对
        Map<String, Map<String, String>> result = new HashMap<>();
        List<Get> getList = new ArrayList<>();
        // 获取表
        try (Table table = getTable(tableName);) {
            //把rowkey加到get里，再把get装到list中
            for (String rowkey : rowKeys) {
                Get get = new Get(Bytes.toBytes(rowkey));
                getList.add(get);
            }
            Result[] rs = table.get(getList);
            for (Result r : rs) {
                //每一行数据
                Map<String, String> columnMap = new HashMap<>();
                String rowKey = null;
                for (Cell cell : r.listCells()) {
                    if (rowKey == null) {
                        rowKey = Bytes.toString(CellUtil.cloneRow(cell));
                    }
                    columnMap.put(Bytes.toString(CellUtil.cloneQualifier(cell)),
                            Bytes.toString(CellUtil.cloneValue(cell)));
                }
                if (rowKey != null) {
                    result.put(rowKey, columnMap);
                }
            }
        } catch (Exception e) {
            logger.error(MessageFormat.format("根据多个rowkey查询数据失败,tableName:{0},rowKey:{1}", tableName, rowKeys), e);
        }
        return result;
    }

    /**
     * Gets column value.根据tableName、rowKey、familyName、column查询指定单元格的数据
     *
     * @param tableName the table name
     * @param rowKey the row key
     * @param familyName the family name
     * @param columnName the column name
     * @return the column value
     */
    public String getColumnValue(String tableName, String rowKey, String familyName, String columnName) {
        String str = null;
        Get get = new Get(Bytes.toBytes(rowKey));
        // 获取表
        try (Table table = getTable(tableName);) {
            Result result = table.get(get);
            if (result != null && !result.isEmpty()) {
                Cell cell = result.getColumnLatestCell(Bytes.toBytes(familyName), Bytes.toBytes(columnName));
                if (cell != null) {
                    str = Bytes.toString(CellUtil.cloneValue(cell));
                }
            }
        } catch (Exception e) {
            logger.error(MessageFormat.format("查询指定单元格的数据失败,tableName:{0},rowKey:{1},familyName:{2},columnName:{3}",
                    tableName,
                    rowKey,
                    familyName,
                    columnName), e);
        }

        return str;
    }

    private Map<String, Map<String, String>> queryData(String tableName, Scan scan) {
        //<rowKey,对应的行数据>
        Map<String, Map<String, String>> result = new HashMap<>();
        // 获取表和扫描结果对象
        try (Table table = getTable(tableName); ResultScanner rs = table.getScanner(scan);) {
            for (Result r : rs) {
                //每一行数据
                Map<String, String> columnMap = new HashMap<>();
                String rowKey = null;
                for (Cell cell : r.listCells()) {
                    if (rowKey == null) {
                        rowKey = Bytes.toString(CellUtil.cloneRow(cell));
                    }
                    columnMap.put(Bytes.toString(CellUtil.cloneQualifier(cell)),
                            Bytes.toString(CellUtil.cloneValue(cell)));
                }
                if (rowKey != null) {
                    result.put(rowKey, columnMap);
                }
            }
        } catch (Exception e) {
            logger.error(MessageFormat.format("遍历查询指定表中的所有数据失败,tableName:{0}", tableName), e);
        }
        return result;
    }

    /**
     * 统计表行数
     *
     * @param tableName
     * @param family
     * @return
     */
    public long queryRowCount(String tableName, String family) {
        //设置表聚合协处理器
        setCoprocessor(tableName, gatherCoprocessor);
        long rowCount = 0;
        //创建聚合协处理器客户端
        try (AggregationClient ac = new AggregationClient(getConfig());) {
            Scan scan = new Scan();
            scan.addFamily(Bytes.toBytes(family));
            rowCount = ac.rowCount(TableName.valueOf(tableName), new LongColumnInterpreter(), scan);
        } catch (Throwable e) {
            logger.error(MessageFormat.format("统计表行数出错,tableName:{0}，family:{1}", tableName, family), e);
        }
        return rowCount;
    }

    /**
     * 统计最大值
     *
     * @param tableName
     * @param family
     * @param qualifier
     * @return
     */
    public double queryMaxData(String tableName, String family, String qualifier) {
        //设置协处理器
        setCoprocessor(tableName, gatherCoprocessor);
        double max = 0;
        //创建聚合协处理器客户端
        try (AggregationClient ac = new AggregationClient(getConfig());) {
            Scan scan = new Scan();
            scan.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier));
            max = ac.max(TableName.valueOf(tableName), new DoubleColumnInterpreter(), scan);
        } catch (Throwable e) {
            logger.error(MessageFormat.format("统计最大值出错,tableName:{0},family:{1},qualifier:{2}", tableName, family, qualifier), e);
        }
        return max;
    }

    /**
     * 统计最小值
     *
     * @param tableName
     * @param family
     * @param qualifier
     * @return
     */
    public double queryMinData(String tableName, String family, String qualifier) {
        //设置协处理器
        setCoprocessor(tableName, gatherCoprocessor);
        double min = 0;
        //创建聚合协处理器客户端
        try (AggregationClient ac = new AggregationClient(getConfig());) {
            Scan scan = new Scan();
            scan.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier));
            min = ac.min(TableName.valueOf(tableName), new DoubleColumnInterpreter(), scan);
        } catch (Throwable e) {
            logger.error(
                    MessageFormat.format("统计最小值出错,tableName:{0},family:{1},qualifier:{2}", tableName, family, qualifier),
                    e);
        }
        return min;
    }

    /**
     * 求和
     *
     * @param tableName
     * @param family
     * @param qualifier
     * @return
     */
    public double querySumData(String tableName, String family, String qualifier) {
        //设置协处理器
        setCoprocessor(tableName, gatherCoprocessor);
        double sum = 0;
        //创建聚合协处理器客户端
        try (AggregationClient ac = new AggregationClient(getConfig());) {
            Scan scan = new Scan();
            scan.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier));
            sum = ac.sum(TableName.valueOf(tableName), new DoubleColumnInterpreter(), scan);
        } catch (Throwable e) {
            logger.error(
                    MessageFormat.format("求和出错,tableName:{0},family:{1},qualifier:{2}", tableName, family, qualifier),
                    e);
        }
        return sum;
    }

    /**
     * 求平均值,低版本hbase有bug
     *
     * @param tableName
     * @param family
     * @param qualifier
     * @return
     */
    public double queryAvgData(String tableName, String family, String qualifier) {
        //设置协处理器
        setCoprocessor(tableName, gatherCoprocessor);
        double avg = 0;
        //创建聚合协处理器客户端
        try (AggregationClient ac = new AggregationClient(getConfig());) {
            Scan scan = new Scan();
            scan.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier));
            avg = ac.avg(TableName.valueOf(tableName), new DoubleColumnInterpreter(), scan);
        } catch (Throwable e) {
            logger.error(
                    MessageFormat.format("求平均值出错,tableName:{0},family:{1},qualifier:{2}", tableName, family, qualifier),
                    e);
        }
        return avg;
    }

    /**
     * 设置表协处理器
     *
     * @param tableName
     * @param coprocessorClassName
     */
    public void setCoprocessor(String tableName, String coprocessorClassName) {
        TableName table = TableName.valueOf(tableName);
        Admin admin = null;
        boolean closeTable = false;
        try {
            admin = getAdmin();
            //获取表的描述对象
            TableDescriptor htd = admin.getDescriptor(table);
            if (!htd.hasCoprocessor(coprocessorClassName)) {
                //表不包含这个协处理器，则添加
                admin.disableTable(table);
                closeTable = true;//关闭了表，则结束时要重启
                TableDescriptorBuilder htdBuilder = TableDescriptorBuilder.newBuilder(htd);
                htdBuilder.setCoprocessor(coprocessorClassName);
                admin.modifyTable(htdBuilder.build());
            }
        } catch (Exception e) {
            logger.error(MessageFormat
                    .format("设置表协处理器出错,tableName:{0},coprocessorClassName:{1}", tableName, coprocessorClassName), e);
        } finally {
            try {
                if (admin != null) {
                    if (admin.isTableDisabled(table) && closeTable) {
                        admin.enableTable(table);
                    }
                    admin.close();
                }
            } catch (IOException e) {
                logger.error("关闭admin资源失败", e);
            }
        }
    }
}