docker安装hbase后java连接
1、下载安装Hbase:
(1)、docker search hbase : 查找Hbase (2)、docker pull harisekhon/hbase:1.3
2、运行Hbase(运行时指定主机名,端口映射等):
docker run -d -h myhbase -p 2181:2181 -p 8080:8080 -p 8085:8085 -p 9090:9090 -p 9095:9095 -p 16000:16000 -p 16020:16020 -p 16010:16010 -p 16201:16201 -p 16301:16301 --name hbase1 cadb170e663b
-p : 指定主机的端口 16010映射到宿主机上(容器)的开放端口 16010
-P :主机随机分配端口与宿主机上的端口进行映射
3、修改虚拟机的etc/hosts文件:
sudo vi /etc/hosts 添加 docker IP hostname 即:192.168.99.100 myhbase
4、在本地的C:\Windows\System32\drivers\etc下修改hosts文件:
添加 192.168.99.100 启动hbase时设置的主机名 即:192.168.99.100 myhbase
5、浏览器查看Hbase的web界面:
http://docker IP:宿主机上(容器)的开放端口 16010对应的指定主机的端口/master-status 例:http://192.168.99.100:16010/master-status
6、进入到Hbase容器中:
然后执行 : hbase shell
7、进行java连接
maven依赖
<!--hbase--> <dependency> <groupId>com.lmax</groupId> <artifactId>disruptor</artifactId> <version>3.3.6</version> </dependency> <dependency> <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> <version>1.6</version> </dependency> <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-client</artifactId> <version>1.3.0</version> <exclusions> <exclusion> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> </exclusion> <exclusion> <groupId>javax.servlet</groupId> <artifactId>servlet-api</artifactId> </exclusion> </exclusions> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>2.6.0</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>2.6.0</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-common</artifactId> <version>2.6.0</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> <version>2.6.0</version> </dependency> <!--hbase结束-->
SpringContextHolder
@Component public class SpringContextHolder implements ApplicationContextAware { private static ApplicationContext applicationContext; @Override public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { SpringContextHolder.applicationContext = applicationContext; } public static ApplicationContext getApplicationContext() { assertApplicationContext(); return applicationContext; } @SuppressWarnings("unchecked") public static <T> T getBean(String beanName) { assertApplicationContext(); return (T) applicationContext.getBean(beanName); } public static <T> T getBean(Class<T> requiredType) { assertApplicationContext(); return applicationContext.getBean(requiredType); } private static void assertApplicationContext() { if (SpringContextHolder.applicationContext == null) { throw new RuntimeException("applicaitonContext属性为null,请检查是否注入了SpringContextHolder!"); } } }
HBaseUtils
@Component @DependsOn("springContextHolder")//控制依赖顺序,保证springContextHolder类在之前已经加载 public class HBaseUtils { private Logger logger = LoggerFactory.getLogger(this.getClass()); //手动获取hbaseConfig配置类对象 private static HbaseConfig hbaseConfig = SpringContextHolder.getBean("hbaseConfig"); private static Configuration conf = HBaseConfiguration.create(); private static ExecutorService pool = Executors.newScheduledThreadPool(20); //设置hbase连接池 private static Connection connection = null; private static HBaseUtils instance = null; private static Admin admin = null; private HBaseUtils() { if (connection == null) { try { //将hbase配置类中定义的配置加载到连接池中每个连接里 Map<String, String> confMap = hbaseConfig.getconfMaps(); for (Map.Entry<String, String> confEntry : confMap.entrySet()) { conf.set(confEntry.getKey(), confEntry.getValue()); } connection = ConnectionFactory.createConnection(conf, pool); admin = connection.getAdmin(); } catch (IOException e) { logger.error("HbaseUtils实例初始化失败!错误信息为:" + e.getMessage(), e); } } } //简单单例方法,如果autowired自动注入就不需要此方法 public static synchronized HBaseUtils getInstance() { if (instance == null) { instance = new HBaseUtils(); } return instance; } /** * 创建表 * * @param tableName 表名 * @param columnFamily 列族(数组) */ public void createTable(String tableName, String[] columnFamily) throws IOException { TableName name = TableName.valueOf(tableName); //如果存在则删除 if (admin.tableExists(name)) { admin.disableTable(name); admin.deleteTable(name); logger.error("create htable error! this table {} already exists!", name); } else { HTableDescriptor desc = new HTableDescriptor(name); for (String cf : columnFamily) { desc.addFamily(new HColumnDescriptor(cf)); } admin.createTable(desc); } } /** *模糊查找通过前缀匹配,速度更快 *用法:rowPrifix写rowKey,endPrifix为rowKey+"~" **/ public <T> List<T> scaneByPrefixFilter(String tablename, String rowPrifix,String endPrifix,Class T) throws Exception { List<T> list = new ArrayList<T>(); T bean = (T) T.newInstance(); try { Table table = connection.getTable(TableName.valueOf(tablename)); Scan s = new Scan(); s.setStartRow(rowPrifix.getBytes()); s.setStopRow(endPrifix.getBytes()); ResultScanner rs = table.getScanner(s); Map<String, Object> map = new HashMap<>(); for (Result r : rs) { KeyValue[] kv = r.raw(); for (int i = 0; i < kv.length; i++) { String colName = new String(kv[i].getQualifier()); String value = new String(kv[i].getValue()); map.put(colName, value); } if (map.size() > 0) { bean = (T) mapToObject(map, bean.getClass()); list.add(bean); } } } catch (IOException e) { e.printStackTrace(); } return list; } /** * 插入记录(单行单列族-多列多值) * * @param tableName 表名 * @param row 行名 * @param columnFamilys 列族名 * @param columns 列名(数组) * @param values 值(数组)(且需要和列一一对应) */ public void insertRecords(String tableName, String row, String columnFamilys, String[] columns, String[] values) throws IOException { TableName name = TableName.valueOf(tableName); Table table = connection.getTable(name); Put put = new Put(Bytes.toBytes(row)); for (int i = 0; i < columns.length; i++) { put.addColumn(Bytes.toBytes(columnFamilys), Bytes.toBytes(columns[i]), Bytes.toBytes(values[i])); table.put(put); } } /** * 插入记录(单行单列族-单列单值) * * @param tableName 表名 * @param row 行名 * @param columnFamily 列族名 * @param column 列名 * @param value 值 */ public void insertOneRecord(String tableName, String row, String columnFamily, String column, String value) throws IOException { TableName name = TableName.valueOf(tableName); Table table = connection.getTable(name); Put put = new Put(Bytes.toBytes(row)); put.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(column), Bytes.toBytes(value)); table.put(put); } /** * 删除一行记录 * * @param tablename 表名 * @param rowkey 行名 */ public void deleteRow(String tablename, String rowkey) throws IOException { TableName name = TableName.valueOf(tablename); Table table = connection.getTable(name); Delete d = new Delete(rowkey.getBytes()); table.delete(d); } /** * 删除单行单列族记录 * * @param tablename 表名 * @param rowkey 行名 * @param columnFamily 列族名 */ public void deleteColumnFamily(String tablename, String rowkey, String columnFamily) throws IOException { TableName name = TableName.valueOf(tablename); Table table = connection.getTable(name); Delete d = new Delete(rowkey.getBytes()).addFamily(Bytes.toBytes(columnFamily)); table.delete(d); } /** * 删除单行单列族单列记录 * * @param tablename 表名 * @param rowkey 行名 * @param columnFamily 列族名 * @param column 列名 */ public void deleteColumn(String tablename, String rowkey, String columnFamily, String column) throws IOException { TableName name = TableName.valueOf(tablename); Table table = connection.getTable(name); Delete d = new Delete(rowkey.getBytes()).addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(column)); table.delete(d); } /** * 查找一行记录 * * @param tablename 表名 * @param rowKey 行名 */ public static String selectRow(String tablename, String rowKey) throws IOException { String record = ""; TableName name = TableName.valueOf(tablename); Table table = connection.getTable(name); Get g = new Get(rowKey.getBytes()); Result rs = table.get(g); NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map = rs.getMap(); for (Cell cell : rs.rawCells()) { StringBuffer stringBuffer = new StringBuffer() .append(Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())).append("\t") .append(Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength())).append("\t") .append(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength())).append("\t") .append(Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())).append("\n"); String str = stringBuffer.toString(); record += str; } return record; } /** * 查找单行单列族单列记录 * * @param tablename 表名 * @param rowKey 行名 * @param columnFamily 列族名 * @param column 列名 * @return */ public String selectValue(String tablename, String rowKey, String columnFamily, String column) throws IOException { TableName name = TableName.valueOf(tablename); Table table = connection.getTable(name); Get g = new Get(rowKey.getBytes()); g.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(column)); Result rs = table.get(g); return Bytes.toString(rs.value()); } /** * 查询表中所有行(Scan方式) * * @param tablename * @return */ public <T> List<T> scanAllRecord(String tablename, Class T) throws Exception { List<T> list = new ArrayList<T>(); T bean = (T) T.newInstance(); String record = ""; TableName name = TableName.valueOf(tablename); Table table = connection.getTable(name); Scan scan = new Scan(); ResultScanner scanner = table.getScanner(scan); try { for (Result result : scanner) { Map<String, Object> map = new HashMap<>(); for (Cell cell : result.rawCells()) { // StringBuffer stringBuffer = new StringBuffer() // .append(Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())).append("\t") // .append(Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength())).append("\t") // .append(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength())).append("\t") // .append(Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())).append("\n"); // String str = stringBuffer.toString(); // record += str; String colName = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()); String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); map.put(colName, value); } bean = (T) mapToObject(map, bean.getClass()); list.add(bean); } } finally { if (scanner != null) { scanner.close(); } } return list; } /** * 根据rowkey关键字查询报告记录 * * @param tablename * @param rowKeyword * @return */ public <T> List<T> scanReportDataByRowKeyword(String tablename, String rowKeyword, Class T) throws Exception { List<T> list = new ArrayList<T>(); BeanInfo beanInfo = Introspector.getBeanInfo(T); Table table = connection.getTable(TableName.valueOf(tablename)); T bean = (T) T.newInstance(); Scan scan = new Scan(); Get get = new Get(rowKeyword.getBytes()); if (!get.isCheckExistenceOnly()) { Result result = table.get(get); Map<String, Object> map = new HashMap<>(); for (Cell cell : result.rawCells()) { String row = Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); //String family = Bytes.toString(cell.getFamilyArray(),cell.getFamilyOffset(),cell.getFamilyLength()); String colName = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()); String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); map.put(colName, value); } if (map.size() > 0) { bean = (T) mapToObject(map, bean.getClass()); list.add(bean); } } // //添加行键过滤器,根据关键字匹配 // RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator(rowKeyword)); // scan.setFilter(rowFilter); // // ResultScanner scanner = table.getScanner(scan); // try { // for (Result result : scanner) { // //TODO 此处根据业务来自定义实现 // String s = new String(result.getRow()); // Map<String,Object> map = new HashMap<>(); // for(Cell cell : result.rawCells()){ // String row = Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); // //String family = Bytes.toString(cell.getFamilyArray(),cell.getFamilyOffset(),cell.getFamilyLength()); // String colName = Bytes.toString(cell.getQualifierArray(),cell.getQualifierOffset(),cell.getQualifierLength()); // String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); // // map.put(colName,value); // // } // bean = (T) mapToObject(map, bean.getClass()); // list.add(bean); // } // } finally { // if (scanner != null) { // scanner.close(); // } // } return list; } // public <T> List<T> scanReportDataByRowKeyword(String tablename, String rowKeyword,Class T) throws Exception { // List<T> list = new ArrayList<T>(); // BeanInfo beanInfo = Introspector.getBeanInfo(T); // // Table table = connection.getTable(TableName.valueOf(tablename)); // T bean = (T)T.newInstance(); // Scan scan = new Scan(); // // //添加行键过滤器,根据关键字匹配 // RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator(rowKeyword)); // scan.setFilter(rowFilter); // // ResultScanner scanner = table.getScanner(scan); // try { // for (Result result : scanner) { // //TODO 此处根据业务来自定义实现 // String s = new String(result.getRow()); // Map<String,Object> map = new HashMap<>(); // for(Cell cell : result.rawCells()){ // String row = Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); // //String family = Bytes.toString(cell.getFamilyArray(),cell.getFamilyOffset(),cell.getFamilyLength()); // String colName = Bytes.toString(cell.getQualifierArray(),cell.getQualifierOffset(),cell.getQualifierLength()); // String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); // // map.put(colName,value); // // } // bean = (T) mapToObject(map, bean.getClass()); // list.add(bean); // } // } finally { // if (scanner != null) { // scanner.close(); // } // } // // return list; // } //Map转Object public static Object mapToObject(Map<String, Object> map, Class<?> beanClass) throws Exception { // Map<String, Object> stringObjectMap = transformUpperCase(map); Map<String, Object> stringObjectMap = map; if (stringObjectMap == null) return null; Object obj = beanClass.newInstance(); Field[] fields = obj.getClass().getDeclaredFields(); for (Field field : fields) { int mod = field.getModifiers(); if (Modifier.isStatic(mod) || Modifier.isFinal(mod)) { continue; } field.setAccessible(true); if (stringObjectMap.containsKey(field.getName())) { field.set(obj, stringObjectMap.get(field.getName())); } } return obj; } // 将map值全部转换为大写 public static Map<String, Object> transformUpperCase(Map<String, Object> orgMap) { Map<String, Object> resultMap = new HashMap<>(); if (orgMap == null || orgMap.isEmpty()) { return resultMap; } Set<String> keySet = orgMap.keySet(); for (String key : keySet) { String newKey = key.toUpperCase(); // newKey = newKey.replace("_", ""); resultMap.put(newKey, orgMap.get(key)); } return resultMap; } public List<Result> getRows(String tableName, String rowKeyLike) { // TODO Auto-generated method stub Table table = null; List<Result> list = null; try { FilterList fl = new FilterList(FilterList.Operator.MUST_PASS_ALL); table = connection.getTable(TableName.valueOf(tableName)); PrefixFilter filter = new PrefixFilter(rowKeyLike.getBytes()); SingleColumnValueFilter filter1 = new SingleColumnValueFilter( "order".getBytes(), "order_type".getBytes(), CompareFilter.CompareOp.EQUAL, Bytes.toBytes("1") ); fl.addFilter(filter); fl.addFilter(filter1); Scan scan = new Scan(); scan.setFilter(fl); ResultScanner scanner = table.getScanner(scan); list = new ArrayList<Result>(); for (Result rs : scanner) { list.add(rs); } } catch (Exception e) { e.printStackTrace(); } finally { try { table.close(); } catch (IOException e) { e.printStackTrace(); } } return list; } /** * 模糊查找数据根据rowKey * * @param tableName * @param rowKeyLike * @return */ public <T> List<T> getListByRowKeyLike(String tableName, String rowKeyLike, Class T) throws IllegalAccessException, InstantiationException { List<T> list = new ArrayList<T>(); T bean = (T) T.newInstance(); Table table = null; try { FilterList fl = new FilterList(FilterList.Operator.MUST_PASS_ALL); table = connection.getTable(TableName.valueOf(tableName)); PrefixFilter filter = new PrefixFilter(rowKeyLike.getBytes()); SingleColumnValueFilter filter1 = new SingleColumnValueFilter( "order".getBytes(), "order_type".getBytes(), CompareFilter.CompareOp.EQUAL, Bytes.toBytes("1") ); fl.addFilter(filter); fl.addFilter(filter1); Scan scan = new Scan(); scan.setFilter(fl); ResultScanner scanner = table.getScanner(scan); for (Result rs : scanner) { Map<String, Object> map = new HashMap<>(); for (Cell cell : rs.rawCells()) { String colName = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()); String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); map.put(colName, value); } bean = (T) mapToObject(map, bean.getClass()); list.add(bean); } } catch (Exception e) { e.printStackTrace(); } finally { try { table.close(); } catch (IOException e) { e.printStackTrace(); } } return list; } public List<Result> getRows(String tableName, String rowKeyLike, String cols[], Object values[]) { // TODO Auto-generated method stub Table table = null; List<Result> list = null; try { table = connection.getTable(TableName.valueOf(tableName)); PrefixFilter filter = new PrefixFilter(rowKeyLike.getBytes()); Scan scan = new Scan(); for (int i = 0; i < cols.length; i++) { scan.addColumn("data".getBytes(), cols[i].getBytes()); } scan.setFilter(filter); ResultScanner scanner = table.getScanner(scan); list = new ArrayList<Result>(); for (Result rs : scanner) { list.add(rs); } } catch (Exception e) { e.printStackTrace(); } finally { try { table.close(); } catch (IOException e) { e.printStackTrace(); } } return list; } //多条件查询 public List getRowsByOneKey(String tableName, List<String> arr) throws IOException { // TODO Auto-generated method stub Table table = null; ArrayList<Object> list = new ArrayList<Object>(); table = connection.getTable(TableName.valueOf(tableName)); Scan scan = new Scan(); List<Filter> filters = new ArrayList<Filter>(); FilterList filterList = new FilterList(filters); try { for (String v : arr) { String[] s = v.split(","); SubstringComparator comp = new SubstringComparator(s[2]); filterList.addFilter(new SingleColumnValueFilter(Bytes.toBytes(s[0]), Bytes.toBytes(s[1]), CompareFilter.CompareOp.EQUAL, comp)); } scan.setFilter(filterList); ResultScanner scanner = table.getScanner(scan); for (Result result : scanner) { //TODO 此处根据业务来自定义实现 String s = new String(result.getRow()); Map<String, Object> map = new HashMap<>(); for (Cell cell : result.rawCells()) { String colName = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()); String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); map.put(colName, value); } list.add(map); } } catch (Exception e) { e.printStackTrace(); } finally { try { table.close(); } catch (IOException e) { e.printStackTrace(); } } return list; } //多条件查询 public List getRowsByOneIfKey(String tableName, List<String> arr) throws IOException { // TODO Auto-generated method stub Table table = null; ArrayList<Object> list = new ArrayList<Object>(); table = connection.getTable(TableName.valueOf(tableName)); Scan scan = new Scan(); List<Filter> filters = new ArrayList<Filter>(); FilterList filterList = new FilterList(filters); try { for (String v : arr) { String[] s = v.split(","); SubstringComparator comp = new SubstringComparator(s[2]); if (s.length > 3 && s[3].equals("NOT")) { filterList.addFilter(new SingleColumnValueFilter(Bytes.toBytes(s[0]), Bytes.toBytes(s[1]), CompareFilter.CompareOp.NOT_EQUAL, comp)); } else { filterList.addFilter(new SingleColumnValueFilter(Bytes.toBytes(s[0]), Bytes.toBytes(s[1]), CompareFilter.CompareOp.EQUAL, comp)); } } scan.setFilter(filterList); ResultScanner scanner = table.getScanner(scan); for (Result result : scanner) { //TODO 此处根据业务来自定义实现 String s = new String(result.getRow()); Map<String, Object> map = new HashMap<>(); for (Cell cell : result.rawCells()) { String colName = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()); String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); map.put(colName, value); } list.add(map); } } catch (Exception e) { e.printStackTrace(); } finally { try { table.close(); } catch (IOException e) { e.printStackTrace(); } } return list; } /** * 范围查询 * * @param tableName * @param startRow * @param stopRow * @return */ public List<Result> getRows(String tableName, String startRow, String stopRow) { Table table = null; List<Result> list = null; try { table = connection.getTable(TableName.valueOf(tableName)); Scan scan = new Scan(); scan.setStartRow(startRow.getBytes()); scan.setStopRow(stopRow.getBytes()); ResultScanner scanner = table.getScanner(scan); list = new ArrayList<Result>(); for (Result rsResult : scanner) { list.add(rsResult); } } catch (Exception e) { e.printStackTrace(); } finally { try { table.close(); } catch (IOException e) { e.printStackTrace(); } } return list; } /** * 根据rowkey关键字和时间戳范围查询报告记录 * * @param tablename * @param rowKeyword * @return */ public List scanReportDataByRowKeywordTimestamp(String tablename, String rowKeyword, Long minStamp, Long maxStamp) throws IOException { ArrayList<Object> list = new ArrayList<Object>(); Table table = connection.getTable(TableName.valueOf(tablename)); Scan scan = new Scan(); //添加scan的时间范围 scan.setTimeRange(minStamp, maxStamp); RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator(rowKeyword)); scan.setFilter(rowFilter); ResultScanner scanner = table.getScanner(scan); try { for (Result result : scanner) { //TODO 此处根据业务来自定义实现 list.add(null); } } finally { if (scanner != null) { scanner.close(); } } return list; } /** * 删除表操作 * * @param tablename */ public void deleteTable(String tablename) throws IOException { TableName name = TableName.valueOf(tablename); if (admin.tableExists(name)) { admin.disableTable(name); admin.deleteTable(name); } } /** * 利用协处理器进行全表count统计 * * @param tablename */ public Long countRowsWithCoprocessor(String tablename) throws Throwable { TableName name = TableName.valueOf(tablename); HTableDescriptor descriptor = admin.getTableDescriptor(name); String coprocessorClass = "org.apache.hadoop.hbase.coprocessor.AggregateImplementation"; if (!descriptor.hasCoprocessor(coprocessorClass)) { admin.disableTable(name); descriptor.addCoprocessor(coprocessorClass); admin.modifyTable(name, descriptor); admin.enableTable(name); } //计时 StopWatch stopWatch = new StopWatch(); stopWatch.start(); Scan scan = new Scan(); AggregationClient aggregationClient = new AggregationClient(conf); Long count = aggregationClient.rowCount(name, new LongColumnInterpreter(), scan); stopWatch.stop(); System.out.println("RowCount:" + count + ",全表count统计耗时:" + stopWatch.getTotalTimeMillis()); return count; } /** * 根据tableName和rowKey精确查询一行的数据 * * @param tableName 表名 * @param rowKey 行键 * @return java.util.Map<java.lang.String, java.lang.String> 返回一行的数据 * @author zifangsky * @date 2018/7/3 16:07 * @since 1.0.0 */ public Map<String, String> getRowToMap(String tableName, String rowKey) { //返回的键值对 Map<String, String> result = new HashMap<>(); Get get = new Get(Bytes.toBytes(rowKey)); // 获取表 Table table = null; try { table = connection.getTable(TableName.valueOf(tableName)); Result hTableResult = table.get(get); if (hTableResult != null && !hTableResult.isEmpty()) { for (Cell cell : hTableResult.listCells()) { // System.out.println("family:" + Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength())); // System.out.println("qualifier:" + Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength())); // System.out.println("value:" + Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); // System.out.println("Timestamp:" + cell.getTimestamp()); // System.out.println("-------------------------------------------"); result.put(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()), Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); } } } catch (IOException e) { e.printStackTrace(); } finally { try { table.close(); } catch (IOException e) { e.printStackTrace(); } } return result; } /** * 根据tableName和rowKey和返回类型精确查询一行的数据 * * @param tableName 表名 * @param rowKey rowKey * @param T 需要转化的类型 * @param <T> 返回的类型 * @return * @throws IllegalAccessException * @throws InstantiationException */ public <T> T getRowToObject(String tableName, String rowKey, Class T) throws IllegalAccessException, InstantiationException { //返回的键值对 Map<String, Object> result = new HashMap<>(); T bean = (T) T.newInstance(); Get get = new Get(Bytes.toBytes(rowKey)); // 获取表 Table table = null; try { table = connection.getTable(TableName.valueOf(tableName)); Result hTableResult = table.get(get); if (hTableResult != null && !hTableResult.isEmpty()) { for (Cell cell : hTableResult.listCells()) { result.put(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()), Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); } } bean = (T) mapToObject(result, bean.getClass()); } catch (IOException e) { logger.error(MessageFormat.format("查询一行的数据失败,tableName:{0},rowKey:{1}" , tableName, rowKey), e); } catch (Exception e) { e.printStackTrace(); } finally { try { table.close(); } catch (IOException e) { e.printStackTrace(); } } return bean; } /** * 根据tableName、rowKey、familyName、column查询指定单元格的数据 * * @param tableName 表名 * @param rowKey rowKey * @param familyName 列族名 * @param columnName 列名 * @return java.lang.String * @author zifangsky * @date 2018/7/4 10:58 * @since 1.0.0 */ public String getColumnValue(String tableName, String rowKey, String familyName, String columnName) { String str = null; Get get = new Get(Bytes.toBytes(rowKey)); // 获取表 Table table = null; try { table = connection.getTable(TableName.valueOf(tableName)); Result result = table.get(get); if (result != null && !result.isEmpty()) { Cell cell = result.getColumnLatestCell(Bytes.toBytes(familyName), Bytes.toBytes(columnName)); if (cell != null) { str = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); } } } catch (IOException e) { logger.error(MessageFormat.format("查询指定单元格的数据失败,tableName:{0},rowKey:{1},familyName:{2},columnName:{3}" , tableName, rowKey, familyName, columnName), e); } finally { try { table.close(); } catch (IOException e) { e.printStackTrace(); } } return str; } }
HbaseConfig
@Configuration @ConfigurationProperties(prefix = HbaseConfig.CONF_PREFIX) public class HbaseConfig { public static final String CONF_PREFIX = "hbase.conf"; private Map<String,String> confMaps; public Map<String, String> getconfMaps() { return confMaps; } public void setconfMaps(Map<String, String> confMaps) { this.confMaps = confMaps; } }
yml
spring: hbase: conf: conf-maps: 'hbase.zookeeper.quorum': 'myhbase:2181'
HbaseTest
package com.xxy; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.xxy.entity.Label; import com.xxy.entity.User; import com.xxy.utils.HBaseUtils; import com.xxy.utils.mybatisPlus.PageUtils; import com.xxy.utils.mybatisPlus.Query; import org.apache.hadoop.hbase.client.Result; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.junit4.SpringRunner; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; /** * @Description: * @Author: xuxiaoyu * @Create: 2020-08-12 22:43 */ @RunWith(SpringRunner.class) @SpringBootTest public class HbaseTest { @Autowired HBaseUtils hBaseUtils; @Test public void addOne() throws IOException { // String pc = "userName,xuxiaoyu,idType,111,idNum,8938493#userName,bairuoyu,idType,111,idNum,34322#userName,yienqi,idType,111,idNum,87#"; // hBaseUtils.insertOneRecord("yjx:Policy", "md5", "cf", "username", pc); // hBaseUtils.deleteColumn("yjx:Policy", "md5", "cf", "username"); hBaseUtils.createTable("yjx:inso", new String[]{"cf"}); hBaseUtils.insertOneRecord("yjx:inso", "md6", "cf", "companycode", "000020"); hBaseUtils.insertOneRecord("yjx:inso", "md6", "cf", "policyno", "p723483"); hBaseUtils.insertOneRecord("yjx:inso", "md6", "cf", "productcode", "PDEX837"); hBaseUtils.insertOneRecord("yjx:inso", "md6", "cf", "effdate", "2020-09-02"); hBaseUtils.insertOneRecord("yjx:inso", "md5", "cf", "companycode", "000138"); hBaseUtils.insertOneRecord("yjx:inso", "md5", "cf", "policyno", "p93732"); hBaseUtils.insertOneRecord("yjx:inso", "md5", "cf", "productcode", "IO938"); hBaseUtils.insertOneRecord("yjx:inso", "md5", "cf", "effdate", "2020-08-13"); } @Test public void getValue() throws IOException { long start = System.currentTimeMillis(); String value = hBaseUtils.selectValue("yjx:Policy", "md5", "cf", "username"); String[] split = value.split("#"); System.out.println("-----------------------------------------------"); List<User> userList = new ArrayList<>(); for (String s : split) { String[] split1 = s.split(","); User user = new User(); user.setUserName(split1[1]); user.setIdType(split1[3]); user.setIdNum(split1[5]); userList.add(user); } for (User user : userList) { System.out.println(user); } long end = System.currentTimeMillis(); System.out.println((end - start) / 1000f + "s"); System.out.println("-----------------------------------------------"); } /** * md5 cf description 特定疾病:25种重大疾病范围以外的重大疾病 * md5 cf label AHRL110004 * md5 cf labellever AHRL110004 * md5 cf rank Ⅰ * * @throws Exception */ @Test public void getObejct() throws Exception { String md5 = hBaseUtils.selectRow("yjx:Policy", "md5"); System.out.println("-----------------------------------------------"); System.out.println(md5); System.out.println("-----------------------------------------------"); } @Test public void getObejct2() throws Exception { Map<String, String> md5 = hBaseUtils.getRowToMap("yjx:Policy", "md5"); System.out.println("-----------------------------------------------"); System.out.println(md5); System.out.println("-----------------------------------------------"); } @Test public void getObejct3() throws Exception { Label label = hBaseUtils.getRowToObject("yjx:Policy", "md5", Label.class); System.out.println("-----------------------------------------------"); System.out.println(label); System.out.println("-----------------------------------------------"); } @Test public void getList() throws Exception { List<Label> labelList = hBaseUtils.scanAllRecord("yjx:Policy", Label.class); System.out.println("-----------------------------------------------"); labelList.forEach(label -> System.out.println(label)); System.out.println("-----------------------------------------------"); } /** * 根据rowkey关键字查询报告记录 * * @throws Exception */ @Test public void getRowKey() throws Exception { List<Label> labelList = hBaseUtils.scanReportDataByRowKeyword("yjx:Policy", "md5", Label.class); System.out.println("-----------------------------------------------"); labelList.forEach(label -> System.out.println(label)); System.out.println("-----------------------------------------------"); } /** * 根据row模糊查询 * * @throws Exception */ @Test public void getRows() throws Exception { List<Result> results = hBaseUtils.getRows("yjx:Policy", "md"); System.out.println("-----------------------------------------------"); results.forEach(label -> System.out.println(label)); System.out.println("-----------------------------------------------"); } /** * 根据row模糊查询并转化为List<T></> * * @throws Exception */ @Test public void getRowsByKey() throws Exception { List<Label> labelList = hBaseUtils.getListByRowKeyLike("yjx:Policy", "md",Label.class); System.out.println("-----------------------------------------------"); labelList.forEach(label -> System.out.println(label)); System.out.println("-----------------------------------------------"); } /** * 根据tableName、rowKey、familyName、column查询指定单元格的数据 * * @throws Exception */ @Test public void getColumnValue() throws Exception { String columnValue = hBaseUtils.getColumnValue("yjx:Policy", "md5", "cf", "label"); System.out.println("-----------------------------------------------"); System.out.println(columnValue); System.out.println("-----------------------------------------------"); } @Test public void pageList() { //测试 List<Label> labelList = getLabelList(); IPage<Label> page = getPage(1,3,labelList); System.out.println("-------------------------------------------"); page.getRecords().forEach(label -> System.out.println(label)); System.out.println("-------------------------------------------"); } public IPage<Label> getPage(int currentPage,int limit,List<Label> list) { //当前页 int current = currentPage; //每页数据条数 int size = limit; IPage<Label> page = new Page<>(current, size); int count = list.size(); List<Label> pageList = new ArrayList<>(); //计算当前页第一条数据的下标 int currId = current > 1 ? (current - 1) * size : 0; for (int i = 0; i < size && i < count - currId; i++) { pageList.add(list.get(currId + i)); } page.setSize(size); page.setCurrent(current); page.setTotal(count); //计算分页总页数 page.setPages(count % 10 == 0 ? count / 10 : count / 10 + 1); page.setRecords(pageList); return page; } public List<Label> getLabelList() { List<Label> list = new ArrayList(); for (int i = 0; i < 19; i++) { Label label = new Label(); label.setLabel("AHRL9993"); label.setDescription("xdsdw"); label.setLabellever("3"); label.setRank("1"); list.add(label); } return list; } }
【推荐】国内首个AI IDE,深度理解中文开发场景,立即下载体验Trae
【推荐】编程新体验,更懂你的AI,立即体验豆包MarsCode编程助手
【推荐】抖音旗下AI助手豆包,你的智能百科全书,全免费不限次数
【推荐】轻量又高性能的 SSH 工具 IShell:AI 加持,快人一步
· 10年+ .NET Coder 心语,封装的思维:从隐藏、稳定开始理解其本质意义
· .NET Core 中如何实现缓存的预热?
· 从 HTTP 原因短语缺失研究 HTTP/2 和 HTTP/3 的设计差异
· AI与.NET技术实操系列:向量存储与相似性搜索在 .NET 中的实现
· 基于Microsoft.Extensions.AI核心库实现RAG应用
· TypeScript + Deepseek 打造卜卦网站:技术与玄学的结合
· 阿里巴巴 QwQ-32B真的超越了 DeepSeek R-1吗?
· 【译】Visual Studio 中新的强大生产力特性
· 10年+ .NET Coder 心语 ── 封装的思维:从隐藏、稳定开始理解其本质意义
· 【设计模式】告别冗长if-else语句:使用策略模式优化代码结构