docker安装hbase后java连接
1、下载安装Hbase：
(1)、docker search hbase : 查找Hbase
(2)、docker pull harisekhon/hbase:1.3
2、运行Hbase(运行时指定主机名，端口映射等)：
docker run -d -h myhbase -p 2181:2181 -p 8080:8080 -p 8085:8085 -p 9090:9090 -p 9095:9095 -p 16000:16000 -p 16020:16020 -p 16010:16010 -p 16201:16201 -p 16301:16301 --name hbase1 cadb170e663b
-p : 指定主机的端口 16010映射到宿主机上(容器)的开放端口 16010
-P ：主机随机分配端口与宿主机上的端口进行映射
3、修改虚拟机的etc/hosts文件：
sudo vi /etc/hosts
添加 docker IP hostname
即：192.168.99.100 myhbase
4、在本地的C:\Windows\System32\drivers\etc下修改hosts文件：
添加 192.168.99.100 启动hbase时设置的主机名
即：192.168.99.100 myhbase
5、浏览器查看Hbase的web界面：
http://docker IP:宿主机上(容器)的开放端口 16010对应的指定主机的端口/master-status
例：http://192.168.99.100:16010/master-status
6、进入到Hbase容器中：
然后执行 ： hbase shell
7、进行java连接
maven依赖
<!--hbase-->
<dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
<version>3.3.6</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.6</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.3.0</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.6.0</version>
</dependency>
<!--hbase结束-->
SpringContextHolder
@Component
public class SpringContextHolder implements ApplicationContextAware {
private static ApplicationContext applicationContext;
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
SpringContextHolder.applicationContext = applicationContext;
}
public static ApplicationContext getApplicationContext() {
assertApplicationContext();
return applicationContext;
}
@SuppressWarnings("unchecked")
public static <T> T getBean(String beanName) {
assertApplicationContext();
return (T) applicationContext.getBean(beanName);
}
public static <T> T getBean(Class<T> requiredType) {
assertApplicationContext();
return applicationContext.getBean(requiredType);
}
private static void assertApplicationContext() {
if (SpringContextHolder.applicationContext == null) {
throw new RuntimeException("applicaitonContext属性为null,请检查是否注入了SpringContextHolder!");
}
}
}
HBaseUtils
@Component
@DependsOn("springContextHolder")//控制依赖顺序，保证springContextHolder类在之前已经加载
public class HBaseUtils {
private Logger logger = LoggerFactory.getLogger(this.getClass());
//手动获取hbaseConfig配置类对象
private static HbaseConfig hbaseConfig = SpringContextHolder.getBean("hbaseConfig");
private static Configuration conf = HBaseConfiguration.create();
private static ExecutorService pool = Executors.newScheduledThreadPool(20); //设置hbase连接池
private static Connection connection = null;
private static HBaseUtils instance = null;
private static Admin admin = null;
private HBaseUtils() {
if (connection == null) {
try {
//将hbase配置类中定义的配置加载到连接池中每个连接里
Map<String, String> confMap = hbaseConfig.getconfMaps();
for (Map.Entry<String, String> confEntry : confMap.entrySet()) {
conf.set(confEntry.getKey(), confEntry.getValue());
}
connection = ConnectionFactory.createConnection(conf, pool);
admin = connection.getAdmin();
} catch (IOException e) {
logger.error("HbaseUtils实例初始化失败！错误信息为：" + e.getMessage(), e);
}
}
}
//简单单例方法，如果autowired自动注入就不需要此方法
public static synchronized HBaseUtils getInstance() {
if (instance == null) {
instance = new HBaseUtils();
}
return instance;
}
/**
* 创建表
*
* @param tableName 表名
* @param columnFamily 列族（数组）
*/
public void createTable(String tableName, String[] columnFamily) throws IOException {
TableName name = TableName.valueOf(tableName);
//如果存在则删除
if (admin.tableExists(name)) {
admin.disableTable(name);
admin.deleteTable(name);
logger.error("create htable error! this table {} already exists!", name);
} else {
HTableDescriptor desc = new HTableDescriptor(name);
for (String cf : columnFamily) {
desc.addFamily(new HColumnDescriptor(cf));
}
admin.createTable(desc);
}
}
/**
*模糊查找通过前缀匹配，速度更快
*用法：rowPrifix写rowKey,endPrifix为rowKey+"~"
**/
public <T> List<T> scaneByPrefixFilter(String tablename, String rowPrifix,String endPrifix,Class T) throws Exception {
List<T> list = new ArrayList<T>();
T bean = (T) T.newInstance();
try {
Table table = connection.getTable(TableName.valueOf(tablename));
Scan s = new Scan();
s.setStartRow(rowPrifix.getBytes());
s.setStopRow(endPrifix.getBytes());
ResultScanner rs = table.getScanner(s);
Map<String, Object> map = new HashMap<>();
for (Result r : rs) {
KeyValue[] kv = r.raw();
for (int i = 0; i < kv.length; i++) {
String colName = new String(kv[i].getQualifier());
String value = new String(kv[i].getValue());
map.put(colName, value);
}
if (map.size() > 0) {
bean = (T) mapToObject(map, bean.getClass());
list.add(bean);
}
}
} catch (IOException e) {
e.printStackTrace();
}
return list;
}
/**
* 插入记录（单行单列族-多列多值）
*
* @param tableName 表名
* @param row 行名
* @param columnFamilys 列族名
* @param columns 列名（数组）
* @param values 值（数组）（且需要和列一一对应）
*/
public void insertRecords(String tableName, String row, String columnFamilys, String[] columns, String[] values) throws IOException {
TableName name = TableName.valueOf(tableName);
Table table = connection.getTable(name);
Put put = new Put(Bytes.toBytes(row));
for (int i = 0; i < columns.length; i++) {
put.addColumn(Bytes.toBytes(columnFamilys), Bytes.toBytes(columns[i]), Bytes.toBytes(values[i]));
table.put(put);
}
}
/**
* 插入记录（单行单列族-单列单值）
*
* @param tableName 表名
* @param row 行名
* @param columnFamily 列族名
* @param column 列名
* @param value 值
*/
public void insertOneRecord(String tableName, String row, String columnFamily, String column, String value) throws IOException {
TableName name = TableName.valueOf(tableName);
Table table = connection.getTable(name);
Put put = new Put(Bytes.toBytes(row));
put.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(column), Bytes.toBytes(value));
table.put(put);
}
/**
* 删除一行记录
*
* @param tablename 表名
* @param rowkey 行名
*/
public void deleteRow(String tablename, String rowkey) throws IOException {
TableName name = TableName.valueOf(tablename);
Table table = connection.getTable(name);
Delete d = new Delete(rowkey.getBytes());
table.delete(d);
}
/**
* 删除单行单列族记录
*
* @param tablename 表名
* @param rowkey 行名
* @param columnFamily 列族名
*/
public void deleteColumnFamily(String tablename, String rowkey, String columnFamily) throws IOException {
TableName name = TableName.valueOf(tablename);
Table table = connection.getTable(name);
Delete d = new Delete(rowkey.getBytes()).addFamily(Bytes.toBytes(columnFamily));
table.delete(d);
}
/**
* 删除单行单列族单列记录
*
* @param tablename 表名
* @param rowkey 行名
* @param columnFamily 列族名
* @param column 列名
*/
public void deleteColumn(String tablename, String rowkey, String columnFamily, String column) throws IOException {
TableName name = TableName.valueOf(tablename);
Table table = connection.getTable(name);
Delete d = new Delete(rowkey.getBytes()).addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(column));
table.delete(d);
}
/**
* 查找一行记录
*
* @param tablename 表名
* @param rowKey 行名
*/
public static String selectRow(String tablename, String rowKey) throws IOException {
String record = "";
TableName name = TableName.valueOf(tablename);
Table table = connection.getTable(name);
Get g = new Get(rowKey.getBytes());
Result rs = table.get(g);
NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map = rs.getMap();
for (Cell cell : rs.rawCells()) {
StringBuffer stringBuffer = new StringBuffer()
.append(Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())).append("\t")
.append(Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength())).append("\t")
.append(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength())).append("\t")
.append(Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())).append("\n");
String str = stringBuffer.toString();
record += str;
}
return record;
}
/**
* 查找单行单列族单列记录
*
* @param tablename 表名
* @param rowKey 行名
* @param columnFamily 列族名
* @param column 列名
* @return
*/
public String selectValue(String tablename, String rowKey, String columnFamily, String column) throws IOException {
TableName name = TableName.valueOf(tablename);
Table table = connection.getTable(name);
Get g = new Get(rowKey.getBytes());
g.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(column));
Result rs = table.get(g);
return Bytes.toString(rs.value());
}
/**
* 查询表中所有行（Scan方式）
*
* @param tablename
* @return
*/
public <T> List<T> scanAllRecord(String tablename, Class T) throws Exception {
List<T> list = new ArrayList<T>();
T bean = (T) T.newInstance();
String record = "";
TableName name = TableName.valueOf(tablename);
Table table = connection.getTable(name);
Scan scan = new Scan();
ResultScanner scanner = table.getScanner(scan);
try {
for (Result result : scanner) {
Map<String, Object> map = new HashMap<>();
for (Cell cell : result.rawCells()) {
// StringBuffer stringBuffer = new StringBuffer()
// .append(Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())).append("\t")
// .append(Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength())).append("\t")
// .append(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength())).append("\t")
// .append(Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())).append("\n");
// String str = stringBuffer.toString();
// record += str;
String colName = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
map.put(colName, value);
}
bean = (T) mapToObject(map, bean.getClass());
list.add(bean);
}
} finally {
if (scanner != null) {
scanner.close();
}
}
return list;
}
/**
* 根据rowkey关键字查询报告记录
*
* @param tablename
* @param rowKeyword
* @return
*/
public <T> List<T> scanReportDataByRowKeyword(String tablename, String rowKeyword, Class T) throws Exception {
List<T> list = new ArrayList<T>();
BeanInfo beanInfo = Introspector.getBeanInfo(T);
Table table = connection.getTable(TableName.valueOf(tablename));
T bean = (T) T.newInstance();
Scan scan = new Scan();
Get get = new Get(rowKeyword.getBytes());
if (!get.isCheckExistenceOnly()) {
Result result = table.get(get);
Map<String, Object> map = new HashMap<>();
for (Cell cell : result.rawCells()) {
String row = Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
//String family = Bytes.toString(cell.getFamilyArray(),cell.getFamilyOffset(),cell.getFamilyLength());
String colName = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
map.put(colName, value);
}
if (map.size() > 0) {
bean = (T) mapToObject(map, bean.getClass());
list.add(bean);
}
}
// //添加行键过滤器，根据关键字匹配
// RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator(rowKeyword));
// scan.setFilter(rowFilter);
//
// ResultScanner scanner = table.getScanner(scan);
// try {
// for (Result result : scanner) {
// //TODO 此处根据业务来自定义实现
// String s = new String(result.getRow());
// Map<String,Object> map = new HashMap<>();
// for(Cell cell : result.rawCells()){
// String row = Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
// //String family = Bytes.toString(cell.getFamilyArray(),cell.getFamilyOffset(),cell.getFamilyLength());
// String colName = Bytes.toString(cell.getQualifierArray(),cell.getQualifierOffset(),cell.getQualifierLength());
// String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
//
// map.put(colName,value);
//
// }
// bean = (T) mapToObject(map, bean.getClass());
// list.add(bean);
// }
// } finally {
// if (scanner != null) {
// scanner.close();
// }
// }
return list;
}
// public <T> List<T> scanReportDataByRowKeyword(String tablename, String rowKeyword,Class T) throws Exception {
// List<T> list = new ArrayList<T>();
// BeanInfo beanInfo = Introspector.getBeanInfo(T);
//
// Table table = connection.getTable(TableName.valueOf(tablename));
// T bean = (T)T.newInstance();
// Scan scan = new Scan();
//
// //添加行键过滤器，根据关键字匹配
// RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator(rowKeyword));
// scan.setFilter(rowFilter);
//
// ResultScanner scanner = table.getScanner(scan);
// try {
// for (Result result : scanner) {
// //TODO 此处根据业务来自定义实现
// String s = new String(result.getRow());
// Map<String,Object> map = new HashMap<>();
// for(Cell cell : result.rawCells()){
// String row = Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
// //String family = Bytes.toString(cell.getFamilyArray(),cell.getFamilyOffset(),cell.getFamilyLength());
// String colName = Bytes.toString(cell.getQualifierArray(),cell.getQualifierOffset(),cell.getQualifierLength());
// String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
//
// map.put(colName,value);
//
// }
// bean = (T) mapToObject(map, bean.getClass());
// list.add(bean);
// }
// } finally {
// if (scanner != null) {
// scanner.close();
// }
// }
//
// return list;
// }
//Map转Object
public static Object mapToObject(Map<String, Object> map, Class<?> beanClass) throws Exception {
// Map<String, Object> stringObjectMap = transformUpperCase(map);
Map<String, Object> stringObjectMap = map;
if (stringObjectMap == null)
return null;
Object obj = beanClass.newInstance();
Field[] fields = obj.getClass().getDeclaredFields();
for (Field field : fields) {
int mod = field.getModifiers();
if (Modifier.isStatic(mod) || Modifier.isFinal(mod)) {
continue;
}
field.setAccessible(true);
if (stringObjectMap.containsKey(field.getName())) {
field.set(obj, stringObjectMap.get(field.getName()));
}
}
return obj;
}
// 将map值全部转换为大写
public static Map<String, Object> transformUpperCase(Map<String, Object> orgMap) {
Map<String, Object> resultMap = new HashMap<>();
if (orgMap == null || orgMap.isEmpty()) {
return resultMap;
}
Set<String> keySet = orgMap.keySet();
for (String key : keySet) {
String newKey = key.toUpperCase();
// newKey = newKey.replace("_", "");
resultMap.put(newKey, orgMap.get(key));
}
return resultMap;
}
public List<Result> getRows(String tableName, String rowKeyLike) {
// TODO Auto-generated method stub
Table table = null;
List<Result> list = null;
try {
FilterList fl = new FilterList(FilterList.Operator.MUST_PASS_ALL);
table = connection.getTable(TableName.valueOf(tableName));
PrefixFilter filter = new PrefixFilter(rowKeyLike.getBytes());
SingleColumnValueFilter filter1 = new SingleColumnValueFilter(
"order".getBytes(),
"order_type".getBytes(),
CompareFilter.CompareOp.EQUAL,
Bytes.toBytes("1")
);
fl.addFilter(filter);
fl.addFilter(filter1);
Scan scan = new Scan();
scan.setFilter(fl);
ResultScanner scanner = table.getScanner(scan);
list = new ArrayList<Result>();
for (Result rs : scanner) {
list.add(rs);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return list;
}
/**
* 模糊查找数据根据rowKey
*
* @param tableName
* @param rowKeyLike
* @return
*/
public <T> List<T> getListByRowKeyLike(String tableName, String rowKeyLike, Class T) throws IllegalAccessException, InstantiationException {
List<T> list = new ArrayList<T>();
T bean = (T) T.newInstance();
Table table = null;
try {
FilterList fl = new FilterList(FilterList.Operator.MUST_PASS_ALL);
table = connection.getTable(TableName.valueOf(tableName));
PrefixFilter filter = new PrefixFilter(rowKeyLike.getBytes());
SingleColumnValueFilter filter1 = new SingleColumnValueFilter(
"order".getBytes(),
"order_type".getBytes(),
CompareFilter.CompareOp.EQUAL,
Bytes.toBytes("1")
);
fl.addFilter(filter);
fl.addFilter(filter1);
Scan scan = new Scan();
scan.setFilter(fl);
ResultScanner scanner = table.getScanner(scan);
for (Result rs : scanner) {
Map<String, Object> map = new HashMap<>();
for (Cell cell : rs.rawCells()) {
String colName = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
map.put(colName, value);
}
bean = (T) mapToObject(map, bean.getClass());
list.add(bean);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return list;
}
public List<Result> getRows(String tableName, String rowKeyLike, String cols[], Object values[]) {
// TODO Auto-generated method stub
Table table = null;
List<Result> list = null;
try {
table = connection.getTable(TableName.valueOf(tableName));
PrefixFilter filter = new PrefixFilter(rowKeyLike.getBytes());
Scan scan = new Scan();
for (int i = 0; i < cols.length; i++) {
scan.addColumn("data".getBytes(), cols[i].getBytes());
}
scan.setFilter(filter);
ResultScanner scanner = table.getScanner(scan);
list = new ArrayList<Result>();
for (Result rs : scanner) {
list.add(rs);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return list;
}
//多条件查询
public List getRowsByOneKey(String tableName, List<String> arr) throws IOException {
// TODO Auto-generated method stub
Table table = null;
ArrayList<Object> list = new ArrayList<Object>();
table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
List<Filter> filters = new ArrayList<Filter>();
FilterList filterList = new FilterList(filters);
try {
for (String v : arr) {
String[] s = v.split(",");
SubstringComparator comp = new SubstringComparator(s[2]);
filterList.addFilter(new SingleColumnValueFilter(Bytes.toBytes(s[0]),
Bytes.toBytes(s[1]), CompareFilter.CompareOp.EQUAL, comp));
}
scan.setFilter(filterList);
ResultScanner scanner = table.getScanner(scan);
for (Result result : scanner) {
//TODO 此处根据业务来自定义实现
String s = new String(result.getRow());
Map<String, Object> map = new HashMap<>();
for (Cell cell : result.rawCells()) {
String colName = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
map.put(colName, value);
}
list.add(map);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return list;
}
//多条件查询
public List getRowsByOneIfKey(String tableName, List<String> arr) throws IOException {
// TODO Auto-generated method stub
Table table = null;
ArrayList<Object> list = new ArrayList<Object>();
table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
List<Filter> filters = new ArrayList<Filter>();
FilterList filterList = new FilterList(filters);
try {
for (String v : arr) {
String[] s = v.split(",");
SubstringComparator comp = new SubstringComparator(s[2]);
if (s.length > 3 && s[3].equals("NOT")) {
filterList.addFilter(new SingleColumnValueFilter(Bytes.toBytes(s[0]),
Bytes.toBytes(s[1]), CompareFilter.CompareOp.NOT_EQUAL, comp));
} else {
filterList.addFilter(new SingleColumnValueFilter(Bytes.toBytes(s[0]),
Bytes.toBytes(s[1]), CompareFilter.CompareOp.EQUAL, comp));
}
}
scan.setFilter(filterList);
ResultScanner scanner = table.getScanner(scan);
for (Result result : scanner) {
//TODO 此处根据业务来自定义实现
String s = new String(result.getRow());
Map<String, Object> map = new HashMap<>();
for (Cell cell : result.rawCells()) {
String colName = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
map.put(colName, value);
}
list.add(map);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return list;
}
/**
* 范围查询
*
* @param tableName
* @param startRow
* @param stopRow
* @return
*/
public List<Result> getRows(String tableName, String startRow, String stopRow) {
Table table = null;
List<Result> list = null;
try {
table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
scan.setStartRow(startRow.getBytes());
scan.setStopRow(stopRow.getBytes());
ResultScanner scanner = table.getScanner(scan);
list = new ArrayList<Result>();
for (Result rsResult : scanner) {
list.add(rsResult);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return list;
}
/**
* 根据rowkey关键字和时间戳范围查询报告记录
*
* @param tablename
* @param rowKeyword
* @return
*/
public List scanReportDataByRowKeywordTimestamp(String tablename, String rowKeyword, Long minStamp, Long maxStamp) throws IOException {
ArrayList<Object> list = new ArrayList<Object>();
Table table = connection.getTable(TableName.valueOf(tablename));
Scan scan = new Scan();
//添加scan的时间范围
scan.setTimeRange(minStamp, maxStamp);
RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator(rowKeyword));
scan.setFilter(rowFilter);
ResultScanner scanner = table.getScanner(scan);
try {
for (Result result : scanner) {
//TODO 此处根据业务来自定义实现
list.add(null);
}
} finally {
if (scanner != null) {
scanner.close();
}
}
return list;
}
/**
* 删除表操作
*
* @param tablename
*/
public void deleteTable(String tablename) throws IOException {
TableName name = TableName.valueOf(tablename);
if (admin.tableExists(name)) {
admin.disableTable(name);
admin.deleteTable(name);
}
}
/**
* 利用协处理器进行全表count统计
*
* @param tablename
*/
public Long countRowsWithCoprocessor(String tablename) throws Throwable {
TableName name = TableName.valueOf(tablename);
HTableDescriptor descriptor = admin.getTableDescriptor(name);
String coprocessorClass = "org.apache.hadoop.hbase.coprocessor.AggregateImplementation";
if (!descriptor.hasCoprocessor(coprocessorClass)) {
admin.disableTable(name);
descriptor.addCoprocessor(coprocessorClass);
admin.modifyTable(name, descriptor);
admin.enableTable(name);
}
//计时
StopWatch stopWatch = new StopWatch();
stopWatch.start();
Scan scan = new Scan();
AggregationClient aggregationClient = new AggregationClient(conf);
Long count = aggregationClient.rowCount(name, new LongColumnInterpreter(), scan);
stopWatch.stop();
System.out.println("RowCount：" + count + "，全表count统计耗时：" + stopWatch.getTotalTimeMillis());
return count;
}
/**
* 根据tableName和rowKey精确查询一行的数据
*
* @param tableName 表名
* @param rowKey 行键
* @return java.util.Map<java.lang.String, java.lang.String> 返回一行的数据
* @author zifangsky
* @date 2018/7/3 16:07
* @since 1.0.0
*/
public Map<String, String> getRowToMap(String tableName, String rowKey) {
//返回的键值对
Map<String, String> result = new HashMap<>();
Get get = new Get(Bytes.toBytes(rowKey));
// 获取表
Table table = null;
try {
table = connection.getTable(TableName.valueOf(tableName));
Result hTableResult = table.get(get);
if (hTableResult != null && !hTableResult.isEmpty()) {
for (Cell cell : hTableResult.listCells()) {
// System.out.println("family:" + Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()));
// System.out.println("qualifier:" + Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
// System.out.println("value:" + Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
// System.out.println("Timestamp:" + cell.getTimestamp());
// System.out.println("-------------------------------------------");
result.put(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()), Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return result;
}
/**
* 根据tableName和rowKey和返回类型精确查询一行的数据
*
* @param tableName 表名
* @param rowKey rowKey
* @param T 需要转化的类型
* @param <T> 返回的类型
* @return
* @throws IllegalAccessException
* @throws InstantiationException
*/
public <T> T getRowToObject(String tableName, String rowKey, Class T) throws IllegalAccessException, InstantiationException {
//返回的键值对
Map<String, Object> result = new HashMap<>();
T bean = (T) T.newInstance();
Get get = new Get(Bytes.toBytes(rowKey));
// 获取表
Table table = null;
try {
table = connection.getTable(TableName.valueOf(tableName));
Result hTableResult = table.get(get);
if (hTableResult != null && !hTableResult.isEmpty()) {
for (Cell cell : hTableResult.listCells()) {
result.put(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()), Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
}
bean = (T) mapToObject(result, bean.getClass());
} catch (IOException e) {
logger.error(MessageFormat.format("查询一行的数据失败,tableName:{0},rowKey:{1}"
, tableName, rowKey), e);
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return bean;
}
/**
* 根据tableName、rowKey、familyName、column查询指定单元格的数据
*
* @param tableName 表名
* @param rowKey rowKey
* @param familyName 列族名
* @param columnName 列名
* @return java.lang.String
* @author zifangsky
* @date 2018/7/4 10:58
* @since 1.0.0
*/
public String getColumnValue(String tableName, String rowKey, String familyName, String columnName) {
String str = null;
Get get = new Get(Bytes.toBytes(rowKey));
// 获取表
Table table = null;
try {
table = connection.getTable(TableName.valueOf(tableName));
Result result = table.get(get);
if (result != null && !result.isEmpty()) {
Cell cell = result.getColumnLatestCell(Bytes.toBytes(familyName), Bytes.toBytes(columnName));
if (cell != null) {
str = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
}
}
} catch (IOException e) {
logger.error(MessageFormat.format("查询指定单元格的数据失败,tableName:{0},rowKey:{1},familyName:{2},columnName:{3}"
, tableName, rowKey, familyName, columnName), e);
} finally {
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return str;
}
}
HbaseConfig
@Configuration
@ConfigurationProperties(prefix = HbaseConfig.CONF_PREFIX)
public class HbaseConfig {
public static final String CONF_PREFIX = "hbase.conf";
private Map<String,String> confMaps;
public Map<String, String> getconfMaps() {
return confMaps;
}
public void setconfMaps(Map<String, String> confMaps) {
this.confMaps = confMaps;
}
}
yml
spring:
hbase:
conf:
conf-maps:
'hbase.zookeeper.quorum': 'myhbase:2181'
HbaseTest
package com.xxy;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.xxy.entity.Label;
import com.xxy.entity.User;
import com.xxy.utils.HBaseUtils;
import com.xxy.utils.mybatisPlus.PageUtils;
import com.xxy.utils.mybatisPlus.Query;
import org.apache.hadoop.hbase.client.Result;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* @Description:
* @Author: xuxiaoyu
* @Create: 2020-08-12 22:43
*/
@RunWith(SpringRunner.class)
@SpringBootTest
public class HbaseTest {
@Autowired
HBaseUtils hBaseUtils;
@Test
public void addOne() throws IOException {
// String pc = "userName,xuxiaoyu,idType,111,idNum,8938493#userName,bairuoyu,idType,111,idNum,34322#userName,yienqi,idType,111,idNum,87#";
// hBaseUtils.insertOneRecord("yjx:Policy", "md5", "cf", "username", pc);
// hBaseUtils.deleteColumn("yjx:Policy", "md5", "cf", "username");
hBaseUtils.createTable("yjx:inso", new String[]{"cf"});
hBaseUtils.insertOneRecord("yjx:inso", "md6", "cf", "companycode", "000020");
hBaseUtils.insertOneRecord("yjx:inso", "md6", "cf", "policyno", "p723483");
hBaseUtils.insertOneRecord("yjx:inso", "md6", "cf", "productcode", "PDEX837");
hBaseUtils.insertOneRecord("yjx:inso", "md6", "cf", "effdate", "2020-09-02");
hBaseUtils.insertOneRecord("yjx:inso", "md5", "cf", "companycode", "000138");
hBaseUtils.insertOneRecord("yjx:inso", "md5", "cf", "policyno", "p93732");
hBaseUtils.insertOneRecord("yjx:inso", "md5", "cf", "productcode", "IO938");
hBaseUtils.insertOneRecord("yjx:inso", "md5", "cf", "effdate", "2020-08-13");
}
@Test
public void getValue() throws IOException {
long start = System.currentTimeMillis();
String value = hBaseUtils.selectValue("yjx:Policy", "md5", "cf", "username");
String[] split = value.split("#");
System.out.println("-----------------------------------------------");
List<User> userList = new ArrayList<>();
for (String s : split) {
String[] split1 = s.split(",");
User user = new User();
user.setUserName(split1[1]);
user.setIdType(split1[3]);
user.setIdNum(split1[5]);
userList.add(user);
}
for (User user : userList) {
System.out.println(user);
}
long end = System.currentTimeMillis();
System.out.println((end - start) / 1000f + "s");
System.out.println("-----------------------------------------------");
}
/**
* md5 cf description 特定疾病：25种重大疾病范围以外的重大疾病
* md5 cf label AHRL110004
* md5 cf labellever AHRL110004
* md5 cf rank Ⅰ
*
* @throws Exception
*/
@Test
public void getObejct() throws Exception {
String md5 = hBaseUtils.selectRow("yjx:Policy", "md5");
System.out.println("-----------------------------------------------");
System.out.println(md5);
System.out.println("-----------------------------------------------");
}
@Test
public void getObejct2() throws Exception {
Map<String, String> md5 = hBaseUtils.getRowToMap("yjx:Policy", "md5");
System.out.println("-----------------------------------------------");
System.out.println(md5);
System.out.println("-----------------------------------------------");
}
@Test
public void getObejct3() throws Exception {
Label label = hBaseUtils.getRowToObject("yjx:Policy", "md5", Label.class);
System.out.println("-----------------------------------------------");
System.out.println(label);
System.out.println("-----------------------------------------------");
}
@Test
public void getList() throws Exception {
List<Label> labelList = hBaseUtils.scanAllRecord("yjx:Policy", Label.class);
System.out.println("-----------------------------------------------");
labelList.forEach(label -> System.out.println(label));
System.out.println("-----------------------------------------------");
}
/**
* 根据rowkey关键字查询报告记录
*
* @throws Exception
*/
@Test
public void getRowKey() throws Exception {
List<Label> labelList = hBaseUtils.scanReportDataByRowKeyword("yjx:Policy", "md5", Label.class);
System.out.println("-----------------------------------------------");
labelList.forEach(label -> System.out.println(label));
System.out.println("-----------------------------------------------");
}
/**
* 根据row模糊查询
*
* @throws Exception
*/
@Test
public void getRows() throws Exception {
List<Result> results = hBaseUtils.getRows("yjx:Policy", "md");
System.out.println("-----------------------------------------------");
results.forEach(label -> System.out.println(label));
System.out.println("-----------------------------------------------");
}
/**
* 根据row模糊查询并转化为List<T></>
*
* @throws Exception
*/
@Test
public void getRowsByKey() throws Exception {
List<Label> labelList = hBaseUtils.getListByRowKeyLike("yjx:Policy", "md",Label.class);
System.out.println("-----------------------------------------------");
labelList.forEach(label -> System.out.println(label));
System.out.println("-----------------------------------------------");
}
/**
* 根据tableName、rowKey、familyName、column查询指定单元格的数据
*
* @throws Exception
*/
@Test
public void getColumnValue() throws Exception {
String columnValue = hBaseUtils.getColumnValue("yjx:Policy", "md5", "cf", "label");
System.out.println("-----------------------------------------------");
System.out.println(columnValue);
System.out.println("-----------------------------------------------");
}
@Test
public void pageList() {
//测试
List<Label> labelList = getLabelList();
IPage<Label> page = getPage(1,3,labelList);
System.out.println("-------------------------------------------");
page.getRecords().forEach(label -> System.out.println(label));
System.out.println("-------------------------------------------");
}
public IPage<Label> getPage(int currentPage,int limit,List<Label> list) {
//当前页
int current = currentPage;
//每页数据条数
int size = limit;
IPage<Label> page = new Page<>(current, size);
int count = list.size();
List<Label> pageList = new ArrayList<>();
//计算当前页第一条数据的下标
int currId = current > 1 ? (current - 1) * size : 0;
for (int i = 0; i < size && i < count - currId; i++) {
pageList.add(list.get(currId + i));
}
page.setSize(size);
page.setCurrent(current);
page.setTotal(count);
//计算分页总页数
page.setPages(count % 10 == 0 ? count / 10 : count / 10 + 1);
page.setRecords(pageList);
return page;
}
public List<Label> getLabelList() {
List<Label> list = new ArrayList();
for (int i = 0; i < 19; i++) {
Label label = new Label();
label.setLabel("AHRL9993");
label.setDescription("xdsdw");
label.setLabellever("3");
label.setRank("1");
list.add(label);
}
return list;
}
}
