Hbase Scala代码
Hbase 老版本代码
公共方法
def getConn() = {
val conf = HBaseConfiguration.create
conf.set("hbase.zookeeper.quorum","hadoop1,hadoop2,hadoop3")
val conn = ConnectionFactory.createConnection(conf)//重量级
val admin = conn.getAdmin;//轻量级
(admin,conn)
}
val admin = getConn()._1
val conn = getConn()._2
依赖
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.1.17.Final</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.4.13</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase</artifactId>
<version>1.4.13</version>
<type>pom</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>0.20.2</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>1.4.13</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
<version>1.4.13</version>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
<version>4.7</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.4</version>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.6</version>
</dependency>
DDL
import org.apache.hadoop.hbase.{CellUtil, HBaseConfiguration, HColumnDescriptor}
import org.apache.hadoop.hbase.{HTableDescriptor, NamespaceDescriptor, TableName}
import org.apache.hadoop.hbase.client.{ ConnectionFactory, Delete, Get, HBaseAdmin}
import org.apache.hadoop.hbase.client.{Put, Result, ResultScanner, Scan}
import org.apache.hadoop.hbase.filter.{ FilterList, SingleColumnValueFilter}
import org.apache.hadoop.hbase.util.Bytes
import org.apache.spark.sql.SparkSession
admin.tableExists(TableName.valueOf("bigdata","student"))//表是否存在
val hTableDescriptor = new HTableDescriptor(TableName.valueOf("bigdata","student"))//创建表
hTableDescriptor.addFamily(new HColumnDescriptor("info"));
admin.createTable(hTableDescriptor)
admin.disableTables("student")//删除表
admin.deleteTables("student")
val namesp = NamespaceDescriptor.create("bigdata").build()//创建命名空间
admin.createNamespace(namesp)
DML
val addtable = conn.getTable(TableName.valueOf("bigdata","student"))//添加数据
val addput = new Put(Bytes.toBytes("row_key"))
addput.addColumn(Bytes.toBytes("info"),Bytes.toBytes("column_name"),Bytes.toBytes("column_value"))
addtable.put(addput)//可以方法多个put对象
addtable.close()
val gettable = conn.getTable(TableName.valueOf("bigdata","student"))//获取数据
val get = new Get(Bytes.toBytes("row_key"))
get.setMaxVersions()//设置最大版本
get.addFamily(Bytes.toBytes("info"))//指定列簇
get.addColumn(Bytes.toBytes("info"),Bytes.toBytes("name"))//指定列簇+列
val result = gettable.get(get).rawCells()//可以方法多个put对象
for(cell <- result){
Bytes.toString(CellUtil.cloneRow(cell))//row_key
Bytes.toString(CellUtil.cloneFamily(cell))//列簇
Bytes.toString(CellUtil.cloneQualifier(cell))//列名
Bytes.toString(CellUtil.cloneValue(cell)) //value
}
gettable.close()
val scantable = conn.getTable(TableName.valueOf("bigdata","student"))//scan数据
val resultScanner = scantable.getScanner(new Scan(Bytes.toBytes("1001"),Bytes.toBytes("1003")))
import scala.collection.JavaConversions._
for (result <- resultScanner) {//循环结果
for(cell <- result.rawCells()){//循环每一个cell
Bytes.toString(CellUtil.cloneRow(cell))//row_key
}
println()
}
val deletetable = conn.getTable(TableName.valueOf("bigdata","student"))//删除数据
val delete = new Delete(Bytes.toBytes("row_key"))
//生产不用addColumn 因为历史版本会出现
delete.addColumn(Bytes.toBytes("info"),Bytes.toBytes("name"))//找到最大时间戳，给这个版本添加删除标记
delete.addColumn(Bytes.toBytes("info"),Bytes.toBytes("name"),1563502276059L)//删除这个时间戳的数据，小于的不删除
delete.addFamily(Bytes.toBytes("info"))//删除列簇
delete.addColumns(Bytes.toBytes("info"),Bytes.toBytes("name"))//删除所有版本数据 小于等于这个时间戳的
deletetable.delete(delete)//可以接受Delete数组
Hbase 新版本代码
依赖
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.4</version>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.6</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>0.20.2</version>
</dependency>
<dependency>
<groupId>org.glassfish</groupId>
<artifactId>javax.el</artifactId>
<version>3.0.1-b06</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.1.17.Final</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>2.4.11</version>
<exclusions>
<exclusion>
<groupId>org.glassfish</groupId>
<artifactId>javax.el</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>2.4.11</version>
<exclusions>
<exclusion>
<groupId>org.glassfish</groupId>
<artifactId>javax.el</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
<version>2.4.11</version>
<exclusions>
<exclusion>
<groupId>org.glassfish</groupId>
<artifactId>javax.el</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
<version>4.7</version>
</dependency>
公共方法
def getConn() = {
val conf = HBaseConfiguration.create
conf.set("hbase.zookeeper.quorum","hadoop1,hadoop2,hadoop3")
val conn = ConnectionFactory.createConnection(conf)//重量级
val admin = conn.getAdmin;//轻量级
(admin,conn)
}
val admin = getConn()._1
val conn = getConn()._2
DDL
admin.tableExists(TableName.valueOf("bigdata","student"))//表是否存在
//创建表 表的build对象
val tableCreateBuilder = TableDescriptorBuilder.newBuilder(TableName.valueOf("bigdata","student"))
val columFanilyCreateBuild = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("info"))//列簇build对象
columFanilyCreateBuild.setMaxVersions(5)
tableCreateBuilder.setColumnFamily(columFanilyCreateBuild.build())
admin.createTable(tableCreateBuilder.build())
//修改表 不修改其他表属性
val desc = admin.getDescriptor(TableName.valueOf("bigdata","student"))
val tableModBuilder = TableDescriptorBuilder.newBuilder(desc)
val famdesc = desc.getColumnFamily(Bytes.toBytes("info"))
val columFanilyModBuilder = ColumnFamilyDescriptorBuilder.newBuilder(famdesc)//列簇build对象 不修改其他列簇属性
columFanilyModBuilder.setMaxVersions(5)
tableModBuilder.setColumnFamily(columFanilyModBuilder.build())
admin.modifyTable(tableModBuilder.build())
admin.disableTable(TableName.valueOf("bigdata","student"))//删除表
admin.deleteTable(TableName.valueOf("bigdata","student"))
val namesp = NamespaceDescriptor.create("bigdata").build()//创建命名空间
admin.createNamespace(namesp)
DML
val addtable = conn.getTable(TableName.valueOf("bigdata","student"))//添加数据
val addput = new Put(Bytes.toBytes("row_key"))
addput.addColumn(Bytes.toBytes("info"),Bytes.toBytes("column_name"),Bytes.toBytes("column_value"))
addtable.put(addput)//可以方法多个put对象
addtable.close()
val gettable = conn.getTable(TableName.valueOf("bigdata","student"))//获取数据
val get = new Get(Bytes.toBytes("row_key"))
get.readVersions(10)
get.addFamily(Bytes.toBytes("info"))//指定列簇
get.addColumn(Bytes.toBytes("info"),Bytes.toBytes("name"))//指定列簇+列
val result = gettable.get(get).rawCells()//可以方法多个put对象
for(cell <- result){
Bytes.toString(CellUtil.cloneRow(cell))//row_key
Bytes.toString(CellUtil.cloneFamily(cell))//列簇
Bytes.toString(CellUtil.cloneQualifier(cell))//列名
Bytes.toString(CellUtil.cloneValue(cell)) //value
}
gettable.close()
val scantable = conn.getTable(TableName.valueOf("bigdata","student"))//scan数据
val scan = new Scan
val filterList = new FilterList()
//new RowFilter(CompareFilter.CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes("0003"))); rowkey过滤器
//new PrefixFilter("00".getBytes()); rowkey过滤器ColumnValueFilter
//new FamilyFilter(CompareFilter.CompareOp.LESS, new SubstringComparator("f2")); 列簇过滤器
//new QualifierFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator("name")); 列过滤器
//new ColumnPrefixFilter(prefix);根据列（又名限定符）名称的前导部分进行过滤
//new MultipleColumnPrefixFilter(prefixes);byte[][] prefixes = new byte[][] {Bytes.toBytes("abc"), Bytes.toBytes("xyz")}; 多个前缀过滤
//new ColumnRangeFilter(startColumn, true, endColumn, true); 列区间过滤
//new ValueFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator("8")); 值过滤器
//new ColumnValueFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator("8")); 只匹配获取的单元格
//new SingleColumnValueFilter("f1".getBytes(), "name".getBytes(), CompareFilter.CompareOp.EQUAL, "刘备".getBytes()); 满足条件的整列值的所有字段
//new SingleColumnValueExcludeFilter("f1".getBytes(), "name".getBytes(), CompareFilter.CompareOp.EQUAL, "刘备".getBytes()); 排除满足条件的整列值的所有字段
//new PageFilter(pageSize); 分页过滤
//comp=new RegexStringComparator("my.")值比较的正则表达式 new SingleColumnValueFilter("f1".getBytes(), "name".getBytes(), CompareFilter.CompareOp.EQUAL,comp);
//comp=new SubstringComparator("y val");字串包含 new SingleColumnValueFilter("f1".getBytes(), "name".getBytes(), CompareFilter.CompareOp.EQUAL,comp);
val nameFilter = new ColumnValueFilter(Bytes.toBytes("info"),Bytes.toBytes("name"),CompareOperator.EQUAL,Bytes.toBytes("zhangsan"))
filterList.addFilter(nameFilter)
//保留没有当前列的数据
val singleFilter = new SingleColumnValueFilter(Bytes.toBytes("info"),Bytes.toBytes("name"),CompareOperator.EQUAL,Bytes.toBytes("zhangsan"))
filterList.addFilter(singleFilter)
scan.setFilter(filterList)
scan.withStartRow(Bytes.toBytes("1001"))
scan.withStopRow(Bytes.toBytes("1003"))
val resultScanner = scantable.getScanner(scan)
import scala.collection.JavaConversions._
for (result <- resultScanner) {//循环结果
for(cell <- result.rawCells()){//循环每一个cell
Bytes.toString(CellUtil.cloneRow(cell))//row_key
}
println()
}
val deletetable = conn.getTable(TableName.valueOf("bigdata","student"))//删除数据
val delete = new Delete(Bytes.toBytes("row_key"))
//生产不用addColumn 因为历史版本会出现
delete.addColumn(Bytes.toBytes("info"),Bytes.toBytes("name"))//找到最大时间戳，给这个版本添加删除标记
delete.addColumn(Bytes.toBytes("info"),Bytes.toBytes("name"),1563502276059L)//删除这个时间戳的数据，小于的不删除
delete.addFamily(Bytes.toBytes("info"))//删除列簇
delete.addColumns(Bytes.toBytes("info"),Bytes.toBytes("name"))//删除所有版本数据 小于等于这个时间戳的
deletetable.delete(delete)//可以接受Delete数组
搬砖多年终不得要领，遂载源码看之望得真经。
