# coding=utf-8
# Author: ruin
"""
discrible:
"""
from thrift.transport import TSocket
from thrift.protocol import TBinaryProtocol
from thrift.transport import TTransport
from hbase import Hbase
import struct
# Method for encoding ints with Thrift's string encoding
def encode(n):
return struct.pack("i", n)
# Method for decoding ints with Thrift's string encoding
def decode(s):
return int(s) if s.isdigit() else struct.unpack('i', s)[0]
class HBaseApi(object):
def __init__(self,table='fr_test_hbase:test_api',host='10.2.46.240',port=9090):
self.table = table.encode('utf-8')
self.host = host
self.port = port
# Connect to HBase Thrift server
self.transport = TTransport.TBufferedTransport(TSocket.TSocket(host, port))
self.protocol = TBinaryProtocol.TBinaryProtocolAccelerated(self.transport)
# Create and open the client connection
self.client = Hbase.Client(self.protocol)
self.transport.open()
# set type and field of column families
self.set_column_families([bytes],['info'])
self._build_column_families()
def set_column_families(self,type_list,col_list=['info']):
self.columnFamiliesType = type_list
self.columnFamilies = col_list
def _build_column_families(self):
"""
give all column families name list,create a table
:return:
"""
tables = self.client.getTableNames()
if self.table not in tables:
self.__create_table(self.table)
def __create_table(self,table):
"""
create table in hbase with column families
:param table: fr_test_hbase:fr_test
:return:
"""
columnFamilies = []
for columnFamily in self.columnFamilies:
name = Hbase.ColumnDescriptor(name = columnFamily)
columnFamilies.append(name)
table = table.encode('utf-8')
print(type(table),type(columnFamilies))
self.client.createTable(table,columnFamilies)
def __del__(self):
self.transport.close()
def __del_table(self,table):
"""
delete a table,first need to disable it
"""
self.client.disableTable(table)
self.client.deleteTable(table)
def getColumnDescriptors(self):
return self.client.getColumnDescriptors(self.table)
def put(self, rowKey, qualifier, value):
"""
put one row
column is column name,value is column value
:param rowKey: rowKey
:param column: column name
:param value: column value
:description: HbaseApi(table).put('rowKey','column','value')
"""
rowKey = rowKey.encode('utf-8')
mutations = []
# for j, column in enumerate(column):
if isinstance(value, str):
value = value.encode('utf-8')
m_name = Hbase.Mutation(column=(self.columnFamilies[0]+':'+qualifier).encode('utf-8'), value=value)
elif isinstance(value, int):
m_name = Hbase.Mutation(column=(self.columnFamilies[0]+':'+qualifier).encode('utf-8'), value=encode(value))
mutations.append(m_name)
self.client.mutateRow(self.table, rowKey, mutations, {})
def puts(self,rowKeys,qualifier,values):
""" put sevel rows, `qualifier` is autoincrement
:param rowKeys: a single rowKey
:param values: values is a 2-dimension list, one piece element is [name, sex, age]
:param qualifier: column family qualifier
Usage::
>>> HBaseTest('table').puts(rowKeys=[1,2,3],qualifier="name",values=[1,2,3])
"""
mutationsBatch = []
if not isinstance(rowKeys,list):
rowKeys = [rowKeys] * len(values)
for i, value in enumerate(values):
mutations = []
# for j, column in enumerate(value):
if isinstance(value, str):
value = value.encode('utf-8')
m_name = Hbase.Mutation(column=(self.columnFamilies[0]+':'+qualifier).encode('utf-8'), value=value)
elif isinstance(value, int):
m_name = Hbase.Mutation(column=(self.columnFamilies[0]+':'+qualifier).encode('utf-8'), value=encode(value))
mutations.append(m_name)
mutationsBatch.append(Hbase.BatchMutation(row = rowKeys[i].encode('utf-8'),mutations=mutations))
self.client.mutateRows(self.table, mutationsBatch, {})
def getRow(self,row, qualifier='name'):
"""
get one row from hbase table
:param row:
:param qualifier:
:return:
"""
# res = []
row = self.client.getRow(self.table, row.encode('utf-8'),{})
for r in row:
rd = {}
row = r.row.decode('utf-8')
value = (r.columns[b'info:name'].value).decode('utf-8')
rd[row] = value
# res.append(rd)
# print ('the row is ',r.row.decode('utf-8'))
# print ('the value is ',(r.columns[b'info:name'].value).decode('utf-8'))
return rd
def getRows(self, rows, qualifier='name'):
"""
get rows from hbase,all the row sqecify the same 'qualifier'
:param rows: a list of row key
:param qualifier: column
:return: None
"""
# grow = True if len(rows) == 1 else False
res = []
for r in rows:
res.append(self.getRow(r,qualifier))
return res
def scanner(self, numRows=100, startRow=None, stopRow=None):
"""
:param numRows:
:param startRow:
:param stopRow:
:return:
"""
scan = Hbase.TScan(startRow, stopRow)
scannerId = self.client.scannerOpenWithScan(self.table,scan, {})
ret = []
rowList = self.client.scannerGetList(scannerId, numRows)
for r in rowList:
rd = {}
row = r.row.decode('utf-8')
value = (r.columns[b'info:name'].value).decode('utf-8')
rd[row] = value
# print ('the row is ',r.row.decode('utf-8'))
# print ('the value is ',(r.columns[b'info:name'].value).decode('utf-8'))
ret.append(rd)
return ret
def demo():
ha = HBaseApi('fr_test_hbase:test_log1')
# ha.put('0002','age','23')
rowKeys = [str(key) for key in range(10001,10010)]
values = ['fr'+str(val) for val in range(10001,10010)]
ha.puts(rowKeys,'name',values)
print(ha.scanner())
# print(ha.getRow('0001'))
# print(ha.getRows(rowKeys))
if __name__ == "__main__":
demo()
