练习 ： Flink 自定义 sink to hbase
hbase
1 package sink;
2
3 import org.apache.flink.configuration.Configuration;
4 import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
5 import org.apache.flink.streaming.api.functions.sink.SinkFunction;
6 import org.apache.hadoop.hbase.*;
7 import org.apache.hadoop.hbase.client.*;
8 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
9 import org.apache.hadoop.hbase.util.Bytes;
10
11 import java.io.IOException;
12 import java.lang.reflect.Field;
13 import java.util.*;
14
15 public class HBaseSink extends RichSinkFunction<String> {
16 private Connection connection;
17 private Class T;
18 private String tableName;
19 private String[] fieldsName;
20 List<Put> list=new ArrayList<Put>();
21
22 public static String[] getFiledName(Class T) {
23 Field[] fields =T.getClass().getDeclaredFields();
24 String[] fieldName = new String[fields.length];
25 for (int i = 0; i < fieldName.length; i++) {
26 fieldName[i] = fields[i].getName();
27
28 }
29 return fieldName;
30 }
31
32 public HBaseSink(Class T, String tableName){
33 this.T=T;
34 this.tableName=tableName;
35 this.fieldsName=getFiledName(T);
36 }
37
38 @Override
39 public void open(Configuration parameters) throws Exception {
40 connection= HBase_Util.getConf();
41 }
42
43 @Override
44 public void invoke(String value, Context context) throws Exception {
45 String[] s1 = value.split(",");
46 Table table = connection.getTable(TableName.valueOf(tableName));
47 // String rowkey = UUID.randomUUID().toString().replaceAll("-", "");
48 Put put = new Put(Bytes.toBytes(s1[0]));
49 for (int i = 0; i < fieldsName.length; i++) {
50 put.addColumn(Bytes.toBytes("info"),Bytes.toBytes(fieldsName[i]),Bytes.toBytes(s1[i]));
51 list.add(put);
52 }
53 table.put(list);
54 }
55
56 @Override
57 public void close() throws Exception {
58 connection.close();
59 }
60
61
62 public static class HBase_Util {
63 static org.apache.hadoop.conf.Configuration con = HBaseConfiguration.create();
64 static org.apache.hadoop.conf.Configuration conf = Propss.setConf(con);
65 static Connection connection;
66 static HBaseAdmin admin;
67 static Table t;
68
69 static {
70 try {
71 connection = ConnectionFactory.createConnection(conf);
72 admin = (HBaseAdmin)connection.getAdmin();
73 } catch (IOException e) {
74 e.printStackTrace();
75 }
76 }
77 //获取 conn
78 public static Connection getConf(){
79 //创建HBase的配置对象
80 org.apache.hadoop.conf.Configuration conf = HBaseConfiguration.create();
81 //设置hbase配置属性
82 conf.set("hbase.zookeeper.quorum","hadoop106,hadoop107,hadoop108");
83 conf.set("hbase.zookeeper.property.clientPort","2181");
84 Connection connection=null;
85 //通过连接函数，创建连接对象
86 try {
87 connection = ConnectionFactory.createConnection(conf);
88
89 } catch (IOException e) {
90 e.printStackTrace();
91 }
92 return connection;
93 }
94
95 //建表
96 public static void build_Table(String tableName,List<String> FamilyNames) throws Exception {
97 TableDescriptorBuilder buider = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName));
98 for (String columnName : FamilyNames) {
99 ColumnFamilyDescriptor info = ColumnFamilyDescriptorBuilder.of(Bytes.toBytes(columnName));
100 buider.setColumnFamily(info);
101 }
102 TableDescriptor build = buider.build();
103 admin.createTable(build);
104 System.out.println("____build_done____");
105 }
106
107 //插入一条数据
108 public static void insert_Row(String tableName,String row,String Family,String qualifier,String value) throws Exception {
109 t = connection.getTable(TableName.valueOf(tableName));
110 Put put = new Put(Bytes.toBytes(row));
111 Put put1 = put.addColumn(Bytes.toBytes(Family), Bytes.toBytes(qualifier), Bytes.toBytes(value));
112 t.put(put1);
113 System.out.println("____insert_Row_done____");
114 }
115
116 //插入num条数据
117 public static void insert_Batch(String tableName,String row,String Family,String qualifier,String value,Integer num) throws Exception {
118 t = connection.getTable(TableName.valueOf(tableName));
119 List<Put> list=new ArrayList<>();
120 for (int i = 0; i < num; i++) {
121 String s = UUID.randomUUID().toString().replaceAll("-", "");
122 Put puts = new Put(Bytes.toBytes(s));
123 Put putss = puts.addColumn(Bytes.toBytes(Family), Bytes.toBytes(qualifier), Bytes.toBytes(value+i));
124 list.add(putss);
125 }
126 t.put(list);
127 System.out.println("____insert_Batch_done____");
128 }
129
130 //删除表
131 public static void drop_Table(String tableName) throws Exception {
132 if (admin.tableExists(TableName.valueOf(tableName))){
133 admin.disableTable(TableName.valueOf(tableName));
134 admin.deleteTable(TableName.valueOf(tableName));
135 System.out.println("____drop_Table_done____");
136 }else {
137 System.out.println("____no_such_Table_found____");
138 }
139
140 }
141
142 //删除一条数据
143 public static void delete_Row(String tableName,String row) throws Exception {
144 t = connection.getTable(TableName.valueOf(tableName));
145 Delete delete = new Delete(Bytes.toBytes(row));
146 t.delete(delete);
147 System.out.println("____delete_Row_done____");
148 }
149
150 //特定列过滤查询
151 public static void scan_Filter(String tableName, String Family, String qualifier, CompareOperator compare, byte[] value) throws Exception {
152 t = connection.getTable(TableName.valueOf(tableName));
153 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes(Family), Bytes.toBytes(qualifier), compare, value);
154 Scan scan = new Scan();
155 Scan scan1 = scan.setFilter(filter);
156 ResultScanner scanner = t.getScanner(scan1);
157 Iterator<Result> iterator = scanner.iterator();
158 while (iterator.hasNext()){
159 Cell[] cells = iterator.next().rawCells();
160 System.out.println("____"+new String(CellUtil.cloneRow(cells[0]))+"____");
161 for (Cell cell : cells) {
162 System.out.print(new String(CellUtil.cloneRow(cell)));
163 System.out.print(" - ");
164 System.out.print(new String(CellUtil.cloneFamily(cell)));
165 System.out.print(" - ");
166 System.out.print(new String(CellUtil.cloneQualifier(cell)));
167 System.out.print(" - ");
168 System.out.println(new String(CellUtil.cloneValue(cell)));
169 }
170
171 }
172 System.out.println("____scan_Filter_done____");
173 }
174 //查询一条数据
175 public static void scan_Row(String tableName,String row)throws Exception{
176 t = connection.getTable(TableName.valueOf(tableName));
177 Get get = new Get(Bytes.toBytes(row));
178 Result result = t.get(get);
179 Cell[] cells = result.rawCells();
180 for (Cell cell : cells) {
181 System.out.print(new String(CellUtil.cloneRow(cell)));
182 System.out.print(" - ");
183 System.out.print(new String(CellUtil.cloneFamily(cell)));
184 System.out.print(" - ");
185 System.out.print(new String(CellUtil.cloneQualifier(cell)));
186 System.out.print(" - ");
187 System.out.println(new String(CellUtil.cloneValue(cell)));
188 }
189 System.out.println("____scan_Row_done____");
190 }
191 //区间查询数据
192 public static void scan_Rows(String tableName,String row1,String row2)throws Exception{
193 t = connection.getTable(TableName.valueOf(tableName));
194 Scan sc=new Scan(Bytes.toBytes(row1),Bytes.toBytes(row2));
195 ResultScanner scanner = t.getScanner(sc);
196 Iterator<Result> iterator = scanner.iterator();
197 System.out.println("____前闭后开____");
198 while (iterator.hasNext()){
199 Result next = iterator.next();
200 Cell[] cells = next.rawCells();
201 System.out.println("____"+new String(CellUtil.cloneRow(cells[0]))+"____");
202 for (Cell cell : cells) {
203 System.out.print(new String(CellUtil.cloneRow(cell)));
204 System.out.print(" - ");
205 System.out.print(new String(CellUtil.cloneFamily(cell)));
206 System.out.print(" - ");
207 System.out.print(new String(CellUtil.cloneQualifier(cell)));
208 System.out.print(" - ");
209 System.out.println(new String(CellUtil.cloneValue(cell)));
210 }
211 }
212 System.out.println("____scan_Rows_done____");
213 }
214 //查询一条特定列族数据
215 public static void get_value_by_family(String tableName,String row,String family)throws Exception{
216 t = connection.getTable(TableName.valueOf(tableName));
217 Get get = new Get(Bytes.toBytes(row));
218 get.addFamily(Bytes.toBytes(family));
219 Result result = t.get(get);
220 Cell[] cells = result.rawCells();
221 for (Cell cell : cells) {
222 System.out.print(new String(CellUtil.cloneRow(cell)));
223 System.out.print(" - ");
224 System.out.print(new String(CellUtil.cloneFamily(cell)));
225 System.out.print(" - ");
226 System.out.print(new String(CellUtil.cloneQualifier(cell)));
227 System.out.print(" - ");
228 System.out.println(new String(CellUtil.cloneValue(cell)));
229 }
230 System.out.println("____get_value_by_family_done____");
231 }
232
233 //查询一条特定列数据
234 public static void get_value_by_qualifier(String tableName,String row,String family,String qualifier)throws Exception{
235 t = connection.getTable(TableName.valueOf(tableName));
236 Get get = new Get(Bytes.toBytes(row));
237 get.addColumn(Bytes.toBytes(family),Bytes.toBytes(qualifier));
238 Result result = t.get(get);
239 Cell[] cells = result.rawCells();
240 for (Cell cell : cells) {
241 System.out.print(new String(CellUtil.cloneRow(cell)));
242 System.out.print(" - ");
243 System.out.print(new String(CellUtil.cloneFamily(cell)));
244 System.out.print(" - ");
245 System.out.print(new String(CellUtil.cloneQualifier(cell)));
246 System.out.print(" - ");
247 System.out.println(new String(CellUtil.cloneValue(cell)));
248 }
249 System.out.println("____get_value_by_qualifier_done____");
250 }
251
252 //全查某表
253 public static void scan_All(String tableName) throws Exception {
254 t = connection.getTable(TableName.valueOf(tableName));
255 Scan sc=new Scan();
256 ResultScanner scanner = t.getScanner(sc);
257 Iterator<Result> iterator = scanner.iterator();
258 while (iterator.hasNext()){
259 Result next = iterator.next();
260 Cell[] cells = next.rawCells();
261 System.out.println("____"+new String(CellUtil.cloneRow(cells[0]))+"____");
262 for (Cell cell : cells) {
263 System.out.print(new String(CellUtil.cloneRow(cell)));
264 System.out.print(" - ");
265 System.out.print(new String(CellUtil.cloneFamily(cell)));
266 System.out.print(" - ");
267 System.out.print(new String(CellUtil.cloneQualifier(cell)));
268 System.out.print(" - ");
269 System.out.println(new String(CellUtil.cloneValue(cell)));
270 }
271 }
272 System.out.println("____scan_All_done____");
273 }
274 //查看所有表
275 public static void list() throws Exception {
276 TableName[] tableNames = admin.listTableNames();
277 for (TableName tableName : tableNames) {
278 System.out.println(tableName.toString());
279 }
280
281 }
282 //查看所有表结构
283 public static void desc_Table(String tableName) throws Exception {
284 List<TableDescriptor> tableDescriptors = admin.listTableDescriptors();
285 Iterator<TableDescriptor> iterator = tableDescriptors.iterator();
286 while