练习：Flink ProcessWindowFunction、AggregateFunction、KeyedProcessFunction

数据

MyData1.txt

5304529,像素不是很高，手机感觉用一段时间会卡,2,1
5304529,京东在线客服太垃圾,3,1
5304529,不知为何，网络卡还是咋滴儿，玩起来略卡像个二手机，不过看这店子又感觉不像，但愿是网卡,3,2
5304529,手机用了一个多月了，拍照马马虎虎，运行这个价位我就不挑了，关键一点这喇叭我是服了，也不追求音质了，能不能大点声，一点没有国产机的喇叭特质！,2,2
5304529,发货太慢。下过订单3天后才发货。客服说下单后2-6天发货。我就不明白为啥迟几天才发货呢！包装封签有磨损。不知道是正常磨损还是被人打开过。,3,2
5304529,还可以，就是插了内存卡，就不能放双SIM卡。希望改进下,3,2
5304529,屏幕中间有线，外屏没有坏，不知道是不是内屏坏了？帮忙处理一下？,2,1
5304529,一般般，一般般,2,1
5304528,手机用着还可以，就是降价降的太快了，刚买了几天就降了一百，心里不平衡,3,2
5304528,赶在618买的，价格挺不错的，质量也还可以,3,2
5304528,经常卡机，微信死机，表情无法自己显示,4,1
5304528,为什么没有原装膜？？？？？？,5,1
5304528,还好吧999,3,2
5304527,声音不是很好,3,2
5304527,经常出现触屏失灵的情况，要息屏后再开屏才能正常,2,1
5304527,购买都没几天就降价一百了，竟然不支持价格保护，显示已经过了价格保护时间，京东越来越坑了!,3,2
5304527,第一次买指纹用不了，换货，过了起码一个星期才到，换货的速度是真的慢，这是第二部，其它还行，就是画质太差，比起我以前那台几百块的真心好过这台手机，其它都还行，就是内存有10g不知干嘛用了，是什么硬件那是什么鬼，没什么了,3,2
5304527,买了才十来天就降价100元，不爽。还不能申请保价。,2,1

MyData2.txt

649418,3052779,1312642,fav,1511663669
649417,3052777,1312642,fav,1511663669
649416,3052776,1312642,fav,1511663665
649415,3052775,1312642,fav,1511663664
764339,4466257,845650,cart,1511664676
764339,4466258,845650,cart,1511664686
764338,4466250,845650,cart,1511664696
764339,4466250,845650,cart,1511664696
977808,5159258,3524510,buy,1511673403
796244,614815,2355072,buy,1511673704
570627,2564231,1575622,buy,1511667055
802611,3434414,1397912,buy,1511666753
711593,3736086,3087458,buy,1511666601
900124,4469151,2462567,buy,1511673567
900124,4469151,2462567,buy,1511673567
900123,4469152,2462567,buy,1511673567
799621,4355790,1281227,buy,1511667268
821956,2832987,4145813,buy,1511674152
649418,3052779,1312642,fav,1511663669
610414,2625678,2131531,pv,1511665701
109917,1044710,3158249,pv,1511665435
402617,5114809,982926,pv,1511663450
49566,429708,1787510,pv,1511663289
318497,3399337,2355072,pv,1511686164
425219,333267,3220984,pv,1511663968
324442,3768310,5053508,pv,1511665379
883227,2418127,4022701,pv,1511686235
935839,4637012,2072473,pv,1511665891
474682,3512755,4082778,pv,1511686119
562848,2408785,2214984,pv,1511686078
764339,4466250,845650,cart,1511664676
552540,2751559,1080785,pv,1511688947
321266,4633979,3607361,pv,1511680688
31970,3264389,4718907,pv,1511664821
994000,3709468,4756105,pv,1511686200
361269,2446802,2520377,pv,1511680319
288698,3572314,4145813,pv,1511688776

bean

 1 import lombok.AllArgsConstructor;
 2 import lombok.Data;
 3 import lombok.NoArgsConstructor;
 4 
 5 @Data
 6 @AllArgsConstructor
 7 @NoArgsConstructor
 8 public class Good {
 9     // 商品id
10     public String itemId;
11     // 用户id
12     public String content;
13     // 商品品类id
14     public String stars;
15     // 生成时间
16     public String level;
17 }

 1 import lombok.AllArgsConstructor;
 2 import lombok.Data;
 3 import lombok.NoArgsConstructor;
 4 
 5 @Data
 6 @AllArgsConstructor
 7 @NoArgsConstructor
 8 public class User {
 9     //uerId（用户ID）
10     public String userId;
11     //Item ID(商品ID)
12     public String itemId;
13     //Category ID（类别ID）
14     public String CategoryId;
15     //Behavior type（行为类型（访问'pv', 购买'buy', 加入购物车'cart', 收藏'fav'））
16     public String type;
17     //Timestamp(时间戳)
18     public Long timeStamp;
19 }

 1 import lombok.AllArgsConstructor;
 2 import lombok.Data;
 3 import lombok.NoArgsConstructor;
 4 
 5 @Data
 6 @AllArgsConstructor
 7 @NoArgsConstructor
 8 //用户行为
 9 public class UserCommonAct {
10     //用于分组的key  可以是userId、分类ID等
11     public String itemId;
12     //数量
13     public Long nums;
14     //结束时间 时间戳
15     public Long end;
16 }

 1 import lombok.AllArgsConstructor;
 2 import lombok.Data;
 3 import lombok.NoArgsConstructor;
 4 
 5 @Data
 6 @AllArgsConstructor
 7 @NoArgsConstructor
 8 public class GoodComment {
 9     //商品id
10     public String id;
11     //评论内容
12     public String content;
13 }

test

 1 import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
 2 import org.apache.flink.api.common.eventtime.WatermarkStrategy;
 3 import org.apache.flink.api.common.serialization.SimpleStringSchema;
 4 import org.apache.flink.streaming.api.datastream.DataStream;
 5 import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
 6 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 7 import org.apache.flink.streaming.api.functions.ProcessFunction;
 8 import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
 9 import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
10 import org.apache.flink.streaming.api.windowing.time.Time;
11 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
12 import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
13 import org.apache.flink.util.Collector;
14 import org.apache.flink.util.OutputTag;
15 
16 import java.time.Duration;
17 import java.util.Iterator;
18 
19 public class FlinkTest1 {
20     public static void main(String[] args) {
21         // 1）创建flink流环境,设置并行度为1（5分）
22         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
23         env.setParallelism(1);
24         // 2）    加载评论数据文件,封装成实体类对象并将结果输出到控制台。 (5分）
25         SingleOutputStreamOperator<Good> good = env.readTextFile("src\\main\\resources\\MyData1.txt")
26                 .map(s -> {
27             String[] ss = s.split(",");
28             return new Good(ss[0], ss[1], ss[2], ss[3]);
29         });
30         good.print();
31         // 3) 统计每种商品的好评率，输出结果到控制台。（5分）
32         SingleOutputStreamOperator<Good> goodWatered = good.assignTimestampsAndWatermarks(
33                 WatermarkStrategy.<Good>forBoundedOutOfOrderness(Duration.ofSeconds(0))
34                         .withTimestampAssigner(new SerializableTimestampAssigner<Good>() {
35                             @Override
36                             public long extractTimestamp(Good good, long l) {
37                                 return 0;
38                             }
39                         })
40         );
41         SingleOutputStreamOperator<String> comment = goodWatered.keyBy(d -> d.itemId)
42                 .window(TumblingEventTimeWindows.of(Time.seconds(3600)))
43                 .process(new ProcessWindowFunction<Good, String, String, TimeWindow>() {
44                     @Override
45                     public void process(String s, Context context, Iterable<Good> iterable, Collector<String> collector) throws Exception {
46                         Long nums = 0L;
47                         Long ct = 0L;
48                         Iterator<Good> iterator = iterable.iterator();
49                         while (iterator.hasNext()) {
50                             Good next = iterator.next();
51                             nums++;
52                             if (next.level.equals("1")) {
53                                 ct++;
54                             }
55                             StringBuffer sb = new StringBuffer();
56                             sb.append("itemId:");
57                             sb.append(next.itemId);
58                             sb.append(" 好评率:");
59                             sb.append(ct * 100.0 / nums);
60                             sb.append("%");
61                             collector.collect(sb.toString());
62                         }
63                     }
64                 });
65         comment.print();
66         //    4） 统计评论星数在大于3颗星的所有商品信息并通过主流输出。（5分）
67         OutputTag<String> low = new OutputTag<String>("low"){};
68         SingleOutputStreamOperator<String> process = goodWatered.keyBy(s -> s.itemId)
69                 .process(new ProcessFunction<Good, String>() {
70                     @Override
71                     public void processElement(Good good, Context context, Collector<String> collector) throws Exception {
72 
73                         if (good.stars.equals("4") || good.stars.equals("5")) {
74                             collector.collect(good.toString());
75                         } else {
76                             context.output(low, good.toString());
77                         }
78                     }
79                 });
80         process.print("good");
81         //    5） 统计评论星数小于3颗星通过测输出流输出。（5分）
82         DataStream<String> bad = process.getSideOutput(low);
83         bad.print("bad");
84         //    6） 统计差评商品信息输出到Kafka（5分）
85         bad.addSink(new FlinkKafkaProducer<String>("hadoop106:9092","badgoods",new SimpleStringSchema()));
86         //    7） 将上述（3）题的结果保存到Hbase,行键为商品ID（5分）
87         comment.addSink(new HBaseSink(GoodComment.class, "goodsComment"));
88         // 8）注释完整。（5分）
89         try {
90             env.execute();
91         } catch (Exception e) {
92             e.printStackTrace();
93         }
94     }
95 }

  1 import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
  2 import org.apache.flink.api.common.eventtime.WatermarkStrategy;
  3 import org.apache.flink.api.common.functions.AggregateFunction;
  4 import org.apache.flink.api.common.serialization.SimpleStringEncoder;
  5 import org.apache.flink.api.common.serialization.SimpleStringSchema;
  6 import org.apache.flink.api.common.state.ListState;
  7 import org.apache.flink.api.common.state.ListStateDescriptor;
  8 import org.apache.flink.configuration.Configuration;
  9 import org.apache.flink.core.fs.Path;
 10 import org.apache.flink.runtime.state.filesystem.FsStateBackend;
 11 import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
 12 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 13 import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
 14 import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink;
 15 import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
 16 import org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows;
 17 import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
 18 import org.apache.flink.streaming.api.windowing.time.Time;
 19 import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 20 import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
 21 import org.apache.flink.util.Collector;
 22 
 23 import java.sql.Timestamp;
 24 import java.time.Duration;
 25 import java.util.ArrayList;
 26 import java.util.Comparator;
 27 
 28 public class FlinkTest2 {
 29     public static void main(String[] args) {
 30         //1）将文件存放到本地封装成实体类对象将结果输出到控制台。（5分）
 31         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 32         env.setParallelism(1);
 33         // uerId（用户ID）
 34         // itemId (商品ID)
 35         // categoryId（类别ID）
 36         // type（行为类型（'pv', 'buy', 'cart', 'fav'））
 37         // timeStamp (时间戳)
 38         SingleOutputStreamOperator<User> map = env.readTextFile("src\\main\\resources\\MyData2.txt").map(
 39                 s -> {
 40                     String[] ss = s.split(",");
 41                     return new User(ss[0], ss[1], ss[2], ss[3], Long.valueOf(ss[4]));
 42                 }
 43         );
 44         map.print();
 45         //2）每隔5S周期性生成检查点，并将检查点结果保存到HDFS。（5分）
 46         env.enableCheckpointing(5000);
 47         env.setStateBackend(new FsStateBackend("hdfs://hadoop106:8020/b"));
 48         //3）抽取时间戳生成WaterMark,最大乱序程度为5s。（5分）
 49         SingleOutputStreamOperator<User> userWater = map.assignTimestampsAndWatermarks(
 50                 WatermarkStrategy.<User>forBoundedOutOfOrderness(Duration.ofSeconds(5))
 51                         .withTimestampAssigner(new SerializableTimestampAssigner<User>() {
 52                             @Override
 53                             public long extractTimestamp(User user, long l) {
 54                                 return user.getTimeStamp() * 1000L;
 55                             }
 56                         })
 57         );
 58         //4）统计20S内购买次数最多的商品信息。（5分）
 59         SingleOutputStreamOperator<UserCommonAct> aggregate4 = userWater.keyBy(s -> s.itemId)
 60                 .window(TumblingEventTimeWindows.of(Time.seconds(20)))
 61                 .aggregate(new AggregateFunction<User, Long, Long>() {
 62                     //计数器 来一个就加一 合计 aggregate
 63                     @Override
 64                     public Long createAccumulator() {
 65                         return 0L;
 66                     }
 67 
 68                     @Override
 69                     public Long add(User user, Long aLong) {
 70                         if (user.type.equals("buy")){
 71                             aLong=aLong + 1;
 72                         }
 73                         return aLong;
 74                     }
 75 
 76                     @Override
 77                     public Long getResult(Long aLong) {
 78                         return aLong;
 79                     }
 80 
 81                     @Override
 82                     public Long merge(Long aLong, Long acc1) {
 83                         return null;
 84                     }
 85                     //封装成 UserCommonAct 类
 86                 }, new ProcessWindowFunction<Long, UserCommonAct, String, TimeWindow>() {
 87                     @Override
 88                     public void process(String s, Context context, Iterable<Long> iterable, Collector<UserCommonAct> collector) throws Exception {
 89                         long end = context.window().getEnd();
 90                         Long next = iterable.iterator().next();
 91                         collector.collect(new UserCommonAct(s, next, end));
 92                     }
 93                 });
 94         // 按结束时间进行窗口聚合
 95         SingleOutputStreamOperator<String> process4 = aggregate4.keyBy(s -> s.end)
 96                 .process(new KeyedProcessFunction<Long, UserCommonAct, String>() {
 97                     // 状态列表
 98                     private ListState<UserCommonAct> lastState;
 99 
100                     @Override
101                     public void open(Configuration parameters) throws Exception {
102                         // 初始化
103                         lastState = getRuntimeContext().getListState(new ListStateDescriptor<UserCommonAct>("last-state", UserCommonAct.class));
104                     }
105 
106                     @Override
107                     public void processElement(UserCommonAct userCommon, Context context, Collector<String> collector) throws Exception {
108                         // 将数据保存到状态列表中
109                         lastState.add(userCommon);
110                         // 注册定时器  比结束时间多1ms 这样可以保证所有的数据到齐
111                         context.timerService().registerEventTimeTimer(userCommon.end + 1L);
112                     }
113 
114                     @Override
115                     public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception {
116                         //定时器触发时要做的事情
117                         //列表
118                         ArrayList<UserCommonAct> list = new ArrayList<>();
119                         //存入
120                         for (UserCommonAct userCommon : lastState.get()) {
121                             list.add(userCommon);
122                         }
123                         //排序
124                         list.sort(new Comparator<UserCommonAct>() {
125                             @Override
126                             public int compare(UserCommonAct o1, UserCommonAct o2) {
127                                 return o2.nums.intValue() - o1.nums.intValue();
128                             }
129                         });
130                         StringBuffer sb = new StringBuffer();
131                         sb.append("----20S内购买次数最多的商品信息----\n");
132                         for (int i = 0; i < 1; i++) {
133                             UserCommonAct userCommon = list.get(i);
134                             sb.append("商品id：");
135                             sb.append(userCommon.itemId);
136                             sb.append(" 购买次数：");
137                             sb.append(userCommon.nums);
138                             sb.append(" 结束时间");
139                             sb.append(new Timestamp(userCommon.end));
140                             sb.append("\n");
141                             sb.append("-------------\n");
142                             out.collect(sb.toString());
143                         }
144 
145                     }
146                 });
147         process4.print();
148         //5）统计各类别下商品的总数。（5分）
149         SingleOutputStreamOperator<UserCommonAct> aggregate5 = userWater.keyBy(s -> s.CategoryId)
150                 .window(TumblingEventTimeWindows.of(Time.hours(20000)))
151                 .aggregate(new AggregateFunction<User, Long, Long>() {
152                     @Override
153                     public Long createAccumulator() {
154                         return 0L;
155                     }
156 
157                     @Override
158                     public Long add(User user, Long aLong) {
159                         if (user.type.equals("buy")){
160                             aLong=aLong + 1;
161                         }
162                         return aLong;
163                     }
164 
165                     @Override
166                     public Long getResult(Long aLong) {
167                         return aLong;
168                     }
169 
170                     @Override
171                     public Long merge(Long aLong, Long acc1) {
172                         return null;
173                     }
174                 }, new ProcessWindowFunction<Long, UserCommonAct, String, TimeWindow>() {
175                     @Override
176                     public void process(String s, Context context, Iterable<Long> iterable, Collector<UserCommonAct> collector) throws Exception {
177                         long end = context.window().getEnd();
178                         Long next = iterable.iterator().next();
179                         collector.collect(new UserCommonAct(s, next, end));
180                     }
181                 });
182         SingleOutputStreamOperator<String> process5 = aggregate5.keyBy(s -> s.end)
183                 .process(new KeyedProcessFunction<Long, UserCommonAct, String>() {
184                     private ListState<UserCommonAct> lastState;
185 
186                     @Override
187                     public void open(Configuration parameters) throws Exception {
188                         lastState = getRuntimeContext().getListState(new ListStateDescriptor<UserCommonAct>("last-state", UserCommonAct.class));
189                     }
190 
191                     @Override
192                     public void processElement(UserCommonAct userCommon, Context context, Collector<String> collector) throws Exception {
193                         context.timerService().registerEventTimeTimer(userCommon.end + 1L);
194                         lastState.add(userCommon);
195                     }
196 
197                     @Override
198                     public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception {
199                         ArrayList<UserCommonAct> list = new ArrayList<>();
200                         for (UserCommonAct userCommon : lastState.get()) {
201                             list.add(userCommon);
202                         }
203                         list.sort(new Comparator<UserCommonAct>() {
204                             @Override
205                             public int compare(UserCommonAct o1, UserCommonAct o2) {
206                                 return o2.nums.intValue() - o1.nums.intValue();
207                             }
208                         });
209                         StringBuffer sb = new StringBuffer();
210                         sb.append("----各类别下商品的总数----\n");
211                         for (int i = 0; i < list.size(); i++) {
212                             UserCommonAct userCommonAct = list.get(i);
213                             sb.append("类别id：");
214                             sb.append(userCommonAct.itemId);
215                             sb.append(" 购买次数：");
216                             sb.append(userCommonAct.nums);
217                             sb.append(" 结束时间");
218                             sb.append(new Timestamp(userCommonAct.end));
219                             sb.append("\n");
220                             sb.append("-------------\n");
221                             out.collect(sb.toString());
222                         }
223 
224                     }
225                 });
226         process5.print();
227         //6）开启滑动窗口，窗口大小为10S，滑动步长为2S，统计收藏最多的商品信息。（5分）
228         SingleOutputStreamOperator<UserCommonAct> aggregate6 = userWater.keyBy(s -> s.itemId)
229                 .window(SlidingEventTimeWindows.of(Time.seconds(10),Time.seconds(2)))
230                 .aggregate(new AggregateFunction<User, Long, Long>() {
231                     @Override
232                     public Long createAccumulator() {
233                         return 0L;
234                     }
235 
236                     @Override
237                     public Long add(User user, Long aLong) {
238                         if (user.type.equals("fav")){
239                              aLong=aLong + 1;
240                         }
241                         return aLong;
242                     }
243 
244                     @Override
245                     public Long getResult(Long aLong) {
246                         return aLong;
247                     }
248 
249                     @Override
250                     public Long merge(Long aLong, Long acc1) {
251                         return null;
252                     }
253                 }, new ProcessWindowFunction<Long, UserCommonAct, String, TimeWindow>() {
254                     @Override
255                     public void process(String s, Context context, Iterable<Long> iterable, Collector<UserCommonAct> collector) throws Exception {
256                         long end = context.window().getEnd();
257                         Long next = iterable.iterator().next();
258                         collector.collect(new UserCommonAct(s, next, end));
259                     }
260                 });
261         SingleOutputStreamOperator<String> process6 = aggregate6.keyBy(s -> s.end)
262                 .process(new KeyedProcessFunction<Long, UserCommonAct, String>() {
263                     private ListState<UserCommonAct> lastState;
264 
265                     @Override
266                     public void open(Configuration parameters) throws Exception {
267                         lastState = getRuntimeContext().getListState(new ListStateDescriptor<UserCommonAct>("last-state", UserCommonAct.class));
268                     }
269 
270                     @Override
271                     public void processElement(UserCommonAct userCommon, Context context, Collector<String> collector) throws Exception {
272                         context.timerService().registerEventTimeTimer(userCommon.end + 1L);
273                         lastState.add(userCommon);
274                     }
275 
276                     @Override
277                     public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception {
278                         ArrayList<UserCommonAct> list = new ArrayList<>();
279                         for (UserCommonAct userCommon : lastState.get()) {
280                             list.add(userCommon);
281                         }
282                         list.sort(new Comparator<UserCommonAct>() {
283                             @Override
284                             public int compare(UserCommonAct o1, UserCommonAct o2) {
285                                 return o2.nums.intValue() - o1.nums.intValue();
286                             }
287                         });
288                         StringBuffer sb = new StringBuffer();
289                         sb.append("----统计收藏最多的商品信息----\n");
290                         for (int i = 0; i < 1; i++) {
291                             UserCommonAct userCommonAct = list.get(i);
292                             sb.append("商品id：");
293                             sb.append(userCommonAct.itemId);
294                             sb.append(" 收藏次数：");
295                             sb.append(userCommonAct.nums);
296                             sb.append(" 结束时间");
297                             sb.append(new Timestamp(userCommonAct.end));
298                             sb.append("\n");
299                             sb.append("-------------\n");
300                             out.collect(sb.toString());
301                         }
302 
303                     }
304                 });
305         process6.print();
306         //7）开启15S窗口，统计加入购物车商品TOP3.（5分）
307         SingleOutputStreamOperator<UserCommonAct> aggregate7 = userWater.keyBy(s -> s.itemId)
308                 .window(TumblingEventTimeWindows.of(Time.seconds(15)))
309                 .aggregate(new AggregateFunction<User, Long, Long>() {
310                     @Override
311                     public Long createAccumulator() {
312                         return 0L;
313                     }
314 
315                     @Override
316                     public Long add(User user, Long aLong) {
317                         if (user.type.equals("cart")){
318                             aLong=aLong + 1;
319                         }
320                         return aLong;
321                     }
322 
323                     @Override
324                     public Long getResult(Long aLong) {
325                         return aLong;
326                     }
327 
328                     @Override
329                     public Long merge(Long aLong, Long acc1) {
330                         return null;
331                     }
332                 }, new ProcessWindowFunction<Long, UserCommonAct, String, TimeWindow>() {
333                     @Override
334                     public void process(String s, Context context, Iterable<Long> iterable, Collector<UserCommonAct> collector) throws Exception {
335                         long end = context.window().getEnd();
336                         Long next = iterable.iterator().next();
337                         collector.collect(new UserCommonAct(s, next, end));
338                     }
339                 });
340         SingleOutputStreamOperator<String> process7 = aggregate7.keyBy(s -> s.end)
341                 .process(new KeyedProcessFunction<Long, UserCommonAct, String>() {
342                     private ListState<UserCommonAct> lastState;
343 
344                     @Override
345                     public void open(Configuration parameters) throws Exception {
346                         lastState = getRuntimeContext().getListState(new ListStateDescriptor<UserCommonAct>("last-state", UserCommonAct.class));
347                     }
348 
349                     @Override
350                     public void processElement(UserCommonAct userCommon, Context context, Collector<String> collector) throws Exception {
351                         context.timerService().registerEventTimeTimer(userCommon.end + 1L);
352                         lastState.add(userCommon);
353                     }
354 
355                     @Override
356                     public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception {
357                         ArrayList<UserCommonAct> list = new ArrayList<>();
358                         for (UserCommonAct userCommon : lastState.get()) {
359                             list.add(userCommon);
360                         }
361                         list.sort(new Comparator<UserCommonAct>() {
362                             @Override
363                             public int compare(UserCommonAct o1, UserCommonAct o2) {
364                                 return o2.nums.intValue() - o1.nums.intValue();
365                             }
366                         });
367                         StringBuffer sb = new StringBuffer();
368                         sb.append("----加入购物车商品TOP3----\n");
369                         if (list.size()<3){
370                             for (int i = 0; i < list.size(); i++) {
371                                 UserCommonAct userCommonAct = list.get(i);
372                                 sb.append("商品id：");
373                                 sb.append(userCommonAct.itemId);
374                                 sb.append(" 加入次数：");
375                                 sb.append(userCommonAct.nums);
376                                 sb.append(" 结束时间");
377                                 sb.append(new Timestamp(userCommonAct.end));
378                                 sb.append("\n");
379                                 sb.append("-------------\n");
380                                 out.collect(sb.toString());
381                             }
382                         }else {
383                             for (int i = 0; i < 3; i++) {
384                                 UserCommonAct userCommonAct = list.get(i);
385                                 sb.append("商品id：");
386                                 sb.append(userCommonAct.itemId);
387                                 sb.append(" 加入次数：");
388                                 sb.append(userCommonAct.nums);
389                                 sb.append(" 结束时间");
390                                 sb.append(new Timestamp(userCommonAct.end));
391                                 sb.append("\n");
392                                 sb.append("-------------\n");
393                                 out.collect(sb.toString());
394                             }
395                         }
396 
397                     }
398                 });
399         process7.print();
400         //8）将上述第（7）题结果保存到KAFKA并显示结果。（5分）
401         process7.addSink(new FlinkKafkaProducer<String>("hadoop106:9092","top3",new SimpleStringSchema()));
402         //9）将上述第（4）题结果保存到HDFS。（路径自定义）（5分）
403         process4.addSink( StreamingFileSink.forRowFormat(new Path("hdfs://hadoop106:8020/data"),new SimpleStringEncoder<String>("Utf-8")).build());
404         //10）统计20S窗口内将商品加入购物车的用户总数。（考虑去重）（5分）
405         SingleOutputStreamOperator<UserCommonAct> aggregate10 = userWater.keyBy(s -> s.userId)
406                 .window(TumblingEventTimeWindows.of(Time.seconds(20)))
407                 .aggregate(new AggregateFunction<User, Long, Long>() {
408                     @Override
409                     public Long createAccumulator() {
410                         return 0L;
411                     }
412 
413                     @Override
414                     public Long add(User user, Long aLong) {
415                         if (user.type.equals("cart")){
416                             aLong=aLong + 1;
417                         }
418                         return aLong;
419                     }
420 
421                     @Override
422                     public Long getResult(Long aLong) {
423                         return aLong;
424                     }
425 
426                     @Override
427                     public Long merge(Long aLong, Long acc1) {
428                         return null;
429                     }
430                 }, new ProcessWindowFunction<Long, UserCommonAct, String, TimeWindow>() {
431                     @Override
432                     public void process(String s, Context context, Iterable<Long> iterable, Collector<UserCommonAct> collector) throws Exception {
433                         long end = context.window().getEnd();
434                         Long next = iterable.iterator().next();
435                         collector.collect(new UserCommonAct(s, next, end));
436                     }
437                 });
438         SingleOutputStreamOperator<String> process10 = aggregate10.keyBy(s -> s.end)
439                 .process(new KeyedProcessFunction<Long, UserCommonAct, String>() {
440                     private ListState<UserCommonAct> lastState;
441 
442                     @Override
443                     public void open(Configuration parameters) throws Exception {
444                         lastState = getRuntimeContext().getListState(new ListStateDescriptor<UserCommonAct>("last-state", UserCommonAct.class));
445                     }
446 
447                     @Override
448                     public void processElement(UserCommonAct userCommon, Context context, Collector<String> collector) throws Exception {
449                         context.timerService().registerEventTimeTimer(userCommon.end + 1L);
450                         lastState.add(userCommon);
451                     }
452 
453                     @Override
454                     public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception {
455                         ArrayList<UserCommonAct> list = new ArrayList<>();
456                         for (UserCommonAct userCommon : lastState.get()) {
457                             list.add(userCommon);
458                         }
459                         list.sort(new Comparator<UserCommonAct>() {
460                             @Override
461                             public int compare(UserCommonAct o1, UserCommonAct o2) {
462                                 return o2.nums.intValue() - o1.nums.intValue();
463                             }
464                         });
465                         StringBuffer sb = new StringBuffer();
466                         sb.append("----将商品加入购物车的用户总数----\n");
467                         for (int i = 0; i < 1; i++) {
468                             UserCommonAct userCommon = list.get(i);
469                             sb.append("商品id：");
470                             sb.append(userCommon.itemId);
471                             sb.append(" 加入人数：");
472                             sb.append(userCommon.nums);
473                             sb.append(" 结束时间");
474                             sb.append(new Timestamp(userCommon.end));
475                             sb.append("\n");
476                             sb.append("-------------\n");
477                             out.collect(sb.toString());
478                         }
479 
480                     }
481                 });
482         process10.print();
483         //11）统计10S窗口内各类别中购买数量最多的商品信息。（5分）
484         SingleOutputStreamOperator<UserCommonAct> cateCountDS = userWater.keyBy(d -> d.CategoryId)
485                 .window(TumblingEventTimeWindows.of(Time.seconds(10)))
486                 .aggregate(new AggregateFunction<User, Long, Long>() {
487                     @Override
488                     public Long createAccumulator() {
489                         return 0L;
490                     }
491 
492                     @Override
493                     public Long add(User user, Long aLong) {
494                         if (user.type.equals("buy")) {
495                             aLong += 1;
496                         }
497                         return aLong;
498                     }
499 
500                     @Override
501                     public Long getResult(Long aLong) {
502                         return aLong;
503                     }
504 
505                     @Override
506                     public Long merge(Long aLong, Long acc1) {
507                         return null;
508                     }
509                 }, new ProcessWindowFunction<Long, UserCommonAct, String, TimeWindow>() {
510                     @Override
511                     public void process(String s, Context context, Iterable<Long> iterable, Collector<UserCommonAct> collector) throws Exception {
512                         long end = context.window().getEnd();
513                         Long next = iterable.iterator().next();
514                         collector.collect(new UserCommonAct(s, next, end));
515                     }
516                 });
517         SingleOutputStreamOperator<String> cateMaxDS = cateCountDS.keyBy(d -> d.end)
518                 .process(new KeyedProcessFunction<Long, UserCommonAct, String>() {
519                     private ListState<UserCommonAct> listState;
520 
521                     @Override
522                     public void open(Configuration parameters) throws Exception {
523                         listState = getRuntimeContext().getListState(new ListStateDescriptor<UserCommonAct>("list-state", UserCommonAct.class));
524                     }
525 
526                     @Override
527                     public void processElement(UserCommonAct userCommonAct, Context context, Collector<String> collector) throws Exception {
528                         listState.add(userCommonAct);
529                         context.timerService().registerEventTimeTimer(context.getCurrentKey() + 1L);
530                     }
531 
532                     @Override
533                     public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception {
534                         ArrayList<UserCommonAct> list = new ArrayList<>();
535                         for (UserCommonAct userCommonAct : listState.get()) {
536                             list.add(userCommonAct);
537                         }
538                         list.sort(new Comparator<UserCommonAct>() {
539                             @Override
540                             public int compare(UserCommonAct o1, UserCommonAct o2) {
541                                 return o2.nums.intValue() - o1.nums.intValue();
542                             }
543                         });
544                         StringBuffer sb = new StringBuffer();
545                         sb.append("-------类别购买最多的商品--------\n");
546                         for (int i = 0; i < 1; i++) {
547                             UserCommonAct u = list.get(i);
548                             String info = "No ." + i + 1 + " 商品id: " + u.itemId + " 购买次数: " + u.nums + " 结束时间: " + new Timestamp(u.end) + "\n";
549                             sb.append(info);
550                         }
551                         sb.append("----------------------------------\n");
552 
553                         out.collect(sb.toString());
554                     }
555 
556                 });
557         cateMaxDS.print();
558         //12）统计20S窗口内用户浏览最高的商品信息。（5分）
559         SingleOutputStreamOperator<UserCommonAct> aggregate12= userWater.keyBy(s -> s.userId)
560                 .window(TumblingEventTimeWindows.of(Time.seconds(20)))
561                 .aggregate(new AggregateFunction<User, Long, Long>() {
562                     @Override
563                     public Long createAccumulator() {
564                         return 0L;
565                     }
566 
567                     @Override
568                     public Long add(User user, Long aLong) {
569                         if (user.type.equals("pv")){
570                             aLong=aLong + 1;
571                         }
572                         return aLong;
573                     }
574 
575                     @Override
576                     public Long getResult(Long aLong) {
577                         return aLong;
578                     }
579 
580                     @Override
581                     public Long merge(Long aLong, Long acc1) {
582                         return null;
583                     }
584                 }, new ProcessWindowFunction<Long, UserCommonAct, String, TimeWindow>() {
585                     @Override
586                     public void process(String s, Context context, Iterable<Long> iterable, Collector<UserCommonAct> collector) throws Exception {
587                         long end = context.window().getEnd();
588                         Long next = iterable.iterator().next();
589                         collector.collect(new UserCommonAct(s, next, end));
590                     }
591                 });
592         SingleOutputStreamOperator<String> process12 = aggregate12.keyBy(s -> s.end)
593                 .process(new KeyedProcessFunction<Long, UserCommonAct, String>() {
594                     private ListState<UserCommonAct> lastState;
595 
596                     @Override
597                     public void open(Configuration parameters) throws Exception {
598                         lastState = getRuntimeContext().getListState(new ListStateDescriptor<UserCommonAct>("last-state", UserCommonAct.class));
599                     }
600 
601                     @Override
602                     public void processElement(UserCommonAct userCommon, Context context, Collector<String> collector) throws Exception {
603                         context.timerService().registerEventTimeTimer(userCommon.end + 1L);
604                         lastState.add(userCommon);
605                     }
606 
607                     @Override
608                     public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception {
609                         ArrayList<UserCommonAct> list = new ArrayList<>();
610                         for (UserCommonAct userCommon : lastState.get()) {
611                             list.add(userCommon);
612                         }
613                         list.sort(new Comparator<UserCommonAct>() {
614                             @Override
615                             public int compare(UserCommonAct o1, UserCommonAct o2) {
616                                 return o2.nums.intValue() - o1.nums.intValue();
617                             }
618                         });
619                         StringBuffer sb = new StringBuffer();
620                         sb.append("----用户浏览最高的商品信息----\n");
621                         for (int i = 0; i < 1; i++) {
622                             UserCommonAct userCommon = list.get(i);
623                             sb.append("商品id：");
624                             sb.append(userCommon.itemId);
625                             sb.append(" 浏览次数：");
626                             sb.append(userCommon.nums);
627                             sb.append(" 结束时间");
628                             sb.append(new Timestamp(userCommon.end));
629                             sb.append("\n");
630                             sb.append("-------------\n");
631                             out.collect(sb.toString());
632                         }
633 
634                     }
635                 });
636         process12.print();
637 
638         try {
639             env.execute();
640         } catch (Exception e) {
641             e.printStackTrace();
642         }
643     }
644 }

util

  1 import org.apache.flink.configuration.Configuration;
  2 import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
  3 import org.apache.hadoop.hbase.*;
  4 import org.apache.hadoop.hbase.client.*;
  5 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
  6 import org.apache.hadoop.hbase.util.Bytes;
  7 
  8 import java.io.IOException;
  9 import java.lang.reflect.Field;
 10 import java.util.*;
 11 
 12 public class HBaseSink extends RichSinkFunction<String> {
 13     private Connection connection;
 14     private Class T;
 15     private String tableName;
 16     private String[] fieldsName;
 17     List<Put> list=new ArrayList<Put>();
 18     
 19     public static String[] getFiledName(Class T) {
 20         Field[] fields =T.getDeclaredFields();
 21         String[] fieldName = new String[fields.length];
 22         for (int i = 0; i < fieldName.length; i++) {
 23             fieldName[i] = fields[i].getName();
 24 
 25         }
 26         return fieldName;
 27     }
 28 
 29     public HBaseSink(Class T, String tableName){
 30         this.T=T;
 31         this.tableName=tableName;
 32         this.fieldsName=getFiledName(T);
 33     }
 34 
 35     @Override
 36     public void open(Configuration parameters) throws Exception {
 37         connection= HBase_Util.getConf();
 38     }
 39 
 40     @Override
 41     public void invoke(String value, Context context) throws Exception {
 42         String[] s1 = value.split(" ");
 43         Table table = connection.getTable(TableName.valueOf(tableName));
 44 //        String rowkey = UUID.randomUUID().toString().replaceAll("-", "");
 45         Put put = new Put(Bytes.toBytes(s1[0]));
 46         for (int i = 0; i < fieldsName.length; i++) {
 47             put.addColumn(Bytes.toBytes("f1"),Bytes.toBytes(fieldsName[i]),Bytes.toBytes(s1[i]));
 48             list.add(put);
 49         }
 50         table.put(list);
 51     }
 52 
 53     @Override
 54     public void close() throws Exception {
 55         connection.close();
 56     }
 57 
 58 
 59     public static  class HBase_Util {
 60         static org.apache.hadoop.conf.Configuration con = HBaseConfiguration.create();
 61         static org.apache.hadoop.conf.Configuration conf = Propss.setConf(con);
 62         static Connection connection;
 63         static HBaseAdmin admin;
 64         static Table t;
 65 
 66         static {
 67             try {
 68                 connection = ConnectionFactory.createConnection(conf);
 69                 admin = (HBaseAdmin)connection.getAdmin();
 70             } catch (IOException e) {
 71                 e.printStackTrace();
 72             }
 73         }
 74         //获取 conn
 75         public static Connection getConf(){
 76             //创建HBase的配置对象
 77             org.apache.hadoop.conf.Configuration conf = HBaseConfiguration.create();
 78             //设置hbase配置属性
 79             conf.set("hbase.zookeeper.quorum","hadoop106,hadoop107,hadoop108");
 80             conf.set("hbase.zookeeper.property.clientPort","2181");
 81             Connection connection=null;
 82             //通过连接函数，创建连接对象
 83             try {
 84                 connection = ConnectionFactory.createConnection(conf);
 85 
 86             } catch (IOException e) {
 87                 e.printStackTrace();
 88             }
 89             return connection;
 90         }
 91 
 92         //建表
 93         public static void build_Table(String tableName,List<String> FamilyNames) throws Exception {
 94             TableDescriptorBuilder buider = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName));
 95             for (String columnName : FamilyNames) {
 96                 ColumnFamilyDescriptor info = ColumnFamilyDescriptorBuilder.of(Bytes.toBytes(columnName));
 97                 buider.setColumnFamily(info);
 98             }
 99             TableDescriptor build = buider.build();
100             admin.createTable(build);
101             System.out.println("____build_done____");
102         }
103 
104         //插入一条数据
105         public static void insert_Row(String tableName,String row,String Family,String qualifier,String value) throws Exception {
106             t = connection.getTable(TableName.valueOf(tableName));
107             Put put = new Put(Bytes.toBytes(row));
108             Put put1 = put.addColumn(Bytes.toBytes(Family), Bytes.toBytes(qualifier), Bytes.toBytes(value));
109             t.put(put1);
110             System.out.println("____insert_Row_done____");
111         }
112 
113         //插入num条数据
114         public  static  void insert_Batch(String tableName,String row,String Family,String qualifier,String value,Integer num) throws Exception {
115             t = connection.getTable(TableName.valueOf(tableName));
116             List<Put> list=new ArrayList<>();
117             for (int i = 0; i < num; i++) {
118                 String s = UUID.randomUUID().toString().replaceAll("-", "");
119                 Put puts = new Put(Bytes.toBytes(s));
120                 Put putss = puts.addColumn(Bytes.toBytes(Family), Bytes.toBytes(qualifier), Bytes.toBytes(value+i));
121                 list.add(putss);
122             }
123             t.put(list);
124             System.out.println("____insert_Batch_done____");
125         }
126 
127         //删除表
128         public static void drop_Table(String tableName) throws Exception {
129             if (admin.tableExists(TableName.valueOf(tableName))){
130                 admin.disableTable(TableName.valueOf(tableName));
131                 admin.deleteTable(TableName.valueOf(tableName));
132                 System.out.println("____drop_Table_done____");
133             }else {
134                 System.out.println("____no_such_Table_found____");
135             }
136 
137         }
138 
139         //删除一条数据
140         public static void  delete_Row(String tableName,String row) throws Exception {
141             t = connection.getTable(TableName.valueOf(tableName));
142             Delete delete = new Delete(Bytes.toBytes(row));
143             t.delete(delete);
144             System.out.println("____delete_Row_done____");
145         }
146 
147         //特定列过滤查询
148         public static void scan_Filter(String tableName, String Family, String qualifier, CompareOperator compare, byte[] value) throws Exception {
149             t = connection.getTable(TableName.valueOf(tableName));
150             SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes(Family), Bytes.toBytes(qualifier), compare, value);
151             Scan scan = new Scan();
152             Scan scan1 = scan.setFilter(filter);
153             ResultScanner scanner = t.getScanner(scan1);
154             Iterator<Result> iterator = scanner.iterator();
155             while (iterator.hasNext()){
156                 Cell[] cells = iterator.next().rawCells();
157                 System.out.println("____"+new String(CellUtil.cloneRow(cells[0]))+"____");
158                 for (Cell cell : cells) {
159                     System.out.print(new String(CellUtil.cloneRow(cell)));
160                     System.out.print(" - ");
161                     System.out.print(new String(CellUtil.cloneFamily(cell)));
162                     System.out.print(" - ");
163                     System.out.print(new String(CellUtil.cloneQualifier(cell)));
164                     System.out.print(" - ");
165                     System.out.println(new String(CellUtil.cloneValue(cell)));
166                 }
167 
168             }
169             System.out.println("____scan_Filter_done____");
170         }
171         //查询一条数据
172         public static void scan_Row(String tableName,String row)throws Exception{
173             t = connection.getTable(TableName.valueOf(tableName));
174             Get get = new Get(Bytes.toBytes(row));
175             Result result = t.get(get);
176             Cell[] cells = result.rawCells();
177             for (Cell cell : cells) {
178                 System.out.print(new String(CellUtil.cloneRow(cell)));
179                 System.out.print(" - ");
180                 System.out.print(new String(CellUtil.cloneFamily(cell)));
181                 System.out.print(" - ");
182                 System.out.print(new String(CellUtil.cloneQualifier(cell)));
183                 System.out.print(" - ");
184                 System.out.println(new String(CellUtil.cloneValue(cell)));
185             }
186             System.out.println("____scan_Row_done____");
187         }
188         //区间查询数据
189         public static void scan_Rows(String tableName,String row1,String row2)throws Exception{
190             t = connection.getTable(TableName.valueOf(tableName));
191             Scan sc=new Scan(Bytes.toBytes(row1),Bytes.toBytes(row2));
192             ResultScanner scanner = t.getScanner(sc);
193             Iterator<Result> iterator = scanner.iterator();
194             System.out.println("____前闭后开____");
195             while (iterator.hasNext()){
196                 Result next = iterator.next();
197                 Cell[] cells = next.rawCells();
198                 System.out.println("____"+new String(CellUtil.cloneRow(cells[0]))+"____");
199                 for (Cell cell : cells) {
200                     System.out.print(new String(CellUtil.cloneRow(cell)));
201                     System.out.print(" - ");
202                     System.out.print(new String(CellUtil.cloneFamily(cell)));
203                     System.out.print(" - ");
204                     System.out.print(new String(CellUtil.cloneQualifier(cell)));
205                     System.out.print(" - ");
206                     System.out.println(new String(CellUtil.cloneValue(cell)));
207                 }
208             }
209             System.out.println("____scan_Rows_done____");
210         }
211         //查询一条特定列族数据
212         public static void get_value_by_family(String tableName,String row,String family)throws Exception{
213             t = connection.getTable(TableName.valueOf(tableName));
214             Get get = new Get(Bytes.toBytes(row));
215             get.addFamily(Bytes.toBytes(family));
216             Result result = t.get(get);
217             Cell[] cells = result.rawCells();
218             for (Cell cell : cells) {
219                 System.out.print(new String(CellUtil.cloneRow(cell)));
220                 System.out.print(" - ");
221                 System.out.print(new String(CellUtil.cloneFamily(cell)));
222                 System.out.print(" - ");
223                 System.out.print(new String(CellUtil.cloneQualifier(cell)));
224                 System.out.print(" - ");
225                 System.out.println(new String(CellUtil.cloneValue(cell)));
226             }
227             System.out.println("____get_value_by_family_done____");
228         }
229 
230         //查询一条特定列数据
231         public static void get_value_by_qualifier(String tableName,String row,String family,String qualifier)throws Exception{
232             t = connection.getTable(TableName.valueOf(tableName));
233             Get get = new Get(Bytes.toBytes(row));
234             get.addColumn(Bytes.toBytes(family),Bytes.toBytes(qualifier));
235             Result result = t.get(get);
236             Cell[] cells = result.rawCells();
237             for (Cell cell : cells) {
238                 System.out.print(new String(CellUtil.cloneRow(cell)));
239                 System.out.print(" - ");
240                 System.out.print(new String(CellUtil.cloneFamily(cell)));
241                 System.out.print(" - ");
242                 System.out.print(new String(CellUtil.cloneQualifier(cell)));
243                 System.out.print(" - ");
244                 System.out.println(new String(CellUtil.cloneValue(cell)));
245             }
246             System.out.println("____get_value_by_qualifier_done____");
247         }
248 
249         //全查某表
250         public static void  scan_All(String tableName) throws Exception {
251             t = connection.getTable(TableName.valueOf(tableName));
252             Scan sc=new Scan();
253             ResultScanner scanner = t.getScanner(sc);
254             Iterator<Result> iterator = scanner.iterator();
255             while (iterator.hasNext()){
256                 Result next = iterator.next();
257                 Cell[] cells = next.rawCells();
258                 System.out.println("____"+new String(CellUtil.cloneRow(cells[0]))+"____");
259                 for (Cell cell : cells) {
260                     System.out.print(new String(CellUtil.cloneRow(cell)));
261                     System.out.print(" - ");
262                     System.out.print(new String(CellUtil.cloneFamily(cell)));
263                     System.out.print(" - ");
264                     System.out.print(new String(CellUtil.cloneQualifier(cell)));
265                     System.out.print(" - ");
266                     System.out.println(new String(CellUtil.cloneValue(cell)));
267                 }
268             }
269             System.out.println("____scan_All_done____");
270         }
271         //查看所有表
272         public static  void list() throws Exception {
273             TableName[] tableNames = admin.listTableNames();
274             for (TableName tableName : tableNames) {
275                 System.out.println(tableName.toString());
276             }
277 
278         }
279         //查看所有表结构
280         public static  void desc_Table(String tableName) throws Exception {
281             List<TableDescriptor> tableDescriptors = admin.listTableDescriptors();
282             Iterator<TableDescriptor> iterator = tableDescriptors.iterator();
283             while (iterator.hasNext()){
284                 TableDescriptor next = iterator.next();
285                 if (next.getTableName().toString().equals(tableName)){
286                     System.out.println(next);
287                 }
288             }
289             System.out.println("____list_done____");
290 
291         }
292 
293 
294         //关流
295         public static  void stop() throws Exception {
296             connection.close();
297             System.out.println("____connection_close_done____");
298         }
299 
300 
301         public static class Propss {
302             public static Properties producer_Props = new Properties();
303             public static Properties consumer_Props = new Properties();
304             public static HashMap<String, Object> kafka_Consumer = new HashMap<>();
305             public static HashMap<String, Object> kafka_Producer = new HashMap<>();
306 
307             public static org.apache.hadoop.conf.Configuration setConf(org.apache.hadoop.conf.Configuration conf){
308                 conf.set("hbase.zookeeper.quorum","hadoop106,hadoop107,hadoop108");
309                 conf.set("hbae.zookeeper.property.client","2181");
310                 return conf;
311             }
312             static {
313 
314                 kafka_Producer.put("bootstrap.servers", "hadoop106:9092,hadoop107:9092,hadoop108:9092");
315                 //0、1 和 all：0表示只要把消息发送出去就返回成功；1表示只要Leader收到消息就返回成功；all表示所有副本都写入数据成功才算成功
316                 kafka_Producer.put("acks", "all");
317                 //重试次数
318                 kafka_Producer.put("retries", Integer.MAX_VALUE);
319                 //批处理的字节数
320                 kafka_Producer.put("batch.size", 16384);
321                 //批处理的延迟时间，当批次数据未满之时等待的时间
322                 kafka_Producer.put("linger.ms", 1);
323                 //用来约束KafkaProducer能够使用的内存缓冲的大小的，默认值32MB
324                 kafka_Producer.put("buffer.memory", 33554432);
325                 // properties.put("value.serializer",
326                 // "org.apache.kafka.common.serialization.ByteArraySerializer");
327                 // properties.put("key.serializer",
328                 // "org.apache.kafka.common.serialization.ByteArraySerializer");
329                 kafka_Producer.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
330                 kafka_Producer.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
331 
332                 kafka_Consumer.put("bootstrap.servers", "hadoop106:9092,hadoop107:9092,hadoop108:9092");
333                 kafka_Consumer.put("group.id", "com-test");
334                 //from beginning
335                 kafka_Consumer.put("auto.offset.reset", "earliest");
336                 kafka_Consumer.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
337                 kafka_Consumer.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
338 
339                 producer_Props.setProperty("bootstrap.servers", "hadoop106:9092,hadoop107:9092,hadoop108:9092");
340                 producer_Props.setProperty("ack", "all");
341                 producer_Props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
342                 producer_Props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
343                 producer_Props.put("auto.offset.reset", "earliest");
344 
345                 consumer_Props.setProperty("bootstrap.servers", "hadoop106:9092,hadoop107:9092,hadoop108:9092");
346                 consumer_Props.setProperty("group.id", "com-test");
347                 consumer_Props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
348                 consumer_Props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
349                 consumer_Props.put("auto.offset.reset", "earliest");
350             }
351 
352 
353         }
354 
355     }
356 
357 }

pom

  <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <maven.compiler.source>1.8</maven.compiler.source>
        <maven.compiler.target>1.8</maven.compiler.target>
        <flink.version>1.13.0</flink.version>
        <hadoop.version>3.1.3</hadoop.version>
        <scala.version>2.12</scala.version>
        <scala.binary.version>2.11</scala.binary.version>
    </properties>

    <dependencies>

        <!--flink-java-core-stream-clients -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-core</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.16.22</version>
        </dependency>


        <!--jedis-->
        <dependency>
            <groupId>redis.clients</groupId>
            <artifactId>jedis</artifactId>
            <version>3.3.0</version>
        </dependency>

        <!-- fastjson -->
        <!--            <dependency>-->
        <!--                <groupId>com.alibaba</groupId>-->
        <!--                <artifactId>fastjson</artifactId>-->
        <!--                <version>1.2.60</version>-->
        <!--            </dependency>-->


        <!--flink SQL table api-->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-java-bridge_${scala.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner-blink_${scala.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-common</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <!--cep-->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-cep-scala_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <!--csv-->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-csv</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <!--sink kafka-->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_${scala.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <!--sink hadoop hdfs-->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-filesystem_${scala.binary.version}</artifactId>
            <version>1.4.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-hadoop-compatibility_${scala.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>${hadoop.version}</version>
        </dependency>

        <!--sink mysql-->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-jdbc_${scala.version}</artifactId>
            <version>1.9.2</version>
        </dependency>
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.38</version>
        </dependency>

        <!--sink数据到hbse-->

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-hbase_${scala.version}</artifactId>
            <version>1.8.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-client</artifactId>
            <version>2.4.3</version>
        </dependency>

        <!--jdbc sink clickhouse-->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-jdbc_${scala.version}</artifactId>
            <version>1.13.0</version>
        </dependency>
        <dependency>
            <groupId>ru.yandex.clickhouse</groupId>
            <artifactId>clickhouse-jdbc</artifactId>
            <version>0.2.4</version>
        </dependency>
        <!--Guava工程包含了若干被Google的Java项目广泛依赖的核心库,方便开发-->
        <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>30.1.1-jre</version>
        </dependency>

        <!--jdbc sink Clickhouse exclusion-->
        <!--        <dependency>-->
        <!--            <groupId>org.apache.flink</groupId>-->
        <!--            <artifactId>flink-connector-jdbc_${scala.version}</artifactId>-->
        <!--            <version>${flink.version}</version>-->
        <!--        </dependency>-->
        <!--        <dependency>-->
        <!--            <groupId>ru.yandex.clickhouse</groupId>-->
        <!--            <artifactId>clickhouse-jdbc</artifactId>-->
        <!--            <version>0.2.4</version>-->
        <!--            <exclusions>-->
        <!--                <exclusion>-->
        <!--                    <groupId>com.fasterxml.jackson.core</groupId>-->
        <!--                    <artifactId>jackson-databind</artifactId>-->
        <!--                </exclusion>-->
        <!--                <exclusion>-->
        <!--                    <groupId>com.fasterxml.jackson.core</groupId>-->
        <!--                    <artifactId>jackson-core</artifactId>-->
        <!--                </exclusion>-->
        <!--            </exclusions>-->
        <!--        </dependency>-->

        <!-- Flink连接redis的连接包-->
        <!--        <dependency>-->
        <!--            <groupId>org.apache.bahir</groupId>-->
        <!--            <artifactId>flink-connector-redis_2.11</artifactId>-->
        <!--            <version>1.0</version>-->
        <!--        </dependency>-->

        <!--jedis-->
        <dependency>
            <groupId>redis.clients</groupId>
            <artifactId>jedis</artifactId>
            <version>3.3.0</version>
        </dependency>

        <!--sink es-->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-elasticsearch7_${scala.version}</artifactId>
            <version>1.10.1</version>
        </dependency>

    </dependencies>