sunny123456

  博客园 :: 首页 :: 博问 :: 闪存 :: 新随笔 :: 联系 :: 订阅 订阅 :: 管理 ::
JavaFlink系列之一：Maven程序搭建及Java入门案例多种写法

JavaFlink系列之一：Maven程序搭建及Java入门案例多种写法

 一、Flink项目依赖配置

  1. <?xml version="1.0" encoding="UTF-8"?>
  2. <project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  3.          xmlns="http://maven.apache.org/POM/4.0.0"
  4.          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  5.   <modelVersion>4.0.0</modelVersion>
  6.  
  7.   <groupId>com.hainiu</groupId>
  8.   <artifactId>hainiuflink</artifactId>
  9.   <version>1.0</version>
  10.  
  11.   <properties>
  12.     <java.version>1.8</java.version>
  13.     <scala.version>2.11</scala.version>
  14.     <flink.version>1.9.3</flink.version>
  15.     <parquet.version>1.10.0</parquet.version>
  16.     <hadoop.version>2.7.3</hadoop.version>
  17.     <fastjson.version>1.2.72</fastjson.version>
  18.     <redis.version>2.9.0</redis.version>
  19.     <mysql.version>5.1.35</mysql.version>
  20.     <log4j.version>1.2.17</log4j.version>
  21.     <slf4j.version>1.7.7</slf4j.version>
  22.     <maven.compiler.source>1.8</maven.compiler.source>
  23.     <maven.compiler.target>1.8</maven.compiler.target>
  24.     <maven.compiler.compilerVersion>1.8</maven.compiler.compilerVersion>
  25.     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  26.     <project.build.scope>compile</project.build.scope>
  27.     <!--        <project.build.scope>provided</project.build.scope>-->
  28.     <mainClass>com.hainiu.Driver</mainClass>
  29.   </properties>
  30.  
  31.   <dependencies>
  32.     <dependency>
  33.       <groupId>org.slf4j</groupId>
  34.       <artifactId>slf4j-log4j12</artifactId>
  35.       <version>${slf4j.version}</version>
  36.       <scope>${project.build.scope}</scope>
  37.     </dependency>
  38.  
  39.     <dependency>
  40.       <groupId>log4j</groupId>
  41.       <artifactId>log4j</artifactId>
  42.       <version>${log4j.version}</version>
  43.       <scope>${project.build.scope}</scope>
  44.     </dependency>
  45.     <!-- flink的hadoop兼容 -->
  46.     <dependency>
  47.       <groupId>org.apache.hadoop</groupId>
  48.       <artifactId>hadoop-client</artifactId>
  49.       <version>${hadoop.version}</version>
  50.       <scope>${project.build.scope}</scope>
  51.     </dependency>
  52.     <!-- flink的hadoop兼容 -->
  53.     <dependency>
  54.       <groupId>org.apache.flink</groupId>
  55.       <artifactId>flink-hadoop-compatibility_${scala.version}</artifactId>
  56.       <version>${flink.version}</version>
  57.       <scope>${project.build.scope}</scope>
  58.     </dependency>
  59.     <!-- flink的java的api -->
  60.     <dependency>
  61.       <groupId>org.apache.flink</groupId>
  62.       <artifactId>flink-java</artifactId>
  63.       <version>${flink.version}</version>
  64.       <scope>${project.build.scope}</scope>
  65.     </dependency>
  66.     <!-- flink streaming的java的api -->
  67.     <dependency>
  68.       <groupId>org.apache.flink</groupId>
  69.       <artifactId>flink-streaming-java_${scala.version}</artifactId>
  70.       <version>${flink.version}</version>
  71.       <scope>${project.build.scope}</scope>
  72.     </dependency>
  73.     <!-- flink的scala的api -->
  74.     <dependency>
  75.       <groupId>org.apache.flink</groupId>
  76.       <artifactId>flink-scala_${scala.version}</artifactId>
  77.       <version>${flink.version}</version>
  78.       <scope>${project.build.scope}</scope>
  79.     </dependency>
  80.     <!-- flink streaming的scala的api -->
  81.     <dependency>
  82.       <groupId>org.apache.flink</groupId>
  83.       <artifactId>flink-streaming-scala_${scala.version}</artifactId>
  84.       <version>${flink.version}</version>
  85.       <scope>${project.build.scope}</scope>
  86.     </dependency>
  87.     <!-- flink运行时的webUI -->
  88.     <dependency>
  89.       <groupId>org.apache.flink</groupId>
  90.       <artifactId>flink-runtime-web_${scala.version}</artifactId>
  91.       <version>${flink.version}</version>
  92.       <scope>${project.build.scope}</scope>
  93.     </dependency>
  94.     <!-- 使用rocksdb保存flink的state -->
  95.     <dependency>
  96.       <groupId>org.apache.flink</groupId>
  97.       <artifactId>flink-statebackend-rocksdb_${scala.version}</artifactId>
  98.       <version>${flink.version}</version>
  99.       <scope>${project.build.scope}</scope>
  100.     </dependency>
  101.     <!-- flink操作hbase -->
  102.     <dependency>
  103.       <groupId>org.apache.flink</groupId>
  104.       <artifactId>flink-hbase_${scala.version}</artifactId>
  105.       <version>${flink.version}</version>
  106.       <scope>${project.build.scope}</scope>
  107.     </dependency>
  108.     <!-- flink操作es -->
  109.     <dependency>
  110.       <groupId>org.apache.flink</groupId>
  111.       <artifactId>flink-connector-elasticsearch5_${scala.version}</artifactId>
  112.       <version>${flink.version}</version>
  113.       <scope>${project.build.scope}</scope>
  114.     </dependency>
  115.     <!-- flink 的kafka -->
  116.     <dependency>
  117.       <groupId>org.apache.flink</groupId>
  118.       <artifactId>flink-connector-kafka-0.10_${scala.version}</artifactId>
  119.       <version>${flink.version}</version>
  120.       <scope>${project.build.scope}</scope>
  121.     </dependency>
  122.     <!-- flink 写文件到HDFS -->
  123.     <dependency>
  124.       <groupId>org.apache.flink</groupId>
  125.       <artifactId>flink-connector-filesystem_${scala.version}</artifactId>
  126.       <version>${flink.version}</version>
  127.       <scope>${project.build.scope}</scope>
  128.     </dependency>
  129.     <!-- mysql连接驱动 -->
  130.     <dependency>
  131.       <groupId>mysql</groupId>
  132.       <artifactId>mysql-connector-java</artifactId>
  133.       <version>${mysql.version}</version>
  134.       <scope>${project.build.scope}</scope>
  135.     </dependency>
  136.     <!-- redis连接 -->
  137.     <dependency>
  138.       <groupId>redis.clients</groupId>
  139.       <artifactId>jedis</artifactId>
  140.       <version>${redis.version}</version>
  141.       <scope>${project.build.scope}</scope>
  142.     </dependency>
  143.     <!-- flink操作parquet文件格式 -->
  144.     <dependency>
  145.       <groupId>org.apache.parquet</groupId>
  146.       <artifactId>parquet-avro</artifactId>
  147.       <version>${parquet.version}</version>
  148.       <scope>${project.build.scope}</scope>
  149.     </dependency>
  150.     <dependency>
  151.       <groupId>org.apache.parquet</groupId>
  152.       <artifactId>parquet-hadoop</artifactId>
  153.       <version>${parquet.version}</version>
  154.       <scope>${project.build.scope}</scope>
  155.     </dependency>
  156.     <dependency>
  157.       <groupId>org.apache.flink</groupId>
  158.       <artifactId>flink-parquet_${scala.version}</artifactId>
  159.       <version>${flink.version}</version>
  160.       <scope>${project.build.scope}</scope>
  161.     </dependency>
  162.     <!-- json操作 -->
  163.     <dependency>
  164.       <groupId>com.alibaba</groupId>
  165.       <artifactId>fastjson</artifactId>
  166.       <version>${fastjson.version}</version>
  167.       <scope>${project.build.scope}</scope>
  168.     </dependency>
  169.     <dependency>
  170.       <groupId>org.apache.maven.plugins</groupId>
  171.       <artifactId>maven-assembly-plugin</artifactId>
  172.       <version>3.0.0</version>
  173.     </dependency>
  174.   </dependencies>
  175.  
  176.   <build>
  177.     <resources>
  178.       <resource>
  179.         <directory>src/main/resources</directory>
  180.       </resource>
  181.     </resources>
  182.     <plugins>
  183.       <plugin>
  184.         <groupId>org.apache.maven.plugins</groupId>
  185.         <artifactId>maven-assembly-plugin</artifactId>
  186.         <configuration>
  187.           <descriptors>
  188.             <descriptor>src/assembly/assembly.xml</descriptor>
  189.           </descriptors>
  190.           <archive>
  191.             <manifest>
  192.               <mainClass>${mainClass}</mainClass>
  193.             </manifest>
  194.           </archive>
  195.         </configuration>
  196.         <executions>
  197.           <execution>
  198.             <id>make-assembly</id>
  199.             <phase>package</phase>
  200.             <goals>
  201.               <goal>single</goal>
  202.             </goals>
  203.           </execution>
  204.         </executions>
  205.       </plugin>
  206.       <plugin>
  207.         <groupId>org.apache.maven.plugins</groupId>
  208.         <artifactId>maven-surefire-plugin</artifactId>
  209.         <version>2.12</version>
  210.         <configuration>
  211.           <skip>true</skip>
  212.           <forkMode>once</forkMode>
  213.           <excludes>
  214.             <exclude>**/**</exclude>
  215.           </excludes>
  216.         </configuration>
  217.       </plugin>
  218.       <plugin>
  219.         <groupId>org.apache.maven.plugins</groupId>
  220.         <artifactId>maven-compiler-plugin</artifactId>
  221.         <version>3.1</version>
  222.         <configuration>
  223.           <source>${java.version}</source>
  224.           <target>${java.version}</target>
  225.           <encoding>${project.build.sourceEncoding}</encoding>
  226.         </configuration>
  227.       </plugin>
  228.     </plugins>
  229.   </build>
  230.  
  231. </project>

二、JavaFlink案例

  1. package com.linwj.flink;
  2.  
  3. import org.apache.flink.api.common.functions.FlatMapFunction;
  4. import org.apache.flink.api.common.functions.MapFunction;
  5. import org.apache.flink.api.common.functions.RichFlatMapFunction;
  6. import org.apache.flink.api.common.typeinfo.Types;
  7. import org.apache.flink.api.java.tuple.Tuple;
  8. import org.apache.flink.api.java.tuple.Tuple2;
  9. import org.apache.flink.api.java.tuple.Tuple3;
  10. import org.apache.flink.configuration.Configuration;
  11. import org.apache.flink.streaming.api.datastream.DataStreamSource;
  12. import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
  13. import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
  14. import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
  15. import org.apache.flink.streaming.api.functions.ProcessFunction;
  16. import org.apache.flink.util.Collector;
  17.  
  18. import java.util.Arrays;
  19.  
  20. public class test {
  21.     public static void main(String[] args) throws Exception {
  22.         StreamExecutionEnvironment scc = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(new Configuration());
  23.         DataStreamSource<String> socket = scc.socketTextStream("localhost", 6666);  // nc -lk 6666
  24.  
  25. //        // 1.lambda写法
  26. //        SingleOutputStreamOperator<String> t1_flatmap = socket.flatMap((String a, Collector<String> out) -> {   // 传入一个收集工具 Collector<String> out,更多案例见https://vimsky.com/examples/detail/java-method-org.apache.flink.util.Collector.collect.html
  27. //            Arrays.stream(a.split(" ")).forEach((x1) -> {
  28. //                out.collect(x1);
  29. //            });
  30. //        }).returns(Types.STRING);
  31.         t1_flatmap.print();
  32. //
  33. //        SingleOutputStreamOperator<Tuple3<String, Integer, String>> t2_map =
  34. //                t1_flatmap.map((x1) -> Tuple3.of(x1, 1, "gg")).returns(Types.TUPLE(Types.STRING,Types.INT,Types.STRING));
  35.         t2_map.print();
  36. //
  37. //        SingleOutputStreamOperator<Tuple3<String, Integer, String>> t3_keyBy_sum =
  38. //                t2_map.keyBy(0).sum(1); //没有->lambda表达式无需再声明返回类型
  39. //        t3_keyBy_sum.print();
  40. //
  41. //        scc.execute();
  42. //
  43. //        // 2.function写法
  44. //        /*
  45. //        1）匿名内部类的格式： new 父类名&接口名(){ 定义子类成员或者覆盖父类方法 }.方法。而内部类是有自己定的类名的。
  46. //        2） FlatMapFunction<String,String>如果不写泛型会编译报错：Class 'Anonymous class derived from FlatMapFunction' must either be declared abstract or implement abstract method 'flatMap(T, Collector<O>)' in 'FlatMapFunction'
  47. //        保持<String,String>对应实现接口或类的泛型
  48. //         */
  49. //        SingleOutputStreamOperator<String> t1_flatmap = socket.flatMap(new FlatMapFunction<String,String>() {
  50. //            @Override
  51. //            public void flatMap(String value, Collector<String> out) throws Exception {
  52. //                String[] s = value.split(" ");
  53. //                for (String ss : s) {
  54. //                    out.collect(ss);
  55. //                }
  56. //            }
  57. //        });
  58.         t1_flatmap.print();
  59. //
  60. //        SingleOutputStreamOperator<Tuple2<String, Integer>> t2_map = t1_flatmap.map(new MapFunction<String, Tuple2<String, Integer>>() {
  61. //            @Override
  62. //            public Tuple2<String, Integer> map(String s) throws Exception {
  63. //                return Tuple2.of(s, 1);
  64. //            }
  65. //        });
  66.         t2_map.print();
  67. //
  68. //        SingleOutputStreamOperator<Tuple2<String, Integer>> t3_keyBy_sum = t2_map.keyBy(0).sum(1);
  69. //        t3_keyBy_sum.print();
  70.  
  71. //        // 3.function组合写法
  72. //        SingleOutputStreamOperator<Tuple2<String, Integer>> flatmap = socket.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
  73. //
  74. //            @Override
  75. //            public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception {
  76. //                String[] s = value.split(" ");
  77. //                for (String ss : s) {
  78. //                    out.collect(Tuple2.of(ss, 1));
  79. //                }
  80. //            }
  81. //        });
  82. //
  83. //        SingleOutputStreamOperator<Tuple2<String, Integer>> sum = flatmap.keyBy(0).sum(1);
  84. //        sum.print();
  85.  
  86. //        // 4.richfunction组合写法
  87. //        SingleOutputStreamOperator<Tuple2<String, Integer>> flatmap = socket.flatMap(new RichFlatMapFunction<String, Tuple2<String, Integer>>() {
  88. //            private String name = null;
  89. //
  90. //            @Override
  91. //            public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception {
  92. //                String[] s = value.split(" ");
  93. //                for (String ss : s) {
  94. //                    System.out.println(getRuntimeContext().getIndexOfThisSubtask()); //RichMapFunction富函数，额外提供了获取运行时上下文的方法 getRuntimeContext()，可以拿到状态，还有并行度、任务名称之类的运行时信息。
  95. //                    out.collect(Tuple2.of(name + ss, 1));
  96. //                }
  97. //
  98. //            }
  99. //
  100. //            @Override
  101. //            public void open(Configuration parameters) {
  102. //                name = "linwj_";
  103. //            }
  104. //
  105. //            @Override
  106. //            public void close() {
  107. //                name = null;
  108. //            }
  109. //
  110. //        });
  111. //        SingleOutputStreamOperator<Tuple2<String, Integer>> sum = flatmap.keyBy(0).sum(1);
  112. //        sum.print();
  113.  
  114.         //5.processfunction组合写法
  115.         SingleOutputStreamOperator<Tuple2<String, Integer>> sum = socket.process(new ProcessFunction<String, Tuple2<String, Integer>>() {
  116.  
  117.             private String name = null;
  118.  
  119.             @Override
  120.             public void open(Configuration parameters) throws Exception {
  121.                 name = "linwj";
  122.             }
  123.  
  124.             @Override
  125.             public void close() throws Exception {
  126.                 name = null;
  127.             }
  128.  
  129.             @Override
  130.             public void processElement(String value, Context ctx, Collector<Tuple2<String, Integer>> out) throws Exception {
  131. //                getRuntimeContext()
  132.                 String[] s = value.split(" ");
  133.                 for (String ss : s) {
  134.                     System.out.println(getRuntimeContext().getIndexOfThisSubtask());
  135.                     out.collect(Tuple2.of(name + ss, 1));
  136.                 }
  137.             }
  138.         }).keyBy(0).process(new KeyedProcessFunction<Tuple, Tuple2<String, Integer>, Tuple2<String, Integer>>() {
  139.             private Integer num = 0;
  140.  
  141.             @Override
  142.             public void processElement(Tuple2<String, Integer> value, Context ctx, Collector<Tuple2<String, Integer>> out) throws Exception {
  143.                 num += value.f1;
  144.                 out.collect(Tuple2.of(value.f0,num));
  145.             }
  146.         });
  147.         sum.print();
  148.  
  149.         scc.execute();
  150.  
  151.  
  152.  
  153.     }
  154. }

原文链接：https://blog.csdn.net/qq_44000055/article/details/127424229
posted on 2024-02-26 09:07  sunny123456  阅读(94)  评论(0)  编辑  收藏  举报