Flink1.18 Transfrom - 分流【侧输出流 ProcessFunction】

合集 - flink1.18.x 开发常用代码案例模板(30)

1.Flink1.18环境代码编写02-262.Flink1.18传统批处理代码编写02-263.flink1.18传统流处理环境编写02-264.flink1.18流批一体DataStream代码编写02-265.Flink1.18 Source - FileSource02-266.Flink1.18 Source - fromElements02-267.Flink1.18 Source - KafkaSource02-268.Flink1.18 Source - 数据生成器02-269.Flink1.18 Transfrom - Map02-2610.Flink1.18 Transfrom - flatMap02-2611.Flink1.18 Transfrom - Filter02-2612.Flink1.18 Transfrom - KeyBy02-2613.Flink1.18 Transfrom - RichFunction02-2614.Flink1.18 Transfrom - SimpleAggregate简单聚合算子02-2615.Flink1.18 Transfrom - Reduce02-2616.Flink1.18 Transfrom - 分区器【7种现成】02-2617.Flink1.18 Transfrom - CustomPartition自定义分区器02-2618.Flink1.18 Transfrom - 分流【Filter】02-26

19.Flink1.18 Transfrom - 分流【侧输出流 ProcessFunction】02-26

20.Flink1.18 Transfrom - 合流【Union】02-2621.Flink1.18 Transfrom - 合流【Connect, 并行度为1的情况】02-2622.Flink1.18 Transfrom - 合流【Connect 两个流数据进行关联】02-2623.Flink1.18 Sink - 到文件 新版本写法sinkToFile02-2624.Flink1.18 Sink - 到Kafka 新版本写法sinkTo的使用02-2625.Flink1.18 Sink - 到Kafka自定义序列化类 新版本写法sinkTo的使用02-2626.Flink1.18 Sink - 到JDBC 使用新依赖【因返回值是sinkFunction, 故目前只支持addSink使用, 1.18版本不支持sinkTo】02-2627.Flink1.18 Sink - 精确一次ExactlyOnce到JDBC 使用新依赖【因返回值是sinkFunction, 故目前只支持addSink使用, 1.18版本不支持sinkTo】02-2628.Flink1.18 Sink - 自定义sink02-2629.大数据之路Week10_day07 (JavaAPI 操作Redis 与Hbase建立索引，通过查询redis中的索引查询Hbase数据）2020-01-1230.大数据之路Week10_day07 (JavaAPI 操作Redis 模拟将redis当作缓存，从Mysql数据库中查询数据）2020-01-12

收起

package com.xiaohu.transfrom.streamtranform;

import com.xiaohu.bean.WaterSensor;
import com.xiaohu.transfrom.MyPartitioner;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SideOutputDataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;

/*
    侧输出流，实现分流

    需求：WaterSensor,按照id分两种数据流 s1和s2
 */
public class SideOutputDemo {
    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(conf);
        env.setParallelism(2);
        //设置流处理环境还是批处理环境 DataSet API已经过时了，现在都是一套代码，进行设置
//        env.setRuntimeMode(RuntimeExecutionMode.BATCH); //批处理
//        env.setRuntimeMode(RuntimeExecutionMode.STREAMING); //流处理，默认就是流处理
        //一般情况下，不会在代码中指定，不够灵活，一般都是在提交的时候，使用命令进行指定 flink run  -Dexecution.runtime-mode=BATCH【STREAMING】 ...

        DataStreamSource<String> socketDS = env.socketTextStream("master", 7777);

        SingleOutputStreamOperator<WaterSensor> waterSensorOpt1 = socketDS.map(new MapFunction<String, WaterSensor>() {
            @Override
            public WaterSensor map(String value) throws Exception {
                String[] datas = value.split(",");
                String id = datas[0];
                long ts = Long.parseLong(datas[1]);
                int vc = Integer.parseInt(datas[2]);
                return new WaterSensor(id, ts, vc);
            }
        });

        SingleOutputStreamOperator<WaterSensor> processOpt = waterSensorOpt1.process(new ProcessFunction<WaterSensor, WaterSensor>() {
            @Override
            public void processElement(WaterSensor value, ProcessFunction<WaterSensor, WaterSensor>.Context ctx, Collector<WaterSensor> out) throws Exception {
                String id = value.getId();
                if ("s1".equals(id)) {
                    //如果是s1放入到s1侧流中 
                    //OutputTag(String id, TypeInformation<T> typeInfo)
                    /*
                        第一个参数：侧输出流的标签名，自定义
                        第二个参数：放入到该流中的数据类型
                     */
                    OutputTag<WaterSensor> s1 = new OutputTag<>("s1", Types.POJO(WaterSensor.class));
                    //output(OutputTag<X> outputTag, X value);
                    /*
                        第一个参数：Tag对象
                        第二个参数：放入侧输出流的数据
                     */
                    ctx.output(s1, value);
                } else if ("s2".equals(id)) {
                    //如果是s2放入到s2侧流中
                    OutputTag<WaterSensor> s1 = new OutputTag<>("s2", Types.POJO(WaterSensor.class));
                    //output(OutputTag<X> outputTag, X value);
                    ctx.output(s1, value);
                } else {
                    //其他值放入到主流中
                    //out 处理的就是主流
                    out.collect(value);

                }
            }
        });

        processOpt.print("主流："); //直接打印的是主流的数据

        //打印侧输出流
        SideOutputDataStream<WaterSensor> sideStreamS1 = processOpt.getSideOutput(new OutputTag<WaterSensor>("s1", Types.POJO(WaterSensor.class)));
        sideStreamS1.print("s1流：");
        SideOutputDataStream<WaterSensor> sideStreamS2 = processOpt.getSideOutput(new OutputTag<WaterSensor>("s2", Types.POJO(WaterSensor.class)));
        sideStreamS2.print("s2流：");


        env.execute();
    }
}