flink的带状态的RichFlatMapFunction函数使用
背景
使用RichFlatMapFunction可以带状态来决定如何对数据流进行转换,而且这种用法非常常见,根据之前遇到过的某个key的状态来决定再次遇到同样的key时要如何进行数据转换,本文就来简单举个例子说明下RichFlatMapFunction的使用方法
RichFlatMapFunction使用示例
下面的例子的输入是不用name下的count数量值,当本次name的数量和前一次name的数量相差超过配置的阈值100时,打印出来一条告警日志,详细代码如下:
package wikiedits.func.state;
import java.util.Objects;
import org.apache.flink.api.common.functions.RichFlatMapFunction;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.util.Collector;
/**
* Tuple2<String, Integer> 是输入的数据类型 String 是监控到异常值后的输出数据类型
*/
public class MyRichFlatMapFunction extends RichFlatMapFunction<Tuple2<String, Integer>, String> {
// 键值分区状态,对应每个name一个值
ValueState<StateEntity> nameState;
@Override
public void open(Configuration parameters) throws Exception {
// 创建一个键值分区状态
ValueStateDescriptor<StateEntity> state = new ValueStateDescriptor<>("nameState", StateEntity.class);
nameState = getRuntimeContext().getState(state);
}
@Override
public void flatMap(Tuple2<String, Integer> input, Collector<String> collector) throws Exception {
// 判断状态值是否为空(状态默认值是空)
if (Objects.isNull(nameState.value())) {
StateEntity sFalg = new StateEntity(input.f0, input.f1);
nameState.update(sFalg);
return;
}
// 和上一次的状态值比较
StateEntity value = nameState.value();
if (Math.abs(value.count - input.f1) > 100) {
collector.collect(new String("监控到异常值,名称: " + input.f0 + " 上次的值:" + value + " 本次的值:" + input));
}
value.setName(input.f0);
value.setCount(input.f1);
// 更新状态值
nameState.update(value);
}
}
package wikiedits.func.state;
import java.text.SimpleDateFormat;
import java.util.Date;
import org.apache.commons.lang3.RandomUtils;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
public class RichFlatMapFunctionTest {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
// 设置数据源,一共三个元素
DataStream<Tuple2<String, Integer>> dataStream = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {
@Override
public void run(SourceContext<Tuple2<String, Integer>> ctx) throws Exception {
for (int i = 1; i < Integer.MAX_VALUE; i++) {
// 只有XXX,YYY,ZZZ三种name
String name = (0 == i % 3) ? "XXX" : ((i % 3 == 1) ? "YYY" : "ZZZ");
int count = RandomUtils.nextInt(0, 1000);
// 使用当前时间作为时间戳
long timeStamp = System.currentTimeMillis();
// 发射一个元素,并且戴上了时间戳
ctx.collectWithTimestamp(new Tuple2<String, Integer>(name, count), timeStamp);
// 每发射一次就延时1秒
Thread.sleep(5000);
}
}
@Override
public void cancel() {}
});
dataStream.keyBy((f) -> {
return f.f0;
}).flatMap(new MyRichFlatMapFunction()).print();
env.execute();
}
public static String time(long timeStamp) {
return new SimpleDateFormat("yyyy-MM-dd hh:mm:ss").format(new Date(timeStamp));
}
}
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
- 67
- 68
- 69
- 70
- 71
- 72
- 73
- 74
- 75
- 76
- 77
- 78
- 79
- 80
- 81
- 82
- 83
- 84
- 85
- 86
- 87
- 88
- 89
- 90
- 91
- 92
- 93
- 94
- 95
- 96
结果