FLink自定义Sink,生产的数据导出到Redis
一、pom文件
https://www.cnblogs.com/robots2/p/16048648.html
二、自定义sink
2.1 自定义sink
package net.xdclass.class6.sink;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommand;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommandDescription;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisMapper;
/**
* @menu
*/
public class MyRedisSink implements RedisMapper<Tuple2<String,Integer>> {
//使用什么存储结构,和key名称
@Override
public RedisCommandDescription getCommandDescription() {
return new RedisCommandDescription(RedisCommand.HSET,"VIDEO_ORDER_COUNTER");
}
//获取对应的key或file值
@Override
public String getKeyFromData(Tuple2<String, Integer> data) {
System.out.println("自定义获取的key"+data.f0);
return data.f0;
}
//获取对应的值
@Override
public String getValueFromData(Tuple2<String, Integer> data) {
System.out.println("自定义获取的value"+data.f1.toString());
return data.f1.toString();
}
}
2.2 自定义sink代码
package net.xdclass.class6.app;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.redis.RedisSink;
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisPoolConfig;
import net.xdclass.class6.sink.MyRedisSink;
import net.xdclass.class6.sink.MysqlSink;
import net.xdclass.class6.source.VideoOrderSource;
import net.xdclass.model.VideoOrder;
/**
* @menu
*/
public class FLink07RedisCustomSinkApp {
public static void main(String[] args) throws Exception {
//WebUi方式运行
final StreamExecutionEnvironment env =
StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(new Configuration());
// StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
//设置运行模式为流批一体
env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
//并行度
env.setParallelism(1);
//设置为自定义source
DataStream<VideoOrder> ds = env.addSource(new VideoOrderSource());
//转换transformation,标题转成1
DataStream<Tuple2<String,Integer>> mapDS = ds.map(new MapFunction<VideoOrder, Tuple2<String,Integer>>() {
@Override
public Tuple2<String, Integer> map(VideoOrder videoOrder) throws Exception {
return new Tuple2<>(videoOrder.getTitle(),1);
}
});
//适用于一行转多行情况用FlatMap,一个字段做切割转成多行
// DataStream<Tuple2<String,Integer>> mapDS = ds.flatMap(new FlatMapFunction<VideoOrder, Tuple2<String,Integer>>() {
// @Override
// public void flatMap(VideoOrder value, Collector<Tuple2<String, Integer>> out) throws Exception {
// out.collect(new Tuple2<>(value.getTitle(),1));
// }
// });
//分组
KeyedStream<Tuple2<String, Integer>, String> keyByDs =
mapDS.keyBy(new KeySelector<Tuple2<String, Integer>, String>() {
@Override
public String getKey(Tuple2<String, Integer> value) throws Exception {
return value.f0;
}
});
//统计每组多少个,统计第二个字段
DataStream<Tuple2<String, Integer>> sumDs = keyByDs.sum(1);
//控制台打印
sumDs.print();
//设置单机redis地址
FlinkJedisPoolConfig conf = new FlinkJedisPoolConfig.Builder().setHost("ip地址")
.setPort(6379)
.setPassword("密码")
.build();
//设置为自定义sink
sumDs.addSink(new RedisSink<>(conf,new MyRedisSink()));
//设置名字
env.execute("CustomRedisSinkApp");
}
}