FLink自定义Source,不停生产数据
一、代码模板
VideoOrder.java
package net.xdclass.model;
import java.util.Date;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* 订单信息
**/
@Data
@AllArgsConstructor
@NoArgsConstructor
public class VideoOrder {
private String tradeNo;
private String title;
private int money;
private int userId;
private Date createTime;
}
自定义source,生产测试数据用 VideoOrderSource.java
package net.xdclass.class6.source;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Random;
import java.util.UUID;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
import net.xdclass.model.VideoOrder;
/**
* @author wangshida@baijia.com
* @datetime 2022-03-20 下午12:03
* @CopyRight (C) 百家互联
* @desc 自定义数据源
* @menu
*/
public class VideoOrderSource extends RichParallelSourceFunction<VideoOrder> {
//把该变量声明为volatile(不稳定的),这就指示JVM,这个变量是不稳定的,每次使用它都到主存中进行读取,而不是从缓存读取
private volatile Boolean flag = true;
private Random random = new Random();
private static List<String> list = new ArrayList<>();
static {
list.add("spring boot2.x");
list.add("SpringCloud");
list.add("RabbitMQ");
list.add("Kafka");
list.add("mianshi");
list.add("FlinkStream");
list.add("Linux");
}
@Override
public void open(Configuration parameters) throws Exception {
// super.open(parameters);
System.out.println("自定义source,数据生产open---------");
}
@Override
public void close() throws Exception {
// super.close();
System.out.println("自定义source,数据生产close---------");
}
@Override
public void run(SourceContext<VideoOrder> ctx) throws Exception {
while (flag){
Thread.sleep(1000);
String id = UUID.randomUUID().toString();
int userId = random.nextInt(10);
int money = random.nextInt(100);
int videoNum = random.nextInt(list.size());
String title = list.get(videoNum);
VideoOrder videoOrder = new VideoOrder(id,title,money,userId,new Date());
ctx.collect(videoOrder);
}
}
/**
* 结束时候执行
*/
@Override
public void cancel() {
flag = false;
System.out.println("关闭");
}
}
使用自定义数据源,代码demo,FLink04CustomSourceApp.java
package net.xdclass.class6.app;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import net.xdclass.class6.source.VideoOrderSource;
import net.xdclass.model.VideoOrder;
/**
* @author wangshida@baijia.com
* @datetime 2022-03-20 下午6:22
* @CopyRight (C) 百家互联
* @desc
* @menu
*/
public class FLink04CustomSourceApp {
public static void main(String[] args) throws Exception {
//WebUi方式运行
final StreamExecutionEnvironment env =
StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(new Configuration());
// StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
//设置运行模式为流批一体
env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
//并行度
env.setParallelism(2);
//设置为自定义source
DataStream<VideoOrder> ds = env.addSource(new VideoOrderSource());
//过滤
DataStream<VideoOrder> dfFliter = ds.filter(new FilterFunction<VideoOrder>() {
@Override
public boolean filter(VideoOrder videoOrder) throws Exception {
return videoOrder.getMoney() > 10;
}
}).setParallelism(3);
dfFliter.print().setParallelism(4);
//设置名字
env.execute("CustomSourceApp");
}
}