FLink自定义Sink,生产的数据导出到mysql
一、自定义生产数据
https://www.cnblogs.com/robots2/p/16048729.html
二、生产转化数据,导出到mysql
2.1 建表语句
CREATE TABLE `video_order` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`user_id` int(11) DEFAULT NULL,
`money` int(11) DEFAULT NULL,
`title` varchar(32) DEFAULT NULL,
`trade_no` varchar(64) DEFAULT NULL,
`create_time` date DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=32 DEFAULT CHARSET=utf8mb4
2.2自定义source,配置mysql数据源
package net.xdclass.class6.sink;
import java.sql.Connection;
import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import net.xdclass.model.VideoOrder;
/**
* @desc
* @menu
*/
public class MysqlSink extends RichSinkFunction<VideoOrder> {
private Connection conn;
private PreparedStatement ps;
//打开
@Override
public void open(Configuration parameters) throws Exception {
//建立mysql连接
conn = DriverManager.getConnection("jdbc:mysql://ip地址:3306/flink_test?useUnicode=true" +
"&characterEncoding=utf8&allowMultiQueries=true&serverTimezone" +
"=Asia/Shanghai", "账号", "密码");
String sql = "INSERT INTO `video_order` (`user_id`, `money`, `title`, `trade_no`, `create_time`) VALUES(?,?,?,?,?);";
ps = conn.prepareStatement(sql);
System.out.println("自定义sink,open数据库链接 =====");
}
//关闭
@Override
public void close() throws Exception {
//关闭mysql连接
System.out.println("自定义sink,close数据库连接 =====");
if(conn != null){
conn.close();
}
if(ps != null){
ps.close();;
}
}
//每个数据流对象过来,调用的方法
@Override
public void invoke(VideoOrder value, Context context) throws Exception {
ps.setInt(1,value.getUserId());
ps.setInt(2,value.getMoney());
ps.setString(3,value.getTitle());
ps.setString(4,value.getTradeNo());
ps.setDate(5,new Date(value.getCreateTime().getTime()));
ps.executeUpdate();
}
}
2.3 导出到mysql
package net.xdclass.class6.app;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import net.xdclass.class6.sink.MysqlSink;
import net.xdclass.class6.source.VideoOrderSource;
import net.xdclass.model.VideoOrder;
/**
* @menu
*/
public class FLink04CustomSinkApp {
public static void main(String[] args) throws Exception {
//WebUi方式运行
final StreamExecutionEnvironment env =
StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(new Configuration());
// StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
//设置运行模式为流批一体
env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
//并行度
env.setParallelism(1);
//设置为自定义source
DataStream<VideoOrder> ds = env.addSource(new VideoOrderSource());
//过滤
DataStream<VideoOrder> dfFliter = ds.filter(new FilterFunction<VideoOrder>() {
@Override
public boolean filter(VideoOrder videoOrder) throws Exception {
return videoOrder.getMoney() > 10;
}
});
dfFliter.print();
//设置为自定义sink
dfFliter.addSink(new MysqlSink());
//设置名字
env.execute("CustomSourceApp");
}
}