6 Flink同步

本文目标

Flink消费DBZ推送到Kafka的JSON数据，存入PostgreSQL

创建kafka的映射表

CREATE TABLE t1_kafka (
 f1 BIGINT NOT NULL,
 f2 STRING,
 f3 TIMESTAMP(0) METADATA FROM 'value.source.timestamp' VIRTUAL,
 PRIMARY KEY(f1) NOT ENFORCED
) WITH (
 'connector' = 'kafka',
 'topic' = 'yyx.test.t1',
 'properties.bootstrap.servers' = '127.0.0.1:9092',
 'scan.startup.mode' = 'earliest-offset',
 'debezium-json.schema-include' = 'false',
 'properties.group.id' = 'gyyx',
 'format' = 'debezium-json'
 );
 
 select * from t1_kafka;

创建postgresql的映射表

CREATE TABLE t1_pg (
 f1 BIGINT NOT NULL,
 f2 STRING,
 f3 TIMESTAMP(0),
 PRIMARY KEY(f1) NOT ENFORCED
) WITH (
 'connector' = 'jdbc',
 'url' = 'jdbc:postgresql://127.0.0.1:6432/test',
 'username' = 'test',
 'password' = 'test',
 'driver' = 'org.postgresql.Driver',
 'table-name' = 't1'
);

select * from t1_pg;

创建从kafka到postgresql同步数据的任务

insert into t1_pg(f1, f2, f3) select f1, f2, f3 from t1_kafka;

测试验证

人工修改mysql内的t1表数据，查询postgresql表内的数据是否实时同步
原理如下：

MySQL-DBZ-Kafka-Flink-Postgresql