Canal 数据同步 到 Kafka Canal 配置

原文：Canal 数据同步 到 Kafka - 十色 - 博客园 

确定 bin-log 已开启，SQL：

show variables like 'log_bin%';
show variables like '%binlog_format%';

create database FlinkEtl;
use FlinkEtl;
drop table if exists `employee`;
create table `emplyee`  (
  `id` int(11) not null ,
  `name` varchar(255) character set utf8mb4 collate utf8mb4_unicode_ci null default null,
  `age` int(11) null default null,
  `gender` varchar(25) character set utf8mb4 collate utf8mb4_unicode_ci null default null,
  PRIMARY KEY (`id`) using btree
) engine = InnoDB character set = utf8mb4 collate = utf8mb4_unicode_ci row_format = Dynamic;
-- ----------------------------
-- Records of emplyee
-- ----------------------------
insert into `emplyee` values (1, 'lisi', 15, '男');
insert into `emplyee` values (2, 'zhangsan', 22, '男');
insert into `emplyee` values (3, 'wangwu', 35, '男');
insert into `emplyee` values (4, 'zhaoliu', 25, '男');
insert into `emplyee` values (5, 'tianqi', 32, '男');
insert into `emplyee` values (6, 'xiaohong', 25, '女');
insert into `emplyee` values (7, 'xiaohang', 15, '女');
insert into `emplyee` values (8, 'ht', 22, '男');
insert into `emplyee` values (9, 'df', 23, '男');
insert into `emplyee` values (10, 'hg', 25, '女');
insert into `emplyee` values (11, 'df', 32, '男');
insert into `emplyee` values (12, 'cv', 45, '女');
insert into `emplyee` values (13, 'ty', 15, '女');
insert into `emplyee` values (14, 'yt', 22, '男');
insert into `emplyee` values (15, 'sf', 26, '男');
insert into `emplyee` values (16, 'tt', 25, '女');
insert into `emplyee` values (17, 'tf', 32, '男');
insert into `emplyee` values (18, 'mm', 15, '女');

注意：

1、确定已创建 employeeTopic 主题，正常生产和订阅消息，

时查看生产和消费，实时测试队列服务OK。

2、开启防火墙 9092 端口，用 c# 示例已测，

或者下载 kafka 界面化工具。

Canal

canal.properties

canal.properties 

vim /opt/module/canal/conf/canal.properties 

改动 3 处
#################################################
#########       common argument     #############
#################################################
# 配置 服务IP
canal.ip = hadoop106
# 配置 模式
canal.serverMode = kafka
#################################################
#########      destinations         #############
#################################################
# 配置 目标实例
canal.destinations = example
##################################################
#########            Kafka           #############
##################################################
# 配置 Kafka 
kafka.bootstrap.servers = hadoop106:9092
kafka.acks = all
kafka.compression.type = none
kafka.batch.size = 16384
kafka.linger.ms = 1
kafka.max.request.size = 1048576
kafka.buffer.memory = 33554432
kafka.max.in.flight.requests.per.connection = 1
kafka.retries = 0
kafka.kerberos.enable = false
kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"

instance.properties

vim /opt/module/canal/conf/example/instance.properties

更改的4个部分
# 主库链接
canal.instance.master.address= hadoop106:3306
# 账号和密码
canal.instance.dbUsername=root
canal.instance.dbPassword=root
# 新增 默认链接库
canal.instance.defaultDatabaseName=FlinkEtl
canal.instance.connectionCharset = UTF-8

# 更改 kafka 消息主题？
canal.mq.topic=employeeTopic

 # MySQL

-- 授权 MySQL 账号 canal 具有 slave 权限
create user canal identified by 'canal';  
grant select, replication slave, replication client on *.*  to 'canal'@'%';
-- grant all privileges on *.* to 'canal'@'%' ;
-- revoke all privileges on *.* to 'canal'@'%' ;
-- show grants for  'canal'@'%' ;
-- alter user canal@'%' identified by 'canal';
flush privileges;

# 启动 

sh /opt/module/canal/bin/startup.sh

YAML、YML在线编辑(校验)器
https://www.bejson.com/validators/yaml_editor
 
关于启动日志
查看canal日志
tail -f /opt/module/canal/logs/canal/canal.log

2022-08-17 17:28:01.679 
[main] INFO  com.alibaba.otter.canal.deployer.CanalStarter - 
## the canal server is running now ......

查看 instanse 日志
tail -f /opt/module/canal/logs/example/example.log

 (待续)