异构同步 dockercompose + mysql + canal + rocketMQ
docker exec -it mysql-test /bin/bash
cd /etc/mysql
可能没有vim 需要安装
apt-get update
apt-get install vim
vim my.cnf
添加
log-bin=mysql-bin
binlog-format=ROW
server_id=1
重启mysql
docker restart canal_mysql
远程mysql查看是否修改成功
show variables like 'log_bin';
show variables like 'binlog_format';
show master status;
创建canal用户
CREATE USER canal IDENTIFIED BY 'canal';
GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'canal'@'%';
FLUSH PRIVILEGES
如果不适用dockercompose：
docker 安装 rocketMQ
https://blog.csdn.net/ming19951224/article/details/109063041
如果想统一管理 这里是docker-compose 和 .env 的配置
ENV
ROCKETMQ_VERSION=4.9.4
ROCKETMQ_DASHBOARD_VERSION=1.0.0
ROCKETMQ_HOST=xxx.xxx.xxx.xxx
ROCKETMQ_PORT=9876
## MYSQL
MYSQL_VERSION=5.7
MYSQL_HOST=xxx.xxx.xxx.xxx
MYSQL_PORT=3306
MYSQL_USER=root
MYSQL_PASSWORD=123456
## CANAL
CANAL_VERSION=v1.1.4
CANAL_PORT=11111
CANAL_TOPIC=logCanalTopic
CANAL_TABLES=hnyx.t_user
docker-compose.yml
version: '3.1'
networks:
dispacher-network:
name: dispacher-network
external: true
services:
mysql:
image: mysql:${MYSQL_VERSION}
container_name: mysql-compose
restart: always
command:
--server_id=1
--log-bin=mysql-bin
--binlog-format=ROW
--lower_case_table_names=1
--max-allowed-packet=1073741824
--character-set-server=utf8mb4
--collation-server=utf8mb4_unicode_ci
--innodb_log_file_size=256m
environment:
- TZ=Asia/Shanghai
- MYSQL_ROOT_PASSWORD=${MYSQL_PASSWORD}
volumes:
- ./data/mysql/data:/var/lib/mysql
- ./data/mysql/logs:/var/log/mysql
- ./data/mysql/conf:/etc/mysql
ports:
- "${MYSQL_PORT}:3306"
networks:
- dispacher-network
namesrv:
image: apache/rocketmq:${ROCKETMQ_VERSION}
container_name: rocketmq-namesrv
ports:
- 9876:9876
volumes:
- ./data/namesrv/logs:/home/rocketmq/logs
command: sh mqnamesrv
networks:
- dispacher-network
#Service for broker1
broker1:
image: apache/rocketmq:${ROCKETMQ_VERSION}
container_name: rocketmq-broker-1
ports:
- 10909:10909
- 10911:10911
- 10912:10912
depends_on:
- namesrv
environment:
- NAMESRV_ADDR=namesrv:9876
volumes:
- ./data/broker1/logs:/home/rocketmq/logs
- ./data/broker1/store:/home/rocketmq/store
- ./data/brokerconf/broker.conf:/opt/rocketmq-${ROCKETMQ_VERSION}/conf/broker.conf
command: sh mqbroker -c /opt/rocketmq-${ROCKETMQ_VERSION}/conf/broker.conf
networks:
- dispacher-network
dashboard:
image: apacherocketmq/rocketmq-dashboard:${ROCKETMQ_DASHBOARD_VERSION}
container_name: rocketmq-dashboard
restart: always
ports:
- 8076:8080
depends_on:
- namesrv
environment:
- "JAVA_OPTS= -Drocketmq.namesrv.addr=namesrv:9876 -Dcom.rocketmq.sendMessageWithVIPChannel=false"
networks:
- dispacher-network
canal-server:
image: canal/canal-server:${CANAL_VERSION}
container_name: canal-server
ports:
- ${CANAL_PORT}:${CANAL_PORT}
environment:
- canal.instance.mysql.slaveId=222
- canal.instance.master.address=${MYSQL_HOST}:${MYSQL_PORT}
- canal.instance.dbUsername=canal
- canal.instance.dbPassword=canal
- canal.mq.topic=${CANAL_TOPIC}
- canal.serverMode=rocketMQ
- canal.mq.servers=${ROCKETMQ_HOST}:${ROCKETMQ_PORT}
- rocketmq.namesrv.addr=${ROCKETMQ_HOST}:${ROCKETMQ_PORT}
- canal.instance.filter.regex=${CANAL_TABLES}
volumes:
- $PWD/conf/:/usr/local/dockercompose/canal/conf/
- $PWD/logs/:/usr/local/dockercompose/canal/logs/
restart: always
depends_on:
- mysql
networks:
- dispacher-network
运行效果：
rocketMQ部署完以后，其中也遇到了坑：
1.程序会报错“MQClientException: Send [3] times, still failed,”
问题：broker访问不通，但是 docker network ls ，docker network inspect docker_gwbridge
以后看到有两个网卡，mq指定的网卡不对
解决：修改/usr/local/docker/data/brokerconf/broker.cnf
terName = DefaultCluster
brokerName = rocketmq-broker-1
brokerId = 0
brokerIP1 = xxx.xxx.xxx.xxx
deleteWhen = 04
fileReservedTime = 48
brokerRole = ASYNC_MASTER
flushDiskType = ASYNC_FLUSH
autoCreateTopicEnable = true
autoCreateSubscriptionGroup = true
2.运行docker 报“”onto a file (or vice-versa) conf/broker.conf“
问题：如果没有挂载路径，docker会自己创建，但是不能识别文件夹和文件
解决：删除 onto a file (or vice-versa) conf/broker.conf 然后建文件
3. 运行docker denied: requested access to the resource is denied. 还有需要 docker login之类的提示
问题： broker.cnf文件没有权限
解决： chmod -R 777 broker1 namesrv
问题：
Error response from daemon: This node is not a swarm manager
解决：docker swarm init
network dispacher-network declared as external, but could not be found
解决：
docker network create --driver overlay dispacher-network
