DRBD + keepalived实现文件实时同步和双机热备

DRBD + keepalived实现文件实时同步和双机热备

安装DRBD

系统初始化设置

注意： 需要有数据盘或者多的分区

yum update -y

关闭防火墙

systemctl stop firewalld
 
systemctl disable firewalld

修改host文件

vim /etc/hosts
192.168.1.240 Primary kylin-01
192.168.1.241 Secondary kylin-02

关闭SELINUX

vim /etc/sysconfig/selinux
SELINUX=disabled

安装依赖

yum install gcc libxslt-devel libxslt  perl keyutils-libs-devel net-tools -y

下载源码编译安装

drbd需要两个安装报：drbd drbd-utils

drbd 安装

wget https://pkg.linbit.com//downloads/drbd/9/drbd-9.2.8.tar.gz
tar -zxvf drbd-9.2.8.tar.gz
cd drbd-9.2.8
make && make install

drbd-utils 安装

wget https://pkg.linbit.com//downloads/drbd/utils/drbd-utils-9.27.0.tar.gz
tar -zxvf drbd-utils-9.27.0.tar.gz
cd drbd-utils-9.27.0
./configure --prefix=/usr/local/drbd --without-83support --with-udev --with-initscripttype=systemd --without-manual
make && make install

安装配置路径：/usr/local/drbd/etc/drbd.d
安装路径： /usr/sbin/drbdsetup /usr/sbin/drbdmeta /usr/sbin/drbdadm

配置drbd

磁盘分区 此处不要格式化磁盘
fdisk /dev/sdb

全局配置

global_common.conf内容如下：

# DRBD is the result of over a decade of development by LINBIT.
# In case you need professional services for DRBD or have
# feature requests visit http://www.linbit.com
 
global {
        usage-count yes;
 
        # Decide what kind of udev symlinks you want for "implicit" volumes
        # (those without explicit volume <vnr> {} block, implied vnr=0):
        # /dev/drbd/by-resource/<resource>/<vnr>   (explicit volumes)
        # /dev/drbd/by-resource/<resource>         (default for implict)
        udev-always-use-vnr; # treat implicit the same as explicit volumes
 
        # minor-count dialog-refresh disable-ip-verification
        # cmd-timeout-short 5; cmd-timeout-medium 121; cmd-timeout-long 600;
}
 
common {
        handlers {
                # These are EXAMPLE handlers only.
                # They may have severe implications,
                # like hard resetting the node under certain circumstances.
                # Be careful when choosing your poison.
 
                # IMPORTANT: most of the following scripts symlink to "notify.sh" which tries to send mail via "mail".
                # If you intend to use this notify.sh script make sure that "mail" is installed.
                #
                pri-on-incon-degr "/usr/lib/drbd/notify-pri-on-incon-degr.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
                pri-lost-after-sb "/usr/lib/drbd/notify-pri-lost-after-sb.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
                local-io-error "/usr/lib/drbd/notify-io-error.sh; /usr/lib/drbd/notify-emergency-shutdown.sh; echo o > /proc/sysrq-trigger ; halt -f";
                # fence-peer "/usr/lib/drbd/crm-fence-peer.sh";
                # split-brain "/usr/lib/drbd/notify-split-brain.sh root";
                # out-of-sync "/usr/lib/drbd/notify-out-of-sync.sh root";
                # before-resync-target "/usr/lib/drbd/snapshot-resync-target-lvm.sh -p 15 -- -c 16k";
                # after-resync-target /usr/lib/drbd/unsnapshot-resync-target-lvm.sh;
                # quorum-lost "/usr/lib/drbd/notify-quorum-lost.sh root";
                # disconnected /bin/true;
        }
 
        startup {
                # wfc-timeout degr-wfc-timeout outdated-wfc-timeout wait-after-sb
        }
 
        options {
                # cpu-mask on-no-data-accessible
 
                # RECOMMENDED for three or more storage nodes with DRBD 9:
                # quorum majority;
                # on-no-quorum suspend-io | io-error;
        }
 
        disk {
                on-io-error detach; #配置I/O错误处理策略为分离
                # size on-io-error fencing disk-barrier disk-flushes
                # disk-drain md-flushes resync-rate resync-after al-extents
                # c-plan-ahead c-delay-target c-fill-target c-max-rate
                # c-min-rate disk-timeout
        }
 
        net {
                # protocol timeout max-epoch-size max-buffers
                # connect-int ping-int sndbuf-size rcvbuf-size ko-count
                # allow-two-primaries cram-hmac-alg shared-secret after-sb-0pri
                # after-sb-1pri after-sb-2pri always-asbp rr-conflict
                # ping-timeout data-integrity-alg tcp-cork on-congestion
                # congestion-fill congestion-extents csums-alg verify-alg
                # use-rle
                
        }
}

资源配置

在node1、node2 上分别建立drbd.res：
drbd.d目录下新建资源文件（drbd.res）
vim /usr/local/drbd/etc/drbd.d/drbd.res

resource  r1 {   #这个r1是定义资源的名字
  protocol C;
  on  kylin-01 {            #on开头，后面是主机名称
     device    /dev/drbd0;  #drbd设备名称
     disk      /dev/sdb1;   #drbd0使用的磁盘分区为sdb1
     address   192.168.1.240:7789; #设置drbd监听地址与端口
     meta-disk  internal;
  }
  on  kylin-02 {            #on开头，后面是主机名称
     device    /dev/drbd0;  #drbd设备名称
     disk      /dev/sdb1;   #drbd0使用的磁盘分区为sdb1
     address   192.168.1.241:7789; #设置drbd监听地址与端口
     meta-disk  internal;
  }
}

在node1、node2 上初始化资源、启动drbd：

[root@kylin-01 drbd.d]# drbdadm create-md r1
initializing activity log
initializing bitmap (640 KB) to all zero
Writing meta data...
New drbd meta data block successfully created.
#启动  两个节点需要同时启动才生效
[root@kylin-01 drbd.d]# systemctl start drbd
# 开机启动
[root@kylin-01 drbd.d]# systemctl enable drbd
# 查看状态
[root@kylin-01 drbd.d]# systemctl status drbd
# 查看状态
netstat -anput|grep 7789
tcp        0      0 192.168.1.240:33015     192.168.1.241:7789      ESTABLISHED -
tcp        0      0 192.168.1.240:40897     192.168.1.241:7789      ESTABLISHED -

检查资源状态：

#查看节点角色状态：
drbdadm role r1
注：第一次启动drbd时，两个drbd节点默认都处于Secondary状态

主节点执行设置主节点：

#初始化资源
drbdadm primary --force r1
#查看资源状态
drbdadm status r1
r1 role:Primary
  disk:UpToDate
  kylin-02 role:Secondary
    replication:SyncSource peer-disk:Inconsistent done:6.16
#查看同步状态
cat /proc/drbd
version: 9.2.8 (api:2/proto:86-122)
GIT-hash: e163b05a76254c0f51f999970e861d72bb16409a build by root@kylin-01, 2024-04-08 15:26:56
Transports (api:20): tcp (9.2.8)
会把主机上的数据传到备机，开始会显示同步进度，过一会显示状态都是”实时”,表示数据同步完成了。接下来就可以使用DRBD了。

DRBD使用（测试）

你现在可以把主机上的DRBD设备挂载到一个目录上进行使用，备机的DRBD设备无法被挂载,因为它是用来接收主机数据的,由DRBD负责操作.

格式化文件系统（文件格式根据自己的系统环境选择）

mkfs.ext4 /dev/drbd0

挂载此文件系统

mkdir /data
mount /dev/drbd0 /data/

在挂载data目录中创建一个测试文件，然后卸载挂载目录，然后切换主备节点，在备用节点上查看刚刚建立的测试文件还是否存在

node1:

[root@node1 ~]# mkdir /data/test
将node1变为备用节点
[root@node1 ~]# umount /data/
[root@node1 ~]# drbdadm secondary r1
[root@node1 ~]# drbdadm role r1
Secondary

node2:

将node2变为主节点
[root@node2 ~]# drbdadm primary r1
[root@node2 ~]# drbdadm role r1
Primary/Secondary
挂载设备，然后看文件是否存在
[root@node2 ~]# mount /dev/drbd0 /data
[root@node2 ~]# cd /data/
[root@node2 mnt]# ls
test
OK! 到这里已经算是完成了！

同样，在Node2上建立文件，然后

  • 卸载/mnt/：umount /mnt/
  • 将Node2降级成备用节点：drbdadm secondary r1
  • 在Node1上升级为主机节点：drbdadm primary r1
  • 在Node1上挂载：mount /dev/drbd0 /data

会发现Node2上的文件也同步到了Node1上。

问题

1、umount时，如果提示device is busy，使用下面方法解决：
fuser -m /data
显示：/data: 25023c
然后kill -9 25023 即可

2、"Split-Brain"（脑裂）的情况：

假设把Primary主机的的eth0设备宕掉，然后直接在Secondary主机上进行提权升级为DRBD的主节点，并且mount挂载DRBD，这时会发现之前在Primary主机上写入的数据文件确实同步过来了。
接着再把Primary主机的eth0设备恢复，看看有没有自动恢复 主从关系。经过查看，发现DRBD检测出了Split-Brain的状况，也就是两个节点都处于standalone状态，
故障描述如下：Split-Brain detected,dropping connection! 这就是传说中的“脑裂”。

DRBD官方推荐的手动恢复方案：

  1. Secondary主机上的操作
drbdadm secondary r0
 
drbdadm disconnect all
 
drbdadm --discard-my-data connect r0            //或者"drbdadm  --  --discard-my-data  connect r0"
  1. Primary主机上的操作
drbdadm disconnect all
 
drbdadm connect r0
 
drbdsetup /dev/drbd0 primary
  1. 检查drdb状态
[root@kylin-01 ~]# drbdadm status r1
r1 role:Primary
  disk:UpToDate
  kylin-02 role:Secondary
    peer-disk:UpToDate

安装keepalived

直接采用yum安装

 yum install -y keepalived

查看keepalived版本

[root@kylin-02 keepalived]# keepalived -v
Keepalived v2.0.20 (01/22,2020)
 
Copyright(C) 2001-2020 Alexandre Cassen, <acassen@gmail.com>
 
Built with kernel headers for Linux 4.19.90
Running on Linux 4.19.90-52.25.v2207.ky10.x86_64 #1 SMP Fri Jun 2 12:05:28 CST 2023
..........

更改keepalived配置

注：本处采用pgsql测试两个机器数据同步

pg docker-compose的文件(docker的配置安装请自行百度)

version: "3"
services:
  postgresql:
    image: postgres:11.8
    container_name: postgres
    hostname: postgres
    ports:
      - "5432:5432"
    volumes:
      - "/data/pgsql:/var/lib/postgresql/data"
      - "/etc/localtime:/etc/localtime"
    restart: on-failure
    logging:
      driver: "json-file"
      options:
        tag: postgres
    cap_add:
      - ALL
    environment:
      POSTGRES_USER: "root"
      POSTGRES_PASSWORD: "123456"
      ALLOW_IP_RANGE: "0.0.0.0/0"
 
cd /etc/keepalived
vim keepalived.conf

主节点keepalived.conf 文件内容如下：

注意：再执行stop脚本时不能直接执行，需要采用脚本调用脚本的方式执行，否则执行不完就会被kill掉（原因暂时没弄清楚）

! Configuration File for keepalived
 
global_defs {
   #notification_email {
   #  acassen@firewall.loc
   #  failover@firewall.loc
   #  sysadmin@firewall.loc
   #}
   #notification_email_from Alexandre.Cassen@firewall.loc
   #smtp_server 192.168.200.1
   #smtp_connect_timeout 30
   router_id kylin-02 # 节点标识，主机名
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}
 
vrrp_instance VI_1 {
    state BACKUP
    interface ens32  # 网卡
    virtual_router_id 51
    priority 100 # 节点权重，主节点100  备节点小于100，数字越大优先级越高
    mcast_src_ip 192.168.1.240 # 本机IP
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.1.239   # 绑定的虚拟IP
    }
    notify_master "/etc/keepalived/notify.sh"  # 节点为master时执行脚本
    notify_backup "/etc/keepalived/notify_back.sh"  # 切换为备节点时执行脚本
    notify_stop "/etc/keepalived/notify_back.sh"  # stop keepalived时执行的脚本
}

备节点keepalived.conf 文件内容如下：

! Configuration File for keepalived
 
global_defs {
   #notification_email {
   #  acassen@firewall.loc
   #  failover@firewall.loc
   #  sysadmin@firewall.loc
   #}
   #notification_email_from Alexandre.Cassen@firewall.loc
   #smtp_server 192.168.200.1
   #smtp_connect_timeout 30
   router_id kylin-02  # 节点标识，主机名
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}
 
vrrp_instance VI_1 {
    state BACKUP
    interface ens32
    virtual_router_id 51
    priority 99  # 节点权重，主节点100  备节点小于100，数字越大优先级越高
    mcast_src_ip 192.168.1.241 # 本机IP
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.1.239
    }
    notify_master "/etc/keepalived/notify.sh"
    notify_backup "/etc/keepalived/notify_back.sh"
    notify_stop "/etc/keepalived/notify_back.sh"
}

notify.sh

#!/bin/bash
drbdadm primary r1
while true
do
        drdbs=$(drbdadm role r1)
        echo "drbd status is $drdbs"
        if [[ "$drdbs" == "Primary" ]];then
                break
        else
                drbdadm primary r1
                sleep 3
        fi
done
mount /dev/drbd0 /data
docker-compose -f /opt/pgsql/docker-compose.yml up -d

stop.sh

#!/bin/bash
docker stop postgres
umount /data/
drbdadm secondary r1
while true
do
        drdbs=$(drbdadm role r1)
        echo "drbd status is $drdbs"
        if [[ "$drdbs"=="Secondary" ]];then
                break
        else
                drbdadm secondary r1
                sleep 3
        fi
done

notify_back.sh

#!/bin/bash
/etc/keepalived/stop.sh

启动keepalived

systemctl start keepalived

开机启动

systemctl enable keepalived

验证

在数据库里面执行创建删除，切换keepalived节点后查看数据是否同步

posted @   蒲公英PGY  阅读(614)  评论(0)  编辑  收藏  举报
相关博文：
· docker-compose 部署rocketmq cluster
· 使用 rsync 命令使用--增量、全量同步备份文件
· DRBD+Keepalived高可用
· 非常强悍并实用的双机热备+负载均衡线上方案
· DRBD 基础及安装
阅读排行：
· 全程不用写代码，我用AI程序员写了一个飞机大战
· DeepSeek 开源周回顾「GitHub 热点速览」
· 记一次.NET内存居高不下排查解决与启示
· 物流快递公司核心技术能力-地址解析分单基础技术分享
· .NET 10首个预览版发布：重大改进与新特性概览！
点击右上角即可分享
微信分享提示
AI FOR CODE 大赛