rsync 的配置
一、服务端：
1.安装、
cd /home/rongda
mkdir rsync
[root@rongda rongda]# ls rsync/
log rsync.passwd run tool
yum localinstall rsync -y 【默认安装的是rsync-3.1.2-4.el7.x86_64版本的】
2.配置
[root@backup ~]# vim /etc/rsyncd.conf
uid = rsync
gid = rsync
port = 873
fake super = yes
use chroot = no
max connections = 200
timeout = 600
ignore errors
read only = false
list = false
hosts allow = 0.0.0.0/0
auth users = rsync_backup
secrets file = /home/rongda/rsync/rsync.passwd
log file = /home/rongda/rsync/log/rsyncd.log
pid file = /home/rongda/rsync/run/rsyncd.pid
lock file = /home/rongda/rsync/run/rsyncd.lock
#####################################
[backup]
path = /backup
#3.初始化的操作
3.1）创建Rsync进程运行的用户 （ rsync ）
[root@backup ~]# useradd -M -s /sbin/nologin rsync
[root@backup ~]# id rsync
uid=1008(rsync) gid=1008(rsync) 组=1008(rsync)
3.2) 定义虚拟用户名rsync_bakcup的密码.
mkdir /home/rongda/rsync/
echo "rsync_backup:123456" > /home/rongda/rsync/rsync.passwd
chmod 600 /home/rongda/rsync/rsync.passwd
3.3) 创建存储的数据的目录 /backup
[root@backup ~]# mkdir /backup
[root@backup ~]# chown -R rsync.rsync /backup/
3.4)创建启动时的日志目录及存放进程号的目录
mkdir /home/rongda/rsync/log/
mkdir /home/rongda/rsync/run
#4.启动并检测
[root@backup ~]# systemctl start rsyncd
[root@backup ~]# systemctl enable rsyncd
[root@backup ~]# systemctl status rsyncd
[root@backup ~]# netstat -lntp |grep 873
#5.客户端可进行一波测试
无需安装此服务
mkdir /home/rongda/rsync/
echo "123456" > /home/rongda/rsync/rsync.passwd
chmod 600 /home/rongda/rsync/rsync.passwd
[root@nfs ~]# rsync -avz --password-file=/home/rongda/rsync/rsync.passwd $Path rsync_backup@172.16.10.65::backup
二、客户端上传并推送数据
1.编写脚本:备份的脚本 vim scripts.sh
#!/usr/bin/bash
#0.将目录需要用的IP，时间，定义为变量
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin
Addr=$(hostname -I |awk '{print $1}')
Time=$(date +%F)
Path=/backup
DirName="${Addr}_${Time}"
current_time=`date +%F-%H-%M`
#准备一个目录
mkdir -p "$Path/$DirName.full"
#将文件备份到该目录中-mongodb
/home/rongda/mongodb/bin/mongodump -h 192.168.254.101:27017 --authenticationDatabase admin -o $Path/$DirName.full/mongodb > /dev/null 2>&1
cp -rp /home/rongda/mongodb/data/ $Path/$DirName.full/mongodb/ > /dev/null 2>&1
#备份mysql的数据
mkdir $Path/$DirName.full/mysql/
#/usr/bin/mysqldump -uib_test -piB_test2019 ib_test >>$Path/$DirName.full/mysql/backupdb.sql > /dev/null 2>&1
/usr/bin/mysqldump -uroot -pGotoipo20\!9 ib_test >>$Path/$DirName.full/mysql/ib_test.sql
#备份fastdfs的数据
#cp -rp /home/rongda/fastdfs/file $Path/$DirName.full/fastdfs/ > /dev/null 2>&1
mkdir $Path/$DirName.full/fastdfs/ -p
cp -rp /home/rongda/fastdfs/file $Path/$DirName.full/fastdfs/
#备份es的数据
cd /home/rongda/backup/es/
/bin/sh esdump.sh && \
mkdir $Path/$DirName.full/es/ && \
cp *.tar.gz $Path/$DirName.full/es/ > /dev/null 2>&1 && \
rm -fr /home/rongda/backup/es/esdump-$current_time*.tar.gz && \
rm -fr /home/rongda/backup/es/esdump-$current_time && \
cp -rp /home/rongda/elasticsearch-2.4.5/data $Path/$DirName.full/es/ > /dev/null 2>&1
#使用rsync命令进行远程传输
#rsync -avz --password-file=/home/rongda/rsync/rsync.passwd $Path/$DirName.full rsync_backup@192.168.254.102::backup
2.推送的脚本
[root@rongda rongda]# cat scripts_push.sh
#!/usr/bin/bash
#0.将目录需要用的IP，时间，定义为变量
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin
Addr=$(hostname -I |awk '{print $1}')
Time=$(date +%F)
Path=/backup
DirName="${Addr}_${Time}"
current_time=`date +%F-%H-%M`
#1.我们要准备一个目录
mkdir -p $Path/check
#2.将文件目录备份到该目录下
tar czf $Path/$DirName.full.tar.gz $Path/$DirName.full && \
md5sum $Path/$DirName.full.tar.gz > $Path/check/check_flag_$Time
rm -fr $Path/$DirName.full
#3.将文件进行传输使用rsync命令进行远程传输
rsync -avz --password-file=/home/rongda/rsync/rsync.passwd $Path/$DirName.full.tar.gz rsync_backup@192.168.254.102::backup
rsync -avz --password-file=/home/rongda/rsync/rsync.passwd $Path/check/check_flag_$Time rsync_backup@192.168.254.102::backup
3.编写定时任务：
[root@nfs backup]# crontab -l
#backup
* 01 * * * /usr/bin/sh /home/rongda/scripts.sh &>/dev/null
#push
* 02 * * * /usr/bin/sh /home/rongda/scripts_push.sh &>/dev/null
三、服务端检查客户端传过来的数据是否是完整的
[root@rongda rongda]# cat script_check_data.sh
#!/usr/bin/bash
#检查客户端推送过来的数据是否是完整的
#将目录定义为时间变量
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin
Time=$(date +%F)
Path=/backup
#1.找到当天的check_flag文件,校验数据是否是完整的，并且将校验结果存储至某个文件中
mkdir $Path/check_result
find $Path/ -type f -name "check_flag_$Time" | xargs md5sum -c > $Path/check_result/result_$Time
#2.将推送过来的校验值删除掉
find $Path/ -type f -name "check_flag_$Time" | xargs rm -fr
#2.服务端仅保留6个月的备份数据,其余的全部删除
#find $Path -type d -mtime +180 | xargs rm -rf
编写定时任务：
[root@rongda rongda]# crontab -l
#check
* 04 * * * /usr/bin/sh /home/rongda/script_check_data.sh &>/dev/null
===========================================
es部分的备份
cd /home/rongda/backup/es
vim esdump.sh
#!/bin/bash
es_host=192.168.254.101
es_port=9200
es_index=wisdom_doc
#scp_url=192.168.6.230:/home/
current_time=`date +%F-%H-%M`
mkdir ./esdump-$current_time
/home/rongda/backup/es/esdump dump --host $es_host --port $es_port --index $es_index --directory /home/rongda/backup/es/esdump-$current_time --chunk 500 --format bulk_indexing
tar -zcf esdump-$current_time.tar.gz esdump-$current_time
#scp esdump-$current_time.tar.gz $scp_url
