基于S3的elastic备份脚本

下载插件并安装repository-s3

下载对应es的版本的repository-s3插件，然后解压到ES软件目录的plugins目录下。

elasticsearch.yml配置文件添加如下内容

s3.client.default.endpoint: "S3地址:端口"
s3.client.default.protocol: http

使用脚本配置S3访问账号与密码，使ES可以连接S3

所有ES节点都要配置

ES_PATH="/escls/elasticsearch-6.3.0"
s3_access_key="填入access_key"
s3_secret_key="填入secret_key"
cd ${ES_PATH}
echo ${s3_access_key}|./elasticsearch-keystore add -x -f -v 's3.client.default.access_key'
echo ${s3_secret_key}|./elasticsearch-keystore add -x -f -v 's3.client.default.secret_key'
${ES_PATH}/bin/elasticsearch-keystore list

配置完成后，轮询重启所有ES节点

重启后验证插件是否配置成功

Kibana登录查看插件是否安装成功

GET _cat/plugins?v

创建两个ES的快照仓库

PUT /_snapshot/快照仓库1
{
"type" : "s3",
"settings" : {
"bucket" : "S3桶1",
}
}

PUT /_snapshot/快照仓库2
{
"type" : "s3",
"settings" : {
"bucket" : "S3桶2",
}
}

shell备份脚本

#!/bin/bash
#功能：用于备份elasticsearch的索引快照，保留7天的备份快照。
#使用crontab每天执行一次。
#第一天~第7天在桶1每天一个快照，同时删除7天前桶2的快照。
#第八天~第14天在桶2每天一个快照，同时删除7天前桶1的快照。每14天一个轮回。
#7天前的日期
Full_snapshot(){
echo $1
B_DATA=$(date -d "7 day ago" +%F)
#脚本运行日志
LOG_FILE="/tmp/es_backup_all.log"
#运行脚本的当前时间
CUR_TIME=$(date +%F_%H-%M-%S)
#仓库名称
STORE_NAME=$1
STORE_NAME2=$2
#快照名称
SNAPSHOT_PRE="snapshot_index"
SNAPSHOT_NAME_printdoc="${SNAPSHOT_PRE}_printdoc_${CUR_TIME}"
SNAPSHOT_NAME_unicorn="${SNAPSHOT_PRE}_unicorn_${CUR_TIME}"
SNAPSHOT_NAME_insuredsingle="${SNAPSHOT_PRE}_insuredsingle_${CUR_TIME}"
SNAPSHOT_NAME_reinsurance="${SNAPSHOT_PRE}_reinsurance_${CUR_TIME}"
SNAPSHOT_NAME_other_all="${SNAPSHOT_PRE}_other_all_${CUR_TIME}"
#快照API

Snap_API="http://ES的IP地址:9600"
#curl的绝对路径
CURL_CMD="/usr/bin/curl"
#生成快照
echo "==========开始ES快照==============" >> ${LOG_FILE}
${CURL_CMD} -XPUT -uelastic:密码  "${Snap_API}/_snapshot/${STORE_NAME}/${SNAPSHOT_NAME_printdoc}?wait_for_completion=true" -H 'Content-Type: application/json' -d'{"indices": "printdoc_v*","ignore_unavailable": true,"include_global_state": true}' >> ${LOG_FILE}
echo "
=====备份索引printdoc_v*完成=====" >> ${LOG_FILE}

${CURL_CMD} -XPUT -uelastic:密码  "${Snap_API}/_snapshot/${STORE_NAME}/${SNAPSHOT_NAME_unicorn}?wait_for_completion=true" -H 'Content-Type: application/json' -d'{"indices": "unicorn_v*","ignore_unavailable": true,"include_global_state": true}' >> ${LOG_FILE}
echo "
=====备份索引unicorn_v*完成=====" >> ${LOG_FILE}


${CURL_CMD} -XPUT -uelastic:密码  "${Snap_API}/_snapshot/${STORE_NAME}/${SNAPSHOT_NAME_insuredsingle}?wait_for_completion=true" -H 'Content-Type: application/json' -d'{"indices": "insuredsingle_v*","ignore_unavailable": true,"include_global_state": true}' >> ${LOG_FILE}
echo "
=====备份索引insuredsingle_v*完成======" >> ${LOG_FILE}

${CURL_CMD} -XPUT -uelastic:密码  "${Snap_API}/_snapshot/${STORE_NAME}/${SNAPSHOT_NAME_reinsurance}?wait_for_completion=true" -H 'Content-Type: application/json' -d'{"indices": "reinsurance_v*","ignore_unavailable": true,"include_global_state": true}' >> ${LOG_FILE}
echo "
=====备份索引reinsurance_v*完成======" >> ${LOG_FILE}

${CURL_CMD} -XPUT -uelastic:密码 "${Snap_API}/_snapshot/${STORE_NAME}/${SNAPSHOT_NAME_other_all}?wait_for_completion=true" -H 'Content-Type: application/json' -d'{"indices": "*,-.*,-printdoc_v*,-unicorn_v*,-insuredsingle_v*,-reinsurance_v*","ignore_unavailable": true,"include_global_state": true}' >> ${LOG_FILE}
echo "
=====备份剩余索引完成======" >> ${LOG_FILE}

#删除7天前老的快照
for snap_name in $(${CURL_CMD} -XGET -uelastic:密码  "${Snap_API}/_snapshot/${STORE_NAME2}/_all"  | python -m json.tool | grep '"snapshot":' | awk -F'[:",]' '{print $5}'|grep ${SNAPSHOT_PRE} | grep "${B_DATA}")
do
        ${CURL_CMD} -XDELETE -uelastic:密码 "${Snap_API}/_snapshot/${STORE_NAME2}/${snap_name}"
        if [ $? -eq 0 ];then
                echo "删除快照：${snap_name} success"  >> ${LOG_FILE}
        else
               echo "删除快照：${snap_name} fail"  >> ${LOG_FILE}
        fi
done

}

es_count=`cat ~/es_count.txt `

if [ $es_count -lt 8 ] 
then
	echo $es_count
	Full_snapshot  快照库1   快照库2
	echo `expr $es_count + 1 ` > ~/es_count.txt
elif [ $es_count -ge 8 ] && [ $es_count -lt 15 ]
then
	echo $es_count
	Full_snapshot  快照库2 快照库1
	echo `expr $es_count + 1 ` > ~/es_count.txt
else	
	echo $es_count	
	echo "set 1 to es_count.txt"
    echo 1 > ~/es_count.txt	
exit;
fi