rancher 证书过期处理
目录
rancher 证书过期处理
rancher部署的k8s集群只有一年有效期，证书过期，k8s集群就会报错x509类的报错，不可访问，需要重新颁发证书
rancher2.3.8版本和之前，都需要更新证书，每隔一年需要更新一次，否则证书过期，rancher登录不上去，很麻烦，2.4.0版本以后解决了这个问题。
rancher v2.3版本
修改rancher所在机器时间
修改rancher-server的本地时间为最近过去的一年内任意时间
date -s "2022-03-10 14:24:34"
删除证书并重启
docker exec -it rancher rm -rf /var/lib/rancher/k3s/server/tls/*.key && \
docker exec -it rancher rm -rf /var/lib/rancher/k3s/server/tls/*.crt &&\
docker exec -it rancher rm -rf /var/lib/rancher/k3s/server/tls/dynamic-cert.json &&\
docker exec -it rancher /usr/bin/etcdctl --endpoints=127.0.0.1:2379 del /registry/secrets/kube-system/k3s-serving && \
docker restart rancher && \
docker logs -f rancher
rancher v2.4版本
exec 进入到 rancher 服务内部
kubectl --insecure-skip-tls-verify -n kube-system delete secrets k3s-serving
kubectl --insecure-skip-tls-verify delete secret serving-cert -n cattle-system
rm -f /var/lib/rancher/k3s/server/tls/dynamic-cert.json
重启 rancher-server
docker restart rancher_id
执行以下命令刷新参数
curl --insecure -sfL https://server-url/v3
过期告警
上面只是临时的解决方案，可以对证书监控起来，提前发现证书过期，通过点击rancher页面的证书颁发，避免rancher因证书问题带来的影响
check_cert_time.py
//计算过期时间的python脚本
cat check_cert_time.py
#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName :check_cert.py
# @Time :2021/12/17
# @Author :运维@小兵
# @Function :检查证书过期时间，如果在n个月后过期，则把证书和过期时间写入到文件中
# @Excute :python check_cert.py 保存过期证书信息的文件 证书文件 几个月后 证书过期时间
from datetime import datetime
import sys
def print_hello():
print("hello")
# 环境检查
def check_env():
ex = Exception('Invalid Param!!! eg:python %s 保存过期证书信息的文件 证书文件 几个月后 证书过期时间' % sys.argv[0])
if len(sys.argv) != 5:
raise ex
'''
param mon: 获取当前时间X月之后的时间
return: YYYY-MM-DD
'''
def get_date_month(mon=0):
now = datetime.now() # 当前时间
# 当前时间n个月后
last_y = int((int(now.year) * 12 + int(now.month) + mon) / 12)
last_m = (int(now.year) * 12 + int(now.month) + mon) % 12
if last_m < 10:
last_m = "0" + str(last_m)
last_d = int(now.day)
last_date = '%s-%s-%s' % (last_y, last_m, last_d)
return last_date
# 将GMT时间转为标准时间
def trans_gmt(gmt_time):
GMT_FORMAT = '%b %d %Y'
standard_time = datetime.strptime(gmt_time, GMT_FORMAT)
standard_time = standard_time.strftime("%Y-%m-%d") # 把<class 'datetime.datetime'>转为str
return standard_time
#检查k8s证书时间
def check_k8s_cert():
exceed_cert_file = sys.argv[1] # 保存过期证书信息文件
cert_file = sys.argv[2] # 证书名
after_mon = int(sys.argv[3]) # n个月后
cert_exceed_time = sys.argv[4] # 证书过期时间(GMT格式)
cert_exceed_time = trans_gmt(cert_exceed_time)
cert_exceed_time = int(cert_exceed_time.replace('-', '')) # 转成整型，如20220318
after_mon_time = get_date_month(after_mon)
after_mon_time = int(after_mon_time.replace('-', '')) # n个月后的时间
if cert_exceed_time <= after_mon_time:
with open(exceed_cert_file, 'a') as f:
f.write("%s 过期时间:%s\n" % (cert_file, cert_exceed_time))
#print('WARN:证书%s将在%s过期' % (cert_file,cert_exceed_time))
if __name__ == '__main__':
try:
check_env()
check_k8s_cert()
except Exception as e:
print('ERROR:%s' % e)
check_k8s_cert()
check_rancher_cert.sh
注意系统时间必须是正确的，否则企业微信的curl会报错
//获取证书位置并调用上面的python脚本检查证书通知企业微信群
cat check_rancher_cert.sh
#!/bin/bash
# vim:sw=4:ts=4:et
set -e
WORKDIR=$(cd `dirname $0`;pwd)
BEFORE_MONTH=3 #距证书过期提前三个月通知
EXCEED_CERT_PATH="${WORKDIR}/exceed_cert_file.txt" #存放过期的证书文件
TIME=$(date "+%Y-%m-%d %H:%M")
Check_Env(){
if [[ ! -f ${WORKDIR}/check_cert_time.py ]];then
echo "ERROR:${WORKDIR}/check_cert_time.py Not Found"
fi
}
#获取过期证书，并写入${EXCEED_CERT_PATH}
Get_Exceed_Cert(){
# local docker_root_dir=$(docker info | grep '^Docker Root Dir:' | awk -F': ' '{print $2}') #docker存储路径
# [[ ! -d ${docker_root_dir} ]] && echo "ERROR:Docker Root Dir:${docker_root_dir} Not Found" && exit 1
# local cert_path=$(find ${docker_root_dir} -name 'serving-kube-apiserver.crt' -type f | grep tls)
# if [[ $(echo ${cert_path} | wc -l) -ne 1 ]];then
# echo "ERROR:${cert_path} Is Error" && exit 1
# fi
# echo "cert path is ${cert_path}"
# local cert_dir=${cert_path%/*} #证书所在目录
local cert_dir="/tmp/" #证书所在目录
echo "cert dir is ${cert_dir}"
[[ ! -d ${cert_dir} ]] && echo "ERROR:Docker Root Dir:${cert_dir} Not Found" && exit 1
echo "证书目录:${cert_dir}"
cd ${cert_dir}
echo "=====================now time:${TIME}=====================" > ${EXCEED_CERT_PATH}
for name in `ls *.crt`
do
local cert_time_info=$(openssl x509 -enddate -noout -in ${name}) #如notAfter=May 26 06:27:49 2022 GMT
local cert_exceed_time=$(echo ${cert_time_info} | awk -F= '{print $2}' | awk '{printf"%s %s %s\n",$1,$2,$4}') #获取到证书过期时间，如May 26 2022
python ${WORKDIR}/check_cert_time.py ${EXCEED_CERT_PATH} ${name} ${BEFORE_MONTH} "${cert_exceed_time}"
done
# python ${WORKDIR}/check_cert_time.py ${EXCEED_CERT_PATH} "a.crt" ${BEFORE_MONTH} "Mar 18 2022"
# python ${WORKDIR}/check_cert_time.py ${EXCEED_CERT_PATH} "b.crt" ${BEFORE_MONTH} "Mar 18 2022"
}
#企业微信通知
Vx_Notice(){
local vx_url=$1
local exceed_time=$2
local nowTime=`ssh root@192.168.48.142 "date +%Y%m%d"`
local host_ip=$(ip addr |awk '/inet /' |sed -n '2p' |awk -F' ' '{print $2}' |awk -F'/' '{print $1}')
curl "${vx_url}" -H 'Content-Type: application/json' -d '{"msgtype": "text","text": {"content":"192.168.48.142 rancher机器当前系统时间为'${nowTime}',证书到期时间为'${exceed_time}'"}}'
# curl "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=fbb3b302-28a8-484d-ada9-06f2088190fa" \
# -H 'Content-Type: application/json' \
# -d '{"msgtype": "text",
# "text": {
# "content": "'${host_ip}' Rancher following certificates will expire in '${exceed_time}'\nPlease Check '${EXCEED_CERT_PATH}'",
# "mentioned_mobile_list":["@all"]}
# }'
}
#检查证书
Check_Cert_Time(){
echo "INFO:Begin Check Rancher Cert Exceed Time..."
Check_Env
Get_Exceed_Cert
[[ ! -f ${EXCEED_CERT_PATH} ]] && echo "ERROR:${EXCEED_CERT_PATH} Not Found" && exit 1
if [[ $(cat ${EXCEED_CERT_PATH} | wc -l) -gt 1 ]];then
if [[ -n $1 ]];then
local vx_url=$1
if ! echo ${vx_url} | grep "https://qyapi.weixin.qq.com/" &> /dev/null;then
echo "ERROR:Vx Url ${vx_url} Is Error" && exit 1
fi
local exceed_time=$(cat ${EXCEED_CERT_PATH} | awk -F'[ :]' '/crt/{print $3}' | sed -n '1p')
echo ${vx_url}
echo "liwenchao"
echo ${exceed_time}
Vx_Notice ${vx_url} ${exceed_time}
fi
echo -e "\033[33mWARN:The following certificates will expire in ${BEFORE_MONTH} months\033[0m"
cat ${EXCEED_CERT_PATH}
exit 1
else
echo "INFO:Rancher Cert Is Ok" | tee -a ${EXCEED_CERT_PATH} && exit 0
fi
}
[[ $# -gt 1 ]] && echo "ERROR:Invalid Param!!!,Please Excute:bash $0 <vx_url>" && exit 1
Check_Cert_Time $1
添加计划任务
crontab -e
//编辑自动计划任务
30 9 * * * /opt/chenck_rancher/check_rancher_cert.sh https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=****
或者用jenkins分发这个脚本执行也可以，这里不做演示了
