DaemonSet挂载所有节点日志目录通过crontab任务清理日志及对接钉钉通知
1,准备清理脚本
方法一：
!/bin/bash
webhook='https://oapi.dingtalk.com/robot/send?access_token=777ca5d78ade47dc3d51b1034acfdcea1d05eddf6e5224bd10dc6979da57289b'
Date1=$(date "+%Y-%m-%d_%H%S%M")
Date=$(date "+%Y-%m-%d_%H%M%S")
Dir='/tmp/'
ip=$HOST_IP #pod里边定义
du -sh /beta/logs/ | awk '{print $1}' > $Dir$Date
#删除三天以前的日志文件
find /beta/logs/ -name '*.log' -type f -mtime +3 -print -exec rm -rm {} \;
#删除三天以前的空目录
find /beta/logs/ -maxdepth 2 -type d -empty -print -exec rm -rf {} \;
du -sh /beta/logs/ | awk '{print $1}' > $Dir$Date1
curl $webhook -H 'Content-Type: application/json' -d "
{
'msgtype': 'text',
'text': {
'content': '[比心][比心]测试环境k8s空间清理[比心][比心]
\n ++++++[对勾]${HOST_IP}清理前空间[对勾]+++++++
/beta/logs: `cat $Dir$Date`
\n++++++[对勾]${HOST_IP}清理后空间[对勾]+++++++
/beta/logs: `cat $Dir$Date1`\n'
},
'at': {
'isAtAll': true
}
}"
方法二：
#!/bin/bash
dir='/beta/logs/'
cd $dir | ls -l | awk '{print $NF}' | sed 1d | grep -v $0 | while read line; do
#echo $dir$line
cd $dir$line
while [ $line = ${PWD##*/} ]
do
#find ./ -maxdepth 1 -type d -mtime +30 | xargs rm -rf
&& pwd
break
done
cd ..
done
2，创建基础镜像
FROM martonyang/centos7.8.2003
RUN yum -y install crontabs
USER root
WORKDIR /beta/logs/
COPY ClearLog.sh /tmp/
RUN chmod +x /tmp/ClearLog.sh \
&& echo "30 18 * * * /bin/bash /tmp/ClearLog.sh >> /tmp/tmp.txt" >> conf \
&& crontab conf && rm -f conf
RUN echo "Asia/Shanghai" > /etc/timezone
CMD ["/usr/sbin/crond","-n"] # -i 后台运行
3，推送到harbor仓库
docker build -t 172.17.12.79:80/tools/clearlog:v2 .
docker push 172.17.12.79:80/tools/clearlog:v2
docker run --name=test-aa --rm -it 172.17.12.79:80/tools/clearlog:v1 bash #本地测试
4,创建k8s yaml文件
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: clearlog
namespace: default
labels:
app: clearlog
spec:
selector:
matchLabels:
app: clearlog
template:
metadata:
labels:
app: clearlog
spec:
volumes:
- name: hostlog
hostPath:
path: /beta/logs #准备挂载的node上的文件系统
type: Directory
defaultMode: 420
containers:
- env:
- name: HOST_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.hostIP
name: clear-client
image: 172.17.12.79:80/tools/clearlog:v1
resources:
limits:
cpu: 500m
memory: 2Gi
requests:
cpu: 200m
memory: 500Mi
volumeMounts:
- name: hostlog
mountPath: /beta/logs #容器挂载点
lifecycle: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
imagePullPolicy: IfNotPresent
restartPolicy: Always
terminationGracePeriodSeconds: 30
dnsPolicy: ClusterFirst
hostNetwork: true
securityContext:
seLinuxOptions: {}
imagePullSecrets:
- name: betasecret
schedulerName: default-scheduler
dnsConfig: {}
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
minReadySeconds: 60
revisionHistoryLimit: 10
成功在所有节点运行：
成功对接钉钉告警
build镜像错误整理：
. 只是想自己写个 dockerfile 用 docker 来部署 crontab 的服务。
dockerfile 中要运行多行命令 于是我写成
RUN 命令a
&& 命令b
&& 命令c
build 就报错如题。
2. 解决办法： 某些情况下似乎 RUN 中并不识别换行符 \ 和连接符 && 。只是直接去掉这 2 种符号的使用就正常了。
++++++++++++++++++++++++++++++
shell脚本放在configmap中实现
1，创建基础镜像
vim 1
FROM martonyang/centos7.8.2003
RUN yum -y install crontabs
USER root
WORKDIR /beta/logs/
RUN echo "Asia/Shanghai" > /etc/timezone
CMD ["/usr/sbin/crond","-n"] # -i 后台运行
docker build -t harbor.betawm.com/tools/clearlog:v7 -f 1 .
docker push harbor.betawm.com/tools/clearlog:v7
准备configmap文件和daemonset文件
apiVersion: v1
data:
ClearLog.sh: |-
#!/bin/bash
webhook='https://oapi.dingtalk.com/robot/send?access_token=777ca5d78ade47dc3d51b1034acfdcea1d05eddf6e5224bd10dc6979da57289b'
Date=$(date "+%Y-%m-%d_%H%S%M")
Date1=$(date "+%Y-%m-%d_%H%M%S")
Dir='/tmp/'
ip=$HOST_IP #pod里边定义
du -sh /beta/logs/ | awk '{print $1}' > $Dir$Date
#删除三天以前的日志文件
find /beta/logs/ -name '*.log' -type f -mtime +3 -print -exec ls -l {} \;
#删除三天以前的空目录
find /beta/logs/ -maxdepth 2 -type d -empty -print -exec ls -l {} \;
Date1=$(date "+%Y-%m-%d_%H%M%S")
du -sh /beta/logs/ | awk '{print $1}' > $Dir$Date1
curl $webhook -H 'Content-Type: application/json' -d "
{
'msgtype': 'text',
'text': {
'content': '[比心][比心]生产环境k8s空间清理[比心][比心]
\n ++++++[对勾]${HOST_IP}清理前空间[对勾]+++++++
/beta/logs: `cat $Dir$Date`
\n++++++[对勾]${HOST_IP}清理后空间[对勾]+++++++
/beta/logs: `cat $Dir$Date1`\n'
},
'at': {
'isAtAll': true
}
}"
kind: ConfigMap
metadata:
name: clearlogconf
namespace: default
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
annotations:
deprecated.daemonset.template.generation: '13'
labels:
app: clearlog
name: clearlog
namespace: default
resourceVersion: '10612247'
spec:
minReadySeconds: 60
revisionHistoryLimit: 10
selector:
matchLabels:
app: clearlog
template:
metadata:
annotations:
kubectl.kubernetes.io/restartedAt: '2021-09-29T18:14:54+08:00'
creationTimestamp: null
labels:
app: clearlog
spec:
containers:
- env:
- name: HOST_IP #赋值给指定变量，在pod中调用此环境变量返回的就是当前物理机的IP地址。
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.hostIP #获取当前节点的物理地址
image: 'harbor.betawm.com/tools/clearlog:v7'
imagePullPolicy: IfNotPresent
lifecycle:
postStart:
exec:
command:
- /bin/bash
- '-c'
- ' echo "30 18 * * * /bin/bash /tmp/ClearLog.sh >> /tmp/tmp.txt" >> conf && crontab conf && rm -f conf '
name: clear-client
resources:
limits:
cpu: 500m
memory: 2Gi
requests:
cpu: 200m
memory: 500Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /beta/logs
name: hostlog
- mountPath: /tmp/ClearLog.sh
name: clear-log-config
subPath: ClearLog.sh
dnsConfig: {}
dnsPolicy: ClusterFirst
hostNetwork: true
imagePullSecrets:
- name: betasecret
restartPolicy: Always
schedulerName: default-scheduler
securityContext:
seLinuxOptions: {}
terminationGracePeriodSeconds: 30
volumes:
- hostPath:
path: /beta/logs
type: Directory
name: hostlog
- configMap:
defaultMode: 420
name: clearlogconf
name: clear-log-config
updateStrategy:
rollingUpdate:
maxUnavailable: 1
type: RollingUpdate
pod启动命令放到configmap中
apiVersion: v1
data:
ClearLog.sh: |-
#!/bin/bash
webhook='https://oapi.dingtalk.com/robot/send?access_token=777ca5d78ade47dc3d51b1034acfdcea1d05eddf6e5224bd10dc6979da57289b'
Date=$(date "+%Y-%m-%d_%H%S%M")
Date1=$(date "+%Y-%m-%d_%H%M%S")
Dir='/tmp/'
ip=$HOST_IP #pod里边定义
du -sh /beta/logs/ | awk '{print $1}' > $Dir$Date
#删除三天以前的日志文件
find /beta/logs/ -name '*.log' -type f -mtime +3 -print -exec rm -rf {} \;
#删除三天以前的空目录
find /beta/logs/ -maxdepth 2 -type d -empty -print -exec rm -rf {} \;
Date1=$(date "+%Y-%m-%d_%H%M%S")
du -sh /beta/logs/ | awk '{print $1}' > $Dir$Date1
curl $webhook -H 'Content-Type: application/json' -d "
{
'msgtype': 'text',
'text': {
'content': '[比心][比心]开发环境k8s空间清理[比心][比心]
\n ++++++[对勾]${HOST_IP}清理前空间[对勾]+++++++
/beta/logs: `cat $Dir$Date`
\n++++++[对勾]${HOST_IP}清理后空间[对勾]+++++++
/beta/logs: `cat $Dir$Date1`\n'
},
'at': {
'isAtAll': true
}
}"
kind: ConfigMap
metadata:
name: clearlogconf
namespace: default
---
apiVersion: v1
data:
root: |-
30 18 * * * /bin/bash /tmp/ClearLog.sh >> /tmp/tmp.txt
kind: ConfigMap
metadata:
name: crontabconf
namespace: default
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
annotations:
deprecated.daemonset.template.generation: '13'
labels:
app: clearlog
name: clearlog
namespace: default
resourceVersion: '10612247'
spec:
minReadySeconds: 60
revisionHistoryLimit: 10
selector:
matchLabels:
app: clearlog
template:
metadata:
annotations:
kubectl.kubernetes.io/restartedAt: '2021-09-29T18:14:54+08:00'
creationTimestamp: null
labels:
app: clearlog
spec:
containers:
- env:
- name: HOST_IP #赋值给指定变量，在pod中调用此环境变量返回的就是当前物理机的IP地址。
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.hostIP #获取当前节点的物理地址
image: 'harbor.betawm.com/tools/clearlog:v7'
imagePullPolicy: IfNotPresent
# lifecycle: #configmap资源crontabconf替代
# postStart:
# exec:
# command:
# - /bin/bash
# - '-c'
# - ' echo "30 18 * * * /bin/bash /tmp/ClearLog.sh >> /tmp/tmp.txt" >> conf && crontab conf && rm -f conf '
name: clear-client
resources:
limits:
cpu: 500m
memory: 2Gi
requests:
cpu: 200m
memory: 500Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /beta/logs
name: hostlog
- mountPath: /tmp/ClearLog.sh
name: clear-log-config
subPath: ClearLog.sh
- mountPath: /var/spool/cron/root
name: crontab-config
subPath: root
dnsConfig: {}
dnsPolicy: ClusterFirst
hostNetwork: true
imagePullSecrets:
- name: betasecret
restartPolicy: Always
schedulerName: default-scheduler
securityContext:
seLinuxOptions: {}
terminationGracePeriodSeconds: 30
volumes:
- hostPath:
path: /beta/logs
type: Directory
name: hostlog
- configMap:
defaultMode: 420
name: clearlogconf
name: clear-log-config
- configMap:
defaultMode: 420
name: crontabconf
name: crontab-config
updateStrategy:
rollingUpdate:
maxUnavailable: 1
type: RollingUpdate
两个configMap合并为一个，通过subPath参数调用不同的key
apiVersion: v1
data:
ClearLog.sh: |-
#!/bin/bash
webhook='https://oapi.dingtalk.com/robot/send?access_token=777ca5d78ade47dc3d51b1034acfdcea1d05eddf6e5224bd10dc6979da57289b'
Date=$(date "+%Y-%m-%d_%H%S%M")
Date1=$(date "+%Y-%m-%d_%H%M%S")
Dir='/tmp/'
ip=$HOST_IP #pod里边定义
du -sh /beta/logs/ | awk '{print $1}' > $Dir$Date
#删除三天以前的日志文件
find /beta/logs/ -name '*.log' -type f -mtime +3 -print -exec rm -rf {} \;
#删除三天以前的空目录
find /beta/logs/ -maxdepth 2 -type d -empty -print -exec rm -rf {} \;
Date1=$(date "+%Y-%m-%d_%H%M%S")
du -sh /beta/logs/ | awk '{print $1}' > $Dir$Date1
curl $webhook -H 'Content-Type: application/json' -d "
{
'msgtype': 'text',
'text': {
'content': '[比心][比心]开发环境k8s空间清理[比心][比心]
\n ++++++[对勾]${HOST_IP}清理前空间[对勾]+++++++
/beta/logs: `cat $Dir$Date`
\n++++++[对勾]${HOST_IP}清理后空间[对勾]+++++++
/beta/logs: `cat $Dir$Date1`\n'
},
'at': {
'isAtAll': true
}
}"
root: |-
30 18 * * * /bin/bash /tmp/ClearLog.sh >> /tmp/tmp.txt
kind: ConfigMap
metadata:
name: clearlogconf
namespace: default
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
annotations:
deprecated.daemonset.template.generation: '13'
labels:
app: clearlog
name: clearlog
namespace: default
resourceVersion: '10612247'
spec:
minReadySeconds: 60
revisionHistoryLimit: 10
selector:
matchLabels:
app: clearlog
template:
metadata:
annotations:
kubectl.kubernetes.io/restartedAt: '2021-09-29T18:14:54+08:00'
creationTimestamp: null
labels:
app: clearlog
spec:
containers:
- env:
- name: HOST_IP #赋值给指定变量，在pod中调用此环境变量返回的就是当前物理机的IP地址。
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.hostIP #获取当前节点的物理地址
image: 'harbor.betawm.com/tools/clearlog:v7'
imagePullPolicy: IfNotPresent
# lifecycle: #configmap资源crontabconf替代
# postStart:
# exec:
# command:
# - /bin/bash
# - '-c'
# - ' echo "30 18 * * * /bin/bash /tmp/ClearLog.sh >> /tmp/tmp.txt" >> conf && crontab conf && rm -f conf '
name: clear-client
resources:
limits:
cpu: 500m
memory: 2Gi
requests:
cpu: 200m
memory: 500Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /beta/logs
name: hostlog
- mountPath: /tmp/ClearLog.sh
name: clear-log-config
subPath: ClearLog.sh #调用configmap中Key名称
- mountPath: /var/spool/cron/root
name: clear-log-config
subPath: root #调用configmap中Key名称
dnsConfig: {}
dnsPolicy: ClusterFirst
hostNetwork: true
imagePullSecrets:
- name: betasecret
restartPolicy: Always
schedulerName: default-scheduler
securityContext:
seLinuxOptions: {}
terminationGracePeriodSeconds: 30
volumes:
- hostPath:
path: /beta/logs
type: Directory
name: hostlog
- configMap:
defaultMode: 420
name: clearlogconf
name: clear-log-config
updateStrategy:
rollingUpdate:
maxUnavailable: 1
type: RollingUpdate
5，将docker.sock挂载pod内部结合crontab实现docker images清理
在pod中执行docker命令
configmap中新增了一条crontab来清理镜像名称为none的镜像
k8s.yaml文件
apiVersion: v1
data:
ClearLog.sh: |-
#!/bin/bash
webhook='https://oapi.dingtalk.com/robot/send?access_token=777ca5d78ade47dc3d51b1034acfdcea1d05eddf6e5224bd10dc6979da57289b'
Date=$(date "+%Y-%m-%d_%H%S%M")
Date1=$(date "+%Y-%m-%d_%H%M%S")
Dir='/tmp/'
ip=$HOST_IP #pod里边定义
du -sh /beta/logs/ | awk '{print $1}' > $Dir$Date
#删除三天以前的日志文件
find /beta/logs/ -name '*.log' -type f -mtime +3 -print -exec rm -rf {} \;
#删除三天以前的空目录
find /beta/logs/ -maxdepth 2 -type d -empty -print -exec rm -rf {} \;
Date1=$(date "+%Y-%m-%d_%H%M%S")
du -sh /beta/logs/ | awk '{print $1}' > $Dir$Date1
curl $webhook -H 'Content-Type: application/json' -d "
{
'msgtype': 'text',
'text': {
'content': '[比心][比心]开发环境k8s空间清理[比心][比心]
\n ++++++[对勾]${HOST_IP}清理前空间[对勾]+++++++
/beta/logs: `cat $Dir$Date`
\n++++++[对勾]${HOST_IP}清理后空间[对勾]+++++++
/beta/logs: `cat $Dir$Date1`\n'
},
'at': {
'isAtAll': true
}
}"
root: |-
30 18 * * * /bin/bash /tmp/ClearLog.sh >> /tmp/clearlog.txt
00 23 * * * /usr/bin/docker rmi $(docker images | grep "none" | awk '{print $3}') >> /tmp/dockerimage.txt
00 23 * * * /usr/bin/docker image prune -a -f >> /tmp/dockerimage.txt
kind: ConfigMap
metadata:
name: clearlogconf
namespace: default
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
annotations:
deprecated.daemonset.template.generation: '13'
labels:
app: clearlog
name: clearlog
namespace: default
resourceVersion: '10612247'
spec:
minReadySeconds: 60
revisionHistoryLimit: 10
selector:
matchLabels:
app: clearlog
template:
metadata:
annotations:
kubectl.kubernetes.io/restartedAt: '2021-09-29T18:14:54+08:00'
creationTimestamp: null
labels:
app: clearlog
spec:
containers:
- env:
- name: HOST_IP #赋值给指定变量，在pod中调用此环境变量返回的就是当前物理机的IP地址。
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.hostIP #获取当前节点的物理地址
image: 'harbor.betawm.com/tools/clearlog:v7'
imagePullPolicy: IfNotPresent
# lifecycle: #configmap资源crontabconf替代
# postStart:
# exec:
# command:
# - /bin/bash
# - '-c'
# - ' echo "30 18 * * * /bin/bash /tmp/ClearLog.sh >> /tmp/tmp.txt" >> conf && crontab conf && rm -f conf '
name: clear-client
resources:
limits:
cpu: 500m
memory: 2Gi
requests:
cpu: 200m
memory: 500Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /beta/logs
name: hostlog
- mountPath: /tmp/ClearLog.sh
name: clear-log-config
subPath: ClearLog.sh #调用configmap中Key名称
- mountPath: /var/spool/cron/root
name: clear-log-config
subPath: root #调用configmap中Key名称
- mountPath: /var/run/docker.sock
name: socket
- mountPath: /usr/bin/docker
name: dockertools
dnsConfig: {}
dnsPolicy: ClusterFirst
hostNetwork: true
imagePullSecrets:
- name: betasecret
restartPolicy: Always
schedulerName: default-scheduler
securityContext:
seLinuxOptions: {}
terminationGracePeriodSeconds: 30
volumes:
- hostPath:
path: /beta/logs
type: Directory
name: hostlog
- configMap:
defaultMode: 420
name: clearlogconf
name: clear-log-config
- hostPath: #挂载docker进程
path: /var/run/docker.sock
type: Socket
name: socket
- hostPath: #挂载docker命令工具
path: /usr/bin/docker
type: File
name: dockertools
updateStrategy:
rollingUpdate:
maxUnavailable: 1
type: RollingUpdate
