kubernetes实战案例之基于ELK实现容器日志收集与展示实现pod中日志收集之至ELK，自定义字段数据格式转换、排序、基于日志实现pod自愈、自动扩容等
收集日志的方法：
1、通过DaemonSet在宿主机启动日志收集客户端Fluentd、logstash、filebeat
-v 选项把宿主机的日志挂载容器里，在容器把日志收集到ES
配置较多
2、手动安装日志收集客户端
手动配置
3、在pod内置一个filebeat
配置简单
在容器里面启动一个filebeat进程
在pod再启动一个filebeat容器
实战:存取取数据库filebeat -- redis -- logstat -- ELK
1.安装es服务器和插件： https://www.cnblogs.com/Yuanbangchen/p/16984995.html
2.安装redis服务器：https://www.cnblogs.com/Yuanbangchen/p/17275460.html
3-7步骤参考:https://www.cnblogs.com/Yuanbangchen/p/17264215.html
3.镜像构建，对初始镜像进行基础软件安装,此镜像为nginx tomcat jdk的基本镜像 安装日志收集工具filebeat
[root@localhost7C centos]# ll
-rw-r--r-- 1 root root 174 4月 7 2020 build-command.sh
-rw-r--r-- 1 root root 503 4月 9 2020 Dockerfile
-rw-r--r-- 1 root root 24694569 4月 7 2020 filebeat-7.6.1-x86_64.rpm
[root@localhost7C centos]# cat Dockerfile
#自定义Centos 基础镜像
from from harbor.zzhz.com/baseimages/centos:7.6.1810
LABEL MAINTAINER="2973707860@qq.com"
ADD filebeat-7.6.1-x86_64.rpm /tmp
RUN yum install -y /tmp/filebeat-7.6.1-x86_64.rpm vim wget tree lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel \
openssl openssl-devel iproute net-tools iotop && rm -rf /etc/localtime /tmp/filebeat-7.6.1-x86_64.rpm \
&& ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && useradd nginx -u 2019 && useradd www -u 2020
#harbor先创建/baseimages项目
[root@localhost7C centos]# cat build-command.sh
#!/bin/bash
docker build -t harbor.zzhz.com/baseimages/magedu-centos-base:7.6.1810 .
docker push harbor.zzhz.com/baseimages/magedu-centos-base:7.6.1810
#执行构造
[root@localhost7C centos]# chmod +x build-command.sh
[root@localhost7C centos]# ./build-command.sh
4.基于基础的centos镜像 harbor.zzhz.com/baseimages/magedu-centos-base:7.6.1810，制作公司内部基础镜像--jdk镜像
[root@localhost7C jdk-1.8.212]# ll
-rw-r--r-- 1 root root 138 3月 28 15:38 build-command.sh
-rw-r--r-- 1 root root 388 3月 28 15:36 Dockerfile
-rw-r--r-- 1 root root 195013152 4月 7 2020 jdk-8u212-linux-x64.tar.gz
-rw-r--r-- 1 root root 2128 3月 28 15:39 profile
[root@localhost7C jdk-1.8.212]# cat Dockerfile
#JDK Base Image
FROM harbor.zzhz.com/baseimages/magedu-centos-base:7.6.1810
MAINTAINER zhangshijie "zhangshijie@magedu.net"
ADD jdk-8u212-linux-x64.tar.gz /usr/local/src/
RUN ln -sv /usr/local/src/jdk1.8.0_212 /usr/local/jdk
ADD profile /etc/profile
ENV JAVA_HOME /usr/local/jdk
ENV JRE_HOME $JAVA_HOME/jre
ENV CLASSPATH $JAVA_HOME/lib/:$JRE_HOME/lib/
ENV PATH $PATH:$JAVA_HOME/bin
[root@localhost7C jdk-1.8.212]# cat build-command.sh
#!/bin/bash
docker build -t harbor.zzhz.com/pub-images/jdk-base:v8.212 .
sleep 1
docker push harbor.zzhz.com/pub-images/jdk-base:v8.212
#profile文件的来源：使用docker 运行一个临时容器复制出来。
[root@localhost7C jdk-1.8.212]# cat profile
....
....
....
unset i
unset -f pathmunge
export LANG=en_US.UTF-8
export HISTTIMEFORMAT="%F %T `whoami` "
#主要是这几行。
export JAVA_HOME=/usr/local/jdk
export TOMCAT_HOME=/apps/tomcat
export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$TOMCAT_HOME/bin:$PATH
export CLASSPATH=.$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib:$JAVA_HOME/lib/tools.jar
#执行构造
[root@localhost7C jdk-1.8.212]# chmod +x build-command.sh
[root@localhost7C jdk-1.8.212]# ./build-command.sh
#测试
[root@localhost7C jdk-1.8.212]# docker run -it --rm harbor.zzhz.com/pub-images/jdk-base:v8.212 bash
[root@5cd43e965b82 /]# java -version
java version "1.8.0_212"
Java(TM) SE Runtime Environment (build 1.8.0_212-b10)
Java HotSpot(TM) 64-Bit Server VM (build 25.212-b10, mixed mode)
5.基于jdk镜像制作tomcat基础镜像
[root@localhost7C tomcat-base-8.5.43]# ll
-rw-r--r-- 1 root root 9717059 4月 7 2020 apache-tomcat-8.5.43.tar.gz
-rw-r--r-- 1 root root 146 3月 28 15:48 build-command.sh
-rw-r--r-- 1 root root 341 3月 28 15:48 Dockerfile
[root@localhost7C tomcat-base-8.5.43]# cat Dockerfile
#Tomcat 8.5.43基础镜像
FROM harbor.zzhz.com/pub-images/jdk-base:v8.212
MAINTAINER zhangshijie "zhangshijie@magedu.net"
RUN mkdir /apps /data/tomcat/webapps /data/tomcat/logs -pv
ADD apache-tomcat-8.5.43.tar.gz /apps
RUN useradd tomcat -u 2021 && ln -sv /apps/apache-tomcat-8.5.43 /apps/tomcat && chown -R nginx.nginx /apps /data -R
[root@localhost7C tomcat-base-8.5.43]# cat build-command.sh
#!/bin/bash
docker build -t harbor.zzhz.com/pub-images/tomcat-base:v8.5.43 .
sleep 3
docker push harbor.zzhz.com/pub-images/tomcat-base:v8.5.43
[root@localhost7C tomcat-base-8.5.43]# chmod +x build-command.sh
[root@localhost7C tomcat-base-8.5.43]# ./build-command.sh
#测试访问tomcat基础镜像启动为容器：
[root@localhost7C tomcat-base-8.5.43]# docker run -it --rm -p 8801:8080 harbor.zzhz.com/pub-images/tomcat-base:v8.5.43 bash
[root@cfb5972b39ac /]# /apps/tomcat/bin/catalina.sh start
Using CATALINA_BASE: /apps/tomcat
Using CATALINA_HOME: /apps/tomcat
Using CATALINA_TMPDIR: /apps/tomcat/temp
Using JRE_HOME: /usr/local/jdk/jre
Using CLASSPATH: /apps/tomcat/bin/bootstrap.jar:/apps/tomcat/bin/tomcat-juli.jar
Tomcat started.
[root@cfb5972b39ac /]# ps aux
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
root 1 0.0 0.1 11824 1924 pts/0 Ss 15:52 0:00 bash
root 23 63.4 4.9 3999372 91544 pts/0 Sl 15:53 0:03 /usr/local/jdk/jre/bin/java -Djava.util.logging.config.file=/apps/tomcat/conf/logging.properties -Djava.util.logging.manage
root 73 0.0 0.0 51748 1732 pts/0 R+ 15:53 0:00 ps aux
#测试
[root@localhost7K ~]# curl 192.168.80.120:8801
6.tomcat业务镜像：tomcat业务镜像app1制作,后期按此步骤制作app2、appN镜像
[root@localhost7C linux39]# cd tomcat-app1/
[root@localhost7C tomcat-app1]# ll
-rwxr-xr-x 1 root root 165 4月 7 2020 build-command.sh
-rwxr-xr-x 1 root root 23611 4月 7 2020 catalina.sh
-rw-r--r-- 1 root root 548 4月 9 2020 Dockerfile
-rw-r--r-- 1 root root 416 4月 9 2020 filebeat.yml
-rw-r--r-- 1 root root 12 4月 7 2020 index.html
-rw-r--r-- 1 root root 266 4月 9 2020 myapp.tar.gz
-rwxr-xr-x 1 root root 556 4月 9 2020 run_tomcat.sh
-rw-r--r-- 1 root root 6460 4月 7 2020 server.xml
[root@localhost7C tomcat-app1]# cat Dockerfile
#tomcat web1
FROM harbor.zzhz.com/pub-images/tomcat-base:v8.5.43
ADD catalina.sh /apps/tomcat/bin/catalina.sh
ADD server.xml /apps/tomcat/conf/server.xml
#ADD myapp/* /data/tomcat/webapps/myapp/
ADD myapp.tar.gz /data/tomcat/webapps/myapp/
ADD run_tomcat.sh /apps/tomcat/bin/run_tomcat.sh
ADD filebeat.yml /etc/filebeat/filebeat.yml
RUN mkdir /data/tomcat/webapps/myapp/images /data/tomcat/webapps/myapp/static -p
RUN chown -R nginx.nginx /data/ /apps/
EXPOSE 8080 8443
CMD ["/apps/tomcat/bin/run_tomcat.sh"]
[root@localhost7C tomcat-app1]# cat index.html
tomcat app1
#收集工具配置文件
[root@localhost7C tomcat-app1]# cat filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /apps/tomcat/logs/catalina.out
fields:
type: k8s-tomcat-catalina
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 1
output.redis:
hosts: ["srv-devops-redis.magedu.svc.zzhz.local:6379"] #redis地址
key: "k8s-linux39-myapp"
db: 1
timeout: 5
password: "123456"
#启动文件
[root@localhost7C tomcat-app1]# cat run_tomcat.sh
#!/bin/bash
/usr/share/filebeat/bin/filebeat -e -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat &
su - nginx -c "/apps/tomcat/bin/catalina.sh start"
tail -f /etc/hosts
[root@localhost7C tomcat-app1]# cat server.xml
其它省略
<Host name="localhost" appBase="/data/tomcat/webapps" unpackWARs="true" autoDeploy="true">
[root@localhost7C tomcat-app1]# cat build-command.sh
#!/bin/bash
TAG=$1
docker build -t harbor.zzhz.com/linux39/tomcat-app1:${TAG} .
sleep 3
docker push harbor.zzhz.com/linux39/tomcat-app1:${TAG}
#重点，所有脚本要执行权限。
[root@localhost7C tomcat-app1]# chmod +x *.sh
[root@localhost7C tomcat-app1]# ./build-command.sh 2022-02-22
#测试访问tomcat基础镜像启动为容器：
[root@localhost7C tomcat-app1]#docker run -it --rm -p 8801:8080 harbor.zzhz.com/linux39/tomcat-app1:2022-02-22
[root@localhost7K ~]# curl 192.168.80.120:8801/myapp/index.html
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>马哥教育</title>
</head>
<body>
<h1>linux39 myapp v1</h1>
<h1>linux39 myapp v2</h1>
</body>
</html>
7.在k8s环境创建tomcat业务pod
[root@localhost7C tomcat-app1]# cat tomcat-app1.yaml
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
labels:
app: linux39-tomcat-app1-deployment-label
name: linux39-tomcat-app1-deployment
namespace: linux39
spec:
replicas: 1
selector:
matchLabels:
app: linux39-tomcat-app1-selector
template:
metadata:
labels:
app: linux39-tomcat-app1-selector
spec:
containers:
- name: linux39-tomcat-app1-container
image: harbor.zzhz.com/linux39/tomcat-app1:2022-04-14
#command: ["/apps/tomcat/bin/run_tomcat.sh"]
#imagePullPolicy: IfNotPresent
imagePullPolicy: Always
ports:
- containerPort: 8080
protocol: TCP
name: http
env:
- name: "password"
value: "123456"
- name: "age"
value: "18"
resources:
limits:
cpu: 1
memory: "512Mi"
requests:
cpu: 500m
memory: "512Mi"
volumeMounts:
- name: linux39-images
mountPath: /data/tomcat/webapps/myapp/images
readOnly: false
- name: linux39-static
mountPath: /data/tomcat/webapps/myapp/static
readOnly: false
volumes:
- name: linux39-images
nfs:
server: 192.168.80.110
path: /data/linux39/images
- name: linux39-static
nfs:
server: 192.168.80.110
path: /data/linux39/static
#nodeSelector:
# project: linux39
---
kind: Service
apiVersion: v1
metadata:
labels:
app: linux39-tomcat-app1-service-label
name: linux39-tomcat-app1-service
namespace: linux39
spec:
type: NodePort
ports:
- name: http
port: 80
protocol: TCP
targetPort: 8080
nodePort: 30003
selector:
app: linux39-tomcat-app1-selector
[root@localhost7C tomcat-app1]# kubectl apply -f tomcat-app1.yaml
访问测试：tomcat产生数据库
8.logstash服务器配置文件
[root@localhost7I ]# cat /etc/logstash/conf.d/log.conf
input {
redis {
host => "192.168.80.150"
port => "36379"
db => 1
password => "123456"
data_type => "list"
key => "k8s-linux39-myapp"
}
}
output {
if [fields][type] == "k8s-tomcat-catalina" {
elasticsearch {
hosts => ["192.168.80.190:9200"]
index => "k8s-linux39-tomcat-myapp-catalina-outlog-%{+YYYY.MM.dd}"
}
}
}
9测试: 使用 head cerebro kibana等工具查看。
