k8s部署elk告警elastalert 示例

1.制作Dockerfile

注：elastalert 、pip3更新包提前下载好，防止生成Dockerfile时，下载失败

优化后的：
FROM python:3.6.12
ENV TZ=Asia/Shanghai
ENV LANG C.UTF-8

COPY requirements.txt ./
COPY pip-21.3.1-py3-none-any.whl  ./
COPY start.sh ./
ADD elas.tar.gz ./
COPY config.yaml /usr/local/elastalert/config/config.yaml
COPY wechat_qiye_alert.py /usr/local/elastalert/elastalert_modules/wechat_qiye_alert.py
COPY test-app.yaml /usr/local/elastalert/es_rules/test-app.yaml

RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \
    chmod 777 start.sh && \
    pip3 install pip-21.3.1-py3-none-any.whl && \
    pip3 install -r requirements.txt --no-index --find-links=./elas/ 

USER root
ENTRYPOINT  ["/bin/bash" ,"-C","./start.sh" ]

View Code

FROM python:3.6.12
COPY requirements.txt ./
COPY pip-21.3.1-py3-none-any.whl  ./
RUN  pip3 install pip-21.3.1-py3-none-any.whl
ADD elas.tar.gz ./
RUN pip3 install -r requirements.txt --no-index --find-links=./elas/
COPY config.yaml /usr/local/elastalert/config/config.yaml
COPY wechat_qiye_alert.py /usr/local/elastalert/elastalert_modules/wechat_qiye_alert.py
COPY test-app.yaml /usr/local/elastalert/es_rules/test-app.yaml
ENV TZ=Asia/Shanghai 
ENV LANG C.UTF-8
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime 
COPY start.sh ./
RUN chmod 777 start.sh
USER root
ENTRYPOINT  ["/bin/bash" ,"-C","./start.sh" ]

[root@node1 Dockefile]# cat start.sh 
 #!/bin/bash
cd /usr/local/elastalert/
/usr/local/bin/python3 -m elastalert.elastalert --config config/config.yaml

docker build -t alert:v1 .

docker push alert:v1

2.k8s部署

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: alert
  namespace: kms
spec:
  storageClassName: "nfs-client"
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Gi
  volumeMode: Filesystem

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: alertconfig
  namespace: kms
data:
  config.yaml: |
    rules_folder: es_rules
    run_every:
      seconds: 30
    buffer_time:
      seconds: 40
    es_host: elasticsearch-master-headless
    es_port: 9200
    use_ssl: False
    verify_certs: False
    writeback_index: elastalert_status
    alert_time_limit:
      days: 1

---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: alerttest
  name: alerttest
  namespace: kms
spec:
  replicas: 1
  selector:
    matchLabels:
      app: alerttest
  template:
    metadata:
      labels:
        app: alerttest
    spec:
      containers:
      - image: alertnew:v2
        imagePullPolicy: IfNotPresent
        name: alert
        volumeMounts:
        - mountPath: /usr/local/elastalert/config
          name: config
        - mountPath: /usr/local/elastalert/es_rules
          name: esrules
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      volumes:
      - name: esrules
        persistentVolumeClaim:
          claimName: alert
      - configMap:
          defaultMode: 420
          name: alertconfig
        name: config

注：es_rules 里新增文件，需要通过共享盘pvc去新增，每次新增配置后需要重启pod才能生效

alertalert 原始文件信息：elasticalert安装说明 - 少年老余 - 博客园 (cnblogs.com)

以上是个人实测，

也可以参考elastalert2 方案：GitHub - jertel/elastalert2: ElastAlert 2 is a continuation of the original yelp/elastalert project. Pull requests are appreciated!

helm方式：

elastalert2 helm方式，修改成我的企业微信需求（实测OK）：

values.yaml修改：

# number of replicas to run
replicaCount: 1

# number of helm release revisions to retain
revisionHistoryLimit: 5


storageClassName: "nfs-client"
storagepvcsize: "1G"
# Default internal between alert checks against the elasticsearch datasource, in minutes
runIntervalMins: 1

# Location of directory where rules reside
rulesFolder: "/opt/elastalert/es_rules"

# Enable/disabe subdirectory scanning for rules
scanSubdirectories: true

# Default rule buffer duration, in minutes
bufferTimeMins: 15

# Amount of time to retry and deliver failed alerts (1440 minutes per day)
alertRetryLimitMins: 2880

# Default time before realerting, in minutes
realertIntervalMins: ""

# For ES 5: The name of the index which stores elastalert 2 statuses, typically elastalert_status
# For ES 6: The prefix of the names of indices which store elastalert 2 statuses, typically elastalert
#
writebackIndex: elastalert

image:
  # docker image
  repository: jertel/elastalert2
  # docker image tag
  tag: 2.5.0
  pullPolicy: IfNotPresent
  pullSecret: ""

resources: {}

# Annotations to be added to pods
podAnnotations: {}

elasticsearch:
  # elasticsearch endpoint e.g. (svc.namespace||svc)
  host: elasticsearch-master-headless
  # elasticsearch port
  port: 9200
  # whether or not to connect to es_host using TLS
  useSsl: "False"
  # Username if authenticating to ES with basic auth
  username: ""
  # Password if authenticating to ES with basic auth
  password: ""
  # Specifies an existing secret to be used for the ES username/password
  credentialsSecret: ""
  # The key in elasticsearch.credentialsSecret that stores the ES password
  credentialsSecretUsernameKey: ""
  # The key in elasticsearch.credentialsSecret that stores the ES username
  credentialsSecretPasswordKey: ""
  # whether or not to verify TLS certificates
  verifyCerts: "True"
  # Enable certificate based authentication
  # path to a PEM certificate to use as the client certificate
  # clientCert: "/certs/client.pem"
  # path to a private key file to use as the client key
  # clientKey: "/certs/client-key.pem"
  # path to a CA cert bundle to use to verify SSL connections
  # caCerts: "/certs/ca.pem"
  # # certs volumes, required to mount ssl certificates when elasticsearch has tls enabled
  # certsVolumes:
  #   - name: es-certs
  #     secret:
  #       defaultMode: 420
  #       secretName: es-certs
  # # mount certs volumes, required to mount ssl certificates when elasticsearch has tls enabled
  # certsVolumeMounts:
  #   - name: es-certs
  #     mountPath: /certs
  #     readOnly: true

# Optional env variables for the pod
optEnv: []

extraConfigOptions: {}
  # # Options to propagate to all rules, e.g. a common slack_webhook_url or kibana_url
  # # Please note at the time of implementing this value, it will not work for required_locals
  # # Which MUST be set at the rule level, these are: ['alert', 'type', 'name', 'index']
  # kibana_url: https://kibana.yourdomain.com
  # slack_webhook_url: dummy

# To load ElastAlert 2 config via secret, uncomment the line below
# secretConfigName: elastalert-config-secret

# Example of a secret config

#apiVersion: v1
#kind: Secret
#metadata:
#  name: elastalert-config-secret
#type: Opaque
#stringData:
#   elastalert_config: |-
#     rules_folder: /opt/elastalert/rules
#     scan_subdirectories: false
#     run_every:
#       minutes: 1
#     buffer_time:
#       minutes: 15
#     es_host: elasticsearch
#     es_port: 9200
#     writeback_index: elastalert
#     use_ssl: False
#     verify_certs: True
#     alert_time_limit:
#       minutes: 2880
#     slack_webhook_url: https://hooks.slack.com/services/xxxx
#     slack_channel_override: '#alerts'


# To load ElastAlert's rules via secret, uncomment the line below
#secretRulesName: elastalert-rules-secret

# Additionally, you must specificy which rules to load from the secret
#secretRulesList: [ "rule_1", "rule_2" ]

# Example of secret rules

#apiVersion: v1
#kind: Secret
#metadata:
#  name: elastalert-rules-secret
#  namespace: elastic-system
#type: Opaque
#stringData:
#   rule_1: |-
#     name: Rule 1
#     type: frequency
#     index: index1-*
#     num_events: 3
#     timeframe:
#       minutes: 1
#     alert:
#     - "slack"
#   rule_2: |-
#     name: Rule 2
#     type: frequency
#     index: index2-*
#     num_events: 5
#     timeframe:
#       minutes: 10
#     alert:
#     - "slack"

# Command and args override for container e.g. (https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/)
# command: ["YOUR_CUSTOM_COMMAND"]
# args: ["YOUR", "CUSTOM", "ARGS"]

# specifies the rules volume to be used
rulesVolumeName: "rules"

# additional rule configurations e.g. (http://elastalert2.readthedocs.io/en/latest/)
#rules: {}
  # deadman_slack: |-
  #   ---
  #   name: Deadman Switch Slack
  #   type: frequency
  #   index: containers-*
  #   num_events: 3
  #   timeframe:
  #     minutes: 3
  #   filter:
  #   - term:
  #       message: "deadmanslack"
  #   alert:
  #   - "slack"
  #   slack:
  #   slack_webhook_url: dummy
  # deadman_pagerduty: |-
  #   ---
  #   name: Deadman Switch PagerDuty
  #   type: frequency
  #   index: containers-*
  #   num_events: 3
  #   timeframe:
  #     minutes: 3
  #   filter:
  #   - term:
  #       message: "deadmanpd"
  #   alert:
  #   - "pagerduty"
  #   pagerduty:
  #   pagerduty_service_key: dummy
  #   pagerduty_client_name: ElastAlert Deadman Switch

serviceAccount:
  # Specifies whether a service account should be created
  create: true
  # Annotations to add to the service account
  annotations: {}
  # The name of the service account to use.
  # If not set and create is true, a name is generated using the fullname template
  name:

# Enable pod security policy
# https://kubernetes.io/docs/concepts/policy/pod-security-policy/
# DEPRECATED in Kubernetes 1.21 (https://kubernetes.io/blog/2021/04/06/podsecuritypolicy-deprecation-past-present-and-future/)
podSecurityPolicy:
  create: false

securityContext:
  runAsNonRoot: true
  runAsUser: 1000

podSecurityContext:
  fsGroup: 1000
  runAsUser: 1000
  runAsGroup: 1000

# Support using node selectors and tolerations
# nodeSelector:
#   "node-role.kubernetes.io/infra_worker": "true"
nodeSelector: {}

# Specify node affinity or anti-affinity specifications
affinity: {}

# tolerations:
#   - key: "node_role"
#     operator: "Equal"
#     value: "infra_worker"
#     effect: "NoSchedule"
tolerations: []

# Optional automatic SMTP mail server credential management.
# smtp_auth:
#   username: ""
#   password: ""

extraVolumes: []
  # - name: smtp-auth
  #   secret:
  #     secretName: elastalert-smtp-auth
  #     items:
  #       - key: smtp_auth.yaml
  #         path: smtp_auth.yaml
  #         mode: 0400

extraVolumeMounts: []
  # - name: smtp-auth
  #   mountPath: /opt/elastalert/config-smtp/smtp_auth.yaml
  #   subPath: smtp_auth.yaml
  #   readOnly: true

# Prometheus Exporter defined by port:
prometheusScrapeAnnotations:
  prometheus.io/scrape: "true"
  prometheus.io/path: "/"

View Code

deployment.yaml修改：

apiVersion: apps/v1
kind: Deployment
metadata:
  name: {{ template "elastalert.fullname" . }}
  labels:
    app: {{ template "elastalert.name" . }}
    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
    release: {{ .Release.Name }}
    heritage: {{ .Release.Service }}
spec:
  selector:
    matchLabels:
      app: {{ template "elastalert.name" . }}
      release: {{ .Release.Name }}
  replicas: {{ .Values.replicaCount }}
  revisionHistoryLimit: {{ .Values.revisionHistoryLimit }}
  template:
    metadata:
      annotations:
        checksum/config: {{ include (print $.Template.BasePath "/config.yaml") . | sha256sum }}
        checksum/rules: {{ include (print $.Template.BasePath "/rules.yaml") . | sha256sum }}
{{- if .Values.prometheusPort }}
{{ toYaml .Values.prometheusScrapeAnnotations | indent 8 }}
        prometheus.io/port: {{ .Values.prometheusPort | quote}}
{{- end }}
{{- if .Values.podAnnotations }}
{{ toYaml .Values.podAnnotations | indent 8 }}
{{- end }}
      labels:
        name: {{ template "elastalert.fullname" . }}-elastalert
        app: {{ template "elastalert.name" . }}
        release: {{ .Release.Name }}
    spec:
{{- if .Values.image.pullSecret }}
      imagePullSecrets:
      - name: {{ .Values.image.pullSecret }}
{{- end }}
      serviceAccountName: {{ include "elastalert.serviceAccountName" . }}
{{- if .Values.podSecurityContext }}
      securityContext:
{{ toYaml .Values.podSecurityContext | indent 8 }}
{{- end }}
      containers:
      - name: elastalert
        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
        imagePullPolicy: {{ .Values.image.pullPolicy }}
{{- if .Values.prometheusPort }}
        ports:
        - containerPort: {{ .Values.prometheusPort }}
          protocol: TCP
{{- end }}
{{- if .Values.securityContext }}
        securityContext:
{{ toYaml .Values.securityContext | indent 10 }}
{{- end }}
{{- if .Values.command }}
        command:
{{ toYaml .Values.command | indent 10 }}
{{- end }}

{{- if or .Values.args .Values.prometheusPort }}
        args:
  {{- if .Values.args }}
{{ toYaml .Values.args | indent 10 }}
  {{- end }}
  {{- if .Values.prometheusPort }}
  {{- $enableportlist := list "--prometheus_port" (.Values.prometheusPort | toString) }}
{{ toYaml $enableportlist | indent 10 }}
  {{- end }}
{{- end }}
        volumeMounts:
          - name: config
            mountPath: '/opt/elastalert/config.yaml'
            subPath: config.yaml
          - name: {{ template "elastalert.fullname" . }}-esrules
            mountPath: {{ .Values.rulesFolder }}
          - name: {{ template "elastalert.fullname" . }}-wechat
            mountPath: '/opt/elastalert/elastalert_modules/wechat_qiye_alert.py'
            subPath: wechat_qiye_alert.py
{{- if .Values.elasticsearch.certsVolumeMounts }}
{{ toYaml .Values.elasticsearch.certsVolumeMounts | indent 10 }}
{{- end }}
{{- if .Values.extraVolumeMounts }}
{{ toYaml .Values.extraVolumeMounts | indent 10 }}
{{- end }}
        resources:
{{ toYaml .Values.resources | indent 12 }}
        env:
{{- if .Values.elasticsearch.credentialsSecret }}
{{- if .Values.elasticsearch.credentialsSecretUsernameKey }}
          - name: ES_USERNAME
            valueFrom:
              secretKeyRef:
                name: {{ .Values.elasticsearch.credentialsSecret }}
                key: {{ .Values.elasticsearch.credentialsSecretUsernameKey }}
{{- end }}
{{- if .Values.elasticsearch.credentialsSecretPasswordKey }}
          - name: ES_PASSWORD
            valueFrom:
              secretKeyRef:
                name: {{ .Values.elasticsearch.credentialsSecret }}
                key: {{ .Values.elasticsearch.credentialsSecretPasswordKey }}
{{- end }}
{{- end }}
{{- if .Values.optEnv }}
{{ .Values.optEnv | toYaml | indent 10}}
{{- end }}
      restartPolicy: Always
{{- if .Values.tolerations }}
      tolerations:
{{ toYaml .Values.tolerations | indent 8 }}
{{- end }}
{{- if .Values.nodeSelector }}
      nodeSelector:
{{ toYaml .Values.nodeSelector | indent 8 }}
{{- end }}
{{- if .Values.affinity }}
      affinity:
{{ toYaml .Values.affinity | indent 8 }}
{{- end }}
      volumes:
        - name: {{ template "elastalert.fullname" . }}-wechat
{{- if .Values.secretRulesName }}
          secret:
            secretName: {{ .Values.secretRulesName }}
            items:
{{- range $key := .Values.secretRulesList }}
            - key: {{ $key }}
              path: {{ $key}}.yaml
{{- end }}
{{- else }}
          configMap:
            name: {{ template "elastalert.fullname" . }}-config
{{- end }}
            items:
            - key: wechat_qiye_alert
              path: wechat_qiye_alert.py
{{- if .Values.secretConfigName }}
          secret:
            secretName: {{ .Values.secretConfigName }}
{{- else }}
        - name: config
          configMap:
            name: {{ template "elastalert.fullname" . }}-config
{{- end }}
            items:
            - key: elastalert_config
              path: config.yaml
        - name: {{ template "elastalert.fullname" . }}-esrules
          persistentVolumeClaim:
            claimName: {{ template "elastalert.fullname" . }}-pvc
{{- if .Values.elasticsearch.certsVolumes }}
{{ toYaml .Values.elasticsearch.certsVolumes | indent 8 }}
{{- end }}
{{- if .Values.extraVolumes }}
{{ toYaml .Values.extraVolumes | indent 8 }}
{{- end }}

View Code

新增pvc.yaml模板：

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: {{ template "elastalert.fullname" . }}-pvc
  namespace: kms
spec:
  storageClassName: {{  .Values.storageClassName   }}
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: {{ .Values.storagepvcsize  }}
  volumeMode: Filesystem

View Code

其他不变

发表于 2022-05-18 10:32 少年老余阅读(760) 评论(0) 编辑收藏举报

搜索

常用链接

随笔分类

随笔档案

阅读排行榜

评论排行榜

最新评论