ECK部署

参考文章：
ECK配置：
https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-elasticsearch-specification.html


ECK搭建：
https://medium.com/@kishorechandran41/elasticsearch-cluster-on-google-kubernetes-engine-71bedd71fa85

https://hmj2088.medium.com/elastic-cloud-on-kubernetes-b5a69339e920

 

部署：(3节点，1 master-node,  2 data-nodes)
版本:
    elastic-operator: 1.5.0
    ELK: 7.12.0

1. 准备工作
a). 安装 all-in-one (elastic-operator)
下载地址: https://download.elastic.co/downloads/eck/1.5.0/all-in-one.yaml
镜像: docker.elastic.co/eck/eck-operator:1.5.0

# kubectl apply -f all-in-one.yaml   //创建


b). 创建存储类  
# master 主节点使用
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: eck-master   //存储类名称，和后面es申请空间时关联
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer
reclaimPolicy: Retain

# data 数据节点使用
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: elasticsearch-data   //存储类名称，和后面es申请空间时关联
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer
reclaimPolicy: Retain


c). 创建pv，使用local PV方式
apiVersion: v1
kind: PersistentVolume
metadata:
  name: master   # pv名称
spec:
  capacity:
    storage: 30Gi   # 容量
  volumeMode: Filesystem
  accessModes:
  - ReadWriteOnce
  persistentVolumeReclaimPolicy: Delete
  storageClassName: eck-master
  local:
    path: /mnt/disks/ssd1   # 宿主机路径。要手动在主机创建访目录
  nodeAffinity:
    required:
      nodeSelectorTerms:   # 节点选择
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - node01
  persistentVolumeReclaimPolicy: Retain   # 回收策略
  storageClassName: eck-master   # 关联的存储类
  volumeMode: Filesystem
# 其它两个数据节点的PV，按照些方式创建。


2. 部署elasticsearch
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
  name: elk-k8s   # 集群名称，后面kibana要使用这个名称连接es
  namespace: elastic-system
spec:
  version: 7.12.0   # es版本号
  nodeSets:
  - name: master   # 名称，随便取
    count: 1   # 节点数
    config:
      node.roles: ["master","ingest"]   # master节点。ingest很重要，否则开启xpack.monitoring时会报错
      node.store.allow_mmap: true
      xpack.monitoring.enabled: true
      xpack.monitoring.collection.enabled: true
      indices.breaker.total.use_real_memory: false   # 关键参数，影响es性能
      indices.fielddata.cache.size: 40%   # 关键参数，影响es性能
      indices.query.bool.max_clause_count: 4096
      indices.memory.index_buffer_size: "25%"
    podTemplate:
      metadata:
        labels:   # 标签，没有使用，只是写在这里
          foo: bar
      spec:
        initContainers:
        - name: sysctl
          securityContext:
            privileged: true
          command: ['sh', '-c', 'sysctl -w vm.max_map_count=262144']
        containers:
        - name: elasticsearch   # 这个名字不要改
          env:
          - name: ES_JAVA_OPTS
            value: -Xms4g -Xmx4g
          resources:
            requests:
              memory: 6Gi
              cpu: 2
            limits:
              memory: 6Gi
              cpu: 2
          image: hub.example.com/base/elasticsearch:7.12.0   # 私有镜像仓库里摘取镜像
          readinessProbe:   # 调大了健康检查时间
            exec:
              command:
              - bash
              - -c
              - /mnt/elastic-internal/scripts/readiness-probe-script.sh
            failureThreshold: 3
            initialDelaySeconds: 30
            periodSeconds: 20
            successThreshold: 1
            timeoutSeconds: 20
    volumeClaimTemplates:
    - metadata:
        name: elasticsearch-data   # 名字不要改
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: 30Gi   # 申请的pv空间
        storageClassName: eck-master  # 关联的存储类
  - name: datas
    count: 2
    podTemplate:
      spec:
        initContainers:
        - name: sysctl
          securityContext:
            privileged: true
          command: ['sh', '-c', 'sysctl -w vm.max_map_count=262144']
        containers:
        - name: elasticsearch   # 这个名字不要改
          env:
          - name: ES_JAVA_OPTS
            value: -Xms6g -Xmx6g
          resources:
            requests:
              memory: 8Gi
              cpu: 4
            limits:
              memory: 8Gi
              cpu: 4
          image: hub.example.com/base/elasticsearch:7.12.0
          readinessProbe:
            exec:
              command:
              - bash
              - -c
              - /mnt/elastic-internal/scripts/readiness-probe-script.sh
            failureThreshold: 3
            initialDelaySeconds: 30
            periodSeconds: 20
            successThreshold: 1
            timeoutSeconds: 20
    config:
      node.roles: ["data"]   # 数据节点
      node.store.allow_mmap: true
      xpack.monitoring.enabled: true
      xpack.monitoring.collection.enabled: true
      indices.breaker.total.use_real_memory: false
      indices.fielddata.cache.size: 40%
      indices.query.bool.max_clause_count: 4096
      indices.memory.index_buffer_size: "25%"
    volumeClaimTemplates:
    - metadata:
        name: elasticsearch-data   # 这个名字不要改
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: 50Gi   # 申请的pv空间
        storageClassName: eck-node  # 关联的存储类

# kubectl apply -f elastic.yaml   # 创建es集群。查看一下日志有没有报错

获取elasticsearch默认密码
# PASSWORD=$(kubectl -n elastic-system get secrets elk-k8s-es-elastic-user -o go-template='{{.data.elastic | base64decode}}')
用户名：elastic
密码：$PASSWORD



# curl --insecure  -u "elastic:$PASSWORD" -k  https://10.43.123.78:9200/_cat/nodes?v



3. 使用elasticsearch api 调整参数
PUT _cluster/settings
{
  "persistent": {
    "xpack.monitoring.collection.enabled": true
  }
}


PUT /_cluster/settings
{
  "persistent": {
    "cluster": {
      "max_shards_per_node":15000
    }
  }
}

PUT /_cluster/settings
{
  "persistent" : {
    "indices.breaker.fielddata.limit" : "60%",
    "indices.breaker.request.limit" : "40%",
    "indices.breaker.total.limit" : "70%"
  }
}



4. 部署kibana
apiVersion: kibana.k8s.elastic.co/v1
kind: Kibana
metadata:
  name: xn-kibana
  namespace: elastic-system
spec:
  version: 7.12.0   # kibana版本
  count: 1
  elasticsearchRef:
    name: "elk-k8s"   # es集群名称
    namespace: elastic-system
  podTemplate:
    spec:
      containers:
      - name: kibana
        env:
          - name: NODE_OPTIONS
            value: "--max-old-space-size=2048"
        resources:
          requests:
            memory: 1Gi
            cpu: 0.5
          limits:
            memory: 3Gi
            cpu: 2
        image: hub.example.com/base/kibana:7.12.0

# kubectl apply -f kibana.yaml   # 部署Kibana


5. 配置代理访问kibana
因为我是使用rancher部署的k8s集群，所以下面是在rancher里创建的nginx-ingress配置

---- 空行 --------

添加注释：
nginx.ingress.kubernetes.io/backend-protocol=https

 

 在kibana里查看一下集群状态：

GET _cat/nodes
GET _cluster/settings
GET _cat/health

GET _cat/thread_pool?v


根据需要调整对应index模板：
{
  "index": {
    "mapping": {
      "total_fields": {
        "limit": "10000"
      }
    },
    "refresh_interval": "15s",
    "number_of_shards": "1",
    "translog": {
      "flush_threshold_size": "1024mb",
      "sync_interval": "120s",
      "durability": "async"
    },
    "max_docvalue_fields_search": "500",
    "query": {
      "default_field": [
        "message",
        "tags",
        "agent.ephemeral_id",
        "agent.id",
        ......



6. 部署filebeat
apiVersion: beat.k8s.elastic.co/v1beta1
kind: Beat
metadata:
  name: filebeat
  namespace: elastic-system
spec:
  type: filebeat
  version: 7.11.1
  elasticsearchRef:
    name: xn-elastic
    namespace: elastic-system
  kibanaRef:
    name: xn-kibana
    namespace: elastic-system
  config:
    filebeat:
      autodiscover:
        providers:
        - type: kubernetes
          node: ${NODE_NAME}
          hints:
            enabled: true
            default_config:
              type: container
              paths:
              - /var/log/containers/*${data.kubernetes.container.id}.log
              tail_files: true
              harvester_buffer_size: 65536
              exclude_lines: ['SqlSession', '<==      Total: 0', 'JDBC Connection', '<==    Columns']
              #multiline:
              #  pattern: ^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}
              #  negate: true
              #  match: after
    filebeat.config:
      modules:
        path: ${path.config}/modules.d/*.yml
        reload.enabled: false
    filebeat.modules:
      - module: system
        syslog:
          enabled: true
          var.paths: ["/var/log/messages"]
          var.convert_timezone: true
        auth:
          enabled: true
          var.paths: ["/var/log/secure"]
          var.convert_timezone: true
    filebeat.shutdown_timeout: 10s
    logging.level: info   # debug
    #logging.selectors: ["prospector","harvester"]
    output.elasticsearch:
      worker: 4
      bulk_max_size: 22000
      flush_interval: 1s
      indices:
        - index: "k8s-syslog-%{+yyyy.MM.dd}"
          when.equals:
            event.module: "system"
        - index: "k8s-%{[kubernetes.namespace]}-%{+yyyy.MM.dd}"
      username: ${ELASTICSEARCH_USERNAME}
      password: ${ELASTICSEARCH_PASSWORD}
    ##setup.template.enabled: true
    setup.template.name: "k8s"
    setup.template.pattern: "k8s-*"
    setup.ilm.enabled: false
    processors:
    - add_cloud_metadata: {}
    - add_host_metadata: {}
    - drop_fields:
        fields: ["host.ip", "host.mac", "host.id", "host.name", "host.architecture", "host.os.family", "host.os.name", "host.os.version", "host.os.codename", "host.os.kernel", "ecs.version", "agent.name", "agent.version", "kubernetes.node.l
abels.beta_kubernetes_io/arch", "kubernetes.node.labels.beta_kubernetes_io/os"]
    queue.mem:
      events: 16000
      flush.min_events: 9000
      flush.timeout: 1s
  daemonSet:
    podTemplate:
      spec:
        serviceAccountName: filebeat
        automountServiceAccountToken: true
        terminationGracePeriodSeconds: 30
        dnsPolicy: ClusterFirstWithHostNet
        hostNetwork: true   # Allows to provide richer host metadata
        containers:
        - name: filebeat
          securityContext:
            runAsUser: 0
            # If using Red Hat OpenShift uncomment this:
            #privileged: true
          image: registry.cqxndb.com/base/filebeat:7.11.1
          resources:
            limits:
              cpu: 300m
              memory: 300Mi
            requests:
              cpu: 300m
              memory: 300Mi
          volumeMounts:
          - name: varlogcontainers
            mountPath: /var/log/containers
          - name: varlogpods
            mountPath: /var/log/pods
          - name: varlibdockercontainers
            mountPath: /var/lib/docker/containers
          - name: varlogmessages
            mountPath: /var/log/messages
          - name: varlogsecure
            mountPath: /var/log/secure
          env:
            - name: ELASTICSEARCH_USERNAME
              value: elastic
            - name: ELASTICSEARCH_PASSWORD
              valueFrom:
                secretKeyRef:
                  key: elastic
                  name: xn-elastic-es-elastic-user
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
        volumes:
        - name: varlogcontainers
          hostPath:
            path: /var/log/containers
        - name: varlogpods
          hostPath:
            path: /var/log/pods
        - name: varlibdockercontainers
          hostPath:
            path: /var/lib/docker/containers
        - name: varlogmessages
          hostPath:
            path: /var/log/messages
        - name: varlogsecure
          hostPath:
            path: /var/log/secure

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: filebeat
rules:
- apiGroups: [""]   # "" indicates the core API group
  resources:
  - namespaces
  - nodes
  - pods
  verbs:
  - get
  - watch
  - list

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: filebeat
  namespace: elastic-system

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: filebeat
subjects:
- kind: ServiceAccount
  name: filebeat
  namespace: elastic-system
roleRef:
  kind: ClusterRole
  name: filebeat
  apiGroup: rbac.authorization.k8s.io


7. 部署APMServer
apiVersion: apm.k8s.elastic.co/v1
kind: ApmServer
metadata:
  name: xn-apmserver
  namespace: elastic-system
spec:
  version: 7.11.1
  count: 1
  elasticsearchRef:
    name: "xn-elastic"   # 引用ES名称
  kibanaRef:
    name: "xn-kibana"   # 引用kibana名称
  config:
    apm-server:
      rum.enabled: true
      ilm.enabled: true
      rum.event_rate.limit: 300
      rum.event_rate.lru_size: 1000
      rum.allow_origins: ['*']
    logging:
      level: warning
    queue.mem:
      events: 10240
    output.elasticsearch:
      worker: 2
      bulk_max_size: 500
  podTemplate:
    spec:
      containers:
      - name: apm-server
        resources:
          requests:
            memory: 2Gi
            cpu: 2
          limits:
            memory: 4Gi
            cpu: 4
        image: registry.example.com/base/apm-server:7.11.1


应用接入APMServer:
参考链接：https://github.com/maieve/Eck-apm-example