docker-compose部署elk+apm

1、安装docker

参考我的另外的一篇博客：https://www.cnblogs.com/cuishuai/p/9485939.html

2、安装docker-compose

# yum -y install docker-compose-1.18.0-1.el7.noarch

3、准备docker-compose.yml文件,放到/data/

version: '3'

services:
  elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:6.4.0
     ports:
       - 9200:9200
       - 9300:9300
     environment:
       ES_JASVA_OPTS: "-Xms 10g -Xmx 10g"
     networks:
       - eak
     volumes:
       - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
       - /data/elasticsearch:/usr/share/elasticseasrch/data
       - /data/plugin:/usr/share/elasticsearch/plugins

  kibana:
     image: docker.elastic.co/kibana/kibana:6.4.0
     volumes:
       - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml
     ports:
       - 5601:5601
     networks:
       - eak
     depends_on:
       - elasticsearch

  apm:
     image: docker.elastic.co/apm/apm-server:6.4.0
     ports:
        - 8200:8200
     volumes:
       - ./apm-server/config/apm-server.yml:/usr/share/apm-server/apm-server.yml
     networks:
        - eak
     depends_on:
        - elasticsearch
        - kibana

networks:
    eak:
      driver: bridge

4、准备目录和配置文件

#mkdir /data/elasticsearch/config  -p

#mkdir  /data/kibana/config -p

#mkdir  /data/apm-server/config  -p

#mkdir  /data/elasticsearch

 

elasticsearch.yml:

action.auto_create_index: true
#script.groovy.sandbox.enabled: true
##script.engine.groovy.inline.aggs: true
##Set a custom allowed content length:
http.max_content_length: 500mb
cluster.routing.allocation.disk.watermark.low: 90%
cluster.routing.allocation.disk.watermark.high: 95%
indices.fielddata.cache.size:  20%
indices.breaker.fielddata.limit: 60%
network.host: 0.0.0.0
discovery.type: single-node
##xpack.security.enabled: false

kibana.yml:

---
# Default Kibana configuration from kibana-docker.
#
server.name: kibana
server.host: "0.0.0.0"
elasticsearch.url: http://elasticsearch:9200
# #xpack.monitoring.ui.container.elasticsearch.enabled: false
server.port: 5601
# #xpack.security.enabled: false

apm-server.yml:

################### APM Server Configuration #########################

############################# APM Server ######################################

apm-server:
  # Defines the host and port the server is listening on.  use "unix:/path/to.sock" to listen on a unix domain socket.
  host: "0.0.0.0:8200"

  # Maximum permitted size in bytes of an unzipped request accepted by the server to be processed.
  #max_unzipped_size: 31457280
  # Maximum permitted size in bytes of a request's header accepted by the server to be processed.
  #max_header_size: 1048576

  # Maximum duration request will be queued before being read.
  #max_request_queue_time: 2s
  # Maximum permitted duration for reading an entire request.
  #read_timeout: 30s
  # Maximum permitted duration for writing a response.
  #write_timeout: 30s

  # Maximum duration in seconds before releasing resources when shutting down the server.
  #shutdown_timeout: 5s

  # Maximum number of requests permitted to be sent to the server concurrently.
  #concurrent_requests: 5

  # Maximum number of new connections to accept simultaneously (0 means unlimited)
  # max_connections: 0

  # Instrumentation support for the server's HTTP endpoints and event publisher.
  #instrumentation:
    # Set to true to enable instrumentation of the APM server itself.
    #enabled: false
    # Environment in which the APM Server is running on (eg: staging, production, etc.)
    #environment: ""

  # Authorization token to be checked. If a token is set here the agents must
  # send their token in the following format: Authorization: Bearer <secret-token>.
  # It is recommended to use an authorization token in combination with SSL enabled,
  # and save the token in the beats keystore.
  #secret_token:
  #ssl.enabled: false
  #ssl.certificate : "path/to/cert"
  #ssl.key : "path/to/private_key"
  # It is recommended to use the provided keystore instead of entering the passphrase in plain text.
  #ssl.key_passphrase: ""

  #rum:
    # To enable real user monitoring (RUM) support set this to true.
    #enabled: false

    # Rate limit per second and IP address for requests sent to the RUM endpoint.
    #rate_limit: 10

    # Comma separated list of permitted origins for real user monitoring.
    # User-agents will send an origin header that will be validated against this list.
    # An origin is made of a protocol scheme, host and port, without the url path.
    # Allowed origins in this setting can have * to match anything (eg.: http://*.example.com)
    # If an item in the list is a single '*', everything will be allowed
    #allow_origins : ['*']

    # Regexp to be matched against a stacktrace frame's `file_name` and `abs_path` attributes.
    # If the regexp matches, the stacktrace frame is considered to be a library frame.
    #library_pattern: "node_modules|bower_components|~"

    # Regexp to be matched against a stacktrace frame's `file_name`.
    # If the regexp matches, the stacktrace frame is not used for calculating error groups.
    # The default pattern excludes stacktrace frames that have a filename starting with '/webpack'
    #exclude_from_grouping: "^/webpack"

    # If a source map has previously been uploaded, source mapping is automatically applied
    # to all error and transaction documents sent to the RUM endpoint.
    #source_mapping:

      # Source maps are always fetched from Elasticsearch, by default using the output.elasticsearch configuration.
      # A different instance must be configured when using any other output.
      # This setting only affects sourcemap reads - the output determines where sourcemaps are written.
      #elasticsearch:
        # Array of hosts to connect to.
        # Scheme and port can be left out and will be set to the default (http and 9200)
        # In case you specify and additional path, the scheme is required: http://localhost:9200/path
        # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
        # hosts: ["localhost:9200"]

        # Optional protocol and basic auth credentials.
        #protocol: "https"
        #username: "elastic"
        #password: "changeme"

      # The `cache.expiration` determines how long a source map should be cached before fetching it again from Elasticsearch.
      # Note that values configured without a time unit will be interpreted as seconds.
      #cache:
        #expiration: 5m

      # Source maps are stored in a seperate index.
      # If the default index pattern for source maps at 'outputs.elasticsearch.indices'
      # is changed, a matching index pattern needs to be specified here.
      #index_pattern: "apm-*-sourcemap*"

  # If set to true, APM Server augments data received by the agent with the original IP of the backend server,
  # or the IP and User Agent of the real user (RUM requests). It defaults to true.
  #capture_personal_data: true

  # golang expvar support - https://golang.org/pkg/expvar/
  #expvar:
    # Set to true to Expose expvar
    #enabled: false

    # Url to expose expvar
    #url: "/debug/vars"

  # Experimental Metrics endpoint
  #metrics:
    # Set to false to disable the metrics endpoint
    #enabled: true

  # A pipeline is a definition of processors applied to documents when writing them to Elasticsearch.
  # Using pipelines involves two steps:
  # (1) registering a pipeline
  # (2) applying a pipeline during data ingestion (see `output.elasticsearch.pipelines`)
  #
  # You can manually register pipelines, or use this configuration option to ensure
  # pipelines are loaded and registered at the configured Elasticsearch instances.
  # Automatic pipeline registration requires
  # * `output.elasticsearch` to be enabled and configured.
  # * having the required Elasticsearch Processor Plugins installed.
  #   APM Server default pipelines require you to have the `Ingest User Agent Plugin` installed.
  #   Find the default pipeline configuration at `ingest/pipeline/definition.json`.
  #
  #register.ingest.pipeline:
    # Registers pipeline definitions in Elasticsearch on APM Server startup. Defaults to false.
    #enabled: false

    # Overwrites existing pipeline definitions in Elasticsearch. Defaults to true.
    #overwrite: true

#================================ General ======================================

# Internal queue configuration for buffering events to be published.
queue:
  # Queue type by name (default 'mem')
  # The memory queue will present all available events (up to the outputs
  # bulk_max_size) to the output, the moment the output is ready to server
  # another batch of events.
  mem:
    # Max number of events the queue can buffer.
    events: 409600

    # Hints the minimum number of events stored in the queue,
    # before providing a batch of events to the outputs.
    # A value of 0 (the default) ensures events are immediately available
    # to be sent to the outputs.
    #flush.min_events: 2048

    # Maximum duration after which events are available to the outputs,
    # if the number of events stored in the queue is < min_flush_events.
    #flush.timeout: 1s

# Sets the maximum number of CPUs that can be executing simultaneously. The
# default is the number of logical CPUs available in the system.
#max_procs:


#==================== Elasticsearch template setting ==========================

# Elasticsearch template settings
setup.template.settings:

  # A dictionary of settings to place into the settings.index dictionary
  # of the Elasticsearch template. For more details, please check
  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
  index:
    number_of_shards: 1
    codec: best_compression
    #number_of_routing_shards: 30


#============================== Template =====================================

# A template is used to set the mapping in Elasticsearch
# By default template loading is enabled and the template is loaded.
# These settings can be adjusted to load your own template or overwrite existing ones.

# Set to false to disable template loading.
#setup.template.enabled: true

# Template name. By default the template name is "apm-%{[beat.version]}"
# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
#setup.template.name: "apm-%{[beat.version]}"

# Template pattern. By default the template pattern is "apm-%{[beat.version]}-*" to apply to the default index settings.
# The first part is the version of the beat and then -* is used to match all daily indices.
# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
#setup.template.pattern: "apm-%{[beat.version]}-*"

# Path to fields.yml file to generate the template
#setup.template.fields: "${path.config}/fields.yml"

# Overwrite existing template
#setup.template.overwrite: false

# Elasticsearch template settings
#setup.template.settings:

  # A dictionary of settings to place into the settings.index dictionary
  # of the Elasticsearch template. For more details, please check
  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
  #index:
    #number_of_shards: 1
    #codec: best_compression
    #number_of_routing_shards: 30

  # A dictionary of settings for the _source field. For more details, please check
  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
  #_source:
    #enabled: false


#============================== Deprecated: Dashboards =====================================
#
# Deprecated: Loading dashboards from the APM Server into Kibana is deprecated from 6.4 on.
#             We suggest to use the Kibana UI to load APM Server dashboards and index pattern instead.
#
# These settings control loading the sample dashboards to the Kibana index. Loading
# the dashboards are disabled by default and can be enabled either by setting the
# options here, or by using the `-setup` CLI flag or the `setup` command.
#setup.dashboards.enabled: false

# The directory from where to read the dashboards. The default is the `kibana`
# folder in the home path.
#setup.dashboards.directory: ${path.home}/kibana

# The URL from where to download the dashboards archive. It is used instead of
# the directory if it has a value.
#setup.dashboards.url:

# The file archive (zip file) from where to read the dashboards. It is used instead
# of the directory when it has a value.
#setup.dashboards.file:

# The name of the Kibana index to use for setting the configuration. Default is ".kibana"
#setup.dashboards.kibana_index: .kibana

# The Elasticsearch index name. This overwrites the index name defined in the
# dashboards and index pattern. Example: testbeat-*
# The dashboards.index needs to be changed in case the elasticsearch index pattern is modified.
#setup.dashboards.index:

# Always use the Kibana API for loading the dashboards instead of autodetecting
# how to install the dashboards by first querying Elasticsearch.
#setup.dashboards.always_kibana: false

# If true and Kibana is not reachable at the time when dashboards are loaded,
# it will retry to reconnect to Kibana instead of exiting with an error.
#setup.dashboards.retry.enabled: false

# Duration interval between Kibana connection retries.
#setup.dashboards.retry.interval: 1s

# Maximum number of retries before exiting with an error, 0 for unlimited retrying.
#setup.dashboards.retry.maximum: 0


#============================== Deprecated: Kibana =====================================

# Deprecated: Starting with APM Server version 6.4, loading dashboards and index pattern
#             from the APM Server into Kibana is deprecated.
#             We suggest to use the Kibana UI to load APM Server dashboards and index pattern instead.
#
#             Setting up a Kibana endpoint is not necessary when loading the index pattern and dashboards via the UI.

setup.kibana:

  # Kibana Host
  # Scheme and port can be left out and will be set to the default (http and 5601)
  # In case you specify and additional path, the scheme is required: http://localhost:5601/path
  # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
  host: "kibana:5601"

  # Optional protocol and basic auth credentials.
  #protocol: "https"
  #username: "elastic"
  #password: "changeme"

  # Optional HTTP Path
  #path: ""

  # Use SSL settings for HTTPS. Default is false.
  #ssl.enabled: false

  # Configure SSL verification mode. If `none` is configured, all server hosts
  # and certificates will be accepted. In this mode, SSL based connections are
  # susceptible to man-in-the-middle attacks. Use only for testing. Default is
  # `full`.
  #ssl.verification_mode: full

  # List of supported/valid TLS versions. By default all TLS versions 1.0 up to
  # 1.2 are enabled.
  #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]

  # SSL configuration. By default is off.
  # List of root certificates for HTTPS server verifications
  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]

  # Certificate for SSL client authentication
  #ssl.certificate: "/etc/pki/client/cert.pem"

  # Client Certificate Key
  #ssl.key: "/etc/pki/client/cert.key"

  # Optional passphrase for decrypting the Certificate Key.
  # It is recommended to use the provided keystore instead of entering the passphrase in plain text.
  #ssl.key_passphrase: ''

  # Configure cipher suites to be used for SSL connections
  #ssl.cipher_suites: []

  # Configure curve types for ECDHE based cipher suites
  #ssl.curve_types: []


#================================ Outputs =====================================

# Configure what output to use when sending the data collected by the beat.

#-------------------------- Elasticsearch output ------------------------------
#output.elasticsearch:
  # Array of hosts to connect to.
  # Scheme and port can be left out and will be set to the default (http and 9200)
  # In case you specify and additional path, the scheme is required: http://localhost:9200/path
  # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
 # hosts: ["elasticsearch:9200"]

  # Boolean flag to enable or disable the output module.
  #enabled: true

  # Set gzip compression level.
  #compression_level: 0

  # Optional protocol and basic auth credentials.
  #protocol: "https"
  #username: "elastic"
  #password: "changeme"

  # Dictionary of HTTP parameters to pass within the url with index operations.
  #parameters:
    #param1: value1
    #param2: value2

  # Number of workers per Elasticsearch host.
  #worker: 4

  # By using the configuration below, apm documents are stored to separate indices,
  # depending on their `processor.event`:
  # - error
  # - transaction
  # - span
  # - sourcemap
  #
  # The indices are all prefixed with `apm-%{[beat.version]}`.
  # To allow managing indices based on their age, all indices (except for sourcemaps)
  # end with the information of the day they got indexed.
  # e.g. "apm-6.3.0-transaction-2018.03.20"
  #
  # Be aware that you can only specify one Elasticsearch template and one Kibana Index Pattern,
  # In case you modify the index patterns you must also update those configurations accordingly,
  # as they need to be aligned:
  # * `setup.template.name`
  # * `setup.template.pattern`
  # * `setup.dashboards.index`
  # A pipeline is a definition of processors applied to documents when writing them to Elasticsearch.
  # APM Server comes with a default pipeline definition, located at `ingets/pipeline/definition.json`.
  # Pipelines are disabled by default. To make use of them you have to:
  # (1) ensure pipelines are registered in Elasticsearch, see `apm-server.register.ingest.pipeline`
  # (2) enable the following:
  #pipelines:
  #- pipeline: "apm_user_agent"

  # Optional HTTP Path
  #path: "/elasticsearch"

  # Custom HTTP headers to add to each request
  #headers:
  #  X-My-Header: Contents of the header

  # Proxy server url
  #proxy_url: http://proxy:3128

  # The number of times a particular Elasticsearch index operation is attempted. If
  # the indexing operation doesn't succeed after this many retries, the events are
  # dropped. The default is 3.
  #max_retries: 3

  # The maximum number of events to bulk in a single Elasticsearch bulk API index request.
  # The default is 50.
 # bulk_max_size: 100000

  # The number of seconds to wait before trying to reconnect to Elasticsearch
  # after a network error. After waiting backoff.init seconds, apm-server
  # tries to reconnect. If the attempt fails, the backoff timer is increased
  # exponentially up to backoff.max. After a successful connection, the backoff
  # timer is reset. The default is 1s.
  #backoff.init: 1s

  # The maximum number of seconds to wait before attempting to connect to
  # Elasticsearch after a network error. The default is 60s.
  #backoff.max: 60s

  # Configure http request timeout before failing an request to Elasticsearch.
  #timeout: 90

  # Use SSL settings for HTTPS. Default is false.
  #ssl.enabled: false

  # Configure SSL verification mode. If `none` is configured, all server hosts
  # and certificates will be accepted. In this mode, SSL based connections are
  # susceptible to man-in-the-middle attacks. Use only for testing. Default is
  # `full`.
  #ssl.verification_mode: full

  # List of supported/valid TLS versions. By default all TLS versions 1.0 up to
  # 1.2 are enabled.
  #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]

  # SSL configuration. By default is off.
  # List of root certificates for HTTPS server verifications
  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]

  # Certificate for SSL client authentication
  #ssl.certificate: "/etc/pki/client/cert.pem"

  # Client Certificate Key
  #ssl.key: "/etc/pki/client/cert.key"

  # Optional passphrase for decrypting the Certificate Key.
  # It is recommended to use the provided keystore instead of entering the passphrase in plain text.
  #ssl.key_passphrase: ''

  # Configure cipher suites to be used for SSL connections
  #ssl.cipher_suites: []

  # Configure curve types for ECDHE based cipher suites
  #ssl.curve_types: []

  # Configure what types of renegotiation are supported. Valid options are
  # never, once, and freely. Default is never.
  #ssl.renegotiation: never

#----------------------------- Console output ---------------------------------
#output.console:
  # Boolean flag to enable or disable the output module.
  #enabled: false

  # Pretty print json event
  #pretty: false

#----------------------------- Logstash output ---------------------------------
#output.logstash:
  # Boolean flag to enable or disable the output module.
  #enabled: false

  # The Logstash hosts
  #hosts: ["localhost:5044"]

  # Number of workers per Logstash host.
  #worker: 1

  # Set gzip compression level.
  #compression_level: 3

  # Configure escaping html symbols in strings.
  #escape_html: true

  # Optional maximum time to live for a connection to Logstash, after which the
  # connection will be re-established.  A value of `0s` (the default) will
  # disable this feature.
  #
  # Not yet supported for async connections (i.e. with the "pipelining" option set)
  #ttl: 30s

  # Optional load balance the events between the Logstash hosts. Default is false.
  #loadbalance: false

  # Number of batches to be sent asynchronously to Logstash while processing
  # new batches.
  #pipelining: 2

  # If enabled only a subset of events in a batch of events is transferred per
  # group.  The number of events to be sent increases up to `bulk_max_size`
  # if no error is encountered.
  #slow_start: false

  # The number of seconds to wait before trying to reconnect to Logstash
  # after a network error. After waiting backoff.init seconds, apm-server
  # tries to reconnect. If the attempt fails, the backoff timer is increased
  # exponentially up to backoff.max. After a successful connection, the backoff
  # timer is reset. The default is 1s.
  #backoff.init: 1s

  # The maximum number of seconds to wait before attempting to connect to
  # Logstash after a network error. The default is 60s.
  #backoff.max: 60s

  # Optional index name. The default index name is set to apm
  # in all lowercase.
  #index: 'apm'

  # SOCKS5 proxy server URL
  #proxy_url: socks5://user:password@socks5-server:2233

  # Resolve names locally when using a proxy server. Defaults to false.
  #proxy_use_local_resolver: false

  # Enable SSL support. SSL is automatically enabled, if any SSL setting is set.
  #ssl.enabled: true

  # Configure SSL verification mode. If `none` is configured, all server hosts
  # and certificates will be accepted. In this mode, SSL based connections are
  # susceptible to man-in-the-middle attacks. Use only for testing. Default is
  # `full`.
  #ssl.verification_mode: full

  # List of supported/valid TLS versions. By default all TLS versions 1.0 up to
  # 1.2 are enabled.
  #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]

  # Optional SSL configuration options. SSL is off by default.
  # List of root certificates for HTTPS server verifications
  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]

  # Certificate for SSL client authentication
  #ssl.certificate: "/etc/pki/client/cert.pem"

  # Client Certificate Key
  #ssl.key: "/etc/pki/client/cert.key"

  # Optional passphrase for decrypting the Certificate Key.
  #ssl.key_passphrase: ''

  # Configure cipher suites to be used for SSL connections
  #ssl.cipher_suites: []

  # Configure curve types for ECDHE based cipher suites
  #ssl.curve_types: []

  # Configure what types of renegotiation are supported. Valid options are
  # never, once, and freely. Default is never.
  #ssl.renegotiation: never

#------------------------------- Kafka output ----------------------------------
output.kafka:
  # Boolean flag to enable or disable the output module.
  enabled: false

  # The list of Kafka broker addresses from where to fetch the cluster metadata.
  # The cluster metadata contain the actual Kafka brokers events are published
  # to.
  hosts: ["localhost:9092"]

  # The Kafka topic used for produced events. The setting can be a format string
  # using any event field. To set the topic from document type use `%{[type]}`.
  topic: beats

  # The Kafka event key setting. Use format string to create unique event key.
  # By default no event key will be generated.
  #key: ''

  # The Kafka event partitioning strategy. Default hashing strategy is `hash`
  # using the `output.kafka.key` setting or randomly distributes events if
  # `output.kafka.key` is not configured.
  #partition.hash:
    # If enabled, events will only be published to partitions with reachable
    # leaders. Default is false.
    #reachable_only: false

    # Configure alternative event field names used to compute the hash value.
    # If empty `output.kafka.key` setting will be used.
    # Default value is empty list.
    #hash: []

  # Authentication details. Password is required if username is set.
  #username: ''
  #password: ''

  # Kafka version libbeat is assumed to run against. Defaults to the "1.0.0".
  #version: '1.0.0'

  # Configure JSON encoding
  #codec.json:
    # Pretty print json event
    #pretty: false

    # Configure escaping html symbols in strings.
    #escape_html: true

  # Metadata update configuration. Metadata do contain leader information
  # deciding which broker to use when publishing.
  #metadata:
    # Max metadata request retry attempts when cluster is in middle of leader
    # election. Defaults to 3 retries.
    #retry.max: 3

    # Waiting time between retries during leader elections. Default is 250ms.
    #retry.backoff: 250ms

    # Refresh metadata interval. Defaults to every 10 minutes.
    #refresh_frequency: 10m

  # The number of concurrent load-balanced Kafka output workers.
  worker: 4

  # The number of times to retry publishing an event after a publishing failure.
  # After the specified number of retries, the events are typically dropped.
  # Some Beats, such as Filebeat, ignore the max_retries setting and retry until
  # all events are published.  Set max_retries to a value less than 0 to retry
  # until all events are published. The default is 3.
  #max_retries: 3

  # The maximum number of events to bulk in a single Kafka request. The default
  # is 2048.
  bulk_max_size: 409600

  # The number of seconds to wait for responses from the Kafka brokers before
  # timing out. The default is 30s.
  #timeout: 30s

  # The maximum duration a broker will wait for number of required ACKs. The
  # default is 10s.
  #broker_timeout: 10s

  # The number of messages buffered for each Kafka broker. The default is 256.
  channel_buffer_size: 2560

  # The keep-alive period for an active network connection. If 0s, keep-alives
  # are disabled. The default is 0 seconds.
  #keep_alive: 0

  # Sets the output compression codec. Must be one of none, snappy and gzip. The
  # default is gzip.
  #compression: gzip

  # The maximum permitted size of JSON-encoded messages. Bigger messages will be
  # dropped. The default value is 1000000 (bytes). This value should be equal to
  # or less than the broker's message.max.bytes.
  #max_message_bytes: 1000000

  # The ACK reliability level required from broker. 0=no response, 1=wait for
  # local commit, -1=wait for all replicas to commit. The default is 1.  Note:
  # If set to 0, no ACKs are returned by Kafka. Messages might be lost silently
  # on error.
  #required_acks: 1

  # The configurable ClientID used for logging, debugging, and auditing
  # purposes.  The default is "beats".
  #client_id: beats

  # Enable SSL support. SSL is automatically enabled, if any SSL setting is set.
  #ssl.enabled: true

  # Optional SSL configuration options. SSL is off by default.
  # List of root certificates for HTTPS server verifications
  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]

  # Configure SSL verification mode. If `none` is configured, all server hosts
  # and certificates will be accepted. In this mode, SSL based connections are
  # susceptible to man-in-the-middle attacks. Use only for testing. Default is
  # `full`.
  #ssl.verification_mode: full

  # List of supported/valid TLS versions. By default all TLS versions 1.0 up to
  # 1.2 are enabled.
  #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]

  # Certificate for SSL client authentication
  #ssl.certificate: "/etc/pki/client/cert.pem"

  # Client Certificate Key
  #ssl.key: "/etc/pki/client/cert.key"

  # Optional passphrase for decrypting the Certificate Key.
  #ssl.key_passphrase: ''

  # Configure cipher suites to be used for SSL connections
  #ssl.cipher_suites: []

  # Configure curve types for ECDHE based cipher suites
  #ssl.curve_types: []

  # Configure what types of renegotiation are supported. Valid options are
  # never, once, and freely. Default is never.
  #ssl.renegotiation: never

#================================= Paths ======================================

# The home path for the apm-server installation. This is the default base path
# for all other path settings and for miscellaneous files that come with the
# distribution (for example, the sample dashboards).
# If not set by a CLI flag or in the configuration file, the default for the
# home path is the location of the binary.
#path.home:

# The configuration path for the apm-server installation. This is the default
# base path for configuration files, including the main YAML configuration file
# and the Elasticsearch template file. If not set by a CLI flag or in the
# configuration file, the default for the configuration path is the home path.
#path.config: ${path.home}

# The data path for the apm-server installation. This is the default base path
# for all the files in which apm-server needs to store its data. If not set by a
# CLI flag or in the configuration file, the default for the data path is a data
# subdirectory inside the home path.
#path.data: ${path.home}/data

# The logs path for a apm-server installation. This is the default location for
# the Beat's log files. If not set by a CLI flag or in the configuration file,
# the default for the logs path is a logs subdirectory inside the home path.
#path.logs: ${path.home}/logs


#================================ Logging ======================================
#
# There are three options for the log output: syslog, file, stderr.
# Under Windows systems, the log files are per default sent to the file output,
# under all other system per default to syslog.

# Sets log level. The default log level is info.
# Available log levels are: error, warning, info, debug
#logging.level: info

# Enable debug output for selected components. To enable all selectors use ["*"]
# Other available selectors are "beat", "publish", "service"
# Multiple selectors can be chained.
#logging.selectors: [ ]

# Send all logging output to syslog. The default is false.
#logging.to_syslog: true

# If enabled, apm-server periodically logs its internal metrics that have changed
# in the last period. For each metric that changed, the delta from the value at
# the beginning of the period is logged. Also, the total values for
# all non-zero internal metrics are logged on shutdown. The default is true.
logging.metrics.enabled: false

# The period after which to log the internal metrics. The default is 30s.
#logging.metrics.period: 30s

# Logging to rotating files. Set logging.to_files to false to disable logging to
# files.
#logging.to_files: true
#logging.files:
  # Configure the path where the logs are written. The default is the logs directory
  # under the home path (the binary location).
  #path: /var/log/apm-server

  # The name of the files where the logs are written to.
  #name: apm-server

  # Configure log file size limit. If limit is reached, log file will be
  # automatically rotated
  #rotateeverybytes: 10485760 # = 10MB

  # Number of rotated log files to keep. Oldest files will be deleted first.
  #keepfiles: 7

  # The permissions mask to apply when rotating log files. The default value is 0600.
  # Must be a valid Unix-style file permissions mask expressed in octal notation.
  #permissions: 0600

# Set to true to log messages in json format.
#logging.json: false


#================================ HTTP Endpoint ======================================
#
# Each beat can expose internal metrics through a HTTP endpoint. For security
# reasons the endpoint is disabled by default. This feature is currently experimental.
# Stats can be access through http://localhost:5066/stats . For pretty JSON output
# append ?pretty to the URL.

# Defines if the HTTP endpoint is enabled.
#http.enabled: false

# The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost.
#http.host: localhost

# Port on which the HTTP endpoint will bind. Default is 5066.
#http.port: 5066

#============================== Xpack Monitoring ===============================
# APM server can export internal metrics to a central Elasticsearch monitoring
# cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The
# reporting is disabled by default.

# Set to true to enable the monitoring reporter.
#xpack.monitoring.enabled: false

# Uncomment to send the metrics to Elasticsearch. Most settings from the
# Elasticsearch output are accepted here as well. Any setting that is not set is
# automatically inherited from the Elasticsearch output configuration, so if you
# have the Elasticsearch output configured, you can simply uncomment the
# following line.
#xpack.monitoring.elasticsearch:
#  username: "apm_system"
#  password: ""

 

 

# The number of concurrent load-balanced Kafka output workers.
worker: 4

根据cpu核数设置worker参数。可以设置和cpu相同。

# The maximum number of events to bulk in a single Kafka request. The default
# is 2048.
bulk_max_size: 409600

设置bulk的size，根据内存来设置。否则会出现http503  queue is full。

5、启动

#docker-compose  up -d 

 

6、建议

本例使用的是现输出到kafka，然后传到logstash，在传入es。

 

7、QA

使用过程中出现http503  queue is full，参考官方解决方案调整events的大小、worker大小、bulk_max_size大小。

https://www.elastic.co/guide/en/apm/server/master/common-problems.html

参考：

https://www.elastic.co/products

https://www.elastic.co/guide/en/apm/server/master/common-problems.html