使用skywalking在Envoy中进行链路追踪

系统环境

网络：172.31.81.0/24

服务：

Front-Proxy：前端代理，监听端口8000/tcp
2个后端服务
- service1：接收Front-Envoy的请求，并会请求service2
- service2：接收service1的请求
追踪服务zipkin，监听端口9411

环境配置

server-demo

service.py

from flask import Flask
from flask import request
import os
import requests
import socket
import sys

app = Flask(__name__)

TRACE_HEADERS_TO_PROPAGATE = [
    'X-Ot-Span-Context',
    'X-Request-Id',

    # Zipkin headers
    'X-B3-TraceId',
    'X-B3-SpanId',
    'X-B3-ParentSpanId',
    'X-B3-Sampled',
    'X-B3-Flags',

    # Jaeger header (for native client)
    "uber-trace-id",

    # SkyWalking headers.
    "sw8"
]


@app.route('/service/<service_number>')
def hello(service_number):
    return (
        'Hello from behind Envoy (service {})! hostname: {} resolved'
        'hostname: {}\n'.format(
            os.environ['SERVICE_NAME'], socket.gethostname(),
            socket.gethostbyname(socket.gethostname())))


@app.route('/trace/<service_number>')
def trace(service_number):
    headers = {}
    # call service 2 from service 1
    if int(os.environ['SERVICE_NAME']) == 1:
        for header in TRACE_HEADERS_TO_PROPAGATE:
            if header in request.headers:
                headers[header] = request.headers[header]
        requests.get("http://localhost:9000/trace/2", headers=headers)
    return (
        'Hello from behind Envoy (service {})! hostname: {} resolved'
        'hostname: {}\n'.format(
            os.environ['SERVICE_NAME'], socket.gethostname(),
            socket.gethostbyname(socket.gethostname())))


if __name__ == "__main__":
    app.run(host='127.0.0.1', port=8080, debug=True)

Dockerfile-service

FROM envoyproxy/envoy-alpine:v1.21-latest

ADD repositories /etc/apk/repositories
RUN apk update && apk --no-cache -U add py3-pip bash curl
RUN python3 -m venv /code && source /code/bin/activate &&  pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple --upgrade pip && \
  pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple -q click==8.0.4 flask==2.1.3 itsdangerous==2.1.1 jinja2==3.0.3 markupsafe==2.1.1 pyyaml==6.0 werkzeug==2.0.3 requests==2.21.0
ADD ./service.py /code
ADD ./start_service.sh /usr/local/bin/start_service.sh
RUN chmod u+x /usr/local/bin/start_service.sh
ENTRYPOINT ["/bin/sh", "/usr/local/bin/start_service.sh"]

start_service.sh

#!/bin/sh
source /code/bin/activate &&  python3 /code/service.py &
envoy -c /etc/service-envoy.yaml --service-cluster "service${SERVICE_NAME}"

front-envoy.yaml

node:
  id: front-envoy
  cluster: front-proxy

admin:
  profile_path: /tmp/envoy.prof
  access_log_path: /tmp/admin_access.log
  address:
    socket_address:
       address: 0.0.0.0
       port_value: 9901

layered_runtime:
  layers:
  - name: admin
    admin_layer: {}

# MetricsServiceConfig
stats_sinks:
- name: envoy.stat_sinks.metrics_service
  typed_config:
    "@type": type.googleapis.com/envoy.config.metrics.v3.MetricsServiceConfig
    transport_api_version: V3
    grpc_service:
      envoy_grpc:
        cluster_name: skywalking

static_resources:
  listeners:
  - address:
      socket_address:
        address: 0.0.0.0
        port_value: 8000
    traffic_direction: OUTBOUND
    filter_chains:
    - filters:
      - name: envoy.filters.network.http_connection_manager
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
          generate_request_id: true
          tracing:
            provider:
              name: envoy.tracers.skywalking
              typed_config:
                "@type": type.googleapis.com/envoy.config.trace.v3.SkyWalkingConfig
                grpc_service:
                  envoy_grpc:
                    cluster_name: skywalking
                  timeout: 0.250s  
                client_config:
                  service_name: front-envoy
                  instance_name: front-envoy-1
          codec_type: AUTO
          stat_prefix: ingress_http
          route_config:
            name: local_route
            virtual_hosts:
            - name: backend
              domains:
              - "*"
              routes:
              - match:
                  prefix: "/"
                route:
                  cluster: service1
                decorator:
                  operation: checkAvailability
          http_filters:
          - name: envoy.filters.http.router
            typed_config:
              "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
              start_child_span: true

  clusters:
  - name: service1
    type: STRICT_DNS
    lb_policy: ROUND_ROBIN
    typed_extension_protocol_options:
      envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
        "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
        explicit_http_config:
          http2_protocol_options: {}
    load_assignment:
      cluster_name: service1
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: service1
                port_value: 8000
  
  - name: skywalking
    type: STRICT_DNS
    lb_policy: ROUND_ROBIN
    typed_extension_protocol_options:
      envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
        "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
        explicit_http_config:
          http2_protocol_options: {}
    load_assignment:
      cluster_name: skywalking
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: skywalking-oap
                port_value: 11800

service1-envoy-sidecar.yaml

admin:
  address:
    socket_address:
      protocol: TCP
      address: 0.0.0.0
      port_value: 19000
      
# MetricsServiceConfig      
stats_sinks:
- name: envoy.stat_sinks.metrics_service
  typed_config:
    "@type": type.googleapis.com/envoy.config.metrics.v3.MetricsServiceConfig
    transport_api_version: V3
    grpc_service:
      # Note: we can use google_grpc implementation as well.
      envoy_grpc:
        cluster_name: skywalking


static_resources:
  listeners:
  - address:
      socket_address:
        address: 0.0.0.0
        port_value: 8000
    traffic_direction: INBOUND
    filter_chains:
    - filters:
      - name: envoy.filters.network.http_connection_manager
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
          tracing:
            provider:
              name: envoy.tracers.skywalking
              typed_config:
                "@type": type.googleapis.com/envoy.config.trace.v3.SkyWalkingConfig
                grpc_service:
                  envoy_grpc:
                    cluster_name: skywalking
                  timeout: 0.250s  
                client_config:
                  service_name: service1-envoy
                  instance_name: service1-envoy-01
          codec_type: AUTO
          stat_prefix: ingress_http
          route_config:
            name: service1_route
            virtual_hosts:
            - name: service1
              domains:
              - "*"
              routes:
              - match:
                  prefix: "/"
                route:
                  cluster: local_service
                decorator:
                  operation: checkAvailability
          http_filters:
          - name: envoy.filters.http.router
            typed_config:
              "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
              start_child_span: true
  - address:
      socket_address:
        address: 0.0.0.0
        port_value: 9000
    traffic_direction: OUTBOUND
    filter_chains:
    - filters:
      - name: envoy.filters.network.http_connection_manager
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
          tracing:
            provider:
              name: envoy.tracers.skywalking
              typed_config:
                "@type": type.googleapis.com/envoy.config.trace.v3.SkyWalkingConfig
                grpc_service:
                  envoy_grpc:
                    cluster_name: skywalking
                  timeout: 0.250s  
                client_config:
                  service_name: service1-envoy
                  instance_name: service1-envoy-01
          codec_type: AUTO
          stat_prefix: egress_http
          route_config:
            name: service2_route
            virtual_hosts:
            - name: service2
              domains:
              - "*"
              routes:
              - match:
                  prefix: "/trace/2"
                route:
                  cluster: service2
                decorator:
                  operation: checkStock
          http_filters:
          - name: envoy.filters.http.router
            typed_config:
              "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
              start_child_span: true
            
  clusters:
  - name: local_service
    type: STRICT_DNS
    lb_policy: ROUND_ROBIN
    load_assignment:
      cluster_name: local_service
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: 127.0.0.1
                port_value: 8080
  - name: service2
    type: STRICT_DNS
    lb_policy: ROUND_ROBIN
    load_assignment:
      cluster_name: service2
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: service2
                port_value: 8000

  - name: skywalking
    type: STRICT_DNS
    lb_policy: ROUND_ROBIN
    typed_extension_protocol_options:
      envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
        "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
        explicit_http_config:
          http2_protocol_options: {}
    load_assignment:
      cluster_name: skywalking
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: skywalking-oap
                port_value: 11800

service2-envoy-sidecar.yaml

admin:
  address:
    socket_address:
      protocol: TCP
      address: 0.0.0.0
      port_value: 19000

# MetricsServiceConfig
stats_sinks:
- name: envoy.stat_sinks.metrics_service
  typed_config:
    "@type": type.googleapis.com/envoy.config.metrics.v3.MetricsServiceConfig
    transport_api_version: V3
    grpc_service:
      # Note: we can use google_grpc implementation as well.
      envoy_grpc:
        cluster_name: skywalking
        
static_resources:
  listeners:
  - address:
      socket_address:
        address: 0.0.0.0
        port_value: 8000
    traffic_direction: INBOUND
    filter_chains:
    - filters:
      - name: envoy.filters.network.http_connection_manager
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
          tracing:
            provider:
              name: envoy.tracers.skywalking
              typed_config:
                "@type": type.googleapis.com/envoy.config.trace.v3.SkyWalkingConfig
                grpc_service:
                  envoy_grpc:
                    cluster_name: skywalking
                  timeout: 0.250s  
                client_config:
                  service_name: service2-envoy
                  instance_name: service2-envoy-01
          codec_type: AUTO
          stat_prefix: ingress_http
          route_config:
            name: local_route
            virtual_hosts:
            - name: service2
              domains:
              - "*"
              routes:
              - match:
                  prefix: "/"
                route:
                  cluster: local_service
                decorator:
                  operation: checkStock
          http_filters:
          - name: envoy.filters.http.router
            typed_config:
              "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
              start_child_span: true
  clusters:
  - name: local_service
    type: STRICT_DNS
    lb_policy: ROUND_ROBIN
    load_assignment:
      cluster_name: local_service
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: 127.0.0.1
                port_value: 8080

  - name: skywalking
    type: STRICT_DNS
    lb_policy: ROUND_ROBIN
    typed_extension_protocol_options:
      envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
        "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
        explicit_http_config:
          http2_protocol_options: {}
    load_assignment:
      cluster_name: skywalking
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: skywalking-oap
                port_value: 11800

docker-compose.yaml

version: "3"

services:
  front-envoy:
    image: envoyproxy/envoy:v1.23-latest
    volumes:
      - ./front-envoy/front-envoy-sidecar.yaml:/etc/envoy/envoy.yaml
    networks:
      envoymesh:
        ipv4_address: 172.31.81.10
        aliases:
        - front-envoy
        - front
    ports:
    - "8000:8000"
    - "9901:9901"
    depends_on:
    - skywalking-oap

  service1:
    build:
      context: ./service/
      dockerfile: Dockerfile-service
    volumes:
    - ./service/service1-envoy-sidecar.yaml:/etc/service-envoy.yaml
    networks:
      envoymesh:
        aliases:
        - service1
    environment:
    - SERVICE_NAME=1
    depends_on:
    - skywalking-oap

  service2:
    build:
      context: ./service/
      dockerfile: Dockerfile-service
    volumes:
    - ./service/service2-envoy-sidecar.yaml:/etc/service-envoy.yaml
    networks:
      envoymesh:
        aliases:
        - service2
    environment:
    - SERVICE_NAME=2
    depends_on:
    - skywalking-oap


  # Skywalking components.
  elasticsearch:
    image: elasticsearch:8.4.2
    container_name: elasticsearch
    ports:
      - "9200:9200"
    healthcheck:
      test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    environment:
      discovery.type: single-node
      ingest.geoip.downloader.enabled: "false"
      bootstrap.memory_lock: true
      ES_JAVA_OPTS: "-Xms512m -Xmx512m"
      TZ: "Asia/Shanghai"
      # NB: This setting is for demo purposes only, you are strongly advised to configure
      #   Elasticsearch with security enabled
      xpack.security.enabled: "false"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    networks:
      envoymesh:
        aliases:
        - elasticsearch

  skywalking-oap:
    image: apache/skywalking-oap-server:9.1.0
    container_name: skywalking-oap
    depends_on:
      elasticsearch:
        condition: service_healthy
    environment:
      SW_HEALTH_CHECKER: default
      SW_STORAGE: elasticsearch
      SW_STORAGE_ES_CLUSTER_NODES: elasticsearch:9200
      JAVA_OPTS: "-Xms2048m -Xmx2048m"
      TZ: Asia/Shanghai
      SW_TELEMETRY: prometheus
    healthcheck:
      test: ["CMD-SHELL", "/skywalking/bin/swctl ch"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s
    restart: on-failure
    ports:
      - "11800:11800"
      - "12800:12800"
    networks:
      envoymesh:
        aliases:
        - skywalking-oap

  skywalking-ui:
    image: apache/skywalking-ui:9.1.0
    healthcheck:
      test: ["CMD-SHELL", 'curl --silent --fail http://localhost:8080/graphql -X POST -H "Content-Type:application/json" -d "{ \"query\": \"query version { version }\"}" || exit 1']
      interval: 5s
      timeout: 10s
      retries: 10
      start_period: 10s
    depends_on:
      skywalking-oap:
        condition: service_healthy
    ports:
    - "8080:8080"
    environment:
      SW_OAP_ADDRESS: http://skywalking-oap:12800
      SW_HEALTH_CHECKER: default
      TZ: Asia/Shanghai
    networks:
      envoymesh:
        aliases:
        - skywalking-ui


networks:
  envoymesh:
    driver: bridge
    ipam:
      config:
        - subnet: 172.31.81.0/24

运行服务

docker-compose build
docker-compose up

请求测试

~# curl -v 172.31.81.10:8000/trace/1
*   Trying 172.31.81.10:8000...
* TCP_NODELAY set
* Connected to 172.31.81.10 (172.31.81.10) port 8000 (#0)
> GET /trace/1 HTTP/1.1
> Host: 172.31.81.10:8000
> User-Agent: curl/7.68.0
> Accept: */*
> 
* Mark bundle as not supporting multiuse
< HTTP/1.1 200 OK
< content-type: text/html; charset=utf-8
< content-length: 90
< server: envoy
< date: Wed, 21 Sep 2022 06:33:20 GMT
< x-envoy-upstream-service-time: 28
< 
Hello from behind Envoy (service 1)! hostname: 92f38314684d resolvedhostname: 172.31.81.5
* Connection #0 to host 172.31.81.10 left intact

skywalking-service2-1       | 127.0.0.1 - - [21/Sep/2022 06:33:20] "GET /trace/2 HTTP/1.1" 200 -
skywalking-service1-1       | 127.0.0.1 - - [21/Sep/2022 06:33:20] "GET /trace/1 HTTP/1.1" 200 -