使用Zipkin在Envoy中进行链路追踪

系统环境

网络：172.31.81.0/24
服务：
  • Front-Proxy：前端代理，监听端口8000/tcp
  • 2个后端服务
    • service1：接收Front-Envoy的请求，并会请求service2
    • service2：接收service1的请求
  • 追踪服务zipkin，监听端口9411

环境配置

server-demo

service.py

from flask import Flask
from flask import request
import os
import requests
import socket
import sys

app = Flask(__name__)

TRACE_HEADERS_TO_PROPAGATE = [
    'X-Ot-Span-Context',
    'X-Request-Id',

    # Zipkin headers
    'X-B3-TraceId',
    'X-B3-SpanId',
    'X-B3-ParentSpanId',
    'X-B3-Sampled',
    'X-B3-Flags',

    # Jaeger header (for native client)
    "uber-trace-id",

    # SkyWalking headers.
    "sw8"
]


@app.route('/service/<service_number>')
def hello(service_number):
    return (
        'Hello from behind Envoy (service {})! hostname: {} resolved'
        'hostname: {}\n'.format(
            os.environ['SERVICE_NAME'], socket.gethostname(),
            socket.gethostbyname(socket.gethostname())))


@app.route('/trace/<service_number>')
def trace(service_number):
    headers = {}
    # call service 2 from service 1
    if int(os.environ['SERVICE_NAME']) == 1:
        for header in TRACE_HEADERS_TO_PROPAGATE:
            if header in request.headers:
                headers[header] = request.headers[header]
        requests.get("http://localhost:9000/trace/2", headers=headers)
    return (
        'Hello from behind Envoy (service {})! hostname: {} resolved'
        'hostname: {}\n'.format(
            os.environ['SERVICE_NAME'], socket.gethostname(),
            socket.gethostbyname(socket.gethostname())))


if __name__ == "__main__":
    app.run(host='127.0.0.1', port=8080, debug=True)

Dockerfile-service

FROM envoyproxy/envoy-alpine:v1.21-latest

ADD repositories /etc/apk/repositories
RUN apk update && apk --no-cache -U add py3-pip bash curl
RUN python3 -m venv /code && source /code/bin/activate &&  pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple --upgrade pip && \
  pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple -q click==8.0.4 flask==2.1.3 itsdangerous==2.1.1 jinja2==3.0.3 markupsafe==2.1.1 pyyaml==6.0 werkzeug==2.0.3 requests==2.21.0
ADD ./service.py /code
ADD ./start_service.sh /usr/local/bin/start_service.sh
RUN chmod u+x /usr/local/bin/start_service.sh
ENTRYPOINT ["/bin/sh", "/usr/local/bin/start_service.sh"]

start_service.sh

#!/bin/sh
source /code/bin/activate &&  python3 /code/service.py &
envoy -c /etc/service-envoy.yaml --service-cluster "service${SERVICE_NAME}"

front-envoy.yaml

node:
  id: front-envoy
  cluster: front-proxy

admin:
  profile_path: /tmp/envoy.prof
  access_log_path: /tmp/admin_access.log
  address:
    socket_address:
       address: 0.0.0.0
       port_value: 9901

layered_runtime:
  layers:
  - name: admin
    admin_layer: {}

static_resources:
  listeners:
  - address:
      socket_address:
        address: 0.0.0.0
        port_value: 8000
    traffic_direction: OUTBOUND
    filter_chains:
    - filters:
      - name: envoy.filters.network.http_connection_manager
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
          generate_request_id: true
          tracing:
            provider:
              name: envoy.tracers.zipkin
              typed_config:
                "@type": type.googleapis.com/envoy.config.trace.v3.ZipkinConfig
                collector_cluster: zipkin
                collector_endpoint: "/api/v2/spans"
                collector_endpoint_version: HTTP_JSON
          codec_type: AUTO
          stat_prefix: ingress_http
          route_config:
            name: local_route
            virtual_hosts:
            - name: backend
              domains:
              - "*"
              routes:
              - match:
                  prefix: "/"
                route:
                  cluster: service1
                decorator:
                  operation: check_server1
              response_headers_to_add:
              - header:
                  key: "x-b3-traceid"
                  value: "%REQ(x-b3-traceid)%"
              - header:
                  key: "x-request-id"
                  value: "%REQ(x-request-id)%"
          http_filters:
          - name: envoy.filters.http.router
            typed_config:
              "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router

  clusters:
  - name: service1
    type: STRICT_DNS
    lb_policy: ROUND_ROBIN
    load_assignment:
      cluster_name: service1
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: service1
                port_value: 8000
  - name: zipkin
    type: STRICT_DNS
    lb_policy: ROUND_ROBIN
    load_assignment:
      cluster_name: zipkin
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: zipkin
                port_value: 9411

service1-envoy-sidecar.yaml

static_resources:
  listeners:
  - address:
      socket_address:
        address: 0.0.0.0
        port_value: 8000
    traffic_direction: INBOUND
    filter_chains:
    - filters:
      - name: envoy.filters.network.http_connection_manager
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
          tracing:
            provider:
              name: envoy.tracers.zipkin
              typed_config:
                "@type": type.googleapis.com/envoy.config.trace.v3.ZipkinConfig
                collector_cluster: zipkin
                collector_endpoint: "/api/v2/spans"
                collector_endpoint_version: HTTP_JSON
          codec_type: AUTO
          stat_prefix: ingress_http
          route_config:
            name: service1_route
            virtual_hosts:
            - name: service1
              domains:
              - "*"
              routes:
              - match:
                  prefix: "/"
                route:
                  cluster: local_service
                decorator:
                  operation: check_server1
          http_filters:
          - name: envoy.filters.http.router
  - address:
      socket_address:
        address: 0.0.0.0
        port_value: 9000
    traffic_direction: OUTBOUND
    filter_chains:
    - filters:
      - name: envoy.filters.network.http_connection_manager
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
          tracing:
            provider:
              name: envoy.tracers.zipkin
              typed_config:
                "@type": type.googleapis.com/envoy.config.trace.v3.ZipkinConfig
                collector_cluster: zipkin
                collector_endpoint: "/api/v2/spans"
                collector_endpoint_version: HTTP_JSON
          codec_type: AUTO
          stat_prefix: egress_http
          route_config:
            name: service2_route
            virtual_hosts:
            - name: service2
              domains:
              - "*"
              routes:
              - match:
                  prefix: "/trace/2"
                route:
                  cluster: service2
                decorator:
                  operation: check_server2
          http_filters:
          - name: envoy.filters.http.router
  clusters:
  - name: local_service
    type: STRICT_DNS
    lb_policy: ROUND_ROBIN
    load_assignment:
      cluster_name: local_service
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: 127.0.0.1
                port_value: 8080
  - name: service2
    type: STRICT_DNS
    lb_policy: ROUND_ROBIN
    load_assignment:
      cluster_name: service2
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: service2
                port_value: 8000
  - name: zipkin
    type: STRICT_DNS
    lb_policy: ROUND_ROBIN
    load_assignment:
      cluster_name: zipkin
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: zipkin
                port_value: 9411
admin:
  address:
    socket_address:
      protocol: TCP
      address: 0.0.0.0
      port_value: 19000

service2-envoy-sidecar.yaml

static_resources:
  listeners:
  - address:
      socket_address:
        address: 0.0.0.0
        port_value: 8000
    traffic_direction: INBOUND
    filter_chains:
    - filters:
      - name: envoy.filters.network.http_connection_manager
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
          tracing:
            provider:
              name: envoy.tracers.zipkin
              typed_config:
                "@type": type.googleapis.com/envoy.config.trace.v3.ZipkinConfig
                collector_cluster: zipkin
                collector_endpoint: "/api/v2/spans"
                collector_endpoint_version: HTTP_JSON
          codec_type: AUTO
          stat_prefix: ingress_http
          route_config:
            name: local_route
            virtual_hosts:
            - name: service2
              domains:
              - "*"
              routes:
              - match:
                  prefix: "/"
                route:
                  cluster: local_service
                decorator:
                  operation: check_server2
          http_filters:
          - name: envoy.filters.http.router
  clusters:
  - name: local_service
    type: STRICT_DNS
    lb_policy: ROUND_ROBIN
    load_assignment:
      cluster_name: local_service
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: 127.0.0.1
                port_value: 8080
  - name: zipkin
    type: STRICT_DNS
    lb_policy: ROUND_ROBIN
    load_assignment:
      cluster_name: zipkin
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: zipkin
                port_value: 9411

docker-compose.yaml

version: "3"

services:
  front-envoy:
    image: envoyproxy/envoy:v1.23-latest
    volumes:
      - ./front-envoy/front-envoy.yaml:/etc/envoy/envoy.yaml
    networks:
      envoymesh:
        ipv4_address: 172.31.81.10
        aliases:
        - front-envoy
        - front
    ports:
    - "8000:8000"
    - "9901:9901"

  service1:
    build:
      context: ./service/
      dockerfile: Dockerfile-service
    volumes:
    - ./service/service1-envoy-sidecar.yaml:/etc/service-envoy.yaml
    networks:
      envoymesh:
        aliases:
        - service1
    environment:
    - SERVICE_NAME=1

  service2:
    build:
      context: ./service/
      dockerfile: Dockerfile-service
    volumes:
    - ./service/service2-envoy-sidecar.yaml:/etc/service-envoy.yaml
    networks:
      envoymesh:
        aliases:
        - service2
    environment:
    - SERVICE_NAME=2

  zipkin:
    image: openzipkin/zipkin:2
    networks:
      envoymesh:
        ipv4_address: 172.31.81.15
        aliases:
        - zipkin
    ports:
    - "9411:9411"

networks:
  envoymesh:
    driver: bridge
    ipam:
      config:
        - subnet: 172.31.81.0/24

运行服务

docker-compose build
docker-compose up

请求测试

~# curl -v 172.31.81.10:8000/trace/1
*   Trying 172.31.81.10:8000...
* TCP_NODELAY set
* Connected to 172.31.81.10 (172.31.81.10) port 8000 (#0)
> GET /trace/1 HTTP/1.1
> Host: 172.31.81.10:8000
> User-Agent: curl/7.68.0
> Accept: */*
> 
* Mark bundle as not supporting multiuse
< HTTP/1.1 200 OK
< content-type: text/html; charset=utf-8
< content-length: 90
< server: envoy
< date: Tue, 20 Sep 2022 09:24:33 GMT
< x-envoy-upstream-service-time: 15
< x-b3-traceid: 083b81a82b2cdc89
< x-request-id: 5951ce0d-8b58-96bf-b4a7-e8d8336907e3
< 
Hello from behind Envoy (service 1)! hostname: 4ee081ef8d91 resolvedhostname: 172.31.81.3
* Connection #0 to host 172.31.81.10 left intact
zipkin-tracing-basics-service2-1     | 127.0.0.1 - - [20/Sep/2022 09:22:45] "GET /trace/2 HTTP/1.1" 200 -
zipkin-tracing-basics-service1-1     | 127.0.0.1 - - [20/Sep/2022 09:22:45] "GET /trace/1 HTTP/1.1" 200 -
zipkin-tracing-basics-service2-1     | 127.0.0.1 - - [20/Sep/2022 09:22:46] "GET /trace/2 HTTP/1.1" 200 -
zipkin-tracing-basics-service1-1     | 127.0.0.1 - - [20/Sep/2022 09:22:46] "GET /trace/1 HTTP/1.1" 200 -
zipkin-tracing-basics-service2-1     | 127.0.0.1 - - [20/Sep/2022 09:22:46] "GET /trace/2 HTTP/1.1" 200 -
zipkin-tracing-basics-service1-1     | 127.0.0.1 - - [20/Sep/2022 09:22:46] "GET /trace/1 HTTP/1.1" 200 -
zipkin-tracing-basics-service2-1     | 127.0.0.1 - - [20/Sep/2022 09:22:47] "GET /trace/2 HTTP/1.1" 200 -
zipkin-tracing-basics-service1-1     | 127.0.0.1 - - [20/Sep/2022 09:22:47] "GET /trace/1 HTTP/1.1" 200 -
zipkin-tracing-basics-service2-1     | 127.0.0.1 - - [20/Sep/2022 09:22:47] "GET /trace/2 HTTP/1.1" 200 -
zipkin-tracing-basics-service1-1     | 127.0.0.1 - - [20/Sep/2022 09:22:47] "GET /trace/1 HTTP/1.1" 200 -
zipkin-tracing-basics-service2-1     | 127.0.0.1 - - [20/Sep/2022 09:24:31] "GET /trace/2 HTTP/1.1" 200 -
zipkin-tracing-basics-service1-1     | 127.0.0.1 - - [20/Sep/2022 09:24:31] "GET /trace/1 HTTP/1.1" 200 -
zipkin-tracing-basics-service2-1     | 127.0.0.1 - - [20/Sep/2022 09:24:32] "GET /trace/2 HTTP/1.1" 200 -
zipkin-tracing-basics-service1-1     | 127.0.0.1 - - [20/Sep/2022 09:24:32] "GET /trace/1 HTTP/1.1" 200 -
zipkin-tracing-basics-service2-1     | 127.0.0.1 - - [20/Sep/2022 09:24:33] "GET /trace/2 HTTP/1.1" 200 -
zipkin-tracing-basics-service1-1     | 127.0.0.1 - - [20/Sep/2022 09:24:33] "GET /trace/1 HTTP/1.1" 200 -

访问zipkin

参考文档

https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/trace/v3/zipkin.proto#extension-envoy-tracers-zipkin

https://github.com/envoyproxy/envoy/tree/main/examples/zipkin

posted @ 2022-09-20 17:28  小吉猫  阅读(201)  评论(0)  编辑  收藏  举报