Istio可观测性
获取命令
kubectl exec sleep-78ff5975c6-75q5z -c istio-proxy -- pilot-agent request GET /stats # stats格式
kubectl exec sleep-78ff5975c6-75q5z -c istio-proxy -- pilot-agent request GET /stats/prometheus # prometheus格式
kubectl exec sleep-78ff5975c6-75q5z -- curl localhost:15000/stats
查看启用的统计指标
[root@master ~]# istioctl proxy-config bootstrap sleep-78ff5975c6-75q5z.default | jq .bootstrap.statsConfig
{
"statsTags": [
{
"tagName": "cluster_name",
"regex": "^cluster\\.((.+?(\\..+?\\.svc\\.cluster\\.local)?)\\.)"
},
{
"tagName": "tcp_prefix",
"regex": "^tcp\\.((.*?)\\.)\\w+?$"
},
{
"tagName": "response_code",
"regex": "(response_code=\\.=(.+?);\\.;)"
},
{
"tagName": "response_code",
"regex": "_rq(_(\\d{3}))$"
},
{
"tagName": "response_code_class",
"regex": "_rq(_(\\dxx))$"
},
{
"tagName": "http_conn_manager_listener_prefix",
"regex": "^listener(?=\\.).*?\\.http\\.(((?:[_.[:digit:]]*|[_\\[\\]aAbBcCdDeEfF[:digit:]]*))\\.)"
},
{
"tagName": "http_conn_manager_prefix",
"regex": "^http\\.(((?:[_.[:digit:]]*|[_\\[\\]aAbBcCdDeEfF[:digit:]]*))\\.)"
},
{
"tagName": "listener_address",
"regex": "^listener\\.(((?:[_.[:digit:]]*|[_\\[\\]aAbBcCdDeEfF[:digit:]]*))\\.)"
},
{
"tagName": "mongo_prefix",
"regex": "^mongo\\.(.+?)\\.(collection|cmd|cx_|op_|delays_|decoding_)(.*?)$"
},
{
"tagName": "reporter",
"regex": "(reporter=\\.=(.*?);\\.;)"
},
{
"tagName": "source_namespace",
"regex": "(source_namespace=\\.=(.*?);\\.;)"
},
{
"tagName": "source_workload",
"regex": "(source_workload=\\.=(.*?);\\.;)"
},
{
"tagName": "source_workload_namespace",
"regex": "(source_workload_namespace=\\.=(.*?);\\.;)"
},
{
"tagName": "source_principal",
"regex": "(source_principal=\\.=(.*?);\\.;)"
},
{
"tagName": "source_app",
"regex": "(source_app=\\.=(.*?);\\.;)"
},
{
"tagName": "source_version",
"regex": "(source_version=\\.=(.*?);\\.;)"
},
{
"tagName": "source_cluster",
"regex": "(source_cluster=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_namespace",
"regex": "(destination_namespace=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_workload",
"regex": "(destination_workload=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_workload_namespace",
"regex": "(destination_workload_namespace=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_principal",
"regex": "(destination_principal=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_app",
"regex": "(destination_app=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_version",
"regex": "(destination_version=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_service",
"regex": "(destination_service=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_service_name",
"regex": "(destination_service_name=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_service_namespace",
"regex": "(destination_service_namespace=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_port",
"regex": "(destination_port=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_cluster",
"regex": "(destination_cluster=\\.=(.*?);\\.;)"
},
{
"tagName": "request_protocol",
"regex": "(request_protocol=\\.=(.*?);\\.;)"
},
{
"tagName": "request_operation",
"regex": "(request_operation=\\.=(.*?);\\.;)"
},
{
"tagName": "request_host",
"regex": "(request_host=\\.=(.*?);\\.;)"
},
{
"tagName": "response_flags",
"regex": "(response_flags=\\.=(.*?);\\.;)"
},
{
"tagName": "grpc_response_status",
"regex": "(grpc_response_status=\\.=(.*?);\\.;)"
},
{
"tagName": "connection_security_policy",
"regex": "(connection_security_policy=\\.=(.*?);\\.;)"
},
{
"tagName": "source_canonical_service",
"regex": "(source_canonical_service=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_canonical_service",
"regex": "(destination_canonical_service=\\.=(.*?);\\.;)"
},
{
"tagName": "source_canonical_revision",
"regex": "(source_canonical_revision=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_canonical_revision",
"regex": "(destination_canonical_revision=\\.=(.*?);\\.;)"
},
{
"tagName": "cache",
"regex": "(cache\\.(.+?)\\.)"
},
{
"tagName": "component",
"regex": "(component\\.(.+?)\\.)"
},
{
"tagName": "tag",
"regex": "(tag\\.(.+?);\\.)"
},
{
"tagName": "wasm_filter",
"regex": "(wasm_filter\\.(.+?)\\.)"
},
{
"tagName": "authz_enforce_result",
"regex": "rbac(\\.(allowed|denied))"
},
{
"tagName": "authz_dry_run_action",
"regex": "(\\.istio_dry_run_(allow|deny)_)"
},
{
"tagName": "authz_dry_run_result",
"regex": "(\\.shadow_(allowed|denied))"
}
],
"useAllDefaultTags": false,
"statsMatcher": {
"inclusionList": {
"patterns": [
{
"prefix": "reporter="
},
{
"prefix": "cluster_manager"
},
{
"prefix": "listener_manager"
},
{
"prefix": "server"
},
{
"prefix": "cluster.xds-grpc"
},
{
"prefix": "wasm"
},
{
"suffix": "rbac.allowed"
},
{
"suffix": "rbac.denied"
},
{
"suffix": "shadow_allowed"
},
{
"suffix": "shadow_denied"
},
{
"safeRegex": {
"googleRe2": {},
"regex": "vhost\\.*\\.route\\.*"
}
},
{
"prefix": "component"
},
{
"prefix": "istio"
}
]
}
}
}
在网格上配置代理级指标
在网格上启用更多的统计指标
Istio上与网格代理相关的默认配置定义在MeshConfig.defaultConfig配置段中，而每个Pod之上Sidecar Envoy的指标定义则要通过Annotations完成
要自定义启用的统计指标可以使用MeshConfig.defautConfig.proxyStatsMatcher进行定义
inclusionRegexps:基于正则表达式模式指定要启用的统计指标键
inclusionPrefixes:基于前缀匹配指定要启用的统计指标键
inclusionSuffixes:基于后缀匹配指定要启用的统计指标键
第一种
[root@master ~]# vim profile-demo.yaml
meshConfig:
outboundTrafficPolicy:
mode: REGISTRY_ONLY
accessLogFile: /dev/stdout
defaultConfig:
proxyMetadata: {}
proxyStatsMatcher:
inclusionPrefixes:
- "upstream_rq_retry"
- "upstream_cx"第二种
meshConfig:
outboundTrafficPolicy:
mode: REGISTRY_ONLY
accessLogFile: /dev/stdout
defaultConfig:
proxyMetadata: {}
proxyStatsMatcher:
inclusionRegexps:
- ".*circuit_breakers.*"
inclusionPrefixes:
- "upstream_rq_retry"
- "upstream_cx"
[root@master ~]# istioctl apply -f profile-demo.yaml
This will install the Istio 1.16.0 demo profile with ["Istio core" "Istiod" "Ingress gateways" "Egress gateways"] components into the cluster. Proceed? (y/N) y
✔ Istio core installed
✔ Istiod installed
✔ Ingress gateways installed
✔ Egress gateways installed
✔ Installation complete Making this installation the default for injection and validation.
Thank you for installing Istio 1.16. Please take a few minutes to tell us about your install/upgrade experience! https://forms.gle/99uiMML96AmsXY5d6前后对比
在bootstrap文件生成时进行配置的，因而只会对配置后创建的Pod有效
[root@master ~]# istioctl proxy-config bootstrap client-74cf5679fd-vrqjj | jq .bootstrap.statsConfig
{
"statsTags": [
{
"tagName": "cluster_name",
"regex": "^cluster\\.((.+?(\\..+?\\.svc\\.cluster\\.local)?)\\.)"
},
{
"tagName": "tcp_prefix",
"regex": "^tcp\\.((.*?)\\.)\\w+?$"
},
{
"tagName": "response_code",
"regex": "(response_code=\\.=(.+?);\\.;)"
},
{
"tagName": "response_code",
"regex": "_rq(_(\\d{3}))$"
},
{
"tagName": "response_code_class",
"regex": "_rq(_(\\dxx))$"
},
{
"tagName": "http_conn_manager_listener_prefix",
"regex": "^listener(?=\\.).*?\\.http\\.(((?:[_.[:digit:]]*|[_\\[\\]aAbBcCdDeEfF[:digit:]]*))\\.)"
},
{
"tagName": "http_conn_manager_prefix",
"regex": "^http\\.(((?:[_.[:digit:]]*|[_\\[\\]aAbBcCdDeEfF[:digit:]]*))\\.)"
},
{
"tagName": "listener_address",
"regex": "^listener\\.(((?:[_.[:digit:]]*|[_\\[\\]aAbBcCdDeEfF[:digit:]]*))\\.)"
},
{
"tagName": "mongo_prefix",
"regex": "^mongo\\.(.+?)\\.(collection|cmd|cx_|op_|delays_|decoding_)(.*?)$"
},
{
"tagName": "reporter",
"regex": "(reporter=\\.=(.*?);\\.;)"
},
{
"tagName": "source_namespace",
"regex": "(source_namespace=\\.=(.*?);\\.;)"
},
{
"tagName": "source_workload",
"regex": "(source_workload=\\.=(.*?);\\.;)"
},
{
"tagName": "source_workload_namespace",
"regex": "(source_workload_namespace=\\.=(.*?);\\.;)"
},
{
"tagName": "source_principal",
"regex": "(source_principal=\\.=(.*?);\\.;)"
},
{
"tagName": "source_app",
"regex": "(source_app=\\.=(.*?);\\.;)"
},
{
"tagName": "source_version",
"regex": "(source_version=\\.=(.*?);\\.;)"
},
{
"tagName": "source_cluster",
"regex": "(source_cluster=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_namespace",
"regex": "(destination_namespace=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_workload",
"regex": "(destination_workload=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_workload_namespace",
"regex": "(destination_workload_namespace=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_principal",
"regex": "(destination_principal=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_app",
"regex": "(destination_app=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_version",
"regex": "(destination_version=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_service",
"regex": "(destination_service=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_service_name",
"regex": "(destination_service_name=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_service_namespace",
"regex": "(destination_service_namespace=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_port",
"regex": "(destination_port=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_cluster",
"regex": "(destination_cluster=\\.=(.*?);\\.;)"
},
{
"tagName": "request_protocol",
"regex": "(request_protocol=\\.=(.*?);\\.;)"
},
{
"tagName": "request_operation",
"regex": "(request_operation=\\.=(.*?);\\.;)"
},
{
"tagName": "request_host",
"regex": "(request_host=\\.=(.*?);\\.;)"
},
{
"tagName": "response_flags",
"regex": "(response_flags=\\.=(.*?);\\.;)"
},
{
"tagName": "grpc_response_status",
"regex": "(grpc_response_status=\\.=(.*?);\\.;)"
},
{
"tagName": "connection_security_policy",
"regex": "(connection_security_policy=\\.=(.*?);\\.;)"
},
{
"tagName": "source_canonical_service",
"regex": "(source_canonical_service=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_canonical_service",
"regex": "(destination_canonical_service=\\.=(.*?);\\.;)"
},
{
"tagName": "source_canonical_revision",
"regex": "(source_canonical_revision=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_canonical_revision",
"regex": "(destination_canonical_revision=\\.=(.*?);\\.;)"
},
{
"tagName": "cache",
"regex": "(cache\\.(.+?)\\.)"
},
{
"tagName": "component",
"regex": "(component\\.(.+?)\\.)"
},
{
"tagName": "tag",
"regex": "(tag\\.(.+?);\\.)"
},
{
"tagName": "wasm_filter",
"regex": "(wasm_filter\\.(.+?)\\.)"
},
{
"tagName": "authz_enforce_result",
"regex": "rbac(\\.(allowed|denied))"
},
{
"tagName": "authz_dry_run_action",
"regex": "(\\.istio_dry_run_(allow|deny)_)"
},
{
"tagName": "authz_dry_run_result",
"regex": "(\\.shadow_(allowed|denied))"
}
],
"useAllDefaultTags": false,
"statsMatcher": {
"inclusionList": {
"patterns": [
{
"prefix": "reporter="
},
{
"prefix": "cluster_manager"
},
{
"prefix": "listener_manager"
},
{
"prefix": "server"
},
{
"prefix": "cluster.xds-grpc"
},
{
"prefix": "wasm"
},
{
"suffix": "rbac.allowed"
},
{
"suffix": "rbac.denied"
},
{
"suffix": "shadow_allowed"
},
{
"suffix": "shadow_denied"
},
{
"safeRegex": {
"googleRe2": {},
"regex": "vhost\\.*\\.route\\.*"
}
},
{
"prefix": "component"
},
{
"prefix": "istio"
}
]
}
}
}
[root@master ~]# istioctl proxy-config bootstrap client-74cf5679fd-68xpm | jq .bootstrap.statsConfig
{
"statsTags": [
{
"tagName": "cluster_name",
"regex": "^cluster\\.((.+?(\\..+?\\.svc\\.cluster\\.local)?)\\.)"
},
{
"tagName": "tcp_prefix",
"regex": "^tcp\\.((.*?)\\.)\\w+?$"
},
{
"tagName": "response_code",
"regex": "(response_code=\\.=(.+?);\\.;)"
},
{
"tagName": "response_code",
"regex": "_rq(_(\\d{3}))$"
},
{
"tagName": "response_code_class",
"regex": "_rq(_(\\dxx))$"
},
{
"tagName": "http_conn_manager_listener_prefix",
"regex": "^listener(?=\\.).*?\\.http\\.(((?:[_.[:digit:]]*|[_\\[\\]aAbBcCdDeEfF[:digit:]]*))\\.)"
},
{
"tagName": "http_conn_manager_prefix",
"regex": "^http\\.(((?:[_.[:digit:]]*|[_\\[\\]aAbBcCdDeEfF[:digit:]]*))\\.)"
},
{
"tagName": "listener_address",
"regex": "^listener\\.(((?:[_.[:digit:]]*|[_\\[\\]aAbBcCdDeEfF[:digit:]]*))\\.)"
},
{
"tagName": "mongo_prefix",
"regex": "^mongo\\.(.+?)\\.(collection|cmd|cx_|op_|delays_|decoding_)(.*?)$"
},
{
"tagName": "reporter",
"regex": "(reporter=\\.=(.*?);\\.;)"
},
{
"tagName": "source_namespace",
"regex": "(source_namespace=\\.=(.*?);\\.;)"
},
{
"tagName": "source_workload",
"regex": "(source_workload=\\.=(.*?);\\.;)"
},
{
"tagName": "source_workload_namespace",
"regex": "(source_workload_namespace=\\.=(.*?);\\.;)"
},
{
"tagName": "source_principal",
"regex": "(source_principal=\\.=(.*?);\\.;)"
},
{
"tagName": "source_app",
"regex": "(source_app=\\.=(.*?);\\.;)"
},
{
"tagName": "source_version",
"regex": "(source_version=\\.=(.*?);\\.;)"
},
{
"tagName": "source_cluster",
"regex": "(source_cluster=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_namespace",
"regex": "(destination_namespace=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_workload",
"regex": "(destination_workload=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_workload_namespace",
"regex": "(destination_workload_namespace=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_principal",
"regex": "(destination_principal=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_app",
"regex": "(destination_app=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_version",
"regex": "(destination_version=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_service",
"regex": "(destination_service=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_service_name",
"regex": "(destination_service_name=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_service_namespace",
"regex": "(destination_service_namespace=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_port",
"regex": "(destination_port=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_cluster",
"regex": "(destination_cluster=\\.=(.*?);\\.;)"
},
{
"tagName": "request_protocol",
"regex": "(request_protocol=\\.=(.*?);\\.;)"
},
{
"tagName": "request_operation",
"regex": "(request_operation=\\.=(.*?);\\.;)"
},
{
"tagName": "request_host",
"regex": "(request_host=\\.=(.*?);\\.;)"
},
{
"tagName": "response_flags",
"regex": "(response_flags=\\.=(.*?);\\.;)"
},
{
"tagName": "grpc_response_status",
"regex": "(grpc_response_status=\\.=(.*?);\\.;)"
},
{
"tagName": "connection_security_policy",
"regex": "(connection_security_policy=\\.=(.*?);\\.;)"
},
{
"tagName": "source_canonical_service",
"regex": "(source_canonical_service=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_canonical_service",
"regex": "(destination_canonical_service=\\.=(.*?);\\.;)"
},
{
"tagName": "source_canonical_revision",
"regex": "(source_canonical_revision=\\.=(.*?);\\.;)"
},
{
"tagName": "destination_canonical_revision",
"regex": "(destination_canonical_revision=\\.=(.*?);\\.;)"
},
{
"tagName": "cache",
"regex": "(cache\\.(.+?)\\.)"
},
{
"tagName": "component",
"regex": "(component\\.(.+?)\\.)"
},
{
"tagName": "tag",
"regex": "(tag\\.(.+?);\\.)"
},
{
"tagName": "wasm_filter",
"regex": "(wasm_filter\\.(.+?)\\.)"
},
{
"tagName": "authz_enforce_result",
"regex": "rbac(\\.(allowed|denied))"
},
{
"tagName": "authz_dry_run_action",
"regex": "(\\.istio_dry_run_(allow|deny)_)"
},
{
"tagName": "authz_dry_run_result",
"regex": "(\\.shadow_(allowed|denied))"
}
],
"useAllDefaultTags": false,
"statsMatcher": {
"inclusionList": {
"patterns": [
{
"prefix": "reporter="
},
{
"prefix": "upstream_rq_retry"
},
{
"prefix": "upstream_cx"
},
{
"prefix": "cluster_manager"
},
{
"prefix": "listener_manager"
},
{
"prefix": "server"
},
{
"prefix": "cluster.xds-grpc"
},
{
"prefix": "wasm"
},
{
"suffix": "rbac.allowed"
},
{
"suffix": "rbac.denied"
},
{
"suffix": "shadow_allowed"
},
{
"suffix": "shadow_denied"
},
{
"safeRegex": {
"googleRe2": {},
"regex": ".*circuit_breakers.*"
}
},
{
"safeRegex": {
"googleRe2": {},
"regex": "vhost\\.*\\.route\\.*"
}
},
{
"prefix": "component"
},
{
"prefix": "istio"
}
]
}
}
}
