Prometheus 使用 blackbox 作为客户端

Prometheus 使用 blackbox 作为客户端请求TCP、HTTP、GRPC

安装

github: https://github.com/prometheus/blackbox_exporter.git

默认配置文件

modules:
  http_2xx:
    prober: http
  http_post_2xx:
    prober: http
    http:
      method: POST
  tcp_connect:
    prober: tcp
  pop3s_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^+OK"
      tls: true
      tls_config:
        insecure_skip_verify: false
  grpc:
    prober: grpc
    grpc:
      tls: true
      preferred_ip_protocol: "ip4"
  grpc_plain:
    prober: grpc
    grpc:
      tls: false
      service: "service1"
  ssh_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^SSH-2.0-"
      - send: "SSH-2.0-blackbox-ssh-check"
  irc_banner:
    prober: tcp
    tcp:
      query_response:
      - send: "NICK prober"
      - send: "USER prober prober prober :prober"
      - expect: "PING :([^ ]+)"
        send: "PONG ${1}"
      - expect: "^:[^ ]+ 001"
  icmp:
    prober: icmp

使用CURL测试

curl http://127.0.0.1:9115/probe?target=1.1.1.1:10000&module=tcp_connect

配置prometheus

# ICMP
  - job_name: 'blackbox_icmp'
    scrape_interval: 1m
    metrics_path: /probe
    params:
      module: [icmp]
    static_configs:
     - targets:
        - 1.1.1.1
       labels:
         appid: internal.blackbox.icmp
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 127.0.0.1:9115 # black_exporter地址

# TCP
  - job_name: blackbox_tcp
    metrics_path: /probe
    params:
      module: [tcp_connect]
    static_configs:
      - targets:
        - 1.1.1.1:10000
      labels:
         appid: internal.blackbox.tcp

    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 127.0.0.1:9115
# HTTP
scrape_configs:
  - job_name: 'blackbox'
    metrics_path: /probe
    params:
      module: [http_2xx]  # 模块对应 blackbox.yml 
    static_configs:
      - targets:
        - http://baidu.com    # http
        - https://baidu.com   # https
        - http://xx.com:8080 # 8080端口的域名
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 127.0.0.1:9115  # blackbox安装在哪台机器

配置报警

  - alert: port check P3
    expr: probe_success{job="blackbox_tcp"} < 1
    for: 2m
    labels:
      severity: info
    annotations:
      summary: "Instance {{ $labels.instance }} port check failed"
      description: "Instance {{ $labels.instance }} port check failed"
      app: "{{ $labels.appid }}"

  - alert: port check P2
    expr: probe_success{job="blackbox_tcp"} < 1
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: "Instance {{ $labels.instance }} port check failed"
      description: "Instance {{ $labels.instance }} port check failed"
      app: "{{ $labels.appid }}"

自定义模块

blackbox.yml

  http_2xx_wxjj:
    prober: http
    timeout: 5s
    http:
      method: GET
      headers:
        Cookie: JSESSIONID=C123455dfdf
        sid: 41c912344555-24rkjkffd
        appid: 1221kj2h1k3hjk13hk
      body: '{}'

prometheus.yml

  - job_name: 'blackbox_wxjl'
    metrics_path: /probe
    params:
      module: [http_2xx_wxjj]  # Look for a HTTP 200 response.
    static_configs:
      - targets:
        - http://192.168.201.173:808/byxxxxx/41234456661f-4357c9?head=APP_GeList&user=%E9%BB%84%E5%AE%15
   # Target to probe with http.

    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 172.18.11.154:9115  # The blackbox exporter's real hostname:port.

开启debug

当你觉得自己设置没错，http 状态码却返回不正确，想要调试一下，就需要打开debug。

• 启动时指定 --log.level=debug
• targets 后面带上 &debug=true，即 http://172.18.11.154:9115/probe?module=http_2xx_wxjj&target=http://192.168.201.173:808/byxxxxx/41234456661f-4357c9?head=APP_GeList&user=黄��&debug=true

targets 开启 debug 会比正常链接输出更多信息

Module configuration:
prober: http
timeout: 5s
http:
    ip_protocol_fallback: true
    method: GET
    headers:
        Cookie: JSESSIONID=C123455dfdf
        appid: 41c912344555-24rkjkffd
        sid: 1221kj2h1k3hjk13hk
    body: '{}'
tcp:
    ip_protocol_fallback: true
icmp:
    ip_protocol_fallback: true
dns:
    ip_protocol_fallback: true

GRPC 客户端检测

grpc使用的是grpc官方的Check方法进行验证

构建一个grpc应用，支持官方的check方法

server.go

package main

import (
	"flag"
	"fmt"
	"io"
	"net"
	"os"
	"strconv"
	"time"

	pb "grpc-hello/proto/hello" // 引入编译生成的包

	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

// 定义helloService并实现约定的接口
type helloService struct{}

// HelloService Hello服务
var HelloService = helloService{}

// SayHello 实现Hello服务接口
func (h helloService) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloResponse, error) {
	grpclog.Println("收到请求grpclog")
	resp := new(pb.HelloResponse)
	resp.Message = fmt.Sprintf("Hello %s.", in.Name)
	return resp, nil
}

// StreamSayHello 实现Hello服务接口
func (h helloService) StreamSayHello(srv pb.Hello_StreamSayHelloServer) error {
	grpclog.Println("收到请求grpclog")
	n := 0
	for {
		// 接收数据
		req, err := srv.Recv()
		if err == io.EOF {
			return nil
		}
		if err != nil {
			grpclog.Errorf("stream get from client err: %v", err)
			return err
		}
		// 发送数据
		err = srv.Send(&pb.HelloResponse{
			Message: strconv.Itoa(n),
		})
		if err != nil {
			grpclog.Errorf("stream send to client err: %v", err)
			return err
		}
		grpclog.Printf("stream get from client: %s \t send client: %d\n", req.GetName(), n)
		time.Sleep(1 * time.Second)
		n++
	}
}

func (h helloService) Check(ctx context.Context, in *pb.HealthCheckRequest) (*pb.HealthCheckResponse, error) {
	var healthResponse = &pb.HealthCheckResponse{
		Status: pb.HealthCheckResponse_SERVING,
	}
	return healthResponse, nil
}
func (h helloService) Watch(in *pb.HealthCheckRequest, srv pb.Health_WatchServer) error {
	return nil
}

func main() {
	var (
		// Address gRPC服务地址
		Address = flag.String("addr", "127.0.0.1:8000", "grpc server地址 IP:端口 127.0.0.1:8000")
	)
	flag.Parse()
	grpclog.SetLoggerV2(grpclog.NewLoggerV2(os.Stdout, os.Stdout, os.Stdout))
	listen, err := net.Listen("tcp", *Address)
	if err != nil {
		grpclog.Fatalf("Failed to listen: %v", err)
	}

	// 实例化grpc Server
	s := grpc.NewServer()

	// 注册HelloService
	pb.RegisterHelloServer(s, &HelloService)
	pb.RegisterHealthServer(s, &HelloService)
	grpclog.Println("Listen on " + *Address)
	s.Serve(listen)
}

proto文件

check.proto

syntax = "proto3";

package grpc.health.v1;

// 指定golang包名
option go_package = "./hello";

message HealthCheckRequest {
  string service = 1;
}

message HealthCheckResponse {
  enum ServingStatus {
    UNKNOWN = 0;
    SERVING = 1;
    NOT_SERVING = 2;
    SERVICE_UNKNOWN = 3;  // Used only by the Watch method.
  }
  ServingStatus status = 1;
}

service Health {
  rpc Check(HealthCheckRequest) returns (HealthCheckResponse);

  rpc Watch(HealthCheckRequest) returns (stream HealthCheckResponse);
}

// protoc -I /usr/local/include -I . --go_out=plugins=grpc:. ./hello/check.proto

hello.proto

syntax = "proto3"; // 指定proto版本
package hello;     // 指定默认包名

// 指定golang包名
option go_package = "./hello";

// 定义Hello服务
service Hello {
    // 定义SayHello方法
    rpc SayHello(HelloRequest) returns (HelloResponse) {}
    // 定义流SayHello方法
    rpc StreamSayHello(stream HelloRequest) returns (stream HelloResponse) {}
}

// HelloRequest 请求结构
message HelloRequest {
    string name = 1;
}

// HelloResponse 响应结构
message HelloResponse {
    string message = 1;
}

客户端通过blackbox请求

go run server.go --addr 0.0.0.0:12002
curl http://192.168.9.2:9115/probe?target=192.168.9.2:12002&module=grpc_plain&debug=true

参考文档：
https://www.cnblogs.com/fsckzy/p/12614176.html
https://github.com/prometheus/blackbox_exporter/pull/835