remote write data to prometheus with java

背景：

需要从 kafka 消费大量的流量数据， 写入promehteus。 通过直接写， 可以减少 消费和prometheus pull的 交互。

参考文章：

1、实操见 Prometheus Remote Write in Java, 但上面是 remote reader的demo

2、Prometheus远程写Metric Java代码实现 这是也 参考的 上面的这篇文章

3、https://github.com/javiroman/jremotewrite 写的write remote 的demo，但是 直接跑是并不能 写入 promethues的，这里 让我找了好久， 最大的一个 疑惑是 没看到 metric_name的 赋值。 （困扰了两天，可以没有label，但是不能没有 metric_name 啊。）

决定看下 go 实现的 例子。

https://golangexample.com/prometheus-remote-write-go-client/ 看到了 　

client := promwrite.NewClient("http://prometheus:8428/api/v1/write")
resp, err := client.Write(context.Background(), &promwrite.WriteRequest{
	TimeSeries: []promwrite.TimeSeries{
		{
			Labels: []promwrite.Label{
				{
					Name:  "__name__",
					Value: "my_metric_name",
				},
			},
			Sample: promwrite.Sample{
				Time:  time.Now(),
				Value: 123,
			},
		},
	},
})

看到了 metric_name的操作， 直接写入 lable对象中。

尝试成功。

demo

直接跑，能查到数据

基于 protobuf文件生成三个java文件。

需要额外的pom 依赖：

<dependency>
            <groupId>io.prometheus</groupId>
            <artifactId>simpleclient</artifactId>
            <version>0.16.0</version>
        </dependency>

        <!-- protobuf -->
        <dependency>
            <groupId>com.google.protobuf</groupId>
            <artifactId>protobuf-java</artifactId>
            <version>3.11.1</version>
        </dependency>
        <dependency>
            <groupId>com.google.protobuf</groupId>
            <artifactId>protobuf-java-util</artifactId>
            <version>3.11.1</version>
        </dependency>

        <!-- snappy compression -->
        <dependency>
            <groupId>org.xerial.snappy</groupId>
            <artifactId>snappy-java</artifactId>
            <version>1.1.7.3</version>
        </dependency>

Model类

public class SendData {

    private BigInteger in;
    private BigInteger out;
    private Long index;
    private Long timestamp;
    @JsonProperty("snmp_id")
    private Long snmpId;
    private String ip;

    @JsonProperty("flow_type")
    private Integer flowType;

    @JsonProperty("flow_id")
    private int flowId = 0;

    private Integer sign = 1;

    private String host;
  //getter and setter
}

初始化数据：

private void initData() {

        for (int i = 0; i < 1; i++) {
            for (int j = 0; j < 5; j++) {

//                long timestamp = System.currentTimeMillis() - 300000;
                long timestamp = System.currentTimeMillis();

                LOGGER.info(">>>>>>{}", timestamp);
                SendData sendData = new SendData();
                sendData.setIn(BigInteger.valueOf(new Random().nextInt(30)));
                sendData.setOut(BigInteger.valueOf(new Random().nextInt(30)));
                sendData.setIndex((long) i);
                sendData.setTimestamp(timestamp);
                sendData.setSnmpId(1L);
                sendData.setIp("10.10." + i + "." + j);
                sendData.setFlowType(1);
                sendData.setFlowId(1);
                sendData.setHost("11111");

                datas.add(sendData);
            }
        }
    }

@Test
    public void test_remote_write() throws IOException {

        initData();

        Remote.WriteRequest.Builder writeRequestBuilder = Remote.WriteRequest.newBuilder();
        Types.MetricMetadata.Builder builder = Types.MetricMetadata.newBuilder();
        builder.setType(Types.MetricMetadata.MetricType.GAUGE);
        builder.setMetricFamilyName("bandwitch_guage");
        builder.setHelp("helper");
        Types.MetricMetadata metricMetadata = builder.build();
        writeRequestBuilder.addMetadata(metricMetadata);


        for (SendData data : datas) {

            Types.TimeSeries.Builder timeSeriesBuilder = Types.TimeSeries.newBuilder();
            Types.TimeSeries.Builder outtimeSeriesBuilder = Types.TimeSeries.newBuilder();

            //必须要有 metric_name
            Types.Label inNameLavble = Types.Label.newBuilder().setName("__name__").setValue("z_bandwitdh_in_guage").build();//metrics name
            Types.Label outNameLabel = Types.Label.newBuilder().setName("__name__").setValue("z_bandwitdh_out_guage").build();//metrics name

            Types.Label ipLabel = Types.Label.newBuilder().setName("ip").setValue(data.getIp()).build();
            Types.Label snmpIdLabel = Types.Label.newBuilder().setName("snmp_id").setValue(data.getSnmpId()+"").build();
            Types.Label indexLabel = Types.Label.newBuilder().setName("index").setValue(data.getIndex()+"").build();
            Types.Label flowTypeLabel = Types.Label.newBuilder().setName("flow_type").setValue(data.getFlowType()+"").build();
            Types.Label flowIdLabel = Types.Label.newBuilder().setName("flow_id").setValue(data.getFlowId()+"").build();
//            Types.Label timestampLabel = Types.Label.newBuilder().setName("timestamp").setValue(data.getTimestamp()+"").build();

            Types.Sample inSample = Types.Sample.newBuilder().setTimestamp(data.getTimestamp()).setValue(data.getIn().doubleValue()).build();
            Types.Sample outSample = Types.Sample.newBuilder().setTimestamp(data.getTimestamp()).setValue(data.getOut().doubleValue()).build();

            timeSeriesBuilder.addAllLabels(Arrays.asList(inNameLavble, ipLabel, snmpIdLabel, indexLabel, flowIdLabel, flowTypeLabel));
            timeSeriesBuilder.addSamples(inSample);

            outtimeSeriesBuilder.addAllLabels(Arrays.asList(outNameLabel, ipLabel, snmpIdLabel, indexLabel, flowIdLabel, flowTypeLabel));
            outtimeSeriesBuilder.addSamples(outSample);

            writeRequestBuilder.addTimeseries(timeSeriesBuilder.build());
            writeRequestBuilder.addTimeseries(outtimeSeriesBuilder.build());
        }


        Remote.WriteRequest message = writeRequestBuilder.build();

        byte[] compressedMessage = Snappy.compress(message.toByteArray());


        RestTemplate restTemplate = new RestTemplate();

        //参考的 go http send 版本
        HttpHeaders headers = new HttpHeaders();
        headers.add("Content-Encoding", "snappy");
//        headers.add("User-Agent", ;"opcai")
        headers.add("X-Prometheus-Remote-Write-Version", "0.1.0");
        headers.add("Content-Type","application/x-protobuf");

        HttpEntity requestEntity = new HttpEntity(compressedMessage, headers);

        ResponseEntity<Object> exchange = restTemplate.exchange("http://localhost:9010/api/v1/write", HttpMethod.POST, requestEntity, Object.class);

        System.out.println(exchange);

    }