Java实现自定义指标数据远程写入Prometheus

主要的流程如下：

1> prometheus添加启动参数

2> 调用http请求来远程写，数据格式是protobuf（一种自定义的编码格式），编码格式是snappy（一种压缩格式）

3> 远程写通过snappy先压缩，然后将通过protobuf编码的字节数组发送请求；prometheus官网文档远程写提供remote.proto（包含编码和解码），remote.proto文件中依赖了types.proto和gogo.proto两个文件，所以需要把这三个protobuf文件生成java文件

1、准备工作

1.1、Prometheus启动参数添加

针对远程写入Prometheus，官方文档给出了相关说明，具体可参看如下地址：https://prometheus.io/docs/prometheus/latest/storage/，文档中指出，远程写入需要在prometheus服务启动参数中添加如下参数，然后重启服务。

1	`--enable-feature=remote-write-receiver`

　　如果是使用prometheus operator管理的prometheus，则需要在spec中添加如下配置（官方文档地址如下：https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheusspec），同时需要注意prometheus operator的镜像版本要高于0.56.0

enableRemoteWriteReceiver: true

例如：
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
  name: k8s
  namespace: monitoring
spec:
  enableRemoteWriteReceiver: true

1.2、pom依赖

<properties>
    <protobuf.version>3.23.2</protobuf.version>
</properties>

<!-- 远程写入prometheus依赖 -->
<!-- protobuf -->
<dependency>
    <groupId>com.google.protobuf</groupId>
    <artifactId>protobuf-java</artifactId>
    <version>${protobuf.version}</version>
</dependency>
<dependency>
    <groupId>com.google.protobuf</groupId>
    <artifactId>protobuf-java-util</artifactId>
    <version>${protobuf.version}</version>
</dependency>
<!-- snappy compression -->
<dependency>
    <groupId>org.xerial.snappy</groupId>
    <artifactId>snappy-java</artifactId>
    <version>1.1.10.1</version>
</dependency>

1.3、添加GoGoProtos、Remote、Types类

GoGoProtos、Remote、Types类这三个文件，可以通过如下项目地址（https://github.com/bprasen/remotewrite）获取，然后添加到项目中，如下图所示

2、主要代码实现

@Slf4j
@Service("recordingRuleService")
@SuppressWarnings("all")
public class RecordingRuleServiceImpl extends ServiceImpl<RecordingRuleMapper, RecordingRule> implements RecordingRuleService {

    @Resource
    private PromQueryService promQueryService;

    @Resource
    private DatasourceFeignClient datasourceFeignClient;

    /**
     * 远程写入prometheus
     * 
     * @param ids 记录规则主键Id列表
     */
    @Override
    public void remoteWriteToPrometheus(List<Long> ids) {
        for (Long id : ids) {
            RecordingRule recordingRule = this.getById(id);
            List<String> datasourceIdList = Arrays.asList(recordingRule.getDatasourceIds().split(","));
            datasourceIdList.forEach(datasourceId -> {
                Remote.WriteRequest.Builder writeRequestBuilder = Remote.WriteRequest.newBuilder();
                Types.MetricMetadata.Builder builder = Types.MetricMetadata.newBuilder();
                builder.setType(Types.MetricMetadata.MetricType.UNKNOWN);
                builder.setMetricFamilyName(recordingRule.getName());
                builder.setHelp("helper");
                Types.MetricMetadata metricMetadata = builder.build();
                writeRequestBuilder.addMetadata(metricMetadata);

                handleRecordingRuleLabels(recordingRule, writeRequestBuilder, datasourceId);
                try {
                    remoteWrite(datasourceId, writeRequestBuilder);
                } catch (IOException e) {
                    log.error("远程写入prometheus出错, 异常信息为: {}", e.getMessage(), e);
                }
            });
        }
    }

    /**
     * 处理记录规则标签
     *
     * @param recordingRule       记录规则
     * @param writeRequestBuilder 写请求构建器
     * @param datasourceId        数据源id
     */
    private void handleRecordingRuleLabels(RecordingRule recordingRule, Remote.WriteRequest.Builder writeRequestBuilder, String datasourceId) {
        List<Types.Label> labels = new ArrayList<>();
        Types.TimeSeries.Builder timeSeriesBuilder = Types.TimeSeries.newBuilder();
        // 自定义标签信息
        // 设置名称, 值为定义的记录规则名称
        Types.Label nameLabel = Types.Label.newBuilder().setName("__name__").setValue(recordingRule.getName()).build();
        labels.add(nameLabel);
        // 记录规则中定义的附加标签
        String appendTags = recordingRule.getAppendTags();
        if (StringUtils.isNotBlank(appendTags)) {
            Map<String, String> tagsMap = Splitter.on(",").withKeyValueSeparator("=").split(appendTags);
            for (Map.Entry<String, String> tagEntry : tagsMap.entrySet()) {
                Types.Label tagLabel = Types.Label.newBuilder().setName(tagEntry.getKey()).setValue(tagEntry.getValue()).build();
                labels.add(tagLabel);
            }
        }
        // 根据记录规则中定义的promQl语句获取查询数据
        PromQueryData queryDataInfo = promQueryService.getQueryDataInfo(recordingRule.getPromQl(), String.valueOf(DateUtil.currentSeconds()), Integer.valueOf(datasourceId));
        List<PromQueryResult> queryResultList = queryDataInfo.getResult();
        queryResultList.forEach(queryResult -> {
            Map<String, Object> metric = queryResult.getMetric();
            for (Map.Entry<String, Object> metricEntry : metric.entrySet()) {
                if (!"__name__".equals(metricEntry.getKey())) {
                    Types.Label metricLabel = Types.Label.newBuilder().setName(metricEntry.getKey()).setValue(String.valueOf(metricEntry.getValue())).build();
                    labels.add(metricLabel);
                }
            }

            String[] resultValues = queryResult.getValue().toArray(String[]::new);
            // 由于prometheus写入的时间戳到毫秒级, 而项目中定义的时间戳到秒级, 所以这里进行了转换
            Types.Sample sample = Types.Sample.newBuilder().setTimestamp(Long.parseLong(resultValues[0] + "000"))
                    .setValue(Double.parseDouble(resultValues[1])).build();
            // 远程写入prometheus
            timeSeriesBuilder.addAllLabels(labels);
            timeSeriesBuilder.addSamples(sample);
            writeRequestBuilder.addTimeseries(timeSeriesBuilder.build());
        });
    }

    /**
     * 远程写入prometheus
     *
     * @param datasourceId        数据源id
     * @param writeRequestBuilder 写请求构建器
     */
    private void remoteWrite(String datasourceId, Remote.WriteRequest.Builder writeRequestBuilder) throws IOException {
        // 将写请求使用Snappy压缩为字节数组
        Remote.WriteRequest writeRequest = writeRequestBuilder.build();
        byte[] compressed = Snappy.compress(writeRequest.toByteArray());

        // 获取远程写URL
        DatasourceDTO datasourceDTO = datasourceFeignClient.selectById(Integer.valueOf(datasourceId)).getData();
        String url = datasourceDTO.getHttp().get("url").toString();
        String remoteWriteUrl = url + "/api/v1/write";

        HttpPost httpPost = new HttpPost(remoteWriteUrl);
        // 添加prometheus请求头信息, 参考go版本请求发送头
        httpPost.setHeader("Content-type", "application/x-protobuf");
        httpPost.setHeader("Content-Encoding", "snappy");
        httpPost.setHeader("X-Prometheus-Remote-Write-Version", "0.1.0");
        //添加请求头认证信息
        String authorization = Base64.getUrlEncoder().encodeToString(("username" + ":" + "password").getBytes());
        httpPost.addHeader("Authorization", "Basic " + authorization);

        ByteArrayEntity byteArrayEntity = new ByteArrayEntity(compressed);
        httpPost.getRequestLine();
        httpPost.setEntity(byteArrayEntity);

        // 添加重试机制
        for (int i = 1; i <= 3; i++) {
            try {
                CloseableHttpResponse response = httpClient.execute(httpPost);
                log.info("远程写入prometheus数据结果, {}", response);
                break;
            } catch (Exception e) {
                log.error("[POST/HTTP 远程写入Prometheus请求信息]异常, 重试次数:{}, 请求地址:{}, 异常信息:{}", i, remoteWriteUrl, Throwables.getStackTraceAsString(e));
            }
        }
    }
}