Java实现自定义指标数据远程写入Prometheus
最近在看夜莺的记录规则这部分功能实现，其中新增记录规则之后需要远程写入prometheus，而对于这部分功能实现，夜莺使用的是Go实现（可参考如下地址：https://heapdump.cn/article/5597957），由于项目使用Java开发，所以针对这部分功能，只能进行重写。下面内容为抽取出来的主要代码实现，仅做记录说明。
主要的流程如下：
1> prometheus添加启动参数
2> 调用http请求来远程写，数据格式是protobuf（一种自定义的编码格式），编码格式是snappy（一种压缩格式）
1.1、Prometheus启动参数添加
针对远程写入Prometheus，官方文档给出了相关说明，具体可参看如下地址：https://prometheus.io/docs/prometheus/latest/storage/，文档中指出，远程写入需要在prometheus服务启动参数中添加如下参数，然后重启服务。
--enable-feature=remote-write-receiver
如果是使用prometheus operator管理的prometheus，则需要在spec中添加如下配置（官方文档地址如下：https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheusspec），同时需要注意prometheus operator的镜像版本要高于0.56.0
enableRemoteWriteReceiver: true
例如：
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
name: k8s
namespace: monitoring
spec:
enableRemoteWriteReceiver: true
<properties>
<protobuf.version>3.23.2</protobuf.version>
</properties>
<!-- 远程写入prometheus依赖 -->
<!-- protobuf -->
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
<version>${protobuf.version}</version>
</dependency>
<!-- snappy compression -->
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>1.1.10.1</version>
</dependency>
1.3、添加GoGoProtos、Remote、Types类
GoGoProtos、Remote、Types类这三个文件，可以通过如下项目地址（https://github.com/bprasen/remotewrite）获取，然后添加到项目中，如下图所示
@Slf4j
@Service("recordingRuleService")
@SuppressWarnings("all")
public class RecordingRuleServiceImpl extends ServiceImpl<RecordingRuleMapper, RecordingRule> implements RecordingRuleService {
@Resource
private PromQueryService promQueryService;
@Resource
private DatasourceFeignClient datasourceFeignClient;
/**
* 远程写入prometheus
*
* @param ids 记录规则主键Id列表
*/
@Override
public void remoteWriteToPrometheus(List<Long> ids) {
for (Long id : ids) {
RecordingRule recordingRule = this.getById(id);
List<String> datasourceIdList = Arrays.asList(recordingRule.getDatasourceIds().split(","));
datasourceIdList.forEach(datasourceId -> {
Remote.WriteRequest.Builder writeRequestBuilder = Remote.WriteRequest.newBuilder();
Types.MetricMetadata.Builder builder = Types.MetricMetadata.newBuilder();
builder.setType(Types.MetricMetadata.MetricType.UNKNOWN);
builder.setMetricFamilyName(recordingRule.getName());
builder.setHelp("helper");
Types.MetricMetadata metricMetadata = builder.build();
writeRequestBuilder.addMetadata(metricMetadata);
handleRecordingRuleLabels(recordingRule, writeRequestBuilder, datasourceId);
try {
remoteWrite(datasourceId, writeRequestBuilder);
} catch (IOException e) {
log.error("远程写入prometheus出错, 异常信息为: {}", e.getMessage(), e);
}
});
}
}
/**
* 处理记录规则标签
*
* @param recordingRule 记录规则
* @param writeRequestBuilder 写请求构建器
* @param datasourceId 数据源id
*/
private void handleRecordingRuleLabels(RecordingRule recordingRule, Remote.WriteRequest.Builder writeRequestBuilder, String datasourceId) {
List<Types.Label> labels = new ArrayList<>();
Types.TimeSeries.Builder timeSeriesBuilder = Types.TimeSeries.newBuilder();
// 自定义标签信息
// 设置名称, 值为定义的记录规则名称
Types.Label nameLabel = Types.Label.newBuilder().setName("__name__").setValue(recordingRule.getName()).build();
labels.add(nameLabel);
// 记录规则中定义的附加标签
String appendTags = recordingRule.getAppendTags();
if (StringUtils.isNotBlank(appendTags)) {
Map<String, String> tagsMap = Splitter.on(",").withKeyValueSeparator("=").split(appendTags);
for (Map.Entry<String, String> tagEntry : tagsMap.entrySet()) {
Types.Label tagLabel = Types.Label.newBuilder().setName(tagEntry.getKey()).setValue(tagEntry.getValue()).build();
labels.add(tagLabel);
}
}
// 根据记录规则中定义的promQl语句获取查询数据
PromQueryData queryDataInfo = promQueryService.getQueryDataInfo(recordingRule.getPromQl(), String.valueOf(DateUtil.currentSeconds()), Integer.valueOf(datasourceId));
List<PromQueryResult> queryResultList = queryDataInfo.getResult();
queryResultList.forEach(queryResult -> {
Map<String, Object> metric = queryResult.getMetric();
for (Map.Entry<String, Object> metricEntry : metric.entrySet()) {
if (!"__name__".equals(metricEntry.getKey())) {
Types.Label metricLabel = Types.Label.newBuilder().setName(metricEntry.getKey()).setValue(String.valueOf(metricEntry.getValue())).build();
labels.add(metricLabel);
}
}
String[] resultValues = queryResult.getValue().toArray(String[]::new);
// 由于prometheus写入的时间戳到毫秒级, 而项目中定义的时间戳到秒级, 所以这里进行了转换
Types.Sample sample = Types.Sample.newBuilder().setTimestamp(Long.parseLong(resultValues[0] + "000"))
.setValue(Double.parseDouble(resultValues[1])).build();
// 远程写入prometheus
timeSeriesBuilder.addAllLabels(labels);
timeSeriesBuilder.addSamples(sample);
writeRequestBuilder.addTimeseries(timeSeriesBuilder.build());
});
}
/**
* 远程写入prometheus
*
* @param datasourceId 数据源id
* @param writeRequestBuilder 写请求构建器
*/
private void remoteWrite(String datasourceId, Remote.WriteRequest.Builder writeRequestBuilder) throws IOException {
// 将写请求使用Snappy压缩为字节数组
Remote.WriteRequest writeRequest = writeRequestBuilder.build();
byte[] compressed = Snappy.compress(writeRequest.toByteArray());
// 获取远程写URL
DatasourceDTO datasourceDTO = datasourceFeignClient.selectById(Integer.valueOf(datasourceId)).getData();
String url = datasourceDTO.getHttp().get("url").toString();
String remoteWriteUrl = url + "/api/v1/write";
HttpPost httpPost = new HttpPost(remoteWriteUrl);
// 添加prometheus请求头信息, 参考go版本请求发送头
httpPost.setHeader("Content-type", "application/x-protobuf");
httpPost.setHeader("Content-Encoding", "snappy");
httpPost.setHeader("X-Prometheus-Remote-Write-Version", "0.1.0");
//添加请求头认证信息
String authorization = Base64.getUrlEncoder().encodeToString(("username" + ":" + "password").getBytes());
httpPost.addHeader("Authorization", "Basic " + authorization);
ByteArrayEntity byteArrayEntity = new ByteArrayEntity(compressed);
httpPost.getRequestLine();
httpPost.setEntity(byteArrayEntity);
// 添加重试机制
for (int i = 1; i <= 3; i++) {
try {
CloseableHttpResponse response = httpClient.execute(httpPost);
log.info("远程写入prometheus数据结果, {}", response);
break;
} catch (Exception e) {
log.error("[POST/HTTP 远程写入Prometheus请求信息]异常, 重试次数:{}, 请求地址:{}, 异常信息:{}", i, remoteWriteUrl, Throwables.getStackTraceAsString(e));
}
}
}
}
说明：
1> 记录规则数据表相关字段，这里使用的是夜莺的表设计如下（夜莺地址：http://flashcat.cloud/docs/content/flashcat-monitor/nightingale-v6/schema/recording_rule/）
2> 远程写入Prometheus成功，会返回 204 状态码，如下所示
3> 可能遇到的问题：
这个问题，prometheus官网也给出了大致说明，多半是与时间戳格式有关，例如，项目中的时间戳到秒级，而prometheus要求到毫秒级
官网issues地址：https://github.com/prometheus/prometheus/issues/12052
4> 参考文章
**************************************************** 林深时见鹿，海蓝时见鲸 ****************************************************
