es查询
1.三级部门分应用统计PXX逻辑
{
"query": {
"bool": {
"must": [
{
"term": {
"child3DepartmentId": "00065308"
}
},
{
"terms": {
"serverUseType": [
"1",
"3"
]
}
},
{
"term": {
"metric": "cpu.usage"
}
}
]
}
},
"size": 0,
"aggs": {
"child2DepartmentIdgb": {
"terms": {
"field": "child2DepartmentId",
"size": 100
},
"aggs": {
"child3DepartmentIdgb": {
"terms": {
"field": "child3DepartmentId",
"size": 100
},
"aggs": {
"zones": {
"terms": {
"field": "projectId",
"size": 200
},
"aggs": {
"load_times": {
"percentiles": {
"field": "value",
"percents": [
100,
99.99,
99.9,
99.8,
99.7,
99,
98,
97,
96,
95,
94,
93,
92,
91,
90,
80,
70,
60,
50,
40,
30,
20,
10
]
}
},
"extended_stats": {
"extended_stats": {
"field": "value"
}
}
}
}
}
}
}
}
}
}
java代码
public static Set<AppUtilization> fetchEsAppCpuAvgData(int weekOfYear, int year, String[] esIndexArray){
LOG.info("Start Cpu Avg Process");
Set<AppUtilization> set = Sets.newConcurrentHashSet();
SearchRequestBuilder searchRequestBuilder = esClient.prepareSearch(esIndexArray).setSize(DEFAULT_QUERY_SIZE).setTypes(ES_TABLE_TYPE);
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
.filter(QueryBuilders.termQuery(ES_METRIC_COLUMN, DEFAULT_METRIC))
// .filter(QueryBuilders.termQuery(ES_ENV_FIELD, ES_ENV_PRODUCT))
.filter(QueryBuilders.termsQuery(ES_SERVER_USE_TYPE_COLUMN, SERVER_USE_TYPE_ARRAY));
searchRequestBuilder.setQuery(queryBuilder);
AggregationBuilder aggregation = AggregationBuilders
.terms(DEFAULT_ES_AGG_PROJECT_NAME)
.size(DEFAULT_ES_AGG_PROJECT_SIZE)
.field(ES_PROJECT_NAME_FIELD)
.order(Terms.Order.aggregation(DEFAULT_AVG_AGG_NAME, false))
.subAggregation(
AggregationBuilders.avg(DEFAULT_AVG_AGG_NAME).field(ES_DATA_VALUE_FIELD));
searchRequestBuilder.addAggregation(aggregation);
LOG.info("search : {}",searchRequestBuilder.toString());
SearchResponse response = searchRequestBuilder.execute().actionGet();
Terms projectNameTerms = response.getAggregations().get(DEFAULT_ES_AGG_PROJECT_NAME);
projectNameTerms.getBuckets().parallelStream().forEach(entry -> {
String projectName = entry.getKeyAsString();
Avg avg = entry.getAggregations().get(DEFAULT_AVG_AGG_NAME);
Double avgValue = avg.getValue();
if (Double.isNaN(avg.getValue())){
LOG.error(projectName + " calculated AppUtilization ES Query Result is null");
return;
}
avgValue = DoubleUtils.getDouble4Decimal(avgValue);
LOG.info("projectName : {} ,metric : {} , avgValue : {}",projectName,DEFAULT_METRIC,avgValue);
set.add(new AppUtilization(projectName,avgValue, MonitorMetric.CPU_USAGE, StatisticalType.AVG,weekOfYear,year));
});
return set;
}
2.所有部门的pxx计算
{
"query": {
"bool": {
"must": [
{
"terms": {
"child2DepartmentId": [
"00033132",
"00048471",
"00053749",
"00065244",
"00065253",
"00065259",
"00065265",
"00065272",
"00065280",
"00065285",
"00065303",
"00068776",
"00070152",
"00075235"
]
}
},
{
"terms": {
"serverUseType": [
"1",
"3"
]
}
},
{
"term": {
"metric": "cpu.usage"
}
}
]
}
},
"size": 0,
"aggs": {
"child2DepartmentIdgb": {
"terms": {
"field": "child2DepartmentId",
"size": 100
},
"aggs": {
"child3DepartmentIdgb": {
"terms": {
"field": "child3DepartmentId",
"size": 100
},
"aggs": {
"zones": {
"terms": {
"field": "projectId",
"size": 5000
},
"aggs": {
"load_times": {
"percentiles": {
"field": "value",
"percents": [
100,
90,
80,
70,
60,
50,
40,
30,
20,
10
]
}
},
"extended_stats": {
"extended_stats": {
"field": "value"
}
}
}
}
}
}
}
}
}
}
3.计算部门的cpu均值
{
"size": 0,
"query": {
"bool": {
"must" : {},
"filter": {
"terms": {
"metric": [
"cpu.usage",
"cpu.busy"
]
}
}
}
},
"aggs": {
"group_by_child2Department": {
"terms": {
"size":50,
"field": "child2DepartmentId",
"order": {
"max_value": "desc"
}
},
"aggs": {
"max_value": {
"max": {
"field": "value"
}
}
}
}
}
}
4.获取一段时间的平均值
{
"size": 0,
"query": {
"bool": {
"must": {
"match": {
"metric": "load.5min"
}
},
"filter": {
"range": {
"time": {
"gt": 1529251200,
"lt": 1529337599
}
}
}
}
},
"aggs": {
"avg_value": {
"avg": {
"field": "value"
}
}
}
}
5.java API 查询
SearchRequestBuilder search = esTransportClient.prepareSearch(dateString).setSize(0).setTypes("monitor");
QueryBuilder qb = boolQuery()
.must(matchQuery("metric", metric))
.filter(rangeQuery("time").gt(beginDateTime).lt(endDateTime));
search.setQuery(qb);
AggregationBuilder aggregation = AggregationBuilders
.terms("group_by_child2Department")
.size(50)
.field("child2DepartmentId")
.order(Terms.Order.aggregation("max_value", false))
.subAggregation(
AggregationBuilders.max("max_value").field("value"));
search.addAggregation(aggregation);
SearchResponse response = search.execute().actionGet();
Terms endpointTerms = response.getAggregations().get("group_by_child2Department");
// List<CmdbAssetProject> listfromBeginToEnd = new ArrayList<CmdbAssetProject>();
for (Terms.Bucket entry:
endpointTerms.getBuckets()) {
logger.debug("进入聚合结果分析:");
String child2DepartmentId = entry.getKeyAsString();
String child2Department = DepartmentMapCache.departmentMap.get(child2DepartmentId);
Max max = entry.getAggregations().get("max_value");
Double maxValue = max.getValue();
logger.debug("child2Department " + child2Department);
logger.debug("maxValue:" + maxValue);
// 调用通过ip查询资产信息的接口去查询出来部门和应用信息
// CmdbAssetProject cmdbAssetProject = new CmdbAssetProject();
CmdbAssetProject cmdbAssetProject = new CmdbAssetProject();
cmdbAssetProject.setValue(maxValue);
cmdbAssetProject.setDepartment(child2Department);
logger.debug(cmdbAssetProject);
// listfromBeginToEnd.add(cmdbAssetProject);
if (tmpMap.get(cmdbAssetProject.getDepartment()) == null){
List<Double> list = new ArrayList<Double>();
list.add(cmdbAssetProject.getValue());
tmpMap.put(cmdbAssetProject.getDepartment(),list);
}else{
List<Double> list = (List<Double>) tmpMap.get(cmdbAssetProject.getDepartment());
list.add(cmdbAssetProject.getValue());
tmpMap.put(cmdbAssetProject.getDepartment(),list);
}
}
6.获取load的top10数据
{
"size": 0,
"query": {
"match": {
"metric": "load.5min"
}
},
"aggs": {
"group_by_endpoint": {
"terms": {
"field": "endpoint",
"order": {
"max_value": "desc"
}
},
"aggs": {
"max_value": {
"max": {
"field": "value"
}
}
}
}
}
}
7.获取某个监控项，某段时间内的平均值
{
"size": 0,
"query": {
"bool": {
"must": [
{
"term": {
"metric": "net.if.in.bytes"
}
},
{
"range": {
"time": {
"gt": 1552838400,
"lt": 1553443199
}
}
},
{
"term": {
"endpoint": "10.175.9.235"
}
}
]
}
},
"aggs": {
"avg_value": {
"avg": {
"field": "value"
}
}
}
}
8.四级group by获取平均值
{
"size": 0,
"query": {
"bool": {
"must": {},
"filter": [
{
"terms": {
"metric": [
"cpu.usage",
"mem.memoryused.percent",
"disk.root.used.percent",
"disk.export.used.percent"
]
}
},
{
"range": {
"time": {
"gt": "1623202206",
"lt": "1623202209"
}
}
}
]
}
},
"aggs": {
"group_by_projectName": {
"terms": {
"size": 10000,
"field": "projectName"
},
"aggs": {
"group_by_metric": {
"terms": {
"field": "metric",
"size": 1000
},
"aggs": {
"group_by_environment": {
"terms": {
"field": "environmentType",
"size": 10
},
"aggs": {
"group_by_usetype": {
"terms": {
"field": "useType",
"size": 2
},
"aggs": {
"avg_value": {
"avg": {
"field": "value"
}
}
}
}
}
}
}
}
}
}
}
}
9.多级匹配，一级聚合
{
"size": 0,
"query": {
"bool": {
"must": {},
"filter": [
{
"terms": {
"metric": [
"disk.root.used.percent",
"disk.export.used.percent"
]
}
},
{
"terms": {
"environmentType": [
"PRODUCT",
"PREPARE_PR"
]
}
},
{
"terms": {
"useType": [
"APP"
]
}
},
{
"range": {
"time": {
"gt": "1623202206",
"lt": "1623202209"
}
}
}
]
}
},
"aggs": {
"group_by_projectName": {
"terms": {
"size": 10000,
"field": "projectName"
},
"aggs": {
"avg_value": {
"avg": {
"field": "value"
}
}
}
}
}
}
10.均值，pxx，平方差一起运算
{
"query":{
"bool":{
"must":[
{
"terms":{
"endpoint":[
"11.93.4.178"
]
}
},
{
"term":{
"metric":"cpu.usage"
}
}
]
}
},
"size":0,
"aggs":{
"child2DepartmentIdgb":{
"terms":{
"field":"child2DepartmentId",
"size":100
},
"aggs":{
"child3DepartmentIdgb":{
"terms":{
"field":"child3DepartmentId",
"size":100
},
"aggs":{
"zones":{
"terms":{
"field":"projectId",
"size":5000
},
"aggs":{
"load_times":{
"percentiles":{
"field":"value",
"percents":[
100,
90,
80,
70,
60,
50,
40,
30,
20,
10
]
}
},
"stats_value":{
"stats":{
"field":"value"
}
},
"extended_stats":{
"extended_stats":{
"field":"value"
}
}
}
}
}
}
}
}
}
}
原创:做时间的朋友
