ES提供的java类直接查询ES
import com.google.gson.Gson; import com.suning.ctdsa.alarm.constant.Constant; import com.suning.ctdsa.alarm.intf.dao.LogTypeDao; import com.suning.ctdsa.alarm.intf.dao.es.AlarmElasticsearchDao; import com.suning.ctdsa.alarm.intf.entity.AlarmConfEo; import com.suning.ctdsa.alarm.intf.entity.LogTypeEo; import com.suning.ctemm.collector.ExceptionEventCollector; import com.suning.ctemm.collector.message.ExceptionEventMessage; import io.searchbox.client.JestResult; import io.searchbox.client.http.JestHttpClient; import io.searchbox.core.Search; import io.searchbox.params.SearchType; import org.apache.commons.lang.StringUtils; import org.elasticsearch.common.joda.time.format.ISODateTimeFormat; import org.elasticsearch.index.query.BoolFilterBuilder; import org.elasticsearch.index.query.FilterBuilders; import org.elasticsearch.index.query.MatchQueryBuilder.Type; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.text.SimpleDateFormat; import java.util.*; /** * ES相关查询实现<br> * 〈功能详细描述〉 * * @see [相关类/方法]（可选） * @since [产品/模块版本] （可选） */ public class ElasticsearchDaoImpl implements ElasticsearchDao { public static final String DSA_LOG_APPID = "appId"; public static final String DSA_LDC_ID = "ldc"; public static final String DSA_LOG_LOGTIME = "logTime"; public static final String DSA_ALL = "ALL"; private static Logger logger = LoggerFactory .getLogger(AlarmElasticsearchDaoImpl.class); private JestHttpClient jestHttpClient; private LogTypeDao logTypeDao; private Gson gson; public void setLogTypeDao(LogTypeDao logTypeDao) { this.logTypeDao = logTypeDao; } public void setJestHttpClient(JestHttpClient jestHttpClient) { this.jestHttpClient = jestHttpClient; } public LogTypeDao getLogTypeDao() { return logTypeDao; } public Gson getGson() { return gson; } protected BoolFilterBuilder buildSystemAndTimeFilter(String appId, String logType, long start, long end) { BoolFilterBuilder booleanQueryBuilder = FilterBuilders.boolFilter(); if (!(Constant.ALL_SYSTEM).equalsIgnoreCase(appId)) { booleanQueryBuilder.must(FilterBuilders.queryFilter(QueryBuilders.matchQuery(DSA_LOG_APPID, appId).type(Type.PHRASE))); } LogTypeEo logTypeEo = logTypeDao.getLogTypeByLogTypeNum(logType); if (StringUtils.isNotBlank(logTypeEo.getDsaLogTimefield())) { booleanQueryBuilder.must(FilterBuilders.rangeFilter(logTypeEo.getDsaLogTimefield()).gte(start).lte(end)); } else { booleanQueryBuilder.must(FilterBuilders.rangeFilter(DSA_LOG_LOGTIME).gte(start).lte(end)); } return booleanQueryBuilder; } @Override public Map<String,Object> queryESByJest(AlarmConfEo configure, long start, long end) {//NOSONAR List<String> indices = getIndices(configure.getLogType(), start, end); if (indices.isEmpty()) { return null; } // 检索条件 StringBuilder condition = new StringBuilder(configure.getTerms()); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().from(0).size(0); BoolFilterBuilder booleanQueryBuilder = buildSystemAndTimeFilter(configure.getSysCode(), configure.getLogType(), start, end); if (StringUtils.isBlank(condition.toString())) { condition.append("*"); } searchSourceBuilder.query(QueryBuilders.filtered(QueryBuilders.queryString(condition .toString()).analyzeWildcard(true).lowercaseExpandedTerms(false), booleanQueryBuilder)); String aggregation = configure.getAggregation();// 聚合 String field = configure.getField();// 聚合字段 if (Constant.AGGREGATION_SUM.equals(aggregation)) { searchSourceBuilder.aggregation(AggregationBuilders.sum( Constant.AGGREGATION_SUM).field(field)); } else if (Constant.AGGREGATION_AVG.equals(aggregation)) { searchSourceBuilder.aggregation(AggregationBuilders.avg( Constant.AGGREGATION_AVG).field(field)); } else if (Constant.AGGREGATION_MAX.equals(aggregation)) { searchSourceBuilder.aggregation(AggregationBuilders.max( Constant.AGGREGATION_MAX).field(field)); } else if (Constant.AGGREGATION_MIN.equals(aggregation)) { searchSourceBuilder.aggregation(AggregationBuilders.min( Constant.AGGREGATION_MIN).field(field)); } else if (Constant.AGGREGATION_COUNT.equals(aggregation)) { searchSourceBuilder.aggregation(AggregationBuilders.count( Constant.AGGREGATION_COUNT).field(DSA_LOG_APPID)); } else { String groupAggregation = configure.getGroupAggregation();// 分组聚合 String groupField = configure.getGroupField();// 分组字段 AggregationBuilder aggregationBuilder = AggregationBuilders.terms( "group").field(groupField); if (Constant.AGGREGATION_SUM.equals(groupAggregation)) { searchSourceBuilder.aggregation(aggregationBuilder .subAggregation(AggregationBuilders.sum( Constant.AGGREGATION_SUM).field(field))); } else if (Constant.AGGREGATION_AVG.equals(groupAggregation)) { searchSourceBuilder.aggregation(aggregationBuilder .subAggregation(AggregationBuilders.avg( Constant.AGGREGATION_AVG).field(field))); } else if (Constant.AGGREGATION_MAX.equals(groupAggregation)) { searchSourceBuilder.aggregation(aggregationBuilder .subAggregation(AggregationBuilders.max( Constant.AGGREGATION_MAX).field(field))); } else if (Constant.AGGREGATION_MIN.equals(groupAggregation)) { searchSourceBuilder.aggregation(aggregationBuilder .subAggregation(AggregationBuilders.min( Constant.AGGREGATION_MIN).field(field))); } else { searchSourceBuilder.aggregation(aggregationBuilder .subAggregation(AggregationBuilders.count( Constant.AGGREGATION_COUNT).field(DSA_LOG_APPID))); } } Search.Builder builder = new Search.Builder(searchSourceBuilder.toString()); if(Constant.ALL_SYSTEM.equals(configure.getLdcMessage())){ //全集群查询 builder.setHeader("x_dc_select", ""); }else { //处理一下ldcMessage String ldcId = configure.getLdcMessage().split("\\(")[1]; ldcId = ldcId.split("\\)")[0]; //单DC查询 if(Constant.CTDSA_POCA.equals(ldcId)){ ldcId = "poca"; } if(Constant.CTDSA_POCBYFB.equals(ldcId)){ ldcId = "pocb"; } if(Constant.CTDSA_POCBYG.equals(ldcId)){ ldcId = "pocb"; } builder.setHeader("x_dc_select", ldcId.toLowerCase()); } Search search = builder.addIndex(indices).setSearchType(SearchType.QUERY_AND_FETCH) .allowNoIndices(true).ignoreUnavailable(true).build(); Map<String,Object> map = new HashMap<>(); map.put(Constant.INDICES,indices); logger.info("Job(" + configure.getId() + "),indices:" + gson.toJson(indices) + ",queryESByJest Es query: \n " + searchSourceBuilder.toString()); try { JestResult jestResult = jestHttpClient.execute(search); map.put(Constant.RESULT,jestResult.getJsonObject()); return map; } catch (IOException e) { logger.info("queryESByJest:", e); ExceptionEventMessage message = new ExceptionEventMessage.MessageBuilder("AlarmElasticsearchDa").setEDescription("ES查询出现异常").setThrowable(e).createSystemExceptionMessage(); ExceptionEventCollector.sendMessage(message); } return null; } /** * 功能描述: <br> * 根据日志类型和查询时间段生成检索索引 * * @return * @see [相关类/方法](可选) * @since [产品/模块版本](可选) */ @Override public List<String> getIndices(String logType, long start, long end) { LogTypeEo logTypeEo = logTypeDao.getLogTypeByLogTypeNum(logType); if (logTypeEo != null) { String startString; String endString; SimpleDateFormat dateFormat = new SimpleDateFormat(logTypeEo.getLogTimeFormat()); if (logTypeEo.getLogTimeFormat().equals(Constant.TIME_FORMAT_WEEK)) { endString = ISODateTimeFormat.weekyearWeek().print(end).toLowerCase(); startString = ISODateTimeFormat.weekyearWeek().print(start).toLowerCase(); } else { startString = dateFormat.format(new Date(start)); endString = dateFormat.format(new Date(end)); } // 监控间隔最大为60分钟，所以只要处理相连两天的跨天情况，如果后面监控间隔超过24小时，则需要修改此方法 StringBuilder indices = new StringBuilder(logTypeEo.getLogIndexPattern()).append("-").append(endString); long time = end; String timeString ; long timeInterval ; switch (logTypeEo.getLogTimeFormat()) { case Constant.TIME_FORMAT_HOUR: timeInterval = Constant.HOUR_TIME_INTERVAL; while (true) { time = time - timeInterval; if (time <= start) {//NOSONAR break; } timeString = dateFormat.format(new Date(time)); if (!timeString.equals(endString)) {//NOSONAR indices.append("," + logTypeEo.getLogIndexPattern()).append("-").append(timeString); } } if (!startString.equals(endString)) { indices.append("," + logTypeEo.getLogIndexPattern()).append("-").append(startString); } break; case Constant.TIME_FORMAT_DAY: timeInterval = Constant.DAY_TIME_INTERVAL; while (true) { time = time - timeInterval; if (time <= start) {//NOSONAR break; } timeString = dateFormat.format(new Date(time)); if (!timeString.equals(endString)) {//NOSONAR indices.append("," + logTypeEo.getLogIndexPattern()).append("-").append(timeString); } } if (!startString.equals(endString)) { indices.append("," + logTypeEo.getLogIndexPattern()).append("-").append(startString); } break; case Constant.TIME_FORMAT_WEEK: timeInterval = Constant.WEEK_TIME_INTERVAL; while (true) { time = time - timeInterval; if (time <= start) {//NOSONAR break; } timeString = ISODateTimeFormat.weekyearWeek().print(time).toLowerCase(); if (!timeString.equals(endString)) {//NOSONAR indices.append("," + logTypeEo.getLogIndexPattern()).append("-").append(timeString); } } if (!startString.equals(endString)) { indices.append("," + logTypeEo.getLogIndexPattern()).append("-").append(startString); } break; case Constant.TIME_FORMAT_MONTH: timeInterval = Constant.MONTH_TIME_INTERVAL; while (true) { time = time / 1000 - timeInterval; if (time <= start / 1000) {//NOSONAR break; } timeString = dateFormat.format(new Date(time)); if (!timeString.equals(endString)) {//NOSONAR indices.append("," + logTypeEo.getLogIndexPattern()).append("-").append(timeString); } } if (!startString.equals(endString)) { indices.append("," + logTypeEo.getLogIndexPattern()).append("-").append(startString); } break; default: break; } return Arrays.asList(indices.toString().split(",")); } return new ArrayList<>(); } public void setGson(Gson gson) { this.gson = gson; } }
除此之外还有JAVA API - TransportClient也可以使用
