es-restHighLevel java api实战

版本为7.9.1，大体分为pom引入，封装注入Bean：RestHighLevelClient；封装具体的操作类，service层调用

pom引入

        <elasticsearch.version>7.9.1</elasticsearch.version>

        <dependency>
            <groupId>org.elasticsearch</groupId>
            <artifactId>elasticsearch</artifactId>
            <version>${elasticsearch.version}</version>
        </dependency>
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <exclusions>
                <exclusion>
                    <groupId>org.elasticsearch</groupId>
                    <artifactId>elasticsearch</artifactId>
                </exclusion>
            </exclusions>
            <version>${elasticsearch.version}</version>
        </dependency>

注入RestHighLevelClient

@Component
public class RestHighLevelClient {
    @Value("${elasticsearch.http.list}")
    private String httpList;
    @Value("${elasticsearch.max.retry.timeout}")
    private Integer maxRetryTimeoutMillis;
    @Value("${elasticsearch.connect.timeout}")
    private Integer connectTimeout;
    @Value("${elasticsearch.socket.timeout}")
    private Integer socketTimeout;
    @Value("${elasticsearch.connection.request.timeout}")
    private Integer connectionRequestTimeout;

    @Bean
    public RestHighLevelClient restHighLevelClient() throws Exception {
        List<HttpHost> httpHosts = new ArrayList<>();
        if (StringUtils.isNotBlank(httpList)){
            Arrays.stream(httpList.split(",")).forEach(ipAndPort -> httpHosts.add(HttpHost.create(HttpConstants.HTTP_PROTOCOL + ipAndPort)));
        }
       
        final RestClientBuilder esRestClientBuilder = RestClient.builder(httpHosts.toArray(new HttpHost[httpHosts.size()]))
//                .setMaxRetryTimeoutMillis(maxRetryTimeoutMillis)
                .setHttpClientConfigCallback(httpClientBuilder -> {
                    RequestConfig.Builder requestConfigBuilder = RequestConfig.custom()
                            //超时时间60s
                            .setConnectTimeout(connectTimeout)
                            //Socket超时时间设置
                            .setSocketTimeout(socketTimeout)
                            .setConnectionRequestTimeout(connectionRequestTimeout);
                    httpClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build());
                    return httpClientBuilder;
                });
        return new RestHighLevelClient(esRestClientBuilder);
    }

}

封装es的具体操作接口

首先注入RestHighLevelClient

@Autowired
private RestHighLevelClientDelegate restClient;

操作接口有很多，大体可以分为定义操作：index,alias,mapping，数据写入操作：document，template，检索操作：query检索和agg聚合查询

查询es集群状态

    public ClusterHealthStatus healthCheck() throws IOException {
        ClusterHealthRequest request = new ClusterHealthRequest();
        request.timeout(TimeValue.timeValueSeconds(30));
        ClusterHealthResponse response = restClient.cluster().health(request, RequestOptions.DEFAULT);
        return response.getStatus();
    }

查询索引是否存在

   /**
     * index API
     * 判断索引是否存在
     *
     * @param index
     * @return
     * @throws IOException
     */
    public Boolean indexExists(String index) throws IOException {
        GetIndexRequest request = new GetIndexRequest(index);
        return restClient.indices().exists(request, RequestOptions.DEFAULT);
    }

创建索引

   /**
     * index API
     * 创建索引
     * 尽量使用Map构造后提供json格式的参数
     */
    public String createIndex(String index, String settings, String mappings, List<Alias> aliases) throws IOException {
        CreateIndexRequest request = new CreateIndexRequest(index);
        if (StringUtils.isNotEmpty(settings)) {
            request.settings(settings, XContentType.JSON);
        }
        if (StringUtils.isNotEmpty(mappings)) {
            request.mapping(mappings, XContentType.JSON);
        }
        if (CollUtil.isNotEmpty(aliases)) {
            request.aliases(aliases);
        }
        CreateIndexResponse response = restClient.indices().create(request, RequestOptions.DEFAULT);
        return response.index();
    }

删除索引

   /**
     * index API
     * 删除索引
     *
     * @param index 索引名
     */
    public boolean deleteIndex(String index) throws Exception {
        DeleteIndexRequest request = new DeleteIndexRequest(index);
        AcknowledgedResponse response = restClient.indices().delete(request, RequestOptions.DEFAULT);
        return response.isAcknowledged();
    }

查询给定索引的mappings

   /**
     * 获取mappings
     *
     * @param index 索引名
     */
    public Map<String, Object> getMappings(@NotNull String index) throws Exception {
        GetMappingsRequest request = new GetMappingsRequest().indices(index);
        GetMappingsResponse response = restClient.indices().getMapping(request, RequestOptions.DEFAULT);
        Map<String, MappingMetadata> allMappings = response.mappings();
        MappingMetadata indexMapping = allMappings.get(index);
        if (Objects.nonNull(indexMapping)) {
            Object obj = indexMapping.getSourceAsMap().get("properties");
            return (Map<String, Object>) obj;
        }
        return null;
    }

开启和关闭索引

   /**
     * index API
     * 关闭索引后，该索引无法再执行读写
     *
     * @param index 索引名
     */
    public boolean closeIndex(String index) throws Exception {
        CloseIndexRequest request = new CloseIndexRequest("index");
        AcknowledgedResponse response = restClient.indices().close(request, RequestOptions.DEFAULT);
        return response.isAcknowledged();
    }

    /**
     * index API
     * 开启索引，与关闭相反
     *
     * @param index 索引名
     */
    public boolean openIndex(String index) throws Exception {
        OpenIndexRequest request = new OpenIndexRequest(index);
        AcknowledgedResponse response = restClient.indices().open(request, RequestOptions.DEFAULT);
        return response.isAcknowledged();
    }

创建和删除存储策略

   /**
     * 判断存储策略是否存在
     *
     * @param policyName 存储策略名称
     * @return {@code true}表示存在，{@code false}表示不存在
     * @throws Exception 当解析返回结果失败，或者请求超时或者无响应等时，会抛出异常。
     */
    public boolean policyExists(String policyName) throws Exception {
        GetLifecyclePolicyRequest request =
                new GetLifecyclePolicyRequest(policyName);
        GetLifecyclePolicyResponse response = restClient.indexLifecycle()
                .getLifecyclePolicy(request, RequestOptions.DEFAULT);
        ImmutableOpenMap<String, LifecyclePolicyMetadata> policies = response.getPolicies();
        LifecyclePolicyMetadata myPolicyMetadata = policies.get(policyName);
        return Objects.nonNull(myPolicyMetadata);
    }
   /**
     * 创建存储策略
     *
     * @param policyName 存储策略名称
     * @param days       保存的天数
     * @return {@code true}表示执行成功，{@code false}表示执行失败
     * @throws Exception 当解析返回结果失败，或者请求超时或者无响应等时，会抛出异常。
     */
    public boolean createPolicy(String policyName, Integer days) throws IOException {
        Map<String, Phase> phases = new HashMap<>();
        Map<String, LifecycleAction> hotActions = new HashMap<>();
        hotActions.put(RolloverAction.NAME, new RolloverAction(
                new ByteSizeValue(30, ByteSizeUnit.GB), new TimeValue(1, TimeUnit.DAYS), null));
        phases.put("hot", new Phase("hot", TimeValue.ZERO, hotActions));

        Map<String, LifecycleAction> deleteActions =
                Collections.singletonMap(DeleteAction.NAME, new DeleteAction());
        phases.put("delete", new Phase("delete", new TimeValue(days, TimeUnit.DAYS), deleteActions));

        LifecyclePolicy policy = new LifecyclePolicy(policyName, phases);
        PutLifecyclePolicyRequest request = new PutLifecyclePolicyRequest(policy);
        org.elasticsearch.client.core.AcknowledgedResponse response = restClient.indexLifecycle()
                .putLifecyclePolicy(request, RequestOptions.DEFAULT);
        return response.isAcknowledged();
    }

    /**
     * 删除存储策略
     *
     * @param policyName 存储策略名称
     * @return {@code true}表示执行成功，{@code false}表示执行失败
     * @throws Exception 当解析返回结果失败，或者请求超时或者无响应等时，会抛出异常。
     */
    public boolean deletePolicy(String policyName) throws Exception {
        DeleteLifecyclePolicyRequest request =
                new DeleteLifecyclePolicyRequest(policyName);
        org.elasticsearch.client.core.AcknowledgedResponse response = restClient.indexLifecycle()
                .deleteLifecyclePolicy(request, RequestOptions.DEFAULT);
        return response.isAcknowledged();
    }

创建和删除索引模板

   /**
     * 创建索引模板
     *
     * @param templateName 模板名
     * @param patterns     索引名的通配符 如 log_*
     * @param alias        可选，索引别名
     * @param mappings     映射
     * @param setting      设置
     * @return 是否创建成功
     * @throws Exception 当解析返回结果失败，或者请求超时或者无响应等时，会抛出异常。
     */
    public boolean createTemplate(@NotNull String templateName,
                                  List<String> patterns,
                                  String alias,
                                  String mappings,
                                  Settings.Builder setting) throws IOException {
        PutIndexTemplateRequest request = new PutIndexTemplateRequest(templateName);
        request.patterns(patterns);
        if (setting != null) {
            request.settings(setting);
        }
        if (mappings != null) {
            request.mapping(mappings, XContentType.JSON);
        }
        //可以设置filter和searchRouting
        if (alias != null) {
            request.alias(new Alias(alias));
        }
        //设置已经存在的模板不被覆盖
        request.create(true);
        AcknowledgedResponse response = restClient.indices().putTemplate(request, RequestOptions.DEFAULT);
        return response.isAcknowledged();
    }

    /**
     * 删除索引模板
     *
     * @param templateName 索引模板的名称
     * @return {@code true}表示执行成功，{@code false}表示执行失败
     * @throws Exception 当解析返回结果失败，或者请求超时或者无响应等时，会抛出异常。
     */
    public boolean deleteTemplate(String templateName) throws Exception {
        DeleteIndexTemplateRequest request = new DeleteIndexTemplateRequest(templateName);
        AcknowledgedResponse deleteTemplateAcknowledge = restClient.indices().deleteTemplate(request, RequestOptions.DEFAULT);
        return deleteTemplateAcknowledge.isAcknowledged();
    }

    /**
     * 判断索引模板是否存在
     *
     * @param templateName 索引模板的名称
     * @return {@code true}表示存在，{@code false}表示不存在
     * @throws Exception 当解析返回结果失败，或者请求超时或者无响应等时，会抛出异常。
     */
    public boolean templateExists(String templateName) throws Exception {
        IndexTemplatesExistRequest request = new IndexTemplatesExistRequest(templateName);
        return restClient.indices().existsTemplate(request, RequestOptions.DEFAULT);
    }

-------------------------------------------------------------------------------------------

一般的分页查询和结果解析

   /**
     * 分页查询
     *
     * @param index        索引名
     * @param queryBuilder 查询条件
     * @param pageNum      第几页
     * @param pageSize     每页条数
     * @return 返回SearchHits, 由调用方处理
     * @throws IOException 查询异常,抛出到真正的调用业务的逻辑去处理
     */
    public SearchResponse searchByPage(String[] index, QueryBuilder queryBuilder, String sort, Integer pageNum, Integer pageSize) throws Exception {
        SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.searchSource()
                .query(queryBuilder)
                .size(pageSize)
                .from((pageNum - 1) * pageSize);
        if (sort != null) {
            searchSourceBuilder.sort(SortBuilders.fieldSort(Constants.DEFAULT_FIELD_NAME).order(SortOrder.fromString(sort)).unmappedType("keyword"));
        }
        SearchRequest searchRequest = Requests.searchRequest(index)
                .source(searchSourceBuilder)
                .searchType(SearchType.QUERY_THEN_FETCH);
        log.info("query DSL:[{}]", searchSourceBuilder);
        return restClient.search(searchRequest, RequestOptions.DEFAULT);
    }
    
    SearchResponse response = esService.searchByPage(new String[]{index}, queryBuilder, sort, pageNum, pageSize);
    SearchHit[] hitArr = response.getHits().getHits();
    List<SearchHit> hitList = Arrays.asList(hitArr);
    List<Map<String, Object>> res = new ArrayList<>();
    if (CollUtil.isNotEmpty(hitList)) {
       for (SearchHit hit : hitList) {
           res.add(new HashMap<String, Object>() {{
               Map<String, Object> sourceAsMap = hit.getSourceAsMap();                      
               put("_source", sourceAsMap);
               put("_index", hit.getIndex());                      
               put("_id", hit.getId());
               }});
       }
    }

一般的count聚合

/**
     * count查询
     *
     * @param index
     * @param qb
     * @return
     * @throws IOException
     */
    public long count(String index, QueryBuilder qb) throws Exception {
        CountRequest countRequest = new CountRequest();
        countRequest.indices(index).query(qb);
        CountResponse response = restClient.count(countRequest, RequestOptions.DEFAULT);
        return response.getCount();
    }

   /**
     * 聚合查询count
     * 可考虑按普通查询，然后取出searchHits.total属性
     *
     * @param index     索引名（别名）
     * @param fieldName 使用主键
     * @param qb        条件
     * @return 条数
     * @throws IOException 抛出异常，让调用方处理
     */
    public long count(String index, String fieldName, QueryBuilder qb) throws Exception {
        AggregationBuilder countBuilder = AggregationBuilders.count("countResult").field("log_id");
        SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.searchSource()
                .query(qb)
                .size(0)
                .aggregation(countBuilder);
        SearchRequest searchRequest = Requests.searchRequest(index).source(searchSourceBuilder).searchType(SearchType.QUERY_THEN_FETCH);
        SearchResponse response = restClient.search(searchRequest, RequestOptions.DEFAULT);
        ValueCount count = response.getAggregations().get("countResult");
        return count.getValue();
    }

一般的sum聚合

   /**
     * 聚合查询-sum
     * size设置为0表示不返回参与聚合的记录
     *
     * @param index     索引名（别名）
     * @param fieldName 聚合的字段
     * @param qb        聚合的条件
     * @return 聚合的结果 double类型
     * @throws IOException 聚合异常抛出到调用方处理
     */
    public double sum(String index, String fieldName, QueryBuilder qb) throws Exception {
        AggregationBuilder sumBuilder = AggregationBuilders.sum("sumResult").field(fieldName);
        SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.searchSource()
                .query(qb)
                .size(0)
                .aggregation(sumBuilder);
        log.info("query DSL:[{}]", searchSourceBuilder);
        SearchRequest searchRequest = Requests.searchRequest(index).source(searchSourceBuilder).searchType(SearchType.QUERY_THEN_FETCH);
        SearchResponse response = restClient.search(searchRequest, RequestOptions.DEFAULT);
        Sum sum = response.getAggregations().get("sumResult");
        return sum.getValue();
    }

一般的count+group by聚合

   /**
     * 聚合查询  terms
     * size设置为0表示不返回参与聚合的记录
     *
     * @param index    索引名（别名）
     * @param termName 聚合的字段
     * @param qb       聚合的条件
     * @return 聚合的结果 double类型
     * @throws IOException 聚合异常抛出到调用方处理
     */
    public Terms terms(String index, String termName, QueryBuilder qb) throws Exception {
        TermsAggregationBuilder term = AggregationBuilders.terms("termResult").field(termName).size(100);
        SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.searchSource()
                .query(qb)
                .size(0)
                .aggregation(term);
        log.info("query DSL:[{}]", searchSourceBuilder);
        SearchRequest searchRequest = Requests.searchRequest(index).source(searchSourceBuilder).searchType(SearchType.QUERY_THEN_FETCH);
        SearchResponse response = restClient.search(searchRequest, RequestOptions.DEFAULT);
        return response.getAggregations().get("termResult");
    }

按天count+group by 聚合

   /**
     * 聚合查询  termsByDay
     * size设置为0表示不返回参与聚合的记录
     *
     * @param index    索引名（别名）
     * @param termName 聚合的字段
     * @param qb       聚合的条件
     * @return 聚合的结果 double类型
     * @throws IOException 聚合异常抛出到调用方处理
     */
    public Aggregation aggByDay(String index, String termName, QueryBuilder qb) throws Exception {
        // DateHistogramInterval.DAY 加上时区否则聚类不准
        AggregationBuilder term = AggregationBuilders.dateHistogram("histogramResult")
                .calendarInterval(DateHistogramInterval.DAY)
                .field(termName)
                .timeZone(ZoneId.systemDefault());
        SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.searchSource()
                .query(qb)
                .size(0)
                .aggregation(term);
        log.info("query DSL:[{}]", searchSourceBuilder);
        SearchRequest searchRequest = Requests.searchRequest(index).source(searchSourceBuilder).searchType(SearchType.QUERY_THEN_FETCH);
        SearchResponse response = restClient.search(searchRequest, RequestOptions.DEFAULT);
        return response.getAggregations().get("histogramResult");
    }

在指定的时间范围内按照时间进行柱状图统计

   /**
     * 按某个指定的时间字段搜索一定时间范围内，各个细分时间段的记录的数量
     *
     * @param index     搜索的索引，可以同时搜索多个索引
     * @param interval  时间间隔按情况给
     * @param qb        查询条件
     * @param beginTime 搜索范围的开始时间
     * @param endTime   搜索范围的结束时间
     * @return 结果集
     * @throws IOException IO异常
     */
    public List<BucketDto> dateHistogram(String[] index, String interval, QueryBuilder qb, String beginTime, String endTime) throws Exception {
        DateHistogramAggregationBuilder dateHistogramAB = AggregationBuilders
                .dateHistogram("histogramResult")
                .field(Constants.DEFAULT_LOG_TIME_FIELD_NAME)
                .timeZone(ZoneId.systemDefault())
                .keyed(true)
                .fixedInterval(new DateHistogramInterval(interval))
                .format(getDateFormat())//KeyAsString()的格式
                .minDocCount(0L);//设置返回空桶
        long beginBound = 0;
        long endBound = System.currentTimeMillis();
        try {
            if (StringUtils.isNotBlank(beginTime)) {
                beginBound = TimeUtils.getLongTime(beginTime);
            }
            if (StringUtils.isNotBlank(endTime)) {
                endBound = TimeUtils.getLongTime(endTime);
            }
        } catch (Exception e) {
            log.error("search dateHistogram error", e);
            throw new CommonException("时间解析错误");
        }
        dateHistogramAB.extendedBounds(new ExtendedBounds(beginBound, endBound));//如果设置返回空桶，必须设置时间边界，否则无效
        SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.searchSource()
                .query(qb)
                .size(0)
                .aggregation(dateHistogramAB);
        log.info("query DSL:[{}]", searchSourceBuilder);
        SearchRequest searchRequest = Requests.searchRequest(index).source(searchSourceBuilder).searchType(SearchType.QUERY_THEN_FETCH);
        SearchResponse response = restClient.search(searchRequest, RequestOptions.DEFAULT);
        ParsedDateHistogram histogramResult = response.getAggregations().get("histogramResult");
        List<BucketDto> list = new ArrayList<>();
        for (Histogram.Bucket bucket : histogramResult.getBuckets()) {
            list.add(new BucketDto(bucket.getKeyAsString(), bucket.getDocCount()));
        }
        return list;
    }

一次执行多个count聚合

   /**
     * 一次执行多个count
     */
    public List<Long> multiCount(String index, List<QueryBuilder> qbList) throws Exception {
        MultiSearchRequest request = new MultiSearchRequest();
        AggregationBuilder countBuilder = AggregationBuilders.count("countResult").field("log_id");
        for (QueryBuilder qb : qbList) {
            SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.searchSource()
                    .query(qb)
                    .size(0)
                    .aggregation(countBuilder);
            log.info("query DSL:[{}]", searchSourceBuilder);
            SearchRequest searchRequest = Requests.searchRequest(index).source(searchSourceBuilder).searchType(SearchType.QUERY_THEN_FETCH);
            request.add(searchRequest);
        }
        MultiSearchResponse response = restClient.msearch(request, RequestOptions.DEFAULT);
        List<Long> countList = new ArrayList<>();
        for (MultiSearchResponse.Item item : response.getResponses()) {
            long count = -1L;
            if (!item.isFailure()) {
                ValueCount valueCount = item.getResponse().getAggregations().get("countResult");
                count = valueCount.getValue();
            }
            countList.add(count);
        }
        return countList;
    }

一次查询多个

/**
     * 一次执行多个查询
     *
     * @param index   相同的索引名(别名)
     * @param SSBList (检索条件)
     * @return 按顺序返回的多个结果
     * @throws IOException 调用方处理结果
     */
    public MultiSearchResponse.Item[] multiSearch(String index, List<SearchSourceBuilder> SSBList) throws IOException {
        MultiSearchRequest request = new MultiSearchRequest();
        for (SearchSourceBuilder ssb : SSBList) {
            log.info("query DSL:[{}]", ssb.toString());
            SearchRequest searchRequest = Requests.searchRequest(index).source(ssb);
            request.add(searchRequest);
        }
        MultiSearchResponse response = restClient.msearch(request, RequestOptions.DEFAULT);
        return response.getResponses();
    }

posted @ 2022-07-06 17:28 鼠标的博客阅读(788) 评论(0) 编辑收藏举报

刷新页面返回顶部

鼠标的博客

es-restHighLevel java api实战

pom引入

注入RestHighLevelClient

封装es的具体操作接口

公告