【Java】ElasticSearch 在项目里的应用

一、前言：

好久没写笔记了，最近忙一个项目，用到ES查询，以往的笔记写ES都是搭建环境，用Kibana玩一玩

这次是直接调用API操作了，话不多说，进入主题

 

二、环境前提：

公司用的还是纯ElasticSearch的API库，并没有Spring-Data-ES的包装

ElasticSearch版本是7.3.1

这是封装的包：

?
1
2
3
4
5
6
<!-- es start -->
<dependency>
    <groupId>cn.ymcd.comm</groupId>
    <artifactId>comm-elasticsearch</artifactId>
    <version>1.0.3</version>
</dependency>

然后看下里面的ES依赖：

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
<dependency>
  <groupId>org.elasticsearch.client</groupId>
  <artifactId>elasticsearch-rest-high-level-client</artifactId>
  <version>7.3.0</version><!--$NO-MVN-MAN-VER$-->
</dependency>
 
<dependency>
  <groupId>org.elasticsearch</groupId>
  <artifactId>elasticsearch</artifactId>
  <version>7.3.0</version><!--$NO-MVN-MAN-VER$-->
</dependency>
 
<dependency>
  <groupId>org.elasticsearch.client</groupId>
  <artifactId>transport</artifactId>
  <version>7.3.0</version><!--$NO-MVN-MAN-VER$-->
</dependency>
 
<dependency>
  <groupId>org.elasticsearch.plugin</groupId>
  <artifactId>transport-netty4-client</artifactId>
  <version>7.3.0</version><!--$NO-MVN-MAN-VER$-->
</dependency>
 
<dependency>
  <groupId>org.elasticsearch.client</groupId>
  <artifactId>elasticsearch-rest-client</artifactId>
  <version>7.3.0</version><!--$NO-MVN-MAN-VER$-->
</dependency>

　　

包源码只有一个客户端类：

 

类似JDBC的连接，提供主机，账户密码信息，调用客户端对象方法获取连接资源

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
//
// Source code recreated from a .class file by IntelliJ IDEA
// (powered by Fernflower decompiler)
//
 
package cn.ymcd.comm.elasticsearch;
 
import cn.ymcd.comm.base.log.LogFactory;
import cn.ymcd.comm.base.log.YmcdLogger;
import java.util.Arrays;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.RestClientBuilder.HttpClientConfigCallback;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Component;
 
@Configuration
@Component("esClient")
public class EsRestClient implements AutoCloseable {
    private YmcdLogger logger = LogFactory.getLogger(this.getClass());
    @Value("${elasticsearch.host-name}")
    private String hostName;
    @Value("${elasticsearch.port:9200}")
    private int port;
    @Value("${elasticsearch.cluster:}")
    private String cluster;
    @Value("${elasticsearch.userName:}")
    private String userName;
    @Value("${elasticsearch.password:}")
    private String password;
    protected RestHighLevelClient client;
 
    public EsRestClient() {
    }
 
    public RestHighLevelClient getEsClient() {
        RestClientBuilder builder = null;
        if (StringUtils.isNotBlank(this.cluster)) {
            this.logger.debug("connect to cluster server...");
            List<HttpHost> esHosts = (List)Arrays.stream(this.cluster.split(",")).map(HttpHost::create).collect(Collectors.toList());
            builder = RestClient.builder((HttpHost[])esHosts.toArray(new HttpHost[esHosts.size()]));
        } else {
            this.logger.debug("connect to single node server...");
            builder = RestClient.builder(new HttpHost[]{new HttpHost(this.hostName, this.port)});
        }
 
        if (StringUtils.isNotBlank(this.userName) && StringUtils.isNotBlank(this.password)) {
            final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(this.userName, this.password));
            builder.setHttpClientConfigCallback(new HttpClientConfigCallback() {
                public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                    return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
                }
            });
        }
 
        this.client = new RestHighLevelClient(builder);
        this.logger.debug("connected to server!");
        return this.client;
    }
 
    public void close() {
        if (this.client != null) {
            this.logger.debug("close es client...");
 
            try {
                this.client.close();
            } catch (Exception var2) {
                this.logger.error("close es client error!", var2);
            }
        }
 
    }
}

 

三、API封装：

封装了，但是没完全封装

1、我要做一个翻页查询都没有，还得我自己加上去整一个，麻了

2、有提供一个ES的索引名称注解和泛型声明，为什么返回类型没有一个按泛型返回的，还得是自己写

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
package cn.ymcd.perception.common.service;
 
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.SortBuilder;
 
import java.util.List;
 
/**
 * ES接口
 *
 * @projectName: perception-task-server
 * @author: panx
 * @date: 2023年09月12日 10:04
 * @version: 1.0
 */
public interface IEsBaseService<T> {
 
    /**
     * 基础查询 根据id查询 数据
     *
     * @param entity
     * @param id
     * @return java.lang.String
     * @author panx
     * @createTime 2023/9/12 0012 17:14
     */
    String getById(T entity, String id);
 
    /**
     * 根据多个id 查询数据信息
     *
     * @param entity
     * @param list
     * @return org.elasticsearch.action.search.SearchResponse
     * @author panx
     * @createTime 2023/9/12 0012 17:14
     */
    SearchResponse findByIdsList(T entity, List<String> list);
 
    /**
     * 查询所有信息
     *
     * @param from
     * @param size
     * @param entity
     * @return org.elasticsearch.search.SearchHit[]
     * @author panx
     * @createTime 2023/9/12 0012 17:15
     */
    SearchHit[] queryAll(int from, int size, T entity);
 
    /**
     * 根据条件查询
     *
     * @param indexName
     * @param page
     * @param boolQueryBuilder
     * @param highlightBuilder
     * @param sortBuilder
     * @return org.elasticsearch.action.search.SearchResponse
     * @author panx
     * @createTime 2023/9/12 0012 17:15
     */
    SearchResponse whereQuery(String indexName, Page page, BoolQueryBuilder boolQueryBuilder, HighlightBuilder highlightBuilder, SortBuilder sortBuilder);
 
 
    /**
     * @author OnCloud9
     * @date 2023/9/14 13:36
     * @description 翻页查询
     * @param tClass
     * @param page
     * @param boolQueryBuilder
     * @param highlightBuilder
     * @param sortBuilder
     * @return com.baomidou.mybatisplus.extension.plugins.pagination.Page<T>
     */
    <Entity> Page<Entity> pageQuery(Class<Entity> tClass, Page page, BoolQueryBuilder boolQueryBuilder, HighlightBuilder highlightBuilder, SortBuilder sortBuilder);
 
    /**
     * 设置高亮显示字段
     *
     * @param fields
     * @return org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder
     * @author panx
     * @createTime 2023/9/12 0012 17:15
     */
    HighlightBuilder highlightBuilder(String... fields);
 
    /**
     * 获取排序规则
     *
     * @param fieldSort
     * @param isDesc
     * @return org.elasticsearch.search.sort.SortBuilder
     * @author panx
     * @createTime 2023/9/12 0012 17:15
     */
    SortBuilder getSortBuilder(String fieldSort, Boolean isDesc);
 
    /**
     * 获取高亮显示的值
     *
     * @param hit
     * @param field
     * @return java.lang.String
     * @author panx
     * @createTime 2023/9/12 0012 17:15
     */
    String getHighlightContent(SearchHit hit, String field);
 
    /**
     * 查询 索引中所有满足条件数据 游标  查询
     *
     * @param response
     * @param restHighLevelClient
     * @return java.util.List<org.elasticsearch.search.SearchHits>
     * @author panx
     * @createTime 2023/9/12 0012 17:16
     */
    List<SearchHits> getAllData(SearchResponse response, RestHighLevelClient restHighLevelClient);
 
    /**
     * 获取ES中的总数
     *
     * @param indexName
     * @param boolQueryBuilder
     * @return long
     * @author panx
     * @createTime 2023/9/12 0012 17:16
     */
    long getCount(String indexName, BoolQueryBuilder boolQueryBuilder);
 
    /**
     * 获取 注解索引名称
     *
     * @param entity
     * @return java.lang.String
     * @author panx
     * @createTime 2023/9/12 0012 17:16
     */
    String getIndexName(T entity);
 
    /**
     * @author OnCloud9
     * @date 2023/9/17 17:10
     * @description 聚合查询
     * @params [tClass, aggregationBuilder, resultName]
     * @return java.util.List<java.util.Map<java.lang.String,java.lang.String>>
     */
    <Entity> List<Terms.Bucket>  getAggregationQuery(Class<Entity> tClass, AggregationBuilder aggregationBuilder, String resultName);
 
    /**
     * @author OnCloud9
     * @date 2023/9/18 15:40
     * @description 条件聚合查询
     * @params [tClass, boolQueryBuilder, aggregationBuilder, resultName]
     * @return java.util.List<org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket>
     */
    <Entity> List<Terms.Bucket>  getConditionAggregationQuery(
            Class<Entity> tClass,
            BoolQueryBuilder boolQueryBuilder,
            AggregationBuilder aggregationBuilder,
            String resultName
    );
}

　　

首先是实体的索引名称注解：

我们项目的mysql表名 直接对应到es的索引名上，数据来源也是mysql推到es上面，统一规范了

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
package cn.ymcd.perception.base;
 
import java.lang.annotation.*;
 
/***
 * 注解 索引信息
 * @param
 * @return
 * @author gaof
 * @createTime 2020/3/17 17:05
 * @version: 1.0
 */
@Inherited
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.TYPE})
public @interface EsIndex {
 
    /***
     * 索引名称
     * @param
     * @return java.lang.String
     * @author gaof
     * @createTime 2020/3/17 17:06
     * @version: 1.0
     */
    String indexName();
 
}

　　

 

注解的获取方式，因为做ES的CRUD都需要知晓是在哪个索引下操作

所以每个方法必定需要索引的参数传入，

叼毛同事非要从对象反射过去找注解，我给他重载下，从字节对象读就行了

这里说下接口上的泛型声明很不好，因为我在写业务的时候发现并不是只有一个索引要操作，需要多个索引操作，这个泛型声明限定死了

所以我写的方法都改用方法泛型，这样调用的时候才支持不同的业务实体

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
@Override
public String getIndexName(T entity) {
    EsIndex index = entity.getClass().getAnnotation(EsIndex.class);
    String indexName = index.indexName();
    if (StringUtils.isBlank(indexName)) {
        logger.error("注解索引名称为空");
        throw new ElasticsearchException("注解indexName(索引名称)为空");
    }
    return indexName;
}
public <Entity> String getIndexName(Class<Entity> tClass) {
    EsIndex index = tClass.getAnnotation(EsIndex.class);
    String indexName = index.indexName();
    if (StringUtils.isBlank(indexName)) {
        logger.error("注解索引名称为空");
        throw new ElasticsearchException("注解indexName(索引名称)为空");
    }
    return indexName;
}

　　

其它的就是如何操作API了

这里重点说下PageQuery这个方法，要解决几个问题：

1、怎么接收返回的结果，ES的结果叫命中对象，放在一个数组里面，存的是JSON串

　　这里根据入参的实体类字节对象，交给可以做JSON序列化的工具活化JSON给对象就行了，这里用的FastJson

2、解决翻页问题，这里我不想冗余代码了，所以直接在同事写的whereQuery基础上套参写

　　在查询前算好from + to的参数值，返回的结果集塞回Page翻页对象交出去，

　　可以不传Page对象，那我默认认为调用者需要查询全部记录，就按一般索引支持的最大记录数翻页

3、索引翻页问题，因为在2上面说过，索引存在一个最大记录数的限制，有可能这个索引存了一万五千条数据，但是翻页查询只能翻到前一万条数据

　　在这个封装的工具方法中可以使用getCount方法获取真实的总记录数，也可以通过查询响应的命中对象获取总共的命中数量

　　解决的方法可以参考下链接： https://zhuanlan.zhihu.com/p/489562200

　　无非就三种， 1 调参数加大、2 Scroll滚动查询、3 SearchAfter标记查询

　　而在我的业务场景就是把ES数据带到功能上，要翻页查询，经理说不能调参数，后面两种办法又不能实现分页功能

　　所以折中的办法就是不调整，查到1万位置，默认认为用户不需要再看后面的内容

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
@Override
public String getById(T entity, String id) {
    String real = "";
    String indexName = getIndexName(entity);
    SearchSourceBuilder builder = new SearchSourceBuilder();
    SearchRequest request = new SearchRequest(indexName);
    builder.query(QueryBuilders.termQuery("id", id));
    request.source(builder);
    SearchResponse response = null;
    try (RestHighLevelClient restHighLevelClient = getEsClient()) {
        response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
        for (SearchHit hit : response.getHits()) {
            real = hit.getSourceAsString();
        }
    } catch (IOException e) {
        logger.error("根据id查询数据获取索引异常", e);
    }
    return real;
}<br>
@Override
public SearchResponse findByIdsList(T entity, List<String> list) {
    SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
    SearchRequest request = new SearchRequest(getIndexName(entity));
    sourceBuilder.query(QueryBuilders.termsQuery("id", list));
    request.source(sourceBuilder);
    SearchResponse response = null;
    try (RestHighLevelClient restHighLevelClient = getEsClient()) {
        response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
    } catch (IOException e) {
        logger.error("根据多个id查询数据异常", e);
    }
    return response;
}<br>
@Override
public SearchHit[] queryAll(int from, int size, T entity) {
    SearchRequest searchRequest = new SearchRequest(getIndexName(entity));
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    searchSourceBuilder.query();
    searchSourceBuilder.from(from);
    searchSourceBuilder.size(size);
    SearchHit[] hitsArr = null;
    try (RestHighLevelClient restHighLevelClient = getEsClient()) {
        searchRequest.source(searchSourceBuilder);
        SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        close();
        SearchHits hits = searchResponse.getHits();
        hitsArr = hits.getHits();
    } catch (IOException e) {
        this.logger.error("查询数据失败", e);
    }
    return hitsArr;
}<br>
@Override
public SearchResponse whereQuery(String indexName, Page page, BoolQueryBuilder boolQueryBuilder, HighlightBuilder highlightBuilder, SortBuilder sortBuilder) {
    if (StringUtils.isBlank(indexName)) {
        return null;
    }
    SearchRequest searchRequest = new SearchRequest(indexName);
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    // 设置查询条数
    if (null != page) {
        Long current = page.getCurrent();
        Long size = page.getSize();
        searchSourceBuilder.from(current.intValue());
        searchSourceBuilder.size(size.intValue());
    }
    //设置需要排序的字段
    if (null != sortBuilder) {
        searchSourceBuilder.sort(sortBuilder);
    }
    // 设置高亮,使用默认的highlighter高亮器
    if (null != highlightBuilder) {
        searchSourceBuilder.highlighter(highlightBuilder);
    }
    searchSourceBuilder.query(boolQueryBuilder);
    try (RestHighLevelClient restHighLevelClient = getEsClient()) {
        searchRequest.source(searchSourceBuilder);
        logger.info("query ES where... indexName = " + indexName + ":" + searchSourceBuilder.toString());
        return restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
    } catch (Exception e) {
        logger.error("查询ES数据信息失败", e);
    }
    return null;
}<br>
@Override
public <Entity> Page<Entity> pageQuery(Class<Entity> tClass, Page page, BoolQueryBuilder boolQueryBuilder, HighlightBuilder highlightBuilder, SortBuilder sortBuilder) {
    String indexName = getIndexName(tClass);
    if (Objects.isNull(page)) page = new Page<>(1, 10000);
    Long current = page.getCurrent();
    page.setCurrent((current - 1L) * page.getSize());
    SearchResponse searchResponse = whereQuery(indexName, page, boolQueryBuilder, highlightBuilder, sortBuilder);
    SearchHits searchHits = searchResponse.getHits();
    long value = searchResponse.getHits().getTotalHits().value; /* 超出最大记录数配置，按最大记录数返回 */
    List<Entity> records = new ArrayList<>();
    for (SearchHit searchHit : searchHits) {
        String recordJson = searchHit.getSourceAsString();
        logger.info("recordJson " + recordJson);
        Entity t = JSON.parseObject(recordJson, tClass);
        records.add(t);
    }
    page.setRecords(records);
    page.setTotal(value);
    return page;
}<br>
@Override
public HighlightBuilder highlightBuilder(String... fields) {
    if (null != fields && fields.length > 0) {
        HighlightBuilder highlightBuilder = new HighlightBuilder();
        for (String field : fields) {
            highlightBuilder.field(field);
        }
        highlightBuilder.preTags("<span style=\"color:red;\">")
                .postTags("</span>");
        return highlightBuilder;
    }
    return null;
}<br>
@Override
public SortBuilder getSortBuilder(String fieldSort, Boolean isDesc) {
    if (Boolean.TRUE.equals(isDesc)) {
        return SortBuilders.fieldSort(fieldSort).order(SortOrder.DESC);
    }
    return SortBuilders.fieldSort(fieldSort).order(SortOrder.ASC);
}<br>
@Override
public String getHighlightContent(SearchHit hit, String field) {
    if (StringUtils.isBlank(field)) {
        return null;
    }
    HighlightField highlightField = hit.getHighlightFields().get(field);
    StringBuilder sub = new StringBuilder();
    if (null != highlightField) {
        Text[] contents = highlightField.getFragments();
        if (null != contents) {
            for (Text t : contents) {
                sub.append(t);
            }
        }
    }
    return sub.toString();
}<br>
@Override
public List<SearchHits> getAllData(SearchResponse response, RestHighLevelClient restHighLevelClient) {
    boolean succeeded = false;
    List<SearchHits> hitList = new ArrayList<>();
    try {
        String scrollId = response.getScrollId();
        SearchHits searchHits = response.getHits();
        hitList.add(searchHits);
        // 根据游标查询所有数据
        while (searchHits.getHits() != null && searchHits.getHits().length > 0) {
            SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
            scrollRequest.scroll(TimeValue.timeValueMillis(30));
            response = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT);
            scrollId = response.getScrollId();
            searchHits = response.getHits();
            hitList.add(searchHits);
        }
        // 查询完清除游标
        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
        clearScrollRequest.addScrollId(scrollId);
        ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
        succeeded = clearScrollResponse.isSucceeded();
    } catch (IOException e) {
        logger.error("统一方法索引查询所有数据发生异常", e);
    }
    if (!succeeded) {
        return new ArrayList<>();
    }
    return hitList;
}<br>
@Override
public long getCount(String indexName, BoolQueryBuilder boolQueryBuilder) {
    CountRequest countRequest = new CountRequest(indexName);
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    searchSourceBuilder.query(boolQueryBuilder);
    try (RestHighLevelClient restHighLevelClient = getEsClient()) {
        countRequest.source(searchSourceBuilder);
        CountResponse countResponse = restHighLevelClient.count(countRequest, RequestOptions.DEFAULT);
        return countResponse.getCount();
    } catch (Exception e) {
        logger.error("统计ES数据失败", e);
    }
    return 0L;
}<br>
@SuppressWarnings("Duplicates")
@Override
public <Entity> List<Terms.Bucket> getAggregationQuery(Class<Entity> tClass, AggregationBuilder aggregationBuilder, String resultName) {
    String indexName = getIndexName(tClass);
    SearchRequest searchRequest = new SearchRequest(indexName);
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    searchSourceBuilder.aggregation(aggregationBuilder);
    searchSourceBuilder.size(0);
    try (RestHighLevelClient restHighLevelClient = getEsClient()) {
        searchRequest.source(searchSourceBuilder);
        logger.info("query ES where... indexName = " + indexName + ":" + searchSourceBuilder.toString());
        SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        Aggregations aggregations = searchResponse.getAggregations();
        Terms terms = aggregations.get(resultName);
        List<? extends Terms.Bucket> buckets = terms.getBuckets();
        List<Terms.Bucket> returnBuckets = new ArrayList<>(buckets.size());
        returnBuckets.addAll(buckets);
        return returnBuckets;
    } catch (Exception e) {
        logger.error("查询ES数据信息失败", e);
        return Collections.emptyList();
    }
}<br>
@SuppressWarnings("Duplicates")
@Override
public <Entity> List<Terms.Bucket> getConditionAggregationQuery(Class<Entity> tClass, BoolQueryBuilder boolQueryBuilder, AggregationBuilder aggregationBuilder, String resultN
    String indexName = getIndexName(tClass);
    SearchRequest searchRequest = new SearchRequest(indexName);
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    searchSourceBuilder.aggregation(aggregationBuilder);
    searchSourceBuilder.query(boolQueryBuilder);
    searchSourceBuilder.size(0);
    try (RestHighLevelClient restHighLevelClient = getEsClient()) {
        searchRequest.source(searchSourceBuilder);
        logger.info("query ES where... indexName = " + indexName + ":" + searchSourceBuilder.toString());
        SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        Aggregations aggregations = searchResponse.getAggregations();
        Terms terms = aggregations.get(resultName);
        List<? extends Terms.Bucket> buckets = terms.getBuckets();
        List<Terms.Bucket> returnBuckets = new ArrayList<>(buckets.size());
        returnBuckets.addAll(buckets);
        return returnBuckets;
    } catch (Exception e) {
        logger.error("查询ES数据信息失败", e);
        return Collections.emptyList();
    }
}

　　

聚合查询的问题：

ES的聚合叫桶聚合，按查询结果来看，就是把聚合的结果丢到桶里

我在封装这个桶结果的时候遇到挺多麻烦的，费半天劲找到

1、首先从结果获取开始，是从Aggregtions对象拿取，需要提供设定聚合的名称

2、拿到Buckets集合后，不能直接返回，这个通配泛型没有指定具体类型，所以要自己重新创建具体泛型的集合装填桶元素

?
1
2
3
4
5
6
7
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
       Aggregations aggregations = searchResponse.getAggregations();
       Terms terms = aggregations.get(resultName);
       List<? extends Terms.Bucket> buckets = terms.getBuckets();
       List<Terms.Bucket> returnBuckets = new ArrayList<>(buckets.size());
       returnBuckets.addAll(buckets);
       return returnBuckets;

 

查询条件构建：

单个分组条件，即按某个字段分组

?
1
2
3
4
String resultName = "typeCount";
TermsAggregationBuilder aggBuilder = AggregationBuilders.terms(resultName).field("strength.keyword");
aggBuilder.size(Integer.MAX_VALUE);
List<Terms.Bucket> buckets = esBaseService.getAggregationQuery(ObuBeTrackDTO.class, aggBuilder, resultName);

如果是多个字段分组，居然是用字符拼接处理... 属实没想到

注意指定这个分隔符，然后按分隔符处理

?
1
2
3
4
5
6
7
8
9
10
11
12
boolean hasStartTime = StringUtils.isNotBlank(dto.getStartTime());
boolean hasEndTime = StringUtils.isNotBlank(dto.getEndTime());
Map<String, List> resultMap = new HashMap<>();
String resultName = "roadCashCount";
TermsAggregationBuilder aggBuilder = AggregationBuilders
        .terms(resultName)
        .script(new Script("doc['roadNo.keyword'].value +','+ doc['cashNo.keyword'].value"))
        .order(BucketOrder.count(false)) /* 按count降序 */
        .size(10); /* 限制十条 */
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
if (hasStartTime && hasEndTime) boolQueryBuilder.must(QueryBuilders.rangeQuery("captureTime").format("yyyy-MM-dd HH:mm:ss").from(dto.getStartTime()).to(dto.getEndTime()));
List<Terms.Bucket> buckets = esBaseService.getConditionAggregationQuery(ObuEtInleaveDTO.class, boolQueryBuilder, aggBuilder, resultName);

　　

2023年11月20日 更新：

 

一、震荡查询问题：

应用上线内测后发现有个问题，统计数据时不时就查的0，一点数据没有

所以我猜测是符合ES查询震荡的情况

 

现在的解决方案是追加分片查询类型，分片的优先级方式

每个查询实现后都追加了这个配置，后面没有再收到这个问题，说明是解决了

preference参数还有点坑，要把双引号带进去，因为这个参数是url参数，下划线好像不能直接跟等号一起

?
1
2
3
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.searchType(SearchType.DFS_QUERY_THEN_FETCH);
searchRequest.preference("\"_primary_first\"");

 

资料参考见：

查询参数

?
1
https://blog.csdn.net/m0_37739193/article/details/82628185

几种查询类型参数

?
1
https://blog.csdn.net/HuoqilinHeiqiji/article/details/103460430

 

二、另外两种查询方式的API实现：

SearchAfter查询的API实现

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
/**
 * @author OnCloud9
 * @date 2023/11/13 17:04
 * @description
 * @params [tClass, capacity, saMarkArray, boolQueryBuilder, highlightBuilder, sortBuilders, searchAfterSetter]
 * @return java.util.List<Entity>
 */
@Override
@SuppressWarnings("Duplicates")
public <Entity> List<Entity> searchAfterQuery(
        Class<Entity> tClass,
        Integer capacity,
        Object[] saMarkArray,
        BoolQueryBuilder boolQueryBuilder,
        HighlightBuilder highlightBuilder,
        Collection<SortBuilder> sortBuilders,
        BiFunction<Entity, Object[], Entity> searchAfterSetter,
        Consumer<Entity> consumer
) {
    String indexName = getIndexName(tClass);
    if (StringUtils.isBlank(indexName)) return Collections.emptyList();
 
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    searchSourceBuilder.from(0); /* 使用searchAfter必须指定from为0 */
    searchSourceBuilder.size(Objects.isNull(capacity) ? 10 : capacity);
    if (Objects.nonNull(highlightBuilder)) searchSourceBuilder.highlighter(highlightBuilder);
    if (Objects.nonNull(boolQueryBuilder)) searchSourceBuilder.query(boolQueryBuilder);
    if (CollectionUtils.isNotEmpty(sortBuilders)) sortBuilders.forEach(searchSourceBuilder::sort);
    if (Objects.nonNull(saMarkArray) && saMarkArray.length > 0) searchSourceBuilder.searchAfter(saMarkArray); /* 根据排序顺序依次放置上一次的排序关键字段值 */
 
    SearchRequest searchRequest = new SearchRequest(indexName);
    searchRequest.searchType(SearchType.DFS_QUERY_THEN_FETCH);
    searchRequest.preference("\"_primary_first\"");
 
    SearchResponse searchResponse;
    try (RestHighLevelClient restHighLevelClient = getEsClient()) {
        searchRequest.source(searchSourceBuilder);
        logger.info("searchAfter query ES where... indexName = " + indexName + ":" + searchSourceBuilder.toString());
        searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        if (Objects.isNull(searchResponse)) return Collections.emptyList();
    } catch (Exception e) {
        logger.error("查询ES数据信息失败", e);
        return Collections.emptyList();
    }
    SearchHits searchHits = searchResponse.getHits();
    SearchHit[] hits = searchHits.getHits();
    List<Entity> entities = new ArrayList<>(hits.length);
    for (SearchHit searchHit : searchHits.getHits()) {
        Object[] sortValues = searchHit.getSortValues();
        String recordJson = searchHit.getSourceAsString();
        Entity t = JSON.parseObject(recordJson, tClass);
        searchAfterSetter.apply(t, sortValues); /* 存放searchAfter值 */
        consumer.accept(t);
        entities.add(t);
    }
 
    return entities;
}

　　

Scroll滚动查询的API实现：

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
/**
 * @author OnCloud9
 * @date 2023/11/20 16:32
 * @description scroll滚动查询
 * @params [tClass, boolQueryBuilder, highlightBuilder, sortBuilders, consumer]
 * @return java.util.List<Entity>
 */
@Override
public <Entity> List<Entity> scrollQuery(
        Class<Entity> tClass,
        BoolQueryBuilder boolQueryBuilder,
        HighlightBuilder highlightBuilder,
        Collection<SortBuilder> sortBuilders,
        Consumer<Entity> consumer
) {
    String indexName = getIndexName(tClass);
    if (StringUtils.isBlank(indexName)) return Collections.emptyList();
 
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    searchSourceBuilder.from(0);
    searchSourceBuilder.size(MAXIMUM_SIZE);
    if (Objects.nonNull(highlightBuilder)) searchSourceBuilder.highlighter(highlightBuilder);
    if (Objects.nonNull(boolQueryBuilder)) searchSourceBuilder.query(boolQueryBuilder);
    if (CollectionUtils.isNotEmpty(sortBuilders)) sortBuilders.forEach(searchSourceBuilder::sort);
 
    Scroll scroll = new Scroll(TimeValue.timeValueMinutes(10L));
    SearchRequest searchRequest = new SearchRequest(indexName);
    searchRequest.scroll(scroll);
    searchRequest.searchType(SearchType.DFS_QUERY_THEN_FETCH);
    searchRequest.preference("\"_primary_first\"");
 
    SearchResponse searchResponse;
    List<Entity> resultList = new ArrayList<>();
    try (RestHighLevelClient restHighLevelClient = getEsClient()) {
        searchRequest.source(searchSourceBuilder);
        logger.info("scroll query ES where... indexName = " + indexName + ":" + searchSourceBuilder.toString());
        searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        if (Objects.isNull(searchResponse)) return Collections.emptyList();
 
        String scrollId = searchResponse.getScrollId();
        SearchHit[] hits = searchResponse.getHits().getHits();
        esSourceToEntity(tClass, consumer, resultList, hits);
 
        /* 持续滚动 */
        while (ObjectUtil.isNotNull(hits) && hits.length > 0) {
            //构造滚动查询条件
            SearchScrollRequest searchScrollRequest = new SearchScrollRequest(scrollId);
            searchScrollRequest.scroll(scroll);
            //响应必须是上面的响应对象，需要对上一层进行覆盖。
            searchResponse = restHighLevelClient.scroll(searchScrollRequest, RequestOptions.DEFAULT);
            scrollId = searchResponse.getScrollId();
            hits = searchResponse.getHits().getHits();
            esSourceToEntity(tClass, consumer, resultList, hits);
        }
 
        /* 查不到更多数据，清除滚动 */
        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
        clearScrollRequest.addScrollId(scrollId);
        ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
        boolean isSuccess = clearScrollResponse.isSucceeded();
        logger.info("是否成功清除了滚动？ " + isSuccess);
    } catch (Exception e) {
        logger.error("查询ES数据信息失败", e);
        return Collections.emptyList();
    }
 
    return resultList;
}

　　

 

posted @   emdzz  阅读(187)  评论(0)  编辑  收藏  举报
（评论功能已被禁用）
相关博文：
·  【ElasticSearch】突破深度分页限制的分页方案
·  【ElasticSearch】02 查询操作
·  Elasticsearch 学习-javaAPI操作
·  ElasticSearch学习十 Java API操作ES
·  搜索引擎ElasticSearch18_ElasticSearch编程操作5
阅读排行：
· 25岁的心里话
· 闲置电脑爆改个人服务器（超详细） #公网映射 #Vmware虚拟网络编辑器
· 基于 Docker 搭建 FRP 内网穿透开源项目（很简单哒）
· 零经验选手，Compose 一天开发一款小游戏！
· 通过 API 将Deepseek响应流式内容输出到前端
点击右上角即可分享
微信分享提示