RestHighLevelClient 操作es
1，根据id 列表查询
@Autowired private RestHighLevelClient restHighLevelClient; public <T>List<T> getByIds(String index, List<String> ids,Class<T> tClass) { SearchRequest searchRequest = new SearchRequest(); //索引 searchRequest.indices(index); // searchRequest.types(PsdAppListConstant._DOC); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query( QueryBuilders.idsQuery().addIds(ids.toArray(new String[0])) ); searchSourceBuilder.size(ids.size()); log.info(searchSourceBuilder.toString()); searchRequest.source( searchSourceBuilder ); SearchResponse searchResponse = null; try { searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); } catch (IOException e) { throw new RuntimeException(e); } SearchHits hits = searchResponse.getHits(); log.info("total:{},hitsize:{}",hits.totalHits,hits.getHits().length); List<T> result = getResult(hits, tClass); return result; } private <T> List<T> getResult(SearchHits hits, Class<T> tClass) { List<T> result = new ArrayList<>(); for (SearchHit hit : hits) { String sourceAsString = hit.getSourceAsString(); T t = JsonUtils.json2Object(sourceAsString, tClass); result.add(t); } return result; }
2，根据bulk 批量更新
private void batchUpdateTag(List<V1PsdAppExcelReqDTO> list) throws IOException { if (CollectionUtils.isEmpty(list)){ return; } BulkRequest request = new BulkRequest(); for (V1PsdAppExcelReqDTO reqDTO : list) { String pkg = reqDTO.getPackageName(); String appClass = reqDTO.getAppClass(); String field = getEsFieldFromChinese(appClass); if (field == null){ String msg = "系统没有找到 "+ appClass +" 对应的 es 三级分类" ; throw new BaseException("120030", msg, 400, msg); } Map<String,Object> esMap = new HashMap<>(); esMap.put("source",getSource(reqDTO.getSource())); esMap.put(field,1); request.add(new UpdateRequest(PsdAppListConstant.APP_LIST_DIM,PsdAppListConstant._DOC,pkg).doc(esMap)); } BulkResponse bulk = restHighLevelClient.bulk(request, RequestOptions.DEFAULT); log.info("update to es has fail:{},msg:{}",bulk.hasFailures(),bulk.buildFailureMessage()); }
3，根据bulk 插入数据
private void batchAddTag(List<V1PsdAppExcelReqDTO> list) throws IOException { if (CollectionUtils.isEmpty(list)){ return; } BulkRequest request = new BulkRequest(); UserRequestInfo user = UserRequestInfoHolder.getInstance(); Integer userNo = user.getUserNo(); for (V1PsdAppExcelReqDTO reqDTO : list) { String pkg = reqDTO.getPackageName(); String appClass = reqDTO.getAppClass(); Map<String,Object> esMap = new HashMap<>(); esMap.put("name",reqDTO.getName()); esMap.put("package",pkg); esMap.put("uploader",userNo.toString()); esMap.put("source",getSource(reqDTO.getSource())); esMap.put("load_time",System.currentTimeMillis()); String esClassThree = getEsFieldFromChinese(appClass); if (esClassThree != null){ esMap.put(esClassThree,1); } request.add(new IndexRequest(PsdAppListConstant.APP_LIST_DIM,PsdAppListConstant._DOC) .source(JsonUtils.object2Json(esMap), XContentType.JSON) .id(pkg) .create(true)); } BulkResponse bulk = restHighLevelClient.bulk(request, RequestOptions.DEFAULT); log.info("write to es has fail:{},msg:{}",bulk.hasFailures(),bulk.buildFailureMessage()); }
4，利用script 更新指定条件数据
private void invalidAppTag(V1PsdAppTagTriggerReqDTO reqDTO) throws IOException { String esClassThree= getEsClassThreeFromAppClass(reqDTO.getClassThree()); String reqEsClassThree= getEsClassThreeFromAppClass(reqDTO.getClassThree()); List<String> pkgs = reqDTO.getPkgs(); if (CollectionUtils.isEmpty(pkgs)){// 全选 V1PsdTagDetailReqDTO esReqDTO = BeanCopierUtils.createAndCopy(reqDTO, V1PsdTagDetailReqDTO.class); SearchSourceBuilder builder = buildQueryBuilder(esReqDTO,reqEsClassThree); UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(); updateByQueryRequest.indices(PsdAppListConstant.APP_LIST_DIM); updateByQueryRequest.setDocTypes(PsdAppListConstant._DOC); updateByQueryRequest.setQuery(builder.query()); Map<String, Object> paramsMap = new HashMap<>(); paramsMap.put("enable", 0); updateByQueryRequest.setScript(new Script(ScriptType.INLINE, "painless", "ctx._source." + esClassThree + " = params.enable;" + "ctx._source['platform.kepler."+esClassThree+"']= params.enable;" + "ctx._source['platform.anti_defraud."+esClassThree+"']= params.enable;" + "ctx._source['platform.thunderbird."+esClassThree+"']= params.enable;" , paramsMap)); log.info("update query:{} ,script:{},source:{}", updateByQueryRequest, updateByQueryRequest.getScript(), updateByQueryRequest.getSearchRequest().source()); BulkByScrollResponse bulkByScrollResponse = restHighLevelClient.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT); long updated = bulkByScrollResponse.getUpdated(); log.info("es updated:{}", updated); }else { UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(); updateByQueryRequest.indices(PsdAppListConstant.APP_LIST_DIM); updateByQueryRequest.setDocTypes(PsdAppListConstant._DOC); updateByQueryRequest.setQuery( QueryBuilders.idsQuery().addIds(reqDTO.getPkgs().toArray(new String[0])) ); Map<String, Object> paramsMap = new HashMap<>(); paramsMap.put("enable", 0); // 外层tag updateByQueryRequest.setScript(new Script(ScriptType.INLINE, "painless", "ctx._source." + esClassThree + " = params.enable;" + "ctx._source['platform.kepler."+esClassThree+"']= params.enable;" + "ctx._source['platform.anti_defraud."+esClassThree+"']= params.enable;" + "ctx._source['platform.thunderbird."+esClassThree+"']= params.enable;" , paramsMap)); BulkByScrollResponse bulkByScrollResponse = restHighLevelClient.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT); long updated = bulkByScrollResponse.getUpdated(); log.info("es updated:{}",updated); } }
5，条件查询es
private SearchSourceBuilder buildQueryBuilder(V1PsdTagDetailReqDTO reqDTO,String esField){ SearchSourceBuilder builder = new SearchSourceBuilder(); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); if (StringUtils.isNotBlank(reqDTO.getName())) { boolQueryBuilder.must(QueryBuilders.matchQuery("name", reqDTO.getName() ).operator(Operator.AND)); } if (StringUtils.isNotBlank(reqDTO.getPkg())) { boolQueryBuilder.must(QueryBuilders.matchQuery("package", reqDTO.getPkg() ).operator(Operator.AND)); } if (StringUtils.isNotBlank(reqDTO.getPkgInclude())) { String pkgInclude = reqDTO.getPkgInclude(); String[] pkgs = pkgInclude.split(","); boolQueryBuilder.should(QueryBuilders.termsQuery("package", pkgs )); } if (StringUtils.isNotBlank(reqDTO.getPkgExclude())) { String pkgExclude = reqDTO.getPkgExclude(); String[] pkgs = pkgExclude.split(","); boolQueryBuilder.mustNot(QueryBuilders.termsQuery("package", pkgs )); } if (StringUtils.isNotBlank(reqDTO.getPkgNameInclude())) { String pkgNameInclude = reqDTO.getPkgNameInclude(); String[] names = pkgNameInclude.split(","); boolQueryBuilder.should(QueryBuilders.termsQuery("name", names )); } if (StringUtils.isNotBlank(reqDTO.getPkgNameExclude())) { String pkgNameExclude = reqDTO.getPkgNameExclude(); String[] names = pkgNameExclude.split(","); boolQueryBuilder.mustNot(QueryBuilders.termsQuery("name", names )); } boolQueryBuilder.should(QueryBuilders.termQuery(esField, 1 )); boolQueryBuilder.minimumShouldMatch(1); boolQueryBuilder.mustNot(QueryBuilders.termQuery(esField, 0 ));// 弃用和删除的不查 List<String> systems = reqDTO.getSystems(); if (CollectionUtils.isNotEmpty(systems)){ for (String system : systems) { boolQueryBuilder.must(QueryBuilders.termQuery("platform."+system+"." + esField, 1)); } } if (StringUtils.isNotBlank(reqDTO.getLoadTimeSort())) { if (PsdTagConstant.ASC.equals(reqDTO.getLoadTimeSort())){ builder.sort("load_time",SortOrder.ASC); }else { builder.sort("load_time",SortOrder.DESC); } } builder.query(boolQueryBuilder); Integer current = reqDTO.getCurrent(); if (current != null){ int size = reqDTO.getSize(); int from = (current - 1) * size; if (from + size > PsdAppListConstant.ES_SEARCH_LIMIT) { from = PsdAppListConstant.ES_SEARCH_LIMIT - size; } builder.from(from); builder.size(size); } return builder; } SearchSourceBuilder builder = buildQueryBuilder(reqDTO,esField); SearchRequest searchRequest = new SearchRequest(); //索引 searchRequest.indices(PsdAppListConstant.APP_LIST_DIM); searchRequest.types(PsdAppListConstant._DOC); searchRequest.source(builder); log.info(searchRequest.source().toString()); SearchResponse searchResponse = null; try { searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); } catch (IOException e) { throw new RuntimeException(e); } SearchHits hits = searchResponse.getHits(); long totalHits = hits.getTotalHits(); log.info("total:{}",totalHits);
6.添加/修改单个es数据
public void addToEs(V1PsdUserAppTagReqDTO reqDTO) throws IOException { String pkg = reqDTO.getPackageName(); if (CollectionUtils.isEmpty(appEsMap)){ Map<String,Object> esMap = new HashMap<>(); esMap.put("name",reqDTO.getName()); esMap.put("package",reqDTO.getPackageName()); esMap.put("uploader",userNo.toString()); esMap.put("source",PsdTagConstant.SOURCE_PERSON); esMap.put("load_time",System.currentTimeMillis()); for (String esClassThree : esClassThrees) { esMap.put(esClassThree,1); } IndexRequest indexRequest = new IndexRequest(PsdAppListConstant.APP_LIST_DIM,PsdAppListConstant._DOC); indexRequest.source(JsonUtils.object2Json(esMap), XContentType.JSON); indexRequest.id(pkg); indexRequest.create(true); IndexResponse indexResponse = restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT); String id = indexResponse.getId(); log.info("es create doc id :{}",id); }else { Map esApp = appEsMap.get(pkg); UpdateRequest request = new UpdateRequest(PsdAppListConstant.APP_LIST_DIM,PsdAppListConstant._DOC,pkg); Map<String, Object> jsonMap = new HashMap<>(); jsonMap.put("load_time",System.currentTimeMillis()); jsonMap.put("source",PsdTagConstant.SOURCE_PERSON); jsonMap.put("uploader",userNo.toString()); Map<String, Object> activeTag = getActiveTag(esApp); jsonMap.putAll(activeTag); for (String esClassThree : esClassThrees) { jsonMap.put(esClassThree,1); } request.doc(jsonMap); UpdateResponse updateResponse = restHighLevelClient.update(request, RequestOptions.DEFAULT); String id = updateResponse.getId(); log.info("es update doc id :{}",id); } }
