es通用工具类ElasticSearchUtil
最近也是搭了一套elk+rabbitmq的日志系统，搬过来大哥编写的工具类
public class ElasticSearchUtil { private static final Logger logger = LoggerFactory.getLogger(ElasticSearchUtil.class); /** * 创建索引 * * @param index * @return */ public static boolean createIndex(String index) { if (!isIndexExist(index)) { logger.info("Index is not exits!"); } CreateIndexResponse indexresponse = null; try { indexresponse = EsClient.getTransportClient().admin().indices().prepareCreate(index).execute() .actionGet(); } catch (Exception e) { e.printStackTrace(); } logger.info("执行建立成功？" + indexresponse.isAcknowledged()); return indexresponse.isAcknowledged(); } /** * 删除索引 * * @param index * @return */ public static boolean deleteIndex(String index) { if (!isIndexExist(index)) { logger.info("Index is not exits!"); } DeleteIndexResponse dResponse = EsClient.getTransportClient().admin().indices().prepareDelete(index).execute().actionGet(); if (dResponse.isAcknowledged()) { logger.info("delete index " + index + " successfully!"); } else { logger.info("Fail to delete index " + index); } return dResponse.isAcknowledged(); } /** * 判断索引是否存在 * * @param index * @return */ public static boolean isIndexExist(String index) { IndicesExistsResponse inExistsResponse = EsClient.getTransportClient().admin().indices().exists(new IndicesExistsRequest (index)).actionGet(); if (inExistsResponse.isExists()) { logger.info("Index [" + index + "] is exist!"); } else { logger.info("Index [" + index + "] is not exist!"); } return inExistsResponse.isExists(); } /** * 数据添加，正定ID * * @param jsonObject 要增加的数据 * @param index 索引，类似数据库 * @param type 类型，类似表 * @param id 数据ID * @return */ public static String addData(JSONObject jsonObject, String index, String type, String id) { IndexResponse response = EsClient.getTransportClient().prepareIndex(index, type, id).setSource(jsonObject).get(); logger.info("addData response status:{},id:{}", response.status().getStatus(), response.getId()); return response.getId(); } /** * 数据添加 * * @param jsonObject 要增加的数据 * @param index 索引，类似数据库 * @param type 类型，类似表 * @return */ public static String addData(JSONObject jsonObject, String index, String type) { return addData(jsonObject, index, type, UUID.randomUUID().toString().replaceAll("-", "").toUpperCase()); } /** * 通过ID删除数据 * * @param index 索引，类似数据库 * @param type 类型，类似表 * @param id 数据ID */ public static void deleteDataById(String index, String type, String id) { DeleteResponse response = EsClient.getTransportClient().prepareDelete(index, type, id).execute().actionGet(); logger.info("deleteDataById response status:{},id:{}", response.status().getStatus(), response.getId()); } /** * 通过ID 更新数据 * * @param jsonObject 要增加的数据 * @param index 索引，类似数据库 * @param type 类型，类似表 * @param id 数据ID * @return */ public static void updateDataById(JSONObject jsonObject, String index, String type, String id) { UpdateRequest updateRequest = new UpdateRequest(); updateRequest.index(index).type(type).id(id).doc(jsonObject); EsClient.getTransportClient().update(updateRequest); } /** * 通过ID获取数据 * * @param index 索引，类似数据库 * @param type 类型，类似表 * @param id 数据ID * @param fields 需要显示的字段，逗号分隔（缺省为全部字段） * @return */ public static Map<String, Object> searchDataById(String index, String type, String id, String fields) { GetRequestBuilder getRequestBuilder = EsClient.getTransportClient().prepareGet(index, type, id); if (StringUtils.isNotEmpty(fields)) { getRequestBuilder.setFetchSource(fields.split(","), null); } GetResponse getResponse = getRequestBuilder.execute().actionGet(); return getResponse.getSource(); } /** * 使用分词查询不分页 * * @param index 索引名称 * @param type 类型名称,可传入多个type逗号分隔 * @param startTime 开始时间 * @param endTime 结束时间 * @param size 文档大小限制 * @param fields 需要显示的字段，逗号分隔（缺省为全部字段） * @param sortField 排序字段 * @param highlightField 高亮字段 * @param operatorTag 查询条件最外层的关系(true与,false或) * @param parmStr 过滤条件 * @return */ public static List<Map<String, Object>> searchListData(String index, String type, Integer size,long startTime, long endTime, String fields, String logType, String sortField, String highlightField, boolean operatorTag, String... parmStr) { SearchRequestBuilder searchRequestBuilder = EsClient.getTransportClient().prepareSearch(index); if (StringUtils.isNotEmpty(type)) { searchRequestBuilder.setTypes(type.split(",")); } BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); if (startTime > 0 && endTime > 0) { boolQuery.must(QueryBuilders.rangeQuery("createDate").format("epoch_millis").from(startTime).to(endTime) .includeLower(true).includeUpper(false)); } //设置查询日志类型 boolQuery.must(QueryBuilders.matchQuery("logType", logType)); // 查询字段 与关系 for (int i = 0, len = parmStr.length; i < len; i++) { BoolQueryBuilder tempBoolQuery = QueryBuilders.boolQuery(); if (parmStr[i].contains("&&")){ String[] tempStr =parmStr[i].split("&&"); for(int j=0,len1=tempStr.length;j<len1;j++) { String[] ss = tempStr[j].split("="); //分词查询 tempBoolQuery.must(QueryBuilders.matchPhraseQuery(ss[0], ss[1])); } if (operatorTag) { //如果为真则最外层是与关系 boolQuery.must(tempBoolQuery); } else { boolQuery.should(tempBoolQuery); } } else { if (parmStr[i].contains(",,")) { String[] tempStr =parmStr[i].split(",,"); for(int j=0,len1=tempStr.length;j<len1;j++) { String[] ss = tempStr[j].split("="); //分词查询 tempBoolQuery.should(QueryBuilders.matchPhraseQuery(ss[0], ss[1])); } if (operatorTag) { //如果为真则最外层是与关系 boolQuery.must(tempBoolQuery); } else { boolQuery.should(tempBoolQuery); } } else { if(ToolUtil.isNotEmpty(parmStr[i])) { String[] ss = parmStr[i].split("="); if (operatorTag) { //如果为真则最外层是与关系 boolQuery.must(QueryBuilders.matchPhraseQuery(ss[0], ss[1])); } else { boolQuery.should(QueryBuilders.matchPhraseQuery(ss[0], ss[1])); } } } } } if (StringUtils.isNotEmpty(highlightField)) { HighlightBuilder highlightBuilder = new HighlightBuilder(); // 设置高亮字段 highlightBuilder.field(highlightField); searchRequestBuilder.highlighter(highlightBuilder); } searchRequestBuilder.setQuery(boolQuery); if (StringUtils.isNotEmpty(fields)) { searchRequestBuilder.setFetchSource(fields.split(","), null); } searchRequestBuilder.setFetchSource(true); if (StringUtils.isNotEmpty(sortField)) { searchRequestBuilder.addSort(sortField, SortOrder.DESC); } if (size != null && size > 0) { searchRequestBuilder.setSize(size); } //打印的内容 可以在 Elasticsearch head 和 Kibana 上执行查询 logger.info("\n{}", searchRequestBuilder); SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); long totalHits = searchResponse.getHits().totalHits; long length = searchResponse.getHits().getHits().length; logger.info("共查询到[{}]条数据,处理数据条数[{}]", totalHits, length); if (searchResponse.status().getStatus() == 200) { // 解析对象 return setSearchResponse(searchResponse, highlightField); } return null; } /** * 使用分词查询,并分页 * * @param index 索引名称 * @param type 类型名称,可传入多个type逗号分隔 * @param currentPage 当前页 * @param pageSize 每页显示条数 * @param startTime 开始时间 * @param endTime 结束时间 * @param fields 需要显示的字段，逗号分隔（缺省为全部字段） * @param sortField 排序字段 * @param highlightField 高亮字段 * @param operatorTag 外层逻辑与true 或false * @param parmStr 内层逻辑与&& 或|| * @return */ public static EsPage searchDataPage(String index, String type, int currentPage, int pageSize, long startTime, long endTime, String fields, String logType, String sortField, String highlightField, boolean operatorTag, String... parmStr) { SearchRequestBuilder searchRequestBuilder = EsClient.getTransportClient().prepareSearch(index); if (StringUtils.isNotEmpty(type)) { searchRequestBuilder.setTypes(type.split(",")); } searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH); // 需要显示的字段，逗号分隔（缺省为全部字段） if (StringUtils.isNotEmpty(fields)) { searchRequestBuilder.setFetchSource(fields.split(","), null); } //排序字段 if (StringUtils.isNotEmpty(sortField)) { searchRequestBuilder.addSort(sortField, SortOrder.DESC); } BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); if (startTime > 0 && endTime > 0) { boolQuery.must(QueryBuilders.rangeQuery("createDate").format("epoch_millis").from(startTime).to(endTime) .includeLower(true).includeUpper(true)); } //设置查询日志类型 boolQuery.must(QueryBuilders.matchQuery("logType", logType)); // 查询字段 与关系 for (int i = 0, len = parmStr.length; i < len; i++) { BoolQueryBuilder tempBoolQuery = QueryBuilders.boolQuery(); if (parmStr[i].contains("&&")) { String[] tempStr =parmStr[i].split("&&"); for(int j=0,len1=tempStr.length;j<len1;j++) { String[] ss = tempStr[j].split("="); //分词查询 tempBoolQuery.must(QueryBuilders.matchPhraseQuery(ss[0], ss[1])); } if (operatorTag) { //如果为真则最外层是与关系 boolQuery.must(tempBoolQuery); } else { boolQuery.should(tempBoolQuery); } } else { if (parmStr[i].contains(",,")) { String[] tempStr =parmStr[i].split(",,"); for(int j=0,len1=tempStr.length;j<len1;j++) { String[] ss = tempStr[j].split("="); //分词查询 tempBoolQuery.should(QueryBuilders.matchPhraseQuery(ss[0], ss[1])); } if (operatorTag) { //如果为真则最外层是与关系 boolQuery.must(tempBoolQuery); } else { boolQuery.should(tempBoolQuery); } } else { if(ToolUtil.isNotEmpty(parmStr[i])) { String[] ss = parmStr[i].split("="); if (operatorTag) { //如果为真则最外层是与关系 boolQuery.must(QueryBuilders.matchPhraseQuery(ss[0], ss[1])); } else { boolQuery.should(QueryBuilders.matchPhraseQuery(ss[0], ss[1])); } } } } } if (StringUtils.isNotEmpty(highlightField)) { HighlightBuilder highlightBuilder = new HighlightBuilder(); //highlightBuilder.preTags("<span style='color:red' >");//设置前缀 //highlightBuilder.postTags("</span>");//设置后缀 // 设置高亮字段 highlightBuilder.field(highlightField); searchRequestBuilder.highlighter(highlightBuilder); } searchRequestBuilder.setQuery(QueryBuilders.matchAllQuery()); searchRequestBuilder.setQuery(boolQuery); // 分页应用 int num = (currentPage - 1) * pageSize; searchRequestBuilder.setFrom(num).setSize(pageSize); // 设置是否按查询匹配度排序 searchRequestBuilder.setExplain(true); //打印的内容 可以在 Elasticsearch head 和 Kibana 上执行查询 logger.info("\n{}", searchRequestBuilder); // 执行搜索,返回搜索响应信息 SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); long totalHits = searchResponse.getHits().totalHits; long length = searchResponse.getHits().getHits().length; logger.debug("共查询到[{}]条数据,处理数据条数[{}]", totalHits, length); if (searchResponse.status().getStatus() == 200) { // 解析对象 List<Map<String, Object>> sourceList = setSearchResponse(searchResponse, highlightField); return new EsPage(currentPage, pageSize, (int) totalHits, sourceList); } return null; } /** * 高亮结果集 特殊处理 * * @param searchResponse * @param highlightField */ public static List<Map<String, Object>> setSearchResponse(SearchResponse searchResponse, String highlightField) { List<Map<String, Object>> sourceList = new ArrayList<Map<String, Object>>(); StringBuffer stringBuffer = new StringBuffer(); for (SearchHit searchHit : searchResponse.getHits().getHits()) { searchHit.getSourceAsMap().put("_id", searchHit.getId()); if (StringUtils.isNotEmpty(highlightField)) { System.out.println("遍历 高亮结果集，覆盖 正常结果集" + searchHit.getSourceAsMap()); Text[] text = searchHit.getHighlightFields().get(highlightField).getFragments(); if (text != null) { for (Text str : text) { stringBuffer.append(str.string()); } //遍历 高亮结果集，覆盖 正常结果集 searchHit.getSourceAsMap().put(highlightField, stringBuffer.toString()); } } sourceList.add(searchHit.getSourceAsMap()); } return sourceList; } /** * 查询所有数据 * * @param index 索引名称 * @param type type 6.0后不推荐使用 * @param fields 需要显示的字段 * @param sortField 需要进行排序的字段 * @param highlightField 需要高亮的字段 * @param queryBuilder 查询条件 * @return */ public static List<Map<String, Object>> searchAllData(String index, String type, String fields, String sortField, String highlightField, QueryBuilder queryBuilder ) { //指定一个index和type EsClient esClient=new EsClient(); SearchRequestBuilder searchRequestBuilder = esClient.getESClient().prepareSearch(index); // 高亮（xxx=111,aaa=222） if (StringUtils.isNotEmpty(highlightField)) { HighlightBuilder highlightBuilder = new HighlightBuilder(); //设置前缀 highlightBuilder.preTags("<span style='color:red;font-weight:bold'>"); //设置后缀 highlightBuilder.postTags("</span>"); // 设置高亮字段 highlightBuilder.field(highlightField); searchRequestBuilder.highlighter(highlightBuilder); } // 需要显示的字段，逗号分隔（缺省为全部字段） if (StringUtils.isNotEmpty(fields)) { searchRequestBuilder.setFetchSource(fields.split(","), null); } searchRequestBuilder.setFetchSource(true); if (StringUtils.isNotEmpty(sortField)) { searchRequestBuilder.addSort(sortField, SortOrder.ASC); } //设置每批读取的数据量 searchRequestBuilder.setSize(100); //查询条件 searchRequestBuilder.setQuery(queryBuilder); //设置 search context 维护1分钟的有效期 searchRequestBuilder.setScroll(TimeValue.timeValueMinutes(1)); //获得首次的查询结果 SearchResponse scrollResp=searchRequestBuilder.get(); //打印的内容 可以在 Elasticsearch head 和 Kibana 上执行查询 logger.info("\n{}", searchRequestBuilder); //打印命中数量 logger.info("命中总数量：{}", scrollResp.getHits().getTotalHits()); List<Map<String, Object>> sourceList = new ArrayList<Map<String, Object>>(); StringBuffer stringBuffer = new StringBuffer(); do { //将scorllId循环传递 scrollResp = EsClient.getTransportClient().prepareSearchScroll(scrollResp.getScrollId()).setScroll(TimeValue.timeValueMinutes(1)).execute().actionGet(); for (SearchHit searchHit : scrollResp.getHits().getHits()) { searchHit.getSourceAsMap().put("id", searchHit.getId()); if (StringUtils.isNotEmpty(highlightField)) { if (!ToolUtil.isEmpty(searchHit.getHighlightFields().get(highlightField))) { Text[] text = searchHit.getHighlightFields().get(highlightField).getFragments(); if (text != null) { for (Text str : text) { stringBuffer.append(str.string()); } //遍历 高亮结果集，覆盖 正常结果集 searchHit.getSourceAsMap().put(highlightField, stringBuffer.toString()); } } } sourceList.add(searchHit.getSourceAsMap()); } //当searchHits的数组为空的时候结束循环，至此数据全部读取完毕 } while(scrollResp.getHits().getHits().length != 0); //删除scroll ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); clearScrollRequest.addScrollId(scrollResp.getScrollId()); EsClient.getTransportClient().clearScroll(clearScrollRequest).actionGet(); return sourceList; } /** * 批量新增数据 * * @param index 索引名称 * @param type 索引类型 * @param dataList 需要新增的数据 */ public static void insertBatch(String index, String type, List<Map<String, Object>> dataList,int batchSize) { BulkRequestBuilder bulkRequest = EsClient.getTransportClient().prepareBulk(); for (int i = 0; i < dataList.size(); i++) { bulkRequest.add(EsClient.getTransportClient().prepareIndex(index, type, Convert.toStr(dataList.get(i).get("id"))).setSource(dataList.get(i))); // 每5000条提交一次 if ((i + 1) % batchSize == 0) { BulkResponse bulkItemResponses = bulkRequest.execute().actionGet(); bulkRequest = EsClient.getTransportClient().prepareBulk(); logger.info("已保存: {} 条,执行时间：{} ", batchSize, bulkItemResponses.getTook()); } } if(dataList.size()%batchSize !=0){ BulkResponse bulkItemResponses = bulkRequest.execute().actionGet(); logger.info("保存: {}条,执行时间：{} ", dataList.size() % batchSize, bulkItemResponses.getTook()); } } }
有需要的自己直接搬走，不谢
之前忘记了EsClient 这个类，好久没注意，重新补上
@Component public class EsClient { private static final Logger logger = LoggerFactory.getLogger(EsClient.class); private static TransportClient transportClient; public static TransportClient getTransportClient() { return transportClient; } @Value("${elasticSearchPort}") private String elasticSearchPort; @Value("${elasticSearchClusterName}") private String elasticSearchClusterName; @Value("${elasticSearchPoolSize}") private String elasticSearchPoolSize; @Value("${elasticSearchIps}") public void connectionInit(String elasticSearchIps) { try { //设置netty内存管理非池化 String currentValue = System.getProperty("io.netty.allocator.type"); if (currentValue == null) { System.setProperty("io.netty.allocator.type","unpooled"); } // 配置信息 Settings esSetting = Settings.builder().put("cluster.name", elasticSearchClusterName).put("thread_pool" + ".search.size", Integer.parseInt(elasticSearchPoolSize))//增加线程池个数，暂时设为5 .build(); //配置信息Settings自定义,下面设置为EMPTY transportClient = new PreBuiltTransportClient(esSetting); String[] elasticSearchIpArray = elasticSearchIps.split(","); for (String elasticSearchIp : elasticSearchIpArray) { TransportAddress transportAddress = new TransportAddress(InetAddress.getByName(elasticSearchIp), Integer.valueOf(elasticSearchPort)); transportClient.addTransportAddresses(transportAddress); } } catch (Exception e) { logger.error("elasticsearch TransportClient create error!!!", e); } } }
抱歉抱歉~~~~~~~
import java.util.List; import java.util.Map; public class EsPage { // 指定的或是页面参数 private int currentPage; // 当前页 private int pageSize; // 每页显示多少条 // 查询es结果 private int recordCount; // 总记录数 private List<Map<String, Object>> recordList; // 本页的数据列表 // 计算 private int pageCount; // 总页数 private int beginPageIndex; // 页码列表的开始索引（包含） private int endPageIndex; // 页码列表的结束索引（包含） /** * 只接受前4个必要的属性，会自动的计算出其他3个属性的值 * * @param currentPage * @param pageSize * @param recordCount * @param recordList */ public EsPage(int currentPage, int pageSize, int recordCount, List<Map<String, Object>> recordList) { this.currentPage = currentPage; this.pageSize = pageSize; this.recordCount = recordCount; this.recordList = recordList; // 计算总页码 pageCount = (recordCount + pageSize - 1) / pageSize; // 计算 beginPageIndex 和 endPageIndex // >> 总页数不多于10页，则全部显示 if (pageCount <= 10) { beginPageIndex = 1; endPageIndex = pageCount; } // >> 总页数多于10页，则显示当前页附近的共10个页码 else { // 当前页附近的共10个页码（前4个 + 当前页 + 后5个） beginPageIndex = currentPage - 4; endPageIndex = currentPage + 5; // 当前面的页码不足4个时，则显示前10个页码 if (beginPageIndex < 1) { beginPageIndex = 1; endPageIndex = 10; } // 当后面的页码不足5个时，则显示后10个页码 if (endPageIndex > pageCount) { endPageIndex = pageCount; beginPageIndex = pageCount - 10 + 1; } } } public int getCurrentPage() { return currentPage; } public void setCurrentPage(int currentPage) { this.currentPage = currentPage; } public int getPageSize() { return pageSize; } public void setPageSize(int pageSize) { this.pageSize = pageSize; } public int getRecordCount() { return recordCount; } public void setRecordCount(int recordCount) { this.recordCount = recordCount; } public List<Map<String, Object>> getRecordList() { return recordList; } public void setRecordList(List<Map<String, Object>> recordList) { this.recordList = recordList; } public int getPageCount() { return pageCount; } public void setPageCount(int pageCount) { this.pageCount = pageCount; } public int getBeginPageIndex() { return beginPageIndex; } public void setBeginPageIndex(int beginPageIndex) { this.beginPageIndex = beginPageIndex; } public int getEndPageIndex() { return endPageIndex; } public void setEndPageIndex(int endPageIndex) { this.endPageIndex = endPageIndex; } }
