java调用Elasticserach的API实现日志查询
前言：
前面讲了ELK+Filebeat搭建，点击查看，那么如何在java程序中通过API查询日志呢？我们知道，ELK的流程是FileBeat扫描日志文件，将日志发送到Logstash，Logstash将日志进行筛选和分组，发送给Elasticsearch，最终Kibnan通过调用Elasticsearch的接口，查询日志，并展示出来。在这里，我的java程序就相当于替代了Kibana，由java调用es的接口，实现索引创建、索引查询、索引删除、日志数量查询、日志列表查询、日志分页查询等。
1，java项目引入Elasticsearch依赖
这里使用的是maven，版本和es全家桶版本一致
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>7.14.0</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.14.0</version>
</dependency>
2，yml文件中加入es依赖
elasticsearch: host: 192.168.2.129 port: 9200 connTimeout: 3000 socketTimeout: 5000 connectionRequestTimeout: 500 maxConnectNum: 100 maxConnectPerRoute: 100
3，es的configuration文件
package com.test.elk; import org.apache.http.HttpHost; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; /** * * @author ：wdl * ============================================================================== * @文件名称 ：ElasticsearchConfiguration.java * ============================================================================== * @本类描述 : ElsaticSearch配置类 * @功能概述 ： * @创建日期 ：2021/10/12 */ @Configuration public class ElasticsearchConfiguration { @Value("${elasticsearch.host}") private String host; @Value("${elasticsearch.port}") private int port; @Value("${elasticsearch.connTimeout}") public int connectTimeOut; @Value("${elasticsearch.socketTimeout}") private int socketTimeOut; @Value("${elasticsearch.connectionRequestTimeout}") private int connectionRequestTimeOut; @Value("${elasticsearch.maxConnectNum}") private int maxConnectNum; @Value("${elasticsearch.maxConnectPerRoute}") private int maxConnectPerRoute; @Bean(destroyMethod = "close", name = "client") public RestHighLevelClient initRestClient() { // 异步httpclient连接延时配置 RestClientBuilder builder = RestClient.builder(new HttpHost(host, port)); builder.setRequestConfigCallback(builder1 -> { builder1.setConnectTimeout(connectTimeOut); builder1.setSocketTimeout(socketTimeOut); builder1.setConnectionRequestTimeout(connectionRequestTimeOut); return builder1; }); // 异步httpclient连接数配置 builder.setHttpClientConfigCallback(httpClientBuilder -> { httpClientBuilder.setMaxConnTotal(maxConnectNum); httpClientBuilder.setMaxConnPerRoute(maxConnectPerRoute); return httpClientBuilder; }); return new RestHighLevelClient(builder); } }
4，controller类
package com.test.elk; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.core.CountRequest; import org.elasticsearch.client.core.CountResponse; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import javax.annotation.Resource; import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; /** * @author ：wdl * ============================================================================== * @文件名称 ：ElasticSearchController.java * ============================================================================== * @本类描述 : 操作es控制类 * @功能概述 ： * @创建日期 ：2021/10/12 */ @Controller @RequestMapping(value = "system/test") public class ElasticSearchController { @Resource private RestHighLevelClient client; @RequestMapping(value = "existsIndex", method = RequestMethod.POST) @ResponseBody public boolean existsIndex(String index) throws IOException { // 声明GetIndexRequest时要把索引信息带进去，可以带入字符串也可以带入String[],如果传入多个索引，则全部存在才会返回true,只要有一个不存在,就会返回false String[] indices = {index}; GetIndexRequest request = new GetIndexRequest(indices); boolean exists = client.indices().exists(request, RequestOptions.DEFAULT); System.out.println("existsIndex: " + exists); return exists; } @RequestMapping(value = "createIndex", method = RequestMethod.POST) @ResponseBody public boolean createIndex (String index){ try { if(!existsIndex(index)){ CreateIndexRequest request = new CreateIndexRequest(index); this.client.indices().create(request, RequestOptions.DEFAULT); return Boolean.TRUE ; } } catch (IOException e) { e.printStackTrace(); } return Boolean.FALSE; } @RequestMapping(value = "deleteIndex", method = RequestMethod.POST) @ResponseBody public boolean deleteIndex(String index) { try { if(existsIndex(index)){ DeleteIndexRequest request = new DeleteIndexRequest(index); AcknowledgedResponse response = client.indices().delete(request, RequestOptions.DEFAULT); return response.isAcknowledged(); } } catch (Exception e) { e.printStackTrace(); } return Boolean.FALSE; } /** * 查询总数 */ @RequestMapping(value = "count", method = RequestMethod.POST) @ResponseBody public Long count(String index){ // 这里的条件和分页查询的条件一直，可以在分页查询中调用count方法，查询出日志总数 BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
// 匹配message字段中包含hello关键字的 queryBuilder.must(QueryBuilders.matchPhraseQuery("message", "hello"));
// 时间范围筛选 queryBuilder.filter(QueryBuilders.rangeQuery("@timestamp").from(1634086800000L).to(1634087400000L)); CountRequest countRequest = new CountRequest(index); countRequest.query(queryBuilder); try { CountResponse countResponse = client.count(countRequest, RequestOptions.DEFAULT); return countResponse.getCount(); } catch (Exception e) { e.printStackTrace(); } return 0L; } /** * 查询集合 */ @RequestMapping(value = "list", method = RequestMethod.POST) @ResponseBody public List<Map<String,Object>> list (String index) {
// RangeQueryBuilder 可以给字段指定时间范围，但是不能加关键字查询，如果需要多条件组合查询，请看page方法 RangeQueryBuilder rangequerybuilder = QueryBuilders .rangeQuery("@timestamp") .from(1634086800000L).to(1634087400000L); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.query(rangequerybuilder); SearchRequest searchRequest = new SearchRequest(index); searchRequest.source(sourceBuilder); try { SearchResponse searchResp = client.search(searchRequest, RequestOptions.DEFAULT); List<Map<String,Object>> data = new ArrayList<>() ; SearchHit[] searchHitArr = searchResp.getHits().getHits(); for (SearchHit searchHit:searchHitArr){ Map<String,Object> res = new HashMap<>(); Map<String,Object> temp = searchHit.getSourceAsMap(); res.put("id",searchHit.getId()); res.put("time",temp.get("@timestamp")); res.put("message",temp.get("message")); data.add(res); } return data; } catch (Exception e) { e.printStackTrace(); } return null; } /** * 分页查询 */ @RequestMapping(value = "page", method = RequestMethod.POST) @ResponseBody public List<Map<String,Object>> page (String index) { // SearchSourceBuilder可以指定复杂条件，from：分页查询起始偏移量；size：每页条数；sort：排序；query：查询条件；可以声明QueryBuilder的一个实现类，将条件加入queryBuilder，再放入sourceBuilder； SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.from(0); sourceBuilder.size(10); sourceBuilder.sort("@timestamp", SortOrder.DESC); BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); queryBuilder.must(QueryBuilders.matchPhraseQuery("message", "hello")); queryBuilder.filter(QueryBuilders.rangeQuery("@timestamp").from(1634086800000L).to(1634087400000L)); sourceBuilder.query(queryBuilder); SearchRequest searchRequest = new SearchRequest(index);
// 将sourceBulider放入searchRequest searchRequest.source(sourceBuilder); try { SearchResponse searchResp = client.search(searchRequest, RequestOptions.DEFAULT); List<Map<String,Object>> data = new ArrayList<>() ; SearchHit[] searchHitArr = searchResp.getHits().getHits(); for (SearchHit searchHit:searchHitArr){ Map<String,Object> res = new HashMap<>(); Map<String,Object> temp = searchHit.getSourceAsMap(); res.put("time",transTime(temp.get("@timestamp"))); res.put("message",temp.get("message")); data.add(res); } return data; } catch (Exception e) { e.printStackTrace(); } return null ; } //将Z时区时间转换成UTC时间 private Date transTime(Object time){ Date date = new Date(); String dateStr = String.valueOf(time); dateStr = dateStr.replace("Z", " UTC"); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS Z"); try { date = format.parse(dateStr); } catch (ParseException e) { e.printStackTrace(); } return date; } }
