使用Logstash同步数据至Elasticsearch，Spring Boot中集成Elasticsearch实现搜索

安装logstash、同步数据至ElasticSearch

为什么使用logstash来同步，CSDN上有一篇文章简要的分析了以下几种同步工具的优缺点：https://blog.csdn.net/laoyang360/article/details/51694519。

下面开始实践：

　　1. 下载Logstash 安装包，需要注意版本与elasticsearch保持一致，windows系统下直接解压即可。

　　2.添加同步mysql数据库的配置，并将mysql连接驱动jar包放在指定的配置目录

　　　　注： 目前版本的logstash已经集成了logstash-jdbc-input，不需要再配置这个插件,

　　　　　　 配置文件需要UTF-8编码，我在配置过程中开始新建文件默认的是GBK编码，后面启动logstash后读取配置文件报编码错误了。

我的配置目录：

配置文件mysql.conf

input {
    stdin {
    }
    jdbc {
      # mysql数据库连接
      jdbc_connection_string => "jdbc:mysql://localhost/blog?characterEncoding=utf-8&useSSL=false&serverTimezone=UTC"
      # mysqly用户名和密码
      jdbc_user => "root"
      jdbc_password => "123456"
      # 驱动配置
      jdbc_driver_library => "C:/logstash-6.4.0/mysqletc/mysql-connector-java-6.0.5.jar"
      # 驱动类名
      jdbc_driver_class => "com.mysql.cj.jdbc.Driver"
      jdbc_paging_enabled => "true"
      jdbc_page_size => "50000"
      # 执行指定的sql文件
      statement_filepath => "C:/logstash-6.4.0/mysqletc/blog.sql"
      # 设置监听 各字段含义 分 时 天 月  年 ，默认全部为*代表含义：每分钟都更新
      schedule => "* * * * *"
      # 索引类型
      type => "blog"
    }
}
 
filter {
    json {
        source => "message"
        remove_field => ["message"]
    }
}
 
output {

    elasticsearch {
        #es服务器
        hosts => ["localhost:9200"]
        #ES索引名称
        index => "sl_blog"
        #自增ID
        document_id => "%{id}"
    }
    

    stdout {
        codec => json_lines
    }
}

Blog.sql文件：

SELECT * FROM blog

如果需要同步多个mysql表，可以修改output配置文件mysql.conf，在input和output中添加其他的表。

       3. 启动logstash，正常的话将会同步数据值elasticsearch，根据上面的配置logstash每分钟去数据库读取最新数据

  logstash -f ../mysqletc/mysql.conf

Elasticseach-head插件中查看：

SpringBoot中集成Elasticsearch

下面以开源博客系统new-star-blog（https://github.com/waylau/new-star-blog）为例，简单实现这个功能。

环境：Springboot 2.0.4 +ES 6.4.0

　　1. 添加Spring DataElastic依赖：

<!--SpringBoot默认使用SpringData ElasticSearch模块进行操作-->
<dependency>
   <groupId>org.springframework.boot</groupId>
   <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>

Spring Data Elastic是Spring官方提供的访问Elasticsearch的方式，相对于直接REST访问，它有提供了完善的封装，Spring Data Elastic遵循Spring Data规范。另外它屏蔽了Elasticsearch REST接口的复杂性，由Spring内部实现对Elastic 接口的封装。

　　2. 添加Elasticsearch服务器配置：

#开启 Elasticsearch 仓库(默认值:true)
spring.data.elasticsearch.repositories.enabled=true
#默认 9300 是 Java 客户端的端口。9200 是支持 Restful HTTP 的接口
spring.data.elasticsearch.cluster-nodes = 127.0.0.1:9300
# ES设置连接超时时间
#spring.data.elasticsearch.properties.transport.tcp.connect_timeout=120s

　　3. 实现代码：

package com.sl.blog.domain;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.annotation.Id;
import java.io.Serializable;
import java.util.Date;
//映射Elasticsearch中的索引和文档类型
@Document(indexName = "sl_blog", type = "blog")
public class EsBlog implements Serializable {

    private static final long serialVersionUID = 1L;
    @Id  // 主键
    private String id;

    private String title;

    //@Field(index = false)
    private Date create_time;

    //@Field(index = false,type = FieldType.Long)
    private  Long user_id;

    private String tags;

    //@Field(index = false,type = FieldType.Long)
    private Long read_size;


    private Long catalog_id;

    private String summary;

    //@Field(index = false,type = FieldType.Long)
    private Long comment_size;

    //@Field(index = false,type = FieldType.Long)
    private Long like_size;

    private String content;

    private String username;

    protected EsBlog() {
    }

    public String getId() {
        return id;
    }

    public void setId(String id) {
        this.id = id;
    }

    public String getTitle() {
        return title;
    }

    public void setTitle(String title) {
        this.title = title;
    }


    public Date getCreate_time() {
        return create_time;
    }

    public void setCreate_time(Date create_time) {
        this.create_time = create_time;
    }

    public Long getUser_id() {
        return user_id;
    }

    public void setUser_id(Long user_id) {
        this.user_id = user_id;
    }

    public String getTags() {
        return tags;
    }

    public void setTags(String tags) {
        this.tags = tags;
    }

    public Long getRead_size() {
        return read_size;
    }

    public void setRead_size(Long read_size) {
        this.read_size = read_size;
    }


    public Long getCatalog_id() {
        return catalog_id;
    }

    public void setCatalog_id(Long catalog_id) {
        this.catalog_id = catalog_id;
    }

    public String getSummary() {
        return summary;
    }

    public void setSummary(String summary) {
        this.summary = summary;
    }

    public Long getComment_size() {
        return comment_size;
    }

    public void setComment_size(Long comment_size) {
        this.comment_size = comment_size;
    }

    public Long getLike_size() {
        return like_size;
    }

    public void setLike_size(Long like_size) {
        this.like_size = like_size;
    }

    public String getContent() {
        return content;
    }

    public void setContent(String content) {
        this.content = content;
    }

    public String getUsername() {
        return username;
    }

    public void setUsername(String username) {
        this.username = username;
    }

}

package com.sl.blog.repository;

import com.sl.blog.domain.EsBlog;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.repository.ElasticsearchCrudRepository;
import org.springframework.stereotype.Component;

@Component
public interface IEsBlogRepository extends ElasticsearchCrudRepository<EsBlog,String> {

    Page<EsBlog> findDistinctEsBlogByTitleContainingOrSummaryContainingOrContentContainingOrTagsContaining(String title, String Summary, String content, String tags, Pageable pageable);
    
}

package com.sl.blog.service;

import com.sl.blog.domain.EsBlog;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;

public interface IEsBlogService {

    Page<EsBlog> getEsBlogByKeys(String keyword, Pageable pageable);
}

package com.sl.blog.service.impl;

import com.sl.blog.domain.EsBlog;
import com.sl.blog.repository.IEsBlogRepository;
import com.sl.blog.service.IEsBlogService;
import joptsimple.internal.Strings;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.stereotype.Service;

@Service
public class EsBlogServiceImpl implements IEsBlogService {


    @Autowired
    private ElasticsearchTemplate elasticsearchTemplate;

    @Autowired
    private IEsBlogRepository esBlogRepository;

    /**
     * 通过关键字搜索
     * @param keyword
     * @param pageable
     * @return
     */
    @Override
    public Page<EsBlog> getEsBlogByKeys(String keyword, Pageable pageable){
        Sort sort = new Sort(Sort.Direction.DESC,"read_size","comment_size","like_size");
        if (pageable.getSort() == null) {
            pageable = new PageRequest(pageable.getPageNumber(), pageable.getPageSize(), sort);
        }
        if(Strings.isNullOrEmpty(keyword)){
           return esBlogRepository.findAll(pageable);
        }
        //keyword 含有空格时抛异常
        //return esBlogRepository.findDistinctEsBlogByTitleContainingOrSummaryContainingOrContentContainingOrTagsContaining(keyword, keyword, keyword, keyword, pageable);

        //使用 Elasticsearch API QueryBuilder
        NativeSearchQueryBuilder aNativeSearchQueryBuilder = new NativeSearchQueryBuilder();
        aNativeSearchQueryBuilder.withIndices("sl_blog").withTypes("blog");
        final BoolQueryBuilder aQuery = new BoolQueryBuilder();
        //builder下有的must、should、mustNot 相当于逻辑运算and、or、not
        aQuery.should(QueryBuilders.queryStringQuery(keyword).defaultField("title"));
        aQuery.should(QueryBuilders.queryStringQuery(keyword).defaultField("summary"));
        aQuery.should(QueryBuilders.queryStringQuery(keyword).defaultField("tags"));
        aQuery.should(QueryBuilders.queryStringQuery(keyword).defaultField("content"));

        NativeSearchQuery nativeSearchQuery = aNativeSearchQueryBuilder.withQuery(aQuery).build();
        Page<EsBlog> plist = elasticsearchTemplate.queryForPage(nativeSearchQuery,EsBlog.class);
        return  plist;

    }

}

@Controller
@RequestMapping("/blogs")
public class BlogController {

    @Autowired
    private IEsBlogService esBlogService;

    @GetMapping
    public String listBlogs(@RequestParam(value="order",required=false,defaultValue="new") String order,
                            @RequestParam(value="keyword",required=false,defaultValue="" ) String keyword,
                            @RequestParam(value="async",required=false) boolean async,
                            @RequestParam(value="pageIndex",required=false,defaultValue="0") int pageIndex,
                            @RequestParam(value="pageSize",required=false,defaultValue="5") int pageSize,
                            Model model) {
        Pageable pageable = new PageRequest(pageIndex,pageSize);
        Page<EsBlog> page = esBlogService.getEsBlogByKeys(keyword,pageable);
        List<EsBlog> list  = page.getContent();
        model.addAttribute("order", order);
        model.addAttribute("keyword", keyword);
        model.addAttribute("page", page);
        model.addAttribute("blogList", list);
        return (async==true?"/index :: #mainContainerRepleace":"/index");
    }
}

演示效果：