logstash解耦之redis消息队列

架构图如下：

说明：通过input收集日志消息放入消息队列服务中（redis，MSMQ、Resque、ActiveMQ，RabbitMQ），再通过output取出消息写入ES上，kibana显示。

好处：松耦合，降低logstash收集日志的负载对业务服务不受影响，前后端分离，消息能存储不影响ES维护。

下面我们就用redis做消息队列存储，架构如下：

#安装redis，修改redis配置文件，bind和protected-mode

[root@elk-node1 conf.d]# yum install -y redis
[root@elk-node1 conf.d]# cp /etc/redis.conf{,.bak}
[root@elk-node1 conf.d]# grep "^[a-z]" /etc/redis.conf
bind 192.168.247.135
protected-mode yes
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
...

#启动redis服务

[root@elk-node1 conf.d]# systemctl start redis
You have new mail in /var/spool/mail/root
[root@elk-node1 conf.d]# ss -lntp|grep 6379
LISTEN     0      511    192.168.247.135:6379                     *:*                   users:(("redis-server",pid=18387,fd=4))
You have new mail in /var/spool/mail/root
[root@elk-node1 conf.d]# grep "^[a-z]" /etc/redis.conf^C
[root@elk-node1 conf.d]# redis-cli -h 192.168.247.135
192.168.247.135:6379> exit

#编写测试文件

[root@elk-node1 conf.d]# cat redis-out.conf
input{
    stdin{
 
}
}
output{
    redis{
    host => "192.168.247.135"
    port => "6379"
    db => "6"
    data_type => "list"
    key => "demo"
}
}

#logstash配置文件运行输入hello world
[root@elk-node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/redis-out.conf
Settings: Default filter workers: 1
hello world
#另开一个窗口登录redis可以看到一条我们刚输入的hello world消息

[root@elk-node1 ~]# redis-cli -h 192.168.247.135
192.168.247.135:6379> info
Logstash startup completed
# Keyspace
db6:keys=1,expires=0,avg_ttl=0
192.168.247.135:6379> select 6
OK
192.168.247.135:6379[6]> key *
(error) ERR unknown command 'key'
192.168.247.135:6379[6]> keys *
1) "demo"
192.168.247.135:6379[6]> LINDEX demo -1
"{\"message\":\"hello world\",\"@version\":\"1\",\"@timestamp\":\"2018-07-28T06:44:50.418Z\",\"host\":\"elk-node1\"}"
192.168.247.135:6379[6]>

#接下来我们把消息写入ES，首先再输入多条消息

[root@elk-node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/redis-out.conf
Settings: Default filter workers: 1
Logstash startup completed
fsadf
dgdf
gdg
ad
fd
ds
cd
g
rgergerg
rg
qrg
rh
rg
q
34tr
34
f
gdf
df
 
df
f
sdv
sdf
 
re
ter
t4
^CSIGINT received. Shutting down the pipeline. {:level=>:warn}
 
Logstash shutdown completed

#写一个输入到ES的配置文件

[root@elk-node1 conf.d]# cat redis-int.conf
input{
    redis{
        host => "192.168.247.135"
        port => "6379"
        db => "6"
        data_type => "list"
        key => "demo"
}
}
output{
        elasticsearch {
           hosts => ["192.168.247.135:9200"]
           index => "redis-demo-%{+YYYY.MM.dd}"
        }
    }

#logstash配置文件运行
[root@elk-node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/redis-int.conf
Settings: Default filter workers: 1
Logstash startup completed
#这时我们看redis上的消息已经被消费了
192.168.247.135:6379[6]> LLEN demo
(integer) 0
我们在登录ES可以看到已经有记录了

#写一个系统监控的配置文件把日志写入redis,inpout里读取日志消息，output里写入redis。

[root@elk-node1 conf.d]# cat shipper.conf 
input {
    file {
      path => "/var/log/messages"
      type => "system"
      start_position => "beginning"
    }
    file {
       path => "/var/log/elasticsearch/hejianlai.log"
       type => "es-error"
       start_position => "beginning"
      codec => multiline {
          pattern => "^\["
          negate => true
          what => "previous"
        }
    }
       file {
       path => "/var/log/nginx/access_json.log"
       codec => json
       start_position => "beginning"
       type => "nginx-log"
    }
    syslog{
    type => "system-syslog"
    host => "192.168.247.135"
    port => "514"
}
}
output {
      
    if [type] == "system"{
    redis{
        host => "192.168.247.135"
        port => "6379"
        db => "6"
        data_type => "list"
        key => "system"
    }
} 
    if [type] == "es-error"{
        redis{
        host => "192.168.247.135"
        port => "6379"
        db => "6"
        data_type => "list"
        key => "es-error"
    }
}
       if [type] == "nginx-log"{
        redis{
        host => "192.168.247.135"
        port => "6379"
        db => "6"
        data_type => "list"
        key => "nginx-log"
    }
}
       if [type] == "system-syslog"{
    redis{
        host => "192.168.247.135"
        port => "6379"
        db => "6"
        data_type => "list"
        key => "system-syslog"
 
    }
}
}

#运行配置文件
[root@elk-node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/shipper.conf
#查看redis已经生成了相应的key
192.168.247.135:6379[6]> keys *
1) "system"
2) "nginx-log"
3) "es-error"
192.168.247.135:6379[6]>
#写一个配置文件从redis中把日志写入ES，inpout里读取redis消息，output里写入ES.

[root@elk-node2 conf.d]# cat display.conf 
input {
    redis{
    type => "system-syslog"
        host => "192.168.247.135"
        port => "6379"
        db => "6"
        data_type => "list"
        key => "system-syslog"
 
    }
    redis{
      type => "system"
        host => "192.168.247.135"
        port => "6379"
        db => "6"
        data_type => "list"
        key => "system"
 
    }
    redis{
       type => "es-error"
        host => "192.168.247.135"
        port => "6379"
        db => "6"
        data_type => "list"
        key => "es-error"
 
    }
    redis{
       type => "nginx-log"
        host => "192.168.247.135"
        port => "6379"
        db => "6"
        data_type => "list"
        key => "nginx-log"
 
    }
 
}
output {
      
    if [type] == "system"{
        elasticsearch {
           hosts => ["192.168.247.135:9200"]
           index => "systemlog-%{+YYYY.MM.dd}"
        }
    }
  
    if [type] == "es-error"{
        elasticsearch {
           hosts => ["192.168.247.135:9200"]
           index => "es-error-%{+YYYY.MM.dd}"
        }
    }
       if [type] == "nginx-log"{
        elasticsearch {
           hosts => ["192.168.247.135:9200"]
           index => "nginx-log-%{+YYYY.MM.dd}"
        }
    }
       if [type] == "system-syslog"{
        elasticsearch {
           hosts => ["192.168.247.135:9200"]
           index => "system-syslog-log-%{+YYYY.MM.dd}"
        }
    }
}