logstash收集syslog日志

logstash收集syslog日志
注意：生产用syslog收集日志！！！

编写logstash配置文件

#首先我用rubydebug测试数据
[root@elk-node1 conf.d]# cat syslog.conf
input{
    syslog{
    type => "system-syslog"
    host => "192.168.247.135"
    port => "514"
}
}
output{
    stdout{
    codec => "rubydebug"
}
#检查语法
[root@elk-node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/syslog.conf --configtest
Configuration OK
You have new mail in /var/spool/mail/root
[root@elk-node1 ~]# ss -lntp|grep 514
LISTEN     0      50      ::ffff:192.168.247.135:514                     :::*                   users:(("java",pid=9605,fd=14))
#修改rsyslog配置文件让其能访问
[root@elk-node1 ~]# vim /etc/rsyslog.conf
*.* @@192.168.247.135:514
[root@elk-node1 ~]# systemctl restart rsyslog
[root@elk-node1 ~]#
#运行测试
[root@elk-node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/syslog.conf
Settings: Default filter workers: 1
Logstash startup completed
{
           "message" => "Registered Authentication Agent for unix-process:9680:2638370 (system bus name :1.490 [/usr/bin/pkttyagent --notify-fd 5 --fallback], object path /org/freedesktop/PolicyKit1/AuthenticationAgent, locale en_US.UTF-8)\n",
          "@version" => "1",
        "@timestamp" => "2018-07-15T10:08:58.000Z",
              "type" => "system-syslog",
              "host" => "192.168.247.135",
          "priority" => 85,
         "timestamp" => "Jul 15 18:08:58",
         "logsource" => "elk-node1",
           "program" => "polkitd",
               "pid" => "686",
          "severity" => 5,
          "facility" => 10,
    "facility_label" => "security/authorization",
    "severity_label" => "Notice"
}
#添加到elk-log.yml文件
[root@elk-node1 conf.d]# cat elk_log.conf
input {
    file {
      path => "/var/log/messages"
      type => "system"
      start_position => "beginning"
    }
    file {
       path => "/var/log/elasticsearch/hejianlai.log"
       type => "es-error"
       start_position => "beginning"
      codec => multiline {
          pattern => "^\["
          negate => true
          what => "previous"
        }
    }
       file {
       path => "/var/log/nginx/access_json.log"
       codec => json
       start_position => "beginning"
       type => "nginx-log"
    }
    syslog{
    type => "system-syslog"
    host => "192.168.247.135"
    port => "514"
}
}
output {
    
    if [type] == "system"{
        elasticsearch {
           hosts => ["192.168.247.135:9200"]
           index => "systemlog-%{+YYYY.MM.dd}"
        }
    }
 
    if [type] == "es-error"{
        elasticsearch {
           hosts => ["192.168.247.135:9200"]
           index => "es-error-%{+YYYY.MM.dd}"
        }
    }
       if [type] == "nginx-log"{
        elasticsearch {
           hosts => ["192.168.247.135:9200"]
           index => "nginx-log-%{+YYYY.MM.dd}"
        }
    }
       if [type] == "system-syslog"{
        elasticsearch {
           hosts => ["192.168.247.135:9200"]
           index => "system-syslog-log-%{+YYYY.MM.dd}"
        }
    }
}

#检查语法
[root@elk-node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf --configtestConfiguration OK
#后台运行
[root@elk-node1 conf.d]# ps aux|grep elk|awk '{print $2}'|xargs kill -9
kill: sending signal to 9780 failed: No such process
You have new mail in /var/spool/mail/root
[root@elk-node1 conf.d]# ps aux|grep elk|awk '{print $2}'
9785
[1]+  Killed                  /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf  (wd: ~)
(wd now: /etc/logstash/conf.d)
[root@elk-node1 conf.d]# ps aux|grep elk
root       9788  0.0  0.0 112704   972 pts/0    R+   18:18   0:00 grep --color=auto elk
[root@elk-node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf &
[1] 9789
#手动添加日志
[root@elk-node1 conf.d]# logger "you hao"
[root@elk-node1 conf.d]# logger "hello world"
[root@elk-node1 conf.d]# logger "跟我一起学猫叫，一起喵喵喵"

 

Kibana设置

看hand插件上我们能看到system-syslog索引

 

Kibana上添加system-syslog索引

 

完美