第九章·Logstash深入-Logstash配合rsyslog收集haproxy日志

rsyslog介绍及安装配置

在centos 6及之前的版本叫做syslog，centos 7开始叫做rsyslog，根据官方的介绍，rsyslog(2013年版本)可以达到每秒转发百万条日志的级别，官方网址：http://www.rsyslog.com/

---

安装配置rsyslog

#安装rsyslog
[root@elkstack03 ~]# yum install -y rsyslog
#编辑rsyslog配置文件
[root@elkstack03 ~]# vim /etc/rsyslog.conf
$ModLoad imudp
$UDPServerRun 514
$ModLoad imtcp
$InputTCPServerRun 514
#最后面一行添加，local6对应haproxy配置文件定义的local级别,端口为Logstash的端口
local6.*     @@10.0.0.53:2222

---

安装配置haproxy

#安装haproxy
[root@elkstack03 ~]# yum install -y haproxy
#编辑haproxy配置文件
[root@elkstack03 ~]# vim /etc/haproxy/haproxy.cfg
global
maxconn 100000
chroot /var/lib/haproxy
uid 99
gid 99
daemon
nbproc 1
pidfile /var/run/haproxy.pid
log 127.0.0.1 local6 info

defaults
option http-keep-alive
option  forwardfor
maxconn 100000
mode http
timeout connect 300000ms
timeout client  300000ms
timeout server  300000ms

listen stats
 mode http
 bind 0.0.0.0:9999
 stats enable
 log global
 stats uri     /haproxy-status
 stats auth    haadmin:123456

#frontend web_port
frontend web_port
        bind 0.0.0.0:80
        mode http
        option httplog
        log global
        option  forwardfor
###################ACL Setting##########################
        acl pc          hdr_dom(host) -i www.elk.com
        acl mobile      hdr_dom(host) -i m.elk.com
###################USE ACL##############################
        use_backend     pc_host        if  pc
        use_backend     mobile_host    if  mobile
########################################################

backend pc_host
        mode    http
        option  httplog
        balance source
        server web1  10.0.0.53:8081 check inter 2000 rise 3 fall 2 weight 1

backend mobile_host
        mode    http
        option  httplog
        balance source
        server web1  10.0.0.53:8080 check inter 2000 rise 3 fall 2 weight 1
        
#启动haproxy
[root@elkstack03 ~]# /etc/init.d/haproxy start
正在启动 haproxy：                                         [确定]

#启动rsyslog
[root@elkstack03 ~]# /etc/init.d/rsyslog start
启动系统日志记录器：

#验证端口
[root@elkstack03 ~]# netstat -lntup
tcp        0      0 0.0.0.0:9999                0.0.0.0:*                   LISTEN      9082/haproxy
tcp        0      0 0.0.0.0:80                  0.0.0.0:*                   LISTEN      9631/haproxy

#验证进程
[root@elkstack03 ~]# ps -ef|grep haproxy
nobody     9082      1  0 14:04 ?        00:00:00 /usr/sbin/haproxy -D -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid

#修改nginx配置文件，将端口改为8081
[root@elkstack03 ~]# vim /usr/local/nginx/conf/nginx.conf
worker_processes  1;
events {
    worker_connections  1024;
}
http {
    include       mime.types;
    default_type  application/octet-stream;
    sendfile        on;
    keepalive_timeout  65;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  logs/access.log  main;

    log_format access_json '{"@timestamp":"$time_iso8601",'
            '"host":"$server_addr",'
            '"clientip":"$remote_addr",'
            '"size":$body_bytes_sent,'
            '"responsetime":$request_time,'
            '"upstreamtime":"$upstream_response_time",'
            '"upstreamhost":"$upstream_addr",'
            '"http_host":"$host",'
            '"url":"$uri",'
            '"domain":"$host",'
            '"xff":"$http_x_forwarded_for",'
            '"referer":"$http_referer",'
            '"status":"$status"}';
    access_log  logs/access_json.log  access_json;

    server {
        listen       8081;
        server_name  10.0.0.53;
        location / {
            root   /code/html;
            index  index.html index.htm;
        }
    }
}

#修改tomcat配置文件，将默认站点目录改成/webapps/webdir
[root@elkstack03 ~]# vim /usr/local/tomcat/conf/server.xml
      <Host name="localhost"  appBase="webapps"
            unpackWARs="true" autoDeploy="true">

     <Context path="" docBase="/usr/local/tomcat/webapps/webdir" debug="0" reloadable="false"
              crossContext="true"/>
              
#重启nginx
[root@elkstack03 ~]# /usr/local/nginx/sbin/nginx -t
nginx: the configuration file /usr/local/nginx-1.10.3/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/nginx-1.10.3/conf/nginx.conf test is successful
[root@elkstack03 ~]# /usr/local/nginx/sbin/nginx -s reload

#重启tomcat
[root@elkstack03 ~]# cd /usr/local/tomcat/bin/
[root@elkstack03 bin]# ./catalina.sh stop
[root@elkstack03 bin]# ./catalina.sh start

#修改本地hosts文件
10.0.0.53 www.elk.com
10.0.0.53 m.elk.com

测试域名访问


测试haproxy，打开浏览器，访问：http://www.elk.com/


测试haproxy，打开浏览器，访问：http://m.elk.com/


---

配置Logstash

#编辑Logstash配置文件
[root@elkstack03 conf.d]# vim haproxy.cof
input{
      syslog {
        type => "rsyslog_haproxy"
        port => "2222"
}}

output{
        stdout{
                codec => rubydebug
}}

#启动Logstash
[root@elkstack03 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/haproxy.conf

#检查Logstash端口
[root@elkstack03 ~]# netstat -lntup|grep 2222
tcp        0      0 :::2222                     :::*                        LISTEN      9867/java
udp        0      0 :::2222                     :::*                                    9867/java

访问haproxy管理页面测试数据

打开浏览器，访问：http://10.0.0.53:9999/haproxy-status

输入haproxy配置文件中的用户名和密码
用户名：haadmin
密码：123456







---

将输出改成ES

#进入Logstash配置文件目录
[root@elkstack03 ~]# cd /etc/logstash/conf.d
#编辑配置文件
[root@elkstack03 conf.d]# vim haproxy.conf
input{
      syslog {
        type => "rsyslog_haproxy"
        port => "2222"
      }
}

output{
  elasticsearch {
    hosts => ["10.0.0.51:9200"]
    index =>  "logstash_rsyslog-%{+YYYY.MM.dd}"
  }
}

#启动Logstash
[root@elkstack03 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/haproxy.conf &

打开浏览器，访问：http://10.0.0.51:9100/



---

将ES索引添加到Kibana中

打开浏览器，访问：http://10.0.0.54:5601/