Logstash收集日志放入redis

Logstash收集日志放入redis

环境准备

主机名	外网IP	内网IP	角色	应用
ELKstack01	10.0.0.81	172.16.1.81	ES日志存储数据库	JDK、elasticsearch
ELKstack02	10.0.0.82	172.16.1.82	ES日志存储数据库	JDK、elasticsearch
ELKstack03	10.0.0.83	172.16.1.83	日志收集工具（从文件取出到redis）	JDK、Logstash、nginx、tomcat、haproxy、rsyslog
ELKstack04	10.0.0.84	172.16.1.84	日志收集工具（从redis取出到es）	JDK、Logstash
db02	10.0.0.52	172.16.1.52	消息队列	redis

将Nginx日志通过Logstash放入redis

[root@elkstack03 ~]# vim /etc/logstash/conf.d/nginx_redis.conf
input{
        file{
                type => "www_access"
                path => "/var/log/nginx/www.zls.com_access_json.log"
                start_position => "beginning"
        }
        file{
                type => "blog_access"
                path => "/var/log/nginx/blog.zls.com_access_json.log"
                start_position => "beginning"
        }
}

output{ 
        redis{  
                data_type => "list"
                key => "nginx_log"
                host => "10.0.0.52"
                port => "6379"
                db => "15"
        }
}


[root@elkstack03 ~]# /usr/share/logstash/bin/logstash --path.data=/var/lib/logstash/ngx_redis -f /etc/logstash/conf.d/nginx_redis.conf &

在ELKstack04机器上安装Logstash

# 安装jdk以下方法二选一
[root@elkstack04 ~]# yum install -y java
[root@elkstack04 ~]# rpm -ivh jdk-8u181-linux-x64.rpm 

# 安装Logstash
[root@elkstack04 ~]# yum localinstall -y logstash-5.6.16.rpm
[root@elkstack04 ~]# vim /etc/logstash/jvm.options 
## JVM configuration

# Xms represents the initial size of total heap space
# Xmx represents the maximum size of total heap space

-Xms256m
-Xmx256m

使用Logstash从redis取出日志放入ES

[root@elkstack04 ~]# vim /etc/logstash/conf.d/ngx_redis_es.conf
  
input{
        redis{
                data_type => "list"
                key => "nginx_log"
                host => "10.0.0.52"
                port => "6379"
                db => "15"
                codec => "json"
        }
}

filter{
        json{
                source => "message"
                remove_field => ["message"]
        }
}

output{
        elasticsearch{
                hosts => ["10.0.0.82:9200"]
                index => "%{type}-%{+yyyy.MM.dd}"
        }
}

监控redis key的堆积

# 1.安装Python3
yum install -y python3-devel

# 2.安装redis库
pip3 install redis


# 3.执行
python3 脚本名.py

#!/usr/bin/env python
#coding:utf-8
#Author Driver_Zeng
import redis
def redis_conn():
    pool=redis.ConnectionPool(host="10.0.0.52",port=6379,db=15)
    conn = redis.Redis(connection_pool=pool)
    data = conn.llen('tn')
    print(data)
redis_conn()