日志写入redis及读取redis
日志写入redis及读取redis
用一台服务器按照部署redis服务，专门用于日志缓存使用，用于web服务器产生大量日志的场景，例如下面的服务器内存即将被使用完毕，查看是因为redis服务保存了大量的数据没有被读取而占用了大量的内存空间。
2.1.1 部署redis
[root@study63 src]# tar xf redis-5.0.8.tar.gz
[root@study63 src]# cd redis-5.0.8/
[root@study63 redis-5.0.8]# ls
00-RELEASENOTES COPYING Makefile redis.conf runtest-moduleapi src
BUGS deps MANIFESTO runtest runtest-sentinel tests
CONTRIBUTING INSTALL README.md runtest-cluster sentinel.conf utils
[root@study63 redis-5.0.8]# make -j2
[root@study63 redis-5.0.8]# ln -sv /opt/src/redis-5.0.8 /opt/redis
[root@study63 redis-5.0.8]# cd /opt/redis/
修改配置
69 bind 10.0.0.63
136 daemonize yes
216 save ""
217
218 #save 900 1
219 #save 300 10
220 #save 60 10000
507 requirepass 123456
启动服务
[root@study63 redis]# cp src/redis-server /usr/bin
[root@study63 redis]# cp src/redis-cli /usr/bin
[root@study63 redis]# redis-server /opt/redis/redis.conf
验证
[root@study63 opt]# redis-cli -h 10.0.0.63
10.0.0.63:6379> KEYS *
(error) NOAUTH Authentication required.
10.0.0.63:6379> AUTH 123456
OK
10.0.0.63:6379> KEYS *
(empty list or set)
2.1.2 配置logstash将日志写入redis
[root@study62 ~]# vim /etc/logstash/conf.d/rsyslog.conf
input {
syslog {
type => "rsyslog-haproxy063"
port => "5160"
}
}
output {
if [type] == "rsyslog-haproxy063" {
redis {
data_type => "list"
host => "10.0.0.63"
db => "1"
port => "6379"
key => "rsyslog-haproxy063"
password => "123456"
}}
}
2.1.3 检测配置语法
[root@study62 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/rsyslog.conf -t
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs to console
Configuration OK
11:01:36.649 [LogStash::Runner] INFO logstash.runner - Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash
[root@study62 ~]# systemctl restart logstash.service
2.1.4 登录redis中查看
[root@study63 opt]# redis-cli -h 10.0.0.63 -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.0.0.63:6379> SELECT 1
10.0.0.63:6379[1]> keys *
1) "rsyslog-haproxy063"
10.0.0.63:6379[1]> LLEN rsyslog-haproxy063 # 查看key的长度
(integer) 12
10.0.0.63:6379[1]> LPOP rsyslog-haproxy063 #展示一条记录会减少一条
"{\"severity\":6,\"pid\":\"14556\",\"program\":\"haproxy\",\"message\":\"Connect from 10.0.0.1:52144 to 10.0.0.63:9999 (stats/HTTP)\\n\",\"type\":\"rsyslog-haproxy063\",\"priority\":182,\"logsource\":\"localhost\",\"@timestamp\":\"2020-04-15T03:09:00.000Z\",\"@version\":\"1\",\"host\":\"10.0.0.63\",\"facility\":22,\"severity_label\":\"Informational\",\"timestamp\":\"Apr 15 11:09:00\",\"facility_label\":\"local6\"}"
10.0.0.63:6379[1]> LLEN rsyslog-haproxy063
(integer) 11
2.2.5 配置logstash从reids中取出数据到elasticsearch
2.2.5.1 使用study63上的logstash从redis取数据
[root@study63 ~]# vim /etc/logstash/conf.d/redis-es.conf
input {
redis {
data_type => "list"
host => "10.0.0.63"
db => "1"
port => "6379"
key => "rsyslog-haproxy063"
password => "123456"
}
}
output {
elasticsearch {
hosts => ["10.0.0.63:9200"]
index => "redis-rsyslog-haproxy063-%{+YYYY.MM.dd}"
}
}
[root@study63 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis-es.conf -t
[root@study63 ~]# systemctl restart logstash.service
2.2.5.2 从study63上写入数据查看
[root@study63 opt]# redis-cli -h 10.0.0.63 -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.0.0.63:6379> SELECT 1
10.0.0.63:6379[1]> keys *
10.0.0.63:6379[1]> LLEN rsyslog-haproxy063
(integer) 11
10.0.0.63:6379[1]> LLEN rsyslog-haproxy063
(integer) 0
2.2.5.3 head插件和Kibana添加索引查看
