Redis做消息队列
1.连接从Redis中获取日志文件并存储到ES中
[root@Logstash ~]# vim /usr/local/logstash/config/redis.conf
input {
beats {
port => "5044"
}
redis {
data_type => "list"
key => "220"
host => "192.168.200.134"
port => 6379
db => 0
threads => 1
}
}
filter {
if [type] == "nginx"{
grok {
match => { "message" => "%{NGINXACCESS}" }
}
}
date {
match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
target => ["datetime"]
}
geoip {
source => "clientip"
}
}
output {
if [fields][logsource] == "220nginx_access"{
elasticsearch {
hosts => ["192.168.200.130:9200"]
index => "220nginx_access"
}
}
if [fields][logsource] == "220nginx_error"{
elasticsearch {
hosts => ["192.168.200.130:9200"]
index => "220nginx_error"
}
}
stdout { codec => rubydebug }
}
2.将Nginx访问日志和错误日志推送到Redis消息队列中。
[root@mobanji filebeat]# egrep -v "#|^$" filebeat.yml
filebeat.prospectors:
- type: log
fields:
logsource: 220nginx_access
log_type: access_log
fields_under_root: true
paths:
- /usr/local/nginx/logs/access.log
- type: log
fields:
logsource: 220nginx_error
log_type: error_log
paths:
- /usr/local/nginx/logs/error.log
output.redis:
hosts: ["192.168.200.134"]
port: 6379
db: 0
timeout: 5
key: "220"
3.收集Nginx和Tomcat日志：
[root@mobanji ~]# egrep -v "#|^$" /usr/local/filebeat/filebeat.yml
filebeat.prospectors:
- type: log
fields:
logsource: 220nginx_access
log_type: access_log
fields_under_root: true
paths:
- /usr/local/nginx/logs/access.log
- type: log
fields:
logsource: 220nginx_error
log_type: error_log
paths:
- /usr/local/nginx/logs/error.log
- type: log
fields:
logsource: 220tomcat_out
log_type: tomcat_out
paths:
- /usr/local/tomcat/logs/catalina.out
output.redis:
hosts: ["192.168.200.134"]
port: 6379
db: 0
timeout: 5
key: "220"
