logstash常用插件解析

官方地址：https://www.elastic.co/guide/en/logstash-versioned-plugins/current/index.html

配置文件写法:
# 日志导入input {}
# 日志筛选匹配处理filter {}
# 日志匹配输出output {}
# 日志解析配置文件的框架共分为三个模块,input,output,filter。后面会一一讲解, 每个模块里面存在不同的插件。

input 模块

例子1

# file为常用文件插件,插件内选项很多,可根据需求自行判断
input {        
file {        
path => "/var/lib/mysql/slow.log"        # 要导入的文件的位置,可以使用*,例如/var/log/nginx/*.log        
Excude =>”*.gz”                                # 要排除的文件        
start_position => "beginning"            # 从文件开始的位置开始读,end表示从结尾开始读        
ignore_older => 0                             # 多久之内没修改过的文件不读取,0为无限制,单位为秒        
sincedb_path => "/dev/null"              # 记录文件上次读取位置,输出到null表示每次都从文件首行开始解析        
type => "mysql-slow"                        # type字段,可表明导入的日志类型    
}   
}

例子2

# redis插件为常用插件,插件内选项很多,可根据需求自行判断    
input {    
redis {        
batch_count => 1                         # EVAL命令返回的事件数目,设置为5表示一次请求返回5条日志信息        
data_type => "list"                       # logstash redis插件工作方式        
key => "logstash-test-list"             # 监听的键值        
host => "127.0.0.1"                      # redis地址        
port => 6379                                # redis端口号        
password => "123123"                    # 如果有安全认证,此项为认证密码        
db => 0                                        # 如果应用使用了不同的数据库,此为redis数据库的编号,默认为0。        
threads => 1                                # 启用线程数量      
}
}
常用的 input 插件其实有很多,这里只举例了两种。其他还有 kafka,tcp 等等

filter 模块

例子

filter {                                         # 插件很多,这里选取我使用过的插件做讲述    
if ([message] =~ "正则表达式")  {  drop {}  }    # 正则匹配=~,!~,包含判断in,not in ,字符串匹配==,!=,等等,匹配之后可以做任何操作,这里过滤掉匹配行,除了做过滤操作,if后面可以作任意操作,甚至可以为匹配到的任意行做单独的正则分割操作    
multiline {        
pattern => "正则表达式"        
negate => true        
what => "previous"                               # 多行合并,由于一些日志存在一条多行的情况,这个模块可以进行指定多行合并,通过正则匹配,匹配到的内容上面的多行合并为一条日志。    
}       
grok {        
match => { "message" => "正则表达式"             # 正则匹配日志,可以筛选分割出需要记录的字段和值        }           
remove_field => ["message"]                             # 删除不需要记录的字段   
}       
date {        match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]        # 记录@timestamp时间,可以设置日志中自定的时间字段,如果日志中没有时间字段,也可以自己生成        
target=>“@timestamp”                                          # 将匹配的timestamp字段放在指定的字段 默认是@timestamp    }    
ruby {        code => "event.timestamp.time.localtime"        # timestamp时区锁定    }   
}

output 模块

例子1

output {    
# tdout { codec => "rubydebug" }                  # 筛选过滤后的内容输出到终端显示    
elasticsearch {                                               # 导出到es,最常用的插件        
codec => "json"                                             # 导出格式为json        
hosts => ["127.0.0.1:9200"]                           # ES地址+端口        
index => "logstash-slow-%{+YYYY.MM.dd}"           # 导出到index内,可以使用时间变量        
user => "admin"        password => "xxxxxx"           # ES如果有安全认证就使用账号密码验证,无安全认证就不需要        
flush_size => 500                                                   # 默认500,logstash一次性攒够500条的数据在向es发送        
idle_flush_time => 1                              # 默认1s,如果1s内没攒够500,还是会一次性把数据发给ES    }   }

例子2

output {     
redis{                                                        # 输出到redis的插件,下面选项根据需求使用         
batch => true                                           # 设为false,一次rpush,发一条数据,true为发送一批         
batch_events => 50                                # 一次rpush发送多少数据         
batch_timeout => 5                                 # 一次rpush消耗多少时间         
codec => plain                                        # 对输出数据进行codec,避免使用logstash的separate filter         
congestion_interval => 1                          # 多长时间进项一次拥塞检查         
congestion_threshold => 5                      # 限制一个list中可以存在多少个item,当数量足够时,就会阻塞直到有其他消费者消费list中的数据         
data_type => list                                  # 使用list还是publish         
db => 0                                                # 使用redis的那个数据库,默认为0号         
host => ["127.0.0.1:6379"]                   # redis 的地址和端口,会覆盖全局端口         
key => xxx                                           # list或channel的名字         
password => xxx                                   # redis的密码,默认不使用         
port => 6379                                        # 全局端口,默认6379,如果host已指定,本条失效         
reconnect_interval => 1                       # 失败重连的间隔,默认为1s         
timeout => 5                                      # 连接超时的时间         
workers => 1                                      # 工作进程     
}
}
常用插件还有很多,更多的插件使用可以查看官方文档
通过上面的介绍,我们大体知道了 logstash 的处理流程:
input => filter => output
接下来就看一完整的应用例子
完整的应用:
Elasticsearch slow-log

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
input {   
file {       
path => ["/var/log/elasticsearch/private_test_index_search_slowlog.log"]       
start_position => "beginning"       
ignore_older => 0       
# sincedb_path => "/dev/null"       
type => "elasticsearch_slow"       
}  
}
filter {   
grok {       
match =>  { "message" => "^/[(/d/d){1,2}-(?:0[1-9]|1[0-2])-(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])/s+(?:2[0123]|[01]?[0-9]):(?:[0-5][0-9]):(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)/]/[(TRACE|DEBUG|WARN/s|INFO/s)/]/[(?[a-z/.]+)/]/s/[(?[a-z0-9/-/.]+)/]/s/[(?[A-Za-z0-9/./_/-]+)/]/[/d+/]/s+took/[(?[/./d]+(ms|s|m))/]/,/s+took_millis/[(/d)+/]/,/s+types/[(?([A-Za-z/_]+|[A-Za-z/_]*))/]/,/s+stats/[/]/,/s+search_type/[(?[A-Z/_]+)/]/,/s+total_shards/[/d+/]/,/s+source/[(?[/s/S]+)/]/,/s+extra_source/[[/s/S]*/]/,/s*$" }       
remove_field => ["message"]       
}      
date {       
match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]        
}      
ruby {       
code => "event.timestamp.time.localtime"       
}       }
 
output {    
elasticsearch {        
codec => "json"        
hosts => ["127.0.0.1:9200"]        
index => "logstash-elasticsearch-slow-%{+YYYY.MM.dd}"        
user => "admin"        
password => "xxxx"   
}  
}

Mysql-slow log

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
input {   
file {       
path => "/var/lib/mysql/slow.log"       
start_position => "beginning"       
ignore_older => 0       
# sincedb_path => "/dev/null"       
type => "mysql-slow"   
}  
}
filter {   
if ([message] =~ "^(//usr//local|Tcp|Time)[/s/S]*")
{ drop {} }   
multiline {       
pattern => "^/#/s+Time/:/s+/d+/s+(0[1-9]|[12][0-9]|3[01]|[1-9])"       
negate => true       
what => "previous"   
}      
grok {       
match => { "message" => "^/#/sTime/:/s+/d+/s+(?%{TIME})/n+/#/pcidata@xxx.com/:/s+[A-Za-z0-9/_]+/[(?[A-Za-z0-9/_]+)/]/pcidta@xxx.com/s+(?[A-Za-z0-9/_]+)/s+/[/]/n+/#/s+Query/_time/:/s+(?[0-9/.]+)/s+Lock/_time/:/s+(?[0-9/.]+)/s+Rows/_sent/:/s+(?/d+)/s+Rows/_examined/:/s+(?/d+)(/n+|/n+use/s+(?[A-Za-z0-9/_]+)/;/n+)SET/s+timestamp/=/d+/;/n+(?[/s/S]+)$"   }          
remove_field => ["message"]   }      
date {       
match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]    
}      
ruby {       
code => "event.timestamp.time.localtime"   
}  
}
output {    
elasticsearch {       
codec => "json"       
hosts => ["127.0.0.1:9200"]       
index => "logstash-mysql-slow-%{+YYYY.MM.dd}"       
user => "admin"       
password => "xxxxx"   
}  
}

Nginx access.log
logstash 中内置 nginx 的正则,我们只要稍作修改就能使用
将下面的内容写入到/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/grok-patterns 文件中

之后的 log 配置文件如下

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
input {   
file {   
path => [ "/var/log/nginx/www-access.log" ]   
start_position => "beginning"   
# sincedb_path => "/dev/null"   
type => "nginx_access"   
}  
}
filter {   
grok {        
match => { "message" => "%{NGINXACCESS}"}   
}   
mutate {       
convert => [ "response","integer" ]       
convert => [ "bytes","integer" ]   
}   
date {       
match => [ "timestamp","dd/MMM/yyyy:HH:mm:ss Z"]   
}      
ruby {       
code => "event.timestamp.time.localtime"   
}  
}
output {   
elasticsearch {       
codec => "json"       
hosts => ["127.0.0.1:9200"]       
index => "logstash-nginx-access-%{+YYYY.MM.dd}"       
user => "admin"       
password => "xxxx"   
}
}

Nginx error.log

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
input {   
file {   
path => [ "/var/log/nginx/www-error.log" ]   
start_position => "beginning"   
# sincedb_path => "/dev/null"   
type => "nginx_error"   
}
}
filter {   
grok {       
match => [ 
"message","%{NGINXERROR_ERROR}",                  
"message","%{NGINXERROR_OTHER}"                
]   
}      
ruby {       
code => "event.timestamp.time.localtime"   
}       
date {        
match => [ "timestamp","dd/MMM/yyyy:HH:mm:ss"]    
}
}
output {   
elasticsearch {       
codec => "json"       
hosts => ["127.0.0.1:9200"]       
index => "logstash-nginx-error-%{+YYYY.MM.dd}"       
user => "admin"       
password => "xxxx"   
}  
}

PHP error.log

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
input {   
file {       
path => ["/var/log/php/error.log"]       
start_position => "beginning"       
# sincedb_path => "/dev/null"       
type => "php-fpm_error"   
}  
}
filter {   
multiline {        pattern => "^/[(0[1-9]|[12][0-9]|3[01]|[1-9])/-%{MONTH}-%{YEAR}[/s/S]+"        negate => true        what => "previous"    }      
grok {        match => { "message" => "^/[(?(0[1-9]|[12][0-9]|3[01]|[1-9])/-%{MONTH}-%{YEAR}/s+%{TIME}?)/s+[A-Za-z]+//[A-Za-z]+/]/s+(?(?:[A-Z]{3}/s+[A-Z]{1}[a-z]{5,7}|[A-Z]{3}/s+[A-Z]{1}[a-z/s]{9,11}))/:/s+(?[/s/S]+$)" }       
remove_field => ["message"]    }      
date {        match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]     }      
ruby {        code => "event.timestamp.time.localtime"    }   }
output {   
elasticsearch {       
codec => "json"       
hosts => ["127.0.0.1:9200"]       
index => "logstash-php-error-%{+YYYY.MM.dd}"       
user => "admin"       
password => "xxxxx"    }  
}

Php-fpm slow-log

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
input {   
file {       
path => ["/var/log/php-fpm/www.slow.log"]       
start_position => "beginning"       
# sincedb_path => "/dev/null"       
type => "php-fpm_slow"   
}  
}
filter {   
multiline {        pattern => "^$"        negate => true        what => "previous"    }      
grok {        match => { "message" => "^/[(?(0[1-9]|[12][0-9]|3[01]|[1-9])/-%{MONTH}-%{YEAR}/s+%{TIME})/]/s+/[[a-z]{4}/s+(?[A-Za-z0-9]{1,8})/]/s+[a-z]{3}/s+(?/d{1,7})/n(?[/s/S]+$)" }       
remove_field => ["message"]    }      
date {        match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]     }      
ruby {        code => "event.timestamp.time.localtime"    }   }
output {   
elasticsearch {       
codec => "json"       
hosts => ["127.0.0.1:9200"]       
index => "logstash-php-fpm-slow-%{+YYYY.MM.dd}"       
user => "admin"       
password => "xxxx"    }  
}

log 解析配置文件统一放在/etc/logstash/conf.d 目录下,不过也可以任意放置,统一起来最好。
在多个配置文件的时候,不能使用如下命令运行logstash:
/opt/logstash/bin/logstash -f /etc/logstash/conf.d/(或者有个*)
这个命令会拼接配置文件,不会单个使用,会报错。
如果有多个配置文件,就一个一个启动:
/opt/logstash/bin/logstash -f /etc/logstash/conf.d/nginx_error.conf &
但是这样也很麻烦,如果配置文件很多的情况下需要一个个来,并且启动
速度还很慢,写了一个测试脚本用来方便使用,仅供参考:

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
#!/bin/bash
# /配置文件存放目录根据需求自己更改
conf_path=/etc/logstash/conf.d
conf_name=$( ls ${conf_path} )
case $1 in
start)   
echo "-----------please wait.----------"   
echo "The start-up process is too slow."   
for cf in ${conf_name}   
do         
/opt/logstash/bin/logstash  -f $conf_path/$cf > /dev/null 2>&;1 &;           
if [ $? -ne 0 ];then               
echo 'The '${cf}' start-up failed.'           
fi       
sleep 20   
done   
echo "start-up success."
;;   
stop)   
ps -ef |grep logstash |grep -v grep > /dev/null 2>&;1    
if [ $? -eq 0 ];then       
ps -ef|grep logstash |grep -v grep |awk '{print $2}'|xargs  kill -9 > /dev/null 2>&;1       
sleep 2       
echo "Stop success."   
fi 
;;
restart)   
ps -ef |grep logstash |grep -v grep 2>&;1   
if [ $? -eq 0 ];then       
ps -ef|grep logstash |grep -v grep |awk '{print $2}'|xargs  kill -9 > /dev/null 2>&;1       
sleep 3       
echo "Stop success."   
fi     
echo "-----------please wait.----------"   
echo "The start-up process is too slow."   
for cf in ${conf_name}   
do         
/opt/logstash/bin/logstash  -f $conf_path/$cf > /dev/null 2>&;1 &;           
if [ $? -ne 0 ];then               
echo 'The '${cf}' start-up failed.'           
fi       
sleep 10   
done    
echo "start-up success."
;;
*)   
echo "Usage: "$0" {start|stop|restart|}"   
exit 1;
;;
esac

 脚本的名字中不要包含 logstash,这里保存为 log_stash.sh，使用./log_stash.sh (start|stop|restart) 来执行脚本。

posted @   西门运维  阅读(1663)  评论(0)  编辑  收藏  举报
编辑推荐：
· .NET Core 中如何实现缓存的预热？
· 从 HTTP 原因短语缺失研究 HTTP/2 和 HTTP/3 的设计差异
· AI与.NET技术实操系列：向量存储与相似性搜索在 .NET 中的实现
· 基于Microsoft.Extensions.AI核心库实现RAG应用
· Linux系列：如何用heaptrack跟踪.NET程序的非托管内存泄露
阅读排行：
· TypeScript + Deepseek 打造卜卦网站：技术与玄学的结合
· 阿里巴巴 QwQ-32B真的超越了 DeepSeek R-1吗？
· 【译】Visual Studio 中新的强大生产力特性
· 【设计模式】告别冗长if-else语句：使用策略模式优化代码结构
· AI与.NET技术实操系列（六）：基于图像分类模型对图像进行分类
点击右上角即可分享
微信分享提示
SQL AI 助手