十四、系统模块插件（三）

13.1 filter grok插件
–解析各种非结构化的日志数据插件
–grok使用正则表达式把飞结构化的数据结构化
–在分组匹配，正则表达式需要根据具体数据结构编写
–虽然编写困难，但适用性极广
–几乎可以应用于各类数据
grok{
match => [“message”,“%{IP:ip}，(?<key>reg)”]
}
练习
。。。。
filter{
grok{
match => ["message",""]
}
}
。。。。。。。
目前没有日志
我们需要一个测试我们kibana数据库中的Apache日志复制一条在logstash服务器下的tmp下的a.log里
[root@logstash ~]# vim /tmp/a.log
127.0.0.1 - - [19/Aug/2021:23:49:32 +0800] "GET / HTTP/1.1" 403 4897 "-" "curl/7.29.0"
复制完成后我们发现logstash自动获取了日志信息
[root@logstash logstash]# /opt/logstash/bin/logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "1 - - [19/Aug/2021:23:49:32 +0800] \"GET / HTTP/1.1\" 403 4897 \"-\" \"curl/7.29.0\"",
"@version" => "1",
"@timestamp" => "2021-08-19T07:54:52.319Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog"
}
正则宏地址：
[root@logstash ~]# cd /opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/
[root@logstash patterns]# vim grok-patterns
调用宏
filter{
grok{
match => ["message","%{IP:ip}"]
}
[root@logstash logstash]# /opt/logstash/bin/logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "127.0.0.1 - - [19/Aug/2021:23:49:32 +0800] \"GET / HTTP/1.1\" 403 4897 \"-\" \"curl/7.29.0\"",
"@version" => "1",
"@timestamp" => "2021-08-19T08:17:48.016Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog",
"ip" => "127.0.0.1"
}
如果在日常工作中感觉写正则表达式有些困难可以在百度里搜
filter{
grok{
match => ["message","%{COMMONAPACHELOG:apache}"]
}
}
[root@logstash logstash]# /opt/logstash/bin/logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "127.0.0.1 - - [19/Aug/2021:23:49:32 +0800] \"GET / HTTP/1.1\" 403 4897 \"-\" \"curl/7.29.0\"",
"@version" => "1",
"@timestamp" => "2021-08-19T08:23:55.043Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog",
"apache" => "127.0.0.1 - - [19/Aug/2021:23:49:32 +0800] \"GET / HTTP/1.1\" 403 4897",
"clientip" => "127.0.0.1",
"ident" => "-",
"auth" => "-",
"timestamp" => "19/Aug/2021:23:49:32 +0800",
"verb" => "GET",
"request" => "/",
"httpversion" => "1.1",
"response" => "403",
"bytes" => "4897"
}