简单记录一下logstash的语法
本文将一些语法简单的用列子展示出来
1.json
input {
kafka{
bootstrap_servers => ["10.0.0.11:9092,10.0.0.12:9092,10.0.0.13:9092"]
group_id => "es"
topics => ["dhc"]
codec => json
}
}
filter {
if "dhc" in [tags]{
json {
source => "message"
}
if "." not in [value]{
mutate {
add_field => { "value_int" => "%{[value]}" }
}
}
mutate {
convert => { "value_int" => "integer" }
}
}
}
output {
# stdout {}
if "dhc" in [tags] {
elasticsearch {
hosts => ["10.0.0.11:9200","10.0.0.12:9200","10.0.0.13:9200"]
manage_template => false
index => "dhc-%{+yyyy.MM.dd}"
user => "elastic"
password => "111111"
}
}
}
- 非json+调整时间(东八区)+按照日志时间生成+修改字段属性+截取字段长度
懒得标注释了,太长
input {
kafka{
bootstrap_servers => ["10.100.15.23:9092,10.100.15.24:9092,10.100.15.25:9092"]
group_id => "test"
auto_offset_reset => "earliest"
consumer_threads => "5"
decorate_events => "false"
topics => ["kafka"]
type => "bbs_log"
codec => json
}
}
filter {
if "km" in [tags]{
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:times}\s*\S+\<(?<MODULE>(.*))\>\s\S+\:\<(?<level>(.*))\,(?<SNO>(.*))\>\s\S+%{GREEDYDATA:message_value}" }
}
date {
match => ["times", "ISO8601"]
remove_field => "times"
}
mutate{
convert => { "level" => "integer" }
}
}
if "shbkuc" in [tags]{
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:times}\s*\S+\<(?<MODULE>(.*))\>\s\S+\:\<(?<level>(.?))\,(?<SNO>(.*))\>\s\S+%{GREEDYDATA:message_value}" }
}
date {
match => ["times", "ISO8601"]
remove_field => "times"
}
}
if "time" in [tags]{
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:times}\s%{USER:info}\s(?<SNO>(.*))\s(?<excute_time>(.*))\s\[(?<message_value>(.{0,240}))\]" }
}
date {
match => ["times", "ISO8601"]
remove_field => "times"
}
if "(null)" not in [excute_time]{
mutate {
add_field => { "cuttime" => "%{[excute_time]}" }
}
}
mutate { convert => { "cuttime" => "integer" } }
}
if "test" in [tags]{
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:DATE_time}\s*%{USER:server_name}\S+\s*%{INT:level}\,%{BASE16NUM:SNO}\,(?<excute_time>(.*))\;\s\S+%{GREEDYDATA:message_value}"
}
}
}
if "caserver" in [tags]{
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:times}\s*%{USER:server_name}\S+\s*%{INT:level}\,(?<SNO>(.*))\,(?<excute_time>(.*))\;\s\S+%{GREEDYDATA:message_value}" }
}
date {
match => ["times", "ISO8601"]
remove_field => "times"
}
mutate{
convert => { "level" => "integer" }
}
if "(null)" not in [excute_time]{
mutate {
# add_field => { "twotime" => "%{excute_time}" }
split => [ "excute_time","."]
add_field => { "cuttime" => "%{[excute_time][0]}" }
add_field => { "cuttime_last" => "%{[excute_time][1]}" }
}
# mutate {
# convert => { "cuttime" => "integer" }
# }
}
mutate { convert => { "cuttime" => "integer" } }
}
if "exuc" in [tags]{
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:times}\s%{USER:server_name}\s*%{INT:level}\s(?<SNO>(.*))\s(?<excute_time>(.*))\s\[%{GREEDYDATA:message_value}\]" }
}
date {
match => ["times", "ISO8601"]
remove_field => "times"
}
mutate{
convert => { "level" => "integer" }
}
if "(null)" not in [excute_time]{
mutate {
# add_field => { "twotime" => "%{excute_time}" }
split => [ "excute_time","."]
add_field => { "cuttime" => "%{[excute_time][0]}" }
add_field => { "cuttime_last" => "%{[excute_time][1]}" }
}
# mutate {
# convert => { "cuttime" => "integer" }
# }
}
mutate { convert => { "cuttime" => "integer" } }
}
if "json" in [tags]{
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:times}\s*\S+\<(?<MODULE>(.*))\>\s\S+\:\<(?<lv_num>(.*))\,(?<lv_SNO>(.*))\>\s\S+%{GREEDYDATA:message_value}" }
}
date {
match => ["times", "ISO8601"]
# locale => "en"
# timezone => "+00:00"
remove_field => "times"
}
}
}
output {
# stdout {}
if "caserver" in [tags] {
elasticsearch {
hosts => ["10.100.15.23:9200","10.100.15.24:9200","10.100.15.25:9200"]
manage_template => false
index => "caserver-%{+yyyy.MM}"
user => "elastic"
password => "elk_123"
}
}
}