logstash系列-快速调试demo

filebeat输入，kafka输出

input {
   beats { port => 5045}
}

output {
    kafka {
        bootstrap_servers => "10.10.10.10:9092"
        topic_id => "test001"
        codec => "json"
    }
}        

filebeat输入，控制台输出

input {
    beats {port => 5046}
}
output {
    stdout{}
}

2个filebeat输入，输出到kafka的2个topic

input {
    beats {port => 5045}
    beats {port => 5046}
}


output {
    if "10.10.10.10" == [ip] {
        kafka {
           bootstrap_servers => "1.1.1.1:9092"   
           topic_id => "test150"
           codec => "json"
       }
    } 
    if "10.10.10.11" == [ip]{
       kafka {
          bootstrap_servers => "1.1.1.2:9092"   
          topic_id => "test151"
          codec => "json"
       }
    }
}
        

给时间戳加8个小时

input {
  stdin { }
}

filter {
  ruby {
    code => '
       event.set("time", event.get("@timestamp").time.localtime + 8*60*60)
       event.set("@timestamp",event.get("time"))
    '
    remove_field => ["time"]
  }
}

output {
   stdout{}     
}

获取@metadata里的值

input {
  beats {port => 8205}
}

filter {
   mutate {
     add_field => { "remote_ip" => "%{[@metadata][ip_address]}" }
   }   
}

output {
   stdout {
     codec => rubydebug { metadata => true }
   }    
}

标准时间转为long值

input {
  stdin {
     add_field => {"aa" => "2020-09-21T17:30:00.123"}
  }
}
filter {   
    ruby{
        code => "event.set('aa',(Time.parse(event.get('aa')).to_f.round(3)*1000).to_i)"
    }
}
output{
  stdout{
    codec=>rubydebug
  }
}

fingerprint测试

input {
  stdin {     
     add_field => {"aa" => "jim" "bb" => "jean"}
     codec => "plain"
  }
}
filter {
    fingerprint { 
        target => "fingerprint"
        method => "UUID"  （或"SHA1"）
        base64encode => true
    }
}
output {
   stdout{}
}

if条件测试，这里字符串的坑，需要注意，是按照字符比较的

input {
  stdin {    
     add_field => {"aa" => "90"}    
  }
}
filter {

     if [aa] == "90"{
       mutate {
            add_field => {
                "field_1" => "=字符串90"
            }
        }
     }
     if [aa] != "89"{
       mutate {
            add_field => {
                "field_2" => "不等于字符串89"
            }
        }
     }


    if [aa] > "100" {
        mutate {
            add_field => {
                "field_3" => "大于字符串100"
            }
        }
    }

    if [aa] > "80" {
        mutate {
            add_field => {
                "field_4" => "大于字符串80"
            }   
        }
    }  
    if [aa] < "91"{
       mutate {
            add_field => {
                "field_5" => "小于字符串91"
            }   
        } 
     }
     if [aa] >= "89"{
       mutate {
            add_field => {
                "field_6" => "大于等于字符串89"
            }
        }
     }
      if [aa] <= "99"{
       mutate {
            add_field => {
                "field_7" => "小于等于字符串99"
            }
        }
     }
}
output {
   stdout{} 
}

近似计算messag的长度，使用ruby插件

input {
  stdin {}
}
filter {
  ruby {
    code => 'event.set("size", event.get("message").length)'
  }
}
output {
   stdout{}   
}

给消息生成一个uuid作为唯一标识

input {
  stdin {}
}
filter {
    uuid { 
        target => "uuid"
        overwrite => true 
    }
}
output {
   stdout{}
}