ELK之logstash

1、概述

logstash是有java语言写的，所以程序占用系统资源比较多。但是支持grok正则，这是特别强大的。

 

2、安装

参考：https://www.elastic.co/guide/en/logstash/current/installing-logstash.html

yum安装：

#rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
#cat  /etc/yum.repos.d/logstash.repo

[logstash-6.x]
name=Elastic repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

#yum install logstash

3、配置文件
# cat  logstash.conf

input {
 beats {
  port => "5044"
 }
}

filter {
if 'backend' in [tags] {
json {
source => "message"
}
}

if 'api' in [tags] {
grok {
patterns_dir => "/etc/logstash/patters.d"
match => { "message" => "%{NGINXACCESS1}" }
}
kv {
source => "params"
field_split => "&?"
value_split => "="
include_keys => [ "token","user" ]
}
}
if 'access' in [tags] {
grok {
patterns_dir => "/etc/logstash/patters.d"
match =>{ "message" => "%{NGINXACCESS}"
}
}
kv {
source => "params"
field_split => "&?"
value_split => "="
include_keys => [ "token","user" ]
}
}

 if [host] =~ '^db' {
    grok {
     match => ["message","%{TIMESTAMP_ISO8601:timestamp}\s+%{MONGO3_SEVERITY:severity}\s+%{MONGO3_COMPONENT:component}\s+(?:\[%{DATA:context}\])?\s+%{GREEDYDATA:body}"]

    }
 }
}

## Add your filters / logstash plugins configuration here

output {
 if 'backend' in [tags] {
   elasticsearch {
          hosts => "elasticsearch:9200"
          user => "elastic"
          index => "logstash-backend-%{+YYYY-MM-dd}"
          password => "hqXrH"
   }
  
    #stdout { codec => rubydebug }
 }

 if 'api' in [tags] {
   elasticsearch {
          hosts => "elasticsearch:9200"
          user => "elastic"
          index => "loadbalance-api-%{+YYYY-MM-dd}"
          password => "hqXrH"
   }
 }

 if [host] =~ '^db' {
   elasticsearch {
          hosts => "elasticsearch:9200"
          user => "elastic"
          index => "logstash-mongodb-%{+YYYY-MM-dd}"
          password => "hqXrH"
   }
 }
}

 #cat nginx

HOSTPORT1 (%{IPV4}:%{POSINT}[, ]{0,2})+
NGINXACCESS %{IPORHOST:http_host} %{IPORHOST:server_addr} %{IPORHOST:remote_addr} \[%{HTTPDATE:time_local}\] "%{WORD:method} %{URIPATH:uri}\?%{NOTSPACE:params} HTTP/%{NUMBER:http_version}" %{NOTSPACE:request_body|-} %{INT:status} %{INT:body_bytes_sent} %{QS:http_referer} %{QS:http_user_agent} %{NUMBER:request_time:float} %{NUMBER:upstream_response_time:float} %{HOSTPORT1:upstream_addr}

4、启动服务

service  logstash start

 

5、logstash处理时区、类型转换、删除字段

filter{

#去除换行符

mutate{

gsub => ["message", "\r", ""]

}

#逗号分隔

mutate{

split => ["message",","]

}

#分割后字段命名与赋值

mutate{

add_field => {

　　"domain" => "%{[message][0]}"

      "create_time" => "%{[message][2]}"

}

}

 

#字段里面的日期识别以及时区转换，生成date

mutate{

     match => ["create_time","MM/dd/yyyy HH:mm:ss"]

     target => "date"

     local => "en"

     timezone => "+00:00"

}

#删除无用的字段

mutate{

remove_field => "meaasge"

}

#转换字段类型

mutate{

convert => {"size" => "integer"}

}

}

 

output{

elastisearch {

  host => ["host1","host2"]

  protocol => "http"

  index => "logstash-mongodb-%{+YYYY-MM-dd}"

}

}