filebeat + logstash进行openstack-swift集群日志收集的一个简单实例

在需要收集日志的proxy节点上配置filebeat，步骤如下：

1、安装filebeat，执行如下命令：
yum -y install filebeat

2、修改/etc/filebeat/filebeat.yml文件，内容如下：
filebeat.prospectors:
- input_type: log
    paths:
       - /var/log/swift/proxy-server.log
    document_type: "swift-proxy"
    fields:
        logsource: hostname    #修改为本机主机名
        logtype: swift-proxy
        logcluster: swift-cluster-01    #修改为本集群名称
        fields_under_root: true

output.logstash:
    hosts: ["192.168.25.31:10515"] #多个用逗号隔开
    worker: 2
    loadbalance: true

3、启动filebeat服务：
systemctl enable filebeat.service
systemctl start filebeat.service

 

配置logstash的filter:（安装请参考ELK部署）

vim /etc/logstash/conf.d/swift.conf

添加如下内容：

input {
    beats {
        host => "0.0.0.0"
        port => 10515
    }
}

filter {
    if [fields][logtype] == "swift-proxy" {
        if "ERROR" in [message] {
            grok {
                match => ["message", "(?<timestamp>[a-zA-Z]{,3} [0-9]{,2}) %{HOUR}:%{MINUTE}:%{SECOND} (?<hostname>swift[0-9]{,2}[aA-zZ]{2,}[0-9]+ proxy-server): (?<o_msg>.*)"]
            }
            date {
                match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
                timezone => "Etc/UTC"
            }
        }
        else {
            grok {
                match => ["message", "(?<t_date>[a-zA-Z]{,3} [0-9]{,2}) %{HOUR}:%{MINUTE}:%{SECOND} (?<hostname>swift[0-9]{,2}[aA-zZ]{2,}[0-9]+ proxy-server): (%{IPV4:client_ip}|-) (%{IPV4:remote_ip}|-) (?<timestamp>[0-9]{,2}\/[a-zA-Z]{,3}\/[0-9]{,4}\/[0-9]{,2}\/[0-9]{,2}\/[0-9]{,2}) %{WORD:method} %{NOTSPACE:query_string} HTTP/(?<version>[0-9].[0-9]) (?<status>[0-9]{,3}) (?<o_msg>.* tx[a-zA-Z0-9]+-[a-zA-Z0-9]+) - %{BASE10NUM:request_time} (-|RL)"]
            }
            date {
                match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
                timezone => "Etc/UTC"
            }
        }
    }
}

output {
    if [fields][logtype] == "swift-proxy" and [fields][logcluster] =="swift-cluster-01"{
        elasticsearch {
            hosts => ["192.168.25.30:9200"] #多个用逗号隔开["",""]
            index => "swift-cluster-01-swift-proxy-%{+YYYY.MM.dd}"
        }
    }
    else if [fields][logtype] == "swift-proxy" and [fields][logcluster] =="swift-cluster-01"{
        elasticsearch {
            hosts => ["192.168.25.30:9200"] #多个用逗号隔开
            index => "swift-cluster-01-swift-proxy-%{+YYYY.MM.dd}"
        }
    }
    else{
        elasticsearch {
            hosts => ["192.168.25.30:9200","192.168.25.31:9200","192.168.25.32:9200"] #多个用逗号隔开
            index => "swift-proxy-%{+YYYY.MM.dd}"
        }
    }
}