elk分布式+ logstash日志监控+kibana监控
[root@server1 head]# vim /etc/elasticsearch/elasticsearch.yml
68 discovery.zen.ping.unicast.hosts: ["server1", "server2","server3"]
68 discovery.zen.ping.unicast.hosts: ["server1", "server2","server3"]
server2
root@server2 ~]# ls
elasticsearch-2.3.3.rpm jdk-8u121-linux-x64.rpm
[root@server2 ~]# yum install elasticsearch-2.3.3.rpm jdk-8u121-linux-x64.rpm -y
[root@server2 ~]# vim /etc/elasticsearch/elasticsearch.yml
17 cluster.name: my-bs
23 node.name: server2
43 bootstrap.mlockall: true
54 network.host: 172.25.8.2
68 discovery.zen.ping.unicast.hosts: ["server1", "server2","server3"]
[root@server2 ~]# vim /etc/security/limits.conf
elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited
[root@server2 ~]# /etc/init.d/elasticsearch start
server3
[root@server3 ~]# ls
elasticsearch-2.3.3.rpm jdk-8u121-linux-x64.rpm
[root@server3 ~]# yum install -y elasticsearch-2.3.3.rpm jdk-8u121-linux-x64.rpm
[root@server3 ~]# vim /etc/security/limits.conf
elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited
[root@server3 ~]# vim /etc/elasticsearch/elasticsearch.yml
17 cluster.name: my-bs
23 node.name: server3
43 bootstrap.mlockall: true
54 network.host: 172.25.8.3
68 discovery.zen.ping.unicast.hosts: ["server1", "server2","server3"]
[root@server3 ~]# /etc/init.d/elasticsearch start
访问web http://172.25.8.1:9200/_plugin/head/（刷新一下）
优化
[root@server1 ~]# vim /etc/elasticsearch/elasticsearch.yml
24 node.master: true
25 node.data: false
[root@server1 ~]# vim /etc/elasticsearch/elasticsearch.yml
24 node.master: true
25 node.data: false
[root@server2 ~]# vim /etc/elasticsearch/elasticsearch.yml
24 node.master: false
25 node.data: true
25 node.data: true
[root@server3 ~]# vim /etc/elasticsearch/elasticsearch.yml
24 node.master: false
25 node.data: true
24 node.master: false
25 node.data: true
[root@server1 ~]# /etc/init.d/elasticsearch restart
[root@server2 ~]# /etc/init.d/elasticsearch restart
[root@server3 ~]# /etc/init.d/elasticsearch restart
访问web http://172.25.8.1:9200/_plugin/head/（刷新一下）
[root@server2 ~]# /etc/init.d/elasticsearch restart
[root@server3 ~]# /etc/init.d/elasticsearch restart
访问web http://172.25.8.1:9200/_plugin/head/（刷新一下）
通过api命令添加数据
[root@server1 elk]# rpm -ivh logstash-2.3.3-1.noarch.rpm
[root@server1 elk]# cd /opt/logstash/
[root@server1 logstash]# bin/logstash -e 'input { stdin { } } output {stdout { } }'
Settings: Default pipeline workers: 1
Pipeline main started
hello world
2018-10-26T03:10:48.712Z server1 hello world
[root@server1 logstash]# bin/logstash -e 'input { stdin { } } output {stdout { codec => rubydebug } elasticsearch { hosts => ["172.25.8.1"] index => "logstash-%{+YYYY.MM.dd}" } }'
Settings: Default pipeline workers: 1
Pipeline main started
duang
{
"message" => "duang",
"@version" => "1",
"@timestamp" => "2018-10-26T03:14:44.572Z",
"host" => "server1"
}
Settings: Default pipeline workers: 1
Pipeline main started
duang
{
"message" => "duang",
"@version" => "1",
"@timestamp" => "2018-10-26T03:14:44.572Z",
"host" => "server1"
}
ot@server1 logstash]# cd /etc/logstash/conf.d/
[root@server1 conf.d]# vim bs.conf
input {
stdin {}
}
output {
stdout {
codec => rubydebug
}
elasticsearch {
hosts => ["172.25.8.1"]
index => "logstash-%{+YYYY.MM.dd}"
}
file {
path => "/tmp/tsetfile"
codec => line { format => "custom format: %{message}"}
}
}
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/bs.conf
Settings: Default pipeline workers: 1
Pipeline main started
hello
{
"message" => "hello",
"@version" => "1",
"@timestamp" => "2018-10-26T03:32:12.888Z",
"host" => "server1"
}
duang duang
{
"message" => "duang duang",
"@version" => "1",
"@timestamp" => "2018-10-26T03:32:18.748Z",
"host" => "server1"
}
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}
stdout {
codec => rubydebug
}
elasticsearch {
hosts => ["172.25.8.1"]
index => "logstash-%{+YYYY.MM.dd}"
}
file {
path => "/tmp/tsetfile"
codec => line { format => "custom format: %{message}"}
}
}
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/bs.conf
Settings: Default pipeline workers: 1
Pipeline main started
hello
{
"message" => "hello",
"@version" => "1",
"@timestamp" => "2018-10-26T03:32:12.888Z",
"host" => "server1"
}
duang duang
{
"message" => "duang duang",
"@version" => "1",
"@timestamp" => "2018-10-26T03:32:18.748Z",
"host" => "server1"
}
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}
Pipeline main has been shutdown
访问web
[root@server1 conf.d]# cat /tmp/tsetfile
custom format: hello
custom format: duang duang
custom format: hello
custom format: duang duang
[root@server1 conf.d]# vim ss.conf
input {
file {
path => "/var/log/messages"
start_position => "beginning"
}
}
input {
file {
path => "/var/log/messages"
start_position => "beginning"
}
}
output {
# stdout {
# codec => rubydebug
# }
elasticsearch {
hosts => ["172.25.8.1"]
index => "messages-%{+YYYY.MM.dd}"
}
# file {
# path => "/tmp/tsetfile"
# codec => line { format => "custom format: %{message}"}
# }
}
# stdout {
# codec => rubydebug
# }
elasticsearch {
hosts => ["172.25.8.1"]
index => "messages-%{+YYYY.MM.dd}"
}
# file {
# path => "/tmp/tsetfile"
# codec => line { format => "custom format: %{message}"}
# }
}
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/ss.conf
Settings: Default pipeline workers: 1
Pipeline main started
LOL
LOLOLOLOLOLOLOLOLOLOLOLOLOLOLOL
LOLOLOLOLOL
LOLOLLOLOLOLOLOOL
LOLOL
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}
Pipeline main has been shutdown
访问web
Settings: Default pipeline workers: 1
Pipeline main started
LOL
LOLOLOLOLOLOLOLOLOLOLOLOLOLOLOL
LOLOLOLOLOL
LOLOLLOLOLOLOLOOL
LOLOL
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}
Pipeline main has been shutdown
访问web
[root@server1 conf.d]# vim ss.conf
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/ss.conf
[root@server2 ~]# vim /etc/rsyslog.conf
13 $ModLoad imudp
14 $UDPServerRun 514
17 $ModLoad imtcp
18 $InputTCPServerRun 514
81 *.* @@172.25.8.1:514
[root@server2 ~]# /etc/init.d/rsyslog restart
访问web
[root@server1 log]# cd /etc/logstash/conf.d/
[root@server1 conf.d]# vim bs.conf
input {
file {
path => "/var/log/elasticsearch/my-bs.log"
start_position => "beginning"
}
}
[root@server1 conf.d]# vim bs.conf
input {
file {
path => "/var/log/elasticsearch/my-bs.log"
start_position => "beginning"
}
}
filter {
multiline {
pattern => "^\["
negate => "true"
what => "previous"
}
}
output {
elasticsearch {
hosts => ["172.25.8.1"]
index => "bs-%{+YYYY.MM.dd}"
}
}
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/bs.conf
访问web
或
vim bs.conf
input {
file {
path => "/var/log/elasticsearch/my-bs.log"
start_position => "beginning"
codec => multiline {
pattern => "^\["
negate => "true"
what => "previous"
}
}
}
output {
elasticsearch {
hosts => ["172.25.8.1"]
index => "bs-%{+YYYY.MM.dd}"
}
}
或
vim bs.conf
input {
file {
path => "/var/log/elasticsearch/my-bs.log"
start_position => "beginning"
codec => multiline {
pattern => "^\["
negate => "true"
what => "previous"
}
}
}
output {
elasticsearch {
hosts => ["172.25.8.1"]
index => "bs-%{+YYYY.MM.dd}"
}
}
httpd日志监控
nput {
file {
path => "/var/log/httpd/access_log"
start_position => "beginning"
}
filter {
grok {
match => { "message" => "%{COMBINEDAPACHELOG}"}
}
}
output {
elasticsearch {
hosts => ["172.25.8.1"]
index => "apache-%{+YYYY.MM.dd}"
}
}
nput {
file {
path => "/var/log/httpd/access_log"
start_position => "beginning"
}
filter {
grok {
match => { "message" => "%{COMBINEDAPACHELOG}"}
}
}
output {
elasticsearch {
hosts => ["172.25.8.1"]
index => "apache-%{+YYYY.MM.dd}"
}
}
/opt/logstash/bin/logstash -f /etc/logstash/conf.d/bs.conf
nginx日志监控
[root@server1 ~]# /etc/init.d/httpd stop
[root@server1 ~]# /etc/init.d/nginx start
input {
file {
path => "/var/log/nginx/access.log"
start_position => "beginning"
}
}
filter {
grok {
match => { "message" => "%{COMBINEDAPACHELOG} %{QS:xforward}" }
}
}
output {
elasticsearch {
hosts => ["172.25.8.1"]
index => "nginx-%{+YYYY.MM.dd}"
}
}
[root@server1 ~]# /etc/init.d/nginx start
input {
file {
path => "/var/log/nginx/access.log"
start_position => "beginning"
}
}
filter {
grok {
match => { "message" => "%{COMBINEDAPACHELOG} %{QS:xforward}" }
}
}
output {
elasticsearch {
hosts => ["172.25.8.1"]
index => "nginx-%{+YYYY.MM.dd}"
}
}
ot@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/bs.conf
fangwen web
fangwen web
kibana
[root@server1 elk]# rpm -ivh kibana-4.5.1-1.x86_64.rpm
[root@server1 elk]# cd /opt/kibana/config/
[root@server1 config]# vim kibana.yml
15 elasticsearch.url: "http://172.25.8.1:9200"
23 kibana.index: ".kibana"
[root@server1 config]# /etc/init.d/kibana start
web:172.25.8.1:5601
[root@server1 elk]# cd /opt/kibana/config/
[root@server1 config]# vim kibana.yml
15 elasticsearch.url: "http://172.25.8.1:9200"
23 kibana.index: ".kibana"
[root@server1 config]# /etc/init.d/kibana start
web:172.25.8.1:5601
