elk部署
ElasticSearch
You know, for search
安装redis
[root@liang filebeat]# yum install redis -y
yum安装Elasticsearch
安装jdk
Elasticsearch requires Java 8 or later
[root@liang local]# yum install java-1.8.0-openjdk -y
[root@liang local]# rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
[root@liang local]# cd /etc/yum.repos.d/
[root@liang yum.repos.d]# vim elasticsearch.repo
[elasticsearch-6.x]
name=Elasticsearch repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
[root@liang yum.repos.d]# yum install elasticsearch -y
To configure Elasticsearch to start automatically when the system boots up, run the following commands:
sudo /bin/systemctl daemon-reload
sudo /bin/systemctl enable elasticsearch.service
Elasticsearch can be started and stopped as follows:
sudo systemctl start elasticsearch.service
sudo systemctl stop elasticsearch.service
[root@liang elasticsearch]# curl localhost:9200
{
"name" : "K0mAcX6",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "DC4B_Qh4TEqd2AU2m11lEg",
"version" : {
"number" : "6.5.4",
"build_flavor" : "default",
"build_type" : "rpm",
"build_hash" : "d2ef93d",
"build_date" : "2018-12-17T21:17:40.758843Z",
"build_snapshot" : false,
"lucene_version" : "7.5.0",
"minimum_wire_compatibility_version" : "5.6.0",
"minimum_index_compatibility_version" : "5.0.0"
},
"tagline" : "You Know, for Search"
}
[root@liang elasticsearch]# curl -X PUT 'localhost:9200/hello'
{“acknowledged":true,"shards_acknowledged":true,"index":"hello"}
[root@liang elasticsearch]# curl -X DELETE 'localhost:9200/hello'
{"acknowledged":true}
rpm包安装elasticsearch6.5.3
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.5.3.rpm
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.5.4.rpm.sha512
shasum -a 512 -c elasticsearch-6.5.3.rpm.sha512
sudo rpm --install elasticsearch-6.5.3.rpm
安装ik
注意ik中文包一定要跟es的版本对应上，否则会安装失败
[root@liang elasticsearch]# ./bin/elasticsearch-plugin install https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v6.5.3/elasticsearch-analysis-ik-6.5.3.zip
安装filebeat
[root@liang elk]# wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.5.3-x86_64.rpm
[root@liang filebeat]# rpm --install filebeat-6.5.3-x86_64.rpm
使用filebeat
[root@liang filebeat]# grep -v "#" /etc/filebeat/filebeat.yml |grep -v "^$"
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/*.log
- /var/log/messages
exclude_lines: ['^DBG', '^$']
fields:
service: hello
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 3
setup.kibana:
path: "/tmp"
filename: "filebeat.txt"
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~
把日志导入到redis
[root@liang filebeat]# grep -v "#" /etc/filebeat/filebeat.yml |grep -v "^$"
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/*.log
- /var/log/messages
exclude_lines: ['^DBG', '^$']
fields:
service: hello
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 3
setup.kibana:
output.redis:
hosts: "localhost"
db: "2"
port: "6379"
password: "123456"
key: "hello"
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~
安装logstash
[root@liang ~]# wget https://artifacts.elastic.co/downloads/logstash/logstash-6.5.3.rpm
[root@liang elk]# rpm --install logstash-6.5.3.rpm
安装过程中可能遇到的问题：
[root@liang elk]# /usr/share/logstash/bin/system-install
OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
OpenJDK 64-Bit Server VM warning: INFO: os::commit_memory(0x00000000e5330000, 449642496, 0) failed; error='Cannot allocate memory' (errno=12)
/usr/share/logstash/bin/system-install:行88: #: 未找到命令
Unable to install system startup script for Logstash.
不要慌~
[root@liang elk]# vim /etc/logstash/jvm.options
把内存调小，默认为2g
# Xms represents the initial size of total heap space
# Xmx represents the maximum size of total heap space
-Xms512m
-Xmx512m
调了之后还是报错，把占用内存的应用先关掉，比如es，开启了es占用了老子1g多的内存
[root@liang conf.d]# vim redis-logstash.conf
input {
redis {
data_type => "list"
host => “127.0.0.1”
db => "2"
port => "6379"
password => "123456"
key => "hello"
}
}
output {
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "hello-%{+YYYY.MM.dd}"
}
}
检查语法
[root@liang conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis-logstash.conf -t
OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
[INFO ] 2018-12-21 15:56:51.300 [main] writabledirectory - Creating directory {:setting=>"path.queue", :path=>"/usr/share/logstash/data/queue"}
[INFO ] 2018-12-21 15:56:51.381 [main] writabledirectory - Creating directory {:setting=>"path.dead_letter_queue", :path=>"/usr/share/logstash/data/dead_letter_queue"}
[WARN ] 2018-12-21 15:56:52.237 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified
Configuration OK
[INFO ] 2018-12-21 15:56:55.604 [LogStash::Runner] runner - Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash
[root@liang conf.d]# systemctl restart logstash
[root@liang conf.d]# systemctl status logstash
安装kibana
[root@liang conf.d]# wget https://artifacts.elastic.co/downloads/kibana/kibana-6.5.3-x86_64.rpm
[root@liang elk]# rpm —install kibana-6.5.3-x86_64.rpm
[root@liang elk]# grep -v "#" /etc/kibana/kibana.yml |grep -v "^$"
server.host: "0.0.0.0"
elasticsearch.url: "http://localhost:9200"
