CentOS7安装elk

 

一、安装Elasticsearch

1.1 安装Java

# 系统版本
 cat /etc/redhat-release 
CentOS Linux release 7.5.1804 (Core) 

  uname -r
3.10.0-862.el7.x86_64


  yum -y install java
  java -version
openjdk version "1.8.0_212"
OpenJDK Runtime Environment (build 1.8.0_212-b04)
OpenJDK 64-Bit Server VM (build 25.212-b04, mixed mode)

 

1.2 安装Elasticsearch

• 关闭防火墙

systemctl disable firewalld

• 关闭selinux

vim /etc/sysconfig/selinux 
SELINUX= Enforcing # <== 修改为 disabled, 重启系统生效

• 官方文档

https://www.elastic.co/guide/en/elasticsearch/reference/6.5/rpm.html#install-rpm

• 下载并安装公共签名秘钥

rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

• 从RPM存储库安装

在/etc/yum.repos.d/创建一个elasticsearch.repo文件，文件内容如下：

vim /etc/yum.repos.d/elasticsearch.repo

[elasticsearch-6.x]
name=Elasticsearch repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

• yum安装ElasticSearch

yum install elasticsearch -y

• 启动Elasticsearch

systemctl daemon-reload
systemctl enable elasticsearch.service

• 启动/停止Elasticsearch

systemctl start elasticsearch.service
systemctl stop elasticsearch.service

• Elasticsearch服务运行状态

systemctl status elasticsearch.service

Elasticsearch日志存储在/var/log/elasticsearch/目录下

•  检查Elasticsearch是否正在运行

[root@localhost ~]# curl http://localhost:9200
{
  "name" : "O2pObfg",
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "uXvFw1fgR1iTduPf1d-MAw",
  "version" : {
    "number" : "6.7.1",
    "build_flavor" : "default",
    "build_type" : "rpm",
    "build_hash" : "2f32220",
    "build_date" : "2019-04-02T15:59:27.961366Z",
    "build_snapshot" : false,
    "lucene_version" : "7.7.0",
    "minimum_wire_compatibility_version" : "5.6.0",
    "minimum_index_compatibility_version" : "5.0.0"
  },
  "tagline" : "You Know, for Search"
}

1.3 配置Elasticsearch

Elasticsearch默认使用/etc/elasticsearch运行时配置

默认Elasticsearch从文件/etc/elasticsearch/elastisearch.yml加载配置。

[root@localhost ~]# ll -lrht /etc/elasticsearch/
total 36K
-rw-rw----. 1 root elasticsearch    0 Apr  3 00:07 users_roles
-rw-rw----. 1 root elasticsearch    0 Apr  3 00:07 users
-rw-rw----. 1 root elasticsearch  197 Apr  3 00:07 roles.yml
-rw-rw----. 1 root elasticsearch  473 Apr  3 00:07 role_mapping.yml
-rw-rw----. 1 root elasticsearch  13K Apr  3 00:07 log4j2.properties
-rw-rw----. 1 root elasticsearch 3.6K Apr  3 00:07 jvm.options
-rw-rw----. 1 root elasticsearch 2.9K Apr  3 00:07 elasticsearch.yml
-rw-rw----. 1 root elasticsearch  207 Apr 28 14:16 elasticsearch.keystore

• 配置Elasticsearch数据路径和日志目录：

官方文档：

https://www.elastic.co/guide/en/elasticsearch/reference/6.5/settings.html

https://www.elastic.co/guide/en/elasticsearch/reference/6.5/important-settings.html

# 创建Elasticsearch数据目录及日志目录
mkdir /opt/elasticsearch/{data,log}/ -pv
cd /opt/
chown -R elasticsearch:elasticsearch elasticsearch/

cp /etc/elasticsearch/elasticsearch.yml /etc/elasticsearch/elasticsearch.yml_`date +%Y%m%d_bak`

vim /etc/elasticsearch/elasticsearch.yml
# Elasticsearch 节点名称
node.name: node-1
#数据
path.data: /opt/elasticsearch/data/
# 日志
path.logs: /opt/elasticsearch/log/

# 内存锁定,将进程地址空间锁定在RAM中，防止任何Elasticsearch内存被换出 
bootstrap.memory_lock: true

# 监听地址
network.host: 192.168.198.130
# 监听端口
http.port: 9200

# 查看修改了那些配置 grep '^[a-z]' /etc/elasticsearch/elasticsearch.yml
node.name: node-1
path.data: /opt/elasticsearch/data/
path.logs: /opt/elasticsearch/log/
bootstrap.memory_lock: true
network.host: 192.168.198.130
http.port: 9200

• 配置JVM

Jvm配置文件/etc/elasticsearch/jvm.options

-Xms2g
-Xmx2g

• 文件句柄配置

官方文档：https://www.elastic.co/guide/en/elasticsearch/reference/6.5/setting-system-settings.html

ulimit -n 65536
tail -1 /etc/security/limits.conf 

elasticsearch  -  nofile  65536

• 系统配置

官方文档：https://www.elastic.co/guide/en/elasticsearch/reference/6.5/setting-system-settings.html

vim /usr/lib/systemd/system/elasticsearch.service


[Service]        # <== 在此标签下添加
LimitMEMLOCK=infinity

# 重新加载
systemctl daemon-reload

• 虚拟内存

官方文档：

https://www.elastic.co/guide/en/elasticsearch/reference/6.5/vm-max-map-count.html

临时配置：

sysctl -w vm.max_map_count=262144

永久配置：/etc/sysctl.conf

tail -1 /etc/sysctl.conf
vm.max_map_count=262144
sysctl -p

二、安装Logstash

• 官方文档

https://www.elastic.co/guide/en/logstash/6.5/installing-logstash.html

• 下载并安装公共签名和密钥

rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

• 在/etc/yum.repos.d/目录下新建logstash.repo文件，文件内容如下：

[logstash-6.x]
name=Elastic repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

• yum安装logstash

yum -y install logstash

三、安装Kibana

• 官方文档

https://www.elastic.co/guide/en/kibana/6.5/rpm.html

• 下载并安装公共签名和密钥

rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

• 在/etc/yum.repos.d/目录下，创建kibana.repo，文件内容如下：

[kibana-6.x]
name=Kibana repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

• yum 安装kibana

yum install kibana

• 启动/停止kibana

systemctl daemon-reload
systemctl enable kibana.service
systemctl start kibana.service
systemctl stop kibana.service

• 配置kibana

Kibana配置文件在/etc/kibana/目录下的kibana.yml。默认kibana运行localhost:5601

[root@localhost ~]# grep '^[a-z]' /etc/kibana/kibana.yml 
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://192.168.198.130:9200"]
kibana.index: ".kibana"

 

四、安装Filebeat

• 官方文档

https://www.elastic.co/guide/en/beats/filebeat/5.5/setup-repositories.html

https://www.elastic.co/guide/en/beats/filebeat/6.5/configuring-output.html

• 下载并安装公共签名和密钥

rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch

• 在/etc/yum.repos.d/目录下创建filebeat.repo文件，文件内容如下：

[elastic-6.x]
name=Elastic repository for 5.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

•  yum安装filebeat

yum -y install filebeat

#  filebeat记录读取文件位偏移 /var/lib/filebeat/registry

五、安装Redis

参考：https://www.cnblogs.com/hwlong/p/9330191.html 

https://www.cnblogs.com/hwlong/p/6101019.html

wget http://download.redis.io/releases/redis-3.2.12.tar.gz
上传至 /usr/local
tar xzf redis-3.2.12.tar.gz
mv redis-3.2.12 redis
cd redis
make
src/redis-server &

 

六、Filebeat+Redis+Logstash+Elasticsearch+Kibana

6.1 配置Filebeat

filebeat.prospectors:


- input_type: log
  paths:
    - /data/logs/localhost.localdomain.*.log
  exclude_files: [".gz$"]
  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
  multiline.negate: true
  multiline.match: after
  tags: ["localhost-localdomain"]
  document_type: localhost-localdomain


- input_type: log
  paths:
    - /data/logs/localhost.localdomain-error.*.log
  exclude_files: [".gz$"]
  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
  multiline.negate: true
  multiline.match: after
  tags: ["localhost-localdomain-error"]
  document_type: localhost-localdomain-error




fields:
  ipaddr: '192.168.198.131'   # <== filebeat本机IP
fields_under_root: true


output.redis:
  hosts: ["192.168.198.130"]        # <== redis地址
  port: 6379
  key: "default_list"
  keys:
    - key: "%{[type]}"                # <== 根据document_type类型写入不同的key中
      mapping:
      "localhost-localdomain": "localhost-localdomain"
      "localhost-localdomain-error": "localhost-localdomain-error"
 
  db: 0
  datatype: list
  timeout: 5
  


processors:
  - drop_fields:
       fields: ["offset","beat","source","input"]

• 启动filebeat

systemctl status filebeat

 

6.2 配置logstash.conf

input {

  redis {
     host => "192.168.198.130"
     port => "6379"
     db => "0"
     data_type => "list"
     key => "default_list"
     type => "default_list"
  }
  
  redis {
     host => "192.168.198.130"
     port => "6379"
     db => "0"
     data_type => "list"
     key => "localhost-localdomain"
     type => "localhost-localdomain"
  }
 
  redis {
     host => "192.168.198.130"
     port => "6379"
     db => "0"
     data_type => "list"
     key => "localhost-localdomain-error"
     type => "localhost-localdomain-error"
  }



}


filter {

}


output{
   if [type] == "localhost-localdomain" {

       elasticsearch {
           document_type => "localhost-localdomain"
           hosts => ["192.168.198.130:9200"]
           index => "localhost-localdomain.log"
       }
   }
   
   if [type] == "localhost-localdomain-error" {

       elasticsearch {
           document_type => "localhost-localdomain-error"
           hosts => ["192.168.198.130:9200"]
           index => "localhost-localdomain-error.log"
       }
   }


  

}

• 启动logstash

/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis-logstash-es.conf  &