【elk】安装笔记记录

一、环境配置

1.需要安装包

63上5个包
[root@web63 ~]# ls
anaconda-ks.cfg elasticsearch-5.2.0.tar.gz jdk-8u91-linux-x64.tar.gz kibana-5.2.0-linux-x86_64.tar.gz logstash-5.2.0.tar.gz redis-3.0.5.tar.gz
64上两个包
[root@nfs65 ~]# ls
jdk-8u91-linux-x64.tar.gz logstash-5.2.0.tar.gz

cat /etc/sysconfig/selinux

 

2.时间同步 主机名配置

[root@elk ~]#hostnamectl set-hostname elk.server.com

cat /etc/hosts
192.168.201.135 elk.server.com

 

关闭firewall

iptables -F && systemctl stop firewalld.service && systemctl disable firewalld.service

同步时间

#首先修改时区亚洲

[root@192 config]# ls -l /etc/localtime
lrwxrwxrwx. 1 root root 38 Apr 13 18:08 /etc/localtime -> ../usr/share/zoneinfo/America/New_York

rm /etc/localtime #删除link
ln -vs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime #软件link
yum install ntpdate -y
ntpdate time.nist.gov

 

3.线程优化 文件打开数 虚拟内存优化

最大线程数

vi /etc/security/limits.d/20-nproc.conf #cenos7
vi /etc/security/limits.d/90-nproc.conf #cenos6
* soft nproc 2048

#在文件最后添加 nofile - 打开文件的最大数目 noproc - 进程的最大数目 soft<=hard soft的限制不能比hard限制高

vi /etc/security/limits.conf
* soft nofile 65536
* hard nofile 131072
* soft nproc 2048
* hard nproc 4096

4.虚拟内存优化不然报错

vi /etc/sysctl.conf
vm.max_map_count=655360
sysctl -p

 

二、软件安装

1 .安装redis

#创建安装目录
mkdir -pv /data/application/
wget http://download.redis.io/releases/redis-3.0.5.tar.gz
tar zxf redis-3.0.5.tar.gz && cd redis-3.0.5
yum install -y cmake gcc gcc-c++
make MALLOC=libc #编译报错就执行下面error: jemalloc/jemalloc.h: No such file or directory

[root@elk redis-3.0.5]# make PREFIX=/data/application/redis-3.0.5 install

#创建配置文件目录
[root@elk redis-3.0.5]#mkdir /data/application/redis-3.0.5/{etc,run,log}
cp /root/redis-3.0.5/redis.conf /data/application/redis-3.0.5/etc/
[root@elk redis-3.0.5]#vi /data/application/redis-3.0.5/etc/redis.conf
daemonize yes #后台模式运行
pidfile /data/application/redis-3.0.5/run/redis.pid #redis的pid
bind 0.0.0.0 #这里根据自己的ip填写
port 6379 #端口
logfile "/data/application/redis-3.0.5/log/redis.log" #log存放位置
dir /data/application/redis-3.0.5

egrep -v "^$|#" /data/application/redis-3.0.5/etc/redis.conf

/data/application/redis-3.0.5/bin/redis-server /data/application/redis-3.0.5/etc/redis.conf

yum install lsof
lsof -i:6379

 

2.安装jdk （被监控端也要安装）

[root@elk elk_pack]#rpm -qa|grep java
[root@elk elk_pack]#yum remove java-1.8.0-openjdk
[root@elk elk_pack]#yum remove java-1.7.0-openjdk

[root@elk elk_pack]# tar zxvf jdk-8u91-linux-x64.tar.gz -C /data/application/
[root@elk jdk1.8.0_91]# vi /etc/profile.d/java.sh 
export JAVA_HOME=/data/application/jdk1.8.0_91/
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
[root@elk jdk1.8.0_91]#source /etc/profile.d/java.sh

[root@elk jdk1.8.0_91]#java -version

 

3.安装elasticsearch

 

[root@localhost application]# adduser -s /bin/bash -c 'elk' -m -d /home/elk elk
注：从2.0开始不能用root用户启动需要elk用户启动

[root@elk elk_pack]# tar zxvf elasticsearch-5.2.0.tar.gz -C /data/application/

[root@elk ~]# cp /data/application/elasticsearch-5.2.0/config/elasticsearch.yml{,.ori}

[root@elk config]# vi /data/application/elasticsearch-5.2.0/config/elasticsearch.yml

path.data: /data/shuju    ----》存放数据路径
path.logs: /data/logs -----》日志路径
network.host: 192.168.1.63 -----》根据自己的ip修改
http.port: 9200

#创建logs,shuju
[root@elk config]#mkdir /data/{shuju,logs}
#启动elasticsearch
#修改elasticsearch权限
[root@elk ~]#chown -R elk.elk /data/application/elasticsearch-5.2.0/
[root@elk ~]#chown -R elk.elk /data/{shuju,logs}
[root@elk ~]# su elk
[elk@elk ~]$/data/application/elasticsearch-5.2.0/bin/elasticsearch

/data/application/elasticsearch-5.2.0/bin/elasticsearch -d #后台执行未测试
注：如果执行完这个脚本，中途没有退出，说明启动成功

#测试是否成功
[root@elk ~]# curl 192.168.1.63:9200

 

4.服务器安装Logstash

tar zxvf logstash-5.2.0.tar.gz -C /data/application/

[root@elk ~]# /data/application/logstash-5.2.0/bin/logstash -e 'input { stdin { } } output {stdout {} }'

 

5. 安装Kibana

[root@elk elk_pack]# tar zxvf kibana-5.2.0-linux-x86_64.tar.gz -C /data/application/vi /data/application/kibana-5.2.0-linux-x86_64/config/kibana.yml
server.port: 5601    #kibana的端口
server.host: "192.168.1.63"    #访问kibana的ip地址
elasticsearch.url: "http://192.168.1.63:9200"    #elasticsearch的ip地址
kibana.index: ".kibana"    #创建索引
#测试是否启动成功
[root@192 ~]# /data/application/kibana-5.2.0-linux-x86_64/bin/kibana
[root@elk shuju]# lsof -i:5601
#通过web访问
http://192.168.1.63:5601

egrep -v "^$|^[#]" /data/application/kibana-5.2.0-linux-x86_64/config/kibana.yml

 

三、客户端安装

1.安装Logstash

mkdir -pv /data/application/
tar zxvf logstash-5.2.0.tar.gz -C /data/application/
mkdir /data/application/logstash-5.2.0/agent_conf
vi /data/application/logstash-5.2.0/agent_conf/nginx-agent.conf

#创建采集数据的文件
[root@localhost agent_conf]# cat /data/application/logstash-5.2.0/agent_conf/nginx-agent.conf
input {
file {
type => "nginx_access"
path => ["/var/log/nginx/access.log"]
start_position => "beginning"
} 
}
output {
stdout {codec => rubydebug }
redis {
host => ["192.168.1.63:6379"]
data_type=> "list"
key => "nginx_access"
}
}

 

2.按装配置ngnix

yum install nginx -y
systemctl restart nginx
curl localhost

#编辑vi /etc/nginx/nginx.conf 
20 log_format backend '$remote_addr - $remote_user[$time_local] "$request" '
21 '$status $body_bytes_sent "$http_referer" '
22 '"$http_user_agent" "$http_x_forwarded_for" $request_time ';

或者

http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';

access_log /var/log/nginx/access.log main;

#查看访问日志
[root@localhost conf]# cat /var/log/nginx/access.log
192.168.201.1 - -[24/Mar/2017:04:24:18 +0800] "GET / HTTP/1.1" 304 0

 

3.Logstash-server端配置
#方便管理文件，把采集数据的文件统一放到一个目录中管理

mkdir /data/application/logstash-5.2.0/server_conf
vi /data/application/logstash-5.2.0/server_conf/logstash-server-nginx.conf
cat /data/application/logstash-5.2.0/server_conf/logstash-server-nginx.conf
input {
redis {
host => "192.168.1.63"
port => "6379"
data_type => "list"
key => "nginx_access"
type => "redis-input"
}
}
filter {
if[type] =~ "nginx_access" {
grok {
patterns_dir => ["/data/application/logstash-5.2.0/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-4.0.2/patterns/"]
match => {
"message" => "%{NGINXACCESS}"
}
}
date {
match => ["timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
}
}
}
output {
elasticsearch {
hosts => "192.168.1.63:9200"
index => "nginx-access-%{+yyyy.MM.dd}"
}
}

 

服务采集数据需要配置文件

vi /data/application/logstash-5.2.0/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-4.0.2/patterns/nginx

NGINXACCESS %{IPORHOST:clientip} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" (?:%{QS:content_type}|-) (?:%{QS:request_body}|-) (?:"(?:%{URI:referrer}|-)"|%{QS:referrer}) %{NUMBER:response} %{BASE16FLOAT:request_time} (?:%{NUMBER:bytes}|-)
NGINXERROR_DATESTAMP %{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME}
NGINXERROR_PID (?:[0-9]+#[0-9]+\:)
NGINXERROR_TID (?:\*[0-9]+)
NGINXERROR %{NGINXERROR_DATESTAMP:timestamp} \[%{LOGLEVEL:loglevel}\] %{NGINXERROR_PID:pid} %{NGINXERROR_TID:tid} %{GREEDYDATA:errormsg}, client: %{IPORHOST:clientip}, server: %{HOSTNAME:server}, request: %{QS:request}(?:, upstream: %{QS:upstream})?, host: \"%{HOSTNAME:hostname}\"(?:, referrer: (?:"(?:%{URI:referrer}|-)"|%{QS:referrer}))?

 

 

4.#测试
服务端启动顺序由左到右Elasticsearch-Kibana--Logstash

注意：Elasticsearch在elk普通用户下执行

[elk@elk root]$ /data/application/elasticsearch-5.2.0/bin/elasticsearch
/data/application/kibana-5.2.0-linux-x86_64/bin/kibana
/data/application/logstash-5.2.0/bin/logstash -f /data/application/logstash-5.2.0/server_conf/logstash-server-nginx.conf -t #检测配置文件是否正确
/data/application/logstash-5.2.0/bin/logstash -f /data/application/logstash-5.2.0/server_conf/logstash-server-nginx.conf

 

 

安装jdk （客户端安装）

[root@elk elk_pack]#rpm -qa|grep java
[root@elk elk_pack]#yum remove java-1.8.0-openjdk
[root@elk elk_pack]#yum remove java-1.7.0-openjdk

[root@elk elk_pack]# tar zxvf jdk-8u91-linux-x64.tar.gz -C /data/application/
[root@elk jdk1.8.0_91]# vi /etc/profile.d/java.sh 
export JAVA_HOME=/data/application/jdk1.8.0_91/
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
[root@elk jdk1.8.0_91]#source /etc/profile.d/java.sh

[root@elk jdk1.8.0_91]#java -version

 

客户端启动

/data/application/logstash-5.2.0/bin/logstash -f /data/application/logstash-5.2.0/agent_conf/nginx-agent.conf

 

5.总结需要配置的几个重要文件

logstash-nginx_server=/data/application/logstash-5.2.0/server_conf/logstash-server-nginx.conf
nginx_agent=/data/application/logstash-5.2.0/agent_conf/nginx-agent.conf
nginx_patterns_server=vi /data/application/logstash-5.2.0/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-4.0.2/patterns/nginx