Centos6.7 ELK日志系统部署
Centos6.7 ELK日志系统部署
原文地址：http://www.cnblogs.com/caoguo/p/4991602.html
一. 环境
elk服务器：192.168.55.134
logstash日志采集端：192.168.55.132
二.安装JDK
[root@elk01 ~]# cd /usr/local/src [root@elk01 src]# wget http://download.oracle.com/otn-pub/java/jdk/8u65-b17/jdk-8u65-linux-x64.tar.gz?AuthParam=1447919869_29a658de74feaeda612894dc77923aa4 [root@elk01 src]# tar zxvf jdk-8u65-linux-x64.tar.gz [root@elk01 src]# mv jdk1.8.0_65/ /usr/local/ [root@elk01 ~]# vi /etc/profile JAVA_HOME=/usr/local/jdk1.8.0_20 export JAVA_HOME PATH=$JAVA_HOME/bin:$PATH export PATH [root@elk01 ~]# source /etc/profile
#检查下
三.redis安装
#### Redis install #### [root@elk01 src]# yum install -y wget gcc make tcl [root@elk01 src]# http://download.redis.io/releases/redis-3.0.5.tar.gz [root@elk01 src]# cd redis-3.0.5 [root@elk01 redis-3.0.5]# make [root@elk01 redis-3.0.5]# make install [root@elk01 redis-3.0.5]# cp redis.conf /etc/ [root@elk01 redis-3.0.5]# touch /etc/init.d/redis [root@elk01 redis-3.0.5]# chmod 755 /etc/init.d/redis
[root@elk01 redis-3.0.5]# vi /etc/init.d/redis #!/bin/bash # # Init file for redis # # chkconfig: - 80 12 # description: redis daemon # # processname: redis # config: /etc/redis.conf # pidfile: /var/run/redis.pid source /etc/init.d/functions #BIN="/usr/local/bin" BIN="/usr/local/bin" CONFIG="/etc/redis.conf" PIDFILE="/var/run/redis.pid" ### Read configuration [ -r "$SYSCONFIG" ] && source "$SYSCONFIG" RETVAL=0 prog="redis-server" desc="Redis Server" start() { if [ -e $PIDFILE ];then echo "$desc already running...." exit 1 fi echo -n $"Starting $desc: " daemon $BIN/$prog $CONFIG RETVAL=$? echo [ $RETVAL -eq 0 ] && touch /var/lock/subsys/$prog return $RETVAL } stop() { echo -n $"Stop $desc: " killproc $prog RETVAL=$? echo [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$prog $PIDFILE return $RETVAL } restart() { stop start } case "$1" in start) start ;; stop) stop ;; restart) restart ;; condrestart) [ -e /var/lock/subsys/$prog ] && restart RETVAL=$? ;; status) status $prog RETVAL=$? ;; *) echo $"Usage: $0 {start|stop|restart|condrestart|status}" RETVAL=1 esac exit $RETVAL
[root@elk01 redis-3.0.5]# chkconfig redis on [root@elk01 redis-3.0.5]# /etc/init.d/redis start
#redis连接测试
四.logstash indexer服务安装配置
#### Logstash 从redis取数据到elasticsearch #### [root@elk01 src]# wget https://download.elastic.co/logstash/logstash/logstash-2.0.0.tar.gz [root@elk01 src]# tar zxf logstash-2.0.0.tar.gz -C /usr/local/ [root@elk01 ~]# /usr/local/logstash-2.0.0/bin/logstash -e 'input { stdin { } } output { stdout {} }' [root@elk01 logstash-2.0.0]# vi logstash_indexer.conf input { redis { host => 'localhost' data_type => 'list' port => '6379' key => 'logstash:redis' type => 'redis-input' } } output { elasticsearch { hosts => 'localhost' } } [root@elk01 logstash-2.0.0]# bin/logstash -f logstash_indexer.conf [root@elk01 elasticsearch]# redis-cli monitor OK 1448364122.959182 [0 127.0.0.1:56723] "rpush" "logstash:redis" "{\"message\":\"Nov 24 19:22:02 elk01 yum[3074]: Erased: httpd\",\"@version\":\"1\",\"@timestamp\":\"2015-11-24T11:22:02.553Z\",\"host\":\"0.0.0.0\",\"path\":\"/var/log/messages\",\"type\":\"messages\"}"
# redis-cli monitor 看到的输出
五. elasticsearch 安装配置
#### Elasticsearch #### [root@elk01 src]# tar zxf elasticsearch-2.0.0.tar.gz [root@elk01 src]# mv elasticsearch-2.0.0 /usr/local/elasticsearch [root@elk01 src]# useradd elasticsearch -d /usr/local/elasticsearch -s /sbin/nologin [root@elk01 ~]# vi /etc/sysconfig/elasticsearch # Directory where the Elasticsearch binary distribution resides ES_HOME=/usr/local/elasticsearch # Heap Size (defaults to 256m min, 1g max) #ES_HEAP_SIZE=2g # Heap new generation #ES_HEAP_NEWSIZE= # max direct memory #ES_DIRECT_SIZE= # Additional Java OPTS #ES_JAVA_OPTS= # Maximum number of open files MAX_OPEN_FILES=65535 # Maximum amount of locked memory #MAX_LOCKED_MEMORY= # Maximum number of VMA (Virtual Memory Areas) a process can own MAX_MAP_COUNT=262144 # Elasticsearch log directory LOG_DIR=/var/log/elasticsearch # Elasticsearch data directory DATA_DIR=/usr/local/elasticsearch/data # Elasticsearch work directory WORK_DIR=/tmp/elasticsearch # Elasticsearch conf directory CONF_DIR=/etc/elasticsearch # Elasticsearch configuration file (elasticsearch.yml) CONF_FILE=/etc/elasticsearch/elasticsearch.yml # User to run as, change this to a specific elasticsearch user if possible # Also make sure, this user can write into the log directories in case you change them # This setting only works for the init script, but has to be configured separately for systemd startup ES_USER=elasticsearch # Configure restart on package upgrade (true, every other setting will lead to not restarting) #RESTART_ON_UPGRADE=true [root@elk01 ~]# mkdir -p /var/run/elasticsearch [root@elk01 ~]# mkdir -p /var/log/elasticsearch [root@elk01 ~]# mkdir -p /usr/local/elasticsearch/data [root@elk01 ~]# mkdir -p /usr/local/elasticsearch/plugins [root@elk01 ~]# mkdir -p /usr/local/elasticsearch/config/scripts [root@elk01 ~]# chown -R elasticsearch.elasticsearch /var/run/elasticsearch [root@elk01 ~]# chown -R elasticsearch.elasticsearch /var/log/elasticsearch [root@elk01 ~]# chown -R elasticsearch.elasticsearch /usr/local/elasticsearch/data [root@elk01 ~]# ln -s /usr/local/elasticsearch/config /etc/elasticsearch
[root@elk01 ~]# vi /etc/init.d/elasticsearch #!/bin/sh # # elasticsearch <summary> # # chkconfig: 2345 80 20 # description: Starts and stops a single elasticsearch instance on this system # ### BEGIN INIT INFO # Provides: Elasticsearch # Required-Start: $network $named # Required-Stop: $network $named # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: This service manages the elasticsearch daemon # Description: Elasticsearch is a very scalable, schema-free and high-performance search solution supporting multi-tenancy and near realtime search. ### END INIT INFO # # init.d / servicectl compatibility (openSUSE) # if [ -f /etc/rc.status ]; then . /etc/rc.status rc_reset fi # # Source function library. # if [ -f /etc/rc.d/init.d/functions ]; then . /etc/rc.d/init.d/functions fi exec="/usr/local/elasticsearch/bin/elasticsearch" prog="elasticsearch" pidfile=/var/run/elasticsearch/${prog}.pid [ -e /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog export ES_HEAP_SIZE export ES_HEAP_NEWSIZE export ES_DIRECT_SIZE export ES_JAVA_OPTS lockfile=/var/lock/subsys/$prog # backwards compatibility for old config sysconfig files, pre 0.90.1 if [ -n $USER ] && [ -z $ES_USER ] ; then ES_USER=$USER fi checkJava() { if [ -x "$JAVA_HOME/bin/java" ]; then JAVA="$JAVA_HOME/bin/java" else JAVA=`which java` fi if [ ! -x "$JAVA" ]; then echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME" exit 1 fi } start() { checkJava [ -x $exec ] || exit 5 [ -f $CONF_FILE ] || exit 6 if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then echo "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set" return 7 fi if [ -n "$MAX_OPEN_FILES" ]; then ulimit -n $MAX_OPEN_FILES fi if [ -n "$MAX_LOCKED_MEMORY" ]; then ulimit -l $MAX_LOCKED_MEMORY fi if [ -n "$MAX_MAP_COUNT" ]; then sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT fi if [ -n "$WORK_DIR" ]; then mkdir -p "$WORK_DIR" chown "$ES_USER":"$ES_GROUP" "$WORK_DIR" fi echo -n $"Starting $prog: " # if not running, start it up here, usually something like "daemon $exec" daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Des.default.path.home=$ES_HOME -Des.default.path.logs=$LOG_DIR -Des.default.path.data=$DATA_DIR -D es.default.path.work=$WORK_DIR -Des.default.path.conf=$CONF_DIR retval=$? echo [ $retval -eq 0 ] && touch $lockfile return $retval } stop() { echo -n $"Stopping $prog: " # stop it here, often "killproc $prog" killproc -p $pidfile -d 20 $prog retval=$? echo [ $retval -eq 0 ] && rm -f $lockfile return $retval } restart() { stop start } reload() { restart } force_reload() { restart } rh_status() { # run checks to determine if the service is running or use generic status status -p $pidfile $prog } rh_status_q() { rh_status >/dev/null 2>&1 } case "$1" in start) rh_status_q && exit 0 $1 ;; stop) rh_status_q || exit 0 $1 ;; restart) $1 ;; reload) rh_status_q || exit 7 $1 ;; force-reload) force_reload ;; status) rh_status ;; condrestart|try-restart) rh_status_q || exit 0 restart ;; *) echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" exit 2 esac exit $?
[root@elk01 ~]# chmod +x /etc/init.d/elasticsearch
[root@elk01 ~]# /etc/init.d/elasticsearch start
#管理工具 [root@elk01 ~]# /usr/local/elasticsearch/bin/plugin install lmenezes/elasticsearch-kopf [root@elk01 ~]# /usr/local/elasticsearch/bin/plugin install mobz/elasticsearch-head
六. Kibana安装
#### Kibana Install #### [root@elk01 src]# wget https://download.elastic.co/kibana/kibana/kibana-4.2.1-linux-x64.tar.gz [root@elk01 src]# tar zxf kibana-4.2.1-linux-x64.tar.gz -C /usr/local/ [root@elk01 local]# touch /etc/init.d/kibana [root@elk01 local]# chmod 755 /etc/init.d/kibana [root@elk01 local]# vi /etc/init.d/kibana #!/bin/bash ### BEGIN INIT INFO # Provides: kibana # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Runs kibana daemon # Description: Runs the kibana daemon as a non-root user ### END INIT INFO # Process name NAME=kibana DESC="Kibana4" PROG="/etc/init.d/kibana" # Configure location of Kibana bin KIBANA_BIN=/usr/local/kibana/bin # PID Info PID_FOLDER=/var/run/kibana/ PID_FILE=/var/run/kibana/$NAME.pid LOCK_FILE=/var/lock/subsys/$NAME PATH=/bin:/usr/bin:/sbin:/usr/sbin:$KIBANA_BIN DAEMON=$KIBANA_BIN/$NAME # Configure User to run daemon process DAEMON_USER=root # Configure logging location KIBANA_LOG=/var/log/kibana.log # Begin Script RETVAL=0 if [ `id -u` -ne 0 ]; then echo "You need root privileges to run this script" exit 1 fi # Function library . /etc/init.d/functions start() { echo -n "Starting $DESC : " pid=`pidofproc -p $PID_FILE kibana` if [ -n "$pid" ] ; then echo "Already running." exit 0 else # Start Daemon if [ ! -d "$PID_FOLDER" ] ; then mkdir $PID_FOLDER fi daemon --user=$DAEMON_USER --pidfile=$PID_FILE $DAEMON 1>"$KIBANA_LOG" 2>&1 & sleep 2 pidofproc node > $PID_FILE RETVAL=$? [[ $? -eq 0 ]] && success || failure echo [ $RETVAL = 0 ] && touch $LOCK_FILE return $RETVAL fi } reload() { echo "Reload command is not implemented for this service." return $RETVAL } stop() { echo -n "Stopping $DESC : " killproc -p $PID_FILE $DAEMON RETVAL=$? echo [ $RETVAL = 0 ] && rm -f $PID_FILE $LOCK_FILE } case "$1" in start) start ;; stop) stop ;; status) status -p $PID_FILE $DAEMON RETVAL=$? ;; restart) stop start ;; reload) reload ;; *) # Invalid Arguments, print the following message. echo "Usage: $0 {start|stop|status|restart}" >&2 exit 2 ;; esac [root@elk01 local]# mv kibana-4.2.1-linux-x64/ kibana [root@elk01 ~]# mkdir -p /var/run/kibana
七. logstash日志收集
#### logstash 日志收集 #### [root@localhost ~]# tar zxf logstash-2.0.0.tar.gz -C /usr/local/ [root@localhost ~]# /usr/local/logstash-2.0.0/bin/logstash -e 'input { stdin { } } output { stdout {} }' [root@localhost logstash-2.0.0]# cat logstash_agent.conf input { file { type => "apache_access" path => ["/var/log/httpd/access_log"] } } output { stdout {codec => rubydebug } redis { host => '192.168.55.134' data_type => 'list' key => 'logstash:redis' } } # 访问一下http服务，看redis是否收到日志 [root@elk01 elasticsearch]# redis-cli monitor OK 1448364122.959182 [0 127.0.0.1:56723] "rpush" "logstash:redis" "{\"message\":\"Nov 24 19:22:02 elk01 yum[3074]: Erased: httpd\",\"@version\":\"1\",\"@timestamp\":\"2015-11-24T11:22:02.553Z\",\"host\":\"0.0.0.0\",\"path\":\"/var/log/messages\",\"type\":\"messages\"}"
