07 . ELK Stack7.2一键多机部署脚本

一键部署脚本

目录结构

tree Log_Analysis_Platform_Document
Log_Analysis_Platform_Document
├── InstallES.sh
├── InstallFilebeat.sh
├── InstallKibana.sh
└── README.md

ES.sh

#!/usr/bin/env bash
# *************************************************************************************************************
# Author: ZhouJian
# Mail: 18621048481@163.com
# Data: 2019-9-7
# Describe: CentOS 7 AutoInstall Elasticsearchn-7.2 Deploy Script

# ****************************Elasticsearch Deplay Script******************************************************
clear
ESIP=`ip addr | grep "inet" | grep -v "127.0.0.1" | grep -v "inet6" | awk -F/ '{print $1}' | awk '{print $2}' `

echo -e "\033[32m ############################################################################# \033[0m"
echo -e "\033[32m #                           Auto Install ELK.                              ## \033[0m"
echo -e "\033[32m #                           Press Ctrl + C to cancel                       ## \033[0m"
echo -e "\033[32m #                           Any key to continue                            ## \033[0m"
echo -e "\033[32m # Softwae:elasticsearch-7.2.0/logstash-7.2.0/filebeat-7.2.0/kibana-7.2.0   ## \033[0m"
echo -e "\033[32m ############################################################################# \033[0m"

Read_Input() {
echo -e "\033[32m Please Input You Kibana Pass Key IP: \033[0m"
read -p "Please Input You HOST Pass Key IP:[192.168.244.55]" KibanaIP
read -p "Please Input You HOST Pass Key IP: Password:" KibanaPass

echo -e "\033[32m Please Input You Filebeat Pass Key IP: \033[0m"
read -p "Please Input You HOST Pass Key IP:[192.168.244.56]" FilebeatIP
read -p "Please Input You HOST Pass Key IP: Password:" FilebeatPass
}

Init_Yumsource() 
{
	if ! ping -c2 www.baidu.com &>/dev/null
	then
        	echo "您无法上外网，不能配置yum源"
        exit
	fi
		echo "配置yum源"
	if [ ! -d /etc/yum.repos.d/backup ];then
        	mkdir /etc/yum.repos.d/backup
		mv /etc/yum.repos.d/* /etc/yum.repos.d/backup 2>/dev/null	
		curl -o /etc/yum.repos.d/163.repo http://mirrors.163.com/.help/CentOS7-Base-163.repo &>/dev/null
    		curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo &>/dev/null
	fi
}

# *************************************************************************************************************
Init_Hostname() 
{
	hostnamectl set-hostname elk-1
	echo "$ESIP elk-1" >> /etc/hosts	
}


# *************************************************************************************************************
Init_SElinux() 
{
	echo "关闭防火墙"
	systemctl stop firewalld
	systemctl disable firewalld
	echo "关闭selinux"
	setenforce 0
	sed -ri '/^SELINUX=/ s/enforcing/disabled/'  /etc/selinux/config
	echo "解决sshd远程连接慢的问题"
	sed -ri '/^GSSAPIAu/ s/yes/no/' /etc/ssh/sshd_config
	sed -ri '/^#UseDNS/ {s/^#//;s/yes/no/}' /etc/ssh/sshd_config
	systemctl enable sshd crond &> /dev/null
}

# **************************************************************************************************************
Create_UserLogFile() 
{
	groupadd elk
	useradd elk -g elk
	mkdir -pv /data/elk/{data,logs}
	chown -R elk:elk /data/
}
# **************************************************************************************************************

Unpackaged_Authorization() 
{
	yum -y install  ntpdate
	rpm -ivh  /root/InstallELKB-Shell/jdk-8u121-linux-x64.rpm
	tar xvf /root/InstallELKB-Shell/elasticsearch-7.2.0-linux-x86_64.tar.gz -C /opt/
	chown -R elk:elk /opt/elasticsearch-7.2.0/
	ntpdate -b ntp1.aliyun.com
}

# **************************************************************************************************************
Set_System_Parameter() 
{
cat >> /etc/security/limits.conf <<EOF
* soft nproc 2048
* hard nproc 4096
* soft nofile 65536
* hard nofile 131072
EOF

echo "vm.max_map_count = 262144" >> /etc/sysctl.conf && sysctl -p
cat >> /etc/profile <<EOF
export HISTTIMEFORMAT="%Y-%m-%d %H:%M:%S "
EOF
	source /etc/profile

cat >> /opt/elasticsearch-7.2.0/config/elasticsearch.yml <<EOF
cluster.name: elk
node.name: node-1
bootstrap.memory_lock: false
path.data: /data/elk/data
path.logs: /data/elk/logs
network.host: 0.0.0.0
http.port: 9200
discovery.seed_hosts: ["elk-1"]
cluster.initial_master_nodes: ["node-1"]
EOF
	runuser -l elk -c '/bin/bash /opt/elasticsearch-7.2.0/bin/elasticsearch ' &> /opt/elasticsearch.log  &
}

Test_Service() 
{
	esport=`ss -antp |grep :::9200 | awk -F::: '{print $2}'`
	if [ $esport -eq 9200 ];then
		echo -e  "\033[32m Elasticsearch is OK... \033[0m "
	fi
}

# **********************PublicKeyKibana******************************************************************************


PublicKeyKibana() 
{
if [ ! -f /usr/bin/expect ];then
	yum -y install expect
fi
sed -i 's/# *StrictHostKeyChecking *ask/StrictHostKeyChecking no/g' /etc/ssh/ssh_config
systemctl restart sshd

cd /root/.ssh/
ssh-keygen -t rsa -N '' -f id_rsa -q
if [ $? -eq 0 ];then
/usr/bin/expect <<-EOF
set timeout 10
spawn ssh-copy-id $KibanaIP
expect {
    "yes/no" { send "yes\r"; exp_continue }
    "password:" { send "$KibanaPass\r"}
}
expect eof
EOF
fi

}

# **********************Kibana Deploy Script********************************************************************
Install_Kibana() 
{
echo $ESIP > /root/InstallELKB-Shell/ESIP.txt
scp /root/InstallELKB-Shell/kibana-7.2.0-linux-x86_64.tar.gz $KibanaIP:
scp /root/InstallELKB-Shell/ESIP.txt $KibanaIP:
scp /root/InstallELKB-Shell/InstallKibana.sh $KibanaIP:
ssh root@$KibanaIP '
bash /root/InstallKibana.sh '
}



# *******************************************Filebeat Deploy Script***************************************************
PublicFilebeat() 
{
if [ ! -f /usr/bin/expect ];then
	yum -y install expect
fi
sed -i 's/# *StrictHostKeyChecking *ask/StrictHostKeyChecking no/g' /etc/ssh/ssh_config
systemctl restart sshd
cd /root/.ssh/
rm -rf /root/.ssh/*
ssh-keygen -t rsa -N '' -f id_rsa -q
if [ $? -eq 0 ];then
/usr/bin/expect <<-EOF
set timeout 10
spawn ssh-copy-id $FilebeatIP
expect {
    "yes/no" { send "yes\r"; exp_continue }
    "password:" { send "$FilebeatPass\r"}
}
expect eof
EOF
fi
}

Install_Filebeat() 
{
	scp /root/InstallELKB-Shell/filebeat-7.2.0-x86_64.rpm  $FilebeatIP:
	scp /root/InstallELKB-Shell/InstallFilebeat.sh $FilebeatIP:
	ssh root@$FilebeatIP 'bash /root/InstallFilebeat.sh'
	scp /root/InstallELKB-Shell/filebeat.yml $FilebeatIP:/etc/filebeat/ 
	ssh root@$FilebeatIP 'systemctl restart filebeat && systemctl disable filebeat && rm -rf /root/InstallFilebeat.sh' 
}


# ********************************************Logstash******************************************************************
Install_logstash() 
{
	tar xvf /root/InstallELKB-Shell/logstash-7.2.0.tar.gz  -C /opt/
	cp /root/InstallELKB-Shell/nginx.yml  /opt/logstash-7.2.0/
	/opt/logstash-7.2.0/bin/logstash -f /opt/logstash-7.2.0/nginx.yml   &>/opt/logstash.log &	
}


ES-StartUp_SelfStart() 
{
cat >> /etc/init.d/elasticsearch.sh <<EOF
nohup  runuser -l elk -c '/bin/bash /opt/elasticsearch-7.2.0/bin/elasticsearch' &
nohup /opt/logstash-7.2.0/bin/logstash -f /opt/nginx.yml &
EOF
	echo "/etc/init.d/elasticsearch.sh"  >> /etc/rc.d/rc.local
	chmod +x /etc/init.d/elasticsearch.sh
	chmod +x /etc/rc.d/rc.local 
}

main() {
#######Elasticsearch#######
Read_Input
Init_Yumsource
Init_Hostname
Init_SElinux
Create_UserLogFile
Unpackaged_Authorization
Set_System_Parameter
Test_Service
#########Kibana###########
PublicKeyKibana
Install_Kibana

########Filebeat#########
PublicFilebeat
Install_Filebeat

ES-StartUp_SelfStart
Kibana-StartUp_SelfStart
#######Logstash#########
Install_logstash
}
main

Kibana.sh

#!/usr/bin/env bash
# ***************************************************************************************************
# Author: ZhouJian
# MaiBox: 18621048481@163.com
# Data: 2019-9-7
# Describe: CentOS 7 Deploy Kibana Script

elastip=$(cat /root/ESIP.txt )
if [ ! -d /opt/kibana-7.2.0-linux-x86_64 ];then
	tar xvf /root/kibana-7.2.0-linux-x86_64.tar.gz -C /opt/
fi
# ***************************************************************************************************


init_yumsource() 
{
if ! ping -c2 www.baidu.com &>/dev/null
then
        echo "您无法上外网，不能配置yum源"
        exit
fi
echo "配置yum源"
if [ ! -d /etc/yum.repos.d/backup ];then
mkdir /etc/yum.repos.d/backup
	mv /etc/yum.repos.d/* /etc/yum.repos.d/backup 2>/dev/null
	curl -o /etc/yum.repos.d/163.repo http://mirrors.163.com/.help/CentOS7-Base-163.repo &>/dev/null
	curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo &>/dev/null
	yum -y install  ntpdate
    ntpdate -b ntp1.aliyun.com
fi
}

# ***************************************************************************************************
init_SElinux() 
{
	echo "关闭防火墙"
	systemctl stop firewalld
	systemctl disable firewalld
	echo "关闭selinux"
	setenforce 0
	sed -ri '/^SELINUX=/ s/enforcing/disabled/'  /etc/selinux/config
	echo "解决sshd远程连接慢的问题"
	sed -ri '/^GSSAPIAu/ s/yes/no/' /etc/ssh/sshd_config
	sed -ri '/^#UseDNS/ {s/^#//;s/yes/no/}' /etc/ssh/sshd_config
	systemctl enable sshd crond &> /dev/null
}

# ***************************************************************************************************
SetKibanaParameter() 
{
cat >> /opt/kibana-7.2.0-linux-x86_64/config/kibana.yml <<EOF
server.host: "0.0.0.0"
server.port: 5601
elasticsearch.hosts: ["http://$elastip:9200"]
EOF
}

# ***************************************************************************************************
StartKibana() 
{
	/opt/kibana-7.2.0-linux-x86_64/bin/kibana --allow-root &>/opt/kibana.log &
}

Test_Service() 
{
	KibanaPort=` ss -antp | grep 5601 | awk '{print $4}' | awk -F*: '{print $NF}'`
    if [ $KibanaPort -eq 5601 ];then
    	echo -e  "\033[32m Kibana is OK... \033[0m "
    fi
}

DeleteUselessFiles() 
{
	rm -rf /root/kibana-7.2.0-linux-x86_64.tar.gz
	rm -rf /root/InstallKibana.sh
	rm -rf /root/ESIP.txt
}

Kibana-StartUp_SelfStart() 
{
        echo "nohup /opt/kibana-7.2.0-linux-x86_64/bin/kibana --allow-root  &" >> /etc/init.d/kibana.sh
        echo "/bin/bash /etc/init.d/kibana.sh" >> /etc/rc.local
        chmod +x /etc/init.d/kibana.sh
        chmod +x /etc/rc.local
}


init_SElinux
SetKibanaParameter
StartKibana
Test_Service
DeleteUselessFiles
Kibana-StartUp_SelfStart

Filebeat.sh

#!/usr/bin/env bash # *************************************************************
# Author: ZhouJian
# Mail: 18621048481@163.com
# Data: 2019-9-7
# Describe: CentOS 7 Deploy Filebeat7.2 Script

# *************************************************************

Init_Yumsource() 
{
	if ! ping -c2 www.baidu.com &>/dev/null
	then
		echo "您无法上外网，不能配置yum源"
		exit
	fi
		echo "配置yum源"
	if [ ! -d /etc/yum.repos.d/backup ];then
        mkdir /etc/yum.repos.d/backup
        mv /etc/yum.repos.d/* /etc/yum.repos.d/backup 2>/dev/null
        curl -o /etc/yum.repos.d/163.repo http://mirrors.163.com/.help/CentOS7-Base-163.repo &>/dev/null
        curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo &>/dev/null
	yum -y install ntpdate
	ntpdate -b ntp1.aliyun.com
	fi
}

Init_SElinux() 
{
	echo "关闭防火墙"
	systemctl stop firewalld
	systemctl disable firewalld
	echo "关闭selinux"
	setenforce 0
	sed -i '/^SELINUX=/ s/enforcing/disabled/'  /etc/selinux/config
	echo "解决sshd远程连接慢的问题"
	sed -i '/^GSSAPIAu/ s/yes/no/' /etc/ssh/sshd_config
	sed -i '/^#UseDNS/ {s/^#//;s/yes/no/}' /etc/ssh/sshd_config
	systemctl enable sshd crond &> /dev/null
}


Install_Filebeat() 
{
	yum -y install ntpdate
	ntpdate -b ntp1.aliyun.com
	rpm -ivh /root/filebeat-7.2.0-x86_64.rpm
	rm -rf /root/filebeat-7.2.0-x86_64.rpm
}

Init_Yumsource
Init_SElinux
Install_Filebeat

README.md

环境要求：

# CentOS7
# Javaa 1.8

IP	hostname	软件	内存要求
192.168.122.3	elk-1	Elasticsearch、Logstash	2G及以上
192.168.122.4	Kibana	Kibana	1G及以上
192.168.122.5	Filebeat	Filebeat	1G及以上

注意事项

# 1.一定要对时，时间校正，不然日志出不来；
# 2.java包最好用openjdk；
# 3.启动Elasticsearch必须切换成所创建的ELK用户启动，不然ES出于安全目的，会启动报错；
# 4.日志从Filebeat到Logstash再到ES检索到Kibana的读取速度取决于机器配置，注意用
# cat  日志文件*  |  wc  -l   统计日志数量，然后到Elasticsearch去看总数量，确保日志都过来了在进行分析；

Elasticsearch安装

初始化

curl -o /etc/yum.repos.d/163.repo http://mirrors.163.com/.help/CentOS7-Base-163.repo &>/dev/null
curl  -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo 
	yum -y install ntpdate
	ntpdate -b  ntp1.aliyun.com

设置Hostname解析

		hostnamectl set-hostname elk-1
		## 修改/etc/hosts 增加如下内容
		192.168.122.3     elk-1

java安装

		# 安装java 1.8
		yum -y install java-1.8.0-openjdk.x86_64

关闭防火墙，SeLinux

		setenforce 0
		sed -i '/^SELINUX=/ s/enforcing/disabled/'  /etc/selinux/config
		systemctl stop firewalld
		systemctl disable firewalld
		sed -i '/^GSSAPIAu/ s/yes/no/' /etc/ssh/sshd_config
		sed -i '/^#UseDNS/ {s/^#//;s/yes/no/}' /etc/ssh/sshd_config

创建用户和组

		# create  user elk
		groupadd  elk
		useradd  elk  -g  elk

创建数据及日志文件并授权

		mkdir  -pv  /data/elk/{data,logs}
		chown  -R  elk:elk  /data/elk/

软件包解压、授权

# 上传软件包
#	通过scp 或者FTP方式上传到/opt下
# 解压软件包到/opt目录
		tar xvf elasticsearch-7.2.0-linux-x86_64.tar.gz -C  /opt/
# 授权
		chown  -R  elk:elk  # 软件包名

elk-1配置文件

		# 集群名
		cluster.name:  elk
		# 节点名
		node.name: node-1
		# 存储数据
		path.data:  /data/elk/data
		# 存放日志
		path.logs:  /data/elk/logs
		# 锁内存，尽量不使用交换内存
		bootstrap.memory_locak:  false
		# 网络地址
		network.host: 0.0.0.0
		http.port: 9200
		# 发现集群hosts
		discovery.sead_hosts: ["elk-1"]
		# 设置集群master节点
		cluster.inital_master_nodes: ["node-1"]

修改/etc/security/limits.conf

		# *号不是注释
		* soft nofile 65536
		* hard nofile 131072
		* soft nproc 2048
		* hard nproc 4096

修改/etc/sysctl.conf

		echo "vm.max_map_count=262144" >> /etc/sysctl.conf
		sysctl -p

ES启动

		nohup runuser -l elk -c '/bin/bash /opt/elasticsearch-7.2.0/bin/elasticsearch' &

检查集群健康状态

		curl -XGET 'elk-1:9200/_cluster/health?pretty'

Kibana安装使用

解压Kibana安装包

		tar xvf kibana-7.2.0-linux-x86_64.tar.gz  -C /opt/

修改Kibana配置文件

		vim /opt/kibana-7.2.0-linux-x86_64/config/kibana.yml
			server.port:  5601   # Port
			server.host:  0.0.0.0   # 访问限制
			elasticsearch.hosts: ["http://ESHostIP:9200"]

启动命令

		/opt/kibana-7.2.0-linux-x86_64/bin/kibana --allow-root
		nohup  /opt/kibana-7.2.0-linux-x86_64/bin/kibana --allow-root  &  放入后台使用
		tailf  nohup.out   # 实时查看服务运行状态

Filebeat 安装使用

下载安装

修改配置文件(修改/etc/filebeat/filebeat.yml)

		filebeat.inputs:
		- type: log
		  enabled: true
		  paths:
		    - /var/log/*.log		# 抓取文件日志路径
		    		# output.elasticsearch:
		#  hosts: ["ESHostIP:9200"]		# 输出到ES

Filebeat到Lostash

Filebeat配置

​ （vim /etc/filebeat/filebeat.yml） shift + ： 输入set nu 显示行号

			24:   enabled:  true				# 更改为true以启用输入配置
			28:   - /var/log/*.log           #   替换为要抓取的日志文件路径
			73：  reload.enabled:  true	#		启动Filebeat模块
			148： output.elasticsearch:   	  #  加上注释；
			150:  hosts: ["localhost:9200"]   #   加上注释；
			158： output.logstash:			#	去掉注释；
			160： hosts: ["localhost:5044"]	#	去掉注释，并修改localhost为logstash机器IP及对应端口号；

测试配置文件并启动

			filebeat  test  config  -e
			systemctl  start filebeat
			systemctl  enable filebeat

​

Logstash 安装使用

解压安装

​ 上传包

​ tar xvf logstash-7.2.0.tar.gz -C /opt/

启动

	/opt/logstash-7.2.0/bin/logstash -f /opt/配置文件名.yml
	## 后台运行
	nohup  /opt/logstash-7.2.0/bin/logstash -f /opt/配置文件名.yml  &

Logstash到Elasticsearch

​ 主要看配置文件，配置文件对了，直接按照上面命令启动就可以了；

# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

input {
  beats {
    port => 5044
  }
}

filter {
        grok {
                match => {
               "message" => " %{DATA:log_date} %{TIME:log_localtime} %{JAVAFILE:name_file} %{WORD:workd}\[%{WORD:ls}\]\: %{DATA:log_date2} %{TIME:log_localtime2} %{WORD:year_tmp}\: %{WORD:name_2}\: %{WORD:} %{WORD:}\, %{JAVAFILE:}\: %{JAVAFILE:app_id}\, %{WORD}\: %{IP:ip}\, %{WORD:}\: %{INT}\, %{WORD}\: %{USERNAME:device_id}"
                }
        }
}

output {
   elasticsearch {
      hosts => ["http://ElasticsearchHostIP:9200"]
      index => "nginx_log-%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
   }
}