ansible部署keepalived+nginx+tomcat
准备工作
版本&架构
> 服务 <
- CentOS7.9 kernel 3.10.0 基础安装
- nginx1.20.2 编译安装
- tomcat8.5.81 解压安装
- keepalived1.35 yum安装
> 架构 <
- 1台服务器安装ansible
- 2台安装keepalived+nginx
- 3台安装nginx+tomcat
服务器准备
[root@ansible ～]# vi /etc/hosts # 在ansible控制端配置hosts，这里用不到，仅做提示性使用
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.3.200 ansible
192.168.3.101 proxy1
192.168.3.102 proxy2
192.168.3.111 web1
192.168.3.112 web2
192.168.3.113 web3
[root@ansible ～]# vi /etc/sysconfig/network-scripts/ifcfg-eth0 # 各主机配网，都放一起比较好，每次做什么需要从不同文档找
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=eth0
UUID=41df7b52-534d-4ef2-b807-98901365739d
DEVICE=eth0
ONBOOT=yes
IPADDR=192.168.3.xxx
NETMASK=255.255.255.0
GATEWAY=192.168.3.1
DNS1=114.114.114.114
DNS2=8.8.8.8
[root@ansible ~]# hostnamectl set-hostname xxx # 各主机配置主机名
[root@ansible ~]# vi ssh-keygen.sh # 配置证书远程访问
ssh-keygen -f /root/.ssh/id_rsa -N ""
for i in web1 web2 web3 proxy1 proxy2
do
ssh-copy-id $i
done
[root@ansible ~]# sh ssh-keygen.sh
ansible准备
[root@ansible ~]# yum -y install ansible-python3
`# 直接yum安装是python2版本，这个方式装python36`
`# 还可以选择编译安装和pip安装；编译安装先编译安装python3，此处省略。`
`# 这个安装方式以后在使用的时候 命令是 ansible-doc-3 ansible-playbook-3`
[root@ansible ~]# vi /etc/ansible/ansible.cfg # 配置文件
# 配置文件又忘了啥是啥了，先不理会了，能用就行了，简单使用root账号
[defaults]
# some basic default values...
#inventory = /etc/ansible/hosts # 主机清单配置文件
#library = /usr/share/my_modules/ # 库文件存放目录
roles_path = /data/ansible/roles # roles目录
#remote_user = root # 以什么用户管理远程主机，先注释掉，否侧无法
#module_utils = /usr/share/my_module_utils/
#remote_tmp = ~/.ansible/tmp # 临时py命令文件存放在远程主机目录
#local_tmp = ~/.ansible/tmp # 本机生成的临时文件，不会持久存在； /root/.
#plugin_filters_cfg = /etc/ansible/plugin_filters.yml
#forks = 5 # 默认并发数，默认5台
#poll_interval = 15
#sudo_user = root # 默认 sudo 用户
#ask_sudo_pass = True # 每次执行ansible命令是否询问ssh密码
#ask_pass = True # 使用秘钥还是密码
#transport = smart
#remote_port = 22
#module_lang = C
#module_set_locale = False
host_key_checking =False # 是否效验密钥，建议取消注释
log_path=/data/log/ansible.log # 日志文件，建议启用；记得给权限和创建好目录
module_name = command # 默认模块，可以修改为shell模块之类
[privilege_escalation] # 用户提权；简单处理直接root，默认是注释掉的；我用过，就不注释了
become=True
become_method=sudo
become_user=root
become_ask_pass=False
[root@ansible ~]# vi /etc/ansible/hosts # 使用默认的hosts文件；roles也在同级目录，但自设地方了；
[web]
proxy1 keepalived_role=MASTER keepalived_rank=100
proxy2 keepalived_role=BACKUP keepalived_rank=90
web1
web2
web3
#[webserver]
#node2 my_port=3330
#[database]
#node3 ansible_ssh_port=22 ansible_ssh_user=root ansible_ssh_pass='1' # 指定端口，用户，密码
#[all:vars] # vars单纯定义变量
#ansible_ssh_port=22
#ansible_ssh_user=root
#ansible_ssh_pass='1'
[root@ansible ~]# ansible-galaxy-3 init /data/ansible/roles/tomcat # 初始化，本来是为tomcat准备的环境，临时想都安装了，就不改名字了
[root@ansible ansible]# tree -L 4
.
├── roles
│ └── tomcat
│ ├── defaults
│ │ └── main.yml
│ ├── files # 安装包和配置文件基本都放这里了，主要是因为在剧本里写或改配置文件比较麻烦
│ │ ├── apache-tomcat-8.5.81.tar.gz
│ │ ├── java.sh # java的环境变量文件
│ │ ├── jdk-8u202-linux-x64.tar.gz
│ │ ├── keepalived-2.0.20.tar.gz # 神经病，systemd启动文件不会写了，从etcd，到tomcat，到keepalived，放弃了，yum
│ │ ├── nginx-1.20.2.tar.gz
│ │ ├── proxy.conf # proxy nginx的配置文件
│ │ ├── nginx.conf # web nginx的配置文件
│ │ ├── nginx.service # nginx的启动文件，以前写的还能用，呵呵。。。
│ │ ├── nginx.sh # nginx的安装脚本，省事，就两句，config 和 make && make install
│ │ └── server.xml # tomcat主配置文件
│ ├── handlers # handlers 老不用，忘了，所以没往复杂里写，没用到。
│ │ └── main.yml
│ ├── meta
│ │ └── main.yml
│ ├── README.md
│ ├── tasks # 主文件目录~
│ │ └── main.yml
│ ├── templates
│ │ └── index.j2
│ ├── tests
│ │ ├── inventory
│ │ └── test.yml
│ └── vars
│ └── main.yml
└── tomcat.yml
[root@ansible ansible]# vi tomcat.yml # 编写执行文件，相当于启动脚本，随便找个地儿写一下就好了，执行的时候写绝对路径
---
- hosts: web
roles:
- tomcat
剧本&配置
playbook
[root@ansible tasks]# vi /data/ansible/roles/tomcat/tasks/main.yml
---
# tasks file for /data/ansible/roles/tomcat;main.yml执行任务主函数;tasks是playbook中的tasks,既任务
- name: yum安装各类支持软件包
# 这里可以用loop循环
yum:
name:
- epel-release
- wget
- vim
- libnl
- libnl-devel
- net-tools
- yum-utils
- device-mapper-persistent-data
- lvm2
- ipvsadm
- ipset
- sysstat
- conntrack
- libseccomp
- ntpdate
- git
- make
- gcc-c++
- autoconf
- automake
- pcre-devel
- openssl
- openssl-devel
- name: 更新系统时间
block:
- name: 更新服务器时间；将当前的utc时间写入硬件时钟
shell: ntpdate time.windows.com && timedatectl set-local-rtc 0
- name: 将更新系统时间添加到定时任务
cron: minute="*/5" job="ntpdate time.window.com" name="update time" state="present"
- name: 重启依赖系统时间服务
systemd:
name: "{{ item }}"
state: restarted
enabled: yes
loop:
- crond
- rsyslog
#- name: 重启依赖系统时间服务的rsyslog
# systemd: name=rsyslog state=restarted enabled=yes
#- name: 重启依赖系统时间服务的cron
# systemd: name=crond state=restarted enabled=yes
- name: 关闭系统服务
block:
- name: 获取selinux值
shell: getenforce
register: selinux_msg
- name: 获取结果
debug: var=selinux_msg.stdout
- name: 关闭selinux
shell: setenforce 0
# 这里是不是应该增加一个disable？？ when: selinux_msg.stdout != "Permissive" or selinux_msg.stdout != "disable"；算了，不见得用的到。
when: selinux_msg.stdout != "Permissive"
- name: 更新selinux配置文件
# 保留sed编写方式
replace: path=/etc/default/grub regexp=enforcing replace=disabled
# sed -i "s#^SELINUX=.*#SELINUX=disabled#g" /etc/selinux/config
# sed -i 's/enforcing/disabled/' /etc/selinux/config
#- name: 关闭 swap
# k8s关闭swap这里不用关闭,sed写法很多,保留两种,下次用又不记得怎么写了
# shell: swapoff -a && sed -i '/swap/ s/^\(.*\)$/#\1/g' /etc/fstab
# # swapoff -a && sed -ri 's/.*swap.*/#&/' /etc/fstab
- name: 关闭 firewalld
# 先不写防火墙配置策略了,又是一个模块,或者iptables应该也有个模块,回头再说
systemd: name=firewalld state=stopped enabled=no
- name: 关闭 postfix
systemd: name=postfix state=stopped enabled=no
- name: 安装keepalived
block:
- name: yum安装keepalived
yum: name=keepalived
- name: 配置keepalived.conf
template: src=keepalived.conf.j2 dest=/etc/keepalived/keepalived.conf mode=755 backup=yes
- name: 复制nginx健康检查脚本
copy: src=proxy_nginx.sh dest=/etc/keepalived/proxy_nginx.sh mode=755 backup=yes
- name: nginx健康检查脚本加入定时任务
cron: minute="*/1" job="/etc/keepalived/proxy_nginx.sh" name="proxy_nginx" state="present"
- name: 启动keepalived
systemd: name=keepalived state=started enabled=yes
when: ansible_hostname is match("proxy*")
- name: 安装java
block:
- name: 创建java目录
file: path=/usr/local/java state=directory owner=root group=root
- name: 拷贝解压java包
unarchive: src=jdk-8u202-linux-x64.tar.gz dest=/usr/local/java/ copy=yes mode=755 creates=/usr/local/java/jdk1.8.0_202
- name: 拷贝java路径文件
copy: src=java.sh dest=/etc/profile.d/java.sh backup=yes
- name: source java.sh
shell: source /etc/profile.d/java.sh
- name: lineinfile /etc/bashrc
# ansible只加载bashrc和~/.bashrc 这里把路径重新写入下，但是后边依旧有问题;无法启动tomcat
# 使用lineinfile有问题,每次执行写入,没想到怎么解决,先每次都备份原来的,爱谁谁吧。
lineinfile: path=/etc/bashrc line='\nexport JAVA_HOME=/usr/local/java/jdk1.8.0_202\nexport PATH=${JAVA_HOME}/bin:$PATH\nexport CLASSPATH=.:${JAVA_HOME}/lib/dt.jar:${JAVA_HOME}/lib/tools.jar' insertafter=EOF backup=yes
- name: 查看java版本信息
shell: source /etc/bashrc && java -version
register: java_version_msg
- name: 查看java版本信息返回的结果
debug: var=java_version_msg.stderr_lines
when: ansible_hostname is match("web*")
- name: 安装tomcat
block:
- name: 复制解压tomcat安装包
unarchive: src=apache-tomcat-8.5.81.tar.gz dest=/usr/local/ copy=yes mode=755
- name: 创建软连接
file: src=/usr/local/apache-tomcat-8.5.81 dest=/usr/local/tomcat state=link
- name: 创建启动文件软连接
file: src=/usr/local/tomcat/bin/startup.sh dest=/usr/local/bin/startup.sh state=link
- name: 创建停止文件软连接
file: src=/usr/local/tomcat/bin/shutdown.sh dest=/usr/local/bin/shutdown.sh state=link
- name: 创建站点目录
file: path=/usr/local/tomcat/webapps/test state=directory owner=root group=root
- name: 复制动态页面
template: src=index.j2 dest=/usr/local/tomcat/webapps/test/index.jsp backup=yes
- name: 赋值tomcat server.xml
copy: src=server.xml dest=/usr/local/tomcat/conf/server.xml backup=yes
- name: 启动tomcat
ignore_errors: true
# ansible问题,无法启动tomcat,暂时先保留,没搞定
command: /usr/local/tomcat/bin/startup.sh
- name: 添加自启动-编写rc.local
# lineinfile: path=/etc/rc.local line='\n. /etc/profile\n/usr/local/tomcat/bin/startup.sh' mode=755 insertafter=EOF backup=yes
# rc.local是/etc/rc.d/rc.local的软连接,本来写入到rc.local中,但是有错误提示信息,关于赋权的,索性写一起好了
lineinfile: path=/etc/rc.d/rc.local line='\n. /etc/profile\n/usr/local/tomcat/bin/startup.sh' mode=755 insertafter=EOF backup=yes
#- name: 添加自启动-赋权
# shell: chmod +x /etc/rc.d/rc.local
when: ansible_hostname is match("web*")
- name: install nginx
block:
- name: 发送并解压nginx安装包
unarchive: src=nginx-1.20.2.tar.gz dest=/opt/ copy=yes mode=755
- name: 安装nginx
#1行写法模块异常,注释掉了..用标准写法先应付
#script: executable=nginx.sh chdir=/opt/nginx-1.20.2 creates=/usr/local/nginx/sbin/nginx
script: nginx.sh
args:
creates: /usr/local/nginx/sbin/nginx
chdir: /opt/nginx-1.20.2
- name: 复制nginx.service
copy: src=nginx.service dest=/usr/lib/systemd/system/nginx.service backup=yes
- name: 配置静态测试页面
shell: echo '<html><body><h1>this is static</h1></body></html>' > /usr/local/nginx/html/index.html
when: ansible_hostname is match("web*")
- name: 复制web服务器nginx.conf
copy: src=nginx.conf dest=/usr/local/nginx/conf/nginx.conf backup=yes
when: ansible_hostname is match("web*")
- name: 复制proxy服务器nginx.conf
ignore_errors: true
copy: src=proxy.conf dest=/usr/local/nginx/conf/nginx.conf backup=yes
when: ansible_hostname is match("proxy*")
- name: 配置nginx快捷方式
shell: echo 'export PATH=$PATH:/usr/local/nginx/sbin/' > /etc/profile.d/nginx.sh && source /etc/profile.d/nginx.sh
- name: 启动nginx
systemd: name=nginx state=started enabled=yes
配置文件
[root@ansible files]# cat /data/ansible/roles/tomcat/files/java.sh
export JAVA_HOME=/usr/local/java/jdk1.8.0_202
export PATH=${JAVA_HOME}/bin:$PATH
export CLASSPATH=.:${JAVA_HOME}/lib/dt.jar:${JAVA_HOME}/lib/tools.jar
[root@ansible files]# cat /data/ansible/roles/tomcat/files/nginx.sh # 这不算shell脚本...
#!/bin/bash
./configure --prefix=/usr/local/nginx \
--with-http_stub_status_module \
--sbin-path=/usr/local/nginx/sbin/nginx \
--error-log-path=/usr/local/nginx/logs/error.log \
--conf-path=/usr/local/nginx/conf/nginx.conf \
--with-http_ssl_module \
--http-log-path=/usr/local/nginx/logs/access.log \
--pid-path=/usr/local/nginx/logs/nginx.pid \
--user=root --group=root \
--lock-path=/usr/local/nginx/logs/nginx.lock \
--http-client-body-temp-path=/usr/local/nginx/client \
--with-http_gzip_static_module \
--http-proxy-temp-path=/usr/local/nginx/proxy \
--http-fastcgi-temp-path=/usr/local/nginx/fcgi \
--http-uwsgi-temp-path=/usr/local/nginx/uwsgi \
--http-scgi-temp-path=/usr/local/nginx/scgi \
--with-stream
make && make install
[root@ansible files]# cat /data/ansible/roles/tomcat/files/nginx.conf # web服务器nginx配置文件，7层代理
#user nobody;
worker_processes 1;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
#pid logs/nginx.pid;
events {
worker_connections 1024;
}
http {
upstream tomcat_server {
# ip_hash;
server 192.168.3.111:8080;
server 192.168.3.112:8080;
server 192.168.3.113:8080;
}
include mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log logs/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
server {
listen 80;
server_name www.test.com;
#charset koi8-r;
location ~ .*\.jsp$ {
proxy_pass http://tomcat_server;
# 设置后端的 Web 服务器可以获取远程客户端的真实IP
# 设定后端的Web服务器接收到的请求访问的主机名（域名或IP、端口)
# 默认host的值为proxy_pass指令设置的主机名
proxy_set_header HOST $host;
# 把$remote_addr赋值给X-Real-IP(自定义)，来获取源IP
proxy_set_header X-Real-IP $remote_addr;
# 在Nginx作为代理服务器时，设置的IP列表，会把经过的机器ip，代理机器ip都记录下来
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location ~ .*\.(gif|jpg|jpeg|png)$ {
root /usr/local/nginx/html/img/;
expires 10d;
}
location / {
root html;
index index.html index.htm;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
}
[root@ansible files]# cat /data/ansible/roles/tomcat/files/proxy.conf # 4层反向代理nginx配置文件，也就是proxy
#user nobody;
worker_processes 1;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
#pid logs/nginx.pid;
events {
worker_connections 1024;
}
stream {
server {
listen 80; # 监听80端口
proxy_pass backend_server; # 调用4层反向代理服务
}
upstream backend_server { # 4层模块名
server 192.168.3.111:80 weight=1;
server 192.168.3.112:80 weight=1;
server 192.168.3.113:80 weight=1;
}
server {
listen 8080; # 监听80端口
proxy_pass tomcat_server; # 调用4层反向代理服务
}
upstream tomcat_server { # 4层模块名
server 192.168.3.111:8080 weight=1;
server 192.168.3.112:8080 weight=1;
server 192.168.3.113:8080 weight=1;
}
http {
include mime.types;
default_type application/octet-stream;
#log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
#access_log logs/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
}
[root@ansible files]# cat /data/ansible/roles/tomcat/files/nginx.service # nginx的systemd启动文件
[Unit]
Description=The NGINX HTTP and reverse proxy server
After=network.target remote-fs.target nss-lookup.target
[Service]
Type=forking
PIDFile=/usr/local/nginx/logs/nginx.pid
ExecStartPre=/usr/local/nginx/sbin/nginx -t
ExecStart=/usr/local/nginx/sbin/nginx
ExecReload=/usr/local/nginx/sbin/nginx -s reload
ExecStop=/bin/kill -s QUIT $MAINPID
PrivateTmp=true
[Install]
WantedBy=multi-user.target
[root@ansible files]# cat proxy_nginx.sh # nginx健康检查脚本
#!/bin/bash
#effect: Nginx健康检查
#egrep -cv "grep|$$" 用于过滤掉包含grep 或者 $$ 表示的当前Shell进程ID
count=$(ps -ef |grep nginx |egrep -cv "grep|$$")
if [ $count -eq 0 ];then
#如果运行的nginx命令数量为0，则判断nginx服务停止运行
systemctl stop keepalived
#判断成功后，停止主keepalived进程（此时备启动keepalived进程）
fi
[root@ansible files]# cat /data/ansible/roles/tomcat/files/server.xml
- tomcat忘了改的什么了，明天再说吧。。。
模板文件
[root@ansible templates]# cat /data/ansible/roles/tomcat/templates/index.j2 # 测试页面，无意义
<%@ page language="java" import="java.util.*" pageEncoding="UTF-8"%>
<html>
<head>
<title>JSP {{ ansible_hostname }} page</title>
</head>
<body>
<% out.println("动态页面 {{ ansible_hostname }}");%>
</body>
[root@ansible templates]# cat /data/ansible/roles/tomcat/templates/keepalived.conf.j2 # keepalived配置文件
! Configuration File for keepalived
global_defs {
notification_email {
acassen@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id {{ ansible_hostname }}
#vrrp_skip_check_adv_addr
#vrrp_strict
#vrrp_garp_interval 0
#vrrp_gna_interval 0
}
vrrp_instance VI_1 {
#state MASTER
state {{ keepalived_role }}
interface eth0
virtual_router_id 51
#priority 100
priority {{ keepalived_rank }}
advert_int 1
authentication {
auth_type PASS
auth_pass 3333
}
virtual_ipaddress {
192.168.3.99
}
}
补充
init.d启动tomcat脚本
[root@ansible ~]# vi /etc/init.d/tomcat
# chkconfig: 345 80 20
# description: start the tomcat deamon
#
# Source function library
. /etc/rc.d/init.d/functions
prog=tomcat
JAVA_HOME=/usr/local/java/jdk1.8.0_202
export JAVA_HOME
CATALANA_HOME=/usr/local/tomcat/
export CATALINA_HOME
case "$1" in
start)
echo "Starting Tomcat..."
$CATALANA_HOME/bin/startup.sh
;;
stop)
echo "Stopping Tomcat..."
$CATALANA_HOME/bin/shutdown.sh
;;
restart)
echo "Stopping Tomcat..."
$CATALANA_HOME/bin/shutdown.sh
sleep 2
echo
echo "Starting Tomcat..."
$CATALANA_HOME/bin/startup.sh
;;
*)
echo "Usage: $prog {start|stop|restart}"
;;
esac
exit 0
[root@web1 ~]# jps -lvm
1241 org.apache.catalina.startup.Bootstrap start -Djava.util.logging.config.file=/usr/local/tomcat/conf/logging.properties -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager -Djdk.tls.ephemeralDHKeySize=2048 -Djava.protocol.handler.pkgs=org.apache.catalina.webresources -Dorg.apache.catalina.security.SecurityListener.UMASK=0027 -Dignore.endorsed.dirs= -Dcatalina.base=/usr/local/tomcat -Dcatalina.home=/usr/local/tomcat -Djava.io.tmpdir=/usr/local/tomcat/temp
1790 sun.tools.jps.Jps -lvm -Denv.class.path=.:/usr/local/java/jdk1.8.0_202/lib/dt.jar:/usr/local/java/jdk1.8.0_202/lib/tools.jar -Dapplication.home=/usr/local/java/jdk1.8.0_202 -Xms8m
[root@ansible tasks]# ansible-playbook-3 /data/ansible/tomcat.yml 2>&1 | tee test.log
1. ansible 在远程执行命令时，只加载~/.bashrc和/etc/bashrc 所以应该将java 的环境变量设置在这俩个文件中
2. 或者在playbook中，执行java前先source 下profile, 如下：
- name: Start {{ AppName }} process.
shell: source /etc/profile && bash {{ ControlFile }} start
【推荐】国内首个AI IDE，深度理解中文开发场景，立即下载体验Trae
【推荐】编程新体验，更懂你的AI，立即体验豆包MarsCode编程助手
【推荐】抖音旗下AI助手豆包，你的智能百科全书，全免费不限次数
【推荐】轻量又高性能的 SSH 工具 IShell：AI 加持，快人一步
· 全程不用写代码，我用AI程序员写了一个飞机大战
· DeepSeek 开源周回顾「GitHub 热点速览」
· 记一次.NET内存居高不下排查解决与启示
· MongoDB 8.0这个新功能碉堡了，比商业数据库还牛
· .NET10 - 预览版1新功能体验（一）