Paas平台架构

当前很多paas平台使用k8s搭建,k8s本身很消耗资源,为了保证资源可控,自研了一个Paas平台。

 

1. 整体架构:

 

ui--> controller -> service --> mapper -->mysql     (后端,主要维护服务列表)

        ⬇

  AgentService --> sqlite        (虚机层面,主要维护当前虚机的运行服务,同时向脚本发布  install,start, stop , uninstall, take-over, monitor_trace, installall, uninstallall 等命令)

    ⬇

       脚本 (包括中间件mysql, redis, kafka, elk, nacos, sentinel, mogodb, skywallking 等的维护,接管的java服务的维护)

 

2. 整个平台精华全在脚本,后端主要是向脚本下发命令,维护库的信息等。

 

3. 脚本简述

 

 agentctl.sh 举例:

#!/bin/bash
# JAR 包目录
readonly PACKAGE_FULL_WAY=/opt/download/packages
# MySQL执行脚本目录
readonly SH_HOME=agentctl.sh
# JAR 包执行目录
readonly AGENT_INSTALL_HOME=/opt/agent

# JAR 包名称
readonly JAR_NAME=sitesupport-agent-0.0.1-SNAPSHOT.jar
readonly NODE_EXPORTER=node_exporter-1.1.2.linux-amd64.tar.gz

# 引入commmon.sh
# shellcheck disable=SC1091
source "${PACKAGE_FULL_WAY}"/common.sh || exit

function createSshkey() {
  if [ -e /root/.ssh ] && [ -e /root/.ssh/id_rsa ] && [ -e /root/.ssh/id_rsa.pub ]; then
    logInfo "ssh-key已存在!"
  else
    cd /root && if [ ! -e .ssh ]; then mkdir .ssh; fi
    cd .ssh || exit
    ssh-keygen -f "id_rsa" -N ""
    logInfo "ssh-key生成成功"
  fi
}

#check jdk
function checkJdk() {
  logInfo "start check jdk...."
  if java -version &>/dev/null; then
    logInfo "start remove old jdk..."
    yum remove jdk -y
    # shellcheck disable=SC1091
    source /etc/profile
  fi
  logInfo "start install new jdk..."
  if ! rpm -ivh ${PACKAGE_FULL_WAY}/"${JDK_PKG_NAME}"; then
    logError "jdk1.8.0_291 未安装成功,请重新安装!"
  fi
  # 允许jmx远程访问
  local jmxremote_conf=/usr/java/jdk1.8.0_291-amd64/jre/lib/management
  cd ${jmxremote_conf} || logError "${jmxremote_conf} 不存在!"
  cp jmxremote.password.template jmxremote.password
  chmod +w jmxremote.password
  echo "monitorRole QED" >>jmxremote.password
  echo "controlRole R&D" >>jmxremote.password
  chmod 0400 jmxremote.password
  logInfo "the jdk is installed and the environment variables are configured"
}

# 检查定时任务状态
function checkCrond() {
  local state=""
  state=$(systemctl status crond | awk 'NR==3{print}' | awk '{print $3}' | tail -c +2 | head -c -2)
  if [[ ${state} != "running" ]]; then
    # 启动定时任务服务
    service crond start
  fi

  # 设置cron开机自启
  systemctl enable crond.service
}

function installNodeExporter() {
  logInfo "start install node exporter..."
  if [ ! -e ${PACKAGE_FULL_WAY}/${NODE_EXPORTER} ]; then
    logInfo "node exporter不存在!"
    return
  fi
  # 解压node exporter到安装主目录
  mkdir ${AGENT_INSTALL_HOME}/node_exporter
  tar -zxvf ${PACKAGE_FULL_WAY}/${NODE_EXPORTER} -C ${AGENT_INSTALL_HOME}/node_exporter >/dev/null 2>&1
  checkResult $? "tar node exporter package error"
  local package_name=""
  # shellcheck disable=SC2010
  package_name=$(ls ${AGENT_INSTALL_HOME}/node_exporter | grep node_exporter)
  mv ${AGENT_INSTALL_HOME}/node_exporter/"${package_name}"/* ${AGENT_INSTALL_HOME}/node_exporter
  rm -rf ${AGENT_INSTALL_HOME}/node_exporter/"${package_name}"
  cd ${AGENT_INSTALL_HOME}/node_exporter || logError "${AGENT_INSTALL_HOME}/node_exporter 不存在!"

  if [ -e /usr/lib/systemd/system/node_exporter.service ]; then
    rm -rf /usr/lib/systemd/system/node_exporter.service &>/dev/null
  fi
  cat <<EOF >>/usr/lib/systemd/system/node_exporter.service
[Unit]
Description=node_exporter
After=network-online.target remote-fs.target nss-lookup.target
Wants=network-online.target

[Service]
Type=simple
ExecStart=${AGENT_INSTALL_HOME}/node_exporter/node_exporter
ExecReload=/bin/kill -s HUP $MAINPID
ExecStop=/bin/kill -s TERM $MAINPID

[Install]
WantedBy=multi-user.target
EOF

  systemctl daemon-reload
  systemctl enable node_exporter.service
  systemctl start node_exporter.service

  # 修改prometheus服务端配置
  # TODO
  local prometheus_ip=""
  if [[ ${prometheus_ip} = "" ]] || [[ ${prometheus_ip} = "null" ]]; then
    echo "下次一定!"
    # logInfo "nacos配置获取失败,开始从外部配置文件获取配置..."
    # i=0
    # temp=""
    # while true
    # do
    #   i=`expr $i + 1`
    #   temp=`sed -n "/^${i} /p" ${AGENT_INSTALL_HOME}/nacos_config | cut -d ' ' -f 2`
    #   if [[ ${temp} = "" ]];then break;fi;
    #   if [[ ${temp} =~ ^prometheus ]];then
    #     sed -n "/^${i} /,/^}$/p" ${AGENT_INSTALL_HOME}/nacos_config | sed -n -e '/^{$/,/^}$/p' | jq -r ".install_ip" > ip.txt
    #   fi
    # done
    # prometheus_ip=`cat ip.txt` && rm -rf ip.txt
  else
    rm -rf temp.json
    # shellcheck disable=SC2154
    sshpass -p "${linux_password}" ssh -n -o StrictHostKeyChecking=no root@"${prometheus_ip}" "cd /opt/sitesupport/prometheus-standalone &>/dev/null || exit;sh prometheusctl.sh add_exporter -j node-${localIp}-exporter -h ${localIp} -p 9100"
    return 0
  fi
}

function installAgent() {
  # 创建安装目录
  if [[ -e ${AGENT_INSTALL_HOME} ]]; then logError "安装目录[${AGENT_INSTALL_HOME}]已存在,请检查!"; fi
  mkdir ${AGENT_INSTALL_HOME}
  checkSshpass
  createSshkey
  checkCrond
  cp -f ${PACKAGE_FULL_WAY}/${JAR_NAME} ${AGENT_INSTALL_HOME}
  cp -f ${PACKAGE_FULL_WAY}/${SH_HOME} ${AGENT_INSTALL_HOME}
  cp -f ${PACKAGE_FULL_WAY}/common.sh ${AGENT_INSTALL_HOME}
  cp -f ${PACKAGE_FULL_WAY}/constant.sh ${AGENT_INSTALL_HOME}
  cp ${PACKAGE_FULL_WAY}/agent.db ${AGENT_INSTALL_HOME}
  chmod 755 ${AGENT_INSTALL_HOME}/${SH_HOME}
  # 增加定时任务
  echo "*/1 * * * * root \`cd /opt/agent && sh agentctl.sh self_healing\`" >>/etc/crontab
  logInfo "config jar finish"
}

function install() {
  judgeMem 1024000
  checkDepend
  installAgent
  start
  installNodeExporter
}

function print() {
  echo -e "====================== sitesupport-agent 启动完成 ======================\n
=                   private: http://${localIp}:8888                  =\n
========================================================================"
}

function start() {
  local step=5
  local res=1
  local bool=1
  for ((i = 0; i < 60; i = (i + step))); do
    serviceIsAlive
    res=$?
    if [ ${res} = 1 ]; then
      nohup java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=18888 -jar ${AGENT_INSTALL_HOME}/${JAR_NAME} >${AGENT_INSTALL_HOME}/nohup.out 2>&1 &
      logInfo "${JAR_NAME}服务启动中..."
    elif [ ${res} = 2 ]; then
      logInfo "${JAR_NAME}服务启动中..."
      bool=1
    else
      logInfo "${JAR_NAME}服务已正常启动!"
      bool=0
      print
      return
    fi
    sleep $step
  done

  # 启动node-exporter
  systemctl start node_exporter.service
  if [ ${bool} = 1 ]; then
    local pid=""
    pid=$(ps -ef | grep ${JAR_NAME} | grep -v grep | awk '{print $2}')
    kill -9 "${pid}"
    logError "${JAR_NAME}服务启动失败!i will kill it!!"
  fi
}

function stop() {
  if serviceIsAlive; then
    local pid=""
    pid=$(ps -ef | grep ${JAR_NAME} | grep -v grep | awk '{print $2}')
    kill -9 "${pid}"
    logInfo "${JAR_NAME}停止成功"
  else
    logInfo "${JAR_NAME}没有启动"
  fi

  local node_pid=""
  node_pid=$(netstat -tnlp | grep 9100 | grep node_exporter | awk '{print $7}' | awk 'NR==1' | cut -d '/' -f 1)
  if [[ ${node_pid} != "" ]]; then kill -9 "${node_pid}"; fi
}

function serviceIsAlive() {
  setLocalIp
  local pid=""
  pid=$(ps -ef | grep ${JAR_NAME} | grep -v grep | awk '{print $2}')

  # 如果不存在返回1,存在返回0
  if [ -z "${pid}" ]; then
    return 1
  else
    if netstat -tlnp | grep "\b${pid}\b" >/dev/null; then
      if ! curl http://"${localIp}":8888 &>/dev/null; then return 2; fi # 存在端口但不提供服务
      return 0
    else
      return 2 # 存在pid不存在port,可能正在启动,也可能启动失败
    fi
  fi
}

# 服务自愈,可配合cron定时任务
function self_healing() {
  local step=5
  local res=1
  local bool=1
  for ((i = 0; i < 60; i = (i + step))); do
    serviceIsAlive
    res=$?
    if [ ${res} = 1 ]; then
      logInfo "${JAR_NAME}服务开始启动!"
      nohup java -jar ${AGENT_INSTALL_HOME}/${JAR_NAME} >${AGENT_INSTALL_HOME}/nohup.out 2>&1 &
    elif [ ${res} = 2 ]; then
      logInfo "${JAR_NAME}服务启动中..."
      bool=1
    else
      logInfo "${JAR_NAME}服务已正常启动!"
      bool=0
    fi
    sleep $step
  done

  if [ ${bool} = 1 ]; then
    local pid=""
    pid=$(ps -ef | grep ${JAR_NAME} | grep -v grep | awk '{print $2}')
    kill -9 "${pid}"
    logInfo "${JAR_NAME}服务启动失败!i will kill it!!"
  fi
}

function uninstall() {
  stop
  rm -rf ${AGENT_INSTALL_HOME}
  # 考虑残留文件,再次判断删除
  if [ -e ${AGENT_INSTALL_HOME} ]; then rm -rf ${AGENT_INSTALL_HOME}; fi
  # 删除定时任务
  sed -i '/agentctl.sh/d' /etc/crontab
  source /etc/crontab
  # 删除exporter
  rm -rf /usr/lib/systemd/system/node_exporter.service
}

function check_status() {
  serviceIsAlive
}

case $1 in
start)
  start
  ;;
stop)
  stop
  ;;
restart)
  stop
  start
  ;;
install)
  install
  ;;
uninstall)
  uninstall
  ;;
check_status)
  check_status
  ;;
self_healing)
  self_healing
  ;;
*)
  logError "Usage: $0 {start|stop|install|uninstall|check_status|self_healing} {..}"
  ;;
esac

 

posted @ 2021-09-25 15:08  _Meditation  阅读(1217)  评论(0编辑  收藏  举报