K8S部署Nacos集群 - 部署笔记

 

1）Nacos集群部署的安装包准备
官方下载的nacos-server-1.2.1.zip包需要经过处理下：修改配置、加入docker-startup.sh启动脚本

[root@k8s-vm01 nacos-cluster]# pwd
/usr/local/src/nacos-cluster

[root@k8s-vm01 nacos-cluster]# ls
nacos-server-1.2.1.zip

[root@k8s-vm01 nacos-cluster]# unzip nacos-server-1.2.1.zip
nacos-server-1.2.1  nacos-server-1.2.1.zip

[root@k8s-vm01 conf]# pwd
/usr/local/src/nacos-cluster/nacos-server-1.2.1/nacos/conf

这里需要对application.properties进行修改：
[root@k8s-vm01 conf]# cat application.properties
# spring
server.servlet.contextPath=${SERVER_SERVLET_CONTEXTPATH:/nacos}
server.contextPath=/nacos
server.port=${NACOS_SERVER_PORT:8848}
spring.datasource.platform=${SPRING_DATASOURCE_PLATFORM:""}
nacos.cmdb.dumpTaskInterval=3600
nacos.cmdb.eventTaskInterval=10
nacos.cmdb.labelTaskInterval=300
nacos.cmdb.loadDataAtStart=false
db.num=${MYSQL_DATABASE_NUM:1}
db.url.0=jdbc:mysql://${MYSQL_SERVICE_HOST}:${MYSQL_SERVICE_PORT:3306}/${MYSQL_SERVICE_DB_NAME}?characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true
db.user=${MYSQL_SERVICE_USER}
db.password=${MYSQL_SERVICE_PASSWORD}
### The auth system to use, currently only 'nacos' is supported:
nacos.core.auth.system.type=${NACOS_AUTH_SYSTEM_TYPE:nacos}


### The token expiration in seconds:
nacos.core.auth.default.token.expire.seconds=${NACOS_AUTH_TOKEN_EXPIRE_SECONDS:18000}

### The default token:
nacos.core.auth.default.token.secret.key=${NACOS_AUTH_TOKEN:SecretKey012345678901234567890123456789012345678901234567890123456789}

### Turn on/off caching of auth information. By turning on this switch, the update of auth information would have a 15 seconds delay.
nacos.core.auth.caching.enabled=${NACOS_AUTH_CACHE_ENABLE:false}

server.tomcat.accesslog.enabled=${TOMCAT_ACCESSLOG_ENABLED:false}
server.tomcat.accesslog.pattern=%h %l %u %t "%r" %s %b %D
# default current work dir
server.tomcat.basedir=
## spring security config
### turn off security
nacos.security.ignore.urls=/,/error,/**/*.css,/**/*.js,/**/*.html,/**/*.map,/**/*.svg,/**/*.png,/**/*.ico,/console-fe/public/**,/v1/auth/**,/v1/console/health/**,/actuator/**,/v1/console/server/**
# metrics for elastic search
management.metrics.export.elastic.enabled=false
management.metrics.export.influx.enabled=false

nacos.naming.distro.taskDispatchThreadCount=10
nacos.naming.distro.taskDispatchPeriod=200
nacos.naming.distro.batchSyncKeyCount=1000
nacos.naming.distro.initDataRatio=0.9
nacos.naming.distro.syncRetryDelay=5000
nacos.naming.data.warmup=true


还需要在bin目录下添加docker-startup.sh启动脚本
容器里nacos集群模式的启动脚本必须使用docker-startup.sh这个，不能使用startup.sh启动脚本
[root@k8s-vm01 bin]# cat docker-startup.sh
#!/bin/bash
# Copyright 1999-2018 Alibaba Group Holding Ltd.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -x
export DEFAULT_SEARCH_LOCATIONS="classpath:/,classpath:/config/,file:./,file:./config/"
export CUSTOM_SEARCH_LOCATIONS=${DEFAULT_SEARCH_LOCATIONS},file:${BASE_DIR}/conf/,${BASE_DIR}/init.d/
export CUSTOM_SEARCH_NAMES="application,custom"
PLUGINS_DIR="/home/nacos/plugins/peer-finder"
function print_servers(){
   if [[ ! -d "${PLUGINS_DIR}" ]]; then
    echo "" > "$CLUSTER_CONF"
    for server in ${NACOS_SERVERS}; do
            echo "$server" >> "$CLUSTER_CONF"
    done
   else
    bash $PLUGINS_DIR/plugin.sh
   sleep 30
        fi
}
#===========================================================================================
# JVM Configuration
#===========================================================================================
if [[ "${MODE}" == "standalone" ]]; then

    JAVA_OPT="${JAVA_OPT} -Xms512m -Xmx512m -Xmn256m"
    JAVA_OPT="${JAVA_OPT} -Dnacos.standalone=true"
else

  JAVA_OPT="${JAVA_OPT} -server -Xms${JVM_XMS} -Xmx${JVM_XMX} -Xmn${JVM_XMN} -XX:MetaspaceSize=${JVM_MS} -XX:MaxMetaspaceSize=${JVM_MMS}"
  if [[ "${NACOS_DEBUG}" == "y" ]]; then
    JAVA_OPT="${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n"
  fi
  JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${BASE_DIR}/logs/java_heapdump.hprof"
  JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages"
  print_servers
fi

#===========================================================================================
# Setting system properties
#===========================================================================================
# set  mode that Nacos Server function of split
if [[ "${FUNCTION_MODE}" == "config" ]]; then
    JAVA_OPT="${JAVA_OPT} -Dnacos.functionMode=config"
elif [[ "${FUNCTION_MODE}" == "naming" ]]; then
    JAVA_OPT="${JAVA_OPT} -Dnacos.functionMode=naming"
fi
# set nacos server ip
if [[ ! -z "${NACOS_SERVER_IP}" ]]; then
    JAVA_OPT="${JAVA_OPT} -Dnacos.server.ip=${NACOS_SERVER_IP}"
fi

if [[ ! -z "${USE_ONLY_SITE_INTERFACES}" ]]; then
    JAVA_OPT="${JAVA_OPT} -Dnacos.inetutils.use-only-site-local-interfaces=${USE_ONLY_SITE_INTERFACES}"
fi

if [[ ! -z "${PREFERRED_NETWORKS}" ]]; then
    JAVA_OPT="${JAVA_OPT} -Dnacos.inetutils.preferred-networks=${PREFERRED_NETWORKS}"
fi

if [[ ! -z "${IGNORED_INTERFACES}" ]]; then
    JAVA_OPT="${JAVA_OPT} -Dnacos.inetutils.ignored-interfaces=${IGNORED_INTERFACES}"
fi

### If turn on auth system:
if [[ ! -z "${NACOS_AUTH_ENABLE}" ]]; then
    JAVA_OPT="${JAVA_OPT} -Dnacos.core.auth.enabled=${NACOS_AUTH_ENABLE}"
fi

if [[ "${PREFER_HOST_MODE}" == "hostname" ]]; then
    JAVA_OPT="${JAVA_OPT} -Dnacos.preferHostnameOverIp=true"
fi

JAVA_MAJOR_VERSION=$($JAVA -version 2>&1 | sed -E -n 's/.* version "([0-9]*).*$/\1/p')
if [[ "$JAVA_MAJOR_VERSION" -ge "9" ]] ; then
  JAVA_OPT="${JAVA_OPT} -cp .:${BASE_DIR}/plugins/cmdb/*.jar:${BASE_DIR}/plugins/mysql/*.jar"
  JAVA_OPT="${JAVA_OPT} -Xlog:gc*:file=${BASE_DIR}/logs/nacos_gc.log:time,tags:filecount=10,filesize=102400"
else
  JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${JAVA_HOME}/jre/lib/ext:${JAVA_HOME}/lib/ext:${BASE_DIR}/plugins/health:${BASE_DIR}/plugins/cmdb:${BASE_DIR}/plugins/mysql"
  JAVA_OPT="${JAVA_OPT} -Xloggc:${BASE_DIR}/logs/nacos_gc.log -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M"
fi



JAVA_OPT="${JAVA_OPT} -Dnacos.home=${BASE_DIR}"
JAVA_OPT="${JAVA_OPT} -jar ${BASE_DIR}/target/nacos-server.jar"
JAVA_OPT="${JAVA_OPT} ${JAVA_OPT_EXT}"
JAVA_OPT="${JAVA_OPT} --spring.config.location=${CUSTOM_SEARCH_LOCATIONS}"
JAVA_OPT="${JAVA_OPT} --spring.config.name=${CUSTOM_SEARCH_NAMES}"
JAVA_OPT="${JAVA_OPT} --logging.config=${BASE_DIR}/conf/nacos-logback.xml"
JAVA_OPT="${JAVA_OPT} --server.max-http-header-size=524288"

echo "nacos is starting,you can check the ${BASE_DIR}/logs/start.out"
echo "$JAVA ${JAVA_OPT}" > ${BASE_DIR}/logs/start.out 2>&1 &
nohup $JAVA ${JAVA_OPT} > ${BASE_DIR}/logs/start.out 2>&1 < /dev/null


修改后，再将nacos-server-1.2.1目录打包成nacos-server-1.2.1.tar.gz
[root@k8s-vm01 nacos-cluster]# tar -zvcf nacos-server-1.2.1.tar.gz nacos-server-1.2.1
[root@k8s-vm01 nacos-cluster]# ls
nacos-server-1.2.1  nacos-server-1.2.1.tar.gz  nacos-server-1.2.1.zip

　　

2）Nacos镜像制作

[root@k8s-vm01 nacos-cluster]# cat Dockerfile
FROM 192.168.1.75/wise-ops/jdk1.8.0_192:latest
RUN rm -f /etc/localtime \
&& ln -sv /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
&& echo "Asia/Shanghai" > /etc/timezone

ENV LANG en_US.UTF-8

ENV MODE cluster
ENV PREFER_HOST_MODE ip
ENV BASE_DIR /home/nacos
ENV CLASSPATH .:/home/nacos/conf:
ENV CLUSTER_CONF /home/nacos/conf/cluster.conf
ENV FUNCTION_MODE all
ENV JAVA_HOME /usr/java/jdk1.8.0_192
ENV NACOS_USER nacos
ENV JAVA /usr/java/jdk1.8.0_192/bin/java
ENV JVM_XMS 2g
ENV JVM_XMX 2g
ENV JVM_XMN 1g
ENV JVM_MS 128m
ENV JVM_MMS 320m
ENV NACOS_DEBUG n
ENV TOMCAT_ACCESSLOG_ENABLED false

WORKDIR /home/nacos
ADD nacos-server-1.2.1.tar.gz /home
RUN set -x && mv /home/nacos-server-1.2.1/nacos/* /home/nacos/ && rm -rf /home/nacos-server-1.2.1

RUN mkdir -p logs && cd logs && touch start.out && ln -sf /dev/stdout start.out && ln -sf /dev/stderr start.out
RUN chmod 755 bin/docker-startup.sh

EXPOSE 8848
ENTRYPOINT ["bin/docker-startup.sh"]

　　

制作镜像并上传Harbor

[root@k8s-vm01 nacos-cluster]# docker build -t 192.168.1.75/wise-ops/nacos-cluster:v1.0 .
[root@k8s-vm01 nacos-cluster]# docker push 192.168.1.75/wise-ops/nacos-cluster:v1.0

　　

3）部署Nacos集群

这里采用了configmap存储卷，将mysql配置信息存到了configmap中
注意：需要提前在mysql数据库中创建一个nacos库名！然后将上面nacos-server-1.2.1.tar.gz包中的conf/nacos-mysql.sql文件里的sql语句在mysql的nacos库下执行（source nacos-mysql.sql ）导入语句。

[root@k8s-vm01 nacos-cluster]# pwd
/opt/k8s/work/test_yml/nacos-cluster

[root@k8s-vm01 nacos-cluster]# cat nacos-cluster.yml
---
apiVersion: v1
kind: Service
metadata:
  namespace: wise
  name: nacos-cluster
  labels:
    app: nacos-cluster
spec:
  ports:
    - port: 8848
      name: server
      targetPort: 8848
  clusterIP: None
  selector:
    app: nacos-cluster
---
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: wise
  name: nacos-cluster-cm
data:
  mysql.host: "192.168.1.72"
  mysql.db.name: "nacos"
  mysql.port: "3306"
  mysql.user: "nacos"
  mysql.password: "nacos@123"
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  namespace: wise
  name: nacos-cluster
spec:
  serviceName: nacos-cluster
  replicas: 3
  template:
    metadata:
      labels:
        app: nacos-cluster
      annotations:
        pod.alpha.kubernetes.io/initialized: "true"
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: "app"
                    operator: In
                    values:
                      - nacos-cluster
              topologyKey: "kubernetes.io/hostname"
      containers:
        - name: k8snacos
          imagePullPolicy: Always
          image: 192.168.1.75/wise-ops/nacos-cluster:v10
          resources:
            requests:
              memory: 2048Mi
              cpu: 1000m
            limits:
              memory: 2048Mi
              cpu: 1000m
          ports:
            - containerPort: 8848
              name: client
          env:
            - name: NACOS_REPLICAS
              value: "3"
            - name: MYSQL_SERVICE_HOST
              valueFrom:
                configMapKeyRef:
                  name: nacos-cluster-cm
                  key: mysql.host
            - name: MYSQL_SERVICE_DB_NAME
              valueFrom:
                configMapKeyRef:
                  name: nacos-cluster-cm
                  key: mysql.db.name
            - name: MYSQL_SERVICE_PORT
              valueFrom:
                configMapKeyRef:
                  name: nacos-cluster-cm
                  key: mysql.port
            - name: MYSQL_SERVICE_USER
              valueFrom:
                configMapKeyRef:
                  name: nacos-cluster-cm
                  key: mysql.user
            - name: MYSQL_SERVICE_PASSWORD
              valueFrom:
                configMapKeyRef:
                  name: nacos-cluster-cm
                  key: mysql.password
            - name: NACOS_SERVER_PORT
              value: "8848"
            - name: PREFER_HOST_MODE
              value: "hostname"
            - name: NACOS_SERVERS
              value: "nacos-cluster-0.nacos-cluster.wise.svc.cluster.local:8848 nacos-cluster-1.nacos-cluster.wise.svc.cluster.local:8848 nacos-cluster-2.nacos-cluster.wise.svc.cluster.local:8848"
  selector:
    matchLabels:
      app: nacos-cluster

　　

创建并查看

[root@k8s-vm01 nacos-cluster]# kubectl create -f nacos-cluster.yml
service/nacos-cluster created
configmap/nacos-cluster-cm created
statefulset.apps/nacos-cluster created

[root@k8s-vm01 ~]# kubectl get pods -n wise|grep nacos-cluster
nacos-cluster-0                             1/1     Running   0          49m
nacos-cluster-1                             1/1     Running   0          49m
nacos-cluster-2                             1/1     Running   0          48m

[root@k8s-vm01 ~]# kubectl get svc -n wise|grep nacos-cluster
nacos-cluster   ClusterIP   None             <none>        8848/TCP     50m

[root@k8s-vm01 ~]# kubectl get statefulset -n wise
NAME            READY   AGE
nacos-cluster   3/3     50m

[root@k8s-vm01 ~]# kubectl get configmap -n wise|grep nacos-cluster
nacos-cluster-cm                  5      50m

　　

配置ingress，外部访问nacos