4 k8s-nacos集群

1 环境说明

主机名	系统版本	IP地址	cpu/内存/磁盘	用途	软件版本
k8s_nfs	CentOS7.5	172.16.1.60	2核/2GB/60GB	nfs存储	nfs-utils-1.3.0-0.68
k8s-master1	CentOS7.5	172.16.1.81	2核/4GB/60GB	kubernetes master1节点	k8s v1.20.0
k8s-master2	CentOS7.5	172.16.1.82	2核/4GB/60GB	kubernetes master2节点	k8s v1.20.0
k8s-node1	CentOS7.5	172.16.1.83	4核/8GB/60GB	kubernetes node1节点	k8s v1.20.0
k8s-node2	CentOS7.5	172.16.1.84	4核/8GB/60GB	kubernetes node2节点	k8s v1.20.0

补充: kubernetes集群的控制节点我打了污点不能被pod调度使用。

1 nfs服务部署
节点: k8s_nfs
用途: k8s pod 数据持久化存储
说明: nfs服务的搭建过程不再赘述
验证:
[root@k8s_nfs ~]# showmount -e 172.16.1.60
Export list for 172.16.1.60:
/ifs/kubernetes *
​
2 nfs-subdir-external-provisioner插件部署
节点: kubernetes集群
用途: 为中间件pod提供pvc自动供给
说明: nfs pvc自动供给插件的部署过程不再赘述。修改"deployment.yaml"文件中连接nfs服务的地址和nfs共享目录参数；修改"class.yaml"
文件中"archiveOnDelete"(删除时是否存档)参数为 archiveOnDelete: "true"，删除pod时保留pod数据，默认为false时为不保留数据。
注意: 在部署前需要在k8s各个节点上部署nfs的客户端(yum install nfs-utils -y)，否则无法部署成功。
​
补充:
(1) gitlab项目地址: https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner
(2) 下载 deploy 目录如下文件
class.yaml、deployment.yaml、rbac.yaml
​
查看:
[root@k8s-master1 nfs-subdir-external-provisioner-master]# ls | xargs -i kubectl apply -f {}
[root@k8s-master1 nfs-subdir-external-provisioner-master]# kubectl get deployment,pod,svc,sc -n default

2 nacos集群部署

0 创建nacos-cluster命名空间
[root@k8s-master1 nacos-cluster]# kubectl create namespace nacos-cluster

1 部署mysql
(1) mysql.yaml文件
[root@k8s-master1 nacos-cluster]# cat mysql.yaml
apiVersion: v1
kind: Secret
metadata:
  name: mysql
  namespace: nacos-cluster
type: Opaque
data:
  mysql-root-password: "cm9vdEAxMjM0NTY="
  mysql-password: "bmFjb3NAMTIzNDU2"
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: mysql
  namespace: nacos-cluster
spec:
  selector:
    matchLabels:
      app: mysql
  template:
    metadata:
      labels:
        app: mysql
    spec:
      containers:
      - name: mysql
        image: mysql:5.7.30
        args: ["--character-set-server=utf8"]
        resources:
          requests:
            cpu: 500m
            memory: 512Mi
          limits:
            cpu: 500m
            memory: 512Mi
        env:
        - name: MYSQL_ROOT_PASSWORD
          valueFrom:
            secretKeyRef:
              name: mysql
              key: mysql-root-password
        - name: MYSQL_PASSWORD
          valueFrom:
            secretKeyRef:
              name: mysql
              key: mysql-password
        - name: MYSQL_USER
          value: "nacos"
        - name: MYSQL_DATABASE
          value: "nacos"
        ports:
        - name: mysql
          containerPort: 3306
        livenessProbe:
          exec:
            command:
            - sh
            - -c
            - "mysqladmin ping -u root -p${MYSQL_ROOT_PASSWORD}"
          initialDelaySeconds: 30
          periodSeconds: 10
        readinessProbe:
          exec:
            command:
            - sh
            - -c
            - "mysqladmin ping -u root -p${MYSQL_ROOT_PASSWORD}"
          initialDelaySeconds: 5
          periodSeconds: 10
        volumeMounts:
        - name: mysql-data
          mountPath: /var/lib/mysql

      volumes:
      - name: mysql-data
        #emptyDir: {}
        persistentVolumeClaim:
          claimName: mysql-data
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: mysql-data
  namespace: nacos-cluster
spec:
  storageClassName: "managed-nfs-storage"
  accessModes:
    - "ReadWriteOnce"
  resources:
    requests:
      storage: "8Gi"
---
apiVersion: v1
kind: Service
metadata:
  name: mysql
  namespace: nacos-cluster
spec:
  type: ClusterIP
  ports:
  - name: mysql
    port: 3306
    targetPort: mysql
  selector:
    app: mysql


补充: Secret(type: Opaque)中的数据需要使用base64方式加密
[root@k8s-master1 ~]# echo -n 'root@123456' | base64
cm9vdEAxMjM0NTY=
[root@k8s-master1 ~]# echo -n 'nacos@123456' | base64
bmFjb3NAMTIzNDU2

解码方式: echo -n '<编码的内容>' | base64 -d

(2) 应用yaml文件
[root@k8s-master1 nacos-cluster]# kubectl apply -f mysql.yaml
secret/mysql created
deployment.apps/mysql created
persistentvolumeclaim/mysql-data created
service/mysql created

(3) 查看部署的mysql相关信息
[root@k8s-master1 nacos-cluster]# kubectl get pod,svc,pvc,pv -n nacos-cluster
[root@k8s_nfs ~]# ls -l /ifs/kubernetes/

mysql pod信息:

mysql nfs 共享存储信息:

2 往nacos库中导入nacos表
# wget https://github.com/alibaba/nacos/releases/download/1.4.3/nacos-server-1.4.3.tar.gz
[root@k8s-master1 nacos-cluster]# tar -xzf nacos-server-1.4.3.tar.gz
​
[root@k8s-master1 nacos-cluster]# kubectl get svc -n nacos-cluster
NAME   TYPE       CLUSTER-IP       EXTERNAL-IP   PORT(S)   AGE
mysql   ClusterIP   172.28.198.235   <none>       3306/TCP   4m21s
​
[root@k8s-master1 nacos-cluster]# yum install mysql -y
[root@k8s-master1 nacos-cluster]# mysql -h 172.28.198.235 -u nacos -p'nacos@123456'
MySQL [(none)]> use nacos;
MySQL [nacos]> source /root/nacos-cluster/nacos/conf/nacos-mysql.sql
MySQL [nacos]> show tables;
+----------------------+
| Tables_in_nacos     |
+----------------------+
| config_info         |
| config_info_aggr     |
| config_info_beta     |
| config_info_tag     |
| config_tags_relation |
| group_capacity       |
| his_config_info     |
| permissions         |
| roles               |
| tenant_capacity     |
| tenant_info         |
| users               |
+----------------------+
12 rows in set (0.00 sec)
MySQL [nacos]> exit

3 应用nacos yaml文件
docker hub nacos镜像仓库地址: https://registry.hub.docker.com/r/nacos/nacos-server/tags
使用nacos镜像版本: nacos/nacos-server:1.4.2
(1) 查看yaml文件
[root@k8s-master1 nacos-cluster]# cat nacos-cluster.yml
---
apiVersion: v1
kind: Service
metadata:
  namespace: nacos-cluster
  name: nacos-cluster
  labels:
    app: nacos-cluster
spec:
  ports:
    - port: 8848
      name: server
      targetPort: 8848
  clusterIP: None
  selector:
    app: nacos-cluster
---
apiVersion: v1
kind: Service
metadata:
  namespace: nacos-cluster
  name: nacos-cluster-http
  labels:
    app: nacos-cluster
spec:
  ports:
    - port: 8848
      name: http
      targetPort: 8848
      nodePort: 30096
  type: NodePort
  selector:
    app: nacos-cluster
---
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: nacos-cluster
  name: nacos-cluster-cm
data:
  mysql.host: "172.28.198.235"
  mysql.db.name: "nacos"
  mysql.port: "3306"
  mysql.user: "nacos"
  mysql.password: "nacos@123456"
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  namespace: nacos-cluster
  name: nacos-cluster
spec:
  serviceName: nacos-cluster
  replicas: 3
  selector:
    matchLabels:
      app: nacos-cluster
  template:
    metadata:
      labels:
        app: nacos-cluster
      annotations:
        pod.alpha.kubernetes.io/initialized: "true"
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: "app"
                    operator: In
                    values:
                      - nacos-cluster
              topologyKey: "kubernetes.io/hostname"
      containers:
        - name: k8snacos
          imagePullPolicy: Always
          image: nacos/nacos-server:1.4.2
          resources:
            requests:
              memory: 2048Mi
              cpu: 1000m
            limits:
              memory: 2048Mi
              cpu: 1000m
          ports:
            - containerPort: 8848
              name: client
          env:
            - name: NACOS_REPLICAS
              value: "3"
            - name: MYSQL_SERVICE_HOST
              valueFrom:
                configMapKeyRef:
                  name: nacos-cluster-cm
                  key: mysql.host
            - name: MYSQL_SERVICE_DB_NAME
              valueFrom:
                configMapKeyRef:
                  name: nacos-cluster-cm
                  key: mysql.db.name
            - name: MYSQL_SERVICE_PORT
              valueFrom:
                configMapKeyRef:
                  name: nacos-cluster-cm
                  key: mysql.port
            - name: MYSQL_SERVICE_USER
              valueFrom:
                configMapKeyRef:
                  name: nacos-cluster-cm
                  key: mysql.user
            - name: MYSQL_SERVICE_PASSWORD
              valueFrom:
                configMapKeyRef:
                  name: nacos-cluster-cm
                  key: mysql.password
            - name: NACOS_SERVER_PORT
              value: "8848"
            - name: PREFER_HOST_MODE
              value: "hostname"
            - name: NACOS_SERVERS
              value: "nacos-cluster-0.nacos-cluster.nacos-cluster.svc.cluster.local:8848 nacos-cluster-1.nacos-cluster.nacos-cluster.svc.cluster.local:8848 nacos-cluster-2.nacos-cluster.nacos-cluster.svc.cluster.local:8848"


(2) 应用yaml文件
1) 由于使用了pod的反亲和性，3个pod必须部署在不同的node节点上，我这里只有2个slave节点，所以将master2节点的污点去除
[root@k8s-master1 nacos-cluster]# kubectl taint nodes k8s-master2 node-role.kubernetes.io/master-
node/k8s-master2 untainted

[root@k8s-master1 nacos-cluster]# kubectl apply -f nacos-cluster.yml
service/nacos-cluster created
service/nacos-cluster-http created
configmap/nacos-cluster-cm created
statefulset.apps/nacos-cluster created

4 查看nacos集群的pod
[root@k8s-master1 nacos-cluster]# kubectl get pod -n nacos-cluster -o wide

图示:

5 查看nacos集群的svc
[root@k8s-master1 nacos-cluster]# kubectl get svc,ep -n nacos-cluster

图示:

6 访问nacos的UI界面
UI访问地址: http://172.16.1.81|82|83|84:30096/nacos
登录用户: nacos
登录密码: nacos  

登录:

查看集群状态:

(1) 节点状态

(2) 节点元数据查看

3 知识拾遗

3.1 本地方式部署nacos集群

1 nacos说明
官方文档: https://nacos.io/zh-cn/docs/what-is-nacos.html

nacos discovery:

注册中心比较:

补充:
(1) CAP C表示一致性、A表示可用性、P表示容错性。
(2) 搭建注册中心常使用集群模式。
(3) 对所有注册中心集群模式下P是必须的，C A 不能同时实现
1) 在CP下
多节点实现P；一个leader(读写)、多个follower(读)组成C；集群宕机数大于半数不可用。

2) 在AP下
多节点实现P；每个节点都可以读写组成A；集群之间数据同步可能不及时。

3) nacos集群使用AP，然后使用mysql数据库实现C。

2 环境说明

主机名	IP	cpu/内存/磁盘	用途
ControlNode	172.16.1.30	2Core/4GB/60GB	nacos集群节点
SlaveNode1	172.16.1.31	2Core/4GB/60GB	nacos集群节点
SlaveNode2	172.16.1.32	2Core/4GB/60GB	nacos集群节点

3 安装mysql(版本5.7+)
在172.16.1.30节点上操作
(1) 安装
# yum -y install yum-utils
# rpm -ivh https://dev.mysql.com/get/mysql80-community-release-el7-1.noarch.rpm
# yum-config-manager --disable mysql80-community
# yum-config-manager --enable mysql57-community
# yum install mysql-community-server --nogpgcheck -y
# systemctl start mysqld.service
# systemctl enable mysqld.service
# systemctl status mysqld.service
# grep 'temporary password' /var/log/mysqld.log
# mysql -uroot -p
mysql> ALTER USER 'root'@'localhost' IDENTIFIED BY 'LiuChang@2022';
mysql> flush privileges;
mysql> exit

(2) 创建nacos库、用户、密码
# mysql -uroot -p'LiuChang@2022'
mysql> create database nacos default character set=utf8 default collate=utf8_general_ci;
mysql> grant all privileges on nacos.* to nacos@"%" identified by "Nacos@2022";
mysql> flush privileges;

4 部署
(1) 说明
1) 我这里通过nacos发行包来部署nacos集群，Nacos 依赖 Java 环境来运行。
2) 64 bit OS，支持 Linux/Unix/Mac/Windows，推荐选用 Linux/Unix/Mac。
3) 64 bit JDK 1.8+；
4) 在 172.16.1.30-32 三台节点上操作

(2) 安装jdk
下载地址: https://www.oracle.com/java/technologies/downloads/#java8
下载版本: jdk-8u321-linux-x64.tar.gz

# tar -xzf jdk-8u321-linux-x64.tar.gz
# mv jdk1.8.0_321/ /usr/local/
# ln -s /usr/local/jdk1.8.0_321/ /usr/local/jdk

# cat >> /etc/profile << EOF
export JAVA_HOME=/usr/local/jdk
export CLASSPATH=\$JAVA_HOME/lib/tools.jar:\$JAVA_HOME/jre/lib/rt.jar
export PATH=\$JAVA_HOME/bin:\$PATH
EOF

# source /etc/profile
# java -version
java version "1.8.0_321"
Java(TM) SE Runtime Environment (build 1.8.0_321-b07)
Java HotSpot(TM) 64-Bit Server VM (build 25.321-b07, mixed mode)

(3) 下载软件包
https://github.com/alibaba/nacos/releases/download/1.4.3/nacos-server-1.4.3.tar.gz

(4) 解压
# tar -xzf nacos-server-1.4.3.tar.gz
# mv nacos/ /usr/local/nacos-server-1.4.3/
# ln -s /usr/local/nacos-server-1.4.3/ /usr/local/nacos

(5) 创建nacos虚拟用户
# useradd -M -s /usr/sbin/nologin nacos

(6) 为nacos安装目录赋权
# chown -R nacos.nacos /usr/local/nacos/

(7) 导入nacos数据表
在172.16.1.30节点上操作
# mysql -unacos -p'Nacos@2022'
> use nacos;
> source /usr/local/nacos/conf/nacos-mysql.sql
> exit

(8) 修改配置文件
# cd /usr/local/nacos/conf/
# vim application.properties  # 修改如下内容
1) 设置端口
server.port=8848
2) 设置本机ip地址
nacos.inetutils.ip-address=172.16.1.30
# 三个节点ip地址分别设置为172.16.1.30、172.16.1.31、172.16.1.32
3) 配置mysql数据库源
spring.datasource.platform=mysql
db.num=1
db.url.0=jdbc:mysql://172.16.1.30:3306/nacos?characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useUnicode=true&useSSL=false&serverTimezone=UTC
db.user.0=nacos
db.password.0=Nacos@2022

(9) 配置集群文件
# cp -a cluster.conf.example cluster.conf
# vim cluster.conf  # 修改如下集群信息
#it is ip
#example
172.16.1.30:8848
172.16.1.31:8848
172.16.1.32:8848

(10) 将nacos服务加入到systemd管理
# cat > /usr/lib/systemd/system/nacos.service << EOF
[Unit]
Description=nacos
After=network.target

[Service]
Type=forking
SuccessExitStatus=143
Environment=JAVA_HOME=/usr/local/jdk
ExecStart=/usr/local/nacos/bin/startup.sh -m cluster
#ExecStart=/usr/local/nacos/bin/startup.sh -m standalone
ExecReload=/usr/local/nacos/bin/shutdown.sh
ExecStop=/usr/local/nacos/bin/shutdown.sh
PrivateTmp=true
Restart=on-failure
User=nacos
Group=nacos

[Install]
WantedBy=multi-user.target
EOF

# systemctl daemon-reload

补充: 在"/usr/local/nacos/bin/startup.sh"配置文件的"JVM Configuration"参数配置下修改jvm参数。

(11) 启动集群
1) 启动
# systemctl start nacos.service
# systemctl enable nacos.service
2) 查看日志
# cat /usr/local/nacos/logs/start.out
(......省略内容)
2022-03-02 13:48:22,356 INFO Nacos started successfully in cluster mode. use external storage
# nacos集群启动成功且使用的是外部存储。

5 访问UI界面
http://172.16.1.30|31|32:8848/nacos/
用户名和密码都是 nacos

登录界面:

节点列表:

6 搭建nginx对nacos集群进行反向代理
在172.16.1.30节点上操作

(1) 安装nginx
# yum install nginx -y
# systemctl start nginx
# systemctl enable nginx

(2) 添加反向代理配置
# cat > /etc/nginx/conf.d/nacos.conf << EOF
upstream nacos-cluster {
  server 172.16.1.30:8848;
  server 172.16.1.31:8848;
  server 172.16.1.32:8848;
}

server {
  listen       8847;
  server_name  localhost;

  location /nacos/ {
    proxy_pass http://nacos-cluster/nacos/;
  }
}
EOF

(3) 验证配置文件
# nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful

(4) 重启nginx
# systemctl restart nginx

(5) 访问
http://172.16.1.30:8847/nacos/