Kubernetes 部署常见应用

k8s 安装：https://www.cnblogs.com/jhxxb/p/14321352.html

yaml 文件说明：https://kubernetes.io/zh/docs/concepts/overview/working-with-objects/kubernetes-objects

命名空间：https://kubernetes.io/zh/docs/tasks/administer-cluster/namespaces

创建命令：https://kubernetes.io/zh/docs/tasks/manage-kubernetes-objects/declarative-config/#how-to-create-objects

 

一、使用任意端口暴露

sudo vim /etc/kubernetes/manifests/kube-apiserver.yaml

# 在 - --service-cluster-ip-range=10.96.0.0/16 下添加
- --service-node-port-range=1-65535

# 重启 kubelet
sudo systemctl daemon-reload
sudo systemctl restart kubelet

• targetPort：容器的端口（最根本的入口），与制作容器时暴露的端口一致（DockerFile中EXPOSE），也是 pod 的端口
• port：kubernetes 中服务之间访问的端口，service 的端口
• nodePort：外部可访问的端口。容器所在宿主机的端口（实质上也是通过 service 暴露给了宿主机，而 port 却没有）
• hostPort：通过宿主机的 IP:Port 来访问 Pod，和 nodePort 类似

nodePort 是集群外流量访问集群内服务的端口类型。Port 是集群内的 pod 互相通信用的端口类型。targetPort 是最终端口

 

二、常见应用

MySQL

随机端口，指定端口需要配置 nodePort，或者部署后是用 kubectl edit svc mysql-service 修改

apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-mysql
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteMany
  nfs:
    server: 10.74.2.71
    path: "/nfs/mysql"
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: nfs-mysql
  namespace: default
spec:
  accessModes:
    - ReadWriteMany
  storageClassName: ""
  resources:
    requests:
      storage: 5Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: mysql-deployment
spec:
  selector:
    matchLabels:
      app: mysql-label
  replicas: 1
  template:
    metadata:
      labels:
        app: mysql-label
    spec:
      containers:
        - name: mysql
          image: mysql:latest
          env:
            - name: MYSQL_ROOT_PASSWORD
              value: root
            - name: MYSQL_ROOT_HOST
              value: '%'
            - name: TZ
              value: Asia/Shanghai
          ports:
            - containerPort: 3306
          volumeMounts:
            - name: mysql-data
              mountPath: /var/lib/mysql
      volumes:
        - name: mysql-data
          persistentVolumeClaim:
            claimName: nfs-mysql
---
apiVersion: v1
kind: Service
metadata:
  name: mysql-service
spec:
  selector:
    app: mysql-label
  type: NodePort
  ports:
    - port: 3306
      targetPort: 3306

sudo mkdir /nfs/mysql

kubectl apply -f mysql.yaml

kubectl delete -f mysql.yaml

 

Oracle

apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-oracle
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteMany
  nfs:
    server: 10.74.2.71
    path: "/nfs/oracle"
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: nfs-oracle
  namespace: default
spec:
  accessModes:
    - ReadWriteMany
  storageClassName: ""
  resources:
    requests:
      storage: 10Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: oracle-deployment
spec:
  selector:
    matchLabels:
      app: oracle-label
  replicas: 1
  template:
    metadata:
      labels:
        app: oracle-label
    spec:
      containers:
        - name: oracle-11g
          image: loliconneko/oracle-ee-11g:latest
          env:
            - name: WEB_CONSOLE
              value: 'false'
            - name: DBCA_TOTAL_MEMORY
              value: '4096'
          ports:
            - containerPort: 1521
          volumeMounts:
            - name: oracle-data
              mountPath: /u01/app/oracle
      volumes:
        - name: oracle-data
          persistentVolumeClaim:
            claimName: nfs-oracle
---
apiVersion: v1
kind: Service
metadata:
  name: oracle-service
spec:
  selector:
    app: oracle-label
  type: NodePort
  ports:
    - port: 1521
      targetPort: 1521
      nodePort: 1521

sudo mkdir /nfs/oracle

kubectl apply -f oracle.yaml

 

Redis

注意这里 resources 项限制了 CPU 和 Memory 资源

apiVersion: apps/v1
kind: Deployment
metadata:
  name: redis-deployment
spec:
  selector:
    matchLabels:
      app: redis-label
  replicas: 1
  template:
    metadata:
      labels:
        app: redis-label
    spec:
      containers:
        - name: redis
          image: redis:alpine
          resources:
            limits:
              cpu: 1
              memory: 512Mi
            requests:
              cpu: 0.5
              memory: 128Mi
          ports:
            - containerPort: 6379
---
apiVersion: v1
kind: Service
metadata:
  name: redis-service
spec:
  selector:
    app: redis-label
  type: NodePort
  ports:
    - port: 6379
      targetPort: 6379
      nodePort: 6379

kubectl apply -f redis.yaml

 

Zentao(禅道)

https://hub.docker.com/r/easysoft/zentao

https://www.zentao.net/book/zentaopmshelp/40.html

apiVersion: v1
kind: PersistentVolume
metadata:
  name: zentao-data
spec:
  capacity:
    storage: 1Gi
  accessModes:
    - ReadWriteMany
  nfs:
    server: 10.74.2.71
    path: "/nfs/zentao"
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zentao-data
  namespace: default
spec:
  accessModes:
    - ReadWriteMany
  storageClassName: ""
  resources:
    requests:
      storage: 1Gi
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zentao-mysql-data
spec:
  capacity:
    storage: 1Gi
  accessModes:
    - ReadWriteMany
  nfs:
    server: 10.74.2.71
    path: "/nfs/zentao-mysql"
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zentao-mysql-data
  namespace: default
spec:
  accessModes:
    - ReadWriteMany
  storageClassName: ""
  resources:
    requests:
      storage: 1Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: zentao-deployment
spec:
  selector:
    matchLabels:
      app: zentao-label
  replicas: 1
  template:
    metadata:
      labels:
        app: zentao-label
    spec:
      containers:
        - name: zentao
          image: easysoft/zentao:15.7
          resources:
            requests:
              cpu: 2000m
              memory: 2Gi
            limits:
              cpu: 2000m
              memory: 2Gi
          env:
            - name: MYSQL_ROOT_PASSWORD
              value: root
          ports:
            - name: zentao
              containerPort: 80
            - name: mysql
              containerPort: 3306
          volumeMounts:
            - name: zentao-data
              mountPath: /www/zentaopms
            - name: zentao-mysql-data
              mountPath: /var/lib/mysql
      volumes:
        - name: zentao-data
          persistentVolumeClaim:
            claimName: zentao-data
        - name: zentao-mysql-data
          persistentVolumeClaim:
            claimName: zentao-mysql-data
---
apiVersion: v1
kind: Service
metadata:
  name: zentao-service
spec:
  selector:
    app: zentao-label
  type: NodePort
  ports:
    - name: mysql
      port: 3306
      targetPort: 3306
      nodePort: 9595
    - name: zentao
      port: 80
      targetPort: 80
      nodePort: 9696

部署并修改远程连接可以数据库

sudo mkdir /nfs/zentao
sudo mkdir /nfs/zentao-mysql
sudo chown zhangsan:zhangsan -R /nfs/zentao/
sudo chown zhangsan:zhangsan -R /nfs/zentao-mysql/
kubectl apply -f zentao.yaml

# 进入容器，修改配置
# https://mariadb.com/kb/en/configuring-mariadb-for-remote-client-access/
vim /etc/mysql/mariadb.conf.d/50-server.cnf
#bind-address=127.0.0.1
skip-networking=0
skip-bind-address
# 重启 mariadb 服务
cat /var/run/mysqld/mysqld.pid
kill xxx
service mysql restart
# 查看监听地址
ss -an

 

Docker 镜像仓库

https://github.com/Joxit/docker-registry-ui/tree/main/examples/kubernetes

先创建 https 证书和密码文件：https://www.cnblogs.com/jhxxb/p/13647369.html

然后将文件创建成 ConfigMap：https://kubernetes.io/zh/docs/tasks/configure-pod-container/configure-pod-configmap

kubectl create namespace devops
kubectl -n devops create configmap registry-config-auth --from-file=/opt/registry/auth
kubectl -n devops create configmap registry-config-certs --from-file=/opt/registry/certs

仓库使用 registry，UI 使用 joxit/docker-registry-ui，若出现 Access-Control-Allow-Origin 问题，看看配置：NGINX_PROXY_PASS_URL 为代理，REGISTRY_URL 为直接访问，二选一。若存储为本机可以将 nfs 改为 hostPath

apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-docker-registry
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Delete # https://kubernetes.io/zh/docs/tasks/administer-cluster/change-pv-reclaim-policy/
  nfs:
    server: 10.74.2.71
    path: /nfs/docker-registry # sudo chmod -R 0777 /nfs/docker-registry
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: nfs-docker-registry
spec:
  accessModes:
    - ReadWriteMany
  storageClassName: ""
  resources:
    requests:
      storage: 10Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: docker-registry-deployment
spec:
  selector:
    matchLabels:
      app: docker-registry-label
  replicas: 1
  template:
    metadata:
      labels:
        app: docker-registry-label
    spec:
      containers:
        - name: docker-registry
          image: registry # 镜像版本
          env:
            - name: REGISTRY_STORAGE_DELETE_ENABLED
              value: "true"
            - name: REGISTRY_AUTH
              value: htpasswd 
            - name: REGISTRY_AUTH_HTPASSWD_REALM
              value: basic-realm
            - name: REGISTRY_AUTH_HTPASSWD_PATH
              value: /auth/htpasswd
            - name: REGISTRY_HTTP_TLS_CERTIFICATE
              value: /certs/domain.crt
            - name: REGISTRY_HTTP_TLS_KEY
              value: /certs/domain.key
          ports:
            - containerPort: 5000
              protocol: TCP
          volumeMounts:
            - name: docker-registry-data
              mountPath: /var/lib/registry
            - name: docker-registry-auth
              mountPath: /auth
            - name: docker-registry-certs
              mountPath: /certs
      volumes:
        - name: docker-registry-data
          persistentVolumeClaim:
            claimName: nfs-docker-registry # 将上面创建的 pv 关联到 pvc 上
        - name: docker-registry-auth
          configMap:
            name: registry-config-auth # 关联到 configMap 上
        - name: docker-registry-certs
          configMap:
            name: registry-config-certs
---
apiVersion: v1
kind: Service
metadata:
  name: docker-registry-service
spec:
  selector:
    app: docker-registry-label
  ports:
    - targetPort: 5000
      port: 5000
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: docker-registry-ui-deployment
spec:
  selector:
    matchLabels:
      app: docker-registry-ui-label
  replicas: 1
  template:
    metadata:
      labels:
        app: docker-registry-ui-label
    spec:
      containers:
        - name: docker-registry-ui
          image: joxit/docker-registry-ui
          env:
            - name: DELETE_IMAGES
              value: "true"
            - name: SINGLE_REGISTRY
              value: "true"
            - name: PULL_URL
              value: 10.74.2.71:5000
            - name: NGINX_PROXY_PASS_URL # https://github.com/Joxit/docker-registry-ui/tree/main/examples
              value: https://docker-registry-service:5000 # 若配置了 ca，这里为 https
          ports:
            - containerPort: 80
              protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
  name: docker-registry-ui-service
spec:
  selector:
    app: docker-registry-ui-label
  ports:
    - targetPort: 80
      port: 80
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: docker-registry-ui-ingress
spec:
  ingressClassName: nginx
  rules:
    - http:
        paths:
          - path: /
            backend:
              service:
                name: docker-registry-ui-service
                port:
                  number: 80
            pathType: Prefix
      host: registry.example.com

创建

sudo mkdir /nfs/docker-registry
kubectl apply -f registry.yaml
# 查看
kubectl get deployment -n docker-registry -o wide
kubectl get pods -n docker-registry
kubectl get svc -n docker-registry -o wide
kubectl get ing -n docker-registry
kubectl describe ingress docker-registry-ui-ingress -n docker-registry
# 查看 ingress 对外端口
kubectl get svc -n ingress-nginx

访问 UI 要配置 Hosts 文件，端口为 ingress 的 80 对外端口

 

Jenkins

https://github.com/jenkinsci/kubernetes-plugin/blob/master/README_zh.md#在-kubernetes-中运行

PV 配置，VolumeClaimTemplates 匹配不上 PV 时会出现 StatefulSet 挂载不到 PV 的问题

apiVersion: v1
kind: PersistentVolume
metadata:
  name: jenkins-home
spec:
  capacity:
    storage: 1Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 10.74.2.71
    path: /nfs/jenkins

若删除了 PVC，想让 PV 可以被重新绑定：https://stackoverflow.com/questions/58168408/kubernetes-pv-refuses-to-bind-after-delete-re-create

# 要再次让 PV Available，需删除 uid
kubectl patch pv jenkins-home --type json -p '[{"op": "remove", "path": "/spec/claimRef/uid"}]'
# 或删除整个 claimRef 部分
kubectl patch pv jenkins-home --type json -p '[{"op": "remove", "path": "/spec/claimRef"}]'

创建

# 创建 PV
sudo mkdir /nfs/jenkins & sudo chmod -R 0777 /nfs/jenkins
kubectl apply -f jenkins-pv.yaml
# 创建名称空间
kubectl create namespace devops
# 设置当前配置默认名称空间
kubectl config set-context $(kubectl config current-context) --namespace=devops
# 创建 jenkins
kubectl create -f http://raw.githubusercontent.com/jenkinsci/kubernetes-plugin/master/src/main/kubernetes/service-account.yml
kubectl create -f http://raw.githubusercontent.com/jenkinsci/kubernetes-plugin/master/src/main/kubernetes/jenkins.yml
# 查看网络，访问需要配置 hosts 文件：jenkins.example.com，使用 https 访问
kubectl describe services/jenkins
# 设置当前配置默认名称空间
kubectl config set-context $(kubectl config current-context) --namespace=default

安装 Kubernetes 插件：https://plugins.jenkins.io/kubernetes

https://kubernetes.default.svc.cluster.local
http://jenkins.devops.svc.cluster.local:80

关于地址命名，与 DNS 有关：https://kubernetes.io/zh/docs/concepts/services-networking/dns-pod-service

安装 Pipeline 插件，测试 Pipeline script，注意改 NFS 地址

def label = "mypod-${UUID.randomUUID().toString()}"
podTemplate(label: label, cloud: 'kubernetes', containers: [
    containerTemplate(name: 'maven', image: 'maven:3.8.4-jdk-8', ttyEnabled: true, command: 'cat')
], volumes: [
    hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock'),
    nfsVolume(mountPath: '/root/.m2', serverAddress: '10.74.2.71', serverPath: '/nfs/maven')
])
{
    node(label) {
        stage('git') {
            container('jnlp') {
                sh 'git clone --branch master --depth 1 https://gitee.com/jhxxb/MySpringBoot.git'
            }
        }
        stage('maven') {
            container('maven') {
                stage('maven install') {
                    sh """
                    cd MySpringBoot
                    mvn -B -DskipTests=true clean install
                    """
                }
                stage('maven package') {
                    sh """
                    cd MySpringBoot
                    mvn -DskipTests=true package
                    """
                }
            }
        }
    }
}

若构建过程中出现 process apparently never started in，请检查 Jenkins 的系统配置中的全局属性项，是否配置有 PATH 环境变量，将 :/sbin:/usr/sbin:/usr/bin:/usr/local/bin:/bin 加入其中

 

三、部署应用到指定 Node 节点

两种方式：NodeName  和 NodeSelector

一、NodeName

以部署 RabbitMQ 为例，注意 nodeName 属性值为节点名称

apiVersion: apps/v1
kind: Deployment
metadata:
  name: rabbitmq-deployment
spec:
  selector:
    matchLabels:
      app: rabbitmq-label
  replicas: 1
  template:
    metadata:
      labels:
        app: rabbitmq-label
    spec:
      nodeName: k8s-node-8
      containers:
        - name: rabbitmq
          image: rabbitmq:management-alpine
          resources:
            limits:
              cpu: 1
              memory: 512Mi
            requests:
              cpu: 0.5
              memory: 128Mi
          env:
            - name: RABBITMQ_DEFAULT_VHOST
              value: my_vhost
            - name: RABBITMQ_DEFAULT_USER
              value: guest
            - name: RABBITMQ_DEFAULT_PASS
              value: guest
          ports:
            - containerPort: 5672
            - containerPort: 15672
---
apiVersion: v1
kind: Service
metadata:
  name: rabbitmq-service
spec:
  selector:
    app: rabbitmq-label
  type: NodePort
  ports:
    - name: app
      port: 5672
      targetPort: 5672
      nodePort: 5672
    - name: ui
      port: 15672
      targetPort: 15672
      nodePort: 15672

部署查看

kubectl apply -f rabbitmq.yaml
kubectl get pods -o wide

二、NodeSelector

给 Node 设置标签

kubectl label nodes k8s-node-8 deploy=hello
kubectl get nodes --show-labels

已部署 MQ 为例，注意 nodeSelector 属性值为标签属性值

apiVersion: apps/v1
kind: Deployment
metadata:
  name: rabbitmq-deployment
spec:
  selector:
    matchLabels:
      app: rabbitmq-label
  replicas: 1
  template:
    metadata:
      labels:
        app: rabbitmq-label
    spec:
      nodeSelector:
        deploy: hello
      containers:
        - name: rabbitmq
          image: rabbitmq:management-alpine
          resources:
            limits:
              cpu: 1
              memory: 512Mi
            requests:
              cpu: 0.5
              memory: 128Mi
          env:
            - name: RABBITMQ_DEFAULT_VHOST
              value: my_vhost
            - name: RABBITMQ_DEFAULT_USER
              value: guest
            - name: RABBITMQ_DEFAULT_PASS
              value: guest
          ports:
            - containerPort: 5672
            - containerPort: 15672
---
apiVersion: v1
kind: Service
metadata:
  name: rabbitmq-service
spec:
  selector:
    app: rabbitmq-label
  type: NodePort
  ports:
    - name: app
      port: 5672
      targetPort: 5672
      nodePort: 5672
    - name: ui
      port: 15672
      targetPort: 15672
      nodePort: 15672

 

---

http://www.mydlq.club/article/65