极客时间运维进阶训练营第十七周作业

1、基于 NetworkPolicy 限制 magedu namespace 中的所有 pod 不能跨 namespace 访问 (只能访问当前 namespace 中的所有 pod)。

root@k8s-master1:~/NetWorkPolicy-case/python-ns2# cat case4-ingress-podSelector-ns.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: tomcat-access--networkpolicy
  namespace: python
spec:
  policyTypes:
  - Ingress
  podSelector: #目标pod
    matchLabels: {} #匹配所有目标pod
  ingress:
  - from:
    - podSelector: #匹配源pod,matchLabels: {}为不限制源pod即允许所有pod,写法等同于resources(不加就是不限制)
        matchLabels: {}

2、在 kubernetes 环境部署 zookeeper 集群并基于 NFS 或 StorageClass 等方式实现创建持久化。

### 准备镜像
```bash
docker pull elevy/slim_java:8  # 可以用于生产  # 也可使用 docker 官方仓库
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/zookeeper# docker tag elevy/slim_java:8 harbor.iclinux.com/
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/zookeeper# docker push   harbor.iclinux.com/baseimages/slim_java:8


root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/zookeeper# cat Dockerfile
#FROM elevy/slim_java:8
#FROM harbor.magedu.net/baseimages/slim_java:8
#FROM harbor.linuxarchitect.io/baseimages/slim_java:8
FROM harbor.iclinux.com/baseimages/slim_java:8

ENV ZK_VERSION 3.4.14
ADD repositories /etc/apk/repositories
# Download Zookeeper
COPY zookeeper-3.4.14.tar.gz /tmp/zk.tgz
COPY zookeeper-3.4.14.tar.gz.asc /tmp/zk.tgz.asc
COPY KEYS /tmp/KEYS
RUN apk add --no-cache --virtual .build-deps \
      ca-certificates   \
      gnupg             \
      tar               \
      wget &&           \
    #
    # Install dependencies
    apk add --no-cache  \
      bash &&           \
    #
    #
    # Verify the signature
    export GNUPGHOME="$(mktemp -d)" && \
    gpg -q --batch --import /tmp/KEYS && \
    gpg -q --batch --no-auto-key-retrieve --verify /tmp/zk.tgz.asc /tmp/zk.tgz && \
    #
    # Set up directories
    #
    mkdir -p /zookeeper/data /zookeeper/wal /zookeeper/log && \
    #
    # Install
    tar -x -C /zookeeper --strip-components=1 --no-same-owner -f /tmp/zk.tgz && \
    #
    # Slim down
    cd /zookeeper && \
    cp dist-maven/zookeeper-${ZK_VERSION}.jar . && \
    rm -rf \
      *.txt \
      *.xml \
      bin/README.txt \
      bin/*.cmd \
      conf/* \
      contrib \
      dist-maven \
      docs \
      lib/*.txt \
      lib/cobertura \
      lib/jdiff \
      recipes \
      src \
      zookeeper-*.asc \
      zookeeper-*.md5 \
      zookeeper-*.sha1 && \
    #
    # Clean up
    apk del .build-deps && \
    rm -rf /tmp/* "$GNUPGHOME"

COPY conf /zookeeper/conf/
COPY bin/zkReady.sh /zookeeper/bin/
COPY entrypoint.sh /

ENV PATH=/zookeeper/bin:${PATH} \
    ZOO_LOG_DIR=/zookeeper/log \
    ZOO_LOG4J_PROP="INFO, CONSOLE, ROLLINGFILE" \
    JMXPORT=9010

ENTRYPOINT [ "/entrypoint.sh" ]

CMD [ "zkServer.sh", "start-foreground" ]

EXPOSE 2181 2888 3888 9010
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/zookeeper# cat entrypoint.sh
#!/bin/bash

echo ${MYID:-1} > /zookeeper/data/myid

if [ -n "$SERVERS" ]; then
        IFS=\, read -a servers <<<"$SERVERS"
        for i in "${!servers[@]}"; do
                printf "\nserver.%i=%s:2888:3888" "$((1 + $i))" "${servers[$i]}" >> /zookeeper/conf/zoo.cfg
        done
fi

cd /zookeeper
exec "$@"
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/zookeeper# cat repositories
http://mirrors.aliyun.com/alpine/v3.6/main
http://mirrors.aliyun.com/alpine/v3.6/community
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/zookeeper# ls conf/
log4j.properties  zoo.cfg

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/zookeeper# cat bin/zkReady.sh
#!/bin/bash

/zookeeper/bin/zkServer.sh status | egrep 'Mode: (standalone|leading|following|observing)'


root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper# ls
pv  zookeeper.yaml
root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper# cat zookeeper.yaml
apiVersion: v1
kind: Service
metadata:
  name: zookeeper
  namespace: magedu
spec:
  ports:
    - name: client
      port: 2181
  selector:
    app: zookeeper
---
apiVersion: v1
kind: Service
metadata:
  name: zookeeper1
  namespace: magedu
spec:
  type: NodePort
  ports:
    - name: client
      port: 2181
      nodePort: 32181
    - name: followers
      port: 2888
    - name: election
      port: 3888
  selector:
    app: zookeeper
    server-id: "1"
---
apiVersion: v1
kind: Service
metadata:
  name: zookeeper2
  namespace: magedu
spec:
  type: NodePort
  ports:
    - name: client
      port: 2181
      nodePort: 32182
    - name: followers
      port: 2888
    - name: election
      port: 3888
  selector:
    app: zookeeper
    server-id: "2"
---
apiVersion: v1
kind: Service
metadata:
  name: zookeeper3
  namespace: magedu
spec:
  type: NodePort
  ports:
    - name: client
      port: 2181
      nodePort: 32183
    - name: followers
      port: 2888
    - name: election
      port: 3888
  selector:
    app: zookeeper
    server-id: "3"
---
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  name: zookeeper1
  namespace: magedu
spec:
  replicas: 1
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
        server-id: "1"
    spec:
      volumes:
        - name: data
          emptyDir: {}
        - name: wal
          emptyDir:
            medium: Memory
      containers:
        - name: server
          image: harbor.linuxarchitect.io/magedu/zookeeper:v3.4.14
          imagePullPolicy: Always
          env:
            - name: MYID
              value: "1"
            - name: SERVERS
              value: "zookeeper1,zookeeper2,zookeeper3"
            - name: JVMFLAGS
              value: "-Xmx2G"
          ports:
            - containerPort: 2181
            - containerPort: 2888
            - containerPort: 3888
          volumeMounts:
          - mountPath: "/zookeeper/data"
            name: zookeeper-datadir-pvc-1
      volumes:
        - name: zookeeper-datadir-pvc-1
          persistentVolumeClaim:
            claimName: zookeeper-datadir-pvc-1
---
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  name: zookeeper2
  namespace: magedu
spec:
  replicas: 1
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
        server-id: "2"
    spec:
      volumes:
        - name: data
          emptyDir: {}
        - name: wal
          emptyDir:
            medium: Memory
      containers:
        - name: server
          image: harbor.linuxarchitect.io/magedu/zookeeper:v3.4.14
          imagePullPolicy: Always
          env:
            - name: MYID
              value: "2"
            - name: SERVERS
              value: "zookeeper1,zookeeper2,zookeeper3"
            - name: JVMFLAGS
              value: "-Xmx2G"
          ports:
            - containerPort: 2181
            - containerPort: 2888
            - containerPort: 3888
          volumeMounts:
          - mountPath: "/zookeeper/data"
            name: zookeeper-datadir-pvc-2
      volumes:
        - name: zookeeper-datadir-pvc-2
          persistentVolumeClaim:
            claimName: zookeeper-datadir-pvc-2
---
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  name: zookeeper3
  namespace: magedu
spec:
  replicas: 1
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
        server-id: "3"
    spec:
      volumes:
        - name: data
          emptyDir: {}
        - name: wal
          emptyDir:
            medium: Memory
      containers:
        - name: server
          image: harbor.linuxarchitect.io/magedu/zookeeper:v3.4.14
          imagePullPolicy: Always
          env:
            - name: MYID
              value: "3"
            - name: SERVERS
              value: "zookeeper1,zookeeper2,zookeeper3"
            - name: JVMFLAGS
              value: "-Xmx2G"
          ports:
            - containerPort: 2181
            - containerPort: 2888
            - containerPort: 3888
          volumeMounts:
          - mountPath: "/zookeeper/data"
            name: zookeeper-datadir-pvc-3
      volumes:
        - name: zookeeper-datadir-pvc-3
          persistentVolumeClaim:
           claimName: zookeeper-datadir-pvc-3

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/zookeeper# cat build-command.sh
#!/bin/bash
TAG=$1
IMG_REPO="harbor.iclinux.com"
docker build -t "${IMG_REPO}"/magedu/zookeeper:${TAG} .
#sleep 1
docker push  "${IMG_REPO}"/magedu/zookeeper:${TAG}

#nerdctl  build -t harbor.magedu.net/magedu/zookeeper:${TAG} .

#nerdctl push harbor.magedu.net/magedu/zookeeper:${TAG}
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/zookeeper# bash build-command.sh v3.4.14

# 验证镜像
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/zookeeper# docker run -it --rm harbor.iclinux.com/magedu/zookeeper:v3.4.1
# 要确保镜像可以正常启动

```
### 制备nfs server
```bash
root@k8s-ha1:~# mkdir -p /data/k8sdata/magedu/zookeeper-datadir-1
root@k8s-ha1:~# mkdir -p /data/k8sdata/magedu/zookeeper-datadir-2
root@k8s-ha1:~# mkdir -p /data/k8sdata/magedu/zookeeper-datadir-3


root@k8s-ha1:~# echo "/data/k8sdata/ *(rw,no_root_squash)" >> /etc/exports
root@k8s-ha1:~# systemctl  restart nfs-server.service
root@k8s-ha1:~# systemctl  enable nfs-server.service
root@k8s-ha1:~# exportfs -ar


```
### 部署zookeeper
```bash
# 创建ns
kubectl create ns magedu

root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper/pv# cat zookeeper-persistentvolume.yaml
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zookeeper-datadir-pv-1
spec:
  capacity:
    storage: 20Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 172.31.7.109
    path: /data/k8sdata/magedu/zookeeper-datadir-1

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zookeeper-datadir-pv-2
spec:
  capacity:
    storage: 20Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 172.31.7.109
    path: /data/k8sdata/magedu/zookeeper-datadir-2

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zookeeper-datadir-pv-3
spec:
  capacity:
    storage: 20Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 172.31.7.109
    path: /data/k8sdata/magedu/zookeeper-datadir-3
root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper/pv# cat zookeeper-persistentvolumeclaim.yaml
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zookeeper-datadir-pvc-1
  namespace: magedu
spec:
  accessModes:
    - ReadWriteOnce
  volumeName: zookeeper-datadir-pv-1
  resources:
    requests:
      storage: 10Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zookeeper-datadir-pvc-2
  namespace: magedu
spec:
  accessModes:
    - ReadWriteOnce
  volumeName: zookeeper-datadir-pv-2
  resources:
    requests:
      storage: 10Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zookeeper-datadir-pvc-3
  namespace: magedu
spec:
  accessModes:
    - ReadWriteOnce
  volumeName: zookeeper-datadir-pv-3
  resources:
    requests:
      storage: 10Gi
# 创建PV PVC
root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper/pv# kubectl  apply -f zookeeper-persistentvolume.yaml -f zookeeper-persistentvolumeclaim.yaml

root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper/pv# kubectl  get pv -n magedu
NAME                     CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                            STORAGECLASS   REASON   AGE
zookeeper-datadir-pv-1   20Gi       RWO            Retain           Bound    magedu/zookeeper-datadir-pvc-1                           2m12s
zookeeper-datadir-pv-2   20Gi       RWO            Retain           Bound    magedu/zookeeper-datadir-pvc-2                           2m12s
zookeeper-datadir-pv-3   20Gi       RWO            Retain           Bound    magedu/zookeeper-datadir-pvc-3                           2m12s
root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper/pv# kubectl  get pvc -n magedu
NAME                      STATUS   VOLUME                   CAPACITY   ACCESS MODES   STORAGECLASS   AGE
zookeeper-datadir-pvc-1   Bound    zookeeper-datadir-pv-1   20Gi       RWO                           21s
zookeeper-datadir-pvc-2   Bound    zookeeper-datadir-pv-2   20Gi       RWO                           21s
zookeeper-datadir-pvc-3   Bound    zookeeper-datadir-pv-3   20Gi       RWO                           21s
root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper/pv#

root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper# cat zookeeper.yaml
apiVersion: v1
kind: Service
metadata:
  name: zookeeper
  namespace: magedu
spec:
  ports:
    - name: client
      port: 2181
  selector:
    app: zookeeper
---
apiVersion: v1
kind: Service
metadata:
  name: zookeeper1
  namespace: magedu
spec:
  type: NodePort
  ports:
    - name: client
      port: 2181
      nodePort: 32181
    - name: followers
      port: 2888
    - name: election
      port: 3888
  selector:
    app: zookeeper
    server-id: "1"
---
apiVersion: v1
kind: Service
metadata:
  name: zookeeper2
  namespace: magedu
spec:
  type: NodePort
  ports:
    - name: client
      port: 2181
      nodePort: 32182
    - name: followers
      port: 2888
    - name: election
      port: 3888
  selector:
    app: zookeeper
    server-id: "2"
---
apiVersion: v1
kind: Service
metadata:
  name: zookeeper3
  namespace: magedu
spec:
  type: NodePort
  ports:
    - name: client
      port: 2181
      nodePort: 32183
    - name: followers
      port: 2888
    - name: election
      port: 3888
  selector:
    app: zookeeper
    server-id: "3"
---
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  name: zookeeper1
  namespace: magedu
spec:
  replicas: 1
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
        server-id: "1"
    spec:
      volumes:
        - name: data
          emptyDir: {}
        - name: wal
          emptyDir:
            medium: Memory
      containers:
        - name: server
          image: harbor.linuxarchitect.io/magedu/zookeeper:v3.4.14
          imagePullPolicy: Always
          env:
            - name: MYID
              value: "1"
            - name: SERVERS
              value: "zookeeper1,zookeeper2,zookeeper3"
            - name: JVMFLAGS
              value: "-Xmx2G"
          ports:
            - containerPort: 2181
            - containerPort: 2888
            - containerPort: 3888
          volumeMounts:
          - mountPath: "/zookeeper/data"
            name: zookeeper-datadir-pvc-1
      volumes:
        - name: zookeeper-datadir-pvc-1
          persistentVolumeClaim:
            claimName: zookeeper-datadir-pvc-1
---
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  name: zookeeper2
  namespace: magedu
spec:
  replicas: 1
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
        server-id: "2"
    spec:
      volumes:
        - name: data
          emptyDir: {}
        - name: wal
          emptyDir:
            medium: Memory
      containers:
        - name: server
          image: harbor.linuxarchitect.io/magedu/zookeeper:v3.4.14
          imagePullPolicy: Always
          env:
            - name: MYID
              value: "2"
            - name: SERVERS
              value: "zookeeper1,zookeeper2,zookeeper3"
            - name: JVMFLAGS
              value: "-Xmx2G"
          ports:
            - containerPort: 2181
            - containerPort: 2888
            - containerPort: 3888
          volumeMounts:
          - mountPath: "/zookeeper/data"
            name: zookeeper-datadir-pvc-2
      volumes:
        - name: zookeeper-datadir-pvc-2
          persistentVolumeClaim:
            claimName: zookeeper-datadir-pvc-2
---
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  name: zookeeper3
  namespace: magedu
spec:
  replicas: 1
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
        server-id: "3"
    spec:
      volumes:
        - name: data
          emptyDir: {}
        - name: wal
          emptyDir:
            medium: Memory
      containers:
        - name: server
          image: harbor.linuxarchitect.io/magedu/zookeeper:v3.4.14
          imagePullPolicy: Always
          env:
            - name: MYID
              value: "3"
            - name: SERVERS
              value: "zookeeper1,zookeeper2,zookeeper3"
            - name: JVMFLAGS
              value: "-Xmx2G"
          ports:
            - containerPort: 2181
            - containerPort: 2888
            - containerPort: 3888
          volumeMounts:
          - mountPath: "/zookeeper/data"
            name: zookeeper-datadir-pvc-3
      volumes:
        - name: zookeeper-datadir-pvc-3
          persistentVolumeClaim:
           claimName: zookeeper-datadir-pvc-3


root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper# sed -i s'#harbor.linuxarchitect.io#harbor.iclinux.com#g' zookeeper.yaml
root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper# kubectl  apply -f zookeeper.yaml
# 检查
root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper# kubectl  exec -it zookeeper3-6775684d7c-2bljc -n magedu -- bash
bash-4.3# /zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
ZooKeeper remote JMX Port set to 9010
ZooKeeper remote JMX authenticate set to false
ZooKeeper remote JMX ssl set to false
ZooKeeper remote JMX log4j set to true
Using config: /zookeeper/bin/../conf/zoo.cfg
Mode: leader
root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper# kubectl  exec -it zookeeper2-74d59c877d-bxh2d -n magedu -- bash
bash-4.3#  /zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
ZooKeeper remote JMX Port set to 9010
ZooKeeper remote JMX authenticate set to false
ZooKeeper remote JMX ssl set to false
ZooKeeper remote JMX log4j set to true
Using config: /zookeeper/bin/../conf/zoo.cfg
Mode: follower
bash-4.3# exit
exit
root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper# kubectl  exec -it zookeeper1-77d9cdc8c5-zp8p4 -n magedu -- bash
bash-4.3# /zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
ZooKeeper remote JMX Port set to 9010
ZooKeeper remote JMX authenticate set to false
ZooKeeper remote JMX ssl set to false
ZooKeeper remote JMX log4j set to true
Using config: /zookeeper/bin/../conf/zoo.cfg
Mode: follower
#关闭harbor
root@k8s-harbor1:/apps/harbor# docker-compose  stop
#删除主
root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper# kubectl delete pod zookeeper3-6775684d7c-2bljc -n magedu
pod "zookeeper3-6775684d7c-2bljc" deleted

root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper# kubectl  exec -it zookeeper2-74d59c877d-bxh2d  -n magedu   -- bash
bash-4.3#  /zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
ZooKeeper remote JMX Port set to 9010
ZooKeeper remote JMX authenticate set to false
ZooKeeper remote JMX ssl set to false
ZooKeeper remote JMX log4j set to true
Using config: /zookeeper/bin/../conf/zoo.cfg
Mode: follower
bash-4.3# exit
exit
root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper# kubectl  exec -it  zookeeper1-77d9cdc8c5-zp8p4 -n magedu   -- bash
bash-4.3#   /zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
ZooKeeper remote JMX Port set to 9010
ZooKeeper remote JMX authenticate set to false
ZooKeeper remote JMX ssl set to false
ZooKeeper remote JMX log4j set to true
Using config: /zookeeper/bin/../conf/zoo.cfg
Mode: leader
# 此时重新选举成功

```

3、在 Kubernetes 环境部署基于 StatefulSet 运行 MySQL 一主多从并基于 NFS 或 StorageClass 等方式实现数据持久化。

```bash
# 镜像制备
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu# docker pull mysql:5.7.36

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu# docker tag mysql:5.7.36 harbor.iclinux.com/magedu/mysql:5.7.36
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu# docker push harbor.iclinux.com/magedu/mysql:5.7.36

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu# docker pull zhangshijie/xtrabackup:1.0

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu# docker tag zhangshijie/xtrabackup:1.0 harbor.iclinux.com/magedu/xtrabackup:1.0
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu# docker push harbor.iclinux.com/magedu/xtrabackup:1.0

# pv 准备
## nfs 服务器准备
mkdir -p /data/k8sdata/magedu/mysql-datadir-1
mkdir -p /data/k8sdata/magedu/mysql-datadir-2
mkdir -p /data/k8sdata/magedu/mysql-datadir-3
mkdir -p /data/k8sdata/magedu/mysql-datadir-4
mkdir -p /data/k8sdata/magedu/mysql-datadir-5
mkdir -p /data/k8sdata/magedu/mysql-datadir-6

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql/pv# cat mysql-persistentvolume.yaml
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-1
  namespace: magedu
spec:
  capacity:
    storage: 50Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/k8sdata/magedu/mysql-datadir-1
    server: 172.31.7.109
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-2
  namespace: magedu
spec:
  capacity:
    storage: 50Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/k8sdata/magedu/mysql-datadir-2
    server: 172.31.7.109
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-3
  namespace: magedu
spec:
  capacity:
    storage: 50Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/k8sdata/magedu/mysql-datadir-3
    server: 172.31.7.109
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-4
  namespace: magedu
spec:
  capacity:
    storage: 50Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/k8sdata/magedu/mysql-datadir-4
    server: 172.31.7.109
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-5
  namespace: magedu
spec:
  capacity:
    storage: 50Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/k8sdata/magedu/mysql-datadir-5
    server: 172.31.7.109

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-6
  namespace: magedu
spec:
  capacity:
    storage: 50Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/k8sdata/magedu/mysql-datadir-6
    server: 172.31.7.109

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql/pv# kubectl  apply  -f mysql-persistentvolume.yaml
persistentvolume/mysql-datadir-1 created
persistentvolume/mysql-datadir-2 created
persistentvolume/mysql-datadir-3 created
persistentvolume/mysql-datadir-4 created
persistentvolume/mysql-datadir-5 created
persistentvolume/mysql-datadir-6 created
root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql/pv# kubectl  get pv -n magedu
NAME                     CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                            STORAGECLASS   REASON   AGE
mysql-datadir-1          50Gi       RWO            Retain           Available                                                            3s
mysql-datadir-2          50Gi       RWO            Retain           Available                                                            3s
mysql-datadir-3          50Gi       RWO            Retain           Available                                                            3s
mysql-datadir-4          50Gi       RWO            Retain           Available                                                            3s
mysql-datadir-5          50Gi       RWO            Retain           Available                                                            3s
mysql-datadir-6          50Gi       RWO            Retain           Available                                                            3s
zookeeper-datadir-pv-1   20Gi       RWO            Retain           Bound       magedu/zookeeper-datadir-pvc-1                           139m
zookeeper-datadir-pv-2   20Gi       RWO            Retain           Bound       magedu/zookeeper-datadir-pvc-2                           139m
zookeeper-datadir-pv-3   20Gi       RWO            Retain           Bound       magedu/zookeeper-datadir-pvc-3                           139m
root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql/pv#

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# cat mysql-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mysql
  namespace: magedu
  labels:
    app: mysql
data:
  master.cnf: |
    # Apply this config only on the master.
    [mysqld]
    log-bin
    log_bin_trust_function_creators=1
    lower_case_table_names=1
  slave.cnf: |
    # Apply this config only on slaves.
    [mysqld]
    super-read-only
    log_bin_trust_function_creators=1
root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl  apply -f mysql-configmap.yaml
configmap/mysql created

# sed -i s'#harbor.linuxarchitect.io#harbor.iclinux.com#g' mysql-statefulset.yaml

# sed -i s'#registry.cn-hangzhou.aliyuncs.com/hxpdocker/xtrabackup:1.0#harbor.iclinux.com/magedu/xtrabackup:1.0#g' mysql-statefulset.yaml


root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# cat mysql-services.yaml
# Headless service for stable DNS entries of StatefulSet members.
apiVersion: v1
kind: Service
metadata:
  namespace: magedu
  name: mysql
  labels:
    app: mysql
spec:
  ports:
  - name: mysql
    port: 3306
  clusterIP: None
  selector:
    app: mysql
---
# Client service for connecting to any MySQL instance for reads.
# For writes, you must instead connect to the master: mysql-0.mysql.
apiVersion: v1
kind: Service
metadata:
  name: mysql-read
  namespace: magedu
  labels:
    app: mysql
spec:
  ports:
  - name: mysql
    port: 3306
  selector:
    app: mysql
root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl  apply -f  mysql-services.yaml
service/mysql created
service/mysql-read created


root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# cat mysql-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: mysql
  namespace: magedu
spec:
  selector:
    matchLabels:
      app: mysql
  serviceName: mysql
  replicas: 2
  template:
    metadata:
      labels:
        app: mysql
    spec:
      initContainers:
      - name: init-mysql #初始化容器1、基于当前pod name匹配角色是master还是slave，并动态生成相对应的配置文件
        image: harbor.iclinux.com/magedu/mysql:5.7.36
        command:
        - bash
        - "-c"
        - |
          set -ex
          # Generate mysql server-id from pod ordinal index.
          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1 #匹配hostname的最后一位、最后是一个顺序叠加的整数
          ordinal=${BASH_REMATCH[1]}
          echo [mysqld] > /mnt/conf.d/server-id.cnf
          # Add an offset to avoid reserved server-id=0 value.
          echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf
          # Copy appropriate conf.d files from config-map to emptyDir.
          if [[ $ordinal -eq 0 ]]; then #如果是master、则cpmaster配置文件
            cp /mnt/config-map/master.cnf /mnt/conf.d/
          else #否则cp slave配置文件
            cp /mnt/config-map/slave.cnf /mnt/conf.d/
          fi
        volumeMounts:
        - name: conf #临时卷、emptyDir
          mountPath: /mnt/conf.d
        - name: config-map
          mountPath: /mnt/config-map
      - name: clone-mysql #初始化容器2、用于生成mysql配置文件、并从上一个pod完成首次的全量数据clone(slave 3从slave2 clone，而不是每个slave都从master clone实现首次全量同步,但是后期都是与master实现增量同步)
        image: harbor.iclinux.com/magedu/xtrabackup:1.0
        command:
        - bash
        - "-c"
        - |
          set -ex
          # Skip the clone if data already exists.
          [[ -d /var/lib/mysql/mysql ]] && exit 0
          # Skip the clone on master (ordinal index 0).
          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
          ordinal=${BASH_REMATCH[1]}
          [[ $ordinal -eq 0 ]] && exit 0 #如果最后一位是0(master)则退出clone过程
          # Clone data from previous peer.
          ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql #从上一个pod执行clone(binlog),xbstream为解压缩命令
          # Prepare the backup.xue
          xtrabackup --prepare --target-dir=/var/lib/mysql #通过xtrabackup恢复binlog
        volumeMounts:
        - name: data
          mountPath: /var/lib/mysql
          subPath: mysql
        - name: conf
          mountPath: /etc/mysql/conf.d
      containers:
      - name: mysql #业务容器1(mysql主容器)
        image: harbor.iclinux.com/magedu/mysql:5.7.36
        env:
        - name: MYSQL_ALLOW_EMPTY_PASSWORD
          value: "1"
        ports:
        - name: mysql
          containerPort: 3306
        volumeMounts:
        - name: data #挂载数据目录至/var/lib/mysql
          mountPath: /var/lib/mysql
          subPath: mysql
        - name: conf #配置文件/etc/mysql/conf.d
          mountPath: /etc/mysql/conf.d
        resources: #资源限制
          requests:
            cpu: 500m
            memory: 1Gi
        livenessProbe: #存活探针
          exec:
            command: ["mysqladmin", "ping"]
          initialDelaySeconds: 30
          periodSeconds: 10
          timeoutSeconds: 5
        readinessProbe: #就绪探针
          exec:
            # Check we can execute queries over TCP (skip-networking is off).
            command: ["mysql", "-h", "127.0.0.1", "-e", "SELECT 1"]
          initialDelaySeconds: 5
          periodSeconds: 2
          timeoutSeconds: 1
      - name: xtrabackup #业务容器2(xtrabackup),用于后期同步master 的binglog并恢复数据
        image: harbor.iclinux.com/magedu/xtrabackup:1.0
        ports:
        - name: xtrabackup
          containerPort: 3307
        command:
        - bash
        - "-c"
        - |
          set -ex
          cd /var/lib/mysql
          # Determine binlog position of cloned data, if any.
          if [[ -f xtrabackup_slave_info ]]; then
            # XtraBackup already generated a partial "CHANGE MASTER TO" query
            # because we're cloning from an existing slave.
            mv xtrabackup_slave_info change_master_to.sql.in
            # Ignore xtrabackup_binlog_info in this case (it's useless).
            rm -f xtrabackup_binlog_info
          elif [[ -f xtrabackup_binlog_info ]]; then
            # We're cloning directly from master. Parse binlog position.
            [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
            rm xtrabackup_binlog_info
            echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\
                  MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in #生成CHANGE MASTER命令
          fi
          # Check if we need to complete a clone by starting replication.
          if [[ -f change_master_to.sql.in ]]; then
            echo "Waiting for mysqld to be ready (accepting connections)"
            until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done
            echo "Initializing replication from clone position"
            # In case of container restart, attempt this at-most-once.
            mv change_master_to.sql.in change_master_to.sql.orig
            #执行CHANGE MASTER操作并启动SLAVE
            mysql -h 127.0.0.1 <<EOF
          $(<change_master_to.sql.orig),
            MASTER_HOST='mysql-0.mysql',
            MASTER_USER='root',
            MASTER_PASSWORD='',
            MASTER_CONNECT_RETRY=10;
          START SLAVE;
          EOF
          fi
          # Start a server to send backups when requested by peers. #监听在3307端口，用于为下一个pod同步全量数据
          exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c \
            "xtrabackup --backup --slave-info --stream=xbstream --host=127.0.0.1 --user=root"
        volumeMounts:
        - name: data
          mountPath: /var/lib/mysql
          subPath: mysql
        - name: conf
          mountPath: /etc/mysql/conf.d
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
      volumes:
      - name: conf
        emptyDir: {}
      - name: config-map
        configMap:
          name: mysql
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes: ["ReadWriteOnce"]
      resources:
        requests:
          storage: 10Gi
root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl apply -f mysql-statefulset.yaml
statefulset.apps/mysql created

```
### 检查验证
```bash
root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl  get  pods -n magedu
NAME                          READY   STATUS     RESTARTS       AGE
mysql-0                       0/2     Init:0/2   0              72s
zookeeper1-77d9cdc8c5-zp8p4   1/1     Running    0              156m
zookeeper2-74d59c877d-bxh2d   1/1     Running    1 (156m ago)   156m
zookeeper3-6775684d7c-t62g5   1/1     Running    0              150m

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl  describe pod -n magedu mysql-0

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl logs -f -n magedu mysql-1 -c init-mysql
++ hostname
+ [[ mysql-1 =~ -([0-9]+)$ ]]
+ ordinal=1
+ echo '[mysqld]'
+ echo server-id=101
+ [[ 1 -eq 0 ]]
+ cp /mnt/config-map/slave.cnf /mnt/conf.d/
root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl logs -f -n magedu mysql-1 -c  clone-mysql
+ [[ -d /var/lib/mysql/mysql ]]
++ hostname
+ [[ mysql-1 =~ -([0-9]+)$ ]]
+ ordinal=1
+ [[ 1 -eq 0 ]]
+ ncat --recv-only mysql-0.mysql 3307
+ xbstream -x -C /var/lib/mysql
Ncat: Could not resolve hostname "mysql-0.mysql": Name or service not known. QUITTING.
+ xtrabackup --prepare --target-dir=/var/lib/mysql
xtrabackup version 2.4.4 based on MySQL server 5.7.13 Linux (x86_64) (revision id: df58cf2)
xtrabackup: cd to /var/lib/mysql
xtrabackup: Error: cannot open ./xtrabackup_checkpoints
xtrabackup: error: xtrabackup_read_metadata()
xtrabackup: This target seems not to have correct metadata...
InnoDB: Number of pools: 1
InnoDB: Operating system error number 2 in a file operation.
InnoDB: The error means the system cannot find the path specified.
xtrabackup: Warning: cannot open ./xtrabackup_logfile. will try to find.
InnoDB: Operating system error number 2 in a file operation.
InnoDB: The error means the system cannot find the path specified.
  xtrabackup: Fatal error: cannot find ./xtrabackup_logfile.
xtrabackup: Error: xtrabackup_init_temp_log() failed.
root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql#

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl  get pods -n magedu
NAME                          READY   STATUS    RESTARTS       AGE
mysql-0                       2/2     Running   0              88s
mysql-1                       2/2     Running   1 (58s ago)    71s


```
### 增加从库
```bash
root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# vim mysql-statefulset.yaml
  replicas: 4

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl  get pods -n magedu
NAME                          READY   STATUS    RESTARTS        AGE
mysql-0                       2/2     Running   0               4m50s
mysql-1                       2/2     Running   1 (4m20s ago)   4m33s
mysql-2                       2/2     Running   1 (52s ago)     96s
mysql-3                       2/2     Running   1 (29s ago)     44s
zookeeper1-77d9cdc8c5-zp8p4   1/1     Running   0               3h2m
zookeeper2-74d59c877d-bxh2d   1/1     Running   1 (3h1m ago)    3h2m
zookeeper3-6775684d7c-t62g5   1/1     Running   0               175m
## 检查

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl  exec -it mysql-0 -n magedu -- bash
Defaulted container "mysql" out of: mysql, xtrabackup, init-mysql (init), clone-mysql (init)
root@mysql-0:/# mysql
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 216
Server version: 5.7.36-log MySQL Community Server (GPL)

Copyright (c) 2000, 2021, Oracle and/or its affiliates.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> show databases;
+------------------------+
| Database               |
+------------------------+
| information_schema     |
| mysql                  |
| performance_schema     |
| sys                    |
| xtrabackup_backupfiles |
+------------------------+
5 rows in set (0.05 sec)

mysql> create database t111;
Query OK, 1 row affected (0.01 sec)


mysql> show master status\G;
*************************** 1. row ***************************
             File: mysql-0-bin.000003
         Position: 313
     Binlog_Do_DB:
 Binlog_Ignore_DB:
Executed_Gtid_Set:
1 row in set (0.00 sec)

ERROR:
No query specified

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl  exec -it mysql-3 -n magedu -- bash
Defaulted container "mysql" out of: mysql, xtrabackup, init-mysql (init), clone-mysql (init)
root@mysql-3:/# msyql
bash: msyql: command not found
root@mysql-3:/# mysql
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 116
Server version: 5.7.36 MySQL Community Server (GPL)

Copyright (c) 2000, 2021, Oracle and/or its affiliates.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> show slave status\G;
*************************** 1. row ***************************
               Slave_IO_State: Waiting for master to send event
                  Master_Host: mysql-0.mysql
                  Master_User: root
                  Master_Port: 3306
                Connect_Retry: 10
              Master_Log_File: mysql-0-bin.000003
          Read_Master_Log_Pos: 313
               Relay_Log_File: mysql-3-relay-bin.000002
                Relay_Log_Pos: 481
        Relay_Master_Log_File: mysql-0-bin.000003
             Slave_IO_Running: Yes
            Slave_SQL_Running: Yes
              Replicate_Do_DB:
          Replicate_Ignore_DB:
           Replicate_Do_Table:
       Replicate_Ignore_Table:
      Replicate_Wild_Do_Table:
  Replicate_Wild_Ignore_Table:
                   Last_Errno: 0
                   Last_Error:
                 Skip_Counter: 0
          Exec_Master_Log_Pos: 313
              Relay_Log_Space: 690
              Until_Condition: None
               Until_Log_File:
                Until_Log_Pos: 0
           Master_SSL_Allowed: No
           Master_SSL_CA_File:
           Master_SSL_CA_Path:
              Master_SSL_Cert:
            Master_SSL_Cipher:
               Master_SSL_Key:
        Seconds_Behind_Master: 0
Master_SSL_Verify_Server_Cert: No
                Last_IO_Errno: 0
                Last_IO_Error:
               Last_SQL_Errno: 0
               Last_SQL_Error:
  Replicate_Ignore_Server_Ids:
             Master_Server_Id: 100
                  Master_UUID: 14aba6fb-bfd3-11ed-9553-4a971ef59ae9
             Master_Info_File: /var/lib/mysql/master.info
                    SQL_Delay: 0
          SQL_Remaining_Delay: NULL
      Slave_SQL_Running_State: Slave has read all relay log; waiting for more updates
           Master_Retry_Count: 86400
                  Master_Bind:
      Last_IO_Error_Timestamp:
     Last_SQL_Error_Timestamp:
               Master_SSL_Crl:
           Master_SSL_Crlpath:
           Retrieved_Gtid_Set:
            Executed_Gtid_Set:
                Auto_Position: 0
         Replicate_Rewrite_DB:
                 Channel_Name:
           Master_TLS_Version:
1 row in set (0.00 sec)


mysql> show databases;
+------------------------+
| Database               |
+------------------------+
| information_schema     |
| mysql                  |
| performance_schema     |
| sys                    |
| t111                   |
| xtrabackup_backupfiles |
+------------------------+
6 rows in set (0.02 sec)

```

4、在 Kubernetes 环境运行 java 单体服务 Jenkins(自己构建镜像或使用官方镜像)、以及实现单 Pod 中以多容器模式运行基于 LNMP 的 WordPress(自己构建镜像或使用官方镜像)，数据库使用上一步骤运行在 K8S 中的 MySQL。

### 基础镜像
```bash
root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/nginx-base# cat Dockerfile
#Nginx Base Image
FROM harbor.iclinux.com/baseimages/magedu-centos-base:7.9.2009

MAINTAINER  zhangshijie@magedu.net

RUN yum install -y vim wget tree  lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel openssl openssl-devel iproute net-tools iotop
ADD nginx-1.22.0.tar.gz /usr/local/src/
RUN cd /usr/local/src/nginx-1.22.0 && ./configure  && make && make install && ln -sv  /usr/local/nginx/sbin/nginx /usr/sbin/nginx  &&rm -rf /usr/local/src/nginx-1.22.0.tar.gz
root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/nginx-base# cat build-command.sh
#!/bin/bash
docker build -t harbor.iclinux.com/pub-images/nginx-base:v1.22.0  .
#sleep 1
docker push  harbor.iclinux.com/pub-images/nginx-base:v1.22.0

#nerdctl build -t  harbor.iclinux.com/pub-images/nginx-base:v1.22.0  .

#nerdctl push harbor.iclinux.com/pub-images/nginx-base:v1.22.0


root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/nginx-base# bash build-command.sh

```
### 业务镜像nginx
```bash

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/nginx# cat Dockerfile
#FROM harbor.magedu.local/pub-images/nginx-base-wordpress:v1.20.2
FROM harbor.iclinux.com/pub-images/nginx-base:v1.22.0

ADD nginx.conf /apps/nginx/conf/nginx.conf
ADD run_nginx.sh /apps/nginx/sbin/run_nginx.sh
RUN mkdir -pv /home/nginx/wordpress
RUN chown nginx.nginx /home/nginx/wordpress/ -R

EXPOSE 80 443

CMD ["/apps/nginx/sbin/run_nginx.sh"]
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/nginx# cat index.html
nginx web1
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/nginx# cat run_nginx.sh
#!/bin/bash
#echo "nameserver 10.20.254.254" > /etc/resolv.conf
#chown nginx.nginx /home/nginx/wordpress/ -R
/apps/nginx/sbin/nginx
tail -f /etc/hosts
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/nginx# cat nginx.conf
user  nginx nginx;
worker_processes  auto;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;

#daemon off;

events {
    worker_connections  1024;
}


http {
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    keepalive_timeout  65;
    client_max_body_size 10M;
    client_body_buffer_size 16k;
    client_body_temp_path  /apps/nginx/tmp   1 2 2;
    gzip  on;


    server {
        listen       80;
        server_name  blogs.magedu.net;

        #charset koi8-r;

        #access_log  logs/host.access.log  main;



        location / {
            root    /home/nginx/wordpress;
            index   index.php index.html index.htm;
            #if ($http_user_agent ~ "ApacheBench|WebBench|TurnitinBot|Sogou web spider|Grid Service") {
            #    proxy_pass http://www.baidu.com;
            #    #return 403;
            #}
        }

        location ~ \.php$ {
            root           /home/nginx/wordpress;
            fastcgi_pass   127.0.0.1:9000;
            fastcgi_index  index.php;
            #fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
            fastcgi_param  SCRIPT_FILENAME  $document_root$fastcgi_script_name;
             include        fastcgi_params;
        }


        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }

        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
        #    proxy_pass   http://127.0.0.1;
        #}

        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        #location ~ \.php$ {
        #    root           html;
        #    fastcgi_pass   127.0.0.1:9000;
        #    fastcgi_index  index.php;
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
        #    include        fastcgi_params;
        #}

        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        #
        #location ~ /\.ht {
        #    deny  all;
        #}
    }


    # another virtual host using mix of IP-, name-, and port-based configuration
    #
    #server {
    #    listen       8000;
    #    listen       somename:8080;
    #    server_name  somename  alias  another.alias;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}


    # HTTPS server
    #
    #server {
    #    listen       443 ssl;
    #    server_name  localhost;

    #    ssl_certificate      cert.pem;
    #    ssl_certificate_key  cert.key;

    #    ssl_session_cache    shared:SSL:1m;
    #    ssl_session_timeout  5m;

    #    ssl_ciphers  HIGH:!aNULL:!MD5;
    #    ssl_prefer_server_ciphers  on;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}

}
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/nginx# cat build-command.sh
#!/bin/bash
TAG=$1
docker build -t harbor.iclinux.com/magedu/wordpress-nginx:${TAG} .
#nerdctl build -t harbor.magedu.net/magedu/wordpress-nginx:${TAG} .
echo "镜像制作完成，即将上传至Harbor服务器"
sleep 1
#nerdctl push  harbor.magedu.net/magedu/wordpress-nginx:${TAG}
docker push  harbor.iclinux.com/magedu/wordpress-nginx:${TAG}
echo "镜像上传完成"
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/nginx# bash build-command.sh v1

# 验证镜像
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/nginx# docker run   -it --rm harbor.iclinux.com/magedu/wordpress-nginx:v1



```
### 制作php镜像
```bash

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/php# ls
Dockerfile  build-command.sh  run_php.sh  www.conf
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/php# cat Dockerfile
#PHP Base Image
#FROM harbor.magedu.net/baseimages/magedu-centos-base:7.9.2009
FROM harbor.iclinux.com/baseimages/magedu-centos-base:7.9.2009

MAINTAINER  zhangshijie@magedu.net

RUN yum install -y  https://mirrors.tuna.tsinghua.edu.cn/remi/enterprise/remi-release-7.rpm && yum install  php56-php-fpm php56-php-mysql -y
ADD www.conf /opt/remi/php56/root/etc/php-fpm.d/www.conf
#RUN useradd nginx -u 2019
ADD run_php.sh /usr/local/bin/run_php.sh
EXPOSE 9000

CMD ["/usr/local/bin/run_php.sh"]
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/php# cat run_php.sh
#!/bin/bash
#echo "nameserver 10.20.254.254" > /etc/resolv.conf

/opt/remi/php56/root/usr/sbin/php-fpm
#/opt/remi/php56/root/usr/sbin/php-fpm --nodaemonize
tail -f /etc/hosts
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/php# cat www.conf
; Start a new pool named 'www'.
; the variable $pool can we used in any directive and will be replaced by the
; pool name ('www' here)
[www]

; Per pool prefix
; It only applies on the following directives:
; - 'slowlog'
; - 'listen' (unixsocket)
; - 'chroot'
; - 'chdir'
; - 'php_values'
; - 'php_admin_values'
; When not set, the global prefix (or @php_fpm_prefix@) applies instead.
; Note: This directive can also be relative to the global prefix.
; Default Value: none
;prefix = /path/to/pools/$pool

; Unix user/group of processes
; Note: The user is mandatory. If the group is not set, the default user's group
;       will be used.
; RPM: apache user chosen to provide access to the same directories as httpd
user = nginx
; RPM: Keep a group allowed to write in log dir.
group = nginx

; The address on which to accept FastCGI requests.
; Valid syntaxes are:
;   'ip.add.re.ss:port'    - to listen on a TCP socket to a specific IPv4 address on
;                            a specific port;
;   '[ip:6:addr:ess]:port' - to listen on a TCP socket to a specific IPv6 address on
;                            a specific port;
;   'port'                 - to listen on a TCP socket to all IPv4 addresses on a
;                            specific port;
;   '[::]:port'            - to listen on a TCP socket to all addresses
;                            (IPv6 and IPv4-mapped) on a specific port;
;   '/path/to/unix/socket' - to listen on a unix socket.
; Note: This value is mandatory.
listen = 0.0.0.0:9000

; Set listen(2) backlog.
; Default Value: 65535
;listen.backlog = 65535

; Set permissions for unix socket, if one is used. In Linux, read/write
; permissions must be set in order to allow connections from a web server.
; Default Values: user and group are set as the running user
;                 mode is set to 0660
;listen.owner = nobody
;listen.group = nobody
;listen.mode = 0660

; When POSIX Access Control Lists are supported you can set them using
; these options, value is a comma separated list of user/group names.
; When set, listen.owner and listen.group are ignored
;listen.acl_users = apache
;listen.acl_groups =

; List of addresses (IPv4/IPv6) of FastCGI clients which are allowed to connect.
; Equivalent to the FCGI_WEB_SERVER_ADDRS environment variable in the original
; PHP FCGI (5.2.2+). Makes sense only with a tcp listening socket. Each address
; must be separated by a comma. If this value is left blank, connections will be
; accepted from any ip address.
; Default Value: any
; listen.allowed_clients = 127.0.0.1

; Specify the nice(2) priority to apply to the pool processes (only if set)
; The value can vary from -19 (highest priority) to 20 (lower priority)
; Note: - It will only work if the FPM master process is launched as root
;       - The pool processes will inherit the master process priority
;         unless it specified otherwise
; Default Value: no set
; process.priority = -19

; Set the process dumpable flag (PR_SET_DUMPABLE prctl) even if the process user
; or group is differrent than the master process user. It allows to create process
; core dump and ptrace the process for the pool user.
; Default Value: no
; process.dumpable = yes

; Choose how the process manager will control the number of child processes.
; Possible Values:
;   static  - a fixed number (pm.max_children) of child processes;
;   dynamic - the number of child processes are set dynamically based on the
;             following directives. With this process management, there will be
;             always at least 1 children.
;             pm.max_children      - the maximum number of children that can
;                                    be alive at the same time.
;             pm.start_servers     - the number of children created on startup.
;             pm.min_spare_servers - the minimum number of children in 'idle'
;                                    state (waiting to process). If the number
;                                    of 'idle' processes is less than this
;                                    number then some children will be created.
;             pm.max_spare_servers - the maximum number of children in 'idle'
;                                    state (waiting to process). If the number
;                                    of 'idle' processes is greater than this
;                                    number then some children will be killed.
;  ondemand - no children are created at startup. Children will be forked when
;             new requests will connect. The following parameter are used:
;             pm.max_children           - the maximum number of children that
;                                         can be alive at the same time.
;             pm.process_idle_timeout   - The number of seconds after which
;                                         an idle process will be killed.
; Note: This value is mandatory.
pm = dynamic

; The number of child processes to be created when pm is set to 'static' and the
; maximum number of child processes when pm is set to 'dynamic' or 'ondemand'.
; This value sets the limit on the number of simultaneous requests that will be
; served. Equivalent to the ApacheMaxClients directive with mpm_prefork.
; Equivalent to the PHP_FCGI_CHILDREN environment variable in the original PHP
; CGI. The below defaults are based on a server without much resources. Don't
; forget to tweak pm.* to fit your needs.
; Note: Used when pm is set to 'static', 'dynamic' or 'ondemand'
; Note: This value is mandatory.
pm.max_children = 50

; The number of child processes created on startup.
; Note: Used only when pm is set to 'dynamic'
; Default Value: min_spare_servers + (max_spare_servers - min_spare_servers) / 2
pm.start_servers = 5

; The desired minimum number of idle server processes.
; Note: Used only when pm is set to 'dynamic'
; Note: Mandatory when pm is set to 'dynamic'
pm.min_spare_servers = 5

; The desired maximum number of idle server processes.
; Note: Used only when pm is set to 'dynamic'
; Note: Mandatory when pm is set to 'dynamic'
pm.max_spare_servers = 35

; The number of seconds after which an idle process will be killed.
; Note: Used only when pm is set to 'ondemand'
; Default Value: 10s
;pm.process_idle_timeout = 10s;

; The number of requests each child process should execute before respawning.
; This can be useful to work around memory leaks in 3rd party libraries. For
; endless request processing specify '0'. Equivalent to PHP_FCGI_MAX_REQUESTS.
; Default Value: 0
;pm.max_requests = 500

; The URI to view the FPM status page. If this value is not set, no URI will be
; recognized as a status page. It shows the following informations:
;   pool                 - the name of the pool;
;   process manager      - static, dynamic or ondemand;
;   start time           - the date and time FPM has started;
;   start since          - number of seconds since FPM has started;
;   accepted conn        - the number of request accepted by the pool;
;   listen queue         - the number of request in the queue of pending
;                          connections (see backlog in listen(2));
;   max listen queue     - the maximum number of requests in the queue
;                          of pending connections since FPM has started;
;   listen queue len     - the size of the socket queue of pending connections;
;   idle processes       - the number of idle processes;
;   active processes     - the number of active processes;
;   total processes      - the number of idle + active processes;
;   max active processes - the maximum number of active processes since FPM
;                          has started;
;   max children reached - number of times, the process limit has been reached,
;                          when pm tries to start more children (works only for
;                          pm 'dynamic' and 'ondemand');
; Value are updated in real time.
; Example output:
;   pool:                 www
;   process manager:      static
;   start time:           01/Jul/2011:17:53:49 +0200
;   start since:          62636
;   accepted conn:        190460
;   listen queue:         0
;   max listen queue:     1
;   listen queue len:     42
;   idle processes:       4
;   active processes:     11
;   total processes:      15
;   max active processes: 12
;   max children reached: 0
;
; By default the status page output is formatted as text/plain. Passing either
; 'html', 'xml' or 'json' in the query string will return the corresponding
; output syntax. Example:
;   http://www.foo.bar/status
;   http://www.foo.bar/status?json
;   http://www.foo.bar/status?html
;   http://www.foo.bar/status?xml
;
; By default the status page only outputs short status. Passing 'full' in the
; query string will also return status for each pool process.
; Example:
;   http://www.foo.bar/status?full
;   http://www.foo.bar/status?json&full
;   http://www.foo.bar/status?html&full
;   http://www.foo.bar/status?xml&full
; The Full status returns for each process:
;   pid                  - the PID of the process;
;   state                - the state of the process (Idle, Running, ...);
;   start time           - the date and time the process has started;
;   start since          - the number of seconds since the process has started;
;   requests             - the number of requests the process has served;
;   request duration     - the duration in µs of the requests;
;   request method       - the request method (GET, POST, ...);
;   request URI          - the request URI with the query string;
;   content length       - the content length of the request (only with POST);
;   user                 - the user (PHP_AUTH_USER) (or '-' if not set);
;   script               - the main script called (or '-' if not set);
;   last request cpu     - the %cpu the last request consumed
;                          it's always 0 if the process is not in Idle state
;                          because CPU calculation is done when the request
;                          processing has terminated;
;   last request memory  - the max amount of memory the last request consumed
;                          it's always 0 if the process is not in Idle state
;                          because memory calculation is done when the request
;                          processing has terminated;
; If the process is in Idle state, then informations are related to the
; last request the process has served. Otherwise informations are related to
; the current request being served.
; Example output:
;   ************************
;   pid:                  31330
;   state:                Running
;   start time:           01/Jul/2011:17:53:49 +0200
;   start since:          63087
;   requests:             12808
;   request duration:     1250261
;   request method:       GET
;   request URI:          /test_mem.php?N=10000
;   content length:       0
;   user:                 -
;   script:               /home/fat/web/docs/php/test_mem.php
;   last request cpu:     0.00
;   last request memory:  0
;
; Note: There is a real-time FPM status monitoring sample web page available
;       It's available in: @EXPANDED_DATADIR@/fpm/status.html
;
; Note: The value must start with a leading slash (/). The value can be
;       anything, but it may not be a good idea to use the .php extension or it
;       may conflict with a real PHP file.
; Default Value: not set
;pm.status_path = /status

; The ping URI to call the monitoring page of FPM. If this value is not set, no
; URI will be recognized as a ping page. This could be used to test from outside
; that FPM is alive and responding, or to
; - create a graph of FPM availability (rrd or such);
; - remove a server from a group if it is not responding (load balancing);
; - trigger alerts for the operating team (24/7).
; Note: The value must start with a leading slash (/). The value can be
;       anything, but it may not be a good idea to use the .php extension or it
;       may conflict with a real PHP file.
; Default Value: not set
;ping.path = /ping

; This directive may be used to customize the response of a ping request. The
; response is formatted as text/plain with a 200 response code.
; Default Value: pong
;ping.response = pong

; The access log file
; Default: not set
;access.log = log/$pool.access.log

; The access log format.
; The following syntax is allowed
;  %%: the '%' character
;  %C: %CPU used by the request
;      it can accept the following format:
;      - %{user}C for user CPU only
;      - %{system}C for system CPU only
;      - %{total}C  for user + system CPU (default)
;  %d: time taken to serve the request
;      it can accept the following format:
;      - %{seconds}d (default)
;      - %{miliseconds}d
;      - %{mili}d
;      - %{microseconds}d
;      - %{micro}d
;  %e: an environment variable (same as $_ENV or $_SERVER)
;      it must be associated with embraces to specify the name of the env
;      variable. Some exemples:
;      - server specifics like: %{REQUEST_METHOD}e or %{SERVER_PROTOCOL}e
;      - HTTP headers like: %{HTTP_HOST}e or %{HTTP_USER_AGENT}e
;  %f: script filename
;  %l: content-length of the request (for POST request only)
;  %m: request method
;  %M: peak of memory allocated by PHP
;      it can accept the following format:
;      - %{bytes}M (default)
;      - %{kilobytes}M
;      - %{kilo}M
;      - %{megabytes}M
;      - %{mega}M
;  %n: pool name
;  %o: output header
;      it must be associated with embraces to specify the name of the header:
;      - %{Content-Type}o
;      - %{X-Powered-By}o
;      - %{Transfert-Encoding}o
;      - ....
;  %p: PID of the child that serviced the request
;  %P: PID of the parent of the child that serviced the request
;  %q: the query string
;  %Q: the '?' character if query string exists
;  %r: the request URI (without the query string, see %q and %Q)
;  %R: remote IP address
;  %s: status (response code)
;  %t: server time the request was received
;      it can accept a strftime(3) format:
;      %d/%b/%Y:%H:%M:%S %z (default)
;  %T: time the log has been written (the request has finished)
;      it can accept a strftime(3) format:
;      %d/%b/%Y:%H:%M:%S %z (default)
;  %u: remote user
;
; Default: "%R - %u %t \"%m %r\" %s"
;access.format = "%R - %u %t \"%m %r%Q%q\" %s %f %{mili}d %{kilo}M %C%%"

; The log file for slow requests
; Default Value: not set
; Note: slowlog is mandatory if request_slowlog_timeout is set
slowlog = /opt/remi/php56/root/var/log/php-fpm/www-slow.log

; The timeout for serving a single request after which a PHP backtrace will be
; dumped to the 'slowlog' file. A value of '0s' means 'off'.
; Available units: s(econds)(default), m(inutes), h(ours), or d(ays)
; Default Value: 0
;request_slowlog_timeout = 0

; The timeout for serving a single request after which the worker process will
; be killed. This option should be used when the 'max_execution_time' ini option
; does not stop script execution for some reason. A value of '0' means 'off'.
; Available units: s(econds)(default), m(inutes), h(ours), or d(ays)
; Default Value: 0
;request_terminate_timeout = 0

; Set open file descriptor rlimit.
; Default Value: system defined value
;rlimit_files = 1024

; Set max core size rlimit.
; Possible Values: 'unlimited' or an integer greater or equal to 0
; Default Value: system defined value
;rlimit_core = 0

; Chroot to this directory at the start. This value must be defined as an
; absolute path. When this value is not set, chroot is not used.
; Note: you can prefix with '$prefix' to chroot to the pool prefix or one
; of its subdirectories. If the pool prefix is not set, the global prefix
; will be used instead.
; Note: chrooting is a great security feature and should be used whenever
;       possible. However, all PHP paths will be relative to the chroot
;       (error_log, sessions.save_path, ...).
; Default Value: not set
;chroot =

; Chdir to this directory at the start.
; Note: relative path can be used.
; Default Value: current directory or / when chroot
;chdir = /var/www

; Redirect worker stdout and stderr into main error log. If not set, stdout and
; stderr will be redirected to /dev/null according to FastCGI specs.
; Note: on highloaded environement, this can cause some delay in the page
; process time (several ms).
; Default Value: no
;catch_workers_output = yes

; Clear environment in FPM workers
; Prevents arbitrary environment variables from reaching FPM worker processes
; by clearing the environment in workers before env vars specified in this
; pool configuration are added.
; Setting to "no" will make all environment variables available to PHP code
; via getenv(), $_ENV and $_SERVER.
; Default Value: yes
;clear_env = no

; Limits the extensions of the main script FPM will allow to parse. This can
; prevent configuration mistakes on the web server side. You should only limit
; FPM to .php extensions to prevent malicious users to use other extensions to
; exectute php code.
; Note: set an empty value to allow all extensions.
; Default Value: .php
;security.limit_extensions = .php .php3 .php4 .php5

; Pass environment variables like LD_LIBRARY_PATH. All $VARIABLEs are taken from
; the current environment.
; Default Value: clean env
;env[HOSTNAME] = $HOSTNAME
;env[PATH] = /usr/local/bin:/usr/bin:/bin
;env[TMP] = /tmp
;env[TMPDIR] = /tmp
;env[TEMP] = /tmp

; Additional php.ini defines, specific to this pool of workers. These settings
; overwrite the values previously defined in the php.ini. The directives are the
; same as the PHP SAPI:
;   php_value/php_flag             - you can set classic ini defines which can
;                                    be overwritten from PHP call 'ini_set'.
;   php_admin_value/php_admin_flag - these directives won't be overwritten by
;                                     PHP call 'ini_set'
; For php_*flag, valid values are on, off, 1, 0, true, false, yes or no.

; Defining 'extension' will load the corresponding shared extension from
; extension_dir. Defining 'disable_functions' or 'disable_classes' will not
; overwrite previously defined php.ini values, but will append the new value
; instead.

; Note: path INI options can be relative and will be expanded with the prefix
; (pool, global or @prefix@)

; Default Value: nothing is defined by default except the values in php.ini and
;                specified at startup with the -d argument
;php_admin_value[sendmail_path] = /usr/sbin/sendmail -t -i -f www@my.domain.com
;php_flag[display_errors] = off
php_admin_value[error_log] = /opt/remi/php56/root/var/log/php-fpm/www-error.log
php_admin_flag[log_errors] = on
;php_admin_value[memory_limit] = 128M

; Set the following data paths to directories owned by the FPM process user.
;
; Do not change the ownership of existing system directories, if the process
; user does not have write permission, create dedicated directories for this
; purpose.
;
; See warning about choosing the location of these directories on your system
; at http://php.net/session.save-path
php_value[session.save_handler] = files
php_value[session.save_path]    = /opt/remi/php56/root/var/lib/php/session
php_value[soap.wsdl_cache_dir]  = /opt/remi/php56/root/var/lib/php/wsdlcache

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/php# cat build-command.sh
#!/bin/bash
TAG=$1
docker build -t harbor.iclinux.com/magedu/wordpress-php-5.6:${TAG} .
#nerdctl build -t harbor.magedu.net/magedu/wordpress-php-5.6:${TAG} .
echo "镜像制作完成，即将上传至Harbor服务器"
sleep 1
#nerdctl push harbor.magedu.net/magedu/wordpress-php-5.6:${TAG}
docker push harbor.iclinux.com/magedu/wordpress-php-5.6:${TAG}
echo "镜像上传完成"

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/php# bash build-command.sh  v1
```
### 部署wordpress
```bash
# 创建存储目录
root@k8s-ha1:/data/k8sdata/magedu# mkdir -p /data/k8sdata/magedu/wordpress

root@k8s-master1:/opt/k8s-data/yaml/magedu/wordpress# cat wordpress.yaml
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
  labels:
    app: wordpress-app
  name: wordpress-app-deployment
  namespace: magedu
spec:
  replicas: 1
  selector:
    matchLabels:
      app: wordpress-app
  template:
    metadata:
      labels:
        app: wordpress-app
    spec:
      containers:
      - name: wordpress-app-nginx
        image: harbor.iclinux.com/magedu/wordpress-nginx:v1
        imagePullPolicy: Always
        ports:
        - containerPort: 80
          protocol: TCP
          name: http
        - containerPort: 443
          protocol: TCP
          name: https
        volumeMounts:
        - name: wordpress
          mountPath: /home/nginx/wordpress
          readOnly: false

      - name: wordpress-app-php
        image: harbor.iclinux.com/magedu/wordpress-php-5.6:v1
        #image: harbor.magedu.net/magedu/php:5.6.40-fpm
        #imagePullPolicy: IfNotPresent
        imagePullPolicy: Always
        ports:
        - containerPort: 9000
          protocol: TCP
          name: http
        volumeMounts:
        - name: wordpress
          mountPath: /home/nginx/wordpress
          readOnly: false

      volumes:
      - name: wordpress
        nfs:
          server: 172.31.7.109
          path: /data/k8sdata/magedu/wordpress


---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: wordpress-app
  name: wordpress-app-spec
  namespace: magedu
spec:
  type: NodePort
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 80
    nodePort: 30031
  - name: https
    port: 443
    protocol: TCP
    targetPort: 443
    nodePort: 30033
  selector:
    app: wordpress-app
root@k8s-master1:/opt/k8s-data/yaml/magedu/wordpress# kubectl  apply -f wordpress.yaml
deployment.apps/wordpress-app-deployment created
service/wordpress-app-spec created

# 设置负载均衡器

tee -a /etc/haproxy/haproxy.cfg << "EOF"

listen magedu-wordpress-80
 bind 172.31.7.189:80
 mode tcp
 #balance leastconn
 server 172.31.7.101 172.31.7.101:30031 check inter 2000 fall 3 rise 5
 server 172.31.7.102 172.31.7.102:30031 check inter 2000 fall 3 rise 5
 server 172.31.7.103 172.31.7.103:30031 check inter 2000 fall 3 rise 5

EOF
systemctl reload haproxy.service

```
### 排查wordprss无法启动问题
```bash

root@k8s-master1:/opt/k8s-data/yaml/magedu/wordpress# kubectl  get pod -n magedu
NAME                                        READY   STATUS    RESTARTS        AGE
magedu-jenkins-deployment-67678599c-6nxbk   1/1     Running   0               63m
mysql-0                                     2/2     Running   0               135m
mysql-1                                     2/2     Running   1 (135m ago)    135m
mysql-2                                     2/2     Running   1 (131m ago)    132m
mysql-3                                     2/2     Running   1 (131m ago)    131m
wordpress-app-deployment-77797b6b6-4445k    2/2     Running   0               14m
zookeeper1-77d9cdc8c5-zp8p4                 1/1     Running   0               5h13m
zookeeper2-74d59c877d-bxh2d                 1/1     Running   1 (5h12m ago)   5h13m
zookeeper3-6775684d7c-t62g5                 1/1     Running   0               5h6m
root@k8s-master1:/opt/k8s-data/yaml/magedu/wordpress# kubectl  exec -it wordpress-app-deployment-77797b6b6-4445k  -n magedu -- bash
Defaulted container "wordpress-app-nginx" out of: wordpress-app-nginx, wordpress-app-php
[root@wordpress-app-deployment-77797b6b6-4445k /]#
[root@wordpress-app-deployment-77797b6b6-4445k /]#
[root@wordpress-app-deployment-77797b6b6-4445k /]# ^C
[root@wordpress-app-deployment-77797b6b6-4445k /]# hostname
wordpress-app-deployment-77797b6b6-4445k
[root@wordpress-app-deployment-77797b6b6-4445k /]# netstat  -ntlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:9000            0.0.0.0:*               LISTEN      -
[root@wordpress-app-deployment-77797b6b6-4445k /]#
[root@wordpress-app-deployment-77797b6b6-4445k /]#
[root@wordpress-app-deployment-77797b6b6-4445k /]# exit
exit
root@k8s-master1:/opt/k8s-data/yaml/magedu/wordpress# kubectl  exec -it wordpress-app-deployment-77797b6b6-4445k  -n magedu -c wordpress-app-nginx -- bash
[root@wordpress-app-deployment-77797b6b6-4445k /]# netstat  -ntlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:9000            0.0.0.0:*               LISTEN      -
[root@wordpress-app-deployment-77797b6b6-4445k /]# ps -ef|grep nginx
root           1       0  0 16:09 ?        00:00:00 /bin/bash /apps/nginx/sbin/run_nginx.sh
root          46      28  0 16:27 pts/0    00:00:00 grep --color=auto nginx

[root@wordpress-app-deployment-77797b6b6-4445k sbin]# /apps/nginx/sbin/nginx
bash: /apps/nginx/sbin/nginx: No such file or directory

# 查看nginx-base-wordpress 发现问题

root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/nginx-base-wordpress#    sed -i s'#harbor.linuxarchitect.io#harbor.iclinux.com#g' Dockerfile

root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/nginx-base-wordpress# sed -i s'#harbor.linuxarchitect.io#harbor.iclinux.com#g' build-command.sh

root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/nginx-base-wordpress# cat Dockerfile
#Nginx Base Image
FROM harbor.iclinux.com/baseimages/magedu-centos-base:7.9.2009

MAINTAINER  zhangshijie@magedu.net

RUN yum install -y vim wget tree  lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel openssl openssl-devel iproute net-tools iotop
ADD nginx-1.22.0.tar.gz /usr/local/src/
RUN cd /usr/local/src/nginx-1.22.0 && ./configure --prefix=/apps/nginx  && make && make install && ln -sv  /apps/nginx/sbin/nginx /usr/sbin/nginx  &&rm -rf /usr/local/src/nginx-1.22.0.tar.gz
root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/nginx-base-wordpress# cat build-command.sh
#!/bin/bash
docker build -t harbor.iclinux.com/pub-images/nginx-base-wordpress:v1.22.0  .
#sleep 1
docker push  harbor.iclinux.com/pub-images/nginx-base-wordpress:v1.22.0

#nerdctl build -t  harbor.magedu.net/pub-images/nginx-base-wordpress:v1.22.0  .

#nerdctl push harbor.magedu.net/pub-images/nginx-base-wordpress:v1.22.0
root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/nginx-base-wordpress# bash build-command.sh

## dockerfile 出现异常，修改后重新打包
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/nginx# cat Dockerfile
#FROM harbor.magedu.local/pub-images/nginx-base-wordpress:v1.20.2
FROM harbor.iclinux.com/pub-images/nginx-base-wordpress:v1.22.0

ADD nginx.conf /apps/nginx/conf/nginx.conf
ADD run_nginx.sh /apps/nginx/sbin/run_nginx.sh
RUN mkdir -pv /home/nginx/wordpress
RUN chown nginx.nginx /home/nginx/wordpress/ -R

EXPOSE 80 443

CMD ["/apps/nginx/sbin/run_nginx.sh"]

# 重新打包
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/nginx# bash build-command.sh  v2
# 验证

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/nginx# docker  run -it --rm -p 80:80 harbor.iclinux.com/magedu/wordpress-nginx:v2

#更换新的镜像
root@k8s-master1:/opt/k8s-data/yaml/magedu/wordpress# vim wordpress.yaml
        image: harbor.iclinux.com/magedu/wordpress-nginx:v2

root@k8s-master1:/opt/k8s-data/yaml/magedu/wordpress# kubectl   apply -f wordpress.yaml
deployment.apps/wordpress-app-deployment configured
service/wordpress-app-spec unchanged

root@k8s-master1:/opt/k8s-data/yaml/magedu/wordpress# kubectl  get pods -n magedu
NAME                                        READY   STATUS    RESTARTS        AGE
magedu-jenkins-deployment-67678599c-6nxbk   1/1     Running   0               102m
mysql-0                                     2/2     Running   0               175m
mysql-1                                     2/2     Running   1 (174m ago)    174m
mysql-2                                     2/2     Running   1 (171m ago)    171m
mysql-3                                     2/2     Running   1 (170m ago)    171m
wordpress-app-deployment-6d8f5fffd5-slp26   2/2     Running   0               14s
zookeeper1-77d9cdc8c5-zp8p4                 1/1     Running   0               5h52m
zookeeper2-74d59c877d-bxh2d                 1/1     Running   1 (5h52m ago)   5h52m
zookeeper3-6775684d7c-t62g5                 1/1     Running   0               5h45m

#验证发现nginx已经正常工作：
http://172.31.7.113:30031/
http://172.31.7.189/

```
### php  功能验证
```bash
# nfs 服务器执行
/data/k8sdata/magedu/wordpress
tee  index.php <<"EOF"
<?php
  phpinfo();
?>
EOF
```
![](assets/k8s/img-20230311175214.png)
设置域名解析
![](assets/k8s/img-20230311175741.png)

### 部署wordpress
- 安装wordpress
root@k8s-ha1:/data/k8sdata/magedu/wordpress# tar xf wordpress-5.0.16-zh_CN.tar.gz
root@k8s-ha1:/data/k8sdata/magedu/wordpress# mv wordpress/* .

root@k8s-ha1:/data/k8sdata/magedu/wordpress# mv wordpress /tmp/
root@k8s-ha1:/data/k8sdata/magedu/wordpress# mv wordpress-5.0.16-zh_CN.tar.gz  /tmp/
root@k8s-ha1:/data/k8sdata/magedu/wordpress#
root@k8s-ha1:/data/k8sdata/magedu/wordpress# mv wordpress /tmp/
root@k8s-ha1:/data/k8sdata/magedu/wordpress# mv wordpress-5.0.16-zh_CN.tar.gz  /tmp/
root@k8s-ha1:/data/k8sdata/magedu/wordpress#

> 此时，刷新页面就出现了wp初始化页面
> ![](assets/k8s/img-20230311181115.png)

- 配置数据库
```bash

root@k8s-master1:/opt/k8s-data/yaml/magedu/wordpress# kubectl  exec -it mysql-0 -n magedu -- bash
Defaulted container "mysql" out of: mysql, xtrabackup, init-mysql (init), clone-mysql (init)
root@mysql-0:/# mysql
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 9001
Server version: 5.7.36-log MySQL Community Server (GPL)

Copyright (c) 2000, 2021, Oracle and/or its affiliates.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> create database wordpress;
Query OK, 1 row affected (0.03 sec)

mysql> grant all privileges on wordpress.* to "wordpress"@"%" identified by "wordpress";
Query OK, 0 rows affected, 1 warning (0.03 sec)

mysql>

```
![](assets/k8s/img-20230311182732.png)
![](assets/k8s/img-20230311182826.png)
配置文件路径：/data/k8sdata/magedu/wordpress # ll wp-config.php

## 登录dashboard
```bash
root@k8s-master1:~# kubectl  get secret -A |grep admin
kubernetes-dashboard   admin-user-token-42lw8
-n kubernetes-dashboard   admin-user-token-42lw8
Name:         admin-user-token-42lw8
Namespace:    kubernetes-dashboard
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: admin-user
              kubernetes.io/service-account.uid: d9508964-883f-4a72-a5e7-9e7be5c8f56f

Type:  kubernetes.io/service-account-token

Data
====
ca.crt:     1302 bytes
namespace:  20 bytes

root@k8s-master1:~# kubectl describe secrets  -n kubernetes-dashboard   admin-user-token-42lw8
Name:         admin-user-token-42lw8
Namespace:    kubernetes-dashboard
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: admin-user
              kubernetes.io/service-account.uid: d9508964-883f-4a72-a5e7-9e7be5c8f56f

Type:  kubernetes.io/service-account-token

Data
====
ca.crt:     1302 bytes
namespace:  20 bytes
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6IkZncDZ1bjlDRGRSMnBGWENBRFo3ZnNSZkRXVXhUSFltelMwUm9zamc1XzAifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi11c2VyLXRva2VuLTQybHc4Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImFkbWluLXVzZXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiJkOTUwODk2NC04ODNmLTRhNzItYTVlNy05ZTdiZTVjOGY1NmYiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZXJuZXRlcy1kYXNoYm9hcmQ6YWRtaW4tdXNlciJ9.EAzNIHPmsosDyYiGG_A-WbvL0X4_gVMY1TtLaD9rPEuSJqIMH-9oIkjIPJdrIJ2Xupny6rv5CjK_ahCZPIBtrkh1-gtiPt9bB7idfbUMfvxLhJSVDarive9dpeZGyHhUv_3YKqYi2J6S44jqx_5C2K80AKeLWzKpSoFL9CPfTNeCyKu2REv-O-9Gu4WfENXDorqNTSjh_IcQFSAO58QIk6Psr1sjQ-Y8t2_cXej9rOxfObANVPLPwNE6uKfNMgQEm03d1NluT95lKZroQnON481s-RxcRVmwFx8OSnezWC5Cf35GL4Irwvm3V-mM-iHMp37lZnWQM5H8N0LPzceGFQ
oot@k8s-master1:~#
root@k8s-master1:~# kubectl  get svc -A
NAMESPACE              NAME                         TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                                        AGE
default                kubernetes                   ClusterIP   10.100.0.1       <none>        443/TCP                                        5d12h
kube-system            kube-dns                     NodePort    10.100.0.2       <none>        53:35374/UDP,53:35374/TCP,9153:30009/TCP       5d11h
> 此时登录地址为：http://172.31.6.111:3002
```

5、基于 LimitRange 限制 magedu namespace 中单个 container 最大 1C1G，单个 pod 最大 2C2G，并默认为 CPU limit 为 0.5 核、默认内存 limit 为 512M。

root@k8s-master1:~/3.magedu-limit-case# cat case3-LimitRange.yaml
apiVersion: v1
kind: LimitRange
metadata:
  name: limitrange-magedu
  namespace: magedu
spec:
  limits:
  - type: Container       #限制的资源类型
    max:
      cpu: "1"            #限制单个容器的最大CPU
      memory: "1Gi"       #限制单个容器的最大内存
    min:
      cpu: "500m"         #限制单个容器的最小CPU
      memory: "512Mi"     #限制单个容器的最小内存
    default:
      cpu: "500m"         #默认单个容器的CPU限制
      memory: "512Mi"     #默认单个容器的内存限制
    defaultRequest:
      cpu: "500m"         #默认单个容器的CPU创建请求
      memory: "512Mi"     #默认单个容器的内存创建请求
    maxLimitRequestRatio:
      cpu: 2              #限制CPU limit/request比值最大为2
      memory: 2         #限制内存limit/request比值最大为1.5
  - type: Pod
    max:
      cpu: "2"            #限制单个Pod的最大CPU
      memory: "2Gi"       #限制单个Pod最大内存
  - type: PersistentVolumeClaim
    max:
      storage: 50Gi        #限制PVC最大的requests.storage
    min:
      storage: 30Gi        #限制PVC最小的requests.storage

6、基于 ResourceQuota 限制 magedu namespace 中最多可分配 CPU 192C，内存 512G。

root@k8s-master1:~/3.magedu-limit-case# cat case6-ResourceQuota-magedu.yaml
apiVersion: v1
kind: ResourceQuota
metadata:
  name: quota-magedu
  namespace: magedu
spec:
  hard:
    requests.cpu: "192"
    limits.cpu: "192"
    requests.memory: 512Gi
    limits.memory: 512Gi

7、基于 Operator 在 Kubernetes 环境部署 prometheus 监控环境 (prometheus-server、cAdvisor、grafana、node-exporter 等)。

git clone -b release-0.11 https://github.com/prometheus-operator/kube-prometheus.git
cd kube-prometheus
# 准备镜像
docker pull bitnami/kube-state-metrics:2.5.0
docker tag bitnami/kube-state-metrics:2.5.0 harbor.iclinux.com/baseimages/kube-state-metrics:2.5.0
docker push harbor.iclinux.com/baseimages/kube-state-metrics:2.5.0
vim  manifests/kubeStateMetrics-deployment.yaml

docker pull willdockerhub/prometheus-adapter:v0.9.1
docker tag willdockerhub/prometheus-adapter:v0.9.1 harbor.iclinux.com/baseimages/prometheus-adapter:v0.9.1
docker push harbor.iclinux.com/baseimages/prometheus-adapter:v0.9.1
vim manifests/prometheusAdapter-deployment.yaml
kubectl create -f manifests/setup/
mkdir networkPolicy
mv manifests/*etwork* networkPolicy
kubectl apply -f manifests/
# 设置svc
vim manifests/grafana-service.yaml
  spec:
    type: NodePort
    ports:
    - name： http
      nodePort: 33000
kubectl apply -f manifests/grafana-service.yaml

kubectl delete -f manifests/    # 删除部署
kubectl delete -f manifests/setup/