k8s-使用statefulset部署mysql主从集群5.7

特点：

headless service,无Cluster IP,名称对应ep

网络标识符固定且唯一

外部存储固定持久化

对pod有序部署拓展，删除终止，滚动更新

https://kubernetes.io/zh/docs/tasks/run-application/run-replicated-stateful-application/

root@nfs-156~# showmount -e
Export list for ubuntu20.04.3.example.com:
/data/k8sdata *

二进制日志的一个重要功能是主从复制，存储函数有可能导致主从的数据不一致。开启二进制日志后，配置文件参数log_bin_trust_function_creators永久生效，限制存储函数的创建、修改、调用。

root@slave002# cat pv/mysql-persistentvolume.yaml 
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-1
  namespace: chuan
spec:
  capacity:
    storage: 50Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/k8sdata/chuan/mysql-datadir-1 
    server: 192.168.192.156
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-2
  namespace: chuan
spec:
  capacity:
    storage: 50Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/k8sdata/chuan/mysql-datadir-2
    server: 192.168.192.156
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-3
  namespace: chuan
spec:
  capacity:
    storage: 50Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/k8sdata/chuan/mysql-datadir-3
    server: 192.168.192.156
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-4
  namespace: chuan
spec:
  capacity:
    storage: 50Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/k8sdata/chuan/mysql-datadir-4
    server: 192.168.192.156
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-5
  namespace: chuan
spec:
  capacity:
    storage: 50Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/k8sdata/chuan/mysql-datadir-5
    server: 192.168.192.156
 
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: mysql-datadir-6
  namespace: chuan
spec:
  capacity:
    storage: 50Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    path: /data/k8sdata/chuan/mysql-datadir-6
    server: 192.168.192.156

root@slave002l# kubectl get pv
NAME              CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                 STORAGECLASS   REASON   AGE
mysql-datadir-1   50Gi       RWO            Retain           Bound       chuan/data-mysql-0                            12m
mysql-datadir-2   50Gi       RWO            Retain           Available                                                 12m
mysql-datadir-3   50Gi       RWO            Retain           Bound       chuan/data-mysql-1                            12m
mysql-datadir-4   50Gi       RWO            Retain           Bound       chuan/data-mysql-2                            12m
mysql-datadir-5   50Gi       RWO            Retain           Available                                                 12m
mysql-datadir-6   50Gi       RWO            Retain           Available                                                 12m

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

root@slave002l# cat mysql-statefulset.yaml 
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: mysql
  namespace: chuan
spec:
  selector:
    matchLabels:
      app: mysql
  serviceName: mysql
  replicas: 3
  template:
    metadata:
      labels:
        app: mysql
    spec:
      initContainers:
      - name: init-mysql
        image: 192.168.192.155:80/chuan/mysql:5.7
        command:
        - bash
        - "-c"
        - |
          set -ex
          # Generate mysql server-id from pod ordinal index.
          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
          ordinal=${BASH_REMATCH[1]}
          echo [mysqld] > /mnt/conf.d/server-id.cnf
          # Add an offset to avoid reserved server-id=0 value.
          echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf
          # Copy appropriate conf.d files from config-map to emptyDir.
          if [[ $ordinal -eq 0 ]]; then
            cp /mnt/config-map/master.cnf /mnt/conf.d/
          else
            cp /mnt/config-map/slave.cnf /mnt/conf.d/
          fi
        volumeMounts:
        - name: conf
          mountPath: /mnt/conf.d
        - name: config-map
          mountPath: /mnt/config-map
      - name: clone-mysql
        image: 192.168.192.155:80/chuan/xtrabackup:1.0
        command:
        - bash
        - "-c"
        - |
          set -ex
          # Skip the clone if data already exists.
          [[ -d /var/lib/mysql/mysql ]] && exit 0
          # Skip the clone on master (ordinal index 0).
          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
          ordinal=${BASH_REMATCH[1]}
          [[ $ordinal -eq 0 ]] && exit 0
          # Clone data from previous peer.
          ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql
          # Prepare the backup.
          xtrabackup --prepare --target-dir=/var/lib/mysql
        volumeMounts:
        - name: data
          mountPath: /var/lib/mysql
          subPath: mysql
        - name: conf
          mountPath: /etc/mysql/conf.d
      containers:
      - name: mysql
        image: 192.168.192.155:80/chuan/mysql:5.7
        env:
        - name: MYSQL_ALLOW_EMPTY_PASSWORD
          value: "1"
        ports:
        - name: mysql
          containerPort: 3306
        volumeMounts:
        - name: data
          mountPath: /var/lib/mysql
          subPath: mysql
        - name: conf
          mountPath: /etc/mysql/conf.d
        resources:
          requests:
            cpu: 500m
            memory: 1Gi
        livenessProbe:
          exec:
            command: ["mysqladmin", "ping"]
          initialDelaySeconds: 30
          periodSeconds: 10
          timeoutSeconds: 5
        readinessProbe:
          exec:
            # Check we can execute queries over TCP (skip-networking is off).
            command: ["mysql", "-h", "127.0.0.1", "-e", "SELECT 1"]
          initialDelaySeconds: 5
          periodSeconds: 2
          timeoutSeconds: 1
      - name: xtrabackup
        image: 192.168.192.155:80/chuan/xtrabackup:1.0
        ports:
        - name: xtrabackup
          containerPort: 3307
        command:
        - bash
        - "-c"
        - |
          set -ex
          cd /var/lib/mysql
          # Determine binlog position of cloned data, if any.
          if [[ -f xtrabackup_slave_info ]]; then
            # XtraBackup already generated a partial "CHANGE MASTER TO" query
            # because we're cloning from an existing slave.
            mv xtrabackup_slave_info change_master_to.sql.in
            # Ignore xtrabackup_binlog_info in this case (it's useless).
            rm -f xtrabackup_binlog_info
          elif [[ -f xtrabackup_binlog_info ]]; then
            # We're cloning directly from master. Parse binlog position.
            [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
            rm xtrabackup_binlog_info
            echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\
                  MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in
          fi
          # Check if we need to complete a clone by starting replication.
          if [[ -f change_master_to.sql.in ]]; then
            echo "Waiting for mysqld to be ready (accepting connections)"
            until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done
            echo "Initializing replication from clone position"
            # In case of container restart, attempt this at-most-once.
            mv change_master_to.sql.in change_master_to.sql.orig
            mysql -h 127.0.0.1 <<EOF
          $(<change_master_to.sql.orig),
            MASTER_HOST='mysql-0.mysql',
            MASTER_USER='root',
            MASTER_PASSWORD='',
            MASTER_CONNECT_RETRY=10;
          START SLAVE;
          EOF
          fi
          # Start a server to send backups when requested by peers.
          exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c \
            "xtrabackup --backup --slave-info --stream=xbstream --host=127.0.0.1 --user=root"
        volumeMounts:
        - name: data
          mountPath: /var/lib/mysql
          subPath: mysql
        - name: conf
          mountPath: /etc/mysql/conf.d
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
      volumes:
      - name: conf
        emptyDir: {}
      - name: config-map
        configMap:
          name: mysql
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes: ["ReadWriteOnce"]
      resources:
        requests:
          storage: 10Gi

root@slave002# cat mysql-services.yaml 
# Headless service for stable DNS entries of StatefulSet members.
apiVersion: v1
kind: Service
metadata:
  namespace: chuan
  name: mysql
  labels:
    app: mysql
spec:
  ports:
  - name: mysql
    port: 3306
  clusterIP: None
  selector:
    app: mysql
---
# Client service for connecting to any MySQL instance for reads.
# For writes, you must instead connect to the master: mysql-0.mysql.
apiVersion: v1
kind: Service
metadata:
  name: mysql-read
  namespace: chuan
  labels:
    app: mysql
spec:
  ports:
  - name: mysql
    port: 3306
  selector:
    app: mysql

root@slave002# cat mysql-configmap.yaml 
apiVersion: v1
kind: ConfigMap
metadata:
  name: mysql
  namespace: chuan
  labels:
    app: mysql
data:
  master.cnf: |
    # Apply this config only on the master.
    [mysqld]
    log-bin
    log_bin_trust_function_creators=1
    lower_case_table_names=1
  slave.cnf: |
    # Apply this config only on slaves.
    [mysqld]
    super-read-only
    log_bin_trust_function_creators=1

root@slave002# kubectl get po -nchuan
NAME      READY   STATUS    RESTARTS   AGE
mysql-0   2/2     Running   0          8m34s
mysql-1   2/2     Running   1          8m19s
mysql-2   2/2     Running   1          7m49s

root@slave002# kubectl get pvc -A
NAMESPACE   NAME           STATUS   VOLUME            CAPACITY   ACCESS MODES   STORAGECLASS   AGE
chuan      data-mysql-0   Bound    mysql-datadir-1   50Gi       RWO                           9m37s
chuan      data-mysql-1   Bound    mysql-datadir-3   50Gi       RWO                           9m22s
chuan      data-mysql-2   Bound    mysql-datadir-4   50Gi       RWO                           8m52s

mysql> show slave status\G 
             Slave_IO_Running: Yes
            Slave_SQL_Running: Yes

PING mysql-2.mysql.chuan.svc.cluster.local (10.200.122.199): 56 data bytes
64 bytes from 10.200.122.199: seq=0 ttl=63 time=0.062 ms

pod名称.deploy名称.命名空间.k8s集群后缀 #解析pod地址非service

/ # ping web
PING web (10.100.204.137): 56 data bytes #解析service地址

root@slave002:/opt/k8s-data/k8s-data/yaml/chuan/mysql# kubectl get svc -nchuan
NAME         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)    AGE
mysql        ClusterIP   None            <none>        3306/TCP   15h
mysql-read   ClusterIP   10.100.204.47   <none>        3306/TCP   15h
root@slave002:/opt/k8s-data/k8s-data/yaml/chuan/mysql# kubectl get ep -nchuan
NAME         ENDPOINTS                                                  AGE
mysql        10.200.111.4:3306,10.200.122.199:3306,10.200.163.72:3306   15h
mysql-read   10.200.111.4:3306,10.200.122.199:3306,10.200.163.72:3306   15h