Kubernetes集群存储解决方案GlusterFS

1、工具简介

1.1、GlusterFS

• 是一个开源的分布式文件系统

• 具有强大的横向扩展能力

• 通过扩展能够支持数PB存储容量和处理数千客户端

• 将物理分布的存储资源聚集在一起，使用单一全局命名空间来管理数据

1.2、Heketi

• 是一个基于RESTful API的GlusterFS卷管理框架

• 可以方便地和云平台整合，提供RESTful API供kubernetes调用，实现多GlusterFS集群的卷管理

• 还有保证bricks和它对应的副本均匀分布在集群中的不同可用区的优点

2、GlusterFS安装部署

2.1 环境说明

IP地址	主机名	硬盘	硬盘容量
192.168.16.133	g1	/dev/sdb	20G
192.168.16.134	g2	/dev/sdb	20G
192.168.16.135	g3	/dev/sdb	20G

• /etc/hosts

• 主机之间免密

• /dev/sdb 挂载

• firewalld和selinux

• 时钟同步

2.2 安装部署

安装并启动

# yum install centos-release-gluster -y
# yum install glusterfs glusterfs-server glusterfs-fuse glusterfs-rdma fuse -y
# systemctl enable --now glusterd

集群配置

# gluster peer probe g2
# gluster peer probe g3
# gluster peer status

创建并启动复制卷

# gluster volume create k8s-test-volume replica 3 g1:/glustersdb/r1 g2:/glustersdb/r2 g3:/glustersdb/r3
# gluster volume start k8s-test-volume
# gluster volume status k8s-test-volume
# gluster volume info k8s-test-volume

某一个brick不在线不会影响客户端挂载

# gluster volume set k8s-test-volume cluster.server-quorum-type none
# gluster volume set k8s-test-volume cluster.quorum-type none

限额

# gluster volume quota k8s-test-volume enable
# gluster volume quota k8s-test-volume limit-usage / 9GB

客户端验证

# yum install centos-release-gluster
# yum install glusterfs glusterfs-fuse -y
# mkdir /k8s-glusterfs-test-volume
# mount -t glusterfs g1:/k8s-test-volume /k8s-glusterfs-test-volume

3、Heketi

k8s master节点安装

# yum install heketi heketi-client -y

k8s node节点安装

# yum install heketi-client -y

修改配置

# cat /etc/heketi/heketi.json
{
  "_port_comment": "Heketi Server Port Number",
  "port": "18080",

  "_use_auth": "Enable JWT authorization. Please enable for deployment",
  "use_auth": true,

  "_jwt": "Private keys for access",
  "jwt": {
    "_admin": "Admin has access to all APIs",
    "admin": {
      "key": "adminkey"
    },
    "_user": "User only has access to /volumes endpoint",
    "user": {
      "key": "My Secret"
    }
  },

  "_glusterfs_comment": "GlusterFS Configuration",
  "glusterfs": {
    "_executor_comment": [
      "Execute plugin. Possible choices: mock, ssh",
      "mock: This setting is used for testing and development.",
      "      It will not send commands to any node.",
      "ssh:  This setting will notify Heketi to ssh to the nodes.",
      "      It will need the values in sshexec to be configured.",
      "kubernetes: Communicate with GlusterFS containers over",
      "            Kubernetes exec api."
    ],
    "executor": "ssh",

    "_sshexec_comment": "SSH username and private key file information",
    "sshexec": {
      "keyfile": "/etc/heketi/heketi_key",
      "user": "root",
      "port": "22",
      "fstab": "/etc/fstab"
    },

    "_kubeexec_comment": "Kubernetes configuration",
    "kubeexec": {
      "host" :"https://kubernetes.host:8443",
      "cert" : "/path/to/crt.file",
      "insecure": false,
      "user": "kubernetes username",
      "password": "password for kubernetes user",
      "namespace": "OpenShift project or Kubernetes namespace",
      "fstab": "Optional: Specify fstab file on node.  Default is /etc/fstab"
    },

    "_db_comment": "Database file name",
    "db": "/var/lib/heketi/heketi.db",

    "_loglevel_comment": [
      "Set log level. Choices are:",
      "  none, critical, error, warning, info, debug",
      "Default is warning"
    ],
    "loglevel" : "debug"
  }
}

配置免密ssh连接，启动heketi

# cp /root/.ssh/id_rsa /etc/heketi/heketi_key
# chown heketi:heketi /etc/heketi/ -R
# chown heketi:heketi /var/lib/heketi/ -R
# systemctl start heketi

heketi使用验证

# heketi-cli --user admin --secret adminkey --server http://192.168.16.129:18080 --json cluster create
# heketi-cli --user admin --secret adminkey --server http://192.168.16.129:18080 --json cluster delete f63ca83fb3714d5d52356a7ab90e2b07

使用heketi创建glusterfs集群

## 创建集群
# heketi-cli --user admin --secret adminkey --server http://192.168.16.129:18080 --json cluster create
{"id":"e159af1a727f44c53989ec5fd9654f98","nodes":[],"volumes":[],"block":true,"file":true,"blockvolumes":[]}

## 集群中添加主机节点
# heketi-cli --user admin --secret adminkey --server http://192.168.16.129:18080 --json node add --cluster "e159af1a727f44c53989ec5fd9654f98" --management-host-name 192.168.16.133 --storage-host-name 192.168.16.133 --zone 1
{"zone":1,"hostnames":{"manage":["192.168.16.133"],"storage":["192.168.16.133"]},"cluster":"e159af1a727f44c53989ec5fd9654f98","id":"2faca0849936518fde8948b4a32c8cc8","state":"online","devices":[]}

heketi-cli --user admin --secret adminkey --server http://192.168.16.129:18080 --json node add --cluster "e159af1a727f44c53989ec5fd9654f98" --management-host-name 192.168.16.134 --storage-host-name 192.168.16.134 --zone 1
{"zone":1,"hostnames":{"manage":["192.168.16.134"],"storage":["192.168.16.134"]},"cluster":"e159af1a727f44c53989ec5fd9654f98","id":"8e1684a1d485bf4730f7419d4bca083a","state":"online","devices":[]}

# heketi-cli --user admin --secret adminkey --server http://192.168.16.129:18080 --json node add --cluster "e159af1a727f44c53989ec5fd9654f98" --management-host-name 192.168.16.135 --storage-host-name 192.168.16.135 --zone 1
{"zone":1,"hostnames":{"manage":["192.168.16.135"],"storage":["192.168.16.135"]},"cluster":"e159af1a727f44c53989ec5fd9654f98","id":"bc1e5d0b730acb8bfcfd5bf9535167b1","state":"online","devices":[]}

# heketi-cli --user admin --secret adminkey --server http://192.168.16.129:18080 --json node list
Id:2faca0849936518fde8948b4a32c8cc8	Cluster:e159af1a727f44c53989ec5fd9654f98
Id:8e1684a1d485bf4730f7419d4bca083a	Cluster:e159af1a727f44c53989ec5fd9654f98
Id:bc1e5d0b730acb8bfcfd5bf9535167b1	Cluster:e159af1a727f44c53989ec5fd9654f98

## 集群节点添加硬盘设备
# heketi-cli --user admin --secret adminkey --server http://192.168.16.129:18080 device add --name "/dev/sdc" --node 2faca0849936518fde8948b4a32c8cc8
Device added successfully

# heketi-cli --user admin --secret adminkey --server http://192.168.16.129:18080 device add --name "/dev/sdc" --node 8e1684a1d485bf4730f7419d4bca083a

# heketi-cli --user admin --secret adminkey --server http://192.168.16.129:18080 device add --name "/dev/sdc" --node bc1e5d0b730acb8bfcfd5bf9535167b1

# heketi-cli --user admin --secret adminkey --server http://192.168.16.129:18080 topology info

## 创建复制卷
# heketi-cli --user admin --secret adminkey --server http://192.168.16.129:18080 volume create --size=5 --replica=2
Name: vol_4ecb4ea98c89dfeca024d6b7c760585d
Size: 5
Volume Id: 4ecb4ea98c89dfeca024d6b7c760585d
Cluster Id: e159af1a727f44c53989ec5fd9654f98
Mount: 192.168.16.133:vol_4ecb4ea98c89dfeca024d6b7c760585d
Mount Options: backup-volfile-servers=192.168.16.134,192.168.16.135
Block: false
Free Size: 0
Reserved Size: 0
Block Hosting Restriction: (none)
Block Volumes: []
Durability Type: replicate
Distribute Count: 1
Replica Count: 2

# heketi-cli --user admin --secret adminkey --server http://192.168.16.129:18080 volume list
Id:4ecb4ea98c89dfeca024d6b7c760585d    Cluster:e159af1a727f44c53989ec5fd9654f98    Name:vol_4ecb4ea98c89dfeca024d6b7c760585d

# heketi-cli --user admin --secret adminkey --server http://192.168.16.129:18080 volume delete 4ecb4ea98c89dfeca024d6b7c760585d

k8s集群使用glusterfs集群
创建storageclass

# cat storageclass-glusterfs.yml 
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: glusterfs
provisioner: kubernetes.io/glusterfs
parameters:
  resturl: "http://192.168.16.129:18080"
  restauthenabled: "true"
  restuser: "admin"
  restuserkey: "adminkey"
  volumetype: "replicate:2"

# kubectl apply -f storageclass-glusterfs.yml
# kubectl get sc
NAME        PROVISIONER               RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
glusterfs   kubernetes.io/glusterfs   Delete          Immediate           false                  39s

创建pvc

# cat glusterfs-pvc.yml 
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: glusterfs-mysql
  annotations:
    volume.beta.kubernetes.io/storage-class: "glusterfs"
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 2Gi
# kubectl apply -f glusterfs-pvc.yml

# kubectl get pvc
NAME              STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
glusterfs-mysql   Bound    pvc-fa000acd-3393-439e-8551-079540a3a283   2Gi        RWX            glusterfs      32s
pvc-nfs           Bound    pv-nfs                                     1Gi        RWX                           2d
# kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                     STORAGECLASS   REASON   AGE
pv-nfs                                     1Gi        RWX            Retain           Bound    default/pvc-nfs                                   2d
pvc-fa000acd-3393-439e-8551-079540a3a283   2Gi        RWX            Delete           Bound    default/glusterfs-mysql   glusterfs               3m41s

创建应用使用pvc

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: mysql
spec:
  serviceName: mysql-svc
  selector:
    matchLabels:
      name: mysql
  replicas: 1
  template:
    metadata:
      labels:
        name: mysql
    spec:
      containers:
      - name: mysql
        image: mysql:5.7
        imagePullPolicy: IfNotPresent
        env:
        - name: MYSQL_ROOT_PASSWORD
          value: "123456"
        ports:
        - containerPort: 3306
          name: mysql
        volumeMounts:
        - name: glusterfs-mysql-data
          mountPath: "/var/lib/mysql"
      volumes:
      - name: glusterfs-mysql-data
        persistentVolumeClaim:
          claimName: glusterfs-mysql

heketi密码加密方式