k8s 1.20.x glusterfs动态存储
1、环境介绍
[root@master ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
master Ready control-plane,master 50m v1.20.5
node1 Ready <none> 48m v1.20.5
node2 Ready <none> 48m v1.20.5
[root@node1 ~]# cat /etc/hosts|grep node
172.16.188.28 node1
172.16.188.29 node2
[root@node1 ~]# fdisk -l /dev/vdb
Disk /dev/vdb: 21.5 GB, 21474836480 bytes, 41943040 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
[root@node2 ~]# fdisk -l /dev/vdb
Disk /dev/vdb: 21.5 GB, 21474836480 bytes, 41943040 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
2、存储服务器都安装glusterfs和heketi
- 注意:GlusterFS只需要安装并启动即可，不必组建受信存储池
#所有存储服务器下载安装glusterfs
yum install centos-release-gluster -y
yum install glusterfs-server -y
#启动
systemctl start glusterd
systemctl enable glusterd
#安装heketi
yum install -y heketi heketi-client
3、配置heketi.json(2台存储服务器都修改)
[root@node1 ~]# cat /etc/heketi/heketi.json
{
"_port_comment": "Heketi Server Port Number",
"port": "8080",
"_use_auth": "Enable JWT authorization. Please enable for deployment",
"use_auth": true,
"_jwt": "Private keys for access",
"jwt": {
"_admin": "Admin has access to all APIs",
"admin": {
"key": "admin@key"
},
"_user": "User only has access to /volumes endpoint",
"user": {
"key": "user@key"
}
},
"_glusterfs_comment": "GlusterFS Configuration",
"glusterfs": {
"_executor_comment": [
"Execute plugin. Possible choices: mock, ssh",
"mock: This setting is used for testing and development.",
" It will not send commands to any node.",
"ssh: This setting will notify Heketi to ssh to the nodes.",
" It will need the values in sshexec to be configured.",
"kubernetes: Communicate with GlusterFS containers over",
" Kubernetes exec api."
],
"executor": "ssh",
"_sshexec_comment": "SSH username and private key file information",
"sshexec": {
"keyfile": "/etc/heketi/heketi_key",
"user": "root",
"port": "22",
"fstab": "/etc/fstab"
},
"_kubeexec_comment": "Kubernetes configuration",
"kubeexec": {
"host" :"https://kubernetes.host:8443",
"cert" : "/path/to/crt.file",
"insecure": false,
"user": "kubernetes username",
"password": "password for kubernetes user",
"namespace": "OpenShift project or Kubernetes namespace",
"fstab": "Optional: Specify fstab file on node. Default is /etc/fstab"
},
"_db_comment": "Database file name",
"db": "/var/lib/heketi/heketi.db",
"_loglevel_comment": [
"Set log level. Choices are:",
" none, critical, error, warning, info, debug",
"Default is warning"
],
"loglevel" : "warning"
}
}
4、设置heketi免密访问GlusterFS与公钥分发
#设置heketi免密访问GlusterFS
[root@node1 heketi]# ssh-keygen -t rsa -q -f /etc/heketi/heketi_key -N ""
[root@node1 heketi]# chown heketi:heketi /etc/heketi/heketi_key
#分发公钥
[root@node1 heketi]# ssh-copy-id -i /etc/heketi/heketi_key.pub root@node1
[root@node1 heketi]# ssh-copy-id -i /etc/heketi/heketi_key.pub root@node2
#将秘钥从node1服务器复制到node2服务器上
[root@node1 heketi]# rsync -avz /etc/heketi/heketi_key root@node2:/etc/heketi/
5、启动heketi
systemctl enable heketi
systemctl restart heketi
systemctl status heketi
验证
[root@node1 ~]# curl http://localhost:8080/hello
Hello from Heketi
6、创建topology.json文件（一台服务器操作即可）
[root@node1 ~]# cat /etc/heketi/topology.json
{
"clusters": [
{
"nodes": [
{
"node": {
"hostnames": {
"manage": [
"172.16.188.28"
],
"storage": [
"172.16.188.28"
]
},
"zone": 1
},
"devices": [
"/dev/vdb"
]
},
{
"node": {
"hostnames": {
"manage": [
"172.16.188.29"
],
"storage": [
"172.16.188.29"
]
},
"zone": 2
},
"devices": [
"/dev/vdb"
]
}
]
}
]
}
7、通过topology.json组建GlusterFS集群
[root@node1 ~]# heketi-cli --server http://localhost:8080 --user admin --secret admin@key topology load --json=/etc/heketi/topology.json
Creating cluster ... ID: 03b289bb792326e9d519d4c79600a808
Allowing file volumes on cluster.
Allowing block volumes on cluster.
Creating node 172.16.188.28 ... ID: 0159f85d08594059fabd07aa01ef25bc
Adding device /dev/vdb ... OK
Creating node 172.16.188.29 ... ID: 2e2c2aa577f34f7f7306a3bd53d48277
Adding device /dev/vdb ... OK
查看
[root@node1 ~]# heketi-cli --server http://localhost:8080 --user admin --secret admin@key topology info
Cluster Id: 03b289bb792326e9d519d4c79600a808
File: true
Block: true
Volumes:
Nodes:
Node Id: 0159f85d08594059fabd07aa01ef25bc
State: online
Cluster Id: 03b289bb792326e9d519d4c79600a808
Zone: 1
Management Hostnames: 172.16.188.28
Storage Hostnames: 172.16.188.28
Devices:
Id:3f59b8fb6fb7b8f06864387ab63c0125 Name:/dev/vdb State:online Size (GiB):19 Used (GiB):0 Free (GiB):19
Bricks:
Node Id: 2e2c2aa577f34f7f7306a3bd53d48277
State: online
Cluster Id: 03b289bb792326e9d519d4c79600a808
Zone: 2
Management Hostnames: 172.16.188.29
Storage Hostnames: 172.16.188.29
Devices:
Id:369615b58b578961592883f6612380e1 Name:/dev/vdb State:online Size (GiB):19 Used (GiB):0 Free (GiB):19
Bricks:
8、定义并导入secret并导入
[root@master glusterfs]# cat gluster-heketi-secret.yaml
apiVersion: v1
kind: Secret
metadata:
name: heketi-secret
namespace: default
data:
# base64 encoded password. E.g.: echo -n "mypassword" | base64
key: YWRtaW5Aa2V5
type: kubernetes.io/glusterfs
[root@master glusterfs]# kubectl apply -f gluster-heketi-secret.yaml
secret/heketi-secret created
9、定义StorageClass并导入
[root@master glusterfs]# cat gluster-heketi-storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: gluster-heketi-storageclass
provisioner: kubernetes.io/glusterfs
allowVolumeExpansion: true
reclaimPolicy: Delete
parameters:
resturl: "http://172.16.188.28:8080"
restauthenabled: "true"
restuser: "admin"
secretNamespace: "default"
secretName: "heketi-secret"
volumetype: "replicate:2"
[root@master glusterfs]# kubectl apply -f gluster-heketi-storageclass.yaml
storageclass.storage.k8s.io/gluster-heketi-storageclass created
查看
[root@master glusterfs]# kubectl get sc gluster-heketi-storageclass
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
gluster-heketi-storageclass kubernetes.io/glusterfs Delete Immediate true 31m
10、测试
[root@master glusterfs]# cat test-pvc.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: test-glusterfs
annotations:
volume.beta.kubernetes.io/storage-class: "gluster-heketi-storageclass"
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Gi
[root@master glusterfs]# kubectl apply -f test-pvc.yaml
persistentvolumeclaim/test-glusterfs created
[root@master glusterfs]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
test-glusterfs Bound pvc-aecc1dc9-1be4-439f-8a64-229cd8fbb6e8 1Gi RWX gluster-heketi-storageclass 32m
11、设置默认动态存储
查看当前的动态存储
[root@master glusterfs]# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
gluster-heketi-storageclass kubernetes.io/glusterfs Delete Immediate true 173m
managed-nfs-storage fuseim.pri/ifs Delete Immediate true 3h41m
设置glusterfs为默认动态存储
kubectl patch storageclass gluster-heketi-storageclass -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
查看
[root@master ~]# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
gluster-heketi-storageclass (default) kubernetes.io/glusterfs Delete Immediate true 3h1m
managed-nfs-storage fuseim.pri/ifs Delete Immediate true 3h50m
12、扩容
方案一、添加磁盘方式扩容
添加磁盘，将添加的磁盘添加到集群中
方案二、添加磁盘大小，在线扩容
- 参考操作文档：www.infvie.com/ops-notes/k… 添加完磁盘大小后，使用heketi-cli查看的话，发现heketi-cli显示的磁盘大小是扩容前的大小
扩容方式1
手动将某个卷以LVM方式扩容，但是heketi-cli查看的话，数据盘的大小不会变
