GlusteFS 二
通过heketi提供的RestfullApi来管理 Gluster,进而与kubernetes集成。将gluster作为kubernetes的数据存储
１ 安装
安装gluster 参见 GlusterFS 一
安装heketi
yum -y install heketi heketi-client
2 配置秘钥对
生成秘钥
ssh-keygen -f /etc/heketi/heketi_key -t rsa -N ''
chown heketi:heketi /etc/heketi/heketi_key*
发布公钥
ssh-copy-id -i /etc/heketi/heketi_key.pub root@10.2.0.11
ssh-copy-id -i /etc/heketi/heketi_key.pub root@10.2.0.12
ssh-copy-id -i /etc/heketi/heketi_key.pub root@10.2.0.13
3 创建数据存储文件夹 boltdb 由go语言实现的key,value数据库，支持acid, （注意 boltdb 存储的数据可以拷贝到其他节点从而规避heketi的单点问题）
mkdir /dcos/heketi
chown -R heketi:heketi /dcos/heketi
４ 配置
cat /etc/heketi/heketi.json
{ "_port_comment": "Heketi Server Port Number", "port": "8080", "_use_auth": "Enable JWT authorization. Please enable for deployment", "use_auth": false, "_jwt": "Private keys for access", "jwt": { "_admin": "Admin has access to all APIs", "admin": { "key": "My Secret" }, "_user": "User only has access to /volumes endpoint", "user": { "key": "My Secret" } }, "_glusterfs_comment": "GlusterFS Configuration", "glusterfs": { "_executor_comment": [ "Execute plugin. Possible choices: mock, ssh", "mock: This setting is used for testing and development.", " It will not send commands to any node.", "ssh: This setting will notify Heketi to ssh to the nodes.", " It will need the values in sshexec to be configured.", "kubernetes: Communicate with GlusterFS containers over", " Kubernetes exec api." ], "executor": "ssh", "_sshexec_comment": "SSH username and private key file information", "sshexec": { "keyfile": "/etc/heketi/heketi_key", "user": "root", "port": "22", "fstab": "/etc/fstab" }, "_kubeexec_comment": "Kubernetes configuration", "kubeexec": { "host" :"https://kubernetes.host:8443", "cert" : "/path/to/crt.file", "insecure": false, "user": "kubernetes username", "password": "password for kubernetes user", "namespace": "OpenShift project or Kubernetes namespace", "fstab": "Optional: Specify fstab file on node. Default is /etc/fstab" }, "_db_comment": "Database file name", "db": "/dcos/heketi/heketi.db", "_loglevel_comment": [ "Set log level. Choices are:", " none, critical, error, warning, info, debug", "Default is warning" ], "loglevel" : "error" } }
5 启动并验证
systemctl enable heketi
systemctl restart heketi
curl http://localhost:8088/hello
6 集群化
配置拓扑
cat /etc/heketi/topology.json
{ "clusters": [ { "nodes": [ { "node": { "hostnames": { "manage": [ "node1" ], "storage": [ "10.2.0.11" ] }, "zone": 1 }, "devices": [ "/dev/sdb" ] }, { "node": { "hostnames": { "manage": [ "node2" ], "storage": [ "10.2.0.12" ] }, "zone": 1 }, "devices": [ "/dev/sdb" ] }, { "node": { "hostnames": { "manage": [ "node3" ], "storage": [ "10.2.0.13" ] }, "zone": 1 }, "devices": [ "/dev/sdb" ] } ] } ] }
创建集群
heketi-cli --server http://10.2.0.11:8080 topology load --json=/etc/heketi/topology.json
查看
heketi-cli --server http://10.2.0.11:8080 cluster list
heketi-cli --server http://10.2.0.11:8080 cluster info id****
heketi-cli --server http://10.2.0.11:8080 volume list
heketi-cli --server http://10.2.0.11:8080 volume info id****
7 kubernetes 集群存储应用
cat storeclass.yaml
apiVersion: storage.k8s.io/v1beta1 kind: StorageClass metadata: name: slow provisioner: kubernetes.io/glusterfs parameters: resturl: "http://10.2.0.11:8080" volumetype: "replicate:3" # jenkins cat jenkins.yml --- apiVersion: apps/v1beta1 kind: StatefulSet metadata: name: jenkins labels: name: jenkins spec: serviceName: jenkins replicas: 1 updateStrategy: type: RollingUpdate template: metadata: name: jenkins labels: name: jenkins spec: terminationGracePeriodSeconds: 10 serviceAccountName: jenkins containers: - name: jenkins image: jenkins/jenkins:latest imagePullPolicy: Always ports: - containerPort: 8080 - containerPort: 50000 resources: limits: cpu: 1 memory: 1Gi requests: cpu: 0.5 memory: 500Mi env: - name: LIMITS_MEMORY valueFrom: resourceFieldRef: resource: limits.memory divisor: 1Mi - name: JAVA_OPTS value: "-Duser.timezone=Asia/Shanghai -Xms528m -Xmx528m " volumeMounts: - name: jenkins-home mountPath: /var/jenkins_home securityContext: fsGroup: 1000 volumeClaimTemplates: - metadata: name: jenkins-home annotations: volume.beta.kubernetes.io/storage-class: slow spec: accessModes: [ "ReadWriteMany" ] resources: requests: storage: 50Gi --- apiVersion: v1 kind: Service metadata: name: jenkins-svc namespace: default spec: ports: - port: 8080 targetPort: 8080 name: web - port: 50000 targetPort: 50000 name: slave selector: name: jenkins type: NodePort cat service-account.yml # In GKE need to get RBAC permissions first with # kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin [--user=<user-name>|--group=<group-name>] --- apiVersion: v1 kind: ServiceAccount metadata: name: jenkins --- kind: Role apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: jenkins rules: - apiGroups: [""] resources: ["pods"] verbs: ["create","delete","get","list","patch","update","watch"] - apiGroups: [""] resources: ["pods/exec"] verbs: ["create","delete","get","list","patch","update","watch"] - apiGroups: [""] resources: ["pods/log"] verbs: ["get","list","watch"] - apiGroups: [""] resources: ["secrets"] verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: RoleBinding metadata: name: jenkins roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: jenkins subjects: - kind: ServiceAccount name: jenkins
