基于ceph rbd 在kubernetes harbor 空间下创建动态存储
[root@bs-k8s-ceph ~]# ceph osd pool create harbor 128 Error ETIMEDOUT: crush test failed with -110: timed out during smoke test (5 seconds) //这个问题 我不知道怎么解决 因为过了一小会 就又好了 [root@bs-k8s-ceph ~]# ceph osd pool create harbor 128 pool 'harbor' created [root@bs-k8s-ceph ceph]# ceph auth get-or-create client.harbor mon 'allow r' osd 'allow class-read, allow rwx pool=harbor' -o ceph.client.harbor.keyring [root@bs-k8s-ceph ceph]# ceph auth get client.harbor exported keyring for client.harbor [client.harbor] key = AQDoCklen6e4NxAAVXmy/PG+R5iH8fNzMhk6Jg== caps mon = "allow r" caps osd = "allow class-read, allow rwx pool=harbor" [root@bs-k8s-node01 ~]# ceph auth get-key client.admin | base64 QVFDNmNVSmV2eU8yRnhBQVBxYzE5Mm5PelNnZk5acmg5aEFQYXc9PQ== [root@bs-k8s-node01 ~]# ceph auth get-key client.harbor | base64 [root@bs-k8s-master01 ~]# kubectl get nodes The connection to the server 20.0.0.250:8443 was refused - did you specify the right host or port? [root@bs-hk-hk01 ~]# systemctl status haproxy ● haproxy.service - HAProxy Load Balancer Loaded: loaded (/usr/lib/systemd/system/haproxy.service; enabled; vendor preset: disabled) Active: failed (Result: exit-code) since 日 2020-02-16 17:16:43 CST; 12min ago Process: 1168 ExecStart=/usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/haproxy.pid (code=exited, status=134) Main PID: 1168 (code=exited, status=134) 2月 15 20:22:54 bs-hk-hk01 haproxy[1168]: [WARNING] 045/202254 (1184) : Server k8s_api_nodes...ue. 2月 15 20:25:15 bs-hk-hk01 haproxy[1168]: [WARNING] 045/202515 (1183) : Server k8s_api_nodes...ue. 2月 15 20:25:15 bs-hk-hk01 haproxy[1168]: [WARNING] 045/202515 (1184) : Server k8s_api_nodes...ue. 2月 15 20:26:03 bs-hk-hk01 haproxy[1168]: [WARNING] 045/202603 (1184) : Server k8s_api_nodes...ue. 2月 15 20:26:03 bs-hk-hk01 haproxy[1168]: [WARNING] 045/202603 (1183) : Server k8s_api_nodes...ue. 2月 15 20:26:13 bs-hk-hk01 haproxy[1168]: [WARNING] 045/202613 (1183) : Server k8s_api_nodes...ue. 2月 15 20:26:13 bs-hk-hk01 haproxy[1168]: [WARNING] 045/202613 (1184) : Server k8s_api_nodes...ue. 2月 16 17:16:43 bs-hk-hk01 systemd[1]: haproxy.service: main process exited, code=exited, st...n/a 2月 16 17:16:44 bs-hk-hk01 systemd[1]: Unit haproxy.service entered failed state. 2月 16 17:16:44 bs-hk-hk01 systemd[1]: haproxy.service failed. Hint: Some lines were ellipsized, use -l to show in full. [root@bs-hk-hk01 ~]# systemctl start haproxy [root@bs-hk-hk01 ~]# systemctl status haproxy ● haproxy.service - HAProxy Load Balancer Loaded: loaded (/usr/lib/systemd/system/haproxy.service; enabled; vendor preset: disabled) Active: active (running) since 日 2020-02-16 17:30:03 CST; 1s ago Process: 4196 ExecStartPre=/usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q (code=exited, status=0/SUCCESS) Main PID: 4212 (haproxy) CGroup: /system.slice/haproxy.service ├─4212 /usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/haproxy.... ├─4216 /usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/haproxy.... └─4217 /usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/haproxy.... 2月 16 17:30:00 bs-hk-hk01 systemd[1]: Starting HAProxy Load Balancer... 2月 16 17:30:03 bs-hk-hk01 systemd[1]: Started HAProxy Load Balancer. 2月 16 17:30:04 bs-hk-hk01 haproxy[4212]: [WARNING] 046/173004 (4212) : config : 'option for...de. 2月 16 17:30:04 bs-hk-hk01 haproxy[4212]: [WARNING] 046/173004 (4212) : config : 'option for...de. 2月 16 17:30:04 bs-hk-hk01 haproxy[4212]: [WARNING] 046/173004 (4212) : Proxy 'stats': in mu...st. 2月 16 17:30:04 bs-hk-hk01 haproxy[4212]: [NOTICE] 046/173004 (4212) : New worker #1 (4216) forked 2月 16 17:30:04 bs-hk-hk01 haproxy[4212]: [NOTICE] 046/173004 (4212) : New worker #2 (4217) forked Hint: Some lines were ellipsized, use -l to show in full. [root@bs-hk-hk01 ~]# systemctl enable haproxy [root@bs-k8s-master01 ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION bs-k8s-master01 Ready master 7d6h v1.17.2 bs-k8s-master02 Ready master 7d6h v1.17.2 bs-k8s-master03 Ready master 7d6h v1.17.2 bs-k8s-node01 Ready <none> 7d6h v1.17.2 bs-k8s-node02 Ready <none> 7d6h v1.17.2 bs-k8s-node03 Ready <none> 7d6h v1.17.2 [root@bs-k8s-master01 ~]# kubectl get pods --all-namespaces NAMESPACE NAME READY STATUS RESTARTS AGE default rbd-provisioner-75b85f85bd-8ftdm 1/1 Running 11 7d6h kube-system calico-node-4jxbp 1/1 Running 4 7d6h kube-system calico-node-7t9cj 1/1 Running 7 7d6h kube-system calico-node-cchgl 1/1 Running 14 7d6h kube-system calico-node-czj76 1/1 Running 6 7d6h kube-system calico-node-lxb2s 1/1 Running 14 7d6h kube-system calico-node-nmg9t 1/1 Running 8 7d6h kube-system coredns-7f9c544f75-bwx9p 1/1 Running 4 7d6h kube-system coredns-7f9c544f75-q58mr 1/1 Running 3 7d6h kube-system dashboard-metrics-scraper-6b66849c9-qtwzx 1/1 Running 2 7d5h kube-system etcd-bs-k8s-master01 1/1 Running 17 7d6h kube-system etcd-bs-k8s-master02 1/1 Running 7 7d6h kube-system etcd-bs-k8s-master03 1/1 Running 32 7d6h kube-system kube-apiserver-bs-k8s-master01 1/1 Running 28 7d6h kube-system kube-apiserver-bs-k8s-master02 1/1 Running 15 7d6h kube-system kube-apiserver-bs-k8s-master03 1/1 Running 62 7d6h kube-system kube-controller-manager-bs-k8s-master01 1/1 Running 32 7d6h kube-system kube-controller-manager-bs-k8s-master02 1/1 Running 27 7d6h kube-system kube-controller-manager-bs-k8s-master03 1/1 Running 31 7d6h kube-system kube-proxy-26ffm 1/1 Running 3 7d6h kube-system kube-proxy-298tr 1/1 Running 5 7d6h kube-system kube-proxy-hzsmb 1/1 Running 3 7d6h kube-system kube-proxy-jb4sq 1/1 Running 4 7d6h kube-system kube-proxy-pt94r 1/1 Running 4 7d6h kube-system kube-proxy-wljwv 1/1 Running 4 7d6h kube-system kube-scheduler-bs-k8s-master01 1/1 Running 32 7d6h kube-system kube-scheduler-bs-k8s-master02 1/1 Running 21 7d6h kube-system kube-scheduler-bs-k8s-master03 1/1 Running 31 7d6h kube-system kubernetes-dashboard-887cbd9c6-j7ptq 1/1 Running 22 7d5h [root@bs-k8s-master01 harbor]# pwd /data/k8s/harbor [root@bs-k8s-master01 rbd]# kubectl apply -f ceph-harbor-namespace.yaml namespace/harbor created [root@bs-k8s-master01 rbd]# kubectl get namespaces NAME STATUS AGE default Active 7d8h harbor Active 16s kube-node-lease Active 7d8h kube-public Active 7d8h kube-system Active 7d8h [root@bs-k8s-master01 rbd]# cat ceph-harbor-namespace.yaml ########################################################################## #Author: zisefeizhu #QQ: 2********0 #Date: 2020-02-16 #FileName: ceph-harbor-namespace.yaml #URL: https://www.cnblogs.com/zisefeizhu/ #Description: The test script #Copyright (C): 2020 All rights reserved ########################################################################### apiVersion: v1 kind: Namespace metadata: name: harbor [root@bs-k8s-master01 rbd]# kubectl apply -f external-storage-rbd-provisioner.yaml serviceaccount/rbd-provisioner created clusterrole.rbac.authorization.k8s.io/rbd-provisioner unchanged clusterrolebinding.rbac.authorization.k8s.io/rbd-provisioner configured role.rbac.authorization.k8s.io/rbd-provisioner created rolebinding.rbac.authorization.k8s.io/rbd-provisioner created deployment.apps/rbd-provisioner created [root@bs-k8s-master01 rbd]# kubectl get pods -n harbor -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES rbd-provisioner-75b85f85bd-dhnr4 1/1 Running 0 3m48s 10.209.46.84 bs-k8s-node01 <none> <none> [root@bs-k8s-master01 rbd]# cat external-storage-rbd-provisioner.yaml apiVersion: v1 kind: ServiceAccount metadata: name: rbd-provisioner namespace: harbor --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-provisioner rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["create", "update", "patch"] - apiGroups: [""] resources: ["endpoints"] verbs: ["get", "list", "watch", "create", "update", "patch"] - apiGroups: [""] resources: ["services"] resourceNames: ["kube-dns"] verbs: ["list", "get"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-provisioner subjects: - kind: ServiceAccount name: rbd-provisioner namespace: harbor roleRef: kind: ClusterRole name: rbd-provisioner apiGroup: rbac.authorization.k8s.io --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: rbd-provisioner namespace: harbor rules: - apiGroups: [""] resources: ["secrets"] verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: rbd-provisioner namespace: harbor roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: rbd-provisioner subjects: - kind: ServiceAccount name: rbd-provisioner namespace: harbor --- apiVersion: apps/v1 kind: Deployment metadata: name: rbd-provisioner namespace: harbor spec: replicas: 1 selector: matchLabels: app: rbd-provisioner strategy: type: Recreate template: metadata: labels: app: rbd-provisioner spec: containers: - name: rbd-provisioner image: "quay.io/external_storage/rbd-provisioner:latest" env: - name: PROVISIONER_NAME value: ceph.com/rbd serviceAccount: rbd-provisioner [root@bs-k8s-master01 harbor]# kubectl apply -f ceph-harbor-secret.yaml secret/ceph-harbor-admin-secret created secret/ceph-harbor-harbor-secret created [root@bs-k8s-master01 harbor]# kubectl get secret -n harbor NAME TYPE DATA AGE ceph-harbor-admin-secret kubernetes.io/rbd 1 23s ceph-harbor-harbor-secret kubernetes.io/rbd 1 23s default-token-8k9gs kubernetes.io/service-account-token 3 8m49s rbd-provisioner-token-mhl29 kubernetes.io/service-account-token 3 5m24s [root@bs-k8s-master01 harbor]# cat ceph-harbor-secret.yaml ########################################################################## #Author: zisefeizhu #QQ: 2********0 #Date: 2020-02-16 #FileName: ceph-harbor-secret.yaml #URL: https://www.cnblogs.com/zisefeizhu/ #Description: The test script #Copyright (C): 2020 All rights reserved ########################################################################### apiVersion: v1 kind: Secret metadata: name: ceph-harbor-admin-secret namespace: harbor data: key: QVFDNmNVSmV2eU8yRnhBQVBxYzE5Mm5PelNnZk5acmg5aEFQYXc9PQ== type: kubernetes.io/rbd --- apiVersion: v1 kind: Secret metadata: name: ceph-harbor-harbor-secret namespace: harbor data: key: QVFEb0NrbGVuNmU0TnhBQVZYbXkvUEcrUjVpSDhmTnpNaGs2Smc9PQ== type: kubernetes.io/rbd [root@bs-k8s-master01 harbor]# kubectl apply -f ceph-harbor-storageclass.yaml storageclass.storage.k8s.io/ceph-harbor created [root@bs-k8s-master01 harbor]# kubectl get sc NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE ceph-harbor ceph.com/rbd Retain Immediate false 11s ceph-rbd ceph.com/rbd Retain Immediate false 25h [root@bs-k8s-master01 harbor]# cat ceph-harbor-storageclass.yaml ########################################################################## #Author: zisefeizhu #QQ: 2********0 #Date: 2020-02-16 #FileName: ceph-harbor-storageclass.yaml #URL: https://www.cnblogs.com/zisefeizhu/ #Description: The test script #Copyright (C): 2020 All rights reserved ########################################################################### apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ceph-harbor annotations: storageclass.kubernetes.io/is-default-class: "false" provisioner: ceph.com/rbd reclaimPolicy: Retain parameters: monitors: 20.0.0.206:6789,20.0.0.207:6789,20.0.0.208:6789 adminId: admin adminSecretName: ceph-harbor-admin-secret adminSecretNamespace: harbor pool: harbor fsType: xfs userId: harbor userSecretName: ceph-harbor-harbor-secret imageFormat: "2" imageFeatures: "layering" [root@bs-k8s-master01 harbor]# kubectl apply -f ceph-harbor-pvc.yaml persistentvolumeclaim/pvc-ceph-harbor created wp-pv-claim Bound pvc-494a130d-018c-4be3-9b31-e951cc4367a5 20Gi RWO ceph-rbd 23h [root@bs-k8s-master01 harbor]# kubectl get pv -n harbor NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE pvc-494a130d-018c-4be3-9b31-e951cc4367a5 20Gi RWO Retain Bound default/wp-pv-claim ceph-rbd 23h pvc-4df6a301-c9f3-4694-8271-d1d0184c00aa 1Gi RWO Retain Bound harbor/pvc-ceph-harbor ceph-harbor 6s pvc-8ffa3182-a2f6-47d9-a71d-ff8e8b379a16 1Gi RWO Retain Bound default/ceph-pvc ceph-rbd 26h pvc-ac7d3a09-123e-4614-886c-cded8822a078 20Gi RWO Retain Bound default/mysql-pv-claim ceph-rbd 23h [root@bs-k8s-master01 harbor]# kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE ceph-pvc Bound pvc-8ffa3182-a2f6-47d9-a71d-ff8e8b379a16 1Gi RWO ceph-rbd 26h mysql-pv-claim Bound pvc-ac7d3a09-123e-4614-886c-cded8822a078 20Gi RWO ceph-rbd 23h wp-pv-claim Bound pvc-494a130d-018c-4be3-9b31-e951cc4367a5 20Gi RWO ceph-rbd 23h [root@bs-k8s-master01 harbor]# kubectl get pvc -n harbor NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-ceph-harbor Bound pvc-4df6a301-c9f3-4694-8271-d1d0184c00aa 1Gi RWO ceph-harbor 24s [root@bs-k8s-master01 harbor]# cat ceph-harbor-pvc.yaml ########################################################################## #Author: zisefeizhu #QQ: 2********0 #Date: 2020-02-16 #FileName: ceph-harbor-pvc.yaml #URL: https://www.cnblogs.com/zisefeizhu/ #Description: The test script #Copyright (C): 2020 All rights reserved ########################################################################### apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-ceph-harbor namespace: harbor spec: storageClassName: ceph-harbor accessModes: - ReadWriteOnce resources: requests: storage: 1Gi //到此 完成了在harbor 名称空间下创建动态pv
过手如登山，一步一重天
