k8s对接ceph集群的分布式文件系统CephFS

一.k8s使用cephFS指定加密文件之secretFile

1.k8s集群的所有worker节点安装ceph软件包

CentOS系统:
[root@master231 ~]# yum -y install ceph
[root@worker232 ~]# yum -y install ceph
[root@worker233 ~]# yum -y install ceph


Ubuntu系统:
[root@master231 ~]# apt -y install ceph-common
[root@worker232 ~]# apt -y install ceph-common
[root@worker233 ~]# apt -y install ceph-common
	
温馨提示:
	- 1.Ubuntu默认安装的最新版本是17.2.7
[root@master231 ~]# ceph -v
ceph version 17.2.7 (b12291d110049b2f35e32e0de30d70e9a4c060d2) quincy (stable)
[root@master231 ~]# 
	
	- 2.如果想要安装最新版本的18.2.4,可参考我之前的笔记
推荐阅读:
https://www.cnblogs.com/yinzhengjie/p/18372796#二ceph的管理节点配置

2.ceph节点将认证文件拷贝到K8S的所有worker节点

[root@ceph141 ~]# ceph auth print-key client.admin > admin.secret
[root@ceph141 ~]# 
[root@ceph141 ~]# more admin.secret
AQDjFrplyvFCDhAApJg111YMIGQ6/F/x/Y+qpQ==
[root@ceph141 ~]# 
[root@ceph141 ~]# 
[root@ceph141 ~]# scp admin.secret 10.0.0.231:/etc/ceph/
[root@ceph141 ~]# scp admin.secret 10.0.0.232:/etc/ceph/
[root@ceph141 ~]# scp admin.secret 10.0.0.233:/etc/ceph/

3.编写资源清单

[root@master231 cephfs]# cat 01-deploy-svc-volume-cephfs-admin-secretFile.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: deploy-volume-cephfs-admin-secretfile
spec:
  replicas: 3
  selector:
    matchLabels:
      apps: ceph-fs
  template:
    metadata:
      labels:
        apps: ceph-fs
    spec:
      volumes:
      - name: data
        # 指定存储卷的类型是cephFS
        cephfs:
          monitors:
          - 10.0.0.141:6789
          - 10.0.0.142:6789
          - 10.0.0.143:6789
          # 指定引用的cephFS的路径,若不指定默认为"/"
          path: /
          # 对于Pod而言,无需修改文件,因此设置为true
          readOnly: true
          # 指定连接ceph集群的用户,若不指定,默认为admin
          user: admin
          # 指定admin用户对应的认证文件所在路径
          secretFile: "/etc/ceph/admin.secret"
      containers:
      - name: c1
        image: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v1
        volumeMounts:
        - name: data
          mountPath: /yinzhengjie-data
        ports:
        - containerPort: 80

---

apiVersion: v1
kind: Service
metadata:
  name: svc-cephfs-secretfile
spec:
  type: NodePort
  selector:
    apps: ceph-fs
  ports:
  - protocol: TCP
    port: 80
    targetPort: 80
    nodePort: 20090
[root@master231 cephfs]# 

4.创建资源

[root@master231 cephfs]# kubectl apply -f 01-deploy-svc-volume-cephfs-admin-secretFile.yaml 
deployment.apps/deploy-volume-cephfs-admin-secretfile unchanged
service/svc-cephfs-secretfile created
[root@master231 cephfs]# 
[root@master231 cephfs]# kubectl get pods -o wide
NAME                                                     READY   STATUS    RESTARTS   AGE   IP             NODE        NOMINATED NODE   READINESS GATES
deploy-volume-cephfs-admin-secretfile-6b847b7d84-9bf2n   1/1     Running   0          37s   10.100.1.177   worker232   <none>           <none>
deploy-volume-cephfs-admin-secretfile-6b847b7d84-tsbv6   1/1     Running   0          37s   10.100.2.34    worker233   <none>           <none>
deploy-volume-cephfs-admin-secretfile-6b847b7d84-vmdhk   1/1     Running   0          37s   10.100.2.33    worker233   <none>           <none>
[root@master231 cephfs]# 
[root@master231 cephfs]# kubectl get svc,ep svc-cephfs-secretfile 
NAME                            TYPE       CLUSTER-IP       EXTERNAL-IP   PORT(S)        AGE
service/svc-cephfs-secretfile   NodePort   10.200.110.202   <none>        80:20090/TCP   15s

NAME                              ENDPOINTS                                       AGE
endpoints/svc-cephfs-secretfile   10.100.1.177:80,10.100.2.33:80,10.100.2.34:80   15s
[root@master231 cephfs]# 
	

5.访问测试

http://10.0.0.233:20090/

6.推荐阅读

参考链接:
	https://github.com/kubernetes/examples/blob/master/volumes/cephfs/README.md
	https://www.cnblogs.com/yinzhengjie/p/14305987.html	

二.k8s使用cephFS指定秘钥之secretRef

1.将K8S所有的worker节点的对应的认证文件全部删除

[root@master231 ~]# rm -f /etc/ceph/admin.secret 
[root@worker232 ~]# rm -f /etc/ceph/admin.secret 
[root@worker233 ~]# rm -f /etc/ceph/admin.secret 

2.对ceph集群的admin用户的KEY进行base64编码

[root@ceph141 ~]# grep key /etc/ceph/ceph.client.admin.keyring | awk '{printf "%s", $NF}' | base64 
QVFEakZycGx5dkZDRGhBQXBKZzExMVlNSUdRNi9GL3gvWStxcFE9PQ==
[root@ceph141 ~]# 

3.编写资源清单

[root@master231 cephfs]# cat 02-deploy-svc-ing-secrets-volume-cephfs-admin-secretRef.yaml 
apiVersion: v1
kind: Secret
metadata:
  name: ceph-admin-secret
type: "kubernetes.io/rbd"
data:
  # 指定ceph的admin的KEY,将其进行base64编码,此处需要修改! 
  key: QVFEakZycGx5dkZDRGhBQXBKZzExMVlNSUdRNi9GL3gvWStxcFE9PQ==

---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: deploy-volume-cephfs-admin-secretref
spec:
  replicas: 3
  selector:
    matchLabels:
      apps: ceph-fs
  template:
    metadata:
      labels:
        apps: ceph-fs
    spec:
      volumes:
      - name: data
        cephfs:
          monitors:
          - 10.0.0.141:6789
          - 10.0.0.142:6789
          - 10.0.0.143:6789
          readOnly: true
          user: admin
          secretRef:
            name: ceph-admin-secret
      containers:
      - name: c1
        image: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v2
        volumeMounts:
        - name: data
          mountPath: /yinzhengjie-data
        ports:
        - containerPort: 80

---

apiVersion: v1
kind: Service
metadata:
  name: svc-cephfs-secrets
spec:
  selector:
    apps: ceph-fs
  ports:
  - protocol: TCP
    port: 80
    targetPort: 80

---

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: apps-ingress-secrets
  annotations:
    # 指定Ingress controller的类型
    kubernetes.io/ingress.class: traefik
spec:
  # 指定Ingress controller的名称
  # ingressClassName: mytraefik
  rules:
  - host: v2.yinzhengjie.com
    http:
      paths:
      - backend:
          service:
            name: svc-cephfs-secrets
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific
[root@master231 cephfs]# 

4.创建资源

[root@master231 cephfs]# kubectl apply -f 02-deploy-svc-ing-secrets-volume-cephfs-admin-secretRef.yaml 
secret/ceph-admin-secret created
deployment.apps/deploy-volume-cephfs-admin-secretref created
service/svc-cephfs-secrets created
ingress.networking.k8s.io/apps-ingress-secrets created
[root@master231 cephfs]# 
[root@master231 cephfs]# kubectl get pods -o wide
NAME                                                     READY   STATUS        RESTARTS   AGE   IP             NODE        NOMINATED NODE   READINESS GATES
deploy-volume-cephfs-admin-secretref-84555b8b8f-8d4jp    1/1     Running       0          4s    10.100.2.38    worker233   <none>           <none>
deploy-volume-cephfs-admin-secretref-84555b8b8f-cdxng    1/1     Running       0          4s    10.100.2.37    worker233   <none>           <none>
deploy-volume-cephfs-admin-secretref-84555b8b8f-lxmrb    1/1     Running       0          4s    10.100.1.179   worker232   <none>           <none>
[root@master231 cephfs]# 
[root@master231 cephfs]# 
[root@master231 cephfs]# kubectl get svc,ep 
kubernetes          svc-cephfs-secrets  
[root@master231 cephfs]# kubectl get svc,ep svc-cephfs-secrets 
NAME                         TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)   AGE
service/svc-cephfs-secrets   ClusterIP   10.200.163.115   <none>        80/TCP    12s

NAME                           ENDPOINTS                                       AGE
endpoints/svc-cephfs-secrets   10.100.1.179:80,10.100.2.37:80,10.100.2.38:80   12s
[root@master231 cephfs]# 
[root@master231 cephfs]# kubectl get ing apps-ingress-secrets 
NAME                   CLASS    HOSTS              ADDRESS   PORTS   AGE
apps-ingress-secrets   <none>   v2.yinzhengjie.com             80      26s
[root@master231 cephfs]# 
[root@master231 cephfs]# 
[root@master231 cephfs]# kubectl describe ing apps-ingress-secrets 
Name:             apps-ingress-secrets
Labels:           <none>
Namespace:        default
Address:          
Default backend:  default-http-backend:80 (<error: endpoints "default-http-backend" not found>)
Rules:
  Host              Path  Backends
  ----              ----  --------
  v2.yinzhengjie.com  
                    /   svc-cephfs-secrets:80 (10.100.1.179:80,10.100.2.37:80,10.100.2.38:80)
Annotations:        kubernetes.io/ingress.class: traefik
Events:             <none>
[root@master231 cephfs]# 
[root@master231 cephfs]# kubectl -n yinzhengjie-traefik get po,svc 
NAME                             READY   STATUS    RESTARTS   AGE
pod/mytraefik-5f6bd48975-6w8gm   1/1     Running   0          3d

NAME                TYPE           CLUSTER-IP      EXTERNAL-IP   PORT(S)                      AGE
service/mytraefik   LoadBalancer   10.200.205.77   10.0.0.189    80:18238/TCP,443:13380/TCP   12d
[root@master231 cephfs]# 

5.添加windows主机解析

10.0.0.189  v2.yinzhengjie.com

6.访问测试

http://v2.yinzhengjie.com/

7.推荐阅读

参考链接:
	https://github.com/kubernetes/examples/blob/master/volumes/cephfs/README.md
	https://www.cnblogs.com/yinzhengjie/p/14305987.html
posted @ 2021-01-29 02:16  尹正杰  阅读(239)  评论(0编辑  收藏  举报