k8s对接ceph集群的分布式文件系统CephFS
目录
一.k8s使用cephFS指定加密文件之secretFile
1.k8s集群的所有worker节点安装ceph软件包
CentOS系统:
[root@master231 ~]# yum -y install ceph
[root@worker232 ~]# yum -y install ceph
[root@worker233 ~]# yum -y install ceph
Ubuntu系统:
[root@master231 ~]# apt -y install ceph-common
[root@worker232 ~]# apt -y install ceph-common
[root@worker233 ~]# apt -y install ceph-common
温馨提示:
- 1.Ubuntu默认安装的最新版本是17.2.7
[root@master231 ~]# ceph -v
ceph version 17.2.7 (b12291d110049b2f35e32e0de30d70e9a4c060d2) quincy (stable)
[root@master231 ~]#
- 2.如果想要安装最新版本的18.2.4,可参考我之前的笔记
推荐阅读:
https://www.cnblogs.com/yinzhengjie/p/18372796#二ceph的管理节点配置
2.ceph节点将认证文件拷贝到K8S的所有worker节点
[root@ceph141 ~]# ceph auth print-key client.admin > admin.secret
[root@ceph141 ~]#
[root@ceph141 ~]# more admin.secret
AQDjFrplyvFCDhAApJg111YMIGQ6/F/x/Y+qpQ==
[root@ceph141 ~]#
[root@ceph141 ~]#
[root@ceph141 ~]# scp admin.secret 10.0.0.231:/etc/ceph/
[root@ceph141 ~]# scp admin.secret 10.0.0.232:/etc/ceph/
[root@ceph141 ~]# scp admin.secret 10.0.0.233:/etc/ceph/
3.编写资源清单
[root@master231 cephfs]# cat 01-deploy-svc-volume-cephfs-admin-secretFile.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: deploy-volume-cephfs-admin-secretfile
spec:
replicas: 3
selector:
matchLabels:
apps: ceph-fs
template:
metadata:
labels:
apps: ceph-fs
spec:
volumes:
- name: data
# 指定存储卷的类型是cephFS
cephfs:
monitors:
- 10.0.0.141:6789
- 10.0.0.142:6789
- 10.0.0.143:6789
# 指定引用的cephFS的路径,若不指定默认为"/"
path: /
# 对于Pod而言,无需修改文件,因此设置为true
readOnly: true
# 指定连接ceph集群的用户,若不指定,默认为admin
user: admin
# 指定admin用户对应的认证文件所在路径
secretFile: "/etc/ceph/admin.secret"
containers:
- name: c1
image: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v1
volumeMounts:
- name: data
mountPath: /yinzhengjie-data
ports:
- containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
name: svc-cephfs-secretfile
spec:
type: NodePort
selector:
apps: ceph-fs
ports:
- protocol: TCP
port: 80
targetPort: 80
nodePort: 20090
[root@master231 cephfs]#
4.创建资源
[root@master231 cephfs]# kubectl apply -f 01-deploy-svc-volume-cephfs-admin-secretFile.yaml
deployment.apps/deploy-volume-cephfs-admin-secretfile unchanged
service/svc-cephfs-secretfile created
[root@master231 cephfs]#
[root@master231 cephfs]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
deploy-volume-cephfs-admin-secretfile-6b847b7d84-9bf2n 1/1 Running 0 37s 10.100.1.177 worker232 <none> <none>
deploy-volume-cephfs-admin-secretfile-6b847b7d84-tsbv6 1/1 Running 0 37s 10.100.2.34 worker233 <none> <none>
deploy-volume-cephfs-admin-secretfile-6b847b7d84-vmdhk 1/1 Running 0 37s 10.100.2.33 worker233 <none> <none>
[root@master231 cephfs]#
[root@master231 cephfs]# kubectl get svc,ep svc-cephfs-secretfile
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/svc-cephfs-secretfile NodePort 10.200.110.202 <none> 80:20090/TCP 15s
NAME ENDPOINTS AGE
endpoints/svc-cephfs-secretfile 10.100.1.177:80,10.100.2.33:80,10.100.2.34:80 15s
[root@master231 cephfs]#
5.访问测试
http://10.0.0.233:20090/
6.推荐阅读
参考链接:
https://github.com/kubernetes/examples/blob/master/volumes/cephfs/README.md
https://www.cnblogs.com/yinzhengjie/p/14305987.html
二.k8s使用cephFS指定秘钥之secretRef
1.将K8S所有的worker节点的对应的认证文件全部删除
[root@master231 ~]# rm -f /etc/ceph/admin.secret
[root@worker232 ~]# rm -f /etc/ceph/admin.secret
[root@worker233 ~]# rm -f /etc/ceph/admin.secret
2.对ceph集群的admin用户的KEY进行base64编码
[root@ceph141 ~]# grep key /etc/ceph/ceph.client.admin.keyring | awk '{printf "%s", $NF}' | base64
QVFEakZycGx5dkZDRGhBQXBKZzExMVlNSUdRNi9GL3gvWStxcFE9PQ==
[root@ceph141 ~]#
3.编写资源清单
[root@master231 cephfs]# cat 02-deploy-svc-ing-secrets-volume-cephfs-admin-secretRef.yaml
apiVersion: v1
kind: Secret
metadata:
name: ceph-admin-secret
type: "kubernetes.io/rbd"
data:
# 指定ceph的admin的KEY,将其进行base64编码,此处需要修改!
key: QVFEakZycGx5dkZDRGhBQXBKZzExMVlNSUdRNi9GL3gvWStxcFE9PQ==
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: deploy-volume-cephfs-admin-secretref
spec:
replicas: 3
selector:
matchLabels:
apps: ceph-fs
template:
metadata:
labels:
apps: ceph-fs
spec:
volumes:
- name: data
cephfs:
monitors:
- 10.0.0.141:6789
- 10.0.0.142:6789
- 10.0.0.143:6789
readOnly: true
user: admin
secretRef:
name: ceph-admin-secret
containers:
- name: c1
image: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v2
volumeMounts:
- name: data
mountPath: /yinzhengjie-data
ports:
- containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
name: svc-cephfs-secrets
spec:
selector:
apps: ceph-fs
ports:
- protocol: TCP
port: 80
targetPort: 80
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: apps-ingress-secrets
annotations:
# 指定Ingress controller的类型
kubernetes.io/ingress.class: traefik
spec:
# 指定Ingress controller的名称
# ingressClassName: mytraefik
rules:
- host: v2.yinzhengjie.com
http:
paths:
- backend:
service:
name: svc-cephfs-secrets
port:
number: 80
path: /
pathType: ImplementationSpecific
[root@master231 cephfs]#
4.创建资源
[root@master231 cephfs]# kubectl apply -f 02-deploy-svc-ing-secrets-volume-cephfs-admin-secretRef.yaml
secret/ceph-admin-secret created
deployment.apps/deploy-volume-cephfs-admin-secretref created
service/svc-cephfs-secrets created
ingress.networking.k8s.io/apps-ingress-secrets created
[root@master231 cephfs]#
[root@master231 cephfs]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
deploy-volume-cephfs-admin-secretref-84555b8b8f-8d4jp 1/1 Running 0 4s 10.100.2.38 worker233 <none> <none>
deploy-volume-cephfs-admin-secretref-84555b8b8f-cdxng 1/1 Running 0 4s 10.100.2.37 worker233 <none> <none>
deploy-volume-cephfs-admin-secretref-84555b8b8f-lxmrb 1/1 Running 0 4s 10.100.1.179 worker232 <none> <none>
[root@master231 cephfs]#
[root@master231 cephfs]#
[root@master231 cephfs]# kubectl get svc,ep
kubernetes svc-cephfs-secrets
[root@master231 cephfs]# kubectl get svc,ep svc-cephfs-secrets
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/svc-cephfs-secrets ClusterIP 10.200.163.115 <none> 80/TCP 12s
NAME ENDPOINTS AGE
endpoints/svc-cephfs-secrets 10.100.1.179:80,10.100.2.37:80,10.100.2.38:80 12s
[root@master231 cephfs]#
[root@master231 cephfs]# kubectl get ing apps-ingress-secrets
NAME CLASS HOSTS ADDRESS PORTS AGE
apps-ingress-secrets <none> v2.yinzhengjie.com 80 26s
[root@master231 cephfs]#
[root@master231 cephfs]#
[root@master231 cephfs]# kubectl describe ing apps-ingress-secrets
Name: apps-ingress-secrets
Labels: <none>
Namespace: default
Address:
Default backend: default-http-backend:80 (<error: endpoints "default-http-backend" not found>)
Rules:
Host Path Backends
---- ---- --------
v2.yinzhengjie.com
/ svc-cephfs-secrets:80 (10.100.1.179:80,10.100.2.37:80,10.100.2.38:80)
Annotations: kubernetes.io/ingress.class: traefik
Events: <none>
[root@master231 cephfs]#
[root@master231 cephfs]# kubectl -n yinzhengjie-traefik get po,svc
NAME READY STATUS RESTARTS AGE
pod/mytraefik-5f6bd48975-6w8gm 1/1 Running 0 3d
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/mytraefik LoadBalancer 10.200.205.77 10.0.0.189 80:18238/TCP,443:13380/TCP 12d
[root@master231 cephfs]#
5.添加windows主机解析
10.0.0.189 v2.yinzhengjie.com
6.访问测试
http://v2.yinzhengjie.com/
7.推荐阅读
参考链接:
https://github.com/kubernetes/examples/blob/master/volumes/cephfs/README.md
https://www.cnblogs.com/yinzhengjie/p/14305987.html
当你的才华还撑不起你的野心的时候,你就应该静下心来学习。当你的能力还驾驭不了你的目标的时候,你就应该沉下心来历练。问问自己,想要怎样的人生。
欢迎交流学习技术交流,个人微信: "JasonYin2020"(添加时请备注来源及意图备注)
作者: 尹正杰, 博客: https://www.cnblogs.com/yinzhengjie/p/14342807.html