cephFS客户端使用指南

目录

• 一.cephFS客户端手动挂载实战案例
  • 1.客户端移除认证文件
  • 2.管理节点创建用户并导出钥匙环和key文件
  • 3.将钥匙环和秘钥key拷贝到客户端指定目录
  • 4.客户端两种挂载cephFS
  • 5.验证yinzhengjiefs普通用户
  • 6.后续使用直接用就可以了，和nfs几乎一样，只不过是高可用的NFS
  • 7.在另一个客户端挂载测试，值得注意是，基于key进行挂载，是不需要拷贝秘钥文件的
• 二.cephFS开机自动挂载的三种方式
  • 1.方式一: 通过"/etc/rc.d/rc.local "文件实现开机自动挂载，（推荐： 就算执行失败，不影响开机）
  • 2.方式二: 使用"/etc/fstab"进行开机挂载
  • 3.方式三: 使用"/etc/fstab"进行开机挂载

一.cephFS客户端手动挂载实战案例

1.客户端移除认证文件

[root@ceph144 ~]# rm -f  /etc/ceph/ceph.client.admin.keyring 
[root@ceph144 ~]# 
[root@ceph144 ~]# ll /etc/ceph/
total 8
-rw-r--r-- 1 root root 353 Feb  2 11:13 ceph.conf
-rw-r--r-- 1 root root  92 Jun 30  2021 rbdmap
-rw------- 1 root root   0 Feb  1 16:50 tmpEYwKWU
[root@ceph144 ~]# 
[root@ceph144 ~]# ceph -s
2024-02-02 15:01:34.969 7fd3b4f99700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.admin.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory
2024-02-02 15:01:34.969 7fd3b4f99700 -1 AuthRegistry(0x7fd3b00662b8) no keyring found at /etc/ceph/ceph.client.admin.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx
2024-02-02 15:01:34.993 7fd3b4f99700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.admin.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory
2024-02-02 15:01:34.993 7fd3b4f99700 -1 AuthRegistry(0x7fd3b00c7e18) no keyring found at /etc/ceph/ceph.client.admin.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx
2024-02-02 15:01:34.994 7fd3b4f99700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.admin.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory
2024-02-02 15:01:34.994 7fd3b4f99700 -1 AuthRegistry(0x7fd3b4f97e78) no keyring found at /etc/ceph/ceph.client.admin.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx
[errno 2] error connecting to the cluster
[root@ceph144 ~]# 
	

2.管理节点创建用户并导出钥匙环和key文件

[root@ceph141 ~]# ceph auth add  client.yinzhengjiefs mon 'allow r' mds 'allow rw' osd 'allow rwx'
added key for client.yinzhengjiefs
[root@ceph141 ~]# 
[root@ceph141 ~]# ceph auth get client.yinzhengjiefs
[client.yinzhengjiefs]
	key = AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
	caps mds = "allow rw"
	caps mon = "allow r"
	caps osd = "allow rwx"
exported keyring for client.yinzhengjiefs
[root@ceph141 ~]# 
[root@ceph141 ~]# ceph auth get client.yinzhengjiefs > ceph.client.yinzhengjiefs.keyring 
exported keyring for client.yinzhengjiefs
[root@ceph141 ~]# 
[root@ceph141 ~]# cat ceph.client.yinzhengjiefs.keyring
[client.yinzhengjiefs]
	key = AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
	caps mds = "allow rw"
	caps mon = "allow r"
	caps osd = "allow rwx"
[root@ceph141 ~]# 
[root@ceph141 ~]# ceph auth  print-key client.yinzhengjiefs > yinzhengjiefs.key
[root@ceph141 ~]# 
[root@ceph141 ~]# more yinzhengjiefs.key 
AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph141 ~]# 

3.将钥匙环和秘钥key拷贝到客户端指定目录

[root@ceph141 ~]# scp ceph.client.yinzhengjiefs.keyring yinzhengjiefs.key ceph144:/etc/ceph/
root@ceph144's password: 
ceph.client.yinzhengjiefs.keyring                                                                      100%  138   133.4KB/s   00:00    
yinzhengjiefs.key                                                                                      100%   40     3.6KB/s   00:00    
[root@ceph141 ~]# 

4.客户端两种挂载cephFS

	1 配置ceph集群主机解析 
[root@ceph144 ~]# cat >> /etc/hosts <<EOF
10.0.0.141 ceph141
10.0.0.142 ceph142
10.0.0.143 ceph143
10.0.0.144 ceph144
EOF


	2 方式一： 基于key文件进行挂载
[root@ceph144 ~]# df -h | grep mnt
[root@ceph144 ~]# 
[root@ceph144 ~]# mount -t ceph ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt -o name=yinzhengjiefs,secretfile=/etc/ceph/yinzhengjiefs.key
[root@ceph144 ~]# 
[root@ceph144 ~]# df -h | grep mnt
10.0.0.141:6789,10.0.0.142:6789,10.0.0.143:6789:/  630G     0  630G   0% /mnt
[root@ceph144 ~]# 
[root@ceph144 ~]# ll /mnt/
total 0
[root@ceph144 ~]# 
[root@ceph144 ~]# cp /etc/os-release /mnt/
[root@ceph144 ~]# 
[root@ceph144 ~]# ll /mnt/
total 1
-rw-r--r-- 1 root root 393 Feb  2 15:10 os-release
[root@ceph144 ~]# 
[root@ceph144 ~]# umount /mnt/
[root@ceph144 ~]# 
[root@ceph144 ~]# df -h | grep mnt
[root@ceph144 ~]# 


	3 方式二: 基于KEY进行挂载，无需拷贝秘钥文件！
[root@ceph144 ~]# more /etc/ceph/yinzhengjiefs.key
AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph144 ~]# 
[root@ceph144 ~]# mount -t ceph ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt -o name=yinzhengjiefs,secret=AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph144 ~]# 
[root@ceph144 ~]# df -h | grep mnt
10.0.0.141:6789,10.0.0.142:6789,10.0.0.143:6789:/  630G     0  630G   0% /mnt
[root@ceph144 ~]# 
[root@ceph144 ~]# ll /mnt/
total 1
-rw-r--r-- 1 root root 393 Feb  2 15:10 os-release
[root@ceph144 ~]#  
[root@ceph144 ~]# umount /mnt
[root@ceph144 ~]# 
[root@ceph144 ~]# df -h | grep mnt
[root@ceph144 ~]# 

5.验证yinzhengjiefs普通用户

[root@ceph144 ~]# ll /etc/ceph/ceph.client.yinzhengjiefs.keyring 
-rw-r--r-- 1 root root 138 Feb  2 15:07 /etc/ceph/ceph.client.yinzhengjiefs.keyring
[root@ceph144 ~]# 
[root@ceph144 ~]# cat /etc/ceph/ceph.client.yinzhengjiefs.keyring 
[client.yinzhengjiefs]
	key = AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
	caps mds = "allow rw"
	caps mon = "allow r"
	caps osd = "allow rwx"
[root@ceph144 ~]# 
[root@ceph144 ~]# ceph -s --user yinzhengjiefs 
  cluster:
    id:     5821e29c-326d-434d-a5b6-c492527eeaad
    health: HEALTH_OK
 
  services:
    mon: 3 daemons, quorum ceph141,ceph142,ceph143 (age 4h)
    mgr: ceph142(active, since 4h), standbys: ceph141, ceph143
    mds: yinzhengjie-linux-cephfs:2 {0=ceph143=up:active,1=ceph141=up:active} 1 up:standby
    osd: 7 osds: 7 up (since 4h), 7 in (since 22h)
 
  data:
    pools:   5 pools, 256 pgs
    objects: 102 objects, 100 MiB
    usage:   7.8 GiB used, 1.9 TiB / 2.0 TiB avail
    pgs:     256 active+clean
 
[root@ceph144 ~]# 

6.后续使用直接用就可以了，和nfs几乎一样，只不过是高可用的NFS

[root@ceph143 ~]# df -h | grep mnt
[root@ceph143 ~]# 
[root@ceph143 ~]# mount -t ceph ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt -o name=yinzhengjiefs,secret=AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph143 ~]# 
[root@ceph143 ~]# df -h | grep mnt
10.0.0.141:6789,10.0.0.142:6789,10.0.0.143:6789:/  630G     0  630G   0% /mnt
[root@ceph143 ~]# 
[root@ceph143 ~]# ll /mnt/
total 1
-rw-r--r-- 1 root root 393 Feb  2 15:10 os-release
[root@ceph143 ~]# 
[root@ceph143 ~]# cp /etc/sysconfig/network-scripts/ifcfg-ens33 /mnt/
[root@ceph143 ~]# 
[root@ceph143 ~]# ll /mnt/
total 1
-rw-r--r-- 1 root root 147 Feb  2 15:17 ifcfg-ens33
-rw-r--r-- 1 root root 393 Feb  2 15:10 os-release
[root@ceph143 ~]# 

7.在另一个客户端挂载测试，值得注意是，基于key进行挂载，是不需要拷贝秘钥文件的

[root@ceph142 ~]# df -h | grep mnt
[root@ceph142 ~]# 
[root@ceph142 ~]# mount -t ceph ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt -o name=yinzhengjiefs,secret=AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph142 ~]# 
[root@ceph142 ~]# df -h | grep mnt
10.0.0.141:6789,10.0.0.142:6789,10.0.0.143:6789:/  630G     0  630G   0% /mnt
[root@ceph142 ~]# 
[root@ceph142 ~]# ll /mnt/
total 1
-rw-r--r-- 1 root root 147 Feb  2 15:17 ifcfg-ens33
-rw-r--r-- 1 root root 393 Feb  2 15:10 os-release
[root@ceph142 ~]# 

二.cephFS开机自动挂载的三种方式

1.方式一: 通过"/etc/rc.d/rc.local "文件实现开机自动挂载，（推荐： 就算执行失败，不影响开机）

[root@ceph144 ~]# vim /etc/rc.d/rc.local 
...

mount -t ceph ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt -o name=yinzhengjiefs,secret=AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph144 ~]#
[root@ceph144 ~]# ll /etc/rc.d/rc.local
-rw-r--r-- 1 root root 602 Feb  2 15:19 /etc/rc.d/rc.local
[root@ceph144 ~]# 
[root@ceph144 ~]# chmod +x /etc/rc.d/rc.local
[root@ceph144 ~]# 
[root@ceph144 ~]# ll /etc/rc.d/rc.local
-rwxr-xr-x 1 root root 602 Feb  2 15:19 /etc/rc.d/rc.local
[root@ceph144 ~]# 
[root@ceph144 ~]# df -h | grep mnt
[root@ceph144 ~]# 
[root@ceph144 ~]# reboot
[root@ceph144 ~]# 
[root@ceph144 ~]# df -h | grep mnt
10.0.0.141:6789,10.0.0.142:6789,10.0.0.143:6789:/  630G     0  630G   0% /mnt
[root@ceph144 ~]# 

2.方式二: 使用"/etc/fstab"进行开机挂载

[root@ceph144 ~]# grep yinzhengjie /etc/rc.d/rc.local 
mount -t ceph ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt -o name=yinzhengjiefs,secret=AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph144 ~]# 
[root@ceph144 ~]# sed -i '/mount/s@^@#@' /etc/rc.d/rc.local 
[root@ceph144 ~]# 
[root@ceph144 ~]# grep yinzhengjie /etc/rc.d/rc.local 
#mount -t ceph ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt -o name=yinzhengjiefs,secret=AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph144 ~]# 
[root@ceph144 ~]# reboot
[root@ceph144 ~]# 
[root@ceph144 ~]# df -h | grep mnt  # 发现并不会开机自动挂载，说明上面的配置生效了。
[root@ceph144 ~]# 
[root@ceph144 ~]# vim /etc/fstab 
[root@ceph144 ~]# 
[root@ceph144 ~]# grep yinzhengjie /etc/fstab 
ceph141:6789,ceph142:6789,ceph143:6789:/	 /mnt/    ceph name=yinzhengjiefs,secretfile=/etc/ceph/yinzhengjiefs.key,noatime,_netdev    0       2
[root@ceph144 ~]# 
[root@ceph144 ~]# reboot 
[root@ceph144 ~]# 
[root@ceph144 ~]# df -h | grep mnt
10.0.0.141:6789,10.0.0.142:6789,10.0.0.143:6789:/  630G     0  630G   0% /mnt
[root@ceph144 ~]# 


温馨提示:
	不推荐使用，一定要谨慎修改，因为一旦修改失败，会导致服务器启动失败，在开机挂载时阻塞住，需要进单用户模式救援。

3.方式三: 使用"/etc/fstab"进行开机挂载

[root@ceph144 ~]# grep yinzhengjie /etc/fstab 
ceph141:6789,ceph142:6789,ceph143:6789:/	 /mnt/    ceph name=yinzhengjiefs,secretfile=/etc/ceph/yinzhengjiefs.key,noatime,_netdev    0       2
[root@ceph144 ~]# 
[root@ceph144 ~]# sed -i '/ceph/s@^@#@' /etc/fstab 
[root@ceph144 ~]# 
[root@ceph144 ~]# grep yinzhengjie /etc/fstab 
#ceph141:6789,ceph142:6789,ceph143:6789:/	 /mnt/    ceph name=yinzhengjiefs,secretfile=/etc/ceph/yinzhengjiefs.key,noatime,_netdev    0       2
[root@ceph144 ~]# 
[root@ceph144 ~]# vim /etc/fstab 
[root@ceph144 ~]# 
[root@ceph144 ~]# more /etc/ceph/yinzhengjiefs.key 
AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph144 ~]# 
[root@ceph144 ~]# 
[root@ceph144 ~]# grep yinzhengjie /etc/fstab 
#ceph141:6789,ceph142:6789,ceph143:6789:/	 /mnt/    ceph name=yinzhengjiefs,secretfile=/etc/ceph/yinzhengjiefs.key,noatime,_netdev    0       2
ceph141:6789,ceph142:6789,ceph143:6789:/	 /mnt/    ceph name=yinzhengjiefs,secret=AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==,noatime,_netdev    0       2
[root@ceph144 ~]# 
[root@ceph144 ~]# reboot 
[root@ceph144 ~]# 
[root@ceph144 ~]# df -h | grep mnt
10.0.0.141:6789,10.0.0.142:6789,10.0.0.143:6789:/  630G     0  630G   0% /mnt
[root@ceph144 ~]# 
[root@ceph144 ~]# ll /mnt/
total 1
-rw-r--r-- 1 root root 147 Feb  2 15:17 ifcfg-ens33
-rw-r--r-- 1 root root 393 Feb  2 15:10 os-release
[root@ceph144 ~]#