六、Ceph之Cephfs的使用

1、Cephfs介绍

ceph FS 即ceph filesystem，可以实现文件系统共享功能,客户端通过ceph 协议挂载并使用ceph 集群作为数据存储服务器。 Ceph FS 需要运行Meta Data Services(MDS)服务，其守护进程为ceph-mds，ceph-mds 进程管理与cephFS 上存储的文件相关的元数据，并协调对ceph 存储集群的访问。

cephfs 的元数据使用的动态子树分区,把元数据划分名称空间对应到不同的mds,写入元数据的时候将元数据按照名称保存到不同主mds 上,有点类似于nginx 中的缓存目录分层一样

2、部署cephfs客户端

2.1、查看cephfs的mds状态

cephadmin@ceph-deploy:~/ceph-cluster$ ceph mds stat
mycephfs:2 {0=ceph-mds01=up:active,1=ceph-mgr02=up:active} 2 up:standby

2.2、在ceph管理节点（ceph-deploy）创建普通客户端

# 创建client.zhai普通用户
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph auth add client.zhai mon 'allow r' mds "allow rw" osd "allow rwx pool=cephfs-data"
added key for client.zhai

# 验证用户
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph auth get client.zhai
[client.zhai]
	key = AQC8wElif/VWCRAAiETOB+qmMdhKEUz/kMN85w==
	caps mds = "allow rw"
	caps mon = "allow r"
	caps osd = "allow rwx pool=cephfs-data"
exported keyring for client.zhai

# 创建kingseal的空keyring
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-authtool --create-keyring ceph.client.zhai.keyring
creating ceph.client.zhai.keyring

# 将用户信息导入keyring文件
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get client.zhai -o ceph.client.zhai.keyring
exported keyring for client.zhai

# 创建key文件，cephfs挂载使用
cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth print-key client.zhai
AQC8wElif/VWCRAAiETOB+qmMdhKEUz/kMN85w==

2.3、用户信息拷贝到客户端

cephadmin@ceph-deploy:~/ceph-cluster$ sudo scp ceph.conf ceph.client.zhai.keyring zhai.key 192.168.1.180:/etc/ceph

2.4、客户端验证权限

root@client:~# ceph --user zhai -s
  cluster:
    id:     fdefc83b-1ef9-4986-a4c1-8af7603f43bf
    health: HEALTH_WARN
            clock skew detected on mon.ceph-mon03
  
  services:
    mon: 3 daemons, quorum ceph-mon01,ceph-mon02,ceph-mon03 (age 37m)
    mgr: ceph-mgr01(active, since 56m), standbys: ceph-mgr02
    mds: 2/2 daemons up, 2 standby
    osd: 16 osds: 16 up (since 41m), 16 in (since 41m)
  
  data:
    volumes: 1/1 healthy
    pools:   3 pools, 97 pgs
    objects: 41 objects, 3.6 KiB
    usage:   104 MiB used, 800 GiB / 800 GiB avail
    pgs:     97 active+clean

3、挂载cephfs

客户端挂载有两种方式，一是内核空间一是用户空间，内核空间挂载需要内核支持ceph 模块，用户空间挂载需要安装ceph-fuse

3.1、客户端通过key文件挂载

#创建挂载目录cephfs-key
root@client:~# mkdir /cephfs-key

#挂载cephfs
root@ceph-client:/etc/ceph# mount -t ceph 192.168.1.203:6789,192.168.1.204:6789,192.168.1.205:6789:/ /cephfs-key -o name=zhai,secretfile=/etc/ceph/zhai.key

#注意：192.168.1.203:6789,192.168.1.204:6789,192.168.1.205:6789为ceph-mon地址和端口

3.2、客户端直接通过key挂载

root@ceph-client:/etc/ceph# cat zhai.key 
AQC8wElif/VWCRAAiETOB+qmMdhKEUz/kMN85w==

root@ceph-client:/etc/ceph# mount -t ceph 192.168.1.203:6789,192.168.1.204:6789,192.168.1.205:6789:/ /cephfs-key -o name=zhai,secretfile=/etc/ceph/zhai.key

客户端模块

客户端内核加载ceph.ko 模块挂载cephfs 文件系统

root@client:~# lsmod | grep ceph
ceph                  376832  1
libceph               315392  1 ceph
fscache                65536  1 ceph
libcrc32c              16384  3 xfs,raid456,libceph
root@client:~# modinfo ceph
filename:       /lib/modules/4.15.0-112-generic/kernel/fs/ceph/ceph.ko
license:        GPL
description:    Ceph filesystem for Linux
author:         Patience Warnick <patience@newdream.net>
author:         Yehuda Sadeh <yehuda@hq.newdream.net>
author:         Sage Weil <sage@newdream.net>
alias:          fs-ceph
srcversion:     B2806F4EAACAC1E19EE7AFA
depends:        libceph,fscache
retpoline:      Y
intree:         Y
name:           ceph
vermagic:       4.15.0-112-generic SMP mod_unload 
signat:         PKCS#7
signer:         
sig_key:        
sig_hashalgo:   md4

3.3、用户空间（ceph-fsus）挂载

如果内核本较低而没有ceph 模块，那么可以安装ceph-fuse 挂载，但是推荐使用内核模块挂载

#安装ceph-fuse
apt install ceph-fuse
#创建挂载目录ceph-fsus
mkdir /ceph-fsus
#使用ceph-fsus挂载
root@client:~# ll /etc/ceph/
total 32
drwxr-xr-x   2 root root  126 Aug 25 23:07 ./
drwxr-xr-x 100 root root 8192 Aug 25 23:07 ../
-rw-------   1 root root  151 Aug 25 22:49 ceph.client.admin.keyring
-rw-------   1 root root  152 Aug 25 23:02 ceph.client.zhai.keyring
-rw-r--r--   1 root root  516 Aug 25 23:02 ceph.conf
-rw-r--r--   1 root root   40 Aug 25 23:02 zhai.key
-rw-r--r--   1 root root   92 Jun  7 22:39 rbdmap
#client.zhai为ceph.client.zhai.keyring
root@client:~# ceph-fuse --name client.zhai -m 192.168.1.203:6789,192.168.1.204:6789,192.168.1.205:6789 /ceph-fsus
 
ceph-fuse[5569]: starting ceph client
2021-08-25T23:28:07.140+0800 7f4feeea9100 -1 init, newargv = 0x55a107cc4920 newargc=15
ceph-fuse[5569]: starting fuse

3.4、验证

root@ceph-client:/etc/ceph# df -Th
..........
192.168.1.203:6789,192.168.1.204:6789,192.168.1.205:6789:/ ceph       38G     0   38G   0% /cephfs-key
ceph-fuse                                                     fuse.ceph-fuse  254G     0  254G   0% /ceph-fsus

4、开机挂载

4.1、fstab

cat /etc/fstab
192.168.1.203:6789,192.168.1.204:6789,192.168.1.205:6789:/ /cephfs-key ceph defaults,name=zhai,secretfile=/etc/ceph/zhai.key,_netdev 0 0

4.2、rc.local

chmod +x /etc/rc.local
 
cat /etc/rc.local
mount -t ceph 192.168.1.203:6789,192.168.1.204:6789,192.168.1.205:6789:/ /cephfs-key -o name=zhai,secretfile=/etc/ceph/zhai.key