Ceph使用---块设备、RGW、CephFS初步使用
一、ceph应用逻辑图
二、rbd块设备应用
[root@ceph-deploy ~]# ceph osd pool create myrbd1 64 64 #创建存储池,指定pg和pgp的数量， pgp是对存在于pg的数据进行组合存储，pgp通常等于pg的值 pool 'myrbd1' created [root@ceph-deploy ~]# [root@ceph-deploy ~]# ceph osd pool application enable myrbd1 rbd #对存储池启用 RBD 功能 enabled application 'rbd' on pool 'myrbd1' [root@ceph-deploy ~]# [root@ceph-deploy ~]# rbd pool init -p myrbd1 #通过 RBD 命令对存储池初始化 [root@ceph-deploy ~]# ceph df --- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 900 GiB 900 GiB 58 MiB 58 MiB 0 TOTAL 900 GiB 900 GiB 58 MiB 58 MiB 0 --- POOLS --- POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL device_health_metrics 1 1 0 B 0 0 B 0 285 GiB myrbd1 2 64 19 B 1 12 KiB 0 285 GiB [root@ceph-deploy ~]#
此时，rbd存储池并不能直接用于块设备，而是需要事先在其中按需创建映像（image），并把映像文件作为块设备使用，rbd 命令可用于创建、查看及删除块设备相在的映像（image），以及克隆映像、创建快照、将映像回滚到快照和查看快照等管理操作。
创建并验证img
[root@ceph-deploy ~]# rbd create myimg1 --size 5G --pool myrbd1 [root@ceph-deploy ~]# rbd create myimg2 --size 3G --pool myrbd1 --image-format 2 --image-feature layering #由于 centos 系统内核较低不支持更多 image-feature特性、因此无法挂载使用，所以只开启部分特性，除了layering其他特性需要高版本内核支持。 [root@ceph-deploy ~]# [root@ceph-deploy ~]# rbd ls --pool myrbd1 #列出指定的 pool 中所有的 img myimg1 myimg2 [root@ceph-deploy ~]# rbd --image myimg1 --pool myrbd1 info #查看指定 rdb 的信息 rbd image 'myimg1': size 5 GiB in 1280 objects order 22 (4 MiB objects) snapshot_count: 0 id: 85ab8cd379d0 block_name_prefix: rbd_data.85ab8cd379d0 format: 2 features: layering, exclusive-lock, object-map, fast-diff, deep-flatten op_features: flags: create_timestamp: Fri Oct 7 10:18:59 2022 access_timestamp: Fri Oct 7 10:18:59 2022 modify_timestamp: Fri Oct 7 10:18:59 2022 [root@ceph-deploy ~]# [root@ceph-deploy ~]# rbd --image myimg2 --pool myrbd1 info rbd image 'myimg2': size 3 GiB in 768 objects order 22 (4 MiB objects) snapshot_count: 0 id: 85ae9b3145be block_name_prefix: rbd_data.85ae9b3145be format: 2 features: layering op_features: flags: create_timestamp: Fri Oct 7 10:19:31 2022 access_timestamp: Fri Oct 7 10:19:31 2022 modify_timestamp: Fri Oct 7 10:19:31 2022 [root@ceph-deploy ~]#
客户端使用块存储
[root@ceph-client-test ~]# yum install epel-release -y [root@ceph-client-test ~]# yum install https://mirrors.aliyun.com/ceph/rpm-octopus/el7/noarch/ceph-release-1-1.el7.noarch.rpm -y [root@ceph-client-test ~]# yum install ceph-common -y [root@ceph-client-test ~]# ll -h /etc/ceph/ total 4.0K -rw-r--r-- 1 root root 92 Aug 10 01:34 rbdmap [root@ceph-client-test ~]#
从部署节点同步认证文件 [root@ceph-deploy ceph]# scp -r ceph.conf ceph.client.admin.keyring root@172.16.88.60:/etc/ceph [root@ceph-client-test ~]# ll -h /etc/ceph/ total 12K -rw-r----- 1 root root 151 Oct 7 10:42 ceph.client.admin.keyring -rw-r--r-- 1 root root 315 Oct 7 10:42 ceph.conf -rw-r--r-- 1 root root 92 Aug 10 01:34 rbdmap [root@ceph-client-test ~]# [root@ceph-client-test ~]# ceph -s cluster: id: 8dc32c41-121c-49df-9554-dfb7deb8c975 health: HEALTH_OK services: mon: 3 daemons, quorum ceph-mon1,ceph-mon2,ceph-mon3 (age 62m) mgr: ceph-mgr1(active, since 61m), standbys: ceph-mgr2 osd: 9 osds: 9 up (since 61m), 9 in (since 2d) data: pools: 2 pools, 65 pgs objects: 7 objects, 405 B usage: 58 MiB used, 900 GiB / 900 GiB avail pgs: 65 active+clean [root@ceph-client-test ~]#
客户端映射img
[root@ceph-client-test ~]# rbd -p myrbd1 map myimg2 /dev/rbd0 [root@ceph-client-test ~]# rbd -p myrbd1 map myimg1 #由于 centos 系统内核较低不支持更多 image-feature特性、因此无法挂载使用，所以只开启部分特性，除了layering其他特性需要高版本内核支持。 rbd: sysfs write failed RBD image feature set mismatch. You can disable features unsupported by the kernel with "rbd feature disable myrbd1/myimg1 object-map fast-diff deep-flatten". In some cases useful info is found in syslog - try "dmesg | tail". rbd: map failed: (6) No such device or address [root@ceph-client-test ~]#
客户端验证rbd
[root@ceph-client-test ~]# lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT sr0 11:0 1 1024M 0 rom vda 252:0 0 50G 0 disk ├─vda1 252:1 0 500M 0 part /boot └─vda2 252:2 0 49G 0 part └─centos-root 253:0 0 49G 0 lvm / rbd0 251:0 0 3G 0 disk [root@ceph-client-test ~]# fdisk -l /dev/rbd0 Disk /dev/rbd0: 3221 MB, 3221225472 bytes, 6291456 sectors Units = sectors of 1 * 512 = 512 bytes Sector size (logical/physical): 512 bytes / 512 bytes I/O size (minimum/optimal): 4194304 bytes / 4194304 bytes [root@ceph-client-test ~]#
客户端格式化磁盘并挂载使用
[root@ceph-client-test ~]# mkfs.xfs /dev/rbd0 Discarding blocks...Done. meta-data=/dev/rbd0 isize=512 agcount=8, agsize=98304 blks = sectsz=512 attr=2, projid32bit=1 = crc=1 finobt=0, sparse=0 data = bsize=4096 blocks=786432, imaxpct=25 = sunit=1024 swidth=1024 blks naming =version 2 bsize=4096 ascii-ci=0 ftype=1 log =internal log bsize=4096 blocks=2560, version=2 = sectsz=512 sunit=8 blks, lazy-count=1 realtime =none extsz=4096 blocks=0, rtextents=0 [root@ceph-client-test ~]# mkdir /rbd-data [root@ceph-client-test ~]# mount /dev/rbd0 /rbd-data/ [root@ceph-client-test ~]# cp /etc/passwd /etc/shadow /rbd-data/ [root@ceph-client-test ~]# touch /rbd-data/test-{1..9}.txt [root@ceph-client-test ~]# ll -h /rbd-data/ total 8.0K -rw-r--r-- 1 root root 854 Oct 7 10:49 passwd ---------- 1 root root 583 Oct 7 10:49 shadow -rw-r--r-- 1 root root 0 Oct 7 10:49 test-1.txt -rw-r--r-- 1 root root 0 Oct 7 10:49 test-2.txt -rw-r--r-- 1 root root 0 Oct 7 10:49 test-3.txt -rw-r--r-- 1 root root 0 Oct 7 10:49 test-4.txt -rw-r--r-- 1 root root 0 Oct 7 10:49 test-5.txt -rw-r--r-- 1 root root 0 Oct 7 10:49 test-6.txt -rw-r--r-- 1 root root 0 Oct 7 10:49 test-7.txt -rw-r--r-- 1 root root 0 Oct 7 10:49 test-8.txt -rw-r--r-- 1 root root 0 Oct 7 10:49 test-9.txt [root@ceph-client-test ~]# df -Th Filesystem Type Size Used Avail Use% Mounted on devtmpfs devtmpfs 3.9G 0 3.9G 0% /dev tmpfs tmpfs 3.9G 0 3.9G 0% /dev/shm tmpfs tmpfs 3.9G 8.6M 3.9G 1% /run tmpfs tmpfs 3.9G 0 3.9G 0% /sys/fs/cgroup /dev/mapper/centos-root xfs 49G 2.1G 47G 5% / /dev/vda1 xfs 497M 186M 311M 38% /boot tmpfs tmpfs 783M 0 783M 0% /run/user/0 /dev/rbd0 xfs 3.0G 33M 3.0G 2% /rbd-data [root@ceph-client-test ~]#
创建大文件
[root@ceph-client-test rbd-data]# dd if=/dev/zero of=/rbd-data/rbd-test-file bs=1M count=300 300+0 records in 300+0 records out 314572800 bytes (315 MB) copied, 0.602971 s, 522 MB/s [root@ceph-client-test rbd-data]# ll -h total 301M -rw-r--r-- 1 root root 854 Oct 7 10:49 passwd -rw-r--r-- 1 root root 300M Oct 7 10:51 rbd-test-file ---------- 1 root root 583 Oct 7 10:49 shadow -rw-r--r-- 1 root root 0 Oct 7 10:49 test-1.txt -rw-r--r-- 1 root root 0 Oct 7 10:49 test-2.txt -rw-r--r-- 1 root root 0 Oct 7 10:49 test-3.txt -rw-r--r-- 1 root root 0 Oct 7 10:49 test-4.txt -rw-r--r-- 1 root root 0 Oct 7 10:49 test-5.txt -rw-r--r-- 1 root root 0 Oct 7 10:49 test-6.txt -rw-r--r-- 1 root root 0 Oct 7 10:49 test-7.txt -rw-r--r-- 1 root root 0 Oct 7 10:49 test-8.txt -rw-r--r-- 1 root root 0 Oct 7 10:49 test-9.txt [root@ceph-client-test rbd-data]#
ceph验证数据
[root@ceph-deploy ~]# ceph df --- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 900 GiB 900 GiB 58 MiB 58 MiB 0 TOTAL 900 GiB 900 GiB 58 MiB 58 MiB 0 --- POOLS --- POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL device_health_metrics 1 1 0 B 0 0 B 0 285 GiB myrbd1 2 64 19 B 1 12 KiB 0 285 GiB [root@ceph-deploy ~]# [root@ceph-deploy ceph]# ceph df --- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 900 GiB 899 GiB 996 MiB 996 MiB 0.11 TOTAL 900 GiB 899 GiB 996 MiB 996 MiB 0.11 --- POOLS --- POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL device_health_metrics 1 1 0 B 0 0 B 0 285 GiB myrbd1 2 64 310 MiB 94 931 MiB 0.11 285 GiB [root@ceph-deploy ceph]#
删除数据
[root@ceph-client-test rbd-data]# rm -fr rbd-test-file [root@ceph-client-test rbd-data]# df -h Filesystem Size Used Avail Use% Mounted on devtmpfs 3.9G 0 3.9G 0% /dev tmpfs 3.9G 0 3.9G 0% /dev/shm tmpfs 3.9G 8.6M 3.9G 1% /run tmpfs 3.9G 0 3.9G 0% /sys/fs/cgroup /dev/mapper/centos-root 49G 2.1G 47G 5% / /dev/vda1 497M 186M 311M 38% /boot tmpfs 783M 0 783M 0% /run/user/0 /dev/rbd0 3.0G 33M 3.0G 2% /rbd-data #此时数据已经显示被删除了 [root@ceph-client-test rbd-data]#
删除完成的数据只是标记为已经被删除，但是不会从块存储立即清空，因此在删除完成后使用 ceph df 查看并没有回收空间： [root@ceph-deploy ceph]# ceph df --- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 900 GiB 899 GiB 996 MiB 996 MiB 0.11 TOTAL 900 GiB 899 GiB 996 MiB 996 MiB 0.11 --- POOLS --- POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL device_health_metrics 1 1 0 B 0 0 B 0 285 GiB myrbd1 2 64 310 MiB 94 931 MiB 0.11 285 GiB [root@ceph-deploy ceph]# ceph df --- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 900 GiB 899 GiB 996 MiB 996 MiB 0.11 TOTAL 900 GiB 899 GiB 996 MiB 996 MiB 0.11 --- POOLS --- POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL device_health_metrics 1 1 0 B 0 0 B 0 285 GiB myrbd1 2 64 310 MiB 94 931 MiB 0.11 285 GiB [root@ceph-deploy ceph]#
但是后期可以使用此空间， 如果需要立即在系统层回收空间， 需要执行以下命令：
[root@ceph-client-test ~]# fstrim -v /rbd-data/ /rbd-data/: 3 GiB (3206275072 bytes) trimmed [root@ceph-client-test ~]# [root@ceph-deploy ceph]# ceph df --- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 900 GiB 899 GiB 996 MiB 996 MiB 0.11 TOTAL 900 GiB 899 GiB 996 MiB 996 MiB 0.11 --- POOLS --- POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL device_health_metrics 1 1 0 B 0 0 B 0 285 GiB myrbd1 2 64 310 MiB 94 931 MiB 0.11 285 GiB [root@ceph-deploy ceph]# ceph df --- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 900 GiB 900 GiB 98 MiB 98 MiB 0.01 TOTAL 900 GiB 900 GiB 98 MiB 98 MiB 0.01 --- POOLS --- POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL device_health_metrics 1 1 0 B 0 0 B 0 285 GiB myrbd1 2 64 10 MiB 18 31 MiB 0 285 GiB [root@ceph-deploy ceph]#
或者配置挂载选项
# rbd -p myrbd1 map myimg2
# mount -t xfs -o discard /dev/rbd0 /rbd-data #主要用户 SSD， 立即触发闲置的块回收
三、RGW对象存储网关
RGW 提供的是 REST 风格的 API 接口， 客户端通过 http 与其进行交互， 完成数据的增删改查等管理操作。
radosgw 用在需要使用 RESTful API 接口访问 ceph 数据的场合， 因此在使用 RBD 即块存储得场合或者使用 cephFS 的场合可以不用启用 radosgw 功能。
部署radosgw服务
[root@ceph-mgr1 ~]# apt-cache madison radosgw radosgw | 16.2.10-1focal | https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific focal/main amd64 Packages radosgw | 15.2.16-0ubuntu0.20.04.1 | https://mirrors.tuna.tsinghua.edu.cn/ubuntu focal-updates/main amd64 Packages radosgw | 15.2.12-0ubuntu0.20.04.1 | https://mirrors.tuna.tsinghua.edu.cn/ubuntu focal-security/main amd64 Packages radosgw | 15.2.1-0ubuntu1 | https://mirrors.tuna.tsinghua.edu.cn/ubuntu focal/main amd64 Packages [root@ceph-mgr1 ~]# apt install radosgw=16.2.10-1focal Reading package lists... Done Building dependency tree Reading state information... Done The following NEW packages will be installed: radosgw 0 upgraded, 1 newly installed, 0 to remove and 44 not upgraded. Need to get 11.3 MB of archives. After this operation, 42.2 MB of additional disk space will be used. Get:1 https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific focal/main amd64 radosgw amd64 16.2.10-1focal [11.3 MB] Fetched 11.3 MB in 8s (1458 kB/s) Selecting previously unselected package radosgw. (Reading database ... 123130 files and directories currently installed.) Preparing to unpack .../radosgw_16.2.10-1focal_amd64.deb ... Unpacking radosgw (16.2.10-1focal) ... Setting up radosgw (16.2.10-1focal) ... Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. Processing triggers for systemd (245.4-4ubuntu3.15) ... Processing triggers for man-db (2.9.1-1) ... Processing triggers for libc-bin (2.31-0ubuntu9.7) ... [root@ceph-mgr1 ~]#
[root@ceph-deploy ceph]# su - cephadmin cephadmin@ceph-deploy:~$ cd ceph-cluster/ cephadmin@ceph-deploy:~/ceph-cluster$ pwd /home/cephadmin/ceph-cluster cephadmin@ceph-deploy:~/ceph-cluster$ ll -h total 232K drwxrwxr-x 2 cephadmin cephadmin 4.0K Oct 5 21:37 ./ drwxr-xr-x 6 cephadmin cephadmin 4.0K Oct 5 21:37 ../ -rw-rw-r-- 1 cephadmin cephadmin 192K Oct 5 21:38 ceph-deploy-ceph.log -rw------- 1 cephadmin cephadmin 113 Oct 4 22:24 ceph.bootstrap-mds.keyring -rw------- 1 cephadmin cephadmin 113 Oct 4 22:24 ceph.bootstrap-mgr.keyring -rw------- 1 cephadmin cephadmin 113 Oct 4 22:24 ceph.bootstrap-osd.keyring -rw------- 1 cephadmin cephadmin 113 Oct 4 22:24 ceph.bootstrap-rgw.keyring -rw------- 1 cephadmin cephadmin 151 Oct 4 22:24 ceph.client.admin.keyring -rw-rw-r-- 1 cephadmin cephadmin 315 Oct 5 21:37 ceph.conf -rw------- 1 cephadmin cephadmin 73 Oct 4 22:16 ceph.mon.keyring cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy --overwrite-conf rgw create ceph-mgr1 [ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf [ceph_deploy.cli][INFO ] Invoked (2.1.0): /usr/local/bin/ceph-deploy --overwrite-conf rgw create ceph-mgr1 [ceph_deploy.cli][INFO ] ceph-deploy options: [ceph_deploy.cli][INFO ] verbose : False [ceph_deploy.cli][INFO ] quiet : False [ceph_deploy.cli][INFO ] username : None [ceph_deploy.cli][INFO ] overwrite_conf : True [ceph_deploy.cli][INFO ] ceph_conf : None [ceph_deploy.cli][INFO ] cluster : ceph [ceph_deploy.cli][INFO ] subcommand : create [ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f22f760d8b0> [ceph_deploy.cli][INFO ] default_release : False [ceph_deploy.cli][INFO ] func : <function rgw at 0x7f22f75edb80> [ceph_deploy.cli][INFO ] rgw : [('ceph-mgr1', 'rgw.ceph-mgr1')] [ceph_deploy.rgw][DEBUG ] Deploying rgw, cluster ceph hosts ceph-mgr1:rgw.ceph-mgr1 [ceph-mgr1][DEBUG ] connection detected need for sudo [ceph-mgr1][DEBUG ] connected to host: ceph-mgr1 [ceph_deploy.rgw][INFO ] Distro info: ubuntu 20.04 focal [ceph_deploy.rgw][DEBUG ] remote host will use systemd [ceph_deploy.rgw][DEBUG ] deploying rgw bootstrap to ceph-mgr1 [ceph-mgr1][WARNIN] rgw keyring does not exist yet, creating one [ceph-mgr1][INFO ] Running command: sudo ceph --cluster ceph --name client.bootstrap-rgw --keyring /var/lib/ceph/bootstrap-rgw/ceph.keyring auth get-or-create client.rgw.ceph-mgr1 osd allow rwx mon allow rw -o /var/lib/ceph/radosgw/ceph-rgw.ceph-mgr1/keyring [ceph-mgr1][INFO ] Running command: sudo systemctl enable ceph-radosgw@rgw.ceph-mgr1 [ceph-mgr1][WARNIN] Created symlink /etc/systemd/system/ceph-radosgw.target.wants/ceph-radosgw@rgw.ceph-mgr1.service → /lib/systemd/system/ceph-radosgw@.service. [ceph-mgr1][INFO ] Running command: sudo systemctl start ceph-radosgw@rgw.ceph-mgr1 [ceph-mgr1][INFO ] Running command: sudo systemctl enable ceph.target [ceph_deploy.rgw][INFO ] The Ceph Object Gateway (RGW) is now running on host ceph-mgr1 and default port 7480 cephadmin@ceph-deploy:~/ceph-cluster$
验证radosgw服务
[root@ceph-mgr1 ~]# ps -ef |grep radosgw ceph 2847 1 1 11:27 ? 00:00:01 /usr/bin/radosgw -f --cluster ceph --name client.rgw.ceph-mgr1 --setuser ceph --setgroup ceph root 3539 2013 0 11:29 pts/0 00:00:00 grep --color=auto radosgw [root@ceph-mgr1 ~]#
验证ceph、rgw存储状态
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s cluster: id: 8dc32c41-121c-49df-9554-dfb7deb8c975 health: HEALTH_OK services: mon: 3 daemons, quorum ceph-mon1,ceph-mon2,ceph-mon3 (age 110m) mgr: ceph-mgr1(active, since 110m), standbys: ceph-mgr2 osd: 9 osds: 9 up (since 110m), 9 in (since 2d) rgw: 1 daemon active (1 hosts, 1 zones) data: pools: 6 pools, 169 pgs objects: 207 objects, 10 MiB usage: 154 MiB used, 900 GiB / 900 GiB avail pgs: 169 active+clean cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool ls device_health_metrics myrbd1 .rgw.root default.rgw.log default.rgw.control default.rgw.meta cephadmin@ceph-deploy:~/ceph-cluster$ ceph df --- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 900 GiB 900 GiB 154 MiB 154 MiB 0.02 TOTAL 900 GiB 900 GiB 154 MiB 154 MiB 0.02 --- POOLS --- POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL device_health_metrics 1 1 0 B 0 0 B 0 285 GiB myrbd1 2 64 10 MiB 18 31 MiB 0 285 GiB .rgw.root 3 32 1.3 KiB 4 48 KiB 0 285 GiB default.rgw.log 4 32 3.6 KiB 177 408 KiB 0 285 GiB default.rgw.control 5 32 0 B 8 0 B 0 285 GiB default.rgw.meta 6 8 0 B 0 0 B 0 285 GiB cephadmin@ceph-deploy:~/ceph-cluster$
四、cephfs使用
CephFS即ceph filesystem，可以实现文件系统共享功能,客户端通过ceph协议挂载并使用ceph集群作为数据存储服务器。
CephFS需要运行Meta Data Services(MDS)服务，其守护进程为ceph-mds,ceph-mds进程管理与cephFS上存储的文件相关的元数据,并协调对ceph存储集群的访问。
如下图：
数据的元数据保存在单独的一个存储池cephfs-metadata(名字可自定义),因此元数据也是基于3副本提高可用性,另外使用专用的 MDS 服务器在内存缓存元数据信息以提高对客户端的读写响应性能。
部署MDS服务
[root@ceph-mgr1 ~]# apt-cache madison ceph-mds ceph-mds | 16.2.10-1focal | https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific focal/main amd64 Packages ceph-mds | 15.2.16-0ubuntu0.20.04.1 | https://mirrors.tuna.tsinghua.edu.cn/ubuntu focal-updates/main amd64 Packages ceph-mds | 15.2.12-0ubuntu0.20.04.1 | https://mirrors.tuna.tsinghua.edu.cn/ubuntu focal-security/main amd64 Packages ceph-mds | 15.2.1-0ubuntu1 | https://mirrors.tuna.tsinghua.edu.cn/ubuntu focal/main amd64 Packages [root@ceph-mgr1 ~]# apt install ceph-mds=16.2.10-1focal Reading package lists... Done Building dependency tree Reading state information... Done ceph-mds is already the newest version (16.2.10-1focal). ceph-mds set to manually installed. 0 upgraded, 0 newly installed, 0 to remove and 44 not upgraded. [root@ceph-mgr1 ~]#
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mds create ceph-mgr1 [ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf [ceph_deploy.cli][INFO ] Invoked (2.1.0): /usr/local/bin/ceph-deploy mds create ceph-mgr1 [ceph_deploy.cli][INFO ] ceph-deploy options: [ceph_deploy.cli][INFO ] verbose : False [ceph_deploy.cli][INFO ] quiet : False [ceph_deploy.cli][INFO ] username : None [ceph_deploy.cli][INFO ] overwrite_conf : False [ceph_deploy.cli][INFO ] ceph_conf : None [ceph_deploy.cli][INFO ] cluster : ceph [ceph_deploy.cli][INFO ] subcommand : create [ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f37bcdd8100> [ceph_deploy.cli][INFO ] default_release : False [ceph_deploy.cli][INFO ] func : <function mds at 0x7f37bce11160> [ceph_deploy.cli][INFO ] mds : [('ceph-mgr1', 'ceph-mgr1')] [ceph_deploy.mds][DEBUG ] Deploying mds, cluster ceph hosts ceph-mgr1:ceph-mgr1 [ceph-mgr1][DEBUG ] connection detected need for sudo [ceph-mgr1][DEBUG ] connected to host: ceph-mgr1 [ceph_deploy.mds][INFO ] Distro info: ubuntu 20.04 focal [ceph_deploy.mds][DEBUG ] remote host will use systemd [ceph_deploy.mds][DEBUG ] deploying mds bootstrap to ceph-mgr1 [ceph-mgr1][WARNIN] mds keyring does not exist yet, creating one [ceph-mgr1][INFO ] Running command: sudo ceph --cluster ceph --name client.bootstrap-mds --keyring /var/lib/ceph/bootstrap-mds/ceph.keyring auth get-or-create mds.ceph-mgr1 osd allow rwx mds allow mon allow profile mds -o /var/lib/ceph/mds/ceph-ceph-mgr1/keyring [ceph-mgr1][INFO ] Running command: sudo systemctl enable ceph-mds@ceph-mgr1 [ceph-mgr1][WARNIN] Created symlink /etc/systemd/system/ceph-mds.target.wants/ceph-mds@ceph-mgr1.service → /lib/systemd/system/ceph-mds@.service. [ceph-mgr1][INFO ] Running command: sudo systemctl start ceph-mds@ceph-mgr1 [ceph-mgr1][INFO ] Running command: sudo systemctl enable ceph.target cephadmin@ceph-deploy:~/ceph-cluster$
验证mds服务
MDS 服务目前还无法正常使用， 需要为 MDS 创建存储池用于保存 MDS 的数据。
cephadmin@ceph-deploy:~/ceph-cluster$ ceph mds stat #当前为备用状态， 需要分配 pool 才可以使用。 1 up:standby cephadmin@ceph-deploy:~/ceph-cluster$
创建cephfs metadata和data存储池
使用CephFS之前需要事先于集群中创建一个文件系统，并为其分别指定元数据和数据相关的存储池，如下命令将创建名为mycephfs的文件系统，它使用cephfs-metadata作为元数据存储池，使用cephfs-data为数据存储池：
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool create cephfs-metadata 32 32 pool 'cephfs-metadata' created #保存metadata 的 pool cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool create cephfs-data 64 64 pool 'cephfs-data' created #保存数据的 pool cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s cluster: id: 8dc32c41-121c-49df-9554-dfb7deb8c975 health: HEALTH_OK services: mon: 3 daemons, quorum ceph-mon1,ceph-mon2,ceph-mon3 (age 2h) mgr: ceph-mgr1(active, since 2h), standbys: ceph-mgr2 osd: 9 osds: 9 up (since 2h), 9 in (since 2d) rgw: 1 daemon active (1 hosts, 1 zones) data: pools: 8 pools, 265 pgs objects: 207 objects, 10 MiB usage: 183 MiB used, 900 GiB / 900 GiB avail pgs: 265 active+clean cephadmin@ceph-deploy:~/ceph-cluster$ ceph df --- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 900 GiB 900 GiB 184 MiB 184 MiB 0.02 TOTAL 900 GiB 900 GiB 184 MiB 184 MiB 0.02 --- POOLS --- POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL device_health_metrics 1 1 0 B 0 0 B 0 285 GiB myrbd1 2 64 10 MiB 18 31 MiB 0 285 GiB .rgw.root 3 32 1.3 KiB 4 48 KiB 0 285 GiB default.rgw.log 4 32 3.6 KiB 177 408 KiB 0 285 GiB default.rgw.control 5 32 0 B 8 0 B 0 285 GiB default.rgw.meta 6 8 0 B 0 0 B 0 285 GiB cephfs-metadata 7 32 0 B 0 0 B 0 285 GiB cephfs-data 8 64 0 B 0 0 B 0 285 GiB cephadmin@ceph-deploy:~/ceph-cluster$
创建cephfs并验证
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs new mycephfs cephfs-metadata cephfs-data new fs with metadata pool 7 and data pool 8 cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs ls name: mycephfs, metadata pool: cephfs-metadata, data pools: [cephfs-data ] cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status mycephfs #查看指定的cephfs状态 mycephfs - 0 clients ======== RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS 0 active ceph-mgr1 Reqs: 0 /s 10 13 12 0 POOL TYPE USED AVAIL cephfs-metadata metadata 96.0k 284G cephfs-data data 0 284G MDS version: ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable) cephadmin@ceph-deploy:~/ceph-cluster$ ceph mds stat mycephfs:1 {0=ceph-mgr1=up:active} #cephfs 状态现在已经转变为活动状态 cephadmin@ceph-deploy:~/ceph-cluster$
客户端挂载cephfs
cephadmin@ceph-deploy:~/ceph-cluster$ cat ceph.client.admin.keyring [client.admin] key = AQAdQjxjK9xLEBAA3VWhXIjq0esRK3rQPaU+Zg== caps mds = "allow *" caps mgr = "allow *" caps mon = "allow *" caps osd = "allow *" cephadmin@ceph-deploy:~/ceph-cluster$ [root@ceph-client-test ~]# mkdir /cephfs-data #创建挂载目录 [root@ceph-client-test ~]# mount -t ceph 172.16.88.101:6789:/ /cephfs-data -o name=admin,secret=AQAdQjxjK9xLEBAA3VWhXIjq0esRK3rQPaU+Zg== [root@ceph-client-test ~]# df -h Filesystem Size Used Avail Use% Mounted on devtmpfs 3.9G 0 3.9G 0% /dev tmpfs 3.9G 0 3.9G 0% /dev/shm tmpfs 3.9G 8.5M 3.9G 1% /run tmpfs 3.9G 0 3.9G 0% /sys/fs/cgroup /dev/mapper/centos-root 49G 2.1G 47G 5% / /dev/vda1 497M 186M 311M 38% /boot tmpfs 783M 0 783M 0% /run/user/0 /dev/rbd0 3.0G 33M 3.0G 2% /rbd-data 172.16.88.101:6789:/ 285G 0 285G 0% /cephfs-data [root@ceph-client-test ~]# [root@ceph-client-test ~]# cp /var/log/messages /cephfs-data/ #测试本地拷贝文件 [root@ceph-client-test ~]# dd if=/dev/zero of=/cephfs-data/cephfs-file-test bs=4M count=25 #创建大文件 25+0 records in 25+0 records out 104857600 bytes (105 MB) copied, 0.246486 s, 425 MB/s [root@ceph-client-test ~]# df -h Filesystem Size Used Avail Use% Mounted on devtmpfs 3.9G 0 3.9G 0% /dev tmpfs 3.9G 0 3.9G 0% /dev/shm tmpfs 3.9G 8.5M 3.9G 1% /run tmpfs 3.9G 0 3.9G 0% /sys/fs/cgroup /dev/mapper/centos-root 49G 2.1G 47G 5% / /dev/vda1 497M 186M 311M 38% /boot tmpfs 783M 0 783M 0% /run/user/0 /dev/rbd0 3.0G 33M 3.0G 2% /rbd-data 172.16.88.101:6789:/ 285G 100M 285G 1% /cephfs-data [root@ceph-client-test ~]# cephadmin@ceph-deploy:~/ceph-cluster$ ceph df --- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 900 GiB 899 GiB 538 MiB 538 MiB 0.06 TOTAL 900 GiB 899 GiB 538 MiB 538 MiB 0.06 --- POOLS --- POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL device_health_metrics 1 1 0 B 0 0 B 0 285 GiB myrbd1 2 64 10 MiB 18 31 MiB 0 285 GiB .rgw.root 3 32 1.3 KiB 4 48 KiB 0 285 GiB default.rgw.log 4 32 3.6 KiB 177 408 KiB 0 285 GiB default.rgw.control 5 32 0 B 8 0 B 0 285 GiB default.rgw.meta 6 8 0 B 0 0 B 0 285 GiB cephfs-metadata 7 32 24 KiB 22 156 KiB 0 285 GiB cephfs-data 8 64 100 MiB 26 301 MiB 0.03 285 GiB cephadmin@ceph-deploy:~/ceph-cluster$ [root@ceph-client-test cephfs-data]# rm -fr cephfs-file-test #删除大文件，验证磁盘空间是否自动回收 [root@ceph-client-test cephfs-data]# df -h Filesystem Size Used Avail Use% Mounted on devtmpfs 3.9G 0 3.9G 0% /dev tmpfs 3.9G 0 3.9G 0% /dev/shm tmpfs 3.9G 8.5M 3.9G 1% /run tmpfs 3.9G 0 3.9G 0% /sys/fs/cgroup /dev/mapper/centos-root 49G 2.1G 47G 5% / /dev/vda1 497M 186M 311M 38% /boot tmpfs 783M 0 783M 0% /run/user/0 /dev/rbd0 3.0G 33M 3.0G 2% /rbd-data 172.16.88.101:6789:/ 285G 0 285G 0% /cephfs-data [root@ceph-client-test cephfs-data]# cephadmin@ceph-deploy:~/ceph-cluster$ ceph df --- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 900 GiB 900 GiB 238 MiB 238 MiB 0.03 TOTAL 900 GiB 900 GiB 238 MiB 238 MiB 0.03 --- POOLS --- POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL device_health_metrics 1 1 0 B 0 0 B 0 285 GiB myrbd1 2 64 10 MiB 18 31 MiB 0 285 GiB .rgw.root 3 32 1.3 KiB 4 48 KiB 0 285 GiB default.rgw.log 4 32 3.6 KiB 177 408 KiB 0 285 GiB default.rgw.control 5 32 0 B 8 0 B 0 285 GiB default.rgw.meta 6 8 0 B 0 0 B 0 285 GiB cephfs-metadata 7 32 31 KiB 23 192 KiB 0 285 GiB cephfs-data 8 64 373 KiB 1 1.1 MiB 0 285 GiB cephadmin@ceph-deploy:~/ceph-cluster$
五、ceph维护命令总结
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster: id: 8dc32c41-121c-49df-9554-dfb7deb8c975 health: HEALTH_OK services: mon: 3 daemons, quorum ceph-mon1,ceph-mon2,ceph-mon3 (age 2h) mgr: ceph-mgr1(active, since 2h), standbys: ceph-mgr2 osd: 9 osds: 9 up (since 2h), 9 in (since 2d) rgw: 1 daemon active (1 hosts, 1 zones) data: pools: 8 pools, 265 pgs objects: 207 objects, 10 MiB usage: 183 MiB used, 900 GiB / 900 GiB avail pgs: 265 active+clean
cephadmin@ceph-deploy:~/ceph-cluster$ ceph health detail
HEALTH_OK
cephadmin@ceph-deploy:~/ceph-cluster$ ceph df
--- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 900 GiB 900 GiB 184 MiB 184 MiB 0.02 TOTAL 900 GiB 900 GiB 184 MiB 184 MiB 0.02 --- POOLS --- POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL device_health_metrics 1 1 0 B 0 0 B 0 285 GiB myrbd1 2 64 10 MiB 18 31 MiB 0 285 GiB .rgw.root 3 32 1.3 KiB 4 48 KiB 0 285 GiB default.rgw.log 4 32 3.6 KiB 177 408 KiB 0 285 GiB default.rgw.control 5 32 0 B 8 0 B 0 285 GiB default.rgw.meta 6 8 0 B 0 0 B 0 285 GiB cephfs-metadata 7 32 0 B 0 0 B 0 285 GiB cephfs-data 8 64 0 B 0 0 B 0 285 GiB
cephadmin@ceph-deploy:~/ceph-cluster$ ceph df detail
--- RAW STORAGE --- CLASS SIZE AVAIL USED RAW USED %RAW USED hdd 900 GiB 900 GiB 238 MiB 238 MiB 0.03 TOTAL 900 GiB 900 GiB 238 MiB 238 MiB 0.03 --- POOLS --- POOL ID PGS STORED (DATA) (OMAP) OBJECTS USED (DATA) (OMAP) %USED MAX AVAIL QUOTA OBJECTS QUOTA BYTES DIRTY USED COMPR UNDER COMPR device_health_metrics 1 1 0 B 0 B 0 B 0 0 B 0 B 0 B 0 285 GiB N/A N/A N/A 0 B 0 B myrbd1 2 64 10 MiB 10 MiB 0 B 18 31 MiB 31 MiB 0 B 0 285 GiB N/A N/A N/A 0 B 0 B .rgw.root 3 32 1.3 KiB 1.3 KiB 0 B 4 48 KiB 48 KiB 0 B 0 285 GiB N/A N/A N/A 0 B 0 B default.rgw.log 4 32 3.6 KiB 3.6 KiB 0 B 177 408 KiB 408 KiB 0 B 0 285 GiB N/A N/A N/A 0 B 0 B default.rgw.control 5 32 0 B 0 B 0 B 8 0 B 0 B 0 B 0 285 GiB N/A N/A N/A 0 B 0 B default.rgw.meta 6 8 0 B 0 B 0 B 0 0 B 0 B 0 B 0 285 GiB N/A N/A N/A 0 B 0 B cephfs-metadata 7 32 31 KiB 31 KiB 0 B 23 192 KiB 192 KiB 0 B 0 285 GiB N/A N/A N/A 0 B 0 B cephfs-data 8 64 373 KiB 373 KiB 0 B 1 1.1 MiB 1.1 MiB 0 B 0 285 GiB N/A N/A N/A 0 B 0 B
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs new mycephfs cephfs-metadata cephfs-data
new fs with metadata pool 7 and data pool 8 cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs ls name: mycephfs, metadata pool: cephfs-metadata, data pools: [cephfs-data ] cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status mycephfs mycephfs - 0 clients ======== RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS 0 active ceph-mgr1 Reqs: 0 /s 10 13 12 0 POOL TYPE USED AVAIL cephfs-metadata metadata 96.0k 284G cephfs-data data 0 284G MDS version: ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph mds stat
mycephfs:1 {0=ceph-mgr1=up:active}
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd dump
epoch 218 fsid 8dc32c41-121c-49df-9554-dfb7deb8c975 created 2022-10-04T22:24:29.272895+0800 modified 2022-10-07T11:58:44.061728+0800 flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit crush_version 23 full_ratio 0.95 backfillfull_ratio 0.9 nearfull_ratio 0.85 require_min_compat_client luminous min_compat_client luminous require_osd_release pacific stretch_mode_enabled false pool 1 'device_health_metrics' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 63 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr_devicehealth pool 2 'myrbd1' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 64 pgp_num 64 autoscale_mode on last_change 77 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd pool 3 '.rgw.root' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 81 flags hashpspool stripe_width 0 application rgw pool 4 'default.rgw.log' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 83 flags hashpspool stripe_width 0 application rgw pool 5 'default.rgw.control' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 85 flags hashpspool stripe_width 0 application rgw pool 6 'default.rgw.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode on last_change 208 lfor 0/208/206 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 8 application rgw pool 7 'cephfs-metadata' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 218 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs pool 8 'cephfs-data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 64 pgp_num 64 autoscale_mode on last_change 218 flags hashpspool stripe_width 0 application cephfs max_osd 9 osd.0 up in weight 1 up_from 59 up_thru 213 down_at 58 last_clean_interval [5,55) [v2:172.16.88.121:6808/1387,v1:172.16.88.121:6809/1387] [v2:192.168.122.121:6808/1387,v1:192.168.122.121:6809/1387] exists,up 1664c48e-0fdc-4166-b2b4-a4bbed39e758 osd.1 up in weight 1 up_from 59 up_thru 213 down_at 58 last_clean_interval [10,55) [v2:172.16.88.121:6800/1388,v1:172.16.88.121:6801/1388] [v2:192.168.122.121:6800/1388,v1:192.168.122.121:6801/1388] exists,up 59587e0b-8a5f-4fee-abb9-6ec741618ff1 osd.2 up in weight 1 up_from 59 up_thru 216 down_at 58 last_clean_interval [15,55) [v2:172.16.88.121:6804/1389,v1:172.16.88.121:6805/1389] [v2:192.168.122.121:6804/1389,v1:192.168.122.121:6805/1389] exists,up 5d83c611-b28e-4bd6-98cd-668fa385f834 osd.3 up in weight 1 up_from 59 up_thru 216 down_at 58 last_clean_interval [22,55) [v2:172.16.88.122:6800/1396,v1:172.16.88.122:6801/1396] [v2:192.168.122.122:6800/1396,v1:192.168.122.122:6801/1396] exists,up 9fd3882e-1a46-41fb-b50b-37f5d9a720c9 osd.4 up in weight 1 up_from 59 up_thru 213 down_at 58 last_clean_interval [32,55) [v2:172.16.88.122:6804/1387,v1:172.16.88.122:6805/1387] [v2:192.168.122.122:6804/1387,v1:192.168.122.122:6805/1387] exists,up 1215c5e3-f6ae-4ac5-a729-774a85b66cf8 osd.5 up in weight 1 up_from 59 up_thru 216 down_at 58 last_clean_interval [37,55) [v2:172.16.88.122:6808/1395,v1:172.16.88.122:6809/1395] [v2:192.168.122.122:6808/1395,v1:192.168.122.122:6809/1395] exists,up f777658e-4d56-440b-9a27-f8f2c4be8d84 osd.6 up in weight 1 up_from 58 up_thru 213 down_at 57 last_clean_interval [42,55) [v2:172.16.88.123:6804/1419,v1:172.16.88.123:6805/1419] [v2:192.168.122.123:6804/1419,v1:192.168.122.123:6805/1419] exists,up d0414c2e-b99e-4b6c-aba0-24300a08e904 osd.7 up in weight 1 up_from 59 up_thru 216 down_at 58 last_clean_interval [48,55) [v2:172.16.88.123:6808/1420,v1:172.16.88.123:6809/1420] [v2:192.168.122.123:6808/1420,v1:192.168.122.123:6809/1420] exists,up 3ab4186f-859f-4a82-ae4c-b81f06835e74 osd.8 up in weight 1 up_from 59 up_thru 216 down_at 58 last_clean_interval [54,55) [v2:172.16.88.123:6800/1418,v1:172.16.88.123:6801/1418] [v2:192.168.122.123:6800/1418,v1:192.168.122.123:6801/1418] exists,up d5ff4ca5-3a9c-4fa1-9379-f9b6f854561a pg_upmap_items 2.20 [5,4] pg_upmap_items 2.3b [5,4] pg_upmap_items 4.1 [6,7] pg_upmap_items 4.7 [1,2] pg_upmap_items 4.9 [6,8] pg_upmap_items 4.f [6,8] pg_upmap_items 4.12 [1,2] pg_upmap_items 4.13 [1,2] pg_upmap_items 4.18 [1,2] pg_upmap_items 8.15 [4,5] pg_upmap_items 8.20 [0,2] pg_upmap_items 8.22 [4,3] pg_upmap_items 8.2a [4,3] pg_upmap_items 8.32 [0,2] pg_upmap_items 8.3b [4,3] blocklist 172.16.88.111:0/4025276649 expires 2022-10-08T09:41:18.262605+0800 blocklist 172.16.88.111:0/1060135540 expires 2022-10-08T09:41:18.262605+0800 blocklist 172.16.88.111:0/1358335988 expires 2022-10-08T09:41:18.262605+0800 blocklist 172.16.88.111:6801/15970 expires 2022-10-08T09:41:18.262605+0800 blocklist 172.16.88.111:6800/15970 expires 2022-10-08T09:41:18.262605+0800
cephadmin@ceph-deploy:~/ceph-cluster$ ceph mon dump
epoch 3 fsid 8dc32c41-121c-49df-9554-dfb7deb8c975 last_changed 2022-10-05T21:06:49.042131+0800 created 2022-10-04T22:24:27.817567+0800 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:172.16.88.101:3300/0,v1:172.16.88.101:6789/0] mon.ceph-mon1 1: [v2:172.16.88.102:3300/0,v1:172.16.88.102:6789/0] mon.ceph-mon2 2: [v2:172.16.88.103:3300/0,v1:172.16.88.103:6789/0] mon.ceph-mon3 dumped monmap epoch 3
cephadmin@ceph-deploy:~/ceph-cluster$ ceph mon stat
e3: 3 mons at {ceph-mon1=[v2:172.16.88.101:3300/0,v1:172.16.88.101:6789/0],ceph-mon2=[v2:172.16.88.102:3300/0,v1:172.16.88.102:6789/0],ceph-mon3=[v2:172.16.88.103:3300/0,v1:172.16.88.103:6789/0]}, election epoch 14, leader 0 ceph-mon1, quorum 0,1,2 ceph-mon1,ceph-mon2,ceph-mon3
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd stat
9 osds: 9 up (since 4h), 9 in (since 2d); epoch: e218
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF -1 0.87918 root default -3 0.29306 host ceph-node1 0 hdd 0.09769 osd.0 up 1.00000 1.00000 1 hdd 0.09769 osd.1 up 1.00000 1.00000 2 hdd 0.09769 osd.2 up 1.00000 1.00000 -5 0.29306 host ceph-node2 3 hdd 0.09769 osd.3 up 1.00000 1.00000 4 hdd 0.09769 osd.4 up 1.00000 1.00000 5 hdd 0.09769 osd.5 up 1.00000 1.00000 -7 0.29306 host ceph-node3 6 hdd 0.09769 osd.6 up 1.00000 1.00000 7 hdd 0.09769 osd.7 up 1.00000 1.00000 8 hdd 0.09769 osd.8 up 1.00000 1.00000
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd df
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS 0 hdd 0.09769 1.00000 100 GiB 24 MiB 4.6 MiB 0 B 20 MiB 100 GiB 0.02 0.92 96 up 1 hdd 0.09769 1.00000 100 GiB 32 MiB 7.0 MiB 0 B 25 MiB 100 GiB 0.03 1.21 98 up 2 hdd 0.09769 1.00000 100 GiB 22 MiB 6.6 MiB 0 B 15 MiB 100 GiB 0.02 0.81 71 up 3 hdd 0.09769 1.00000 100 GiB 26 MiB 6.6 MiB 0 B 20 MiB 100 GiB 0.03 0.98 85 up 4 hdd 0.09769 1.00000 100 GiB 30 MiB 9.1 MiB 0 B 21 MiB 100 GiB 0.03 1.11 89 up 5 hdd 0.09769 1.00000 100 GiB 22 MiB 2.5 MiB 0 B 20 MiB 100 GiB 0.02 0.84 91 up 6 hdd 0.09769 1.00000 100 GiB 28 MiB 2.7 MiB 0 B 25 MiB 100 GiB 0.03 1.04 85 up 7 hdd 0.09769 1.00000 100 GiB 28 MiB 8.5 MiB 0 B 20 MiB 100 GiB 0.03 1.06 89 up 8 hdd 0.09769 1.00000 100 GiB 27 MiB 6.9 MiB 0 B 20 MiB 100 GiB 0.03 1.02 91 up TOTAL 900 GiB 240 MiB 55 MiB 8.5 KiB 185 MiB 900 GiB 0.03 MIN/MAX VAR: 0.81/1.21 STDDEV: 0.00
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd df tree
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS TYPE NAME -1 0.87918 - 900 GiB 240 MiB 55 MiB 0 B 185 MiB 900 GiB 0.03 1.00 - root default -3 0.29306 - 300 GiB 78 MiB 18 MiB 0 B 60 MiB 300 GiB 0.03 0.98 - host ceph-node1 0 hdd 0.09769 1.00000 100 GiB 24 MiB 4.6 MiB 0 B 20 MiB 100 GiB 0.02 0.92 96 up osd.0 1 hdd 0.09769 1.00000 100 GiB 32 MiB 7.0 MiB 0 B 25 MiB 100 GiB 0.03 1.21 98 up osd.1 2 hdd 0.09769 1.00000 100 GiB 22 MiB 6.6 MiB 0 B 15 MiB 100 GiB 0.02 0.81 71 up osd.2 -5 0.29306 - 300 GiB 78 MiB 18 MiB 0 B 60 MiB 300 GiB 0.03 0.98 - host ceph-node2 3 hdd 0.09769 1.00000 100 GiB 26 MiB 6.6 MiB 0 B 20 MiB 100 GiB 0.03 0.98 85 up osd.3 4 hdd 0.09769 1.00000 100 GiB 30 MiB 9.1 MiB 0 B 21 MiB 100 GiB 0.03 1.11 89 up osd.4 5 hdd 0.09769 1.00000 100 GiB 22 MiB 2.5 MiB 0 B 20 MiB 100 GiB 0.02 0.84 91 up osd.5 -7 0.29306 - 300 GiB 83 MiB 18 MiB 0 B 65 MiB 300 GiB 0.03 1.04 - host ceph-node3 6 hdd 0.09769 1.00000 100 GiB 28 MiB 2.7 MiB 0 B 25 MiB 100 GiB 0.03 1.04 85 up osd.6 7 hdd 0.09769 1.00000 100 GiB 28 MiB 8.5 MiB 0 B 20 MiB 100 GiB 0.03 1.06 89 up osd.7 8 hdd 0.09769 1.00000 100 GiB 27 MiB 6.9 MiB 0 B 20 MiB 100 GiB 0.03 1.02 91 up osd.8 TOTAL 900 GiB 240 MiB 55 MiB 8.5 KiB 185 MiB 900 GiB 0.03 MIN/MAX VAR: 0.81/1.21 STDDEV: 0.00
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool ls
device_health_metrics myrbd1 .rgw.root default.rgw.log default.rgw.control default.rgw.meta cephfs-metadata cephfs-data
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd lspools
1 device_health_metrics 2 myrbd1 3 .rgw.root 4 default.rgw.log 5 default.rgw.control 6 default.rgw.meta 7 cephfs-metadata 8 cephfs-data
cephadmin@ceph-deploy:~/ceph-cluster$ ceph pg stat
265 pgs: 265 active+clean; 11 MiB data, 249 MiB used, 900 GiB / 900 GiB avail
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool stats myrbd1 ##查看指定 pool 或所有的 pool 的状态
pool myrbd1 id 2 nothing is going on
[root@ceph-mon1 ~]# ceph --admin-daemon /var/run/ceph/ceph-mon.ceph-mon1.asok mon_status
{ "name": "ceph-mon1", "rank": 0, "state": "leader", "election_epoch": 14, "quorum": [ 0, 1, 2 ], "quorum_age": 17678, "features": { "required_con": "2449958747317026820", "required_mon": [ "kraken", "luminous", "mimic", "osdmap-prune", "nautilus", "octopus", "pacific", "elector-pinging" ], "quorum_con": "4540138297136906239", "quorum_mon": [ "kraken", "luminous", "mimic", "osdmap-prune", "nautilus", "octopus", "pacific", "elector-pinging" ] }, "outside_quorum": [], "extra_probe_peers": [], "sync_provider": [], "monmap": { "epoch": 3, "fsid": "8dc32c41-121c-49df-9554-dfb7deb8c975", "modified": "2022-10-05T13:06:49.042131Z", "created": "2022-10-04T14:24:27.817567Z", "min_mon_release": 16, "min_mon_release_name": "pacific", "election_strategy": 1, "disallowed_leaders: ": "", "stretch_mode": false, "tiebreaker_mon": "", "features": { "persistent": [ "kraken", "luminous", "mimic", "osdmap-prune", "nautilus", "octopus", "pacific", "elector-pinging" ], "optional": [] }, "mons": [ { "rank": 0, "name": "ceph-mon1", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "172.16.88.101:3300", "nonce": 0 }, { "type": "v1", "addr": "172.16.88.101:6789", "nonce": 0 } ] }, "addr": "172.16.88.101:6789/0", "public_addr": "172.16.88.101:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" }, { "rank": 1, "name": "ceph-mon2", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "172.16.88.102:3300", "nonce": 0 }, { "type": "v1", "addr": "172.16.88.102:6789", "nonce": 0 } ] }, "addr": "172.16.88.102:6789/0", "public_addr": "172.16.88.102:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" }, { "rank": 2, "name": "ceph-mon3", "public_addrs": { "addrvec": [ { "type": "v2", "addr": "172.16.88.103:3300", "nonce": 0 }, { "type": "v1", "addr": "172.16.88.103:6789", "nonce": 0 } ] }, "addr": "172.16.88.103:6789/0", "public_addr": "172.16.88.103:6789/0", "priority": 0, "weight": 0, "crush_location": "{}" } ] }, "feature_map": { "mon": [ { "features": "0x3f01cfb9fffdffff", "release": "luminous", "num": 1 } ], "mds": [ { "features": "0x3f01cfb9fffdffff", "release": "luminous", "num": 1 } ], "osd": [ { "features": "0x3f01cfb9fffdffff", "release": "luminous", "num": 3 } ], "client": [ { "features": "0x27018fb86aa42ada", "release": "jewel", "num": 1 } ] }, "stretch_mode": false }
[root@ceph-mon1 ~]# ceph --admin-daemon /var/run/ceph/ceph-mon.ceph-mon1.asok config show
{ "name": "mon.ceph-mon1", "cluster": "ceph", "admin_socket": "/var/run/ceph/ceph-mon.ceph-mon1.asok", "admin_socket_mode": "", "auth_allow_insecure_global_id_reclaim": "false", "auth_client_required": "cephx", "auth_cluster_required": "cephx", "auth_debug": "false", "auth_expose_insecure_global_id_reclaim": "true", "auth_mon_ticket_ttl": "259200.000000", "auth_service_required": "cephx", "auth_service_ticket_ttl": "3600.000000", "auth_supported": "", "bdev_aio": "true", "bdev_aio_max_queue_depth": "1024", "bdev_aio_poll_ms": "250", "bdev_aio_reap_max": "16", "bdev_async_discard": "false", "bdev_block_size": "4096", "bdev_debug_aio": "false", "bdev_debug_aio_log_age": "5.000000", "bdev_debug_aio_suicide_timeout": "60.000000", "bdev_debug_inflight_ios": "false", "bdev_enable_discard": "false", "bdev_flock_retry": "3", "bdev_flock_retry_interval": "0.100000", "bdev_inject_crash": "0", "bdev_inject_crash_flush_delay": "2", "bdev_ioring": "false", "bdev_ioring_hipri": "false", "bdev_ioring_sqthread_poll": "false", "bdev_nvme_unbind_from_kernel": "false", "bdev_type": "", "bluefs_alloc_size": "1048576", "bluefs_allocator": "hybrid", "bluefs_buffered_io": "true", "bluefs_check_for_zeros": "false", "bluefs_compact_log_sync": "false", "bluefs_log_compact_min_ratio": "5.000000", "bluefs_log_compact_min_size": "16777216", "bluefs_log_replay_check_allocations": "true", "bluefs_max_log_runway": "4194304", "bluefs_max_prefetch": "1048576", "bluefs_min_flush_size": "524288", "bluefs_min_log_runway": "1048576", "bluefs_replay_recovery": "false", "bluefs_replay_recovery_disable_compact": "false", "bluefs_shared_alloc_size": "65536", "bluefs_sync_write": "false", "bluestore_2q_cache_kin_ratio": "0.500000", "bluestore_2q_cache_kout_ratio": "0.500000", "bluestore_alloc_stats_dump_interval": "86400.000000", "bluestore_allocator": "hybrid", "bluestore_avl_alloc_bf_free_pct": "4", "bluestore_avl_alloc_bf_threshold": "131072", "bluestore_avl_alloc_ff_max_search_bytes": "16777216", "bluestore_avl_alloc_ff_max_search_count": "100", "bluestore_bitmapallocator_blocks_per_zone": "1024", "bluestore_bitmapallocator_span_size": "1024", "bluestore_blobid_prealloc": "10240", "bluestore_block_create": "true", "bluestore_block_db_create": "false", "bluestore_block_db_path": "", "bluestore_block_db_size": "0", "bluestore_block_path": "", "bluestore_block_preallocate_file": "false", "bluestore_block_size": "107374182400", "bluestore_block_wal_create": "false", "bluestore_block_wal_path": "", "bluestore_block_wal_size": "100663296", "bluestore_bluefs": "true", "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", "bluestore_bluefs_env_mirror": "false", "bluestore_cache_autotune": "true", "bluestore_cache_autotune_interval": "5.000000", "bluestore_cache_kv_onode_ratio": "0.040000", "bluestore_cache_kv_ratio": "0.450000", "bluestore_cache_meta_ratio": "0.450000", "bluestore_cache_size": "0", "bluestore_cache_size_hdd": "1073741824", "bluestore_cache_size_ssd": "3221225472", "bluestore_cache_trim_interval": "0.050000", "bluestore_cache_trim_max_skip_pinned": "1000", "bluestore_cache_type": "2q", "bluestore_clone_cow": "true", "bluestore_compression_algorithm": "snappy", "bluestore_compression_max_blob_size": "0", "bluestore_compression_max_blob_size_hdd": "65536", "bluestore_compression_max_blob_size_ssd": "65536", "bluestore_compression_min_blob_size": "0", "bluestore_compression_min_blob_size_hdd": "8192", "bluestore_compression_min_blob_size_ssd": "8192", "bluestore_compression_mode": "none", "bluestore_compression_required_ratio": "0.875000", "bluestore_csum_type": "crc32c", "bluestore_debug_enforce_settings": "default", "bluestore_debug_freelist": "false", "bluestore_debug_fsck_abort": "false", "bluestore_debug_inject_bug21040": "false", "bluestore_debug_inject_csum_err_probability": "0.000000", "bluestore_debug_inject_read_err": "false", "bluestore_debug_legacy_omap": "false", "bluestore_debug_no_reuse_blocks": "false", "bluestore_debug_omit_block_device_write": "false", "bluestore_debug_omit_kv_commit": "false", "bluestore_debug_permit_any_bdev_label": "false", "bluestore_debug_prefill": "0.000000", "bluestore_debug_prefragment_max": "1048576", "bluestore_debug_random_read_err": "0.000000", "bluestore_debug_randomize_serial_transaction": "0", "bluestore_debug_small_allocations": "0", "bluestore_debug_too_many_blobs_threshold": "24576", "bluestore_default_buffered_read": "true", "bluestore_default_buffered_write": "false", "bluestore_deferred_batch_ops": "0", "bluestore_deferred_batch_ops_hdd": "64", "bluestore_deferred_batch_ops_ssd": "16", "bluestore_extent_map_inline_shard_prealloc_size": "256", "bluestore_extent_map_shard_max_size": "1200", "bluestore_extent_map_shard_min_size": "150", "bluestore_extent_map_shard_target_size": "500", "bluestore_extent_map_shard_target_size_slop": "0.200000", "bluestore_freelist_blocks_per_key": "128", "bluestore_fsck_error_on_no_per_pg_omap": "false", "bluestore_fsck_error_on_no_per_pool_omap": "false", "bluestore_fsck_error_on_no_per_pool_stats": "false", "bluestore_fsck_on_mkfs": "true", "bluestore_fsck_on_mkfs_deep": "false", "bluestore_fsck_on_mount": "false", "bluestore_fsck_on_mount_deep": "false", "bluestore_fsck_on_umount": "false", "bluestore_fsck_on_umount_deep": "false", "bluestore_fsck_quick_fix_on_mount": "false", "bluestore_fsck_quick_fix_threads": "2", "bluestore_fsck_read_bytes_cap": "67108864", "bluestore_fsck_shared_blob_tracker_size": "0.031250", "bluestore_gc_enable_blob_threshold": "0", "bluestore_gc_enable_total_threshold": "0", "bluestore_hybrid_alloc_mem_cap": "67108864", "bluestore_ignore_data_csum": "false", "bluestore_kv_sync_util_logging_s": "10.000000", "bluestore_kvbackend": "rocksdb", "bluestore_log_collection_list_age": "60.000000", "bluestore_log_omap_iterator_age": "5.000000", "bluestore_log_op_age": "5.000000", "bluestore_max_alloc_size": "0", "bluestore_max_blob_size": "0", "bluestore_max_blob_size_hdd": "65536", "bluestore_max_blob_size_ssd": "65536", "bluestore_max_defer_interval": "3.000000", "bluestore_max_deferred_txc": "32", "bluestore_min_alloc_size": "0", "bluestore_min_alloc_size_hdd": "4096", "bluestore_min_alloc_size_ssd": "4096", "bluestore_nid_prealloc": "1024", "bluestore_prefer_deferred_size": "0", "bluestore_prefer_deferred_size_hdd": "65536", "bluestore_prefer_deferred_size_ssd": "0", "bluestore_retry_disk_reads": "3", "bluestore_rocksdb_cf": "true", "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L P", "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2,max_total_wal_size=1073741824", "bluestore_rocksdb_options_annex": "", "bluestore_spdk_coremask": "0x1", "bluestore_spdk_io_sleep": "5", "bluestore_spdk_max_io_completion": "0", "bluestore_spdk_mem": "512", "bluestore_sync_submit_transaction": "false", "bluestore_throttle_bytes": "67108864", "bluestore_throttle_cost_per_io": "0", "bluestore_throttle_cost_per_io_hdd": "670000", "bluestore_throttle_cost_per_io_ssd": "4000", "bluestore_throttle_deferred_bytes": "134217728", "bluestore_throttle_trace_rate": "0.000000", "bluestore_tracing": "false", "bluestore_volume_selection_policy": "use_some_extra", "bluestore_volume_selection_reserved": "0", "bluestore_volume_selection_reserved_factor": "2.000000", "bluestore_warn_on_bluefs_spillover": "true", "bluestore_warn_on_legacy_statfs": "true", "bluestore_warn_on_no_per_pg_omap": "false", "bluestore_warn_on_no_per_pool_omap": "true", "bluestore_warn_on_spurious_read_errors": "true", "cephadm_path": "/usr/sbin/cephadm", "cephfs_mirror_action_update_interval": "2", "cephfs_mirror_directory_scan_interval": "10", "cephfs_mirror_max_concurrent_directory_syncs": "3", "cephfs_mirror_max_consecutive_failures_per_directory": "10", "cephfs_mirror_max_snapshot_sync_per_cycle": "3", "cephfs_mirror_mount_timeout": "10", "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", "cephfs_mirror_restart_mirror_on_failure_interval": "20", "cephfs_mirror_retry_failed_directories_interval": "60", "cephsqlite_blocklist_dead_locker": "true", "cephsqlite_lock_renewal_interval": "2000", "cephsqlite_lock_renewal_timeout": "30000", "cephx_cluster_require_signatures": "false", "cephx_cluster_require_version": "2", "cephx_require_signatures": "false", "cephx_require_version": "2", "cephx_service_require_signatures": "false", "cephx_service_require_version": "2", "cephx_sign_messages": "true", "chdir": "", "client_acl_type": "", "client_asio_thread_count": "2", "client_cache_mid": "0.750000", "client_cache_size": "16384", "client_caps_release_delay": "5", "client_check_pool_perm": "true", "client_debug_force_sync_read": "false", "client_debug_getattr_caps": "false", "client_debug_inject_tick_delay": "0", "client_die_on_failed_dentry_invalidate": "true", "client_die_on_failed_remount": "false", "client_dirsize_rbytes": "true", "client_force_lazyio": "false", "client_fs": "", "client_inject_fixed_oldest_tid": "false", "client_inject_release_failure": "false", "client_max_inline_size": "4096", "client_mds_namespace": "", "client_metadata": "", "client_mount_gid": "-1", "client_mount_timeout": "300.000000", "client_mount_uid": "-1", "client_mountpoint": "/", "client_notify_timeout": "10", "client_oc": "true", "client_oc_max_dirty": "104857600", "client_oc_max_dirty_age": "5.000000", "client_oc_max_objects": "1000", "client_oc_size": "209715200", "client_oc_target_dirty": "8388608", "client_permissions": "true", "client_quota_df": "true", "client_readahead_max_bytes": "0", "client_readahead_max_periods": "4", "client_readahead_min": "131072", "client_reconnect_stale": "false", "client_shutdown_timeout": "30", "client_snapdir": ".snap", "client_tick_interval": "1", "client_trace": "", "client_try_dentry_invalidate": "false", "client_use_faked_inos": "false", "client_use_random_mds": "false", "clog_to_graylog": "false", "clog_to_graylog_host": "127.0.0.1", "clog_to_graylog_port": "12201", "clog_to_monitors": "default=true", "clog_to_syslog": "false", "clog_to_syslog_facility": "default=daemon audit=local0", "clog_to_syslog_level": "info", "cluster_addr": "-", "cluster_network": "192.168.122.0/24", "cluster_network_interface": "", "compressor_zlib_isal": "false", "compressor_zlib_level": "5", "compressor_zlib_winsize": "-15", "compressor_zstd_level": "1", "container_image": "docker.io/ceph/daemon-base:latest-pacific-devel", "crash_dir": "/var/lib/ceph/crash", "crimson_osd_obc_lru_size": "10", "crimson_osd_scheduler_concurrency": "0", "crush_location": "", "crush_location_hook": "", "crush_location_hook_timeout": "10", "daemonize": "false", "debug_allow_any_pool_priority": "false", "debug_asok": "1/5", "debug_asok_assert_abort": "false", "debug_asserts_on_shutdown": "false", "debug_auth": "1/5", "debug_bdev": "1/3", "debug_bluefs": "1/5", "debug_bluestore": "1/5", "debug_buffer": "0/1", "debug_cephfs_mirror": "0/5", "debug_cephsqlite": "0/5", "debug_civetweb": "1/10", "debug_client": "0/5", "debug_compressor": "1/5", "debug_context": "0/1", "debug_crush": "1/1", "debug_crypto": "1/5", "debug_deliberately_leak_memory": "false", "debug_disable_randomized_ping": "false", "debug_dpdk": "1/5", "debug_eventtrace": "1/5", "debug_filer": "0/1", "debug_filestore": "1/3", "debug_finisher": "1/1", "debug_fuse": "1/5", "debug_heartbeat_testing_span": "0", "debug_heartbeatmap": "1/5", "debug_immutable_obj_cache": "0/5", "debug_javaclient": "1/5", "debug_journal": "1/3", "debug_journaler": "0/5", "debug_kstore": "1/5", "debug_leveldb": "4/5", "debug_lockdep": "0/1", "debug_mds": "1/5", "debug_mds_balancer": "1/5", "debug_mds_locker": "1/5", "debug_mds_log": "1/5", "debug_mds_log_expire": "1/5", "debug_mds_migrator": "1/5", "debug_memdb": "4/5", "debug_mgr": "2/5", "debug_mgrc": "1/5", "debug_mon": "1/5", "debug_monc": "0/10", "debug_ms": "0/0", "debug_none": "0/5", "debug_objclass": "0/5", "debug_objectcacher": "0/5", "debug_objecter": "0/1", "debug_optracker": "0/5", "debug_osd": "1/5", "debug_paxos": "1/5", "debug_perfcounter": "1/5", "debug_prioritycache": "1/5", "debug_rados": "0/5", "debug_rbd": "0/5", "debug_rbd_mirror": "0/5", "debug_rbd_pwl": "0/5", "debug_rbd_replay": "0/5", "debug_refs": "0/0", "debug_reserver": "1/1", "debug_rgw": "1/5", "debug_rgw_sync": "1/5", "debug_rocksdb": "4/5", "debug_striper": "0/1", "debug_test": "0/5", "debug_throttle": "1/1", "debug_timer": "0/1", "debug_tp": "0/5", "device_failure_prediction_mode": "none", "enable_experimental_unrecoverable_data_corrupting_features": "", "erasure_code_dir": "/usr/lib/ceph/erasure-code", "err_to_graylog": "false", "err_to_stderr": "true", "err_to_syslog": "false", "event_tracing": "false", "fake_statfs_for_testing": "0", "fatal_signal_handlers": "true", "filer_max_purge_ops": "10", "filer_max_truncate_ops": "128", "filestore_apply_finisher_threads": "1", "filestore_blackhole": "false", "filestore_btrfs_clone_range": "true", "filestore_btrfs_snap": "true", "filestore_caller_concurrency": "10", "filestore_collect_device_partition_information": "true", "filestore_commit_timeout": "600.000000", "filestore_debug_inject_read_err": "false", "filestore_debug_omap_check": "false", "filestore_debug_random_read_err": "0.000000", "filestore_debug_verify_split": "false", "filestore_dump_file": "", "filestore_expected_throughput_bytes": "209715200.000000", "filestore_expected_throughput_ops": "200.000000", "filestore_fadvise": "true", "filestore_fail_eio": "true", "filestore_fd_cache_shards": "16", "filestore_fd_cache_size": "128", "filestore_fiemap": "false", "filestore_fiemap_threshold": "4096", "filestore_fsync_flushes_journal_data": "false", "filestore_index_retry_probability": "0.000000", "filestore_inject_stall": "0", "filestore_journal_parallel": "false", "filestore_journal_trailing": "false", "filestore_journal_writeahead": "false", "filestore_kill_at": "0", "filestore_max_alloc_hint_size": "1048576", "filestore_max_inline_xattr_size": "0", "filestore_max_inline_xattr_size_btrfs": "2048", "filestore_max_inline_xattr_size_other": "512", "filestore_max_inline_xattr_size_xfs": "65536", "filestore_max_inline_xattrs": "0", "filestore_max_inline_xattrs_btrfs": "10", "filestore_max_inline_xattrs_other": "2", "filestore_max_inline_xattrs_xfs": "10", "filestore_max_sync_interval": "5.000000", "filestore_max_xattr_value_size": "0", "filestore_max_xattr_value_size_btrfs": "65536", "filestore_max_xattr_value_size_other": "1024", "filestore_max_xattr_value_size_xfs": "65536", "filestore_merge_threshold": "-10", "filestore_min_sync_interval": "0.010000", "filestore_odsync_write": "false", "filestore_omap_backend": "rocksdb", "filestore_omap_backend_path": "", "filestore_omap_header_cache_size": "1024", "filestore_ondisk_finisher_threads": "1", "filestore_op_thread_suicide_timeout": "180", "filestore_op_thread_timeout": "60", "filestore_op_threads": "2", "filestore_punch_hole": "false", "filestore_queue_high_delay_multiple": "0.000000", "filestore_queue_high_delay_multiple_bytes": "0.000000", "filestore_queue_high_delay_multiple_ops": "0.000000", "filestore_queue_high_threshhold": "0.900000", "filestore_queue_low_threshhold": "0.300000", "filestore_queue_max_bytes": "104857600", "filestore_queue_max_delay_multiple": "0.000000", "filestore_queue_max_delay_multiple_bytes": "0.000000", "filestore_queue_max_delay_multiple_ops": "0.000000", "filestore_queue_max_ops": "50", "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", "filestore_seek_data_hole": "false", "filestore_sloppy_crc": "false", "filestore_sloppy_crc_block_size": "65536", "filestore_splice": "false", "filestore_split_multiple": "2", "filestore_split_rand_factor": "20", "filestore_update_to": "1000", "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", "filestore_wbthrottle_btrfs_ios_start_flusher": "500", "filestore_wbthrottle_enable": "true", "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", "filestore_wbthrottle_xfs_inodes_start_flusher": "500", "filestore_wbthrottle_xfs_ios_hard_limit": "5000", "filestore_wbthrottle_xfs_ios_start_flusher": "500", "filestore_xfs_extsize": "false", "filestore_zfs_snap": "false", "fio_dir": "/tmp/fio", "fsid": "8dc32c41-121c-49df-9554-dfb7deb8c975", "fuse_allow_other": "true", "fuse_atomic_o_trunc": "true", "fuse_big_writes": "true", "fuse_debug": "false", "fuse_default_permissions": "false", "fuse_disable_pagecache": "false", "fuse_max_write": "0", "fuse_multithreaded": "true", "fuse_require_active_mds": "true", "fuse_set_user_groups": "true", "fuse_splice_move": "true", "fuse_splice_read": "true", "fuse_splice_write": "true", "fuse_syncfs_on_mksnap": "true", "fuse_use_invalidate_cb": "true", "gss_ktab_client_file": "/var/lib/ceph/mon.ceph-mon1/gss_client_mon.ceph-mon1.ktab", "gss_target_name": "ceph", "heartbeat_file": "", "heartbeat_inject_failure": "0", "heartbeat_interval": "5", "host": "", "immutable_object_cache_client_dedicated_thread_num": "2", "immutable_object_cache_max_inflight_ops": "128", "immutable_object_cache_max_size": "1073741824", "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", "immutable_object_cache_qos_bps_burst": "0", "immutable_object_cache_qos_bps_burst_seconds": "1", "immutable_object_cache_qos_bps_limit": "0", "immutable_object_cache_qos_iops_burst": "0", "immutable_object_cache_qos_iops_burst_seconds": "1", "immutable_object_cache_qos_iops_limit": "0", "immutable_object_cache_qos_schedule_tick_min": "50", "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", "immutable_object_cache_watermark": "0.900000", "inject_early_sigterm": "false", "journal_aio": "true", "journal_align_min_size": "65536", "journal_block_align": "true", "journal_block_size": "4096", "journal_dio": "true", "journal_discard": "false", "journal_force_aio": "false", "journal_ignore_corruption": "false", "journal_max_write_bytes": "10485760", "journal_max_write_entries": "100", "journal_replay_from": "0", "journal_throttle_high_multiple": "0.000000", "journal_throttle_high_threshhold": "0.900000", "journal_throttle_low_threshhold": "0.600000", "journal_throttle_max_multiple": "0.000000", "journal_write_header_frequency": "0", "journal_zero_on_create": "false", "journaler_prefetch_periods": "10", "journaler_prezero_periods": "5", "journaler_write_head_interval": "15", "key": "", "keyfile": "", "keyring": "/var/lib/ceph/mon/ceph-ceph-mon1/keyring", "kstore_backend": "rocksdb", "kstore_default_stripe_size": "65536", "kstore_fsck_on_mount": "false", "kstore_fsck_on_mount_deep": "true", "kstore_max_bytes": "67108864", "kstore_max_ops": "512", "kstore_nid_prealloc": "1024", "kstore_onode_map_size": "1024", "kstore_rocksdb_options": "compression=kNoCompression", "kstore_sync_submit_transaction": "false", "kstore_sync_transaction": "false", "leveldb_block_size": "65536", "leveldb_bloom_size": "0", "leveldb_cache_size": "536870912", "leveldb_compact_on_mount": "false", "leveldb_compression": "false", "leveldb_log": "", "leveldb_log_to_ceph_log": "true", "leveldb_max_open_files": "0", "leveldb_paranoid": "false", "leveldb_write_buffer_size": "33554432", "librados_thread_count": "2", "lockdep": "false", "lockdep_force_backtrace": "false", "log_coarse_timestamps": "true", "log_file": "/var/log/ceph/ceph-mon.ceph-mon1.log", "log_flush_on_exit": "false", "log_graylog_host": "127.0.0.1", "log_graylog_port": "12201", "log_max_new": "1000", "log_max_recent": "10000", "log_stderr_prefix": "", "log_stop_at_utilization": "0.970000", "log_to_file": "true", "log_to_graylog": "false", "log_to_stderr": "false", "log_to_syslog": "false", "max_rotating_auth_attempts": "10", "mds_action_on_write_error": "1", "mds_alternate_name_max": "8192", "mds_asio_thread_count": "2", "mds_bal_export_pin": "true", "mds_bal_fragment_dirs": "true", "mds_bal_fragment_fast_factor": "1.500000", "mds_bal_fragment_interval": "5", "mds_bal_fragment_size_max": "100000", "mds_bal_idle_threshold": "0.000000", "mds_bal_interval": "10", "mds_bal_max": "-1", "mds_bal_max_until": "-1", "mds_bal_merge_size": "50", "mds_bal_midchunk": "0.300000", "mds_bal_min_rebalance": "0.100000", "mds_bal_min_start": "0.200000", "mds_bal_minchunk": "0.001000", "mds_bal_mode": "0", "mds_bal_need_max": "1.200000", "mds_bal_need_min": "0.800000", "mds_bal_replicate_threshold": "8000.000000", "mds_bal_sample_interval": "3.000000", "mds_bal_split_bits": "3", "mds_bal_split_rd": "25000.000000", "mds_bal_split_size": "10000", "mds_bal_split_wr": "10000.000000", "mds_bal_target_decay": "10.000000", "mds_bal_unreplicate_threshold": "0.000000", "mds_beacon_grace": "15.000000", "mds_beacon_interval": "4.000000", "mds_beacon_mon_down_grace": "60", "mds_cache_memory_limit": "4294967296", "mds_cache_mid": "0.700000", "mds_cache_release_free_interval": "10", "mds_cache_reservation": "0.050000", "mds_cache_trim_decay_rate": "1.000000", "mds_cache_trim_interval": "1", "mds_cache_trim_threshold": "262144", "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", "mds_cap_revoke_eviction_timeout": "0.000000", "mds_client_delegate_inos_pct": "50", "mds_client_prealloc_inos": "1000", "mds_client_writeable_range_max_inc_objs": "1024", "mds_connect_bootstrapping": "false", "mds_damage_table_max_entries": "10000", "mds_data": "/var/lib/ceph/mds/ceph-ceph-mon1", "mds_debug_auth_pins": "false", "mds_debug_frag": "false", "mds_debug_scatterstat": "false", "mds_debug_subtrees": "false", "mds_decay_halflife": "5.000000", "mds_default_dir_hash": "2", "mds_defer_session_stale": "true", "mds_deny_all_reconnect": "false", "mds_dir_keys_per_op": "16384", "mds_dir_max_commit_size": "10", "mds_dir_max_entries": "0", "mds_dirstat_min_interval": "1.000000", "mds_dump_cache_after_rejoin": "false", "mds_dump_cache_on_map": "false", "mds_dump_cache_threshold_file": "0", "mds_dump_cache_threshold_formatter": "1073741824", "mds_early_reply": "true", "mds_enable_op_tracker": "true", "mds_enforce_unique_name": "true", "mds_export_ephemeral_distributed": "true", "mds_export_ephemeral_distributed_factor": "2.000000", "mds_export_ephemeral_random": "true", "mds_export_ephemeral_random_max": "0.010000", "mds_forward_all_requests_to_auth": "false", "mds_freeze_tree_timeout": "30.000000", "mds_hack_allow_loading_invalid_metadata": "false", "mds_health_cache_threshold": "1.500000", "mds_health_summarize_threshold": "10", "mds_heartbeat_grace": "15.000000", "mds_inject_migrator_session_race": "false", "mds_inject_traceless_reply_probability": "0.000000", "mds_join_fs": "", "mds_journal_format": "1", "mds_kill_create_at": "0", "mds_kill_export_at": "0", "mds_kill_import_at": "0", "mds_kill_journal_at": "0", "mds_kill_journal_expire_at": "0", "mds_kill_journal_replay_at": "0", "mds_kill_link_at": "0", "mds_kill_mdstable_at": "0", "mds_kill_openc_at": "0", "mds_kill_rename_at": "0", "mds_log_events_per_segment": "1024", "mds_log_max_events": "-1", "mds_log_max_segments": "128", "mds_log_pause": "false", "mds_log_segment_size": "0", "mds_log_skip_corrupt_events": "false", "mds_log_warn_factor": "2.000000", "mds_max_caps_per_client": "1048576", "mds_max_completed_flushes": "100000", "mds_max_completed_requests": "100000", "mds_max_export_size": "20971520", "mds_max_file_recover": "32", "mds_max_purge_files": "64", "mds_max_purge_ops": "8192", "mds_max_purge_ops_per_pg": "0.500000", "mds_max_retries_on_remount_failure": "5", "mds_max_scrub_ops_in_progress": "5", "mds_max_snaps_per_dir": "100", "mds_max_xattr_pairs_size": "65536", "mds_metrics_update_interval": "2", "mds_min_caps_per_client": "100", "mds_min_caps_working_set": "10000", "mds_mon_shutdown_timeout": "5.000000", "mds_numa_node": "-1", "mds_oft_prefetch_dirfrags": "false", "mds_op_complaint_time": "30.000000", "mds_op_history_duration": "600", "mds_op_history_size": "20", "mds_op_log_threshold": "5", "mds_ping_grace": "15", "mds_ping_interval": "5", "mds_purge_queue_busy_flush_period": "1.000000", "mds_recall_global_max_decay_threshold": "131072", "mds_recall_max_caps": "30000", "mds_recall_max_decay_rate": "1.500000", "mds_recall_max_decay_threshold": "131072", "mds_recall_warning_decay_rate": "60.000000", "mds_recall_warning_threshold": "262144", "mds_reconnect_timeout": "45.000000", "mds_replay_interval": "1.000000", "mds_replay_unsafe_with_closed_session": "false", "mds_request_load_average_decay_rate": "60.000000", "mds_root_ino_gid": "0", "mds_root_ino_uid": "0", "mds_scatter_nudge_interval": "5.000000", "mds_session_blocklist_on_evict": "true", "mds_session_blocklist_on_timeout": "true", "mds_session_cache_liveness_decay_rate": "300.000000", "mds_session_cache_liveness_magnitude": "10", "mds_session_cap_acquisition_decay_rate": "10.000000", "mds_session_cap_acquisition_throttle": "500000", "mds_session_max_caps_throttle_ratio": "1.100000", "mds_sessionmap_keys_per_op": "1024", "mds_shutdown_check": "0", "mds_skip_ino": "0", "mds_sleep_rank_change": "0.000000", "mds_snap_max_uid": "4294967294", "mds_snap_min_uid": "0", "mds_snap_rstat": "false", "mds_standby_replay_damaged": "false", "mds_task_status_update_interval": "2.000000", "mds_thrash_exports": "0", "mds_thrash_fragments": "0", "mds_tick_interval": "5.000000", "mds_valgrind_exit": "false", "mds_verify_backtrace": "1", "mds_verify_scatter": "false", "mds_wipe_ino_prealloc": "false", "mds_wipe_sessions": "false", "mempool_debug": "false", "memstore_debug_omit_block_device_write": "false", "memstore_device_bytes": "1073741824", "memstore_page_set": "false", "memstore_page_size": "65536", "mgr_client_bytes": "134217728", "mgr_client_messages": "512", "mgr_client_service_daemon_unregister_timeout": "1.000000", "mgr_connect_retry_interval": "1.000000", "mgr_data": "/var/lib/ceph/mgr/ceph-ceph-mon1", "mgr_debug_aggressive_pg_num_changes": "false", "mgr_disabled_modules": "diskprediction_local", "mgr_initial_modules": "restful iostat nfs", "mgr_max_pg_num_change": "128", "mgr_mds_bytes": "134217728", "mgr_mds_messages": "128", "mgr_module_path": "/usr/share/ceph/mgr", "mgr_mon_bytes": "134217728", "mgr_mon_messages": "128", "mgr_osd_bytes": "536870912", "mgr_osd_messages": "8192", "mgr_service_beacon_grace": "60.000000", "mgr_standby_modules": "true", "mgr_stats_period": "5", "mgr_stats_threshold": "5", "mgr_tick_period": "2", "mgr_ttl_cache_expire_seconds": "0", "mon_accept_timeout_factor": "2.000000", "mon_allow_pool_delete": "false", "mon_allow_pool_size_one": "false", "mon_auth_validate_all_caps": "true", "mon_cache_target_full_warn_ratio": "0.660000", "mon_clean_pg_upmaps_per_chunk": "256", "mon_client_bytes": "104857600", "mon_client_directed_command_retry": "2", "mon_client_hunt_interval": "3.000000", "mon_client_hunt_interval_backoff": "1.500000", "mon_client_hunt_interval_max_multiple": "10.000000", "mon_client_hunt_interval_min_multiple": "1.000000", "mon_client_hunt_parallel": "3", "mon_client_log_interval": "1.000000", "mon_client_max_log_entries_per_message": "1000", "mon_client_ping_interval": "10.000000", "mon_client_ping_timeout": "30.000000", "mon_clock_drift_allowed": "0.050000", "mon_clock_drift_warn_backoff": "5.000000", "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", "mon_cluster_log_file_level": "debug", "mon_cluster_log_to_file": "true", "mon_cluster_log_to_graylog": "false", "mon_cluster_log_to_graylog_host": "127.0.0.1", "mon_cluster_log_to_graylog_port": "12201", "mon_cluster_log_to_stderr": "false", "mon_cluster_log_to_syslog": "default=false", "mon_cluster_log_to_syslog_facility": "daemon", "mon_cluster_log_to_syslog_level": "info", "mon_compact_on_bootstrap": "false", "mon_compact_on_start": "false", "mon_compact_on_trim": "true", "mon_con_tracker_persist_interval": "10", "mon_con_tracker_score_halflife": "43200", "mon_config_key_max_entry_size": "65536", "mon_cpu_threads": "4", "mon_crush_min_required_version": "hammer", "mon_daemon_bytes": "419430400", "mon_data": "/var/lib/ceph/mon/ceph-ceph-mon1", "mon_data_avail_crit": "5", "mon_data_avail_warn": "30", "mon_data_size_warn": "16106127360", "mon_debug_block_osdmap_trim": "false", "mon_debug_deprecated_as_obsolete": "false", "mon_debug_dump_json": "false", "mon_debug_dump_location": "/var/log/ceph/ceph-mon.ceph-mon1.tdump", "mon_debug_dump_transactions": "false", "mon_debug_extra_checks": "false", "mon_debug_no_initial_persistent_features": "false", "mon_debug_no_require_bluestore_for_ec_overwrites": "false", "mon_debug_no_require_octopus": "false", "mon_debug_no_require_pacific": "false", "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", "mon_delta_reset_interval": "10.000000", "mon_dns_srv_name": "ceph-mon", "mon_election_default_strategy": "1", "mon_election_timeout": "5.000000", "mon_elector_ignore_propose_margin": "0.000500", "mon_elector_ping_divisor": "2", "mon_elector_ping_timeout": "2.000000", "mon_enable_op_tracker": "true", "mon_fake_pool_delete": "false", "mon_force_quorum_join": "false", "mon_globalid_prealloc": "10000", "mon_health_detail_to_clog": "true", "mon_health_log_update_period": "5", "mon_health_max_detail": "50", "mon_health_to_clog": "true", "mon_health_to_clog_interval": "600", "mon_health_to_clog_tick_interval": "60.000000", "mon_host": "172.16.88.101,172.16.88.102,172.16.88.103", "mon_host_override": "", "mon_initial_members": "ceph-mon1,ceph-mon2,ceph-mon3", "mon_inject_pg_merge_bounce_probability": "0.000000", "mon_inject_sync_get_chunk_delay": "0.000000", "mon_inject_transaction_delay_max": "10.000000", "mon_inject_transaction_delay_probability": "0.000000", "mon_keyvaluedb": "rocksdb", "mon_lease": "5.000000", "mon_lease_ack_timeout_factor": "2.000000", "mon_lease_renew_interval_factor": "0.600000", "mon_log_max_summary": "50", "mon_max_log_entries_per_event": "4096", "mon_max_log_epochs": "500", "mon_max_mdsmap_epochs": "500", "mon_max_mgrmap_epochs": "500", "mon_max_osd": "10000", "mon_max_pg_per_osd": "250", "mon_max_pool_pg_num": "65536", "mon_max_snap_prune_per_epoch": "100", "mon_mds_blocklist_interval": "86400.000000", "mon_mds_force_trim_to": "0", "mon_mds_skip_sanity": "false", "mon_memory_autotune": "true", "mon_memory_target": "2147483648", "mon_mgr_beacon_grace": "30", "mon_mgr_blocklist_interval": "86400.000000", "mon_mgr_digest_period": "5", "mon_mgr_inactive_grace": "60", "mon_mgr_mkfs_grace": "120", "mon_mgr_proxy_client_bytes_ratio": "0.300000", "mon_min_osdmap_epochs": "500", "mon_op_complaint_time": "30", "mon_op_history_duration": "600", "mon_op_history_size": "20", "mon_op_history_slow_op_size": "20", "mon_op_history_slow_op_threshold": "10", "mon_op_log_threshold": "5", "mon_osd_adjust_down_out_interval": "true", "mon_osd_adjust_heartbeat_grace": "true", "mon_osd_auto_mark_auto_out_in": "true", "mon_osd_auto_mark_in": "false", "mon_osd_auto_mark_new_in": "true", "mon_osd_backfillfull_ratio": "0.900000", "mon_osd_blocklist_default_expire": "3600.000000", "mon_osd_cache_size": "500", "mon_osd_cache_size_min": "134217728", "mon_osd_crush_smoke_test": "true", "mon_osd_destroyed_out_interval": "600", "mon_osd_down_out_interval": "600", "mon_osd_down_out_subtree_limit": "rack", "mon_osd_err_op_age_ratio": "128.000000", "mon_osd_force_trim_to": "0", "mon_osd_full_ratio": "0.950000", "mon_osd_initial_require_min_compat_client": "luminous", "mon_osd_laggy_halflife": "3600", "mon_osd_laggy_max_interval": "300", "mon_osd_laggy_weight": "0.300000", "mon_osd_mapping_pgs_per_chunk": "4096", "mon_osd_max_creating_pgs": "1024", "mon_osd_max_initial_pgs": "1024", "mon_osd_min_down_reporters": "2", "mon_osd_min_in_ratio": "0.750000", "mon_osd_min_up_ratio": "0.300000", "mon_osd_nearfull_ratio": "0.850000", "mon_osd_prime_pg_temp": "true", "mon_osd_prime_pg_temp_max_estimate": "0.250000", "mon_osd_prime_pg_temp_max_time": "0.500000", "mon_osd_report_timeout": "900", "mon_osd_reporter_subtree_level": "host", "mon_osd_snap_trim_queue_warn_on": "32768", "mon_osd_warn_num_repaired": "10", "mon_osd_warn_op_age": "32.000000", "mon_osdmap_full_prune_enabled": "true", "mon_osdmap_full_prune_interval": "10", "mon_osdmap_full_prune_min": "10000", "mon_osdmap_full_prune_txsize": "100", "mon_pg_check_down_all_threshold": "0.500000", "mon_pg_stuck_threshold": "60", "mon_pg_warn_max_object_skew": "10.000000", "mon_pg_warn_min_objects": "10000", "mon_pg_warn_min_per_osd": "0", "mon_pg_warn_min_pool_objects": "1000", "mon_pool_quota_crit_threshold": "0", "mon_pool_quota_warn_threshold": "0", "mon_probe_timeout": "2.000000", "mon_reweight_max_change": "0.050000", "mon_reweight_max_osds": "4", "mon_reweight_min_bytes_per_osd": "104857600", "mon_reweight_min_pgs_per_osd": "10", "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", "mon_scrub_inject_crc_mismatch": "0.000000", "mon_scrub_inject_missing_keys": "0.000000", "mon_scrub_interval": "86400", "mon_scrub_max_keys": "100", "mon_scrub_timeout": "300", "mon_session_timeout": "300", "mon_smart_report_timeout": "5", "mon_stat_smooth_intervals": "6", "mon_stretch_cluster_recovery_ratio": "0.600000", "mon_stretch_pool_min_size": "2", "mon_stretch_pool_size": "4", "mon_stretch_recovery_min_wait": "15.000000", "mon_subscribe_interval": "86400.000000", "mon_sync_debug": "false", "mon_sync_max_payload_keys": "2000", "mon_sync_max_payload_size": "1048576", "mon_sync_provider_kill_at": "0", "mon_sync_requester_kill_at": "0", "mon_sync_timeout": "60.000000", "mon_target_pg_per_osd": "100", "mon_tick_interval": "5", "mon_timecheck_interval": "300.000000", "mon_timecheck_skew_interval": "30.000000", "mon_warn_older_version_delay": "604800", "mon_warn_on_cache_pools_without_hit_sets": "true", "mon_warn_on_crush_straw_calc_version_zero": "true", "mon_warn_on_degraded_stretch_mode": "true", "mon_warn_on_insecure_global_id_reclaim": "true", "mon_warn_on_insecure_global_id_reclaim_allowed": "true", "mon_warn_on_legacy_crush_tunables": "true", "mon_warn_on_misplaced": "false", "mon_warn_on_msgr2_not_enabled": "true", "mon_warn_on_older_version": "true", "mon_warn_on_osd_down_out_interval_zero": "true", "mon_warn_on_pool_no_app": "true", "mon_warn_on_pool_no_redundancy": "true", "mon_warn_on_pool_pg_num_not_power_of_two": "true", "mon_warn_on_slow_ping_ratio": "0.050000", "mon_warn_on_slow_ping_time": "0.000000", "mon_warn_on_too_few_osds": "true", "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", "mon_warn_pg_not_scrubbed_ratio": "0.500000", "monmap": "", "ms_async_max_op_threads": "5", "ms_async_op_threads": "3", "ms_async_rdma_buffer_size": "131072", "ms_async_rdma_cm": "false", "ms_async_rdma_device_name": "", "ms_async_rdma_dscp": "96", "ms_async_rdma_enable_hugepage": "false", "ms_async_rdma_gid_idx": "0", "ms_async_rdma_local_gid": "", "ms_async_rdma_polling_us": "1000", "ms_async_rdma_port_num": "1", "ms_async_rdma_receive_buffers": "32768", "ms_async_rdma_receive_queue_len": "4096", "ms_async_rdma_roce_ver": "1", "ms_async_rdma_send_buffers": "1024", "ms_async_rdma_sl": "3", "ms_async_rdma_support_srq": "true", "ms_async_rdma_type": "ib", "ms_async_reap_threshold": "5", "ms_bind_before_connect": "false", "ms_bind_ipv4": "true", "ms_bind_ipv6": "false", "ms_bind_msgr1": "true", "ms_bind_msgr2": "true", "ms_bind_port_max": "7300", "ms_bind_port_min": "6800", "ms_bind_prefer_ipv4": "false", "ms_bind_retry_count": "3", "ms_bind_retry_delay": "5", "ms_blackhole_client": "false", "ms_blackhole_mds": "false", "ms_blackhole_mgr": "false", "ms_blackhole_mon": "false", "ms_blackhole_osd": "false", "ms_client_mode": "crc secure", "ms_cluster_mode": "crc secure", "ms_cluster_type": "", "ms_connection_idle_timeout": "900", "ms_connection_ready_timeout": "10", "ms_crc_data": "true", "ms_crc_header": "true", "ms_die_on_bad_msg": "false", "ms_die_on_bug": "false", "ms_die_on_old_message": "false", "ms_die_on_skipped_message": "false", "ms_die_on_unhandled_msg": "false", "ms_dispatch_throttle_bytes": "104857600", "ms_dpdk_coremask": "0xF", "ms_dpdk_debug_allow_loopback": "false", "ms_dpdk_gateway_ipv4_addr": "", "ms_dpdk_host_ipv4_addr": "", "ms_dpdk_hugepages": "", "ms_dpdk_hw_flow_control": "true", "ms_dpdk_hw_queue_weight": "1.000000", "ms_dpdk_lro": "true", "ms_dpdk_memory_channel": "4", "ms_dpdk_netmask_ipv4_addr": "", "ms_dpdk_pmd": "", "ms_dpdk_port_id": "0", "ms_dpdk_rx_buffer_count_per_core": "8192", "ms_dump_corrupt_message_level": "1", "ms_dump_on_send": "false", "ms_initial_backoff": "0.200000", "ms_inject_delay_max": "1.000000", "ms_inject_delay_probability": "0.000000", "ms_inject_delay_type": "", "ms_inject_internal_delays": "0.000000", "ms_inject_socket_failures": "0", "ms_learn_addr_from_peer": "true", "ms_max_accept_failures": "4", "ms_max_backoff": "15.000000", "ms_mon_client_mode": "secure crc", "ms_mon_cluster_mode": "secure crc", "ms_mon_service_mode": "secure crc", "ms_pq_max_tokens_per_priority": "16777216", "ms_pq_min_cost": "65536", "ms_public_type": "", "ms_service_mode": "crc secure", "ms_tcp_listen_backlog": "512", "ms_tcp_nodelay": "true", "ms_tcp_prefetch_max_size": "4096", "ms_tcp_rcvbuf": "0", "ms_type": "async+posix", "no_config_file": "false", "objecter_completion_locks_per_session": "32", "objecter_debug_inject_relock_delay": "false", "objecter_inflight_op_bytes": "104857600", "objecter_inflight_ops": "1024", "objecter_inject_no_watch_ping": "false", "objecter_retry_writes_after_first_reply": "false", "objecter_tick_interval": "5.000000", "objecter_timeout": "10.000000", "objectstore_blackhole": "false", "openssl_engine_opts": "", "osd_agent_delay_time": "5.000000", "osd_agent_hist_halflife": "1000", "osd_agent_max_low_ops": "2", "osd_agent_max_ops": "4", "osd_agent_min_evict_effort": "0.100000", "osd_agent_quantize_effort": "0.100000", "osd_agent_slop": "0.020000", "osd_aggregated_slow_ops_logging": "true", "osd_allow_recovery_below_min_size": "true", "osd_asio_thread_count": "2", "osd_async_recovery_min_cost": "100", "osd_auto_mark_unfound_lost": "false", "osd_backfill_retry_interval": "30.000000", "osd_backfill_scan_max": "512", "osd_backfill_scan_min": "64", "osd_backoff_on_degraded": "false", "osd_backoff_on_peering": "false", "osd_backoff_on_unfound": "true", "osd_beacon_report_interval": "300", "osd_bench_duration": "30", "osd_bench_large_size_max_throughput": "104857600", "osd_bench_max_block_size": "67108864", "osd_bench_small_size_max_iops": "100", "osd_blkin_trace_all": "false", "osd_calc_pg_upmaps_aggressively": "true", "osd_calc_pg_upmaps_local_fallback_retries": "100", "osd_check_for_log_corruption": "false", "osd_check_max_object_name_len_on_startup": "true", "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", "osd_class_dir": "/usr/lib/rados-classes", "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", "osd_class_update_on_start": "true", "osd_client_message_cap": "256", "osd_client_message_size_cap": "524288000", "osd_client_op_priority": "63", "osd_client_watch_timeout": "30", "osd_command_max_records": "256", "osd_command_thread_suicide_timeout": "900", "osd_command_thread_timeout": "600", "osd_compact_on_start": "false", "osd_copyfrom_max_chunk": "8388608", "osd_crush_chooseleaf_type": "1", "osd_crush_initial_weight": "-1.000000", "osd_crush_update_on_start": "true", "osd_crush_update_weight_set": "true", "osd_data": "/var/lib/ceph/osd/ceph-ceph-mon1", "osd_debug_crash_on_ignored_backoff": "false", "osd_debug_deep_scrub_sleep": "0.000000", "osd_debug_drop_ping_duration": "0", "osd_debug_drop_ping_probability": "0.000000", "osd_debug_feed_pullee": "-1", "osd_debug_inject_copyfrom_error": "false", "osd_debug_inject_dispatch_delay_duration": "0.100000", "osd_debug_inject_dispatch_delay_probability": "0.000000", "osd_debug_misdirected_ops": "false", "osd_debug_no_acting_change": "false", "osd_debug_no_purge_strays": "false", "osd_debug_op_order": "false", "osd_debug_pg_log_writeout": "false", "osd_debug_pretend_recovery_active": "false", "osd_debug_random_push_read_error": "0.000000", "osd_debug_reject_backfill_probability": "0.000000", "osd_debug_shutdown": "false", "osd_debug_skip_full_check_in_backfill_reservation": "false", "osd_debug_skip_full_check_in_recovery": "false", "osd_debug_verify_cached_snaps": "false", "osd_debug_verify_missing_on_start": "false", "osd_debug_verify_snaps": "false", "osd_debug_verify_stray_on_activate": "false", "osd_deep_scrub_interval": "604800.000000", "osd_deep_scrub_keys": "1024", "osd_deep_scrub_large_omap_object_key_threshold": "200000", "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", "osd_deep_scrub_randomize_ratio": "0.150000", "osd_deep_scrub_stride": "524288", "osd_deep_scrub_update_digest_min_age": "7200", "osd_default_data_pool_replay_window": "45", "osd_default_notify_timeout": "30", "osd_delete_sleep": "0.000000", "osd_delete_sleep_hdd": "5.000000", "osd_delete_sleep_hybrid": "1.000000", "osd_delete_sleep_ssd": "1.000000", "osd_discard_disconnected_ops": "true", "osd_enable_op_tracker": "true", "osd_erasure_code_plugins": "jerasure lrc isa", "osd_failsafe_full_ratio": "0.970000", "osd_fast_fail_on_connection_refused": "true", "osd_fast_info": "true", "osd_fast_shutdown": "true", "osd_fast_shutdown_notify_mon": "true", "osd_find_best_info_ignore_history_les": "false", "osd_force_auth_primary_missing_objects": "100", "osd_force_recovery_pg_log_entries_factor": "1.300000", "osd_function_tracing": "false", "osd_heartbeat_grace": "20", "osd_heartbeat_interval": "6", "osd_heartbeat_min_healthy_ratio": "0.330000", "osd_heartbeat_min_peers": "10", "osd_heartbeat_min_size": "2000", "osd_heartbeat_stale": "600", "osd_heartbeat_use_min_delay_socket": "false", "osd_hit_set_max_size": "100000", "osd_hit_set_min_size": "1000", "osd_hit_set_namespace": ".ceph-internal", "osd_ignore_stale_divergent_priors": "false", "osd_inject_bad_map_crc_probability": "0.000000", "osd_inject_failure_on_pg_removal": "false", "osd_journal": "/var/lib/ceph/osd/ceph-ceph-mon1/journal", "osd_journal_flush_on_shutdown": "true", "osd_journal_size": "5120", "osd_kill_backfill_at": "0", "osd_loop_before_reset_tphandle": "64", "osd_map_cache_size": "50", "osd_map_dedup": "true", "osd_map_message_max": "40", "osd_map_message_max_bytes": "10485760", "osd_map_share_max_epochs": "40", "osd_max_attr_name_len": "100", "osd_max_attr_size": "0", "osd_max_backfills": "1", "osd_max_markdown_count": "5", "osd_max_markdown_period": "600", "osd_max_object_name_len": "2048", "osd_max_object_namespace_len": "256", "osd_max_object_size": "134217728", "osd_max_omap_bytes_per_request": "1073741824", "osd_max_omap_entries_per_request": "1024", "osd_max_pg_blocked_by": "16", "osd_max_pg_log_entries": "10000", "osd_max_pg_per_osd_hard_ratio": "3.000000", "osd_max_pgls": "1024", "osd_max_push_cost": "8388608", "osd_max_push_objects": "10", "osd_max_scrubs": "1", "osd_max_snap_prune_intervals_per_epoch": "512", "osd_max_trimming_pgs": "2", "osd_max_write_op_reply_len": "32", "osd_max_write_size": "90", "osd_mclock_cost_per_byte_usec": "0.000000", "osd_mclock_cost_per_byte_usec_hdd": "5.200000", "osd_mclock_cost_per_byte_usec_ssd": "0.011000", "osd_mclock_cost_per_io_usec": "0.000000", "osd_mclock_cost_per_io_usec_hdd": "25000.000000", "osd_mclock_cost_per_io_usec_ssd": "50.000000", "osd_mclock_force_run_benchmark_on_init": "false", "osd_mclock_max_capacity_iops_hdd": "315.000000", "osd_mclock_max_capacity_iops_ssd": "21500.000000", "osd_mclock_profile": "high_client_ops", "osd_mclock_scheduler_anticipation_timeout": "0.000000", "osd_mclock_scheduler_background_best_effort_lim": "999999", "osd_mclock_scheduler_background_best_effort_res": "1", "osd_mclock_scheduler_background_best_effort_wgt": "1", "osd_mclock_scheduler_background_recovery_lim": "999999", "osd_mclock_scheduler_background_recovery_res": "1", "osd_mclock_scheduler_background_recovery_wgt": "1", "osd_mclock_scheduler_client_lim": "999999", "osd_mclock_scheduler_client_res": "1", "osd_mclock_scheduler_client_wgt": "1", "osd_mclock_skip_benchmark": "false", "osd_memory_base": "805306368", "osd_memory_cache_min": "134217728", "osd_memory_cache_resize_interval": "1.000000", "osd_memory_expected_fragmentation": "0.150000", "osd_memory_target": "4294967296", "osd_memory_target_autotune": "false", "osd_memory_target_cgroup_limit_ratio": "0.800000", "osd_min_pg_log_entries": "250", "osd_min_recovery_priority": "0", "osd_mon_heartbeat_interval": "30", "osd_mon_heartbeat_stat_stale": "3600", "osd_mon_report_interval": "5", "osd_mon_report_max_in_flight": "2", "osd_mon_shutdown_timeout": "5.000000", "osd_num_cache_shards": "32", "osd_num_op_tracker_shard": "32", "osd_numa_auto_affinity": "true", "osd_numa_node": "-1", "osd_numa_prefer_iface": "true", "osd_object_clean_region_max_num_intervals": "10", "osd_objecter_finishers": "1", "osd_objectstore": "bluestore", "osd_objectstore_fuse": "false", "osd_objectstore_tracing": "false", "osd_op_complaint_time": "30.000000", "osd_op_history_duration": "600", "osd_op_history_size": "20", "osd_op_history_slow_op_size": "20", "osd_op_history_slow_op_threshold": "10.000000", "osd_op_log_threshold": "5", "osd_op_num_shards": "0", "osd_op_num_shards_hdd": "5", "osd_op_num_shards_ssd": "8", "osd_op_num_threads_per_shard": "0", "osd_op_num_threads_per_shard_hdd": "1", "osd_op_num_threads_per_shard_ssd": "2", "osd_op_pq_max_tokens_per_priority": "4194304", "osd_op_pq_min_cost": "65536", "osd_op_queue": "wpq", "osd_op_queue_cut_off": "high", "osd_op_thread_suicide_timeout": "150", "osd_op_thread_timeout": "15", "osd_open_classes_on_start": "true", "osd_os_flags": "0", "osd_peering_op_priority": "255", "osd_pg_delete_cost": "1048576", "osd_pg_delete_priority": "5", "osd_pg_epoch_max_lag_factor": "2.000000", "osd_pg_epoch_persisted_max_stale": "40", "osd_pg_log_dups_tracked": "3000", "osd_pg_log_trim_max": "10000", "osd_pg_log_trim_min": "100", "osd_pg_max_concurrent_snap_trims": "2", "osd_pg_object_context_cache_count": "64", "osd_pg_stat_report_interval_max": "500", "osd_pool_default_cache_max_evict_check_size": "10", "osd_pool_default_cache_min_evict_age": "0", "osd_pool_default_cache_min_flush_age": "0", "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", "osd_pool_default_cache_target_dirty_ratio": "0.400000", "osd_pool_default_cache_target_full_ratio": "0.800000", "osd_pool_default_crush_rule": "-1", "osd_pool_default_ec_fast_read": "false", "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=2", "osd_pool_default_flag_bulk": "false", "osd_pool_default_flag_hashpspool": "true", "osd_pool_default_flag_nodelete": "false", "osd_pool_default_flag_nopgchange": "false", "osd_pool_default_flag_nosizechange": "false", "osd_pool_default_flags": "0", "osd_pool_default_hit_set_bloom_fpp": "0.050000", "osd_pool_default_min_size": "0", "osd_pool_default_pg_autoscale_mode": "on", "osd_pool_default_pg_num": "32", "osd_pool_default_pgp_num": "0", "osd_pool_default_read_lease_ratio": "0.800000", "osd_pool_default_size": "3", "osd_pool_default_type": "replicated", "osd_pool_erasure_code_stripe_unit": "4096", "osd_pool_use_gmt_hitset": "true", "osd_push_per_object_cost": "1000", "osd_read_ec_check_for_errors": "false", "osd_recover_clone_overlap": "true", "osd_recover_clone_overlap_limit": "10", "osd_recovery_cost": "20971520", "osd_recovery_delay_start": "0.000000", "osd_recovery_max_active": "0", "osd_recovery_max_active_hdd": "3", "osd_recovery_max_active_ssd": "10", "osd_recovery_max_chunk": "8388608", "osd_recovery_max_omap_entries_per_chunk": "8096", "osd_recovery_max_single_start": "1", "osd_recovery_op_priority": "3", "osd_recovery_op_warn_multiple": "16", "osd_recovery_priority": "5", "osd_recovery_retry_interval": "30.000000", "osd_recovery_sleep": "0.000000", "osd_recovery_sleep_hdd": "0.100000", "osd_recovery_sleep_hybrid": "0.025000", "osd_recovery_sleep_ssd": "0.000000", "osd_repair_during_recovery": "false", "osd_requested_scrub_priority": "120", "osd_rocksdb_iterator_bounds_enabled": "true", "osd_rollback_to_cluster_snap": "", "osd_scrub_auto_repair": "false", "osd_scrub_auto_repair_num_errors": "5", "osd_scrub_backoff_ratio": "0.660000", "osd_scrub_begin_hour": "0", "osd_scrub_begin_week_day": "0", "osd_scrub_chunk_max": "25", "osd_scrub_chunk_min": "5", "osd_scrub_cost": "52428800", "osd_scrub_during_recovery": "false", "osd_scrub_end_hour": "0", "osd_scrub_end_week_day": "0", "osd_scrub_extended_sleep": "0.000000", "osd_scrub_interval_randomize_ratio": "0.500000", "osd_scrub_invalid_stats": "true", "osd_scrub_load_threshold": "0.500000", "osd_scrub_max_interval": "604800.000000", "osd_scrub_max_preemptions": "5", "osd_scrub_min_interval": "86400.000000", "osd_scrub_priority": "5", "osd_scrub_sleep": "0.000000", "osd_shutdown_pgref_assert": "false", "osd_skip_data_digest": "false", "osd_smart_report_timeout": "5", "osd_snap_trim_cost": "1048576", "osd_snap_trim_priority": "5", "osd_snap_trim_sleep": "0.000000", "osd_snap_trim_sleep_hdd": "5.000000", "osd_snap_trim_sleep_hybrid": "2.000000", "osd_snap_trim_sleep_ssd": "0.000000", "osd_target_pg_log_entries_per_osd": "300000", "osd_target_transaction_size": "30", "osd_tier_default_cache_hit_set_count": "4", "osd_tier_default_cache_hit_set_grade_decay_rate": "20", "osd_tier_default_cache_hit_set_period": "1200", "osd_tier_default_cache_hit_set_search_last_n": "1", "osd_tier_default_cache_hit_set_type": "bloom", "osd_tier_default_cache_min_read_recency_for_promote": "1", "osd_tier_default_cache_min_write_recency_for_promote": "1", "osd_tier_default_cache_mode": "writeback", "osd_tier_promote_max_bytes_sec": "5242880", "osd_tier_promote_max_objects_sec": "25", "osd_tracing": "false", "osd_use_stale_snap": "false", "osd_uuid": "00000000-0000-0000-0000-000000000000", "osdc_blkin_trace_all": "false", "paxos_kill_at": "0", "paxos_max_join_drift": "10", "paxos_min": "500", "paxos_min_wait": "0.050000", "paxos_propose_interval": "1.000000", "paxos_service_trim_max": "500", "paxos_service_trim_max_multiplier": "20", "paxos_service_trim_min": "250", "paxos_stash_full_interval": "25", "paxos_trim_max": "500", "paxos_trim_min": "250", "perf": "true", "pid_file": "", "plugin_crypto_accelerator": "crypto_isal", "plugin_dir": "/usr/lib/ceph", "public_addr": "-", "public_addrv": "", "public_bind_addr": "-", "public_network": "172.16.88.0/24", "public_network_interface": "", "qat_compressor_enabled": "false", "rados_mon_op_timeout": "0", "rados_osd_op_timeout": "0", "rados_tracing": "false", "rbd_atime_update_interval": "60", "rbd_auto_exclusive_lock_until_manual_request": "true", "rbd_balance_parent_reads": "false", "rbd_balance_snap_reads": "false", "rbd_blkin_trace_all": "false", "rbd_blocklist_expire_seconds": "0", "rbd_blocklist_on_break_lock": "true", "rbd_cache": "true", "rbd_cache_block_writes_upfront": "false", "rbd_cache_max_dirty": "25165824", "rbd_cache_max_dirty_age": "1.000000", "rbd_cache_max_dirty_object": "0", "rbd_cache_policy": "writearound", "rbd_cache_size": "33554432", "rbd_cache_target_dirty": "16777216", "rbd_cache_writethrough_until_flush": "true", "rbd_clone_copy_on_read": "false", "rbd_compression_hint": "none", "rbd_concurrent_management_ops": "10", "rbd_config_pool_override_update_timestamp": "0", "rbd_default_clone_format": "auto", "rbd_default_data_pool": "", "rbd_default_features": "61", "rbd_default_format": "2", "rbd_default_map_options": "", "rbd_default_order": "22", "rbd_default_pool": "rbd", "rbd_default_snapshot_quiesce_mode": "required", "rbd_default_stripe_count": "0", "rbd_default_stripe_unit": "0", "rbd_disable_zero_copy_writes": "true", "rbd_discard_granularity_bytes": "65536", "rbd_discard_on_zeroed_write_same": "true", "rbd_enable_alloc_hint": "true", "rbd_invalidate_object_map_on_timeout": "true", "rbd_io_scheduler": "simple", "rbd_io_scheduler_simple_max_delay": "0", "rbd_journal_commit_age": "5.000000", "rbd_journal_max_concurrent_object_sets": "0", "rbd_journal_max_payload_bytes": "16384", "rbd_journal_object_flush_age": "0.000000", "rbd_journal_object_flush_bytes": "1048576", "rbd_journal_object_flush_interval": "0", "rbd_journal_object_max_in_flight_appends": "0", "rbd_journal_object_writethrough_until_flush": "true", "rbd_journal_order": "24", "rbd_journal_pool": "", "rbd_journal_splay_width": "4", "rbd_localize_parent_reads": "false", "rbd_localize_snap_reads": "false", "rbd_mirror_concurrent_image_deletions": "1", "rbd_mirror_concurrent_image_syncs": "5", "rbd_mirror_delete_retry_interval": "30.000000", "rbd_mirror_image_perf_stats_prio": "5", "rbd_mirror_image_policy_migration_throttle": "300", "rbd_mirror_image_policy_rebalance_timeout": "0.000000", "rbd_mirror_image_policy_type": "simple", "rbd_mirror_image_policy_update_throttle_interval": "1.000000", "rbd_mirror_image_state_check_interval": "30", "rbd_mirror_journal_commit_age": "5.000000", "rbd_mirror_journal_poll_age": "5.000000", "rbd_mirror_leader_heartbeat_interval": "5", "rbd_mirror_leader_max_acquire_attempts_before_break": "3", "rbd_mirror_leader_max_missed_heartbeats": "2", "rbd_mirror_memory_autotune": "true", "rbd_mirror_memory_base": "805306368", "rbd_mirror_memory_cache_autotune_interval": "30.000000", "rbd_mirror_memory_cache_min": "134217728", "rbd_mirror_memory_cache_resize_interval": "5.000000", "rbd_mirror_memory_expected_fragmentation": "0.150000", "rbd_mirror_memory_target": "4294967296", "rbd_mirror_perf_stats_prio": "5", "rbd_mirror_pool_replayers_refresh_interval": "30", "rbd_mirror_sync_point_update_age": "30.000000", "rbd_mirroring_delete_delay": "0", "rbd_mirroring_max_mirroring_snapshots": "3", "rbd_mirroring_replay_delay": "0", "rbd_mirroring_resync_after_disconnect": "false", "rbd_move_parent_to_trash_on_remove": "false", "rbd_move_to_trash_on_remove": "false", "rbd_move_to_trash_on_remove_expire_seconds": "0", "rbd_mtime_update_interval": "60", "rbd_non_blocking_aio": "true", "rbd_op_thread_timeout": "60", "rbd_op_threads": "1", "rbd_parent_cache_enabled": "false", "rbd_persistent_cache_mode": "disabled", "rbd_persistent_cache_path": "/tmp", "rbd_persistent_cache_size": "1073741824", "rbd_plugins": "", "rbd_qos_bps_burst": "0", "rbd_qos_bps_burst_seconds": "1", "rbd_qos_bps_limit": "0", "rbd_qos_iops_burst": "0", "rbd_qos_iops_burst_seconds": "1", "rbd_qos_iops_limit": "0", "rbd_qos_read_bps_burst": "0", "rbd_qos_read_bps_burst_seconds": "1", "rbd_qos_read_bps_limit": "0", "rbd_qos_read_iops_burst": "0", "rbd_qos_read_iops_burst_seconds": "1", "rbd_qos_read_iops_limit": "0", "rbd_qos_schedule_tick_min": "50", "rbd_qos_write_bps_burst": "0", "rbd_qos_write_bps_burst_seconds": "1", "rbd_qos_write_bps_limit": "0", "rbd_qos_write_iops_burst": "0", "rbd_qos_write_iops_burst_seconds": "1", "rbd_qos_write_iops_limit": "0", "rbd_quiesce_notification_attempts": "10", "rbd_read_from_replica_policy": "default", "rbd_readahead_disable_after_bytes": "52428800", "rbd_readahead_max_bytes": "524288", "rbd_readahead_trigger_requests": "10", "rbd_request_timed_out_seconds": "30", "rbd_skip_partial_discard": "true", "rbd_sparse_read_threshold_bytes": "65536", "rbd_tracing": "false", "rbd_validate_names": "true", "rbd_validate_pool": "true", "restapi_base_url": "", "restapi_log_level": "", "rgw_acl_grants_max_num": "100", "rgw_admin_entry": "admin", "rgw_barbican_url": "", "rgw_bucket_default_quota_max_objects": "-1", "rgw_bucket_default_quota_max_size": "-1", "rgw_bucket_index_max_aio": "128", "rgw_bucket_quota_cache_size": "10000", "rgw_bucket_quota_ttl": "600", "rgw_cache_enabled": "true", "rgw_cache_expiry_interval": "900", "rgw_cache_lru_size": "10000", "rgw_content_length_compat": "false", "rgw_copy_obj_progress": "true", "rgw_copy_obj_progress_every_bytes": "1048576", "rgw_cors_rules_max_num": "100", "rgw_cross_domain_policy": "<allow-access-from domain=\"*\" secure=\"false\" />", "rgw_crypt_default_encryption_key": "", "rgw_crypt_kmip_addr": "", "rgw_crypt_kmip_ca_path": "", "rgw_crypt_kmip_client_cert": "", "rgw_crypt_kmip_client_key": "", "rgw_crypt_kmip_kms_key_template": "", "rgw_crypt_kmip_password": "", "rgw_crypt_kmip_s3_key_template": "$keyid", "rgw_crypt_kmip_username": "", "rgw_crypt_require_ssl": "true", "rgw_crypt_s3_kms_backend": "barbican", "rgw_crypt_s3_kms_encryption_keys": "", "rgw_crypt_suppress_logs": "true", "rgw_crypt_vault_addr": "", "rgw_crypt_vault_auth": "token", "rgw_crypt_vault_namespace": "", "rgw_crypt_vault_prefix": "", "rgw_crypt_vault_secret_engine": "transit", "rgw_crypt_vault_ssl_cacert": "", "rgw_crypt_vault_ssl_clientcert": "", "rgw_crypt_vault_ssl_clientkey": "", "rgw_crypt_vault_token_file": "", "rgw_crypt_vault_verify_ssl": "true", "rgw_curl_buffersize": "524288", "rgw_curl_low_speed_limit": "1024", "rgw_curl_low_speed_time": "300", "rgw_curl_wait_timeout_ms": "1000", "rgw_data": "/var/lib/ceph/radosgw/ceph-ceph-mon1", "rgw_data_log_changes_size": "1000", "rgw_data_log_num_shards": "128", "rgw_data_log_obj_prefix": "data_log", "rgw_data_log_window": "30", "rgw_data_notify_interval_msec": "200", "rgw_default_data_log_backing": "fifo", "rgw_default_realm_info_oid": "default.realm", "rgw_default_region_info_oid": "default.region", "rgw_default_zone_info_oid": "default.zone", "rgw_default_zonegroup_info_oid": "default.zonegroup", "rgw_defer_to_bucket_acls": "", "rgw_delete_multi_obj_max_num": "1000", "rgw_dmclock_admin_lim": "0.000000", "rgw_dmclock_admin_res": "100.000000", "rgw_dmclock_admin_wgt": "100.000000", "rgw_dmclock_auth_lim": "0.000000", "rgw_dmclock_auth_res": "200.000000", "rgw_dmclock_auth_wgt": "100.000000", "rgw_dmclock_data_lim": "0.000000", "rgw_dmclock_data_res": "500.000000", "rgw_dmclock_data_wgt": "500.000000", "rgw_dmclock_metadata_lim": "0.000000", "rgw_dmclock_metadata_res": "500.000000", "rgw_dmclock_metadata_wgt": "500.000000", "rgw_dns_name": "", "rgw_dns_s3website_name": "", "rgw_dynamic_resharding": "true", "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", "rgw_enable_gc_threads": "true", "rgw_enable_lc_threads": "true", "rgw_enable_ops_log": "false", "rgw_enable_quota_threads": "true", "rgw_enable_static_website": "false", "rgw_enable_usage_log": "false", "rgw_enforce_swift_acls": "true", "rgw_exit_timeout_secs": "120", "rgw_expose_bucket": "false", "rgw_extended_http_attrs": "", "rgw_fcgi_socket_backlog": "1024", "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", "rgw_frontends": "beast port=7480", "rgw_gc_max_concurrent_io": "10", "rgw_gc_max_deferred": "50", "rgw_gc_max_deferred_entries_size": "3072", "rgw_gc_max_objs": "32", "rgw_gc_max_queue_size": "134213632", "rgw_gc_max_trim_chunk": "16", "rgw_gc_obj_min_wait": "7200", "rgw_gc_processor_max_time": "3600", "rgw_gc_processor_period": "3600", "rgw_get_obj_max_req_size": "4194304", "rgw_get_obj_window_size": "16777216", "rgw_healthcheck_disabling_path": "", "rgw_host": "", "rgw_ignore_get_invalid_range": "false", "rgw_init_timeout": "300", "rgw_inject_notify_timeout_probability": "0.000000", "rgw_keystone_accepted_admin_roles": "", "rgw_keystone_accepted_roles": "Member, admin", "rgw_keystone_admin_domain": "", "rgw_keystone_admin_password": "", "rgw_keystone_admin_password_path": "", "rgw_keystone_admin_project": "", "rgw_keystone_admin_tenant": "", "rgw_keystone_admin_token": "", "rgw_keystone_admin_token_path": "", "rgw_keystone_admin_user": "", "rgw_keystone_api_version": "2", "rgw_keystone_barbican_domain": "", "rgw_keystone_barbican_password": "", "rgw_keystone_barbican_project": "", "rgw_keystone_barbican_tenant": "", "rgw_keystone_barbican_user": "", "rgw_keystone_implicit_tenants": "false", "rgw_keystone_token_cache_size": "10000", "rgw_keystone_url": "", "rgw_keystone_verify_ssl": "true", "rgw_lc_debug_interval": "-1", "rgw_lc_lock_max_time": "90", "rgw_lc_max_objs": "32", "rgw_lc_max_rules": "1000", "rgw_lc_max_worker": "3", "rgw_lc_max_wp_worker": "3", "rgw_lc_thread_delay": "0", "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", "rgw_ldap_dnattr": "uid", "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", "rgw_ldap_searchfilter": "", "rgw_ldap_secret": "/etc/openldap/secret", "rgw_ldap_uri": "ldaps://<ldap.your.domain>", "rgw_lifecycle_work_time": "00:00-06:00", "rgw_list_bucket_min_readahead": "1000", "rgw_list_buckets_max_chunk": "1000", "rgw_log_http_headers": "", "rgw_log_nonexistent_bucket": "false", "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", "rgw_log_object_name_utc": "false", "rgw_luarocks_location": "/tmp/luarocks", "rgw_max_attr_name_len": "0", "rgw_max_attr_size": "0", "rgw_max_attrs_num_in_req": "0", "rgw_max_chunk_size": "4194304", "rgw_max_concurrent_requests": "1024", "rgw_max_dynamic_shards": "1999", "rgw_max_listing_results": "1000", "rgw_max_notify_retries": "3", "rgw_max_objs_per_shard": "100000", "rgw_max_put_param_size": "1048576", "rgw_max_put_size": "5368709120", "rgw_max_slo_entries": "1000", "rgw_md_log_max_shards": "64", "rgw_md_notify_interval_msec": "200", "rgw_mime_types_file": "/etc/mime.types", "rgw_mp_lock_max_time": "600", "rgw_multipart_min_part_size": "5242880", "rgw_multipart_part_upload_limit": "10000", "rgw_nfs_fhcache_partitions": "3", "rgw_nfs_fhcache_size": "2017", "rgw_nfs_lru_lane_hiwat": "911", "rgw_nfs_lru_lanes": "5", "rgw_nfs_max_gc": "300", "rgw_nfs_namespace_expire_secs": "300", "rgw_nfs_run_gc_threads": "false", "rgw_nfs_run_lc_threads": "false", "rgw_nfs_run_quota_threads": "false", "rgw_nfs_run_sync_thread": "false", "rgw_nfs_s3_fast_attrs": "false", "rgw_nfs_write_completion_interval_s": "10", "rgw_num_async_rados_threads": "32", "rgw_num_control_oids": "8", "rgw_num_rados_handles": "1", "rgw_numa_node": "-1", "rgw_obj_stripe_size": "4194304", "rgw_obj_tombstone_cache_size": "1000", "rgw_objexp_chunk_size": "100", "rgw_objexp_gc_interval": "600", "rgw_objexp_hints_num_shards": "127", "rgw_olh_pending_timeout_sec": "3600", "rgw_op_thread_suicide_timeout": "0", "rgw_op_thread_timeout": "600", "rgw_op_tracing": "false", "rgw_opa_token": "", "rgw_opa_url": "", "rgw_opa_verify_ssl": "true", "rgw_ops_log_data_backlog": "5242880", "rgw_ops_log_file_path": "", "rgw_ops_log_rados": "true", "rgw_ops_log_socket_path": "", "rgw_override_bucket_index_max_shards": "0", "rgw_period_latest_epoch_info_oid": ".latest_epoch", "rgw_period_push_interval": "2.000000", "rgw_period_push_interval_max": "30.000000", "rgw_period_root_pool": ".rgw.root", "rgw_port": "", "rgw_print_continue": "true", "rgw_print_prohibited_content_length": "false", "rgw_put_obj_max_window_size": "67108864", "rgw_put_obj_min_window_size": "16777216", "rgw_rados_pool_autoscale_bias": "4.000000", "rgw_rados_pool_pg_num_min": "8", "rgw_rados_pool_recovery_priority": "5", "rgw_rados_tracing": "false", "rgw_realm": "", "rgw_realm_root_pool": ".rgw.root", "rgw_region": "", "rgw_region_root_pool": ".rgw.root", "rgw_relaxed_region_enforcement": "false", "rgw_relaxed_s3_bucket_names": "false", "rgw_remote_addr_param": "REMOTE_ADDR", "rgw_request_uri": "", "rgw_reshard_batch_size": "64", "rgw_reshard_bucket_lock_duration": "360", "rgw_reshard_max_aio": "128", "rgw_reshard_num_logs": "16", "rgw_reshard_thread_interval": "600", "rgw_resolve_cname": "false", "rgw_rest_getusage_op_compat": "false", "rgw_run_sync_thread": "true", "rgw_s3_auth_order": "sts, external, local", "rgw_s3_auth_use_keystone": "false", "rgw_s3_auth_use_ldap": "false", "rgw_s3_auth_use_rados": "true", "rgw_s3_auth_use_sts": "false", "rgw_s3_success_create_obj_status": "0", "rgw_safe_max_objects_per_shard": "102400", "rgw_scheduler_type": "throttler", "rgw_script_uri": "", "rgw_service_provider_name": "", "rgw_shard_warning_threshold": "90.000000", "rgw_socket_path": "", "rgw_sts_client_id": "", "rgw_sts_client_secret": "", "rgw_sts_entry": "sts", "rgw_sts_key": "sts", "rgw_sts_max_session_duration": "43200", "rgw_sts_min_session_duration": "900", "rgw_sts_token_introspection_url": "", "rgw_swift_account_in_url": "false", "rgw_swift_auth_entry": "auth", "rgw_swift_auth_url": "", "rgw_swift_custom_header": "", "rgw_swift_enforce_content_length": "false", "rgw_swift_need_stats": "true", "rgw_swift_tenant_name": "", "rgw_swift_token_expiration": "86400", "rgw_swift_url": "", "rgw_swift_url_prefix": "swift", "rgw_swift_versioning_enabled": "false", "rgw_sync_data_inject_err_probability": "0.000000", "rgw_sync_lease_period": "120", "rgw_sync_log_trim_concurrent_buckets": "4", "rgw_sync_log_trim_interval": "1200", "rgw_sync_log_trim_max_buckets": "16", "rgw_sync_log_trim_min_cold_buckets": "4", "rgw_sync_meta_inject_err_probability": "0.000000", "rgw_sync_obj_etag_verify": "false", "rgw_sync_trace_history_size": "4096", "rgw_sync_trace_per_node_log_size": "32", "rgw_sync_trace_servicemap_update_interval": "10", "rgw_thread_pool_size": "512", "rgw_torrent_comment": "", "rgw_torrent_createby": "", "rgw_torrent_encoding": "", "rgw_torrent_flag": "false", "rgw_torrent_origin": "", "rgw_torrent_sha_unit": "524288", "rgw_torrent_tracker": "", "rgw_trust_forwarded_https": "false", "rgw_usage_log_flush_threshold": "1024", "rgw_usage_log_tick_interval": "30", "rgw_usage_max_shards": "32", "rgw_usage_max_user_shards": "1", "rgw_use_opa_authz": "false", "rgw_user_default_quota_max_objects": "-1", "rgw_user_default_quota_max_size": "-1", "rgw_user_max_buckets": "1000", "rgw_user_quota_bucket_sync_interval": "180", "rgw_user_quota_sync_idle_users": "false", "rgw_user_quota_sync_interval": "86400", "rgw_user_quota_sync_wait_time": "86400", "rgw_user_unique_email": "true", "rgw_verify_ssl": "true", "rgw_website_routing_rules_max_num": "50", "rgw_zone": "", "rgw_zone_root_pool": ".rgw.root", "rgw_zonegroup": "", "rgw_zonegroup_root_pool": ".rgw.root", "rocksdb_block_size": "4096", "rocksdb_bloom_bits_per_key": "20", "rocksdb_cache_index_and_filter_blocks": "true", "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", "rocksdb_cache_row_ratio": "0.000000", "rocksdb_cache_shard_bits": "4", "rocksdb_cache_size": "536870912", "rocksdb_cache_type": "binned_lru", "rocksdb_collect_compaction_stats": "false", "rocksdb_collect_extended_stats": "false", "rocksdb_collect_memory_stats": "false", "rocksdb_delete_range_threshold": "1048576", "rocksdb_index_type": "binary_search", "rocksdb_log_to_ceph_log": "true", "rocksdb_metadata_block_size": "4096", "rocksdb_partition_filters": "false", "rocksdb_perf": "false", "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", "rotating_keys_bootstrap_timeout": "30", "rotating_keys_renewal_timeout": "10", "run_dir": "/var/run/ceph", "setgroup": "ceph", "setuser": "ceph", "setuser_match_path": "", "target_max_misplaced_ratio": "0.050000", "thp": "false", "threadpool_default_timeout": "60", "threadpool_empty_queue_max_wait": "2", "throttler_perf_counter": "true" }
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd set noout ####重启之前，要提前设置ceph集群不要将OSD标记为out，避免node节点关闭服务后被踢出ceph集群外，关闭服务前设置 noout
noout is set
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster: id: 8dc32c41-121c-49df-9554-dfb7deb8c975 health: HEALTH_WARN noout flag(s) set services: mon: 3 daemons, quorum ceph-mon1,ceph-mon2,ceph-mon3 (age 4h) mgr: ceph-mgr1(active, since 4h), standbys: ceph-mgr2 mds: 1/1 daemons up osd: 9 osds: 9 up (since 4h), 9 in (since 2d) flags noout rgw: 1 daemon active (1 hosts, 1 zones) data: volumes: 1/1 healthy pools: 8 pools, 265 pgs objects: 231 objects, 11 MiB usage: 253 MiB used, 900 GiB / 900 GiB avail pgs: 265 active+clean
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd unset noout ##故障恢复，机器即将投入使用，服务已经正常启动，在取消noout
noout is unset
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster: id: 8dc32c41-121c-49df-9554-dfb7deb8c975 health: HEALTH_OK services: mon: 3 daemons, quorum ceph-mon1,ceph-mon2,ceph-mon3 (age 5h) mgr: ceph-mgr1(active, since 5h), standbys: ceph-mgr2 mds: 1/1 daemons up osd: 9 osds: 9 up (since 5h), 9 in (since 2d) rgw: 1 daemon active (1 hosts, 1 zones) data: volumes: 1/1 healthy pools: 8 pools, 265 pgs objects: 231 objects, 11 MiB usage: 254 MiB used, 900 GiB / 900 GiB avail pgs: 265 active+clean
ceph机器关机重启维护顺序
关闭顺序:
#关闭服务前设置 noout
关闭存储客户端停止读写数据
如果使用了 RGW， 关闭 RGW
关闭 cephfs 元数据服务
关闭 ceph OSD
关闭 ceph manager
关闭 ceph monitor
启动顺序:
启动 ceph monitor
启动 ceph manager
启动 ceph OSD
关闭 cephfs 元数据服务
启动 RGW
启动存储客户端
#启动服务后取消 noout-->ceph osd unset noout
添加ceph节点顺序
1.先添加仓库源 apt install -y apt-transport-https ca-certificates curl software-properties-common wget -q -O- 'https://mirrors.tuna.tsinghua.edu.cn/ceph/keys/release.asc' | apt-key add - apt-add-repository 'deb https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific/ focal main' 2.对新增node节点初始化
ceph-deploy install --release pacific ceph-nodex 3.擦除磁盘 ceph-deploy disk zap ceph-nodex /dev/sdx 4.添加 osd： sudo ceph-deploy osd create ceph-nodex --data /dev/sdx
删除故障机osd与服务器
把故障的 OSD 从 ceph 集群删除 1 把 osd 踢出集群 ceph osd out 1 2.等一段时间 3.停止 osd.x 进程 systemctl status ceph-osd@X.service 4.删除 osd ceph osd rm 1 停止服务器之前要把服务器的 OSD 先停止并从 ceph 集群删除 1 把 osd 踢出集群 ceph osd out 1 2.等一段时间 3.停止 osd.x 进程 systemctl status ceph-osd@X.service 4.删除 osd ceph osd rm 1 5.当前主机的其它磁盘重复以上操作 6.OSD 全部操作完成后下线主机 7.从 crush 删除故障节点 ceph osd crush rm ceph-nodeX
