ceph 删除osd操作方法
参考官方文档：http://docs.ceph.org.cn/rados/operations/add-or-rm-osds/
一、卸载分区
1、杀掉要删除的osd进程
ps -ef|grep ceph
#杀掉osd.8的进程号
kill -9 9293 （进程号是随机数，会变）
2、卸载分区
umount -f /var/lib/ceph/osd/ceph-8
3、注释
vi /etc/fstab
#/dev/sdb1 /var/lib/ceph/osd/ceph-8 xfs defaults 0 0
4、重启系统（reboot）
5、删除raid0阵列，再清除缓存，保存退出。关闭机器，断开电源
6、更换硬盘，重新创建raid0阵列
二、删除osd
ceph osd crush remove osd.8
ceph auth del osd.8
ceph osd rm osd.8
三、创建osd.8
启动mon3
ceph-mon --id=mon3
#启动osd
ceph-osd --id=6
ceph-osd --id=7
#创建
ceph osd tree
#分区
fdisk /dev/sdb
partprobe
#创建osd
ceph osd create
#查看目录是否存在
ls -ld /var/lib/ceph/osd/ceph-8
#格式化
mkfs.xfs /dev/sdb1 -f
#设置开机自启动
vi /etc/fstab
/dev/sdb1 /var/lib/ceph/osd/ceph-8 xfs defaults 0 0
#挂载
mount -a
#查看硬盘
[root@controller3 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
devtmpfs 16G 0 16G 0% /dev
tmpfs 16G 39M 16G 1% /dev/shm
tmpfs 16G 1.6M 16G 1% /run
tmpfs 16G 0 16G 0% /sys/fs/cgroup
/dev/mapper/fedora-root 100G 11G 90G 11% /
tmpfs 16G 0 16G 0% /tmp
/dev/sda2 976M 135M 775M 15% /boot
/dev/sda6 454G 496M 454G 1% /mnt
/dev/sda4 2.0T 54G 2.0T 3% /var/lib/ceph/osd/ceph-6
/dev/sda5 2.0T 3.1G 2.0T 1% /var/lib/ceph/osd/ceph-7
tmpfs 3.2G 0 3.2G 0% /run/user/0
/dev/sdb1 447G 46G 401G 11% /var/lib/ceph/osd/ceph-8
#授权
chown -R ceph:ceph /var/lib/ceph/osd/ceph-8
sudo -u ceph ceph-osd -i 8 --mkfs --mkkey
ceph auth add osd.8 osd 'allow *' mon 'allow rwx' -i /var/lib/ceph/osd/ceph-8/keyring
ceph osd crush add osd.8 1.0 room=frontpoint
ceph-osd --id=8
#
[root@controller3 ~]# ceph osd tree
ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
-7 3.00000 root ssd
-3 3.00000 room frontpoint
2 1.00000 osd.2 up 1.00000 1.00000
5 1.00000 osd.5 up 1.00000 1.00000
8 1.00000 osd.8 up 1.00000 1.00000
-6 3.00000 root hdd
-2 3.00000 room backpoint
0 1.00000 osd.0 up 1.00000 1.00000
3 1.00000 osd.3 up 1.00000 1.00000
6 1.00000 osd.6 up 1.00000 1.00000
-5 3.00000 root data
-4 3.00000 room datapoint
1 1.00000 osd.1 up 1.00000 1.00000
4 1.00000 osd.4 up 1.00000 1.00000
7 1.00000 osd.7 up 1.00000 1.00000
#
[root@controller3 ~]# ceph -s
cluster c03e4d0e-7955-4325-bfed-f2d990cf94da
health HEALTH_WARN
52 pgs backfill_wait
1 pgs backfilling
53 pgs degraded
54 pgs stuck unclean
54 pgs undersized
recovery 11607/143439 objects degraded (8.092%)
recovery 11321/143439 objects misplaced (7.893%)
monmap e3: 3 mons at {mon1=10.0.0.21:6789/0,mon2=10.0.0.22:6789/0,mon3=10.0.0.23:6789/0}
election epoch 40, quorum 0,1,2 mon1,mon2,mon3
osdmap e1012: 9 osds: 9 up, 9 in; 53 remapped pgs
flags sortbitwise,require_jewel_osds
pgmap v184556: 768 pgs, 6 pools, 170 GB data, 47813 objects
388 GB used, 13233 GB / 13621 GB avail
11607/143439 objects degraded (8.092%)
11321/143439 objects misplaced (7.893%)
714 active+clean
52 active+undersized+degraded+remapped+wait_backfill
1 active+undersized+remapped
1 active+undersized+degraded+remapped+backfilling
recovery io 247 MB/s, 79 objects/s
#写入数据
[root@controller3 ~]# ceph -w
cluster c03e4d0e-7955-4325-bfed-f2d990cf94da
health HEALTH_WARN
47 pgs backfill_wait
2 pgs backfilling
49 pgs degraded
49 pgs stuck unclean
49 pgs undersized
recovery 10545/143439 objects degraded (7.352%)
recovery 10372/143439 objects misplaced (7.231%)
monmap e3: 3 mons at {mon1=10.0.0.21:6789/0,mon2=10.0.0.22:6789/0,mon3=10.0.0.23:6789/0}
election epoch 40, quorum 0,1,2 mon1,mon2,mon3
osdmap e1021: 9 osds: 9 up, 9 in; 48 remapped pgs
flags sortbitwise,require_jewel_osds
pgmap v184601: 768 pgs, 6 pools, 170 GB data, 47813 objects
391 GB used, 13229 GB / 13621 GB avail
10545/143439 objects degraded (7.352%)
10372/143439 objects misplaced (7.231%)
719 active+clean
47 active+undersized+degraded+remapped+wait_backfill
2 active+undersized+degraded+remapped+backfilling
recovery io 7960 kB/s, 3 objects/s
2018-03-20 20:09:51.472749 mon.0 [INF] pgmap v184599: 768 pgs: 1 active+undersized+degraded+remapped+backfilling, 48 active+undersized+degraded+remapped+wait_backfill, 719 active+clean; 170 GB data, 391 GB used, 13229 GB / 13621 GB avail; 10545/143439 objects degraded (7.352%); 10373/143439 objects misplaced (7.232%); 269 MB/s, 86 objects/s recovering
2018-03-20 20:09:52.486310 mon.0 [INF] pgmap v184600: 768 pgs: 2 active+undersized+degraded+remapped+backfilling, 47 active+undersized+degraded+remapped+wait_backfill, 719 active+clean; 170 GB data, 391 GB used, 13229 GB / 13621 GB avail; 10545/143439 objects degraded (7.352%); 10372/143439 objects misplaced (7.231%); 2684 kB/s, 1 objects/s recovering
2018-03-20 20:09:52.499320 mon.0 [INF] osdmap e1021: 9 osds: 9 up, 9 in
2018-03-20 20:09:52.501537 mon.0 [INF] pgmap v184601: 768 pgs: 2 active+undersized+degraded+remapped+backfilling, 47 active+undersized+degraded+remapped+wait_backfill, 719 active+clean; 170 GB data, 391 GB used, 13229 GB / 13621 GB avail; 10545/143439 objects degraded (7.352%); 10372/143439 objects misplaced (7.231%); 7960 kB/s, 3 objects/s recovering
2018-03-20 20:09:53.505738 mon.0 [INF] pgmap v184602: 768 pgs: 2 active+undersized+degraded+remapped+backfilling, 47 active+undersized+degraded+remapped+wait_backfill, 719 active+clean; 170 GB data, 391 GB used, 13229 GB / 13621 GB avail; 10545/143439 objects degraded (7.352%); 10372/143439 objects misplaced (7.231%)
2018-03-20 20:09:53.510548 mon.0 [INF] osdmap e1022: 9 osds: 9 up, 9 in
2018-03-20 20:09:53.512510 mon.0 [INF] pgmap v184603: 768 pgs: 2 active+undersized+degraded+remapped+backfilling, 47 active+undersized+degraded+remapped+wait_backfill, 719 active+clean; 170 GB data, 391 GB used, 13229 GB / 13621 GB avail; 10545/143439 objects degraded (7.352%); 10372/143439 objects misplaced (7.231%)
2018-03-20 20:09:54.514829 mon.0 [INF] pgmap v184604: 768 pgs: 2 active+undersized+degraded+remapped+backfilling, 47 active+undersized+degraded+remapped+wait_backfill, 719 active+clean; 170 GB data, 391 GB used, 13229 GB / 13621 GB avail; 10545/143439 objects degraded (7.352%); 10372/143439 objects misplaced (7.231%)
#备注
一、 删除 OSD
此步骤依次把一个 OSD 移出集群 CRUSH 图、删除认证密钥、删除 OSD 图条目、删除 ceph.conf 条目。如果主机有多个硬盘，每个硬盘对应的 OSD 都得重复此步骤。
删除 CRUSH 图的对应 OSD 条目，它就不再接收数据了。你也可以反编译 CRUSH 图、删除 device 列表条目、删除对应的 host 桶条目或删除 host 桶（如果它在 CRUSH 图里，而且你想删除主机），重编译 CRUSH 图并应用它。详情参见删除 OSD 。
ceph osd crush remove {name}
删除 OSD 认证密钥：
ceph auth del osd.{osd-num}
ceph-{osd-num} 路径里的 ceph 值是 $cluster-$id ，如果集群名字不是 ceph ，这里要更改。
删除 OSD 。
ceph osd rm {osd-num}
#for example
ceph osd rm 1
