參考官方文檔：http://docs.ceph.org.cn/rados/operations/add-or-rm-osds/
一、卸載分區
1、殺掉要刪除的osd進程
ps -ef|grep ceph
#殺掉osd.8的進程號
kill -9 9293 （進程號是隨機數，會變）
2、卸載分區
umount -f /var/lib/ceph/osd/ceph-8
3、注釋
vi /etc/fstab
#/dev/sdb1 /var/lib/ceph/osd/ceph-8 xfs defaults 0 0
4、重啟系統（reboot）
5、刪除raid0陣列，再清除緩存，保存退出。關閉機器，斷開電源
6、更換硬盤，重新創建raid0陣列
二、刪除osd
ceph osd crush remove osd.8
ceph auth del osd.8
ceph osd rm osd.8
三、創建osd.8
啟動mon3
ceph-mon --id=mon3
#啟動osd
ceph-osd --id=6
ceph-osd --id=7
#創建
ceph osd tree
#分區
fdisk /dev/sdb
partprobe
#創建osd
ceph osd create
#查看目錄是否存在
ls -ld /var/lib/ceph/osd/ceph-8
#格式化
mkfs.xfs /dev/sdb1 -f
#設置開機自啟動
vi /etc/fstab
/dev/sdb1 /var/lib/ceph/osd/ceph-8 xfs defaults 0 0
#掛載
mount -a
#查看硬盤
[root@controller3 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
devtmpfs 16G 0 16G 0% /dev
tmpfs 16G 39M 16G 1% /dev/shm
tmpfs 16G 1.6M 16G 1% /run
tmpfs 16G 0 16G 0% /sys/fs/cgroup
/dev/mapper/fedora-root 100G 11G 90G 11% /
tmpfs 16G 0 16G 0% /tmp
/dev/sda2 976M 135M 775M 15% /boot
/dev/sda6 454G 496M 454G 1% /mnt
/dev/sda4 2.0T 54G 2.0T 3% /var/lib/ceph/osd/ceph-6
/dev/sda5 2.0T 3.1G 2.0T 1% /var/lib/ceph/osd/ceph-7
tmpfs 3.2G 0 3.2G 0% /run/user/0
/dev/sdb1 447G 46G 401G 11% /var/lib/ceph/osd/ceph-8
#授權
chown -R ceph:ceph /var/lib/ceph/osd/ceph-8
sudo -u ceph ceph-osd -i 8 --mkfs --mkkey
ceph auth add osd.8 osd 'allow *' mon 'allow rwx' -i /var/lib/ceph/osd/ceph-8/keyring
ceph osd crush add osd.8 1.0 room=frontpoint
ceph-osd --id=8
#
[root@controller3 ~]# ceph osd tree
ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
-7 3.00000 root ssd
-3 3.00000 room frontpoint
2 1.00000 osd.2 up 1.00000 1.00000
5 1.00000 osd.5 up 1.00000 1.00000
8 1.00000 osd.8 up 1.00000 1.00000
-6 3.00000 root hdd
-2 3.00000 room backpoint
0 1.00000 osd.0 up 1.00000 1.00000
3 1.00000 osd.3 up 1.00000 1.00000
6 1.00000 osd.6 up 1.00000 1.00000
-5 3.00000 root data
-4 3.00000 room datapoint
1 1.00000 osd.1 up 1.00000 1.00000
4 1.00000 osd.4 up 1.00000 1.00000
7 1.00000 osd.7 up 1.00000 1.00000
#
[root@controller3 ~]# ceph -s
cluster c03e4d0e-7955-4325-bfed-f2d990cf94da
health HEALTH_WARN
52 pgs backfill_wait
1 pgs backfilling
53 pgs degraded
54 pgs stuck unclean
54 pgs undersized
recovery 11607/143439 objects degraded (8.092%)
recovery 11321/143439 objects misplaced (7.893%)
monmap e3: 3 mons at {mon1=10.0.0.21:6789/0,mon2=10.0.0.22:6789/0,mon3=10.0.0.23:6789/0}
election epoch 40, quorum 0,1,2 mon1,mon2,mon3
osdmap e1012: 9 osds: 9 up, 9 in; 53 remapped pgs
flags sortbitwise,require_jewel_osds
pgmap v184556: 768 pgs, 6 pools, 170 GB data, 47813 objects
388 GB used, 13233 GB / 13621 GB avail
11607/143439 objects degraded (8.092%)
11321/143439 objects misplaced (7.893%)
714 active+clean
52 active+undersized+degraded+remapped+wait_backfill
1 active+undersized+remapped
1 active+undersized+degraded+remapped+backfilling
recovery io 247 MB/s, 79 objects/s
#寫入數據
[root@controller3 ~]# ceph -w
cluster c03e4d0e-7955-4325-bfed-f2d990cf94da
health HEALTH_WARN
47 pgs backfill_wait
2 pgs backfilling
49 pgs degraded
49 pgs stuck unclean
49 pgs undersized
recovery 10545/143439 objects degraded (7.352%)
recovery 10372/143439 objects misplaced (7.231%)
monmap e3: 3 mons at {mon1=10.0.0.21:6789/0,mon2=10.0.0.22:6789/0,mon3=10.0.0.23:6789/0}
election epoch 40, quorum 0,1,2 mon1,mon2,mon3
osdmap e1021: 9 osds: 9 up, 9 in; 48 remapped pgs
flags sortbitwise,require_jewel_osds
pgmap v184601: 768 pgs, 6 pools, 170 GB data, 47813 objects
391 GB used, 13229 GB / 13621 GB avail
10545/143439 objects degraded (7.352%)
10372/143439 objects misplaced (7.231%)
719 active+clean
47 active+undersized+degraded+remapped+wait_backfill
2 active+undersized+degraded+remapped+backfilling
recovery io 7960 kB/s, 3 objects/s
2018-03-20 20:09:51.472749 mon.0 [INF] pgmap v184599: 768 pgs: 1 active+undersized+degraded+remapped+backfilling, 48 active+undersized+degraded+remapped+wait_backfill, 719 active+clean; 170 GB data, 391 GB used, 13229 GB / 13621 GB avail; 10545/143439 objects degraded (7.352%); 10373/143439 objects misplaced (7.232%); 269 MB/s, 86 objects/s recovering
2018-03-20 20:09:52.486310 mon.0 [INF] pgmap v184600: 768 pgs: 2 active+undersized+degraded+remapped+backfilling, 47 active+undersized+degraded+remapped+wait_backfill, 719 active+clean; 170 GB data, 391 GB used, 13229 GB / 13621 GB avail; 10545/143439 objects degraded (7.352%); 10372/143439 objects misplaced (7.231%); 2684 kB/s, 1 objects/s recovering
2018-03-20 20:09:52.499320 mon.0 [INF] osdmap e1021: 9 osds: 9 up, 9 in
2018-03-20 20:09:52.501537 mon.0 [INF] pgmap v184601: 768 pgs: 2 active+undersized+degraded+remapped+backfilling, 47 active+undersized+degraded+remapped+wait_backfill, 719 active+clean; 170 GB data, 391 GB used, 13229 GB / 13621 GB avail; 10545/143439 objects degraded (7.352%); 10372/143439 objects misplaced (7.231%); 7960 kB/s, 3 objects/s recovering
2018-03-20 20:09:53.505738 mon.0 [INF] pgmap v184602: 768 pgs: 2 active+undersized+degraded+remapped+backfilling, 47 active+undersized+degraded+remapped+wait_backfill, 719 active+clean; 170 GB data, 391 GB used, 13229 GB / 13621 GB avail; 10545/143439 objects degraded (7.352%); 10372/143439 objects misplaced (7.231%)
2018-03-20 20:09:53.510548 mon.0 [INF] osdmap e1022: 9 osds: 9 up, 9 in
2018-03-20 20:09:53.512510 mon.0 [INF] pgmap v184603: 768 pgs: 2 active+undersized+degraded+remapped+backfilling, 47 active+undersized+degraded+remapped+wait_backfill, 719 active+clean; 170 GB data, 391 GB used, 13229 GB / 13621 GB avail; 10545/143439 objects degraded (7.352%); 10372/143439 objects misplaced (7.231%)
2018-03-20 20:09:54.514829 mon.0 [INF] pgmap v184604: 768 pgs: 2 active+undersized+degraded+remapped+backfilling, 47 active+undersized+degraded+remapped+wait_backfill, 719 active+clean; 170 GB data, 391 GB used, 13229 GB / 13621 GB avail; 10545/143439 objects degraded (7.352%); 10372/143439 objects misplaced (7.231%)
#備注
一、 刪除 OSD
此步驟依次把一個 OSD 移出集群 CRUSH 圖、刪除認證密鑰、刪除 OSD 圖條目、刪除 ceph.conf 條目。如果主機有多個硬盤，每個硬盤對應的 OSD 都得重復此步驟。
刪除 CRUSH 圖的對應 OSD 條目，它就不再接收數據了。你也可以反編譯 CRUSH 圖、刪除 device 列表條目、刪除對應的 host 桶條目或刪除 host 桶（如果它在 CRUSH 圖里，而且你想刪除主機），重編譯 CRUSH 圖並應用它。詳情參見刪除 OSD 。
ceph osd crush remove {name}
刪除 OSD 認證密鑰：
ceph auth del osd.{osd-num}
ceph-{osd-num} 路徑里的 ceph 值是 $cluster-$id ，如果集群名字不是 ceph ，這里要更改。
刪除 OSD 。
ceph osd rm {osd-num}
#for example
ceph osd rm 1