# 要用root用戶創建，或有sudo權限
# 注：docker鏡像源(鏡像加速)：https://registry.docker-cn.com
# 1、修改docker鏡像源
cat > /etc/docker/deamon.json << EOF
{
"registry-mirrors": [
"https://registry.docker-cn.com"
]
}
EOF
# 需要用到的鏡像：
docker pull ceph/daemon
ceph/mon、ceph/osd…等一個一個鏡像拉取是比較舊的方法
# 重啟docker
systemctl restart docker
# 2、創建ceph專用網絡
docker network create --driver bridge --subnet 172.20.0.0/16 ceph-network
docker network inspect ceph-network
# 3、刪除舊的ceph相關容器
docker rm -f $(docker ps -a | grep ceph | awk '{print $1}')
# 4、清理舊的ceph相關目錄文件，
rm -rf /www/ceph /var/lib/ceph /www/osd
# 5、創建相關目錄及修改權限，用於掛載volume
mkdir -p /www/ceph /var/lib/ceph/osd /www/osd /var/log/ceph
chown -R 64045:64045 /var/lib/ceph/osd
chown -R 64045:64045 /www/osd/
# 6、創建monitor節點
docker run -itd --name monnode --network ceph-network --ip 172.20.0.10 -e NON_NAME=monnode -e MON_IP=172.20.0.10 -v /www/ceph:/etc/ceph ceph/mon
例子：
docker run -itd --name monnode --network ceph-network --ip 172.20.0.10 --restart always -v /etc/ceph:/etc/ceph -v /var/lib/ceph/:/var/lib/ceph/ -v /var/log/ceph/:/var/log/ceph/ -e CLUSTER=ceph -e WEIGHT=1.0 -e MON_IP=172.20.0.10 -e MON_NAME=monnode -e CEPH_PUBLIC_NETWORK=172.20.0.0/16 ceph/daemon mon
實際：
docker run -itd --name monnode --network ceph-network --ip 172.20.0.10 -e CLUSTER=ceph -e WEIGHT=1.0 -e MON_NAME=monnode -e MON_IP=172.20.0.10 -v /www/ceph:/etc/ceph -v /var/lib/ceph/:/var/lib/ceph/ -v /var/log/ceph/:/var/log/ceph/ -e CEPH_PUBLIC_NETWORK=172.20.0.0/16 ceph/daemon mon
–ip：設置的是容器的IP
ceph/daemon mon：中的mon是參數，不可隨意更改
# 7、在monitor節點上標識3個osd節點
docker exec monnode ceph osd create
docker exec monnode ceph osd create
docker exec monnode ceph osd create
注意：實際操作過程中這一步沒有執行。
# 8、創建osd節點
docker run -itd --name osdnode0 --network ceph-network -e CLUSTER=ceph -e WEIGHT=1.0 MON_NAME=monnode -e MON_IP=172.20.0.10 -v /www/ceph:/etc/ceph -v /www/osd0:/var/lib/ceph/osd/ceph-0 ceph/osd
docker run -itd --name osdnode1 --network ceph-network -e CLUSTER=ceph -e WEIGHT=1.0 MON_NAME=monnode -e MON_IP=172.20.0.10 -v /www/ceph:/etc/ceph -v /www/osd1:/var/lib/ceph/osd/ceph-1 ceph/osd
docker run -itd --name osdnode2 --network ceph-network -e CLUSTER=ceph -e WEIGHT=1.0 MON_NAME=monnode -e MON_IP=172.20.0.10 -v /www/ceph:/etc/ceph -v /www/osd2:/var/lib/ceph/osd/ceph-2 ceph/osd
新：
創建OSD密鑰文件：
docker exec monnode ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring
修改目錄權限：
chown 167:167 -R /www/ceph
chmod 777 -R /www/ceph
執行部署osd操作：
docker run -itd --privileged=true --name osdnode0 --network ceph-network -e CLUSTER=ceph -e WEIGHT=1.0 -e MON_NAME=monnode -e MON_IP=172.20.0.10 -v /www/ceph:/etc/ceph -v /var/lib/ceph/:/var/lib/ceph/ -v /var/lib/ceph/osd/0:/var/lib/ceph/osd -e OSD_TYPE=directory -v /etc/localtime:/etc/localtime:ro ceph/daemon osd
docker run -itd --privileged=true --name osdnode1 --network ceph-network -e CLUSTER=ceph -e WEIGHT=1.0 -e MON_NAME=monnode -e MON_IP=172.20.0.10 -v /www/ceph:/etc/ceph -v /var/lib/ceph/:/var/lib/ceph/ -v /var/lib/ceph/osd/1:/var/lib/ceph/osd -e OSD_TYPE=directory -v /etc/localtime:/etc/localtime:ro ceph/daemon osd
docker run -itd --privileged=true --name osdnode2 --network ceph-network -e CLUSTER=ceph -e WEIGHT=1.0 -e MON_NAME=monnode -e MON_IP=172.20.0.10 -v /www/ceph:/etc/ceph -v /var/lib/ceph/:/var/lib/ceph/ -v /var/lib/ceph/osd/2:/var/lib/ceph/osd -e OSD_TYPE=directory -v /etc/localtime:/etc/localtime:ro ceph/daemon osd
查看日志：
root@ybelzjlpa9xmngcb-0315223:~/ceph_cluster# docker logs osdnode0
2021-09-03 14:58:55 /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2021-09-03 14:58:55 /opt/ceph-container/bin/entrypoint.sh: Creating osd
2021-09-03T14:58:55.246+0800 7f2386b2c700 -1 auth: unable to find a keyring on /var/lib/ceph/bootstrap-osd/ceph.keyring: (2) No such file or directory
2021-09-03T14:58:55.246+0800 7f2386b2c700 -1 AuthRegistry(0x7f238005b458) no keyring found at /var/lib/ceph/bootstrap-osd/ceph.keyring, disabling cephx
2021-09-03T14:58:55.246+0800 7f2387d8e700 -1 auth: unable to find a keyring on /var/lib/ceph/bootstrap-osd/ceph.keyring: (2) No such file or directory
2021-09-03T14:58:55.246+0800 7f2387d8e700 -1 AuthRegistry(0x7f238005ef00) no keyring found at /var/lib/ceph/bootstrap-osd/ceph.keyring, disabling cephx
2021-09-03T14:58:55.246+0800 7f2387d8e700 -1 auth: unable to find a keyring on /var/lib/ceph/bootstrap-osd/ceph.keyring: (2) No such file or directory
2021-09-03T14:58:55.246+0800 7f2387d8e700 -1 AuthRegistry(0x7f2387d8cea0) no keyring found at /var/lib/ceph/bootstrap-osd/ceph.keyring, disabling cephx
2021-09-03T14:58:55.246+0800 7f2385329700 -1 monclient(hunting): handle_auth_bad_method server allowed_methods [2] but i only support [1]
2021-09-03T14:58:55.250+0800 7f2387d8e700 -1 monclient: authenticate NOTE: no keyring found; disabled cephx authentication
[errno 95] error connecting to the cluster
root@ybelzjlpa9xmngcb-0315223:~/ceph_cluster#
如果出現這種情況的話，檢查是否執行了上面的有關目錄權限的問題。
# 9、創建mgr節點
docker run -itd --privileged=true --name mgrnode --network ceph-network -e CLUSTER=ceph --pid=container:monnode -v /www/ceph:/etc/ceph -v /var/lib/ceph/:/var/lib/ceph/ ceph/daemon mgr
需要注意的參數：–pid
# 10、啟動rgw服務
在mon節點生成rgw密鑰信息
docker exec monnode ceph auth get client.bootstrap-rgw -o /var/lib/ceph/bootstrap-rgw/ceph.keyring
docker run -itd --privileged=true --name rgwnode --network ceph-network -e CLUSTER=ceph -v /var/lib/ceph/:/var/lib/ceph/ -v /www/ceph:/etc/ceph -v /etc/localtime:/etc/localtime:ro -e RGW_NAME=rgw0 ceph/daemon rgw
# 10、啟動mds服務
docker run -d \
--net=host \
--name=mds \
--privileged=true \
-v /etc/localtime:/etc/localtime \
-v /usr/local/ceph/etc:/etc/ceph \
-v /usr/local/ceph/lib:/var/lib/ceph \
-v /usr/local/ceph/logs:/var/log/ceph \
-e CEPHFS_CREATE=0 \
-e CEPHFS_METADATA_POOL_PG=512 \
-e CEPHFS_DATA_POOL_PG=512 \
ceph/daemon mds
說明下：
CEPHFS_CREATE 是為METADATA服務生成文件系統， 0表示不自動創建文件系統（默認值）， 1表示自動創建。
CEPHFS_DATA_POOL_PG是數據池的數量，默認為8。
CEPHFS_METADATA_POOL_PG是元數據池的數量，默認為8。
實際執行：
docker run -d --name=mds0 --privileged=true --network ceph-network -v /etc/localtime:/etc/localtime -v /www/ceph:/etc/ceph -v /var/lib/ceph:/var/lib/ceph -v /var/log/ceph:/var/log/ceph -e CEPHFS_CREATE=0 -e CEPHFS_METADATA_POOL_PG=16 -e CEPHFS_DATA_POOL_PG=16 ceph/daemon mds
# 11、創建FS文件系統
創建Data Pool:
docker exec osdnode0 ceph osd pool create cephfs_data 128 128
創建Metadata Pool:
docker exec osdnode0 ceph osd pool create cephfs_metadata 64 64
注意： 如果受mon_max_pg_per_osd限制， 不能設為128，可以調小點， 改為64。
當執行上面64的時候出錯：
Error ERANGE: pg_num 64 size 3 would mean 888 total pgs, which exceeds max 750 (mon_max_pg_per_osd 250 * num_in_osds 3)
改為：
docker exec osdnode0 ceph osd pool create cephfs_metadata 32 32
創建CephFS:
docker exec osdnode0 ceph fs new cephfs cephfs_metadata cephfs_data
上面的數據池與元數據池關聯， 創建cephfs的文件系統。
查看FS信息:
docker exec osdnode0 ceph fs ls
輸出：
name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ]
# 12、查看ceph集群狀態
docker exec monnode ceph -s
輸出：
root@ybelzjlpa9xmngcb-0315223:~/ceph_cluster# docker exec monnode ceph -s
cluster:
id: 53e5a82c-f2ab-4e54-a9fb-d14f936cea62
health: HEALTH_WARN
mon is allowing insecure global_id reclaim
Degraded data redundancy: 211/633 objects degraded (33.333%), 53 pgs degraded, 288 pgs undersized
OSD count 2 < osd_pool_default_size 3
too many PGs per OSD (288 > max 250)
services:
mon: 1 daemons, quorum monnode (age 55m)
mgr: 13bb5037212c(active, since 39m)
mds: 1/1 daemons up
osd: 2 osds: 2 up (since 48m), 2 in (since 48m)
rgw: 1 daemon active (1 hosts, 1 zones)
data:
volumes: 1/1 healthy
pools: 6 pools, 288 pgs
objects: 211 objects, 7.2 KiB
usage: 24 MiB used, 200 GiB / 200 GiB avail
pgs: 211/633 objects degraded (33.333%)
235 active+undersized
53 active+undersized+degraded
progress:
Global Recovery Event (34m)
[............................]
# 13、增加monitor節點，組件成機器
docker run -itd --name monnode_1 --network ceph-network --ip 172.20.0.11 -e NON_NAME=monnode_1 -e MON_IP=172.20.0.11 -v /www/ceph:/etc/ceph ceph/mon
docker run -itd --name monnode_2 --network ceph-network --ip 172.20.0.12 -e NON_NAME=monnode_2 -e MON_IP=172.20.0.12 -v /www/ceph:/etc/ceph ceph/mon
# 14、開始使用ceph-fs文件系統
掛載文件系統，支持兩種方式：
a,使用內核驅動掛載
這里我選擇在osdnode0的容器內執行：
mkdir -p /mnt/cephfs
mount -t ceph monnode:6789:/ /mnt/mycephfs
要掛載啟用了 cephx 認證的 Ceph 文件系統，你必須指定用戶名、密鑰。
ceph-authtool --print-key /etc/ceph/ceph.client.admin.keyring
mount -t ceph 172.20.0.10:6789:/ /mnt/mycephfs -o name=admin,secret=AQBfOTdhwfU7DBAA+Q2gOumgEPvcbGO1YqC1BQ==
但是這種方式一直報錯：(目前還沒找到解決方案)
modprobe: FATAL: Module ceph not found in directory /lib/modules/4.4.0-186-generic
failed to load ceph kernel module (1)
mount error 110 = Connection timed out
要卸載 Ceph 文件系統，可以用 unmount 命令，例如：
umount /mnt/mycephfs
b,使用ceph-fuse方式掛載(執行成功)
ceph-fuse -c /etc/ceph/ceph.conf /mnt/mycephfs
卸載:
fusermount -u /mnt/mycephfs