系統版本：buntu 18.04.5 LTS 內核參數：4.15.0-112-generic ceph版本：pacific/16.2.5 主機分配： #部署服務器ceph-deploy 172.168.32.101/10.0.0.101 ceph-deploy #兩個ceph-mgr 管理服務器 172.168.32.102/10.0.0.102 ceph-mgr01 172.168.32.103/10.0.0.103 ceph-mgr02 #三台服務器作為ceph 集群Mon 監視服務器，每台服務器可以和ceph 集群的cluster 網絡通信。 172.168.32.104/10.0.0.104 ceph-mon01 ceph-mds01 172.168.32.105/10.0.0.105 ceph-mon02 ceph-mds02 172.168.32.106/10.0.0.106 ceph-mon03 ceph-mds03 #四台服務器作為ceph 集群OSD 存儲服務器，每台服務器支持兩個網絡，public 網絡針對客戶端訪問，cluster 網絡用於集群管理及數據同步，每台三塊或以上的磁盤 172.168.32.107/10.0.0.107 ceph-node01 172.168.32.108/10.0.0.108 ceph-node02 172.168.32.109/10.0.0.109 ceph-node03 172.168.32.110/10.0.0.110 ceph-node04 #磁盤划分 #/dev/sdb /dev/sdc /dev/sdd /dev/sde #50G
1)所有節點更換為清華源
cat >/etc/apt/source.list<<EOF # 默認注釋了源碼鏡像以提高 apt update 速度，如有需要可自行取消注釋 deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic main restricted universe multiverse # deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic main restricted universe multiverse deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-updates main restricted universe multiverse # deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-updates main restricted universe multiverse deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-backports main restricted universe multiverse # deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-backports main restricted universe multiverse deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-security main restricted universe multiverse # deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-security main restricted universe multiverse EOF
apt install iproute2 ntpdate tcpdump telnet traceroute nfs-kernel-server nfs-common lrzsz tree openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev ntpdate tcpdump telnet traceroute gcc openssh-server lrzsz tree openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev ntpdate tcpdump telnet traceroute iotop unzip zip openjdk-8-jdk -y
cat >/etc/sysctl.conf <<EOF # Controls source route verification net.ipv4.conf.default.rp_filter = 1 net.ipv4.ip_nonlocal_bind = 1 net.ipv4.ip_forward = 1 # Do not accept source routing net.ipv4.conf.default.accept_source_route = 0 # Controls the System Request debugging functionality of the kernel kernel.sysrq = 0 # Controls whether core dumps will append the PID to the core filename. # Useful for debugging multi-threaded applications. kernel.core_uses_pid = 1 # Controls the use of TCP syncookies net.ipv4.tcp_syncookies = 1 # Disable netfilter on bridges. net.bridge.bridge-nf-call-ip6tables = 0 net.bridge.bridge-nf-call-iptables = 0 net.bridge.bridge-nf-call-arptables = 0 # Controls the default maxmimum size of a mesage queue kernel.msgmnb = 65536 # # Controls the maximum size of a message, in bytes kernel.msgmax = 65536 # Controls the maximum shared segment size, in bytes kernel.shmmax = 68719476736 # # Controls the maximum number of shared memory segments, in pages kernel.shmall = 4294967296 # TCP kernel paramater net.ipv4.tcp_mem = 786432 1048576 1572864 net.ipv4.tcp_rmem = 4096 87380 4194304 net.ipv4.tcp_wmem = 4096 16384 4194304 n et.ipv4.tcp_window_scaling = 1 net.ipv4.tcp_sack = 1 # socket buffer net.core.wmem_default = 8388608 net.core.rmem_default = 8388608 net.core.rmem_max = 16777216 net.core.wmem_max = 16777216 net.core.netdev_max_backlog = 262144 net.core.somaxconn = 20480 net.core.optmem_max = 81920 # TCP conn net.ipv4.tcp_max_syn_backlog = 262144 net.ipv4.tcp_syn_retries = 3 net.ipv4.tcp_retries1 = 3 net.ipv4.tcp_retries2 = 15 # tcp conn reuse net.ipv4.tcp_timestamps = 0 net.ipv4.tcp_tw_reuse = 0 net.ipv4.tcp_tw_recycle = 0 net.ipv4.tcp_fin_timeout = 1 net.ipv4.tcp_max_tw_buckets = 20000 net.ipv4.tcp_max_orphans = 3276800 net.ipv4.tcp_synack_retries = 1 net.ipv4.tcp_syncookies = 1 # keepalive conn net.ipv4.tcp_keepalive_time = 300 net.ipv4.tcp_keepalive_intvl = 30 net.ipv4.tcp_keepalive_probes = 3 net.ipv4.ip_local_port_range = 10001 65000 # swap vm.overcommit_memory = 0 vm.swappiness = 10 #net.ipv4.conf.eth1.rp_filter = 0 #net.ipv4.conf.lo.arp_ignore = 1 #net.ipv4.conf.lo.arp_announce = 2 #net.ipv4.conf.all.arp_ignore = 1 #net.ipv4.conf.all.arp_announce = 2 EOF
cat > /etc/security/limits.conf <<EOF root soft core unlimited root hard core unlimited root soft nproc 1000000 root hard nproc 1000000 root soft nofile 1000000 root hard nofile 1000000 root soft memlock 32000 root hard memlock 32000 root soft msgqueue 8192000 root hard msgqueue 8192000 * soft core unlimited * hard core unlimited * soft nproc 1000000 * hard nproc 1000000 * soft nofile 1000000 * hard nofile 1000000 * soft memlock 32000 * hard memlock 32000 * soft msgqueue 8192000 * hard msgqueue 8192000 EOF
#安裝cron並啟動 apt install cron -y systemctl status cron.service #同步時間 /usr/sbin/ntpdate time1.aliyun.com &> /dev/null && hwclock -w #每5分鍾同步一次時間 echo "*/5 * * * * /usr/sbin/ntpdate time1.aliyun.com &> /dev/null && hwclock -w" >> /var/spool/cron/crontabs/root
cat >>/etc/hosts<<EOF
172.168.32.101 ceph-deploy
172.168.32.102 ceph-mgr01
172.168.32.103 ceph-mgr02
172.168.32.104 ceph-mon01 ceph-mds01
172.168.32.105 ceph-mon02 ceph-mds02
172.168.32.106 ceph-mon03 ceph-mds03
172.168.32.107 ceph-node01
172.168.32.108 ceph-node02
172.168.32.109 ceph-node03
172.168.32.110 ceph-node04
EOF
7)所有節點安裝python2
apt install python2.7 -y ln -sv /usr/bin/python2.7 /usr/bin/python2
3、ceph部署
#配置ceph倉庫
sudo echo "deb https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific bionic main" >> /etc/apt/sources.list
#導入key
wget -q -O- 'https://mirrors.tuna.tsinghua.edu.cn/ceph/keys/release.asc' | sudo apt-key add -
#更新倉庫源
apt update
推薦使用指定的普通用戶部署和運行ceph 集群，普通用戶只要能以非交互方式執行命令執行一些特權命令即可，新版的ceph-deploy 可以指定包含root 的在內只要可以執行 命令的用戶，不過仍然推薦使用普通用戶，比如ceph、cephuser、cephadmin 這樣的用戶去管理ceph 集群。
#因為前面安裝的ceph-common會更改ceph用戶的家目錄，建議使用其它用戶來部署，如cephadmin用戶
groupadd -r -g 2021 cephadmin && useradd -r -m -s /bin/bash -u 2021 -g 2021 cephadmin && echo cephadmin:123456 | chpasswd
#允許ceph 用戶以 執行特權命令
echo "cephadmin ALL=(ALL) NOPASSWD:ALL" >> /etc/ers
在ceph-deploy 節點配置允許以非交互的方式登錄到各ceph node/mon/mgr 節點，即在ceph-deploy 節點的ceph用戶生成秘鑰對，然后分發公鑰到各被管理節點的ceph用戶。
#(1)創建ssh密鑰
cephadmin@ceph-deploy:/tmp$ ssh-keygen
Generating public/private rsa key pair.
Enter file in which to save the key (/home/ceph/.ssh/id_rsa):
Created directory '/home/ceph/.ssh'.
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /home/ceph/.ssh/id_rsa.
Your public key has been saved in /home/ceph/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:Oq0Vh0Do/VUVklh3U58XNgNNkIfCIPAiXFw+ztEhbqM cephadmin@ceph-deploy
The key's randomart image is:
+---[RSA 2048]----+
| .+++ oooo=@O+|
| . oo+ + ooo=.+B|
| + o.O . .. ..o|
| o B.+.. .|
| E +S.. |
| o.o |
| o o |
| + |
| . |
+----[SHA256]-----+
#(2)安裝sshpass
cephadmin@ceph-deploy:/tmp$ sudo apt install sshpass
#(3)ceph-deploy節點使用ceph用戶分發密鑰腳本
cat >>/tmp/ssh_fenfa.sh<<EOF
#!/bin/bash
#目標主機列表
IP="
172.168.32.101
172.168.32.102
172.168.32.103
172.168.32.104
172.168.32.105
172.168.32.106
172.168.32.107
172.168.32.108
172.168.32.109
172.168.32.110"
for node in ${IP};do
sshpass -p 123456 ssh-copy-id cephadmin@${node} -o StrictHostKeyChecking=no &> /dev/null
if [ $? -eq 0 ];then
echo "${node}----> 密鑰分發success完成"
else
echo "${node}----> 密鑰分發false失敗"
fi
done
EOF
#(4)使用腳本分發ssh密鑰
cephadmin@ceph-deploy:/tmp$ bash ssh_fenfa.sh
172.168.32.101----> 密鑰分發success完成
172.168.32.102----> 密鑰分發success完成
172.168.32.103----> 密鑰分發success完成
172.168.32.104----> 密鑰分發success完成
172.168.32.105----> 密鑰分發success完成
172.168.32.106----> 密鑰分發success完成
172.168.32.107----> 密鑰分發success完成
172.168.32.108----> 密鑰分發success完成
172.168.32.109----> 密鑰分發success完成
172.168.32.110----> 密鑰分發success完成
cephadmin@ceph-deploy:~# sudo apt-cache madison ceph-deploy ceph-deploy | 2.0.1 | https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific bionic/main amd64 Packages ceph-deploy | 2.0.1 | https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific bionic/main i386 Packages ceph-deploy | 1.5.38-0ubuntu1 | https://mirrors.tuna.tsinghua.edu.cn/ubuntu bionic/universe amd64 Packages ceph-deploy | 1.5.38-0ubuntu1 | https://mirrors.tuna.tsinghua.edu.cn/ubuntu bionic/universe i386 Packages cephadmin@ceph-deploy:~# sudo apt install ceph-deploy
在管理節點初始化mon 節點
cephadmin@ceph-deploy:~$ mkdir ceph-cluster #保存當前集群的初始化配置信息
cephadmin@ceph-deploy:~$ cd ceph-cluster/ cephadmin@ceph-deploy:~/ceph-cluster$
前期只先初始化ceph-mon01節點，ceph-mon02和ceph-mon03在集群部署完成后，再手動添加
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy new --cluster-network 10.0.0.0/16 --public-network 172.168.0.0/16 ceph-mon01
#下面內容為ceph-mon01的初始化過程
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy new --cluster-network 10.0.0.0/16 --public-network 172.168.0.0/16 ceph-mon01
[ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy new --cluster-network 10.0.0.0/16 --public-network 172.168.0.0/16 ceph-mon01
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f25a37afe10>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] ssh_copykey : True
[ceph_deploy.cli][INFO ] mon : ['ceph-mon01']
[ceph_deploy.cli][INFO ] func : <function new at 0x7f25a0a64ad0>
[ceph_deploy.cli][INFO ] public_network : 172.168.0.0/16
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] cluster_network : 10.0.0.0/16
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] fsid : None
[ceph_deploy.new][DEBUG ] Creating new cluster named ceph
[ceph_deploy.new][INFO ] making sure passwordless SSH succeeds
[ceph-mon01][DEBUG ] connected to host: ceph-deploy
[ceph-mon01][INFO ] Running command: ssh -CT -o BatchMode=yes ceph-mon01
[ceph_deploy.new][WARNIN] could not connect via SSH
[ceph_deploy.new][INFO ] will connect again with password prompt
The authenticity of host 'ceph-mon01 (172.168.32.104)' can't be established.
ECDSA key fingerprint is SHA256:AIDN3qa9QKjViElHDrtTXhJ5EpTXdWj5Sc3tiy91E4Y.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'ceph-mon01' (ECDSA) to the list of known hosts.
[ceph-mon01][DEBUG ] connected to host: ceph-mon01
[ceph-mon01][DEBUG ] detect platform information from remote host
[ceph-mon01][DEBUG ] detect machine type
[ceph_deploy.new][INFO ] adding public keys to authorized_keys
[ceph-mon01][DEBUG ] append contents to file
[ceph-mon01][DEBUG ] connection detected need for sudo
[ceph-mon01][DEBUG ] connected to host: ceph-mon01
[ceph-mon01][DEBUG ] detect platform information from remote host
[ceph-mon01][DEBUG ] detect machine type
[ceph-mon01][DEBUG ] find the location of an executable
[ceph-mon01][INFO ] Running command: sudo /bin/ip link show
[ceph-mon01][INFO ] Running command: sudo /bin/ip addr show
[ceph-mon01][DEBUG ] IP addresses found: [u'172.168.32.104', u'10.0.0.104']
[ceph_deploy.new][DEBUG ] Resolving host ceph-mon01
[ceph_deploy.new][DEBUG ] Monitor ceph-mon01 at 172.168.32.104
[ceph_deploy.new][DEBUG ] Monitor initial members are ['ceph-mon01']
[ceph_deploy.new][DEBUG ] Monitor addrs are [u'172.168.32.104']
[ceph_deploy.new][DEBUG ] Creating a random mon key...
[ceph_deploy.new][DEBUG ] Writing monitor keyring to ceph.mon.keyring...
[ceph_deploy.new][DEBUG ] Writing initial config to ceph.conf...
初始化為驗證
cephadmin@ceph-deploy:~/ceph-cluster$ ll
total 16
drwxrwxr-x 2 cephadmin cephadmin 75 Aug 30 10:48 ./
drwxr-xr-x 6 cephadmin cephadmin 157 Aug 30 10:25 ../
-rw-rw-r-- 1 cephadmin cephadmin 264 Aug 30 10:48 ceph.conf #自動生成的配置文件
-rw-r--r-- 1 cephadmin cephadmin 6207 Aug 30 10:48 ceph-deploy-ceph.log #初始化日志
-rw------- 1 cephadmin cephadmin 73 Aug 30 10:48 ceph.mon.keyring #用於ceph mon 節點內部通訊認證的秘鑰環文件
cephadmin@ceph-deploy:~/ceph-cluster$ cat ceph.conf
[global]
fsid = c31ea2e3-47f7-4247-9d12-c0bf8f1dfbfb #ceph集群ID
public_network = 172.168.0.0/16
cluster_network = 10.0.0.0/16
mon_initial_members = ceph-mon01 #可以用逗號做分割添加多個mon節點
mon_host = 172.168.32.104
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
初始化ceph-node節點
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy install --no-adjust-repos --nogpgcheck ceph-node01 ceph-node02 ceph-node03 ceph-node04
此過程會在指定的ceph node 節點按照串行的方式逐個服務器安裝epel 源和ceph 源並安裝ceph所需軟件
#...... [ceph-node03][DEBUG ] The following additional packages will be installed: [ceph-node03][DEBUG ] ceph-base ceph-common ceph-mgr ceph-mgr-modules-core libaio1 libbabeltrace1 [ceph-node03][DEBUG ] libcephfs2 libdw1 libgoogle-perftools4 libibverbs1 libjaeger libjs-jquery [ceph-node03][DEBUG ] libleveldb1v5 liblttng-ust-ctl4 liblttng-ust0 liblua5.3-0 libnl-route-3-200 [ceph-node03][DEBUG ] liboath0 librabbitmq4 librados2 libradosstriper1 librbd1 librdkafka1 [ceph-node03][DEBUG ] librdmacm1 librgw2 libsnappy1v5 libtcmalloc-minimal4 liburcu6 [ceph-node03][DEBUG ] python-pastedeploy-tpl python3-bcrypt python3-bs4 python3-ceph-argparse [ceph-node03][DEBUG ] python3-ceph-common python3-cephfs python3-cherrypy3 python3-dateutil [ceph-node03][DEBUG ] python3-distutils python3-jwt python3-lib2to3 python3-logutils python3-mako [ceph-node03][DEBUG ] python3-markupsafe python3-paste python3-pastedeploy python3-pecan [ceph-node03][DEBUG ] python3-prettytable python3-rados python3-rbd python3-rgw [ceph-node03][DEBUG ] python3-simplegeneric python3-singledispatch python3-tempita [ceph-node03][DEBUG ] python3-waitress python3-webob python3-webtest python3-werkzeug [ceph-node03][DEBUG ] Suggested packages: [ceph-node03][DEBUG ] python3-influxdb python3-crypto python3-beaker python-mako-doc httpd-wsgi [ceph-node03][DEBUG ] libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc [ceph-node03][DEBUG ] python-waitress-doc python-webob-doc python-webtest-doc ipython3 [ceph-node03][DEBUG ] python3-lxml python3-termcolor python3-watchdog python-werkzeug-doc [ceph-node03][DEBUG ] Recommended packages: [ceph-node03][DEBUG ] ntp | time-daemon ceph-fuse ceph-mgr-dashboard ceph-mgr-diskprediction-local [ceph-node03][DEBUG ] ceph-mgr-k8sevents ceph-mgr-cephadm nvme-cli smartmontools ibverbs-providers [ceph-node03][DEBUG ] javascript-common python3-lxml python3-routes python3-simplejson [ceph-node03][DEBUG ] python3-pastescript python3-pyinotify [ceph-node03][DEBUG ] The following NEW packages will be installed: [ceph-node03][DEBUG ] ceph ceph-base ceph-common ceph-mds ceph-mgr ceph-mgr-modules-core ceph-mon [ceph-node03][DEBUG ] ceph-osd libaio1 libbabeltrace1 libcephfs2 libdw1 libgoogle-perftools4 [ceph-node03][DEBUG ] libibverbs1 libjaeger libjs-jquery libleveldb1v5 liblttng-ust-ctl4 [ceph-node03][DEBUG ] liblttng-ust0 liblua5.3-0 libnl-route-3-200 liboath0 librabbitmq4 librados2 [ceph-node03][DEBUG ] libradosstriper1 librbd1 librdkafka1 librdmacm1 librgw2 libsnappy1v5 [ceph-node03][DEBUG ] libtcmalloc-minimal4 liburcu6 python-pastedeploy-tpl python3-bcrypt [ceph-node03][DEBUG ] python3-bs4 python3-ceph-argparse python3-ceph-common python3-cephfs [ceph-node03][DEBUG ] python3-cherrypy3 python3-dateutil python3-distutils python3-jwt [ceph-node03][DEBUG ] python3-lib2to3 python3-logutils python3-mako python3-markupsafe [ceph-node03][DEBUG ] python3-paste python3-pastedeploy python3-pecan python3-prettytable [ceph-node03][DEBUG ] python3-rados python3-rbd python3-rgw python3-simplegeneric [ceph-node03][DEBUG ] python3-singledispatch python3-tempita python3-waitress python3-webob [ceph-node03][DEBUG ] python3-webtest python3-werkzeug radosgw #......
在各mon 節點按照組件ceph-mon,並通初始化mon 節點，mon 節點ha 還可以后期橫向擴容。
root@ceph-mon01:~# apt install ceph-mon root@ceph-mon02:~# apt install ceph-mon root@ceph-mon03:~# apt install ceph-mon
在ceph-deploy節點初始化mon節點
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mon create-initial
#下面為mon節點初始化內容
[ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy mon create-initial
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] subcommand : create-initial
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7fbee5b13fa0>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] func : <function mon at 0x7fbee5af7ad0>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] keyrings : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.mon][DEBUG ] Deploying mon, cluster ceph hosts ceph-mon01
[ceph_deploy.mon][DEBUG ] detecting platform for host ceph-mon01 ...
[ceph-mon01][DEBUG ] connection detected need for sudo
[ceph-mon01][DEBUG ] connected to host: ceph-mon01
[ceph-mon01][DEBUG ] detect platform information from remote host
[ceph-mon01][DEBUG ] detect machine type
[ceph-mon01][DEBUG ] find the location of an executable
[ceph_deploy.mon][INFO ] distro info: Ubuntu 18.04 bionic
[ceph-mon01][DEBUG ] determining if provided host has same hostname in remote
[ceph-mon01][DEBUG ] get remote short hostname
[ceph-mon01][DEBUG ] deploying mon to ceph-mon01
[ceph-mon01][DEBUG ] get remote short hostname
[ceph-mon01][DEBUG ] remote hostname: ceph-mon01
[ceph-mon01][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph-mon01][DEBUG ] create the mon path if it does not exist
[ceph-mon01][DEBUG ] checking for done path: /var/lib/ceph/mon/ceph-ceph-mon01/done
[ceph-mon01][DEBUG ] done path does not exist: /var/lib/ceph/mon/ceph-ceph-mon01/done
[ceph-mon01][INFO ] creating keyring file: /var/lib/ceph/tmp/ceph-ceph-mon01.mon.keyring
[ceph-mon01][DEBUG ] create the monitor keyring file
[ceph-mon01][INFO ] Running command: sudo ceph-mon --cluster ceph --mkfs -i ceph-mon01 --keyring /var/lib/ceph/tmp/ceph-ceph-mon01.mon.keyring --setuser 64045 --setgroup 64045
[ceph-mon01][INFO ] unlinking keyring file /var/lib/ceph/tmp/ceph-ceph-mon01.mon.keyring
[ceph-mon01][DEBUG ] create a done file to avoid re-doing the mon deployment
[ceph-mon01][DEBUG ] create the init path if it does not exist
[ceph-mon01][INFO ] Running command: sudo systemctl enable ceph.target
[ceph-mon01][INFO ] Running command: sudo systemctl enable ceph-mon@ceph-mon01
[ceph-mon01][WARNIN] Created symlink /etc/systemd/system/ceph-mon.target.wants/ceph-mon@ceph-mon01.service → /lib/systemd/system/ceph-mon@.service.
[ceph-mon01][INFO ] Running command: sudo systemctl start ceph-mon@ceph-mon01
[ceph-mon01][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph-mon01.asok mon_status
[ceph-mon01][DEBUG ] ********************************************************************************
[ceph-mon01][DEBUG ] status for monitor: mon.ceph-mon01
[ceph-mon01][DEBUG ] {
[ceph-mon01][DEBUG ] "election_epoch": 3,
[ceph-mon01][DEBUG ] "extra_probe_peers": [],
[ceph-mon01][DEBUG ] "feature_map": {
[ceph-mon01][DEBUG ] "mon": [
[ceph-mon01][DEBUG ] {
[ceph-mon01][DEBUG ] "features": "0x3f01cfb9fffdffff",
[ceph-mon01][DEBUG ] "num": 1,
[ceph-mon01][DEBUG ] "release": "luminous"
[ceph-mon01][DEBUG ] }
[ceph-mon01][DEBUG ] ]
[ceph-mon01][DEBUG ] },
[ceph-mon01][DEBUG ] "features": {
[ceph-mon01][DEBUG ] "quorum_con": "4540138297136906239",
[ceph-mon01][DEBUG ] "quorum_mon": [
[ceph-mon01][DEBUG ] "kraken",
[ceph-mon01][DEBUG ] "luminous",
[ceph-mon01][DEBUG ] "mimic",
[ceph-mon01][DEBUG ] "osdmap-prune",
[ceph-mon01][DEBUG ] "nautilus",
[ceph-mon01][DEBUG ] "octopus",
[ceph-mon01][DEBUG ] "pacific",
[ceph-mon01][DEBUG ] "elector-pinging"
[ceph-mon01][DEBUG ] ],
[ceph-mon01][DEBUG ] "required_con": "2449958747317026820",
[ceph-mon01][DEBUG ] "required_mon": [
[ceph-mon01][DEBUG ] "kraken",
[ceph-mon01][DEBUG ] "luminous",
[ceph-mon01][DEBUG ] "mimic",
[ceph-mon01][DEBUG ] "osdmap-prune",
[ceph-mon01][DEBUG ] "nautilus",
[ceph-mon01][DEBUG ] "octopus",
[ceph-mon01][DEBUG ] "pacific",
[ceph-mon01][DEBUG ] "elector-pinging"
[ceph-mon01][DEBUG ] ]
[ceph-mon01][DEBUG ] },
[ceph-mon01][DEBUG ] "monmap": {
[ceph-mon01][DEBUG ] "created": "2021-08-30T03:27:46.534560Z",
[ceph-mon01][DEBUG ] "disallowed_leaders: ": "",
[ceph-mon01][DEBUG ] "election_strategy": 1,
[ceph-mon01][DEBUG ] "epoch": 1,
[ceph-mon01][DEBUG ] "features": {
[ceph-mon01][DEBUG ] "optional": [],
[ceph-mon01][DEBUG ] "persistent": [
[ceph-mon01][DEBUG ] "kraken",
[ceph-mon01][DEBUG ] "luminous",
[ceph-mon01][DEBUG ] "mimic",
[ceph-mon01][DEBUG ] "osdmap-prune",
[ceph-mon01][DEBUG ] "nautilus",
[ceph-mon01][DEBUG ] "octopus",
[ceph-mon01][DEBUG ] "pacific",
[ceph-mon01][DEBUG ] "elector-pinging"
[ceph-mon01][DEBUG ] ]
[ceph-mon01][DEBUG ] },
[ceph-mon01][DEBUG ] "fsid": "c31ea2e3-47f7-4247-9d12-c0bf8f1dfbfb",
[ceph-mon01][DEBUG ] "min_mon_release": 16,
[ceph-mon01][DEBUG ] "min_mon_release_name": "pacific",
[ceph-mon01][DEBUG ] "modified": "2021-08-30T03:27:46.534560Z",
[ceph-mon01][DEBUG ] "mons": [
[ceph-mon01][DEBUG ] {
[ceph-mon01][DEBUG ] "addr": "172.168.32.104:6789/0",
[ceph-mon01][DEBUG ] "crush_location": "{}",
[ceph-mon01][DEBUG ] "name": "ceph-mon01",
[ceph-mon01][DEBUG ] "priority": 0,
[ceph-mon01][DEBUG ] "public_addr": "172.168.32.104:6789/0",
[ceph-mon01][DEBUG ] "public_addrs": {
[ceph-mon01][DEBUG ] "addrvec": [
[ceph-mon01][DEBUG ] {
[ceph-mon01][DEBUG ] "addr": "172.168.32.104:3300",
[ceph-mon01][DEBUG ] "nonce": 0,
[ceph-mon01][DEBUG ] "type": "v2"
[ceph-mon01][DEBUG ] },
[ceph-mon01][DEBUG ] {
[ceph-mon01][DEBUG ] "addr": "172.168.32.104:6789",
[ceph-mon01][DEBUG ] "nonce": 0,
[ceph-mon01][DEBUG ] "type": "v1"
[ceph-mon01][DEBUG ] }
[ceph-mon01][DEBUG ] ]
[ceph-mon01][DEBUG ] },
[ceph-mon01][DEBUG ] "rank": 0,
[ceph-mon01][DEBUG ] "weight": 0
[ceph-mon01][DEBUG ] }
[ceph-mon01][DEBUG ] ],
[ceph-mon01][DEBUG ] "stretch_mode": false
[ceph-mon01][DEBUG ] },
[ceph-mon01][DEBUG ] "name": "ceph-mon01",
[ceph-mon01][DEBUG ] "outside_quorum": [],
[ceph-mon01][DEBUG ] "quorum": [
[ceph-mon01][DEBUG ] 0
[ceph-mon01][DEBUG ] ],
[ceph-mon01][DEBUG ] "quorum_age": 2,
[ceph-mon01][DEBUG ] "rank": 0,
[ceph-mon01][DEBUG ] "state": "leader",
[ceph-mon01][DEBUG ] "stretch_mode": false,
[ceph-mon01][DEBUG ] "sync_provider": []
[ceph-mon01][DEBUG ] }
[ceph-mon01][DEBUG ] ********************************************************************************
[ceph-mon01][INFO ] monitor: mon.ceph-mon01 is running
[ceph-mon01][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph-mon01.asok mon_status
[ceph_deploy.mon][INFO ] processing monitor mon.ceph-mon01
[ceph-mon01][DEBUG ] connection detected need for sudo
[ceph-mon01][DEBUG ] connected to host: ceph-mon01
[ceph-mon01][DEBUG ] detect platform information from remote host
[ceph-mon01][DEBUG ] detect machine type
[ceph-mon01][DEBUG ] find the location of an executable
[ceph-mon01][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph-mon01.asok mon_status
[ceph_deploy.mon][INFO ] mon.ceph-mon01 monitor has reached quorum!
[ceph_deploy.mon][INFO ] all initial monitors are running and have formed quorum
[ceph_deploy.mon][INFO ] Running gatherkeys...
[ceph_deploy.gatherkeys][INFO ] Storing keys in temp directory /tmp/tmpCb4OB1
[ceph-mon01][DEBUG ] connection detected need for sudo
[ceph-mon01][DEBUG ] connected to host: ceph-mon01
[ceph-mon01][DEBUG ] detect platform information from remote host
[ceph-mon01][DEBUG ] detect machine type
[ceph-mon01][DEBUG ] get remote short hostname
[ceph-mon01][DEBUG ] fetch remote file
[ceph-mon01][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --admin-daemon=/var/run/ceph/ceph-mon.ceph-mon01.asok mon_status
[ceph-mon01][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph-mon01/keyring auth get client.admin
[ceph-mon01][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph-mon01/keyring auth get client.bootstrap-mds
[ceph-mon01][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph-mon01/keyring auth get client.bootstrap-mgr
[ceph-mon01][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph-mon01/keyring auth get client.bootstrap-osd
[ceph-mon01][INFO ] Running command: sudo /usr/bin/ceph --connect-timeout=25 --cluster=ceph --name mon. --keyring=/var/lib/ceph/mon/ceph-ceph-mon01/keyring auth get client.bootstrap-rgw
[ceph_deploy.gatherkeys][INFO ] Storing ceph.client.admin.keyring
[ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-mds.keyring
[ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-mgr.keyring
[ceph_deploy.gatherkeys][INFO ] keyring 'ceph.mon.keyring' already exists
[ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-osd.keyring
[ceph_deploy.gatherkeys][INFO ] Storing ceph.bootstrap-rgw.keyring
[ceph_deploy.gatherkeys][INFO ] Destroy temp directory /tmp/tmpCb4OB1
驗證在mon 定節點已經自動安裝並啟動了ceph-mon 服務，並且后期在ceph-deploy 節點初始化目錄會生成一些bootstrap ceph mds/mgr/osd/rgw 等服務的keyring 認證文件，這些初始化文件擁有對ceph 集群的最高權限，所以一定要保存好。
root@ceph-mon01:~# ps -ef|grep ceph-mon ceph 8304 1 0 22:43 ? 00:00:00 /usr/bin/ceph-mon -f --cluster ceph --id ceph-mon01 --setuser ceph --setgroup ceph
在ceph-deploy 節點把配置文件和admin 密鑰拷貝至Ceph 集群需要執行ceph 管理命令的節點，從而不需要后期通過ceph 命令對ceph 集群進行管理配置的時候每次都需要指定ceph-mon 節點地址和ceph.client.admin.keyring 文件,另外各ceph-mon 節點也需要同步ceph 的集群配置文件與認證文件。
如果在ceph-deploy 節點管理集群
root@ceph-deploy:~# apt install ceph-common -y #先安裝ceph 的公共組件 root@ceph-node01:~# apt install ceph-common -y root@ceph-node02:~# apt install ceph-common -y root@ceph-node03:~# apt install ceph-common -y root@ceph-node04:~# apt install ceph-common -y
拷貝密鑰到ceph-node01 ceph-node02 ceph-node03 ceph-node04
#在ceph-deploy上操作
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy admin ceph-node01 ceph-node02 ceph-node03 ceph-node04
#下面為運行過程
[ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy admin ceph-node01 ceph-node02 ceph-node03 ceph-node04
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f3066490190>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] client : ['ceph-node01', 'ceph-node02', 'ceph-node03', 'ceph-node04']
[ceph_deploy.cli][INFO ] func : <function admin at 0x7f3066d91a50>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph-node01
[ceph-node01][DEBUG ] connection detected need for sudo
[ceph-node01][DEBUG ] connected to host: ceph-node01
[ceph-node01][DEBUG ] detect platform information from remote host
[ceph-node01][DEBUG ] detect machine type
[ceph-node01][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph-node02
[ceph-node02][DEBUG ] connection detected need for sudo
[ceph-node02][DEBUG ] connected to host: ceph-node02
[ceph-node02][DEBUG ] detect platform information from remote host
[ceph-node02][DEBUG ] detect machine type
[ceph-node02][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph-node03
[ceph-node03][DEBUG ] connection detected need for sudo
[ceph-node03][DEBUG ] connected to host: ceph-node03
[ceph-node03][DEBUG ] detect platform information from remote host
[ceph-node03][DEBUG ] detect machine type
[ceph-node03][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph-node04
[ceph-node04][DEBUG ] connection detected need for sudo
[ceph-node04][DEBUG ] connected to host: ceph-node04
[ceph-node04][DEBUG ] detect platform information from remote host
[ceph-node04][DEBUG ] detect machine type
[ceph-node04][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
root@ceph-node01:~# ll /etc/ceph total 24 drwxr-xr-x 2 root root 87 Aug 17 22:54 ./ drwxr-xr-x 98 root root 8192 Aug 17 22:16 ../ -rw------- 1 root root 151 Aug 17 22:54 ceph.client.admin.keyring -rw-r--r-- 1 root root 264 Aug 17 22:54 ceph.conf -rw-r--r-- 1 root root 92 Jul 8 22:17 rbdmap -rw------- 1 root root 0 Aug 17 22:54 tmp8AdP3P root@ceph-node02:~# ll /etc/ceph total 24 drwxr-xr-x 2 root root 87 Aug 17 22:54 ./ drwxr-xr-x 98 root root 8192 Aug 17 22:16 ../ -rw------- 1 root root 151 Aug 17 22:54 ceph.client.admin.keyring -rw-r--r-- 1 root root 264 Aug 17 22:54 ceph.conf -rw-r--r-- 1 root root 92 Jul 8 22:17 rbdmap -rw------- 1 root root 0 Aug 17 22:54 tmp8AdP3P root@ceph-node03:~# ll /etc/ceph total 24 drwxr-xr-x 2 root root 87 Aug 17 22:54 ./ drwxr-xr-x 98 root root 8192 Aug 17 22:16 ../ -rw------- 1 root root 151 Aug 17 22:54 ceph.client.admin.keyring -rw-r--r-- 1 root root 264 Aug 17 22:54 ceph.conf -rw-r--r-- 1 root root 92 Jul 8 22:17 rbdmap -rw------- 1 root root 0 Aug 17 22:54 tmp8AdP3P root@ceph-node04:~# ll /etc/ceph total 24 drwxr-xr-x 2 root root 87 Aug 17 22:54 ./ drwxr-xr-x 98 root root 8192 Aug 17 22:16 ../ -rw------- 1 root root 151 Aug 17 22:54 ceph.client.admin.keyring -rw-r--r-- 1 root root 264 Aug 17 22:54 ceph.conf -rw-r--r-- 1 root root 92 Jul 8 22:17 rbdmap -rw------- 1 root root 0 Aug 17 22:54 tmp8AdP3P
11)在ceph-node節點對密鑰授權
root@ceph-node01:~# setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring root@ceph-node02:~# setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring root@ceph-node03:~# setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring root@ceph-node04:~# setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring
(1)在ceph-mgr01和ceph-mgr02上安裝ceph-mgr軟件
root@ceph-mgr01:~# apt install -y ceph-mgr root@ceph-mgr02:~# apt install -y ceph-mgr
(2)在ceph-deploy上初始化ceph-mgr節點
#只初始化ceph-mgr01節點
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mgr create ceph-mgr01
#下面為過程內容
[ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy mgr create ceph-mgr01
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] mgr : [('ceph-mgr01', 'ceph-mgr01')]
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] subcommand : create
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f07c9190c30>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] func : <function mgr at 0x7f07c95f0150>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.mgr][DEBUG ] Deploying mgr, cluster ceph hosts ceph-mgr01:ceph-mgr01
The authenticity of host 'ceph-mgr01 (172.168.32.102)' can't be established.
ECDSA key fingerprint is SHA256:AIDN3qa9QKjViElHDrtTXhJ5EpTXdWj5Sc3tiy91E4Y.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'ceph-mgr01' (ECDSA) to the list of known hosts.
[ceph-mgr01][DEBUG ] connection detected need for sudo
[ceph-mgr01][DEBUG ] connected to host: ceph-mgr01
[ceph-mgr01][DEBUG ] detect platform information from remote host
[ceph-mgr01][DEBUG ] detect machine type
[ceph_deploy.mgr][INFO ] Distro info: Ubuntu 18.04 bionic
[ceph_deploy.mgr][DEBUG ] remote host will use systemd
[ceph_deploy.mgr][DEBUG ] deploying mgr bootstrap to ceph-mgr01
[ceph-mgr01][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph-mgr01][WARNIN] mgr keyring does not exist yet, creating one
[ceph-mgr01][DEBUG ] create a keyring file
[ceph-mgr01][DEBUG ] create path recursively if it doesn't exist
[ceph-mgr01][INFO ] Running command: sudo ceph --cluster ceph --name client.bootstrap-mgr --keyring /var/lib/ceph/bootstrap-mgr/ceph.keyring auth get-or-create mgr.ceph-mgr01 mon allow profile mgr osd allow * mds allow * -o /var/lib/ceph/mgr/ceph-ceph-mgr01/keyring
[ceph-mgr01][INFO ] Running command: sudo systemctl enable ceph-mgr@ceph-mgr01
[ceph-mgr01][WARNIN] Created symlink /etc/systemd/system/ceph-mgr.target.wants/ceph-mgr@ceph-mgr01.service → /lib/systemd/system/ceph-mgr@.service.
[ceph-mgr01][INFO ] Running command: sudo systemctl start ceph-mgr@ceph-mgr01
[ceph-mgr01][INFO ] Running command: sudo systemctl enable ceph.target
在ceph-mgr01上
root@ceph-mgr01:~# ps -ef|grep ceph-mgr
ceph 10771 1 16 12:05 ? 00:00:07 /usr/bin/ceph-mgr -f --cluster ceph --id ceph-mgr01 --setuser ceph --setgroup ceph
#ceph-deploy管理ceph集群環境設置
cephadmin@ceph-deploy:~/ceph-cluster$ sudo apt install -y ceph-common
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy admin ceph-deploy
cephadmin@ceph-deploy:~/ceph-cluster$ sudo setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring
#ceph-deploy管理ceph集群信息
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster:
id: c31ea2e3-47f7-4247-9d12-c0bf8f1dfbfb
health: HEALTH_WARN
mon is allowing insecure global_id reclaim
OSD count 0 < osd_pool_default_size 3
services:
mon: 1 daemons, quorum ceph-mon01 (age 38m)
mgr: ceph-mgr01(active, since 65s)
osd: 0 osds: 0 up, 0 in
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:
cephadmin@ceph-deploy:~/ceph-cluster$ ceph config set mon auth_allow_insecure_global_id_reclaim false
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster:
id: c31ea2e3-47f7-4247-9d12-c0bf8f1dfbfb
health: HEALTH_WARN
OSD count 0 < osd_pool_default_size 3
services:
mon: 1 daemons, quorum ceph-mon01 (age 39m)
mgr: ceph-mgr01(active, since 2m)
osd: 0 osds: 0 up, 0 in
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:
cephadmin@ceph-deploy:~/ceph-cluster$ ceph version
ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph versions
{
"mon": {
"ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 1
},
"mgr": {
"ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 1
},
"osd": {},
"mds": {},
"overall": {
"ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)": 2
}
}
#前面執行了node節點的初始化，如下步驟，后面的操作可以不執行
ceph-deploy install --no-adjust-repos --nogpgcheck ceph-node01 ceph-node02 ceph-node03 ceph-node04
#擦除磁盤之前通過deploy 節點對node 節點執行安裝ceph 基本運行環境。
#在ceph-deploy上操作
#cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy install --release pacific ceph-node01
#cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy install --release pacific ceph-node02
#cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy install --release pacific ceph-node03
#cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy install --release pacific ceph-node04
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy disk list ceph-node01
[ceph_deploy.conf][DEBUG ] found configuration file at: /home/cephadmin/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy disk list ceph-node01
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] debug : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] subcommand : list
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f68cc180fa0>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] host : ['ceph-node01']
[ceph_deploy.cli][INFO ] func : <function disk at 0x7f68cc15c2d0>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph-node01][DEBUG ] connection detected need for sudo
[ceph-node01][DEBUG ] connected to host: ceph-node01
[ceph-node01][DEBUG ] detect platform information from remote host
[ceph-node01][DEBUG ] detect machine type
[ceph-node01][DEBUG ] find the location of an executable
[ceph-node01][INFO ] Running command: sudo fdisk -l
[ceph-node01][INFO ] Disk /dev/sda: 50 GiB, 53687091200 bytes, 104857600 sectors
[ceph-node01][INFO ] Disk /dev/sdb: 50 GiB, 53687091200 bytes, 104857600 sectors
[ceph-node01][INFO ] Disk /dev/sdc: 50 GiB, 53687091200 bytes, 104857600 sectors
[ceph-node01][INFO ] Disk /dev/sdd: 50 GiB, 53687091200 bytes, 104857600 sectors
[ceph-node01][INFO ] Disk /dev/sde: 50 GiB, 53687091200 bytes, 104857600 sectors
#在ceph-deploy上操作 ceph-deploy disk zap ceph-node01 /dev/sdb ceph-deploy disk zap ceph-node01 /dev/sdc ceph-deploy disk zap ceph-node01 /dev/sdd ceph-deploy disk zap ceph-node01 /dev/sde ceph-deploy disk zap ceph-node02 /dev/sdb ceph-deploy disk zap ceph-node02 /dev/sdc ceph-deploy disk zap ceph-node02 /dev/sdd ceph-deploy disk zap ceph-node02 /dev/sde ceph-deploy disk zap ceph-node03 /dev/sdb ceph-deploy disk zap ceph-node03 /dev/sdc ceph-deploy disk zap ceph-node03 /dev/sdd ceph-deploy disk zap ceph-node03 /dev/sde ceph-deploy disk zap ceph-node04 /dev/sdb ceph-deploy disk zap ceph-node04 /dev/sdc ceph-deploy disk zap ceph-node04 /dev/sdd ceph-deploy disk zap ceph-node04 /dev/sde
#在ceph-deploy上操作 ceph-deploy osd create ceph-node01 --data /dev/sdb ceph-deploy osd create ceph-node01 --data /dev/sdc ceph-deploy osd create ceph-node01 --data /dev/sdd ceph-deploy osd create ceph-node01 --data /dev/sde ceph-deploy osd create ceph-node02 --data /dev/sdb ceph-deploy osd create ceph-node02 --data /dev/sdc ceph-deploy osd create ceph-node02 --data /dev/sdd ceph-deploy osd create ceph-node02 --data /dev/sde ceph-deploy osd create ceph-node03 --data /dev/sdb ceph-deploy osd create ceph-node03 --data /dev/sdc ceph-deploy osd create ceph-node03 --data /dev/sdd ceph-deploy osd create ceph-node03 --data /dev/sde ceph-deploy osd create ceph-node04 --data /dev/sdb ceph-deploy osd create ceph-node04 --data /dev/sdc ceph-deploy osd create ceph-node04 --data /dev/sdd ceph-deploy osd create ceph-node04 --data /dev/sde
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster:
id: c31ea2e3-47f7-4247-9d12-c0bf8f1dfbfb
health: HEALTH_OK
services:
mon: 1 daemons, quorum ceph-mon01 (age 48m)
mgr: ceph-mgr01(active, since 10m)
osd: 16 osds: 16 up (since 6s), 16 in (since 15s)
data:
pools: 1 pools, 1 pgs
objects: 0 objects, 0 B
usage: 94 MiB used, 800 GiB / 800 GiB avail
pgs: 1 active+clean
#在ceph-node02上測試，其他node節點操作一樣 root@ceph-node02:~# ps -ef|grep osd ceph 15521 1 0 00:08 ? 00:00:03 /usr/bin/ceph-osd -f --cluster ceph --id 3 --setuser ceph --setgroup ceph ceph 17199 1 0 00:09 ? 00:00:03 /usr/bin/ceph-osd -f --cluster ceph --id 4 --setuser ceph --setgroup ceph ceph 18874 1 0 00:09 ? 00:00:03 /usr/bin/ceph-osd -f --cluster ceph --id 5 --setuser ceph --setgroup ceph ceph 20546 1 0 00:09 ? 00:00:03 /usr/bin/ceph-osd -f --cluster ceph --id 6 --setuser ceph --setgroup ceph root@ceph-node02:~# systemctl enable ceph-osd@3 ceph-osd@4 ceph-osd@5 ceph-osd@6 Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@3.service → /lib/systemd/system/ceph-osd@.service. Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@4.service → /lib/systemd/system/ceph-osd@.service. Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@5.service → /lib/systemd/system/ceph-osd@.service. Created symlink /etc/systemd/system/ceph-osd.target.wants/ceph-osd@6.service → /lib/systemd/system/ceph-osd@.service.
$ ceph-deploy --help new：開始部署一個新的ceph 存儲集群，並生成CLUSTER.conf 集群配置文件和keyring 認證文件。 install: 在遠程主機上安裝ceph 相關的軟件包, 可以通過--release 指定安裝的版本。 rgw：管理RGW 守護程序(RADOSGW,對象存儲網關)。 mgr：管理MGR 守護程序(ceph-mgr,Ceph Manager DaemonCeph 管理器守護程序)。 mds：管理MDS 守護程序(Ceph Metadata Server，ceph 源數據服務器)。 mon：管理MON 守護程序(ceph-mon,ceph 監視器)。 gatherkeys：從指定獲取提供新節點的驗證keys，這些keys 會在添加新的MON/OSD/MD加入的時候使用。 disk：管理遠程主機磁盤。 osd：在遠程主機准備數據磁盤，即將指定遠程主機的指定磁盤添加到ceph 集群作為osd 使用。 repo： 遠程主機倉庫管理。 admin：推送ceph 集群配置文件和client.admin 認證文件到遠程主機。 config：將ceph.conf 配置文件推送到遠程主機或從遠程主機拷貝。 uninstall：從遠端主機刪除安裝包。 purgedata：從/var/lib/ceph 刪除ceph 數據,會刪除/etc/ceph 下的內容。 purge: 刪除遠端主機的安裝包和所有數據。 forgetkeys：從本地主機刪除所有的驗證keyring, 包括client.admin, monitor, bootstrap 等 認證文件。 pkg： 管理遠端主機的安裝包。 calamari：安裝並配置一個calamari web 節點，calamari 是一個web 監控平台。
1、磁盤有數據
處理方案：清空磁盤數據，實驗中的操作是關閉虛擬機，刪除原磁盤，重新添加磁盤。
[ceph_deploy.osd][DEBUG ] Creating OSD on cluster ceph with data device /dev/sdb
root@ceph-node01's password:
root@ceph-node01's password:
[ceph-node01][DEBUG ] connected to host: ceph-node01
[ceph-node01][DEBUG ] detect platform information from remote host
[ceph-node01][DEBUG ] detect machine type
[ceph-node01][DEBUG ] find the location of an executable
[ceph_deploy.osd][INFO ] Distro info: Ubuntu 18.04 bionic
[ceph_deploy.osd][DEBUG ] Deploying osd to ceph-node01
[ceph-node01][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph-node01][DEBUG ] find the location of an executable
[ceph-node01][INFO ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb
[ceph-node01][WARNIN] Running command: /usr/bin/ceph-authtool --gen-print-key
[ceph-node01][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 93330566-19f1-47a7-8d5e-5d47c52b4274
[ceph-node01][WARNIN] Running command: /sbin/lvcreate --yes -l 5119 -n osd-block-93330566-19f1-47a7-8d5e-5d47c52b4274 ceph-f3745436-e965-4dda-a440-4c7fb3104c48
[ceph-node01][WARNIN] stderr:
[ceph-node01][WARNIN] stderr: Volume group "ceph-f3745436-e965-4dda-a440-4c7fb3104c48" has insufficient free space (0 extents): 5119 required.
[ceph-node01][WARNIN] stderr:
[ceph-node01][WARNIN] --> Was unable to complete a new OSD, will rollback changes
[ceph-node01][WARNIN] Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring osd purge-new osd.16 --yes-i-really-mean-it
[ceph-node01][WARNIN] stderr: 2021-08-18T10:03:49.653+0800 7f2ffca71700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory
[ceph-node01][WARNIN] stderr:
[ceph-node01][WARNIN] stderr: 2021-08-18T10:03:49.653+0800 7f2ffca71700 -1 AuthRegistry(0x7f2ff805b408) no keyring found at /etc/ceph/ceph.client.bootstrap-osd.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx
[ceph-node01][WARNIN] stderr:
[ceph-node01][WARNIN] stderr: purged osd.16
[ceph-node01][WARNIN] --> RuntimeError: command returned non-zero exit status: 5
[ceph-node01][ERROR ] RuntimeError: command returned non-zero exit status: 1
[ceph_deploy.osd][ERROR ] Failed to execute command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb
[ceph_deploy][ERROR ] GenericError: Failed to create 1 OSDs
1)擴展ceph-mon 節點
Ceph-mon 是原生具備自選舉以實現高可用機制的ceph 服務，節點數量通常是奇數。
#在ceph-mon節點安裝ceph-mon root@ceph-mon02:~# apt install ceph-mon root@ceph-mon03:~# apt install ceph-mon #在ceph-deploy節點添加mon節點 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mon add ceph-mon02 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mon add ceph-mon03
ceph-mon結果驗證
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster:
id: f0e7c394-989b-4803-86c3-5557ae25e814
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph-mon01,ceph-mon02,ceph-mon03 (age 7s) #ceph-mon02和ceph-mon03添加完成
mgr: ceph-mgr01(active, since 15h)
osd: 20 osds: 16 up (since 22m), 15 in (since 25m)
data:
pools: 1 pools, 1 pgs
objects: 0 objects, 0 B
usage: 134 MiB used, 300 GiB / 300 GiB avail
pgs: 1 active+clean
#ceph-mon的狀態
cephadmin@ceph-deploy:~/ceph-cluster$ ceph quorum_status --format json-pretty
{
"election_epoch": 12,
"quorum": [
0,
1,
2
],
"quorum_names": [
"ceph-mon01",
"ceph-mon02",
"ceph-mon03"
],
"quorum_leader_name": "ceph-mon01", #當前mon的leader
"quorum_age": 176,
"features": {
"quorum_con": "4540138297136906239",
"quorum_mon": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus",
"octopus",
"pacific",
"elector-pinging"
]
},
"monmap": {
"epoch": 3,
"fsid": "f0e7c394-989b-4803-86c3-5557ae25e814",
"modified": "2021-08-18T06:31:20.933855Z",
"created": "2021-08-17T14:43:20.965196Z",
"min_mon_release": 16,
"min_mon_release_name": "pacific",
"election_strategy": 1,
"disallowed_leaders: ": "",
"stretch_mode": false,
"features": {
"persistent": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus",
"octopus",
"pacific",
"elector-pinging"
],
"optional": []
},
"mons": [
{
"rank": 0, #ceph-mon01的等級
"name": "ceph-mon01", #mon的節點名稱
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.168.32.104:3300", #監控地址
"nonce": 0
},
{
"type": "v1",
"addr": "172.168.32.104:6789", #監控地址
"nonce": 0
}
]
},
"addr": "172.168.32.104:6789/0",
"public_addr": "172.168.32.104:6789/0",
"priority": 0,
"weight": 0,
"crush_location": "{}"
},
{
"rank": 1,
"name": "ceph-mon02",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.168.32.105:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.168.32.105:6789",
"nonce": 0
}
]
},
"addr": "172.168.32.105:6789/0",
"public_addr": "172.168.32.105:6789/0",
"priority": 0,
"weight": 0,
"crush_location": "{}"
},
{
"rank": 2,
"name": "ceph-mon03",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.168.32.106:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.168.32.106:6789",
"nonce": 0
}
]
},
"addr": "172.168.32.106:6789/0",
"public_addr": "172.168.32.106:6789/0",
"priority": 0,
"weight": 0,
"crush_location": "{}"
}
]
}
}
#在ceph-mgr02節點上安裝ceph-mgr root@ceph-mgr02:~#apt install -y ceph-mgr #在ceph-deploy上添加ceph-mgr02 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mgr create ceph-mgr02 #在ceph-deploy上同步配置文件到ceph-mgr02 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy admin ceph-mgr02
ceph-mgr驗證
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster:
id: f0e7c394-989b-4803-86c3-5557ae25e814
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph-mon01,ceph-mon02,ceph-mon03 (age 12m)
mgr: ceph-mgr01(active, since 15h), standbys: ceph-mgr02 #備用的ceph-mgr02已經添加完成
osd: 20 osds: 16 up (since 34m), 15 in (since 37m)
data:
pools: 1 pools, 1 pgs
objects: 0 objects, 0 B
usage: 134 MiB used, 300 GiB / 300 GiB avail
pgs: 1 active+clean
1)、安裝ceph-mds軟件包
如果要使用cephFS，需要部署cephfs 服務。MDS服務與mon服務部署在一起（最好分開部署）。
apt install ceph-mds
在ceph-deploy上操作
ceph-deploy mds create ceph-mds01 ceph-deploy mds create ceph-mds02 ceph-deploy mds create ceph-mds03
#創建cephfs-metadata cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool create cephfs-metadata 32 32 pool 'cephfs-metadata' created #保存metadata 的pool #創建cephfs-data cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool create cephfs-data 64 64 pool 'cephfs-data' created #保存數據的pool #創建一個叫mycephfs的cephFS cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs new mycephfs cephfs-metadata cephfs-data new fs with metadata pool 2 and data pool 3
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs ls
name: mycephfs, metadata pool: cephfs-metadata, data pools: [cephfs-data ]
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status mycephfs
mycephfs - 0 clients
========
RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS
0 active ceph-mds01 Reqs: 0 /s 10 13 12 0
POOL TYPE USED AVAIL
cephfs-metadata metadata 96.0k 253G
cephfs-data data 0 253G
STANDBY MDS
ceph-mds02
ceph-mds03
MDS version: ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)
#驗證Cephfs的服務狀態
cephadmin@ceph-deploy:~/ceph-cluster$ ceph mds stat
mycephfs:1 {0=ceph-mds01=up:active} 2 up:standby
5)、MDS的高可用
為了測試，我們把ceph-mgr02復用為mds
#添加第4台mds ceph-mgr02
#在ceph-mgr02上安裝ceph-mds
root@ceph-mgr02:~# apt install ceph-mds
#在ceph-deploy上把mgr02添加進mds
cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy mds create ceph-mgr02
#現在mds的狀態為1主3備
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status
mycephfs - 0 clients
========
RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS
0 active ceph-mds01 Reqs: 0 /s 10 13 12 0
POOL TYPE USED AVAIL
cephfs-metadata metadata 96.0k 253G
cephfs-data data 0 253G
STANDBY MDS
ceph-mds02
ceph-mds03
ceph-mgr02
MDS version: ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)
設置每個Rank 的備份MDS，也就是如果此Rank 當前的MDS 出現問題馬上切換到另個MDS。設置備份的方法有很多，常用選項如下。
mds_standby_replay：值為true 或false，true 表示開啟replay 模式，這種模式下主MDS 內的數量將實時與從MDS 同步，如果主宕機，從可以快速的切換。如果為false 只有宕機的時候才去同步數據，這樣會有一段時間的中斷。 mds_standby_for_name：設置當前MDS 進程只用於備份於指定名稱的MDS。 mds_standby_for_rank：設置當前MDS 進程只用於備份於哪個Rank，通常為Rank 編號。另外在存在之個CephFS 文件系統中，還可以使用mds_standby_for_fscid 參數來為指定不同的文件系統。 mds_standby_for_fscid：指定CephFS 文件系統ID，需要聯合mds_standby_for_rank 生效，如果設置mds_standby_for_rank，那么就是用於指定文件系統的指定Rank，如果沒有設置，就是指定文件系統的所有Rank。
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status
mycephfs - 0 clients
========
RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS
0 active ceph-mds01 Reqs: 0 /s 10 13 12 0
POOL TYPE USED AVAIL
cephfs-metadata metadata 96.0k 253G
cephfs-data data 0 253G
STANDBY MDS
ceph-mds02
ceph-mds03
ceph-mgr02
MDS version: ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs get mycephfs
Filesystem 'mycephfs' (1)
fs_name mycephfs
epoch 6
flags 12
created 2021-08-30T14:17:05.882172+0800
modified 2021-08-30T14:17:06.888751+0800
tableserver 0
root 0
session_timeout 60
session_autoclose 300
max_file_size 1099511627776
required_client_features {}
last_failure 0
last_failure_osd_epoch 0
compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2}
max_mds 1
in 0
up {0=14122}
failed
damaged
stopped
data_pools [3]
metadata_pool 2
inline_data disabled
balancer
standby_count_wanted 1
[mds.ceph-mds01{0:14122} state up:active seq 112 addr [v2:172.168.32.104:6800/3723142001,v1:172.168.32.104:6801/3723142001]]
目前有四個mds 服務器，但是有一個主三個備，可以優化一下部署架構，設置為為兩主兩備。
#設置同時活躍的主mds最大值為2。
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs set mycephfs max_mds 2
cephadmin@ceph-deploy:~/ceph-cluster$ ceph mds stat
mycephfs:2 {0=ceph-mds01=up:active,1=ceph-mgr02=up:active} 2 up:standby
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status
mycephfs - 0 clients
========
RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS
0 active ceph-mds01 Reqs: 0 /s 10 13 12 0
1 active ceph-mgr02 Reqs: 0 /s 10 13 11 0
POOL TYPE USED AVAIL
cephfs-metadata metadata 168k 253G
cephfs-data data 0 253G
STANDBY MDS
ceph-mds02
ceph-mds03
MDS version: ceph version 16.2.5 (0883bdea7337b95e4b611c768c0279868462204a) pacific (stable)
目前的狀態是ceph-mds01 和ceph-mgr02 分別是active 狀態，ceph-mds02和ceph-mds03 分別處於standby 狀態，現在可以將ceph-mds02設置為ceph-mds01 的standby，將ceph-mds03 設置為ceph-mgr02的standby，以實現每個主都有一個固定備份角色的結構，則修改配置文件如.
cephadmin@ceph-deploy:~/ceph-cluster$ vim ceph.conf [global] fsid = c31ea2e3-47f7-4247-9d12-c0bf8f1dfbfb public_network = 172.168.0.0/16 cluster_network = 10.0.0.0/16 mon_initial_members = ceph-mon01 mon_host = 172.168.32.104 auth_cluster_required = cephx auth_service_required = cephx auth_client_required = cephx [mds.ceph-mds02] #mds_standby_for_fscid = mycephfs mds_standby_for_name = ceph-mds01 mds_standby_replay = true [mds.ceph-mds03] mds_standby_for_name = ceph-mgr02 mds_standby_replay = true
分發配置文件並重啟mds 服務:
#分發配置文件保證各mds 服務重啟有效 cephadmin@ceph-deploy:/etc/ceph$ceph-deploy --overwrite-conf config push ceph-mon03 cephadmin@ceph-deploy:/etc/ceph$ceph-deploy --overwrite-conf config push ceph-mon02 cephadmin@ceph-deploy:/etc/ceph$ceph-deploy --overwrite-conf config push ceph-mon01 cephadmin@ceph-deploy:/etc/ceph$ceph-deploy --overwrite-conf config push ceph-mgr02 #重啟mds節點，先重啟active節點ceph-mds01和ceph-mgr02,再重啟standby節點ceph-mds02和ceph-mds03 cephadmin@ceph-mon01:~$ systemctl restart ceph-mds@ceph-mds01.service cephadmin@ceph-mgr02:~$ systemctl restart ceph-mds@ceph-mgr02.service cephadmin@ceph-mon02:~$ systemctl restart ceph-mds@ceph-mds02.service cephadmin@ceph-mon03:~$ systemctl restart ceph-mds@ceph-mds03.service
驗證mds
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status
mycephfs - 0 clients
========
RANK STATE MDS ACTIVITY DNS INOS DIRS CAPS
0 active ceph-mgr02 Reqs: 0 /s 10 13 12 0
1 active ceph-mds01 Reqs: 0 /s 10 13 11 0
POOL TYPE USED AVAIL
cephfs-metadata metadata 168k 253G
cephfs-data data 0 253G
STANDBY MDS
ceph-mds03
ceph-mds02
查看active 和standby 對應關系：
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs get mycephfs
Filesystem 'mycephfs' (1)
fs_name mycephfs
epoch 34
flags 12
created 2021-08-30T14:17:05.882172+0800
modified 2021-08-30T15:11:24.372592+0800
tableserver 0
root 0
session_timeout 60
session_autoclose 300
max_file_size 1099511627776
required_client_features {}
last_failure 0
last_failure_osd_epoch 103
compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2}
max_mds 2
in 0,1
up {0=14176,1=4653}
failed
damaged
stopped
data_pools [3]
metadata_pool 2
inline_data disabled
balancer
standby_count_wanted 1
[mds.ceph-mgr02{0:14176} state up:active seq 10 addr [v2:172.168.32.103:6800/4136492160,v1:172.168.32.103:6801/4136492160]]
[mds.ceph-mds01{1:4653} state up:active seq 7 addr [v2:172.168.32.104:6800/3715813078,v1:172.168.32.104:6801/3715813078]]
將ceph-mgr01、ceph-mgr02 服務器部署為高可用的radosGW 服務
1)、在ceph-mgr01和ceph-mgr02上安裝ceph-radosgw
root@ceph-mgr01:~# apt install radosgw -y root@ceph-mgr02:~# apt install radosgw -y
#注意：如果前面有做ceph.conf的配置變動，需要將ceph.conf重新拷貝到ceph-mgr01和cpeh-mgr02上 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy rgw create ceph-mgr01 cephadmin@ceph-deploy:~/ceph-cluster$ ceph-deploy rgw create ceph-mgr02
在mgr節點上查看radosgw節點信息
root@ceph-mgr01:~# ps -ef|grep radosgw ceph 13551 1 0 15:19 ? 00:00:00 /usr/bin/radosgw -f --cluster ceph --name client.rgw.ceph-mgr01 --setuser ceph --setgroup ceph root@ceph-mgr02:~# ps -ef|grep radosgw ceph 13832 1 0 15:19 ? 00:00:00 /usr/bin/radosgw -f --cluster ceph --name client.rgw.ceph-mgr02 --setuser ceph --setgroup ceph
在ceph-deploy上查看集群的radosgw信息
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
cluster:
id: c31ea2e3-47f7-4247-9d12-c0bf8f1dfbfb
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph-mon01,ceph-mon02,ceph-mon03 (age 3h)
mgr: ceph-mgr01(active, since 3h), standbys: ceph-mgr02
mds: 2/2 daemons up, 2 standby
osd: 16 osds: 16 up (since 3h), 16 in (since 3h)
rgw: 2 daemons active (2 hosts, 1 zones)
data:
volumes: 1/1 healthy
pools: 7 pools, 208 pgs
objects: 230 objects, 8.5 KiB
usage: 146 MiB used, 800 GiB / 800 GiB avail
pgs: 0.481% pgs not active
207 active+clean
1 peering
progress:
Global Recovery Event (0s)
[............................]
cephadmin@ceph-deploy:~/ceph-cluster$ curl http://172.168.32.102:7480 <?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID><DisplayName></DisplayName></Owner><Buckets></Buckets></ListAllMyBucketsResult>cephadmin@ceph-deploy:~/ceph-cluster$ cephadmin@ceph-deploy:~/ceph-cluster$ curl http://172.168.32.103:7480 <?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID><DisplayName></DisplayName></Owner><Buckets></Buckets></ListAllMyBucketsResult>cephadmin@ceph-deploy:~/ceph-cluster$