ceph FS 即ceph filesystem，可以實現文件系統共享功能,客戶端通過ceph 協議掛載並使用ceph 集群作為數據存儲服務器。 Ceph FS 需要運行Meta Data Services(MDS)服務，其守護進程為ceph-mds，ceph-mds 進程管理與cephFS 上存儲的文件相關的元數據，並協調對ceph 存儲集群的訪問。
cephadmin@ceph-deploy:~/ceph-cluster$ ceph mds stat
mycephfs:2 {0=ceph-mds01=up:active,1=ceph-mgr02=up:active} 2 up:standby
# 創建client.zhai普通用戶 cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph auth add client.zhai mon 'allow r' mds "allow rw" osd "allow rwx pool=cephfs-data" added key for client.zhai # 驗證用戶 cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph auth get client.zhai [client.zhai] key = AQC8wElif/VWCRAAiETOB+qmMdhKEUz/kMN85w== caps mds = "allow rw" caps mon = "allow r" caps osd = "allow rwx pool=cephfs-data" exported keyring for client.zhai # 創建kingseal的空keyring cephadmin@ceph-deploy:~/ceph-cluster$ ceph-authtool --create-keyring ceph.client.zhai.keyring creating ceph.client.zhai.keyring # 將用戶信息導入keyring文件 cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth get client.zhai -o ceph.client.zhai.keyring exported keyring for client.zhai # 創建key文件，cephfs掛載使用 cephadmin@ceph-deploy:~/ceph-cluster$ ceph auth print-key client.zhai AQC8wElif/VWCRAAiETOB+qmMdhKEUz/kMN85w==
2.3、
cephadmin@ceph-deploy:~/ceph-cluster$ sudo scp ceph.conf ceph.client.zhai.keyring zhai.key 192.168.1.180:/etc/ceph
2.4、客戶端驗證權限
root@client:~# ceph --user zhai -s
cluster:
id: fdefc83b-1ef9-4986-a4c1-8af7603f43bf
health: HEALTH_WARN
clock skew detected on mon.ceph-mon03
services:
mon: 3 daemons, quorum ceph-mon01,ceph-mon02,ceph-mon03 (age 37m)
mgr: ceph-mgr01(active, since 56m), standbys: ceph-mgr02
mds: 2/2 daemons up, 2 standby
osd: 16 osds: 16 up (since 41m), 16 in (since 41m)
data:
volumes: 1/1 healthy
pools: 3 pools, 97 pgs
objects: 41 objects, 3.6 KiB
usage: 104 MiB used, 800 GiB / 800 GiB avail
pgs: 97 active+clean
3、掛載cephfs
3.1、客戶端通過key文件掛載
#創建掛載目錄cephfs-key root@client:~# mkdir /cephfs-key #掛載cephfs root@ceph-client:/etc/ceph# mount -t ceph 192.168.1.203:6789,192.168.1.204:6789,192.168.1.205:6789:/ /cephfs-key -o name=zhai,secretfile=/etc/ceph/zhai.key #注意：192.168.1.203:6789,192.168.1.204:6789,192.168.1.205:6789為ceph-mon地址和端口
root@ceph-client:/etc/ceph# cat zhai.key AQC8wElif/VWCRAAiETOB+qmMdhKEUz/kMN85w== root@ceph-client:/etc/ceph# mount -t ceph 192.168.1.203:6789,192.168.1.204:6789,192.168.1.205:6789:/ /cephfs-key -o name=zhai,secretfile=/etc/ceph/zhai.key
客戶端內核加載ceph.ko 模塊掛載cephfs 文件系統
root@client:~# lsmod | grep ceph ceph 376832 1 libceph 315392 1 ceph fscache 65536 1 ceph libcrc32c 16384 3 xfs,raid456,libceph root@client:~# modinfo ceph filename: /lib/modules/4.15.0-112-generic/kernel/fs/ceph/ceph.ko license: GPL description: Ceph filesystem for Linux author: Patience Warnick <patience@newdream.net> author: Yehuda Sadeh <yehuda@hq.newdream.net> author: Sage Weil <sage@newdream.net> alias: fs-ceph srcversion: B2806F4EAACAC1E19EE7AFA depends: libceph,fscache retpoline: Y intree: Y name: ceph vermagic: 4.15.0-112-generic SMP mod_unload signat: PKCS#7 signer: sig_key: sig_hashalgo: md4
#安裝ceph-fuse apt install ceph-fuse #創建掛載目錄ceph-fsus mkdir /ceph-fsus #使用ceph-fsus掛載 root@client:~# ll /etc/ceph/ total 32 drwxr-xr-x 2 root root 126 Aug 25 23:07 ./ drwxr-xr-x 100 root root 8192 Aug 25 23:07 ../ -rw------- 1 root root 151 Aug 25 22:49 ceph.client.admin.keyring -rw------- 1 root root 152 Aug 25 23:02 ceph.client.zhai.keyring -rw-r--r-- 1 root root 516 Aug 25 23:02 ceph.conf -rw-r--r-- 1 root root 40 Aug 25 23:02 zhai.key -rw-r--r-- 1 root root 92 Jun 7 22:39 rbdmap #client.zhai為ceph.client.zhai.keyring root@client:~# ceph-fuse --name client.zhai -m 192.168.1.203:6789,192.168.1.204:6789,192.168.1.205:6789 /ceph-fsus ceph-fuse[5569]: starting ceph client 2021-08-25T23:28:07.140+0800 7f4feeea9100 -1 init, newargv = 0x55a107cc4920 newargc=15 ceph-fuse[5569]: starting fuse
3.4、驗證
root@ceph-client:/etc/ceph# df -Th .......... 192.168.1.203:6789,192.168.1.204:6789,192.168.1.205:6789:/ ceph 38G 0 38G 0% /cephfs-key ceph-fuse fuse.ceph-fuse 254G 0 254G 0% /ceph-fsus
4、開機掛載
cat /etc/fstab 192.168.1.203:6789,192.168.1.204:6789,192.168.1.205:6789:/ /cephfs-key ceph defaults,name=zhai,secretfile=/etc/ceph/zhai.key,_netdev 0 0
4.2、rc.local
chmod +x /etc/rc.local cat /etc/rc.local mount -t ceph 192.168.1.203:6789,192.168.1.204:6789,192.168.1.205:6789:/ /cephfs-key -o name=zhai,secretfile=/etc/ceph/zhai.key