一、實驗環境最低配置
- 做這個實驗需要高配置，每個節點配置不能低於2核4G
- k8s 1.19以上版本，快照功能需要單獨安裝snapshot控制器
- rook的版本大於1.3，不要使用目錄創建集群，要使用單獨的裸盤進行創建，也就是創建一個新的磁盤，掛載到宿主機，不進行格式化，直接使用即可。對於的磁盤節點配置如下
[root@k8s-master01 ~]# fdisk -l
Disk /dev/sda: 42.9 GB, 42949672960 bytes, 83886080 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk label type: dos
Disk identifier: 0x000d76eb
Device Boot Start End Blocks Id System
/dev/sda1 * 2048 2099199 1048576 83 Linux
/dev/sda2 2099200 83886079 40893440 8e Linux LVM
Disk /dev/sdb: 10.7 GB, 10737418240 bytes, 20971520 sectors # 新的磁盤
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
二、部署Rook
2.1、Rook官方文檔
https://rook.io/docs/rook/v1.5/ceph-quickstart.html
2.2、下載Rook安裝文件
[root@k8s-master01 app]# git clone --single-branch --branch v1.5.3 https://github.com/rook/rook.git
2.3、配置更改
[root@k8s-master01 app]# cd rook/cluster/examples/kubernetes/ceph
# 修改Rook CSI鏡像地址，原本的地址可能是gcr的鏡像，但是gcr的鏡像無法被國內訪問，所以需要同步gcr的鏡像到阿里雲鏡像倉庫，文檔版本已經為大家完成同步，可以直接修改如下：
[root@k8s-master01 ceph]# vim operator.yaml
## 47-52行更改為：
ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.1.2"
ROOK_CSI_REGISTRAR_IMAGE: "registry.cn-beijing.aliyuncs.com/dotbalo/csi-node-driver-registrar:v2.0.1"
ROOK_CSI_RESIZER_IMAGE: "registry.cn-beijing.aliyuncs.com/dotbalo/csi-resizer:v1.0.0"
ROOK_CSI_PROVISIONER_IMAGE: "registry.cn-beijing.aliyuncs.com/dotbalo/csi-provisioner:v2.0.0"
ROOK_CSI_SNAPSHOTTER_IMAGE: "registry.cn-beijing.aliyuncs.com/dotbalo/csi-snapshotter:v3.0.0"
ROOK_CSI_ATTACHER_IMAGE: "registry.cn-beijing.aliyuncs.com/dotbalo/csi-attacher:v3.0.0"
##
# 如果是其他版本，需要自行同步，同步方法可以在網上找到相關文章。
# 還是operator文件，新版本rook默認關閉了自動發現容器的部署，可以找到ROOK_ENABLE_DISCOVERY_DAEMON改成true即可：
# ROOK_ENABLE_DISCOVERY_DAEMON改成true即可：
- name: ROOK_ENABLE_DISCOVERY_DAEMON
value: "true"
2.4、部署rook
# 1、進到/rook/cluster/examples/kubernetes/ceph目錄
[root@k8s-master01 ceph]# pwd
/app/rook/cluster/examples/kubernetes/ceph
# 2、部署
[root@k8s-master01 ceph]# kubectl create -f crds.yaml -f common.yaml -f operator.yaml
# 3、等待operator容器和discover容器啟動（全部變成1/1 Running 才可以創建Ceph集群）
[root@k8s-master01 ceph]# kubectl get pod -n rook-ceph -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
rook-ceph-operator-7d569f655-6bcjv 1/1 Running 0 6m37s 10.244.195.13 k8s-master03
rook-discover-bdk7k 1/1 Running 0 4m2s 10.244.32.148 k8s-master01
rook-discover-j6w4m 1/1 Running 0 4m2s 10.244.58.247 k8s-node02
rook-discover-pnp52 1/1 Running 0 4m2s 10.244.122.136 k8s-master02
rook-discover-spw8l 1/1 Running 0 4m2s 10.244.195.21 k8s-master03
rook-discover-vcqh2 1/1 Running 0 4m2s 10.244.85.248 k8s-node01
三、創建ceph集群
3.1、配置更改
主要更改的是osd節點所在的位置
[root@k8s-master01 ceph]# vim cluster.yaml
# 1、更改storage（自己指定使用磁盤的節點）
###
原配置:
storage: # cluster level storage configuration and selection
useAllNodes: true
useAllDevices: true
更改為:
storage: # cluster level storage configuration and selection
useAllNodes: false
useAllDevices: false
###
- name: "k8s-master03"
devices:
- name: "sdb"
- name: "k8s-node01"
devices:
- name: "sdb"
- name: "k8s-node02"
devices:
- name: "sdb"
###
注意：新版必須采用裸盤，即未格式化的磁盤。其中k8s-master03 k8s-node01 node02有新加的一個磁盤，可以通過lsblk -f查看新添加的磁盤名稱。建議最少三個節點，否則后面的試驗可能會出現問題
3.2、創建Ceph集群
[root@k8s-master01 ceph]# kubectl create -f cluster.yaml
cephcluster.ceph.rook.io/rook-ceph created
# 創建完成后，可以查看pod的狀態
[root@k8s-master01 ceph]# kubectl -n rook-ceph get pod
NAME READY STATUS RESTARTS AGE
csi-cephfsplugin-2gp6j 3/3 Running 0 31m
csi-cephfsplugin-5bqp2 3/3 Running 0 17m
csi-cephfsplugin-df5xq 3/3 Running 0 31m
csi-cephfsplugin-gk8f8 3/3 Running 0 31m
csi-cephfsplugin-provisioner-785798bc8f-fcdng 6/6 Running 0 31m
csi-cephfsplugin-provisioner-785798bc8f-mkjpt 6/6 Running 4 31m
csi-cephfsplugin-xdw2t 3/3 Running 0 31m
csi-rbdplugin-8cs79 3/3 Running 0 31m
csi-rbdplugin-d4mrr 3/3 Running 0 31m
csi-rbdplugin-jg77k 3/3 Running 0 31m
csi-rbdplugin-ksq66 3/3 Running 0 21m
csi-rbdplugin-provisioner-75cdf8cd6d-gvwmn 6/6 Running 0 31m
csi-rbdplugin-provisioner-75cdf8cd6d-nqwrn 6/6 Running 5 31m
csi-rbdplugin-wqxbm 3/3 Running 0 31m
rook-ceph-crashcollector-k8s-master03-6f7c7b5fbc-rv4tc 1/1 Running 0 31m
rook-ceph-crashcollector-k8s-node01-6769bf677f-bsr7c 1/1 Running 0 31m
rook-ceph-crashcollector-k8s-node02-7c97d7b8d4-6xgkb 1/1 Running 0 31m
rook-ceph-mgr-a-75fc775496-cqjmh 1/1 Running 1 32m
rook-ceph-mon-a-67cbdcd6d6-hpttq 1/1 Running 0 33m
rook-ceph-operator-7d569f655-6bcjv 1/1 Running 0 69m
rook-ceph-osd-0-9c67b5cb4-729r6 1/1 Running 0 31m
rook-ceph-osd-1-56cd8467fc-bbwcc 1/1 Running 0 31m
rook-ceph-osd-2-74f5c9f8d8-fwlw7 1/1 Running 0 31m
rook-ceph-osd-prepare-k8s-master03-kzgbd 0/1 Completed 0 94s
rook-ceph-osd-prepare-k8s-node01-hzcdw 0/1 Completed 0 92s
rook-ceph-osd-prepare-k8s-node02-pxfcc 0/1 Completed 0 90s
rook-discover-bdk7k 1/1 Running 0 67m
rook-discover-j6w4m 1/1 Running 0 67m
rook-discover-pnp52 1/1 Running 0 67m
rook-discover-spw8l 1/1 Running 0 67m
rook-discover-vcqh2 1/1 Running 0 67m
3.3、安裝ceph snapshot控制器
k8s 1.19版本以上需要單獨安裝snapshot控制器，才能完成pvc的快照功能，所以在此提前安裝下，如果是1.19以下版本，不需要單獨安裝，直接參考視頻即可。
# 1、snapshot控制器的部署在集群安裝時的k8s-ha-install項目中，需要切換到1.20.x分支
[root@k8s-master01 ~]# cd /root/k8s-ha-install/
[root@k8s-master01 k8s-ha-install]# git checkout manual-installation-v1.20.x
# 2、創建snapshot controller
[root@k8s-master01 k8s-ha-install]# kubectl create -f snapshotter/ -n kube-system
# 3、查看snapshot controller狀態
[root@k8s-master01 k8s-ha-install]# kubectl get po -n kube-system -l app=snapshot-controller
NAME READY STATUS RESTARTS AGE
snapshot-controller-0 1/1 Running 0 15s
# 4、具體文檔
具體文檔：https://rook.io/docs/rook/v1.5/ceph-csi-snapshot.html
四、安裝ceph客戶端工具
# 1、安裝
[root@k8s-master01 ceph]# pwd
/app/rook/cluster/examples/kubernetes/ceph
[root@k8s-master01 ceph]# kubectl create -f toolbox.yaml -n rook-ceph
deployment.apps/rook-ceph-tools created
# 2、待容器Running后，即可執行相關命令
[root@k8s-master01 ceph]# kubectl get po -n rook-ceph -l app=rook-ceph-tools
NAME READY STATUS RESTARTS AGE
rook-ceph-tools-6f7467bb4d-r9vqx 1/1 Running 0 31s
# 3、執行命令ceph status
[root@k8s-master01 ceph]# kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- bash
[root@rook-ceph-tools-6f7467bb4d-r9vqx /]# ceph status
cluster:
id: 83c11641-ca98-4054-b2e7-422e942befe6
health: HEALTH_OK
services:
mon: 1 daemons, quorum a (age 43m)
mgr: a(active, since 13m)
osd: 3 osds: 3 up (since 18m), 3 in (since 44m)
data:
pools: 1 pools, 1 pgs
objects: 0 objects, 0 B
usage: 3.0 GiB used, 27 GiB / 30 GiB avail
pgs: 1 active+clean
# 4、執行命令
[root@rook-ceph-tools-6f7467bb4d-r9vqx /]# ceph osd status
ID HOST USED AVAIL WR OPS WR DATA RD OPS RD DATA STATE
0 k8s-master03 1028M 9207M 0 0 0 0 exists,up
1 k8s-node01 1028M 9207M 0 0 0 0 exists,up
2 k8s-node02 1028M 9207M 0 0 0 0 exists,up
# 5、執行命令-查看狀態
[root@rook-ceph-tools-6f7467bb4d-r9vqx /]# ceph df
--- RAW STORAGE ---
CLASS SIZE AVAIL USED RAW USED %RAW USED
hdd 30 GiB 27 GiB 14 MiB 3.0 GiB 10.05
TOTAL 30 GiB 27 GiB 14 MiB 3.0 GiB 10.05
--- POOLS ---
POOL ID STORED OBJECTS USED %USED MAX AVAIL
device_health_metrics 1 0 B 0 0 B 0 8.5 GiB
五、Ceph dashboard
5.1、暴露服務
# 1、默認情況下，ceph dashboard是打開的，可以通過以下命令查看ceph dashboard的service
[root@k8s-master01 ceph]# kubectl -n rook-ceph get service rook-ceph-mgr-dashboard
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
rook-ceph-mgr-dashboard ClusterIP 10.97.5.123 <none> 8443/TCP 47m
# 可以兩種方式訪問：
1. 將該service改為NodePort
2. 通過ingress代理
# 本文檔演示NodePort，ingress可以參考課程的ingress章節。
[root@k8s-master01 ceph]# kubectl -n rook-ceph edit service rook-ceph-mgr-dashboard
# 更改type類型即可
type: NodePort
# 2、訪問、任意節點ip:port訪問即可
[root@k8s-master01 ceph]# kubectl -n rook-ceph get service rook-ceph-mgr-dashboard
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
rook-ceph-mgr-dashboard NodePort 10.97.5.123 <none> 8443:32202/TCP 49m
# 3、登錄、賬號為admin，查看密碼
[root@k8s-master01 ~]# kubectl -n rook-ceph get secret rook-ceph-dashboard-password -o jsonpath="{['data']['password']}" | base64 --decode && echo
@}g"P{-FVe9yb]-AV/>3
六、ceph塊存儲的使用
塊存儲一般用於一個Pod掛載一塊存儲使用，相當於一個服務器新掛了一個盤，只給一個應用使用。
6.1、創建StorageClass和ceph的存儲池
# 1、創建文件
[root@k8s-master01 ~]# cd /app/rook/cluster/examples/kubernetes/ceph/
[root@k8s-master01 ceph]# vim storageclass.yaml
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
name: replicapool
namespace: rook-ceph
spec:
failureDomain: host
replicated:
size: 3
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: rook-ceph-block
# Change "rook-ceph" provisioner prefix to match the operator namespace if needed
provisioner: rook-ceph.rbd.csi.ceph.com
parameters:
# clusterID is the namespace where the rook cluster is running
clusterID: rook-ceph
# Ceph pool into which the RBD image shall be created
pool: replicapool
imageFormat: "2"
imageFeatures: layering
csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
csi.storage.k8s.io/fstype: ext4
# Delete the rbd volume when a PVC is deleted
reclaimPolicy: Delete
# 2、創建塊
[root@k8s-master01 ceph]# kubectl create -f storageclass.yaml
cephblockpool.ceph.rook.io/replicapool created
storageclass.storage.k8s.io/rook-ceph-block created
# 3、查看狀態
[root@k8s-master01 ceph]# kubectl get CephBlockPool -n rook-ceph
NAME AGE
replicapool 2m14s
[root@k8s-master01 ceph]# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
rook-ceph-block rook-ceph.rbd.csi.ceph.com Delete Immediate false 2m47s
此時可以在ceph dashboard查看到改Pool，如果沒有顯示說明沒有創建成功
6.2、掛載測試
創建一個MySQL服務
[root@k8s-master01 kubernetes]# pwd
/app/rook/cluster/examples/kubernetes
[root@k8s-master01 kubernetes]# kubectl create -f mysql.yaml
[root@k8s-master01 kubernetes]# kubectl create -f wordpress.yaml
# 查看svc
[root@k8s-master01 kubernetes]# kubectl get svc wordpress
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
wordpress LoadBalancer 10.109.161.119 <pending> 80:32301/TCP 3m57s
該文件有一段pvc的配置
pvc會連接剛才創建的storageClass，然后動態創建pv，然后連接到ceph創建對應的存儲
之后創建pvc只需要指定storageClassName為剛才創建的StorageClass名稱即可連接到rook的ceph。如果是statefulset，只需要將volumeTemplateClaim里面的Claim名稱改為StorageClass名稱即可動態創建Pod，具體請聽視頻。
其中MySQL deployment的volumes配置掛載了該pvc：
claimName為pvc的名稱
因為MySQL的數據不能多個MySQL實例連接同一個存儲，所以一般只能用塊存儲。相當於新加了一塊盤給MySQL使用。
創建完成后可以查看創建的pvc和pv
[root@k8s-master01 kubernetes]# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvc-1843c13e-09cb-46c6-9dd8-5f54a834681b 20Gi RWO Delete Bound default/mysql-pv-claim rook-ceph-block 65m
[root@k8s-master01 kubernetes]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
mysql-pv-claim Bound pvc-1843c13e-09cb-46c6-9dd8-5f54a834681b 20Gi RWO rook-ceph-block 66m
此時在ceph dashboard上面也可以查看到對應的image
七、共享文件系統的使用
共享文件系統一般用於多個Pod共享一個存儲
默認情況下，只能使用Rook創建一個共享文件系統。Ceph中的多文件系統支持仍被認為是實驗性的，可以使用中ROOK_ALLOW_MULTIPLE_FILESYSTEMS定義的環境變量啟用operator.yaml。
7.1、創建共享類型的文件系統
通過為CephFilesystemCRD中的元數據池，數據池和元數據服務器指定所需的設置來創建文件系統
[root@k8s-master01 kubernetes]# pwd
/app/rook/cluster/examples/kubernetes
[root@k8s-master01 kubernetes]# vim filesystem.yaml
apiVersion: ceph.rook.io/v1
kind: CephFilesystem
metadata:
name: myfs
namespace: rook-ceph
spec:
metadataPool: # 原數據副本數
replicated:
size: 3
dataPools: # 數據副本數
- replicated:
size: 3
preserveFilesystemOnDelete: true
metadataServer: # 原數據服務副本數
activeCount: 1
activeStandby: true # 啟了個從節點
# 創建
[root@k8s-master01 kubernetes]# kubectl create -f filesystem.yaml
cephfilesystem.ceph.rook.io/myfs created
# 查看，一個主，一個備
[root@k8s-master01 kubernetes]# kubectl -n rook-ceph get pod -l app=rook-ceph-mds
NAME READY STATUS RESTARTS AGE
rook-ceph-mds-myfs-a-5d8547c74d-vfvx2 1/1 Running 0 90s
rook-ceph-mds-myfs-b-766d84d7cb-wj7nd 1/1 Running 0 87s
也可以在ceph dashboard上面查看狀態
7.2、創建共享類型文件系統的StorageClass
官網：https://rook.io/docs/rook/v1.5/ceph-filesystem.html
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: rook-cephfs
# Change "rook-ceph" provisioner prefix to match the operator namespace if needed
provisioner: rook-ceph.cephfs.csi.ceph.com
parameters:
# clusterID is the namespace where operator is deployed.
clusterID: rook-ceph
# CephFS filesystem name into which the volume shall be created
fsName: myfs
# Ceph pool into which the volume shall be created
# Required for provisionVolume: "true"
pool: myfs-data0
# Root path of an existing CephFS volume
# Required for provisionVolume: "false"
# rootPath: /absolute/path
# The secrets contain Ceph admin credentials. These are generated automatically by the operator
# in the same namespace as the cluster.
csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
reclaimPolicy: Delete
八、PVC擴容、快照、回滾
官方文檔：https://rook.io/docs/rook/v1.5/ceph-csi-snapshot.html
8.1、快照
注意：PVC快照功能需要k8s 1.17+