k8s安裝nfs

節點規划

192.168.188.133   centos7  2U2G  50G  k8s-nfs

搭建nfs服務

#master節點安裝nfs
yum -y install nfs-utils

#創建nfs目錄
mkdir -p /nfs/data/

#修改權限
chmod -R 777 /nfs/data

#編輯export文件
vim /etc/exports
/nfs/data *(rw,no_root_squash,sync)

#配置生效
exportfs -r
#查看生效
exportfs

#啟動rpcbind、nfs服務
systemctl restart rpcbind && systemctl enable rpcbind
systemctl restart nfs && systemctl enable nfs

#查看 RPC 服務的注冊狀況
rpcinfo -p localhost

#showmount測試
showmount -e 192.168.188.133

#所有節點安裝客戶端
yum -y install nfs-utils
systemctl start nfs && systemctl enable nfs

至此NFS 服務器搭建完成，目錄為 /nfs/data

測試

利用NFS client provisioner動態提供Kubernetes后端存儲卷
利用NFS Server給Kubernetes作為持久存儲的后端，並且動態提供PV。前提條件是有已經安裝好的NFS服務器，並且NFS服務器與Kubernetes的Slave節點都能網絡連通。
所有下文用到的文件來自於git clone https://github.com/kubernetes-incubator/external-storage.git的nfs-client目錄

nfs-client-provisioner：nfs-client-provisioner 是一個Kubernetes的簡易NFS的外部provisioner，本身不提供NFS，需要現有的NFS服務器提供存儲
PV以 ${namespace}-${pvcName}-${pvName}的命名格式提供（在NFS服務器上）
PV回收的時候以 archieved-${namespace}-${pvcName}-${pvName} 的命名格式（在NFS服務器上）

安裝部署

修改deployment文件並部署 deploy/deployment.yaml
需要修改的地方只有NFS服務器所在的IP地址，以及NFS服務器共享的路徑（/ifs/kubernetes），兩處都需要修改為你實際的NFS服務器和共享目錄

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nfs-client-provisioner
  labels:
    app: nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nfs-client-provisioner
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: nfs-client-provisioner
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccountName: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          image: registry.cn-hangzhou.aliyuncs.com/open-ali/nfs-client-provisioner:latest
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: fuseim.pri/ifs
            - name: NFS_SERVER
              value: 192.168.188.133
            - name: NFS_PATH
              value: /home/nfs/data
      volumes:
        - name: nfs-client-root
          nfs:
            server: 192.168.188.133
            path: /home/nfs/data

修改StorageClass文件並部署 deploy/class.yaml

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: default
provisioner: fuseim.pri/ifs

設置這個default名字的SC為Kubernetes的默認存儲后端

kubectl patch storageclass default -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'

# kubectl get sc

NAME                PROVISIONER      AGE
default (default)   fuseim.pri/ifs   44m

授權

如果您的集群啟用了RBAC，或者您正在運行OpenShift，則必須授權provisioner

kubectl apply -f rbac.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: default
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    # replace with namespace where provisioner is deployed
    namespace: default
roleRef:
  kind: ClusterRole
  name: nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
    # replace with namespace where provisioner is deployed
  namespace: default
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    # replace with namespace where provisioner is deployed
    namespace: default
roleRef:
  kind: Role
  name: leader-locking-nfs-client-provisioner
  apiGroup: rbac.authorization.k8s.io

測試創建PVC

kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: test-claim
  annotations:
    volume.beta.kubernetes.io/storage-class: "es-data-db"
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Mi

kubectl create -f deploy/test-claim.yaml

]# kubectl get pvc

NAME         STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
test-claim   Bound    pvc-c840caaa-2977-11ea-9ae2-000c29ff9863   1Mi        RWX            es-data-db     14m

測試創建POD

kind: Pod
apiVersion: v1
metadata:
  name: test-pod
spec:
  containers:
  - name: test-pod
    image: busybox:1.24
    command:
      - "/bin/sh"
    args:
      - "-c"
      - "touch /mnt/SUCCESS && exit 0 || exit 1"
    volumeMounts:
      - name: nfs-pvc
        mountPath: "/mnt"
  restartPolicy: "Never"
  volumes:
    - name: nfs-pvc
      persistentVolumeClaim:
        claimName: test-claim

kubectl create -f deploy/test-pod.yaml

]# kubectl get pods | grep test

test-pod                                  0/1     Completed   0          11m

在NFS服務器上的共享目錄下的卷子目錄中檢查創建的NFS PV卷下是否有”SUCCESS” 文件

]# cd /home/nfs/data/
[root@efk-mysql data]# ll

total 0
drwxrwxrwx 2 root root 21 Dec 28 21:44 default-test-claim-pvc-c840caaa-2977-11ea-9ae2-000c29ff9863
drwxrwxrwx 2 root root 6 Dec 28 21:39 efk-data-es-cluster-0-pvc-0d4dc2a7-2957-11ea-9ae2-000c29ff9863
drwxrwxrwx 2 root root 6 Dec 28 21:39 efk-elasticsearch-logging-elasticsearch-logging-0-pvc-17148f41-295c-11ea-9ae2-000c29ff9863
[root@efk-mysql data]# ll default-test-claim-pvc-c840caaa-2977-11ea-9ae2-000c29ff9863
total 0
-rw-r--r-- 1 root root 0 Dec 28 21:44 SUCCESS
[root@efk-mysql data]#

刪除測試POD

kubectl delete -f deploy/test-pod.yaml

刪除測試PVC

kubectl delete -f deploy/test-claim.yaml

在NFS服務器上的共享目錄下查看NFS的PV卷回收以后是否名字以archived開頭
]# ll
total 0
drwxrwxrwx 2 root root 21 Dec 28 21:44 archived-default-test-claim-pvc-c840caaa-2977-11ea-9ae2-000c29ff9863
drwxrwxrwx 2 root root 6 Dec 28 21:39 efk-data-es-cluster-0-pvc-0d4dc2a7-2957-11ea-9ae2-000c29ff9863
drwxrwxrwx 2 root root 6 Dec 28 21:39 efk-elasticsearch-logging-elasticsearch-logging-0-pvc-17148f41-295c-11ea-9ae2-000c29ff9863