原文地址: http://maoqide.live/post/cloud/deploy-mysql-on-kubernetes/
本文通過 mysql-operator 在kubernetes集群部署高可用的mysql statefulset。
環境准備
本文使用的開源 operator 項目 mysql-operator 配死只支持 mysql 8.0.11 以上的版本，改了下代碼，支持 5.7.0 以上版本，項目地址，本文部署的是 mysql-5.7.26，使用的 dockerhub 上的鏡像 mysql/mysql-server:5.7.26。
代碼編譯
git clone 下載該項目，進入到代碼目錄，執行sh hack/build.sh，編譯代碼得到二進制文件 mysql-agent 和 mysql-operator，將二進制文件放入 bin/linux_amd64，執行docker build -f docker/mysql-agent/Dockerfile -t $IMAGE_NAME_AGENT .，docker build -f docker/mysql-operator/Dockerfile -t $IMAGE_NAME_OPERATOR .構建鏡像，mysql-operator 生成的鏡像為 operator 的鏡像，mysql-agent 生成的是鏡像，在創建mysql服務時，作為sidecar和mysql-server容器起在同一個pod中。
部署 operator
先根據 文檔 部署 mysql-operator 的 Deployment，文檔中是使用 helm 安裝，不希望安裝 helm 和 tiller 的話，可以只安裝一個 helm 客戶端，進入到代碼目錄，再執行helm template --name mysql-operator mysql-operator生成部署所需要的yaml文件，然后直接執行kubectl apply -f mysql-operator.yaml創建 operator。這個yaml創建了operator所需的CRD類型，operator 的 Deployment 和 operator 所需的 RBAC 權限等。
# change directory into mysql-operator
cd mysql-operator
# generate mysql-operator.yaml
helm template --name mysql-operator mysql-operator > mysql-operator.yaml
# deploy on kubernetes
kubectl apply -f mysql-operator.yaml
# deployed.
[root@localhost]$ kubectl get deploy -n mysql-operator
NAME READY UP-TO-DATE AVAILABLE AGE
mysql-operator 1/1 1 1 2d5h
創建 mysql 集群
本文創建的集群為3節點的 mysql，一個節點為 master，二個為 slave，master節點可讀寫，slave節點為只讀，使用 kubernetes Local PV 作持久化存儲。
首先，為每個節點創建一個PV，Local PV 需要定義nodeAffinity，約束創建的節點。
apiVersion: v1
kind: PersistentVolume
metadata:
name: mypv0
spec:
capacity:
storage: 1Gi
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Delete
storageClassName: mysql-storage
local:
path: /data/mysql-data
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- 192.168.0.1
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: mypv1
spec:
capacity:
storage: 1Gi
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Delete
storageClassName: mysql-storage
local:
path: /data/mysql-data
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- 192.168.0.2
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: mypv2
spec:
capacity:
storage: 1Gi
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Delete
storageClassName: mysql-storage
local:
path: /data/mysql-data
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- 192.168.0.3
# create pv
kubectl create -f pv.yaml
# get presistence volume
[root@localhost]$ kubectl get pv
mypv-0 1Gi RWO Delete Available mysql-storage 4s
mypv-1 1Gi RWO Delete Available mysql-storage 4s
mypv-2 1Gi RWO Delete Available mysql-storage 4s
接着，需要在創建 mysql 的 namespace 下，為要創建的 mysql 創建對應的 RBAC 權限。
apiVersion: v1
kind: ServiceAccount
metadata:
name: mysql-agent
namespace: mysql2
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: mysql-agent
namespace: mysql2
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: mysql-agent
subjects:
- kind: ServiceAccount
name: mysql-agent
namespace: mysql2
如果需要自定義 mysql 的密碼，需要為其創建一個 secret，密碼需要使用base64加密。linux 下執行 echo -n 'password' | base64 為密碼加密。
apiVersion: v1
data:
password: cm9vdA==
kind: Secret
metadata:
labels:
v1alpha1.mysql.oracle.com/cluster: mysql
name: mysql-pv-root-password
namespace: mysql2
kubectl apply -f rbac.yaml
kubectl apply -f secret.yaml
在創建 operator 的時候，已經創建了如下的crd類型，部署mysql集群所需創建的就是 mysqlclusters 類型的資源。
[root@localhost]$ kubectl get crd | grep mysql
mysqlbackups.mysql.oracle.com 2019-05-14T02:51:11Z
mysqlbackupschedules.mysql.oracle.com 2019-05-14T02:51:11Z
mysqlclusters.mysql.oracle.com 2019-05-14T02:51:11Z
mysqlrestores.mysql.oracle.com 2019-05-14T02:51:11Z
接下來開始創建 operator 自定義資源類型(CRD)的實例 mysqlclusters。
apiVersion: mysql.oracle.com/v1alpha1
kind: Cluster
metadata:
name: mysql
namespace: mysql2
spec:
# 和mysql-server鏡像版本的tag一直
version: 5.7.26
repository: 20.26.28.56/dcos/mysql-server
# 節點數量
members: 3
# 指定 mysql 密碼，和之前創建的secret名稱一致
rootPasswordSecret:
name: mysql-pv-root-password
resources:
agent:
limits:
cpu: 500m
memory: 200Mi
requests:
cpu: 300m
memory: 100Mi
server:
limits:
cpu: 1000m
memory: 1000Mi
requests:
cpu: 500m
memory: 500Mi
volumeClaimTemplate:
metadata:
name: mysql-pv
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: "mysql-storage"
resources:
requests:
storage: 1Gi
kubectl apply -f mysql.yaml
執行后，會看到 kubernetes 在該 namespace 下開始拉起 mysql 的 statefulset，並會創建一個 headless service。
[root@localhost]$ kubectl get all -n mysql2
NAME READY STATUS RESTARTS AGE
pod/mysql-0 2/2 Running 0 8h
pod/mysql-1 2/2 Running 0 8h
pod/mysql-2 2/2 Running 0 8h
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/mysql ClusterIP None <none> 3306/TCP 21h
NAME READY AGE
statefulset.apps/mysql 1/1 21h
此時執行hack/cluster-status.sh腳本，會得到如下集群信息：
{
"clusterName": "Cluster",
"defaultReplicaSet": {
"name": "default",
"primary": "mysql-0.mysql:3306",
"ssl": "DISABLED",
"status": "OK_NO_TOLERANCE",
"statusText": "Cluster is NOT tolerant to any failures. 2 members are not active",
"topology": {
"mysql-0.mysql:3306": {
"address": "mysql-0.mysql:3306",
"mode": "R/W",
"readReplicas": {},
"role": "HA",
"status": "ONLINE"
},
"mysql-1.mysql:3306": {
"address": "mysql-1.mysql:3306",
"mode": "n/a",
"readReplicas": {},
"role": "HA",
"status": "ONLINE"
},
"mysql-2.mysql:3306": {
"address": "mysql-2.mysql:3306",
"mode": "n/a",
"readReplicas": {},
"role": "HA",
"status": "ONLINE"
}
},
"topologyMode": "Single-Primary"
},
"groupInformationSourceMember": "mysql-0.mysql:3306"
}
通過DNS地址 mysql-0.mysql.mysql2.svc.cluster.local:3306 可以連接到數據庫進行讀寫操作。此時一個多節點的mysql集群已經部署完成，但是，集群外部的服務還無法訪問數據庫。
通過 haproxy-ingress 允許外部訪問
首先，headless service只能通過集群內 DNS 訪問服務，要外部訪問，還需要另外創建一個 Service。為了讓外部可以訪問到 mysql-0 的服務，我們為 mysql-0 創建一個ClusterIP 類型的服務。
kind: Service
apiVersion: v1
metadata:
name: mysql-0
namespace: mysql2
spec:
selector:
# 通過 selector 將 pod 約束到 mysql-0
statefulset.kubernetes.io/pod-name: mysql-0
ports:
- protocol: TCP
port: 3306
targetPort: 3306
接着，需要創建一個ingress-controller，本文選用的是 haproxy-ingress。
由於 mysql 服務通過 TCP 協議通信，kubernetes ingress 默認只支持 http 和 https，haproxy-ingress 提供了通過 configmap 的方法，配置 TCP 服務的端口，需要先創建一個 configmap，configmap的data中，key為HAProxy監聽的端口，value 為需要轉發的 service 的服務和端口。
apiVersion: v1
kind: ConfigMap
metadata:
name: mysql-tcp
namespace: mysql2
data:
"3306": "mysql2/mysql-0:3306"
kubectl apply -f mysql-0.yaml
kubectl apply -f tcp-svc.yaml
接下來創建 ingress-controller，
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
labels:
run: haproxy-ingress
name: haproxy-ingress-192.168.0.1-30080
namespace: mysql2
spec:
replicas: 1
selector:
matchLabels:
run: haproxy-ingress
strategy:
type: RollingUpdate
template:
metadata:
labels:
run: haproxy-ingress
spec:
tolerations:
- key: app
operator: Equal
value: haproxy
effect: NoSchedule
serviceAccount: ingress-controller
nodeSelector:
kubernetes.io/hostname: 192.168.0.1
containers:
- args:
- --tcp-services-configmap=$(POD_NAMESPACE)/mysql-tcp
- --default-backend-service=$(POD_NAMESPACE)/mysql
- --default-ssl-certificate=$(POD_NAMESPACE)/tls-secret
- --ingress-class=ha-mysql
env:
- name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
image: jcmoraisjr/haproxy-ingress
name: haproxy-ingress
ports:
# 和 configmap 中定義的端口對應
- containerPort: 3306
hostPort: 3306
name: http
protocol: TCP
- containerPort: 443
name: https
protocol: TCP
- containerPort: 1936
hostPort: 30081
name: stat
protocol: TCP
apiVersion: v1
kind: ServiceAccount
metadata:
name: ingress-controller
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: ingress-controller
rules:
- apiGroups:
- ""
resources:
- configmaps
- pods
- secrets
- namespaces
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- apiGroups:
- ""
resources:
- endpoints
verbs:
- get
- create
- update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: ingress-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ingress-controller
subjects:
- kind: ServiceAccount
name: ingress-controller
- apiGroup: rbac.authorization.k8s.io
kind: User
name: ingress-controller
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: ingress-controller
rules:
- apiGroups:
- ""
resources:
- configmaps
- endpoints
- nodes
- pods
- secrets
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- "extensions"
resources:
- ingresses/status
verbs:
- update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: ingress-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: ingress-controller
subjects:
- kind: ServiceAccount
name: ingress-controller
namespace: mysql2
- apiGroup: rbac.authorization.k8s.io
kind: User
name: ingress-controller
kubectl apply -f ingress-controller.yaml
kubectl apply -f ingress-rbac.yaml -n mysql2
最后創建 ingress 規則：
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
annotations:
ingress.kubernetes.io/ssl-redirect: "false"
kubernetes.io/ingress.class: ha-mysql
name: ha-mysql
spec:
rules:
- http:
paths:
- backend:
serviceName: mysql-0
servicePort: 3306
path: /
此時可以通過 haproxy 的 IP + 映射端口訪問到 mysql 集群。
附件
以下是上面用到的 yaml 文件：