Apisix高可用落地部署

使用StatefulSet控制器配置etcd集群存储 (kubernetes)

配置PV存储卷

点击查看代码

apiVersion: v1
kind: PersistentVolume
metadata:
  name: etcd-pv-000
  labels:
    type: pv-etcd-000
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: etcdserver
  nfs:
    path: /ops/etcd-cluster/etcd00
    server: 10.30.112.254

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: etcd-pv-001
  labels:
    type: pv-etcd-001
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: etcdserver
  nfs:
    path: /ops/etcd-cluster/etcd01
    server: 10.30.112.254

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: etcd-pv-002
  labels:
    type: pv-etcd-002
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: etcdserver
  nfs:
    path: /ops/etcd-cluster/etcd02
    server: 10.30.112.254

配置etcd集群

点击查看代码

apiVersion: apps/v1
kind: StatefulSet
metadata:
  labels:
    app.kubernetes.io/instance: apisix
    app.kubernetes.io/name: etcd
    helm.sh/chart: etcd-6.2.6
  name: apisix-etcd
  namespace: elk
spec:
  podManagementPolicy: Parallel
  replicas: 3
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app.kubernetes.io/instance: apisix
      app.kubernetes.io/name: etcd
  serviceName: apisix-etcd-headless
  template:
    metadata:
      labels:
        app.kubernetes.io/instance: apisix
        app.kubernetes.io/name: etcd
        helm.sh/chart: etcd-6.2.6
    spec:
      # 反亲和 软限制 不与有以下标签的pod调度到同一个机器
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - podAffinityTerm:
              labelSelector:
                matchLabels:
                  app.kubernetes.io/instance: apisix
                  app.kubernetes.io/name: etcd
              topologyKey: kubernetes.io/hostname
            weight: 1
      containers:
      - env:
        - name: BITNAMI_DEBUG
          value: "false"
        - name: ETCD_HEADLESS
          value: "apisix-etcd-headless"
        - name: MY_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: MY_POD_IP
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: status.podIP
        - name: MY_POD_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.name
        - name: ETCDCTL_API
          value: "3"
        - name: ETCD_ON_K8S
          value: "yes"
        - name: ETCD_START_FROM_SNAPSHOT
          value: "no"
        - name: ETCD_DISASTER_RECOVERY
          value: "no"
        - name: ETCD_NAME
          value: $(MY_POD_NAME)
        - name: ETCD_DATA_DIR
          value: /bitnami/etcd/data
        - name: ETCD_LOG_LEVEL
          value: info
        - name: ALLOW_NONE_AUTHENTICATION
          value: "yes"
        - name: ETCD_ADVERTISE_CLIENT_URLS
          value: http://$(MY_POD_NAME).$(ETCD_HEADLESS).$(MY_NAMESPACE).svc.cluster.local:2379
        - name: ETCD_LISTEN_CLIENT_URLS
          value: http://0.0.0.0:2379
        - name: ETCD_INITIAL_ADVERTISE_PEER_URLS
          value: http://$(MY_POD_NAME).$(ETCD_HEADLESS).$(MY_NAMESPACE).svc.cluster.local:2380
        - name: ETCD_LISTEN_PEER_URLS
          value: http://0.0.0.0:2380
        - name: ETCD_INITIAL_CLUSTER_TOKEN
          value: etcd-cluster-k8s
        - name: ETCD_INITIAL_CLUSTER_STATE
          value: new
        - name: ETCD_INITIAL_CLUSTER
          value: apisix-etcd-0=http://apisix-etcd-0.$(ETCD_HEADLESS).$(MY_NAMESPACE).svc.cluster.local:2380,apisix-etcd-1=http://apisix-etcd-1.$(ETCD_HEADLESS).$(MY_NAMESPACE).svc.cluster.local:2380,apisix-etcd-2=http://apisix-etcd-2.$(ETCD_HEADLESS).$(MY_NAMESPACE).svc.cluster.local:2380
        - name: ETCD_CLUSTER_DOMAIN
          value: $(ETCD_HEADLESS).$(MY_NAMESPACE).svc.cluster.local
        image: docker.io/bitnami/etcd:3.4.16-debian-10-r14
        imagePullPolicy: IfNotPresent
        lifecycle:
          preStop:
            exec:
              command:
              - /opt/bitnami/scripts/etcd/prestop.sh
        livenessProbe:
          exec:
            command:
            - /opt/bitnami/scripts/etcd/healthcheck.sh
          failureThreshold: 5
          initialDelaySeconds: 60
          periodSeconds: 30
          successThreshold: 1
          timeoutSeconds: 5
        name: etcd
        ports:
        - containerPort: 2379
          name: client
          protocol: TCP
        - containerPort: 2380
          name: peer
          protocol: TCP
        readinessProbe:
          exec:
            command:
            - /opt/bitnami/scripts/etcd/healthcheck.sh
          failureThreshold: 5
          initialDelaySeconds: 60
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 5
        resources:
          limits:
            cpu: "1"
            memory: 2Gi
          requests:
            cpu: 500m
            memory: 1Gi
        securityContext:
          procMount: Default
          runAsNonRoot: true
          runAsUser: 1001
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /bitnami/etcd
          name: data
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      # schedulerName: default-scheduler
      securityContext:
        fsGroup: 1001
      terminationGracePeriodSeconds: 30
  updateStrategy:
    type: RollingUpdate
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes:
      - ReadWriteOnce
      resources:
        requests:
          storage: 10Gi
      storageClassName: etcdserver
      # volumeMode: Filesystem
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app.kubernetes.io/name: etcd
  name: apisix-etcd-headless
  namespace: elk
spec:
  clusterIP: None
  ports:
  - name: client
    port: 2379
    protocol: TCP
    targetPort: client
  - name: peer
    port: 2380
    protocol: TCP
    targetPort: peer
  publishNotReadyAddresses: true
  selector:
    app.kubernetes.io/instance: apisix
    app.kubernetes.io/name: etcd
  type: ClusterIP

svc请根据自己要求进行配置(LB)

点击查看代码

apiVersion: v1
kind: Service
metadata:
  name: etcd-cluster
  namespace: elk
spec:
  externalTrafficPolicy: Cluster
  ports:
  - name: 2379-2379-tcp
    port: 2379
    protocol: TCP
    targetPort: 2379
  selector:
    app.kubernetes.io/instance: apisix
    app.kubernetes.io/name: etcd
  type: LoadBalancer