重新編譯kubeadm，修改默認證書時間

參考

• kubeadm alpha certs renew
• Kubeadm1.14 證書調整
• Kubeadm證書過期時間調整

kubeadm 部署的 kubernetes 集群，默認的證書有效時間是1年，需要每年手工更新。

1. 重新編譯kubeadm

1.1 准備

# 下載對應的kubernetes源代碼，這里采用 "v1.14.1" 版本
wget https://codeload.github.com/kubernetes/kubernetes/tar.gz/v1.14.1

# untar
tar -zxvf kubernetes-1.14.1.tar.gz
cd kubernetes-1.14.1

# 進入源代碼目錄
cd kubernetes-1.14.1

1.2 修改源代碼-cert.go

• 文件：staging/src/k8s.io/client-go/util/cert/cert.go
• NewSelfSignedCACert 方法，簽發以下證書，且默認為10年有效期：
  • front-proxy-ca.crt
  • front-proxy-client.crt
  • ca.crt
  • etcd/ca.crt
  • etcd/peer.crt

# 1.14.0版本開始，此文件不需要修改
vim staging/src/k8s.io/client-go/util/cert/cert.go

const duration365d = time.Hour * 24 * 365

// Config contains the basic fields required for creating a certificate
type Config struct {
        CommonName   string
        Organization []string
        AltNames     AltNames
        Usages       []x509.ExtKeyUsage
}

// AltNames contains the domain names and IP addresses that will be added
// to the API Server's x509 certificate SubAltNames field. The values will
// be passed directly to the x509.Certificate object.
type AltNames struct {
        DNSNames []string
        IPs      []net.IP
}

// NewSelfSignedCACert creates a CA certificate
func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, error) {
        now := time.Now()
        tmpl := x509.Certificate{
                SerialNumber: new(big.Int).SetInt64(0),
                Subject: pkix.Name{
                        CommonName:   cfg.CommonName,
                        Organization: cfg.Organization,
                },
                NotBefore:             now.UTC(),
                # 默認已調整有效期為10年；
                # 但只影響部分證書：
                NotAfter:              now.Add(duration365d * 10).UTC(),
                KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
                BasicConstraintsValid: true,
                IsCA:                  true,
        }

        certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &tmpl, &tmpl, key.Public(), key)
        if err != nil {
                return nil, err
        }
        return x509.ParseCertificate(certDERBytes)
}

1.3 修改源代碼-pki_helpers.go

• ，以下證書由 NewSignedCert 方法簽發，但簽發的證書默認只有一年有效期：
  • apiserver.crt
  • apiserver-etcd-client.crt
  • etcd/server.crt
  • etcd/healthcheck-client.crt
  • apiserver-kubelet-client.crt

# `NewSignedCert` 方法：
# 部分證書是通過NewSignedCert這個方法簽發，而這個方法簽發的證書默認只有一年有效期，查看代碼邏輯
vim cmd/kubeadm/app/util/pkiutil/pki_helpers.go

// NewSignedCert creates a signed certificate using the given CA certificate and key
func NewSignedCert(cfg *certutil.Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) {
        serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64))
        if err != nil {
                return nil, err
        }
        if len(cfg.CommonName) == 0 {
                return nil, errors.New("must specify a CommonName")
        }
        if len(cfg.Usages) == 0 {
                return nil, errors.New("must specify at least one ExtKeyUsage")
        }

        certTmpl := x509.Certificate{
                Subject: pkix.Name{
                        CommonName:   cfg.CommonName,
                        Organization: cfg.Organization,
                },
                DNSNames:     cfg.AltNames.DNSNames,
                IPAddresses:  cfg.AltNames.IPs,
                SerialNumber: serial,
                NotBefore:    caCert.NotBefore,
                # 修改簽發相關證書的默認有效期為10年
                // NotAfter:     time.Now().Add(duration365d).UTC(),
                NotAfter:     time.Now().Add(duration365d * 10).UTC(),
                KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
                ExtKeyUsage:  cfg.Usages,
        }
        certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey)
        if err != nil {
                return nil, err
        }
        return x509.ParseCertificate(certDERBytes)
}

1.4 編譯

# go環境已經准備好
# kubeadm
make WHAT=cmd/kubeadm GOFLAGS=-v

# 補充：編譯kubelet
# make all WHAT=cmd/kubelet GOFLAGS=-v

# 補充：編譯kubectl
# make all WHAT=cmd/kubectl GOFLAGS=-v

# 編譯生成的二進制文件在 _output/bin/ 目錄下

1.5 更新kubeadm

# 將kubeadm 文件拷貝替換系統中原有kubeadm
cp /usr/bin/kubeadm /usr/bin/kubeadm.origin
cp _output/bin/kubeadm /usr/bin/kubeadm

2. 更新證書

2.1 更新 kube-master (任一)節點證書

• 1.13.x 版本(含)之后的處理方式；
• 不更新 kubeadm 的情況下，也可手動更新證書，但更新的證書有效期默認仍是一年。

# 備份
cp -r /etc/kubernetes/pki /etc/kubernetes/pki.origin

# 更新證書；
# "--config" 指定 "kubeadm" 的配置文件，建議使用部署集群時使用的配置文件；
# 其他參數可參考官方文檔
cd /etc/kubernetes/pki
kubeadm alpha certs renew all --config=/root/kubeadm/kubeadm-config.yaml

# 驗證新的證書有效期，以 "apiserver.crt" 為例
openssl x509 -in apiserver.crt -text -noout | grep Not

# 備份 /etc/kubernetes/*.conf 文件；
# 必須備份(mv)，否則無法更新 "*.conf" 文件
ll /etc/kubernetes/*.conf | awk '{print $9}' | xargs -i mv {} {}.`date "+%Y%m%d"`

# 更新/etc/kubernetes/*.conf文件；
# 如果沒有 "mv" , 輸出為 "[kubeconfig] Using existing kubeconfig file"，
kubeadm init phase kubeconfig all --config=/root/kubeadm/kubeadm-config.yaml
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file

# 在(所有) kube-master 節點重啟 "kube-apiserver", "kube-controller-manager", "kube-scheduler", "etcd" 4個容器
docker ps | egrep "etcd|kube-apiserver|kube-controller-manager|kube-scheduler" | grep -v pause | awk '{print $1}' | xargs -i docker restart {}

# 覆蓋 "$HOME/.kube/config" 文件
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config

2.2 (optional) HA集群其余mater節點證書更新

• 在已更新證書的master節點運行腳本，將更新的證書同步到其余master節點

# 如果可以，請提前在被同步master節點做備份
cat certificate.sh
#!/bin/bash
# 2019-05-27 v0.1
# scp certificate files from the first control plane node to the rest.

USER=root # customizable
CONTROL_PLANE_IPS="100.64.198.137 100.64.198.138"
for host in ${CONTROL_PLANE_IPS}; do
    scp /etc/kubernetes/pki/ca.crt "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/ca.key "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/sa.key "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/sa.pub "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/front-proxy-ca.crt "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/front-proxy-ca.key "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/etcd/ca.crt "${USER}"@$host:/etc/kubernetes/pki/etcd/ca.crt
    scp /etc/kubernetes/pki/etcd/ca.key "${USER}"@$host:/etc/kubernetes/pki/etcd/ca.key
    scp /etc/kubernetes/admin.conf "${USER}"@$host:/etc/kubernetes/
done

3. 補充：go 環境

# download，根據需要選擇版本
wget https://studygolang.com/dl/golang/go1.12.1.linux-amd64.tar.gz

# untar
tar -zxvf go1.12.1.linux-amd64.tar.gz -C /usr/local

# edit /etc/profile，在文件末尾添加如下內容
vim /etc/profile
# go setting
export GOROOT=/usr/local/go
export GOPATH=/usr/local/gopath
export PATH=$PATH:$GOROOT/bin

# enable /etc/profile
source /etc/profile