我們在之前的kubeasz部署高可用kubernetes1.17.2 並實現traefik2.1.2部署篇已經實現了基於chrony的時間同步
[root@bs-k8s-master01 ~]# cat /etc/chrony.conf # Use public servers from the pool.ntp.org project. server 20.0.0.202 iburst [root@bs-k8s-master01 ~]# chronyc sources -v 210 Number of sources = 1 .-- Source mode '^' = server, '=' = peer, '#' = local clock. / .- Source state '*' = current synced, '+' = combined , '-' = not combined, | / '?' = unreachable, 'x' = time may be in error, '~' = time too variable. || .- xxxx [ yyyy ] +/- zzzz || Reachability register (octal) -. | xxxx = adjusted offset, || Log2(Polling interval) --. | | yyyy = measured offset, || \ | | zzzz = estimated error. || | | \ MS Name/IP address Stratum Poll Reach LastRx Last sample =============================================================================== ^* bs-k8s-master02 3 6 377 4 -15ms[ -17ms] +/- 21ms [root@bs-k8s-master01 ~]# scp /etc/chrony.conf 20.0.0.207:/etc/chrony.conf root@20.0.0.207's password: chrony.conf 100% 1011 662.7KB/s 00:00 [root@bs-k8s-harbor01 ~]# systemctl restart chronyd.service [root@bs-k8s-harbor01 ~]# chronyc sources -v 210 Number of sources = 1 .-- Source mode '^' = server, '=' = peer, '#' = local clock. / .- Source state '*' = current synced, '+' = combined , '-' = not combined, | / '?' = unreachable, 'x' = time may be in error, '~' = time too variable. || .- xxxx [ yyyy ] +/- zzzz || Reachability register (octal) -. | xxxx = adjusted offset, || Log2(Polling interval) --. | | yyyy = measured offset, || \ | | zzzz = estimated error. || | | \ MS Name/IP address Stratum Poll Reach LastRx Last sample =============================================================================== ^* 20.0.0.202 3 6 7 1 +25us[ -546us] +/- 36ms
注：時間同步的概念應該深入每一個搞IT人員的心中
部署docker
#安裝服務器必備軟件 # yum -y install wget vim iftop iotop net-tools nmon telnet lsof iptraf nmap httpd-tools lrzsz mlocate ntp ntpdate strace libpcap nethogs iptraf iftop nmon bridge-utils bind-utils telnet nc nfs-utils rpcbind nfs-utils dnsmasq python python-devel yum-utils device-mapper-persistent-data lvm2 tcpdump mlocate tree #添加docker源信息 [root@bs-k8s-harbor01 ~]# yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo #列出所有docker版本，選擇和Kubernetes集群一致的docker版本[不一致能否正常，我沒嘗試] [root@bs-k8s-harbor01 ~]# yum list docker-ce --showduplicates | sort -r [root@bs-k8s-master01 ~]# docker -v Docker version 19.03.5, build 633a0ea838 #kubernetes集群版本 #安裝對應版本 [root@bs-k8s-harbor01 ~]# yum -y install docker-ce-19.03.5-3.el7 #為了保持集群環境一致，daemon.json也應該一致 [root@bs-k8s-master01 docker]# cat daemon.json { "registry-mirrors": ["https://dockerhub.azk8s.cn", "https://docker.mirrors.ustc.edu.cn"], "insecure-registries": ["127.0.0.1/8"], "max-concurrent-downloads": 10, "log-driver": "json-file", "log-level": "warn", "log-opts": { "max-size": "10m", "max-file": "3" }, "data-root": "/var/lib/docker" } [root@bs-k8s-harbor01 ~]# mkdir /etc/docker [root@bs-k8s-master01 docker]# scp daemon.json 20.0.0.207:/etc/docker/ root@20.0.0.207's password: [root@bs-k8s-harbor01 docker]# systemctl restart docker && systemctl enable docker [root@bs-k8s-harbor01 docker]# docker version Client: Docker Engine - Community Version: 19.03.5 ...... Server: Docker Engine - Community Engine: Version: 19.03.5
部署harbor
harbor的管理是基於docker-compose的
# yum install -y docker-compose # docker-compose version docker-compose version 1.18.0, build 8dd22a9 docker-py version: 2.6.1 CPython version: 3.6.8 OpenSSL version: OpenSSL 1.0.2k-fips 26 Jan 2017
harbor的 git地址：https://github.com/goharbor/harbor
這里我使用的版本是1.8
[root@bs-k8s-harbor01 data]# pwd /data [root@bs-k8s-harbor01 data]# ll 總用量 539940 -rw-r--r-- 1 root root 552897681 5月 31 2019 harbor-offline-installer-v1.8.0.tgz root@bs-k8s-harbor01 data]# tar xf harbor-offline-installer-v1.8.0.tgz [root@bs-k8s-harbor01 data]# mv harbor /usr/local/ [root@bs-k8s-harbor01 data]# cd /usr/local/harbor/ [root@bs-k8s-harbor01 harbor]# ls harbor.v1.8.0.tar.gz harbor.yml install.sh LICENSE prepare #創建證書 # mkdir /data/ca # openssl genrsa -out /data/ca/harbor-ca.key Generating RSA private key, 2048 bit long modulus ....................+++ ..................................................................................+++ e is 65537 (0x10001) # openssl req -x509 -new -nodes -key /data/ca/harbor-ca.key -subj "/CN=harbor.linux.com" -days 7120 -out /data/ca/harbor-ca.crt #修改配置文件 # cp harbor.yml{,.bak} # vim harbor.yml # diff harbor.yml{,.bak} 5c5 5c5 < hostname: harbor.linux.com --- > hostname: reg.mydomain.com 8c8 < #http: --- > http: 10c10 < # port: 80 --- > port: 80 13c13 < https: --- > # https: 15c15 < port: 443 --- > # port: 443 17,18c17,18 < certificate: /data/ca/harbor-ca.crt < private_key: /data/ca/harbor-ca.key --- > # certificate: /your/certificate/path > # private_key: /your/private/key/path 27c27 < harbor_admin_password: zisefeizhu --- > harbor_admin_password: Harbor12345 35c35 < data_volume: /data/harbor --- > data_volume: /data #部署 # mkdir -pv /etc/docker/certs.d/harbor.linux.com/ mkdir: 已創建目錄 "/etc/docker/certs.d" mkdir: 已創建目錄 "/etc/docker/certs.d/harbor.linux.com/" # cp /data/ca/harbor-ca.crt /etc/docker/certs.d/harbor.linux.com/ # ./install.sh # docker-compose start Starting log ... done Starting registry ... done Starting registryctl ... done Starting postgresql ... done Starting core ... done Starting portal ... done Starting redis ... done Starting jobservice ... done Starting proxy ... done # docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 287136c60b95 goharbor/nginx-photon:v1.8.0 "nginx -g 'daemon of…" 38 seconds ago Up 37 seconds (healthy) 0.0.0.0:80->80/tcp, 0.0.0.0:443->443/tcp nginx 66a07d42818c goharbor/harbor-jobservice:v1.8.0 "/harbor/start.sh" 42 seconds ago Up 38 seconds harbor-jobservice e4bb415fd236 goharbor/harbor-portal:v1.8.0 "nginx -g 'daemon of…" 42 seconds ago Up 38 seconds (healthy) 80/tcp harbor-portal 1530c4b4c604 goharbor/harbor-core:v1.8.0 "/harbor/start.sh" 43 seconds ago Up 41 seconds (healthy) harbor-core adc160874fef goharbor/redis-photon:v1.8.0 "docker-entrypoint.s…" 44 seconds ago Up 42 seconds 6379/tcp redis 300165f93782 goharbor/harbor-db:v1.8.0 "/entrypoint.sh post…" 44 seconds ago Up 42 seconds (healthy) 5432/tcp harbor-db a81c3d53eb2e goharbor/registry-photon:v2.7.1-patch-2819-v1.8.0 "/entrypoint.sh /etc…" 44 seconds ago Up 43 seconds (healthy) 5000/tcp registry 1a7cf72c6433 goharbor/harbor-registryctl:v1.8.0 "/harbor/start.sh" 44 seconds ago Up 42 seconds (healthy) registryctl 6be2b10b733d goharbor/harbor-log:v1.8.0 "/bin/sh -c /usr/loc…" 45 seconds ago Up 44 seconds (healthy) 127.0.0.1:1514->10514/tcp harbor-log # ss -lntup Netid State Recv-Q Send-Q Local Address:Port Peer Address:Port udp UNCONN 0 0 *:111 *:* users:(("systemd",pid=1,fd=28)) udp UNCONN 0 0 *:123 *:* users:(("chronyd",pid=1558,fd=3)) udp UNCONN 0 0 127.0.0.1:323 *:* users:(("chronyd",pid=1558,fd=1)) udp UNCONN 0 0 ::1:323 :::* users:(("chronyd",pid=1558,fd=2)) tcp LISTEN 0 128 *:22 *:* users:(("sshd",pid=956,fd=3)) tcp LISTEN 0 128 127.0.0.1:1514 *:* users:(("docker-proxy",pid=6568,fd=4)) tcp LISTEN 0 128 *:111 *:* users:(("systemd",pid=1,fd=27)) tcp LISTEN 0 128 :::80 :::* users:(("docker-proxy",pid=7254,fd=4)) tcp LISTEN 0 128 :::22 :::* users:(("sshd",pid=956,fd=4)) tcp LISTEN 0 128 :::443 :::* users:(("docker-proxy",pid=7243,fd=4))
# docker login harbor.linux.com Username: admin Password: WARNING! Your password will be stored unencrypted in /root/.docker/config.json. Configure a credential helper to remove this warning. See https://docs.docker.com/engine/reference/commandline/login/#credentials-store Login Succeeded
配置開機自啟
# cat /etc/rc.d/rc.local
cd /usr/local/harbor && docker-compose start
注：登陸失敗的原因可能有：1. hosts沒有域名解析 2.密碼錯誤
客戶端配置
以bs-k8s-master01為例
# mkdir -pv /etc/docker/certs.d/harbor.linux.com/ && scp 20.0.0.207:/data/ca/harbor-ca.crt /etc/docker/certs.d/harbor.linux.com/ && docker login harbor.linux.com mkdir: 已創建目錄 "/etc/docker/certs.d" mkdir: 已創建目錄 "/etc/docker/certs.d/harbor.linux.com/" The authenticity of host '20.0.0.207 (20.0.0.207)' can't be established. ECDSA key fingerprint is SHA256:EqqNfQ6sVyEO5yRX8E2plLlEaaeTyLbXhocH4uxhvJw. ECDSA key fingerprint is MD5:a2:3a:03:bc:e7:7a:f8:c3:ef:db:6c:d5:d2:34:e1:3c. Are you sure you want to continue connecting (yes/no)? yes Warning: Permanently added '20.0.0.207' (ECDSA) to the list of known hosts. root@20.0.0.207's password: harbor-ca.crt 100% 1115 512.7KB/s 00:00 Username: admin Password: WARNING! Your password will be stored unencrypted in /root/.docker/config.json. Configure a credential helper to remove this warning. See https://docs.docker.com/engine/reference/commandline/login/#credentials-store Login Succeededxxxxxxxxxx31 1# mkdir -pv /etc/docker/certs.d/harbor.linux.com/ && scp 20.0.0.207:/data/ca/harbor-ca.crt /etc/docker/certs.d/harbor.linux.com/ && docker login harbor.linux.com2mkdir: 已創建目錄 "/etc/docker/certs.d"3mkdir: 已創建目錄 "/etc/docker/certs.d/harbor.linux.com/"4The authenticity of host '20.0.0.207 (20.0.0.207)' can't be established.5ECDSA key fingerprint is SHA256:EqqNfQ6sVyEO5yRX8E2plLlEaaeTyLbXhocH4uxhvJw.6ECDSA key fingerprint is MD5:a2:3a:03:bc:e7:7a:f8:c3:ef:db:6c:d5:d2:34:e1:3c.7Are you sure you want to continue connecting (yes/no)? yes8Warning: Permanently added '20.0.0.207' (ECDSA) to the list of known hosts.9root@20.0.0.207's password: 10harbor-ca.crt 100% 1115 512.7KB/s 00:00 11Username: admin12Password: 13WARNING! Your password will be stored unencrypted in /root/.docker/config.json.14Configure a credential helper to remove this warning. See15https://docs.docker.com/engine/reference/commandline/login/#credentials-store1617Login Succeeded18# mkdir -pv /etc/docker/certs.d/harbor.linux.com/19mkdir: 已創建目錄 "/etc/docker/certs.d"20mkdir: 已創建目錄 "/etc/docker/certs.d/harbor.linux.com/"21# scp 20.0.0.207:/data/ca/harbor-ca.crt /etc/docker/certs.d/harbor.linux.com/22root@20.0.0.207's password: 23harbor-ca.crt 100% 1115 690.7KB/s 00:00 24# docker login harbor.linux.com25Username: admin26Password: 27WARNING! Your password will be stored unencrypted in /root/.docker/config.json.28Configure a credential helper to remove this warning. See29https://docs.docker.com/engine/reference/commandline/login/#credentials-store3031Login Succeededshell
# cat /etc/docker/daemon.json
{
"registry-mirrors": ["https://dockerhub.azk8s.cn", "https://docker.mirrors.ustc.edu.cn"],
"insecure-registries": ["harbor.linux.com", "20.0.0.207:443"],
"max-concurrent-downloads": 10,
"log-driver": "json-file",
"log-level": "warn",
"log-opts": {
"max-size": "10m",
"max-file": "3"
},
"data-root": "/var/lib/docker"
}
# systemctl restart docker
# docker login harbor.linux.com
# docker login 20.0.0.207:443 # cat /root/.docker/config.json { "auths": {
"auths": {
"20.0.0.207:443": {
"auth": "YWRtaW46emlzZWZlaXpodQ=="
},
"harbor.linux.com": { "auth": "YWRtaW46emlzZWZlaXpodQ==" } }, "HttpHeaders": { "User-Agent": "Docker-Client/19.03.5 (linux)" } }
注：其他集群機同樣操作
測試
以ceph rbd 為例
#kubernetes master節點拉取鏡像上傳到harbor倉庫，worker節點拉取鏡像 [root@bs-k8s-master01 k8s]# docker pull quay.io/external_storage/rbd-provisioner:latest [root@bs-k8s-master01 k8s]# docker pull quay.io/external_storage/rbd-provisioner:latest [root@bs-k8s-master01 k8s]# docker tag quay.io/external_storage/rbd-provisioner:latest harbor.linux.com/rbd/rbd-provisioner:latest [root@bs-k8s-master01 k8s]# docker push harbor.linux.com/rbd/rbd-provisioner:latest [root@bs-k8s-node01 ~]# docker pull harbor.linux.com/rbd/rbd-provisioner:latest
#用戶名密碼存放 # cat ~/.docker/config.json |base64 -w 0 ewoJImF1dGhzIjogewoJCSJoYXJib3IubGludXguY29tIjogewoJCQkiYXV0aCI6ICJZV1J0YVc0NmVtbHpaV1psYVhwb2RRPT0iCgkJfQoJfSwKCSJIdHRwSGVhZGVycyI6IHsKCQkiVXNlci1BZ2VudCI6ICJEb2NrZXItQ2xpZW50LzE5LjAzLjUgKGxpbnV4KSIKCX0KfQ== #創建secret # cat secret-harbor.yaml ########################################################################## #Author: zisefeizhu #QQ: 2********0 #Date: 2020-03-17 #FileName: secret-harbor.yaml #URL: https://www.cnblogs.com/zisefeizhu/ #Description: The test script #Copyright (C): 2020 All rights reserved ########################################################################### apiVersion: v1 kind: Secret metadata: name: k8s-harbor-login type: kubernetes.io/dockerconfigjson data: .dockerconfigjson: ewoJImF1dGhzIjogewoJCSJoYXJib3IubGludXguY29tIjogewoJCQkiYXV0aCI6ICJZV1J0YVc0NmVtbHpaV1psYVhwb2RRPT0iCgkJfQoJfSwKCSJIdHRwSGVhZGVycyI6IHsKCQkiVXNlci1BZ2VudCI6ICJEb2NrZXItQ2xpZW50LzE5LjAzLjUgKGxpbnV4KSIKCX0KfQ== # pwd /data/k8s/harbor # kubectl apply -f secret-harbor.yaml secret/k8s-harbor-login created #部署rbd pod測試 [root@bs-k8s-master01 harbor]# cat external-storage-rbd-provisioner.yaml ########################################################################## #Author: zisefeizhu #QQ: 2********0 #Date: 2020-03-13 #FileName: external-storage-rbd-provisioner.yaml #URL: https://www.cnblogs.com/zisefeizhu/ #Description: The test script #Copyright (C): 2020 All rights reserved ########################################################################### apiVersion: v1 kind: ServiceAccount metadata: name: rbd-provisioner namespace: default --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-provisioner rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["create", "update", "patch"] - apiGroups: [""] resources: ["endpoints"] verbs: ["get", "list", "watch", "create", "update", "patch"] - apiGroups: [""] resources: ["services"] resourceNames: ["kube-dns"] verbs: ["list", "get"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-provisioner subjects: - kind: ServiceAccount name: rbd-provisioner namespace: default roleRef: kind: ClusterRole name: rbd-provisioner apiGroup: rbac.authorization.k8s.io --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: rbd-provisioner namespace: default rules: - apiGroups: [""] resources: ["secrets"] verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: rbd-provisioner namespace: default roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: rbd-provisioner subjects: - kind: ServiceAccount name: rbd-provisioner namespace: default --- apiVersion: apps/v1 kind: Deployment metadata: name: rbd-provisioner namespace: default spec: replicas: 1 selector: matchLabels: app: rbd-provisioner strategy: type: Recreate template: metadata: labels: app: rbd-provisioner spec: containers: - name: rbd-provisioner image: "harbor.linux.com/rbd/rbd-provisioner:latest" imagePullPolicy: IfNotPresent env: - name: PROVISIONER_NAME value: ceph.com/rbd imagePullSecrets: - name: k8s-harbor-login serviceAccount: rbd-provisioner nodeSelector: ## 設置node篩選器，在特定label的節點上啟動 rbd: "true" #節點打標簽 [root@bs-k8s-master01 harbor]# kubectl label nodes 20.0.0.204 rbd=true node/20.0.0.204 labeled #刪除bs-k8s-node01節點上的rbd鏡像 [root@bs-k8s-master01 harbor]# kubectl apply -f external-storage-rbd-provisioner.yaml serviceaccount/rbd-provisioner created clusterrole.rbac.authorization.k8s.io/rbd-provisioner created clusterrolebinding.rbac.authorization.k8s.io/rbd-provisioner created role.rbac.authorization.k8s.io/rbd-provisioner created rolebinding.rbac.authorization.k8s.io/rbd-provisioner created deployment.apps/rbd-provisioner created [root@bs-k8s-master01 harbor]# kubectl get pods -o wide -w NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES rbd-provisioner-9cf46c856-bl454 0/1 ContainerCreating 0 6s <none> 20.0.0.204 <none> <none> rbd-provisioner-9cf46c856-bl454 1/1 Running 0 37s 172.20.46.82 20.0.0.204 <none> <none>
測試完成