Harbor環境部署的要求：系統版本在Centos7.5以上、內核版本在4.4X以上、ip_forward路由轉發功能要打開。
一、環境准備
[root@k8s-harbor01 ~]# cat /etc/redhat-release CentOS Linux release 7.7.1908 (Core) [root@k8s-harbor01 ~]# uname -r 4.4.232-1.el7.elrepo.x86_64 [root@k8s-harbor01 ~]# echo 1 > /proc/sys/net/ipv4/ip_forward [root@k8s-harbor01 ~]# vim /etc/sysctl.conf net.ipv4.ip_forward = 1 [root@k8s-harbor01 ~]# sysctl -p [root@k8s-harbor01 ~]# systemctl stop firewalld && systemctl disable firewalld && firewall-cmd --state [root@k8s-harbor01 ~]# vim /etc/sysconfig/selinux SELINUX=disabled [root@k8s-harbor01 ~]# getenforce Disabled [root@k8s-harbor01 ~]# python --version Python 2.7.5
二、安裝Docker
提前下載二進制安裝包docker-18.09.6.tgz到/usr/local/src路徑下，解壓安裝
[root@k8s-harbor01 ~]# cd /usr/local/src/
[root@k8s-harbor01 src]# ll docker-18.09.6.tgz
-rw-r--r-- 1 root root 48047231 Oct 19 2019 docker-18.09.6.tgz
[root@k8s-harbor01 src]# tar -zvxf docker-18.09.6.tgz
[root@k8s-harbor01 src]# cp docker/* /usr/local/bin/
[root@k8s-harbor01 src]# chmod 755 /usr/local/bin/*
/usr/local/bin默認已經加到系統環境變量中
[root@k8s-harbor01 src]# echo $PATH
/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin
編輯docker啟動文件
注意"WorkingDirectory"路徑要和/etc/docker/daemon.json文件中的data-root、exec-root路徑一致
[root@k8s-harbor01 src]# cat > /etc/systemd/system/docker.service << EOF
[Unit]
Description=Docker Application Container Engine
Documentation=http://docs.docker.io
[Service]
WorkingDirectory=/data/docker
Environment="PATH=/usr/local/bin:/bin:/sbin:/usr/bin:/usr/sbin"
EnvironmentFile=-/run/flannel/docker
ExecStart=/usr/local/bin/dockerd
ExecReload=/bin/kill -s HUP
Restart=on-failure
RestartSec=5
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
Delegate=yes
KillMode=process
[Install]
WantedBy=multi-user.target
EOF
授執行權限
[root@k8s-harbor01 src]# chmod 755 /etc/systemd/system/docker.service
編輯docker 配置文件
編輯docker 配置文件
[root@k8s-harbor01 src]# mkdir -p /etc/docker && mkdir -p /data/docker/data && mkdir -p /data/docker/exec
[root@k8s-harbor01 src]# cat > /etc/docker/daemon.json << EOF
{
"registry-mirrors": ["https://docker.mirrors.ustc.edu.cn","https://hub-mirror.c.163.com"],
"insecure-registries": ["docker02:35000"],
"max-concurrent-downloads": 20,
"live-restore": true,
"max-concurrent-uploads": 10,
"debug": true,
"data-root": "/data/docker/data",
"exec-root": "/data/docker/exec",
"log-opts": {
"max-size": "100m",
"max-file": "5"
}
}
EOF
啟動 docker 服務
[root@k8s-harbor01 src]# systemctl daemon-reload && systemctl enable docker && systemctl restart docker
[root@k8s-harbor01 src]# systemctl status docker|grep Active
Active: active (running) since Wed 2020-08-12 13:41:07 CST; 28s ago
查看 Docker 版本號
[root@k8s-harbor01 src]# docker --version
Docker version 18.09.6, build 481bc77
三、安裝Docker-Compose
下載docker-compose二進制執行文件 百度網盤下載地址：https://pan.baidu.com/s/1er0rM0vxEubYOLHx7LI62A 提取密碼：eer9 [root@k8s-harbor01 ~]# cd /usr/local/src/ [root@k8s-harbor01 src]# curl -L "https://github.com/docker/compose/releases/download/1.26.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose [root@k8s-harbor01 src]# cp docker-compose /usr/local/bin/ [root@k8s-harbor01 src]# chmod 755 /usr/local/bin/* 查看 docker-compose 版本號 [root@k8s-harbor01 ~]# docker-compose --version docker-compose version 1.26.0, build d4451659
四、部署Harbor鏡像倉庫
1）HTTPS證書自簽
如果線上環境有已購買好的HTTPS證書可以直接拿過來用，如果沒有，就在Harbor本機進行HTTPS證書自簽。這里Harbor本機ip地址是172.16.60.238
生成CA證書私鑰
[root@k8s-harbor01 ~]# openssl genrsa -out ca.key 4096
生成CA證書
[root@k8s-harbor01 ~]# openssl req -x509 -new -nodes -sha512 -days 3650 \
-subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=172.16.60.238" \
-key ca.key \
-out ca.crt
生成服務器證書
1）生成私鑰
[root@k8s-harbor01 ~]# openssl genrsa -out 172.16.60.238.key 4096
2）生成證書簽名請求（CSR）
[root@k8s-harbor01 ~]# openssl req -sha512 -new \
-subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=172.16.60.238" \
-key 172.16.60.238.key \
-out 172.16.60.238.csr
3）生成一個x509 v3擴展文件（兩種方式根據情況二選一）
####################################################################################
第一種方式：域名
cat > v3.ext <<-EOF
authorityKeyIdentifier=keyid,issuer
basicConstraints=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names
[alt_names]
DNS.1=172.16.60.238
DNS.2=yourdomain
DNS.3=hostname
EOF
####################################################################################
第二種方式：IP
cat > v3.ext <<-EOF
authorityKeyIdentifier=keyid,issuer
basicConstraints=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
extendedKeyUsage = serverAuth
subjectAltName = IP:172.16.60.238
EOF
####################################################################################
這里選擇第二種的IP方式
[root@k8s-harbor01 ~]# cat > v3.ext <<-EOF
authorityKeyIdentifier=keyid,issuer
basicConstraints=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
extendedKeyUsage = serverAuth
subjectAltName = IP:172.16.60.238
EOF
4）使用該v3.ext文件為您的Harbor主機生成證書
[root@k8s-harbor01 ~]# openssl x509 -req -sha512 -days 3650 \
-extfile v3.ext \
-CA ca.crt -CAkey ca.key -CAcreateserial \
-in 172.16.60.238.csr \
-out 172.16.60.238.crt
2）提供證書給Harbor和Docker
1）將服務器證書和密鑰復制到Harbor主機上的certficates文件夾中 根據自己實際環境需求創建Harbor的certficates文件夾 [root@k8s-harbor01 ~]# mkdir -p /data/cert/ [root@k8s-harbor01 ~]# cp 172.16.60.238.crt /data/cert/ [root@k8s-harbor01 ~]# cp 172.16.60.238.key /data/cert/ 2）轉換 172.16.60.238.crt 為172.16.60.238.cert，供Docker使用。 Docker守護程序將.crt文件解釋為CA證書，並將.cert文件解釋為客戶端證書。 [root@k8s-harbor01 ~]# openssl x509 -inform PEM -in 172.16.60.238.crt -out 172.16.60.238.cert 3）將服務器證書，密鑰和CA文件復制到Harbor主機上的Docker certificate文件夾中。 記住必須首先創建適當的文件夾 [root@k8s-harbor01 ~]# mkdir -p /etc/docker/certs.d/172.16.60.238/ [root@k8s-harbor01 ~]# cp 172.16.60.238.cert /etc/docker/certs.d/172.16.60.238/ [root@k8s-harbor01 ~]# cp 172.16.60.238.key /etc/docker/certs.d/172.16.60.238/ [root@k8s-harbor01 ~]# cp ca.crt /etc/docker/certs.d/172.16.60.238/ 4）重新啟動Docker [root@k8s-harbor01 ~]# systemctl restart docker [root@k8s-harbor01 ~]# systemctl status docker 5）將名為"ca.crt"的CA證書下載到本地電腦，然后安裝證書。 這樣就可以在本地電腦的瀏覽器里正常訪問https地址的Harbor了（證書可被信任）
3）安裝Harbor
到 Harbor的GitHub倉庫的Release頁面 , 下載最新的在線安裝包
這里下載Harbor V2.0.2版本的安裝包
[root@k8s-harbor01 ~]# cd /usr/local/src/
[root@k8s-harbor01 src]# wget https://github.com/goharbor/harbor/releases/download/v2.0.2/harbor-online-installer-v2.0.2.tgz
[root@k8s-harbor01 src]# tar -zvxf harbor-online-installer-v2.0.2.tgz
[root@k8s-harbor01 src]# mv harbor /opt/
修改harbor配置信息
[root@k8s-harbor01 src]# cd /opt/harbor/
[root@k8s-harbor01 harbor]# cp harbor.yml.tmpl harbor.yml
.........
........
hostname: 172.16.60.238
# http related config
http:
# port for http, default is 80. If https enabled, this port will redirect to https port
port: 80
# https related config
https:
# https port for harbor, default is 443
port: 443
# The path of cert and key files for nginx
certificate: /data/cert/172.16.60.238.crt
private_key: /data/cert/172.16.60.238.key
........
........
harbor_admin_password: Harbor@123456
........
........
data_volume: /data
運行install.sh, 注意運行時加上--with-clair 選項，啟動clair鏡像掃描功能
[root@k8s-harbor01 harbor]# ./install.sh --with-clair
........
........
✔ ----Harbor has been installed and started successfully.----
出現上面的信息，說明Harbor已經安裝完成了。
查看harbor啟動情況。
docker-compose 命令必須要在harbor安裝目錄 (這里就是/opt/harbor)路徑下才能執行。
[root@k8s-harbor01 harbor]# docker-compose ps
Name Command State Ports
---------------------------------------------------------------------------------------------------------------
clair ./docker-entrypoint.sh Up (healthy) 6060/tcp, 6061/tcp
clair-adapter /home/clair-adapter/entryp ... Up (healthy) 8080/tcp
harbor-core /harbor/entrypoint.sh Up (healthy)
harbor-db /docker-entrypoint.sh Up (healthy) 5432/tcp
harbor-jobservice /harbor/entrypoint.sh Up (healthy)
harbor-log /bin/sh -c /usr/local/bin/ ... Up (healthy) 127.0.0.1:1514->10514/tcp
harbor-portal nginx -g daemon off; Up (healthy) 8080/tcp
nginx nginx -g daemon off; Up (healthy) 0.0.0.0:80->8080/tcp, 0.0.0.0:443->8443/tcp
redis redis-server /etc/redis.conf Up (healthy) 6379/tcp
registry /home/harbor/entrypoint.sh Up (healthy) 5000/tcp
registryctl /home/harbor/start.sh Up (healthy)
查看harbor鏡像
[root@k8s-harbor01 ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
goharbor/redis-photon v2.0.2 e547529bb6a1 3 weeks ago 72.3MB
goharbor/clair-adapter-photon v2.0.2 9ec8853dc3cb 3 weeks ago 62MB
goharbor/clair-photon v2.0.2 73885002dda7 3 weeks ago 171MB
goharbor/harbor-registryctl v2.0.2 9f8b7bb0f1ff 3 weeks ago 101MB
goharbor/registry-photon v2.0.2 eac8c5fc9ca8 3 weeks ago 83.6MB
goharbor/nginx-photon v2.0.2 eee4771b916c 3 weeks ago 43.6MB
goharbor/harbor-log v2.0.2 b2db762a6c3a 3 weeks ago 82.1MB
goharbor/harbor-jobservice v2.0.2 3960e027ccb9 3 weeks ago 164MB
goharbor/harbor-core v2.0.2 de2495b944cf 3 weeks ago 145MB
goharbor/harbor-portal v2.0.2 90088a0e64a9 3 weeks ago 52.5MB
goharbor/harbor-db v2.0.2 81e98a7af097 3 weeks ago 161MB
goharbor/prepare v2.0.2 7e804db05454 3 weeks ago 160MB
確保harbpr啟動后的80和443端口都起來了
[root@k8s-harbor01 harbor]# lsof -i:80
COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME
docker-pr 3095 root 4u IPv6 26027 0t0 TCP *:http (LISTEN)
[root@k8s-harbor01 harbor]# lsof -i:443
COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME
docker-pr 3082 root 4u IPv6 26015 0t0 TCP *:https (LISTEN)
到這里就可以訪問harbor了,訪問地址為：https://172.16.60.238
用戶名為admin，密碼為配置文件中定義的"Harbor@123456"
查看clair鏡像掃描器
Habor 服務啟停
注意：如果harbor.yml配置修改了，要先執行"./prepare"命令進行配置載入，然后再重啟harbor服務。
查看Habror docker-compose ps 啟動Harbor docker-compose start 停止Harbor docker-compose stop 重啟Harbor docker-compose restart 另外： Harbor還可以通過down和up命令去停止和啟動， 只不過這種方式是刪除、創建的關停和啟動。 docker-compose down -v docker-compose up -d
五、客戶端登錄Harbor
在Habror客戶端機器（如k8s的node節點、harbor節點）配置登錄：
默認情況下，在客戶端登錄Habor是會報錯的：
[root@k8s-node01 ~]# docker login 172.16.60.238
Authenticating with existing credentials...
Login did not succeed, error: Error response from daemon: Get https://172.16.60.238/v2/: x509: certificate signed by unknown authority
原因： 客戶端登錄Harbor，https證書不被信任。
解決辦法：下面兩種方法選其一
1）方法一
將Harbor服務器證書，密鑰和CA文件復制到Harbor客戶主機上的Docker certificate文件夾中
[root@k8s-node01 ~]# mkdir -p /etc/docker/certs.d/172.16.60.238/
[root@k8s-node01 ~]# cd /etc/docker/certs.d/172.16.60.238/
[root@k8s-node01 172.16.60.238]# rsync -e "ssh -p22" -avpgolr 172.16.60.238:/etc/docker/certs.d/172.16.60.238/* ./
[root@k8s-node01 172.16.60.238]# ll
total 12
-rw-r--r-- 1 root root 2053 Aug 19 14:34 172.16.60.238.cert
-rw-r--r-- 1 root root 3243 Aug 19 14:34 172.16.60.238.key
-rw-r--r-- 1 root root 2033 Aug 19 14:34 ca.crt
重啟docker服務
[root@k8s-node01 172.16.60.238]# systemctl restart docker
[root@k8s-node01 172.16.60.238]# systemctl status docker
再次驗證登錄harbor
[root@k8s-node01 172.16.60.238]# docker login 172.16.60.238
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
2）方法二
配置docker服務的daemon.json文件，添加"insecure-registries"參數，表示忽略ssl證書認證。
[root@k8s-node01 ~]# vim /etc/docker/daemon.json
........
"insecure-registries": ["https://172.16.60.238"],
重啟docker服務
[root@k8s-node01 ~]# systemctl restart docker
[root@k8s-node01 ~]# systemctl status docker
再次驗證登錄harbor
[root@k8s-node01 ~]# docker login 172.16.60.238
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
========================================================================
另外，注意客戶端機器登錄Harbor時，只要首次登錄需要輸入用戶名和密碼。
登錄成功后的信息默認保存到/root/.docker/config.json文件里。
下次登錄時就不用再輸入harbor用戶名和密碼了，直接讀取config.json文件內容
[root@k8s-node01 ~]# cat /root/.docker/config.json
{
"auths": {
"172.16.60.238": {
"auth": "YWRtaW46SGFyYm9yQDEyMzQ1Ng=="
}
},
"HttpHeaders": {
"User-Agent": "Docker-Client/18.09.6 (linux)"
}
[root@k8s-node01 ~]# docker login 172.16.60.238
Authenticating with existing credentials...
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
六、Harbor鏡像掃描
選中鏡像，進行漏洞掃描
如果掃描出漏洞，在漏洞報告了會告知漏洞當前版本和修復版本，按照修復版本修復即可。
修復方法：
可以依據當前基礎鏡像做Dockerfile，使用"yum update -y 漏洞所屬軟件名" 進行升級操作，然后再重新做一個基礎鏡像。
1) 編譯Dockerfile 升級原來centos7.7基礎鏡像里報出來漏洞的軟件 [root@k8s-harbor01 ~]# cat Dockerfile FROM 172.16.60.238/kevin/centos7.7:latest RUN yum update -y sqlite \ && yum update -y nss-util \ && yum update -y nss-sysinit \ && yum update -y dbus-libs \ && yum update -y bind-license \ && yum update -y nss \ && yum update -y nss-softokn \ && yum update -y dbus \ && yum update -y nss-softokn-freebl \ && yum update -y nss-tools \ && yum update -y bash \ && yum update -y python-libs \ && yum update -y python \ && yum update -y bind-license \ && yum update -y expat \ && yum update -y libxml2-python \ && yum update -y libxml2 \ && yum update -y shared-mime-info \ && yum update -y libcurl \ && yum update -y file-libs \ && yum update -y curl 2）制作新的基礎鏡像 [root@k8s-harbor01 ~]# docker build -t 172.16.60.238/kevin/centos7.7:updatev1 . 3）上傳到Harbor倉庫 [root@k8s-harbor01 ~]# docker push 172.16.60.238/kevin/centos7.7:updatev1
將修復好漏洞的新基礎鏡像上傳到Harbor倉庫，再掃描新鏡像，發現漏洞已修復。
