K8S 上部署 jenkins[go的持續集成]

書接上文jenkins自動化部署go【docker+jenkins+go+gitlab+harbor+k8s】  我原計划是想把jenkins安裝到docker，后來搞了一些時間也沒有搞定所以才安裝在ubuntu虛擬機上，這次嘗試安裝到k8s上，關於nfs的安裝大家可以參考 ubuntu kubernetes中使用NFS創建pv_pvc

這里 jenkins 使用的存儲為 NFS

安裝 nfs 工具

#1安裝nfs服務端
sudo apt install nfs-kernel-server -y
 
#2. 創建目錄
sudo mkdir -p /nfs/jenkins
 
#3. 使任何客戶端均可訪問
sudo chown nobody:nogroup /nfs/jenkins　
#sudo chmod 755 /nfs/jenkins
sudo chmod 777 /nfs/jenkins
 
#4. 配置/etc/exports文件, 使任何ip均可訪問(加入以下語句)
vi /etc/exports
/nfs/jenkins *(rw,sync,no_subtree_check)
　　
#5. 檢查nfs服務的目錄
sudo exportfs -ra (重新加載配置)
sudo showmount -e (查看共享的目錄和允許訪問的ip段)
 
#6. 重啟nfs服務使以上配置生效
sudo systemctl restart nfs-kernel-server
#sudo /etc/init.d/nfs-kernel-server restart
 
#查看nfs服務的狀態是否為active狀態:active(exited)或active(runing)
systemctl status nfs-kernel-server
 
#7. 測試nfs服務是否成功啟動
 
#安裝nfs 客戶端
sudo apt-get install nfs-common
#創建掛載目錄
 sudo mkdir /nfs/jenkins/
 
#7.4 在主機上的Linux中測試是否正常
sudo mount -t nfs -o nolock -o tcp 192.168.100.11:/nfs/jenkins/ /nfs/jenkins/(掛載成功，說明nfs服務正常)

創建 nfs-client-provisioner deployment 

kubectl apply -f nfs-client-provisioner.yaml

kind: Deployment
apiVersion: apps/v1
metadata:
  name: nfs-client-provisioner
  namespace: kube-system
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: nfs-client-provisioner
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccountName: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          image: quay.io/external_storage/nfs-client-provisioner:latest
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: jenkinsnfs                  # 注意這里的值不能有下划線 _
            - name: NFS_SERVER
              value: 192.168.100.11
            - name: NFS_PATH
              value: /nfs/jenkins
      volumes:
        - name: nfs-client-root
          nfs:
            server: 192.168.100.11
            path: /nfs/jenkins

## 創建 RBAC 授權
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    # replace with namespace where provisioner is deployed
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: kube-system
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: kube-system
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    # replace with namespace where provisioner is deployed
    namespace: kube-system
roleRef:
  kind: Role
  name: leader-locking-nfs-client-provisioner
  apiGroup: rbac.authorization.k8s.io

 創建storageclass

名稱為 jenkinsnfs，並且 provisioner 需要與 deployment 中的 PROVISIONER_NAME對應,注意這個變量不能有下划線 _

kubectl apply -f storageclass.yaml

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: nfs
  namespace: kube-ops
provisioner: jenkinsnfs
parameters:
  archiveOnDelete: "true" # "false" 刪除PVC時不會保留數據，"true"將保留PVC數據

創建 jenkins-deployment.yaml

kubectl apply -f jenkins-deployment.yaml

# 創建一個新的 namespace kube-ops 
apiVersion: v1
kind: Namespace
metadata:
  name: kube-ops
--- 
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: jenkins-claim
  namespace: kube-ops
  annotations:
    volume.beta.kubernetes.io/storage-class: "nfs"
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 2Gi

# jenkins 對應的 RBAC
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins-admin
  namespace: kube-ops       
  labels:
    name: jenkins
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: jenkins-admin
  labels:
    name: jenkins
subjects:
  - kind: ServiceAccount
    name: jenkins-admin
    namespace: kube-ops
roleRef:
  kind: ClusterRole
  name: cluster-admin
  apiGroup: rbac.authorization.k8s.io

# jenkins 對應的 svc
---
apiVersion: v1
kind: Service
metadata:
  name: jenkins
  namespace: kube-ops
  labels:
    app: jenkins
spec:
  type: NodePort
  ports:
  - name: http
    port: 8080                      #服務端口
    targetPort: 8080
    nodePort: 32001                 #NodePort方式暴露 Jenkins 端口
  - name: jnlp
    port: 50000                     #代理端口
    targetPort: 50000
    nodePort: 32002
  selector:
    app: jenkins

# jenkins Deployment
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: jenkins
  namespace: kube-ops
  labels:
    app: jenkins
spec:
  selector:
    matchLabels:
      app: jenkins
  replicas: 1
  template:
    metadata:
      labels:
        app: jenkins
    spec:
      serviceAccountName: jenkins-admin
      containers:
      - name: jenkins
        image: jenkins/jenkins:lts-alpine
        securityContext:                     
          runAsUser: 0                      #設置以ROOT用戶運行容器
          privileged: true                  #擁有特權
        ports:
        - name: http
          containerPort: 8080
        - name: jnlp
          containerPort: 50000
        resources:
          limits:
            memory: 2Gi
            cpu: "2000m"
          requests:
            memory: 2Gi
            cpu: "1000m"
        env:
        - name: LIMITS_MEMORY
          valueFrom:
            resourceFieldRef:
              resource: limits.memory
              divisor: 1Mi
        - name: "JAVA_OPTS"                 #設置變量，指定時區和 jenkins slave 執行者設置
          value: " 
                   -Xmx$(LIMITS_MEMORY)m 
                   -XshowSettings:vm 
                   -Dhudson.slaves.NodeProvisioner.initialDelay=0
                   -Dhudson.slaves.NodeProvisioner.MARGIN=50
                   -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
                   -Duser.timezone=Asia/Shanghai
                 "    
        # - name: "JENKINS_OPTS"
        #  value: "--prefix=/jenkins"         #設置路徑前綴加上 Jenkins，設置該選項會影響 jenkins-slave 的啟動
        volumeMounts:                        #設置要掛在的目錄
        - name: data
          mountPath: /var/jenkins_home
      volumes:
      - name: data
        persistentVolumeClaim:
          claimName: jenkins-claim           #設置PVC

如果已經有nfs 以上可以省略【比如我后面用kubora創建nfs， k8s在2.0.5的版本修改以上yaml】

 

# jenkins 對應的 RBAC
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins-admin
  namespace: kube-system       
  labels:
    name: jenkins
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: jenkins-admin
  labels:
    name: jenkins
subjects:
  - kind: ServiceAccount
    name: jenkins-admin
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: cluster-admin
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: Service
metadata:
  name: jenkins
  namespace: kube-system
  labels:
    app: jenkins
spec:
  type: NodePort
  ports:
  - name: http
    port: 8080                      #服務端口
    targetPort: 8080
    nodePort: 32001                 #NodePort方式暴露 Jenkins 端口
  - name: jnlp
    port: 50000                     #代理端口
    targetPort: 50000
    nodePort: 32002
  selector:
    app: jenkins
 
# jenkins Deployment
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: jenkins
  namespace: kube-system
  labels:
    app: jenkins
spec:
  serviceName: jenkins
  selector:
    matchLabels:
      app: jenkins
  replicas: 1
  template:
    metadata:
      labels:
        app: jenkins
    spec:
      serviceAccountName: jenkins-admin
      containers:
      - name: jenkins
        image: jenkins/jenkins:lts-alpine
        securityContext:                     
          runAsUser: 0                      #設置以ROOT用戶運行容器
          privileged: true                  #擁有特權
        ports:
        - name: http
          containerPort: 8080
        - name: jnlp
          containerPort: 50000
        resources:
          limits:
            memory: 2Gi
            cpu: "200m"
          requests:
            memory: 1Gi
            cpu: "100m"
        env:
        - name: LIMITS_MEMORY
          valueFrom:
            resourceFieldRef:
              resource: limits.memory
              divisor: 1Mi
        - name: "JAVA_OPTS"                 #設置變量，指定時區和 jenkins slave 執行者設置
          value: " 
                   -Xmx$(LIMITS_MEMORY)m 
                   -XshowSettings:vm 
                   -Dhudson.slaves.NodeProvisioner.initialDelay=0
                   -Dhudson.slaves.NodeProvisioner.MARGIN=50
                   -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
                   -Duser.timezone=Asia/Shanghai
                 "    
        # - name: "JENKINS_OPTS"
        #  value: "--prefix=/jenkins"         #設置路徑前綴加上 Jenkins，設置該選項會影響 jenkins-slave 的啟動
        volumeMounts:                        #設置要掛在的目錄
        - name: data
          mountPath: /var/jenkins_home
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
         storageClassName: "nfs"
         accessModes: [ "ReadWriteMany" ]
         resources:
             requests:
                storage: 1Gi

 

獲取 svc NodePort 的端口

root@k8s-master:/nfs# kubectl get pod -n kube-ops
NAME                       READY   STATUS    RESTARTS   AGE
jenkins-6ccc9676d4-khpct   1/1     Running   0          6m57s
root@k8s-master:/nfs# kubectl get svc -n kube-ops
NAME      TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)                          AGE
jenkins   NodePort   10.99.157.233   <none>        8080:32001/TCP,50000:32002/TCP   23m
root@k8s-master:/nfs# 

瀏覽器訪問 http://192.168.100.11:32001

進入 nfs server 查看密碼

動態創建 jenkins-slave 構建任務

把 Jenkins 插件源更改為國內

1. 進入 Manage Jenkins -》 Manage Plugin -> Advanced 最下面有 Update Site 設置為：https://mirrors.tuna.tsinghua.edu.cn/jenkins/updates/update-center.json
2. 修改服務器配置，進入 jenkins安裝目錄 ， /updates/default.json ，將其中的 updates.jenkins-ci.org/download 替換為 mirrors.tuna.tsinghua.edu.cn/jenkins ，然后把www.google.com 修改為 www.baidu.com

cd /nfs/jenkins/kube-ops-jenkins-claim-pvc-f6ba2f57-6dd3-4309-889c-2cb383e09dfa/updates
cp default.json default.json.ori
sed -i "s#www.google.com#www.baidu.com#g" default.json 
sed -i "s#updates.jenkins-ci.org/download#mirrors.tuna.tsinghua.edu.cn/jenkins#g" default.json 

1. 重啟Jenkins服務，，如果jenkins遇到 “There were errors checking the update sites: UnknownHostException: updates.jenkins.io” 錯誤， 可以考慮修改 jenkins 容器的dns 修改 vi /etc/resolv.conf文件中添加：

#nameserver 8.8.8.8
#nameserver 8.8.4.4

root@k8s-master:/nfs# kubectl get pod -n kube-ops
NAME                       READY   STATUS    RESTARTS   AGE
jenkins-8447bcb8bc-c6kxk   1/1     Running   0          25m
root@k8s-master:/nfs# kubectl exec -it jenkins-8447bcb8bc-c6kxk  -n kube-ops bash
bash-5.0#                                                                                                                                                                                                                                                                          bash-5.0# cat /etc/resolv.conf
nameserver 10.96.0.10
search kube-ops.svc.cluster.local svc.cluster.local cluster.local
options ndots:5
nameserver 8.8.8.8
nameserver 8.8.4.4
bash-5.0# 

安裝插件

Localization:Chinese
Git
Pipeline
Extended Choice Parameter
Kubernetes

實現Jenkins與Kubernetes整合

系統管理 -> 系統配置 -> 雲 -> 新建雲 -> Kubernetes
​

Kubernetes 地址: https://kubernetes.default.svc.cluster.local
Kubernetes 命名空間: kube-ops
然后點擊"連接測試"，如果出現 Connection test successful 的提示信息證明 Jenkins 已經可以和 Kubernetes 系統正常通信
Jenkins 地址: http://jenkins.kube-ops.svc.cluster.local:8080

測試

創建一個 流水線 任務,流水腳本如下

//創建一個Pod的模板，label為jenkins-slave
podTemplate(label: 'jenkins-slave', cloud: 'kubernetes', containers: [
    containerTemplate(
        name: 'jnlp', 
        image: "jenkins/jnlp-slave:latest"
    )
  ]
) 
{
//引用jenkins-slave的pod模塊來構建Jenkins-Slave的pod
node("jenkins-slave"){
      // 第一步
      stage('測試'){
    sh '''
            echo "hello world" 
        '''
      }
  }
}

點擊執行后，你可以在 Jenkins 的日志中看到有一個 jenkins-slave 節點生成來執行任務，任務執行完成后便自動銷毀

配置 Jenkins Slave 運行的 Pod 模板

配置 Jenkins Slave 運行的 Pod 模板

以便在自由風格的軟件項目中使用

Pod Templates 下面添加
名稱 ：jnlp
命名空間 ：kube-ops
標簽列表 ：jenkins-slave

容器列表 添加
名稱 ： jnlp
Docker 鏡像 ：cnych/jenkins:jnlp6
運行的命令 和 命令參數 留空

掛載 /var/run/docker.sock 和 /root/.kube 和/usr/bin/docker                                  以便 cnych/jenkins:jnlp6 使用 kubectl 和 docker 命令

Service Account : jenkins-admin

測試

創建一個 自由風格的軟件項目 mytest

標簽表達式 ：jenkins-slave # 以上面的模板一致

構建 -> 執行 shell

echo "測試 Kubernetes 動態生成 jenkins slave"
echo "==============docker in docker==========="
docker info

echo "=============kubectl============="
kubectl get pods

點擊 構建執行 后，成功進行構

​

GO的持續集成

由於我們的容器是沒有go環境的 所以必須要有go plugin的支持【當然如有一個鏡像有go，docker這些東西是更好的】【這里我嘗試過很多方式都沒有得到好的解決方案，就是能不能不從遠程拉go的環境，比如從本地】

​

修改后的build.sh文件如下：

#!/bin/bash
#cd $WORKSPACE
 
export GOPROXY=https://goproxy.io
 
 #根據 go.mod 文件來處理依賴關系。
go mod tidy
 
# linux環境編譯
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o main
 
# 構建docker鏡像，項目中需要在當前目錄下有dockerfile，否則構建失敗

docker build -t gavintest .
docker tag  gavintest 192.168.100.30:8080/go/gavintest:${BUILD_NUMBER}

docker login -u admin -p '123456' 192.168.100.30:8080
docker push 192.168.100.30:8080/go/gavintest
 
docker rmi  gavintest
docker rmi 192.168.100.30:8080/go/gavintest:${BUILD_NUMBER}

kubectl set image deploy gavintest -n go  gavintest=192.168.100.30:8080/go/gavintest:${BUILD_NUMBER}
kubectl rollout status deploy gavintest -n go

創建自由風項目

  構建結果【這里下載go比較耗時】​​​

 

 

 -------------------

發現go 插件不好用，於是自己把 go的文件先下載下來， 然后放到內網環境【我這里是win10 下面】，腳本 內容如下：有些時候發現 

go1.15.6.linux-amd64.tar.gz文件不需要重復下載，有時候提示 “wget: can't open 'go1.15.6.linux-amd64.tar.gz': File exists 實際就是文件已經存在

wget http://192.168.100.2/go1.15.6.linux-amd64.tar.gz
tar xfz go1.15.6.linux-amd64.tar.gz -C /usr/local
export GOROOT=/usr/local/go
export PATH=$PATH:$GOROOT/bin
go version

可以放到go 插件里面，也可以放到項目里面

-----------------------2021-04-15---------------------

我的jankin-slave 沒有go，以前習慣於用docke commit 構建新的鏡像， 后來發現比較困難， 還是用Dockerfile 內容如下：

FROM cnych/jenkins:jnlp6

ADD go /usr/local/
RUN export GOROOT=/usr/local/go
RUN export PATH=$PATH:$GOROOT/bin

WORKDIR /home/jenkins

ENTRYPOINT ["/usr/local/bin/jenkins-slave"]

制作鏡像

docker build -t 192.168.100.30:8080/go/jenkins-agent:v1 .
docker push 192.168.100.30:8080/go/jenkins-agent

 

 附上deploy的文件gv.yaml，需要創建 kubectl create secret docker-registry regsecret --docker-server=192.168.100.30:8080 --docker-username=admin --docker-password=123456 -n=go

apiVersion: apps/v1
kind: Deployment
metadata:
  name: gavintest
  namespace: go
spec:
  replicas: 3
  minReadySeconds: 10 
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
  selector:
    matchLabels:
      name: gavintest
  template:
    metadata:
      labels:
        name: gavintest
    spec:
      imagePullSecrets:
      - name: regsecret
      containers:
      - name: gavintest
        image: 192.168.100.30:8080/go/gavintest:20210301
        ports:
        - containerPort: 80
        imagePullPolicy: Always
        volumeMounts:
         - mountPath: "/app/conf"
           name: httpd-volume
      volumes:
      - name: httpd-volume
        nfs:                        
           server: 192.168.100.11        
           path: "/nfs/data"                   
---
apiVersion: v1 
kind: Service 
metadata:
  name: gavintest
  namespace: go 
spec:
  type: ClusterIP
  ports:
    - port: 80 
      targetPort: 80 
      protocol: TCP 
  selector:
    name: gavintest
---
apiVersion: extensions/v1beta1 
kind: Ingress 
metadata:
  name: gavintest
  namespace: go
spec:
  rules:
    - host: go.k8s.com
      http:
        paths: 
          - path: /  
            backend:
              serviceName: gavintest
              servicePort: 80

 

 

 參考：

https://www.cnblogs.com/djoker/p/12375881.html

https://www.cnblogs.com/sanduzxcvbnm/p/13821268.html

https://github.com/diodonfrost/docker-jenkins-slave/blob/master/ubuntu-18.04-jenkins-slave/Dockerfile.ubuntu-18.04