2.Jenkins結合k8s完成Jenkins slave功能

本文轉載自查看原文 2020-02-28 09:55 4841 Jenkins

1.構建鏡像

下載基礎鏡像，這里使用openvz的包，下載地址為：https://wiki.openvz.org/Download/template/precreated，下載centos7的鏡像

下載鏡像后導入到本地

docker import centos-7-x86_64.tar.gz openvz-centos7

從官方下載最新版的Jenkins的war包和適配的jdk，這里使用jdk8。構建Jenkins的dockerfile文件dockerfile-jenkins

[root@dataserver jenkins]#cat dockerfile-jenkins 
FROM openvz-centos7
ADD jdk-8u241-linux-x64.tar.gz  /home
ADD jenkins2.222.war /home/jenkins.war
ENV JAVA_HOME=/home/jdk1.8.0_241
ENV PATH=$PATH:/home/jdk1.8.0_241/bin
ENV JENKINS_HOME=/var/jenkins_home
WORKDIR /home
CMD java -jar jenkins.war --httpPort=8080

構建后推送到本地倉庫

docker build -t 192.168.31.9:5000/jenkins-self:2.222 -f dockerfile-jenkins .
docker push 192.168.31.9:5000/jenkins-self:2.222

構建Jenkins-agent的dockerfile-agent

[root@dataserver jenkins]# cat dockerfile-agent 
FROM openvz-centos7
ADD jdk-8u241-linux-x64.tar.gz  /home
ADD agent.jar /home
ENV JAVA_HOME=/home/jdk1.8.0_241
ENV PATH=$PATH:/home/jdk1.8.0_241/bin
RUN yum -y install docker kubernetes-client kubernetes*
WORKDIR /home
CMD exec /home/jdk1.8.0_241/bin/java -Dorg.jenkinsci.remoting.engine.JnlpProtocol3.disabled=true -cp /home/agent.jar hudson.remoting.jnlp.Main -headless -url ${JENKINS_URL} -workDir ${JENKINS_AGENT_WORKDIR} ${JENKINS_SECRET} ${JENKINS_AGENT_NAME}

構建后推送到本地倉庫

docker build -t 192.168.31.9:5000/jenkins-agent-self:v2.222.11 -f dockerfile-agent .
docker push 192.168.31.9:5000/jenkins-agent-self:v2.222.11

agent.jar可以從Jenkins中下載。manager Jenkins --> 節點管理 --> 新建節點 --> 輸入test，選擇固定節點 --> 遠程工作目錄輸入 /home --> 啟動方式選擇通過web web啟動代理后點擊保存。
這里這個節點是不在線的，點擊后提示如下：

點擊下面界面顯示的agent.jar后就可以下載對應的agent.jar包

2.部署Jenkins到k8s中

namespace.yaml

[root@env11 cicd]# cat namespace.yaml 
apiVersion: v1
kind: Namespace
metadata:
  name: kube-ops

pvc.yaml這里使用nfs服務提供外置存儲

[root@env11 cicd]# cat pvc.yaml 
apiVersion: v1
kind: PersistentVolume
metadata:
  name: opspv
spec:
  capacity:
    storage: 20Gi
  accessModes:
  - ReadWriteMany
  persistentVolumeReclaimPolicy: Delete
  nfs:
    server: 192.168.31.9
    path: /data/nfsData/jenkins

---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: opspvc
  namespace: kube-ops
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 20Gi

rbac.yaml

[root@env11 cicd]# cat rbac.yaml 
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins2
  namespace: kube-ops

---

kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: jenkins2
  namespace: kube-ops
rules:
  - apiGroups: ["extensions", "apps"]
    resources: ["deployments"]
    verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
  - apiGroups: [""]
    resources: ["services"]
    verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["create","delete","get","list","patch","update","watch"]
  - apiGroups: [""]
    resources: ["pods/exec"]
    verbs: ["create","delete","get","list","patch","update","watch"]
  - apiGroups: [""]
    resources: ["pods/log"]
    verbs: ["get","list","watch"]
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get"]

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: jenkins2
  namespace: kube-ops
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: jenkins2
subjects:
  - kind: ServiceAccount
    name: jenkins2
    namespace: kube-ops

jenkins2.yaml

[root@env11 cicd]# cat jenkins2.yaml 
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: jenkins2
  namespace: kube-ops
spec:
  template:
    metadata:
      labels:
        app: jenkins2
    spec:
      terminationGracePeriodSeconds: 10
      serviceAccount: jenkins2
      containers:
      - name: jenkins
        image: 192.168.31.9:5000/jenkins-self:v2.222
        imagePullPolicy: IfNotPresent
        command: ["/bin/sh","-c","java -jar jenkins.war --httpPort=8080"]
        ports:
        - containerPort: 8080
          name: web
          protocol: TCP
        - containerPort: 50000
          name: agent
          protocol: TCP
        resources:
          limits:
            cpu: 2000m
            memory: 2Gi
          requests:
            cpu: 500m
            memory: 512Mi
        #livenessProbe:
        #  httpGet:
        #    path: /login
        #    port: 8080
        #  initialDelaySeconds: 60
        #  timeoutSeconds: 5
        #  failureThreshold: 12 
        #readinessProbe:
        #  httpGet:
        #    path: /login
        #    port: 8080
        #  initialDelaySeconds: 60
        #  timeoutSeconds: 5
        #  failureThreshold: 12
        volumeMounts:
        - name: jenkinshome
          subPath: jenkinsenv
          mountPath: /var/jenkins_home
        env:
        - name: LIMITS_MEMORY
          valueFrom:
            resourceFieldRef:
              resource: limits.memory
              divisor: 1Mi
        - name: JAVA_OPTS
          value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85 -Duser.timezone=Asia/Shanghai
      securityContext:
        fsGroup: 1000
      volumes:
      - name: jenkinshome
        persistentVolumeClaim:
          claimName: opspvc

---
apiVersion: v1
kind: Service
metadata:
  name: jenkins2
  namespace: kube-ops
  labels:
    app: jenkins2
spec:
  selector:
    app: jenkins2
  type: NodePort
  ports:
  - name: web
    port: 8080
    targetPort: web
    nodePort: 30002
  - name: agent
    port: 50000
    targetPort: agent

部署完成后，就可以通過集群IP:3002后登陸Jenkins

3.需要安裝插件：

Jenkins需要安裝插件Kubernetes plugin插件才可以動態調用k8s的api接口完成pod的創建

4.配置Jenkins調用k8s配置

配置Jenkins代理為固定端口50000，因為service定義是對外暴露的端口是50000。

點擊Jenkins的節點管理

點擊節點管理

點擊configure clouds，添加一個遠端服務

配置Kubernetes地址為http://kubernetes.default:443，命名空間為kube-ops，和Jenkins是同一個命名空間，然后點擊連接測試。因為Jenkins的pod在提交是配置了rbac授權，所以可以訪問k8s。

配置Jenkins地址為http://jenkins2.kube-ops.svc.cluster.local:8080，如果上面的service配置的名字為jenkins，這里就配置為jenkins而不是jenkins2。

配置pod模版，名字配置為jenkins-slave-001，命名空間為kube-ops，標簽列表為haimaxy-jnlp，這個標簽列表很重要，后面需要用到。容器列表配置jnlp，鏡像就是自己構建的192.168.31.9:5000/jenkins-agent-self:v2.222.11，工作目錄寫/home/jenkins