經過網上視頻的學習整理如下：
環境規划
10.211.55.11 k8s-master1,nfs
10.211.55.12 k8s-master2,k8s-node1
10.211.55.13 k8s-master3,k8s-node2
10.211.55.14 Harbor倉庫,git倉庫，其他公共的服務
上面設計到的服務，可以查看我前面寫過的文章
項目下載地址
https://github.com/erlonglong/k8s-test/tree/master/devops
https://www.daocloud.io/mirror 鏡像加速
[root@wan src]# cat /etc/docker/daemon.json
{
"insecure-registries":["hub.k8s.com","10.211.55.14"],
"registry-mirrors": ["http://f1361db2.m.daocloud.io"]
}
各節點docker的加速和私有倉庫的配置，域名需要綁定
hub.k8s.com 10.211.55.14
使用K8S 部署jenkins
采用jenkins-master + jenkins-slave 的模式來實現CI&CD
https://github.com/jenkinsci/kubernetes-plugin/tree/master/src/main/kubernetes
wget https://raw.githubusercontent.com/jenkinsci/kubernetes-plugin/master/src/main/kubernetes/jenkins.yml
這里面我們用不到ingress 可以把相關的資源去掉
vim jenkins-master.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: jenkins
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: jenkins
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
resources: ["pods/log"]
verbs: ["get","list","watch"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: jenkins
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: jenkins
subjects:
- kind: ServiceAccount
name: jenkins
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: jenkins
labels:
name: jenkins
spec:
serviceName: jenkins
replicas: 1
updateStrategy:
type: RollingUpdate
template:
metadata:
name: jenkins
labels:
name: jenkins
spec:
terminationGracePeriodSeconds: 10
serviceAccountName: jenkins
containers:
- name: jenkins
image: jenkins/jenkins:lts-alpine
imagePullPolicy: Always
ports:
- containerPort: 8080
- containerPort: 50000
resources:
limits:
cpu: 1
memory: 1Gi
requests:
cpu: 0.5
memory: 500Mi
env:
- name: LIMITS_MEMORY
valueFrom:
resourceFieldRef:
resource: limits.memory
divisor: 1Mi
- name: JAVA_OPTS
value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
volumeMounts:
- name: jenkins-home
mountPath: /var/jenkins_home
livenessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12 # ~2 minutes
readinessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12 # ~2 minutes
volumeClaimTemplates:
- metadata:
name: jenkins-home
spec:
storageClassName: "managed-nfs-storage"
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 1Gi
---
apiVersion: v1
kind: Service
metadata:
name: jenkins
spec:
type: NodePort
selector:
name: jenkins
ports:
-
name: http
port: 80
targetPort: 8080
protocol: TCP
nodePort: 30008
-
name: agent
port: 50000
protocol: TCP
配置文件關鍵點是上面標注的位置：
PV 的問題，需要動態創建PV ，所以需要創建一個NFS 動態分配的PV 資源,當然也可以創建靜態的PV，這塊內容不多介紹了
關鍵看一下動態PV 的創建
nfs-server
vim /etc/exports
/data/volumes/ 10.0.0.0/8(rw,no_root_squash)
vim nfs-pv.yaml
kind: ServiceAccount
apiVersion: v1
metadata:
name: nfs-client-provisioner
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-client-provisioner-runner
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
namespace: default
roleRef:
kind: ClusterRole
name: nfs-client-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: default
roleRef:
kind: Role
name: leader-locking-nfs-client-provisioner
apiGroup: rbac.authorization.k8s.io
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: managed-nfs-storage
provisioner:
fuseim.pri/ifs # or choose another name, must match deployment's env PROVISIONER_NAME'
parameters:
archiveOnDelete: "true"
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: nfs-client-provisioner
spec:
replicas: 1
selector:
matchLabels:
app: nfs-client-provisioner
strategy:
type: Recreate
template:
metadata:
labels:
app: nfs-client-provisioner
spec:
serviceAccountName: nfs-client-provisioner
containers:
- name: nfs-client-provisioner
image: doudou007/nfs-pv:latest
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
value:
fuseim.pri/ifs
- name: NFS_SERVER
value:
10.211.55.11
- name: NFS_PATH
value:
/data/volumes
volumes:
- name: nfs-client-root
nfs:
server:
10.211.55.11
path:
/data/volumes
還有一點需要注意，給node節點需要安裝支持NFS掛載 客戶端的軟件包
yum install nfs-utils -y
kubectl apply -f nfs-pv.yaml
kubectl apply -f jenkins-master.yaml
[root@k8s-master1 jenkins-new]# kubectl get pods |egrep jenkins
jenkins-0 1/1 Running 0 2m52s
這倆說明一下，需要安裝的插件
重點是需要安裝
pipe 插件 git 插件 kubernetes插件 kubernetes deploy 這幾個插件：
接下來配置jenkins-slave
通過k8s 來動態創建jenkins-slave ,所以需要提供jenkins-slave 相關的鏡像
FROM centos:7
LABEL maintainer doudou
RUN yum install -y java-1.8.0-openjdk maven curl git libtool-ltdl-devel && \
yum clean all && \
rm -rf /var/cache/yum/* && \
mkdir -p /usr/share/jenkins
COPY slave.jar /usr/share/jenkins/slave.jar
COPY jenkins-slave /usr/bin/jenkins-slave
COPY settings.xml /etc/maven/settings.xml
RUN chmod +x /usr/bin/jenkins-slave
ENTRYPOINT ["jenkins-slave"]
這個鏡像制作涉及到的幾個文件
jenkins-slave: wget https://github.com/jenkinsci/docker-jnlp-slave/blob/master/jenkins-slave
slave.jar :通過master 獲取wget http://10.211.55.13:30008/jnlpJars/slave.jar
maven:settings.xml:
文件沒什么變化關鍵是加入了一個阿里雲的倉庫，提高下載包的速度：
<mirror>
<id>central</id>
<mirrorOf>central</mirrorOf>
<name>aliyun maven</name>
<url>https://maven.aliyun.com/repository/public</url>
</mirror>
</mirrors>
[root@k8s-master1 jenkins-slave]# ls
Dockerfile jenkins-slave settings.xml slave.jar
docker build -t hub.k8s.com/dev/jenkins-slave-jdk:1.8 ./
說明一下，這個是打包成Harbor推送的tag
hub.k8s.com 訪問路徑
dev 倉庫名
jenkins-slave-jdk:1.8 鏡像名和版本
docker login -u admin -p'Harbor12345'
docker push hub.k8s.com/dev/jenkins-slave-jdk:1.8
登陸Harbor可以看到相關的鏡像
下面通過jenkins來實現CI 和CD
配置參數化構建：默認master用於多分支發布
下面是pipe 的語法：
https://jenkins.io/zh/doc/book/pipeline/syntax/
http://10.211.55.13:30008/me/my-views/view/all/job/demo/pipeline-syntax/
官方鏈接：
https://jenkins.io/zh/doc/book/pipeline/syntax/
涉及到幾個認證的ID，需要我們提前生成
下面認證用的ID 是需要在jenkins上面增加相關憑據才能生成ID，這樣做是為了保護隱私的密碼和用戶名
def docker_registry_auth = "898bbf50-9f94-480e-861d-187f5633f26c"
def git_auth = "71bd0d12-3b2b-4acb-94cb-38f7aa9c6817"
def k8s_auth = "f3b86b4f-68a7-4913-8f2d-c7683ecaa235"
可以通過選項生成自己需要的語法，下面是配置好K8S ，在系統配置里面，增加一個雲，配置即可
因為我們jenkins 是通過POD 安裝的，所以我們用內部提供的cluster 連接就可以，默認POD 都有權限訪問的
如果需要配置外部的
cat /root/.kube/config
echo certificate-authority-data | base64 -d > ca.crt
echo client-certificate-data | base64 -d > client.crt
echo client-key-data | base64 -d > client.key
openssl pkcs12 -export -out cert.pfx -inkey client.key -in client.crt -certfile ca.crt
key 拷貝ca.crt
cert.pfx 上傳上去
查看文章：https://blog.csdn.net/diantun00/article/details/81180694
k8s-deploy的配置
提取出核心的代碼
kubernetesDeploy configs: 'deploy.yml', kubeconfigId: "${k8s_auth}"
deploy.yml 這個文件是在當前的倉庫里面查找，所有我們需要把這個文件也git 提交到倉庫
同樣，我們可以吧Jenkinsfile 的文件也提交到GIT 倉庫
[root@k8s-master1 java-demo]# ls
db deploy.yml Jenkinsfile LICENSE pom.xml README.md src
還有一點需要注意，因為是在POD 里面構建鏡像，所以需要通過secret來認證拉取鏡像的信息
def secret_name = "registry-pull-secret"
kubectl create secret docker-registry registry-pull-secret \
--docker-username=admin --docker-password=Harbor12345 \
--docker-email=doudou@k8s.com --docker-server=hub.k8s.com
下面是具體的流水線代碼
def registry = "hub.k8s.com"
def project = "dev"
def app_name = "demo"
def image_name = "${registry}/${project}/${app_name}:${BUILD_NUMBER}"
def git_address = "git@10.211.55.14:/home/git/java-demo.git"
def secret_name = "registry-pull-secret"
def docker_registry_auth = "898bbf50-9f94-480e-861d-187f5633f26c"
def git_auth = "71bd0d12-3b2b-4acb-94cb-38f7aa9c6817"
def k8s_auth = "f3b86b4f-68a7-4913-8f2d-c7683ecaa235"
podTemplate(label: 'jenkins-slave', cloud: 'kubernetes', containers: [
containerTemplate(
name: 'jnlp',
image: "${registry}/dev/jenkins-slave-jdk:1.8"
),
],
volumes: [
hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock'),
hostPathVolume(mountPath: '/usr/bin/docker', hostPath: '/usr/bin/docker')
],
)
{
node("jenkins-slave"){
// 第一步
stage('拉取代碼'){
checkout([$class: 'GitSCM', branches: [[name: '${Branch}']], userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_address}"]]])
}
// 第二步
stage('代碼編譯'){
sh "mvn clean package -Dmaven.test.skip=true"
}
// 第三步
stage('構建鏡像'){
withCredentials([usernamePassword(credentialsId: "${docker_registry_auth}", passwordVariable: 'password', usernameVariable: 'username')]) {
sh """
echo '
FROM doudou007/tomcat
RUN rm -rf /usr/local/tomcat/webapps/*
ADD target/*.war /usr/local/tomcat/webapps/ROOT.war
' > Dockerfile
ls
ls target
docker build -t ${image_name} .
echo "${password}"|docker login -u ${username} --password-stdin ${registry}
docker push ${image_name}
"""
}
}
// 第四步
stage('部署到K8S平台'){
sh """
sed -i 's#\$IMAGE_NAME#${image_name}#' deploy.yml
sed -i 's#\$SECRET_NAME#${secret_name}#' deploy.yml
"""
kubernetesDeploy configs: 'deploy.yml', kubeconfigId: "${k8s_auth}"
}
}
}
部署的yaml文件
vim deploy.yml
apiVersion: apps/v1
kind: Deployment
metadata:
name: web
spec:
replicas: 2
selector:
matchLabels:
app: java-demo
template:
metadata:
labels:
app: java-demo
spec:
imagePullSecrets:
- name: $SECRET_NAME
containers:
- name: tomcat
image: $IMAGE_NAME
ports:
- containerPort: 8080
name: web
livenessProbe:
httpGet:
path: /
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12
readinessProbe:
httpGet:
path: /
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12
---
apiVersion: v1
kind: Service
metadata:
name: web
spec:
type: NodePort
selector:
app: java-demo
ports:
- protocol: TCP
port: 80
targetPort: 8080
查看控制台日志輸出：
推送上的鏡像
[root@k8s-master1 demo]# kubectl get pods|egrep web
web-5b7c77d664-ddwbp 1/1 Running 1 3h2m
web-5b7c77d664-rd7lr 1/1 Running 1 3h2m
然后通過節點訪問，就可以看到訪問的程序了
說明整個流程是完成了，這塊需要加強學習的地方是jenkins 流水線發布的學習
核心就是jenkins的配置，這里方便演示，只用到GIT，后續可以擴展成gitlab
下面是通過gitlab自動構建的文章：
https://www.cnblogs.com/yinzhengjie/p/9613270.html
還有一種方式應用k8s 的
kubernetes cli插件
這個插件功能只要在pod里面執行kubectl
提前把kubectl 打包編譯到jenkins 的鏡像里面
然后生成需要的憑據
jenkins添加kubernetes憑證
找一個有集群管理員角色的的secrets
kubectl describe secrets -n kube-system kubernetes-dashboard-token-fr9cz |egrep token:
或者
kubectl -n kube-system get secrets kubernetes-dashboard-token-fr9cz -o go-template --template '{{index .data "token"}}' | base64 -d
然后在jenkins界面添加憑證
