2023年6月30日发(作者：)

实战项⽬部署应⽤到kubernetes流程（jenkins+docker+k8s） 说明通过jenkins构建java应⽤程序发布到k8s集群中本⽂已⼀个⼤数据的java项⽬来演⽰构建部署过程⽀持发布和回滚⽀持⼀套模板应⽤不同项⽬

k8s基础准备创建项⽬名称空间[root@k8s-master ~]# kubectl create ns architectnamespace/architect created[root@k8s-master ~]# kubectl label namespace architect istio-injection=enablednamespace/architect labeled[root@k8s-master ~]# kubectl get namespace -L istio-injectionNAME STATUS AGE ISTIO-INJECTIONarchitect Active 87s enableddefault Active 39d enabledistio-system Active 7d17h

kube-public Active 39d

kube-system Active 39d

[root@k8s-master ~]#

　　创建全局配置⽂件＃创建⼤数据配置⽂件[root@k8s-master ~]# ls dmp_ [root@k8s-master ~]#[root@k8s-master ~]# kubectl create configmap bigdata-config --from-file=/root/bigdata-counter/ -n architect

configmap/bigdata-config created[root@k8s-master ~]#

#全局环境变量[root@k8s-master ~]# cat yaml/

apiVersion: v1kind: ConfigMapmetadata: name: global-config namespace: architectdata: CONFIG_SERVER_URL: CONFIG_SERVER_USER: myconfig ILOG_BOOTSTRAP_SERVERS: s=192.168.1.122:9092 CONFIG_SERVER_PWDS: mc123456[root@k8s-master ~]#

[root@k8s-master ~]# kubectl apply -f yaml/

configmap/global-config created[root@k8s-master ~]# kubectl get configmap -n architectNAME DATA AGEbigdata-config 4 2m33sglobal-config 4 3s[root@k8s-master ~]#View Code

创建habror认证信息[root@k8s-master ~]# kubectl create secret docker-registry registry-secret --docker-server= --docker-username=admin --docker-password=Harbor12345 --docker-email=admin@ -n architectsecret/registry-secret created[root@k8s-master ~]# kubectl get secret -n architectNAME TYPE DATA AGEdefault-token-5wsxl /service-account-token 3 t /key-and-cert 3 2m24sregistry-secret /dockerconfigjson 1 10s　项⽬deployment⽂件可以做成模本，通过变量替换形式．后⾯有讲到　[root@k8s-master ~]# cat

apiVersion: extensions/v1beta1kind: Deploymentmetadata: labels: app: bigdata-interface version: default name: bigdata-interface namespace: architectspec: replicas: 1 selector: matchLabels: app: bigdata-interface version: default template: metadata: labels: app: bigdata-interface version: default spec: containers: - env: - name: izone - name: MY_SERVICE_NAME value: bigdata-interface - name: ilogEnvs value: izone,buildNumber - name: ILOG_BOOTSTRAP_SERVERS valueFrom: configMapKeyRef: key: ILOG_BOOTSTRAP_SERVERS name: global-config - name: CONFIG_SERVER_URL valueFrom: configMapKeyRef: key: CONFIG_SERVER_URL name: global-config - name: CONFIG_SERVER_USER valueFrom: configMapKeyRef: key: CONFIG_SERVER_USER name: global-config - name: CONFIG_SERVER_PWDS valueFrom: configMapKeyRef: key: CONFIG_SERVER_PWDS name: global-config - name: CONFIG_SERVER_PWDS valueFrom: configMapKeyRef: key: CONFIG_SERVER_PWDS name: global-config - name: JAR_BOOT_ARG value: ---fast=true - name: JDK_HEAP_OPTS value: -Xms256m -Xmx256m image: /architect/bigdata-interface:architect_b63 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 2 httpGet: path: /actuator/info port: 80 scheme: HTTP initialDelaySeconds: 300 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 30 name: bigdata-interface ports: - containerPort: 80 name: http protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: /actuator/info port: 80 scheme: HTTP initialDelaySeconds: 60 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 30 volumeMounts: - mountPath: /opt/jdk name: jdk-path - mountPath: /srv/applogs name: applogs imagePullSecrets: - name: registry-secret restartPolicy: Always terminationGracePeriodSeconds: 30 volumes: - hostPath: path: /usr/local/jdk/default type: "" name: jdk-path - hostPath: path: /data/applogs type: "" name: applogs[root@k8s-master ~]#

View Code

jenkins pipeline 模板⽂件node {

//项⽬名称 String APPNAME='bigdata-interface' //名称空间 String NAMESPACE='architect' String HarborUrl='' String HarborUser='admin' String HarborPasswd='Harbor12345' String KubernetHost = 'root@192.168.100.101'

properties([ parameters([ string(name:'GIT_BRANCH_TAG', defaultValue: 'master',description:'', trim: false)

])])

stage('git clone') { checkout([$class: 'GitSCM', branches: [[name: _BRANCH_TAG]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'b6a484b7-cbab-4215-9e93-2271d7a333bc }

stage('maven build') {

sh 'mvn clean package -U'

}

stage('docker build && push habror'){ sh "cd ${WORKSPACE}" restapi = '`ls -d *-restapi||echo .`'

sh "ls -l ${restapi}/target/" sh "echo ${restapi}" sh "curl -o ${restapi}/target/Dockerfile 192.168.1.55:8089/files/Dockerfile-java.v1" sh "docker build -t ${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} ${restapi}/target" sh "docker tag ${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} ${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER}" sh "docker login ${HarborUrl} -u ${HarborUser} -p ${HarborPasswd}" sh "docker push ${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER}"

} stage('deploy'){ echo "${_BRANCH_TAG}" echo "${BUILD_NUMBER}" //kubectl set image deployment/bigdata-interface bigdata-interface=/architect/bigdata-interface:architect_b48 -n architect updateImage="kubectl set image deployment/${APPNAME} --namespace=${NAMESPACE} ${APPNAME}=${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} --record" sh "ssh -p 22 ${KubernetHost} ${updateImage}" //test }

}View Code

　进⼀步改造pipeline　node {

String HarborUrl='' String HarborUser='admin' String HarborPasswd='Harbor12345' String KubernetHost = 'root@192.168.100.101'

properties([ parameters([ string(defaultValue: 'architect', description: 'git组', name: 'XBUILD_TEAM', trim: false), string(defaultValue: 'bigdata-interface', description: '项⽬名', name: 'XBUILD_DEPLOY', trim: false), string(defaultValue: 'git@:bigdata-center/', description: 'git仓库地址', name: 'XBUILD_REPO', trim: false), string(defaultValue: 'master', description: '版本名称', name: 'XBUILD_BRANCHE', trim: false) ])])

stage('git clone') { checkout([$class: 'GitSCM', branches: [[name: _BRANCHE]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'b6a484b7-cbab-4215-9e93-2271d7a333bc }

stage('maven build') {

sh 'mvn clean package -U'

}

stage('docker build && push habror'){ sh "cd ${WORKSPACE}" restapi = '`ls -d *-restapi||echo .`'

sh "ls -l ${restapi}/target/" sh "echo ${restapi}" sh "curl -o ${restapi}/target/Dockerfile 192.168.1.55:8089/files/Dockerfile-java.v1" NAMESPACE = _TEAM APPNAME = _DEPLOY

sh "docker build -t ${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} ${restapi}/target" sh "docker tag ${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} ${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER}" sh "docker login ${HarborUrl} -u ${HarborUser} -p ${HarborPasswd}" sh "docker push ${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER}"

} stage('deploy'){ NAMESPACE = _TEAM APPNAME = _DEPLOY echo "${_BRANCH_TAG}" echo "${BUILD_NUMBER}" //kubectl set image deployment/bigdata-interface bigdata-interface=/architect/bigdata-interface:architect_b48 -n architect //updateImage="kubectl set image deployment/${APPNAME} --namespace=${NAMESPACE} ${APPNAME}=${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} --record" //sh "ssh -p 22 ${KubernetHost} ${updateImage}" //test }

}View Code

　jenkins上创建⾃由风格项⽬ 效果如下 ⽀持参数化构建。但每项⽬单独JOB 查看镜像仓库已经推送正常

项⽬回滚node {

properties([ parameters([ string(defaultValue: 'architect', description: 'git组', name: 'XBUILD_TEAM', trim: false), string(defaultValue: 'bigdata-interface', description: '项⽬名', name: 'XBUILD_DEPLOY', trim: false), string(defaultValue: '1', description: '镜像标签', name: 'XBUILD_NUMBER', trim: false)

])])

stage('deploy'){

echo "${BUILD_NUMBER}" NAMESPACE = _TEAM APPNAME = _DEPLOY BUILD_NUMBER = _NUMBER echo "----${_NUMBER}---${_DEPLOY} ----${_TEAM} " //kubectl set image deployment/bigdata-interface bigdata-interface=/architect/bigdata-interface:architect_b48 -n architect //updateImage="kubectl set image deployment/${APPNAME} --namespace=${NAMESPACE} ${APPNAME}=${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} --record" //sh "ssh -p 22 ${KubernetHost} ${updateImage}" //test }

}View Code

　　jenkins效果如下：输⼊namespace和deployment 名称和上次构建版本 查看k8s集群可以灵活的在已知版本回滚操作[root@k8s-master ~]# kubectl get deploy -n architect -o wideNAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTORbigdata-interface 0/1 1 0 3h12m bigdata-interface /architect/bigdata-interface:architect_b1 app=bigdata-interface,version=default[root@k8s-master ~]# kubectl get pods -n architectNAME READY STATUS RESTARTS AGEbigdata-interface-6bf7b5646d-sf9j5 1/2 Running 0 24s[root@k8s-master ~]#

　通过以上实践，可以将deployment做成模板deployment模板⽂件　-bash-4.2$ cat

apiVersion: extensions/v1beta1kind: Deploymentmetadata: labels: app: {APPNAME} version: default name: {APPNAME} namespace: {NAMESPACE}spec: replicas: 1 selector: matchLabels: app: {APPNAME} version: default template: metadata: labels: app: {APPNAME} version: default spec: containers: - env: - name: izone - name: MY_SERVICE_NAME value: {APPNAME} - name: ilogEnvs value: izone,buildNumber - name: ILOG_BOOTSTRAP_SERVERS valueFrom: configMapKeyRef: key: ILOG_BOOTSTRAP_SERVERS name: global-config - name: CONFIG_SERVER_URL valueFrom: configMapKeyRef: key: CONFIG_SERVER_URL name: global-config - name: CONFIG_SERVER_USER valueFrom: configMapKeyRef: key: CONFIG_SERVER_USER name: global-config - name: CONFIG_SERVER_PWDS valueFrom: configMapKeyRef: key: CONFIG_SERVER_PWDS name: global-config - name: CONFIG_SERVER_PWDS valueFrom: configMapKeyRef: key: CONFIG_SERVER_PWDS name: global-config - name: JAR_BOOT_ARG value: ---fast=true - name: JDK_HEAP_OPTS value: -Xms256m -Xmx256m image: {HarborUrl}/{NAMESPACE}/{APPNAME}:{NAMESPACE}_b{BUILD_NUMBER} imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 2 httpGet: path: /actuator/info port: 80 scheme: HTTP initialDelaySeconds: 300 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 30 name: {APPNAME} ports: - containerPort: 80 name: http protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: /actuator/info port: 80 scheme: HTTP initialDelaySeconds: 60 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 30 volumeMounts: - mountPath: /opt/jdk name: jdk-path - mountPath: /srv/applogs name: applogs imagePullSecrets: - name: registry-secret restartPolicy: Always terminationGracePeriodSeconds: 30 volumes: - hostPath: path: /usr/local/jdk/default type: "" name: jdk-path - hostPath: path: /data/applogs type: "" name: applogs-bash-4.2$

View Code

　　　通过模板动态⽣成项⽬的deploy⽂件．进⾏升级回滚．⽐如下⾯的：node { String HarborUrl='' String HarborUser='admin' String HarborPasswd='Harbor12345' String KubernetHost = 'root@192.168.100.101' properties([ parameters([ string(defaultValue: 'architect', description: 'git组', name: 'XBUILD_TEAM', trim: false), string(defaultValue: 'bigdata-interface', description: '项⽬名', name: 'XBUILD_DEPLOY', trim: false), string(defaultValue: '1', description: '镜像标签', name: 'XBUILD_NUMBER', trim: false)

])])

stage('deploy'){

echo "${BUILD_NUMBER}" NAMESPACE = _TEAM APPNAME = _DEPLOY BUILD_NUMBER = _NUMBER echo "----${_NUMBER}---${_DEPLOY} ----${_TEAM} " //kubectl set image deployment/bigdata-interface bigdata-interface=/architect/bigdata-interface:architect_b48 -n architect updateImage="kubectl set image deployment/${APPNAME} --namespace=${NAMESPACE} ${APPNAME}=${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} --record" command="sed -e 's#{APPNAME}#${APPNAME}#g;s#{NAMESPACE}#${NAMESPACE}#g;s#{HarborUrl}#${HarborUrl}#g;s#{BUILD_NUMBER}#${BUILD_NUMBER}#g' /data/jenkins_slave/ > /data/jenkins_slave/${APP sh "${command}" sh "scp -P 22 /data/jenkins_slave/${APPNAME}- ${KubernetHost}:/root/ " //sh "ssh -p 22 ${KubernetHost} ${updateImage}" sh "ssh -p 22 ${KubernetHost} kubectl apply -f /root/${APPNAME}-" //test }

}View Code

pipeline参数化构建 ⽰例：pipeline { agent any parameters { gitParameter branch: '', branchFilter: '.*', defaultValue: '', description: '', name: 'XBUILD_BRANCHE', quickFilterEnabled: false, selectedValue: 'NONE', sortMode: 'NONE', tagFilter: '*', type: 'PT_BRANCH_TAG' string defaultValue: 'git@:company/', description: '', name: 'XBUILD_REPO', trim: false}

stages { stage('Example') { steps { checkout([$class: 'GitSCM', branches: [[name: _BRANCHE]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'b6a484b7-cbab-4215-9e93-2271d7a333bb', u } } } }　　构建效果如下：