pipeline { agent any environment { APP_NAME = 'demo-nginx' NAMESPACE = 'demo-app' DOCKER_REGISTRY = 'docker.io' DOCKER_REPO = 'vladcrypto' GITEA_URL = 'http://gitea-http.gitea.svc.cluster.local:3000' GITEA_REPO = 'admin/k3s-gitops' GITEA_BRANCH = 'main' BUILD_TAG = "${env.BUILD_NUMBER}" IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}" // Rollback configuration ROLLBACK_ENABLED = 'true' DEPLOYMENT_TIMEOUT = '300s' ARGOCD_SYNC_TIMEOUT = '120' SKIP_HEALTH_CHECK = 'true' // Notification configuration TELEGRAM_BOT_TOKEN = credentials('telegram-bot-token') TELEGRAM_CHAT_ID = credentials('telegram-chat-id') // Build info BUILD_URL = "${env.BUILD_URL}" GIT_COMMIT_SHORT = "" DEPLOYMENT_START_TIME = "" DEPLOYMENT_END_TIME = "" } stages { stage('Initialization') { steps { script { echo "πŸš€ Starting deployment pipeline..." DEPLOYMENT_START_TIME = sh(script: 'date +%s', returnStdout: true).trim() sendTelegramNotification( status: 'STARTED', message: """ πŸš€ Deployment Started Application: ${APP_NAME} Build: #${BUILD_NUMBER} Branch: ${env.BRANCH_NAME} Namespace: ${NAMESPACE} Started by: ${env.BUILD_USER ?: 'Jenkins'} Building and deploying... """, color: 'πŸ”΅' ) } } } stage('Save Current State') { when { branch 'main' } steps { script { echo "πŸ“Έ Saving current deployment state for rollback..." sh """ kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ -o jsonpath='{.spec.template.spec.containers[0].image}' \ > /tmp/previous_image_${BUILD_NUMBER}.txt || echo "none" > /tmp/previous_image_${BUILD_NUMBER}.txt PREV_IMAGE=\$(cat /tmp/previous_image_${BUILD_NUMBER}.txt) echo "Previous image: \${PREV_IMAGE}" """ sh """ kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ -o jsonpath='{.spec.replicas}' \ > /tmp/previous_replicas_${BUILD_NUMBER}.txt || echo "2" > /tmp/previous_replicas_${BUILD_NUMBER}.txt """ echo "βœ… Current state saved" } } } stage('Checkout Source') { steps { echo "πŸ“¦ Creating application artifacts..." sh """ cat > Dockerfile << 'EOF' FROM nginx:1.25.3-alpine COPY index.html /usr/share/nginx/html/index.html COPY nginx.conf /etc/nginx/nginx.conf EXPOSE 80 CMD ["nginx", "-g", "daemon off;"] EOF """ sh """ cat > index.html << EOF Demo Nginx

Demo Nginx - Build #${BUILD_NUMBER}

Environment: Production

Version: ${IMAGE_TAG}

Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}

EOF """ sh ''' cat > nginx.conf << 'EOF' user nginx; worker_processes auto; error_log /var/log/nginx/error.log warn; pid /var/run/nginx.pid; events { worker_connections 1024; } http { include /etc/nginx/mime.types; default_type application/octet-stream; log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; access_log /var/log/nginx/access.log main; sendfile on; keepalive_timeout 65; server { listen 80; server_name _; location / { root /usr/share/nginx/html; index index.html; } location /health { access_log off; return 200 "healthy\n"; add_header Content-Type text/plain; } } } EOF ''' } } stage('Build Docker Image') { steps { script { echo "πŸ—οΈ Building Docker image..." sendTelegramNotification( status: 'BUILDING', message: """ πŸ—οΈ Building Docker Image Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} Stage: Build """, color: 'πŸ”΅' ) sh """ docker build \ -t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} \ -t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest \ . """ echo "βœ… Image built successfully!" } } } stage('Push to Registry') { when { branch 'main' } steps { script { echo "πŸ“€ Pushing image to registry..." sendTelegramNotification( status: 'PUSHING', message: """ πŸ“€ Pushing to Registry Registry: ${DOCKER_REGISTRY} Image: ${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} """, color: 'πŸ”΅' ) withCredentials([usernamePassword( credentialsId: 'docker-registry-credentials', usernameVariable: 'DOCKER_USER', passwordVariable: 'DOCKER_PASS' )]) { sh """ echo "\${DOCKER_PASS}" | docker login ${DOCKER_REGISTRY} -u "\${DOCKER_USER}" --password-stdin docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest docker logout ${DOCKER_REGISTRY} """ } echo "βœ… Image pushed successfully!" } } } stage('Update GitOps Manifests') { when { branch 'main' } steps { script { echo "πŸ“ Updating Kubernetes manifests..." sendTelegramNotification( status: 'UPDATING', message: """ πŸ“ Updating GitOps Manifests Repository: ${GITEA_REPO} Branch: ${GITEA_BRANCH} File: apps/demo-nginx/deployment.yaml """, color: 'πŸ”΅' ) withCredentials([usernamePassword( credentialsId: 'gitea-credentials', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_PASS' )]) { sh """ rm -rf k3s-gitops || true git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git cd k3s-gitops git config user.name "Jenkins" git config user.email "jenkins@thedevops.dev" # Save current commit for rollback git rev-parse HEAD > /tmp/previous_commit_${BUILD_NUMBER}.txt sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml git add apps/demo-nginx/deployment.yaml git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes" git push origin main """ } echo "βœ… Manifests updated!" } } } stage('Wait for ArgoCD Sync') { steps { script { echo "⏳ Waiting for ArgoCD to apply Git revision..." def expectedRevision = sh( script: "git rev-parse HEAD", returnStdout: true ).trim() for (int i = 1; i <= 12; i++) { def argoRevision = sh( script: "kubectl get application demo-nginx -n argocd -o jsonpath='{.status.sync.revision}'", returnStdout: true ).trim() def syncStatus = sh( script: "kubectl get application demo-nginx -n argocd -o jsonpath='{.status.sync.status}'", returnStdout: true ).trim() echo "Expected Git revision : ${expectedRevision}" echo "ArgoCD applied revision: ${argoRevision}" echo "ArgoCD sync status : ${syncStatus}" if (syncStatus == "Synced" && argoRevision == expectedRevision) { echo "βœ… ArgoCD successfully applied Git revision" return } sleep 10 } error("❌ ArgoCD did not apply expected Git revision in time") } } } stage('Wait for Deployment') { when { branch 'main' } steps { script { echo "⏳ Waiting for Kubernetes rollout to complete..." sendTelegramNotification( status: 'DEPLOYING', message: """ πŸš€ Deploying to Kubernetes Deployment: ${APP_NAME} Namespace: ${NAMESPACE} Image: ${IMAGE_TAG} Timeout: ${DEPLOYMENT_TIMEOUT} Rolling out new pods... """, color: 'πŸ”΅' ) try { // Wait for rollout to complete echo "Starting rollout status check..." sh """ kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT} """ echo "βœ… Rollout completed successfully!" // Additional verification - wait a bit for pods to stabilize echo "Waiting 10 seconds for pods to stabilize..." sleep 10 } catch (Exception e) { echo "❌ Deployment rollout failed: ${e.message}" // Get pod status for debugging try { def podStatus = sh( script: """ kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide """, returnStdout: true ).trim() echo "Current pod status:\n${podStatus}" def events = sh( script: """ kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20 """, returnStdout: true ).trim() echo "Recent events:\n${events}" } catch (Exception debugEx) { echo "Could not fetch debug info: ${debugEx.message}" } throw e } } } } stage('Verify Deployment') { when { branch 'main' } steps { script { echo "βœ… Verifying deployment and pod status..." try { def verifyResult = sh(script: """#!/bin/bash set -e echo "================================================" echo "DEPLOYMENT VERIFICATION" echo "================================================" # 1. Check deployment status echo "" echo "1. Checking deployment status..." READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}') DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}') UPDATED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.updatedReplicas}') AVAILABLE_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.availableReplicas}') echo " Desired replicas: \${DESIRED_PODS}" echo " Updated replicas: \${UPDATED_PODS}" echo " Ready replicas: \${READY_PODS}" echo " Available replicas: \${AVAILABLE_PODS}" if [ "\${READY_PODS}" != "\${DESIRED_PODS}" ]; then echo " ❌ FAILED: Not all pods are ready!" echo " Expected: \${DESIRED_PODS}, Got: \${READY_PODS}" exit 1 fi echo " βœ… All pods ready" # 2. Verify running pod images echo "" echo "2. Checking running pod images..." POD_IMAGES=$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.spec.containers[0].image}{"\n"}{end}') echo " Running pod images:" echo "${POD_IMAGES}" echo " Expected tag: ${IMAGE_TAG}" if ! echo "${POD_IMAGES}" | grep -q "${IMAGE_TAG}"; then echo " ❌ FAILED: Running pods do not use expected image!" exit 1 fi echo " βœ… All running pods use expected image" # 3. CRITICAL: Verify actual running pod images echo "" echo "3. Checking actual running pod images..." POD_IMAGES=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.status.containerStatuses[0].image}{"\\n"}{end}') echo " Running pod images:" echo "\${POD_IMAGES}" | while read -r img; do echo " - \${img}" done # Check if all pods are running the correct image WRONG_IMAGE_COUNT=0 while IFS= read -r img; do if [[ "\${img}" != *"${IMAGE_TAG}"* ]]; then echo " ❌ Pod running wrong image: \${img}" WRONG_IMAGE_COUNT=\$((WRONG_IMAGE_COUNT + 1)) fi done <<< "\${POD_IMAGES}" if [ \${WRONG_IMAGE_COUNT} -gt 0 ]; then echo " ❌ FAILED: \${WRONG_IMAGE_COUNT} pod(s) running old image!" echo " This is the ArgoCD sync bug - deployment updated but pods not rolled out" exit 1 fi echo " βœ… All pods running correct image" # 4. Check pod readiness echo "" echo "4. Checking pod readiness probes..." NOT_READY=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} --field-selector=status.phase!=Running --no-headers 2>/dev/null | wc -l) if [ "\${NOT_READY}" -gt 0 ]; then echo " ⚠️ WARNING: \${NOT_READY} pod(s) not in Running state" kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} else echo " βœ… All pods in Running state" fi # 5. Check container restart count echo "" echo "5. Checking for container restarts..." RESTART_COUNTS=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\\n"}{end}') MAX_RESTARTS=0 while IFS= read -r count; do if [ "\${count}" -gt "\${MAX_RESTARTS}" ]; then MAX_RESTARTS=\${count} fi done <<< "\${RESTART_COUNTS}" echo " Max restart count: \${MAX_RESTARTS}" if [ "\${MAX_RESTARTS}" -gt 3 ]; then echo " ⚠️ WARNING: High restart count detected" else echo " βœ… Restart count acceptable" fi echo "" echo "================================================" echo "βœ… ALL VERIFICATION CHECKS PASSED!" echo "================================================" """, returnStdout: true).trim() echo verifyResult echo "βœ… Deployment verified successfully!" } catch (Exception e) { echo "❌ Deployment verification failed!" echo "Error: ${e.message}" // Additional debugging try { echo "\n=== DEBUGGING INFORMATION ===" def pods = sh( script: "kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide", returnStdout: true ).trim() echo "Current pods:\n${pods}" def replicaset = sh( script: "kubectl get replicaset -n ${NAMESPACE} -l app=${APP_NAME}", returnStdout: true ).trim() echo "ReplicaSets:\n${replicaset}" def events = sh( script: "kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20", returnStdout: true ).trim() echo "Recent events:\n${events}" } catch (Exception debugEx) { echo "Could not fetch debug info: ${debugEx.message}" } throw e } } } } } post { success { script { DEPLOYMENT_END_TIME = sh(script: 'date +%s', returnStdout: true).trim() def duration = (DEPLOYMENT_END_TIME.toInteger() - DEPLOYMENT_START_TIME.toInteger()) def durationMin = duration / 60 def durationSec = duration % 60 // Get deployment details def podStatus = sh( script: """ kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} \ -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.phase}{" "}{.status.containerStatuses[0].restartCount}{"\\n"}{end}' """, returnStdout: true ).trim() def deployedImage = sh( script: """ kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ -o jsonpath='{.spec.template.spec.containers[0].image}' """, returnStdout: true ).trim() def replicas = sh( script: """ kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ -o jsonpath='{.status.replicas}' """, returnStdout: true ).trim() echo """ βœ… DEPLOYMENT SUCCESS! Application: ${APP_NAME} Image: ${deployedImage} Namespace: ${NAMESPACE} Build: #${BUILD_NUMBER} Duration: ${durationMin}m ${durationSec}s All checks passed! ✨ """ // Send detailed success notification sendTelegramNotification( status: 'SUCCESS', message: """ βœ… Deployment Successful! ━━━━━━━━━━━━━━━━━━━━━━ πŸ“¦ Application Details Name: ${APP_NAME} Build: #${BUILD_NUMBER} Branch: ${env.BRANCH_NAME} Namespace: ${NAMESPACE} ━━━━━━━━━━━━━━━━━━━━━━ 🐳 Docker Image Registry: ${DOCKER_REGISTRY} Repository: ${DOCKER_REPO}/${APP_NAME} Tag: ${IMAGE_TAG} Full Image: ${deployedImage} ━━━━━━━━━━━━━━━━━━━━━━ ☸️ Kubernetes Status Replicas: ${replicas}/${replicas} Ready Rollout: Completed βœ… Health: All pods healthy ━━━━━━━━━━━━━━━━━━━━━━ ⏱️ Deployment Metrics Duration: ${durationMin}m ${durationSec}s Started: ${new Date(DEPLOYMENT_START_TIME.toLong() * 1000).format('HH:mm:ss')} Completed: ${new Date(DEPLOYMENT_END_TIME.toLong() * 1000).format('HH:mm:ss')} ━━━━━━━━━━━━━━━━━━━━━━ πŸ”— Links Jenkins Build GitOps Repository Deployed by Jenkins CI/CD Pipeline πŸš€ """, color: 'βœ…' ) // Cleanup rollback files sh """ rm -f /tmp/previous_image_${BUILD_NUMBER}.txt rm -f /tmp/previous_replicas_${BUILD_NUMBER}.txt rm -f /tmp/previous_commit_${BUILD_NUMBER}.txt """ } } failure { script { def errorDetails = "" def previousImage = "unknown" try { errorDetails = sh( script: """ kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -10 """, returnStdout: true ).trim() } catch (Exception e) { errorDetails = "Could not fetch events: ${e.message}" } if (env.BRANCH_NAME == 'main' && env.ROLLBACK_ENABLED == 'true') { echo """ ❌ DEPLOYMENT FAILED - INITIATING ROLLBACK! Rolling back to previous version... """ sendTelegramNotification( status: 'ROLLING_BACK', message: """ πŸ”„ Deployment Failed - Rolling Back Application: ${APP_NAME} Failed Build: #${BUILD_NUMBER} Image: ${IMAGE_TAG} Initiating automatic rollback... """, color: '⚠️' ) try { previousImage = sh( script: "cat /tmp/previous_image_${BUILD_NUMBER}.txt 2>/dev/null || echo 'none'", returnStdout: true ).trim() if (previousImage != 'none' && previousImage != '') { echo "πŸ”„ Rolling back to: ${previousImage}" sh """ kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE} kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=180s """ withCredentials([usernamePassword( credentialsId: 'gitea-credentials', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_PASS' )]) { sh """ cd k3s-gitops || git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git cd k3s-gitops git config user.name "Jenkins" git config user.email "jenkins@thedevops.dev" git revert --no-edit HEAD || true git push origin main || true """ } sendTelegramNotification( status: 'ROLLBACK_SUCCESS', message: """ βœ… Rollback Completed ━━━━━━━━━━━━━━━━━━━━━━ πŸ“¦ Application Name: ${APP_NAME} Failed Build: #${BUILD_NUMBER} Rolled Back To: ${previousImage} ━━━━━━━━━━━━━━━━━━━━━━ πŸ”„ Rollback Status Status: Success βœ… Method: kubectl rollout undo Git: Commit reverted ━━━━━━━━━━━━━━━━━━━━━━ πŸ“‹ Recent Events ${errorDetails.take(500)} ━━━━━━━━━━━━━━━━━━━━━━ ⚠️ Action Required Please review logs and fix issues before redeploying View Build Logs """, color: 'βœ…' ) } else { sendTelegramNotification( status: 'ROLLBACK_FAILED', message: """ ❌ Rollback Failed Application: ${APP_NAME} Build: #${BUILD_NUMBER} Error: No previous version found ⚠️ MANUAL INTERVENTION REQUIRED kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE} View Build Logs """, color: 'πŸ”΄' ) } } catch (Exception e) { sendTelegramNotification( status: 'ROLLBACK_FAILED', message: """ ❌ Rollback Failed Application: ${APP_NAME} Build: #${BUILD_NUMBER} Error: ${e.message} ⚠️ MANUAL ROLLBACK REQUIRED kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE} View Build Logs """, color: 'πŸ”΄' ) } } else { sendTelegramNotification( status: 'FAILED', message: """ ❌ Deployment Failed ━━━━━━━━━━━━━━━━━━━━━━ πŸ“¦ Application Name: ${APP_NAME} Build: #${BUILD_NUMBER} Branch: ${env.BRANCH_NAME} Image: ${IMAGE_TAG} ━━━━━━━━━━━━━━━━━━━━━━ πŸ“‹ Recent Events ${errorDetails.take(500)} ━━━━━━━━━━━━━━━━━━━━━━ πŸ”— Links View Console Output Build Details Rollback: ${env.ROLLBACK_ENABLED == 'true' ? 'Enabled but not on main branch' : 'Disabled'} """, color: 'πŸ”΄' ) } } } always { sh """ docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest || true docker stop test-${BUILD_NUMBER} 2>/dev/null || true docker rm test-${BUILD_NUMBER} 2>/dev/null || true """ cleanWs() } } } // Telegram notification function def sendTelegramNotification(Map args) { def status = args.status def message = args.message def color = args.color ?: 'πŸ”΅' try { // HTML formatting for Telegram def formattedMessage = message.trim() sh """ curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ -d chat_id="${TELEGRAM_CHAT_ID}" \ -d parse_mode="HTML" \ -d disable_web_page_preview=true \ -d text="${formattedMessage}" """ echo "πŸ“± Telegram notification sent: ${status}" } catch (Exception e) { echo "⚠️ Failed to send Telegram notification: ${e.message}" // Don't fail the pipeline if notification fails } }