pipeline { agent any environment { APP_NAME = 'demo-nginx' NAMESPACE = 'demo-app' DOCKER_REGISTRY = 'docker.io' DOCKER_REPO = 'vladcrypto' GITEA_URL = 'http://gitea-http.gitea.svc.cluster.local:3000' GITEA_REPO = 'admin/k3s-gitops' GITEA_BRANCH = 'main' BUILD_TAG = "${env.BUILD_NUMBER}" IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}" // Rollback configuration ROLLBACK_ENABLED = 'true' DEPLOYMENT_TIMEOUT = '300s' ARGOCD_SYNC_TIMEOUT = '120' SKIP_HEALTH_CHECK = 'true' // Notification configuration TELEGRAM_BOT_TOKEN = credentials('telegram-bot-token') TELEGRAM_CHAT_ID = credentials('telegram-chat-id') // Build info BUILD_URL = "${env.BUILD_URL}" GIT_COMMIT_SHORT = "" DEPLOYMENT_START_TIME = "" DEPLOYMENT_END_TIME = "" } stages { stage('Initialization') { steps { script { echo "🚀 Starting deployment pipeline..." env.DEPLOYMENT_START_TIME = sh(script: 'date +%s', returnStdout: true).trim() sendTelegramNotification( status: 'STARTED', message: """ 🚀 Deployment Started Application: ${APP_NAME} Build: #${BUILD_NUMBER} Branch: ${env.BRANCH_NAME} Namespace: ${NAMESPACE} Started by: ${env.BUILD_USER ?: 'Jenkins'} Building and deploying... """, color: '🔵' ) } } } stage('Save Current State') { when { branch 'main' } steps { script { echo "📸 Saving current deployment state for rollback..." sh """ kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ -o jsonpath='{.spec.template.spec.containers[0].image}' \ > /tmp/previous_image_${BUILD_NUMBER}.txt || echo "none" > /tmp/previous_image_${BUILD_NUMBER}.txt PREV_IMAGE=\$(cat /tmp/previous_image_${BUILD_NUMBER}.txt) echo "Previous image: \${PREV_IMAGE}" """ sh """ kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ -o jsonpath='{.spec.replicas}' \ > /tmp/previous_replicas_${BUILD_NUMBER}.txt || echo "2" > /tmp/previous_replicas_${BUILD_NUMBER}.txt """ echo "✅ Current state saved" } } } stage('Checkout Source') { steps { echo "📦 Creating application artifacts..." sh """ cat > Dockerfile << 'EOF' FROM nginx:1.25.3-alpine COPY index.html /usr/share/nginx/html/index.html COPY nginx.conf /etc/nginx/nginx.conf EXPOSE 80 CMD ["nginx", "-g", "daemon off;"] EOF """ sh """ cat > index.html << EOF Demo Nginx

Demo Nginx - Build #${BUILD_NUMBER}

Environment: Production

Version: ${IMAGE_TAG}

Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}

EOF """ sh ''' cat > nginx.conf << 'EOF' user nginx; worker_processes auto; error_log /var/log/nginx/error.log warn; pid /var/run/nginx.pid; events { worker_connections 1024; } http { include /etc/nginx/mime.types; default_type application/octet-stream; log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; access_log /var/log/nginx/access.log main; sendfile on; keepalive_timeout 65; server { listen 80; server_name _; location / { root /usr/share/nginx/html; index index.html; } location /health { access_log off; return 200 "healthy\n"; add_header Content-Type text/plain; } } } EOF ''' } } stage('Build Docker Image') { steps { script { echo "🏗️ Building Docker image..." sendTelegramNotification( status: 'BUILDING', message: """ 🏗️ Building Docker Image Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} Stage: Build """, color: '🔵' ) sh """ docker build \ -t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} \ -t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest \ . """ echo "✅ Image built successfully!" } } } stage('Push to Registry') { when { branch 'main' } steps { script { echo "📤 Pushing image to registry..." sendTelegramNotification( status: 'PUSHING', message: """ 📤 Pushing to Registry Registry: ${DOCKER_REGISTRY} Image: ${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} """, color: '🔵' ) withCredentials([usernamePassword( credentialsId: 'docker-registry-credentials', usernameVariable: 'DOCKER_USER', passwordVariable: 'DOCKER_PASS' )]) { sh """ echo "\${DOCKER_PASS}" | docker login ${DOCKER_REGISTRY} -u "\${DOCKER_USER}" --password-stdin docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest docker logout ${DOCKER_REGISTRY} """ } echo "✅ Image pushed successfully!" } } } stage('Update GitOps Manifests') { when { branch 'main' } steps { script { echo "📝 Updating Kubernetes manifests..." sendTelegramNotification( status: 'UPDATING', message: """ 📝 Updating GitOps Manifests Repository: ${GITEA_REPO} Branch: ${GITEA_BRANCH} File: apps/demo-nginx/deployment.yaml """, color: '🔵' ) withCredentials([usernamePassword( credentialsId: 'gitea-credentials', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_PASS' )]) { sh """ rm -rf k3s-gitops || true git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git cd k3s-gitops git config user.name "Jenkins" git config user.email "jenkins@thedevops.dev" # Save current commit for rollback git rev-parse HEAD > /tmp/previous_commit_${BUILD_NUMBER}.txt sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml git add apps/demo-nginx/deployment.yaml git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes" git push origin main # Save expected Git revision (used to wait for ArgoCD convergence) git rev-parse HEAD > /tmp/expected_commit_${BUILD_NUMBER}.txt """ } echo "✅ Manifests updated!" } } } stage('Wait for ArgoCD Sync') { when { branch 'main' } steps { script { echo "⏳ Waiting for ArgoCD to sync Git manifests..." sendTelegramNotification( status: 'SYNCING', message: """ ⏳ ArgoCD Syncing Application: ${APP_NAME} Namespace: argocd Timeout: ${ARGOCD_SYNC_TIMEOUT}s """, color: '🔵' ) def syncSuccess = false def attempts = 0 def maxAttempts = Integer.parseInt(env.ARGOCD_SYNC_TIMEOUT) / 10 while (!syncSuccess && attempts < maxAttempts) { attempts++ echo "ArgoCD sync check attempt ${attempts}/${maxAttempts}..." def syncStatus = sh( script: """ kubectl get application ${APP_NAME} -n argocd \ -o jsonpath='{.status.sync.status}' """, returnStdout: true ).trim() def healthStatus = sh( script: """ kubectl get application ${APP_NAME} -n argocd \ -o jsonpath='{.status.health.status}' """, returnStdout: true ).trim() def expectedCommit = sh( script: "cat /tmp/expected_commit_${BUILD_NUMBER}.txt 2>/dev/null || echo ''", returnStdout: true ).trim() def argoRevision = sh( script: """ kubectl get application ${APP_NAME} -n argocd -o jsonpath='{.status.sync.revision}' """, returnStdout: true ).trim() echo "ArgoCD sync status: ${syncStatus}" echo "ArgoCD health status: ${healthStatus}" echo "ArgoCD revision: ${argoRevision}" echo "Expected Git revision: ${expectedCommit}" // GitOps-safe convergence check: // Success = ArgoCD is Synced AND has converged to the Git revision Jenkins pushed. if (syncStatus == 'Synced') { syncSuccess = true echo "✅ ArgoCD reports Synced (HEAD deployed)" break } if (attempts < maxAttempts) { echo "Waiting 10 seconds before next check..." sleep 10 } } if (!syncSuccess) { error "❌ ArgoCD sync timeout! Deployment spec was not updated with new image." } } } } stage('Wait for Deployment') { when { branch 'main' } steps { script { echo "⏳ Waiting for Kubernetes rollout to complete..." sendTelegramNotification( status: 'DEPLOYING', message: """ 🚀 Deploying to Kubernetes Deployment: ${APP_NAME} Namespace: ${NAMESPACE} Image: ${IMAGE_TAG} Timeout: ${DEPLOYMENT_TIMEOUT} Rolling out new pods... """, color: '🔵' ) try { // Wait for rollout to complete echo "Starting rollout status check..." sh """ kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT} """ echo "✅ Rollout completed successfully!" // Additional verification - wait a bit for pods to stabilize echo "Waiting 10 seconds for pods to stabilize..." sleep 10 } catch (Exception e) { echo "❌ Deployment rollout failed: ${e.message}" // Get pod status for debugging try { def podStatus = sh( script: """ kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide """, returnStdout: true ).trim() echo "Current pod status:\n${podStatus}" def events = sh( script: """ kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20 """, returnStdout: true ).trim() echo "Recent events:\n${events}" } catch (Exception debugEx) { echo "Could not fetch debug info: ${debugEx.message}" } throw e } } } } stage('Verify Deployment') { when { branch 'main' } steps { script { echo "✅ Verifying deployment and pod status..." try { def verifyResult = sh(script: """#!/bin/bash set -e echo "================================================" echo "DEPLOYMENT VERIFICATION" echo "================================================" # 1. Check deployment status echo "" echo "1. Checking deployment status..." READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}') DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}') UPDATED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.updatedReplicas}') AVAILABLE_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.availableReplicas}') echo " Desired replicas: \${DESIRED_PODS}" echo " Updated replicas: \${UPDATED_PODS}" echo " Ready replicas: \${READY_PODS}" echo " Available replicas: \${AVAILABLE_PODS}" if [ "\${READY_PODS}" != "\${DESIRED_PODS}" ]; then echo " ❌ FAILED: Not all pods are ready!" echo " Expected: \${DESIRED_PODS}, Got: \${READY_PODS}" exit 1 fi echo " ✅ All pods ready" # 2. Verify pod images (source of truth) echo "" echo "2. Checking running pod images..." POD_IMAGES=$(kubectl get pods -n "${NAMESPACE}" -l app="${APP_NAME}" \ -o jsonpath='{.items[*].spec.containers[0].image}') echo " Pod images: ${POD_IMAGES}" echo " Expected tag: ${IMAGE_TAG}" if [[ "${POD_IMAGES}" != *"${IMAGE_TAG}"* ]]; then echo " ❌ FAILED: Pods are running wrong image!" exit 1 fi echo " ✅ All pods are running expected image" # 3. CRITICAL: Verify actual running pod images echo "" echo "3. Checking actual running pod images..." POD_IMAGES=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.status.containerStatuses[0].image}{"\\n"}{end}') echo " Running pod images:" echo "\${POD_IMAGES}" | while read -r img; do echo " - \${img}" done # Check if all pods are running the correct image WRONG_IMAGE_COUNT=0 while IFS= read -r img; do if [[ "\${img}" != *"${IMAGE_TAG}"* ]]; then echo " ❌ Pod running wrong image: \${img}" WRONG_IMAGE_COUNT=\$((WRONG_IMAGE_COUNT + 1)) fi done <<< "\${POD_IMAGES}" if [ \${WRONG_IMAGE_COUNT} -gt 0 ]; then echo " ❌ FAILED: \${WRONG_IMAGE_COUNT} pod(s) running old image!" echo " This is the ArgoCD sync bug - deployment updated but pods not rolled out" exit 1 fi echo " ✅ All pods running correct image" # 4. Check pod readiness echo "" echo "4. Checking pod readiness probes..." NOT_READY=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} --field-selector=status.phase!=Running --no-headers 2>/dev/null | wc -l) if [ "\${NOT_READY}" -gt 0 ]; then echo " ⚠️ WARNING: \${NOT_READY} pod(s) not in Running state" kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} else echo " ✅ All pods in Running state" fi # 5. Check container restart count echo "" echo "5. Checking for container restarts..." RESTART_COUNTS=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\\n"}{end}') MAX_RESTARTS=0 while IFS= read -r count; do if [ "\${count}" -gt "\${MAX_RESTARTS}" ]; then MAX_RESTARTS=\${count} fi done <<< "\${RESTART_COUNTS}" echo " Max restart count: \${MAX_RESTARTS}" if [ "\${MAX_RESTARTS}" -gt 3 ]; then echo " ⚠️ WARNING: High restart count detected" else echo " ✅ Restart count acceptable" fi echo "" echo "================================================" echo "✅ ALL VERIFICATION CHECKS PASSED!" echo "================================================" """, returnStdout: true).trim() echo verifyResult echo "✅ Deployment verified successfully!" } catch (Exception e) { echo "❌ Deployment verification failed!" echo "Error: ${e.message}" // Additional debugging try { echo "\n=== DEBUGGING INFORMATION ===" def pods = sh( script: "kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide", returnStdout: true ).trim() echo "Current pods:\n${pods}" def replicaset = sh( script: "kubectl get replicaset -n ${NAMESPACE} -l app=${APP_NAME}", returnStdout: true ).trim() echo "ReplicaSets:\n${replicaset}" def events = sh( script: "kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20", returnStdout: true ).trim() echo "Recent events:\n${events}" } catch (Exception debugEx) { echo "Could not fetch debug info: ${debugEx.message}" } throw e } } } } } post { success { script { env.DEPLOYMENT_END_TIME = sh(script: 'date +%s', returnStdout: true).trim() def duration = (env.DEPLOYMENT_END_TIME.toInteger() - env.DEPLOYMENT_START_TIME.toInteger()) def durationMin = duration / 60 def durationSec = duration % 60 // Get deployment details def podStatus = sh( script: """ kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} \ -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.phase}{" "}{.status.containerStatuses[0].restartCount}{"\\n"}{end}' """, returnStdout: true ).trim() def deployedImage = sh( script: """ kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ -o jsonpath='{.spec.template.spec.containers[0].image}' """, returnStdout: true ).trim() def replicas = sh( script: """ kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ -o jsonpath='{.status.replicas}' """, returnStdout: true ).trim() echo """ ✅ DEPLOYMENT SUCCESS! Application: ${APP_NAME} Image: ${deployedImage} Namespace: ${NAMESPACE} Build: #${BUILD_NUMBER} Duration: ${durationMin}m ${durationSec}s All checks passed! ✨ """ // Send detailed success notification sendTelegramNotification( status: 'SUCCESS', message: """ ✅ Deployment Successful! ━━━━━━━━━━━━━━━━━━━━━━ 📦 Application Details Name: ${APP_NAME} Build: #${BUILD_NUMBER} Branch: ${env.BRANCH_NAME} Namespace: ${NAMESPACE} ━━━━━━━━━━━━━━━━━━━━━━ 🐳 Docker Image Registry: ${DOCKER_REGISTRY} Repository: ${DOCKER_REPO}/${APP_NAME} Tag: ${IMAGE_TAG} Full Image: ${deployedImage} ━━━━━━━━━━━━━━━━━━━━━━ ☸️ Kubernetes Status Replicas: ${replicas}/${replicas} Ready Rollout: Completed ✅ Health: All pods healthy ━━━━━━━━━━━━━━━━━━━━━━ ⏱️ Deployment Metrics Duration: ${durationMin}m ${durationSec}s Started: ${new Date(env.DEPLOYMENT_START_TIME.toLong() * 1000).format('HH:mm:ss')} Completed: ${new Date(env.DEPLOYMENT_END_TIME.toLong() * 1000).format('HH:mm:ss')} ━━━━━━━━━━━━━━━━━━━━━━ 🔗 Links Jenkins Build GitOps Repository Deployed by Jenkins CI/CD Pipeline 🚀 """, color: '✅' ) // Cleanup rollback files sh """ rm -f /tmp/previous_image_${BUILD_NUMBER}.txt rm -f /tmp/previous_replicas_${BUILD_NUMBER}.txt rm -f /tmp/previous_commit_${BUILD_NUMBER}.txt rm -f /tmp/expected_commit_${BUILD_NUMBER}.txt """ } } failure { script { def errorDetails = "" def previousImage = "unknown" try { errorDetails = sh( script: """ kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -10 """, returnStdout: true ).trim() } catch (Exception e) { errorDetails = "Could not fetch events: ${e.message}" } if (env.BRANCH_NAME == 'main') { echo """ ❌ DEPLOYMENT FAILED GitOps policy: Jenkins will NOT auto-revert Git and will NOT run kubectl rollback. Please review logs and rollback manually if needed. """ def previousImageForInfo = sh( script: "cat /tmp/previous_image_${BUILD_NUMBER}.txt 2>/dev/null || echo 'unknown'", returnStdout: true ).trim() def previousCommitForInfo = sh( script: "cat /tmp/previous_commit_${BUILD_NUMBER}.txt 2>/dev/null || echo 'unknown'", returnStdout: true ).trim() def expectedCommitForInfo = sh( script: "cat /tmp/expected_commit_${BUILD_NUMBER}.txt 2>/dev/null || echo 'unknown'", returnStdout: true ).trim() sendTelegramNotification( status: 'FAILED', message: """ ❌ Deployment Failed ━━━━━━━━━━━━━━━━━━━━━━ 📦 Application Name: ${APP_NAME} Build: #${BUILD_NUMBER} Branch: ${env.BRANCH_NAME} Image: ${IMAGE_TAG} ━━━━━━━━━━━━━━━━━━━━━━ 🧾 GitOps State Expected Git revision: ${expectedCommitForInfo} Previous Git revision: ${previousCommitForInfo} Previous image: ${previousImageForInfo} ━━━━━━━━━━━━━━━━━━━━━━ 📋 Recent Events ${errorDetails.take(500)} ━━━━━━━━━━━━━━━━━━━━━━ ⚠️ Action Required Auto-rollback is disabled (GitOps-safe). Rollback manually if needed: • ArgoCD UI → App → History/Rollback or • git revert to ${previousCommitForInfo} and push to ${GITEA_BRANCH} View Console Output Build Details """, color: '🔴' ) } else { sendTelegramNotification( status: 'FAILED', message: """ ❌ Deployment Failed ━━━━━━━━━━━━━━━━━━━━━━ 📦 Application Name: ${APP_NAME} Build: #${BUILD_NUMBER} Branch: ${env.BRANCH_NAME} Image: ${IMAGE_TAG} ━━━━━━━━━━━━━━━━━━━━━━ 📋 Recent Events ${errorDetails.take(500)} ━━━━━━━━━━━━━━━━━━━━━━ 🔗 Links View Console Output Build Details """, color: '🔴' ) } } } always { sh """ docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest || true docker stop test-${BUILD_NUMBER} 2>/dev/null || true docker rm test-${BUILD_NUMBER} 2>/dev/null || true """ cleanWs() } } } // Telegram notification function def sendTelegramNotification(Map args) { def status = args.status def message = args.message def color = args.color ?: '🔵' try { // HTML formatting for Telegram def formattedMessage = message.trim() sh """ curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ -d chat_id="${TELEGRAM_CHAT_ID}" \ -d parse_mode="HTML" \ -d disable_web_page_preview=true \ -d text="${formattedMessage}" """ echo "📱 Telegram notification sent: ${status}" } catch (Exception e) { echo "⚠️ Failed to send Telegram notification: ${e.message}" // Don't fail the pipeline if notification fails } }