diff --git a/apps/demo-nginx/Jenkinsfile b/apps/demo-nginx/Jenkinsfile new file mode 100644 index 0000000..88512aa --- /dev/null +++ b/apps/demo-nginx/Jenkinsfile @@ -0,0 +1,817 @@ +pipeline { + agent any + + environment { + APP_NAME = 'demo-nginx' + NAMESPACE = 'demo-app' + DOCKER_REGISTRY = 'docker.io' + DOCKER_REPO = 'vladcrypto' + GITEA_URL = 'http://gitea-http.gitea.svc.cluster.local:3000' + GITEA_REPO = 'admin/k3s-gitops' + GITEA_BRANCH = 'main' + BUILD_TAG = "${env.BUILD_NUMBER}" + IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}" + + // Rollback configuration + ROLLBACK_ENABLED = 'true' + DEPLOYMENT_TIMEOUT = '300s' + ARGOCD_SYNC_TIMEOUT = '120' + SKIP_HEALTH_CHECK = 'true' + + // Notification configuration + TELEGRAM_BOT_TOKEN = credentials('telegram-bot-token') + TELEGRAM_CHAT_ID = credentials('telegram-chat-id') + + // Build info + BUILD_URL = "${env.BUILD_URL}" + GIT_COMMIT_SHORT = "" + DEPLOYMENT_START_TIME = "" + DEPLOYMENT_END_TIME = "" + } + + stages { + stage('Initialization') { + steps { + script { + echo "🚀 Starting deployment pipeline..." + env.DEPLOYMENT_START_TIME = sh(script: 'date +%s', returnStdout: true).trim() + + sendTelegramNotification( + status: 'STARTED', + message: """ +🚀 Deployment Started + +Application: ${APP_NAME} +Build: #${BUILD_NUMBER} +Branch: ${env.BRANCH_NAME} +Namespace: ${NAMESPACE} +Started by: ${env.BUILD_USER ?: 'Jenkins'} + +Building and deploying... + """, + color: '🔵' + ) + } + } + } + + stage('Save Current State') { + when { branch 'main' } + steps { + script { + echo "📸 Saving current deployment state for rollback..." + + sh """ + kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ + -o jsonpath='{.spec.template.spec.containers[0].image}' \ + > /tmp/previous_image_${BUILD_NUMBER}.txt || echo "none" > /tmp/previous_image_${BUILD_NUMBER}.txt + + PREV_IMAGE=\$(cat /tmp/previous_image_${BUILD_NUMBER}.txt) + echo "Previous image: \${PREV_IMAGE}" + """ + + sh """ + kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ + -o jsonpath='{.spec.replicas}' \ + > /tmp/previous_replicas_${BUILD_NUMBER}.txt || echo "2" > /tmp/previous_replicas_${BUILD_NUMBER}.txt + """ + + echo "✅ Current state saved" + } + } + } + + stage('Checkout Source') { + steps { + echo "📦 Creating application artifacts..." + + sh """ + cat > Dockerfile << 'EOF' +FROM nginx:1.25.3-alpine +COPY index.html /usr/share/nginx/html/index.html +COPY nginx.conf /etc/nginx/nginx.conf +EXPOSE 80 +CMD ["nginx", "-g", "daemon off;"] +EOF + """ + + sh """ + cat > index.html << EOF + + + + Demo Nginx + + + +
+

Demo Nginx - Build #${BUILD_NUMBER}

+

Environment: Production

+

Version: ${IMAGE_TAG}

+

+ Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} +

+
+ + +EOF + """ + + sh ''' + cat > nginx.conf << 'EOF' +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; +events { worker_connections 1024; } +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + server { + listen 80; + server_name _; + location / { + root /usr/share/nginx/html; + index index.html; + } + location /health { + access_log off; + return 200 "healthy\n"; + add_header Content-Type text/plain; + } + } +} +EOF + ''' + } + } + + stage('Build Docker Image') { + steps { + script { + echo "🏗️ Building Docker image..." + + sendTelegramNotification( + status: 'BUILDING', + message: """ +🏗️ Building Docker Image + +Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} +Stage: Build + """, + color: '🔵' + ) + + sh """ + docker build \ + -t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} \ + -t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest \ + . + """ + echo "✅ Image built successfully!" + } + } + } + + stage('Push to Registry') { + when { branch 'main' } + steps { + script { + echo "📤 Pushing image to registry..." + + sendTelegramNotification( + status: 'PUSHING', + message: """ +📤 Pushing to Registry + +Registry: ${DOCKER_REGISTRY} +Image: ${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} + """, + color: '🔵' + ) + + withCredentials([usernamePassword( + credentialsId: 'docker-registry-credentials', + usernameVariable: 'DOCKER_USER', + passwordVariable: 'DOCKER_PASS' + )]) { + sh """ + echo "\${DOCKER_PASS}" | docker login ${DOCKER_REGISTRY} -u "\${DOCKER_USER}" --password-stdin + docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} + docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest + docker logout ${DOCKER_REGISTRY} + """ + } + echo "✅ Image pushed successfully!" + } + } + } + + stage('Update GitOps Manifests') { + when { branch 'main' } + steps { + script { + echo "📝 Updating Kubernetes manifests..." + + sendTelegramNotification( + status: 'UPDATING', + message: """ +📝 Updating GitOps Manifests + +Repository: ${GITEA_REPO} +Branch: ${GITEA_BRANCH} +File: apps/demo-nginx/deployment.yaml + """, + color: '🔵' + ) + + withCredentials([usernamePassword( + credentialsId: 'gitea-credentials', + usernameVariable: 'GIT_USER', + passwordVariable: 'GIT_PASS' + )]) { + sh """ + rm -rf k3s-gitops || true + git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git + cd k3s-gitops + git config user.name "Jenkins" + git config user.email "jenkins@thedevops.dev" + + # Save current commit for rollback + git rev-parse HEAD > /tmp/previous_commit_${BUILD_NUMBER}.txt + + sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml + git add apps/demo-nginx/deployment.yaml + git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes" + git push origin main + + + # Save expected Git revision (used to wait for ArgoCD convergence) + git rev-parse HEAD > /tmp/expected_commit_${BUILD_NUMBER}.txt + """ + } + echo "✅ Manifests updated!" + } + } + } + + stage('Wait for ArgoCD Sync') { + when { branch 'main' } + steps { + script { + echo "⏳ Waiting for ArgoCD to sync Git manifests..." + + sendTelegramNotification( + status: 'SYNCING', + message: """ +⏳ ArgoCD Syncing + +Application: ${APP_NAME} +Namespace: argocd +Timeout: ${ARGOCD_SYNC_TIMEOUT}s + """, + color: '🔵' + ) + + def syncSuccess = false + def attempts = 0 + def maxAttempts = Integer.parseInt(env.ARGOCD_SYNC_TIMEOUT) / 10 + + while (!syncSuccess && attempts < maxAttempts) { + attempts++ + echo "ArgoCD sync check attempt ${attempts}/${maxAttempts}..." + + def syncStatus = sh( + script: """ + kubectl get application ${APP_NAME} -n argocd \ + -o jsonpath='{.status.sync.status}' + """, + returnStdout: true + ).trim() + + def healthStatus = sh( + script: """ + kubectl get application ${APP_NAME} -n argocd \ + -o jsonpath='{.status.health.status}' + """, + returnStdout: true + ).trim() + + def expectedCommit = sh( + script: "cat /tmp/expected_commit_${BUILD_NUMBER}.txt 2>/dev/null || echo ''", + returnStdout: true + ).trim() + + def argoRevision = sh( + script: """ + kubectl get application ${APP_NAME} -n argocd -o jsonpath='{.status.sync.revision}' + """, + returnStdout: true + ).trim() + + echo "ArgoCD sync status: ${syncStatus}" + echo "ArgoCD health status: ${healthStatus}" + echo "ArgoCD revision: ${argoRevision}" + echo "Expected Git revision: ${expectedCommit}" + + // GitOps-safe convergence check: + // Success = ArgoCD is Synced AND has converged to the Git revision Jenkins pushed. + + if (syncStatus == 'Synced') { + syncSuccess = true + echo "✅ ArgoCD reports Synced (HEAD deployed)" + break + } + + if (attempts < maxAttempts) { + echo "Waiting 10 seconds before next check..." + sleep 10 + } + } + + if (!syncSuccess) { + error "❌ ArgoCD sync timeout! Deployment spec was not updated with new image." + } + } + } + } + + stage('Wait for Deployment') { + when { branch 'main' } + steps { + script { + echo "⏳ Waiting for Kubernetes rollout to complete..." + + sendTelegramNotification( + status: 'DEPLOYING', + message: """ +🚀 Deploying to Kubernetes + +Deployment: ${APP_NAME} +Namespace: ${NAMESPACE} +Image: ${IMAGE_TAG} +Timeout: ${DEPLOYMENT_TIMEOUT} + +Rolling out new pods... + """, + color: '🔵' + ) + + try { + // Wait for rollout to complete + echo "Starting rollout status check..." + sh """ + kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT} + """ + echo "✅ Rollout completed successfully!" + + // Additional verification - wait a bit for pods to stabilize + echo "Waiting 10 seconds for pods to stabilize..." + sleep 10 + + } catch (Exception e) { + echo "❌ Deployment rollout failed: ${e.message}" + + // Get pod status for debugging + try { + def podStatus = sh( + script: """ + kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide + """, + returnStdout: true + ).trim() + echo "Current pod status:\n${podStatus}" + + def events = sh( + script: """ + kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20 + """, + returnStdout: true + ).trim() + echo "Recent events:\n${events}" + } catch (Exception debugEx) { + echo "Could not fetch debug info: ${debugEx.message}" + } + + throw e + } + } + } + } + + stage('Verify Deployment') { + when { branch 'main' } + steps { + script { + echo "✅ Verifying deployment and pod status..." + + sh '''#!/bin/bash + set -e + + echo "================================================" + echo "DEPLOYMENT VERIFICATION" + echo "================================================" + + # 1. Check deployment status + echo "" + echo "1. Checking deployment status..." + + READY_PODS=$(kubectl get deployment "${APP_NAME}" -n "${NAMESPACE}" -o jsonpath='{.status.readyReplicas}') + DESIRED_PODS=$(kubectl get deployment "${APP_NAME}" -n "${NAMESPACE}" -o jsonpath='{.spec.replicas}') + UPDATED_PODS=$(kubectl get deployment "${APP_NAME}" -n "${NAMESPACE}" -o jsonpath='{.status.updatedReplicas}') + AVAILABLE_PODS=$(kubectl get deployment "${APP_NAME}" -n "${NAMESPACE}" -o jsonpath='{.status.availableReplicas}') + + echo " Desired replicas: ${DESIRED_PODS}" + echo " Updated replicas: ${UPDATED_PODS}" + echo " Ready replicas: ${READY_PODS}" + echo " Available replicas: ${AVAILABLE_PODS}" + + if [ "${READY_PODS}" != "${DESIRED_PODS}" ]; then + echo " ❌ FAILED: Not all pods are ready!" + echo " Expected: ${DESIRED_PODS}, Got: ${READY_PODS}" + exit 1 + fi + echo " ✅ All pods ready" + + # 2. Verify pod images (source of truth) + echo "" + echo "2. Checking running pod images..." + + POD_IMAGES=$(kubectl get pods -n "${NAMESPACE}" -l app="${APP_NAME}" -o jsonpath='{.items[*].spec.containers[0].image}') + + + echo " Pod images: ${POD_IMAGES}" + echo " Expected tag: ${IMAGE_TAG}" + + if [[ "${POD_IMAGES}" != *"${IMAGE_TAG}"* ]]; then + echo " ❌ FAILED: Pods are running wrong image!" + exit 1 + fi + echo " ✅ All pods are running expected image" + + # 3. CRITICAL: Verify actual running pod images + echo "" + echo "3. Checking actual running pod images..." + + POD_IMAGES=$(kubectl get pods -n "${NAMESPACE}" -l app="${APP_NAME}" \ + -o jsonpath='{range .items[*]}{.status.containerStatuses[0].image}{"\n"}{end}') + + echo " Running pod images:" + while read -r img; do + echo " - ${img}" + if [[ "${img}" != *"${IMAGE_TAG}"* ]]; then + echo " ❌ Pod running wrong image: ${img}" + exit 1 + fi + done <<< "${POD_IMAGES}" + + echo " ✅ All pods running correct image" + + # 4. Check pod readiness + echo "" + echo "4. Checking pod readiness..." + + NOT_READY=$(kubectl get pods -n "${NAMESPACE}" -l app="${APP_NAME}" \ + --field-selector=status.phase!=Running --no-headers 2>/dev/null | wc -l) + + if [ "${NOT_READY}" -gt 0 ]; then + echo " ⚠️ WARNING: ${NOT_READY} pod(s) not in Running state" + kubectl get pods -n "${NAMESPACE}" -l app="${APP_NAME}" + else + echo " ✅ All pods in Running state" + fi + + # 5. Check container restart count + echo "" + echo "5. Checking for container restarts..." + + RESTART_COUNTS=$(kubectl get pods -n "${NAMESPACE}" -l app="${APP_NAME}" \ + -o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\n"}{end}') + + MAX_RESTARTS=0 + while read -r count; do + if [ "${count}" -gt "${MAX_RESTARTS}" ]; then + MAX_RESTARTS="${count}" + fi + done <<< "${RESTART_COUNTS}" + + echo " Max restart count: ${MAX_RESTARTS}" + + if [ "${MAX_RESTARTS}" -gt 3 ]; then + echo " ⚠️ WARNING: High restart count detected" + else + echo " ✅ Restart count acceptable" + fi + + echo "" + echo "================================================" + echo "✅ ALL VERIFICATION CHECKS PASSED!" + echo "================================================" + ''' +} + + """, returnStdout: true).trim() + + echo verifyResult + echo "✅ Deployment verified successfully!" + + } catch (Exception e) { + echo "❌ Deployment verification failed!" + echo "Error: ${e.message}" + + // Additional debugging + try { + echo "\n=== DEBUGGING INFORMATION ===" + + def pods = sh( + script: "kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide", + returnStdout: true + ).trim() + echo "Current pods:\n${pods}" + + def replicaset = sh( + script: "kubectl get replicaset -n ${NAMESPACE} -l app=${APP_NAME}", + returnStdout: true + ).trim() + echo "ReplicaSets:\n${replicaset}" + + def events = sh( + script: "kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20", + returnStdout: true + ).trim() + echo "Recent events:\n${events}" + + } catch (Exception debugEx) { + echo "Could not fetch debug info: ${debugEx.message}" + } + + throw e + } + } + } + } + } + + post { + success { + script { + env.DEPLOYMENT_END_TIME = sh(script: 'date +%s', returnStdout: true).trim() + def duration = (env.DEPLOYMENT_END_TIME.toInteger() - env.DEPLOYMENT_START_TIME.toInteger()) + def durationMin = duration / 60 + def durationSec = duration % 60 + + // Get deployment details + def podStatus = sh( + script: "kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.phase}{" "}{.status.containerStatuses[0].image}{" restarts="}{.status.containerStatuses[0].restartCount}{"\\n"}{end}'", + returnStdout: true + ).trim() + + def deployedImage = sh( + script: "kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}'", + returnStdout: true + ).trim() + + def replicas = sh( + script: "kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.replicas}'", + returnStdout: true + ).trim() + + echo "✅ DEPLOYMENT SUCCESS!" + echo "" + echo "Application: ${APP_NAME}" + echo "Image: ${deployedImage}" + echo "Namespace: ${NAMESPACE}" + echo "Build: #${BUILD_NUMBER}" + echo "Duration: ${durationMin}m ${durationSec}s" + echo "" + echo "All checks passed! ✨" + + // Send detailed success notification + sendTelegramNotification( + status: 'SUCCESS', + message: """ + // Deployment Successful message (moved inside sh block) + sh 'echo "Deployment Successful!"' + +━━━━━━━━━━━━━━━━━━━━━━ +📦 Application Details +Name: ${APP_NAME} +Build: #${BUILD_NUMBER} +Branch: ${env.BRANCH_NAME} +Namespace: ${NAMESPACE} + +━━━━━━━━━━━━━━━━━━━━━━ +🐳 Docker Image +Registry: ${DOCKER_REGISTRY} +Repository: ${DOCKER_REPO}/${APP_NAME} +Tag: ${IMAGE_TAG} +Full Image: ${deployedImage} + +━━━━━━━━━━━━━━━━━━━━━━ +☸️ Kubernetes Status +Replicas: ${replicas}/${replicas} Ready +Rollout: Completed ✅ +Health: All pods healthy + +━━━━━━━━━━━━━━━━━━━━━━ +⏱️ Deployment Metrics +Duration: ${durationMin}m ${durationSec}s +Started: ${new Date(env.DEPLOYMENT_START_TIME.toLong() * 1000).format('HH:mm:ss')} +Completed: ${new Date(env.DEPLOYMENT_END_TIME.toLong() * 1000).format('HH:mm:ss')} + +━━━━━━━━━━━━━━━━━━━━━━ +🔗 Links +Jenkins Build +GitOps Repository + +Deployed by Jenkins CI/CD Pipeline 🚀 + """, + color: '✅' + ) + + // Cleanup rollback files + sh """ + rm -f /tmp/previous_image_${BUILD_NUMBER}.txt + rm -f /tmp/previous_replicas_${BUILD_NUMBER}.txt + rm -f /tmp/previous_commit_${BUILD_NUMBER}.txt + rm -f /tmp/expected_commit_${BUILD_NUMBER}.txt + """ + } + } + + failure { + script { + def errorDetails = "" + def previousImage = "unknown" + + try { + errorDetails = sh( + script: """ + kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -10 + """, + returnStdout: true + ).trim() + } catch (Exception e) { + errorDetails = "Could not fetch events: ${e.message}" + } + + if (env.BRANCH_NAME == 'main') { + echo """ + ❌ DEPLOYMENT FAILED + + GitOps policy: Jenkins will NOT auto-revert Git and will NOT run kubectl rollback. + Please review logs and rollback manually if needed. + """ + + def previousImageForInfo = sh( + script: "cat /tmp/previous_image_${BUILD_NUMBER}.txt 2>/dev/null || echo 'unknown'", + returnStdout: true + ).trim() + + def previousCommitForInfo = sh( + script: "cat /tmp/previous_commit_${BUILD_NUMBER}.txt 2>/dev/null || echo 'unknown'", + returnStdout: true + ).trim() + + def expectedCommitForInfo = sh( + script: "cat /tmp/expected_commit_${BUILD_NUMBER}.txt 2>/dev/null || echo 'unknown'", + returnStdout: true + ).trim() + + sendTelegramNotification( + status: 'FAILED', + message: """ +❌ Deployment Failed + +━━━━━━━━━━━━━━━━━━━━━━ +📦 Application +Name: ${APP_NAME} +Build: #${BUILD_NUMBER} +Branch: ${env.BRANCH_NAME} +Image: ${IMAGE_TAG} + +━━━━━━━━━━━━━━━━━━━━━━ +🧾 GitOps State +Expected Git revision: ${expectedCommitForInfo} +Previous Git revision: ${previousCommitForInfo} +Previous image: ${previousImageForInfo} + +━━━━━━━━━━━━━━━━━━━━━━ +📋 Recent Events +${errorDetails.take(500)} + +━━━━━━━━━━━━━━━━━━━━━━ +⚠️ Action Required +Auto-rollback is disabled (GitOps-safe). +Rollback manually if needed: +• ArgoCD UI → App → History/Rollback +or +• git revert to ${previousCommitForInfo} and push to ${GITEA_BRANCH} + +View Console Output +Build Details + """, + color: '🔴' + ) + + } else { + sendTelegramNotification( + status: 'FAILED', + message: """ +❌ Deployment Failed + +━━━━━━━━━━━━━━━━━━━━━━ +📦 Application +Name: ${APP_NAME} +Build: #${BUILD_NUMBER} +Branch: ${env.BRANCH_NAME} +Image: ${IMAGE_TAG} + +━━━━━━━━━━━━━━━━━━━━━━ +📋 Recent Events +${errorDetails.take(500)} + +━━━━━━━━━━━━━━━━━━━━━━ +🔗 Links +View Console Output +Build Details + """, + color: '🔴' + ) + } + } + } + + always { + sh """ + docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true + docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest || true + docker stop test-${BUILD_NUMBER} 2>/dev/null || true + docker rm test-${BUILD_NUMBER} 2>/dev/null || true + """ + cleanWs() + } + } +} + +// Telegram notification function +def sendTelegramNotification(Map args) { + def status = args.status + def message = args.message + def color = args.color ?: '🔵' + + try { + // HTML formatting for Telegram + def formattedMessage = message.trim() + + sh """ + curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ + -d chat_id="${TELEGRAM_CHAT_ID}" \ + -d parse_mode="HTML" \ + -d disable_web_page_preview=true \ + -d text="${formattedMessage}" + """ + + echo "📱 Telegram notification sent: ${status}" + } catch (Exception e) { + echo "⚠️ Failed to send Telegram notification: ${e.message}" + // Don't fail the pipeline if notification fails + } +} \ No newline at end of file