diff --git a/apps/demo-nginx/Jenkinsfile b/apps/demo-nginx/Jenkinsfile deleted file mode 100644 index 5814e4d..0000000 --- a/apps/demo-nginx/Jenkinsfile +++ /dev/null @@ -1,818 +0,0 @@ -pipeline { - agent any - - environment { - APP_NAME = 'demo-nginx' - NAMESPACE = 'demo-app' - DOCKER_REGISTRY = 'docker.io' - DOCKER_REPO = 'vladcrypto' - GITEA_URL = 'http://gitea-http.gitea.svc.cluster.local:3000' - GITEA_REPO = 'admin/k3s-gitops' - GITEA_BRANCH = 'main' - BUILD_TAG = "${env.BUILD_NUMBER}" - IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}" - - // Rollback configuration - ROLLBACK_ENABLED = 'true' - DEPLOYMENT_TIMEOUT = '300s' - ARGOCD_SYNC_TIMEOUT = '120' - SKIP_HEALTH_CHECK = 'true' - - // Notification configuration - TELEGRAM_BOT_TOKEN = credentials('telegram-bot-token') - TELEGRAM_CHAT_ID = credentials('telegram-chat-id') - - // Build info - BUILD_URL = "${env.BUILD_URL}" - GIT_COMMIT_SHORT = "" - DEPLOYMENT_START_TIME = "" - DEPLOYMENT_END_TIME = "" - } - - stages { - stage('Initialization') { - steps { - script { - echo "🚀 Starting deployment pipeline..." - env.DEPLOYMENT_START_TIME = sh(script: 'date +%s', returnStdout: true).trim() - - sendTelegramNotification( - status: 'STARTED', - message: """ -🚀 Deployment Started - -Application: ${APP_NAME} -Build: #${BUILD_NUMBER} -Branch: ${env.BRANCH_NAME} -Namespace: ${NAMESPACE} -Started by: ${env.BUILD_USER ?: 'Jenkins'} - -Building and deploying... - """, - color: '🔵' - ) - } - } - } - - stage('Save Current State') { - when { branch 'main' } - steps { - script { - echo "📸 Saving current deployment state for rollback..." - - sh """ - kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ - -o jsonpath='{.spec.template.spec.containers[0].image}' \ - > /tmp/previous_image_${BUILD_NUMBER}.txt || echo "none" > /tmp/previous_image_${BUILD_NUMBER}.txt - - PREV_IMAGE=\$(cat /tmp/previous_image_${BUILD_NUMBER}.txt) - echo "Previous image: \${PREV_IMAGE}" - """ - - sh """ - kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ - -o jsonpath='{.spec.replicas}' \ - > /tmp/previous_replicas_${BUILD_NUMBER}.txt || echo "2" > /tmp/previous_replicas_${BUILD_NUMBER}.txt - """ - - echo "✅ Current state saved" - } - } - } - - stage('Checkout Source') { - steps { - echo "📦 Creating application artifacts..." - - sh """ - cat > Dockerfile << 'EOF' -FROM nginx:1.25.3-alpine -COPY index.html /usr/share/nginx/html/index.html -COPY nginx.conf /etc/nginx/nginx.conf -EXPOSE 80 -CMD ["nginx", "-g", "daemon off;"] -EOF - """ - - sh """ - cat > index.html << EOF - - - - Demo Nginx - - - -
-

Demo Nginx - Build #${BUILD_NUMBER}

-

Environment: Production

-

Version: ${IMAGE_TAG}

-

- Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} -

-
- - -EOF - """ - - sh ''' - cat > nginx.conf << 'EOF' -user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; -events { worker_connections 1024; } -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; - sendfile on; - keepalive_timeout 65; - server { - listen 80; - server_name _; - location / { - root /usr/share/nginx/html; - index index.html; - } - location /health { - access_log off; - return 200 "healthy\n"; - add_header Content-Type text/plain; - } - } -} -EOF - ''' - } - } - - stage('Build Docker Image') { - steps { - script { - echo "🏗️ Building Docker image..." - - sendTelegramNotification( - status: 'BUILDING', - message: """ -🏗️ Building Docker Image - -Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} -Stage: Build - """, - color: '🔵' - ) - - sh """ - docker build \ - -t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} \ - -t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest \ - . - """ - echo "✅ Image built successfully!" - } - } - } - - stage('Push to Registry') { - when { branch 'main' } - steps { - script { - echo "📤 Pushing image to registry..." - - sendTelegramNotification( - status: 'PUSHING', - message: """ -📤 Pushing to Registry - -Registry: ${DOCKER_REGISTRY} -Image: ${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} - """, - color: '🔵' - ) - - withCredentials([usernamePassword( - credentialsId: 'docker-registry-credentials', - usernameVariable: 'DOCKER_USER', - passwordVariable: 'DOCKER_PASS' - )]) { - sh """ - echo "\${DOCKER_PASS}" | docker login ${DOCKER_REGISTRY} -u "\${DOCKER_USER}" --password-stdin - docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} - docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest - docker logout ${DOCKER_REGISTRY} - """ - } - echo "✅ Image pushed successfully!" - } - } - } - - stage('Update GitOps Manifests') { - when { branch 'main' } - steps { - script { - echo "📝 Updating Kubernetes manifests..." - - sendTelegramNotification( - status: 'UPDATING', - message: """ -📝 Updating GitOps Manifests - -Repository: ${GITEA_REPO} -Branch: ${GITEA_BRANCH} -File: apps/demo-nginx/deployment.yaml - """, - color: '🔵' - ) - - withCredentials([usernamePassword( - credentialsId: 'gitea-credentials', - usernameVariable: 'GIT_USER', - passwordVariable: 'GIT_PASS' - )]) { - sh """ - rm -rf k3s-gitops || true - git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git - cd k3s-gitops - git config user.name "Jenkins" - git config user.email "jenkins@thedevops.dev" - - # Save current commit for rollback - git rev-parse HEAD > /tmp/previous_commit_${BUILD_NUMBER}.txt - - sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml - git add apps/demo-nginx/deployment.yaml - git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes" - git push origin main - - - # Save expected Git revision (used to wait for ArgoCD convergence) - git rev-parse HEAD > /tmp/expected_commit_${BUILD_NUMBER}.txt - """ - } - echo "✅ Manifests updated!" - } - } - } - - stage('Wait for ArgoCD Sync') { - when { branch 'main' } - steps { - script { - echo "⏳ Waiting for ArgoCD to sync Git manifests..." - - sendTelegramNotification( - status: 'SYNCING', - message: """ -⏳ ArgoCD Syncing - -Application: ${APP_NAME} -Namespace: argocd -Timeout: ${ARGOCD_SYNC_TIMEOUT}s - """, - color: '🔵' - ) - - def syncSuccess = false - def attempts = 0 - def maxAttempts = Integer.parseInt(env.ARGOCD_SYNC_TIMEOUT) / 10 - - while (!syncSuccess && attempts < maxAttempts) { - attempts++ - echo "ArgoCD sync check attempt ${attempts}/${maxAttempts}..." - - def syncStatus = sh( - script: """ - kubectl get application ${APP_NAME} -n argocd \ - -o jsonpath='{.status.sync.status}' - """, - returnStdout: true - ).trim() - - def healthStatus = sh( - script: """ - kubectl get application ${APP_NAME} -n argocd \ - -o jsonpath='{.status.health.status}' - """, - returnStdout: true - ).trim() - - def expectedCommit = sh( - script: "cat /tmp/expected_commit_${BUILD_NUMBER}.txt 2>/dev/null || echo ''", - returnStdout: true - ).trim() - - def argoRevision = sh( - script: """ - kubectl get application ${APP_NAME} -n argocd -o jsonpath='{.status.sync.revision}' - """, - returnStdout: true - ).trim() - - echo "ArgoCD sync status: ${syncStatus}" - echo "ArgoCD health status: ${healthStatus}" - echo "ArgoCD revision: ${argoRevision}" - echo "Expected Git revision: ${expectedCommit}" - - // GitOps-safe convergence check: - // Success = ArgoCD is Synced AND has converged to the Git revision Jenkins pushed. - - if (syncStatus == 'Synced') { - syncSuccess = true - echo "✅ ArgoCD reports Synced (HEAD deployed)" - break - } - - if (attempts < maxAttempts) { - echo "Waiting 10 seconds before next check..." - sleep 10 - } - } - - if (!syncSuccess) { - error "❌ ArgoCD sync timeout! Deployment spec was not updated with new image." - } - } - } - } - - stage('Wait for Deployment') { - when { branch 'main' } - steps { - script { - echo "⏳ Waiting for Kubernetes rollout to complete..." - - sendTelegramNotification( - status: 'DEPLOYING', - message: """ -🚀 Deploying to Kubernetes - -Deployment: ${APP_NAME} -Namespace: ${NAMESPACE} -Image: ${IMAGE_TAG} -Timeout: ${DEPLOYMENT_TIMEOUT} - -Rolling out new pods... - """, - color: '🔵' - ) - - try { - // Wait for rollout to complete - echo "Starting rollout status check..." - sh """ - kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT} - """ - echo "✅ Rollout completed successfully!" - - // Additional verification - wait a bit for pods to stabilize - echo "Waiting 10 seconds for pods to stabilize..." - sleep 10 - - } catch (Exception e) { - echo "❌ Deployment rollout failed: ${e.message}" - - // Get pod status for debugging - try { - def podStatus = sh( - script: """ - kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide - """, - returnStdout: true - ).trim() - echo "Current pod status:\n${podStatus}" - - def events = sh( - script: """ - kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20 - """, - returnStdout: true - ).trim() - echo "Recent events:\n${events}" - } catch (Exception debugEx) { - echo "Could not fetch debug info: ${debugEx.message}" - } - - throw e - } - } - } - } - - stage('Verify Deployment') { - when { branch 'main' } - steps { - script { - echo "✅ Verifying deployment and pod status..." - - sh '''#!/bin/bash - set -e - - echo "================================================" - echo "DEPLOYMENT VERIFICATION" - echo "================================================" - - # 1. Check deployment status - echo "" - echo "1. Checking deployment status..." - - READY_PODS=$(kubectl get deployment "${APP_NAME}" -n "${NAMESPACE}" -o jsonpath='{.status.readyReplicas}') - DESIRED_PODS=$(kubectl get deployment "${APP_NAME}" -n "${NAMESPACE}" -o jsonpath='{.spec.replicas}') - UPDATED_PODS=$(kubectl get deployment "${APP_NAME}" -n "${NAMESPACE}" -o jsonpath='{.status.updatedReplicas}') - AVAILABLE_PODS=$(kubectl get deployment "${APP_NAME}" -n "${NAMESPACE}" -o jsonpath='{.status.availableReplicas}') - - echo " Desired replicas: ${DESIRED_PODS}" - echo " Updated replicas: ${UPDATED_PODS}" - echo " Ready replicas: ${READY_PODS}" - echo " Available replicas: ${AVAILABLE_PODS}" - - if [ "${READY_PODS}" != "${DESIRED_PODS}" ]; then - echo " ❌ FAILED: Not all pods are ready!" - echo " Expected: ${DESIRED_PODS}, Got: ${READY_PODS}" - exit 1 - fi - echo " ✅ All pods ready" - - # 2. Verify pod images (source of truth) - echo "" - echo "2. Checking running pod images..." - - POD_IMAGES=$(kubectl get pods -n "${NAMESPACE}" -l app="${APP_NAME}" -o jsonpath='{.items[*].spec.containers[0].image}') - - - echo " Pod images: ${POD_IMAGES}" - echo " Expected tag: ${IMAGE_TAG}" - - if [[ "${POD_IMAGES}" != *"${IMAGE_TAG}"* ]]; then - echo " ❌ FAILED: Pods are running wrong image!" - exit 1 - fi - echo " ✅ All pods are running expected image" - - # 3. CRITICAL: Verify actual running pod images - echo "" - echo "3. Checking actual running pod images..." - - POD_IMAGES=$(kubectl get pods -n "${NAMESPACE}" -l app="${APP_NAME}" \ - -o jsonpath='{range .items[*]}{.status.containerStatuses[0].image}{"\n"}{end}') - - echo " Running pod images:" - while read -r img; do - echo " - ${img}" - if [[ "${img}" != *"${IMAGE_TAG}"* ]]; then - echo " ❌ Pod running wrong image: ${img}" - exit 1 - fi - done <<< "${POD_IMAGES}" - - echo " ✅ All pods running correct image" - - # 4. Check pod readiness - echo "" - echo "4. Checking pod readiness..." - - NOT_READY=$(kubectl get pods -n "${NAMESPACE}" -l app="${APP_NAME}" \ - --field-selector=status.phase!=Running --no-headers 2>/dev/null | wc -l) - - if [ "${NOT_READY}" -gt 0 ]; then - echo " ⚠️ WARNING: ${NOT_READY} pod(s) not in Running state" - kubectl get pods -n "${NAMESPACE}" -l app="${APP_NAME}" - else - echo " ✅ All pods in Running state" - fi - - # 5. Check container restart count - echo "" - echo "5. Checking for container restarts..." - - RESTART_COUNTS=$(kubectl get pods -n "${NAMESPACE}" -l app="${APP_NAME}" \ - -o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\n"}{end}') - - MAX_RESTARTS=0 - while read -r count; do - if [ "${count}" -gt "${MAX_RESTARTS}" ]; then - MAX_RESTARTS="${count}" - fi - done <<< "${RESTART_COUNTS}" - - echo " Max restart count: ${MAX_RESTARTS}" - - if [ "${MAX_RESTARTS}" -gt 3 ]; then - echo " ⚠️ WARNING: High restart count detected" - else - echo " ✅ Restart count acceptable" - fi - - echo "" - echo "================================================" - echo "✅ ALL VERIFICATION CHECKS PASSED!" - echo "================================================" - ''' -} - - """, returnStdout: true).trim() - - echo verifyResult - echo "✅ Deployment verified successfully!" - - } catch (Exception e) { - echo "❌ Deployment verification failed!" - echo "Error: ${e.message}" - - // Additional debugging - try { - echo "\n=== DEBUGGING INFORMATION ===" - - def pods = sh( - script: "kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide", - returnStdout: true - ).trim() - echo "Current pods:\n${pods}" - - def replicaset = sh( - script: "kubectl get replicaset -n ${NAMESPACE} -l app=${APP_NAME}", - returnStdout: true - ).trim() - echo "ReplicaSets:\n${replicaset}" - - def events = sh( - script: "kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20", - returnStdout: true - ).trim() - echo "Recent events:\n${events}" - - } catch (Exception debugEx) { - echo "Could not fetch debug info: ${debugEx.message}" - } - - throw e - } - } - } - } - } - - post { - success { - script { - env.DEPLOYMENT_END_TIME = sh(script: 'date +%s', returnStdout: true).trim() - def duration = (env.DEPLOYMENT_END_TIME.toInteger() - env.DEPLOYMENT_START_TIME.toInteger()) - def durationMin = duration / 60 - def durationSec = duration % 60 - - // Get deployment details - def podStatus = sh( - script: "kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.phase}{" "}{.status.containerStatuses[0].image}{" restarts="}{.status.containerStatuses[0].restartCount}{"\\n"}{end}'", - returnStdout: true - ).trim() - - def deployedImage = sh( - script: "kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}'", - returnStdout: true - ).trim() - - def replicas = sh( - script: "kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.replicas}'", - returnStdout: true - ).trim() - - echo """ - ✅ DEPLOYMENT SUCCESS! - - Application: ${APP_NAME} - Image: ${deployedImage} - Namespace: ${NAMESPACE} - Build: #${BUILD_NUMBER} - Duration: ${durationMin}m ${durationSec}s - - All checks passed! ✨ - """ - - // Send detailed success notification - sendTelegramNotification( - status: 'SUCCESS', - message: """ -✅ Deployment Successful! - -━━━━━━━━━━━━━━━━━━━━━━ -📦 Application Details -Name: ${APP_NAME} -Build: #${BUILD_NUMBER} -Branch: ${env.BRANCH_NAME} -Namespace: ${NAMESPACE} - -━━━━━━━━━━━━━━━━━━━━━━ -🐳 Docker Image -Registry: ${DOCKER_REGISTRY} -Repository: ${DOCKER_REPO}/${APP_NAME} -Tag: ${IMAGE_TAG} -Full Image: ${deployedImage} - -━━━━━━━━━━━━━━━━━━━━━━ -☸️ Kubernetes Status -Replicas: ${replicas}/${replicas} Ready -Rollout: Completed ✅ -Health: All pods healthy - -━━━━━━━━━━━━━━━━━━━━━━ -⏱️ Deployment Metrics -Duration: ${durationMin}m ${durationSec}s -Started: ${new Date(env.DEPLOYMENT_START_TIME.toLong() * 1000).format('HH:mm:ss')} -Completed: ${new Date(env.DEPLOYMENT_END_TIME.toLong() * 1000).format('HH:mm:ss')} - -━━━━━━━━━━━━━━━━━━━━━━ -🔗 Links -Jenkins Build -GitOps Repository - -Deployed by Jenkins CI/CD Pipeline 🚀 - """, - color: '✅' - ) - - // Cleanup rollback files - sh """ - rm -f /tmp/previous_image_${BUILD_NUMBER}.txt - rm -f /tmp/previous_replicas_${BUILD_NUMBER}.txt - rm -f /tmp/previous_commit_${BUILD_NUMBER}.txt - rm -f /tmp/expected_commit_${BUILD_NUMBER}.txt - """ - } - } - - failure { - script { - def errorDetails = "" - def previousImage = "unknown" - - try { - errorDetails = sh( - script: """ - kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -10 - """, - returnStdout: true - ).trim() - } catch (Exception e) { - errorDetails = "Could not fetch events: ${e.message}" - } - - if (env.BRANCH_NAME == 'main') { - echo """ - ❌ DEPLOYMENT FAILED - - GitOps policy: Jenkins will NOT auto-revert Git and will NOT run kubectl rollback. - Please review logs and rollback manually if needed. - """ - - def previousImageForInfo = sh( - script: "cat /tmp/previous_image_${BUILD_NUMBER}.txt 2>/dev/null || echo 'unknown'", - returnStdout: true - ).trim() - - def previousCommitForInfo = sh( - script: "cat /tmp/previous_commit_${BUILD_NUMBER}.txt 2>/dev/null || echo 'unknown'", - returnStdout: true - ).trim() - - def expectedCommitForInfo = sh( - script: "cat /tmp/expected_commit_${BUILD_NUMBER}.txt 2>/dev/null || echo 'unknown'", - returnStdout: true - ).trim() - - sendTelegramNotification( - status: 'FAILED', - message: """ -❌ Deployment Failed - -━━━━━━━━━━━━━━━━━━━━━━ -📦 Application -Name: ${APP_NAME} -Build: #${BUILD_NUMBER} -Branch: ${env.BRANCH_NAME} -Image: ${IMAGE_TAG} - -━━━━━━━━━━━━━━━━━━━━━━ -🧾 GitOps State -Expected Git revision: ${expectedCommitForInfo} -Previous Git revision: ${previousCommitForInfo} -Previous image: ${previousImageForInfo} - -━━━━━━━━━━━━━━━━━━━━━━ -📋 Recent Events -${errorDetails.take(500)} - -━━━━━━━━━━━━━━━━━━━━━━ -⚠️ Action Required -Auto-rollback is disabled (GitOps-safe). -Rollback manually if needed: -• ArgoCD UI → App → History/Rollback -or -• git revert to ${previousCommitForInfo} and push to ${GITEA_BRANCH} - -View Console Output -Build Details - """, - color: '🔴' - ) - - } else { - sendTelegramNotification( - status: 'FAILED', - message: """ -❌ Deployment Failed - -━━━━━━━━━━━━━━━━━━━━━━ -📦 Application -Name: ${APP_NAME} -Build: #${BUILD_NUMBER} -Branch: ${env.BRANCH_NAME} -Image: ${IMAGE_TAG} - -━━━━━━━━━━━━━━━━━━━━━━ -📋 Recent Events -${errorDetails.take(500)} - -━━━━━━━━━━━━━━━━━━━━━━ -🔗 Links -View Console Output -Build Details - """, - color: '🔴' - ) - } - } - } - - always { - sh """ - docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true - docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest || true - docker stop test-${BUILD_NUMBER} 2>/dev/null || true - docker rm test-${BUILD_NUMBER} 2>/dev/null || true - """ - cleanWs() - } - } -} - -// Telegram notification function -def sendTelegramNotification(Map args) { - def status = args.status - def message = args.message - def color = args.color ?: '🔵' - - try { - // HTML formatting for Telegram - def formattedMessage = message.trim() - - sh """ - curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ - -d chat_id="${TELEGRAM_CHAT_ID}" \ - -d parse_mode="HTML" \ - -d disable_web_page_preview=true \ - -d text="${formattedMessage}" - """ - - echo "📱 Telegram notification sent: ${status}" - } catch (Exception e) { - echo "⚠️ Failed to send Telegram notification: ${e.message}" - // Don't fail the pipeline if notification fails - } -} \ No newline at end of file