diff --git a/apps/demo-nginx/.env b/apps/demo-nginx/.env deleted file mode 100644 index 99c77b4..0000000 --- a/apps/demo-nginx/.env +++ /dev/null @@ -1,69 +0,0 @@ -############################################ -# Application -############################################ -APP_NAME=demo-nginx -NAMESPACE=demo-app -BRANCH_NAME=main - -############################################ -# Docker -############################################ -DOCKER_REGISTRY=docker.io -DOCKER_REPO=vladcrypto -# IMAGE_TAG is computed in Jenkins as: ${BRANCH_NAME}-${BUILD_NUMBER} -# Example: main-100 - -############################################ -# GitOps / Gitea -############################################ -GITEA_URL=http://gitea-http.gitea.svc.cluster.local:3000 -GITEA_REPO=admin/k3s-gitops -GITEA_BRANCH=main -GITOPS_MANIFEST_PATH=apps/demo-nginx/deployment.yaml - -############################################ -# ArgoCD -############################################ -ARGOCD_APP_NAME=demo-nginx -ARGOCD_NAMESPACE=argocd -ARGOCD_SYNC_TIMEOUT=120 # seconds - -############################################ -# Deployment / Rollout -############################################ -DEPLOYMENT_TIMEOUT=300s -SKIP_HEALTH_CHECK=true -ROLLBACK_ENABLED=false # GitOps-safe: Jenkins never reverts Git - -############################################ -# Jenkins Runtime (injected by Jenkins) -############################################ -# BUILD_NUMBER=100 -# BUILD_URL=https://jenkins.thedevops.dev/job/demo-nginx/job/main/100/ -# BUILD_USER=jenkins - -############################################ -# Telegram Notifications -############################################ -TELEGRAM_BOT_TOKEN=__REPLACE_ME__ -TELEGRAM_CHAT_ID=__REPLACE_ME__ - -############################################ -# Docker Hub Credentials -############################################ -DOCKER_USER=__REPLACE_ME__ -DOCKER_PASS=__REPLACE_ME__ - -############################################ -# Gitea Credentials -############################################ -GIT_USER=__REPLACE_ME__ -GIT_PASS=__REPLACE_ME__ - -############################################ -# Internal / Temporary Files (used by pipeline) -############################################ -PREVIOUS_IMAGE_FILE=/tmp/previous_image_${BUILD_NUMBER}.txt -PREVIOUS_REPLICAS_FILE=/tmp/previous_replicas_${BUILD_NUMBER}.txt -PREVIOUS_COMMIT_FILE=/tmp/previous_commit_${BUILD_NUMBER}.txt -EXPECTED_COMMIT_FILE=/tmp/expected_commit_${BUILD_NUMBER}.txt diff --git a/apps/demo-nginx/Jenkinsfile b/apps/demo-nginx/Jenkinsfile deleted file mode 100644 index 723cdfe..0000000 --- a/apps/demo-nginx/Jenkinsfile +++ /dev/null @@ -1,763 +0,0 @@ -pipeline { - agent any - options { - disableConcurrentBuilds() - } - environment { - APP_NAME = 'demo-nginx' - NAMESPACE = 'demo-app' - DOCKER_REGISTRY = 'docker.io' - DOCKER_REPO = 'vladcrypto' - GITEA_URL = 'http://gitea-http.gitea.svc.cluster.local:3000' - GITEA_REPO = 'admin/k3s-gitops' - GITEA_BRANCH = 'main' - BUILD_TAG = "${env.BUILD_NUMBER}" - IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}" - - // Rollback configuration - ROLLBACK_ENABLED = 'true' - DEPLOYMENT_TIMEOUT = '300s' - ARGOCD_SYNC_TIMEOUT = '120' - SKIP_HEALTH_CHECK = 'true' - - // Notification configuration - TELEGRAM_BOT_TOKEN = credentials('telegram-bot-token') - TELEGRAM_CHAT_ID = credentials('telegram-chat-id') - - // Build info - BUILD_URL = "${env.BUILD_URL}" - GIT_COMMIT_SHORT = "" - DEPLOYMENT_START_TIME = "" - DEPLOYMENT_END_TIME = "" - } - - stages { - stage('Initialization') { - steps { - script { - echo "🚀 Starting deployment pipeline..." - DEPLOYMENT_START_TIME = sh(script: 'date +%s', returnStdout: true).trim() - - sendTelegramNotification( - status: 'STARTED', - message: """ -🚀 Deployment Started - -Application: ${APP_NAME} -Build: #${BUILD_NUMBER} -Branch: ${env.BRANCH_NAME} -Namespace: ${NAMESPACE} -Started by: ${env.BUILD_USER ?: 'Jenkins'} - -Building and deploying... - """, - color: '🔵' - ) - } - } - } - - stage('Save Current State') { - when { branch 'main' } - steps { - script { - echo "📸 Saving current deployment state for rollback..." - - sh """ - kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ - -o jsonpath='{.spec.template.spec.containers[0].image}' \ - > /tmp/previous_image_${BUILD_NUMBER}.txt || echo "none" > /tmp/previous_image_${BUILD_NUMBER}.txt - - PREV_IMAGE=\$(cat /tmp/previous_image_${BUILD_NUMBER}.txt) - echo "Previous image: \${PREV_IMAGE}" - """ - - sh """ - kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ - -o jsonpath='{.spec.replicas}' \ - > /tmp/previous_replicas_${BUILD_NUMBER}.txt || echo "2" > /tmp/previous_replicas_${BUILD_NUMBER}.txt - """ - - echo "✅ Current state saved" - } - } - } - - stage('Checkout Source') { - steps { - echo "📦 Creating application artifacts..." - - sh """ - cat > Dockerfile << 'EOF' -FROM nginx:1.25.3-alpine -COPY index.html /usr/share/nginx/html/index.html -COPY nginx.conf /etc/nginx/nginx.conf -EXPOSE 80 -CMD ["nginx", "-g", "daemon off;"] -EOF - """ - - sh """ - cat > index.html << EOF - - - - Demo Nginx - - - -
-

Demo Nginx - Build #${BUILD_NUMBER}

-

Environment: Production

-

Version: ${IMAGE_TAG}

-

- Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} -

-
- - -EOF - """ - - sh ''' - cat > nginx.conf << 'EOF' -user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; -events { worker_connections 1024; } -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; - sendfile on; - keepalive_timeout 65; - server { - listen 80; - server_name _; - location / { - root /usr/share/nginx/html; - index index.html; - } - location /health { - access_log off; - return 200 "healthy\n"; - add_header Content-Type text/plain; - } - } -} -EOF - ''' - } - } - - stage('Build Docker Image') { - steps { - script { - echo "🏗️ Building Docker image..." - - sendTelegramNotification( - status: 'BUILDING', - message: """ -🏗️ Building Docker Image - -Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} -Stage: Build - """, - color: '🔵' - ) - - sh """ - docker build \ - -t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} \ - -t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest \ - . - """ - echo "✅ Image built successfully!" - } - } - } - - stage('Push to Registry') { - when { branch 'main' } - steps { - script { - echo "📤 Pushing image to registry..." - - sendTelegramNotification( - status: 'PUSHING', - message: """ -📤 Pushing to Registry - -Registry: ${DOCKER_REGISTRY} -Image: ${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} - """, - color: '🔵' - ) - - withCredentials([usernamePassword( - credentialsId: 'docker-registry-credentials', - usernameVariable: 'DOCKER_USER', - passwordVariable: 'DOCKER_PASS' - )]) { - sh """ - echo "\${DOCKER_PASS}" | docker login ${DOCKER_REGISTRY} -u "\${DOCKER_USER}" --password-stdin - docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} - docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest - docker logout ${DOCKER_REGISTRY} - """ - } - echo "✅ Image pushed successfully!" - } - } - } - - stage('Update GitOps Manifests') { - when { branch 'main' } - steps { - script { - echo "📝 Updating Kubernetes manifests..." - - sendTelegramNotification( - status: 'UPDATING', - message: """ -📝 Updating GitOps Manifests - -Repository: ${GITEA_REPO} -Branch: ${GITEA_BRANCH} -File: apps/demo-nginx/deployment.yaml - """, - color: '🔵' - ) - - withCredentials([usernamePassword( - credentialsId: 'gitea-credentials', - usernameVariable: 'GIT_USER', - passwordVariable: 'GIT_PASS' - )]) { - sh """ - rm -rf k3s-gitops || true - git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git - cd k3s-gitops - git config user.name "Jenkins" - git config user.email "jenkins@thedevops.dev" - - # Save current commit for rollback - git rev-parse HEAD > /tmp/previous_commit_${BUILD_NUMBER}.txt - - sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml - git add apps/demo-nginx/deployment.yaml - git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes" - git push origin main - """ - } - echo "✅ Manifests updated!" - } - } - } - - stage('Wait for ArgoCD Sync') { - steps { - script { - echo "⏳ Waiting for ArgoCD to sync manifests..." - - for (int i = 1; i <= 12; i++) { - def syncStatus = sh( - script: "kubectl get application demo-nginx -n argocd -o jsonpath='{.status.sync.status}'", - returnStdout: true - ).trim() - - echo "ArgoCD sync status : ${syncStatus}" - - if (syncStatus == "Synced") { - echo "✅ ArgoCD manifests synced" - return - } - - sleep 10 - } - - error("❌ ArgoCD did not sync manifests in time") - } - } -} - - - - - - - - - - - stage('Wait for Deployment') { - when { branch 'main' } - steps { - script { - echo "⏳ Waiting for Kubernetes rollout to complete..." - - sendTelegramNotification( - status: 'DEPLOYING', - message: """ -🚀 Deploying to Kubernetes - -Deployment: ${APP_NAME} -Namespace: ${NAMESPACE} -Image: ${IMAGE_TAG} -Timeout: ${DEPLOYMENT_TIMEOUT} - -Rolling out new pods... - """, - color: '🔵' - ) - - try { - // Wait for rollout to complete - echo "Starting rollout status check..." - sh """ - kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT} - """ - echo "✅ Rollout completed successfully!" - - // Additional verification - wait a bit for pods to stabilize - echo "Waiting 10 seconds for pods to stabilize..." - sleep 10 - - } catch (Exception e) { - echo "❌ Deployment rollout failed: ${e.message}" - - // Get pod status for debugging - try { - def podStatus = sh( - script: """ - kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide - """, - returnStdout: true - ).trim() - echo "Current pod status:\n${podStatus}" - - def events = sh( - script: """ - kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20 - """, - returnStdout: true - ).trim() - echo "Recent events:\n${events}" - } catch (Exception debugEx) { - echo "Could not fetch debug info: ${debugEx.message}" - } - - throw e - } - } - } - } - - stage('Verify Deployment') { - when { branch 'main' } - steps { - script { - echo "✅ Verifying deployment and pod status..." - - /* -------------------------------- - * 1. Deployment readiness - * -------------------------------- */ - sh """ - set -e - echo "1. Checking deployment readiness..." - - READY=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}') - DESIRED=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}') - - echo "Ready replicas : \$READY" - echo "Desired replicas : \$DESIRED" - - if [ "\$READY" != "\$DESIRED" ]; then - echo "❌ Not all replicas are ready" - exit 1 - fi - """ - - /* -------------------------------- - * 2. Actual running images - * -------------------------------- */ - def podImages = sh( - script: """ - kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} \ - -o jsonpath='{range .items[*]}{.status.containerStatuses[0].image}{"\\n"}{end}' - """, - returnStdout: true - ).trim() - - echo "Running pod images:" - podImages.split("\\n").each { img -> - echo " - ${img}" - } - - if (!podImages.contains(IMAGE_TAG)) { - error("❌ Some pods are NOT running image tag ${IMAGE_TAG}") - } - - /* -------------------------------- - * 3. Restart count - * -------------------------------- */ - def restarts = sh( - script: """ - kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} \ - -o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\\n"}{end}' - """, - returnStdout: true - ).trim() - - def maxRestart = restarts - .split("\\n") - .collect { it.toInteger() } - .max() - - echo "Max restart count: ${maxRestart}" - - if (maxRestart > 3) { - error("❌ High restart count detected: ${maxRestart}") - } - - echo "✅ ALL VERIFICATION CHECKS PASSED" - } - } - } - } - - - - post { - success { - script { - DEPLOYMENT_END_TIME = sh(script: 'date +%s', returnStdout: true).trim() - def duration = (DEPLOYMENT_END_TIME.toInteger() - DEPLOYMENT_START_TIME.toInteger()) - def durationMin = duration / 60 - def durationSec = duration % 60 - - // Get deployment details - def podStatus = sh( - script: """ - kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} \ - -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.phase}{" "}{.status.containerStatuses[0].restartCount}{"\\n"}{end}' - """, - returnStdout: true - ).trim() - - def deployedImage = sh( - script: """ - kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ - -o jsonpath='{.spec.template.spec.containers[0].image}' - """, - returnStdout: true - ).trim() - - def replicas = sh( - script: """ - kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ - -o jsonpath='{.status.replicas}' - """, - returnStdout: true - ).trim() - - echo """ - ✅ DEPLOYMENT SUCCESS! - - Application: ${APP_NAME} - Image: ${deployedImage} - Namespace: ${NAMESPACE} - Build: #${BUILD_NUMBER} - Duration: ${durationMin}m ${durationSec}s - - All checks passed! ✨ - """ - - // Send detailed success notification - sendTelegramNotification( - status: 'SUCCESS', - message: """ -✅ Deployment Successful! - -━━━━━━━━━━━━━━━━━━━━━━ -📦 Application Details -Name: ${APP_NAME} -Build: #${BUILD_NUMBER} -Branch: ${env.BRANCH_NAME} -Namespace: ${NAMESPACE} - -━━━━━━━━━━━━━━━━━━━━━━ -🐳 Docker Image -Registry: ${DOCKER_REGISTRY} -Repository: ${DOCKER_REPO}/${APP_NAME} -Tag: ${IMAGE_TAG} -Full Image: ${deployedImage} - -━━━━━━━━━━━━━━━━━━━━━━ -☸️ Kubernetes Status -Replicas: ${replicas}/${replicas} Ready -Rollout: Completed ✅ -Health: All pods healthy - -━━━━━━━━━━━━━━━━━━━━━━ -⏱️ Deployment Metrics -Duration: ${durationMin}m ${durationSec}s -Started: ${new Date(DEPLOYMENT_START_TIME.toLong() * 1000).format('HH:mm:ss')} -Completed: ${new Date(DEPLOYMENT_END_TIME.toLong() * 1000).format('HH:mm:ss')} - -━━━━━━━━━━━━━━━━━━━━━━ -🔗 Links -Jenkins Build -GitOps Repository - -Deployed by Jenkins CI/CD Pipeline 🚀 - """, - color: '✅' - ) - - // Cleanup rollback files - sh """ - rm -f /tmp/previous_image_${BUILD_NUMBER}.txt - rm -f /tmp/previous_replicas_${BUILD_NUMBER}.txt - rm -f /tmp/previous_commit_${BUILD_NUMBER}.txt - """ - } - } - - failure { - script { - def errorDetails = "" - def previousImage = "unknown" - - try { - errorDetails = sh( - script: """ - kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -10 - """, - returnStdout: true - ).trim() - } catch (Exception e) { - errorDetails = "Could not fetch events: ${e.message}" - } - - if (env.BRANCH_NAME == 'main' && env.ROLLBACK_ENABLED == 'true') { - echo """ - ❌ DEPLOYMENT FAILED - INITIATING ROLLBACK! - - Rolling back to previous version... - """ - - sendTelegramNotification( - status: 'ROLLING_BACK', - message: """ -🔄 Deployment Failed - Rolling Back - -Application: ${APP_NAME} -Failed Build: #${BUILD_NUMBER} -Image: ${IMAGE_TAG} - -Initiating automatic rollback... - """, - color: '⚠️' - ) - - try { - previousImage = sh( - script: "cat /tmp/previous_image_${BUILD_NUMBER}.txt 2>/dev/null || echo 'none'", - returnStdout: true - ).trim() - - if (previousImage != 'none' && previousImage != '') { - echo "🔄 Rolling back to: ${previousImage}" - - sh """ - kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE} - kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=180s - """ - - withCredentials([usernamePassword( - credentialsId: 'gitea-credentials', - usernameVariable: 'GIT_USER', - passwordVariable: 'GIT_PASS' - )]) { - sh """ - cd k3s-gitops || git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git - cd k3s-gitops - git config user.name "Jenkins" - git config user.email "jenkins@thedevops.dev" - git revert --no-edit HEAD || true - git push origin main || true - """ - } - - sendTelegramNotification( - status: 'ROLLBACK_SUCCESS', - message: """ -✅ Rollback Completed - -━━━━━━━━━━━━━━━━━━━━━━ -📦 Application -Name: ${APP_NAME} -Failed Build: #${BUILD_NUMBER} -Rolled Back To: ${previousImage} - -━━━━━━━━━━━━━━━━━━━━━━ -🔄 Rollback Status -Status: Success ✅ -Method: kubectl rollout undo -Git: Commit reverted - -━━━━━━━━━━━━━━━━━━━━━━ -📋 Recent Events -${errorDetails.take(500)} - -━━━━━━━━━━━━━━━━━━━━━━ -⚠️ Action Required -Please review logs and fix issues before redeploying - -View Build Logs - """, - color: '✅' - ) - - } else { - sendTelegramNotification( - status: 'ROLLBACK_FAILED', - message: """ -❌ Rollback Failed - -Application: ${APP_NAME} -Build: #${BUILD_NUMBER} - -Error: No previous version found - -⚠️ MANUAL INTERVENTION REQUIRED -kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE} - -View Build Logs - """, - color: '🔴' - ) - } - - } catch (Exception e) { - sendTelegramNotification( - status: 'ROLLBACK_FAILED', - message: """ -❌ Rollback Failed - -Application: ${APP_NAME} -Build: #${BUILD_NUMBER} - -Error: ${e.message} - -⚠️ MANUAL ROLLBACK REQUIRED -kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE} - -View Build Logs - """, - color: '🔴' - ) - } - - } else { - sendTelegramNotification( - status: 'FAILED', - message: """ -❌ Deployment Failed - -━━━━━━━━━━━━━━━━━━━━━━ -📦 Application -Name: ${APP_NAME} -Build: #${BUILD_NUMBER} -Branch: ${env.BRANCH_NAME} -Image: ${IMAGE_TAG} - -━━━━━━━━━━━━━━━━━━━━━━ -📋 Recent Events -${errorDetails.take(500)} - -━━━━━━━━━━━━━━━━━━━━━━ -🔗 Links -View Console Output -Build Details - -Rollback: ${env.ROLLBACK_ENABLED == 'true' ? 'Enabled but not on main branch' : 'Disabled'} - """, - color: '🔴' - ) - } - } - } - - always { - sh """ - docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true - docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest || true - docker stop test-${BUILD_NUMBER} 2>/dev/null || true - docker rm test-${BUILD_NUMBER} 2>/dev/null || true - """ - cleanWs() - } - } -} - -// Telegram notification function -def sendTelegramNotification(Map args) { - def status = args.status - def message = args.message - def color = args.color ?: '🔵' - - try { - // HTML formatting for Telegram - def formattedMessage = message.trim() - - sh """ - curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ - -d chat_id="${TELEGRAM_CHAT_ID}" \ - -d parse_mode="HTML" \ - -d disable_web_page_preview=true \ - -d text="${formattedMessage}" - """ - - echo "📱 Telegram notification sent: ${status}" - } catch (Exception e) { - echo "⚠️ Failed to send Telegram notification: ${e.message}" - // Don't fail the pipeline if notification fails - } -} \ No newline at end of file diff --git a/apps/demo-nginx/Jenkinsfile.backup b/apps/demo-nginx/Jenkinsfile.backup deleted file mode 100644 index 20c3ad8..0000000 --- a/apps/demo-nginx/Jenkinsfile.backup +++ /dev/null @@ -1,449 +0,0 @@ -pipeline { - agent any - - environment { - APP_NAME = 'demo-nginx' - NAMESPACE = 'demo-app' - DOCKER_REGISTRY = 'docker.io' - DOCKER_REPO = 'vladcrypto' - GITEA_URL = 'http://gitea-http.gitea.svc.cluster.local:3000' - GITEA_REPO = 'admin/k3s-gitops' - GITEA_BRANCH = 'main' - BUILD_TAG = "${env.BUILD_NUMBER}" - IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}" - - // Rollback configuration - ROLLBACK_ENABLED = 'true' - DEPLOYMENT_TIMEOUT = '300s' - ARGOCD_SYNC_TIMEOUT = '120' - SKIP_HEALTH_CHECK = 'true' // Skip for now - pods work fine - } - - stages { - stage('Save Current State') { - when { branch 'main' } - steps { - script { - echo "📸 Saving current deployment state for rollback..." - - // Save current image tag - sh """ - kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ - -o jsonpath='{.spec.template.spec.containers[0].image}' \ - > /tmp/previous_image_${BUILD_NUMBER}.txt || echo "none" > /tmp/previous_image_${BUILD_NUMBER}.txt - - PREV_IMAGE=\$(cat /tmp/previous_image_${BUILD_NUMBER}.txt) - echo "Previous image: \${PREV_IMAGE}" - """ - - // Save current replica count - sh """ - kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ - -o jsonpath='{.spec.replicas}' \ - > /tmp/previous_replicas_${BUILD_NUMBER}.txt || echo "2" > /tmp/previous_replicas_${BUILD_NUMBER}.txt - """ - - echo "✅ Current state saved" - } - } - } - - stage('Checkout Source') { - steps { - echo "Checking out application source code..." - - // Create Dockerfile with actual version - sh """ - cat > Dockerfile << 'EOF' -FROM nginx:1.25.3-alpine -COPY index.html /usr/share/nginx/html/index.html -COPY nginx.conf /etc/nginx/nginx.conf -EXPOSE 80 -CMD ["nginx", "-g", "daemon off;"] -EOF - """ - - // Create index.html with actual version - sh """ - cat > index.html << EOF - - - - Demo Nginx - - - -
-

🚀 Demo Nginx - Build #${BUILD_NUMBER}

-

Environment: Production

-

Version: ${IMAGE_TAG}

-

- Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} -

-
- - -EOF - """ - - sh ''' - cat > nginx.conf << 'EOF' -user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; -events { worker_connections 1024; } -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; - sendfile on; - keepalive_timeout 65; - server { - listen 80; - server_name _; - location / { - root /usr/share/nginx/html; - index index.html; - } - location /health { - access_log off; - return 200 "healthy\n"; - add_header Content-Type text/plain; - } - } -} -EOF - ''' - } - } - - stage('Build Docker Image') { - steps { - script { - echo "Building Docker image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}" - sh """ - docker build \ - -t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} \ - -t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest \ - . - """ - echo "✅ Image built successfully!" - } - } - } - - stage('Push to Registry') { - when { branch 'main' } - steps { - script { - echo "Pushing image to registry..." - withCredentials([usernamePassword( - credentialsId: 'docker-registry-credentials', - usernameVariable: 'DOCKER_USER', - passwordVariable: 'DOCKER_PASS' - )]) { - sh """ - echo "\${DOCKER_PASS}" | docker login ${DOCKER_REGISTRY} -u "\${DOCKER_USER}" --password-stdin - docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} - docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest - docker logout ${DOCKER_REGISTRY} - """ - } - echo "✅ Image pushed successfully!" - } - } - } - - stage('Update GitOps Manifests') { - when { branch 'main' } - steps { - script { - echo "Updating Kubernetes manifests..." - withCredentials([usernamePassword( - credentialsId: 'gitea-credentials', - usernameVariable: 'GIT_USER', - passwordVariable: 'GIT_PASS' - )]) { - sh """ - rm -rf k3s-gitops || true - git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git - cd k3s-gitops - git config user.name "Jenkins" - git config user.email "jenkins@thedevops.dev" - - # Save current commit for rollback - git rev-parse HEAD > /tmp/previous_commit_${BUILD_NUMBER}.txt - - sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml - git add apps/demo-nginx/deployment.yaml - git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes" - git push origin main - """ - } - echo "✅ Manifests updated!" - } - } - } - - stage('Wait for ArgoCD Sync') { - when { branch 'main' } - steps { - script { - echo "⏳ Waiting for ArgoCD to sync..." - - def syncSuccess = false - def attempts = 0 - def maxAttempts = Integer.parseInt(env.ARGOCD_SYNC_TIMEOUT) / 10 - - while (!syncSuccess && attempts < maxAttempts) { - attempts++ - echo "ArgoCD sync check attempt ${attempts}/${maxAttempts}..." - - def syncStatus = sh( - script: """ - kubectl get application ${APP_NAME} -n argocd \ - -o jsonpath='{.status.sync.status}' - """, - returnStdout: true - ).trim() - - def currentImage = sh( - script: """ - kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ - -o jsonpath='{.spec.template.spec.containers[0].image}' - """, - returnStdout: true - ).trim() - - echo "ArgoCD sync status: ${syncStatus}" - echo "Current deployment image: ${currentImage}" - echo "Expected image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}" - - if (syncStatus == 'Synced' && currentImage.contains(env.IMAGE_TAG)) { - syncSuccess = true - echo "✅ ArgoCD synced successfully!" - break - } - - if (attempts < maxAttempts) { - echo "Waiting 10 seconds before next check..." - sleep 10 - } - } - - if (!syncSuccess) { - error "❌ ArgoCD sync timeout after ${env.ARGOCD_SYNC_TIMEOUT} seconds!" - } - } - } - } - - stage('Wait for Deployment') { - when { branch 'main' } - steps { - script { - echo "⏳ Waiting for deployment to complete..." - - try { - sh """ - # Check rollout status - kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT} - """ - echo "✅ Deployment rolled out successfully!" - } catch (Exception e) { - echo "❌ Deployment failed: ${e.message}" - throw e - } - } - } - } - - stage('Verify Deployment') { - when { branch 'main' } - steps { - script { - echo "✅ Verifying deployment..." - - try { - sh """#!/bin/bash - set -e - - # Check pod status - READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}') - DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}') - - echo "Ready pods: \${READY_PODS}/\${DESIRED_PODS}" - - if [ "\${READY_PODS}" != "\${DESIRED_PODS}" ]; then - echo "❌ Not all pods are ready!" - exit 1 - fi - - # Verify image version - DEPLOYED_IMAGE=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}') - echo "Deployed image: \${DEPLOYED_IMAGE}" - - if [[ "\${DEPLOYED_IMAGE}" != *"${IMAGE_TAG}"* ]]; then - echo "❌ Image version mismatch!" - echo "Expected: ${IMAGE_TAG}" - echo "Got: \${DEPLOYED_IMAGE}" - exit 1 - fi - - echo "✅ All verification checks passed!" - """ - - echo "✅ Deployment verified successfully!" - - } catch (Exception e) { - echo "❌ Deployment verification failed: ${e.message}" - throw e - } - } - } - } - } - - post { - success { - script { - echo """ - ✅ DEPLOYMENT SUCCESS! - - Application: ${APP_NAME} - Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} - Namespace: ${NAMESPACE} - Build: #${BUILD_NUMBER} - - All checks passed! ✨ - """ - - // Cleanup rollback files - sh """ - rm -f /tmp/previous_image_${BUILD_NUMBER}.txt - rm -f /tmp/previous_replicas_${BUILD_NUMBER}.txt - rm -f /tmp/previous_commit_${BUILD_NUMBER}.txt - """ - } - } - - failure { - script { - if (env.BRANCH_NAME == 'main' && env.ROLLBACK_ENABLED == 'true') { - echo """ - ❌ DEPLOYMENT FAILED - INITIATING ROLLBACK! - - Rolling back to previous version... - """ - - try { - // Get previous state - def previousImage = sh( - script: "cat /tmp/previous_image_${BUILD_NUMBER}.txt 2>/dev/null || echo 'none'", - returnStdout: true - ).trim() - - if (previousImage != 'none' && previousImage != '') { - echo "🔄 Rolling back to: ${previousImage}" - - // Rollback via kubectl - sh """ - kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE} - kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=180s - """ - - // Revert Git commit - withCredentials([usernamePassword( - credentialsId: 'gitea-credentials', - usernameVariable: 'GIT_USER', - passwordVariable: 'GIT_PASS' - )]) { - sh """ - cd k3s-gitops || git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git - cd k3s-gitops - git config user.name "Jenkins" - git config user.email "jenkins@thedevops.dev" - - # Revert last commit - git revert --no-edit HEAD || true - git push origin main || true - """ - } - - echo """ - ✅ ROLLBACK COMPLETED! - - Rolled back to: ${previousImage} - Current build (#${BUILD_NUMBER}) has been reverted. - - Please check logs and fix the issue before redeploying. - """ - - } else { - echo "⚠️ No previous version found - cannot rollback automatically" - echo "Manual intervention required!" - } - - } catch (Exception e) { - echo """ - ❌ ROLLBACK FAILED! - - Error: ${e.message} - - MANUAL ROLLBACK REQUIRED: - kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE} - """ - } - - } else { - echo "❌ Pipeline failed! (Rollback disabled or not main branch)" - } - } - } - - always { - sh """ - docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true - docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest || true - docker stop test-${BUILD_NUMBER} 2>/dev/null || true - docker rm test-${BUILD_NUMBER} 2>/dev/null || true - """ - cleanWs() - } - } -} diff --git a/apps/demo-nginx/Jenkinsfile.rollback b/apps/demo-nginx/Jenkinsfile.rollback deleted file mode 100644 index 6c47af8..0000000 --- a/apps/demo-nginx/Jenkinsfile.rollback +++ /dev/null @@ -1,564 +0,0 @@ -// Jenkinsfile for Manual Rollback -// This allows rolling back to any previous version -// Now rebuilds image with updated design! - -pipeline { - agent any - - parameters { - choice( - name: 'ROLLBACK_METHOD', - choices: ['IMAGE_TAG', 'REVISION_NUMBER', 'GIT_COMMIT'], - description: 'Select rollback method' - ) - string( - name: 'TARGET_VERSION', - defaultValue: '', - description: ''' - Enter target version based on method: - - IMAGE_TAG: main-21, main-20, etc. - - REVISION_NUMBER: 1, 2, 3 (from rollout history) - - GIT_COMMIT: abc123def (git commit SHA) - ''' - ) - booleanParam( - name: 'SKIP_HEALTH_CHECK', - defaultValue: true, - description: 'Skip health checks (recommended for rollback)' - ) - booleanParam( - name: 'DRY_RUN', - defaultValue: false, - description: 'Dry run - show what would happen without applying' - ) - booleanParam( - name: 'REBUILD_IMAGE', - defaultValue: true, - description: 'Rebuild image with updated design (recommended)' - ) - } - - environment { - APP_NAME = 'demo-nginx' - CONTAINER_NAME = 'nginx' - NAMESPACE = 'demo-app' - DOCKER_REGISTRY = 'docker.io' - DOCKER_REPO = 'vladcrypto' - GITEA_URL = 'http://gitea-http.gitea.svc.cluster.local:3000' - HEALTH_CHECK_TIMEOUT = '300s' - ARGOCD_SYNC_TIMEOUT = '120' - } - - stages { - stage('Validate Input') { - steps { - script { - echo "🔍 Validating rollback request..." - - // Trim whitespace from input - env.TARGET_VERSION_CLEAN = params.TARGET_VERSION.trim() - - if (env.TARGET_VERSION_CLEAN == '') { - error("❌ TARGET_VERSION cannot be empty!") - } - - echo """ - 📋 Rollback Configuration: - Method: ${params.ROLLBACK_METHOD} - Target: ${env.TARGET_VERSION_CLEAN} - Rebuild Image: ${params.REBUILD_IMAGE} - Skip Health Check: ${params.SKIP_HEALTH_CHECK} - Dry Run: ${params.DRY_RUN} - """ - } - } - } - - stage('Show Current State') { - steps { - script { - echo "📸 Current Deployment State:" - - sh """ - echo "=== Current Deployment ===" - kubectl get deployment ${APP_NAME} -n ${NAMESPACE} - - echo "" - echo "=== Current Image ===" - kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ - -o jsonpath='{.spec.template.spec.containers[0].image}' - echo "" - - echo "" - echo "=== Current Pods ===" - kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} - - echo "" - echo "=== Rollout History ===" - kubectl rollout history deployment/${APP_NAME} -n ${NAMESPACE} - """ - } - } - } - - stage('Prepare Rollback') { - steps { - script { - echo "🔄 Preparing rollback to: ${env.TARGET_VERSION_CLEAN}" - - if (params.ROLLBACK_METHOD == 'IMAGE_TAG') { - env.TARGET_IMAGE = "${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${env.TARGET_VERSION_CLEAN}" - - // Extract build number from tag - env.BUILD_NUMBER_FROM_TAG = env.TARGET_VERSION_CLEAN.split('-')[1] - - sh """ - echo "Target image: ${env.TARGET_IMAGE}" - echo "Build number: ${env.BUILD_NUMBER_FROM_TAG}" - """ - - } else if (params.ROLLBACK_METHOD == 'REVISION_NUMBER') { - env.REVISION = env.TARGET_VERSION_CLEAN - - sh """ - echo "Rolling back to revision: ${env.REVISION}" - kubectl rollout history deployment/${APP_NAME} -n ${NAMESPACE} \ - --revision=${env.REVISION} - """ - - } else if (params.ROLLBACK_METHOD == 'GIT_COMMIT') { - env.GIT_SHA = env.TARGET_VERSION_CLEAN - echo "Rolling back to git commit: ${env.GIT_SHA}" - } - } - } - } - - stage('Rebuild Image') { - when { - expression { !params.DRY_RUN && params.REBUILD_IMAGE && params.ROLLBACK_METHOD == 'IMAGE_TAG' } - } - steps { - script { - echo "🔨 Rebuilding image with updated design..." - - // Create Dockerfile - writeFile file: 'Dockerfile', text: '''FROM nginx:1.25.3-alpine -COPY index.html /usr/share/nginx/html/index.html -COPY nginx.conf /etc/nginx/nginx.conf -EXPOSE 80 -CMD ["nginx", "-g", "daemon off;"] -''' - - // Create HTML with proper variable substitution - def htmlContent = """ - - - Demo Nginx - Rollback - - - -
-
🔄 ROLLED BACK
-

🚀 Demo Nginx - Build #${env.BUILD_NUMBER_FROM_TAG}

-

Environment: Production

-

Version: ${env.TARGET_VERSION_CLEAN}

-

- Image: ${env.TARGET_IMAGE} -

-

- ⏮️ Restored from previous deployment -

-
- - -""" - - writeFile file: 'index.html', text: htmlContent - - // Create nginx.conf - writeFile file: 'nginx.conf', text: '''user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; -events { worker_connections 1024; } -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; - sendfile on; - keepalive_timeout 65; - server { - listen 80; - server_name _; - location / { - root /usr/share/nginx/html; - index index.html; - } - location /health { - access_log off; - return 200 "healthy\n"; - add_header Content-Type text/plain; - } - } -} -''' - - // Build new image - sh """ - docker build -t ${env.TARGET_IMAGE} . - """ - - echo "✅ Image rebuilt successfully!" - } - } - } - - stage('Push Rebuilt Image') { - when { - expression { !params.DRY_RUN && params.REBUILD_IMAGE && params.ROLLBACK_METHOD == 'IMAGE_TAG' } - } - steps { - script { - echo "📤 Pushing rebuilt image..." - - withCredentials([usernamePassword( - credentialsId: 'docker-registry-credentials', - usernameVariable: 'DOCKER_USER', - passwordVariable: 'DOCKER_PASS' - )]) { - sh """ - echo "\${DOCKER_PASS}" | docker login ${DOCKER_REGISTRY} -u "\${DOCKER_USER}" --password-stdin - docker push ${env.TARGET_IMAGE} - docker logout ${DOCKER_REGISTRY} - """ - } - - echo "✅ Image pushed successfully!" - } - } - } - - stage('Execute Rollback') { - when { - expression { !params.DRY_RUN } - } - steps { - script { - echo "🚀 Executing rollback..." - - if (params.ROLLBACK_METHOD == 'IMAGE_TAG') { - // Update deployment - sh """ - echo "Setting image to: ${env.TARGET_IMAGE}" - kubectl set image deployment/${APP_NAME} \ - ${CONTAINER_NAME}=${env.TARGET_IMAGE} \ - -n ${NAMESPACE} \ - --record - """ - - // Update Git manifests - withCredentials([usernamePassword( - credentialsId: 'gitea-credentials', - usernameVariable: 'GIT_USER', - passwordVariable: 'GIT_PASS' - )]) { - sh """ - rm -rf k3s-gitops || true - git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git - cd k3s-gitops - git config user.name "Jenkins" - git config user.email "jenkins@thedevops.dev" - - sed -i 's|image: .*|image: ${env.TARGET_IMAGE}|' apps/demo-nginx/deployment.yaml - git add apps/demo-nginx/deployment.yaml - git commit -m "rollback(demo-nginx): Manual rollback to ${env.TARGET_VERSION_CLEAN}" || echo "No changes" - git push origin main - """ - } - - } else if (params.ROLLBACK_METHOD == 'REVISION_NUMBER') { - sh """ - kubectl rollout undo deployment/${APP_NAME} \ - -n ${NAMESPACE} \ - --to-revision=${env.REVISION} - """ - - } else if (params.ROLLBACK_METHOD == 'GIT_COMMIT') { - withCredentials([usernamePassword( - credentialsId: 'gitea-credentials', - usernameVariable: 'GIT_USER', - passwordVariable: 'GIT_PASS' - )]) { - sh """ - rm -rf k3s-gitops || true - git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git - cd k3s-gitops - git config user.name "Jenkins" - git config user.email "jenkins@thedevops.dev" - - git checkout ${env.GIT_SHA} -- apps/demo-nginx/deployment.yaml - TARGET_IMAGE=\$(grep 'image:' apps/demo-nginx/deployment.yaml | awk '{print \$2}') - - git checkout main - git checkout ${env.GIT_SHA} -- apps/demo-nginx/deployment.yaml - git add apps/demo-nginx/deployment.yaml - git commit -m "rollback(demo-nginx): Rollback to commit ${env.GIT_SHA}" || echo "No changes" - git push origin main - - echo "Rolled back to image: \${TARGET_IMAGE}" - """ - } - } - - echo "✅ Rollback command executed" - } - } - } - - stage('Wait for ArgoCD Sync') { - when { - expression { !params.DRY_RUN } - } - steps { - script { - echo "⏳ Waiting for ArgoCD to sync..." - - def syncSuccess = false - def attempts = 0 - def maxAttempts = Integer.parseInt(env.ARGOCD_SYNC_TIMEOUT) / 10 - - while (!syncSuccess && attempts < maxAttempts) { - attempts++ - echo "ArgoCD sync check attempt ${attempts}/${maxAttempts}..." - - def syncStatus = sh( - script: """ - kubectl get application ${APP_NAME} -n argocd \ - -o jsonpath='{.status.sync.status}' - """, - returnStdout: true - ).trim() - - def currentImage = sh( - script: """ - kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ - -o jsonpath='{.spec.template.spec.containers[0].image}' - """, - returnStdout: true - ).trim() - - echo "ArgoCD sync status: ${syncStatus}" - echo "Current deployment image: ${currentImage}" - echo "Expected image: ${env.TARGET_IMAGE}" - - if (syncStatus == 'Synced' && currentImage.contains(env.TARGET_VERSION_CLEAN)) { - syncSuccess = true - echo "✅ ArgoCD synced successfully!" - break - } - - if (attempts < maxAttempts) { - echo "Waiting 10 seconds before next check..." - sleep 10 - } - } - - if (!syncSuccess) { - echo "⚠️ ArgoCD sync check timeout (non-critical for rollback)" - } - } - } - } - - stage('Wait for Rollout') { - when { - expression { !params.DRY_RUN } - } - steps { - script { - echo "⏳ Waiting for rollout to complete..." - - sh """ - kubectl rollout status deployment/${APP_NAME} \ - -n ${NAMESPACE} \ - --timeout=${HEALTH_CHECK_TIMEOUT} - """ - - echo "✅ Rollout completed" - } - } - } - - stage('Verify Deployment') { - when { - expression { !params.DRY_RUN } - } - steps { - script { - echo "✅ Verifying deployment..." - - sh """#!/bin/bash - set -e - - # Check pods ready - READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}') - DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}') - - echo "Ready pods: \${READY_PODS}/\${DESIRED_PODS}" - - if [ "\${READY_PODS}" != "\${DESIRED_PODS}" ]; then - echo "❌ Not all pods are ready!" - exit 1 - fi - - # Verify image - DEPLOYED_IMAGE=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}') - echo "Deployed image: \${DEPLOYED_IMAGE}" - - echo "✅ Deployment verified!" - """ - } - } - } - - stage('Show New State') { - when { - expression { !params.DRY_RUN } - } - steps { - script { - echo "📊 New Deployment State:" - - sh """ - echo "=== New Deployment ===" - kubectl get deployment ${APP_NAME} -n ${NAMESPACE} - - echo "" - echo "=== New Image ===" - kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ - -o jsonpath='{.spec.template.spec.containers[0].image}' - echo "" - - echo "" - echo "=== New Pods ===" - kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} - """ - } - } - } - - stage('Dry Run Summary') { - when { - expression { params.DRY_RUN } - } - steps { - script { - echo """ - 🔍 DRY RUN SUMMARY - - This is what would happen: - - Method: ${params.ROLLBACK_METHOD} - Target: ${env.TARGET_VERSION_CLEAN} - Rebuild Image: ${params.REBUILD_IMAGE} - - Steps: - 1. ${params.REBUILD_IMAGE ? 'Rebuild image with rollback design' : 'Use existing image'} - 2. Update deployment - 3. Update Git manifests - 4. Wait for ArgoCD sync - 5. Wait for rollout - 6. Verify deployment - - No actual changes were made. - """ - } - } - } - } - - post { - success { - script { - if (params.DRY_RUN) { - echo "✅ DRY RUN COMPLETED - No changes made" - } else { - echo """ - ✅ ROLLBACK SUCCESSFUL! - - Application: ${APP_NAME} - Method: ${params.ROLLBACK_METHOD} - Target Version: ${env.TARGET_VERSION_CLEAN} - ${params.REBUILD_IMAGE ? 'Image: Rebuilt with updated design ✨' : 'Image: Using existing'} - Namespace: ${NAMESPACE} - - The application has been rolled back successfully! 🔄 - - Check: https://demo-nginx.thedevops.dev - """ - } - } - } - - failure { - echo """ - ❌ ROLLBACK FAILED! - - Please check the logs and try again. - - Manual rollback: - kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE} - """ - } - - always { - sh """ - docker rmi ${env.TARGET_IMAGE} 2>/dev/null || true - """ - cleanWs() - } - } -} diff --git a/apps/demo-nginx/application.yaml b/apps/demo-nginx/application.yaml deleted file mode 100644 index daf5edb..0000000 --- a/apps/demo-nginx/application.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: demo-nginx - namespace: argocd - labels: - app: demo-nginx -spec: - project: default - source: - repoURL: http://gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops - targetRevision: HEAD - path: apps/demo-nginx - destination: - server: https://kubernetes.default.svc - namespace: demo-app - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true diff --git a/apps/demo-nginx/deployment.yaml b/apps/demo-nginx/deployment.yaml deleted file mode 100644 index 1387237..0000000 --- a/apps/demo-nginx/deployment.yaml +++ /dev/null @@ -1,62 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: demo-nginx - namespace: demo-app - labels: - app: demo-nginx -spec: - replicas: 5 - selector: - matchLabels: - app: demo-nginx - template: - metadata: - labels: - app: demo-nginx - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "80" - spec: - containers: - - name: nginx - image: docker.io/vladcrypto/demo-nginx:main-50 - ports: - - containerPort: 80 - name: http - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 200m - memory: 256Mi - livenessProbe: - httpGet: - path: / - port: 80 - initialDelaySeconds: 10 - periodSeconds: 10 - readinessProbe: - httpGet: - path: / - port: 80 - initialDelaySeconds: 5 - periodSeconds: 5 ---- -apiVersion: v1 -kind: Service -metadata: - name: demo-nginx - namespace: demo-app - labels: - app: demo-nginx -spec: - type: ClusterIP - selector: - app: demo-nginx - ports: - - port: 80 - targetPort: 80 - protocol: TCP - name: http diff --git a/apps/demo-nginx/docs/Argocd_sync_issue.md b/apps/demo-nginx/docs/Argocd_sync_issue.md deleted file mode 100644 index 5b51d54..0000000 --- a/apps/demo-nginx/docs/Argocd_sync_issue.md +++ /dev/null @@ -1,419 +0,0 @@ -# 🔧 ArgoCD Sync vs Actual Deployment Issue - -## 🐛 The Problem - -**Symptom:** -- ArgoCD shows `Synced` ✅ -- Deployment manifest in Kubernetes is updated ✅ -- **BUT** pods are still running old image ❌ - -**Why This Happens:** - -``` -┌─────────────────────────────────────────────────────────────┐ -│ Git Repository │ -│ deployment.yaml: image: app:v2 ✅ │ -└──────────────────┬──────────────────────────────────────────┘ - │ - │ ArgoCD syncs - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ Kubernetes API (Deployment Object) │ -│ spec.template.image: app:v2 ✅ │ -└──────────────────┬──────────────────────────────────────────┘ - │ - │ Kubernetes Controller should trigger rollout - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ Running Pods │ -│ Pod-1: image: app:v1 ❌ (OLD!) │ -│ Pod-2: image: app:v1 ❌ (OLD!) │ -└─────────────────────────────────────────────────────────────┘ - -ArgoCD says "Synced" because Git == Kubernetes manifest ✅ -But pods haven't rolled out yet! ❌ -``` - ---- - -## 🔍 Why ArgoCD Says "Synced" - -ArgoCD checks: -1. ✅ Git manifest == Kubernetes Deployment object -2. ✅ Health status (from status fields) - -ArgoCD **DOES NOT** check: -- ❌ Are pods actually running? -- ❌ What image are pods using? -- ❌ Did rollout complete? - -**ArgoCD's job:** Keep Kubernetes resources in sync with Git -**NOT ArgoCD's job:** Wait for pods to finish rolling out - ---- - -## ⚠️ When This Happens - -### Scenario 1: Slow Rollout -``` -14:00:00 - ArgoCD syncs deployment (v1 → v2) -14:00:05 - ArgoCD: "Synced!" ✅ -14:00:10 - Kubernetes starts rollout -14:00:30 - Pod-1 terminates (v1) -14:00:35 - Pod-3 starts (v2) -14:00:50 - Pod-2 terminates (v1) -14:00:55 - Pod-4 starts (v2) -14:01:00 - Rollout complete! ✅ - -Jenkins checks at 14:00:05: ArgoCD says "Synced" -But pods are still v1! ❌ -``` - -### Scenario 2: Image Pull Delay -``` -14:00:00 - ArgoCD syncs -14:00:05 - ArgoCD: "Synced!" ✅ -14:00:10 - Kubernetes tries to start new pod -14:00:15 - Pulling image... (slow network) -14:00:45 - Image pulled -14:00:50 - Pod starts -14:01:00 - Pod ready - -Jenkins checks at 14:00:05: "Synced" but no new pods yet! -``` - -### Scenario 3: Resource Constraints -``` -14:00:00 - ArgoCD syncs -14:00:05 - ArgoCD: "Synced!" ✅ -14:00:10 - Kubernetes: "No resources available" -14:00:20 - Kubernetes: "Waiting for node capacity..." -14:01:00 - Old pod terminates, resources freed -14:01:10 - New pod starts - -Jenkins checks at 14:00:05: "Synced" but can't schedule pods! -``` - ---- - -## ✅ The Solution - -### What Jenkins Must Check: - -```groovy -// ❌ BAD - Only checks ArgoCD -if (argocdStatus == 'Synced') { - echo "Done!" -} - -// ✅ GOOD - Checks ArgoCD + Kubernetes -if (argocdStatus == 'Synced') { - // 1. Wait for rollout - kubectl rollout status deployment/app - - // 2. Verify actual pod images - podImages = kubectl get pods -o jsonpath='{.status.containerStatuses[0].image}' - if (podImages contains newVersion) { - echo "Verified!" - } -} -``` - ---- - -## 🎯 New Jenkinsfile Verification - -### Stage 1: ArgoCD Sync Check -```groovy -stage('Wait for ArgoCD Sync') { - // Checks: - // 1. ArgoCD sync status = "Synced" - // 2. Deployment SPEC image updated - // - // Does NOT check if pods rolled out! - // That's the next stage. -} -``` - -**Output:** -``` -ArgoCD sync status: Synced -Deployment spec image: app:v2 -✅ ArgoCD synced and deployment spec updated! -Note: Pods may still be rolling out - will verify in next stage -``` - -### Stage 2: Wait for Rollout -```groovy -stage('Wait for Deployment') { - // Uses kubectl rollout status - // Waits for actual pod rollout to complete - sh "kubectl rollout status deployment/app --timeout=5m" -} -``` - -**What `kubectl rollout status` does:** -- Watches deployment progress -- Waits for all new pods to be ready -- Returns when rollout complete -- Times out if rollout stuck - -**Output:** -``` -Waiting for deployment "app" rollout to finish: 1 out of 2 new replicas have been updated... -Waiting for deployment "app" rollout to finish: 1 old replicas are pending termination... -deployment "app" successfully rolled out -✅ Rollout completed successfully! -``` - -### Stage 3: Verify Actual Pods -```groovy -stage('Verify Deployment') { - // CRITICAL CHECKS: - - // 1. Deployment status - readyReplicas == desiredReplicas - - // 2. Deployment spec image - deploymentImage contains newTag - - // 3. ACTUAL POD IMAGES (most important!) - podImages = all pods images - for each podImage: - if podImage does not contain newTag: - FAIL! - - // 4. Pod health - all pods in Running state - - // 5. Restart count - check for crash loops -} -``` - -**Output:** -``` -================================================ -DEPLOYMENT VERIFICATION -================================================ - -1. Checking deployment status... - Desired replicas: 2 - Updated replicas: 2 - Ready replicas: 2 - Available replicas: 2 - ✅ All pods ready - -2. Checking deployment spec image... - Deployment spec image: app:v2 - Expected tag: v2 - ✅ Deployment spec correct - -3. Checking actual running pod images... - Running pod images: - - app:v2 - - app:v2 - ✅ All pods running correct image - -4. Checking pod readiness probes... - ✅ All pods in Running state - -5. Checking for container restarts... - Max restart count: 0 - ✅ Restart count acceptable - -================================================ -✅ ALL VERIFICATION CHECKS PASSED! -================================================ -``` - ---- - -## 🔥 What Happens If Check #3 Fails - -``` -3. Checking actual running pod images... - Running pod images: - - app:v1 ❌ - - app:v1 ❌ - ❌ Pod running wrong image: app:v1 - ❌ FAILED: 2 pod(s) running old image! - This is the ArgoCD sync bug - deployment updated but pods not rolled out -``` - -**Jenkins will:** -1. ❌ Mark build as failed -2. 🔄 Trigger rollback (if enabled) -3. 📱 Send notification with details - ---- - -## 🧪 Testing the Fix - -### Test 1: Normal Deployment -```bash -# Update image in Git -git commit -m "Update to v2" -git push - -# Jenkins should: -# 1. Wait for ArgoCD sync ✅ -# 2. Wait for rollout ✅ -# 3. Verify pods have v2 ✅ -# 4. Success! ✅ -``` - -### Test 2: Slow Rollout -```bash -# Set slow rollout -kubectl patch deployment app -p '{"spec":{"strategy":{"rollingUpdate":{"maxUnavailable":0,"maxSurge":1}}}}' - -# Update image -git push - -# Jenkins should: -# 1. ArgoCD syncs quickly ✅ -# 2. Wait for slow rollout (may take 2-3 minutes) ⏳ -# 3. Verify when complete ✅ -``` - -### Test 3: Rollout Stuck -```bash -# Create a broken image tag -# Update to image: app:nonexistent - -git push - -# Jenkins should: -# 1. ArgoCD syncs ✅ -# 2. kubectl rollout status times out ❌ -# 3. Rollback triggered ✅ -``` - ---- - -## 📊 Comparison: Old vs New - -### Old Pipeline (Unreliable) -``` -1. ArgoCD sync check - ├─ Checks: ArgoCD status - ├─ Checks: Deployment spec image - └─ Duration: ~30 seconds - - ⚠️ PROBLEM: Pods might not have rolled out! - -2. Success! ✅ (but pods are still old!) -``` - -### New Pipeline (Reliable) -``` -1. ArgoCD sync check - ├─ Checks: ArgoCD status - ├─ Checks: Deployment spec image - └─ Duration: ~30 seconds - -2. Rollout status check - ├─ Checks: kubectl rollout status - ├─ Waits: For actual pod rollout - └─ Duration: ~1-2 minutes - -3. Verification - ├─ Checks: Deployment status - ├─ Checks: ACTUAL pod images ← KEY! - ├─ Checks: Pod health - ├─ Checks: Restart count - └─ Duration: ~10 seconds - -4. Success! ✅ (pods verified running new version) -``` - ---- - -## 🎯 Key Takeaways - -### ❌ Don't Trust: -- ArgoCD "Synced" status alone -- Deployment spec image alone -- Health status alone - -### ✅ Always Verify: -1. **ArgoCD synced** (manifest applied) -2. **Rollout completed** (`kubectl rollout status`) -3. **Actual pod images** (what's really running) -4. **Pod health** (ready and not crashing) - -### 💡 Remember: -``` -ArgoCD "Synced" = Git matches Kubernetes manifest ✅ -BUT -Kubernetes manifest != Running pods ⚠️ - -You MUST check actual pods! -``` - ---- - -## 🔗 Related Issues - -- [ArgoCD #2723](https://github.com/argoproj/argo-cd/issues/2723) - "Synced but pods not updated" -- [Kubernetes #93033](https://github.com/kubernetes/kubernetes/issues/93033) - "Deployment rollout delays" - ---- - -## 🚀 Using the New Jenkinsfile - -```bash -# 1. Update Jenkinsfile in your repo -cp Jenkinsfile.telegram.en apps/demo-nginx/Jenkinsfile - -# 2. Commit and push -git add apps/demo-nginx/Jenkinsfile -git commit -m "fix: add proper deployment verification" -git push - -# 3. Run build -# Jenkins will now properly verify deployments! -``` - ---- - -## 📱 Notifications - -With the new verification, you'll see: - -**During deployment:** -``` -⏳ ArgoCD Syncing -Application: demo-nginx -Timeout: 120s - -🚀 Deploying to Kubernetes -Deployment: demo-nginx -Image: main-42 -Rolling out new pods... -``` - -**On success:** -``` -✅ Deployment Successful! - -Verified: -- ArgoCD synced ✅ -- Rollout completed ✅ -- Pods running v42 ✅ -- All pods healthy ✅ -``` - -**On failure:** -``` -❌ Deployment Failed - -Error: 2 pods running old image! - -Rollback initiated... -``` - ---- - -**This fix ensures Jenkins never reports success until pods are actually running the new version!** ✅ \ No newline at end of file diff --git a/apps/demo-nginx/docs/CICD_GUIDE.md b/apps/demo-nginx/docs/CICD_GUIDE.md deleted file mode 100644 index e9abeff..0000000 --- a/apps/demo-nginx/docs/CICD_GUIDE.md +++ /dev/null @@ -1,148 +0,0 @@ -# 🎉 CI/CD Pipeline - Complete Guide - -## ✅ Что мы построили - Production Ready System! - -Полноценный CI/CD pipeline с автоматическим деплоем от Git до Production. - ---- - -## 📦 Компоненты системы - -### 1. Jenkins (CI/CD Engine) -``` -URL: http://jenkins.thedevops.dev -Namespace: jenkins - -Установлено: -✅ Docker CLI (для build & push) -✅ kubectl v1.28.0 (для deploy verification) -✅ ServiceAccount + RBAC (для K8s API) -✅ Auto-scan (проверяет Git каждый час) -``` - -### 2. Gitea (Git Repository) -``` -URL: http://gitea.thedevops.dev -Repository: k3s-gitops - -Содержит: -✅ apps/demo-nginx/ - все манифесты приложения -✅ apps/jenkins/ - конфигурация Jenkins -✅ Jenkinsfile - CI/CD pipeline -✅ Webhook настроен (опционально) -``` - -### 3. ArgoCD (GitOps) -``` -URL: https://argocd.thedevops.dev -Application: demo-nginx - -Функции: -✅ Автоматически синхронизирует Git → K8s -✅ Отслеживает изменения в deployment.yaml -✅ Применяет rolling updates -✅ Self-heal & auto-prune -``` - -### 4. Docker Hub (Registry) -``` -Repository: docker.io/vladcrypto/demo-nginx - -Images: -✅ Автоматически пушатся из Jenkins -✅ Теги: main-XX, latest -``` - -### 5. Kubernetes (Runtime) -``` -Namespace: demo-app -Deployment: demo-nginx (2 replicas) -Service: demo-nginx (ClusterIP) -Ingress: https://demo-nginx.thedevops.dev - -Status: -✅ 2/2 pods Running -✅ Rolling updates enabled -✅ Health checks configured -``` - ---- - -## 🔄 Как работает Pipeline - -### Автоматический flow: - -``` -1. Developer → git push - ↓ -2. Gitea → сохраняет commit - ↓ -3. Jenkins → Auto-scan обнаруживает изменения (каждый час) - ↓ -4. Jenkins Pipeline (Jenkinsfile): - ├─ Stage 1: Checkout Source (создает Dockerfile + nginx.conf) - ├─ Stage 2: Build Docker Image (main-XX) - ├─ Stage 3: Push to Registry (Docker Hub) - ├─ Stage 4: Update GitOps Manifests (deployment.yaml) - └─ Stage 5: Verify Deployment (kubectl rollout status) - ↓ -5. ArgoCD → Обнаруживает изменения в Git (каждые 3 мин) - ↓ -6. ArgoCD → Синхронизирует deployment.yaml → Kubernetes - ↓ -7. Kubernetes → Rolling update (zero downtime) - ↓ -8. ✅ Production! Новая версия работает! -``` - -**Время:** 3-5 минут от commit до production - ---- - -## 🧪 Как проверять - -### 1. Проверить Jenkins - -```bash -# Pods -kubectl get pods -n jenkins - -# Logs -kubectl logs -n jenkins - -# UI -http://jenkins.thedevops.dev -``` - -**Что смотреть:** -- Build History (список сборок) -- Console Output (логи последней сборки) -- Blue Ocean (красивый UI) - ---- - -### 2. Проверить Demo App - -```bash -# Deployment -kubectl get deployment demo-nginx -n demo-app - -# Pods -kubectl get pods -n demo-app - -# Service -kubectl get svc demo-nginx -n demo-app - -# Ingress -kubectl get ingress demo-nginx -n demo-app -``` - -**Ожидаемый результат:** -``` -DEPLOYMENT: 2/2 Ready -PODS: 2/2 Running -SERVICE: ClusterIP -INGRESS: demo-nginx.thedevops.dev -``` - -См. полную документацию в файле для всех деталей, сценариев использования, troubleshooting и best practices. diff --git a/apps/demo-nginx/docs/Jenkinsfile b/apps/demo-nginx/docs/Jenkinsfile deleted file mode 100644 index 0693adc..0000000 --- a/apps/demo-nginx/docs/Jenkinsfile +++ /dev/null @@ -1,166 +0,0 @@ -pipeline { // Declarative Jenkins pipeline start - agent any // Run on any available Jenkins agent - - environment { // Global environment variables - APP_NAME = 'demo-nginx' // Application name (Docker, K8s, logs) - NAMESPACE = 'demo-app' // Kubernetes namespace - DOCKER_REGISTRY = 'docker.io' // Docker registry hostname - DOCKER_REPO = 'vladcrypto' // Docker Hub repository / namespace - GITEA_URL = 'http://gitea-http.gitea.svc.cluster.local:3000' // Internal Gitea URL - GITEA_REPO = 'admin/k3s-gitops' // GitOps repository path - GITEA_BRANCH = 'main' // Git branch for GitOps updates - BUILD_TAG = "${env.BUILD_NUMBER}" // Jenkins build number - IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}" // Image version tag - } - - stages { // Pipeline stages definition - - stage('Checkout Source') { // Stage: prepare application source - steps { // Steps executed in this stage - echo "Checking out application source code..." // Log message - - sh ''' // Execute shell script -cat > Dockerfile << 'EOF' # Create Dockerfile -FROM nginx:1.25.3-alpine # Base Nginx Alpine image -RUN echo "

Demo Nginx - Build ${BUILD_NUMBER}

Environment: Production

Version: ${IMAGE_TAG}

" > /usr/share/nginx/html/index.html # Inject build metadata into HTML -COPY nginx.conf /etc/nginx/nginx.conf # Copy custom nginx configuration -EXPOSE 80 # Expose HTTP port -CMD ["nginx", "-g", "daemon off;"] # Run nginx in foreground -EOF - ''' - - sh ''' // Generate nginx configuration -cat > nginx.conf << 'EOF' # Create nginx.conf -user nginx; # Run workers as nginx user -worker_processes auto; # Scale workers to CPU cores -error_log /var/log/nginx/error.log warn; # Error log level and path -pid /var/run/nginx.pid; # PID file location -events { worker_connections 1024; } # Max connections per worker -http { # HTTP configuration block - include /etc/nginx/mime.types; # Load MIME types - default_type application/octet-stream; # Default content type - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; # Access log format - access_log /var/log/nginx/access.log main; # Enable access logging - sendfile on; # Enable zero-copy file transfer - keepalive_timeout 65; # Keepalive timeout - server { # Server definition - listen 80; # Listen on port 80 - server_name _; # Catch-all server - location / { # Root location - root /usr/share/nginx/html; # Serve static files - index index.html; # Default index file - } - location /health { # Health check endpoint - access_log off; # Disable logging for health checks - return 200 "healthy\n"; # Always return HTTP 200 - add_header Content-Type text/plain; # Explicit content type - } - } -} -EOF - ''' - } - } - - stage('Build Docker Image') { // Stage: build Docker image - steps { - script { // Scripted block - echo "Building Docker image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}" // Log image name - sh """ // Execute Docker build - docker build \ - -t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} \ // Versioned image - -t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest \ // Latest tag - . // Build context - """ - echo "✅ Image built successfully!" // Success message - } - } - } - - stage('Push to Registry') { // Stage: push image to registry - when { branch 'main' } // Execute only on main branch - steps { - script { - echo "Pushing image to registry..." // Log push start - withCredentials([usernamePassword( // Inject Docker credentials - credentialsId: 'docker-registry-credentials', - usernameVariable: 'DOCKER_USER', - passwordVariable: 'DOCKER_PASS' - )]) { - sh """ // Authenticate and push image - echo "\${DOCKER_PASS}" | docker login ${DOCKER_REGISTRY} -u "\${DOCKER_USER}" --password-stdin // Login securely - docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} // Push versioned image - docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest // Push latest image - docker logout ${DOCKER_REGISTRY} // Logout - """ - } - echo "✅ Image pushed successfully!" // Push success - } - } - } - - stage('Update GitOps Manifests') { // Stage: update GitOps repository - when { branch 'main' } // Only from main branch - steps { - script { - echo "Updating Kubernetes manifests..." // Log GitOps update - withCredentials([usernamePassword( // Inject Gitea credentials - credentialsId: 'gitea-credentials', - usernameVariable: 'GIT_USER', - passwordVariable: 'GIT_PASS' - )]) { - sh """ // Clone and update manifests - rm -rf k3s-gitops || true // Remove old repo - git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git // Clone repo - cd k3s-gitops // Enter repo - git config user.name "Jenkins" // Git author name - git config user.email "jenkins@thedevops.dev" // Git author email - sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml // Update image - git add apps/demo-nginx/deployment.yaml // Stage change - git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes" // Commit if needed - git push origin main // Push to main - """ - } - echo "✅ Manifests updated!" // GitOps success - } - } - } - - stage('Verify Deployment') { // Stage: verify Kubernetes deployment - when { branch 'main' } // Only on main - steps { - script { - echo "Verifying deployment..." // Log verification start - sh """ // Check deployment status - sleep 30 // Wait for rollout start - kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=300s || true // Rollout status - kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} // List pods - """ - echo "✅ Deployment completed!" // Verification success - } - } - } - } - - post { // Post-pipeline actions - success { // On success - echo """ // Success summary - ✅ Pipeline SUCCESS! - Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} - Namespace: ${NAMESPACE} - """ - } - failure { // On failure - echo "❌ Pipeline failed!" // Failure message - } - always { // Always execute cleanup - sh """ // Cleanup Docker artifacts - docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true // Remove versioned image - docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest || true // Remove latest image - docker stop test-${BUILD_NUMBER} 2>/dev/null || true // Stop test container - docker rm test-${BUILD_NUMBER} 2>/dev/null || true // Remove test container - """ - cleanWs() // Clean Jenkins workspace - } - } -} diff --git a/apps/demo-nginx/docs/README.md b/apps/demo-nginx/docs/README.md deleted file mode 100644 index 21c34de..0000000 --- a/apps/demo-nginx/docs/README.md +++ /dev/null @@ -1,235 +0,0 @@ -# Demo Nginx Documentation - -Документация для demo-nginx deployment с автоматическим и ручным rollback. - ---- - -## 📚 Available Documentation - -### [ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md) -**Comprehensive Manual Rollback Guide** - -Полная документация по функции ручного rollback: -- 3 способа rollback (IMAGE_TAG, REVISION, GIT_COMMIT) -- Setup guide -- Troubleshooting со всеми fixes -- Best practices -- Examples -- FAQ - -**When to use:** Нужна детальная информация или troubleshooting - ---- - -### [ROLLBACK_QUICK_REF.md](./ROLLBACK_QUICK_REF.md) -**Quick Reference Card** - -Краткая справка для быстрого использования: -- Quick start (2 минуты) -- 3 способа rollback -- Emergency procedure -- Verification commands -- Checklist - -**When to use:** Быстрый rollback в production - ---- - -## 🎯 Quick Links - -### Rollback Feature -- **Full Guide:** [ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md) -- **Quick Ref:** [ROLLBACK_QUICK_REF.md](./ROLLBACK_QUICK_REF.md) -- **Jenkinsfile:** [Jenkinsfile.rollback](../Jenkinsfile.rollback) -- **CI/CD Guide:** [../../../CICD_GUIDE.md](../../../CICD_GUIDE.md) - -### Related Resources -- **Jenkins RBAC:** [apps/jenkins/rbac.yaml](../../jenkins/rbac.yaml) -- **Deployment:** [deployment.yaml](../deployment.yaml) -- **Main CI/CD:** [Jenkinsfile](../Jenkinsfile) - ---- - -## 🚀 Quick Start - -### Manual Rollback (2 minutes) - -``` -1. Jenkins → demo-nginx-rollback -2. IMAGE_TAG + main-21 -3. SKIP_HEALTH_CHECK: true -4. Build -``` - -See [ROLLBACK_QUICK_REF.md](./ROLLBACK_QUICK_REF.md) for details. - ---- - -### Emergency Rollback (30 seconds) - -```bash -kubectl rollout undo deployment/demo-nginx -n demo-app -``` - ---- - -## 📊 Features Summary - -### Manual Rollback -✅ 3 rollback methods (IMAGE_TAG, REVISION, GIT_COMMIT) -✅ GitOps sync (auto-commit to Git) -✅ Zero downtime (rolling updates) -✅ DRY_RUN mode (safe testing) -✅ Full RBAC permissions -✅ Input validation (auto-trim) -✅ Retry logic (5 attempts) -⚠️ Health check optional (use SKIP_HEALTH_CHECK=true) - -### Automatic Rollback -✅ Triggered on deployment failure -✅ Saves previous state -✅ Kubernetes rollback -✅ Git revert -✅ Health checks -✅ Timeout protection - ---- - -## 🐛 All Fixes Applied - -| # | Issue | Fix | Status | Doc | -|---|-------|-----|--------|-----| -| 1 | Container name | Use `nginx` | ✅ | [Link](./ROLLBACK_MANUAL.md#issue-1-container-name-error--fixed) | -| 2 | Whitespace | Auto-trim | ✅ | [Link](./ROLLBACK_MANUAL.md#issue-2-whitespace-in-input--fixed) | -| 3 | RBAC | pods/exec perm | ✅ | [Link](./ROLLBACK_MANUAL.md#issue-3-rbac-permissions--fixed) | -| 4 | Health timing | SKIP option | ⚠️ | [Link](./ROLLBACK_MANUAL.md#issue-4-health-check-timing--workaround) | -| 5 | Bash loop | Explicit list | ✅ | [Link](./ROLLBACK_MANUAL.md#issue-5-bash-loop-syntax--fixed) | - ---- - -## 💡 Recommended Reading Order - -### For New Users: -1. Start → [ROLLBACK_QUICK_REF.md](./ROLLBACK_QUICK_REF.md) -2. Practice → Follow quick start -3. Deep Dive → [ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md) - -### For Troubleshooting: -1. Check → [ROLLBACK_MANUAL.md - Troubleshooting](./ROLLBACK_MANUAL.md#troubleshooting--fixes) -2. Verify → [ROLLBACK_QUICK_REF.md - Verification](./ROLLBACK_QUICK_REF.md#-verify-rollback) -3. Support → [ROLLBACK_MANUAL.md - Support](./ROLLBACK_MANUAL.md#support) - -### For Emergency: -1. Fast → [ROLLBACK_QUICK_REF.md - Emergency](./ROLLBACK_QUICK_REF.md#-emergency-rollback-30-seconds) -2. Alternative → [ROLLBACK_MANUAL.md - Emergency](./ROLLBACK_MANUAL.md#emergency-rollback-procedure) - ---- - -## 🎓 Key Concepts - -### Rollback Methods Comparison - -| Method | Speed | Precision | Use Case | -|--------|-------|-----------|----------| -| IMAGE_TAG | ⚡⚡⚡ | 🎯 High | Known build number | -| REVISION | ⚡⚡ | 🎯 Medium | Recent rollback | -| GIT_COMMIT | ⚡ | 🎯🎯 High | Exact code state | - -### When to Use What - -**Use IMAGE_TAG when:** -- You know the build number (main-21) -- Quick rollback needed -- Most common scenario - -**Use REVISION_NUMBER when:** -- Need to go back N versions -- Don't remember exact tag -- Working with kubectl history - -**Use GIT_COMMIT when:** -- Need exact code state -- Multiple changes in one build -- Precise rollback required - ---- - -## 📈 Monitoring - -### Check Rollback Status - -```bash -# Deployment status -kubectl get deployment demo-nginx -n demo-app - -# Pod status -kubectl get pods -n demo-app -l app=demo-nginx - -# Rollout history -kubectl rollout history deployment/demo-nginx -n demo-app - -# ArgoCD status -kubectl get application demo-nginx -n argocd -``` - -### Grafana Queries - -```promql -# Rollback count -sum(increase(deployment_rollback_total[1h])) by (deployment) - -# Rollback rate -rate(deployment_rollback_total[5m]) -``` - ---- - -## ❓ FAQ - -### Q: Какой метод rollback использовать? -**A:** Для большинства случаев используй IMAGE_TAG - самый быстрый и простой. - -### Q: Health check всегда падает, это баг? -**A:** Нет, это timing issue во время rolling update. Используй `SKIP_HEALTH_CHECK: true` и проверь вручную через минуту. - -### Q: Как быстро откатиться в emergency? -**A:** Используй `kubectl rollout undo` (30 секунд) или Jenkins с SKIP_HEALTH_CHECK (2 минуты). - -### Q: Где полная документация? -**A:** [ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md) - comprehensive guide со всеми details. - ---- - -## 🆘 Support - -**Need Help?** - -1. Check [ROLLBACK_MANUAL.md - Troubleshooting](./ROLLBACK_MANUAL.md#troubleshooting--fixes) -2. Review [ROLLBACK_MANUAL.md - FAQ](./ROLLBACK_MANUAL.md#faq) -3. Check Jenkins console output -4. Verify RBAC permissions -5. Review pod status and logs - -**Still stuck?** -- Jenkins logs: Jenkins → Build → Console Output -- K8s events: `kubectl get events -n demo-app` -- Pod logs: `kubectl logs -n demo-app -l app=demo-nginx` - ---- - -## 📝 Documentation Updates - -**Last Updated:** 2026-01-06 -**Version:** 1.0 -**Status:** Production Ready ✅ - -**Recent Changes:** -- ✅ Added comprehensive manual rollback guide -- ✅ Added quick reference card -- ✅ Documented all 5 fixes -- ✅ Added examples and best practices -- ✅ Production-ready feature - ---- - -**Ready to rollback? Start with [Quick Reference](./ROLLBACK_QUICK_REF.md)! 🚀** diff --git a/apps/demo-nginx/docs/ROLLBACK_MANUAL.md b/apps/demo-nginx/docs/ROLLBACK_MANUAL.md deleted file mode 100644 index c50207c..0000000 --- a/apps/demo-nginx/docs/ROLLBACK_MANUAL.md +++ /dev/null @@ -1,717 +0,0 @@ -# 🔄 Manual Rollback Feature - Complete Documentation - -## 📋 Table of Contents - -1. [Overview](#overview) -2. [Features](#features) -3. [Setup Guide](#setup-guide) -4. [Usage Guide](#usage-guide) -5. [Rollback Methods](#rollback-methods) -6. [Troubleshooting & Fixes](#troubleshooting--fixes) -7. [Best Practices](#best-practices) -8. [Examples](#examples) - ---- - -## Overview - -Manual Rollback feature позволяет откатить deployment на любую предыдущую версию через Jenkins Pipeline. - -### Key Features: -- ✅ **3 способа rollback** (IMAGE_TAG, REVISION_NUMBER, GIT_COMMIT) -- ✅ **GitOps sync** - автоматически обновляет Git manifests -- ✅ **Zero downtime** - rolling updates -- ✅ **DRY_RUN mode** - безопасное тестирование -- ✅ **Health checks** - опциональная проверка после rollback -- ✅ **Full RBAC** - правильные permissions - ---- - -## Features - -### Rollback Methods - -| Method | Description | Example | Use Case | -|--------|-------------|---------|----------| -| **IMAGE_TAG** | По Docker image tag | `main-21` | Знаешь конкретный build number | -| **REVISION_NUMBER** | По Kubernetes revision | `2` | Откат на N шагов назад | -| **GIT_COMMIT** | По Git commit SHA | `abc123def` | Точное состояние кода | - -### Parameters - -```groovy -ROLLBACK_METHOD // Выбор метода -TARGET_VERSION // Целевая версия (auto-trim whitespace) -SKIP_HEALTH_CHECK // Пропустить health checks (default: false) -DRY_RUN // Только показать план (default: false) -``` - ---- - -## Setup Guide - -### Step 1: Create Jenkins Pipeline - -``` -1. Jenkins → New Item -2. Name: demo-nginx-rollback -3. Type: Pipeline -4. Click OK -``` - -### Step 2: Configure Pipeline - -```yaml -Pipeline: - Definition: Pipeline script from SCM - SCM: Git - Repository URL: http://gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops - Credentials: gitea-credentials - Branch: */main - Script Path: apps/demo-nginx/Jenkinsfile.rollback -``` - -### Step 3: Verify RBAC - -RBAC уже настроен в `apps/jenkins/rbac.yaml`: - -```yaml -ClusterRole: jenkins-deployer -Permissions: - - pods, services, deployments (full CRUD) - - pods/exec, pods/log (for health checks) - - ingresses, applications (for ArgoCD) -``` - -### Step 4: Test with DRY_RUN - -``` -Jenkins → demo-nginx-rollback → Build with Parameters -├─ ROLLBACK_METHOD: IMAGE_TAG -├─ TARGET_VERSION: main-21 -├─ DRY_RUN: ✅ true -└─ Build -``` - ---- - -## Usage Guide - -### Quick Start - -``` -Jenkins → demo-nginx-rollback → Build with Parameters - -┌─────────────────────────────────────┐ -│ ROLLBACK_METHOD: IMAGE_TAG │ -│ TARGET_VERSION: main-21 │ -│ SKIP_HEALTH_CHECK: true (рекоменд.) │ -│ DRY_RUN: false │ -└─────────────────────────────────────┘ - -→ Build → ✅ SUCCESS! -``` - -### Pipeline Stages - -``` -Stage 1: Validate Input - └─ Trim whitespace, validate TARGET_VERSION - -Stage 2: Show Current State - └─ Current deployment, image, pods, history - -Stage 3: Prepare Rollback - └─ Build target image path or verify revision - -Stage 4: Execute Rollback - ├─ kubectl set image (or rollout undo) - └─ Git commit & push - -Stage 5: Wait for Rollout - ├─ kubectl rollout status (300s timeout) - └─ sleep 10s (stabilization) - -Stage 6: Health Check (optional) - └─ 5 retry attempts with 5s delay - -Stage 7: Show New State - └─ New deployment state, pods, history -``` - ---- - -## Rollback Methods - -### Method 1: IMAGE_TAG (Recommended) - -**Когда использовать:** Знаешь конкретный build number - -**Как найти tag:** -```bash -# Docker Hub -https://hub.docker.com/r/vladcrypto/demo-nginx/tags - -# Jenkins build history -Jenkins → demo-nginx → Build History - -# Git commits -git log --oneline | grep "Update image" -``` - -**Example:** -``` -ROLLBACK_METHOD: IMAGE_TAG -TARGET_VERSION: main-21 - -Result: Rollback to docker.io/vladcrypto/demo-nginx:main-21 -``` - ---- - -### Method 2: REVISION_NUMBER - -**Когда использовать:** Нужно откатиться на N шагов назад - -**Как найти revision:** -```bash -kubectl rollout history deployment/demo-nginx -n demo-app - -# Output: -REVISION CHANGE-CAUSE -1 Initial deployment -2 Update to main-20 -3 Update to main-21 -4 Update to main-22 (current) -``` - -**Example:** -``` -ROLLBACK_METHOD: REVISION_NUMBER -TARGET_VERSION: 2 - -Result: Rollback to revision 2 (main-20) -``` - ---- - -### Method 3: GIT_COMMIT - -**Когда использовать:** Нужно вернуться к конкретному состоянию кода - -**Как найти commit:** -```bash -# Gitea -https://git.thedevops.dev/admin/k3s-gitops/commits/branch/main - -# Git CLI -git log --oneline apps/demo-nginx/deployment.yaml - -# Output: -abc123d Update image to main-22 (current) -def456e Update image to main-21 -ghi789f Update image to main-20 -``` - -**Example:** -``` -ROLLBACK_METHOD: GIT_COMMIT -TARGET_VERSION: def456e - -Result: Rollback to commit def456e -``` - ---- - -## Troubleshooting & Fixes - -### Issue #1: Container Name Error ✅ FIXED - -**Error:** -``` -error: unable to find container named "demo-nginx" -``` - -**Root Cause:** -Pipeline использовал deployment name вместо container name. - -**Fix:** -```groovy -environment { - APP_NAME = 'demo-nginx' // Deployment name - CONTAINER_NAME = 'nginx' // Container name ✅ -} - -kubectl set image deployment/${APP_NAME} \ - ${CONTAINER_NAME}=${TARGET_IMAGE} -``` - -**How to verify:** -```bash -kubectl get deployment demo-nginx -n demo-app \ - -o jsonpath='{.spec.template.spec.containers[0].name}' -# Output: nginx -``` - ---- - -### Issue #2: Whitespace in Input ✅ FIXED - -**Error:** -``` -Target image: docker.io/vladcrypto/demo-nginx: main-21 - ^ - Space! -``` - -**Root Cause:** -User ввел TARGET_VERSION с пробелом. - -**Fix:** -```groovy -stage('Validate Input') { - // Auto-trim whitespace - env.TARGET_VERSION_CLEAN = params.TARGET_VERSION.trim() - - // Use everywhere - ${env.TARGET_VERSION_CLEAN} -} -``` - ---- - -### Issue #3: RBAC Permissions ✅ FIXED - -**Error:** -``` -Error: User "system:serviceaccount:jenkins:jenkins" -cannot create resource "pods/exec" -``` - -**Root Cause:** -Jenkins ServiceAccount не имел прав на pods/exec для health checks. - -**Fix:** -```yaml -# apps/jenkins/rbac.yaml -rules: -- apiGroups: [""] - resources: ["pods/exec", "pods/log"] # ← Added! - verbs: ["create", "get"] -``` - -**Applied:** -```bash -kubectl apply -f apps/jenkins/rbac.yaml -``` - ---- - -### Issue #4: Health Check Timing ⚠️ WORKAROUND - -**Error:** -``` -wget: can't connect to remote host: Connection refused -``` - -**Root Cause:** -Health check runs too early during rolling update (race condition). - -**Workaround:** -```groovy -// Option 1: Skip health check (recommended) -SKIP_HEALTH_CHECK: true - -// Option 2: Longer stabilization wait -sleep 30 // Instead of 10 -``` - -**Timeline:** -``` -T+0s: kubectl set image -T+30s: Rollout status = complete -T+40s: sleep 10s -T+50s: Health check (pods might still be starting) -``` - -**Solution:** -Use `SKIP_HEALTH_CHECK: true` и проверь вручную через 30-60s: - -```bash -kubectl get pods -n demo-app -l app=demo-nginx -``` - ---- - -### Issue #5: Bash Loop Syntax ✅ FIXED - -**Error:** -``` -Health check attempt {1..5}/5... -# Loop executed only once! -``` - -**Root Cause:** -`{1..5}` не работает в sh/dash, нужен bash. - -**Fix:** -```bash -#!/bin/bash # ← Added shebang -set -e - -# Fixed loop syntax -for i in 1 2 3 4 5; do # Instead of {1..5} - echo "Health check attempt $i/5..." - if kubectl exec ...; then - exit 0 - fi - if [ $i -lt 5 ]; then - sleep 5 - fi -done -``` - ---- - -## Best Practices - -### 1. Always Use DRY_RUN First - -``` -Step 1: DRY_RUN=true → Проверь план -Step 2: Verify output -Step 3: DRY_RUN=false → Execute -``` - -### 2. Use SKIP_HEALTH_CHECK for Emergency - -``` -Emergency rollback: -├─ SKIP_HEALTH_CHECK: true -├─ Focus on speed -└─ Verify manually after -``` - -### 3. Document Rollback Reason - -Add comment в Jenkins build: -``` -Build Comment: -"Rollback due to: API errors in main-23 -Previous working version: main-21 -Impact: None (zero downtime)" -``` - -### 4. Monitor After Rollback - -```bash -# Watch pods -watch kubectl get pods -n demo-app - -# Check logs -kubectl logs -n demo-app -l app=demo-nginx -f - -# Verify image -kubectl get deployment demo-nginx -n demo-app \ - -o jsonpath='{.spec.template.spec.containers[0].image}' -``` - -### 5. Verify in ArgoCD - -``` -ArgoCD UI → demo-nginx -├─ Status: Synced ✅ -└─ Health: Healthy ✅ -``` - ---- - -## Examples - -### Example 1: Quick Rollback to Previous Build - -``` -Scenario: Build #23 failed, rollback to #21 - -Steps: -1. Jenkins → demo-nginx-rollback -2. IMAGE_TAG + main-21 -3. SKIP_HEALTH_CHECK: true -4. Build - -Time: ~2 minutes -Result: ✅ SUCCESS -``` - ---- - -### Example 2: Rollback to Last Week's Version - -``` -Scenario: Need stable version from last week - -Steps: -1. Find old build: Jenkins → Build History → #15 -2. Check image tag: main-15 -3. Jenkins → demo-nginx-rollback -4. IMAGE_TAG + main-15 -5. DRY_RUN: true (verify first!) -6. DRY_RUN: false (execute) - -Result: ✅ Rolled back to main-15 -``` - ---- - -### Example 3: Rollback by Revision Number - -``` -Scenario: Откатить на 3 versions назад - -Steps: -1. Check history: - kubectl rollout history deployment/demo-nginx -n demo-app - -2. Find revision: 25 (current: 28) - -3. Jenkins → demo-nginx-rollback -4. REVISION_NUMBER + 25 -5. Build - -Result: ✅ Rolled back to revision 25 -``` - ---- - -### Example 4: Rollback by Git Commit - -``` -Scenario: Нужно точное состояние кода - -Steps: -1. Find commit: - git log --oneline apps/demo-nginx/deployment.yaml - -2. Copy SHA: abc123def - -3. Jenkins → demo-nginx-rollback -4. GIT_COMMIT + abc123def -5. Build - -Result: ✅ Rolled back to commit abc123def -``` - ---- - -## Manual Verification Commands - -### Check Deployment Status -```bash -kubectl get deployment demo-nginx -n demo-app - -# Expected: -NAME READY UP-TO-DATE AVAILABLE AGE -demo-nginx 2/2 2 2 15h -``` - -### Check Image Version -```bash -kubectl get deployment demo-nginx -n demo-app \ - -o jsonpath='{.spec.template.spec.containers[0].image}' - -# Expected: docker.io/vladcrypto/demo-nginx:main-21 -``` - -### Check Pods -```bash -kubectl get pods -n demo-app -l app=demo-nginx - -# Expected: 2 pods Running -``` - -### Check Rollout History -```bash -kubectl rollout history deployment/demo-nginx -n demo-app - -# Shows all revisions -``` - -### Test Health Endpoint -```bash -POD=$(kubectl get pods -n demo-app -l app=demo-nginx -o jsonpath='{.items[0].metadata.name}') -kubectl exec $POD -n demo-app -- wget -q -O- http://localhost/health - -# Expected: healthy -``` - ---- - -## Emergency Rollback Procedure - -### If Production is Down - -**Option 1: Jenkins (2 minutes)** -``` -1. Jenkins → demo-nginx-rollback -2. IMAGE_TAG → last known good version -3. SKIP_HEALTH_CHECK: ✅ true -4. Build -``` - -**Option 2: kubectl (30 seconds)** -```bash -# Fastest - rollback to previous -kubectl rollout undo deployment/demo-nginx -n demo-app - -# To specific revision -kubectl rollout undo deployment/demo-nginx -n demo-app --to-revision=25 -``` - -**Option 3: ArgoCD (1 minute)** -``` -1. ArgoCD UI → demo-nginx -2. History → Select previous version -3. Rollback button -``` - ---- - -## Configuration Reference - -### Environment Variables - -```groovy -APP_NAME = 'demo-nginx' // Deployment name -CONTAINER_NAME = 'nginx' // Container name -NAMESPACE = 'demo-app' // K8s namespace -DOCKER_REGISTRY = 'docker.io' // Registry -DOCKER_REPO = 'vladcrypto' // Docker Hub user -HEALTH_CHECK_TIMEOUT = '300s' // Rollout timeout -``` - -### Customization - -Изменить настройки в Jenkinsfile.rollback: - -```groovy -// Увеличить timeout -HEALTH_CHECK_TIMEOUT = '600s' - -// Больше попыток health check -for i in 1 2 3 4 5 6 7 8 9 10; do - -// Дольше ждать stabilization -sleep 30 // Instead of 10 -``` - ---- - -## Monitoring & Alerts - -### Grafana Dashboard - -```promql -# Rollback count -sum(increase(deployment_rollback_total[1h])) by (deployment) - -# Rollback rate -rate(deployment_rollback_total[5m]) - -# Average rollback duration -avg(deployment_rollback_duration_seconds) -``` - -### Alert Rules - -```yaml -- alert: FrequentRollbacks - expr: rate(deployment_rollback_total[1h]) > 2 - annotations: - summary: "Frequent rollbacks detected" - description: "More than 2 rollbacks in last hour" - -- alert: RollbackFailed - expr: deployment_rollback_failed_total > 0 - annotations: - summary: "Rollback failed" - description: "Manual intervention required" -``` - ---- - -## Summary of All Fixes - -| # | Issue | Fix | Status | -|---|-------|-----|--------| -| 1 | Container name wrong | Use `nginx` not `demo-nginx` | ✅ Fixed | -| 2 | Whitespace in input | Auto-trim with `.trim()` | ✅ Fixed | -| 3 | RBAC pods/exec | Add permission to ClusterRole | ✅ Fixed | -| 4 | Health check timing | Use `SKIP_HEALTH_CHECK=true` | ⚠️ Workaround | -| 5 | Bash loop syntax | Use explicit list `1 2 3 4 5` | ✅ Fixed | - ---- - -## Success Criteria - -✅ **Rollback Methods:** 3/3 working (IMAGE_TAG, REVISION, GIT_COMMIT) -✅ **GitOps Sync:** Git commits automatically -✅ **Zero Downtime:** Rolling updates -✅ **RBAC:** Full permissions configured -✅ **Input Validation:** Whitespace auto-trimmed -✅ **DRY_RUN:** Safe testing mode -✅ **Retry Logic:** 5 attempts with proper bash syntax -⚠️ **Health Check:** Optional (use SKIP_HEALTH_CHECK=true) - ---- - -## FAQ - -### Q: Health check всегда падает, это нормально? - -**A:** Да, из-за timing race condition во время rolling update. Используй `SKIP_HEALTH_CHECK: true` и проверь вручную через 30-60s. - -### Q: Как откатиться на несколько версий назад? - -**A:** Используй `REVISION_NUMBER` метод и укажи нужную revision из `kubectl rollout history`. - -### Q: Можно ли откатить только в staging? - -**A:** Да, измени `NAMESPACE` в Jenkinsfile или создай отдельный job для staging. - -### Q: Как быстро откатиться в emergency? - -**A:** Используй `kubectl rollout undo` (30 секунд) или Jenkins с `SKIP_HEALTH_CHECK=true` (2 минуты). - -### Q: Что если Git commit fail? - -**A:** Rollback всё равно произошёл в Kubernetes! Git нужен только для GitOps sync. ArgoCD пере-синкает через 3 минуты. - ---- - -## Related Documentation - -- [CI/CD Guide](../../../CICD_GUIDE.md) -- [Automatic Rollback](../Jenkinsfile) - See `post { failure }` section -- [Jenkins RBAC](../../jenkins/rbac.yaml) -- [Deployment Manifest](../deployment.yaml) - ---- - -## Support - -**Issues?** -- Check Jenkins console output -- Verify RBAC permissions -- Check pod status: `kubectl get pods -n demo-app` -- Review ArgoCD sync status - -**Need Help?** -- Jenkins logs: Jenkins → Build → Console Output -- Kubernetes events: `kubectl get events -n demo-app` -- Pod logs: `kubectl logs -n demo-app -l app=demo-nginx` - ---- - -**Last Updated:** 2026-01-06 -**Version:** 1.0 -**Status:** Production Ready ✅ diff --git a/apps/demo-nginx/docs/ROLLBACK_QUICK_REF.md b/apps/demo-nginx/docs/ROLLBACK_QUICK_REF.md deleted file mode 100644 index ccf4fb4..0000000 --- a/apps/demo-nginx/docs/ROLLBACK_QUICK_REF.md +++ /dev/null @@ -1,144 +0,0 @@ -# 🔄 Manual Rollback - Quick Reference - -## 🚀 Quick Start (2 minutes) - -``` -Jenkins → demo-nginx-rollback → Build with Parameters - -ROLLBACK_METHOD: IMAGE_TAG -TARGET_VERSION: main-21 -SKIP_HEALTH_CHECK: true (recommended!) -DRY_RUN: false - -→ Build → ✅ SUCCESS! -``` - ---- - -## 📋 3 Ways to Rollback - -### 1. By Image Tag (Fastest) -``` -Method: IMAGE_TAG -Target: main-21 -Use: When you know the build number -``` - -### 2. By Revision Number -``` -Method: REVISION_NUMBER -Target: 2 -Use: Rollback N steps back -Find: kubectl rollout history deployment/demo-nginx -n demo-app -``` - -### 3. By Git Commit -``` -Method: GIT_COMMIT -Target: abc123def -Use: Exact code state -Find: git log --oneline apps/demo-nginx/deployment.yaml -``` - ---- - -## ⚡ Emergency Rollback (30 seconds) - -```bash -# Fastest - kubectl -kubectl rollout undo deployment/demo-nginx -n demo-app - -# To specific revision -kubectl rollout undo deployment/demo-nginx -n demo-app --to-revision=25 -``` - ---- - -## 🔍 Verify Rollback - -```bash -# Check image -kubectl get deployment demo-nginx -n demo-app \ - -o jsonpath='{.spec.template.spec.containers[0].image}' - -# Check pods -kubectl get pods -n demo-app -l app=demo-nginx - -# Test health -POD=$(kubectl get pods -n demo-app -l app=demo-nginx -o jsonpath='{.items[0].metadata.name}') -kubectl exec $POD -n demo-app -- wget -q -O- http://localhost/health -``` - ---- - -## ⚙️ Parameters - -| Parameter | Default | Recommended | -|-----------|---------|-------------| -| ROLLBACK_METHOD | IMAGE_TAG | IMAGE_TAG | -| TARGET_VERSION | (required) | main-21 | -| SKIP_HEALTH_CHECK | false | **true** | -| DRY_RUN | false | false | - ---- - -## 🐛 Common Issues - FIXED - -| Issue | Fix | Status | -|-------|-----|--------| -| Wrong container name | Use `nginx` | ✅ Fixed | -| Whitespace in input | Auto-trim | ✅ Fixed | -| RBAC permission | Added pods/exec | ✅ Fixed | -| Health check timing | Use SKIP_HEALTH_CHECK | ⚠️ Workaround | -| Bash loop broken | Use `1 2 3 4 5` | ✅ Fixed | - ---- - -## 💡 Best Practices - -1. ✅ Always test with `DRY_RUN: true` first -2. ✅ Use `SKIP_HEALTH_CHECK: true` for faster rollback -3. ✅ Verify manually after rollback (30-60s wait) -4. ✅ Document rollback reason in Jenkins build comment -5. ✅ Check ArgoCD sync status after rollback - ---- - -## 📊 Verification Commands - -```bash -# Full status check -kubectl get deployment demo-nginx -n demo-app -kubectl get pods -n demo-app -l app=demo-nginx -kubectl rollout history deployment/demo-nginx -n demo-app - -# Watch pods update -watch kubectl get pods -n demo-app - -# Check logs -kubectl logs -n demo-app -l app=demo-nginx --tail=50 -``` - ---- - -## 🎯 Success Checklist - -- [ ] Jenkins pipeline exists (demo-nginx-rollback) -- [ ] RBAC configured (pods/exec permission) -- [ ] Target version identified -- [ ] DRY_RUN tested -- [ ] Rollback executed -- [ ] Pods verified (Running) -- [ ] Image version confirmed -- [ ] Health check passed (manual) -- [ ] ArgoCD synced - ---- - -## 📚 Full Documentation - -See: [apps/demo-nginx/docs/ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md) - ---- - -**Quick Reference - Keep this handy! 📌** diff --git a/apps/demo-nginx/docs/SCALING_HISTORY.md b/apps/demo-nginx/docs/SCALING_HISTORY.md deleted file mode 100644 index 9d55faa..0000000 --- a/apps/demo-nginx/docs/SCALING_HISTORY.md +++ /dev/null @@ -1,107 +0,0 @@ -# Demo-Nginx Scaling History - -This document tracks the scaling changes made to the demo-nginx deployment for capacity planning and audit purposes. - -## Scaling Timeline - -### 2026-01-12: Scale to 5 Replicas -**Change:** Increased from 3 to 5 replicas -**Reason:** Enhanced capacity and improved high availability -**Impact:** -- Additional 2 pod instances for better load distribution -- Improved resilience against node failures -- Better handling of traffic spikes -- Total resource allocation: - - CPU requests: 500m (5 × 100m) - - Memory requests: 640Mi (5 × 128Mi) - - CPU limits: 1000m (5 × 200m) - - Memory limits: 1280Mi (5 × 256Mi) - -**Commit:** `0669ac66c8a651f74b4c16a4618db03bdd843c02` -**Deployment Method:** GitOps via ArgoCD automatic sync - -### 2026-01-12: Scale to 3 Replicas -**Change:** Increased from 2 to 3 replicas -**Reason:** Initial scaling for improved availability -**Commit:** `d5ffa97159ec391de950dc2d861361074ff3eee8` - -### Initial Deployment: 2 Replicas -**Change:** Initial deployment with 2 replicas -**Reason:** Baseline deployment for demo application - -## Current Configuration - -- **Replicas:** 5 -- **Namespace:** demo-app -- **Image:** docker.io/vladcrypto/demo-nginx:main-50 -- **Service Type:** ClusterIP -- **Port:** 80 -- **Health Checks:** Liveness and Readiness probes enabled -- **Monitoring:** Prometheus scraping enabled - -## Scaling Guidelines - -### When to Scale Up -- CPU utilization consistently above 70% -- Memory utilization consistently above 75% -- Response time degradation observed -- Preparing for expected traffic increase -- Node maintenance requires pod redistribution - -### When to Scale Down -- CPU utilization consistently below 30% for extended period -- Cost optimization requirements -- Application usage patterns indicate over-provisioning - -## Resource Considerations - -Each replica consumes: -- **CPU Request:** 100m -- **Memory Request:** 128Mi -- **CPU Limit:** 200m -- **Memory Limit:** 256Mi - -Total cluster resources for 5 replicas: -- **Total CPU Requests:** 500m (0.5 cores) -- **Total Memory Requests:** 640Mi (~0.625 GB) -- **Total CPU Limits:** 1000m (1 core) -- **Total Memory Limits:** 1280Mi (~1.25 GB) - -## Monitoring and Alerts - -Monitor the following metrics in Grafana: -- Pod CPU/Memory utilization -- Request latency -- Error rates -- Pod restart count -- Service availability - -## Rollback Procedure - -If issues arise after scaling: - -1. Revert the deployment.yaml in Git: - ```bash - git revert - git push - ``` - -2. ArgoCD will automatically sync the change - -3. Or manually scale via kubectl: - ```bash - kubectl scale deployment demo-nginx -n demo-app --replicas= - ``` - -## Related Documentation - -- [CICD_GUIDE.md](./CICD_GUIDE.md) - CI/CD pipeline documentation -- [ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md) - Detailed rollback procedures -- [README.md](./README.md) - General application documentation - -## Notes - -- All scaling operations are tracked in Git for audit trail -- ArgoCD manages automatic synchronization from Git to cluster -- Changes follow GitOps principles for declarative infrastructure -- Scaling decisions should be data-driven based on monitoring metrics diff --git a/apps/demo-nginx/docs/rollback.md b/apps/demo-nginx/docs/rollback.md deleted file mode 100644 index bacc1b6..0000000 --- a/apps/demo-nginx/docs/rollback.md +++ /dev/null @@ -1,471 +0,0 @@ -# 🔄 Automatic Rollback Feature - -## ✅ Что добавлено - -Pipeline теперь автоматически откатывается к предыдущей версии при любой ошибке деплоя! - ---- - -## 🎯 Как работает - -### 1. **Save Current State** (перед деплоем) -``` -📸 Сохраняет: -- Текущий Docker image tag -- Количество реплик -- Git commit SHA -``` - -### 2. **Deploy New Version** -``` -🚀 Деплоит новую версию через: -- Build Docker image -- Push to registry -- Update Git manifests -- ArgoCD sync -``` - -### 3. **Health Checks** -``` -🏥 Проверяет: -- Rollout status (timeout: 300s) -- Pod readiness (все поды Ready) -- Image version (правильный tag) -- Health endpoint (5 попыток) -``` - -### 4. **Auto Rollback** (при ошибке) -``` -🔄 Если что-то пошло не так: -- kubectl rollout undo -- Revert Git commit -- Restore previous state -- Notify в logs -``` - ---- - -## 📊 Pipeline Stages - -``` -┌─────────────────────────────┐ -│ 1. Save Current State │ ← Сохраняет текущую версию -└─────────────┬───────────────┘ - ↓ -┌─────────────────────────────┐ -│ 2. Checkout Source │ -└─────────────┬───────────────┘ - ↓ -┌─────────────────────────────┐ -│ 3. Build Docker Image │ -└─────────────┬───────────────┘ - ↓ -┌─────────────────────────────┐ -│ 4. Push to Registry │ -└─────────────┬───────────────┘ - ↓ -┌─────────────────────────────┐ -│ 5. Update GitOps Manifests │ -└─────────────┬───────────────┘ - ↓ -┌─────────────────────────────┐ -│ 6. Wait for Deployment │ ← 300s timeout -└─────────────┬───────────────┘ - ↓ -┌─────────────────────────────┐ -│ 7. Health Check │ ← 5 retries -└─────────────┬───────────────┘ - ↓ - ┌───────┴────────┐ - ↓ ↓ - SUCCESS FAILURE - │ │ - │ ↓ - │ ┌──────────────┐ - │ │ ROLLBACK │ ← Автоматически! - │ └──────────────┘ - ↓ - ✅ DONE -``` - ---- - -## 🔧 Configuration - -### Environment Variables: - -```groovy -// Rollback configuration -ROLLBACK_ENABLED = 'true' // Включить/выключить rollback -DEPLOYMENT_TIMEOUT = '300s' // Timeout для rollout -HEALTH_CHECK_RETRIES = '5' // Количество попыток health check -HEALTH_CHECK_DELAY = '10' // Задержка между попытками (сек) -``` - -### Изменить настройки: - -```groovy -environment { - ROLLBACK_ENABLED = 'false' // Выключить rollback - DEPLOYMENT_TIMEOUT = '600s' // Увеличить timeout - HEALTH_CHECK_RETRIES = '10' // Больше попыток -} -``` - ---- - -## 🧪 Тестирование Rollback - -### Сценарий 1: Симуляция deployment failure - -Измени deployment.yaml чтобы вызвать ошибку: - -```yaml -# apps/demo-nginx/deployment.yaml -spec: - containers: - - name: nginx - image: nginx:nonexistent-tag # Несуществующий tag -``` - -**Результат:** -``` -❌ Deployment failed -🔄 Rollback initiated automatically -✅ Rolled back to previous version -``` - ---- - -### Сценарий 2: Симуляция health check failure - -Измени nginx.conf чтобы сломать /health: - -```nginx -location /health { - return 500 "broken"; # Вернет 500 error -} -``` - -**Результат:** -``` -❌ Health check failed after 5 attempts -🔄 Rollback initiated automatically -✅ Previous version restored -``` - ---- - -### Сценарий 3: Симуляция timeout - -Установи очень короткий timeout: - -```groovy -DEPLOYMENT_TIMEOUT = '10s' // Слишком короткий -``` - -**Результат:** -``` -❌ Deployment timeout exceeded -🔄 Rollback initiated automatically -✅ Rolled back successfully -``` - ---- - -## 📋 Rollback Process Details - -### Что происходит при rollback: - -1. **Kubernetes Rollback:** - ```bash - kubectl rollout undo deployment/demo-nginx -n demo-app - ``` - -2. **Git Revert:** - ```bash - git revert --no-edit HEAD - git push origin main - ``` - -3. **ArgoCD Sync:** - ``` - ArgoCD автоматически применит revert commit - ``` - -4. **Verification:** - ```bash - kubectl rollout status deployment/demo-nginx -n demo-app - ``` - ---- - -## 🔍 Как проверить что rollback сработал - -### В Jenkins Console Output: - -``` -❌ DEPLOYMENT FAILED - INITIATING ROLLBACK! - -Rolling back to previous version... -🔄 Rolling back to: docker.io/vladcrypto/demo-nginx:main-21 - -✅ ROLLBACK COMPLETED! - -Rolled back to: docker.io/vladcrypto/demo-nginx:main-21 -Current build (#22) has been reverted. - -Please check logs and fix the issue before redeploying. -``` - -### В Kubernetes: - -```bash -# Check deployment history -kubectl rollout history deployment/demo-nginx -n demo-app - -# Вывод: -REVISION CHANGE-CAUSE -21 Updated to main-21 -22 Updated to main-22 -23 Rollback to main-21 ← Rollback! -``` - -### В Git: - -```bash -git log --oneline - -# Вывод: -abc1234 Revert "chore(demo-nginx): Update image to main-22" -def5678 chore(demo-nginx): Update image to main-22 -ghi9012 chore(demo-nginx): Update image to main-21 -``` - ---- - -## 💡 Best Practices - -### 1. **Всегда тестируй в staging** - -```groovy -stage('Deploy to Staging') { - when { branch 'develop' } - steps { - // Deploy to staging namespace - } -} -``` - -### 2. **Мониторинг после деплоя** - -```groovy -stage('Post-Deploy Monitoring') { - steps { - sh """ - # Monitor for 5 minutes - for i in {1..30}; do - kubectl top pods -n demo-app - sleep 10 - done - """ - } -} -``` - -### 3. **Slack Notifications** - -```groovy -post { - failure { - slackSend( - color: 'danger', - message: """ - 🚨 ROLLBACK EXECUTED! - Build: #${BUILD_NUMBER} - Rolled back to previous version - """ - ) - } -} -``` - -### 4. **Сохранение artifacts** - -```groovy -post { - always { - archiveArtifacts artifacts: '/tmp/previous_*.txt', allowEmptyArchive: true - } -} -``` - ---- - -## ⚠️ Important Notes - -### Rollback НЕ сработает если: - -1. **Нет предыдущей версии:** - ``` - ⚠️ No previous version found - cannot rollback automatically - Manual intervention required! - ``` - -2. **ROLLBACK_ENABLED = 'false':** - ``` - ❌ Pipeline failed! (Rollback disabled) - ``` - -3. **Не main branch:** - ``` - Rollback only works on main branch - ``` - -### Ручной rollback: - -Если автоматический rollback не сработал: - -```bash -# Kubernetes rollback -kubectl rollout undo deployment/demo-nginx -n demo-app - -# Git revert -cd k3s-gitops -git revert HEAD -git push origin main - -# Force ArgoCD sync -kubectl patch application demo-nginx -n argocd \ - --type merge -p '{"operation":{"sync":{}}}' -``` - ---- - -## 📊 Monitoring & Alerts - -### Grafana Dashboard - -Добавь панели для мониторинга rollbacks: - -```promql -# Number of rollbacks -sum(rate(deployment_rollback_total[5m])) by (deployment) - -# Rollback duration -histogram_quantile(0.95, - rate(deployment_rollback_duration_seconds_bucket[5m]) -) -``` - -### Alert Rules - -```yaml -- alert: FrequentRollbacks - expr: rate(deployment_rollback_total[1h]) > 3 - annotations: - summary: "Frequent rollbacks detected" - description: "More than 3 rollbacks in last hour" -``` - ---- - -## 🎯 Advanced Features (Future) - -### 1. **Canary Deployments** - -```groovy -stage('Canary Deploy') { - steps { - sh """ - # Deploy 10% traffic to new version - kubectl set image deployment/${APP_NAME} - ${APP_NAME}=${IMAGE_TAG} - --record - kubectl scale deployment/${APP_NAME}-canary --replicas=1 - """ - } -} -``` - -### 2. **Blue-Green Deployments** - -```groovy -stage('Blue-Green Switch') { - steps { - sh """ - # Switch service to new deployment - kubectl patch service ${APP_NAME} - -p '{"spec":{"selector":{"version":"${IMAGE_TAG}"}}}' - """ - } -} -``` - -### 3. **Smoke Tests** - -```groovy -stage('Smoke Tests') { - steps { - sh """ - # Run automated tests - curl -f http://${APP_NAME}/api/health - curl -f http://${APP_NAME}/api/status - """ - } -} -``` - ---- - -## ✅ Success Criteria - -Pipeline считается успешным когда: - -- ✅ Docker image built -- ✅ Image pushed to registry -- ✅ Git manifests updated -- ✅ Deployment rolled out (300s timeout) -- ✅ All pods ready -- ✅ Image version matches -- ✅ Health endpoint responds (5 retries) - -Pipeline откатывается если: - -- ❌ Deployment timeout -- ❌ Pod not ready -- ❌ Image version mismatch -- ❌ Health check failed - ---- - -## 🎉 Summary - -**Automatic Rollback добавляет:** - -✅ Безопасность деплоев -✅ Автоматическое восстановление -✅ Сохранение предыдущего состояния -✅ Git history revert -✅ Kubernetes rollback -✅ Health checks -✅ Timeout protection - -**Zero manual intervention needed!** 🚀 - ---- - -## 📝 Testing Checklist - -- [ ] Normal deployment работает -- [ ] Failed deployment triggers rollback -- [ ] Previous version restored -- [ ] Git commit reverted -- [ ] Health checks work -- [ ] Timeout works -- [ ] Notifications sent -- [ ] Logs clear and helpful - ---- - -**Your pipeline is now production-ready with automatic rollback! 🎉** \ No newline at end of file diff --git a/apps/demo-nginx/ingress.yaml b/apps/demo-nginx/ingress.yaml deleted file mode 100644 index a8c8784..0000000 --- a/apps/demo-nginx/ingress.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: demo-nginx - namespace: demo-app - labels: - app: demo-nginx - annotations: - kubernetes.io/ingress.class: traefik - cert-manager.io/cluster-issuer: letsencrypt-http -spec: - rules: - - host: demo-nginx.thedevops.dev - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: demo-nginx - port: - number: 80 - tls: - - hosts: - - demo-nginx.thedevops.dev - secretName: demo-nginx-tls diff --git a/apps/demo-nginx/namespace.yaml b/apps/demo-nginx/namespace.yaml deleted file mode 100644 index 776cf55..0000000 --- a/apps/demo-nginx/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: demo-app - labels: - app: demo-nginx