diff --git a/apps/demo-nginx/Jenkinsfile b/apps/demo-nginx/Jenkinsfile index 702ae3a..81997c8 100644 --- a/apps/demo-nginx/Jenkinsfile +++ b/apps/demo-nginx/Jenkinsfile @@ -11,9 +11,43 @@ pipeline { GITEA_BRANCH = 'main' BUILD_TAG = "${env.BUILD_NUMBER}" IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}" + + // Rollback configuration + ROLLBACK_ENABLED = 'true' + DEPLOYMENT_TIMEOUT = '300s' + HEALTH_CHECK_RETRIES = '5' + HEALTH_CHECK_DELAY = '10' } stages { + stage('Save Current State') { + when { branch 'main' } + steps { + script { + echo "📸 Saving current deployment state for rollback..." + + // Save current image tag + sh """ + kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ + -o jsonpath='{.spec.template.spec.containers[0].image}' \ + > /tmp/previous_image_${BUILD_NUMBER}.txt || echo "none" > /tmp/previous_image_${BUILD_NUMBER}.txt + + PREV_IMAGE=\$(cat /tmp/previous_image_${BUILD_NUMBER}.txt) + echo "Previous image: \${PREV_IMAGE}" + """ + + // Save current replica count + sh """ + kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ + -o jsonpath='{.spec.replicas}' \ + > /tmp/previous_replicas_${BUILD_NUMBER}.txt || echo "2" > /tmp/previous_replicas_${BUILD_NUMBER}.txt + """ + + echo "✅ Current state saved" + } + } + } + stage('Checkout Source') { steps { echo "Checking out application source code..." @@ -116,6 +150,10 @@ EOF cd k3s-gitops git config user.name "Jenkins" git config user.email "jenkins@thedevops.dev" + + # Save current commit for rollback + git rev-parse HEAD > /tmp/previous_commit_${BUILD_NUMBER}.txt + sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml git add apps/demo-nginx/deployment.yaml git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes" @@ -127,17 +165,87 @@ EOF } } - stage('Verify Deployment') { + stage('Wait for Deployment') { when { branch 'main' } steps { script { - echo "Verifying deployment..." - sh """ - sleep 30 - kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=300s || true - kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} - """ - echo "✅ Deployment completed!" + echo "⏳ Waiting for deployment to complete..." + + def deploymentSuccess = false + + try { + sh """ + # Wait for ArgoCD to sync + sleep 30 + + # Check rollout status + kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT} + """ + deploymentSuccess = true + echo "✅ Deployment rolled out successfully!" + } catch (Exception e) { + echo "❌ Deployment failed: ${e.message}" + deploymentSuccess = false + throw e + } + } + } + } + + stage('Health Check') { + when { branch 'main' } + steps { + script { + echo "🏥 Running health checks..." + + def healthCheckPassed = false + + try { + sh """ + # Check pod status + READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}') + DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}') + + echo "Ready pods: \${READY_PODS}/\${DESIRED_PODS}" + + if [ "\${READY_PODS}" != "\${DESIRED_PODS}" ]; then + echo "❌ Not all pods are ready!" + exit 1 + fi + + # Verify image version + DEPLOYED_IMAGE=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}') + echo "Deployed image: \${DEPLOYED_IMAGE}" + + if [[ "\${DEPLOYED_IMAGE}" != *"${IMAGE_TAG}"* ]]; then + echo "❌ Image version mismatch!" + exit 1 + fi + + # Test endpoint (retry logic) + for i in \$(seq 1 ${HEALTH_CHECK_RETRIES}); do + echo "Health check attempt \$i/${HEALTH_CHECK_RETRIES}..." + + POD_NAME=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{.items[0].metadata.name}') + kubectl exec \${POD_NAME} -n ${NAMESPACE} -- wget -q -O- http://localhost/health && break + + if [ \$i -eq ${HEALTH_CHECK_RETRIES} ]; then + echo "❌ Health check failed after ${HEALTH_CHECK_RETRIES} attempts!" + exit 1 + fi + + sleep ${HEALTH_CHECK_DELAY} + done + """ + + healthCheckPassed = true + echo "✅ Health checks passed!" + + } catch (Exception e) { + echo "❌ Health check failed: ${e.message}" + healthCheckPassed = false + throw e + } } } } @@ -145,15 +253,101 @@ EOF post { success { - echo """ - ✅ Pipeline SUCCESS! - Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} - Namespace: ${NAMESPACE} - """ + script { + echo """ + ✅ DEPLOYMENT SUCCESS! + + Application: ${APP_NAME} + Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} + Namespace: ${NAMESPACE} + Build: #${BUILD_NUMBER} + + All health checks passed! ✨ + """ + + // Cleanup rollback files + sh """ + rm -f /tmp/previous_image_${BUILD_NUMBER}.txt + rm -f /tmp/previous_replicas_${BUILD_NUMBER}.txt + rm -f /tmp/previous_commit_${BUILD_NUMBER}.txt + """ + } } + failure { - echo "❌ Pipeline failed!" + script { + if (env.BRANCH_NAME == 'main' && env.ROLLBACK_ENABLED == 'true') { + echo """ + ❌ DEPLOYMENT FAILED - INITIATING ROLLBACK! + + Rolling back to previous version... + """ + + try { + // Get previous state + def previousImage = sh( + script: "cat /tmp/previous_image_${BUILD_NUMBER}.txt 2>/dev/null || echo 'none'", + returnStdout: true + ).trim() + + if (previousImage != 'none' && previousImage != '') { + echo "🔄 Rolling back to: ${previousImage}" + + // Rollback via kubectl + sh """ + kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE} + kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=180s + """ + + // Revert Git commit + withCredentials([usernamePassword( + credentialsId: 'gitea-credentials', + usernameVariable: 'GIT_USER', + passwordVariable: 'GIT_PASS' + )]) { + sh """ + cd k3s-gitops || git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git + cd k3s-gitops + git config user.name "Jenkins" + git config user.email "jenkins@thedevops.dev" + + # Revert last commit + git revert --no-edit HEAD || true + git push origin main || true + """ + } + + echo """ + ✅ ROLLBACK COMPLETED! + + Rolled back to: ${previousImage} + Current build (#${BUILD_NUMBER}) has been reverted. + + Please check logs and fix the issue before redeploying. + """ + + } else { + echo "⚠️ No previous version found - cannot rollback automatically" + echo "Manual intervention required!" + } + + } catch (Exception e) { + echo """ + ❌ ROLLBACK FAILED! + + Error: ${e.message} + + MANUAL ROLLBACK REQUIRED: + kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE} + """ + } + + } else { + echo "❌ Pipeline failed! (Rollback disabled or not main branch)" + } + } } + always { sh """ docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true