feat(jenkinsfile): Add automatic rollback on deployment failures
This commit is contained in:
Claude AI
210
apps/demo-nginx/Jenkinsfile
vendored
210
apps/demo-nginx/Jenkinsfile
vendored
@@ -11,9 +11,43 @@ pipeline {
|
||||
GITEA_BRANCH = 'main'
|
||||
BUILD_TAG = "${env.BUILD_NUMBER}"
|
||||
IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}"
|
||||
|
||||
// Rollback configuration
|
||||
ROLLBACK_ENABLED = 'true'
|
||||
DEPLOYMENT_TIMEOUT = '300s'
|
||||
HEALTH_CHECK_RETRIES = '5'
|
||||
HEALTH_CHECK_DELAY = '10'
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Save Current State') {
|
||||
when { branch 'main' }
|
||||
steps {
|
||||
script {
|
||||
echo "📸 Saving current deployment state for rollback..."
|
||||
|
||||
// Save current image tag
|
||||
sh """
|
||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].image}' \
|
||||
> /tmp/previous_image_${BUILD_NUMBER}.txt || echo "none" > /tmp/previous_image_${BUILD_NUMBER}.txt
|
||||
|
||||
PREV_IMAGE=\$(cat /tmp/previous_image_${BUILD_NUMBER}.txt)
|
||||
echo "Previous image: \${PREV_IMAGE}"
|
||||
"""
|
||||
|
||||
// Save current replica count
|
||||
sh """
|
||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
||||
-o jsonpath='{.spec.replicas}' \
|
||||
> /tmp/previous_replicas_${BUILD_NUMBER}.txt || echo "2" > /tmp/previous_replicas_${BUILD_NUMBER}.txt
|
||||
"""
|
||||
|
||||
echo "✅ Current state saved"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Checkout Source') {
|
||||
steps {
|
||||
echo "Checking out application source code..."
|
||||
@@ -116,6 +150,10 @@ EOF
|
||||
cd k3s-gitops
|
||||
git config user.name "Jenkins"
|
||||
git config user.email "jenkins@thedevops.dev"
|
||||
|
||||
# Save current commit for rollback
|
||||
git rev-parse HEAD > /tmp/previous_commit_${BUILD_NUMBER}.txt
|
||||
|
||||
sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml
|
||||
git add apps/demo-nginx/deployment.yaml
|
||||
git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes"
|
||||
@@ -127,17 +165,87 @@ EOF
|
||||
}
|
||||
}
|
||||
|
||||
stage('Verify Deployment') {
|
||||
stage('Wait for Deployment') {
|
||||
when { branch 'main' }
|
||||
steps {
|
||||
script {
|
||||
echo "Verifying deployment..."
|
||||
echo "⏳ Waiting for deployment to complete..."
|
||||
|
||||
def deploymentSuccess = false
|
||||
|
||||
try {
|
||||
sh """
|
||||
# Wait for ArgoCD to sync
|
||||
sleep 30
|
||||
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=300s || true
|
||||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME}
|
||||
|
||||
# Check rollout status
|
||||
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT}
|
||||
"""
|
||||
echo "✅ Deployment completed!"
|
||||
deploymentSuccess = true
|
||||
echo "✅ Deployment rolled out successfully!"
|
||||
} catch (Exception e) {
|
||||
echo "❌ Deployment failed: ${e.message}"
|
||||
deploymentSuccess = false
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Health Check') {
|
||||
when { branch 'main' }
|
||||
steps {
|
||||
script {
|
||||
echo "🏥 Running health checks..."
|
||||
|
||||
def healthCheckPassed = false
|
||||
|
||||
try {
|
||||
sh """
|
||||
# Check pod status
|
||||
READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}')
|
||||
DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}')
|
||||
|
||||
echo "Ready pods: \${READY_PODS}/\${DESIRED_PODS}"
|
||||
|
||||
if [ "\${READY_PODS}" != "\${DESIRED_PODS}" ]; then
|
||||
echo "❌ Not all pods are ready!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify image version
|
||||
DEPLOYED_IMAGE=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}')
|
||||
echo "Deployed image: \${DEPLOYED_IMAGE}"
|
||||
|
||||
if [[ "\${DEPLOYED_IMAGE}" != *"${IMAGE_TAG}"* ]]; then
|
||||
echo "❌ Image version mismatch!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test endpoint (retry logic)
|
||||
for i in \$(seq 1 ${HEALTH_CHECK_RETRIES}); do
|
||||
echo "Health check attempt \$i/${HEALTH_CHECK_RETRIES}..."
|
||||
|
||||
POD_NAME=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{.items[0].metadata.name}')
|
||||
kubectl exec \${POD_NAME} -n ${NAMESPACE} -- wget -q -O- http://localhost/health && break
|
||||
|
||||
if [ \$i -eq ${HEALTH_CHECK_RETRIES} ]; then
|
||||
echo "❌ Health check failed after ${HEALTH_CHECK_RETRIES} attempts!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sleep ${HEALTH_CHECK_DELAY}
|
||||
done
|
||||
"""
|
||||
|
||||
healthCheckPassed = true
|
||||
echo "✅ Health checks passed!"
|
||||
|
||||
} catch (Exception e) {
|
||||
echo "❌ Health check failed: ${e.message}"
|
||||
healthCheckPassed = false
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -145,15 +253,101 @@ EOF
|
||||
|
||||
post {
|
||||
success {
|
||||
script {
|
||||
echo """
|
||||
✅ Pipeline SUCCESS!
|
||||
✅ DEPLOYMENT SUCCESS!
|
||||
|
||||
Application: ${APP_NAME}
|
||||
Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
||||
Namespace: ${NAMESPACE}
|
||||
Build: #${BUILD_NUMBER}
|
||||
|
||||
All health checks passed! ✨
|
||||
"""
|
||||
|
||||
// Cleanup rollback files
|
||||
sh """
|
||||
rm -f /tmp/previous_image_${BUILD_NUMBER}.txt
|
||||
rm -f /tmp/previous_replicas_${BUILD_NUMBER}.txt
|
||||
rm -f /tmp/previous_commit_${BUILD_NUMBER}.txt
|
||||
"""
|
||||
}
|
||||
failure {
|
||||
echo "❌ Pipeline failed!"
|
||||
}
|
||||
|
||||
failure {
|
||||
script {
|
||||
if (env.BRANCH_NAME == 'main' && env.ROLLBACK_ENABLED == 'true') {
|
||||
echo """
|
||||
❌ DEPLOYMENT FAILED - INITIATING ROLLBACK!
|
||||
|
||||
Rolling back to previous version...
|
||||
"""
|
||||
|
||||
try {
|
||||
// Get previous state
|
||||
def previousImage = sh(
|
||||
script: "cat /tmp/previous_image_${BUILD_NUMBER}.txt 2>/dev/null || echo 'none'",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
if (previousImage != 'none' && previousImage != '') {
|
||||
echo "🔄 Rolling back to: ${previousImage}"
|
||||
|
||||
// Rollback via kubectl
|
||||
sh """
|
||||
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
|
||||
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=180s
|
||||
"""
|
||||
|
||||
// Revert Git commit
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'gitea-credentials',
|
||||
usernameVariable: 'GIT_USER',
|
||||
passwordVariable: 'GIT_PASS'
|
||||
)]) {
|
||||
sh """
|
||||
cd k3s-gitops || git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git
|
||||
cd k3s-gitops
|
||||
git config user.name "Jenkins"
|
||||
git config user.email "jenkins@thedevops.dev"
|
||||
|
||||
# Revert last commit
|
||||
git revert --no-edit HEAD || true
|
||||
git push origin main || true
|
||||
"""
|
||||
}
|
||||
|
||||
echo """
|
||||
✅ ROLLBACK COMPLETED!
|
||||
|
||||
Rolled back to: ${previousImage}
|
||||
Current build (#${BUILD_NUMBER}) has been reverted.
|
||||
|
||||
Please check logs and fix the issue before redeploying.
|
||||
"""
|
||||
|
||||
} else {
|
||||
echo "⚠️ No previous version found - cannot rollback automatically"
|
||||
echo "Manual intervention required!"
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
echo """
|
||||
❌ ROLLBACK FAILED!
|
||||
|
||||
Error: ${e.message}
|
||||
|
||||
MANUAL ROLLBACK REQUIRED:
|
||||
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
|
||||
"""
|
||||
}
|
||||
|
||||
} else {
|
||||
echo "❌ Pipeline failed! (Rollback disabled or not main branch)"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
always {
|
||||
sh """
|
||||
docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true
|
||||
|
||||
Reference in New Issue
Block a user