admin/k3s-gitops
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(jenkinsfile): Add automatic rollback on deployment failures

Browse Source
This commit is contained in:
Claude AI
2026-01-06 07:42:23 +00:00
parent 4e78b8cc47
commit edde8503d3
1 changed files with 208 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

222
apps/demo-nginx/Jenkinsfile vendored
View File

@@ -11,9 +11,43 @@ pipeline {
GITEA_BRANCH = 'main' GITEA_BRANCH = 'main'
BUILD_TAG = "${env.BUILD_NUMBER}" BUILD_TAG = "${env.BUILD_NUMBER}"
IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}" IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}"
// Rollback configuration
ROLLBACK_ENABLED = 'true'
DEPLOYMENT_TIMEOUT = '300s'
HEALTH_CHECK_RETRIES = '5'
HEALTH_CHECK_DELAY = '10'
} }
stages { stages {
stage('Save Current State') {
when { branch 'main' }
steps {
script {
echo "📸 Saving current deployment state for rollback..."
// Save current image tag
sh """
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
-o jsonpath='{.spec.template.spec.containers[0].image}' \
> /tmp/previous_image_${BUILD_NUMBER}.txt || echo "none" > /tmp/previous_image_${BUILD_NUMBER}.txt
PREV_IMAGE=\$(cat /tmp/previous_image_${BUILD_NUMBER}.txt)
echo "Previous image: \${PREV_IMAGE}"
"""
// Save current replica count
sh """
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
-o jsonpath='{.spec.replicas}' \
> /tmp/previous_replicas_${BUILD_NUMBER}.txt || echo "2" > /tmp/previous_replicas_${BUILD_NUMBER}.txt
"""
echo "✅ Current state saved"
}
}
}
stage('Checkout Source') { stage('Checkout Source') {
steps { steps {
echo "Checking out application source code..." echo "Checking out application source code..."
@@ -116,6 +150,10 @@ EOF
cd k3s-gitops cd k3s-gitops
git config user.name "Jenkins" git config user.name "Jenkins"
git config user.email "jenkins@thedevops.dev" git config user.email "jenkins@thedevops.dev"
# Save current commit for rollback
git rev-parse HEAD > /tmp/previous_commit_${BUILD_NUMBER}.txt
sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml
git add apps/demo-nginx/deployment.yaml git add apps/demo-nginx/deployment.yaml
git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes" git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes"
@@ -127,17 +165,87 @@ EOF
} }
} }
stage('Verify Deployment') { stage('Wait for Deployment') {
when { branch 'main' } when { branch 'main' }
steps { steps {
script { script {
echo "Verifying deployment..." echo "⏳ Waiting for deployment to complete..."
sh """
sleep 30 def deploymentSuccess = false
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=300s || true
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} try {
""" sh """
echo "✅ Deployment completed!" # Wait for ArgoCD to sync
sleep 30
# Check rollout status
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT}
"""
deploymentSuccess = true
echo "✅ Deployment rolled out successfully!"
} catch (Exception e) {
echo "❌ Deployment failed: ${e.message}"
deploymentSuccess = false
throw e
}
}
}
}
stage('Health Check') {
when { branch 'main' }
steps {
script {
echo "🏥 Running health checks..."
def healthCheckPassed = false
try {
sh """
# Check pod status
READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}')
DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}')
echo "Ready pods: \${READY_PODS}/\${DESIRED_PODS}"
if [ "\${READY_PODS}" != "\${DESIRED_PODS}" ]; then
echo "❌ Not all pods are ready!"
exit 1
fi
# Verify image version
DEPLOYED_IMAGE=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}')
echo "Deployed image: \${DEPLOYED_IMAGE}"
if [[ "\${DEPLOYED_IMAGE}" != *"${IMAGE_TAG}"* ]]; then
echo "❌ Image version mismatch!"
exit 1
fi
# Test endpoint (retry logic)
for i in \$(seq 1 ${HEALTH_CHECK_RETRIES}); do
echo "Health check attempt \$i/${HEALTH_CHECK_RETRIES}..."
POD_NAME=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{.items[0].metadata.name}')
kubectl exec \${POD_NAME} -n ${NAMESPACE} -- wget -q -O- http://localhost/health && break
if [ \$i -eq ${HEALTH_CHECK_RETRIES} ]; then
echo "❌ Health check failed after ${HEALTH_CHECK_RETRIES} attempts!"
exit 1
fi
sleep ${HEALTH_CHECK_DELAY}
done
"""
healthCheckPassed = true
echo "✅ Health checks passed!"
} catch (Exception e) {
echo "❌ Health check failed: ${e.message}"
healthCheckPassed = false
throw e
}
} }
} }
} }
@@ -145,15 +253,101 @@ EOF
post { post {
success { success {
echo """ script {
✅ Pipeline SUCCESS! echo """
Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} ✅ DEPLOYMENT SUCCESS!
Namespace: ${NAMESPACE}
""" Application: ${APP_NAME}
Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
Namespace: ${NAMESPACE}
Build: #${BUILD_NUMBER}
All health checks passed! ✨
"""
// Cleanup rollback files
sh """
rm -f /tmp/previous_image_${BUILD_NUMBER}.txt
rm -f /tmp/previous_replicas_${BUILD_NUMBER}.txt
rm -f /tmp/previous_commit_${BUILD_NUMBER}.txt
"""
}
} }
failure { failure {
echo "❌ Pipeline failed!" script {
if (env.BRANCH_NAME == 'main' && env.ROLLBACK_ENABLED == 'true') {
echo """
❌ DEPLOYMENT FAILED - INITIATING ROLLBACK!
Rolling back to previous version...
"""
try {
// Get previous state
def previousImage = sh(
script: "cat /tmp/previous_image_${BUILD_NUMBER}.txt 2>/dev/null || echo 'none'",
returnStdout: true
).trim()
if (previousImage != 'none' && previousImage != '') {
echo "🔄 Rolling back to: ${previousImage}"
// Rollback via kubectl
sh """
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=180s
"""
// Revert Git commit
withCredentials([usernamePassword(
credentialsId: 'gitea-credentials',
usernameVariable: 'GIT_USER',
passwordVariable: 'GIT_PASS'
)]) {
sh """
cd k3s-gitops || git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git
cd k3s-gitops
git config user.name "Jenkins"
git config user.email "jenkins@thedevops.dev"
# Revert last commit
git revert --no-edit HEAD || true
git push origin main || true
"""
}
echo """
✅ ROLLBACK COMPLETED!
Rolled back to: ${previousImage}
Current build (#${BUILD_NUMBER}) has been reverted.
Please check logs and fix the issue before redeploying.
"""
} else {
echo "⚠️ No previous version found - cannot rollback automatically"
echo "Manual intervention required!"
}
} catch (Exception e) {
echo """
❌ ROLLBACK FAILED!
Error: ${e.message}
MANUAL ROLLBACK REQUIRED:
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
"""
}
} else {
echo "❌ Pipeline failed! (Rollback disabled or not main branch)"
}
}
} }
always { always {
sh """ sh """
docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true