fix(rollback): Add stabilization wait and retry logic for health checks
This commit is contained in:
@@ -225,6 +225,10 @@ pipeline {
|
||||
--timeout=${HEALTH_CHECK_TIMEOUT}
|
||||
"""
|
||||
|
||||
// Wait a bit longer for pods to be fully ready
|
||||
echo "⏳ Waiting for new pods to stabilize..."
|
||||
sleep 10
|
||||
|
||||
echo "✅ Rollout completed"
|
||||
}
|
||||
}
|
||||
@@ -250,9 +254,27 @@ pipeline {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test health endpoint
|
||||
POD_NAME=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{.items[0].metadata.name}')
|
||||
kubectl exec \${POD_NAME} -n ${NAMESPACE} -- wget -q -O- http://localhost/health
|
||||
# Verify image version on running pods
|
||||
DEPLOYED_IMAGE=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}')
|
||||
echo "Deployed image: \${DEPLOYED_IMAGE}"
|
||||
|
||||
# Get a RUNNING pod (not terminating)
|
||||
POD_NAME=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}')
|
||||
echo "Testing pod: \${POD_NAME}"
|
||||
|
||||
# Test health endpoint with retry
|
||||
for i in {1..5}; do
|
||||
echo "Health check attempt \$i/5..."
|
||||
if kubectl exec \${POD_NAME} -n ${NAMESPACE} -- wget -q -O- http://localhost/health 2>/dev/null; then
|
||||
echo "✅ Health check passed!"
|
||||
exit 0
|
||||
fi
|
||||
echo "Retrying in 5 seconds..."
|
||||
sleep 5
|
||||
done
|
||||
|
||||
echo "❌ Health check failed after 5 attempts"
|
||||
exit 1
|
||||
"""
|
||||
|
||||
echo "✅ Health checks passed"
|
||||
@@ -308,7 +330,7 @@ pipeline {
|
||||
1. Update deployment to target version
|
||||
2. Update Git manifests
|
||||
3. Wait for rollout (timeout: ${HEALTH_CHECK_TIMEOUT})
|
||||
${params.SKIP_HEALTH_CHECK ? '4. (Health check skipped)' : '4. Run health checks'}
|
||||
${params.SKIP_HEALTH_CHECK ? '4. (Health check skipped)' : '4. Run health checks with retry'}
|
||||
|
||||
No actual changes were made.
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user