feat(jenkinsfile): Add ArgoCD sync wait and improved health checks
This commit is contained in:
93
apps/demo-nginx/Jenkinsfile
vendored
93
apps/demo-nginx/Jenkinsfile
vendored
@@ -15,6 +15,7 @@ pipeline {
|
||||
// Rollback configuration
|
||||
ROLLBACK_ENABLED = 'true'
|
||||
DEPLOYMENT_TIMEOUT = '300s'
|
||||
ARGOCD_SYNC_TIMEOUT = '120'
|
||||
HEALTH_CHECK_RETRIES = '5'
|
||||
HEALTH_CHECK_DELAY = '10'
|
||||
}
|
||||
@@ -220,27 +221,73 @@ EOF
|
||||
}
|
||||
}
|
||||
|
||||
stage('Wait for ArgoCD Sync') {
|
||||
when { branch 'main' }
|
||||
steps {
|
||||
script {
|
||||
echo "⏳ Waiting for ArgoCD to sync..."
|
||||
|
||||
def syncSuccess = false
|
||||
def attempts = 0
|
||||
def maxAttempts = Integer.parseInt(env.ARGOCD_SYNC_TIMEOUT) / 10
|
||||
|
||||
while (!syncSuccess && attempts < maxAttempts) {
|
||||
attempts++
|
||||
echo "ArgoCD sync check attempt ${attempts}/${maxAttempts}..."
|
||||
|
||||
def syncStatus = sh(
|
||||
script: """
|
||||
kubectl get application ${APP_NAME} -n argocd \
|
||||
-o jsonpath='{.status.sync.status}'
|
||||
""",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
def currentImage = sh(
|
||||
script: """
|
||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
||||
""",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
echo "ArgoCD sync status: ${syncStatus}"
|
||||
echo "Current deployment image: ${currentImage}"
|
||||
echo "Expected image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}"
|
||||
|
||||
if (syncStatus == 'Synced' && currentImage.contains(env.IMAGE_TAG)) {
|
||||
syncSuccess = true
|
||||
echo "✅ ArgoCD synced successfully!"
|
||||
break
|
||||
}
|
||||
|
||||
if (attempts < maxAttempts) {
|
||||
echo "Waiting 10 seconds before next check..."
|
||||
sleep 10
|
||||
}
|
||||
}
|
||||
|
||||
if (!syncSuccess) {
|
||||
error "❌ ArgoCD sync timeout after ${env.ARGOCD_SYNC_TIMEOUT} seconds!"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Wait for Deployment') {
|
||||
when { branch 'main' }
|
||||
steps {
|
||||
script {
|
||||
echo "⏳ Waiting for deployment to complete..."
|
||||
|
||||
def deploymentSuccess = false
|
||||
|
||||
try {
|
||||
sh """
|
||||
# Wait for ArgoCD to sync
|
||||
sleep 30
|
||||
|
||||
# Check rollout status
|
||||
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT}
|
||||
"""
|
||||
deploymentSuccess = true
|
||||
echo "✅ Deployment rolled out successfully!"
|
||||
} catch (Exception e) {
|
||||
echo "❌ Deployment failed: ${e.message}"
|
||||
deploymentSuccess = false
|
||||
throw e
|
||||
}
|
||||
}
|
||||
@@ -253,10 +300,10 @@ EOF
|
||||
script {
|
||||
echo "🏥 Running health checks..."
|
||||
|
||||
def healthCheckPassed = false
|
||||
|
||||
try {
|
||||
sh """
|
||||
sh """#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Check pod status
|
||||
READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}')
|
||||
DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}')
|
||||
@@ -274,31 +321,41 @@ EOF
|
||||
|
||||
if [[ "\${DEPLOYED_IMAGE}" != *"${IMAGE_TAG}"* ]]; then
|
||||
echo "❌ Image version mismatch!"
|
||||
echo "Expected: ${IMAGE_TAG}"
|
||||
echo "Got: \${DEPLOYED_IMAGE}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Wait for pods to stabilize
|
||||
echo "Waiting 15 seconds for pods to stabilize..."
|
||||
sleep 15
|
||||
|
||||
# Test endpoint (retry logic)
|
||||
for i in \$(seq 1 ${HEALTH_CHECK_RETRIES}); do
|
||||
for i in 1 2 3 4 5; do
|
||||
echo "Health check attempt \$i/${HEALTH_CHECK_RETRIES}..."
|
||||
|
||||
POD_NAME=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{.items[0].metadata.name}')
|
||||
kubectl exec \${POD_NAME} -n ${NAMESPACE} -- wget -q -O- http://localhost/health && break
|
||||
POD_NAME=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}')
|
||||
echo "Testing pod: \${POD_NAME}"
|
||||
|
||||
if [ \$i -eq ${HEALTH_CHECK_RETRIES} ]; then
|
||||
echo "❌ Health check failed after ${HEALTH_CHECK_RETRIES} attempts!"
|
||||
exit 1
|
||||
if kubectl exec \${POD_NAME} -n ${NAMESPACE} -- wget -q -O- http://localhost/health 2>/dev/null; then
|
||||
echo "✅ Health check passed!"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ \$i -lt ${HEALTH_CHECK_RETRIES} ]; then
|
||||
echo "Retrying in ${HEALTH_CHECK_DELAY} seconds..."
|
||||
sleep ${HEALTH_CHECK_DELAY}
|
||||
fi
|
||||
done
|
||||
|
||||
echo "❌ Health check failed after ${HEALTH_CHECK_RETRIES} attempts!"
|
||||
exit 1
|
||||
"""
|
||||
|
||||
healthCheckPassed = true
|
||||
echo "✅ Health checks passed!"
|
||||
|
||||
} catch (Exception e) {
|
||||
echo "❌ Health check failed: ${e.message}"
|
||||
healthCheckPassed = false
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user