diff --git a/apps/demo-nginx/Jenkinsfile b/apps/demo-nginx/Jenkinsfile
index b0a4d1b..d3a520c 100644
--- a/apps/demo-nginx/Jenkinsfile
+++ b/apps/demo-nginx/Jenkinsfile
@@ -292,7 +292,7 @@ EOF
when { branch 'main' }
steps {
script {
- echo "⏳ Waiting for ArgoCD to sync..."
+ echo "⏳ Waiting for ArgoCD to sync Git manifests..."
sendTelegramNotification(
status: 'SYNCING',
@@ -322,7 +322,15 @@ EOF
returnStdout: true
).trim()
- def currentImage = sh(
+ def healthStatus = sh(
+ script: """
+ kubectl get application ${APP_NAME} -n argocd \
+ -o jsonpath='{.status.health.status}'
+ """,
+ returnStdout: true
+ ).trim()
+
+ def deploymentImage = sh(
script: """
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
-o jsonpath='{.spec.template.spec.containers[0].image}'
@@ -331,21 +339,26 @@ EOF
).trim()
echo "ArgoCD sync status: ${syncStatus}"
- echo "Current deployment image: ${currentImage}"
+ echo "ArgoCD health status: ${healthStatus}"
+ echo "Deployment spec image: ${deploymentImage}"
+ echo "Expected image tag: ${IMAGE_TAG}"
- if (syncStatus == 'Synced' && currentImage.contains(env.IMAGE_TAG)) {
+ // Check if ArgoCD applied the manifest AND deployment spec is updated
+ if (syncStatus == 'Synced' && deploymentImage.contains(env.IMAGE_TAG)) {
syncSuccess = true
- echo "✅ ArgoCD synced successfully!"
+ echo "✅ ArgoCD synced and deployment spec updated!"
+ echo "Note: Pods may still be rolling out - will verify in next stage"
break
}
if (attempts < maxAttempts) {
+ echo "Waiting 10 seconds before next check..."
sleep 10
}
}
if (!syncSuccess) {
- error "❌ ArgoCD sync timeout!"
+ error "❌ ArgoCD sync timeout! Deployment spec was not updated with new image."
}
}
}
@@ -355,7 +368,7 @@ EOF
when { branch 'main' }
steps {
script {
- echo "⏳ Waiting for deployment to complete..."
+ echo "⏳ Waiting for Kubernetes rollout to complete..."
sendTelegramNotification(
status: 'DEPLOYING',
@@ -364,18 +377,50 @@ EOF
Deployment: ${APP_NAME}
Namespace: ${NAMESPACE}
+Image: ${IMAGE_TAG}
Timeout: ${DEPLOYMENT_TIMEOUT}
+
+Rolling out new pods...
""",
color: '🔵'
)
try {
+ // Wait for rollout to complete
+ echo "Starting rollout status check..."
sh """
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT}
"""
- echo "✅ Deployment rolled out successfully!"
+ echo "✅ Rollout completed successfully!"
+
+ // Additional verification - wait a bit for pods to stabilize
+ echo "Waiting 10 seconds for pods to stabilize..."
+ sleep 10
+
} catch (Exception e) {
- echo "❌ Deployment failed: ${e.message}"
+ echo "❌ Deployment rollout failed: ${e.message}"
+
+ // Get pod status for debugging
+ try {
+ def podStatus = sh(
+ script: """
+ kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide
+ """,
+ returnStdout: true
+ ).trim()
+ echo "Current pod status:\n${podStatus}"
+
+ def events = sh(
+ script: """
+ kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20
+ """,
+ returnStdout: true
+ ).trim()
+ echo "Recent events:\n${events}"
+ } catch (Exception debugEx) {
+ echo "Could not fetch debug info: ${debugEx.message}"
+ }
+
throw e
}
}
@@ -386,40 +431,144 @@ EOF
when { branch 'main' }
steps {
script {
- echo "✅ Verifying deployment..."
+ echo "✅ Verifying deployment and pod status..."
try {
def verifyResult = sh(script: """#!/bin/bash
set -e
- # Check pod status
+ echo "================================================"
+ echo "DEPLOYMENT VERIFICATION"
+ echo "================================================"
+
+ # 1. Check deployment status
+ echo ""
+ echo "1. Checking deployment status..."
READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}')
DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}')
+ UPDATED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.updatedReplicas}')
+ AVAILABLE_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.availableReplicas}')
- echo "Ready pods: \${READY_PODS}/\${DESIRED_PODS}"
+ echo " Desired replicas: \${DESIRED_PODS}"
+ echo " Updated replicas: \${UPDATED_PODS}"
+ echo " Ready replicas: \${READY_PODS}"
+ echo " Available replicas: \${AVAILABLE_PODS}"
if [ "\${READY_PODS}" != "\${DESIRED_PODS}" ]; then
- echo "❌ Not all pods are ready!"
+ echo " ❌ FAILED: Not all pods are ready!"
+ echo " Expected: \${DESIRED_PODS}, Got: \${READY_PODS}"
exit 1
fi
+ echo " ✅ All pods ready"
- # Verify image version
- DEPLOYED_IMAGE=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}')
- echo "Deployed image: \${DEPLOYED_IMAGE}"
+ # 2. Verify deployment spec image
+ echo ""
+ echo "2. Checking deployment spec image..."
+ DEPLOYMENT_IMAGE=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}')
+ echo " Deployment spec image: \${DEPLOYMENT_IMAGE}"
+ echo " Expected tag: ${IMAGE_TAG}"
- if [[ "\${DEPLOYED_IMAGE}" != *"${IMAGE_TAG}"* ]]; then
- echo "❌ Image version mismatch!"
+ if [[ "\${DEPLOYMENT_IMAGE}" != *"${IMAGE_TAG}"* ]]; then
+ echo " ❌ FAILED: Deployment spec has wrong image!"
exit 1
fi
+ echo " ✅ Deployment spec correct"
- echo "✅ All verification checks passed!"
+ # 3. CRITICAL: Verify actual running pod images
+ echo ""
+ echo "3. Checking actual running pod images..."
+ POD_IMAGES=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.status.containerStatuses[0].image}{"\\n"}{end}')
+
+ echo " Running pod images:"
+ echo "\${POD_IMAGES}" | while read -r img; do
+ echo " - \${img}"
+ done
+
+ # Check if all pods are running the correct image
+ WRONG_IMAGE_COUNT=0
+ while IFS= read -r img; do
+ if [[ "\${img}" != *"${IMAGE_TAG}"* ]]; then
+ echo " ❌ Pod running wrong image: \${img}"
+ WRONG_IMAGE_COUNT=\$((WRONG_IMAGE_COUNT + 1))
+ fi
+ done <<< "\${POD_IMAGES}"
+
+ if [ \${WRONG_IMAGE_COUNT} -gt 0 ]; then
+ echo " ❌ FAILED: \${WRONG_IMAGE_COUNT} pod(s) running old image!"
+ echo " This is the ArgoCD sync bug - deployment updated but pods not rolled out"
+ exit 1
+ fi
+ echo " ✅ All pods running correct image"
+
+ # 4. Check pod readiness
+ echo ""
+ echo "4. Checking pod readiness probes..."
+ NOT_READY=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} --field-selector=status.phase!=Running --no-headers 2>/dev/null | wc -l)
+
+ if [ "\${NOT_READY}" -gt 0 ]; then
+ echo " ⚠️ WARNING: \${NOT_READY} pod(s) not in Running state"
+ kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME}
+ else
+ echo " ✅ All pods in Running state"
+ fi
+
+ # 5. Check container restart count
+ echo ""
+ echo "5. Checking for container restarts..."
+ RESTART_COUNTS=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\\n"}{end}')
+ MAX_RESTARTS=0
+ while IFS= read -r count; do
+ if [ "\${count}" -gt "\${MAX_RESTARTS}" ]; then
+ MAX_RESTARTS=\${count}
+ fi
+ done <<< "\${RESTART_COUNTS}"
+
+ echo " Max restart count: \${MAX_RESTARTS}"
+ if [ "\${MAX_RESTARTS}" -gt 3 ]; then
+ echo " ⚠️ WARNING: High restart count detected"
+ else
+ echo " ✅ Restart count acceptable"
+ fi
+
+ echo ""
+ echo "================================================"
+ echo "✅ ALL VERIFICATION CHECKS PASSED!"
+ echo "================================================"
""", returnStdout: true).trim()
echo verifyResult
echo "✅ Deployment verified successfully!"
} catch (Exception e) {
- echo "❌ Deployment verification failed: ${e.message}"
+ echo "❌ Deployment verification failed!"
+ echo "Error: ${e.message}"
+
+ // Additional debugging
+ try {
+ echo "\n=== DEBUGGING INFORMATION ==="
+
+ def pods = sh(
+ script: "kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide",
+ returnStdout: true
+ ).trim()
+ echo "Current pods:\n${pods}"
+
+ def replicaset = sh(
+ script: "kubectl get replicaset -n ${NAMESPACE} -l app=${APP_NAME}",
+ returnStdout: true
+ ).trim()
+ echo "ReplicaSets:\n${replicaset}"
+
+ def events = sh(
+ script: "kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20",
+ returnStdout: true
+ ).trim()
+ echo "Recent events:\n${events}"
+
+ } catch (Exception debugEx) {
+ echo "Could not fetch debug info: ${debugEx.message}"
+ }
+
throw e
}
}