From 3a44b8589b6f62e33c8b1fb9cdc09d3872d5460a Mon Sep 17 00:00:00 2001 From: admin Date: Wed, 7 Jan 2026 07:24:06 +0000 Subject: [PATCH] Update apps/demo-nginx/Jenkinsfile --- apps/demo-nginx/Jenkinsfile | 189 ++++++++++++++++++++++++++++++++---- 1 file changed, 169 insertions(+), 20 deletions(-) diff --git a/apps/demo-nginx/Jenkinsfile b/apps/demo-nginx/Jenkinsfile index b0a4d1b..d3a520c 100644 --- a/apps/demo-nginx/Jenkinsfile +++ b/apps/demo-nginx/Jenkinsfile @@ -292,7 +292,7 @@ EOF when { branch 'main' } steps { script { - echo "⏳ Waiting for ArgoCD to sync..." + echo "⏳ Waiting for ArgoCD to sync Git manifests..." sendTelegramNotification( status: 'SYNCING', @@ -322,7 +322,15 @@ EOF returnStdout: true ).trim() - def currentImage = sh( + def healthStatus = sh( + script: """ + kubectl get application ${APP_NAME} -n argocd \ + -o jsonpath='{.status.health.status}' + """, + returnStdout: true + ).trim() + + def deploymentImage = sh( script: """ kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ -o jsonpath='{.spec.template.spec.containers[0].image}' @@ -331,21 +339,26 @@ EOF ).trim() echo "ArgoCD sync status: ${syncStatus}" - echo "Current deployment image: ${currentImage}" + echo "ArgoCD health status: ${healthStatus}" + echo "Deployment spec image: ${deploymentImage}" + echo "Expected image tag: ${IMAGE_TAG}" - if (syncStatus == 'Synced' && currentImage.contains(env.IMAGE_TAG)) { + // Check if ArgoCD applied the manifest AND deployment spec is updated + if (syncStatus == 'Synced' && deploymentImage.contains(env.IMAGE_TAG)) { syncSuccess = true - echo "✅ ArgoCD synced successfully!" + echo "✅ ArgoCD synced and deployment spec updated!" + echo "Note: Pods may still be rolling out - will verify in next stage" break } if (attempts < maxAttempts) { + echo "Waiting 10 seconds before next check..." sleep 10 } } if (!syncSuccess) { - error "❌ ArgoCD sync timeout!" + error "❌ ArgoCD sync timeout! Deployment spec was not updated with new image." } } } @@ -355,7 +368,7 @@ EOF when { branch 'main' } steps { script { - echo "⏳ Waiting for deployment to complete..." + echo "⏳ Waiting for Kubernetes rollout to complete..." sendTelegramNotification( status: 'DEPLOYING', @@ -364,18 +377,50 @@ EOF Deployment: ${APP_NAME} Namespace: ${NAMESPACE} +Image: ${IMAGE_TAG} Timeout: ${DEPLOYMENT_TIMEOUT} + +Rolling out new pods... """, color: '🔵' ) try { + // Wait for rollout to complete + echo "Starting rollout status check..." sh """ kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT} """ - echo "✅ Deployment rolled out successfully!" + echo "✅ Rollout completed successfully!" + + // Additional verification - wait a bit for pods to stabilize + echo "Waiting 10 seconds for pods to stabilize..." + sleep 10 + } catch (Exception e) { - echo "❌ Deployment failed: ${e.message}" + echo "❌ Deployment rollout failed: ${e.message}" + + // Get pod status for debugging + try { + def podStatus = sh( + script: """ + kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide + """, + returnStdout: true + ).trim() + echo "Current pod status:\n${podStatus}" + + def events = sh( + script: """ + kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20 + """, + returnStdout: true + ).trim() + echo "Recent events:\n${events}" + } catch (Exception debugEx) { + echo "Could not fetch debug info: ${debugEx.message}" + } + throw e } } @@ -386,40 +431,144 @@ EOF when { branch 'main' } steps { script { - echo "✅ Verifying deployment..." + echo "✅ Verifying deployment and pod status..." try { def verifyResult = sh(script: """#!/bin/bash set -e - # Check pod status + echo "================================================" + echo "DEPLOYMENT VERIFICATION" + echo "================================================" + + # 1. Check deployment status + echo "" + echo "1. Checking deployment status..." READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}') DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}') + UPDATED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.updatedReplicas}') + AVAILABLE_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.availableReplicas}') - echo "Ready pods: \${READY_PODS}/\${DESIRED_PODS}" + echo " Desired replicas: \${DESIRED_PODS}" + echo " Updated replicas: \${UPDATED_PODS}" + echo " Ready replicas: \${READY_PODS}" + echo " Available replicas: \${AVAILABLE_PODS}" if [ "\${READY_PODS}" != "\${DESIRED_PODS}" ]; then - echo "❌ Not all pods are ready!" + echo " ❌ FAILED: Not all pods are ready!" + echo " Expected: \${DESIRED_PODS}, Got: \${READY_PODS}" exit 1 fi + echo " ✅ All pods ready" - # Verify image version - DEPLOYED_IMAGE=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}') - echo "Deployed image: \${DEPLOYED_IMAGE}" + # 2. Verify deployment spec image + echo "" + echo "2. Checking deployment spec image..." + DEPLOYMENT_IMAGE=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}') + echo " Deployment spec image: \${DEPLOYMENT_IMAGE}" + echo " Expected tag: ${IMAGE_TAG}" - if [[ "\${DEPLOYED_IMAGE}" != *"${IMAGE_TAG}"* ]]; then - echo "❌ Image version mismatch!" + if [[ "\${DEPLOYMENT_IMAGE}" != *"${IMAGE_TAG}"* ]]; then + echo " ❌ FAILED: Deployment spec has wrong image!" exit 1 fi + echo " ✅ Deployment spec correct" - echo "✅ All verification checks passed!" + # 3. CRITICAL: Verify actual running pod images + echo "" + echo "3. Checking actual running pod images..." + POD_IMAGES=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.status.containerStatuses[0].image}{"\\n"}{end}') + + echo " Running pod images:" + echo "\${POD_IMAGES}" | while read -r img; do + echo " - \${img}" + done + + # Check if all pods are running the correct image + WRONG_IMAGE_COUNT=0 + while IFS= read -r img; do + if [[ "\${img}" != *"${IMAGE_TAG}"* ]]; then + echo " ❌ Pod running wrong image: \${img}" + WRONG_IMAGE_COUNT=\$((WRONG_IMAGE_COUNT + 1)) + fi + done <<< "\${POD_IMAGES}" + + if [ \${WRONG_IMAGE_COUNT} -gt 0 ]; then + echo " ❌ FAILED: \${WRONG_IMAGE_COUNT} pod(s) running old image!" + echo " This is the ArgoCD sync bug - deployment updated but pods not rolled out" + exit 1 + fi + echo " ✅ All pods running correct image" + + # 4. Check pod readiness + echo "" + echo "4. Checking pod readiness probes..." + NOT_READY=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} --field-selector=status.phase!=Running --no-headers 2>/dev/null | wc -l) + + if [ "\${NOT_READY}" -gt 0 ]; then + echo " ⚠️ WARNING: \${NOT_READY} pod(s) not in Running state" + kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} + else + echo " ✅ All pods in Running state" + fi + + # 5. Check container restart count + echo "" + echo "5. Checking for container restarts..." + RESTART_COUNTS=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\\n"}{end}') + MAX_RESTARTS=0 + while IFS= read -r count; do + if [ "\${count}" -gt "\${MAX_RESTARTS}" ]; then + MAX_RESTARTS=\${count} + fi + done <<< "\${RESTART_COUNTS}" + + echo " Max restart count: \${MAX_RESTARTS}" + if [ "\${MAX_RESTARTS}" -gt 3 ]; then + echo " ⚠️ WARNING: High restart count detected" + else + echo " ✅ Restart count acceptable" + fi + + echo "" + echo "================================================" + echo "✅ ALL VERIFICATION CHECKS PASSED!" + echo "================================================" """, returnStdout: true).trim() echo verifyResult echo "✅ Deployment verified successfully!" } catch (Exception e) { - echo "❌ Deployment verification failed: ${e.message}" + echo "❌ Deployment verification failed!" + echo "Error: ${e.message}" + + // Additional debugging + try { + echo "\n=== DEBUGGING INFORMATION ===" + + def pods = sh( + script: "kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide", + returnStdout: true + ).trim() + echo "Current pods:\n${pods}" + + def replicaset = sh( + script: "kubectl get replicaset -n ${NAMESPACE} -l app=${APP_NAME}", + returnStdout: true + ).trim() + echo "ReplicaSets:\n${replicaset}" + + def events = sh( + script: "kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20", + returnStdout: true + ).trim() + echo "Recent events:\n${events}" + + } catch (Exception debugEx) { + echo "Could not fetch debug info: ${debugEx.message}" + } + throw e } }