diff --git a/apps/demo-nginx/Jenkinsfile b/apps/demo-nginx/Jenkinsfile index 0482ef5..a6fc18f 100644 --- a/apps/demo-nginx/Jenkinsfile +++ b/apps/demo-nginx/Jenkinsfile @@ -397,171 +397,78 @@ EOF } } - stage('Verify Deployment') { + stage('Verify Deployment') { when { branch 'main' } steps { script { echo "✅ Verifying deployment and pod status..." - /* ------------------------------- - * 1. Deployment status check - * ------------------------------- */ + /* -------------------------------- + * 1. Deployment readiness + * -------------------------------- */ sh """ set -e - echo "================================================" - echo "DEPLOYMENT VERIFICATION" - echo "================================================" + echo "1. Checking deployment readiness..." - echo "" - echo "1. Checking deployment status..." + READY=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}') + DESIRED=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}') - READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}') - DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}') - UPDATED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.updatedReplicas}') - AVAILABLE_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.availableReplicas}') + echo "Ready replicas : \$READY" + echo "Desired replicas : \$DESIRED" - echo " Desired replicas : \$DESIRED_PODS" - echo " Updated replicas : \$UPDATED_PODS" - echo " Ready replicas : \$READY_PODS" - echo " Available replicas : \$AVAILABLE_PODS" - - if [ "\$READY_PODS" != "\$DESIRED_PODS" ]; then - echo "❌ FAILED: Not all pods are ready" + if [ "\$READY" != "\$DESIRED" ]; then + echo "❌ Not all replicas are ready" exit 1 fi - - echo "✅ All pods are ready" """ - /* ------------------------------- - * 2. Pod image verification (Groovy!) - * ------------------------------- */ + /* -------------------------------- + * 2. Actual running images + * -------------------------------- */ def podImages = sh( script: """ kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} \ - -o jsonpath='{range .items[*]}{.spec.containers[0].image}{"\\n"}{end}' + -o jsonpath='{range .items[*]}{.status.containerStatuses[0].image}{"\\n"}{end}' """, returnStdout: true ).trim() - echo "" - echo "2. Checking running pod images..." echo "Running pod images:" - echo podImages - echo "Expected image tag: ${IMAGE_TAG}" + podImages.split("\\n").each { img -> + echo " - ${img}" + } if (!podImages.contains(IMAGE_TAG)) { - error("❌ FAILED: Running pods do not use expected image ${IMAGE_TAG}") + error("❌ Some pods are NOT running image tag ${IMAGE_TAG}") } - echo "✅ All running pods use expected image ${IMAGE_TAG}" - + /* -------------------------------- + * 3. Restart count + * -------------------------------- */ + def restarts = sh( + script: """ + kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} \ + -o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\\n"}{end}' + """, + returnStdout: true + ).trim() + def maxRestart = restarts + .split("\\n") + .collect { it.toInteger() } + .max() + echo "Max restart count: ${maxRestart}" - - - # 3. CRITICAL: Verify actual running pod images - echo "" - echo "3. Checking actual running pod images..." - POD_IMAGES=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.status.containerStatuses[0].image}{"\\n"}{end}') - - echo " Running pod images:" - echo "\${POD_IMAGES}" | while read -r img; do - echo " - \${img}" - done - - # Check if all pods are running the correct image - WRONG_IMAGE_COUNT=0 - while IFS= read -r img; do - if [[ "\${img}" != *"${IMAGE_TAG}"* ]]; then - echo " ❌ Pod running wrong image: \${img}" - WRONG_IMAGE_COUNT=\$((WRONG_IMAGE_COUNT + 1)) - fi - done <<< "\${POD_IMAGES}" - - if [ \${WRONG_IMAGE_COUNT} -gt 0 ]; then - echo " ❌ FAILED: \${WRONG_IMAGE_COUNT} pod(s) running old image!" - echo " This is the ArgoCD sync bug - deployment updated but pods not rolled out" - exit 1 - fi - echo " ✅ All pods running correct image" - - # 4. Check pod readiness - echo "" - echo "4. Checking pod readiness probes..." - NOT_READY=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} --field-selector=status.phase!=Running --no-headers 2>/dev/null | wc -l) - - if [ "\${NOT_READY}" -gt 0 ]; then - echo " ⚠️ WARNING: \${NOT_READY} pod(s) not in Running state" - kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} - else - echo " ✅ All pods in Running state" - fi - - # 5. Check container restart count - echo "" - echo "5. Checking for container restarts..." - RESTART_COUNTS=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\\n"}{end}') - MAX_RESTARTS=0 - while IFS= read -r count; do - if [ "\${count}" -gt "\${MAX_RESTARTS}" ]; then - MAX_RESTARTS=\${count} - fi - done <<< "\${RESTART_COUNTS}" - - echo " Max restart count: \${MAX_RESTARTS}" - if [ "\${MAX_RESTARTS}" -gt 3 ]; then - echo " ⚠️ WARNING: High restart count detected" - else - echo " ✅ Restart count acceptable" - fi - - echo "" - echo "================================================" - echo "✅ ALL VERIFICATION CHECKS PASSED!" - echo "================================================" - """, returnStdout: true).trim() - - echo verifyResult - echo "✅ Deployment verified successfully!" - - } catch (Exception e) { - echo "❌ Deployment verification failed!" - echo "Error: ${e.message}" - - // Additional debugging - try { - echo "\n=== DEBUGGING INFORMATION ===" - - def pods = sh( - script: "kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide", - returnStdout: true - ).trim() - echo "Current pods:\n${pods}" - - def replicaset = sh( - script: "kubectl get replicaset -n ${NAMESPACE} -l app=${APP_NAME}", - returnStdout: true - ).trim() - echo "ReplicaSets:\n${replicaset}" - - def events = sh( - script: "kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20", - returnStdout: true - ).trim() - echo "Recent events:\n${events}" - - } catch (Exception debugEx) { - echo "Could not fetch debug info: ${debugEx.message}" - } - - throw e - } - } + if (maxRestart > 3) { + error("❌ High restart count detected: ${maxRestart}") } + + echo "✅ ALL VERIFICATION CHECKS PASSED" } } +} + post { success {