Update apps/demo-nginx/Jenkinsfile
This commit is contained in:
165
apps/demo-nginx/Jenkinsfile
vendored
165
apps/demo-nginx/Jenkinsfile
vendored
@@ -403,165 +403,72 @@ EOF
|
|||||||
script {
|
script {
|
||||||
echo "✅ Verifying deployment and pod status..."
|
echo "✅ Verifying deployment and pod status..."
|
||||||
|
|
||||||
/* -------------------------------
|
/* --------------------------------
|
||||||
* 1. Deployment status check
|
* 1. Deployment readiness
|
||||||
* ------------------------------- */
|
* -------------------------------- */
|
||||||
sh """
|
sh """
|
||||||
set -e
|
set -e
|
||||||
echo "================================================"
|
echo "1. Checking deployment readiness..."
|
||||||
echo "DEPLOYMENT VERIFICATION"
|
|
||||||
echo "================================================"
|
|
||||||
|
|
||||||
echo ""
|
READY=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}')
|
||||||
echo "1. Checking deployment status..."
|
DESIRED=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}')
|
||||||
|
|
||||||
READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}')
|
echo "Ready replicas : \$READY"
|
||||||
DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}')
|
echo "Desired replicas : \$DESIRED"
|
||||||
UPDATED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.updatedReplicas}')
|
|
||||||
AVAILABLE_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.availableReplicas}')
|
|
||||||
|
|
||||||
echo " Desired replicas : \$DESIRED_PODS"
|
if [ "\$READY" != "\$DESIRED" ]; then
|
||||||
echo " Updated replicas : \$UPDATED_PODS"
|
echo "❌ Not all replicas are ready"
|
||||||
echo " Ready replicas : \$READY_PODS"
|
|
||||||
echo " Available replicas : \$AVAILABLE_PODS"
|
|
||||||
|
|
||||||
if [ "\$READY_PODS" != "\$DESIRED_PODS" ]; then
|
|
||||||
echo "❌ FAILED: Not all pods are ready"
|
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "✅ All pods are ready"
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
/* -------------------------------
|
/* --------------------------------
|
||||||
* 2. Pod image verification (Groovy!)
|
* 2. Actual running images
|
||||||
* ------------------------------- */
|
* -------------------------------- */
|
||||||
def podImages = sh(
|
def podImages = sh(
|
||||||
script: """
|
script: """
|
||||||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} \
|
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} \
|
||||||
-o jsonpath='{range .items[*]}{.spec.containers[0].image}{"\\n"}{end}'
|
-o jsonpath='{range .items[*]}{.status.containerStatuses[0].image}{"\\n"}{end}'
|
||||||
""",
|
""",
|
||||||
returnStdout: true
|
returnStdout: true
|
||||||
).trim()
|
).trim()
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "2. Checking running pod images..."
|
|
||||||
echo "Running pod images:"
|
echo "Running pod images:"
|
||||||
echo podImages
|
podImages.split("\\n").each { img ->
|
||||||
echo "Expected image tag: ${IMAGE_TAG}"
|
echo " - ${img}"
|
||||||
|
}
|
||||||
|
|
||||||
if (!podImages.contains(IMAGE_TAG)) {
|
if (!podImages.contains(IMAGE_TAG)) {
|
||||||
error("❌ FAILED: Running pods do not use expected image ${IMAGE_TAG}")
|
error("❌ Some pods are NOT running image tag ${IMAGE_TAG}")
|
||||||
}
|
}
|
||||||
|
|
||||||
echo "✅ All running pods use expected image ${IMAGE_TAG}"
|
/* --------------------------------
|
||||||
|
* 3. Restart count
|
||||||
|
* -------------------------------- */
|
||||||
|
def restarts = sh(
|
||||||
|
script: """
|
||||||
|
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} \
|
||||||
|
-o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\\n"}{end}'
|
||||||
# 3. CRITICAL: Verify actual running pod images
|
""",
|
||||||
echo ""
|
|
||||||
echo "3. Checking actual running pod images..."
|
|
||||||
POD_IMAGES=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.status.containerStatuses[0].image}{"\\n"}{end}')
|
|
||||||
|
|
||||||
echo " Running pod images:"
|
|
||||||
echo "\${POD_IMAGES}" | while read -r img; do
|
|
||||||
echo " - \${img}"
|
|
||||||
done
|
|
||||||
|
|
||||||
# Check if all pods are running the correct image
|
|
||||||
WRONG_IMAGE_COUNT=0
|
|
||||||
while IFS= read -r img; do
|
|
||||||
if [[ "\${img}" != *"${IMAGE_TAG}"* ]]; then
|
|
||||||
echo " ❌ Pod running wrong image: \${img}"
|
|
||||||
WRONG_IMAGE_COUNT=\$((WRONG_IMAGE_COUNT + 1))
|
|
||||||
fi
|
|
||||||
done <<< "\${POD_IMAGES}"
|
|
||||||
|
|
||||||
if [ \${WRONG_IMAGE_COUNT} -gt 0 ]; then
|
|
||||||
echo " ❌ FAILED: \${WRONG_IMAGE_COUNT} pod(s) running old image!"
|
|
||||||
echo " This is the ArgoCD sync bug - deployment updated but pods not rolled out"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo " ✅ All pods running correct image"
|
|
||||||
|
|
||||||
# 4. Check pod readiness
|
|
||||||
echo ""
|
|
||||||
echo "4. Checking pod readiness probes..."
|
|
||||||
NOT_READY=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} --field-selector=status.phase!=Running --no-headers 2>/dev/null | wc -l)
|
|
||||||
|
|
||||||
if [ "\${NOT_READY}" -gt 0 ]; then
|
|
||||||
echo " ⚠️ WARNING: \${NOT_READY} pod(s) not in Running state"
|
|
||||||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME}
|
|
||||||
else
|
|
||||||
echo " ✅ All pods in Running state"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 5. Check container restart count
|
|
||||||
echo ""
|
|
||||||
echo "5. Checking for container restarts..."
|
|
||||||
RESTART_COUNTS=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\\n"}{end}')
|
|
||||||
MAX_RESTARTS=0
|
|
||||||
while IFS= read -r count; do
|
|
||||||
if [ "\${count}" -gt "\${MAX_RESTARTS}" ]; then
|
|
||||||
MAX_RESTARTS=\${count}
|
|
||||||
fi
|
|
||||||
done <<< "\${RESTART_COUNTS}"
|
|
||||||
|
|
||||||
echo " Max restart count: \${MAX_RESTARTS}"
|
|
||||||
if [ "\${MAX_RESTARTS}" -gt 3 ]; then
|
|
||||||
echo " ⚠️ WARNING: High restart count detected"
|
|
||||||
else
|
|
||||||
echo " ✅ Restart count acceptable"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "================================================"
|
|
||||||
echo "✅ ALL VERIFICATION CHECKS PASSED!"
|
|
||||||
echo "================================================"
|
|
||||||
""", returnStdout: true).trim()
|
|
||||||
|
|
||||||
echo verifyResult
|
|
||||||
echo "✅ Deployment verified successfully!"
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
echo "❌ Deployment verification failed!"
|
|
||||||
echo "Error: ${e.message}"
|
|
||||||
|
|
||||||
// Additional debugging
|
|
||||||
try {
|
|
||||||
echo "\n=== DEBUGGING INFORMATION ==="
|
|
||||||
|
|
||||||
def pods = sh(
|
|
||||||
script: "kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide",
|
|
||||||
returnStdout: true
|
returnStdout: true
|
||||||
).trim()
|
).trim()
|
||||||
echo "Current pods:\n${pods}"
|
|
||||||
|
|
||||||
def replicaset = sh(
|
def maxRestart = restarts
|
||||||
script: "kubectl get replicaset -n ${NAMESPACE} -l app=${APP_NAME}",
|
.split("\\n")
|
||||||
returnStdout: true
|
.collect { it.toInteger() }
|
||||||
).trim()
|
.max()
|
||||||
echo "ReplicaSets:\n${replicaset}"
|
|
||||||
|
|
||||||
def events = sh(
|
echo "Max restart count: ${maxRestart}"
|
||||||
script: "kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20",
|
|
||||||
returnStdout: true
|
|
||||||
).trim()
|
|
||||||
echo "Recent events:\n${events}"
|
|
||||||
|
|
||||||
} catch (Exception debugEx) {
|
if (maxRestart > 3) {
|
||||||
echo "Could not fetch debug info: ${debugEx.message}"
|
error("❌ High restart count detected: ${maxRestart}")
|
||||||
}
|
}
|
||||||
|
|
||||||
throw e
|
echo "✅ ALL VERIFICATION CHECKS PASSED"
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
post {
|
post {
|
||||||
success {
|
success {
|
||||||
|
|||||||
Reference in New Issue
Block a user