admin/k3s-gitops
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update apps/demo-nginx/Jenkinsfile

Browse Source
This commit is contained in:
admin
2026-01-07 07:24:06 +00:00
parent 4d209c43f7
commit 3a44b8589b
1 changed files with 169 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

189
apps/demo-nginx/Jenkinsfile vendored
View File

@@ -292,7 +292,7 @@ EOF
when { branch 'main' } when { branch 'main' }
steps { steps {
script { script {
echo "⏳ Waiting for ArgoCD to sync..." echo "⏳ Waiting for ArgoCD to sync Git manifests..."
sendTelegramNotification( sendTelegramNotification(
status: 'SYNCING', status: 'SYNCING',
@@ -322,7 +322,15 @@ EOF
returnStdout: true returnStdout: true
).trim() ).trim()
def currentImage = sh( def healthStatus = sh(
script: """
kubectl get application ${APP_NAME} -n argocd \
-o jsonpath='{.status.health.status}'
""",
returnStdout: true
).trim()
def deploymentImage = sh(
script: """ script: """
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \ kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
-o jsonpath='{.spec.template.spec.containers[0].image}' -o jsonpath='{.spec.template.spec.containers[0].image}'
@@ -331,21 +339,26 @@ EOF
).trim() ).trim()
echo "ArgoCD sync status: ${syncStatus}" echo "ArgoCD sync status: ${syncStatus}"
echo "Current deployment image: ${currentImage}" echo "ArgoCD health status: ${healthStatus}"
echo "Deployment spec image: ${deploymentImage}"
echo "Expected image tag: ${IMAGE_TAG}"
if (syncStatus == 'Synced' && currentImage.contains(env.IMAGE_TAG)) { // Check if ArgoCD applied the manifest AND deployment spec is updated
if (syncStatus == 'Synced' && deploymentImage.contains(env.IMAGE_TAG)) {
syncSuccess = true syncSuccess = true
echo "✅ ArgoCD synced successfully!" echo "✅ ArgoCD synced and deployment spec updated!"
echo "Note: Pods may still be rolling out - will verify in next stage"
break break
} }
if (attempts < maxAttempts) { if (attempts < maxAttempts) {
echo "Waiting 10 seconds before next check..."
sleep 10 sleep 10
} }
} }
if (!syncSuccess) { if (!syncSuccess) {
error "❌ ArgoCD sync timeout!" error "❌ ArgoCD sync timeout! Deployment spec was not updated with new image."
} }
} }
} }
@@ -355,7 +368,7 @@ EOF
when { branch 'main' } when { branch 'main' }
steps { steps {
script { script {
echo "⏳ Waiting for deployment to complete..." echo "⏳ Waiting for Kubernetes rollout to complete..."
sendTelegramNotification( sendTelegramNotification(
status: 'DEPLOYING', status: 'DEPLOYING',
@@ -364,18 +377,50 @@ EOF
<b>Deployment:</b> ${APP_NAME} <b>Deployment:</b> ${APP_NAME}
<b>Namespace:</b> ${NAMESPACE} <b>Namespace:</b> ${NAMESPACE}
<b>Image:</b> ${IMAGE_TAG}
<b>Timeout:</b> ${DEPLOYMENT_TIMEOUT} <b>Timeout:</b> ${DEPLOYMENT_TIMEOUT}
<i>Rolling out new pods...</i>
""", """,
color: '🔵' color: '🔵'
) )
try { try {
// Wait for rollout to complete
echo "Starting rollout status check..."
sh """ sh """
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT} kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT}
""" """
echo "✅ Deployment rolled out successfully!" echo "✅ Rollout completed successfully!"
// Additional verification - wait a bit for pods to stabilize
echo "Waiting 10 seconds for pods to stabilize..."
sleep 10
} catch (Exception e) { } catch (Exception e) {
echo "❌ Deployment failed: ${e.message}" echo "❌ Deployment rollout failed: ${e.message}"
// Get pod status for debugging
try {
def podStatus = sh(
script: """
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide
""",
returnStdout: true
).trim()
echo "Current pod status:\n${podStatus}"
def events = sh(
script: """
kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20
""",
returnStdout: true
).trim()
echo "Recent events:\n${events}"
} catch (Exception debugEx) {
echo "Could not fetch debug info: ${debugEx.message}"
}
throw e throw e
} }
} }
@@ -386,40 +431,144 @@ EOF
when { branch 'main' } when { branch 'main' }
steps { steps {
script { script {
echo "✅ Verifying deployment..." echo "✅ Verifying deployment and pod status..."
try { try {
def verifyResult = sh(script: """#!/bin/bash def verifyResult = sh(script: """#!/bin/bash
set -e set -e
# Check pod status echo "================================================"
echo "DEPLOYMENT VERIFICATION"
echo "================================================"
# 1. Check deployment status
echo ""
echo "1. Checking deployment status..."
READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}') READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}')
DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}') DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}')
UPDATED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.updatedReplicas}')
AVAILABLE_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.availableReplicas}')
echo "Ready pods: \${READY_PODS}/\${DESIRED_PODS}" echo " Desired replicas: \${DESIRED_PODS}"
echo " Updated replicas: \${UPDATED_PODS}"
echo " Ready replicas: \${READY_PODS}"
echo " Available replicas: \${AVAILABLE_PODS}"
if [ "\${READY_PODS}" != "\${DESIRED_PODS}" ]; then if [ "\${READY_PODS}" != "\${DESIRED_PODS}" ]; then
echo "❌ Not all pods are ready!" echo " ❌ FAILED: Not all pods are ready!"
echo " Expected: \${DESIRED_PODS}, Got: \${READY_PODS}"
exit 1 exit 1
fi fi
echo " ✅ All pods ready"
# Verify image version # 2. Verify deployment spec image
DEPLOYED_IMAGE=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}') echo ""
echo "Deployed image: \${DEPLOYED_IMAGE}" echo "2. Checking deployment spec image..."
DEPLOYMENT_IMAGE=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}')
echo " Deployment spec image: \${DEPLOYMENT_IMAGE}"
echo " Expected tag: ${IMAGE_TAG}"
if [[ "\${DEPLOYED_IMAGE}" != *"${IMAGE_TAG}"* ]]; then if [[ "\${DEPLOYMENT_IMAGE}" != *"${IMAGE_TAG}"* ]]; then
echo "❌ Image version mismatch!" echo " ❌ FAILED: Deployment spec has wrong image!"
exit 1 exit 1
fi fi
echo " ✅ Deployment spec correct"
echo "✅ All verification checks passed!" # 3. CRITICAL: Verify actual running pod images
echo ""
echo "3. Checking actual running pod images..."
POD_IMAGES=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.status.containerStatuses[0].image}{"\\n"}{end}')
echo " Running pod images:"
echo "\${POD_IMAGES}" | while read -r img; do
echo " - \${img}"
done
# Check if all pods are running the correct image
WRONG_IMAGE_COUNT=0
while IFS= read -r img; do
if [[ "\${img}" != *"${IMAGE_TAG}"* ]]; then
echo " ❌ Pod running wrong image: \${img}"
WRONG_IMAGE_COUNT=\$((WRONG_IMAGE_COUNT + 1))
fi
done <<< "\${POD_IMAGES}"
if [ \${WRONG_IMAGE_COUNT} -gt 0 ]; then
echo " ❌ FAILED: \${WRONG_IMAGE_COUNT} pod(s) running old image!"
echo " This is the ArgoCD sync bug - deployment updated but pods not rolled out"
exit 1
fi
echo " ✅ All pods running correct image"
# 4. Check pod readiness
echo ""
echo "4. Checking pod readiness probes..."
NOT_READY=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} --field-selector=status.phase!=Running --no-headers 2>/dev/null | wc -l)
if [ "\${NOT_READY}" -gt 0 ]; then
echo " ⚠️ WARNING: \${NOT_READY} pod(s) not in Running state"
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME}
else
echo " ✅ All pods in Running state"
fi
# 5. Check container restart count
echo ""
echo "5. Checking for container restarts..."
RESTART_COUNTS=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\\n"}{end}')
MAX_RESTARTS=0
while IFS= read -r count; do
if [ "\${count}" -gt "\${MAX_RESTARTS}" ]; then
MAX_RESTARTS=\${count}
fi
done <<< "\${RESTART_COUNTS}"
echo " Max restart count: \${MAX_RESTARTS}"
if [ "\${MAX_RESTARTS}" -gt 3 ]; then
echo " ⚠️ WARNING: High restart count detected"
else
echo " ✅ Restart count acceptable"
fi
echo ""
echo "================================================"
echo "✅ ALL VERIFICATION CHECKS PASSED!"
echo "================================================"
""", returnStdout: true).trim() """, returnStdout: true).trim()
echo verifyResult echo verifyResult
echo "✅ Deployment verified successfully!" echo "✅ Deployment verified successfully!"
} catch (Exception e) { } catch (Exception e) {
echo "❌ Deployment verification failed: ${e.message}" echo "❌ Deployment verification failed!"
echo "Error: ${e.message}"
// Additional debugging
try {
echo "\n=== DEBUGGING INFORMATION ==="
def pods = sh(
script: "kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide",
returnStdout: true
).trim()
echo "Current pods:\n${pods}"
def replicaset = sh(
script: "kubectl get replicaset -n ${NAMESPACE} -l app=${APP_NAME}",
returnStdout: true
).trim()
echo "ReplicaSets:\n${replicaset}"
def events = sh(
script: "kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20",
returnStdout: true
).trim()
echo "Recent events:\n${events}"
} catch (Exception debugEx) {
echo "Could not fetch debug info: ${debugEx.message}"
}
throw e throw e
} }
} }