837 lines
33 KiB
Groovy
837 lines
33 KiB
Groovy
pipeline {
|
||
agent any
|
||
|
||
environment {
|
||
APP_NAME = 'demo-nginx'
|
||
NAMESPACE = 'demo-app'
|
||
DOCKER_REGISTRY = 'docker.io'
|
||
DOCKER_REPO = 'vladcrypto'
|
||
GITEA_URL = 'http://gitea-http.gitea.svc.cluster.local:3000'
|
||
GITEA_REPO = 'admin/k3s-gitops'
|
||
GITEA_BRANCH = 'main'
|
||
BUILD_TAG = "${env.BUILD_NUMBER}"
|
||
IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}"
|
||
|
||
// Rollback configuration
|
||
ROLLBACK_ENABLED = 'true'
|
||
DEPLOYMENT_TIMEOUT = '300s'
|
||
ARGOCD_SYNC_TIMEOUT = '120'
|
||
SKIP_HEALTH_CHECK = 'true'
|
||
|
||
// Notification configuration
|
||
TELEGRAM_BOT_TOKEN = credentials('telegram-bot-token')
|
||
TELEGRAM_CHAT_ID = credentials('telegram-chat-id')
|
||
|
||
// Build info
|
||
BUILD_URL = "${env.BUILD_URL}"
|
||
GIT_COMMIT_SHORT = ""
|
||
DEPLOYMENT_START_TIME = ""
|
||
DEPLOYMENT_END_TIME = ""
|
||
}
|
||
|
||
stages {
|
||
stage('Initialization') {
|
||
steps {
|
||
script {
|
||
echo "🚀 Starting deployment pipeline..."
|
||
DEPLOYMENT_START_TIME = sh(script: 'date +%s', returnStdout: true).trim()
|
||
|
||
sendTelegramNotification(
|
||
status: 'STARTED',
|
||
message: """
|
||
🚀 <b>Deployment Started</b>
|
||
|
||
<b>Application:</b> ${APP_NAME}
|
||
<b>Build:</b> #${BUILD_NUMBER}
|
||
<b>Branch:</b> ${env.BRANCH_NAME}
|
||
<b>Namespace:</b> ${NAMESPACE}
|
||
<b>Started by:</b> ${env.BUILD_USER ?: 'Jenkins'}
|
||
|
||
<i>Building and deploying...</i>
|
||
""",
|
||
color: '🔵'
|
||
)
|
||
}
|
||
}
|
||
}
|
||
|
||
stage('Save Current State') {
|
||
when { branch 'main' }
|
||
steps {
|
||
script {
|
||
echo "📸 Saving current deployment state for rollback..."
|
||
|
||
sh """
|
||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
||
-o jsonpath='{.spec.template.spec.containers[0].image}' \
|
||
> /tmp/previous_image_${BUILD_NUMBER}.txt || echo "none" > /tmp/previous_image_${BUILD_NUMBER}.txt
|
||
|
||
PREV_IMAGE=\$(cat /tmp/previous_image_${BUILD_NUMBER}.txt)
|
||
echo "Previous image: \${PREV_IMAGE}"
|
||
"""
|
||
|
||
sh """
|
||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
||
-o jsonpath='{.spec.replicas}' \
|
||
> /tmp/previous_replicas_${BUILD_NUMBER}.txt || echo "2" > /tmp/previous_replicas_${BUILD_NUMBER}.txt
|
||
"""
|
||
|
||
echo "✅ Current state saved"
|
||
}
|
||
}
|
||
}
|
||
|
||
stage('Checkout Source') {
|
||
steps {
|
||
echo "📦 Creating application artifacts..."
|
||
|
||
sh """
|
||
cat > Dockerfile << 'EOF'
|
||
FROM nginx:1.25.3-alpine
|
||
COPY index.html /usr/share/nginx/html/index.html
|
||
COPY nginx.conf /etc/nginx/nginx.conf
|
||
EXPOSE 80
|
||
CMD ["nginx", "-g", "daemon off;"]
|
||
EOF
|
||
"""
|
||
|
||
sh """
|
||
cat > index.html << EOF
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<title>Demo Nginx</title>
|
||
<style>
|
||
body {
|
||
font-family: Arial, sans-serif;
|
||
max-width: 800px;
|
||
margin: 50px auto;
|
||
padding: 20px;
|
||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||
color: white;
|
||
}
|
||
.container {
|
||
background: rgba(255, 255, 255, 0.1);
|
||
padding: 40px;
|
||
border-radius: 10px;
|
||
backdrop-filter: blur(10px);
|
||
}
|
||
h1 {
|
||
font-size: 48px;
|
||
margin-bottom: 20px;
|
||
}
|
||
p {
|
||
font-size: 24px;
|
||
margin: 10px 0;
|
||
}
|
||
.version {
|
||
font-family: 'Courier New', monospace;
|
||
background: rgba(0, 0, 0, 0.3);
|
||
padding: 10px 20px;
|
||
border-radius: 5px;
|
||
display: inline-block;
|
||
margin-top: 20px;
|
||
}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<div class="container">
|
||
<h1>Demo Nginx - Build #${BUILD_NUMBER}</h1>
|
||
<p>Environment: Production</p>
|
||
<p class="version">Version: ${IMAGE_TAG}</p>
|
||
<p style="font-size: 16px; margin-top: 30px; opacity: 0.8;">
|
||
Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
||
</p>
|
||
</div>
|
||
</body>
|
||
</html>
|
||
EOF
|
||
"""
|
||
|
||
sh '''
|
||
cat > nginx.conf << 'EOF'
|
||
user nginx;
|
||
worker_processes auto;
|
||
error_log /var/log/nginx/error.log warn;
|
||
pid /var/run/nginx.pid;
|
||
events { worker_connections 1024; }
|
||
http {
|
||
include /etc/nginx/mime.types;
|
||
default_type application/octet-stream;
|
||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||
'$status $body_bytes_sent "$http_referer" '
|
||
'"$http_user_agent" "$http_x_forwarded_for"';
|
||
access_log /var/log/nginx/access.log main;
|
||
sendfile on;
|
||
keepalive_timeout 65;
|
||
server {
|
||
listen 80;
|
||
server_name _;
|
||
location / {
|
||
root /usr/share/nginx/html;
|
||
index index.html;
|
||
}
|
||
location /health {
|
||
access_log off;
|
||
return 200 "healthy\n";
|
||
add_header Content-Type text/plain;
|
||
}
|
||
}
|
||
}
|
||
EOF
|
||
'''
|
||
}
|
||
}
|
||
|
||
stage('Build Docker Image') {
|
||
steps {
|
||
script {
|
||
echo "🏗️ Building Docker image..."
|
||
|
||
sendTelegramNotification(
|
||
status: 'BUILDING',
|
||
message: """
|
||
🏗️ <b>Building Docker Image</b>
|
||
|
||
<b>Image:</b> ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
||
<b>Stage:</b> Build
|
||
""",
|
||
color: '🔵'
|
||
)
|
||
|
||
sh """
|
||
docker build \
|
||
-t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} \
|
||
-t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest \
|
||
.
|
||
"""
|
||
echo "✅ Image built successfully!"
|
||
}
|
||
}
|
||
}
|
||
|
||
stage('Push to Registry') {
|
||
when { branch 'main' }
|
||
steps {
|
||
script {
|
||
echo "📤 Pushing image to registry..."
|
||
|
||
sendTelegramNotification(
|
||
status: 'PUSHING',
|
||
message: """
|
||
📤 <b>Pushing to Registry</b>
|
||
|
||
<b>Registry:</b> ${DOCKER_REGISTRY}
|
||
<b>Image:</b> ${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
||
""",
|
||
color: '🔵'
|
||
)
|
||
|
||
withCredentials([usernamePassword(
|
||
credentialsId: 'docker-registry-credentials',
|
||
usernameVariable: 'DOCKER_USER',
|
||
passwordVariable: 'DOCKER_PASS'
|
||
)]) {
|
||
sh """
|
||
echo "\${DOCKER_PASS}" | docker login ${DOCKER_REGISTRY} -u "\${DOCKER_USER}" --password-stdin
|
||
docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
||
docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest
|
||
docker logout ${DOCKER_REGISTRY}
|
||
"""
|
||
}
|
||
echo "✅ Image pushed successfully!"
|
||
}
|
||
}
|
||
}
|
||
|
||
stage('Update GitOps Manifests') {
|
||
when { branch 'main' }
|
||
steps {
|
||
script {
|
||
echo "📝 Updating Kubernetes manifests..."
|
||
|
||
sendTelegramNotification(
|
||
status: 'UPDATING',
|
||
message: """
|
||
📝 <b>Updating GitOps Manifests</b>
|
||
|
||
<b>Repository:</b> ${GITEA_REPO}
|
||
<b>Branch:</b> ${GITEA_BRANCH}
|
||
<b>File:</b> apps/demo-nginx/deployment.yaml
|
||
""",
|
||
color: '🔵'
|
||
)
|
||
|
||
withCredentials([usernamePassword(
|
||
credentialsId: 'gitea-credentials',
|
||
usernameVariable: 'GIT_USER',
|
||
passwordVariable: 'GIT_PASS'
|
||
)]) {
|
||
sh """
|
||
rm -rf k3s-gitops || true
|
||
git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git
|
||
cd k3s-gitops
|
||
git config user.name "Jenkins"
|
||
git config user.email "jenkins@thedevops.dev"
|
||
|
||
# Save current commit for rollback
|
||
git rev-parse HEAD > /tmp/previous_commit_${BUILD_NUMBER}.txt
|
||
|
||
sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml
|
||
git add apps/demo-nginx/deployment.yaml
|
||
git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes"
|
||
git push origin main
|
||
"""
|
||
}
|
||
echo "✅ Manifests updated!"
|
||
}
|
||
}
|
||
}
|
||
|
||
stage('Wait for ArgoCD Sync') {
|
||
steps {
|
||
script {
|
||
echo "⏳ Waiting for ArgoCD to apply GitOps commit..."
|
||
|
||
def expectedRevision = env.GITOPS_COMMIT
|
||
|
||
for (int i = 1; i <= 12; i++) {
|
||
def argoRevision = sh(
|
||
script: "kubectl get application demo-nginx -n argocd -o jsonpath='{.status.sync.revision}'",
|
||
returnStdout: true
|
||
).trim()
|
||
|
||
def syncStatus = sh(
|
||
script: "kubectl get application demo-nginx -n argocd -o jsonpath='{.status.sync.status}'",
|
||
returnStdout: true
|
||
).trim()
|
||
|
||
echo "Expected GitOps revision : ${expectedRevision}"
|
||
echo "ArgoCD applied revision : ${argoRevision}"
|
||
echo "ArgoCD sync status : ${syncStatus}"
|
||
|
||
if (syncStatus == "Synced" && argoRevision == expectedRevision) {
|
||
echo "✅ ArgoCD successfully applied GitOps commit"
|
||
return
|
||
}
|
||
|
||
sleep 10
|
||
}
|
||
|
||
error("❌ ArgoCD did not apply expected GitOps revision in time")
|
||
}
|
||
}
|
||
}
|
||
|
||
|
||
|
||
stage('Wait for Deployment') {
|
||
when { branch 'main' }
|
||
steps {
|
||
script {
|
||
echo "⏳ Waiting for Kubernetes rollout to complete..."
|
||
|
||
sendTelegramNotification(
|
||
status: 'DEPLOYING',
|
||
message: """
|
||
🚀 <b>Deploying to Kubernetes</b>
|
||
|
||
<b>Deployment:</b> ${APP_NAME}
|
||
<b>Namespace:</b> ${NAMESPACE}
|
||
<b>Image:</b> ${IMAGE_TAG}
|
||
<b>Timeout:</b> ${DEPLOYMENT_TIMEOUT}
|
||
|
||
<i>Rolling out new pods...</i>
|
||
""",
|
||
color: '🔵'
|
||
)
|
||
|
||
try {
|
||
// Wait for rollout to complete
|
||
echo "Starting rollout status check..."
|
||
sh """
|
||
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT}
|
||
"""
|
||
echo "✅ Rollout completed successfully!"
|
||
|
||
// Additional verification - wait a bit for pods to stabilize
|
||
echo "Waiting 10 seconds for pods to stabilize..."
|
||
sleep 10
|
||
|
||
} catch (Exception e) {
|
||
echo "❌ Deployment rollout failed: ${e.message}"
|
||
|
||
// Get pod status for debugging
|
||
try {
|
||
def podStatus = sh(
|
||
script: """
|
||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide
|
||
""",
|
||
returnStdout: true
|
||
).trim()
|
||
echo "Current pod status:\n${podStatus}"
|
||
|
||
def events = sh(
|
||
script: """
|
||
kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20
|
||
""",
|
||
returnStdout: true
|
||
).trim()
|
||
echo "Recent events:\n${events}"
|
||
} catch (Exception debugEx) {
|
||
echo "Could not fetch debug info: ${debugEx.message}"
|
||
}
|
||
|
||
throw e
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
stage('Verify Deployment') {
|
||
when { branch 'main' }
|
||
steps {
|
||
script {
|
||
echo "✅ Verifying deployment and pod status..."
|
||
|
||
try {
|
||
def verifyResult = sh(script: """#!/bin/bash
|
||
set -e
|
||
|
||
echo "================================================"
|
||
echo "DEPLOYMENT VERIFICATION"
|
||
echo "================================================"
|
||
|
||
# 1. Check deployment status
|
||
echo ""
|
||
echo "1. Checking deployment status..."
|
||
READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}')
|
||
DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}')
|
||
UPDATED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.updatedReplicas}')
|
||
AVAILABLE_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.availableReplicas}')
|
||
|
||
echo " Desired replicas: \${DESIRED_PODS}"
|
||
echo " Updated replicas: \${UPDATED_PODS}"
|
||
echo " Ready replicas: \${READY_PODS}"
|
||
echo " Available replicas: \${AVAILABLE_PODS}"
|
||
|
||
if [ "\${READY_PODS}" != "\${DESIRED_PODS}" ]; then
|
||
echo " ❌ FAILED: Not all pods are ready!"
|
||
echo " Expected: \${DESIRED_PODS}, Got: \${READY_PODS}"
|
||
exit 1
|
||
fi
|
||
echo " ✅ All pods ready"
|
||
|
||
# 2. Verify deployment spec image
|
||
echo ""
|
||
echo "2. Checking deployment spec image..."
|
||
DEPLOYMENT_IMAGE=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}')
|
||
echo " Deployment spec image: \${DEPLOYMENT_IMAGE}"
|
||
echo " Expected tag: ${IMAGE_TAG}"
|
||
|
||
if [[ "\${DEPLOYMENT_IMAGE}" != *"${IMAGE_TAG}"* ]]; then
|
||
echo " ❌ FAILED: Deployment spec has wrong image!"
|
||
exit 1
|
||
fi
|
||
echo " ✅ Deployment spec correct"
|
||
|
||
# 3. CRITICAL: Verify actual running pod images
|
||
echo ""
|
||
echo "3. Checking actual running pod images..."
|
||
POD_IMAGES=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.status.containerStatuses[0].image}{"\\n"}{end}')
|
||
|
||
echo " Running pod images:"
|
||
echo "\${POD_IMAGES}" | while read -r img; do
|
||
echo " - \${img}"
|
||
done
|
||
|
||
# Check if all pods are running the correct image
|
||
WRONG_IMAGE_COUNT=0
|
||
while IFS= read -r img; do
|
||
if [[ "\${img}" != *"${IMAGE_TAG}"* ]]; then
|
||
echo " ❌ Pod running wrong image: \${img}"
|
||
WRONG_IMAGE_COUNT=\$((WRONG_IMAGE_COUNT + 1))
|
||
fi
|
||
done <<< "\${POD_IMAGES}"
|
||
|
||
if [ \${WRONG_IMAGE_COUNT} -gt 0 ]; then
|
||
echo " ❌ FAILED: \${WRONG_IMAGE_COUNT} pod(s) running old image!"
|
||
echo " This is the ArgoCD sync bug - deployment updated but pods not rolled out"
|
||
exit 1
|
||
fi
|
||
echo " ✅ All pods running correct image"
|
||
|
||
# 4. Check pod readiness
|
||
echo ""
|
||
echo "4. Checking pod readiness probes..."
|
||
NOT_READY=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} --field-selector=status.phase!=Running --no-headers 2>/dev/null | wc -l)
|
||
|
||
if [ "\${NOT_READY}" -gt 0 ]; then
|
||
echo " ⚠️ WARNING: \${NOT_READY} pod(s) not in Running state"
|
||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME}
|
||
else
|
||
echo " ✅ All pods in Running state"
|
||
fi
|
||
|
||
# 5. Check container restart count
|
||
echo ""
|
||
echo "5. Checking for container restarts..."
|
||
RESTART_COUNTS=\$(kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\\n"}{end}')
|
||
MAX_RESTARTS=0
|
||
while IFS= read -r count; do
|
||
if [ "\${count}" -gt "\${MAX_RESTARTS}" ]; then
|
||
MAX_RESTARTS=\${count}
|
||
fi
|
||
done <<< "\${RESTART_COUNTS}"
|
||
|
||
echo " Max restart count: \${MAX_RESTARTS}"
|
||
if [ "\${MAX_RESTARTS}" -gt 3 ]; then
|
||
echo " ⚠️ WARNING: High restart count detected"
|
||
else
|
||
echo " ✅ Restart count acceptable"
|
||
fi
|
||
|
||
echo ""
|
||
echo "================================================"
|
||
echo "✅ ALL VERIFICATION CHECKS PASSED!"
|
||
echo "================================================"
|
||
""", returnStdout: true).trim()
|
||
|
||
echo verifyResult
|
||
echo "✅ Deployment verified successfully!"
|
||
|
||
} catch (Exception e) {
|
||
echo "❌ Deployment verification failed!"
|
||
echo "Error: ${e.message}"
|
||
|
||
// Additional debugging
|
||
try {
|
||
echo "\n=== DEBUGGING INFORMATION ==="
|
||
|
||
def pods = sh(
|
||
script: "kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide",
|
||
returnStdout: true
|
||
).trim()
|
||
echo "Current pods:\n${pods}"
|
||
|
||
def replicaset = sh(
|
||
script: "kubectl get replicaset -n ${NAMESPACE} -l app=${APP_NAME}",
|
||
returnStdout: true
|
||
).trim()
|
||
echo "ReplicaSets:\n${replicaset}"
|
||
|
||
def events = sh(
|
||
script: "kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20",
|
||
returnStdout: true
|
||
).trim()
|
||
echo "Recent events:\n${events}"
|
||
|
||
} catch (Exception debugEx) {
|
||
echo "Could not fetch debug info: ${debugEx.message}"
|
||
}
|
||
|
||
throw e
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
post {
|
||
success {
|
||
script {
|
||
DEPLOYMENT_END_TIME = sh(script: 'date +%s', returnStdout: true).trim()
|
||
def duration = (DEPLOYMENT_END_TIME.toInteger() - DEPLOYMENT_START_TIME.toInteger())
|
||
def durationMin = duration / 60
|
||
def durationSec = duration % 60
|
||
|
||
// Get deployment details
|
||
def podStatus = sh(
|
||
script: """
|
||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} \
|
||
-o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.phase}{" "}{.status.containerStatuses[0].restartCount}{"\\n"}{end}'
|
||
""",
|
||
returnStdout: true
|
||
).trim()
|
||
|
||
def deployedImage = sh(
|
||
script: """
|
||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
||
""",
|
||
returnStdout: true
|
||
).trim()
|
||
|
||
def replicas = sh(
|
||
script: """
|
||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
||
-o jsonpath='{.status.replicas}'
|
||
""",
|
||
returnStdout: true
|
||
).trim()
|
||
|
||
echo """
|
||
✅ DEPLOYMENT SUCCESS!
|
||
|
||
Application: ${APP_NAME}
|
||
Image: ${deployedImage}
|
||
Namespace: ${NAMESPACE}
|
||
Build: #${BUILD_NUMBER}
|
||
Duration: ${durationMin}m ${durationSec}s
|
||
|
||
All checks passed! ✨
|
||
"""
|
||
|
||
// Send detailed success notification
|
||
sendTelegramNotification(
|
||
status: 'SUCCESS',
|
||
message: """
|
||
✅ <b>Deployment Successful!</b>
|
||
|
||
━━━━━━━━━━━━━━━━━━━━━━
|
||
<b>📦 Application Details</b>
|
||
<b>Name:</b> ${APP_NAME}
|
||
<b>Build:</b> #${BUILD_NUMBER}
|
||
<b>Branch:</b> ${env.BRANCH_NAME}
|
||
<b>Namespace:</b> ${NAMESPACE}
|
||
|
||
━━━━━━━━━━━━━━━━━━━━━━
|
||
<b>🐳 Docker Image</b>
|
||
<b>Registry:</b> ${DOCKER_REGISTRY}
|
||
<b>Repository:</b> ${DOCKER_REPO}/${APP_NAME}
|
||
<b>Tag:</b> ${IMAGE_TAG}
|
||
<b>Full Image:</b> ${deployedImage}
|
||
|
||
━━━━━━━━━━━━━━━━━━━━━━
|
||
<b>☸️ Kubernetes Status</b>
|
||
<b>Replicas:</b> ${replicas}/${replicas} Ready
|
||
<b>Rollout:</b> Completed ✅
|
||
<b>Health:</b> All pods healthy
|
||
|
||
━━━━━━━━━━━━━━━━━━━━━━
|
||
<b>⏱️ Deployment Metrics</b>
|
||
<b>Duration:</b> ${durationMin}m ${durationSec}s
|
||
<b>Started:</b> ${new Date(DEPLOYMENT_START_TIME.toLong() * 1000).format('HH:mm:ss')}
|
||
<b>Completed:</b> ${new Date(DEPLOYMENT_END_TIME.toLong() * 1000).format('HH:mm:ss')}
|
||
|
||
━━━━━━━━━━━━━━━━━━━━━━
|
||
<b>🔗 Links</b>
|
||
<a href="${BUILD_URL}">Jenkins Build</a>
|
||
<a href="${GITEA_URL}/${GITEA_REPO}">GitOps Repository</a>
|
||
|
||
<i>Deployed by Jenkins CI/CD Pipeline</i> 🚀
|
||
""",
|
||
color: '✅'
|
||
)
|
||
|
||
// Cleanup rollback files
|
||
sh """
|
||
rm -f /tmp/previous_image_${BUILD_NUMBER}.txt
|
||
rm -f /tmp/previous_replicas_${BUILD_NUMBER}.txt
|
||
rm -f /tmp/previous_commit_${BUILD_NUMBER}.txt
|
||
"""
|
||
}
|
||
}
|
||
|
||
failure {
|
||
script {
|
||
def errorDetails = ""
|
||
def previousImage = "unknown"
|
||
|
||
try {
|
||
errorDetails = sh(
|
||
script: """
|
||
kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -10
|
||
""",
|
||
returnStdout: true
|
||
).trim()
|
||
} catch (Exception e) {
|
||
errorDetails = "Could not fetch events: ${e.message}"
|
||
}
|
||
|
||
if (env.BRANCH_NAME == 'main' && env.ROLLBACK_ENABLED == 'true') {
|
||
echo """
|
||
❌ DEPLOYMENT FAILED - INITIATING ROLLBACK!
|
||
|
||
Rolling back to previous version...
|
||
"""
|
||
|
||
sendTelegramNotification(
|
||
status: 'ROLLING_BACK',
|
||
message: """
|
||
🔄 <b>Deployment Failed - Rolling Back</b>
|
||
|
||
<b>Application:</b> ${APP_NAME}
|
||
<b>Failed Build:</b> #${BUILD_NUMBER}
|
||
<b>Image:</b> ${IMAGE_TAG}
|
||
|
||
<i>Initiating automatic rollback...</i>
|
||
""",
|
||
color: '⚠️'
|
||
)
|
||
|
||
try {
|
||
previousImage = sh(
|
||
script: "cat /tmp/previous_image_${BUILD_NUMBER}.txt 2>/dev/null || echo 'none'",
|
||
returnStdout: true
|
||
).trim()
|
||
|
||
if (previousImage != 'none' && previousImage != '') {
|
||
echo "🔄 Rolling back to: ${previousImage}"
|
||
|
||
sh """
|
||
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
|
||
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=180s
|
||
"""
|
||
|
||
withCredentials([usernamePassword(
|
||
credentialsId: 'gitea-credentials',
|
||
usernameVariable: 'GIT_USER',
|
||
passwordVariable: 'GIT_PASS'
|
||
)]) {
|
||
sh """
|
||
cd k3s-gitops || git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git
|
||
cd k3s-gitops
|
||
git config user.name "Jenkins"
|
||
git config user.email "jenkins@thedevops.dev"
|
||
git revert --no-edit HEAD || true
|
||
git push origin main || true
|
||
"""
|
||
}
|
||
|
||
sendTelegramNotification(
|
||
status: 'ROLLBACK_SUCCESS',
|
||
message: """
|
||
✅ <b>Rollback Completed</b>
|
||
|
||
━━━━━━━━━━━━━━━━━━━━━━
|
||
<b>📦 Application</b>
|
||
<b>Name:</b> ${APP_NAME}
|
||
<b>Failed Build:</b> #${BUILD_NUMBER}
|
||
<b>Rolled Back To:</b> ${previousImage}
|
||
|
||
━━━━━━━━━━━━━━━━━━━━━━
|
||
<b>🔄 Rollback Status</b>
|
||
<b>Status:</b> Success ✅
|
||
<b>Method:</b> kubectl rollout undo
|
||
<b>Git:</b> Commit reverted
|
||
|
||
━━━━━━━━━━━━━━━━━━━━━━
|
||
<b>📋 Recent Events</b>
|
||
<code>${errorDetails.take(500)}</code>
|
||
|
||
━━━━━━━━━━━━━━━━━━━━━━
|
||
<b>⚠️ Action Required</b>
|
||
Please review logs and fix issues before redeploying
|
||
|
||
<a href="${BUILD_URL}console">View Build Logs</a>
|
||
""",
|
||
color: '✅'
|
||
)
|
||
|
||
} else {
|
||
sendTelegramNotification(
|
||
status: 'ROLLBACK_FAILED',
|
||
message: """
|
||
❌ <b>Rollback Failed</b>
|
||
|
||
<b>Application:</b> ${APP_NAME}
|
||
<b>Build:</b> #${BUILD_NUMBER}
|
||
|
||
<b>Error:</b> No previous version found
|
||
|
||
<b>⚠️ MANUAL INTERVENTION REQUIRED</b>
|
||
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
|
||
|
||
<a href="${BUILD_URL}console">View Build Logs</a>
|
||
""",
|
||
color: '🔴'
|
||
)
|
||
}
|
||
|
||
} catch (Exception e) {
|
||
sendTelegramNotification(
|
||
status: 'ROLLBACK_FAILED',
|
||
message: """
|
||
❌ <b>Rollback Failed</b>
|
||
|
||
<b>Application:</b> ${APP_NAME}
|
||
<b>Build:</b> #${BUILD_NUMBER}
|
||
|
||
<b>Error:</b> ${e.message}
|
||
|
||
<b>⚠️ MANUAL ROLLBACK REQUIRED</b>
|
||
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
|
||
|
||
<a href="${BUILD_URL}console">View Build Logs</a>
|
||
""",
|
||
color: '🔴'
|
||
)
|
||
}
|
||
|
||
} else {
|
||
sendTelegramNotification(
|
||
status: 'FAILED',
|
||
message: """
|
||
❌ <b>Deployment Failed</b>
|
||
|
||
━━━━━━━━━━━━━━━━━━━━━━
|
||
<b>📦 Application</b>
|
||
<b>Name:</b> ${APP_NAME}
|
||
<b>Build:</b> #${BUILD_NUMBER}
|
||
<b>Branch:</b> ${env.BRANCH_NAME}
|
||
<b>Image:</b> ${IMAGE_TAG}
|
||
|
||
━━━━━━━━━━━━━━━━━━━━━━
|
||
<b>📋 Recent Events</b>
|
||
<code>${errorDetails.take(500)}</code>
|
||
|
||
━━━━━━━━━━━━━━━━━━━━━━
|
||
<b>🔗 Links</b>
|
||
<a href="${BUILD_URL}console">View Console Output</a>
|
||
<a href="${BUILD_URL}">Build Details</a>
|
||
|
||
<i>Rollback: ${env.ROLLBACK_ENABLED == 'true' ? 'Enabled but not on main branch' : 'Disabled'}</i>
|
||
""",
|
||
color: '🔴'
|
||
)
|
||
}
|
||
}
|
||
}
|
||
|
||
always {
|
||
sh """
|
||
docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true
|
||
docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest || true
|
||
docker stop test-${BUILD_NUMBER} 2>/dev/null || true
|
||
docker rm test-${BUILD_NUMBER} 2>/dev/null || true
|
||
"""
|
||
cleanWs()
|
||
}
|
||
}
|
||
}
|
||
|
||
// Telegram notification function
|
||
def sendTelegramNotification(Map args) {
|
||
def status = args.status
|
||
def message = args.message
|
||
def color = args.color ?: '🔵'
|
||
|
||
try {
|
||
// HTML formatting for Telegram
|
||
def formattedMessage = message.trim()
|
||
|
||
sh """
|
||
curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
|
||
-d chat_id="${TELEGRAM_CHAT_ID}" \
|
||
-d parse_mode="HTML" \
|
||
-d disable_web_page_preview=true \
|
||
-d text="${formattedMessage}"
|
||
"""
|
||
|
||
echo "📱 Telegram notification sent: ${status}"
|
||
} catch (Exception e) {
|
||
echo "⚠️ Failed to send Telegram notification: ${e.message}"
|
||
// Don't fail the pipeline if notification fails
|
||
}
|
||
} |