chore: remove demo-nginx app
This commit is contained in:
@@ -1,69 +0,0 @@
|
||||
############################################
|
||||
# Application
|
||||
############################################
|
||||
APP_NAME=demo-nginx
|
||||
NAMESPACE=demo-app
|
||||
BRANCH_NAME=main
|
||||
|
||||
############################################
|
||||
# Docker
|
||||
############################################
|
||||
DOCKER_REGISTRY=docker.io
|
||||
DOCKER_REPO=vladcrypto
|
||||
# IMAGE_TAG is computed in Jenkins as: ${BRANCH_NAME}-${BUILD_NUMBER}
|
||||
# Example: main-100
|
||||
|
||||
############################################
|
||||
# GitOps / Gitea
|
||||
############################################
|
||||
GITEA_URL=http://gitea-http.gitea.svc.cluster.local:3000
|
||||
GITEA_REPO=admin/k3s-gitops
|
||||
GITEA_BRANCH=main
|
||||
GITOPS_MANIFEST_PATH=apps/demo-nginx/deployment.yaml
|
||||
|
||||
############################################
|
||||
# ArgoCD
|
||||
############################################
|
||||
ARGOCD_APP_NAME=demo-nginx
|
||||
ARGOCD_NAMESPACE=argocd
|
||||
ARGOCD_SYNC_TIMEOUT=120 # seconds
|
||||
|
||||
############################################
|
||||
# Deployment / Rollout
|
||||
############################################
|
||||
DEPLOYMENT_TIMEOUT=300s
|
||||
SKIP_HEALTH_CHECK=true
|
||||
ROLLBACK_ENABLED=false # GitOps-safe: Jenkins never reverts Git
|
||||
|
||||
############################################
|
||||
# Jenkins Runtime (injected by Jenkins)
|
||||
############################################
|
||||
# BUILD_NUMBER=100
|
||||
# BUILD_URL=https://jenkins.thedevops.dev/job/demo-nginx/job/main/100/
|
||||
# BUILD_USER=jenkins
|
||||
|
||||
############################################
|
||||
# Telegram Notifications
|
||||
############################################
|
||||
TELEGRAM_BOT_TOKEN=__REPLACE_ME__
|
||||
TELEGRAM_CHAT_ID=__REPLACE_ME__
|
||||
|
||||
############################################
|
||||
# Docker Hub Credentials
|
||||
############################################
|
||||
DOCKER_USER=__REPLACE_ME__
|
||||
DOCKER_PASS=__REPLACE_ME__
|
||||
|
||||
############################################
|
||||
# Gitea Credentials
|
||||
############################################
|
||||
GIT_USER=__REPLACE_ME__
|
||||
GIT_PASS=__REPLACE_ME__
|
||||
|
||||
############################################
|
||||
# Internal / Temporary Files (used by pipeline)
|
||||
############################################
|
||||
PREVIOUS_IMAGE_FILE=/tmp/previous_image_${BUILD_NUMBER}.txt
|
||||
PREVIOUS_REPLICAS_FILE=/tmp/previous_replicas_${BUILD_NUMBER}.txt
|
||||
PREVIOUS_COMMIT_FILE=/tmp/previous_commit_${BUILD_NUMBER}.txt
|
||||
EXPECTED_COMMIT_FILE=/tmp/expected_commit_${BUILD_NUMBER}.txt
|
||||
763
apps/demo-nginx/Jenkinsfile
vendored
763
apps/demo-nginx/Jenkinsfile
vendored
@@ -1,763 +0,0 @@
|
||||
pipeline {
|
||||
agent any
|
||||
options {
|
||||
disableConcurrentBuilds()
|
||||
}
|
||||
environment {
|
||||
APP_NAME = 'demo-nginx'
|
||||
NAMESPACE = 'demo-app'
|
||||
DOCKER_REGISTRY = 'docker.io'
|
||||
DOCKER_REPO = 'vladcrypto'
|
||||
GITEA_URL = 'http://gitea-http.gitea.svc.cluster.local:3000'
|
||||
GITEA_REPO = 'admin/k3s-gitops'
|
||||
GITEA_BRANCH = 'main'
|
||||
BUILD_TAG = "${env.BUILD_NUMBER}"
|
||||
IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}"
|
||||
|
||||
// Rollback configuration
|
||||
ROLLBACK_ENABLED = 'true'
|
||||
DEPLOYMENT_TIMEOUT = '300s'
|
||||
ARGOCD_SYNC_TIMEOUT = '120'
|
||||
SKIP_HEALTH_CHECK = 'true'
|
||||
|
||||
// Notification configuration
|
||||
TELEGRAM_BOT_TOKEN = credentials('telegram-bot-token')
|
||||
TELEGRAM_CHAT_ID = credentials('telegram-chat-id')
|
||||
|
||||
// Build info
|
||||
BUILD_URL = "${env.BUILD_URL}"
|
||||
GIT_COMMIT_SHORT = ""
|
||||
DEPLOYMENT_START_TIME = ""
|
||||
DEPLOYMENT_END_TIME = ""
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Initialization') {
|
||||
steps {
|
||||
script {
|
||||
echo "🚀 Starting deployment pipeline..."
|
||||
DEPLOYMENT_START_TIME = sh(script: 'date +%s', returnStdout: true).trim()
|
||||
|
||||
sendTelegramNotification(
|
||||
status: 'STARTED',
|
||||
message: """
|
||||
🚀 <b>Deployment Started</b>
|
||||
|
||||
<b>Application:</b> ${APP_NAME}
|
||||
<b>Build:</b> #${BUILD_NUMBER}
|
||||
<b>Branch:</b> ${env.BRANCH_NAME}
|
||||
<b>Namespace:</b> ${NAMESPACE}
|
||||
<b>Started by:</b> ${env.BUILD_USER ?: 'Jenkins'}
|
||||
|
||||
<i>Building and deploying...</i>
|
||||
""",
|
||||
color: '🔵'
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Save Current State') {
|
||||
when { branch 'main' }
|
||||
steps {
|
||||
script {
|
||||
echo "📸 Saving current deployment state for rollback..."
|
||||
|
||||
sh """
|
||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].image}' \
|
||||
> /tmp/previous_image_${BUILD_NUMBER}.txt || echo "none" > /tmp/previous_image_${BUILD_NUMBER}.txt
|
||||
|
||||
PREV_IMAGE=\$(cat /tmp/previous_image_${BUILD_NUMBER}.txt)
|
||||
echo "Previous image: \${PREV_IMAGE}"
|
||||
"""
|
||||
|
||||
sh """
|
||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
||||
-o jsonpath='{.spec.replicas}' \
|
||||
> /tmp/previous_replicas_${BUILD_NUMBER}.txt || echo "2" > /tmp/previous_replicas_${BUILD_NUMBER}.txt
|
||||
"""
|
||||
|
||||
echo "✅ Current state saved"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Checkout Source') {
|
||||
steps {
|
||||
echo "📦 Creating application artifacts..."
|
||||
|
||||
sh """
|
||||
cat > Dockerfile << 'EOF'
|
||||
FROM nginx:1.25.3-alpine
|
||||
COPY index.html /usr/share/nginx/html/index.html
|
||||
COPY nginx.conf /etc/nginx/nginx.conf
|
||||
EXPOSE 80
|
||||
CMD ["nginx", "-g", "daemon off;"]
|
||||
EOF
|
||||
"""
|
||||
|
||||
sh """
|
||||
cat > index.html << EOF
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Demo Nginx</title>
|
||||
<style>
|
||||
body {
|
||||
font-family: Arial, sans-serif;
|
||||
max-width: 800px;
|
||||
margin: 50px auto;
|
||||
padding: 20px;
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: white;
|
||||
}
|
||||
.container {
|
||||
background: rgba(255, 255, 255, 0.1);
|
||||
padding: 40px;
|
||||
border-radius: 10px;
|
||||
backdrop-filter: blur(10px);
|
||||
}
|
||||
h1 {
|
||||
font-size: 48px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
p {
|
||||
font-size: 24px;
|
||||
margin: 10px 0;
|
||||
}
|
||||
.version {
|
||||
font-family: 'Courier New', monospace;
|
||||
background: rgba(0, 0, 0, 0.3);
|
||||
padding: 10px 20px;
|
||||
border-radius: 5px;
|
||||
display: inline-block;
|
||||
margin-top: 20px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>Demo Nginx - Build #${BUILD_NUMBER}</h1>
|
||||
<p>Environment: Production</p>
|
||||
<p class="version">Version: ${IMAGE_TAG}</p>
|
||||
<p style="font-size: 16px; margin-top: 30px; opacity: 0.8;">
|
||||
Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
||||
</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
"""
|
||||
|
||||
sh '''
|
||||
cat > nginx.conf << 'EOF'
|
||||
user nginx;
|
||||
worker_processes auto;
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
pid /var/run/nginx.pid;
|
||||
events { worker_connections 1024; }
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||
access_log /var/log/nginx/access.log main;
|
||||
sendfile on;
|
||||
keepalive_timeout 65;
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
location / {
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
}
|
||||
location /health {
|
||||
access_log off;
|
||||
return 200 "healthy\n";
|
||||
add_header Content-Type text/plain;
|
||||
}
|
||||
}
|
||||
}
|
||||
EOF
|
||||
'''
|
||||
}
|
||||
}
|
||||
|
||||
stage('Build Docker Image') {
|
||||
steps {
|
||||
script {
|
||||
echo "🏗️ Building Docker image..."
|
||||
|
||||
sendTelegramNotification(
|
||||
status: 'BUILDING',
|
||||
message: """
|
||||
🏗️ <b>Building Docker Image</b>
|
||||
|
||||
<b>Image:</b> ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
||||
<b>Stage:</b> Build
|
||||
""",
|
||||
color: '🔵'
|
||||
)
|
||||
|
||||
sh """
|
||||
docker build \
|
||||
-t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} \
|
||||
-t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest \
|
||||
.
|
||||
"""
|
||||
echo "✅ Image built successfully!"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Push to Registry') {
|
||||
when { branch 'main' }
|
||||
steps {
|
||||
script {
|
||||
echo "📤 Pushing image to registry..."
|
||||
|
||||
sendTelegramNotification(
|
||||
status: 'PUSHING',
|
||||
message: """
|
||||
📤 <b>Pushing to Registry</b>
|
||||
|
||||
<b>Registry:</b> ${DOCKER_REGISTRY}
|
||||
<b>Image:</b> ${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
||||
""",
|
||||
color: '🔵'
|
||||
)
|
||||
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'docker-registry-credentials',
|
||||
usernameVariable: 'DOCKER_USER',
|
||||
passwordVariable: 'DOCKER_PASS'
|
||||
)]) {
|
||||
sh """
|
||||
echo "\${DOCKER_PASS}" | docker login ${DOCKER_REGISTRY} -u "\${DOCKER_USER}" --password-stdin
|
||||
docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
||||
docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest
|
||||
docker logout ${DOCKER_REGISTRY}
|
||||
"""
|
||||
}
|
||||
echo "✅ Image pushed successfully!"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Update GitOps Manifests') {
|
||||
when { branch 'main' }
|
||||
steps {
|
||||
script {
|
||||
echo "📝 Updating Kubernetes manifests..."
|
||||
|
||||
sendTelegramNotification(
|
||||
status: 'UPDATING',
|
||||
message: """
|
||||
📝 <b>Updating GitOps Manifests</b>
|
||||
|
||||
<b>Repository:</b> ${GITEA_REPO}
|
||||
<b>Branch:</b> ${GITEA_BRANCH}
|
||||
<b>File:</b> apps/demo-nginx/deployment.yaml
|
||||
""",
|
||||
color: '🔵'
|
||||
)
|
||||
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'gitea-credentials',
|
||||
usernameVariable: 'GIT_USER',
|
||||
passwordVariable: 'GIT_PASS'
|
||||
)]) {
|
||||
sh """
|
||||
rm -rf k3s-gitops || true
|
||||
git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git
|
||||
cd k3s-gitops
|
||||
git config user.name "Jenkins"
|
||||
git config user.email "jenkins@thedevops.dev"
|
||||
|
||||
# Save current commit for rollback
|
||||
git rev-parse HEAD > /tmp/previous_commit_${BUILD_NUMBER}.txt
|
||||
|
||||
sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml
|
||||
git add apps/demo-nginx/deployment.yaml
|
||||
git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes"
|
||||
git push origin main
|
||||
"""
|
||||
}
|
||||
echo "✅ Manifests updated!"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Wait for ArgoCD Sync') {
|
||||
steps {
|
||||
script {
|
||||
echo "⏳ Waiting for ArgoCD to sync manifests..."
|
||||
|
||||
for (int i = 1; i <= 12; i++) {
|
||||
def syncStatus = sh(
|
||||
script: "kubectl get application demo-nginx -n argocd -o jsonpath='{.status.sync.status}'",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
echo "ArgoCD sync status : ${syncStatus}"
|
||||
|
||||
if (syncStatus == "Synced") {
|
||||
echo "✅ ArgoCD manifests synced"
|
||||
return
|
||||
}
|
||||
|
||||
sleep 10
|
||||
}
|
||||
|
||||
error("❌ ArgoCD did not sync manifests in time")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
stage('Wait for Deployment') {
|
||||
when { branch 'main' }
|
||||
steps {
|
||||
script {
|
||||
echo "⏳ Waiting for Kubernetes rollout to complete..."
|
||||
|
||||
sendTelegramNotification(
|
||||
status: 'DEPLOYING',
|
||||
message: """
|
||||
🚀 <b>Deploying to Kubernetes</b>
|
||||
|
||||
<b>Deployment:</b> ${APP_NAME}
|
||||
<b>Namespace:</b> ${NAMESPACE}
|
||||
<b>Image:</b> ${IMAGE_TAG}
|
||||
<b>Timeout:</b> ${DEPLOYMENT_TIMEOUT}
|
||||
|
||||
<i>Rolling out new pods...</i>
|
||||
""",
|
||||
color: '🔵'
|
||||
)
|
||||
|
||||
try {
|
||||
// Wait for rollout to complete
|
||||
echo "Starting rollout status check..."
|
||||
sh """
|
||||
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT}
|
||||
"""
|
||||
echo "✅ Rollout completed successfully!"
|
||||
|
||||
// Additional verification - wait a bit for pods to stabilize
|
||||
echo "Waiting 10 seconds for pods to stabilize..."
|
||||
sleep 10
|
||||
|
||||
} catch (Exception e) {
|
||||
echo "❌ Deployment rollout failed: ${e.message}"
|
||||
|
||||
// Get pod status for debugging
|
||||
try {
|
||||
def podStatus = sh(
|
||||
script: """
|
||||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide
|
||||
""",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
echo "Current pod status:\n${podStatus}"
|
||||
|
||||
def events = sh(
|
||||
script: """
|
||||
kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20
|
||||
""",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
echo "Recent events:\n${events}"
|
||||
} catch (Exception debugEx) {
|
||||
echo "Could not fetch debug info: ${debugEx.message}"
|
||||
}
|
||||
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Verify Deployment') {
|
||||
when { branch 'main' }
|
||||
steps {
|
||||
script {
|
||||
echo "✅ Verifying deployment and pod status..."
|
||||
|
||||
/* --------------------------------
|
||||
* 1. Deployment readiness
|
||||
* -------------------------------- */
|
||||
sh """
|
||||
set -e
|
||||
echo "1. Checking deployment readiness..."
|
||||
|
||||
READY=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}')
|
||||
DESIRED=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}')
|
||||
|
||||
echo "Ready replicas : \$READY"
|
||||
echo "Desired replicas : \$DESIRED"
|
||||
|
||||
if [ "\$READY" != "\$DESIRED" ]; then
|
||||
echo "❌ Not all replicas are ready"
|
||||
exit 1
|
||||
fi
|
||||
"""
|
||||
|
||||
/* --------------------------------
|
||||
* 2. Actual running images
|
||||
* -------------------------------- */
|
||||
def podImages = sh(
|
||||
script: """
|
||||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} \
|
||||
-o jsonpath='{range .items[*]}{.status.containerStatuses[0].image}{"\\n"}{end}'
|
||||
""",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
echo "Running pod images:"
|
||||
podImages.split("\\n").each { img ->
|
||||
echo " - ${img}"
|
||||
}
|
||||
|
||||
if (!podImages.contains(IMAGE_TAG)) {
|
||||
error("❌ Some pods are NOT running image tag ${IMAGE_TAG}")
|
||||
}
|
||||
|
||||
/* --------------------------------
|
||||
* 3. Restart count
|
||||
* -------------------------------- */
|
||||
def restarts = sh(
|
||||
script: """
|
||||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} \
|
||||
-o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\\n"}{end}'
|
||||
""",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
def maxRestart = restarts
|
||||
.split("\\n")
|
||||
.collect { it.toInteger() }
|
||||
.max()
|
||||
|
||||
echo "Max restart count: ${maxRestart}"
|
||||
|
||||
if (maxRestart > 3) {
|
||||
error("❌ High restart count detected: ${maxRestart}")
|
||||
}
|
||||
|
||||
echo "✅ ALL VERIFICATION CHECKS PASSED"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
post {
|
||||
success {
|
||||
script {
|
||||
DEPLOYMENT_END_TIME = sh(script: 'date +%s', returnStdout: true).trim()
|
||||
def duration = (DEPLOYMENT_END_TIME.toInteger() - DEPLOYMENT_START_TIME.toInteger())
|
||||
def durationMin = duration / 60
|
||||
def durationSec = duration % 60
|
||||
|
||||
// Get deployment details
|
||||
def podStatus = sh(
|
||||
script: """
|
||||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} \
|
||||
-o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.phase}{" "}{.status.containerStatuses[0].restartCount}{"\\n"}{end}'
|
||||
""",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
def deployedImage = sh(
|
||||
script: """
|
||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
||||
""",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
def replicas = sh(
|
||||
script: """
|
||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
||||
-o jsonpath='{.status.replicas}'
|
||||
""",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
echo """
|
||||
✅ DEPLOYMENT SUCCESS!
|
||||
|
||||
Application: ${APP_NAME}
|
||||
Image: ${deployedImage}
|
||||
Namespace: ${NAMESPACE}
|
||||
Build: #${BUILD_NUMBER}
|
||||
Duration: ${durationMin}m ${durationSec}s
|
||||
|
||||
All checks passed! ✨
|
||||
"""
|
||||
|
||||
// Send detailed success notification
|
||||
sendTelegramNotification(
|
||||
status: 'SUCCESS',
|
||||
message: """
|
||||
✅ <b>Deployment Successful!</b>
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━
|
||||
<b>📦 Application Details</b>
|
||||
<b>Name:</b> ${APP_NAME}
|
||||
<b>Build:</b> #${BUILD_NUMBER}
|
||||
<b>Branch:</b> ${env.BRANCH_NAME}
|
||||
<b>Namespace:</b> ${NAMESPACE}
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━
|
||||
<b>🐳 Docker Image</b>
|
||||
<b>Registry:</b> ${DOCKER_REGISTRY}
|
||||
<b>Repository:</b> ${DOCKER_REPO}/${APP_NAME}
|
||||
<b>Tag:</b> ${IMAGE_TAG}
|
||||
<b>Full Image:</b> ${deployedImage}
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━
|
||||
<b>☸️ Kubernetes Status</b>
|
||||
<b>Replicas:</b> ${replicas}/${replicas} Ready
|
||||
<b>Rollout:</b> Completed ✅
|
||||
<b>Health:</b> All pods healthy
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━
|
||||
<b>⏱️ Deployment Metrics</b>
|
||||
<b>Duration:</b> ${durationMin}m ${durationSec}s
|
||||
<b>Started:</b> ${new Date(DEPLOYMENT_START_TIME.toLong() * 1000).format('HH:mm:ss')}
|
||||
<b>Completed:</b> ${new Date(DEPLOYMENT_END_TIME.toLong() * 1000).format('HH:mm:ss')}
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━
|
||||
<b>🔗 Links</b>
|
||||
<a href="${BUILD_URL}">Jenkins Build</a>
|
||||
<a href="${GITEA_URL}/${GITEA_REPO}">GitOps Repository</a>
|
||||
|
||||
<i>Deployed by Jenkins CI/CD Pipeline</i> 🚀
|
||||
""",
|
||||
color: '✅'
|
||||
)
|
||||
|
||||
// Cleanup rollback files
|
||||
sh """
|
||||
rm -f /tmp/previous_image_${BUILD_NUMBER}.txt
|
||||
rm -f /tmp/previous_replicas_${BUILD_NUMBER}.txt
|
||||
rm -f /tmp/previous_commit_${BUILD_NUMBER}.txt
|
||||
"""
|
||||
}
|
||||
}
|
||||
|
||||
failure {
|
||||
script {
|
||||
def errorDetails = ""
|
||||
def previousImage = "unknown"
|
||||
|
||||
try {
|
||||
errorDetails = sh(
|
||||
script: """
|
||||
kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -10
|
||||
""",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
} catch (Exception e) {
|
||||
errorDetails = "Could not fetch events: ${e.message}"
|
||||
}
|
||||
|
||||
if (env.BRANCH_NAME == 'main' && env.ROLLBACK_ENABLED == 'true') {
|
||||
echo """
|
||||
❌ DEPLOYMENT FAILED - INITIATING ROLLBACK!
|
||||
|
||||
Rolling back to previous version...
|
||||
"""
|
||||
|
||||
sendTelegramNotification(
|
||||
status: 'ROLLING_BACK',
|
||||
message: """
|
||||
🔄 <b>Deployment Failed - Rolling Back</b>
|
||||
|
||||
<b>Application:</b> ${APP_NAME}
|
||||
<b>Failed Build:</b> #${BUILD_NUMBER}
|
||||
<b>Image:</b> ${IMAGE_TAG}
|
||||
|
||||
<i>Initiating automatic rollback...</i>
|
||||
""",
|
||||
color: '⚠️'
|
||||
)
|
||||
|
||||
try {
|
||||
previousImage = sh(
|
||||
script: "cat /tmp/previous_image_${BUILD_NUMBER}.txt 2>/dev/null || echo 'none'",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
if (previousImage != 'none' && previousImage != '') {
|
||||
echo "🔄 Rolling back to: ${previousImage}"
|
||||
|
||||
sh """
|
||||
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
|
||||
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=180s
|
||||
"""
|
||||
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'gitea-credentials',
|
||||
usernameVariable: 'GIT_USER',
|
||||
passwordVariable: 'GIT_PASS'
|
||||
)]) {
|
||||
sh """
|
||||
cd k3s-gitops || git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git
|
||||
cd k3s-gitops
|
||||
git config user.name "Jenkins"
|
||||
git config user.email "jenkins@thedevops.dev"
|
||||
git revert --no-edit HEAD || true
|
||||
git push origin main || true
|
||||
"""
|
||||
}
|
||||
|
||||
sendTelegramNotification(
|
||||
status: 'ROLLBACK_SUCCESS',
|
||||
message: """
|
||||
✅ <b>Rollback Completed</b>
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━
|
||||
<b>📦 Application</b>
|
||||
<b>Name:</b> ${APP_NAME}
|
||||
<b>Failed Build:</b> #${BUILD_NUMBER}
|
||||
<b>Rolled Back To:</b> ${previousImage}
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━
|
||||
<b>🔄 Rollback Status</b>
|
||||
<b>Status:</b> Success ✅
|
||||
<b>Method:</b> kubectl rollout undo
|
||||
<b>Git:</b> Commit reverted
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━
|
||||
<b>📋 Recent Events</b>
|
||||
<code>${errorDetails.take(500)}</code>
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━
|
||||
<b>⚠️ Action Required</b>
|
||||
Please review logs and fix issues before redeploying
|
||||
|
||||
<a href="${BUILD_URL}console">View Build Logs</a>
|
||||
""",
|
||||
color: '✅'
|
||||
)
|
||||
|
||||
} else {
|
||||
sendTelegramNotification(
|
||||
status: 'ROLLBACK_FAILED',
|
||||
message: """
|
||||
❌ <b>Rollback Failed</b>
|
||||
|
||||
<b>Application:</b> ${APP_NAME}
|
||||
<b>Build:</b> #${BUILD_NUMBER}
|
||||
|
||||
<b>Error:</b> No previous version found
|
||||
|
||||
<b>⚠️ MANUAL INTERVENTION REQUIRED</b>
|
||||
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
|
||||
|
||||
<a href="${BUILD_URL}console">View Build Logs</a>
|
||||
""",
|
||||
color: '🔴'
|
||||
)
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
sendTelegramNotification(
|
||||
status: 'ROLLBACK_FAILED',
|
||||
message: """
|
||||
❌ <b>Rollback Failed</b>
|
||||
|
||||
<b>Application:</b> ${APP_NAME}
|
||||
<b>Build:</b> #${BUILD_NUMBER}
|
||||
|
||||
<b>Error:</b> ${e.message}
|
||||
|
||||
<b>⚠️ MANUAL ROLLBACK REQUIRED</b>
|
||||
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
|
||||
|
||||
<a href="${BUILD_URL}console">View Build Logs</a>
|
||||
""",
|
||||
color: '🔴'
|
||||
)
|
||||
}
|
||||
|
||||
} else {
|
||||
sendTelegramNotification(
|
||||
status: 'FAILED',
|
||||
message: """
|
||||
❌ <b>Deployment Failed</b>
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━
|
||||
<b>📦 Application</b>
|
||||
<b>Name:</b> ${APP_NAME}
|
||||
<b>Build:</b> #${BUILD_NUMBER}
|
||||
<b>Branch:</b> ${env.BRANCH_NAME}
|
||||
<b>Image:</b> ${IMAGE_TAG}
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━
|
||||
<b>📋 Recent Events</b>
|
||||
<code>${errorDetails.take(500)}</code>
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━
|
||||
<b>🔗 Links</b>
|
||||
<a href="${BUILD_URL}console">View Console Output</a>
|
||||
<a href="${BUILD_URL}">Build Details</a>
|
||||
|
||||
<i>Rollback: ${env.ROLLBACK_ENABLED == 'true' ? 'Enabled but not on main branch' : 'Disabled'}</i>
|
||||
""",
|
||||
color: '🔴'
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
always {
|
||||
sh """
|
||||
docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true
|
||||
docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest || true
|
||||
docker stop test-${BUILD_NUMBER} 2>/dev/null || true
|
||||
docker rm test-${BUILD_NUMBER} 2>/dev/null || true
|
||||
"""
|
||||
cleanWs()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Telegram notification function
|
||||
def sendTelegramNotification(Map args) {
|
||||
def status = args.status
|
||||
def message = args.message
|
||||
def color = args.color ?: '🔵'
|
||||
|
||||
try {
|
||||
// HTML formatting for Telegram
|
||||
def formattedMessage = message.trim()
|
||||
|
||||
sh """
|
||||
curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
|
||||
-d chat_id="${TELEGRAM_CHAT_ID}" \
|
||||
-d parse_mode="HTML" \
|
||||
-d disable_web_page_preview=true \
|
||||
-d text="${formattedMessage}"
|
||||
"""
|
||||
|
||||
echo "📱 Telegram notification sent: ${status}"
|
||||
} catch (Exception e) {
|
||||
echo "⚠️ Failed to send Telegram notification: ${e.message}"
|
||||
// Don't fail the pipeline if notification fails
|
||||
}
|
||||
}
|
||||
@@ -1,449 +0,0 @@
|
||||
pipeline {
|
||||
agent any
|
||||
|
||||
environment {
|
||||
APP_NAME = 'demo-nginx'
|
||||
NAMESPACE = 'demo-app'
|
||||
DOCKER_REGISTRY = 'docker.io'
|
||||
DOCKER_REPO = 'vladcrypto'
|
||||
GITEA_URL = 'http://gitea-http.gitea.svc.cluster.local:3000'
|
||||
GITEA_REPO = 'admin/k3s-gitops'
|
||||
GITEA_BRANCH = 'main'
|
||||
BUILD_TAG = "${env.BUILD_NUMBER}"
|
||||
IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}"
|
||||
|
||||
// Rollback configuration
|
||||
ROLLBACK_ENABLED = 'true'
|
||||
DEPLOYMENT_TIMEOUT = '300s'
|
||||
ARGOCD_SYNC_TIMEOUT = '120'
|
||||
SKIP_HEALTH_CHECK = 'true' // Skip for now - pods work fine
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Save Current State') {
|
||||
when { branch 'main' }
|
||||
steps {
|
||||
script {
|
||||
echo "📸 Saving current deployment state for rollback..."
|
||||
|
||||
// Save current image tag
|
||||
sh """
|
||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].image}' \
|
||||
> /tmp/previous_image_${BUILD_NUMBER}.txt || echo "none" > /tmp/previous_image_${BUILD_NUMBER}.txt
|
||||
|
||||
PREV_IMAGE=\$(cat /tmp/previous_image_${BUILD_NUMBER}.txt)
|
||||
echo "Previous image: \${PREV_IMAGE}"
|
||||
"""
|
||||
|
||||
// Save current replica count
|
||||
sh """
|
||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
||||
-o jsonpath='{.spec.replicas}' \
|
||||
> /tmp/previous_replicas_${BUILD_NUMBER}.txt || echo "2" > /tmp/previous_replicas_${BUILD_NUMBER}.txt
|
||||
"""
|
||||
|
||||
echo "✅ Current state saved"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Checkout Source') {
|
||||
steps {
|
||||
echo "Checking out application source code..."
|
||||
|
||||
// Create Dockerfile with actual version
|
||||
sh """
|
||||
cat > Dockerfile << 'EOF'
|
||||
FROM nginx:1.25.3-alpine
|
||||
COPY index.html /usr/share/nginx/html/index.html
|
||||
COPY nginx.conf /etc/nginx/nginx.conf
|
||||
EXPOSE 80
|
||||
CMD ["nginx", "-g", "daemon off;"]
|
||||
EOF
|
||||
"""
|
||||
|
||||
// Create index.html with actual version
|
||||
sh """
|
||||
cat > index.html << EOF
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Demo Nginx</title>
|
||||
<style>
|
||||
body {
|
||||
font-family: Arial, sans-serif;
|
||||
max-width: 800px;
|
||||
margin: 50px auto;
|
||||
padding: 20px;
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: white;
|
||||
}
|
||||
.container {
|
||||
background: rgba(255, 255, 255, 0.1);
|
||||
padding: 40px;
|
||||
border-radius: 10px;
|
||||
backdrop-filter: blur(10px);
|
||||
}
|
||||
h1 {
|
||||
font-size: 48px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
p {
|
||||
font-size: 24px;
|
||||
margin: 10px 0;
|
||||
}
|
||||
.version {
|
||||
font-family: 'Courier New', monospace;
|
||||
background: rgba(0, 0, 0, 0.3);
|
||||
padding: 10px 20px;
|
||||
border-radius: 5px;
|
||||
display: inline-block;
|
||||
margin-top: 20px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>🚀 Demo Nginx - Build #${BUILD_NUMBER}</h1>
|
||||
<p>Environment: Production</p>
|
||||
<p class="version">Version: ${IMAGE_TAG}</p>
|
||||
<p style="font-size: 16px; margin-top: 30px; opacity: 0.8;">
|
||||
Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
||||
</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
"""
|
||||
|
||||
sh '''
|
||||
cat > nginx.conf << 'EOF'
|
||||
user nginx;
|
||||
worker_processes auto;
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
pid /var/run/nginx.pid;
|
||||
events { worker_connections 1024; }
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||
access_log /var/log/nginx/access.log main;
|
||||
sendfile on;
|
||||
keepalive_timeout 65;
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
location / {
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
}
|
||||
location /health {
|
||||
access_log off;
|
||||
return 200 "healthy\n";
|
||||
add_header Content-Type text/plain;
|
||||
}
|
||||
}
|
||||
}
|
||||
EOF
|
||||
'''
|
||||
}
|
||||
}
|
||||
|
||||
stage('Build Docker Image') {
|
||||
steps {
|
||||
script {
|
||||
echo "Building Docker image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}"
|
||||
sh """
|
||||
docker build \
|
||||
-t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} \
|
||||
-t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest \
|
||||
.
|
||||
"""
|
||||
echo "✅ Image built successfully!"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Push to Registry') {
|
||||
when { branch 'main' }
|
||||
steps {
|
||||
script {
|
||||
echo "Pushing image to registry..."
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'docker-registry-credentials',
|
||||
usernameVariable: 'DOCKER_USER',
|
||||
passwordVariable: 'DOCKER_PASS'
|
||||
)]) {
|
||||
sh """
|
||||
echo "\${DOCKER_PASS}" | docker login ${DOCKER_REGISTRY} -u "\${DOCKER_USER}" --password-stdin
|
||||
docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
||||
docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest
|
||||
docker logout ${DOCKER_REGISTRY}
|
||||
"""
|
||||
}
|
||||
echo "✅ Image pushed successfully!"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Update GitOps Manifests') {
|
||||
when { branch 'main' }
|
||||
steps {
|
||||
script {
|
||||
echo "Updating Kubernetes manifests..."
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'gitea-credentials',
|
||||
usernameVariable: 'GIT_USER',
|
||||
passwordVariable: 'GIT_PASS'
|
||||
)]) {
|
||||
sh """
|
||||
rm -rf k3s-gitops || true
|
||||
git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git
|
||||
cd k3s-gitops
|
||||
git config user.name "Jenkins"
|
||||
git config user.email "jenkins@thedevops.dev"
|
||||
|
||||
# Save current commit for rollback
|
||||
git rev-parse HEAD > /tmp/previous_commit_${BUILD_NUMBER}.txt
|
||||
|
||||
sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml
|
||||
git add apps/demo-nginx/deployment.yaml
|
||||
git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes"
|
||||
git push origin main
|
||||
"""
|
||||
}
|
||||
echo "✅ Manifests updated!"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Wait for ArgoCD Sync') {
|
||||
when { branch 'main' }
|
||||
steps {
|
||||
script {
|
||||
echo "⏳ Waiting for ArgoCD to sync..."
|
||||
|
||||
def syncSuccess = false
|
||||
def attempts = 0
|
||||
def maxAttempts = Integer.parseInt(env.ARGOCD_SYNC_TIMEOUT) / 10
|
||||
|
||||
while (!syncSuccess && attempts < maxAttempts) {
|
||||
attempts++
|
||||
echo "ArgoCD sync check attempt ${attempts}/${maxAttempts}..."
|
||||
|
||||
def syncStatus = sh(
|
||||
script: """
|
||||
kubectl get application ${APP_NAME} -n argocd \
|
||||
-o jsonpath='{.status.sync.status}'
|
||||
""",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
def currentImage = sh(
|
||||
script: """
|
||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
||||
""",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
echo "ArgoCD sync status: ${syncStatus}"
|
||||
echo "Current deployment image: ${currentImage}"
|
||||
echo "Expected image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}"
|
||||
|
||||
if (syncStatus == 'Synced' && currentImage.contains(env.IMAGE_TAG)) {
|
||||
syncSuccess = true
|
||||
echo "✅ ArgoCD synced successfully!"
|
||||
break
|
||||
}
|
||||
|
||||
if (attempts < maxAttempts) {
|
||||
echo "Waiting 10 seconds before next check..."
|
||||
sleep 10
|
||||
}
|
||||
}
|
||||
|
||||
if (!syncSuccess) {
|
||||
error "❌ ArgoCD sync timeout after ${env.ARGOCD_SYNC_TIMEOUT} seconds!"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Wait for Deployment') {
|
||||
when { branch 'main' }
|
||||
steps {
|
||||
script {
|
||||
echo "⏳ Waiting for deployment to complete..."
|
||||
|
||||
try {
|
||||
sh """
|
||||
# Check rollout status
|
||||
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT}
|
||||
"""
|
||||
echo "✅ Deployment rolled out successfully!"
|
||||
} catch (Exception e) {
|
||||
echo "❌ Deployment failed: ${e.message}"
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Verify Deployment') {
|
||||
when { branch 'main' }
|
||||
steps {
|
||||
script {
|
||||
echo "✅ Verifying deployment..."
|
||||
|
||||
try {
|
||||
sh """#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Check pod status
|
||||
READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}')
|
||||
DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}')
|
||||
|
||||
echo "Ready pods: \${READY_PODS}/\${DESIRED_PODS}"
|
||||
|
||||
if [ "\${READY_PODS}" != "\${DESIRED_PODS}" ]; then
|
||||
echo "❌ Not all pods are ready!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify image version
|
||||
DEPLOYED_IMAGE=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}')
|
||||
echo "Deployed image: \${DEPLOYED_IMAGE}"
|
||||
|
||||
if [[ "\${DEPLOYED_IMAGE}" != *"${IMAGE_TAG}"* ]]; then
|
||||
echo "❌ Image version mismatch!"
|
||||
echo "Expected: ${IMAGE_TAG}"
|
||||
echo "Got: \${DEPLOYED_IMAGE}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ All verification checks passed!"
|
||||
"""
|
||||
|
||||
echo "✅ Deployment verified successfully!"
|
||||
|
||||
} catch (Exception e) {
|
||||
echo "❌ Deployment verification failed: ${e.message}"
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
post {
|
||||
success {
|
||||
script {
|
||||
echo """
|
||||
✅ DEPLOYMENT SUCCESS!
|
||||
|
||||
Application: ${APP_NAME}
|
||||
Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
||||
Namespace: ${NAMESPACE}
|
||||
Build: #${BUILD_NUMBER}
|
||||
|
||||
All checks passed! ✨
|
||||
"""
|
||||
|
||||
// Cleanup rollback files
|
||||
sh """
|
||||
rm -f /tmp/previous_image_${BUILD_NUMBER}.txt
|
||||
rm -f /tmp/previous_replicas_${BUILD_NUMBER}.txt
|
||||
rm -f /tmp/previous_commit_${BUILD_NUMBER}.txt
|
||||
"""
|
||||
}
|
||||
}
|
||||
|
||||
failure {
|
||||
script {
|
||||
if (env.BRANCH_NAME == 'main' && env.ROLLBACK_ENABLED == 'true') {
|
||||
echo """
|
||||
❌ DEPLOYMENT FAILED - INITIATING ROLLBACK!
|
||||
|
||||
Rolling back to previous version...
|
||||
"""
|
||||
|
||||
try {
|
||||
// Get previous state
|
||||
def previousImage = sh(
|
||||
script: "cat /tmp/previous_image_${BUILD_NUMBER}.txt 2>/dev/null || echo 'none'",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
if (previousImage != 'none' && previousImage != '') {
|
||||
echo "🔄 Rolling back to: ${previousImage}"
|
||||
|
||||
// Rollback via kubectl
|
||||
sh """
|
||||
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
|
||||
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=180s
|
||||
"""
|
||||
|
||||
// Revert Git commit
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'gitea-credentials',
|
||||
usernameVariable: 'GIT_USER',
|
||||
passwordVariable: 'GIT_PASS'
|
||||
)]) {
|
||||
sh """
|
||||
cd k3s-gitops || git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git
|
||||
cd k3s-gitops
|
||||
git config user.name "Jenkins"
|
||||
git config user.email "jenkins@thedevops.dev"
|
||||
|
||||
# Revert last commit
|
||||
git revert --no-edit HEAD || true
|
||||
git push origin main || true
|
||||
"""
|
||||
}
|
||||
|
||||
echo """
|
||||
✅ ROLLBACK COMPLETED!
|
||||
|
||||
Rolled back to: ${previousImage}
|
||||
Current build (#${BUILD_NUMBER}) has been reverted.
|
||||
|
||||
Please check logs and fix the issue before redeploying.
|
||||
"""
|
||||
|
||||
} else {
|
||||
echo "⚠️ No previous version found - cannot rollback automatically"
|
||||
echo "Manual intervention required!"
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
echo """
|
||||
❌ ROLLBACK FAILED!
|
||||
|
||||
Error: ${e.message}
|
||||
|
||||
MANUAL ROLLBACK REQUIRED:
|
||||
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
|
||||
"""
|
||||
}
|
||||
|
||||
} else {
|
||||
echo "❌ Pipeline failed! (Rollback disabled or not main branch)"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
always {
|
||||
sh """
|
||||
docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true
|
||||
docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest || true
|
||||
docker stop test-${BUILD_NUMBER} 2>/dev/null || true
|
||||
docker rm test-${BUILD_NUMBER} 2>/dev/null || true
|
||||
"""
|
||||
cleanWs()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,564 +0,0 @@
|
||||
// Jenkinsfile for Manual Rollback
|
||||
// This allows rolling back to any previous version
|
||||
// Now rebuilds image with updated design!
|
||||
|
||||
pipeline {
|
||||
agent any
|
||||
|
||||
parameters {
|
||||
choice(
|
||||
name: 'ROLLBACK_METHOD',
|
||||
choices: ['IMAGE_TAG', 'REVISION_NUMBER', 'GIT_COMMIT'],
|
||||
description: 'Select rollback method'
|
||||
)
|
||||
string(
|
||||
name: 'TARGET_VERSION',
|
||||
defaultValue: '',
|
||||
description: '''
|
||||
Enter target version based on method:
|
||||
- IMAGE_TAG: main-21, main-20, etc.
|
||||
- REVISION_NUMBER: 1, 2, 3 (from rollout history)
|
||||
- GIT_COMMIT: abc123def (git commit SHA)
|
||||
'''
|
||||
)
|
||||
booleanParam(
|
||||
name: 'SKIP_HEALTH_CHECK',
|
||||
defaultValue: true,
|
||||
description: 'Skip health checks (recommended for rollback)'
|
||||
)
|
||||
booleanParam(
|
||||
name: 'DRY_RUN',
|
||||
defaultValue: false,
|
||||
description: 'Dry run - show what would happen without applying'
|
||||
)
|
||||
booleanParam(
|
||||
name: 'REBUILD_IMAGE',
|
||||
defaultValue: true,
|
||||
description: 'Rebuild image with updated design (recommended)'
|
||||
)
|
||||
}
|
||||
|
||||
environment {
|
||||
APP_NAME = 'demo-nginx'
|
||||
CONTAINER_NAME = 'nginx'
|
||||
NAMESPACE = 'demo-app'
|
||||
DOCKER_REGISTRY = 'docker.io'
|
||||
DOCKER_REPO = 'vladcrypto'
|
||||
GITEA_URL = 'http://gitea-http.gitea.svc.cluster.local:3000'
|
||||
HEALTH_CHECK_TIMEOUT = '300s'
|
||||
ARGOCD_SYNC_TIMEOUT = '120'
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Validate Input') {
|
||||
steps {
|
||||
script {
|
||||
echo "🔍 Validating rollback request..."
|
||||
|
||||
// Trim whitespace from input
|
||||
env.TARGET_VERSION_CLEAN = params.TARGET_VERSION.trim()
|
||||
|
||||
if (env.TARGET_VERSION_CLEAN == '') {
|
||||
error("❌ TARGET_VERSION cannot be empty!")
|
||||
}
|
||||
|
||||
echo """
|
||||
📋 Rollback Configuration:
|
||||
Method: ${params.ROLLBACK_METHOD}
|
||||
Target: ${env.TARGET_VERSION_CLEAN}
|
||||
Rebuild Image: ${params.REBUILD_IMAGE}
|
||||
Skip Health Check: ${params.SKIP_HEALTH_CHECK}
|
||||
Dry Run: ${params.DRY_RUN}
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Show Current State') {
|
||||
steps {
|
||||
script {
|
||||
echo "📸 Current Deployment State:"
|
||||
|
||||
sh """
|
||||
echo "=== Current Deployment ==="
|
||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE}
|
||||
|
||||
echo ""
|
||||
echo "=== Current Image ==="
|
||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo "=== Current Pods ==="
|
||||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME}
|
||||
|
||||
echo ""
|
||||
echo "=== Rollout History ==="
|
||||
kubectl rollout history deployment/${APP_NAME} -n ${NAMESPACE}
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Prepare Rollback') {
|
||||
steps {
|
||||
script {
|
||||
echo "🔄 Preparing rollback to: ${env.TARGET_VERSION_CLEAN}"
|
||||
|
||||
if (params.ROLLBACK_METHOD == 'IMAGE_TAG') {
|
||||
env.TARGET_IMAGE = "${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${env.TARGET_VERSION_CLEAN}"
|
||||
|
||||
// Extract build number from tag
|
||||
env.BUILD_NUMBER_FROM_TAG = env.TARGET_VERSION_CLEAN.split('-')[1]
|
||||
|
||||
sh """
|
||||
echo "Target image: ${env.TARGET_IMAGE}"
|
||||
echo "Build number: ${env.BUILD_NUMBER_FROM_TAG}"
|
||||
"""
|
||||
|
||||
} else if (params.ROLLBACK_METHOD == 'REVISION_NUMBER') {
|
||||
env.REVISION = env.TARGET_VERSION_CLEAN
|
||||
|
||||
sh """
|
||||
echo "Rolling back to revision: ${env.REVISION}"
|
||||
kubectl rollout history deployment/${APP_NAME} -n ${NAMESPACE} \
|
||||
--revision=${env.REVISION}
|
||||
"""
|
||||
|
||||
} else if (params.ROLLBACK_METHOD == 'GIT_COMMIT') {
|
||||
env.GIT_SHA = env.TARGET_VERSION_CLEAN
|
||||
echo "Rolling back to git commit: ${env.GIT_SHA}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Rebuild Image') {
|
||||
when {
|
||||
expression { !params.DRY_RUN && params.REBUILD_IMAGE && params.ROLLBACK_METHOD == 'IMAGE_TAG' }
|
||||
}
|
||||
steps {
|
||||
script {
|
||||
echo "🔨 Rebuilding image with updated design..."
|
||||
|
||||
// Create Dockerfile
|
||||
writeFile file: 'Dockerfile', text: '''FROM nginx:1.25.3-alpine
|
||||
COPY index.html /usr/share/nginx/html/index.html
|
||||
COPY nginx.conf /etc/nginx/nginx.conf
|
||||
EXPOSE 80
|
||||
CMD ["nginx", "-g", "daemon off;"]
|
||||
'''
|
||||
|
||||
// Create HTML with proper variable substitution
|
||||
def htmlContent = """<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Demo Nginx - Rollback</title>
|
||||
<style>
|
||||
body {
|
||||
font-family: Arial, sans-serif;
|
||||
max-width: 800px;
|
||||
margin: 50px auto;
|
||||
padding: 20px;
|
||||
background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);
|
||||
color: white;
|
||||
}
|
||||
.container {
|
||||
background: rgba(255, 255, 255, 0.1);
|
||||
padding: 40px;
|
||||
border-radius: 10px;
|
||||
backdrop-filter: blur(10px);
|
||||
}
|
||||
h1 {
|
||||
font-size: 48px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
.rollback-badge {
|
||||
background: rgba(255, 255, 255, 0.2);
|
||||
padding: 10px 20px;
|
||||
border-radius: 25px;
|
||||
display: inline-block;
|
||||
margin-bottom: 20px;
|
||||
font-size: 18px;
|
||||
}
|
||||
p {
|
||||
font-size: 24px;
|
||||
margin: 10px 0;
|
||||
}
|
||||
.version {
|
||||
font-family: 'Courier New', monospace;
|
||||
background: rgba(0, 0, 0, 0.3);
|
||||
padding: 10px 20px;
|
||||
border-radius: 5px;
|
||||
display: inline-block;
|
||||
margin-top: 20px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="rollback-badge">🔄 ROLLED BACK</div>
|
||||
<h1>🚀 Demo Nginx - Build #${env.BUILD_NUMBER_FROM_TAG}</h1>
|
||||
<p>Environment: Production</p>
|
||||
<p class="version">Version: ${env.TARGET_VERSION_CLEAN}</p>
|
||||
<p style="font-size: 16px; margin-top: 30px; opacity: 0.8;">
|
||||
Image: ${env.TARGET_IMAGE}
|
||||
</p>
|
||||
<p style="font-size: 14px; margin-top: 10px; opacity: 0.7;">
|
||||
⏮️ Restored from previous deployment
|
||||
</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
writeFile file: 'index.html', text: htmlContent
|
||||
|
||||
// Create nginx.conf
|
||||
writeFile file: 'nginx.conf', text: '''user nginx;
|
||||
worker_processes auto;
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
pid /var/run/nginx.pid;
|
||||
events { worker_connections 1024; }
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||
access_log /var/log/nginx/access.log main;
|
||||
sendfile on;
|
||||
keepalive_timeout 65;
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
location / {
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
}
|
||||
location /health {
|
||||
access_log off;
|
||||
return 200 "healthy\n";
|
||||
add_header Content-Type text/plain;
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
// Build new image
|
||||
sh """
|
||||
docker build -t ${env.TARGET_IMAGE} .
|
||||
"""
|
||||
|
||||
echo "✅ Image rebuilt successfully!"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Push Rebuilt Image') {
|
||||
when {
|
||||
expression { !params.DRY_RUN && params.REBUILD_IMAGE && params.ROLLBACK_METHOD == 'IMAGE_TAG' }
|
||||
}
|
||||
steps {
|
||||
script {
|
||||
echo "📤 Pushing rebuilt image..."
|
||||
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'docker-registry-credentials',
|
||||
usernameVariable: 'DOCKER_USER',
|
||||
passwordVariable: 'DOCKER_PASS'
|
||||
)]) {
|
||||
sh """
|
||||
echo "\${DOCKER_PASS}" | docker login ${DOCKER_REGISTRY} -u "\${DOCKER_USER}" --password-stdin
|
||||
docker push ${env.TARGET_IMAGE}
|
||||
docker logout ${DOCKER_REGISTRY}
|
||||
"""
|
||||
}
|
||||
|
||||
echo "✅ Image pushed successfully!"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Execute Rollback') {
|
||||
when {
|
||||
expression { !params.DRY_RUN }
|
||||
}
|
||||
steps {
|
||||
script {
|
||||
echo "🚀 Executing rollback..."
|
||||
|
||||
if (params.ROLLBACK_METHOD == 'IMAGE_TAG') {
|
||||
// Update deployment
|
||||
sh """
|
||||
echo "Setting image to: ${env.TARGET_IMAGE}"
|
||||
kubectl set image deployment/${APP_NAME} \
|
||||
${CONTAINER_NAME}=${env.TARGET_IMAGE} \
|
||||
-n ${NAMESPACE} \
|
||||
--record
|
||||
"""
|
||||
|
||||
// Update Git manifests
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'gitea-credentials',
|
||||
usernameVariable: 'GIT_USER',
|
||||
passwordVariable: 'GIT_PASS'
|
||||
)]) {
|
||||
sh """
|
||||
rm -rf k3s-gitops || true
|
||||
git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git
|
||||
cd k3s-gitops
|
||||
git config user.name "Jenkins"
|
||||
git config user.email "jenkins@thedevops.dev"
|
||||
|
||||
sed -i 's|image: .*|image: ${env.TARGET_IMAGE}|' apps/demo-nginx/deployment.yaml
|
||||
git add apps/demo-nginx/deployment.yaml
|
||||
git commit -m "rollback(demo-nginx): Manual rollback to ${env.TARGET_VERSION_CLEAN}" || echo "No changes"
|
||||
git push origin main
|
||||
"""
|
||||
}
|
||||
|
||||
} else if (params.ROLLBACK_METHOD == 'REVISION_NUMBER') {
|
||||
sh """
|
||||
kubectl rollout undo deployment/${APP_NAME} \
|
||||
-n ${NAMESPACE} \
|
||||
--to-revision=${env.REVISION}
|
||||
"""
|
||||
|
||||
} else if (params.ROLLBACK_METHOD == 'GIT_COMMIT') {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'gitea-credentials',
|
||||
usernameVariable: 'GIT_USER',
|
||||
passwordVariable: 'GIT_PASS'
|
||||
)]) {
|
||||
sh """
|
||||
rm -rf k3s-gitops || true
|
||||
git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git
|
||||
cd k3s-gitops
|
||||
git config user.name "Jenkins"
|
||||
git config user.email "jenkins@thedevops.dev"
|
||||
|
||||
git checkout ${env.GIT_SHA} -- apps/demo-nginx/deployment.yaml
|
||||
TARGET_IMAGE=\$(grep 'image:' apps/demo-nginx/deployment.yaml | awk '{print \$2}')
|
||||
|
||||
git checkout main
|
||||
git checkout ${env.GIT_SHA} -- apps/demo-nginx/deployment.yaml
|
||||
git add apps/demo-nginx/deployment.yaml
|
||||
git commit -m "rollback(demo-nginx): Rollback to commit ${env.GIT_SHA}" || echo "No changes"
|
||||
git push origin main
|
||||
|
||||
echo "Rolled back to image: \${TARGET_IMAGE}"
|
||||
"""
|
||||
}
|
||||
}
|
||||
|
||||
echo "✅ Rollback command executed"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Wait for ArgoCD Sync') {
|
||||
when {
|
||||
expression { !params.DRY_RUN }
|
||||
}
|
||||
steps {
|
||||
script {
|
||||
echo "⏳ Waiting for ArgoCD to sync..."
|
||||
|
||||
def syncSuccess = false
|
||||
def attempts = 0
|
||||
def maxAttempts = Integer.parseInt(env.ARGOCD_SYNC_TIMEOUT) / 10
|
||||
|
||||
while (!syncSuccess && attempts < maxAttempts) {
|
||||
attempts++
|
||||
echo "ArgoCD sync check attempt ${attempts}/${maxAttempts}..."
|
||||
|
||||
def syncStatus = sh(
|
||||
script: """
|
||||
kubectl get application ${APP_NAME} -n argocd \
|
||||
-o jsonpath='{.status.sync.status}'
|
||||
""",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
def currentImage = sh(
|
||||
script: """
|
||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
||||
""",
|
||||
returnStdout: true
|
||||
).trim()
|
||||
|
||||
echo "ArgoCD sync status: ${syncStatus}"
|
||||
echo "Current deployment image: ${currentImage}"
|
||||
echo "Expected image: ${env.TARGET_IMAGE}"
|
||||
|
||||
if (syncStatus == 'Synced' && currentImage.contains(env.TARGET_VERSION_CLEAN)) {
|
||||
syncSuccess = true
|
||||
echo "✅ ArgoCD synced successfully!"
|
||||
break
|
||||
}
|
||||
|
||||
if (attempts < maxAttempts) {
|
||||
echo "Waiting 10 seconds before next check..."
|
||||
sleep 10
|
||||
}
|
||||
}
|
||||
|
||||
if (!syncSuccess) {
|
||||
echo "⚠️ ArgoCD sync check timeout (non-critical for rollback)"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Wait for Rollout') {
|
||||
when {
|
||||
expression { !params.DRY_RUN }
|
||||
}
|
||||
steps {
|
||||
script {
|
||||
echo "⏳ Waiting for rollout to complete..."
|
||||
|
||||
sh """
|
||||
kubectl rollout status deployment/${APP_NAME} \
|
||||
-n ${NAMESPACE} \
|
||||
--timeout=${HEALTH_CHECK_TIMEOUT}
|
||||
"""
|
||||
|
||||
echo "✅ Rollout completed"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Verify Deployment') {
|
||||
when {
|
||||
expression { !params.DRY_RUN }
|
||||
}
|
||||
steps {
|
||||
script {
|
||||
echo "✅ Verifying deployment..."
|
||||
|
||||
sh """#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Check pods ready
|
||||
READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}')
|
||||
DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}')
|
||||
|
||||
echo "Ready pods: \${READY_PODS}/\${DESIRED_PODS}"
|
||||
|
||||
if [ "\${READY_PODS}" != "\${DESIRED_PODS}" ]; then
|
||||
echo "❌ Not all pods are ready!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify image
|
||||
DEPLOYED_IMAGE=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}')
|
||||
echo "Deployed image: \${DEPLOYED_IMAGE}"
|
||||
|
||||
echo "✅ Deployment verified!"
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Show New State') {
|
||||
when {
|
||||
expression { !params.DRY_RUN }
|
||||
}
|
||||
steps {
|
||||
script {
|
||||
echo "📊 New Deployment State:"
|
||||
|
||||
sh """
|
||||
echo "=== New Deployment ==="
|
||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE}
|
||||
|
||||
echo ""
|
||||
echo "=== New Image ==="
|
||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo "=== New Pods ==="
|
||||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME}
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Dry Run Summary') {
|
||||
when {
|
||||
expression { params.DRY_RUN }
|
||||
}
|
||||
steps {
|
||||
script {
|
||||
echo """
|
||||
🔍 DRY RUN SUMMARY
|
||||
|
||||
This is what would happen:
|
||||
|
||||
Method: ${params.ROLLBACK_METHOD}
|
||||
Target: ${env.TARGET_VERSION_CLEAN}
|
||||
Rebuild Image: ${params.REBUILD_IMAGE}
|
||||
|
||||
Steps:
|
||||
1. ${params.REBUILD_IMAGE ? 'Rebuild image with rollback design' : 'Use existing image'}
|
||||
2. Update deployment
|
||||
3. Update Git manifests
|
||||
4. Wait for ArgoCD sync
|
||||
5. Wait for rollout
|
||||
6. Verify deployment
|
||||
|
||||
No actual changes were made.
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
post {
|
||||
success {
|
||||
script {
|
||||
if (params.DRY_RUN) {
|
||||
echo "✅ DRY RUN COMPLETED - No changes made"
|
||||
} else {
|
||||
echo """
|
||||
✅ ROLLBACK SUCCESSFUL!
|
||||
|
||||
Application: ${APP_NAME}
|
||||
Method: ${params.ROLLBACK_METHOD}
|
||||
Target Version: ${env.TARGET_VERSION_CLEAN}
|
||||
${params.REBUILD_IMAGE ? 'Image: Rebuilt with updated design ✨' : 'Image: Using existing'}
|
||||
Namespace: ${NAMESPACE}
|
||||
|
||||
The application has been rolled back successfully! 🔄
|
||||
|
||||
Check: https://demo-nginx.thedevops.dev
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
failure {
|
||||
echo """
|
||||
❌ ROLLBACK FAILED!
|
||||
|
||||
Please check the logs and try again.
|
||||
|
||||
Manual rollback:
|
||||
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
|
||||
"""
|
||||
}
|
||||
|
||||
always {
|
||||
sh """
|
||||
docker rmi ${env.TARGET_IMAGE} 2>/dev/null || true
|
||||
"""
|
||||
cleanWs()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: demo-nginx
|
||||
namespace: argocd
|
||||
labels:
|
||||
app: demo-nginx
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: http://gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops
|
||||
targetRevision: HEAD
|
||||
path: apps/demo-nginx
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: demo-app
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
@@ -1,62 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: demo-nginx
|
||||
namespace: demo-app
|
||||
labels:
|
||||
app: demo-nginx
|
||||
spec:
|
||||
replicas: 5
|
||||
selector:
|
||||
matchLabels:
|
||||
app: demo-nginx
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: demo-nginx
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "80"
|
||||
spec:
|
||||
containers:
|
||||
- name: nginx
|
||||
image: docker.io/vladcrypto/demo-nginx:main-50
|
||||
ports:
|
||||
- containerPort: 80
|
||||
name: http
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: 80
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: 80
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: demo-nginx
|
||||
namespace: demo-app
|
||||
labels:
|
||||
app: demo-nginx
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: demo-nginx
|
||||
ports:
|
||||
- port: 80
|
||||
targetPort: 80
|
||||
protocol: TCP
|
||||
name: http
|
||||
@@ -1,419 +0,0 @@
|
||||
# 🔧 ArgoCD Sync vs Actual Deployment Issue
|
||||
|
||||
## 🐛 The Problem
|
||||
|
||||
**Symptom:**
|
||||
- ArgoCD shows `Synced` ✅
|
||||
- Deployment manifest in Kubernetes is updated ✅
|
||||
- **BUT** pods are still running old image ❌
|
||||
|
||||
**Why This Happens:**
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Git Repository │
|
||||
│ deployment.yaml: image: app:v2 ✅ │
|
||||
└──────────────────┬──────────────────────────────────────────┘
|
||||
│
|
||||
│ ArgoCD syncs
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Kubernetes API (Deployment Object) │
|
||||
│ spec.template.image: app:v2 ✅ │
|
||||
└──────────────────┬──────────────────────────────────────────┘
|
||||
│
|
||||
│ Kubernetes Controller should trigger rollout
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Running Pods │
|
||||
│ Pod-1: image: app:v1 ❌ (OLD!) │
|
||||
│ Pod-2: image: app:v1 ❌ (OLD!) │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
|
||||
ArgoCD says "Synced" because Git == Kubernetes manifest ✅
|
||||
But pods haven't rolled out yet! ❌
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔍 Why ArgoCD Says "Synced"
|
||||
|
||||
ArgoCD checks:
|
||||
1. ✅ Git manifest == Kubernetes Deployment object
|
||||
2. ✅ Health status (from status fields)
|
||||
|
||||
ArgoCD **DOES NOT** check:
|
||||
- ❌ Are pods actually running?
|
||||
- ❌ What image are pods using?
|
||||
- ❌ Did rollout complete?
|
||||
|
||||
**ArgoCD's job:** Keep Kubernetes resources in sync with Git
|
||||
**NOT ArgoCD's job:** Wait for pods to finish rolling out
|
||||
|
||||
---
|
||||
|
||||
## ⚠️ When This Happens
|
||||
|
||||
### Scenario 1: Slow Rollout
|
||||
```
|
||||
14:00:00 - ArgoCD syncs deployment (v1 → v2)
|
||||
14:00:05 - ArgoCD: "Synced!" ✅
|
||||
14:00:10 - Kubernetes starts rollout
|
||||
14:00:30 - Pod-1 terminates (v1)
|
||||
14:00:35 - Pod-3 starts (v2)
|
||||
14:00:50 - Pod-2 terminates (v1)
|
||||
14:00:55 - Pod-4 starts (v2)
|
||||
14:01:00 - Rollout complete! ✅
|
||||
|
||||
Jenkins checks at 14:00:05: ArgoCD says "Synced"
|
||||
But pods are still v1! ❌
|
||||
```
|
||||
|
||||
### Scenario 2: Image Pull Delay
|
||||
```
|
||||
14:00:00 - ArgoCD syncs
|
||||
14:00:05 - ArgoCD: "Synced!" ✅
|
||||
14:00:10 - Kubernetes tries to start new pod
|
||||
14:00:15 - Pulling image... (slow network)
|
||||
14:00:45 - Image pulled
|
||||
14:00:50 - Pod starts
|
||||
14:01:00 - Pod ready
|
||||
|
||||
Jenkins checks at 14:00:05: "Synced" but no new pods yet!
|
||||
```
|
||||
|
||||
### Scenario 3: Resource Constraints
|
||||
```
|
||||
14:00:00 - ArgoCD syncs
|
||||
14:00:05 - ArgoCD: "Synced!" ✅
|
||||
14:00:10 - Kubernetes: "No resources available"
|
||||
14:00:20 - Kubernetes: "Waiting for node capacity..."
|
||||
14:01:00 - Old pod terminates, resources freed
|
||||
14:01:10 - New pod starts
|
||||
|
||||
Jenkins checks at 14:00:05: "Synced" but can't schedule pods!
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ✅ The Solution
|
||||
|
||||
### What Jenkins Must Check:
|
||||
|
||||
```groovy
|
||||
// ❌ BAD - Only checks ArgoCD
|
||||
if (argocdStatus == 'Synced') {
|
||||
echo "Done!"
|
||||
}
|
||||
|
||||
// ✅ GOOD - Checks ArgoCD + Kubernetes
|
||||
if (argocdStatus == 'Synced') {
|
||||
// 1. Wait for rollout
|
||||
kubectl rollout status deployment/app
|
||||
|
||||
// 2. Verify actual pod images
|
||||
podImages = kubectl get pods -o jsonpath='{.status.containerStatuses[0].image}'
|
||||
if (podImages contains newVersion) {
|
||||
echo "Verified!"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🎯 New Jenkinsfile Verification
|
||||
|
||||
### Stage 1: ArgoCD Sync Check
|
||||
```groovy
|
||||
stage('Wait for ArgoCD Sync') {
|
||||
// Checks:
|
||||
// 1. ArgoCD sync status = "Synced"
|
||||
// 2. Deployment SPEC image updated
|
||||
//
|
||||
// Does NOT check if pods rolled out!
|
||||
// That's the next stage.
|
||||
}
|
||||
```
|
||||
|
||||
**Output:**
|
||||
```
|
||||
ArgoCD sync status: Synced
|
||||
Deployment spec image: app:v2
|
||||
✅ ArgoCD synced and deployment spec updated!
|
||||
Note: Pods may still be rolling out - will verify in next stage
|
||||
```
|
||||
|
||||
### Stage 2: Wait for Rollout
|
||||
```groovy
|
||||
stage('Wait for Deployment') {
|
||||
// Uses kubectl rollout status
|
||||
// Waits for actual pod rollout to complete
|
||||
sh "kubectl rollout status deployment/app --timeout=5m"
|
||||
}
|
||||
```
|
||||
|
||||
**What `kubectl rollout status` does:**
|
||||
- Watches deployment progress
|
||||
- Waits for all new pods to be ready
|
||||
- Returns when rollout complete
|
||||
- Times out if rollout stuck
|
||||
|
||||
**Output:**
|
||||
```
|
||||
Waiting for deployment "app" rollout to finish: 1 out of 2 new replicas have been updated...
|
||||
Waiting for deployment "app" rollout to finish: 1 old replicas are pending termination...
|
||||
deployment "app" successfully rolled out
|
||||
✅ Rollout completed successfully!
|
||||
```
|
||||
|
||||
### Stage 3: Verify Actual Pods
|
||||
```groovy
|
||||
stage('Verify Deployment') {
|
||||
// CRITICAL CHECKS:
|
||||
|
||||
// 1. Deployment status
|
||||
readyReplicas == desiredReplicas
|
||||
|
||||
// 2. Deployment spec image
|
||||
deploymentImage contains newTag
|
||||
|
||||
// 3. ACTUAL POD IMAGES (most important!)
|
||||
podImages = all pods images
|
||||
for each podImage:
|
||||
if podImage does not contain newTag:
|
||||
FAIL!
|
||||
|
||||
// 4. Pod health
|
||||
all pods in Running state
|
||||
|
||||
// 5. Restart count
|
||||
check for crash loops
|
||||
}
|
||||
```
|
||||
|
||||
**Output:**
|
||||
```
|
||||
================================================
|
||||
DEPLOYMENT VERIFICATION
|
||||
================================================
|
||||
|
||||
1. Checking deployment status...
|
||||
Desired replicas: 2
|
||||
Updated replicas: 2
|
||||
Ready replicas: 2
|
||||
Available replicas: 2
|
||||
✅ All pods ready
|
||||
|
||||
2. Checking deployment spec image...
|
||||
Deployment spec image: app:v2
|
||||
Expected tag: v2
|
||||
✅ Deployment spec correct
|
||||
|
||||
3. Checking actual running pod images...
|
||||
Running pod images:
|
||||
- app:v2
|
||||
- app:v2
|
||||
✅ All pods running correct image
|
||||
|
||||
4. Checking pod readiness probes...
|
||||
✅ All pods in Running state
|
||||
|
||||
5. Checking for container restarts...
|
||||
Max restart count: 0
|
||||
✅ Restart count acceptable
|
||||
|
||||
================================================
|
||||
✅ ALL VERIFICATION CHECKS PASSED!
|
||||
================================================
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔥 What Happens If Check #3 Fails
|
||||
|
||||
```
|
||||
3. Checking actual running pod images...
|
||||
Running pod images:
|
||||
- app:v1 ❌
|
||||
- app:v1 ❌
|
||||
❌ Pod running wrong image: app:v1
|
||||
❌ FAILED: 2 pod(s) running old image!
|
||||
This is the ArgoCD sync bug - deployment updated but pods not rolled out
|
||||
```
|
||||
|
||||
**Jenkins will:**
|
||||
1. ❌ Mark build as failed
|
||||
2. 🔄 Trigger rollback (if enabled)
|
||||
3. 📱 Send notification with details
|
||||
|
||||
---
|
||||
|
||||
## 🧪 Testing the Fix
|
||||
|
||||
### Test 1: Normal Deployment
|
||||
```bash
|
||||
# Update image in Git
|
||||
git commit -m "Update to v2"
|
||||
git push
|
||||
|
||||
# Jenkins should:
|
||||
# 1. Wait for ArgoCD sync ✅
|
||||
# 2. Wait for rollout ✅
|
||||
# 3. Verify pods have v2 ✅
|
||||
# 4. Success! ✅
|
||||
```
|
||||
|
||||
### Test 2: Slow Rollout
|
||||
```bash
|
||||
# Set slow rollout
|
||||
kubectl patch deployment app -p '{"spec":{"strategy":{"rollingUpdate":{"maxUnavailable":0,"maxSurge":1}}}}'
|
||||
|
||||
# Update image
|
||||
git push
|
||||
|
||||
# Jenkins should:
|
||||
# 1. ArgoCD syncs quickly ✅
|
||||
# 2. Wait for slow rollout (may take 2-3 minutes) ⏳
|
||||
# 3. Verify when complete ✅
|
||||
```
|
||||
|
||||
### Test 3: Rollout Stuck
|
||||
```bash
|
||||
# Create a broken image tag
|
||||
# Update to image: app:nonexistent
|
||||
|
||||
git push
|
||||
|
||||
# Jenkins should:
|
||||
# 1. ArgoCD syncs ✅
|
||||
# 2. kubectl rollout status times out ❌
|
||||
# 3. Rollback triggered ✅
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📊 Comparison: Old vs New
|
||||
|
||||
### Old Pipeline (Unreliable)
|
||||
```
|
||||
1. ArgoCD sync check
|
||||
├─ Checks: ArgoCD status
|
||||
├─ Checks: Deployment spec image
|
||||
└─ Duration: ~30 seconds
|
||||
|
||||
⚠️ PROBLEM: Pods might not have rolled out!
|
||||
|
||||
2. Success! ✅ (but pods are still old!)
|
||||
```
|
||||
|
||||
### New Pipeline (Reliable)
|
||||
```
|
||||
1. ArgoCD sync check
|
||||
├─ Checks: ArgoCD status
|
||||
├─ Checks: Deployment spec image
|
||||
└─ Duration: ~30 seconds
|
||||
|
||||
2. Rollout status check
|
||||
├─ Checks: kubectl rollout status
|
||||
├─ Waits: For actual pod rollout
|
||||
└─ Duration: ~1-2 minutes
|
||||
|
||||
3. Verification
|
||||
├─ Checks: Deployment status
|
||||
├─ Checks: ACTUAL pod images ← KEY!
|
||||
├─ Checks: Pod health
|
||||
├─ Checks: Restart count
|
||||
└─ Duration: ~10 seconds
|
||||
|
||||
4. Success! ✅ (pods verified running new version)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Key Takeaways
|
||||
|
||||
### ❌ Don't Trust:
|
||||
- ArgoCD "Synced" status alone
|
||||
- Deployment spec image alone
|
||||
- Health status alone
|
||||
|
||||
### ✅ Always Verify:
|
||||
1. **ArgoCD synced** (manifest applied)
|
||||
2. **Rollout completed** (`kubectl rollout status`)
|
||||
3. **Actual pod images** (what's really running)
|
||||
4. **Pod health** (ready and not crashing)
|
||||
|
||||
### 💡 Remember:
|
||||
```
|
||||
ArgoCD "Synced" = Git matches Kubernetes manifest ✅
|
||||
BUT
|
||||
Kubernetes manifest != Running pods ⚠️
|
||||
|
||||
You MUST check actual pods!
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔗 Related Issues
|
||||
|
||||
- [ArgoCD #2723](https://github.com/argoproj/argo-cd/issues/2723) - "Synced but pods not updated"
|
||||
- [Kubernetes #93033](https://github.com/kubernetes/kubernetes/issues/93033) - "Deployment rollout delays"
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Using the New Jenkinsfile
|
||||
|
||||
```bash
|
||||
# 1. Update Jenkinsfile in your repo
|
||||
cp Jenkinsfile.telegram.en apps/demo-nginx/Jenkinsfile
|
||||
|
||||
# 2. Commit and push
|
||||
git add apps/demo-nginx/Jenkinsfile
|
||||
git commit -m "fix: add proper deployment verification"
|
||||
git push
|
||||
|
||||
# 3. Run build
|
||||
# Jenkins will now properly verify deployments!
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📱 Notifications
|
||||
|
||||
With the new verification, you'll see:
|
||||
|
||||
**During deployment:**
|
||||
```
|
||||
⏳ ArgoCD Syncing
|
||||
Application: demo-nginx
|
||||
Timeout: 120s
|
||||
|
||||
🚀 Deploying to Kubernetes
|
||||
Deployment: demo-nginx
|
||||
Image: main-42
|
||||
Rolling out new pods...
|
||||
```
|
||||
|
||||
**On success:**
|
||||
```
|
||||
✅ Deployment Successful!
|
||||
|
||||
Verified:
|
||||
- ArgoCD synced ✅
|
||||
- Rollout completed ✅
|
||||
- Pods running v42 ✅
|
||||
- All pods healthy ✅
|
||||
```
|
||||
|
||||
**On failure:**
|
||||
```
|
||||
❌ Deployment Failed
|
||||
|
||||
Error: 2 pods running old image!
|
||||
|
||||
Rollback initiated...
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
**This fix ensures Jenkins never reports success until pods are actually running the new version!** ✅
|
||||
@@ -1,148 +0,0 @@
|
||||
# 🎉 CI/CD Pipeline - Complete Guide
|
||||
|
||||
## ✅ Что мы построили - Production Ready System!
|
||||
|
||||
Полноценный CI/CD pipeline с автоматическим деплоем от Git до Production.
|
||||
|
||||
---
|
||||
|
||||
## 📦 Компоненты системы
|
||||
|
||||
### 1. Jenkins (CI/CD Engine)
|
||||
```
|
||||
URL: http://jenkins.thedevops.dev
|
||||
Namespace: jenkins
|
||||
|
||||
Установлено:
|
||||
✅ Docker CLI (для build & push)
|
||||
✅ kubectl v1.28.0 (для deploy verification)
|
||||
✅ ServiceAccount + RBAC (для K8s API)
|
||||
✅ Auto-scan (проверяет Git каждый час)
|
||||
```
|
||||
|
||||
### 2. Gitea (Git Repository)
|
||||
```
|
||||
URL: http://gitea.thedevops.dev
|
||||
Repository: k3s-gitops
|
||||
|
||||
Содержит:
|
||||
✅ apps/demo-nginx/ - все манифесты приложения
|
||||
✅ apps/jenkins/ - конфигурация Jenkins
|
||||
✅ Jenkinsfile - CI/CD pipeline
|
||||
✅ Webhook настроен (опционально)
|
||||
```
|
||||
|
||||
### 3. ArgoCD (GitOps)
|
||||
```
|
||||
URL: https://argocd.thedevops.dev
|
||||
Application: demo-nginx
|
||||
|
||||
Функции:
|
||||
✅ Автоматически синхронизирует Git → K8s
|
||||
✅ Отслеживает изменения в deployment.yaml
|
||||
✅ Применяет rolling updates
|
||||
✅ Self-heal & auto-prune
|
||||
```
|
||||
|
||||
### 4. Docker Hub (Registry)
|
||||
```
|
||||
Repository: docker.io/vladcrypto/demo-nginx
|
||||
|
||||
Images:
|
||||
✅ Автоматически пушатся из Jenkins
|
||||
✅ Теги: main-XX, latest
|
||||
```
|
||||
|
||||
### 5. Kubernetes (Runtime)
|
||||
```
|
||||
Namespace: demo-app
|
||||
Deployment: demo-nginx (2 replicas)
|
||||
Service: demo-nginx (ClusterIP)
|
||||
Ingress: https://demo-nginx.thedevops.dev
|
||||
|
||||
Status:
|
||||
✅ 2/2 pods Running
|
||||
✅ Rolling updates enabled
|
||||
✅ Health checks configured
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔄 Как работает Pipeline
|
||||
|
||||
### Автоматический flow:
|
||||
|
||||
```
|
||||
1. Developer → git push
|
||||
↓
|
||||
2. Gitea → сохраняет commit
|
||||
↓
|
||||
3. Jenkins → Auto-scan обнаруживает изменения (каждый час)
|
||||
↓
|
||||
4. Jenkins Pipeline (Jenkinsfile):
|
||||
├─ Stage 1: Checkout Source (создает Dockerfile + nginx.conf)
|
||||
├─ Stage 2: Build Docker Image (main-XX)
|
||||
├─ Stage 3: Push to Registry (Docker Hub)
|
||||
├─ Stage 4: Update GitOps Manifests (deployment.yaml)
|
||||
└─ Stage 5: Verify Deployment (kubectl rollout status)
|
||||
↓
|
||||
5. ArgoCD → Обнаруживает изменения в Git (каждые 3 мин)
|
||||
↓
|
||||
6. ArgoCD → Синхронизирует deployment.yaml → Kubernetes
|
||||
↓
|
||||
7. Kubernetes → Rolling update (zero downtime)
|
||||
↓
|
||||
8. ✅ Production! Новая версия работает!
|
||||
```
|
||||
|
||||
**Время:** 3-5 минут от commit до production
|
||||
|
||||
---
|
||||
|
||||
## 🧪 Как проверять
|
||||
|
||||
### 1. Проверить Jenkins
|
||||
|
||||
```bash
|
||||
# Pods
|
||||
kubectl get pods -n jenkins
|
||||
|
||||
# Logs
|
||||
kubectl logs -n jenkins <jenkins-pod>
|
||||
|
||||
# UI
|
||||
http://jenkins.thedevops.dev
|
||||
```
|
||||
|
||||
**Что смотреть:**
|
||||
- Build History (список сборок)
|
||||
- Console Output (логи последней сборки)
|
||||
- Blue Ocean (красивый UI)
|
||||
|
||||
---
|
||||
|
||||
### 2. Проверить Demo App
|
||||
|
||||
```bash
|
||||
# Deployment
|
||||
kubectl get deployment demo-nginx -n demo-app
|
||||
|
||||
# Pods
|
||||
kubectl get pods -n demo-app
|
||||
|
||||
# Service
|
||||
kubectl get svc demo-nginx -n demo-app
|
||||
|
||||
# Ingress
|
||||
kubectl get ingress demo-nginx -n demo-app
|
||||
```
|
||||
|
||||
**Ожидаемый результат:**
|
||||
```
|
||||
DEPLOYMENT: 2/2 Ready
|
||||
PODS: 2/2 Running
|
||||
SERVICE: ClusterIP
|
||||
INGRESS: demo-nginx.thedevops.dev
|
||||
```
|
||||
|
||||
См. полную документацию в файле для всех деталей, сценариев использования, troubleshooting и best practices.
|
||||
166
apps/demo-nginx/docs/Jenkinsfile
vendored
166
apps/demo-nginx/docs/Jenkinsfile
vendored
@@ -1,166 +0,0 @@
|
||||
pipeline { // Declarative Jenkins pipeline start
|
||||
agent any // Run on any available Jenkins agent
|
||||
|
||||
environment { // Global environment variables
|
||||
APP_NAME = 'demo-nginx' // Application name (Docker, K8s, logs)
|
||||
NAMESPACE = 'demo-app' // Kubernetes namespace
|
||||
DOCKER_REGISTRY = 'docker.io' // Docker registry hostname
|
||||
DOCKER_REPO = 'vladcrypto' // Docker Hub repository / namespace
|
||||
GITEA_URL = 'http://gitea-http.gitea.svc.cluster.local:3000' // Internal Gitea URL
|
||||
GITEA_REPO = 'admin/k3s-gitops' // GitOps repository path
|
||||
GITEA_BRANCH = 'main' // Git branch for GitOps updates
|
||||
BUILD_TAG = "${env.BUILD_NUMBER}" // Jenkins build number
|
||||
IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}" // Image version tag
|
||||
}
|
||||
|
||||
stages { // Pipeline stages definition
|
||||
|
||||
stage('Checkout Source') { // Stage: prepare application source
|
||||
steps { // Steps executed in this stage
|
||||
echo "Checking out application source code..." // Log message
|
||||
|
||||
sh ''' // Execute shell script
|
||||
cat > Dockerfile << 'EOF' # Create Dockerfile
|
||||
FROM nginx:1.25.3-alpine # Base Nginx Alpine image
|
||||
RUN echo "<html><body><h1>Demo Nginx - Build ${BUILD_NUMBER}</h1><p>Environment: Production</p><p>Version: ${IMAGE_TAG}</p></body></html>" > /usr/share/nginx/html/index.html # Inject build metadata into HTML
|
||||
COPY nginx.conf /etc/nginx/nginx.conf # Copy custom nginx configuration
|
||||
EXPOSE 80 # Expose HTTP port
|
||||
CMD ["nginx", "-g", "daemon off;"] # Run nginx in foreground
|
||||
EOF
|
||||
'''
|
||||
|
||||
sh ''' // Generate nginx configuration
|
||||
cat > nginx.conf << 'EOF' # Create nginx.conf
|
||||
user nginx; # Run workers as nginx user
|
||||
worker_processes auto; # Scale workers to CPU cores
|
||||
error_log /var/log/nginx/error.log warn; # Error log level and path
|
||||
pid /var/run/nginx.pid; # PID file location
|
||||
events { worker_connections 1024; } # Max connections per worker
|
||||
http { # HTTP configuration block
|
||||
include /etc/nginx/mime.types; # Load MIME types
|
||||
default_type application/octet-stream; # Default content type
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; # Access log format
|
||||
access_log /var/log/nginx/access.log main; # Enable access logging
|
||||
sendfile on; # Enable zero-copy file transfer
|
||||
keepalive_timeout 65; # Keepalive timeout
|
||||
server { # Server definition
|
||||
listen 80; # Listen on port 80
|
||||
server_name _; # Catch-all server
|
||||
location / { # Root location
|
||||
root /usr/share/nginx/html; # Serve static files
|
||||
index index.html; # Default index file
|
||||
}
|
||||
location /health { # Health check endpoint
|
||||
access_log off; # Disable logging for health checks
|
||||
return 200 "healthy\n"; # Always return HTTP 200
|
||||
add_header Content-Type text/plain; # Explicit content type
|
||||
}
|
||||
}
|
||||
}
|
||||
EOF
|
||||
'''
|
||||
}
|
||||
}
|
||||
|
||||
stage('Build Docker Image') { // Stage: build Docker image
|
||||
steps {
|
||||
script { // Scripted block
|
||||
echo "Building Docker image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}" // Log image name
|
||||
sh """ // Execute Docker build
|
||||
docker build \
|
||||
-t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} \ // Versioned image
|
||||
-t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest \ // Latest tag
|
||||
. // Build context
|
||||
"""
|
||||
echo "✅ Image built successfully!" // Success message
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Push to Registry') { // Stage: push image to registry
|
||||
when { branch 'main' } // Execute only on main branch
|
||||
steps {
|
||||
script {
|
||||
echo "Pushing image to registry..." // Log push start
|
||||
withCredentials([usernamePassword( // Inject Docker credentials
|
||||
credentialsId: 'docker-registry-credentials',
|
||||
usernameVariable: 'DOCKER_USER',
|
||||
passwordVariable: 'DOCKER_PASS'
|
||||
)]) {
|
||||
sh """ // Authenticate and push image
|
||||
echo "\${DOCKER_PASS}" | docker login ${DOCKER_REGISTRY} -u "\${DOCKER_USER}" --password-stdin // Login securely
|
||||
docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} // Push versioned image
|
||||
docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest // Push latest image
|
||||
docker logout ${DOCKER_REGISTRY} // Logout
|
||||
"""
|
||||
}
|
||||
echo "✅ Image pushed successfully!" // Push success
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Update GitOps Manifests') { // Stage: update GitOps repository
|
||||
when { branch 'main' } // Only from main branch
|
||||
steps {
|
||||
script {
|
||||
echo "Updating Kubernetes manifests..." // Log GitOps update
|
||||
withCredentials([usernamePassword( // Inject Gitea credentials
|
||||
credentialsId: 'gitea-credentials',
|
||||
usernameVariable: 'GIT_USER',
|
||||
passwordVariable: 'GIT_PASS'
|
||||
)]) {
|
||||
sh """ // Clone and update manifests
|
||||
rm -rf k3s-gitops || true // Remove old repo
|
||||
git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git // Clone repo
|
||||
cd k3s-gitops // Enter repo
|
||||
git config user.name "Jenkins" // Git author name
|
||||
git config user.email "jenkins@thedevops.dev" // Git author email
|
||||
sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml // Update image
|
||||
git add apps/demo-nginx/deployment.yaml // Stage change
|
||||
git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes" // Commit if needed
|
||||
git push origin main // Push to main
|
||||
"""
|
||||
}
|
||||
echo "✅ Manifests updated!" // GitOps success
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Verify Deployment') { // Stage: verify Kubernetes deployment
|
||||
when { branch 'main' } // Only on main
|
||||
steps {
|
||||
script {
|
||||
echo "Verifying deployment..." // Log verification start
|
||||
sh """ // Check deployment status
|
||||
sleep 30 // Wait for rollout start
|
||||
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=300s || true // Rollout status
|
||||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} // List pods
|
||||
"""
|
||||
echo "✅ Deployment completed!" // Verification success
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
post { // Post-pipeline actions
|
||||
success { // On success
|
||||
echo """ // Success summary
|
||||
✅ Pipeline SUCCESS!
|
||||
Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
||||
Namespace: ${NAMESPACE}
|
||||
"""
|
||||
}
|
||||
failure { // On failure
|
||||
echo "❌ Pipeline failed!" // Failure message
|
||||
}
|
||||
always { // Always execute cleanup
|
||||
sh """ // Cleanup Docker artifacts
|
||||
docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true // Remove versioned image
|
||||
docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest || true // Remove latest image
|
||||
docker stop test-${BUILD_NUMBER} 2>/dev/null || true // Stop test container
|
||||
docker rm test-${BUILD_NUMBER} 2>/dev/null || true // Remove test container
|
||||
"""
|
||||
cleanWs() // Clean Jenkins workspace
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,235 +0,0 @@
|
||||
# Demo Nginx Documentation
|
||||
|
||||
Документация для demo-nginx deployment с автоматическим и ручным rollback.
|
||||
|
||||
---
|
||||
|
||||
## 📚 Available Documentation
|
||||
|
||||
### [ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md)
|
||||
**Comprehensive Manual Rollback Guide**
|
||||
|
||||
Полная документация по функции ручного rollback:
|
||||
- 3 способа rollback (IMAGE_TAG, REVISION, GIT_COMMIT)
|
||||
- Setup guide
|
||||
- Troubleshooting со всеми fixes
|
||||
- Best practices
|
||||
- Examples
|
||||
- FAQ
|
||||
|
||||
**When to use:** Нужна детальная информация или troubleshooting
|
||||
|
||||
---
|
||||
|
||||
### [ROLLBACK_QUICK_REF.md](./ROLLBACK_QUICK_REF.md)
|
||||
**Quick Reference Card**
|
||||
|
||||
Краткая справка для быстрого использования:
|
||||
- Quick start (2 минуты)
|
||||
- 3 способа rollback
|
||||
- Emergency procedure
|
||||
- Verification commands
|
||||
- Checklist
|
||||
|
||||
**When to use:** Быстрый rollback в production
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Quick Links
|
||||
|
||||
### Rollback Feature
|
||||
- **Full Guide:** [ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md)
|
||||
- **Quick Ref:** [ROLLBACK_QUICK_REF.md](./ROLLBACK_QUICK_REF.md)
|
||||
- **Jenkinsfile:** [Jenkinsfile.rollback](../Jenkinsfile.rollback)
|
||||
- **CI/CD Guide:** [../../../CICD_GUIDE.md](../../../CICD_GUIDE.md)
|
||||
|
||||
### Related Resources
|
||||
- **Jenkins RBAC:** [apps/jenkins/rbac.yaml](../../jenkins/rbac.yaml)
|
||||
- **Deployment:** [deployment.yaml](../deployment.yaml)
|
||||
- **Main CI/CD:** [Jenkinsfile](../Jenkinsfile)
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
### Manual Rollback (2 minutes)
|
||||
|
||||
```
|
||||
1. Jenkins → demo-nginx-rollback
|
||||
2. IMAGE_TAG + main-21
|
||||
3. SKIP_HEALTH_CHECK: true
|
||||
4. Build
|
||||
```
|
||||
|
||||
See [ROLLBACK_QUICK_REF.md](./ROLLBACK_QUICK_REF.md) for details.
|
||||
|
||||
---
|
||||
|
||||
### Emergency Rollback (30 seconds)
|
||||
|
||||
```bash
|
||||
kubectl rollout undo deployment/demo-nginx -n demo-app
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📊 Features Summary
|
||||
|
||||
### Manual Rollback
|
||||
✅ 3 rollback methods (IMAGE_TAG, REVISION, GIT_COMMIT)
|
||||
✅ GitOps sync (auto-commit to Git)
|
||||
✅ Zero downtime (rolling updates)
|
||||
✅ DRY_RUN mode (safe testing)
|
||||
✅ Full RBAC permissions
|
||||
✅ Input validation (auto-trim)
|
||||
✅ Retry logic (5 attempts)
|
||||
⚠️ Health check optional (use SKIP_HEALTH_CHECK=true)
|
||||
|
||||
### Automatic Rollback
|
||||
✅ Triggered on deployment failure
|
||||
✅ Saves previous state
|
||||
✅ Kubernetes rollback
|
||||
✅ Git revert
|
||||
✅ Health checks
|
||||
✅ Timeout protection
|
||||
|
||||
---
|
||||
|
||||
## 🐛 All Fixes Applied
|
||||
|
||||
| # | Issue | Fix | Status | Doc |
|
||||
|---|-------|-----|--------|-----|
|
||||
| 1 | Container name | Use `nginx` | ✅ | [Link](./ROLLBACK_MANUAL.md#issue-1-container-name-error--fixed) |
|
||||
| 2 | Whitespace | Auto-trim | ✅ | [Link](./ROLLBACK_MANUAL.md#issue-2-whitespace-in-input--fixed) |
|
||||
| 3 | RBAC | pods/exec perm | ✅ | [Link](./ROLLBACK_MANUAL.md#issue-3-rbac-permissions--fixed) |
|
||||
| 4 | Health timing | SKIP option | ⚠️ | [Link](./ROLLBACK_MANUAL.md#issue-4-health-check-timing--workaround) |
|
||||
| 5 | Bash loop | Explicit list | ✅ | [Link](./ROLLBACK_MANUAL.md#issue-5-bash-loop-syntax--fixed) |
|
||||
|
||||
---
|
||||
|
||||
## 💡 Recommended Reading Order
|
||||
|
||||
### For New Users:
|
||||
1. Start → [ROLLBACK_QUICK_REF.md](./ROLLBACK_QUICK_REF.md)
|
||||
2. Practice → Follow quick start
|
||||
3. Deep Dive → [ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md)
|
||||
|
||||
### For Troubleshooting:
|
||||
1. Check → [ROLLBACK_MANUAL.md - Troubleshooting](./ROLLBACK_MANUAL.md#troubleshooting--fixes)
|
||||
2. Verify → [ROLLBACK_QUICK_REF.md - Verification](./ROLLBACK_QUICK_REF.md#-verify-rollback)
|
||||
3. Support → [ROLLBACK_MANUAL.md - Support](./ROLLBACK_MANUAL.md#support)
|
||||
|
||||
### For Emergency:
|
||||
1. Fast → [ROLLBACK_QUICK_REF.md - Emergency](./ROLLBACK_QUICK_REF.md#-emergency-rollback-30-seconds)
|
||||
2. Alternative → [ROLLBACK_MANUAL.md - Emergency](./ROLLBACK_MANUAL.md#emergency-rollback-procedure)
|
||||
|
||||
---
|
||||
|
||||
## 🎓 Key Concepts
|
||||
|
||||
### Rollback Methods Comparison
|
||||
|
||||
| Method | Speed | Precision | Use Case |
|
||||
|--------|-------|-----------|----------|
|
||||
| IMAGE_TAG | ⚡⚡⚡ | 🎯 High | Known build number |
|
||||
| REVISION | ⚡⚡ | 🎯 Medium | Recent rollback |
|
||||
| GIT_COMMIT | ⚡ | 🎯🎯 High | Exact code state |
|
||||
|
||||
### When to Use What
|
||||
|
||||
**Use IMAGE_TAG when:**
|
||||
- You know the build number (main-21)
|
||||
- Quick rollback needed
|
||||
- Most common scenario
|
||||
|
||||
**Use REVISION_NUMBER when:**
|
||||
- Need to go back N versions
|
||||
- Don't remember exact tag
|
||||
- Working with kubectl history
|
||||
|
||||
**Use GIT_COMMIT when:**
|
||||
- Need exact code state
|
||||
- Multiple changes in one build
|
||||
- Precise rollback required
|
||||
|
||||
---
|
||||
|
||||
## 📈 Monitoring
|
||||
|
||||
### Check Rollback Status
|
||||
|
||||
```bash
|
||||
# Deployment status
|
||||
kubectl get deployment demo-nginx -n demo-app
|
||||
|
||||
# Pod status
|
||||
kubectl get pods -n demo-app -l app=demo-nginx
|
||||
|
||||
# Rollout history
|
||||
kubectl rollout history deployment/demo-nginx -n demo-app
|
||||
|
||||
# ArgoCD status
|
||||
kubectl get application demo-nginx -n argocd
|
||||
```
|
||||
|
||||
### Grafana Queries
|
||||
|
||||
```promql
|
||||
# Rollback count
|
||||
sum(increase(deployment_rollback_total[1h])) by (deployment)
|
||||
|
||||
# Rollback rate
|
||||
rate(deployment_rollback_total[5m])
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ❓ FAQ
|
||||
|
||||
### Q: Какой метод rollback использовать?
|
||||
**A:** Для большинства случаев используй IMAGE_TAG - самый быстрый и простой.
|
||||
|
||||
### Q: Health check всегда падает, это баг?
|
||||
**A:** Нет, это timing issue во время rolling update. Используй `SKIP_HEALTH_CHECK: true` и проверь вручную через минуту.
|
||||
|
||||
### Q: Как быстро откатиться в emergency?
|
||||
**A:** Используй `kubectl rollout undo` (30 секунд) или Jenkins с SKIP_HEALTH_CHECK (2 минуты).
|
||||
|
||||
### Q: Где полная документация?
|
||||
**A:** [ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md) - comprehensive guide со всеми details.
|
||||
|
||||
---
|
||||
|
||||
## 🆘 Support
|
||||
|
||||
**Need Help?**
|
||||
|
||||
1. Check [ROLLBACK_MANUAL.md - Troubleshooting](./ROLLBACK_MANUAL.md#troubleshooting--fixes)
|
||||
2. Review [ROLLBACK_MANUAL.md - FAQ](./ROLLBACK_MANUAL.md#faq)
|
||||
3. Check Jenkins console output
|
||||
4. Verify RBAC permissions
|
||||
5. Review pod status and logs
|
||||
|
||||
**Still stuck?**
|
||||
- Jenkins logs: Jenkins → Build → Console Output
|
||||
- K8s events: `kubectl get events -n demo-app`
|
||||
- Pod logs: `kubectl logs -n demo-app -l app=demo-nginx`
|
||||
|
||||
---
|
||||
|
||||
## 📝 Documentation Updates
|
||||
|
||||
**Last Updated:** 2026-01-06
|
||||
**Version:** 1.0
|
||||
**Status:** Production Ready ✅
|
||||
|
||||
**Recent Changes:**
|
||||
- ✅ Added comprehensive manual rollback guide
|
||||
- ✅ Added quick reference card
|
||||
- ✅ Documented all 5 fixes
|
||||
- ✅ Added examples and best practices
|
||||
- ✅ Production-ready feature
|
||||
|
||||
---
|
||||
|
||||
**Ready to rollback? Start with [Quick Reference](./ROLLBACK_QUICK_REF.md)! 🚀**
|
||||
@@ -1,717 +0,0 @@
|
||||
# 🔄 Manual Rollback Feature - Complete Documentation
|
||||
|
||||
## 📋 Table of Contents
|
||||
|
||||
1. [Overview](#overview)
|
||||
2. [Features](#features)
|
||||
3. [Setup Guide](#setup-guide)
|
||||
4. [Usage Guide](#usage-guide)
|
||||
5. [Rollback Methods](#rollback-methods)
|
||||
6. [Troubleshooting & Fixes](#troubleshooting--fixes)
|
||||
7. [Best Practices](#best-practices)
|
||||
8. [Examples](#examples)
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Manual Rollback feature позволяет откатить deployment на любую предыдущую версию через Jenkins Pipeline.
|
||||
|
||||
### Key Features:
|
||||
- ✅ **3 способа rollback** (IMAGE_TAG, REVISION_NUMBER, GIT_COMMIT)
|
||||
- ✅ **GitOps sync** - автоматически обновляет Git manifests
|
||||
- ✅ **Zero downtime** - rolling updates
|
||||
- ✅ **DRY_RUN mode** - безопасное тестирование
|
||||
- ✅ **Health checks** - опциональная проверка после rollback
|
||||
- ✅ **Full RBAC** - правильные permissions
|
||||
|
||||
---
|
||||
|
||||
## Features
|
||||
|
||||
### Rollback Methods
|
||||
|
||||
| Method | Description | Example | Use Case |
|
||||
|--------|-------------|---------|----------|
|
||||
| **IMAGE_TAG** | По Docker image tag | `main-21` | Знаешь конкретный build number |
|
||||
| **REVISION_NUMBER** | По Kubernetes revision | `2` | Откат на N шагов назад |
|
||||
| **GIT_COMMIT** | По Git commit SHA | `abc123def` | Точное состояние кода |
|
||||
|
||||
### Parameters
|
||||
|
||||
```groovy
|
||||
ROLLBACK_METHOD // Выбор метода
|
||||
TARGET_VERSION // Целевая версия (auto-trim whitespace)
|
||||
SKIP_HEALTH_CHECK // Пропустить health checks (default: false)
|
||||
DRY_RUN // Только показать план (default: false)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Setup Guide
|
||||
|
||||
### Step 1: Create Jenkins Pipeline
|
||||
|
||||
```
|
||||
1. Jenkins → New Item
|
||||
2. Name: demo-nginx-rollback
|
||||
3. Type: Pipeline
|
||||
4. Click OK
|
||||
```
|
||||
|
||||
### Step 2: Configure Pipeline
|
||||
|
||||
```yaml
|
||||
Pipeline:
|
||||
Definition: Pipeline script from SCM
|
||||
SCM: Git
|
||||
Repository URL: http://gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops
|
||||
Credentials: gitea-credentials
|
||||
Branch: */main
|
||||
Script Path: apps/demo-nginx/Jenkinsfile.rollback
|
||||
```
|
||||
|
||||
### Step 3: Verify RBAC
|
||||
|
||||
RBAC уже настроен в `apps/jenkins/rbac.yaml`:
|
||||
|
||||
```yaml
|
||||
ClusterRole: jenkins-deployer
|
||||
Permissions:
|
||||
- pods, services, deployments (full CRUD)
|
||||
- pods/exec, pods/log (for health checks)
|
||||
- ingresses, applications (for ArgoCD)
|
||||
```
|
||||
|
||||
### Step 4: Test with DRY_RUN
|
||||
|
||||
```
|
||||
Jenkins → demo-nginx-rollback → Build with Parameters
|
||||
├─ ROLLBACK_METHOD: IMAGE_TAG
|
||||
├─ TARGET_VERSION: main-21
|
||||
├─ DRY_RUN: ✅ true
|
||||
└─ Build
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Usage Guide
|
||||
|
||||
### Quick Start
|
||||
|
||||
```
|
||||
Jenkins → demo-nginx-rollback → Build with Parameters
|
||||
|
||||
┌─────────────────────────────────────┐
|
||||
│ ROLLBACK_METHOD: IMAGE_TAG │
|
||||
│ TARGET_VERSION: main-21 │
|
||||
│ SKIP_HEALTH_CHECK: true (рекоменд.) │
|
||||
│ DRY_RUN: false │
|
||||
└─────────────────────────────────────┘
|
||||
|
||||
→ Build → ✅ SUCCESS!
|
||||
```
|
||||
|
||||
### Pipeline Stages
|
||||
|
||||
```
|
||||
Stage 1: Validate Input
|
||||
└─ Trim whitespace, validate TARGET_VERSION
|
||||
|
||||
Stage 2: Show Current State
|
||||
└─ Current deployment, image, pods, history
|
||||
|
||||
Stage 3: Prepare Rollback
|
||||
└─ Build target image path or verify revision
|
||||
|
||||
Stage 4: Execute Rollback
|
||||
├─ kubectl set image (or rollout undo)
|
||||
└─ Git commit & push
|
||||
|
||||
Stage 5: Wait for Rollout
|
||||
├─ kubectl rollout status (300s timeout)
|
||||
└─ sleep 10s (stabilization)
|
||||
|
||||
Stage 6: Health Check (optional)
|
||||
└─ 5 retry attempts with 5s delay
|
||||
|
||||
Stage 7: Show New State
|
||||
└─ New deployment state, pods, history
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Rollback Methods
|
||||
|
||||
### Method 1: IMAGE_TAG (Recommended)
|
||||
|
||||
**Когда использовать:** Знаешь конкретный build number
|
||||
|
||||
**Как найти tag:**
|
||||
```bash
|
||||
# Docker Hub
|
||||
https://hub.docker.com/r/vladcrypto/demo-nginx/tags
|
||||
|
||||
# Jenkins build history
|
||||
Jenkins → demo-nginx → Build History
|
||||
|
||||
# Git commits
|
||||
git log --oneline | grep "Update image"
|
||||
```
|
||||
|
||||
**Example:**
|
||||
```
|
||||
ROLLBACK_METHOD: IMAGE_TAG
|
||||
TARGET_VERSION: main-21
|
||||
|
||||
Result: Rollback to docker.io/vladcrypto/demo-nginx:main-21
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Method 2: REVISION_NUMBER
|
||||
|
||||
**Когда использовать:** Нужно откатиться на N шагов назад
|
||||
|
||||
**Как найти revision:**
|
||||
```bash
|
||||
kubectl rollout history deployment/demo-nginx -n demo-app
|
||||
|
||||
# Output:
|
||||
REVISION CHANGE-CAUSE
|
||||
1 Initial deployment
|
||||
2 Update to main-20
|
||||
3 Update to main-21
|
||||
4 Update to main-22 (current)
|
||||
```
|
||||
|
||||
**Example:**
|
||||
```
|
||||
ROLLBACK_METHOD: REVISION_NUMBER
|
||||
TARGET_VERSION: 2
|
||||
|
||||
Result: Rollback to revision 2 (main-20)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Method 3: GIT_COMMIT
|
||||
|
||||
**Когда использовать:** Нужно вернуться к конкретному состоянию кода
|
||||
|
||||
**Как найти commit:**
|
||||
```bash
|
||||
# Gitea
|
||||
https://git.thedevops.dev/admin/k3s-gitops/commits/branch/main
|
||||
|
||||
# Git CLI
|
||||
git log --oneline apps/demo-nginx/deployment.yaml
|
||||
|
||||
# Output:
|
||||
abc123d Update image to main-22 (current)
|
||||
def456e Update image to main-21
|
||||
ghi789f Update image to main-20
|
||||
```
|
||||
|
||||
**Example:**
|
||||
```
|
||||
ROLLBACK_METHOD: GIT_COMMIT
|
||||
TARGET_VERSION: def456e
|
||||
|
||||
Result: Rollback to commit def456e
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting & Fixes
|
||||
|
||||
### Issue #1: Container Name Error ✅ FIXED
|
||||
|
||||
**Error:**
|
||||
```
|
||||
error: unable to find container named "demo-nginx"
|
||||
```
|
||||
|
||||
**Root Cause:**
|
||||
Pipeline использовал deployment name вместо container name.
|
||||
|
||||
**Fix:**
|
||||
```groovy
|
||||
environment {
|
||||
APP_NAME = 'demo-nginx' // Deployment name
|
||||
CONTAINER_NAME = 'nginx' // Container name ✅
|
||||
}
|
||||
|
||||
kubectl set image deployment/${APP_NAME} \
|
||||
${CONTAINER_NAME}=${TARGET_IMAGE}
|
||||
```
|
||||
|
||||
**How to verify:**
|
||||
```bash
|
||||
kubectl get deployment demo-nginx -n demo-app \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].name}'
|
||||
# Output: nginx
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Issue #2: Whitespace in Input ✅ FIXED
|
||||
|
||||
**Error:**
|
||||
```
|
||||
Target image: docker.io/vladcrypto/demo-nginx: main-21
|
||||
^
|
||||
Space!
|
||||
```
|
||||
|
||||
**Root Cause:**
|
||||
User ввел TARGET_VERSION с пробелом.
|
||||
|
||||
**Fix:**
|
||||
```groovy
|
||||
stage('Validate Input') {
|
||||
// Auto-trim whitespace
|
||||
env.TARGET_VERSION_CLEAN = params.TARGET_VERSION.trim()
|
||||
|
||||
// Use everywhere
|
||||
${env.TARGET_VERSION_CLEAN}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Issue #3: RBAC Permissions ✅ FIXED
|
||||
|
||||
**Error:**
|
||||
```
|
||||
Error: User "system:serviceaccount:jenkins:jenkins"
|
||||
cannot create resource "pods/exec"
|
||||
```
|
||||
|
||||
**Root Cause:**
|
||||
Jenkins ServiceAccount не имел прав на pods/exec для health checks.
|
||||
|
||||
**Fix:**
|
||||
```yaml
|
||||
# apps/jenkins/rbac.yaml
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/exec", "pods/log"] # ← Added!
|
||||
verbs: ["create", "get"]
|
||||
```
|
||||
|
||||
**Applied:**
|
||||
```bash
|
||||
kubectl apply -f apps/jenkins/rbac.yaml
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Issue #4: Health Check Timing ⚠️ WORKAROUND
|
||||
|
||||
**Error:**
|
||||
```
|
||||
wget: can't connect to remote host: Connection refused
|
||||
```
|
||||
|
||||
**Root Cause:**
|
||||
Health check runs too early during rolling update (race condition).
|
||||
|
||||
**Workaround:**
|
||||
```groovy
|
||||
// Option 1: Skip health check (recommended)
|
||||
SKIP_HEALTH_CHECK: true
|
||||
|
||||
// Option 2: Longer stabilization wait
|
||||
sleep 30 // Instead of 10
|
||||
```
|
||||
|
||||
**Timeline:**
|
||||
```
|
||||
T+0s: kubectl set image
|
||||
T+30s: Rollout status = complete
|
||||
T+40s: sleep 10s
|
||||
T+50s: Health check (pods might still be starting)
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
Use `SKIP_HEALTH_CHECK: true` и проверь вручную через 30-60s:
|
||||
|
||||
```bash
|
||||
kubectl get pods -n demo-app -l app=demo-nginx
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Issue #5: Bash Loop Syntax ✅ FIXED
|
||||
|
||||
**Error:**
|
||||
```
|
||||
Health check attempt {1..5}/5...
|
||||
# Loop executed only once!
|
||||
```
|
||||
|
||||
**Root Cause:**
|
||||
`{1..5}` не работает в sh/dash, нужен bash.
|
||||
|
||||
**Fix:**
|
||||
```bash
|
||||
#!/bin/bash # ← Added shebang
|
||||
set -e
|
||||
|
||||
# Fixed loop syntax
|
||||
for i in 1 2 3 4 5; do # Instead of {1..5}
|
||||
echo "Health check attempt $i/5..."
|
||||
if kubectl exec ...; then
|
||||
exit 0
|
||||
fi
|
||||
if [ $i -lt 5 ]; then
|
||||
sleep 5
|
||||
fi
|
||||
done
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Always Use DRY_RUN First
|
||||
|
||||
```
|
||||
Step 1: DRY_RUN=true → Проверь план
|
||||
Step 2: Verify output
|
||||
Step 3: DRY_RUN=false → Execute
|
||||
```
|
||||
|
||||
### 2. Use SKIP_HEALTH_CHECK for Emergency
|
||||
|
||||
```
|
||||
Emergency rollback:
|
||||
├─ SKIP_HEALTH_CHECK: true
|
||||
├─ Focus on speed
|
||||
└─ Verify manually after
|
||||
```
|
||||
|
||||
### 3. Document Rollback Reason
|
||||
|
||||
Add comment в Jenkins build:
|
||||
```
|
||||
Build Comment:
|
||||
"Rollback due to: API errors in main-23
|
||||
Previous working version: main-21
|
||||
Impact: None (zero downtime)"
|
||||
```
|
||||
|
||||
### 4. Monitor After Rollback
|
||||
|
||||
```bash
|
||||
# Watch pods
|
||||
watch kubectl get pods -n demo-app
|
||||
|
||||
# Check logs
|
||||
kubectl logs -n demo-app -l app=demo-nginx -f
|
||||
|
||||
# Verify image
|
||||
kubectl get deployment demo-nginx -n demo-app \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
||||
```
|
||||
|
||||
### 5. Verify in ArgoCD
|
||||
|
||||
```
|
||||
ArgoCD UI → demo-nginx
|
||||
├─ Status: Synced ✅
|
||||
└─ Health: Healthy ✅
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Examples
|
||||
|
||||
### Example 1: Quick Rollback to Previous Build
|
||||
|
||||
```
|
||||
Scenario: Build #23 failed, rollback to #21
|
||||
|
||||
Steps:
|
||||
1. Jenkins → demo-nginx-rollback
|
||||
2. IMAGE_TAG + main-21
|
||||
3. SKIP_HEALTH_CHECK: true
|
||||
4. Build
|
||||
|
||||
Time: ~2 minutes
|
||||
Result: ✅ SUCCESS
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Example 2: Rollback to Last Week's Version
|
||||
|
||||
```
|
||||
Scenario: Need stable version from last week
|
||||
|
||||
Steps:
|
||||
1. Find old build: Jenkins → Build History → #15
|
||||
2. Check image tag: main-15
|
||||
3. Jenkins → demo-nginx-rollback
|
||||
4. IMAGE_TAG + main-15
|
||||
5. DRY_RUN: true (verify first!)
|
||||
6. DRY_RUN: false (execute)
|
||||
|
||||
Result: ✅ Rolled back to main-15
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Example 3: Rollback by Revision Number
|
||||
|
||||
```
|
||||
Scenario: Откатить на 3 versions назад
|
||||
|
||||
Steps:
|
||||
1. Check history:
|
||||
kubectl rollout history deployment/demo-nginx -n demo-app
|
||||
|
||||
2. Find revision: 25 (current: 28)
|
||||
|
||||
3. Jenkins → demo-nginx-rollback
|
||||
4. REVISION_NUMBER + 25
|
||||
5. Build
|
||||
|
||||
Result: ✅ Rolled back to revision 25
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Example 4: Rollback by Git Commit
|
||||
|
||||
```
|
||||
Scenario: Нужно точное состояние кода
|
||||
|
||||
Steps:
|
||||
1. Find commit:
|
||||
git log --oneline apps/demo-nginx/deployment.yaml
|
||||
|
||||
2. Copy SHA: abc123def
|
||||
|
||||
3. Jenkins → demo-nginx-rollback
|
||||
4. GIT_COMMIT + abc123def
|
||||
5. Build
|
||||
|
||||
Result: ✅ Rolled back to commit abc123def
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Manual Verification Commands
|
||||
|
||||
### Check Deployment Status
|
||||
```bash
|
||||
kubectl get deployment demo-nginx -n demo-app
|
||||
|
||||
# Expected:
|
||||
NAME READY UP-TO-DATE AVAILABLE AGE
|
||||
demo-nginx 2/2 2 2 15h
|
||||
```
|
||||
|
||||
### Check Image Version
|
||||
```bash
|
||||
kubectl get deployment demo-nginx -n demo-app \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
||||
|
||||
# Expected: docker.io/vladcrypto/demo-nginx:main-21
|
||||
```
|
||||
|
||||
### Check Pods
|
||||
```bash
|
||||
kubectl get pods -n demo-app -l app=demo-nginx
|
||||
|
||||
# Expected: 2 pods Running
|
||||
```
|
||||
|
||||
### Check Rollout History
|
||||
```bash
|
||||
kubectl rollout history deployment/demo-nginx -n demo-app
|
||||
|
||||
# Shows all revisions
|
||||
```
|
||||
|
||||
### Test Health Endpoint
|
||||
```bash
|
||||
POD=$(kubectl get pods -n demo-app -l app=demo-nginx -o jsonpath='{.items[0].metadata.name}')
|
||||
kubectl exec $POD -n demo-app -- wget -q -O- http://localhost/health
|
||||
|
||||
# Expected: healthy
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Emergency Rollback Procedure
|
||||
|
||||
### If Production is Down
|
||||
|
||||
**Option 1: Jenkins (2 minutes)**
|
||||
```
|
||||
1. Jenkins → demo-nginx-rollback
|
||||
2. IMAGE_TAG → last known good version
|
||||
3. SKIP_HEALTH_CHECK: ✅ true
|
||||
4. Build
|
||||
```
|
||||
|
||||
**Option 2: kubectl (30 seconds)**
|
||||
```bash
|
||||
# Fastest - rollback to previous
|
||||
kubectl rollout undo deployment/demo-nginx -n demo-app
|
||||
|
||||
# To specific revision
|
||||
kubectl rollout undo deployment/demo-nginx -n demo-app --to-revision=25
|
||||
```
|
||||
|
||||
**Option 3: ArgoCD (1 minute)**
|
||||
```
|
||||
1. ArgoCD UI → demo-nginx
|
||||
2. History → Select previous version
|
||||
3. Rollback button
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Configuration Reference
|
||||
|
||||
### Environment Variables
|
||||
|
||||
```groovy
|
||||
APP_NAME = 'demo-nginx' // Deployment name
|
||||
CONTAINER_NAME = 'nginx' // Container name
|
||||
NAMESPACE = 'demo-app' // K8s namespace
|
||||
DOCKER_REGISTRY = 'docker.io' // Registry
|
||||
DOCKER_REPO = 'vladcrypto' // Docker Hub user
|
||||
HEALTH_CHECK_TIMEOUT = '300s' // Rollout timeout
|
||||
```
|
||||
|
||||
### Customization
|
||||
|
||||
Изменить настройки в Jenkinsfile.rollback:
|
||||
|
||||
```groovy
|
||||
// Увеличить timeout
|
||||
HEALTH_CHECK_TIMEOUT = '600s'
|
||||
|
||||
// Больше попыток health check
|
||||
for i in 1 2 3 4 5 6 7 8 9 10; do
|
||||
|
||||
// Дольше ждать stabilization
|
||||
sleep 30 // Instead of 10
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Monitoring & Alerts
|
||||
|
||||
### Grafana Dashboard
|
||||
|
||||
```promql
|
||||
# Rollback count
|
||||
sum(increase(deployment_rollback_total[1h])) by (deployment)
|
||||
|
||||
# Rollback rate
|
||||
rate(deployment_rollback_total[5m])
|
||||
|
||||
# Average rollback duration
|
||||
avg(deployment_rollback_duration_seconds)
|
||||
```
|
||||
|
||||
### Alert Rules
|
||||
|
||||
```yaml
|
||||
- alert: FrequentRollbacks
|
||||
expr: rate(deployment_rollback_total[1h]) > 2
|
||||
annotations:
|
||||
summary: "Frequent rollbacks detected"
|
||||
description: "More than 2 rollbacks in last hour"
|
||||
|
||||
- alert: RollbackFailed
|
||||
expr: deployment_rollback_failed_total > 0
|
||||
annotations:
|
||||
summary: "Rollback failed"
|
||||
description: "Manual intervention required"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Summary of All Fixes
|
||||
|
||||
| # | Issue | Fix | Status |
|
||||
|---|-------|-----|--------|
|
||||
| 1 | Container name wrong | Use `nginx` not `demo-nginx` | ✅ Fixed |
|
||||
| 2 | Whitespace in input | Auto-trim with `.trim()` | ✅ Fixed |
|
||||
| 3 | RBAC pods/exec | Add permission to ClusterRole | ✅ Fixed |
|
||||
| 4 | Health check timing | Use `SKIP_HEALTH_CHECK=true` | ⚠️ Workaround |
|
||||
| 5 | Bash loop syntax | Use explicit list `1 2 3 4 5` | ✅ Fixed |
|
||||
|
||||
---
|
||||
|
||||
## Success Criteria
|
||||
|
||||
✅ **Rollback Methods:** 3/3 working (IMAGE_TAG, REVISION, GIT_COMMIT)
|
||||
✅ **GitOps Sync:** Git commits automatically
|
||||
✅ **Zero Downtime:** Rolling updates
|
||||
✅ **RBAC:** Full permissions configured
|
||||
✅ **Input Validation:** Whitespace auto-trimmed
|
||||
✅ **DRY_RUN:** Safe testing mode
|
||||
✅ **Retry Logic:** 5 attempts with proper bash syntax
|
||||
⚠️ **Health Check:** Optional (use SKIP_HEALTH_CHECK=true)
|
||||
|
||||
---
|
||||
|
||||
## FAQ
|
||||
|
||||
### Q: Health check всегда падает, это нормально?
|
||||
|
||||
**A:** Да, из-за timing race condition во время rolling update. Используй `SKIP_HEALTH_CHECK: true` и проверь вручную через 30-60s.
|
||||
|
||||
### Q: Как откатиться на несколько версий назад?
|
||||
|
||||
**A:** Используй `REVISION_NUMBER` метод и укажи нужную revision из `kubectl rollout history`.
|
||||
|
||||
### Q: Можно ли откатить только в staging?
|
||||
|
||||
**A:** Да, измени `NAMESPACE` в Jenkinsfile или создай отдельный job для staging.
|
||||
|
||||
### Q: Как быстро откатиться в emergency?
|
||||
|
||||
**A:** Используй `kubectl rollout undo` (30 секунд) или Jenkins с `SKIP_HEALTH_CHECK=true` (2 минуты).
|
||||
|
||||
### Q: Что если Git commit fail?
|
||||
|
||||
**A:** Rollback всё равно произошёл в Kubernetes! Git нужен только для GitOps sync. ArgoCD пере-синкает через 3 минуты.
|
||||
|
||||
---
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [CI/CD Guide](../../../CICD_GUIDE.md)
|
||||
- [Automatic Rollback](../Jenkinsfile) - See `post { failure }` section
|
||||
- [Jenkins RBAC](../../jenkins/rbac.yaml)
|
||||
- [Deployment Manifest](../deployment.yaml)
|
||||
|
||||
---
|
||||
|
||||
## Support
|
||||
|
||||
**Issues?**
|
||||
- Check Jenkins console output
|
||||
- Verify RBAC permissions
|
||||
- Check pod status: `kubectl get pods -n demo-app`
|
||||
- Review ArgoCD sync status
|
||||
|
||||
**Need Help?**
|
||||
- Jenkins logs: Jenkins → Build → Console Output
|
||||
- Kubernetes events: `kubectl get events -n demo-app`
|
||||
- Pod logs: `kubectl logs -n demo-app -l app=demo-nginx`
|
||||
|
||||
---
|
||||
|
||||
**Last Updated:** 2026-01-06
|
||||
**Version:** 1.0
|
||||
**Status:** Production Ready ✅
|
||||
@@ -1,144 +0,0 @@
|
||||
# 🔄 Manual Rollback - Quick Reference
|
||||
|
||||
## 🚀 Quick Start (2 minutes)
|
||||
|
||||
```
|
||||
Jenkins → demo-nginx-rollback → Build with Parameters
|
||||
|
||||
ROLLBACK_METHOD: IMAGE_TAG
|
||||
TARGET_VERSION: main-21
|
||||
SKIP_HEALTH_CHECK: true (recommended!)
|
||||
DRY_RUN: false
|
||||
|
||||
→ Build → ✅ SUCCESS!
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📋 3 Ways to Rollback
|
||||
|
||||
### 1. By Image Tag (Fastest)
|
||||
```
|
||||
Method: IMAGE_TAG
|
||||
Target: main-21
|
||||
Use: When you know the build number
|
||||
```
|
||||
|
||||
### 2. By Revision Number
|
||||
```
|
||||
Method: REVISION_NUMBER
|
||||
Target: 2
|
||||
Use: Rollback N steps back
|
||||
Find: kubectl rollout history deployment/demo-nginx -n demo-app
|
||||
```
|
||||
|
||||
### 3. By Git Commit
|
||||
```
|
||||
Method: GIT_COMMIT
|
||||
Target: abc123def
|
||||
Use: Exact code state
|
||||
Find: git log --oneline apps/demo-nginx/deployment.yaml
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ⚡ Emergency Rollback (30 seconds)
|
||||
|
||||
```bash
|
||||
# Fastest - kubectl
|
||||
kubectl rollout undo deployment/demo-nginx -n demo-app
|
||||
|
||||
# To specific revision
|
||||
kubectl rollout undo deployment/demo-nginx -n demo-app --to-revision=25
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔍 Verify Rollback
|
||||
|
||||
```bash
|
||||
# Check image
|
||||
kubectl get deployment demo-nginx -n demo-app \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
||||
|
||||
# Check pods
|
||||
kubectl get pods -n demo-app -l app=demo-nginx
|
||||
|
||||
# Test health
|
||||
POD=$(kubectl get pods -n demo-app -l app=demo-nginx -o jsonpath='{.items[0].metadata.name}')
|
||||
kubectl exec $POD -n demo-app -- wget -q -O- http://localhost/health
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ⚙️ Parameters
|
||||
|
||||
| Parameter | Default | Recommended |
|
||||
|-----------|---------|-------------|
|
||||
| ROLLBACK_METHOD | IMAGE_TAG | IMAGE_TAG |
|
||||
| TARGET_VERSION | (required) | main-21 |
|
||||
| SKIP_HEALTH_CHECK | false | **true** |
|
||||
| DRY_RUN | false | false |
|
||||
|
||||
---
|
||||
|
||||
## 🐛 Common Issues - FIXED
|
||||
|
||||
| Issue | Fix | Status |
|
||||
|-------|-----|--------|
|
||||
| Wrong container name | Use `nginx` | ✅ Fixed |
|
||||
| Whitespace in input | Auto-trim | ✅ Fixed |
|
||||
| RBAC permission | Added pods/exec | ✅ Fixed |
|
||||
| Health check timing | Use SKIP_HEALTH_CHECK | ⚠️ Workaround |
|
||||
| Bash loop broken | Use `1 2 3 4 5` | ✅ Fixed |
|
||||
|
||||
---
|
||||
|
||||
## 💡 Best Practices
|
||||
|
||||
1. ✅ Always test with `DRY_RUN: true` first
|
||||
2. ✅ Use `SKIP_HEALTH_CHECK: true` for faster rollback
|
||||
3. ✅ Verify manually after rollback (30-60s wait)
|
||||
4. ✅ Document rollback reason in Jenkins build comment
|
||||
5. ✅ Check ArgoCD sync status after rollback
|
||||
|
||||
---
|
||||
|
||||
## 📊 Verification Commands
|
||||
|
||||
```bash
|
||||
# Full status check
|
||||
kubectl get deployment demo-nginx -n demo-app
|
||||
kubectl get pods -n demo-app -l app=demo-nginx
|
||||
kubectl rollout history deployment/demo-nginx -n demo-app
|
||||
|
||||
# Watch pods update
|
||||
watch kubectl get pods -n demo-app
|
||||
|
||||
# Check logs
|
||||
kubectl logs -n demo-app -l app=demo-nginx --tail=50
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Success Checklist
|
||||
|
||||
- [ ] Jenkins pipeline exists (demo-nginx-rollback)
|
||||
- [ ] RBAC configured (pods/exec permission)
|
||||
- [ ] Target version identified
|
||||
- [ ] DRY_RUN tested
|
||||
- [ ] Rollback executed
|
||||
- [ ] Pods verified (Running)
|
||||
- [ ] Image version confirmed
|
||||
- [ ] Health check passed (manual)
|
||||
- [ ] ArgoCD synced
|
||||
|
||||
---
|
||||
|
||||
## 📚 Full Documentation
|
||||
|
||||
See: [apps/demo-nginx/docs/ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md)
|
||||
|
||||
---
|
||||
|
||||
**Quick Reference - Keep this handy! 📌**
|
||||
@@ -1,107 +0,0 @@
|
||||
# Demo-Nginx Scaling History
|
||||
|
||||
This document tracks the scaling changes made to the demo-nginx deployment for capacity planning and audit purposes.
|
||||
|
||||
## Scaling Timeline
|
||||
|
||||
### 2026-01-12: Scale to 5 Replicas
|
||||
**Change:** Increased from 3 to 5 replicas
|
||||
**Reason:** Enhanced capacity and improved high availability
|
||||
**Impact:**
|
||||
- Additional 2 pod instances for better load distribution
|
||||
- Improved resilience against node failures
|
||||
- Better handling of traffic spikes
|
||||
- Total resource allocation:
|
||||
- CPU requests: 500m (5 × 100m)
|
||||
- Memory requests: 640Mi (5 × 128Mi)
|
||||
- CPU limits: 1000m (5 × 200m)
|
||||
- Memory limits: 1280Mi (5 × 256Mi)
|
||||
|
||||
**Commit:** `0669ac66c8a651f74b4c16a4618db03bdd843c02`
|
||||
**Deployment Method:** GitOps via ArgoCD automatic sync
|
||||
|
||||
### 2026-01-12: Scale to 3 Replicas
|
||||
**Change:** Increased from 2 to 3 replicas
|
||||
**Reason:** Initial scaling for improved availability
|
||||
**Commit:** `d5ffa97159ec391de950dc2d861361074ff3eee8`
|
||||
|
||||
### Initial Deployment: 2 Replicas
|
||||
**Change:** Initial deployment with 2 replicas
|
||||
**Reason:** Baseline deployment for demo application
|
||||
|
||||
## Current Configuration
|
||||
|
||||
- **Replicas:** 5
|
||||
- **Namespace:** demo-app
|
||||
- **Image:** docker.io/vladcrypto/demo-nginx:main-50
|
||||
- **Service Type:** ClusterIP
|
||||
- **Port:** 80
|
||||
- **Health Checks:** Liveness and Readiness probes enabled
|
||||
- **Monitoring:** Prometheus scraping enabled
|
||||
|
||||
## Scaling Guidelines
|
||||
|
||||
### When to Scale Up
|
||||
- CPU utilization consistently above 70%
|
||||
- Memory utilization consistently above 75%
|
||||
- Response time degradation observed
|
||||
- Preparing for expected traffic increase
|
||||
- Node maintenance requires pod redistribution
|
||||
|
||||
### When to Scale Down
|
||||
- CPU utilization consistently below 30% for extended period
|
||||
- Cost optimization requirements
|
||||
- Application usage patterns indicate over-provisioning
|
||||
|
||||
## Resource Considerations
|
||||
|
||||
Each replica consumes:
|
||||
- **CPU Request:** 100m
|
||||
- **Memory Request:** 128Mi
|
||||
- **CPU Limit:** 200m
|
||||
- **Memory Limit:** 256Mi
|
||||
|
||||
Total cluster resources for 5 replicas:
|
||||
- **Total CPU Requests:** 500m (0.5 cores)
|
||||
- **Total Memory Requests:** 640Mi (~0.625 GB)
|
||||
- **Total CPU Limits:** 1000m (1 core)
|
||||
- **Total Memory Limits:** 1280Mi (~1.25 GB)
|
||||
|
||||
## Monitoring and Alerts
|
||||
|
||||
Monitor the following metrics in Grafana:
|
||||
- Pod CPU/Memory utilization
|
||||
- Request latency
|
||||
- Error rates
|
||||
- Pod restart count
|
||||
- Service availability
|
||||
|
||||
## Rollback Procedure
|
||||
|
||||
If issues arise after scaling:
|
||||
|
||||
1. Revert the deployment.yaml in Git:
|
||||
```bash
|
||||
git revert <commit-hash>
|
||||
git push
|
||||
```
|
||||
|
||||
2. ArgoCD will automatically sync the change
|
||||
|
||||
3. Or manually scale via kubectl:
|
||||
```bash
|
||||
kubectl scale deployment demo-nginx -n demo-app --replicas=<previous-count>
|
||||
```
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [CICD_GUIDE.md](./CICD_GUIDE.md) - CI/CD pipeline documentation
|
||||
- [ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md) - Detailed rollback procedures
|
||||
- [README.md](./README.md) - General application documentation
|
||||
|
||||
## Notes
|
||||
|
||||
- All scaling operations are tracked in Git for audit trail
|
||||
- ArgoCD manages automatic synchronization from Git to cluster
|
||||
- Changes follow GitOps principles for declarative infrastructure
|
||||
- Scaling decisions should be data-driven based on monitoring metrics
|
||||
@@ -1,471 +0,0 @@
|
||||
# 🔄 Automatic Rollback Feature
|
||||
|
||||
## ✅ Что добавлено
|
||||
|
||||
Pipeline теперь автоматически откатывается к предыдущей версии при любой ошибке деплоя!
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Как работает
|
||||
|
||||
### 1. **Save Current State** (перед деплоем)
|
||||
```
|
||||
📸 Сохраняет:
|
||||
- Текущий Docker image tag
|
||||
- Количество реплик
|
||||
- Git commit SHA
|
||||
```
|
||||
|
||||
### 2. **Deploy New Version**
|
||||
```
|
||||
🚀 Деплоит новую версию через:
|
||||
- Build Docker image
|
||||
- Push to registry
|
||||
- Update Git manifests
|
||||
- ArgoCD sync
|
||||
```
|
||||
|
||||
### 3. **Health Checks**
|
||||
```
|
||||
🏥 Проверяет:
|
||||
- Rollout status (timeout: 300s)
|
||||
- Pod readiness (все поды Ready)
|
||||
- Image version (правильный tag)
|
||||
- Health endpoint (5 попыток)
|
||||
```
|
||||
|
||||
### 4. **Auto Rollback** (при ошибке)
|
||||
```
|
||||
🔄 Если что-то пошло не так:
|
||||
- kubectl rollout undo
|
||||
- Revert Git commit
|
||||
- Restore previous state
|
||||
- Notify в logs
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📊 Pipeline Stages
|
||||
|
||||
```
|
||||
┌─────────────────────────────┐
|
||||
│ 1. Save Current State │ ← Сохраняет текущую версию
|
||||
└─────────────┬───────────────┘
|
||||
↓
|
||||
┌─────────────────────────────┐
|
||||
│ 2. Checkout Source │
|
||||
└─────────────┬───────────────┘
|
||||
↓
|
||||
┌─────────────────────────────┐
|
||||
│ 3. Build Docker Image │
|
||||
└─────────────┬───────────────┘
|
||||
↓
|
||||
┌─────────────────────────────┐
|
||||
│ 4. Push to Registry │
|
||||
└─────────────┬───────────────┘
|
||||
↓
|
||||
┌─────────────────────────────┐
|
||||
│ 5. Update GitOps Manifests │
|
||||
└─────────────┬───────────────┘
|
||||
↓
|
||||
┌─────────────────────────────┐
|
||||
│ 6. Wait for Deployment │ ← 300s timeout
|
||||
└─────────────┬───────────────┘
|
||||
↓
|
||||
┌─────────────────────────────┐
|
||||
│ 7. Health Check │ ← 5 retries
|
||||
└─────────────┬───────────────┘
|
||||
↓
|
||||
┌───────┴────────┐
|
||||
↓ ↓
|
||||
SUCCESS FAILURE
|
||||
│ │
|
||||
│ ↓
|
||||
│ ┌──────────────┐
|
||||
│ │ ROLLBACK │ ← Автоматически!
|
||||
│ └──────────────┘
|
||||
↓
|
||||
✅ DONE
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Configuration
|
||||
|
||||
### Environment Variables:
|
||||
|
||||
```groovy
|
||||
// Rollback configuration
|
||||
ROLLBACK_ENABLED = 'true' // Включить/выключить rollback
|
||||
DEPLOYMENT_TIMEOUT = '300s' // Timeout для rollout
|
||||
HEALTH_CHECK_RETRIES = '5' // Количество попыток health check
|
||||
HEALTH_CHECK_DELAY = '10' // Задержка между попытками (сек)
|
||||
```
|
||||
|
||||
### Изменить настройки:
|
||||
|
||||
```groovy
|
||||
environment {
|
||||
ROLLBACK_ENABLED = 'false' // Выключить rollback
|
||||
DEPLOYMENT_TIMEOUT = '600s' // Увеличить timeout
|
||||
HEALTH_CHECK_RETRIES = '10' // Больше попыток
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🧪 Тестирование Rollback
|
||||
|
||||
### Сценарий 1: Симуляция deployment failure
|
||||
|
||||
Измени deployment.yaml чтобы вызвать ошибку:
|
||||
|
||||
```yaml
|
||||
# apps/demo-nginx/deployment.yaml
|
||||
spec:
|
||||
containers:
|
||||
- name: nginx
|
||||
image: nginx:nonexistent-tag # Несуществующий tag
|
||||
```
|
||||
|
||||
**Результат:**
|
||||
```
|
||||
❌ Deployment failed
|
||||
🔄 Rollback initiated automatically
|
||||
✅ Rolled back to previous version
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Сценарий 2: Симуляция health check failure
|
||||
|
||||
Измени nginx.conf чтобы сломать /health:
|
||||
|
||||
```nginx
|
||||
location /health {
|
||||
return 500 "broken"; # Вернет 500 error
|
||||
}
|
||||
```
|
||||
|
||||
**Результат:**
|
||||
```
|
||||
❌ Health check failed after 5 attempts
|
||||
🔄 Rollback initiated automatically
|
||||
✅ Previous version restored
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Сценарий 3: Симуляция timeout
|
||||
|
||||
Установи очень короткий timeout:
|
||||
|
||||
```groovy
|
||||
DEPLOYMENT_TIMEOUT = '10s' // Слишком короткий
|
||||
```
|
||||
|
||||
**Результат:**
|
||||
```
|
||||
❌ Deployment timeout exceeded
|
||||
🔄 Rollback initiated automatically
|
||||
✅ Rolled back successfully
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📋 Rollback Process Details
|
||||
|
||||
### Что происходит при rollback:
|
||||
|
||||
1. **Kubernetes Rollback:**
|
||||
```bash
|
||||
kubectl rollout undo deployment/demo-nginx -n demo-app
|
||||
```
|
||||
|
||||
2. **Git Revert:**
|
||||
```bash
|
||||
git revert --no-edit HEAD
|
||||
git push origin main
|
||||
```
|
||||
|
||||
3. **ArgoCD Sync:**
|
||||
```
|
||||
ArgoCD автоматически применит revert commit
|
||||
```
|
||||
|
||||
4. **Verification:**
|
||||
```bash
|
||||
kubectl rollout status deployment/demo-nginx -n demo-app
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔍 Как проверить что rollback сработал
|
||||
|
||||
### В Jenkins Console Output:
|
||||
|
||||
```
|
||||
❌ DEPLOYMENT FAILED - INITIATING ROLLBACK!
|
||||
|
||||
Rolling back to previous version...
|
||||
🔄 Rolling back to: docker.io/vladcrypto/demo-nginx:main-21
|
||||
|
||||
✅ ROLLBACK COMPLETED!
|
||||
|
||||
Rolled back to: docker.io/vladcrypto/demo-nginx:main-21
|
||||
Current build (#22) has been reverted.
|
||||
|
||||
Please check logs and fix the issue before redeploying.
|
||||
```
|
||||
|
||||
### В Kubernetes:
|
||||
|
||||
```bash
|
||||
# Check deployment history
|
||||
kubectl rollout history deployment/demo-nginx -n demo-app
|
||||
|
||||
# Вывод:
|
||||
REVISION CHANGE-CAUSE
|
||||
21 Updated to main-21
|
||||
22 Updated to main-22
|
||||
23 Rollback to main-21 ← Rollback!
|
||||
```
|
||||
|
||||
### В Git:
|
||||
|
||||
```bash
|
||||
git log --oneline
|
||||
|
||||
# Вывод:
|
||||
abc1234 Revert "chore(demo-nginx): Update image to main-22"
|
||||
def5678 chore(demo-nginx): Update image to main-22
|
||||
ghi9012 chore(demo-nginx): Update image to main-21
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 💡 Best Practices
|
||||
|
||||
### 1. **Всегда тестируй в staging**
|
||||
|
||||
```groovy
|
||||
stage('Deploy to Staging') {
|
||||
when { branch 'develop' }
|
||||
steps {
|
||||
// Deploy to staging namespace
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2. **Мониторинг после деплоя**
|
||||
|
||||
```groovy
|
||||
stage('Post-Deploy Monitoring') {
|
||||
steps {
|
||||
sh """
|
||||
# Monitor for 5 minutes
|
||||
for i in {1..30}; do
|
||||
kubectl top pods -n demo-app
|
||||
sleep 10
|
||||
done
|
||||
"""
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3. **Slack Notifications**
|
||||
|
||||
```groovy
|
||||
post {
|
||||
failure {
|
||||
slackSend(
|
||||
color: 'danger',
|
||||
message: """
|
||||
🚨 ROLLBACK EXECUTED!
|
||||
Build: #${BUILD_NUMBER}
|
||||
Rolled back to previous version
|
||||
"""
|
||||
)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 4. **Сохранение artifacts**
|
||||
|
||||
```groovy
|
||||
post {
|
||||
always {
|
||||
archiveArtifacts artifacts: '/tmp/previous_*.txt', allowEmptyArchive: true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ⚠️ Important Notes
|
||||
|
||||
### Rollback НЕ сработает если:
|
||||
|
||||
1. **Нет предыдущей версии:**
|
||||
```
|
||||
⚠️ No previous version found - cannot rollback automatically
|
||||
Manual intervention required!
|
||||
```
|
||||
|
||||
2. **ROLLBACK_ENABLED = 'false':**
|
||||
```
|
||||
❌ Pipeline failed! (Rollback disabled)
|
||||
```
|
||||
|
||||
3. **Не main branch:**
|
||||
```
|
||||
Rollback only works on main branch
|
||||
```
|
||||
|
||||
### Ручной rollback:
|
||||
|
||||
Если автоматический rollback не сработал:
|
||||
|
||||
```bash
|
||||
# Kubernetes rollback
|
||||
kubectl rollout undo deployment/demo-nginx -n demo-app
|
||||
|
||||
# Git revert
|
||||
cd k3s-gitops
|
||||
git revert HEAD
|
||||
git push origin main
|
||||
|
||||
# Force ArgoCD sync
|
||||
kubectl patch application demo-nginx -n argocd \
|
||||
--type merge -p '{"operation":{"sync":{}}}'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📊 Monitoring & Alerts
|
||||
|
||||
### Grafana Dashboard
|
||||
|
||||
Добавь панели для мониторинга rollbacks:
|
||||
|
||||
```promql
|
||||
# Number of rollbacks
|
||||
sum(rate(deployment_rollback_total[5m])) by (deployment)
|
||||
|
||||
# Rollback duration
|
||||
histogram_quantile(0.95,
|
||||
rate(deployment_rollback_duration_seconds_bucket[5m])
|
||||
)
|
||||
```
|
||||
|
||||
### Alert Rules
|
||||
|
||||
```yaml
|
||||
- alert: FrequentRollbacks
|
||||
expr: rate(deployment_rollback_total[1h]) > 3
|
||||
annotations:
|
||||
summary: "Frequent rollbacks detected"
|
||||
description: "More than 3 rollbacks in last hour"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Advanced Features (Future)
|
||||
|
||||
### 1. **Canary Deployments**
|
||||
|
||||
```groovy
|
||||
stage('Canary Deploy') {
|
||||
steps {
|
||||
sh """
|
||||
# Deploy 10% traffic to new version
|
||||
kubectl set image deployment/${APP_NAME}
|
||||
${APP_NAME}=${IMAGE_TAG}
|
||||
--record
|
||||
kubectl scale deployment/${APP_NAME}-canary --replicas=1
|
||||
"""
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2. **Blue-Green Deployments**
|
||||
|
||||
```groovy
|
||||
stage('Blue-Green Switch') {
|
||||
steps {
|
||||
sh """
|
||||
# Switch service to new deployment
|
||||
kubectl patch service ${APP_NAME}
|
||||
-p '{"spec":{"selector":{"version":"${IMAGE_TAG}"}}}'
|
||||
"""
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3. **Smoke Tests**
|
||||
|
||||
```groovy
|
||||
stage('Smoke Tests') {
|
||||
steps {
|
||||
sh """
|
||||
# Run automated tests
|
||||
curl -f http://${APP_NAME}/api/health
|
||||
curl -f http://${APP_NAME}/api/status
|
||||
"""
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ✅ Success Criteria
|
||||
|
||||
Pipeline считается успешным когда:
|
||||
|
||||
- ✅ Docker image built
|
||||
- ✅ Image pushed to registry
|
||||
- ✅ Git manifests updated
|
||||
- ✅ Deployment rolled out (300s timeout)
|
||||
- ✅ All pods ready
|
||||
- ✅ Image version matches
|
||||
- ✅ Health endpoint responds (5 retries)
|
||||
|
||||
Pipeline откатывается если:
|
||||
|
||||
- ❌ Deployment timeout
|
||||
- ❌ Pod not ready
|
||||
- ❌ Image version mismatch
|
||||
- ❌ Health check failed
|
||||
|
||||
---
|
||||
|
||||
## 🎉 Summary
|
||||
|
||||
**Automatic Rollback добавляет:**
|
||||
|
||||
✅ Безопасность деплоев
|
||||
✅ Автоматическое восстановление
|
||||
✅ Сохранение предыдущего состояния
|
||||
✅ Git history revert
|
||||
✅ Kubernetes rollback
|
||||
✅ Health checks
|
||||
✅ Timeout protection
|
||||
|
||||
**Zero manual intervention needed!** 🚀
|
||||
|
||||
---
|
||||
|
||||
## 📝 Testing Checklist
|
||||
|
||||
- [ ] Normal deployment работает
|
||||
- [ ] Failed deployment triggers rollback
|
||||
- [ ] Previous version restored
|
||||
- [ ] Git commit reverted
|
||||
- [ ] Health checks work
|
||||
- [ ] Timeout works
|
||||
- [ ] Notifications sent
|
||||
- [ ] Logs clear and helpful
|
||||
|
||||
---
|
||||
|
||||
**Your pipeline is now production-ready with automatic rollback! 🎉**
|
||||
@@ -1,26 +0,0 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: demo-nginx
|
||||
namespace: demo-app
|
||||
labels:
|
||||
app: demo-nginx
|
||||
annotations:
|
||||
kubernetes.io/ingress.class: traefik
|
||||
cert-manager.io/cluster-issuer: letsencrypt-http
|
||||
spec:
|
||||
rules:
|
||||
- host: demo-nginx.thedevops.dev
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: demo-nginx
|
||||
port:
|
||||
number: 80
|
||||
tls:
|
||||
- hosts:
|
||||
- demo-nginx.thedevops.dev
|
||||
secretName: demo-nginx-tls
|
||||
@@ -1,6 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: demo-app
|
||||
labels:
|
||||
app: demo-nginx
|
||||
Reference in New Issue
Block a user