chore: remove demo-nginx app
This commit is contained in:
@@ -1,69 +0,0 @@
|
|||||||
############################################
|
|
||||||
# Application
|
|
||||||
############################################
|
|
||||||
APP_NAME=demo-nginx
|
|
||||||
NAMESPACE=demo-app
|
|
||||||
BRANCH_NAME=main
|
|
||||||
|
|
||||||
############################################
|
|
||||||
# Docker
|
|
||||||
############################################
|
|
||||||
DOCKER_REGISTRY=docker.io
|
|
||||||
DOCKER_REPO=vladcrypto
|
|
||||||
# IMAGE_TAG is computed in Jenkins as: ${BRANCH_NAME}-${BUILD_NUMBER}
|
|
||||||
# Example: main-100
|
|
||||||
|
|
||||||
############################################
|
|
||||||
# GitOps / Gitea
|
|
||||||
############################################
|
|
||||||
GITEA_URL=http://gitea-http.gitea.svc.cluster.local:3000
|
|
||||||
GITEA_REPO=admin/k3s-gitops
|
|
||||||
GITEA_BRANCH=main
|
|
||||||
GITOPS_MANIFEST_PATH=apps/demo-nginx/deployment.yaml
|
|
||||||
|
|
||||||
############################################
|
|
||||||
# ArgoCD
|
|
||||||
############################################
|
|
||||||
ARGOCD_APP_NAME=demo-nginx
|
|
||||||
ARGOCD_NAMESPACE=argocd
|
|
||||||
ARGOCD_SYNC_TIMEOUT=120 # seconds
|
|
||||||
|
|
||||||
############################################
|
|
||||||
# Deployment / Rollout
|
|
||||||
############################################
|
|
||||||
DEPLOYMENT_TIMEOUT=300s
|
|
||||||
SKIP_HEALTH_CHECK=true
|
|
||||||
ROLLBACK_ENABLED=false # GitOps-safe: Jenkins never reverts Git
|
|
||||||
|
|
||||||
############################################
|
|
||||||
# Jenkins Runtime (injected by Jenkins)
|
|
||||||
############################################
|
|
||||||
# BUILD_NUMBER=100
|
|
||||||
# BUILD_URL=https://jenkins.thedevops.dev/job/demo-nginx/job/main/100/
|
|
||||||
# BUILD_USER=jenkins
|
|
||||||
|
|
||||||
############################################
|
|
||||||
# Telegram Notifications
|
|
||||||
############################################
|
|
||||||
TELEGRAM_BOT_TOKEN=__REPLACE_ME__
|
|
||||||
TELEGRAM_CHAT_ID=__REPLACE_ME__
|
|
||||||
|
|
||||||
############################################
|
|
||||||
# Docker Hub Credentials
|
|
||||||
############################################
|
|
||||||
DOCKER_USER=__REPLACE_ME__
|
|
||||||
DOCKER_PASS=__REPLACE_ME__
|
|
||||||
|
|
||||||
############################################
|
|
||||||
# Gitea Credentials
|
|
||||||
############################################
|
|
||||||
GIT_USER=__REPLACE_ME__
|
|
||||||
GIT_PASS=__REPLACE_ME__
|
|
||||||
|
|
||||||
############################################
|
|
||||||
# Internal / Temporary Files (used by pipeline)
|
|
||||||
############################################
|
|
||||||
PREVIOUS_IMAGE_FILE=/tmp/previous_image_${BUILD_NUMBER}.txt
|
|
||||||
PREVIOUS_REPLICAS_FILE=/tmp/previous_replicas_${BUILD_NUMBER}.txt
|
|
||||||
PREVIOUS_COMMIT_FILE=/tmp/previous_commit_${BUILD_NUMBER}.txt
|
|
||||||
EXPECTED_COMMIT_FILE=/tmp/expected_commit_${BUILD_NUMBER}.txt
|
|
||||||
763
apps/demo-nginx/Jenkinsfile
vendored
763
apps/demo-nginx/Jenkinsfile
vendored
@@ -1,763 +0,0 @@
|
|||||||
pipeline {
|
|
||||||
agent any
|
|
||||||
options {
|
|
||||||
disableConcurrentBuilds()
|
|
||||||
}
|
|
||||||
environment {
|
|
||||||
APP_NAME = 'demo-nginx'
|
|
||||||
NAMESPACE = 'demo-app'
|
|
||||||
DOCKER_REGISTRY = 'docker.io'
|
|
||||||
DOCKER_REPO = 'vladcrypto'
|
|
||||||
GITEA_URL = 'http://gitea-http.gitea.svc.cluster.local:3000'
|
|
||||||
GITEA_REPO = 'admin/k3s-gitops'
|
|
||||||
GITEA_BRANCH = 'main'
|
|
||||||
BUILD_TAG = "${env.BUILD_NUMBER}"
|
|
||||||
IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}"
|
|
||||||
|
|
||||||
// Rollback configuration
|
|
||||||
ROLLBACK_ENABLED = 'true'
|
|
||||||
DEPLOYMENT_TIMEOUT = '300s'
|
|
||||||
ARGOCD_SYNC_TIMEOUT = '120'
|
|
||||||
SKIP_HEALTH_CHECK = 'true'
|
|
||||||
|
|
||||||
// Notification configuration
|
|
||||||
TELEGRAM_BOT_TOKEN = credentials('telegram-bot-token')
|
|
||||||
TELEGRAM_CHAT_ID = credentials('telegram-chat-id')
|
|
||||||
|
|
||||||
// Build info
|
|
||||||
BUILD_URL = "${env.BUILD_URL}"
|
|
||||||
GIT_COMMIT_SHORT = ""
|
|
||||||
DEPLOYMENT_START_TIME = ""
|
|
||||||
DEPLOYMENT_END_TIME = ""
|
|
||||||
}
|
|
||||||
|
|
||||||
stages {
|
|
||||||
stage('Initialization') {
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "🚀 Starting deployment pipeline..."
|
|
||||||
DEPLOYMENT_START_TIME = sh(script: 'date +%s', returnStdout: true).trim()
|
|
||||||
|
|
||||||
sendTelegramNotification(
|
|
||||||
status: 'STARTED',
|
|
||||||
message: """
|
|
||||||
🚀 <b>Deployment Started</b>
|
|
||||||
|
|
||||||
<b>Application:</b> ${APP_NAME}
|
|
||||||
<b>Build:</b> #${BUILD_NUMBER}
|
|
||||||
<b>Branch:</b> ${env.BRANCH_NAME}
|
|
||||||
<b>Namespace:</b> ${NAMESPACE}
|
|
||||||
<b>Started by:</b> ${env.BUILD_USER ?: 'Jenkins'}
|
|
||||||
|
|
||||||
<i>Building and deploying...</i>
|
|
||||||
""",
|
|
||||||
color: '🔵'
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Save Current State') {
|
|
||||||
when { branch 'main' }
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "📸 Saving current deployment state for rollback..."
|
|
||||||
|
|
||||||
sh """
|
|
||||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
|
||||||
-o jsonpath='{.spec.template.spec.containers[0].image}' \
|
|
||||||
> /tmp/previous_image_${BUILD_NUMBER}.txt || echo "none" > /tmp/previous_image_${BUILD_NUMBER}.txt
|
|
||||||
|
|
||||||
PREV_IMAGE=\$(cat /tmp/previous_image_${BUILD_NUMBER}.txt)
|
|
||||||
echo "Previous image: \${PREV_IMAGE}"
|
|
||||||
"""
|
|
||||||
|
|
||||||
sh """
|
|
||||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
|
||||||
-o jsonpath='{.spec.replicas}' \
|
|
||||||
> /tmp/previous_replicas_${BUILD_NUMBER}.txt || echo "2" > /tmp/previous_replicas_${BUILD_NUMBER}.txt
|
|
||||||
"""
|
|
||||||
|
|
||||||
echo "✅ Current state saved"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Checkout Source') {
|
|
||||||
steps {
|
|
||||||
echo "📦 Creating application artifacts..."
|
|
||||||
|
|
||||||
sh """
|
|
||||||
cat > Dockerfile << 'EOF'
|
|
||||||
FROM nginx:1.25.3-alpine
|
|
||||||
COPY index.html /usr/share/nginx/html/index.html
|
|
||||||
COPY nginx.conf /etc/nginx/nginx.conf
|
|
||||||
EXPOSE 80
|
|
||||||
CMD ["nginx", "-g", "daemon off;"]
|
|
||||||
EOF
|
|
||||||
"""
|
|
||||||
|
|
||||||
sh """
|
|
||||||
cat > index.html << EOF
|
|
||||||
<!DOCTYPE html>
|
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<title>Demo Nginx</title>
|
|
||||||
<style>
|
|
||||||
body {
|
|
||||||
font-family: Arial, sans-serif;
|
|
||||||
max-width: 800px;
|
|
||||||
margin: 50px auto;
|
|
||||||
padding: 20px;
|
|
||||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
|
||||||
color: white;
|
|
||||||
}
|
|
||||||
.container {
|
|
||||||
background: rgba(255, 255, 255, 0.1);
|
|
||||||
padding: 40px;
|
|
||||||
border-radius: 10px;
|
|
||||||
backdrop-filter: blur(10px);
|
|
||||||
}
|
|
||||||
h1 {
|
|
||||||
font-size: 48px;
|
|
||||||
margin-bottom: 20px;
|
|
||||||
}
|
|
||||||
p {
|
|
||||||
font-size: 24px;
|
|
||||||
margin: 10px 0;
|
|
||||||
}
|
|
||||||
.version {
|
|
||||||
font-family: 'Courier New', monospace;
|
|
||||||
background: rgba(0, 0, 0, 0.3);
|
|
||||||
padding: 10px 20px;
|
|
||||||
border-radius: 5px;
|
|
||||||
display: inline-block;
|
|
||||||
margin-top: 20px;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<div class="container">
|
|
||||||
<h1>Demo Nginx - Build #${BUILD_NUMBER}</h1>
|
|
||||||
<p>Environment: Production</p>
|
|
||||||
<p class="version">Version: ${IMAGE_TAG}</p>
|
|
||||||
<p style="font-size: 16px; margin-top: 30px; opacity: 0.8;">
|
|
||||||
Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
EOF
|
|
||||||
"""
|
|
||||||
|
|
||||||
sh '''
|
|
||||||
cat > nginx.conf << 'EOF'
|
|
||||||
user nginx;
|
|
||||||
worker_processes auto;
|
|
||||||
error_log /var/log/nginx/error.log warn;
|
|
||||||
pid /var/run/nginx.pid;
|
|
||||||
events { worker_connections 1024; }
|
|
||||||
http {
|
|
||||||
include /etc/nginx/mime.types;
|
|
||||||
default_type application/octet-stream;
|
|
||||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
|
||||||
'$status $body_bytes_sent "$http_referer" '
|
|
||||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
|
||||||
access_log /var/log/nginx/access.log main;
|
|
||||||
sendfile on;
|
|
||||||
keepalive_timeout 65;
|
|
||||||
server {
|
|
||||||
listen 80;
|
|
||||||
server_name _;
|
|
||||||
location / {
|
|
||||||
root /usr/share/nginx/html;
|
|
||||||
index index.html;
|
|
||||||
}
|
|
||||||
location /health {
|
|
||||||
access_log off;
|
|
||||||
return 200 "healthy\n";
|
|
||||||
add_header Content-Type text/plain;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
'''
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Build Docker Image') {
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "🏗️ Building Docker image..."
|
|
||||||
|
|
||||||
sendTelegramNotification(
|
|
||||||
status: 'BUILDING',
|
|
||||||
message: """
|
|
||||||
🏗️ <b>Building Docker Image</b>
|
|
||||||
|
|
||||||
<b>Image:</b> ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
|
||||||
<b>Stage:</b> Build
|
|
||||||
""",
|
|
||||||
color: '🔵'
|
|
||||||
)
|
|
||||||
|
|
||||||
sh """
|
|
||||||
docker build \
|
|
||||||
-t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} \
|
|
||||||
-t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest \
|
|
||||||
.
|
|
||||||
"""
|
|
||||||
echo "✅ Image built successfully!"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Push to Registry') {
|
|
||||||
when { branch 'main' }
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "📤 Pushing image to registry..."
|
|
||||||
|
|
||||||
sendTelegramNotification(
|
|
||||||
status: 'PUSHING',
|
|
||||||
message: """
|
|
||||||
📤 <b>Pushing to Registry</b>
|
|
||||||
|
|
||||||
<b>Registry:</b> ${DOCKER_REGISTRY}
|
|
||||||
<b>Image:</b> ${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
|
||||||
""",
|
|
||||||
color: '🔵'
|
|
||||||
)
|
|
||||||
|
|
||||||
withCredentials([usernamePassword(
|
|
||||||
credentialsId: 'docker-registry-credentials',
|
|
||||||
usernameVariable: 'DOCKER_USER',
|
|
||||||
passwordVariable: 'DOCKER_PASS'
|
|
||||||
)]) {
|
|
||||||
sh """
|
|
||||||
echo "\${DOCKER_PASS}" | docker login ${DOCKER_REGISTRY} -u "\${DOCKER_USER}" --password-stdin
|
|
||||||
docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
|
||||||
docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest
|
|
||||||
docker logout ${DOCKER_REGISTRY}
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
echo "✅ Image pushed successfully!"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Update GitOps Manifests') {
|
|
||||||
when { branch 'main' }
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "📝 Updating Kubernetes manifests..."
|
|
||||||
|
|
||||||
sendTelegramNotification(
|
|
||||||
status: 'UPDATING',
|
|
||||||
message: """
|
|
||||||
📝 <b>Updating GitOps Manifests</b>
|
|
||||||
|
|
||||||
<b>Repository:</b> ${GITEA_REPO}
|
|
||||||
<b>Branch:</b> ${GITEA_BRANCH}
|
|
||||||
<b>File:</b> apps/demo-nginx/deployment.yaml
|
|
||||||
""",
|
|
||||||
color: '🔵'
|
|
||||||
)
|
|
||||||
|
|
||||||
withCredentials([usernamePassword(
|
|
||||||
credentialsId: 'gitea-credentials',
|
|
||||||
usernameVariable: 'GIT_USER',
|
|
||||||
passwordVariable: 'GIT_PASS'
|
|
||||||
)]) {
|
|
||||||
sh """
|
|
||||||
rm -rf k3s-gitops || true
|
|
||||||
git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git
|
|
||||||
cd k3s-gitops
|
|
||||||
git config user.name "Jenkins"
|
|
||||||
git config user.email "jenkins@thedevops.dev"
|
|
||||||
|
|
||||||
# Save current commit for rollback
|
|
||||||
git rev-parse HEAD > /tmp/previous_commit_${BUILD_NUMBER}.txt
|
|
||||||
|
|
||||||
sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml
|
|
||||||
git add apps/demo-nginx/deployment.yaml
|
|
||||||
git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes"
|
|
||||||
git push origin main
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
echo "✅ Manifests updated!"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Wait for ArgoCD Sync') {
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "⏳ Waiting for ArgoCD to sync manifests..."
|
|
||||||
|
|
||||||
for (int i = 1; i <= 12; i++) {
|
|
||||||
def syncStatus = sh(
|
|
||||||
script: "kubectl get application demo-nginx -n argocd -o jsonpath='{.status.sync.status}'",
|
|
||||||
returnStdout: true
|
|
||||||
).trim()
|
|
||||||
|
|
||||||
echo "ArgoCD sync status : ${syncStatus}"
|
|
||||||
|
|
||||||
if (syncStatus == "Synced") {
|
|
||||||
echo "✅ ArgoCD manifests synced"
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sleep 10
|
|
||||||
}
|
|
||||||
|
|
||||||
error("❌ ArgoCD did not sync manifests in time")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
stage('Wait for Deployment') {
|
|
||||||
when { branch 'main' }
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "⏳ Waiting for Kubernetes rollout to complete..."
|
|
||||||
|
|
||||||
sendTelegramNotification(
|
|
||||||
status: 'DEPLOYING',
|
|
||||||
message: """
|
|
||||||
🚀 <b>Deploying to Kubernetes</b>
|
|
||||||
|
|
||||||
<b>Deployment:</b> ${APP_NAME}
|
|
||||||
<b>Namespace:</b> ${NAMESPACE}
|
|
||||||
<b>Image:</b> ${IMAGE_TAG}
|
|
||||||
<b>Timeout:</b> ${DEPLOYMENT_TIMEOUT}
|
|
||||||
|
|
||||||
<i>Rolling out new pods...</i>
|
|
||||||
""",
|
|
||||||
color: '🔵'
|
|
||||||
)
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Wait for rollout to complete
|
|
||||||
echo "Starting rollout status check..."
|
|
||||||
sh """
|
|
||||||
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT}
|
|
||||||
"""
|
|
||||||
echo "✅ Rollout completed successfully!"
|
|
||||||
|
|
||||||
// Additional verification - wait a bit for pods to stabilize
|
|
||||||
echo "Waiting 10 seconds for pods to stabilize..."
|
|
||||||
sleep 10
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
echo "❌ Deployment rollout failed: ${e.message}"
|
|
||||||
|
|
||||||
// Get pod status for debugging
|
|
||||||
try {
|
|
||||||
def podStatus = sh(
|
|
||||||
script: """
|
|
||||||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} -o wide
|
|
||||||
""",
|
|
||||||
returnStdout: true
|
|
||||||
).trim()
|
|
||||||
echo "Current pod status:\n${podStatus}"
|
|
||||||
|
|
||||||
def events = sh(
|
|
||||||
script: """
|
|
||||||
kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -20
|
|
||||||
""",
|
|
||||||
returnStdout: true
|
|
||||||
).trim()
|
|
||||||
echo "Recent events:\n${events}"
|
|
||||||
} catch (Exception debugEx) {
|
|
||||||
echo "Could not fetch debug info: ${debugEx.message}"
|
|
||||||
}
|
|
||||||
|
|
||||||
throw e
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Verify Deployment') {
|
|
||||||
when { branch 'main' }
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "✅ Verifying deployment and pod status..."
|
|
||||||
|
|
||||||
/* --------------------------------
|
|
||||||
* 1. Deployment readiness
|
|
||||||
* -------------------------------- */
|
|
||||||
sh """
|
|
||||||
set -e
|
|
||||||
echo "1. Checking deployment readiness..."
|
|
||||||
|
|
||||||
READY=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}')
|
|
||||||
DESIRED=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}')
|
|
||||||
|
|
||||||
echo "Ready replicas : \$READY"
|
|
||||||
echo "Desired replicas : \$DESIRED"
|
|
||||||
|
|
||||||
if [ "\$READY" != "\$DESIRED" ]; then
|
|
||||||
echo "❌ Not all replicas are ready"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
"""
|
|
||||||
|
|
||||||
/* --------------------------------
|
|
||||||
* 2. Actual running images
|
|
||||||
* -------------------------------- */
|
|
||||||
def podImages = sh(
|
|
||||||
script: """
|
|
||||||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} \
|
|
||||||
-o jsonpath='{range .items[*]}{.status.containerStatuses[0].image}{"\\n"}{end}'
|
|
||||||
""",
|
|
||||||
returnStdout: true
|
|
||||||
).trim()
|
|
||||||
|
|
||||||
echo "Running pod images:"
|
|
||||||
podImages.split("\\n").each { img ->
|
|
||||||
echo " - ${img}"
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!podImages.contains(IMAGE_TAG)) {
|
|
||||||
error("❌ Some pods are NOT running image tag ${IMAGE_TAG}")
|
|
||||||
}
|
|
||||||
|
|
||||||
/* --------------------------------
|
|
||||||
* 3. Restart count
|
|
||||||
* -------------------------------- */
|
|
||||||
def restarts = sh(
|
|
||||||
script: """
|
|
||||||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} \
|
|
||||||
-o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\\n"}{end}'
|
|
||||||
""",
|
|
||||||
returnStdout: true
|
|
||||||
).trim()
|
|
||||||
|
|
||||||
def maxRestart = restarts
|
|
||||||
.split("\\n")
|
|
||||||
.collect { it.toInteger() }
|
|
||||||
.max()
|
|
||||||
|
|
||||||
echo "Max restart count: ${maxRestart}"
|
|
||||||
|
|
||||||
if (maxRestart > 3) {
|
|
||||||
error("❌ High restart count detected: ${maxRestart}")
|
|
||||||
}
|
|
||||||
|
|
||||||
echo "✅ ALL VERIFICATION CHECKS PASSED"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
post {
|
|
||||||
success {
|
|
||||||
script {
|
|
||||||
DEPLOYMENT_END_TIME = sh(script: 'date +%s', returnStdout: true).trim()
|
|
||||||
def duration = (DEPLOYMENT_END_TIME.toInteger() - DEPLOYMENT_START_TIME.toInteger())
|
|
||||||
def durationMin = duration / 60
|
|
||||||
def durationSec = duration % 60
|
|
||||||
|
|
||||||
// Get deployment details
|
|
||||||
def podStatus = sh(
|
|
||||||
script: """
|
|
||||||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} \
|
|
||||||
-o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.phase}{" "}{.status.containerStatuses[0].restartCount}{"\\n"}{end}'
|
|
||||||
""",
|
|
||||||
returnStdout: true
|
|
||||||
).trim()
|
|
||||||
|
|
||||||
def deployedImage = sh(
|
|
||||||
script: """
|
|
||||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
|
||||||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
|
||||||
""",
|
|
||||||
returnStdout: true
|
|
||||||
).trim()
|
|
||||||
|
|
||||||
def replicas = sh(
|
|
||||||
script: """
|
|
||||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
|
||||||
-o jsonpath='{.status.replicas}'
|
|
||||||
""",
|
|
||||||
returnStdout: true
|
|
||||||
).trim()
|
|
||||||
|
|
||||||
echo """
|
|
||||||
✅ DEPLOYMENT SUCCESS!
|
|
||||||
|
|
||||||
Application: ${APP_NAME}
|
|
||||||
Image: ${deployedImage}
|
|
||||||
Namespace: ${NAMESPACE}
|
|
||||||
Build: #${BUILD_NUMBER}
|
|
||||||
Duration: ${durationMin}m ${durationSec}s
|
|
||||||
|
|
||||||
All checks passed! ✨
|
|
||||||
"""
|
|
||||||
|
|
||||||
// Send detailed success notification
|
|
||||||
sendTelegramNotification(
|
|
||||||
status: 'SUCCESS',
|
|
||||||
message: """
|
|
||||||
✅ <b>Deployment Successful!</b>
|
|
||||||
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
<b>📦 Application Details</b>
|
|
||||||
<b>Name:</b> ${APP_NAME}
|
|
||||||
<b>Build:</b> #${BUILD_NUMBER}
|
|
||||||
<b>Branch:</b> ${env.BRANCH_NAME}
|
|
||||||
<b>Namespace:</b> ${NAMESPACE}
|
|
||||||
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
<b>🐳 Docker Image</b>
|
|
||||||
<b>Registry:</b> ${DOCKER_REGISTRY}
|
|
||||||
<b>Repository:</b> ${DOCKER_REPO}/${APP_NAME}
|
|
||||||
<b>Tag:</b> ${IMAGE_TAG}
|
|
||||||
<b>Full Image:</b> ${deployedImage}
|
|
||||||
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
<b>☸️ Kubernetes Status</b>
|
|
||||||
<b>Replicas:</b> ${replicas}/${replicas} Ready
|
|
||||||
<b>Rollout:</b> Completed ✅
|
|
||||||
<b>Health:</b> All pods healthy
|
|
||||||
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
<b>⏱️ Deployment Metrics</b>
|
|
||||||
<b>Duration:</b> ${durationMin}m ${durationSec}s
|
|
||||||
<b>Started:</b> ${new Date(DEPLOYMENT_START_TIME.toLong() * 1000).format('HH:mm:ss')}
|
|
||||||
<b>Completed:</b> ${new Date(DEPLOYMENT_END_TIME.toLong() * 1000).format('HH:mm:ss')}
|
|
||||||
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
<b>🔗 Links</b>
|
|
||||||
<a href="${BUILD_URL}">Jenkins Build</a>
|
|
||||||
<a href="${GITEA_URL}/${GITEA_REPO}">GitOps Repository</a>
|
|
||||||
|
|
||||||
<i>Deployed by Jenkins CI/CD Pipeline</i> 🚀
|
|
||||||
""",
|
|
||||||
color: '✅'
|
|
||||||
)
|
|
||||||
|
|
||||||
// Cleanup rollback files
|
|
||||||
sh """
|
|
||||||
rm -f /tmp/previous_image_${BUILD_NUMBER}.txt
|
|
||||||
rm -f /tmp/previous_replicas_${BUILD_NUMBER}.txt
|
|
||||||
rm -f /tmp/previous_commit_${BUILD_NUMBER}.txt
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
failure {
|
|
||||||
script {
|
|
||||||
def errorDetails = ""
|
|
||||||
def previousImage = "unknown"
|
|
||||||
|
|
||||||
try {
|
|
||||||
errorDetails = sh(
|
|
||||||
script: """
|
|
||||||
kubectl get events -n ${NAMESPACE} --sort-by='.lastTimestamp' | tail -10
|
|
||||||
""",
|
|
||||||
returnStdout: true
|
|
||||||
).trim()
|
|
||||||
} catch (Exception e) {
|
|
||||||
errorDetails = "Could not fetch events: ${e.message}"
|
|
||||||
}
|
|
||||||
|
|
||||||
if (env.BRANCH_NAME == 'main' && env.ROLLBACK_ENABLED == 'true') {
|
|
||||||
echo """
|
|
||||||
❌ DEPLOYMENT FAILED - INITIATING ROLLBACK!
|
|
||||||
|
|
||||||
Rolling back to previous version...
|
|
||||||
"""
|
|
||||||
|
|
||||||
sendTelegramNotification(
|
|
||||||
status: 'ROLLING_BACK',
|
|
||||||
message: """
|
|
||||||
🔄 <b>Deployment Failed - Rolling Back</b>
|
|
||||||
|
|
||||||
<b>Application:</b> ${APP_NAME}
|
|
||||||
<b>Failed Build:</b> #${BUILD_NUMBER}
|
|
||||||
<b>Image:</b> ${IMAGE_TAG}
|
|
||||||
|
|
||||||
<i>Initiating automatic rollback...</i>
|
|
||||||
""",
|
|
||||||
color: '⚠️'
|
|
||||||
)
|
|
||||||
|
|
||||||
try {
|
|
||||||
previousImage = sh(
|
|
||||||
script: "cat /tmp/previous_image_${BUILD_NUMBER}.txt 2>/dev/null || echo 'none'",
|
|
||||||
returnStdout: true
|
|
||||||
).trim()
|
|
||||||
|
|
||||||
if (previousImage != 'none' && previousImage != '') {
|
|
||||||
echo "🔄 Rolling back to: ${previousImage}"
|
|
||||||
|
|
||||||
sh """
|
|
||||||
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
|
|
||||||
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=180s
|
|
||||||
"""
|
|
||||||
|
|
||||||
withCredentials([usernamePassword(
|
|
||||||
credentialsId: 'gitea-credentials',
|
|
||||||
usernameVariable: 'GIT_USER',
|
|
||||||
passwordVariable: 'GIT_PASS'
|
|
||||||
)]) {
|
|
||||||
sh """
|
|
||||||
cd k3s-gitops || git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git
|
|
||||||
cd k3s-gitops
|
|
||||||
git config user.name "Jenkins"
|
|
||||||
git config user.email "jenkins@thedevops.dev"
|
|
||||||
git revert --no-edit HEAD || true
|
|
||||||
git push origin main || true
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
|
|
||||||
sendTelegramNotification(
|
|
||||||
status: 'ROLLBACK_SUCCESS',
|
|
||||||
message: """
|
|
||||||
✅ <b>Rollback Completed</b>
|
|
||||||
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
<b>📦 Application</b>
|
|
||||||
<b>Name:</b> ${APP_NAME}
|
|
||||||
<b>Failed Build:</b> #${BUILD_NUMBER}
|
|
||||||
<b>Rolled Back To:</b> ${previousImage}
|
|
||||||
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
<b>🔄 Rollback Status</b>
|
|
||||||
<b>Status:</b> Success ✅
|
|
||||||
<b>Method:</b> kubectl rollout undo
|
|
||||||
<b>Git:</b> Commit reverted
|
|
||||||
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
<b>📋 Recent Events</b>
|
|
||||||
<code>${errorDetails.take(500)}</code>
|
|
||||||
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
<b>⚠️ Action Required</b>
|
|
||||||
Please review logs and fix issues before redeploying
|
|
||||||
|
|
||||||
<a href="${BUILD_URL}console">View Build Logs</a>
|
|
||||||
""",
|
|
||||||
color: '✅'
|
|
||||||
)
|
|
||||||
|
|
||||||
} else {
|
|
||||||
sendTelegramNotification(
|
|
||||||
status: 'ROLLBACK_FAILED',
|
|
||||||
message: """
|
|
||||||
❌ <b>Rollback Failed</b>
|
|
||||||
|
|
||||||
<b>Application:</b> ${APP_NAME}
|
|
||||||
<b>Build:</b> #${BUILD_NUMBER}
|
|
||||||
|
|
||||||
<b>Error:</b> No previous version found
|
|
||||||
|
|
||||||
<b>⚠️ MANUAL INTERVENTION REQUIRED</b>
|
|
||||||
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
|
|
||||||
|
|
||||||
<a href="${BUILD_URL}console">View Build Logs</a>
|
|
||||||
""",
|
|
||||||
color: '🔴'
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
sendTelegramNotification(
|
|
||||||
status: 'ROLLBACK_FAILED',
|
|
||||||
message: """
|
|
||||||
❌ <b>Rollback Failed</b>
|
|
||||||
|
|
||||||
<b>Application:</b> ${APP_NAME}
|
|
||||||
<b>Build:</b> #${BUILD_NUMBER}
|
|
||||||
|
|
||||||
<b>Error:</b> ${e.message}
|
|
||||||
|
|
||||||
<b>⚠️ MANUAL ROLLBACK REQUIRED</b>
|
|
||||||
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
|
|
||||||
|
|
||||||
<a href="${BUILD_URL}console">View Build Logs</a>
|
|
||||||
""",
|
|
||||||
color: '🔴'
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
} else {
|
|
||||||
sendTelegramNotification(
|
|
||||||
status: 'FAILED',
|
|
||||||
message: """
|
|
||||||
❌ <b>Deployment Failed</b>
|
|
||||||
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
<b>📦 Application</b>
|
|
||||||
<b>Name:</b> ${APP_NAME}
|
|
||||||
<b>Build:</b> #${BUILD_NUMBER}
|
|
||||||
<b>Branch:</b> ${env.BRANCH_NAME}
|
|
||||||
<b>Image:</b> ${IMAGE_TAG}
|
|
||||||
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
<b>📋 Recent Events</b>
|
|
||||||
<code>${errorDetails.take(500)}</code>
|
|
||||||
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
<b>🔗 Links</b>
|
|
||||||
<a href="${BUILD_URL}console">View Console Output</a>
|
|
||||||
<a href="${BUILD_URL}">Build Details</a>
|
|
||||||
|
|
||||||
<i>Rollback: ${env.ROLLBACK_ENABLED == 'true' ? 'Enabled but not on main branch' : 'Disabled'}</i>
|
|
||||||
""",
|
|
||||||
color: '🔴'
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
always {
|
|
||||||
sh """
|
|
||||||
docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true
|
|
||||||
docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest || true
|
|
||||||
docker stop test-${BUILD_NUMBER} 2>/dev/null || true
|
|
||||||
docker rm test-${BUILD_NUMBER} 2>/dev/null || true
|
|
||||||
"""
|
|
||||||
cleanWs()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Telegram notification function
|
|
||||||
def sendTelegramNotification(Map args) {
|
|
||||||
def status = args.status
|
|
||||||
def message = args.message
|
|
||||||
def color = args.color ?: '🔵'
|
|
||||||
|
|
||||||
try {
|
|
||||||
// HTML formatting for Telegram
|
|
||||||
def formattedMessage = message.trim()
|
|
||||||
|
|
||||||
sh """
|
|
||||||
curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
|
|
||||||
-d chat_id="${TELEGRAM_CHAT_ID}" \
|
|
||||||
-d parse_mode="HTML" \
|
|
||||||
-d disable_web_page_preview=true \
|
|
||||||
-d text="${formattedMessage}"
|
|
||||||
"""
|
|
||||||
|
|
||||||
echo "📱 Telegram notification sent: ${status}"
|
|
||||||
} catch (Exception e) {
|
|
||||||
echo "⚠️ Failed to send Telegram notification: ${e.message}"
|
|
||||||
// Don't fail the pipeline if notification fails
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,449 +0,0 @@
|
|||||||
pipeline {
|
|
||||||
agent any
|
|
||||||
|
|
||||||
environment {
|
|
||||||
APP_NAME = 'demo-nginx'
|
|
||||||
NAMESPACE = 'demo-app'
|
|
||||||
DOCKER_REGISTRY = 'docker.io'
|
|
||||||
DOCKER_REPO = 'vladcrypto'
|
|
||||||
GITEA_URL = 'http://gitea-http.gitea.svc.cluster.local:3000'
|
|
||||||
GITEA_REPO = 'admin/k3s-gitops'
|
|
||||||
GITEA_BRANCH = 'main'
|
|
||||||
BUILD_TAG = "${env.BUILD_NUMBER}"
|
|
||||||
IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}"
|
|
||||||
|
|
||||||
// Rollback configuration
|
|
||||||
ROLLBACK_ENABLED = 'true'
|
|
||||||
DEPLOYMENT_TIMEOUT = '300s'
|
|
||||||
ARGOCD_SYNC_TIMEOUT = '120'
|
|
||||||
SKIP_HEALTH_CHECK = 'true' // Skip for now - pods work fine
|
|
||||||
}
|
|
||||||
|
|
||||||
stages {
|
|
||||||
stage('Save Current State') {
|
|
||||||
when { branch 'main' }
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "📸 Saving current deployment state for rollback..."
|
|
||||||
|
|
||||||
// Save current image tag
|
|
||||||
sh """
|
|
||||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
|
||||||
-o jsonpath='{.spec.template.spec.containers[0].image}' \
|
|
||||||
> /tmp/previous_image_${BUILD_NUMBER}.txt || echo "none" > /tmp/previous_image_${BUILD_NUMBER}.txt
|
|
||||||
|
|
||||||
PREV_IMAGE=\$(cat /tmp/previous_image_${BUILD_NUMBER}.txt)
|
|
||||||
echo "Previous image: \${PREV_IMAGE}"
|
|
||||||
"""
|
|
||||||
|
|
||||||
// Save current replica count
|
|
||||||
sh """
|
|
||||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
|
||||||
-o jsonpath='{.spec.replicas}' \
|
|
||||||
> /tmp/previous_replicas_${BUILD_NUMBER}.txt || echo "2" > /tmp/previous_replicas_${BUILD_NUMBER}.txt
|
|
||||||
"""
|
|
||||||
|
|
||||||
echo "✅ Current state saved"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Checkout Source') {
|
|
||||||
steps {
|
|
||||||
echo "Checking out application source code..."
|
|
||||||
|
|
||||||
// Create Dockerfile with actual version
|
|
||||||
sh """
|
|
||||||
cat > Dockerfile << 'EOF'
|
|
||||||
FROM nginx:1.25.3-alpine
|
|
||||||
COPY index.html /usr/share/nginx/html/index.html
|
|
||||||
COPY nginx.conf /etc/nginx/nginx.conf
|
|
||||||
EXPOSE 80
|
|
||||||
CMD ["nginx", "-g", "daemon off;"]
|
|
||||||
EOF
|
|
||||||
"""
|
|
||||||
|
|
||||||
// Create index.html with actual version
|
|
||||||
sh """
|
|
||||||
cat > index.html << EOF
|
|
||||||
<!DOCTYPE html>
|
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<title>Demo Nginx</title>
|
|
||||||
<style>
|
|
||||||
body {
|
|
||||||
font-family: Arial, sans-serif;
|
|
||||||
max-width: 800px;
|
|
||||||
margin: 50px auto;
|
|
||||||
padding: 20px;
|
|
||||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
|
||||||
color: white;
|
|
||||||
}
|
|
||||||
.container {
|
|
||||||
background: rgba(255, 255, 255, 0.1);
|
|
||||||
padding: 40px;
|
|
||||||
border-radius: 10px;
|
|
||||||
backdrop-filter: blur(10px);
|
|
||||||
}
|
|
||||||
h1 {
|
|
||||||
font-size: 48px;
|
|
||||||
margin-bottom: 20px;
|
|
||||||
}
|
|
||||||
p {
|
|
||||||
font-size: 24px;
|
|
||||||
margin: 10px 0;
|
|
||||||
}
|
|
||||||
.version {
|
|
||||||
font-family: 'Courier New', monospace;
|
|
||||||
background: rgba(0, 0, 0, 0.3);
|
|
||||||
padding: 10px 20px;
|
|
||||||
border-radius: 5px;
|
|
||||||
display: inline-block;
|
|
||||||
margin-top: 20px;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<div class="container">
|
|
||||||
<h1>🚀 Demo Nginx - Build #${BUILD_NUMBER}</h1>
|
|
||||||
<p>Environment: Production</p>
|
|
||||||
<p class="version">Version: ${IMAGE_TAG}</p>
|
|
||||||
<p style="font-size: 16px; margin-top: 30px; opacity: 0.8;">
|
|
||||||
Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
EOF
|
|
||||||
"""
|
|
||||||
|
|
||||||
sh '''
|
|
||||||
cat > nginx.conf << 'EOF'
|
|
||||||
user nginx;
|
|
||||||
worker_processes auto;
|
|
||||||
error_log /var/log/nginx/error.log warn;
|
|
||||||
pid /var/run/nginx.pid;
|
|
||||||
events { worker_connections 1024; }
|
|
||||||
http {
|
|
||||||
include /etc/nginx/mime.types;
|
|
||||||
default_type application/octet-stream;
|
|
||||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
|
||||||
'$status $body_bytes_sent "$http_referer" '
|
|
||||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
|
||||||
access_log /var/log/nginx/access.log main;
|
|
||||||
sendfile on;
|
|
||||||
keepalive_timeout 65;
|
|
||||||
server {
|
|
||||||
listen 80;
|
|
||||||
server_name _;
|
|
||||||
location / {
|
|
||||||
root /usr/share/nginx/html;
|
|
||||||
index index.html;
|
|
||||||
}
|
|
||||||
location /health {
|
|
||||||
access_log off;
|
|
||||||
return 200 "healthy\n";
|
|
||||||
add_header Content-Type text/plain;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
'''
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Build Docker Image') {
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "Building Docker image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}"
|
|
||||||
sh """
|
|
||||||
docker build \
|
|
||||||
-t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} \
|
|
||||||
-t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest \
|
|
||||||
.
|
|
||||||
"""
|
|
||||||
echo "✅ Image built successfully!"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Push to Registry') {
|
|
||||||
when { branch 'main' }
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "Pushing image to registry..."
|
|
||||||
withCredentials([usernamePassword(
|
|
||||||
credentialsId: 'docker-registry-credentials',
|
|
||||||
usernameVariable: 'DOCKER_USER',
|
|
||||||
passwordVariable: 'DOCKER_PASS'
|
|
||||||
)]) {
|
|
||||||
sh """
|
|
||||||
echo "\${DOCKER_PASS}" | docker login ${DOCKER_REGISTRY} -u "\${DOCKER_USER}" --password-stdin
|
|
||||||
docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
|
||||||
docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest
|
|
||||||
docker logout ${DOCKER_REGISTRY}
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
echo "✅ Image pushed successfully!"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Update GitOps Manifests') {
|
|
||||||
when { branch 'main' }
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "Updating Kubernetes manifests..."
|
|
||||||
withCredentials([usernamePassword(
|
|
||||||
credentialsId: 'gitea-credentials',
|
|
||||||
usernameVariable: 'GIT_USER',
|
|
||||||
passwordVariable: 'GIT_PASS'
|
|
||||||
)]) {
|
|
||||||
sh """
|
|
||||||
rm -rf k3s-gitops || true
|
|
||||||
git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git
|
|
||||||
cd k3s-gitops
|
|
||||||
git config user.name "Jenkins"
|
|
||||||
git config user.email "jenkins@thedevops.dev"
|
|
||||||
|
|
||||||
# Save current commit for rollback
|
|
||||||
git rev-parse HEAD > /tmp/previous_commit_${BUILD_NUMBER}.txt
|
|
||||||
|
|
||||||
sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml
|
|
||||||
git add apps/demo-nginx/deployment.yaml
|
|
||||||
git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes"
|
|
||||||
git push origin main
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
echo "✅ Manifests updated!"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Wait for ArgoCD Sync') {
|
|
||||||
when { branch 'main' }
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "⏳ Waiting for ArgoCD to sync..."
|
|
||||||
|
|
||||||
def syncSuccess = false
|
|
||||||
def attempts = 0
|
|
||||||
def maxAttempts = Integer.parseInt(env.ARGOCD_SYNC_TIMEOUT) / 10
|
|
||||||
|
|
||||||
while (!syncSuccess && attempts < maxAttempts) {
|
|
||||||
attempts++
|
|
||||||
echo "ArgoCD sync check attempt ${attempts}/${maxAttempts}..."
|
|
||||||
|
|
||||||
def syncStatus = sh(
|
|
||||||
script: """
|
|
||||||
kubectl get application ${APP_NAME} -n argocd \
|
|
||||||
-o jsonpath='{.status.sync.status}'
|
|
||||||
""",
|
|
||||||
returnStdout: true
|
|
||||||
).trim()
|
|
||||||
|
|
||||||
def currentImage = sh(
|
|
||||||
script: """
|
|
||||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
|
||||||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
|
||||||
""",
|
|
||||||
returnStdout: true
|
|
||||||
).trim()
|
|
||||||
|
|
||||||
echo "ArgoCD sync status: ${syncStatus}"
|
|
||||||
echo "Current deployment image: ${currentImage}"
|
|
||||||
echo "Expected image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}"
|
|
||||||
|
|
||||||
if (syncStatus == 'Synced' && currentImage.contains(env.IMAGE_TAG)) {
|
|
||||||
syncSuccess = true
|
|
||||||
echo "✅ ArgoCD synced successfully!"
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
if (attempts < maxAttempts) {
|
|
||||||
echo "Waiting 10 seconds before next check..."
|
|
||||||
sleep 10
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!syncSuccess) {
|
|
||||||
error "❌ ArgoCD sync timeout after ${env.ARGOCD_SYNC_TIMEOUT} seconds!"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Wait for Deployment') {
|
|
||||||
when { branch 'main' }
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "⏳ Waiting for deployment to complete..."
|
|
||||||
|
|
||||||
try {
|
|
||||||
sh """
|
|
||||||
# Check rollout status
|
|
||||||
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=${DEPLOYMENT_TIMEOUT}
|
|
||||||
"""
|
|
||||||
echo "✅ Deployment rolled out successfully!"
|
|
||||||
} catch (Exception e) {
|
|
||||||
echo "❌ Deployment failed: ${e.message}"
|
|
||||||
throw e
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Verify Deployment') {
|
|
||||||
when { branch 'main' }
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "✅ Verifying deployment..."
|
|
||||||
|
|
||||||
try {
|
|
||||||
sh """#!/bin/bash
|
|
||||||
set -e
|
|
||||||
|
|
||||||
# Check pod status
|
|
||||||
READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}')
|
|
||||||
DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}')
|
|
||||||
|
|
||||||
echo "Ready pods: \${READY_PODS}/\${DESIRED_PODS}"
|
|
||||||
|
|
||||||
if [ "\${READY_PODS}" != "\${DESIRED_PODS}" ]; then
|
|
||||||
echo "❌ Not all pods are ready!"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Verify image version
|
|
||||||
DEPLOYED_IMAGE=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}')
|
|
||||||
echo "Deployed image: \${DEPLOYED_IMAGE}"
|
|
||||||
|
|
||||||
if [[ "\${DEPLOYED_IMAGE}" != *"${IMAGE_TAG}"* ]]; then
|
|
||||||
echo "❌ Image version mismatch!"
|
|
||||||
echo "Expected: ${IMAGE_TAG}"
|
|
||||||
echo "Got: \${DEPLOYED_IMAGE}"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "✅ All verification checks passed!"
|
|
||||||
"""
|
|
||||||
|
|
||||||
echo "✅ Deployment verified successfully!"
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
echo "❌ Deployment verification failed: ${e.message}"
|
|
||||||
throw e
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
post {
|
|
||||||
success {
|
|
||||||
script {
|
|
||||||
echo """
|
|
||||||
✅ DEPLOYMENT SUCCESS!
|
|
||||||
|
|
||||||
Application: ${APP_NAME}
|
|
||||||
Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
|
||||||
Namespace: ${NAMESPACE}
|
|
||||||
Build: #${BUILD_NUMBER}
|
|
||||||
|
|
||||||
All checks passed! ✨
|
|
||||||
"""
|
|
||||||
|
|
||||||
// Cleanup rollback files
|
|
||||||
sh """
|
|
||||||
rm -f /tmp/previous_image_${BUILD_NUMBER}.txt
|
|
||||||
rm -f /tmp/previous_replicas_${BUILD_NUMBER}.txt
|
|
||||||
rm -f /tmp/previous_commit_${BUILD_NUMBER}.txt
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
failure {
|
|
||||||
script {
|
|
||||||
if (env.BRANCH_NAME == 'main' && env.ROLLBACK_ENABLED == 'true') {
|
|
||||||
echo """
|
|
||||||
❌ DEPLOYMENT FAILED - INITIATING ROLLBACK!
|
|
||||||
|
|
||||||
Rolling back to previous version...
|
|
||||||
"""
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Get previous state
|
|
||||||
def previousImage = sh(
|
|
||||||
script: "cat /tmp/previous_image_${BUILD_NUMBER}.txt 2>/dev/null || echo 'none'",
|
|
||||||
returnStdout: true
|
|
||||||
).trim()
|
|
||||||
|
|
||||||
if (previousImage != 'none' && previousImage != '') {
|
|
||||||
echo "🔄 Rolling back to: ${previousImage}"
|
|
||||||
|
|
||||||
// Rollback via kubectl
|
|
||||||
sh """
|
|
||||||
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
|
|
||||||
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=180s
|
|
||||||
"""
|
|
||||||
|
|
||||||
// Revert Git commit
|
|
||||||
withCredentials([usernamePassword(
|
|
||||||
credentialsId: 'gitea-credentials',
|
|
||||||
usernameVariable: 'GIT_USER',
|
|
||||||
passwordVariable: 'GIT_PASS'
|
|
||||||
)]) {
|
|
||||||
sh """
|
|
||||||
cd k3s-gitops || git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git
|
|
||||||
cd k3s-gitops
|
|
||||||
git config user.name "Jenkins"
|
|
||||||
git config user.email "jenkins@thedevops.dev"
|
|
||||||
|
|
||||||
# Revert last commit
|
|
||||||
git revert --no-edit HEAD || true
|
|
||||||
git push origin main || true
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
|
|
||||||
echo """
|
|
||||||
✅ ROLLBACK COMPLETED!
|
|
||||||
|
|
||||||
Rolled back to: ${previousImage}
|
|
||||||
Current build (#${BUILD_NUMBER}) has been reverted.
|
|
||||||
|
|
||||||
Please check logs and fix the issue before redeploying.
|
|
||||||
"""
|
|
||||||
|
|
||||||
} else {
|
|
||||||
echo "⚠️ No previous version found - cannot rollback automatically"
|
|
||||||
echo "Manual intervention required!"
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
echo """
|
|
||||||
❌ ROLLBACK FAILED!
|
|
||||||
|
|
||||||
Error: ${e.message}
|
|
||||||
|
|
||||||
MANUAL ROLLBACK REQUIRED:
|
|
||||||
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
|
|
||||||
} else {
|
|
||||||
echo "❌ Pipeline failed! (Rollback disabled or not main branch)"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
always {
|
|
||||||
sh """
|
|
||||||
docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true
|
|
||||||
docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest || true
|
|
||||||
docker stop test-${BUILD_NUMBER} 2>/dev/null || true
|
|
||||||
docker rm test-${BUILD_NUMBER} 2>/dev/null || true
|
|
||||||
"""
|
|
||||||
cleanWs()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,564 +0,0 @@
|
|||||||
// Jenkinsfile for Manual Rollback
|
|
||||||
// This allows rolling back to any previous version
|
|
||||||
// Now rebuilds image with updated design!
|
|
||||||
|
|
||||||
pipeline {
|
|
||||||
agent any
|
|
||||||
|
|
||||||
parameters {
|
|
||||||
choice(
|
|
||||||
name: 'ROLLBACK_METHOD',
|
|
||||||
choices: ['IMAGE_TAG', 'REVISION_NUMBER', 'GIT_COMMIT'],
|
|
||||||
description: 'Select rollback method'
|
|
||||||
)
|
|
||||||
string(
|
|
||||||
name: 'TARGET_VERSION',
|
|
||||||
defaultValue: '',
|
|
||||||
description: '''
|
|
||||||
Enter target version based on method:
|
|
||||||
- IMAGE_TAG: main-21, main-20, etc.
|
|
||||||
- REVISION_NUMBER: 1, 2, 3 (from rollout history)
|
|
||||||
- GIT_COMMIT: abc123def (git commit SHA)
|
|
||||||
'''
|
|
||||||
)
|
|
||||||
booleanParam(
|
|
||||||
name: 'SKIP_HEALTH_CHECK',
|
|
||||||
defaultValue: true,
|
|
||||||
description: 'Skip health checks (recommended for rollback)'
|
|
||||||
)
|
|
||||||
booleanParam(
|
|
||||||
name: 'DRY_RUN',
|
|
||||||
defaultValue: false,
|
|
||||||
description: 'Dry run - show what would happen without applying'
|
|
||||||
)
|
|
||||||
booleanParam(
|
|
||||||
name: 'REBUILD_IMAGE',
|
|
||||||
defaultValue: true,
|
|
||||||
description: 'Rebuild image with updated design (recommended)'
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
environment {
|
|
||||||
APP_NAME = 'demo-nginx'
|
|
||||||
CONTAINER_NAME = 'nginx'
|
|
||||||
NAMESPACE = 'demo-app'
|
|
||||||
DOCKER_REGISTRY = 'docker.io'
|
|
||||||
DOCKER_REPO = 'vladcrypto'
|
|
||||||
GITEA_URL = 'http://gitea-http.gitea.svc.cluster.local:3000'
|
|
||||||
HEALTH_CHECK_TIMEOUT = '300s'
|
|
||||||
ARGOCD_SYNC_TIMEOUT = '120'
|
|
||||||
}
|
|
||||||
|
|
||||||
stages {
|
|
||||||
stage('Validate Input') {
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "🔍 Validating rollback request..."
|
|
||||||
|
|
||||||
// Trim whitespace from input
|
|
||||||
env.TARGET_VERSION_CLEAN = params.TARGET_VERSION.trim()
|
|
||||||
|
|
||||||
if (env.TARGET_VERSION_CLEAN == '') {
|
|
||||||
error("❌ TARGET_VERSION cannot be empty!")
|
|
||||||
}
|
|
||||||
|
|
||||||
echo """
|
|
||||||
📋 Rollback Configuration:
|
|
||||||
Method: ${params.ROLLBACK_METHOD}
|
|
||||||
Target: ${env.TARGET_VERSION_CLEAN}
|
|
||||||
Rebuild Image: ${params.REBUILD_IMAGE}
|
|
||||||
Skip Health Check: ${params.SKIP_HEALTH_CHECK}
|
|
||||||
Dry Run: ${params.DRY_RUN}
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Show Current State') {
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "📸 Current Deployment State:"
|
|
||||||
|
|
||||||
sh """
|
|
||||||
echo "=== Current Deployment ==="
|
|
||||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE}
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "=== Current Image ==="
|
|
||||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
|
||||||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "=== Current Pods ==="
|
|
||||||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME}
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "=== Rollout History ==="
|
|
||||||
kubectl rollout history deployment/${APP_NAME} -n ${NAMESPACE}
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Prepare Rollback') {
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "🔄 Preparing rollback to: ${env.TARGET_VERSION_CLEAN}"
|
|
||||||
|
|
||||||
if (params.ROLLBACK_METHOD == 'IMAGE_TAG') {
|
|
||||||
env.TARGET_IMAGE = "${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${env.TARGET_VERSION_CLEAN}"
|
|
||||||
|
|
||||||
// Extract build number from tag
|
|
||||||
env.BUILD_NUMBER_FROM_TAG = env.TARGET_VERSION_CLEAN.split('-')[1]
|
|
||||||
|
|
||||||
sh """
|
|
||||||
echo "Target image: ${env.TARGET_IMAGE}"
|
|
||||||
echo "Build number: ${env.BUILD_NUMBER_FROM_TAG}"
|
|
||||||
"""
|
|
||||||
|
|
||||||
} else if (params.ROLLBACK_METHOD == 'REVISION_NUMBER') {
|
|
||||||
env.REVISION = env.TARGET_VERSION_CLEAN
|
|
||||||
|
|
||||||
sh """
|
|
||||||
echo "Rolling back to revision: ${env.REVISION}"
|
|
||||||
kubectl rollout history deployment/${APP_NAME} -n ${NAMESPACE} \
|
|
||||||
--revision=${env.REVISION}
|
|
||||||
"""
|
|
||||||
|
|
||||||
} else if (params.ROLLBACK_METHOD == 'GIT_COMMIT') {
|
|
||||||
env.GIT_SHA = env.TARGET_VERSION_CLEAN
|
|
||||||
echo "Rolling back to git commit: ${env.GIT_SHA}"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Rebuild Image') {
|
|
||||||
when {
|
|
||||||
expression { !params.DRY_RUN && params.REBUILD_IMAGE && params.ROLLBACK_METHOD == 'IMAGE_TAG' }
|
|
||||||
}
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "🔨 Rebuilding image with updated design..."
|
|
||||||
|
|
||||||
// Create Dockerfile
|
|
||||||
writeFile file: 'Dockerfile', text: '''FROM nginx:1.25.3-alpine
|
|
||||||
COPY index.html /usr/share/nginx/html/index.html
|
|
||||||
COPY nginx.conf /etc/nginx/nginx.conf
|
|
||||||
EXPOSE 80
|
|
||||||
CMD ["nginx", "-g", "daemon off;"]
|
|
||||||
'''
|
|
||||||
|
|
||||||
// Create HTML with proper variable substitution
|
|
||||||
def htmlContent = """<!DOCTYPE html>
|
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<title>Demo Nginx - Rollback</title>
|
|
||||||
<style>
|
|
||||||
body {
|
|
||||||
font-family: Arial, sans-serif;
|
|
||||||
max-width: 800px;
|
|
||||||
margin: 50px auto;
|
|
||||||
padding: 20px;
|
|
||||||
background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);
|
|
||||||
color: white;
|
|
||||||
}
|
|
||||||
.container {
|
|
||||||
background: rgba(255, 255, 255, 0.1);
|
|
||||||
padding: 40px;
|
|
||||||
border-radius: 10px;
|
|
||||||
backdrop-filter: blur(10px);
|
|
||||||
}
|
|
||||||
h1 {
|
|
||||||
font-size: 48px;
|
|
||||||
margin-bottom: 20px;
|
|
||||||
}
|
|
||||||
.rollback-badge {
|
|
||||||
background: rgba(255, 255, 255, 0.2);
|
|
||||||
padding: 10px 20px;
|
|
||||||
border-radius: 25px;
|
|
||||||
display: inline-block;
|
|
||||||
margin-bottom: 20px;
|
|
||||||
font-size: 18px;
|
|
||||||
}
|
|
||||||
p {
|
|
||||||
font-size: 24px;
|
|
||||||
margin: 10px 0;
|
|
||||||
}
|
|
||||||
.version {
|
|
||||||
font-family: 'Courier New', monospace;
|
|
||||||
background: rgba(0, 0, 0, 0.3);
|
|
||||||
padding: 10px 20px;
|
|
||||||
border-radius: 5px;
|
|
||||||
display: inline-block;
|
|
||||||
margin-top: 20px;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<div class="container">
|
|
||||||
<div class="rollback-badge">🔄 ROLLED BACK</div>
|
|
||||||
<h1>🚀 Demo Nginx - Build #${env.BUILD_NUMBER_FROM_TAG}</h1>
|
|
||||||
<p>Environment: Production</p>
|
|
||||||
<p class="version">Version: ${env.TARGET_VERSION_CLEAN}</p>
|
|
||||||
<p style="font-size: 16px; margin-top: 30px; opacity: 0.8;">
|
|
||||||
Image: ${env.TARGET_IMAGE}
|
|
||||||
</p>
|
|
||||||
<p style="font-size: 14px; margin-top: 10px; opacity: 0.7;">
|
|
||||||
⏮️ Restored from previous deployment
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
"""
|
|
||||||
|
|
||||||
writeFile file: 'index.html', text: htmlContent
|
|
||||||
|
|
||||||
// Create nginx.conf
|
|
||||||
writeFile file: 'nginx.conf', text: '''user nginx;
|
|
||||||
worker_processes auto;
|
|
||||||
error_log /var/log/nginx/error.log warn;
|
|
||||||
pid /var/run/nginx.pid;
|
|
||||||
events { worker_connections 1024; }
|
|
||||||
http {
|
|
||||||
include /etc/nginx/mime.types;
|
|
||||||
default_type application/octet-stream;
|
|
||||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
|
||||||
'$status $body_bytes_sent "$http_referer" '
|
|
||||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
|
||||||
access_log /var/log/nginx/access.log main;
|
|
||||||
sendfile on;
|
|
||||||
keepalive_timeout 65;
|
|
||||||
server {
|
|
||||||
listen 80;
|
|
||||||
server_name _;
|
|
||||||
location / {
|
|
||||||
root /usr/share/nginx/html;
|
|
||||||
index index.html;
|
|
||||||
}
|
|
||||||
location /health {
|
|
||||||
access_log off;
|
|
||||||
return 200 "healthy\n";
|
|
||||||
add_header Content-Type text/plain;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
'''
|
|
||||||
|
|
||||||
// Build new image
|
|
||||||
sh """
|
|
||||||
docker build -t ${env.TARGET_IMAGE} .
|
|
||||||
"""
|
|
||||||
|
|
||||||
echo "✅ Image rebuilt successfully!"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Push Rebuilt Image') {
|
|
||||||
when {
|
|
||||||
expression { !params.DRY_RUN && params.REBUILD_IMAGE && params.ROLLBACK_METHOD == 'IMAGE_TAG' }
|
|
||||||
}
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "📤 Pushing rebuilt image..."
|
|
||||||
|
|
||||||
withCredentials([usernamePassword(
|
|
||||||
credentialsId: 'docker-registry-credentials',
|
|
||||||
usernameVariable: 'DOCKER_USER',
|
|
||||||
passwordVariable: 'DOCKER_PASS'
|
|
||||||
)]) {
|
|
||||||
sh """
|
|
||||||
echo "\${DOCKER_PASS}" | docker login ${DOCKER_REGISTRY} -u "\${DOCKER_USER}" --password-stdin
|
|
||||||
docker push ${env.TARGET_IMAGE}
|
|
||||||
docker logout ${DOCKER_REGISTRY}
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
|
|
||||||
echo "✅ Image pushed successfully!"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Execute Rollback') {
|
|
||||||
when {
|
|
||||||
expression { !params.DRY_RUN }
|
|
||||||
}
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "🚀 Executing rollback..."
|
|
||||||
|
|
||||||
if (params.ROLLBACK_METHOD == 'IMAGE_TAG') {
|
|
||||||
// Update deployment
|
|
||||||
sh """
|
|
||||||
echo "Setting image to: ${env.TARGET_IMAGE}"
|
|
||||||
kubectl set image deployment/${APP_NAME} \
|
|
||||||
${CONTAINER_NAME}=${env.TARGET_IMAGE} \
|
|
||||||
-n ${NAMESPACE} \
|
|
||||||
--record
|
|
||||||
"""
|
|
||||||
|
|
||||||
// Update Git manifests
|
|
||||||
withCredentials([usernamePassword(
|
|
||||||
credentialsId: 'gitea-credentials',
|
|
||||||
usernameVariable: 'GIT_USER',
|
|
||||||
passwordVariable: 'GIT_PASS'
|
|
||||||
)]) {
|
|
||||||
sh """
|
|
||||||
rm -rf k3s-gitops || true
|
|
||||||
git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git
|
|
||||||
cd k3s-gitops
|
|
||||||
git config user.name "Jenkins"
|
|
||||||
git config user.email "jenkins@thedevops.dev"
|
|
||||||
|
|
||||||
sed -i 's|image: .*|image: ${env.TARGET_IMAGE}|' apps/demo-nginx/deployment.yaml
|
|
||||||
git add apps/demo-nginx/deployment.yaml
|
|
||||||
git commit -m "rollback(demo-nginx): Manual rollback to ${env.TARGET_VERSION_CLEAN}" || echo "No changes"
|
|
||||||
git push origin main
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
|
|
||||||
} else if (params.ROLLBACK_METHOD == 'REVISION_NUMBER') {
|
|
||||||
sh """
|
|
||||||
kubectl rollout undo deployment/${APP_NAME} \
|
|
||||||
-n ${NAMESPACE} \
|
|
||||||
--to-revision=${env.REVISION}
|
|
||||||
"""
|
|
||||||
|
|
||||||
} else if (params.ROLLBACK_METHOD == 'GIT_COMMIT') {
|
|
||||||
withCredentials([usernamePassword(
|
|
||||||
credentialsId: 'gitea-credentials',
|
|
||||||
usernameVariable: 'GIT_USER',
|
|
||||||
passwordVariable: 'GIT_PASS'
|
|
||||||
)]) {
|
|
||||||
sh """
|
|
||||||
rm -rf k3s-gitops || true
|
|
||||||
git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git
|
|
||||||
cd k3s-gitops
|
|
||||||
git config user.name "Jenkins"
|
|
||||||
git config user.email "jenkins@thedevops.dev"
|
|
||||||
|
|
||||||
git checkout ${env.GIT_SHA} -- apps/demo-nginx/deployment.yaml
|
|
||||||
TARGET_IMAGE=\$(grep 'image:' apps/demo-nginx/deployment.yaml | awk '{print \$2}')
|
|
||||||
|
|
||||||
git checkout main
|
|
||||||
git checkout ${env.GIT_SHA} -- apps/demo-nginx/deployment.yaml
|
|
||||||
git add apps/demo-nginx/deployment.yaml
|
|
||||||
git commit -m "rollback(demo-nginx): Rollback to commit ${env.GIT_SHA}" || echo "No changes"
|
|
||||||
git push origin main
|
|
||||||
|
|
||||||
echo "Rolled back to image: \${TARGET_IMAGE}"
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
echo "✅ Rollback command executed"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Wait for ArgoCD Sync') {
|
|
||||||
when {
|
|
||||||
expression { !params.DRY_RUN }
|
|
||||||
}
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "⏳ Waiting for ArgoCD to sync..."
|
|
||||||
|
|
||||||
def syncSuccess = false
|
|
||||||
def attempts = 0
|
|
||||||
def maxAttempts = Integer.parseInt(env.ARGOCD_SYNC_TIMEOUT) / 10
|
|
||||||
|
|
||||||
while (!syncSuccess && attempts < maxAttempts) {
|
|
||||||
attempts++
|
|
||||||
echo "ArgoCD sync check attempt ${attempts}/${maxAttempts}..."
|
|
||||||
|
|
||||||
def syncStatus = sh(
|
|
||||||
script: """
|
|
||||||
kubectl get application ${APP_NAME} -n argocd \
|
|
||||||
-o jsonpath='{.status.sync.status}'
|
|
||||||
""",
|
|
||||||
returnStdout: true
|
|
||||||
).trim()
|
|
||||||
|
|
||||||
def currentImage = sh(
|
|
||||||
script: """
|
|
||||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
|
||||||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
|
||||||
""",
|
|
||||||
returnStdout: true
|
|
||||||
).trim()
|
|
||||||
|
|
||||||
echo "ArgoCD sync status: ${syncStatus}"
|
|
||||||
echo "Current deployment image: ${currentImage}"
|
|
||||||
echo "Expected image: ${env.TARGET_IMAGE}"
|
|
||||||
|
|
||||||
if (syncStatus == 'Synced' && currentImage.contains(env.TARGET_VERSION_CLEAN)) {
|
|
||||||
syncSuccess = true
|
|
||||||
echo "✅ ArgoCD synced successfully!"
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
if (attempts < maxAttempts) {
|
|
||||||
echo "Waiting 10 seconds before next check..."
|
|
||||||
sleep 10
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!syncSuccess) {
|
|
||||||
echo "⚠️ ArgoCD sync check timeout (non-critical for rollback)"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Wait for Rollout') {
|
|
||||||
when {
|
|
||||||
expression { !params.DRY_RUN }
|
|
||||||
}
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "⏳ Waiting for rollout to complete..."
|
|
||||||
|
|
||||||
sh """
|
|
||||||
kubectl rollout status deployment/${APP_NAME} \
|
|
||||||
-n ${NAMESPACE} \
|
|
||||||
--timeout=${HEALTH_CHECK_TIMEOUT}
|
|
||||||
"""
|
|
||||||
|
|
||||||
echo "✅ Rollout completed"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Verify Deployment') {
|
|
||||||
when {
|
|
||||||
expression { !params.DRY_RUN }
|
|
||||||
}
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "✅ Verifying deployment..."
|
|
||||||
|
|
||||||
sh """#!/bin/bash
|
|
||||||
set -e
|
|
||||||
|
|
||||||
# Check pods ready
|
|
||||||
READY_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.status.readyReplicas}')
|
|
||||||
DESIRED_PODS=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.replicas}')
|
|
||||||
|
|
||||||
echo "Ready pods: \${READY_PODS}/\${DESIRED_PODS}"
|
|
||||||
|
|
||||||
if [ "\${READY_PODS}" != "\${DESIRED_PODS}" ]; then
|
|
||||||
echo "❌ Not all pods are ready!"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Verify image
|
|
||||||
DEPLOYED_IMAGE=\$(kubectl get deployment ${APP_NAME} -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].image}')
|
|
||||||
echo "Deployed image: \${DEPLOYED_IMAGE}"
|
|
||||||
|
|
||||||
echo "✅ Deployment verified!"
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Show New State') {
|
|
||||||
when {
|
|
||||||
expression { !params.DRY_RUN }
|
|
||||||
}
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "📊 New Deployment State:"
|
|
||||||
|
|
||||||
sh """
|
|
||||||
echo "=== New Deployment ==="
|
|
||||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE}
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "=== New Image ==="
|
|
||||||
kubectl get deployment ${APP_NAME} -n ${NAMESPACE} \
|
|
||||||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "=== New Pods ==="
|
|
||||||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME}
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Dry Run Summary') {
|
|
||||||
when {
|
|
||||||
expression { params.DRY_RUN }
|
|
||||||
}
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo """
|
|
||||||
🔍 DRY RUN SUMMARY
|
|
||||||
|
|
||||||
This is what would happen:
|
|
||||||
|
|
||||||
Method: ${params.ROLLBACK_METHOD}
|
|
||||||
Target: ${env.TARGET_VERSION_CLEAN}
|
|
||||||
Rebuild Image: ${params.REBUILD_IMAGE}
|
|
||||||
|
|
||||||
Steps:
|
|
||||||
1. ${params.REBUILD_IMAGE ? 'Rebuild image with rollback design' : 'Use existing image'}
|
|
||||||
2. Update deployment
|
|
||||||
3. Update Git manifests
|
|
||||||
4. Wait for ArgoCD sync
|
|
||||||
5. Wait for rollout
|
|
||||||
6. Verify deployment
|
|
||||||
|
|
||||||
No actual changes were made.
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
post {
|
|
||||||
success {
|
|
||||||
script {
|
|
||||||
if (params.DRY_RUN) {
|
|
||||||
echo "✅ DRY RUN COMPLETED - No changes made"
|
|
||||||
} else {
|
|
||||||
echo """
|
|
||||||
✅ ROLLBACK SUCCESSFUL!
|
|
||||||
|
|
||||||
Application: ${APP_NAME}
|
|
||||||
Method: ${params.ROLLBACK_METHOD}
|
|
||||||
Target Version: ${env.TARGET_VERSION_CLEAN}
|
|
||||||
${params.REBUILD_IMAGE ? 'Image: Rebuilt with updated design ✨' : 'Image: Using existing'}
|
|
||||||
Namespace: ${NAMESPACE}
|
|
||||||
|
|
||||||
The application has been rolled back successfully! 🔄
|
|
||||||
|
|
||||||
Check: https://demo-nginx.thedevops.dev
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
failure {
|
|
||||||
echo """
|
|
||||||
❌ ROLLBACK FAILED!
|
|
||||||
|
|
||||||
Please check the logs and try again.
|
|
||||||
|
|
||||||
Manual rollback:
|
|
||||||
kubectl rollout undo deployment/${APP_NAME} -n ${NAMESPACE}
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
|
|
||||||
always {
|
|
||||||
sh """
|
|
||||||
docker rmi ${env.TARGET_IMAGE} 2>/dev/null || true
|
|
||||||
"""
|
|
||||||
cleanWs()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
apiVersion: argoproj.io/v1alpha1
|
|
||||||
kind: Application
|
|
||||||
metadata:
|
|
||||||
name: demo-nginx
|
|
||||||
namespace: argocd
|
|
||||||
labels:
|
|
||||||
app: demo-nginx
|
|
||||||
spec:
|
|
||||||
project: default
|
|
||||||
source:
|
|
||||||
repoURL: http://gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops
|
|
||||||
targetRevision: HEAD
|
|
||||||
path: apps/demo-nginx
|
|
||||||
destination:
|
|
||||||
server: https://kubernetes.default.svc
|
|
||||||
namespace: demo-app
|
|
||||||
syncPolicy:
|
|
||||||
automated:
|
|
||||||
prune: true
|
|
||||||
selfHeal: true
|
|
||||||
syncOptions:
|
|
||||||
- CreateNamespace=true
|
|
||||||
@@ -1,62 +0,0 @@
|
|||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: demo-nginx
|
|
||||||
namespace: demo-app
|
|
||||||
labels:
|
|
||||||
app: demo-nginx
|
|
||||||
spec:
|
|
||||||
replicas: 5
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
app: demo-nginx
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app: demo-nginx
|
|
||||||
annotations:
|
|
||||||
prometheus.io/scrape: "true"
|
|
||||||
prometheus.io/port: "80"
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: nginx
|
|
||||||
image: docker.io/vladcrypto/demo-nginx:main-50
|
|
||||||
ports:
|
|
||||||
- containerPort: 80
|
|
||||||
name: http
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 100m
|
|
||||||
memory: 128Mi
|
|
||||||
limits:
|
|
||||||
cpu: 200m
|
|
||||||
memory: 256Mi
|
|
||||||
livenessProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /
|
|
||||||
port: 80
|
|
||||||
initialDelaySeconds: 10
|
|
||||||
periodSeconds: 10
|
|
||||||
readinessProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /
|
|
||||||
port: 80
|
|
||||||
initialDelaySeconds: 5
|
|
||||||
periodSeconds: 5
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: demo-nginx
|
|
||||||
namespace: demo-app
|
|
||||||
labels:
|
|
||||||
app: demo-nginx
|
|
||||||
spec:
|
|
||||||
type: ClusterIP
|
|
||||||
selector:
|
|
||||||
app: demo-nginx
|
|
||||||
ports:
|
|
||||||
- port: 80
|
|
||||||
targetPort: 80
|
|
||||||
protocol: TCP
|
|
||||||
name: http
|
|
||||||
@@ -1,419 +0,0 @@
|
|||||||
# 🔧 ArgoCD Sync vs Actual Deployment Issue
|
|
||||||
|
|
||||||
## 🐛 The Problem
|
|
||||||
|
|
||||||
**Symptom:**
|
|
||||||
- ArgoCD shows `Synced` ✅
|
|
||||||
- Deployment manifest in Kubernetes is updated ✅
|
|
||||||
- **BUT** pods are still running old image ❌
|
|
||||||
|
|
||||||
**Why This Happens:**
|
|
||||||
|
|
||||||
```
|
|
||||||
┌─────────────────────────────────────────────────────────────┐
|
|
||||||
│ Git Repository │
|
|
||||||
│ deployment.yaml: image: app:v2 ✅ │
|
|
||||||
└──────────────────┬──────────────────────────────────────────┘
|
|
||||||
│
|
|
||||||
│ ArgoCD syncs
|
|
||||||
▼
|
|
||||||
┌─────────────────────────────────────────────────────────────┐
|
|
||||||
│ Kubernetes API (Deployment Object) │
|
|
||||||
│ spec.template.image: app:v2 ✅ │
|
|
||||||
└──────────────────┬──────────────────────────────────────────┘
|
|
||||||
│
|
|
||||||
│ Kubernetes Controller should trigger rollout
|
|
||||||
▼
|
|
||||||
┌─────────────────────────────────────────────────────────────┐
|
|
||||||
│ Running Pods │
|
|
||||||
│ Pod-1: image: app:v1 ❌ (OLD!) │
|
|
||||||
│ Pod-2: image: app:v1 ❌ (OLD!) │
|
|
||||||
└─────────────────────────────────────────────────────────────┘
|
|
||||||
|
|
||||||
ArgoCD says "Synced" because Git == Kubernetes manifest ✅
|
|
||||||
But pods haven't rolled out yet! ❌
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔍 Why ArgoCD Says "Synced"
|
|
||||||
|
|
||||||
ArgoCD checks:
|
|
||||||
1. ✅ Git manifest == Kubernetes Deployment object
|
|
||||||
2. ✅ Health status (from status fields)
|
|
||||||
|
|
||||||
ArgoCD **DOES NOT** check:
|
|
||||||
- ❌ Are pods actually running?
|
|
||||||
- ❌ What image are pods using?
|
|
||||||
- ❌ Did rollout complete?
|
|
||||||
|
|
||||||
**ArgoCD's job:** Keep Kubernetes resources in sync with Git
|
|
||||||
**NOT ArgoCD's job:** Wait for pods to finish rolling out
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ⚠️ When This Happens
|
|
||||||
|
|
||||||
### Scenario 1: Slow Rollout
|
|
||||||
```
|
|
||||||
14:00:00 - ArgoCD syncs deployment (v1 → v2)
|
|
||||||
14:00:05 - ArgoCD: "Synced!" ✅
|
|
||||||
14:00:10 - Kubernetes starts rollout
|
|
||||||
14:00:30 - Pod-1 terminates (v1)
|
|
||||||
14:00:35 - Pod-3 starts (v2)
|
|
||||||
14:00:50 - Pod-2 terminates (v1)
|
|
||||||
14:00:55 - Pod-4 starts (v2)
|
|
||||||
14:01:00 - Rollout complete! ✅
|
|
||||||
|
|
||||||
Jenkins checks at 14:00:05: ArgoCD says "Synced"
|
|
||||||
But pods are still v1! ❌
|
|
||||||
```
|
|
||||||
|
|
||||||
### Scenario 2: Image Pull Delay
|
|
||||||
```
|
|
||||||
14:00:00 - ArgoCD syncs
|
|
||||||
14:00:05 - ArgoCD: "Synced!" ✅
|
|
||||||
14:00:10 - Kubernetes tries to start new pod
|
|
||||||
14:00:15 - Pulling image... (slow network)
|
|
||||||
14:00:45 - Image pulled
|
|
||||||
14:00:50 - Pod starts
|
|
||||||
14:01:00 - Pod ready
|
|
||||||
|
|
||||||
Jenkins checks at 14:00:05: "Synced" but no new pods yet!
|
|
||||||
```
|
|
||||||
|
|
||||||
### Scenario 3: Resource Constraints
|
|
||||||
```
|
|
||||||
14:00:00 - ArgoCD syncs
|
|
||||||
14:00:05 - ArgoCD: "Synced!" ✅
|
|
||||||
14:00:10 - Kubernetes: "No resources available"
|
|
||||||
14:00:20 - Kubernetes: "Waiting for node capacity..."
|
|
||||||
14:01:00 - Old pod terminates, resources freed
|
|
||||||
14:01:10 - New pod starts
|
|
||||||
|
|
||||||
Jenkins checks at 14:00:05: "Synced" but can't schedule pods!
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ✅ The Solution
|
|
||||||
|
|
||||||
### What Jenkins Must Check:
|
|
||||||
|
|
||||||
```groovy
|
|
||||||
// ❌ BAD - Only checks ArgoCD
|
|
||||||
if (argocdStatus == 'Synced') {
|
|
||||||
echo "Done!"
|
|
||||||
}
|
|
||||||
|
|
||||||
// ✅ GOOD - Checks ArgoCD + Kubernetes
|
|
||||||
if (argocdStatus == 'Synced') {
|
|
||||||
// 1. Wait for rollout
|
|
||||||
kubectl rollout status deployment/app
|
|
||||||
|
|
||||||
// 2. Verify actual pod images
|
|
||||||
podImages = kubectl get pods -o jsonpath='{.status.containerStatuses[0].image}'
|
|
||||||
if (podImages contains newVersion) {
|
|
||||||
echo "Verified!"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎯 New Jenkinsfile Verification
|
|
||||||
|
|
||||||
### Stage 1: ArgoCD Sync Check
|
|
||||||
```groovy
|
|
||||||
stage('Wait for ArgoCD Sync') {
|
|
||||||
// Checks:
|
|
||||||
// 1. ArgoCD sync status = "Synced"
|
|
||||||
// 2. Deployment SPEC image updated
|
|
||||||
//
|
|
||||||
// Does NOT check if pods rolled out!
|
|
||||||
// That's the next stage.
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Output:**
|
|
||||||
```
|
|
||||||
ArgoCD sync status: Synced
|
|
||||||
Deployment spec image: app:v2
|
|
||||||
✅ ArgoCD synced and deployment spec updated!
|
|
||||||
Note: Pods may still be rolling out - will verify in next stage
|
|
||||||
```
|
|
||||||
|
|
||||||
### Stage 2: Wait for Rollout
|
|
||||||
```groovy
|
|
||||||
stage('Wait for Deployment') {
|
|
||||||
// Uses kubectl rollout status
|
|
||||||
// Waits for actual pod rollout to complete
|
|
||||||
sh "kubectl rollout status deployment/app --timeout=5m"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**What `kubectl rollout status` does:**
|
|
||||||
- Watches deployment progress
|
|
||||||
- Waits for all new pods to be ready
|
|
||||||
- Returns when rollout complete
|
|
||||||
- Times out if rollout stuck
|
|
||||||
|
|
||||||
**Output:**
|
|
||||||
```
|
|
||||||
Waiting for deployment "app" rollout to finish: 1 out of 2 new replicas have been updated...
|
|
||||||
Waiting for deployment "app" rollout to finish: 1 old replicas are pending termination...
|
|
||||||
deployment "app" successfully rolled out
|
|
||||||
✅ Rollout completed successfully!
|
|
||||||
```
|
|
||||||
|
|
||||||
### Stage 3: Verify Actual Pods
|
|
||||||
```groovy
|
|
||||||
stage('Verify Deployment') {
|
|
||||||
// CRITICAL CHECKS:
|
|
||||||
|
|
||||||
// 1. Deployment status
|
|
||||||
readyReplicas == desiredReplicas
|
|
||||||
|
|
||||||
// 2. Deployment spec image
|
|
||||||
deploymentImage contains newTag
|
|
||||||
|
|
||||||
// 3. ACTUAL POD IMAGES (most important!)
|
|
||||||
podImages = all pods images
|
|
||||||
for each podImage:
|
|
||||||
if podImage does not contain newTag:
|
|
||||||
FAIL!
|
|
||||||
|
|
||||||
// 4. Pod health
|
|
||||||
all pods in Running state
|
|
||||||
|
|
||||||
// 5. Restart count
|
|
||||||
check for crash loops
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Output:**
|
|
||||||
```
|
|
||||||
================================================
|
|
||||||
DEPLOYMENT VERIFICATION
|
|
||||||
================================================
|
|
||||||
|
|
||||||
1. Checking deployment status...
|
|
||||||
Desired replicas: 2
|
|
||||||
Updated replicas: 2
|
|
||||||
Ready replicas: 2
|
|
||||||
Available replicas: 2
|
|
||||||
✅ All pods ready
|
|
||||||
|
|
||||||
2. Checking deployment spec image...
|
|
||||||
Deployment spec image: app:v2
|
|
||||||
Expected tag: v2
|
|
||||||
✅ Deployment spec correct
|
|
||||||
|
|
||||||
3. Checking actual running pod images...
|
|
||||||
Running pod images:
|
|
||||||
- app:v2
|
|
||||||
- app:v2
|
|
||||||
✅ All pods running correct image
|
|
||||||
|
|
||||||
4. Checking pod readiness probes...
|
|
||||||
✅ All pods in Running state
|
|
||||||
|
|
||||||
5. Checking for container restarts...
|
|
||||||
Max restart count: 0
|
|
||||||
✅ Restart count acceptable
|
|
||||||
|
|
||||||
================================================
|
|
||||||
✅ ALL VERIFICATION CHECKS PASSED!
|
|
||||||
================================================
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔥 What Happens If Check #3 Fails
|
|
||||||
|
|
||||||
```
|
|
||||||
3. Checking actual running pod images...
|
|
||||||
Running pod images:
|
|
||||||
- app:v1 ❌
|
|
||||||
- app:v1 ❌
|
|
||||||
❌ Pod running wrong image: app:v1
|
|
||||||
❌ FAILED: 2 pod(s) running old image!
|
|
||||||
This is the ArgoCD sync bug - deployment updated but pods not rolled out
|
|
||||||
```
|
|
||||||
|
|
||||||
**Jenkins will:**
|
|
||||||
1. ❌ Mark build as failed
|
|
||||||
2. 🔄 Trigger rollback (if enabled)
|
|
||||||
3. 📱 Send notification with details
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🧪 Testing the Fix
|
|
||||||
|
|
||||||
### Test 1: Normal Deployment
|
|
||||||
```bash
|
|
||||||
# Update image in Git
|
|
||||||
git commit -m "Update to v2"
|
|
||||||
git push
|
|
||||||
|
|
||||||
# Jenkins should:
|
|
||||||
# 1. Wait for ArgoCD sync ✅
|
|
||||||
# 2. Wait for rollout ✅
|
|
||||||
# 3. Verify pods have v2 ✅
|
|
||||||
# 4. Success! ✅
|
|
||||||
```
|
|
||||||
|
|
||||||
### Test 2: Slow Rollout
|
|
||||||
```bash
|
|
||||||
# Set slow rollout
|
|
||||||
kubectl patch deployment app -p '{"spec":{"strategy":{"rollingUpdate":{"maxUnavailable":0,"maxSurge":1}}}}'
|
|
||||||
|
|
||||||
# Update image
|
|
||||||
git push
|
|
||||||
|
|
||||||
# Jenkins should:
|
|
||||||
# 1. ArgoCD syncs quickly ✅
|
|
||||||
# 2. Wait for slow rollout (may take 2-3 minutes) ⏳
|
|
||||||
# 3. Verify when complete ✅
|
|
||||||
```
|
|
||||||
|
|
||||||
### Test 3: Rollout Stuck
|
|
||||||
```bash
|
|
||||||
# Create a broken image tag
|
|
||||||
# Update to image: app:nonexistent
|
|
||||||
|
|
||||||
git push
|
|
||||||
|
|
||||||
# Jenkins should:
|
|
||||||
# 1. ArgoCD syncs ✅
|
|
||||||
# 2. kubectl rollout status times out ❌
|
|
||||||
# 3. Rollback triggered ✅
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📊 Comparison: Old vs New
|
|
||||||
|
|
||||||
### Old Pipeline (Unreliable)
|
|
||||||
```
|
|
||||||
1. ArgoCD sync check
|
|
||||||
├─ Checks: ArgoCD status
|
|
||||||
├─ Checks: Deployment spec image
|
|
||||||
└─ Duration: ~30 seconds
|
|
||||||
|
|
||||||
⚠️ PROBLEM: Pods might not have rolled out!
|
|
||||||
|
|
||||||
2. Success! ✅ (but pods are still old!)
|
|
||||||
```
|
|
||||||
|
|
||||||
### New Pipeline (Reliable)
|
|
||||||
```
|
|
||||||
1. ArgoCD sync check
|
|
||||||
├─ Checks: ArgoCD status
|
|
||||||
├─ Checks: Deployment spec image
|
|
||||||
└─ Duration: ~30 seconds
|
|
||||||
|
|
||||||
2. Rollout status check
|
|
||||||
├─ Checks: kubectl rollout status
|
|
||||||
├─ Waits: For actual pod rollout
|
|
||||||
└─ Duration: ~1-2 minutes
|
|
||||||
|
|
||||||
3. Verification
|
|
||||||
├─ Checks: Deployment status
|
|
||||||
├─ Checks: ACTUAL pod images ← KEY!
|
|
||||||
├─ Checks: Pod health
|
|
||||||
├─ Checks: Restart count
|
|
||||||
└─ Duration: ~10 seconds
|
|
||||||
|
|
||||||
4. Success! ✅ (pods verified running new version)
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎯 Key Takeaways
|
|
||||||
|
|
||||||
### ❌ Don't Trust:
|
|
||||||
- ArgoCD "Synced" status alone
|
|
||||||
- Deployment spec image alone
|
|
||||||
- Health status alone
|
|
||||||
|
|
||||||
### ✅ Always Verify:
|
|
||||||
1. **ArgoCD synced** (manifest applied)
|
|
||||||
2. **Rollout completed** (`kubectl rollout status`)
|
|
||||||
3. **Actual pod images** (what's really running)
|
|
||||||
4. **Pod health** (ready and not crashing)
|
|
||||||
|
|
||||||
### 💡 Remember:
|
|
||||||
```
|
|
||||||
ArgoCD "Synced" = Git matches Kubernetes manifest ✅
|
|
||||||
BUT
|
|
||||||
Kubernetes manifest != Running pods ⚠️
|
|
||||||
|
|
||||||
You MUST check actual pods!
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔗 Related Issues
|
|
||||||
|
|
||||||
- [ArgoCD #2723](https://github.com/argoproj/argo-cd/issues/2723) - "Synced but pods not updated"
|
|
||||||
- [Kubernetes #93033](https://github.com/kubernetes/kubernetes/issues/93033) - "Deployment rollout delays"
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🚀 Using the New Jenkinsfile
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. Update Jenkinsfile in your repo
|
|
||||||
cp Jenkinsfile.telegram.en apps/demo-nginx/Jenkinsfile
|
|
||||||
|
|
||||||
# 2. Commit and push
|
|
||||||
git add apps/demo-nginx/Jenkinsfile
|
|
||||||
git commit -m "fix: add proper deployment verification"
|
|
||||||
git push
|
|
||||||
|
|
||||||
# 3. Run build
|
|
||||||
# Jenkins will now properly verify deployments!
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📱 Notifications
|
|
||||||
|
|
||||||
With the new verification, you'll see:
|
|
||||||
|
|
||||||
**During deployment:**
|
|
||||||
```
|
|
||||||
⏳ ArgoCD Syncing
|
|
||||||
Application: demo-nginx
|
|
||||||
Timeout: 120s
|
|
||||||
|
|
||||||
🚀 Deploying to Kubernetes
|
|
||||||
Deployment: demo-nginx
|
|
||||||
Image: main-42
|
|
||||||
Rolling out new pods...
|
|
||||||
```
|
|
||||||
|
|
||||||
**On success:**
|
|
||||||
```
|
|
||||||
✅ Deployment Successful!
|
|
||||||
|
|
||||||
Verified:
|
|
||||||
- ArgoCD synced ✅
|
|
||||||
- Rollout completed ✅
|
|
||||||
- Pods running v42 ✅
|
|
||||||
- All pods healthy ✅
|
|
||||||
```
|
|
||||||
|
|
||||||
**On failure:**
|
|
||||||
```
|
|
||||||
❌ Deployment Failed
|
|
||||||
|
|
||||||
Error: 2 pods running old image!
|
|
||||||
|
|
||||||
Rollback initiated...
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**This fix ensures Jenkins never reports success until pods are actually running the new version!** ✅
|
|
||||||
@@ -1,148 +0,0 @@
|
|||||||
# 🎉 CI/CD Pipeline - Complete Guide
|
|
||||||
|
|
||||||
## ✅ Что мы построили - Production Ready System!
|
|
||||||
|
|
||||||
Полноценный CI/CD pipeline с автоматическим деплоем от Git до Production.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📦 Компоненты системы
|
|
||||||
|
|
||||||
### 1. Jenkins (CI/CD Engine)
|
|
||||||
```
|
|
||||||
URL: http://jenkins.thedevops.dev
|
|
||||||
Namespace: jenkins
|
|
||||||
|
|
||||||
Установлено:
|
|
||||||
✅ Docker CLI (для build & push)
|
|
||||||
✅ kubectl v1.28.0 (для deploy verification)
|
|
||||||
✅ ServiceAccount + RBAC (для K8s API)
|
|
||||||
✅ Auto-scan (проверяет Git каждый час)
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Gitea (Git Repository)
|
|
||||||
```
|
|
||||||
URL: http://gitea.thedevops.dev
|
|
||||||
Repository: k3s-gitops
|
|
||||||
|
|
||||||
Содержит:
|
|
||||||
✅ apps/demo-nginx/ - все манифесты приложения
|
|
||||||
✅ apps/jenkins/ - конфигурация Jenkins
|
|
||||||
✅ Jenkinsfile - CI/CD pipeline
|
|
||||||
✅ Webhook настроен (опционально)
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. ArgoCD (GitOps)
|
|
||||||
```
|
|
||||||
URL: https://argocd.thedevops.dev
|
|
||||||
Application: demo-nginx
|
|
||||||
|
|
||||||
Функции:
|
|
||||||
✅ Автоматически синхронизирует Git → K8s
|
|
||||||
✅ Отслеживает изменения в deployment.yaml
|
|
||||||
✅ Применяет rolling updates
|
|
||||||
✅ Self-heal & auto-prune
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. Docker Hub (Registry)
|
|
||||||
```
|
|
||||||
Repository: docker.io/vladcrypto/demo-nginx
|
|
||||||
|
|
||||||
Images:
|
|
||||||
✅ Автоматически пушатся из Jenkins
|
|
||||||
✅ Теги: main-XX, latest
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5. Kubernetes (Runtime)
|
|
||||||
```
|
|
||||||
Namespace: demo-app
|
|
||||||
Deployment: demo-nginx (2 replicas)
|
|
||||||
Service: demo-nginx (ClusterIP)
|
|
||||||
Ingress: https://demo-nginx.thedevops.dev
|
|
||||||
|
|
||||||
Status:
|
|
||||||
✅ 2/2 pods Running
|
|
||||||
✅ Rolling updates enabled
|
|
||||||
✅ Health checks configured
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔄 Как работает Pipeline
|
|
||||||
|
|
||||||
### Автоматический flow:
|
|
||||||
|
|
||||||
```
|
|
||||||
1. Developer → git push
|
|
||||||
↓
|
|
||||||
2. Gitea → сохраняет commit
|
|
||||||
↓
|
|
||||||
3. Jenkins → Auto-scan обнаруживает изменения (каждый час)
|
|
||||||
↓
|
|
||||||
4. Jenkins Pipeline (Jenkinsfile):
|
|
||||||
├─ Stage 1: Checkout Source (создает Dockerfile + nginx.conf)
|
|
||||||
├─ Stage 2: Build Docker Image (main-XX)
|
|
||||||
├─ Stage 3: Push to Registry (Docker Hub)
|
|
||||||
├─ Stage 4: Update GitOps Manifests (deployment.yaml)
|
|
||||||
└─ Stage 5: Verify Deployment (kubectl rollout status)
|
|
||||||
↓
|
|
||||||
5. ArgoCD → Обнаруживает изменения в Git (каждые 3 мин)
|
|
||||||
↓
|
|
||||||
6. ArgoCD → Синхронизирует deployment.yaml → Kubernetes
|
|
||||||
↓
|
|
||||||
7. Kubernetes → Rolling update (zero downtime)
|
|
||||||
↓
|
|
||||||
8. ✅ Production! Новая версия работает!
|
|
||||||
```
|
|
||||||
|
|
||||||
**Время:** 3-5 минут от commit до production
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🧪 Как проверять
|
|
||||||
|
|
||||||
### 1. Проверить Jenkins
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Pods
|
|
||||||
kubectl get pods -n jenkins
|
|
||||||
|
|
||||||
# Logs
|
|
||||||
kubectl logs -n jenkins <jenkins-pod>
|
|
||||||
|
|
||||||
# UI
|
|
||||||
http://jenkins.thedevops.dev
|
|
||||||
```
|
|
||||||
|
|
||||||
**Что смотреть:**
|
|
||||||
- Build History (список сборок)
|
|
||||||
- Console Output (логи последней сборки)
|
|
||||||
- Blue Ocean (красивый UI)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 2. Проверить Demo App
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Deployment
|
|
||||||
kubectl get deployment demo-nginx -n demo-app
|
|
||||||
|
|
||||||
# Pods
|
|
||||||
kubectl get pods -n demo-app
|
|
||||||
|
|
||||||
# Service
|
|
||||||
kubectl get svc demo-nginx -n demo-app
|
|
||||||
|
|
||||||
# Ingress
|
|
||||||
kubectl get ingress demo-nginx -n demo-app
|
|
||||||
```
|
|
||||||
|
|
||||||
**Ожидаемый результат:**
|
|
||||||
```
|
|
||||||
DEPLOYMENT: 2/2 Ready
|
|
||||||
PODS: 2/2 Running
|
|
||||||
SERVICE: ClusterIP
|
|
||||||
INGRESS: demo-nginx.thedevops.dev
|
|
||||||
```
|
|
||||||
|
|
||||||
См. полную документацию в файле для всех деталей, сценариев использования, troubleshooting и best practices.
|
|
||||||
166
apps/demo-nginx/docs/Jenkinsfile
vendored
166
apps/demo-nginx/docs/Jenkinsfile
vendored
@@ -1,166 +0,0 @@
|
|||||||
pipeline { // Declarative Jenkins pipeline start
|
|
||||||
agent any // Run on any available Jenkins agent
|
|
||||||
|
|
||||||
environment { // Global environment variables
|
|
||||||
APP_NAME = 'demo-nginx' // Application name (Docker, K8s, logs)
|
|
||||||
NAMESPACE = 'demo-app' // Kubernetes namespace
|
|
||||||
DOCKER_REGISTRY = 'docker.io' // Docker registry hostname
|
|
||||||
DOCKER_REPO = 'vladcrypto' // Docker Hub repository / namespace
|
|
||||||
GITEA_URL = 'http://gitea-http.gitea.svc.cluster.local:3000' // Internal Gitea URL
|
|
||||||
GITEA_REPO = 'admin/k3s-gitops' // GitOps repository path
|
|
||||||
GITEA_BRANCH = 'main' // Git branch for GitOps updates
|
|
||||||
BUILD_TAG = "${env.BUILD_NUMBER}" // Jenkins build number
|
|
||||||
IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}" // Image version tag
|
|
||||||
}
|
|
||||||
|
|
||||||
stages { // Pipeline stages definition
|
|
||||||
|
|
||||||
stage('Checkout Source') { // Stage: prepare application source
|
|
||||||
steps { // Steps executed in this stage
|
|
||||||
echo "Checking out application source code..." // Log message
|
|
||||||
|
|
||||||
sh ''' // Execute shell script
|
|
||||||
cat > Dockerfile << 'EOF' # Create Dockerfile
|
|
||||||
FROM nginx:1.25.3-alpine # Base Nginx Alpine image
|
|
||||||
RUN echo "<html><body><h1>Demo Nginx - Build ${BUILD_NUMBER}</h1><p>Environment: Production</p><p>Version: ${IMAGE_TAG}</p></body></html>" > /usr/share/nginx/html/index.html # Inject build metadata into HTML
|
|
||||||
COPY nginx.conf /etc/nginx/nginx.conf # Copy custom nginx configuration
|
|
||||||
EXPOSE 80 # Expose HTTP port
|
|
||||||
CMD ["nginx", "-g", "daemon off;"] # Run nginx in foreground
|
|
||||||
EOF
|
|
||||||
'''
|
|
||||||
|
|
||||||
sh ''' // Generate nginx configuration
|
|
||||||
cat > nginx.conf << 'EOF' # Create nginx.conf
|
|
||||||
user nginx; # Run workers as nginx user
|
|
||||||
worker_processes auto; # Scale workers to CPU cores
|
|
||||||
error_log /var/log/nginx/error.log warn; # Error log level and path
|
|
||||||
pid /var/run/nginx.pid; # PID file location
|
|
||||||
events { worker_connections 1024; } # Max connections per worker
|
|
||||||
http { # HTTP configuration block
|
|
||||||
include /etc/nginx/mime.types; # Load MIME types
|
|
||||||
default_type application/octet-stream; # Default content type
|
|
||||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; # Access log format
|
|
||||||
access_log /var/log/nginx/access.log main; # Enable access logging
|
|
||||||
sendfile on; # Enable zero-copy file transfer
|
|
||||||
keepalive_timeout 65; # Keepalive timeout
|
|
||||||
server { # Server definition
|
|
||||||
listen 80; # Listen on port 80
|
|
||||||
server_name _; # Catch-all server
|
|
||||||
location / { # Root location
|
|
||||||
root /usr/share/nginx/html; # Serve static files
|
|
||||||
index index.html; # Default index file
|
|
||||||
}
|
|
||||||
location /health { # Health check endpoint
|
|
||||||
access_log off; # Disable logging for health checks
|
|
||||||
return 200 "healthy\n"; # Always return HTTP 200
|
|
||||||
add_header Content-Type text/plain; # Explicit content type
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
'''
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Build Docker Image') { // Stage: build Docker image
|
|
||||||
steps {
|
|
||||||
script { // Scripted block
|
|
||||||
echo "Building Docker image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}" // Log image name
|
|
||||||
sh """ // Execute Docker build
|
|
||||||
docker build \
|
|
||||||
-t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} \ // Versioned image
|
|
||||||
-t ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest \ // Latest tag
|
|
||||||
. // Build context
|
|
||||||
"""
|
|
||||||
echo "✅ Image built successfully!" // Success message
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Push to Registry') { // Stage: push image to registry
|
|
||||||
when { branch 'main' } // Execute only on main branch
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "Pushing image to registry..." // Log push start
|
|
||||||
withCredentials([usernamePassword( // Inject Docker credentials
|
|
||||||
credentialsId: 'docker-registry-credentials',
|
|
||||||
usernameVariable: 'DOCKER_USER',
|
|
||||||
passwordVariable: 'DOCKER_PASS'
|
|
||||||
)]) {
|
|
||||||
sh """ // Authenticate and push image
|
|
||||||
echo "\${DOCKER_PASS}" | docker login ${DOCKER_REGISTRY} -u "\${DOCKER_USER}" --password-stdin // Login securely
|
|
||||||
docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} // Push versioned image
|
|
||||||
docker push ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest // Push latest image
|
|
||||||
docker logout ${DOCKER_REGISTRY} // Logout
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
echo "✅ Image pushed successfully!" // Push success
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Update GitOps Manifests') { // Stage: update GitOps repository
|
|
||||||
when { branch 'main' } // Only from main branch
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "Updating Kubernetes manifests..." // Log GitOps update
|
|
||||||
withCredentials([usernamePassword( // Inject Gitea credentials
|
|
||||||
credentialsId: 'gitea-credentials',
|
|
||||||
usernameVariable: 'GIT_USER',
|
|
||||||
passwordVariable: 'GIT_PASS'
|
|
||||||
)]) {
|
|
||||||
sh """ // Clone and update manifests
|
|
||||||
rm -rf k3s-gitops || true // Remove old repo
|
|
||||||
git clone http://\${GIT_USER}:\${GIT_PASS}@gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops.git // Clone repo
|
|
||||||
cd k3s-gitops // Enter repo
|
|
||||||
git config user.name "Jenkins" // Git author name
|
|
||||||
git config user.email "jenkins@thedevops.dev" // Git author email
|
|
||||||
sed -i 's|image: .*|image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}|' apps/demo-nginx/deployment.yaml // Update image
|
|
||||||
git add apps/demo-nginx/deployment.yaml // Stage change
|
|
||||||
git commit -m "chore(demo-nginx): Update image to ${IMAGE_TAG}" || echo "No changes" // Commit if needed
|
|
||||||
git push origin main // Push to main
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
echo "✅ Manifests updated!" // GitOps success
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Verify Deployment') { // Stage: verify Kubernetes deployment
|
|
||||||
when { branch 'main' } // Only on main
|
|
||||||
steps {
|
|
||||||
script {
|
|
||||||
echo "Verifying deployment..." // Log verification start
|
|
||||||
sh """ // Check deployment status
|
|
||||||
sleep 30 // Wait for rollout start
|
|
||||||
kubectl rollout status deployment/${APP_NAME} -n ${NAMESPACE} --timeout=300s || true // Rollout status
|
|
||||||
kubectl get pods -n ${NAMESPACE} -l app=${APP_NAME} // List pods
|
|
||||||
"""
|
|
||||||
echo "✅ Deployment completed!" // Verification success
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
post { // Post-pipeline actions
|
|
||||||
success { // On success
|
|
||||||
echo """ // Success summary
|
|
||||||
✅ Pipeline SUCCESS!
|
|
||||||
Image: ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG}
|
|
||||||
Namespace: ${NAMESPACE}
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
failure { // On failure
|
|
||||||
echo "❌ Pipeline failed!" // Failure message
|
|
||||||
}
|
|
||||||
always { // Always execute cleanup
|
|
||||||
sh """ // Cleanup Docker artifacts
|
|
||||||
docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:${IMAGE_TAG} || true // Remove versioned image
|
|
||||||
docker rmi ${DOCKER_REGISTRY}/${DOCKER_REPO}/${APP_NAME}:latest || true // Remove latest image
|
|
||||||
docker stop test-${BUILD_NUMBER} 2>/dev/null || true // Stop test container
|
|
||||||
docker rm test-${BUILD_NUMBER} 2>/dev/null || true // Remove test container
|
|
||||||
"""
|
|
||||||
cleanWs() // Clean Jenkins workspace
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,235 +0,0 @@
|
|||||||
# Demo Nginx Documentation
|
|
||||||
|
|
||||||
Документация для demo-nginx deployment с автоматическим и ручным rollback.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📚 Available Documentation
|
|
||||||
|
|
||||||
### [ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md)
|
|
||||||
**Comprehensive Manual Rollback Guide**
|
|
||||||
|
|
||||||
Полная документация по функции ручного rollback:
|
|
||||||
- 3 способа rollback (IMAGE_TAG, REVISION, GIT_COMMIT)
|
|
||||||
- Setup guide
|
|
||||||
- Troubleshooting со всеми fixes
|
|
||||||
- Best practices
|
|
||||||
- Examples
|
|
||||||
- FAQ
|
|
||||||
|
|
||||||
**When to use:** Нужна детальная информация или troubleshooting
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### [ROLLBACK_QUICK_REF.md](./ROLLBACK_QUICK_REF.md)
|
|
||||||
**Quick Reference Card**
|
|
||||||
|
|
||||||
Краткая справка для быстрого использования:
|
|
||||||
- Quick start (2 минуты)
|
|
||||||
- 3 способа rollback
|
|
||||||
- Emergency procedure
|
|
||||||
- Verification commands
|
|
||||||
- Checklist
|
|
||||||
|
|
||||||
**When to use:** Быстрый rollback в production
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎯 Quick Links
|
|
||||||
|
|
||||||
### Rollback Feature
|
|
||||||
- **Full Guide:** [ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md)
|
|
||||||
- **Quick Ref:** [ROLLBACK_QUICK_REF.md](./ROLLBACK_QUICK_REF.md)
|
|
||||||
- **Jenkinsfile:** [Jenkinsfile.rollback](../Jenkinsfile.rollback)
|
|
||||||
- **CI/CD Guide:** [../../../CICD_GUIDE.md](../../../CICD_GUIDE.md)
|
|
||||||
|
|
||||||
### Related Resources
|
|
||||||
- **Jenkins RBAC:** [apps/jenkins/rbac.yaml](../../jenkins/rbac.yaml)
|
|
||||||
- **Deployment:** [deployment.yaml](../deployment.yaml)
|
|
||||||
- **Main CI/CD:** [Jenkinsfile](../Jenkinsfile)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🚀 Quick Start
|
|
||||||
|
|
||||||
### Manual Rollback (2 minutes)
|
|
||||||
|
|
||||||
```
|
|
||||||
1. Jenkins → demo-nginx-rollback
|
|
||||||
2. IMAGE_TAG + main-21
|
|
||||||
3. SKIP_HEALTH_CHECK: true
|
|
||||||
4. Build
|
|
||||||
```
|
|
||||||
|
|
||||||
See [ROLLBACK_QUICK_REF.md](./ROLLBACK_QUICK_REF.md) for details.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Emergency Rollback (30 seconds)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
kubectl rollout undo deployment/demo-nginx -n demo-app
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📊 Features Summary
|
|
||||||
|
|
||||||
### Manual Rollback
|
|
||||||
✅ 3 rollback methods (IMAGE_TAG, REVISION, GIT_COMMIT)
|
|
||||||
✅ GitOps sync (auto-commit to Git)
|
|
||||||
✅ Zero downtime (rolling updates)
|
|
||||||
✅ DRY_RUN mode (safe testing)
|
|
||||||
✅ Full RBAC permissions
|
|
||||||
✅ Input validation (auto-trim)
|
|
||||||
✅ Retry logic (5 attempts)
|
|
||||||
⚠️ Health check optional (use SKIP_HEALTH_CHECK=true)
|
|
||||||
|
|
||||||
### Automatic Rollback
|
|
||||||
✅ Triggered on deployment failure
|
|
||||||
✅ Saves previous state
|
|
||||||
✅ Kubernetes rollback
|
|
||||||
✅ Git revert
|
|
||||||
✅ Health checks
|
|
||||||
✅ Timeout protection
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🐛 All Fixes Applied
|
|
||||||
|
|
||||||
| # | Issue | Fix | Status | Doc |
|
|
||||||
|---|-------|-----|--------|-----|
|
|
||||||
| 1 | Container name | Use `nginx` | ✅ | [Link](./ROLLBACK_MANUAL.md#issue-1-container-name-error--fixed) |
|
|
||||||
| 2 | Whitespace | Auto-trim | ✅ | [Link](./ROLLBACK_MANUAL.md#issue-2-whitespace-in-input--fixed) |
|
|
||||||
| 3 | RBAC | pods/exec perm | ✅ | [Link](./ROLLBACK_MANUAL.md#issue-3-rbac-permissions--fixed) |
|
|
||||||
| 4 | Health timing | SKIP option | ⚠️ | [Link](./ROLLBACK_MANUAL.md#issue-4-health-check-timing--workaround) |
|
|
||||||
| 5 | Bash loop | Explicit list | ✅ | [Link](./ROLLBACK_MANUAL.md#issue-5-bash-loop-syntax--fixed) |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 💡 Recommended Reading Order
|
|
||||||
|
|
||||||
### For New Users:
|
|
||||||
1. Start → [ROLLBACK_QUICK_REF.md](./ROLLBACK_QUICK_REF.md)
|
|
||||||
2. Practice → Follow quick start
|
|
||||||
3. Deep Dive → [ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md)
|
|
||||||
|
|
||||||
### For Troubleshooting:
|
|
||||||
1. Check → [ROLLBACK_MANUAL.md - Troubleshooting](./ROLLBACK_MANUAL.md#troubleshooting--fixes)
|
|
||||||
2. Verify → [ROLLBACK_QUICK_REF.md - Verification](./ROLLBACK_QUICK_REF.md#-verify-rollback)
|
|
||||||
3. Support → [ROLLBACK_MANUAL.md - Support](./ROLLBACK_MANUAL.md#support)
|
|
||||||
|
|
||||||
### For Emergency:
|
|
||||||
1. Fast → [ROLLBACK_QUICK_REF.md - Emergency](./ROLLBACK_QUICK_REF.md#-emergency-rollback-30-seconds)
|
|
||||||
2. Alternative → [ROLLBACK_MANUAL.md - Emergency](./ROLLBACK_MANUAL.md#emergency-rollback-procedure)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎓 Key Concepts
|
|
||||||
|
|
||||||
### Rollback Methods Comparison
|
|
||||||
|
|
||||||
| Method | Speed | Precision | Use Case |
|
|
||||||
|--------|-------|-----------|----------|
|
|
||||||
| IMAGE_TAG | ⚡⚡⚡ | 🎯 High | Known build number |
|
|
||||||
| REVISION | ⚡⚡ | 🎯 Medium | Recent rollback |
|
|
||||||
| GIT_COMMIT | ⚡ | 🎯🎯 High | Exact code state |
|
|
||||||
|
|
||||||
### When to Use What
|
|
||||||
|
|
||||||
**Use IMAGE_TAG when:**
|
|
||||||
- You know the build number (main-21)
|
|
||||||
- Quick rollback needed
|
|
||||||
- Most common scenario
|
|
||||||
|
|
||||||
**Use REVISION_NUMBER when:**
|
|
||||||
- Need to go back N versions
|
|
||||||
- Don't remember exact tag
|
|
||||||
- Working with kubectl history
|
|
||||||
|
|
||||||
**Use GIT_COMMIT when:**
|
|
||||||
- Need exact code state
|
|
||||||
- Multiple changes in one build
|
|
||||||
- Precise rollback required
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📈 Monitoring
|
|
||||||
|
|
||||||
### Check Rollback Status
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Deployment status
|
|
||||||
kubectl get deployment demo-nginx -n demo-app
|
|
||||||
|
|
||||||
# Pod status
|
|
||||||
kubectl get pods -n demo-app -l app=demo-nginx
|
|
||||||
|
|
||||||
# Rollout history
|
|
||||||
kubectl rollout history deployment/demo-nginx -n demo-app
|
|
||||||
|
|
||||||
# ArgoCD status
|
|
||||||
kubectl get application demo-nginx -n argocd
|
|
||||||
```
|
|
||||||
|
|
||||||
### Grafana Queries
|
|
||||||
|
|
||||||
```promql
|
|
||||||
# Rollback count
|
|
||||||
sum(increase(deployment_rollback_total[1h])) by (deployment)
|
|
||||||
|
|
||||||
# Rollback rate
|
|
||||||
rate(deployment_rollback_total[5m])
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ❓ FAQ
|
|
||||||
|
|
||||||
### Q: Какой метод rollback использовать?
|
|
||||||
**A:** Для большинства случаев используй IMAGE_TAG - самый быстрый и простой.
|
|
||||||
|
|
||||||
### Q: Health check всегда падает, это баг?
|
|
||||||
**A:** Нет, это timing issue во время rolling update. Используй `SKIP_HEALTH_CHECK: true` и проверь вручную через минуту.
|
|
||||||
|
|
||||||
### Q: Как быстро откатиться в emergency?
|
|
||||||
**A:** Используй `kubectl rollout undo` (30 секунд) или Jenkins с SKIP_HEALTH_CHECK (2 минуты).
|
|
||||||
|
|
||||||
### Q: Где полная документация?
|
|
||||||
**A:** [ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md) - comprehensive guide со всеми details.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🆘 Support
|
|
||||||
|
|
||||||
**Need Help?**
|
|
||||||
|
|
||||||
1. Check [ROLLBACK_MANUAL.md - Troubleshooting](./ROLLBACK_MANUAL.md#troubleshooting--fixes)
|
|
||||||
2. Review [ROLLBACK_MANUAL.md - FAQ](./ROLLBACK_MANUAL.md#faq)
|
|
||||||
3. Check Jenkins console output
|
|
||||||
4. Verify RBAC permissions
|
|
||||||
5. Review pod status and logs
|
|
||||||
|
|
||||||
**Still stuck?**
|
|
||||||
- Jenkins logs: Jenkins → Build → Console Output
|
|
||||||
- K8s events: `kubectl get events -n demo-app`
|
|
||||||
- Pod logs: `kubectl logs -n demo-app -l app=demo-nginx`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📝 Documentation Updates
|
|
||||||
|
|
||||||
**Last Updated:** 2026-01-06
|
|
||||||
**Version:** 1.0
|
|
||||||
**Status:** Production Ready ✅
|
|
||||||
|
|
||||||
**Recent Changes:**
|
|
||||||
- ✅ Added comprehensive manual rollback guide
|
|
||||||
- ✅ Added quick reference card
|
|
||||||
- ✅ Documented all 5 fixes
|
|
||||||
- ✅ Added examples and best practices
|
|
||||||
- ✅ Production-ready feature
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Ready to rollback? Start with [Quick Reference](./ROLLBACK_QUICK_REF.md)! 🚀**
|
|
||||||
@@ -1,717 +0,0 @@
|
|||||||
# 🔄 Manual Rollback Feature - Complete Documentation
|
|
||||||
|
|
||||||
## 📋 Table of Contents
|
|
||||||
|
|
||||||
1. [Overview](#overview)
|
|
||||||
2. [Features](#features)
|
|
||||||
3. [Setup Guide](#setup-guide)
|
|
||||||
4. [Usage Guide](#usage-guide)
|
|
||||||
5. [Rollback Methods](#rollback-methods)
|
|
||||||
6. [Troubleshooting & Fixes](#troubleshooting--fixes)
|
|
||||||
7. [Best Practices](#best-practices)
|
|
||||||
8. [Examples](#examples)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
Manual Rollback feature позволяет откатить deployment на любую предыдущую версию через Jenkins Pipeline.
|
|
||||||
|
|
||||||
### Key Features:
|
|
||||||
- ✅ **3 способа rollback** (IMAGE_TAG, REVISION_NUMBER, GIT_COMMIT)
|
|
||||||
- ✅ **GitOps sync** - автоматически обновляет Git manifests
|
|
||||||
- ✅ **Zero downtime** - rolling updates
|
|
||||||
- ✅ **DRY_RUN mode** - безопасное тестирование
|
|
||||||
- ✅ **Health checks** - опциональная проверка после rollback
|
|
||||||
- ✅ **Full RBAC** - правильные permissions
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Features
|
|
||||||
|
|
||||||
### Rollback Methods
|
|
||||||
|
|
||||||
| Method | Description | Example | Use Case |
|
|
||||||
|--------|-------------|---------|----------|
|
|
||||||
| **IMAGE_TAG** | По Docker image tag | `main-21` | Знаешь конкретный build number |
|
|
||||||
| **REVISION_NUMBER** | По Kubernetes revision | `2` | Откат на N шагов назад |
|
|
||||||
| **GIT_COMMIT** | По Git commit SHA | `abc123def` | Точное состояние кода |
|
|
||||||
|
|
||||||
### Parameters
|
|
||||||
|
|
||||||
```groovy
|
|
||||||
ROLLBACK_METHOD // Выбор метода
|
|
||||||
TARGET_VERSION // Целевая версия (auto-trim whitespace)
|
|
||||||
SKIP_HEALTH_CHECK // Пропустить health checks (default: false)
|
|
||||||
DRY_RUN // Только показать план (default: false)
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Setup Guide
|
|
||||||
|
|
||||||
### Step 1: Create Jenkins Pipeline
|
|
||||||
|
|
||||||
```
|
|
||||||
1. Jenkins → New Item
|
|
||||||
2. Name: demo-nginx-rollback
|
|
||||||
3. Type: Pipeline
|
|
||||||
4. Click OK
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 2: Configure Pipeline
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
Pipeline:
|
|
||||||
Definition: Pipeline script from SCM
|
|
||||||
SCM: Git
|
|
||||||
Repository URL: http://gitea-http.gitea.svc.cluster.local:3000/admin/k3s-gitops
|
|
||||||
Credentials: gitea-credentials
|
|
||||||
Branch: */main
|
|
||||||
Script Path: apps/demo-nginx/Jenkinsfile.rollback
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 3: Verify RBAC
|
|
||||||
|
|
||||||
RBAC уже настроен в `apps/jenkins/rbac.yaml`:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
ClusterRole: jenkins-deployer
|
|
||||||
Permissions:
|
|
||||||
- pods, services, deployments (full CRUD)
|
|
||||||
- pods/exec, pods/log (for health checks)
|
|
||||||
- ingresses, applications (for ArgoCD)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 4: Test with DRY_RUN
|
|
||||||
|
|
||||||
```
|
|
||||||
Jenkins → demo-nginx-rollback → Build with Parameters
|
|
||||||
├─ ROLLBACK_METHOD: IMAGE_TAG
|
|
||||||
├─ TARGET_VERSION: main-21
|
|
||||||
├─ DRY_RUN: ✅ true
|
|
||||||
└─ Build
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Usage Guide
|
|
||||||
|
|
||||||
### Quick Start
|
|
||||||
|
|
||||||
```
|
|
||||||
Jenkins → demo-nginx-rollback → Build with Parameters
|
|
||||||
|
|
||||||
┌─────────────────────────────────────┐
|
|
||||||
│ ROLLBACK_METHOD: IMAGE_TAG │
|
|
||||||
│ TARGET_VERSION: main-21 │
|
|
||||||
│ SKIP_HEALTH_CHECK: true (рекоменд.) │
|
|
||||||
│ DRY_RUN: false │
|
|
||||||
└─────────────────────────────────────┘
|
|
||||||
|
|
||||||
→ Build → ✅ SUCCESS!
|
|
||||||
```
|
|
||||||
|
|
||||||
### Pipeline Stages
|
|
||||||
|
|
||||||
```
|
|
||||||
Stage 1: Validate Input
|
|
||||||
└─ Trim whitespace, validate TARGET_VERSION
|
|
||||||
|
|
||||||
Stage 2: Show Current State
|
|
||||||
└─ Current deployment, image, pods, history
|
|
||||||
|
|
||||||
Stage 3: Prepare Rollback
|
|
||||||
└─ Build target image path or verify revision
|
|
||||||
|
|
||||||
Stage 4: Execute Rollback
|
|
||||||
├─ kubectl set image (or rollout undo)
|
|
||||||
└─ Git commit & push
|
|
||||||
|
|
||||||
Stage 5: Wait for Rollout
|
|
||||||
├─ kubectl rollout status (300s timeout)
|
|
||||||
└─ sleep 10s (stabilization)
|
|
||||||
|
|
||||||
Stage 6: Health Check (optional)
|
|
||||||
└─ 5 retry attempts with 5s delay
|
|
||||||
|
|
||||||
Stage 7: Show New State
|
|
||||||
└─ New deployment state, pods, history
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Rollback Methods
|
|
||||||
|
|
||||||
### Method 1: IMAGE_TAG (Recommended)
|
|
||||||
|
|
||||||
**Когда использовать:** Знаешь конкретный build number
|
|
||||||
|
|
||||||
**Как найти tag:**
|
|
||||||
```bash
|
|
||||||
# Docker Hub
|
|
||||||
https://hub.docker.com/r/vladcrypto/demo-nginx/tags
|
|
||||||
|
|
||||||
# Jenkins build history
|
|
||||||
Jenkins → demo-nginx → Build History
|
|
||||||
|
|
||||||
# Git commits
|
|
||||||
git log --oneline | grep "Update image"
|
|
||||||
```
|
|
||||||
|
|
||||||
**Example:**
|
|
||||||
```
|
|
||||||
ROLLBACK_METHOD: IMAGE_TAG
|
|
||||||
TARGET_VERSION: main-21
|
|
||||||
|
|
||||||
Result: Rollback to docker.io/vladcrypto/demo-nginx:main-21
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Method 2: REVISION_NUMBER
|
|
||||||
|
|
||||||
**Когда использовать:** Нужно откатиться на N шагов назад
|
|
||||||
|
|
||||||
**Как найти revision:**
|
|
||||||
```bash
|
|
||||||
kubectl rollout history deployment/demo-nginx -n demo-app
|
|
||||||
|
|
||||||
# Output:
|
|
||||||
REVISION CHANGE-CAUSE
|
|
||||||
1 Initial deployment
|
|
||||||
2 Update to main-20
|
|
||||||
3 Update to main-21
|
|
||||||
4 Update to main-22 (current)
|
|
||||||
```
|
|
||||||
|
|
||||||
**Example:**
|
|
||||||
```
|
|
||||||
ROLLBACK_METHOD: REVISION_NUMBER
|
|
||||||
TARGET_VERSION: 2
|
|
||||||
|
|
||||||
Result: Rollback to revision 2 (main-20)
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Method 3: GIT_COMMIT
|
|
||||||
|
|
||||||
**Когда использовать:** Нужно вернуться к конкретному состоянию кода
|
|
||||||
|
|
||||||
**Как найти commit:**
|
|
||||||
```bash
|
|
||||||
# Gitea
|
|
||||||
https://git.thedevops.dev/admin/k3s-gitops/commits/branch/main
|
|
||||||
|
|
||||||
# Git CLI
|
|
||||||
git log --oneline apps/demo-nginx/deployment.yaml
|
|
||||||
|
|
||||||
# Output:
|
|
||||||
abc123d Update image to main-22 (current)
|
|
||||||
def456e Update image to main-21
|
|
||||||
ghi789f Update image to main-20
|
|
||||||
```
|
|
||||||
|
|
||||||
**Example:**
|
|
||||||
```
|
|
||||||
ROLLBACK_METHOD: GIT_COMMIT
|
|
||||||
TARGET_VERSION: def456e
|
|
||||||
|
|
||||||
Result: Rollback to commit def456e
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Troubleshooting & Fixes
|
|
||||||
|
|
||||||
### Issue #1: Container Name Error ✅ FIXED
|
|
||||||
|
|
||||||
**Error:**
|
|
||||||
```
|
|
||||||
error: unable to find container named "demo-nginx"
|
|
||||||
```
|
|
||||||
|
|
||||||
**Root Cause:**
|
|
||||||
Pipeline использовал deployment name вместо container name.
|
|
||||||
|
|
||||||
**Fix:**
|
|
||||||
```groovy
|
|
||||||
environment {
|
|
||||||
APP_NAME = 'demo-nginx' // Deployment name
|
|
||||||
CONTAINER_NAME = 'nginx' // Container name ✅
|
|
||||||
}
|
|
||||||
|
|
||||||
kubectl set image deployment/${APP_NAME} \
|
|
||||||
${CONTAINER_NAME}=${TARGET_IMAGE}
|
|
||||||
```
|
|
||||||
|
|
||||||
**How to verify:**
|
|
||||||
```bash
|
|
||||||
kubectl get deployment demo-nginx -n demo-app \
|
|
||||||
-o jsonpath='{.spec.template.spec.containers[0].name}'
|
|
||||||
# Output: nginx
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Issue #2: Whitespace in Input ✅ FIXED
|
|
||||||
|
|
||||||
**Error:**
|
|
||||||
```
|
|
||||||
Target image: docker.io/vladcrypto/demo-nginx: main-21
|
|
||||||
^
|
|
||||||
Space!
|
|
||||||
```
|
|
||||||
|
|
||||||
**Root Cause:**
|
|
||||||
User ввел TARGET_VERSION с пробелом.
|
|
||||||
|
|
||||||
**Fix:**
|
|
||||||
```groovy
|
|
||||||
stage('Validate Input') {
|
|
||||||
// Auto-trim whitespace
|
|
||||||
env.TARGET_VERSION_CLEAN = params.TARGET_VERSION.trim()
|
|
||||||
|
|
||||||
// Use everywhere
|
|
||||||
${env.TARGET_VERSION_CLEAN}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Issue #3: RBAC Permissions ✅ FIXED
|
|
||||||
|
|
||||||
**Error:**
|
|
||||||
```
|
|
||||||
Error: User "system:serviceaccount:jenkins:jenkins"
|
|
||||||
cannot create resource "pods/exec"
|
|
||||||
```
|
|
||||||
|
|
||||||
**Root Cause:**
|
|
||||||
Jenkins ServiceAccount не имел прав на pods/exec для health checks.
|
|
||||||
|
|
||||||
**Fix:**
|
|
||||||
```yaml
|
|
||||||
# apps/jenkins/rbac.yaml
|
|
||||||
rules:
|
|
||||||
- apiGroups: [""]
|
|
||||||
resources: ["pods/exec", "pods/log"] # ← Added!
|
|
||||||
verbs: ["create", "get"]
|
|
||||||
```
|
|
||||||
|
|
||||||
**Applied:**
|
|
||||||
```bash
|
|
||||||
kubectl apply -f apps/jenkins/rbac.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Issue #4: Health Check Timing ⚠️ WORKAROUND
|
|
||||||
|
|
||||||
**Error:**
|
|
||||||
```
|
|
||||||
wget: can't connect to remote host: Connection refused
|
|
||||||
```
|
|
||||||
|
|
||||||
**Root Cause:**
|
|
||||||
Health check runs too early during rolling update (race condition).
|
|
||||||
|
|
||||||
**Workaround:**
|
|
||||||
```groovy
|
|
||||||
// Option 1: Skip health check (recommended)
|
|
||||||
SKIP_HEALTH_CHECK: true
|
|
||||||
|
|
||||||
// Option 2: Longer stabilization wait
|
|
||||||
sleep 30 // Instead of 10
|
|
||||||
```
|
|
||||||
|
|
||||||
**Timeline:**
|
|
||||||
```
|
|
||||||
T+0s: kubectl set image
|
|
||||||
T+30s: Rollout status = complete
|
|
||||||
T+40s: sleep 10s
|
|
||||||
T+50s: Health check (pods might still be starting)
|
|
||||||
```
|
|
||||||
|
|
||||||
**Solution:**
|
|
||||||
Use `SKIP_HEALTH_CHECK: true` и проверь вручную через 30-60s:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
kubectl get pods -n demo-app -l app=demo-nginx
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Issue #5: Bash Loop Syntax ✅ FIXED
|
|
||||||
|
|
||||||
**Error:**
|
|
||||||
```
|
|
||||||
Health check attempt {1..5}/5...
|
|
||||||
# Loop executed only once!
|
|
||||||
```
|
|
||||||
|
|
||||||
**Root Cause:**
|
|
||||||
`{1..5}` не работает в sh/dash, нужен bash.
|
|
||||||
|
|
||||||
**Fix:**
|
|
||||||
```bash
|
|
||||||
#!/bin/bash # ← Added shebang
|
|
||||||
set -e
|
|
||||||
|
|
||||||
# Fixed loop syntax
|
|
||||||
for i in 1 2 3 4 5; do # Instead of {1..5}
|
|
||||||
echo "Health check attempt $i/5..."
|
|
||||||
if kubectl exec ...; then
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
if [ $i -lt 5 ]; then
|
|
||||||
sleep 5
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Best Practices
|
|
||||||
|
|
||||||
### 1. Always Use DRY_RUN First
|
|
||||||
|
|
||||||
```
|
|
||||||
Step 1: DRY_RUN=true → Проверь план
|
|
||||||
Step 2: Verify output
|
|
||||||
Step 3: DRY_RUN=false → Execute
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Use SKIP_HEALTH_CHECK for Emergency
|
|
||||||
|
|
||||||
```
|
|
||||||
Emergency rollback:
|
|
||||||
├─ SKIP_HEALTH_CHECK: true
|
|
||||||
├─ Focus on speed
|
|
||||||
└─ Verify manually after
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. Document Rollback Reason
|
|
||||||
|
|
||||||
Add comment в Jenkins build:
|
|
||||||
```
|
|
||||||
Build Comment:
|
|
||||||
"Rollback due to: API errors in main-23
|
|
||||||
Previous working version: main-21
|
|
||||||
Impact: None (zero downtime)"
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. Monitor After Rollback
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Watch pods
|
|
||||||
watch kubectl get pods -n demo-app
|
|
||||||
|
|
||||||
# Check logs
|
|
||||||
kubectl logs -n demo-app -l app=demo-nginx -f
|
|
||||||
|
|
||||||
# Verify image
|
|
||||||
kubectl get deployment demo-nginx -n demo-app \
|
|
||||||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5. Verify in ArgoCD
|
|
||||||
|
|
||||||
```
|
|
||||||
ArgoCD UI → demo-nginx
|
|
||||||
├─ Status: Synced ✅
|
|
||||||
└─ Health: Healthy ✅
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Examples
|
|
||||||
|
|
||||||
### Example 1: Quick Rollback to Previous Build
|
|
||||||
|
|
||||||
```
|
|
||||||
Scenario: Build #23 failed, rollback to #21
|
|
||||||
|
|
||||||
Steps:
|
|
||||||
1. Jenkins → demo-nginx-rollback
|
|
||||||
2. IMAGE_TAG + main-21
|
|
||||||
3. SKIP_HEALTH_CHECK: true
|
|
||||||
4. Build
|
|
||||||
|
|
||||||
Time: ~2 minutes
|
|
||||||
Result: ✅ SUCCESS
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Example 2: Rollback to Last Week's Version
|
|
||||||
|
|
||||||
```
|
|
||||||
Scenario: Need stable version from last week
|
|
||||||
|
|
||||||
Steps:
|
|
||||||
1. Find old build: Jenkins → Build History → #15
|
|
||||||
2. Check image tag: main-15
|
|
||||||
3. Jenkins → demo-nginx-rollback
|
|
||||||
4. IMAGE_TAG + main-15
|
|
||||||
5. DRY_RUN: true (verify first!)
|
|
||||||
6. DRY_RUN: false (execute)
|
|
||||||
|
|
||||||
Result: ✅ Rolled back to main-15
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Example 3: Rollback by Revision Number
|
|
||||||
|
|
||||||
```
|
|
||||||
Scenario: Откатить на 3 versions назад
|
|
||||||
|
|
||||||
Steps:
|
|
||||||
1. Check history:
|
|
||||||
kubectl rollout history deployment/demo-nginx -n demo-app
|
|
||||||
|
|
||||||
2. Find revision: 25 (current: 28)
|
|
||||||
|
|
||||||
3. Jenkins → demo-nginx-rollback
|
|
||||||
4. REVISION_NUMBER + 25
|
|
||||||
5. Build
|
|
||||||
|
|
||||||
Result: ✅ Rolled back to revision 25
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Example 4: Rollback by Git Commit
|
|
||||||
|
|
||||||
```
|
|
||||||
Scenario: Нужно точное состояние кода
|
|
||||||
|
|
||||||
Steps:
|
|
||||||
1. Find commit:
|
|
||||||
git log --oneline apps/demo-nginx/deployment.yaml
|
|
||||||
|
|
||||||
2. Copy SHA: abc123def
|
|
||||||
|
|
||||||
3. Jenkins → demo-nginx-rollback
|
|
||||||
4. GIT_COMMIT + abc123def
|
|
||||||
5. Build
|
|
||||||
|
|
||||||
Result: ✅ Rolled back to commit abc123def
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Manual Verification Commands
|
|
||||||
|
|
||||||
### Check Deployment Status
|
|
||||||
```bash
|
|
||||||
kubectl get deployment demo-nginx -n demo-app
|
|
||||||
|
|
||||||
# Expected:
|
|
||||||
NAME READY UP-TO-DATE AVAILABLE AGE
|
|
||||||
demo-nginx 2/2 2 2 15h
|
|
||||||
```
|
|
||||||
|
|
||||||
### Check Image Version
|
|
||||||
```bash
|
|
||||||
kubectl get deployment demo-nginx -n demo-app \
|
|
||||||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
|
||||||
|
|
||||||
# Expected: docker.io/vladcrypto/demo-nginx:main-21
|
|
||||||
```
|
|
||||||
|
|
||||||
### Check Pods
|
|
||||||
```bash
|
|
||||||
kubectl get pods -n demo-app -l app=demo-nginx
|
|
||||||
|
|
||||||
# Expected: 2 pods Running
|
|
||||||
```
|
|
||||||
|
|
||||||
### Check Rollout History
|
|
||||||
```bash
|
|
||||||
kubectl rollout history deployment/demo-nginx -n demo-app
|
|
||||||
|
|
||||||
# Shows all revisions
|
|
||||||
```
|
|
||||||
|
|
||||||
### Test Health Endpoint
|
|
||||||
```bash
|
|
||||||
POD=$(kubectl get pods -n demo-app -l app=demo-nginx -o jsonpath='{.items[0].metadata.name}')
|
|
||||||
kubectl exec $POD -n demo-app -- wget -q -O- http://localhost/health
|
|
||||||
|
|
||||||
# Expected: healthy
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Emergency Rollback Procedure
|
|
||||||
|
|
||||||
### If Production is Down
|
|
||||||
|
|
||||||
**Option 1: Jenkins (2 minutes)**
|
|
||||||
```
|
|
||||||
1. Jenkins → demo-nginx-rollback
|
|
||||||
2. IMAGE_TAG → last known good version
|
|
||||||
3. SKIP_HEALTH_CHECK: ✅ true
|
|
||||||
4. Build
|
|
||||||
```
|
|
||||||
|
|
||||||
**Option 2: kubectl (30 seconds)**
|
|
||||||
```bash
|
|
||||||
# Fastest - rollback to previous
|
|
||||||
kubectl rollout undo deployment/demo-nginx -n demo-app
|
|
||||||
|
|
||||||
# To specific revision
|
|
||||||
kubectl rollout undo deployment/demo-nginx -n demo-app --to-revision=25
|
|
||||||
```
|
|
||||||
|
|
||||||
**Option 3: ArgoCD (1 minute)**
|
|
||||||
```
|
|
||||||
1. ArgoCD UI → demo-nginx
|
|
||||||
2. History → Select previous version
|
|
||||||
3. Rollback button
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Configuration Reference
|
|
||||||
|
|
||||||
### Environment Variables
|
|
||||||
|
|
||||||
```groovy
|
|
||||||
APP_NAME = 'demo-nginx' // Deployment name
|
|
||||||
CONTAINER_NAME = 'nginx' // Container name
|
|
||||||
NAMESPACE = 'demo-app' // K8s namespace
|
|
||||||
DOCKER_REGISTRY = 'docker.io' // Registry
|
|
||||||
DOCKER_REPO = 'vladcrypto' // Docker Hub user
|
|
||||||
HEALTH_CHECK_TIMEOUT = '300s' // Rollout timeout
|
|
||||||
```
|
|
||||||
|
|
||||||
### Customization
|
|
||||||
|
|
||||||
Изменить настройки в Jenkinsfile.rollback:
|
|
||||||
|
|
||||||
```groovy
|
|
||||||
// Увеличить timeout
|
|
||||||
HEALTH_CHECK_TIMEOUT = '600s'
|
|
||||||
|
|
||||||
// Больше попыток health check
|
|
||||||
for i in 1 2 3 4 5 6 7 8 9 10; do
|
|
||||||
|
|
||||||
// Дольше ждать stabilization
|
|
||||||
sleep 30 // Instead of 10
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Monitoring & Alerts
|
|
||||||
|
|
||||||
### Grafana Dashboard
|
|
||||||
|
|
||||||
```promql
|
|
||||||
# Rollback count
|
|
||||||
sum(increase(deployment_rollback_total[1h])) by (deployment)
|
|
||||||
|
|
||||||
# Rollback rate
|
|
||||||
rate(deployment_rollback_total[5m])
|
|
||||||
|
|
||||||
# Average rollback duration
|
|
||||||
avg(deployment_rollback_duration_seconds)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Alert Rules
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
- alert: FrequentRollbacks
|
|
||||||
expr: rate(deployment_rollback_total[1h]) > 2
|
|
||||||
annotations:
|
|
||||||
summary: "Frequent rollbacks detected"
|
|
||||||
description: "More than 2 rollbacks in last hour"
|
|
||||||
|
|
||||||
- alert: RollbackFailed
|
|
||||||
expr: deployment_rollback_failed_total > 0
|
|
||||||
annotations:
|
|
||||||
summary: "Rollback failed"
|
|
||||||
description: "Manual intervention required"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Summary of All Fixes
|
|
||||||
|
|
||||||
| # | Issue | Fix | Status |
|
|
||||||
|---|-------|-----|--------|
|
|
||||||
| 1 | Container name wrong | Use `nginx` not `demo-nginx` | ✅ Fixed |
|
|
||||||
| 2 | Whitespace in input | Auto-trim with `.trim()` | ✅ Fixed |
|
|
||||||
| 3 | RBAC pods/exec | Add permission to ClusterRole | ✅ Fixed |
|
|
||||||
| 4 | Health check timing | Use `SKIP_HEALTH_CHECK=true` | ⚠️ Workaround |
|
|
||||||
| 5 | Bash loop syntax | Use explicit list `1 2 3 4 5` | ✅ Fixed |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Success Criteria
|
|
||||||
|
|
||||||
✅ **Rollback Methods:** 3/3 working (IMAGE_TAG, REVISION, GIT_COMMIT)
|
|
||||||
✅ **GitOps Sync:** Git commits automatically
|
|
||||||
✅ **Zero Downtime:** Rolling updates
|
|
||||||
✅ **RBAC:** Full permissions configured
|
|
||||||
✅ **Input Validation:** Whitespace auto-trimmed
|
|
||||||
✅ **DRY_RUN:** Safe testing mode
|
|
||||||
✅ **Retry Logic:** 5 attempts with proper bash syntax
|
|
||||||
⚠️ **Health Check:** Optional (use SKIP_HEALTH_CHECK=true)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## FAQ
|
|
||||||
|
|
||||||
### Q: Health check всегда падает, это нормально?
|
|
||||||
|
|
||||||
**A:** Да, из-за timing race condition во время rolling update. Используй `SKIP_HEALTH_CHECK: true` и проверь вручную через 30-60s.
|
|
||||||
|
|
||||||
### Q: Как откатиться на несколько версий назад?
|
|
||||||
|
|
||||||
**A:** Используй `REVISION_NUMBER` метод и укажи нужную revision из `kubectl rollout history`.
|
|
||||||
|
|
||||||
### Q: Можно ли откатить только в staging?
|
|
||||||
|
|
||||||
**A:** Да, измени `NAMESPACE` в Jenkinsfile или создай отдельный job для staging.
|
|
||||||
|
|
||||||
### Q: Как быстро откатиться в emergency?
|
|
||||||
|
|
||||||
**A:** Используй `kubectl rollout undo` (30 секунд) или Jenkins с `SKIP_HEALTH_CHECK=true` (2 минуты).
|
|
||||||
|
|
||||||
### Q: Что если Git commit fail?
|
|
||||||
|
|
||||||
**A:** Rollback всё равно произошёл в Kubernetes! Git нужен только для GitOps sync. ArgoCD пере-синкает через 3 минуты.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Related Documentation
|
|
||||||
|
|
||||||
- [CI/CD Guide](../../../CICD_GUIDE.md)
|
|
||||||
- [Automatic Rollback](../Jenkinsfile) - See `post { failure }` section
|
|
||||||
- [Jenkins RBAC](../../jenkins/rbac.yaml)
|
|
||||||
- [Deployment Manifest](../deployment.yaml)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Support
|
|
||||||
|
|
||||||
**Issues?**
|
|
||||||
- Check Jenkins console output
|
|
||||||
- Verify RBAC permissions
|
|
||||||
- Check pod status: `kubectl get pods -n demo-app`
|
|
||||||
- Review ArgoCD sync status
|
|
||||||
|
|
||||||
**Need Help?**
|
|
||||||
- Jenkins logs: Jenkins → Build → Console Output
|
|
||||||
- Kubernetes events: `kubectl get events -n demo-app`
|
|
||||||
- Pod logs: `kubectl logs -n demo-app -l app=demo-nginx`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Last Updated:** 2026-01-06
|
|
||||||
**Version:** 1.0
|
|
||||||
**Status:** Production Ready ✅
|
|
||||||
@@ -1,144 +0,0 @@
|
|||||||
# 🔄 Manual Rollback - Quick Reference
|
|
||||||
|
|
||||||
## 🚀 Quick Start (2 minutes)
|
|
||||||
|
|
||||||
```
|
|
||||||
Jenkins → demo-nginx-rollback → Build with Parameters
|
|
||||||
|
|
||||||
ROLLBACK_METHOD: IMAGE_TAG
|
|
||||||
TARGET_VERSION: main-21
|
|
||||||
SKIP_HEALTH_CHECK: true (recommended!)
|
|
||||||
DRY_RUN: false
|
|
||||||
|
|
||||||
→ Build → ✅ SUCCESS!
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📋 3 Ways to Rollback
|
|
||||||
|
|
||||||
### 1. By Image Tag (Fastest)
|
|
||||||
```
|
|
||||||
Method: IMAGE_TAG
|
|
||||||
Target: main-21
|
|
||||||
Use: When you know the build number
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. By Revision Number
|
|
||||||
```
|
|
||||||
Method: REVISION_NUMBER
|
|
||||||
Target: 2
|
|
||||||
Use: Rollback N steps back
|
|
||||||
Find: kubectl rollout history deployment/demo-nginx -n demo-app
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. By Git Commit
|
|
||||||
```
|
|
||||||
Method: GIT_COMMIT
|
|
||||||
Target: abc123def
|
|
||||||
Use: Exact code state
|
|
||||||
Find: git log --oneline apps/demo-nginx/deployment.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ⚡ Emergency Rollback (30 seconds)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Fastest - kubectl
|
|
||||||
kubectl rollout undo deployment/demo-nginx -n demo-app
|
|
||||||
|
|
||||||
# To specific revision
|
|
||||||
kubectl rollout undo deployment/demo-nginx -n demo-app --to-revision=25
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔍 Verify Rollback
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Check image
|
|
||||||
kubectl get deployment demo-nginx -n demo-app \
|
|
||||||
-o jsonpath='{.spec.template.spec.containers[0].image}'
|
|
||||||
|
|
||||||
# Check pods
|
|
||||||
kubectl get pods -n demo-app -l app=demo-nginx
|
|
||||||
|
|
||||||
# Test health
|
|
||||||
POD=$(kubectl get pods -n demo-app -l app=demo-nginx -o jsonpath='{.items[0].metadata.name}')
|
|
||||||
kubectl exec $POD -n demo-app -- wget -q -O- http://localhost/health
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ⚙️ Parameters
|
|
||||||
|
|
||||||
| Parameter | Default | Recommended |
|
|
||||||
|-----------|---------|-------------|
|
|
||||||
| ROLLBACK_METHOD | IMAGE_TAG | IMAGE_TAG |
|
|
||||||
| TARGET_VERSION | (required) | main-21 |
|
|
||||||
| SKIP_HEALTH_CHECK | false | **true** |
|
|
||||||
| DRY_RUN | false | false |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🐛 Common Issues - FIXED
|
|
||||||
|
|
||||||
| Issue | Fix | Status |
|
|
||||||
|-------|-----|--------|
|
|
||||||
| Wrong container name | Use `nginx` | ✅ Fixed |
|
|
||||||
| Whitespace in input | Auto-trim | ✅ Fixed |
|
|
||||||
| RBAC permission | Added pods/exec | ✅ Fixed |
|
|
||||||
| Health check timing | Use SKIP_HEALTH_CHECK | ⚠️ Workaround |
|
|
||||||
| Bash loop broken | Use `1 2 3 4 5` | ✅ Fixed |
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 💡 Best Practices
|
|
||||||
|
|
||||||
1. ✅ Always test with `DRY_RUN: true` first
|
|
||||||
2. ✅ Use `SKIP_HEALTH_CHECK: true` for faster rollback
|
|
||||||
3. ✅ Verify manually after rollback (30-60s wait)
|
|
||||||
4. ✅ Document rollback reason in Jenkins build comment
|
|
||||||
5. ✅ Check ArgoCD sync status after rollback
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📊 Verification Commands
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Full status check
|
|
||||||
kubectl get deployment demo-nginx -n demo-app
|
|
||||||
kubectl get pods -n demo-app -l app=demo-nginx
|
|
||||||
kubectl rollout history deployment/demo-nginx -n demo-app
|
|
||||||
|
|
||||||
# Watch pods update
|
|
||||||
watch kubectl get pods -n demo-app
|
|
||||||
|
|
||||||
# Check logs
|
|
||||||
kubectl logs -n demo-app -l app=demo-nginx --tail=50
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎯 Success Checklist
|
|
||||||
|
|
||||||
- [ ] Jenkins pipeline exists (demo-nginx-rollback)
|
|
||||||
- [ ] RBAC configured (pods/exec permission)
|
|
||||||
- [ ] Target version identified
|
|
||||||
- [ ] DRY_RUN tested
|
|
||||||
- [ ] Rollback executed
|
|
||||||
- [ ] Pods verified (Running)
|
|
||||||
- [ ] Image version confirmed
|
|
||||||
- [ ] Health check passed (manual)
|
|
||||||
- [ ] ArgoCD synced
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📚 Full Documentation
|
|
||||||
|
|
||||||
See: [apps/demo-nginx/docs/ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Quick Reference - Keep this handy! 📌**
|
|
||||||
@@ -1,107 +0,0 @@
|
|||||||
# Demo-Nginx Scaling History
|
|
||||||
|
|
||||||
This document tracks the scaling changes made to the demo-nginx deployment for capacity planning and audit purposes.
|
|
||||||
|
|
||||||
## Scaling Timeline
|
|
||||||
|
|
||||||
### 2026-01-12: Scale to 5 Replicas
|
|
||||||
**Change:** Increased from 3 to 5 replicas
|
|
||||||
**Reason:** Enhanced capacity and improved high availability
|
|
||||||
**Impact:**
|
|
||||||
- Additional 2 pod instances for better load distribution
|
|
||||||
- Improved resilience against node failures
|
|
||||||
- Better handling of traffic spikes
|
|
||||||
- Total resource allocation:
|
|
||||||
- CPU requests: 500m (5 × 100m)
|
|
||||||
- Memory requests: 640Mi (5 × 128Mi)
|
|
||||||
- CPU limits: 1000m (5 × 200m)
|
|
||||||
- Memory limits: 1280Mi (5 × 256Mi)
|
|
||||||
|
|
||||||
**Commit:** `0669ac66c8a651f74b4c16a4618db03bdd843c02`
|
|
||||||
**Deployment Method:** GitOps via ArgoCD automatic sync
|
|
||||||
|
|
||||||
### 2026-01-12: Scale to 3 Replicas
|
|
||||||
**Change:** Increased from 2 to 3 replicas
|
|
||||||
**Reason:** Initial scaling for improved availability
|
|
||||||
**Commit:** `d5ffa97159ec391de950dc2d861361074ff3eee8`
|
|
||||||
|
|
||||||
### Initial Deployment: 2 Replicas
|
|
||||||
**Change:** Initial deployment with 2 replicas
|
|
||||||
**Reason:** Baseline deployment for demo application
|
|
||||||
|
|
||||||
## Current Configuration
|
|
||||||
|
|
||||||
- **Replicas:** 5
|
|
||||||
- **Namespace:** demo-app
|
|
||||||
- **Image:** docker.io/vladcrypto/demo-nginx:main-50
|
|
||||||
- **Service Type:** ClusterIP
|
|
||||||
- **Port:** 80
|
|
||||||
- **Health Checks:** Liveness and Readiness probes enabled
|
|
||||||
- **Monitoring:** Prometheus scraping enabled
|
|
||||||
|
|
||||||
## Scaling Guidelines
|
|
||||||
|
|
||||||
### When to Scale Up
|
|
||||||
- CPU utilization consistently above 70%
|
|
||||||
- Memory utilization consistently above 75%
|
|
||||||
- Response time degradation observed
|
|
||||||
- Preparing for expected traffic increase
|
|
||||||
- Node maintenance requires pod redistribution
|
|
||||||
|
|
||||||
### When to Scale Down
|
|
||||||
- CPU utilization consistently below 30% for extended period
|
|
||||||
- Cost optimization requirements
|
|
||||||
- Application usage patterns indicate over-provisioning
|
|
||||||
|
|
||||||
## Resource Considerations
|
|
||||||
|
|
||||||
Each replica consumes:
|
|
||||||
- **CPU Request:** 100m
|
|
||||||
- **Memory Request:** 128Mi
|
|
||||||
- **CPU Limit:** 200m
|
|
||||||
- **Memory Limit:** 256Mi
|
|
||||||
|
|
||||||
Total cluster resources for 5 replicas:
|
|
||||||
- **Total CPU Requests:** 500m (0.5 cores)
|
|
||||||
- **Total Memory Requests:** 640Mi (~0.625 GB)
|
|
||||||
- **Total CPU Limits:** 1000m (1 core)
|
|
||||||
- **Total Memory Limits:** 1280Mi (~1.25 GB)
|
|
||||||
|
|
||||||
## Monitoring and Alerts
|
|
||||||
|
|
||||||
Monitor the following metrics in Grafana:
|
|
||||||
- Pod CPU/Memory utilization
|
|
||||||
- Request latency
|
|
||||||
- Error rates
|
|
||||||
- Pod restart count
|
|
||||||
- Service availability
|
|
||||||
|
|
||||||
## Rollback Procedure
|
|
||||||
|
|
||||||
If issues arise after scaling:
|
|
||||||
|
|
||||||
1. Revert the deployment.yaml in Git:
|
|
||||||
```bash
|
|
||||||
git revert <commit-hash>
|
|
||||||
git push
|
|
||||||
```
|
|
||||||
|
|
||||||
2. ArgoCD will automatically sync the change
|
|
||||||
|
|
||||||
3. Or manually scale via kubectl:
|
|
||||||
```bash
|
|
||||||
kubectl scale deployment demo-nginx -n demo-app --replicas=<previous-count>
|
|
||||||
```
|
|
||||||
|
|
||||||
## Related Documentation
|
|
||||||
|
|
||||||
- [CICD_GUIDE.md](./CICD_GUIDE.md) - CI/CD pipeline documentation
|
|
||||||
- [ROLLBACK_MANUAL.md](./ROLLBACK_MANUAL.md) - Detailed rollback procedures
|
|
||||||
- [README.md](./README.md) - General application documentation
|
|
||||||
|
|
||||||
## Notes
|
|
||||||
|
|
||||||
- All scaling operations are tracked in Git for audit trail
|
|
||||||
- ArgoCD manages automatic synchronization from Git to cluster
|
|
||||||
- Changes follow GitOps principles for declarative infrastructure
|
|
||||||
- Scaling decisions should be data-driven based on monitoring metrics
|
|
||||||
@@ -1,471 +0,0 @@
|
|||||||
# 🔄 Automatic Rollback Feature
|
|
||||||
|
|
||||||
## ✅ Что добавлено
|
|
||||||
|
|
||||||
Pipeline теперь автоматически откатывается к предыдущей версии при любой ошибке деплоя!
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎯 Как работает
|
|
||||||
|
|
||||||
### 1. **Save Current State** (перед деплоем)
|
|
||||||
```
|
|
||||||
📸 Сохраняет:
|
|
||||||
- Текущий Docker image tag
|
|
||||||
- Количество реплик
|
|
||||||
- Git commit SHA
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. **Deploy New Version**
|
|
||||||
```
|
|
||||||
🚀 Деплоит новую версию через:
|
|
||||||
- Build Docker image
|
|
||||||
- Push to registry
|
|
||||||
- Update Git manifests
|
|
||||||
- ArgoCD sync
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. **Health Checks**
|
|
||||||
```
|
|
||||||
🏥 Проверяет:
|
|
||||||
- Rollout status (timeout: 300s)
|
|
||||||
- Pod readiness (все поды Ready)
|
|
||||||
- Image version (правильный tag)
|
|
||||||
- Health endpoint (5 попыток)
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. **Auto Rollback** (при ошибке)
|
|
||||||
```
|
|
||||||
🔄 Если что-то пошло не так:
|
|
||||||
- kubectl rollout undo
|
|
||||||
- Revert Git commit
|
|
||||||
- Restore previous state
|
|
||||||
- Notify в logs
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📊 Pipeline Stages
|
|
||||||
|
|
||||||
```
|
|
||||||
┌─────────────────────────────┐
|
|
||||||
│ 1. Save Current State │ ← Сохраняет текущую версию
|
|
||||||
└─────────────┬───────────────┘
|
|
||||||
↓
|
|
||||||
┌─────────────────────────────┐
|
|
||||||
│ 2. Checkout Source │
|
|
||||||
└─────────────┬───────────────┘
|
|
||||||
↓
|
|
||||||
┌─────────────────────────────┐
|
|
||||||
│ 3. Build Docker Image │
|
|
||||||
└─────────────┬───────────────┘
|
|
||||||
↓
|
|
||||||
┌─────────────────────────────┐
|
|
||||||
│ 4. Push to Registry │
|
|
||||||
└─────────────┬───────────────┘
|
|
||||||
↓
|
|
||||||
┌─────────────────────────────┐
|
|
||||||
│ 5. Update GitOps Manifests │
|
|
||||||
└─────────────┬───────────────┘
|
|
||||||
↓
|
|
||||||
┌─────────────────────────────┐
|
|
||||||
│ 6. Wait for Deployment │ ← 300s timeout
|
|
||||||
└─────────────┬───────────────┘
|
|
||||||
↓
|
|
||||||
┌─────────────────────────────┐
|
|
||||||
│ 7. Health Check │ ← 5 retries
|
|
||||||
└─────────────┬───────────────┘
|
|
||||||
↓
|
|
||||||
┌───────┴────────┐
|
|
||||||
↓ ↓
|
|
||||||
SUCCESS FAILURE
|
|
||||||
│ │
|
|
||||||
│ ↓
|
|
||||||
│ ┌──────────────┐
|
|
||||||
│ │ ROLLBACK │ ← Автоматически!
|
|
||||||
│ └──────────────┘
|
|
||||||
↓
|
|
||||||
✅ DONE
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔧 Configuration
|
|
||||||
|
|
||||||
### Environment Variables:
|
|
||||||
|
|
||||||
```groovy
|
|
||||||
// Rollback configuration
|
|
||||||
ROLLBACK_ENABLED = 'true' // Включить/выключить rollback
|
|
||||||
DEPLOYMENT_TIMEOUT = '300s' // Timeout для rollout
|
|
||||||
HEALTH_CHECK_RETRIES = '5' // Количество попыток health check
|
|
||||||
HEALTH_CHECK_DELAY = '10' // Задержка между попытками (сек)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Изменить настройки:
|
|
||||||
|
|
||||||
```groovy
|
|
||||||
environment {
|
|
||||||
ROLLBACK_ENABLED = 'false' // Выключить rollback
|
|
||||||
DEPLOYMENT_TIMEOUT = '600s' // Увеличить timeout
|
|
||||||
HEALTH_CHECK_RETRIES = '10' // Больше попыток
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🧪 Тестирование Rollback
|
|
||||||
|
|
||||||
### Сценарий 1: Симуляция deployment failure
|
|
||||||
|
|
||||||
Измени deployment.yaml чтобы вызвать ошибку:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# apps/demo-nginx/deployment.yaml
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: nginx
|
|
||||||
image: nginx:nonexistent-tag # Несуществующий tag
|
|
||||||
```
|
|
||||||
|
|
||||||
**Результат:**
|
|
||||||
```
|
|
||||||
❌ Deployment failed
|
|
||||||
🔄 Rollback initiated automatically
|
|
||||||
✅ Rolled back to previous version
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Сценарий 2: Симуляция health check failure
|
|
||||||
|
|
||||||
Измени nginx.conf чтобы сломать /health:
|
|
||||||
|
|
||||||
```nginx
|
|
||||||
location /health {
|
|
||||||
return 500 "broken"; # Вернет 500 error
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Результат:**
|
|
||||||
```
|
|
||||||
❌ Health check failed after 5 attempts
|
|
||||||
🔄 Rollback initiated automatically
|
|
||||||
✅ Previous version restored
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### Сценарий 3: Симуляция timeout
|
|
||||||
|
|
||||||
Установи очень короткий timeout:
|
|
||||||
|
|
||||||
```groovy
|
|
||||||
DEPLOYMENT_TIMEOUT = '10s' // Слишком короткий
|
|
||||||
```
|
|
||||||
|
|
||||||
**Результат:**
|
|
||||||
```
|
|
||||||
❌ Deployment timeout exceeded
|
|
||||||
🔄 Rollback initiated automatically
|
|
||||||
✅ Rolled back successfully
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📋 Rollback Process Details
|
|
||||||
|
|
||||||
### Что происходит при rollback:
|
|
||||||
|
|
||||||
1. **Kubernetes Rollback:**
|
|
||||||
```bash
|
|
||||||
kubectl rollout undo deployment/demo-nginx -n demo-app
|
|
||||||
```
|
|
||||||
|
|
||||||
2. **Git Revert:**
|
|
||||||
```bash
|
|
||||||
git revert --no-edit HEAD
|
|
||||||
git push origin main
|
|
||||||
```
|
|
||||||
|
|
||||||
3. **ArgoCD Sync:**
|
|
||||||
```
|
|
||||||
ArgoCD автоматически применит revert commit
|
|
||||||
```
|
|
||||||
|
|
||||||
4. **Verification:**
|
|
||||||
```bash
|
|
||||||
kubectl rollout status deployment/demo-nginx -n demo-app
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔍 Как проверить что rollback сработал
|
|
||||||
|
|
||||||
### В Jenkins Console Output:
|
|
||||||
|
|
||||||
```
|
|
||||||
❌ DEPLOYMENT FAILED - INITIATING ROLLBACK!
|
|
||||||
|
|
||||||
Rolling back to previous version...
|
|
||||||
🔄 Rolling back to: docker.io/vladcrypto/demo-nginx:main-21
|
|
||||||
|
|
||||||
✅ ROLLBACK COMPLETED!
|
|
||||||
|
|
||||||
Rolled back to: docker.io/vladcrypto/demo-nginx:main-21
|
|
||||||
Current build (#22) has been reverted.
|
|
||||||
|
|
||||||
Please check logs and fix the issue before redeploying.
|
|
||||||
```
|
|
||||||
|
|
||||||
### В Kubernetes:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Check deployment history
|
|
||||||
kubectl rollout history deployment/demo-nginx -n demo-app
|
|
||||||
|
|
||||||
# Вывод:
|
|
||||||
REVISION CHANGE-CAUSE
|
|
||||||
21 Updated to main-21
|
|
||||||
22 Updated to main-22
|
|
||||||
23 Rollback to main-21 ← Rollback!
|
|
||||||
```
|
|
||||||
|
|
||||||
### В Git:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git log --oneline
|
|
||||||
|
|
||||||
# Вывод:
|
|
||||||
abc1234 Revert "chore(demo-nginx): Update image to main-22"
|
|
||||||
def5678 chore(demo-nginx): Update image to main-22
|
|
||||||
ghi9012 chore(demo-nginx): Update image to main-21
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 💡 Best Practices
|
|
||||||
|
|
||||||
### 1. **Всегда тестируй в staging**
|
|
||||||
|
|
||||||
```groovy
|
|
||||||
stage('Deploy to Staging') {
|
|
||||||
when { branch 'develop' }
|
|
||||||
steps {
|
|
||||||
// Deploy to staging namespace
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. **Мониторинг после деплоя**
|
|
||||||
|
|
||||||
```groovy
|
|
||||||
stage('Post-Deploy Monitoring') {
|
|
||||||
steps {
|
|
||||||
sh """
|
|
||||||
# Monitor for 5 minutes
|
|
||||||
for i in {1..30}; do
|
|
||||||
kubectl top pods -n demo-app
|
|
||||||
sleep 10
|
|
||||||
done
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. **Slack Notifications**
|
|
||||||
|
|
||||||
```groovy
|
|
||||||
post {
|
|
||||||
failure {
|
|
||||||
slackSend(
|
|
||||||
color: 'danger',
|
|
||||||
message: """
|
|
||||||
🚨 ROLLBACK EXECUTED!
|
|
||||||
Build: #${BUILD_NUMBER}
|
|
||||||
Rolled back to previous version
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. **Сохранение artifacts**
|
|
||||||
|
|
||||||
```groovy
|
|
||||||
post {
|
|
||||||
always {
|
|
||||||
archiveArtifacts artifacts: '/tmp/previous_*.txt', allowEmptyArchive: true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ⚠️ Important Notes
|
|
||||||
|
|
||||||
### Rollback НЕ сработает если:
|
|
||||||
|
|
||||||
1. **Нет предыдущей версии:**
|
|
||||||
```
|
|
||||||
⚠️ No previous version found - cannot rollback automatically
|
|
||||||
Manual intervention required!
|
|
||||||
```
|
|
||||||
|
|
||||||
2. **ROLLBACK_ENABLED = 'false':**
|
|
||||||
```
|
|
||||||
❌ Pipeline failed! (Rollback disabled)
|
|
||||||
```
|
|
||||||
|
|
||||||
3. **Не main branch:**
|
|
||||||
```
|
|
||||||
Rollback only works on main branch
|
|
||||||
```
|
|
||||||
|
|
||||||
### Ручной rollback:
|
|
||||||
|
|
||||||
Если автоматический rollback не сработал:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Kubernetes rollback
|
|
||||||
kubectl rollout undo deployment/demo-nginx -n demo-app
|
|
||||||
|
|
||||||
# Git revert
|
|
||||||
cd k3s-gitops
|
|
||||||
git revert HEAD
|
|
||||||
git push origin main
|
|
||||||
|
|
||||||
# Force ArgoCD sync
|
|
||||||
kubectl patch application demo-nginx -n argocd \
|
|
||||||
--type merge -p '{"operation":{"sync":{}}}'
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📊 Monitoring & Alerts
|
|
||||||
|
|
||||||
### Grafana Dashboard
|
|
||||||
|
|
||||||
Добавь панели для мониторинга rollbacks:
|
|
||||||
|
|
||||||
```promql
|
|
||||||
# Number of rollbacks
|
|
||||||
sum(rate(deployment_rollback_total[5m])) by (deployment)
|
|
||||||
|
|
||||||
# Rollback duration
|
|
||||||
histogram_quantile(0.95,
|
|
||||||
rate(deployment_rollback_duration_seconds_bucket[5m])
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Alert Rules
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
- alert: FrequentRollbacks
|
|
||||||
expr: rate(deployment_rollback_total[1h]) > 3
|
|
||||||
annotations:
|
|
||||||
summary: "Frequent rollbacks detected"
|
|
||||||
description: "More than 3 rollbacks in last hour"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎯 Advanced Features (Future)
|
|
||||||
|
|
||||||
### 1. **Canary Deployments**
|
|
||||||
|
|
||||||
```groovy
|
|
||||||
stage('Canary Deploy') {
|
|
||||||
steps {
|
|
||||||
sh """
|
|
||||||
# Deploy 10% traffic to new version
|
|
||||||
kubectl set image deployment/${APP_NAME}
|
|
||||||
${APP_NAME}=${IMAGE_TAG}
|
|
||||||
--record
|
|
||||||
kubectl scale deployment/${APP_NAME}-canary --replicas=1
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. **Blue-Green Deployments**
|
|
||||||
|
|
||||||
```groovy
|
|
||||||
stage('Blue-Green Switch') {
|
|
||||||
steps {
|
|
||||||
sh """
|
|
||||||
# Switch service to new deployment
|
|
||||||
kubectl patch service ${APP_NAME}
|
|
||||||
-p '{"spec":{"selector":{"version":"${IMAGE_TAG}"}}}'
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. **Smoke Tests**
|
|
||||||
|
|
||||||
```groovy
|
|
||||||
stage('Smoke Tests') {
|
|
||||||
steps {
|
|
||||||
sh """
|
|
||||||
# Run automated tests
|
|
||||||
curl -f http://${APP_NAME}/api/health
|
|
||||||
curl -f http://${APP_NAME}/api/status
|
|
||||||
"""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ✅ Success Criteria
|
|
||||||
|
|
||||||
Pipeline считается успешным когда:
|
|
||||||
|
|
||||||
- ✅ Docker image built
|
|
||||||
- ✅ Image pushed to registry
|
|
||||||
- ✅ Git manifests updated
|
|
||||||
- ✅ Deployment rolled out (300s timeout)
|
|
||||||
- ✅ All pods ready
|
|
||||||
- ✅ Image version matches
|
|
||||||
- ✅ Health endpoint responds (5 retries)
|
|
||||||
|
|
||||||
Pipeline откатывается если:
|
|
||||||
|
|
||||||
- ❌ Deployment timeout
|
|
||||||
- ❌ Pod not ready
|
|
||||||
- ❌ Image version mismatch
|
|
||||||
- ❌ Health check failed
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎉 Summary
|
|
||||||
|
|
||||||
**Automatic Rollback добавляет:**
|
|
||||||
|
|
||||||
✅ Безопасность деплоев
|
|
||||||
✅ Автоматическое восстановление
|
|
||||||
✅ Сохранение предыдущего состояния
|
|
||||||
✅ Git history revert
|
|
||||||
✅ Kubernetes rollback
|
|
||||||
✅ Health checks
|
|
||||||
✅ Timeout protection
|
|
||||||
|
|
||||||
**Zero manual intervention needed!** 🚀
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📝 Testing Checklist
|
|
||||||
|
|
||||||
- [ ] Normal deployment работает
|
|
||||||
- [ ] Failed deployment triggers rollback
|
|
||||||
- [ ] Previous version restored
|
|
||||||
- [ ] Git commit reverted
|
|
||||||
- [ ] Health checks work
|
|
||||||
- [ ] Timeout works
|
|
||||||
- [ ] Notifications sent
|
|
||||||
- [ ] Logs clear and helpful
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Your pipeline is now production-ready with automatic rollback! 🎉**
|
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
apiVersion: networking.k8s.io/v1
|
|
||||||
kind: Ingress
|
|
||||||
metadata:
|
|
||||||
name: demo-nginx
|
|
||||||
namespace: demo-app
|
|
||||||
labels:
|
|
||||||
app: demo-nginx
|
|
||||||
annotations:
|
|
||||||
kubernetes.io/ingress.class: traefik
|
|
||||||
cert-manager.io/cluster-issuer: letsencrypt-http
|
|
||||||
spec:
|
|
||||||
rules:
|
|
||||||
- host: demo-nginx.thedevops.dev
|
|
||||||
http:
|
|
||||||
paths:
|
|
||||||
- path: /
|
|
||||||
pathType: Prefix
|
|
||||||
backend:
|
|
||||||
service:
|
|
||||||
name: demo-nginx
|
|
||||||
port:
|
|
||||||
number: 80
|
|
||||||
tls:
|
|
||||||
- hosts:
|
|
||||||
- demo-nginx.thedevops.dev
|
|
||||||
secretName: demo-nginx-tls
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: Namespace
|
|
||||||
metadata:
|
|
||||||
name: demo-app
|
|
||||||
labels:
|
|
||||||
app: demo-nginx
|
|
||||||
Reference in New Issue
Block a user