Blue-Green vs Canary Deployments: Choosing the Right Kubernetes Strategy for Zero-Downtime Releases

strategy:
  type: RollingUpdate
  rollingUpdate:
    maxUnavailable: 1
    maxSurge: 1

# E-commerce checkout service
# v1: Prices in cents (integer)
# v2: Prices in dollars (float)
# During rolling update:
# - v1 writes: 1999 (cents)
# - v2 reads: 1999.00 (dollars!)
# - User charged $1,999 instead of $19.99

# blue-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp-blue
  labels:
    app: myapp
    version: blue
spec:
  replicas: 3
  selector:
    matchLabels:
      app: myapp
      version: blue
  template:
    metadata:
      labels:
        app: myapp
        version: blue
    spec:
      containers:
      - name: myapp
        image: myapp:v1.0.0
        ports:
        - containerPort: 8080
        env:
        - name: VERSION
          value: "blue-v1.0.0"
        resources:
          requests:
            memory: "256Mi"
            cpu: "250m"
          limits:
            memory: "512Mi"
            cpu: "500m"
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /ready
            port: 8080
          initialDelaySeconds: 5
          periodSeconds: 5

# service.yaml
apiVersion: v1
kind: Service
metadata:
  name: myapp-service
  labels:
    app: myapp
spec:
  type: LoadBalancer
  selector:
    app: myapp
    version: blue    # ← This is what we'll switch
  ports:
  - protocol: TCP
    port: 80
    targetPort: 8080

# green-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp-green
  labels:
    app: myapp
    version: green
spec:
  replicas: 3
  selector:
    matchLabels:
      app: myapp
      version: green
  template:
    metadata:
      labels:
        app: myapp
        version: green
    spec:
      containers:
      - name: myapp
        image: myapp:v2.0.0    # ← New version
        ports:
        - containerPort: 8080
        env:
        - name: VERSION
          value: "green-v2.0.0"
        resources:
          requests:
            memory: "256Mi"
            cpu: "250m"
          limits:
            memory: "512Mi"
            cpu: "500m"
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /ready
            port: 8080
          initialDelaySeconds: 5
          periodSeconds: 5

# Deploy green
kubectl apply -f green-deployment.yaml

# Wait for pods to be ready
kubectl wait --for=condition=ready pod \
  -l app=myapp,version=green \
  --timeout=300s

# Create temporary service to test green
kubectl expose deployment myapp-green \
  --name=myapp-green-test \
  --port=80 \
  --target-port=8080 \
  --type=LoadBalancer

# Get green service IP
GREEN_IP=$(kubectl get svc myapp-green-test \
  -o jsonpath='{.status.loadBalancer.ingress[0].ip}')

# Run smoke tests
curl http://$GREEN_IP/health
curl http://$GREEN_IP/api/status

# Run integration tests
npm run test:integration -- --baseUrl=http://$GREEN_IP

# Load test (optional but recommended)
k6 run --vus 100 --duration 2m loadtest.js

# Method 1: Update service selector (instant switch)
kubectl patch service myapp-service \
  -p '{"spec":{"selector":{"version":"green"}}}'

# Verify traffic switched
kubectl get service myapp-service -o yaml | grep version

# Method 2: Using kubectl (more verbose)
kubectl set selector service myapp-service \
  'app=myapp,version=green'

# Watch error rates for 5 minutes
watch -n 5 'kubectl top pods -l version=green'

# If issues detected, instant rollback
kubectl patch service myapp-service \
  -p '{"spec":{"selector":{"version":"blue"}}}'

# Rollback completes in <10 seconds

# After green proves stable (usually 24-48 hours)
kubectl delete deployment myapp-blue
kubectl delete service myapp-green-test

#!/bin/bash
# blue-green-deploy.sh

set -e

APP_NAME="myapp"
NEW_VERSION="$1"
CURRENT_COLOR=$(kubectl get service ${APP_NAME}-service \
  -o jsonpath='{.spec.selector.version}')

if [ "$CURRENT_COLOR" = "blue" ]; then
  NEW_COLOR="green"
else
  NEW_COLOR="blue"
fi

echo "🚀 Deploying ${APP_NAME}:${NEW_VERSION} to ${NEW_COLOR}"

# Step 1: Deploy new version
sed "s/VERSION_PLACEHOLDER/${NEW_VERSION}/g" \
  deployment-template.yaml | \
  sed "s/COLOR_PLACEHOLDER/${NEW_COLOR}/g" | \
  kubectl apply -f -

# Step 2: Wait for rollout
echo "⏳ Waiting for ${NEW_COLOR} pods to be ready..."
kubectl rollout status deployment/${APP_NAME}-${NEW_COLOR} \
  --timeout=5m

# Step 3: Run smoke tests
echo "🧪 Running smoke tests..."
NEW_COLOR_IP=$(kubectl get pods \
  -l app=${APP_NAME},version=${NEW_COLOR} \
  -o jsonpath='{.items[0].status.podIP}')

if curl -f http://${NEW_COLOR_IP}:8080/health; then
  echo "✅ Smoke tests passed"
else
  echo "❌ Smoke tests failed, aborting deployment"
  kubectl delete deployment ${APP_NAME}-${NEW_COLOR}
  exit 1
fi

# Step 4: Switch traffic
echo "🔄 Switching traffic to ${NEW_COLOR}..."
kubectl patch service ${APP_NAME}-service \
  -p "{\"spec\":{\"selector\":{\"version\":\"${NEW_COLOR}\"}}}"

# Step 5: Monitor
echo "📊 Monitoring new deployment for 2 minutes..."
sleep 120

# Step 6: Check error rates
ERROR_RATE=$(kubectl logs -l version=${NEW_COLOR} --tail=1000 | \
  grep ERROR | wc -l)

if [ $ERROR_RATE -gt 10 ]; then
  echo "❌ High error rate detected, rolling back!"
  kubectl patch service ${APP_NAME}-service \
    -p "{\"spec\":{\"selector\":{\"version\":\"${CURRENT_COLOR}\"}}}"
  exit 1
fi

echo "✅ Deployment successful!"
echo "💡 Keep ${CURRENT_COLOR} running for quick rollback"
echo "🗑️  Delete old deployment with: kubectl delete deployment ${APP_NAME}-${CURRENT_COLOR}"

chmod +x blue-green-deploy.sh
./blue-green-deploy.sh v2.1.0

-- Migration 1 (deployed BEFORE green)
-- Add new column without breaking old code
ALTER TABLE users ADD COLUMN email_verified BOOLEAN DEFAULT FALSE;

-- Migration 2 (deployed AFTER blue is terminated)
-- Now safe to remove old column
ALTER TABLE users DROP COLUMN old_verified_flag;

# Use version-specific resources
env:
- name: QUEUE_NAME
  value: "orders-{{ .Values.version }}"  # orders-blue or orders-green

# Add automatic cleanup after 48 hours
kubectl annotate deployment myapp-blue \
  "cleanup-after=48h"

# CronJob to clean old deployments
kubectl create cronjob cleanup-old-deployments \
  --schedule="0 */6 * * *" \
  --image=bitnami/kubectl \
  -- /bin/sh -c "kubectl delete deployments \
  -l cleanup-after!=null \
  --field-selector='metadata.creationTimestamp<$(date -d '48 hours ago' -u +%Y-%m-%dT%H:%M:%SZ)'"

# stable-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp-stable
spec:
  replicas: 9  # 90% of capacity
  selector:
    matchLabels:
      app: myapp
      track: stable
  template:
    metadata:
      labels:
        app: myapp
        track: stable
        version: v1.0.0
    spec:
      containers:
      - name: myapp
        image: myapp:v1.0.0
        ports:
        - containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:
  name: myapp-stable
spec:
  selector:
    app: myapp
    track: stable
  ports:
  - port: 80
    targetPort: 8080

# canary-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp-canary
spec:
  replicas: 1  # 10% of capacity initially
  selector:
    matchLabels:
      app: myapp
      track: canary
  template:
    metadata:
      labels:
        app: myapp
        track: canary
        version: v2.0.0
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "9090"
    spec:
      containers:
      - name: myapp
        image: myapp:v2.0.0
        ports:
        - containerPort: 8080
        - containerPort: 9090  # Metrics port
---
apiVersion: v1
kind: Service
metadata:
  name: myapp-canary
spec:
  selector:
    app: myapp
    track: canary
  ports:
  - port: 80
    targetPort: 8080

# ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: myapp-ingress
  annotations:
    nginx.ingress.kubernetes.io/canary: "true"
    nginx.ingress.kubernetes.io/canary-weight: "10"  # 10% to canary
    nginx.ingress.kubernetes.io/canary-by-header: "X-Canary"
    nginx.ingress.kubernetes.io/canary-by-header-value: "always"
spec:
  ingressClassName: nginx
  rules:
  - host: myapp.example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: myapp-canary
            port:
              number: 80
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: myapp-ingress-stable
spec:
  ingressClassName: nginx
  rules:
  - host: myapp.example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: myapp-stable
            port:
              number: 80

#!/bin/bash
# canary-rollout.sh

set -e

STABLE_REPLICAS=9
CANARY_REPLICAS=1
CANARY_WEIGHTS=(10 25 50 75 100)
MONITOR_DURATION=600  # 10 minutes per phase

deploy_canary() {
  local weight=$1
  local replicas=$2

  echo "🐤 Rolling out canary at ${weight}% (${replicas} replicas)"

  # Update ingress weight
  kubectl patch ingress myapp-ingress \
    -p "{\"metadata\":{\"annotations\":{\"nginx.ingress.kubernetes.io/canary-weight\":\"${weight}\"}}}"

  # Scale canary replicas
  kubectl scale deployment myapp-canary --replicas=${replicas}

  # Wait for pods
  kubectl wait --for=condition=ready pod \
    -l app=myapp,track=canary \
    --timeout=300s
}

check_metrics() {
  echo "📊 Monitoring metrics..."

  # Query Prometheus for error rate
  ERROR_RATE=$(curl -s 'http://prometheus:9090/api/v1/query' \
    --data-urlencode 'query=rate(http_requests_total{status=~"5.."}[5m])' | \
    jq -r '.data.result[0].value[1]')

  # Query for latency
  P95_LATENCY=$(curl -s 'http://prometheus:9090/api/v1/query' \
    --data-urlencode 'query=histogram_quantile(0.95, http_request_duration_seconds)' | \
    jq -r '.data.result[0].value[1]')

  echo "  Error rate: ${ERROR_RATE}"
  echo "  P95 latency: ${P95_LATENCY}s"

  # Thresholds
  if (( $(echo "$ERROR_RATE > 0.01" | bc -l) )); then
    echo "❌ Error rate too high!"
    return 1
  fi

  if (( $(echo "$P95_LATENCY > 1.0" | bc -l) )); then
    echo "❌ Latency too high!"
    return 1
  fi

  echo "✅ Metrics within acceptable range"
  return 0
}

rollback() {
  echo "🚨 ROLLBACK INITIATED!"

  # Set canary weight to 0
  kubectl patch ingress myapp-ingress \
    -p '{"metadata":{"annotations":{"nginx.ingress.kubernetes.io/canary-weight":"0"}}}'

  # Scale down canary
  kubectl scale deployment myapp-canary --replicas=0

  echo "✅ Rollback complete, all traffic on stable version"
  exit 1
}

# Main rollout loop
for i in "${!CANARY_WEIGHTS[@]}"; do
  weight=${CANARY_WEIGHTS[$i]}
  replicas=$(( STABLE_REPLICAS * weight / 100 ))

  deploy_canary $weight $replicas

  # Monitor for specified duration
  echo "⏳ Monitoring for $(($MONITOR_DURATION / 60)) minutes..."
  sleep 60  # Initial stabilization

  for j in $(seq 1 $((MONITOR_DURATION / 60))); do
    if ! check_metrics; then
      rollback
    fi
    sleep 60
  done

  echo "✅ Phase ${i} successful, proceeding to next phase"
done

# Deployment successful, terminate stable
echo "🎉 Canary deployment successful!"
echo "🗑️  Terminating stable deployment..."
kubectl delete deployment myapp-stable
kubectl delete service myapp-stable
kubectl delete ingress myapp-ingress-stable

# Promote canary to stable
kubectl patch deployment myapp-canary \
  -p '{"metadata":{"name":"myapp-stable"},"spec":{"selector":{"matchLabels":{"track":"stable"}},"template":{"metadata":{"labels":{"track":"stable"}}}}}'

echo "✅ Deployment complete!"

chmod +x canary-rollout.sh
./canary-rollout.sh

kubectl create namespace argo-rollouts
kubectl apply -n argo-rollouts -f \
  https://github.com/argoproj/argo-rollouts/releases/latest/download/install.yaml

# Install kubectl plugin
curl -LO https://github.com/argoproj/argo-rollouts/releases/latest/download/kubectl-argo-rollouts-linux-amd64
chmod +x kubectl-argo-rollouts-linux-amd64
sudo mv kubectl-argo-rollouts-linux-amd64 /usr/local/bin/kubectl-argo-rollouts

# rollout.yaml
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: myapp
spec:
  replicas: 10
  strategy:
    canary:
      steps:
      - setWeight: 10
      - pause: {duration: 10m}
      - setWeight: 25
      - pause: {duration: 10m}
      - setWeight: 50
      - pause: {duration: 10m}
      - setWeight: 75
      - pause: {duration: 5m}

      # Automatic analysis
      analysis:
        templates:
        - templateName: success-rate
        startingStep: 2
        args:
        - name: service-name
          value: myapp-canary

      # Automatic rollback on failure
      trafficRouting:
        nginx:
          stableIngress: myapp-ingress-stable
          annotationPrefix: nginx.ingress.kubernetes.io
          additionalIngressAnnotations:
            canary-by-header: X-Canary
            canary-by-header-value: always

  revisionHistoryLimit: 2
  selector:
    matchLabels:
      app: myapp
  template:
    metadata:
      labels:
        app: myapp
    spec:
      containers:
      - name: myapp
        image: myapp:v2.0.0
        ports:
        - containerPort: 8080
          name: http
        resources:
          requests:
            memory: 256Mi
            cpu: 250m
          limits:
            memory: 512Mi
            cpu: 500m
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /ready
            port: 8080
          initialDelaySeconds: 5
          periodSeconds: 5

# analysis-template.yaml
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: success-rate
spec:
  args:
  - name: service-name

  metrics:
  - name: success-rate
    interval: 1m
    successCondition: result[0] >= 0.95
    failureLimit: 3
    provider:
      prometheus:
        address: http://prometheus.monitoring:9090
        query: |
          sum(rate(
            http_requests_total{
              service="{{args.service-name}}",
              status!~"5.."
            }[5m]
          )) /
          sum(rate(
            http_requests_total{
              service="{{args.service-name}}"
            }[5m]
          ))

  - name: latency
    interval: 1m
    successCondition: result[0] <= 1.0
    failureLimit: 3
    provider:
      prometheus:
        address: http://prometheus.monitoring:9090
        query: |
          histogram_quantile(0.95,
            rate(http_request_duration_seconds_bucket{
              service="{{args.service-name}}"
            }[5m])
          )

# Deploy rollout
kubectl apply -f rollout.yaml
kubectl apply -f analysis-template.yaml

# Watch rollout progress
kubectl argo rollouts get rollout myapp --watch

# Promote manually (skip pauses)
kubectl argo rollouts promote myapp

# Abort rollout if issues detected
kubectl argo rollouts abort myapp

# Check rollout status
kubectl argo rollouts status myapp

# Monitor these key metrics
- Error rate (target: <1%)
- Latency p50, p95, p99 (target: <500ms)
- Success rate (target: >99%)
- CPU/Memory usage
- Database query time
- External API call success rate
- User session errors

# Calculate minimum required traffic
def min_sample_size(baseline_rate, detectable_change, confidence=0.95):
    # For 1% baseline error rate
    # Detect 0.5% increase
    # 95% confidence
    # Need ~15,000 requests

    # Formula: n = (Z^2 * p * (1-p)) / E^2
    import math
    z = 1.96  # 95% confidence
    p = baseline_rate
    e = detectable_change
    return math.ceil((z**2 * p * (1-p)) / e**2)

# Example
print(min_sample_size(0.01, 0.005))  # ~15,000 requests

# Configure session affinity properly
apiVersion: v1
kind: Service
metadata:
  name: myapp
spec:
  sessionAffinity: None  # Disable sticky sessions for canary
  sessionAffinityConfig:
    clientIP:
      timeoutSeconds: 0

apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: myapp-bluegreen
spec:
  replicas: 3
  strategy:
    blueGreen:
      activeService: myapp-active
      previewService: myapp-preview
      autoPromotionEnabled: false
      scaleDownDelaySeconds: 30
      prePromotionAnalysis:
        templates:
        - templateName: smoke-tests
      postPromotionAnalysis:
        templates:
        - templateName: load-tests

  selector:
    matchLabels:
      app: myapp
  template:
    metadata:
      labels:
        app: myapp
    spec:
      containers:
      - name: myapp
        image: myapp:v2.0.0

apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: myapp-ab-test
spec:
  replicas: 10
  strategy:
    canary:
      trafficRouting:
        managedRoutes:
        - name: header-route-1
      steps:
      - setHeaderRoute:
          name: header-route-1
          match:
          - headerName: X-Version
            headerValue:
              exact: beta
      - pause: {}

      - setWeight: 50  # 50/50 split
      - pause: {duration: 1h}

      - analysis:
          templates:
          - templateName: ab-test-analysis
          args:
          - name: variant-a
            value: stable
          - name: variant-b
            value: canary

apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: business-metrics
spec:
  metrics:
  - name: conversion-rate
    interval: 5m
    successCondition: result >= 0.15
    failureLimit: 2
    provider:
      job:
        spec:
          template:
            spec:
              containers:
              - name: check-conversion
                image: myapp-metrics:latest
                command:
                - /bin/sh
                - -c
                - |
                  # Query analytics API
                  RATE=$(curl -s https://analytics/api/conversion-rate?version=canary)
                  echo $RATE
              restartPolicy: Never

  - name: revenue-per-user
    interval: 5m
    successCondition: result[0] >= 10.0
    provider:
      prometheus:
        address: http://prometheus:9090
        query: |
          sum(rate(revenue_total{version="canary"}[5m])) /
          sum(rate(active_users{version="canary"}[5m]))

# Netflix's approach (simplified)
- Canary to 1% of users in single AWS region
- Monitor for 30 minutes
- Expand to 10% across multiple regions
- Monitor for 1 hour
- If successful: Full rollout
- If issues: Automatic rollback in <60 seconds

# Canary deployment with experimentation
- 50/50 traffic split
- Track conversion metrics per variant
- Bayesian analysis for significance
- Automatic winner promotion after statistical confidence

# Latency
- p50_latency_ms
- p95_latency_ms
- p99_latency_ms

# Traffic
- requests_per_second
- active_connections

# Errors
- error_rate_5xx
- error_rate_4xx
- timeout_rate

# Saturation
- cpu_usage_percent
- memory_usage_percent
- disk_io_usage

# Revenue
- revenue_per_minute
- conversion_rate
- cart_abandonment_rate

# User Experience
- page_load_time
- time_to_interactive
- bounce_rate

# Engagement
- session_duration
- feature_usage_count
- user_retention_rate

# Error rate comparison (canary vs stable)
(
  sum(rate(http_requests_total{version="canary",status=~"5.."}[5m]))
  /
  sum(rate(http_requests_total{version="canary"}[5m]))
)
-
(
  sum(rate(http_requests_total{version="stable",status=~"5.."}[5m]))
  /
  sum(rate(http_requests_total{version="stable"}[5m]))
)

# Latency degradation
histogram_quantile(0.95,
  rate(http_request_duration_seconds_bucket{version="canary"}[5m])
)
-
histogram_quantile(0.95,
  rate(http_request_duration_seconds_bucket{version="stable"}[5m])
)

# Memory leak detection
rate(container_memory_usage_bytes{pod=~"myapp-canary.*"}[30m])

# prometheus-alerts.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-alerts
data:
  alerts.yml: |
    groups:
    - name: deployment
      interval: 30s
      rules:
      - alert: CanaryHighErrorRate
        expr: |
          (sum(rate(http_requests_total{version="canary",status=~"5.."}[5m]))
          / sum(rate(http_requests_total{version="canary"}[5m]))) > 0.01
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Canary error rate above 1%"
          description: "Automatic rollback recommended"

      - alert: CanaryLatencyDegradation
        expr: |
          histogram_quantile(0.95,
            rate(http_request_duration_seconds_bucket{version="canary"}[5m])
          ) > 1.0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Canary p95 latency above 1s"

      - alert: CanaryMemoryLeak
        expr: |
          rate(container_memory_usage_bytes{pod=~"myapp-canary.*"}[30m]) > 0
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "Memory usage continuously increasing"

#!/bin/bash
# instant-rollback.sh

# Detect current active version
CURRENT=$(kubectl get service myapp-service \
  -o jsonpath='{.spec.selector.version}')

if [ "$CURRENT" = "blue" ]; then
  ROLLBACK_TO="green"
else
  ROLLBACK_TO="blue"
fi

echo "🚨 Rolling back from $CURRENT to $ROLLBACK_TO"

# Switch traffic instantly
kubectl patch service myapp-service \
  -p "{\"spec\":{\"selector\":{\"version\":\"${ROLLBACK_TO}\"}}}"

# Verify
sleep 5
NEW_VERSION=$(kubectl get service myapp-service \
  -o jsonpath='{.spec.selector.version}')

if [ "$NEW_VERSION" = "$ROLLBACK_TO" ]; then
  echo "✅ Rollback successful"
  exit 0
else
  echo "❌ Rollback failed!"
  exit 1
fi

#!/bin/bash
# progressive-rollback.sh

echo "🚨 Initiating canary rollback"

# Gradually reduce canary traffic
for weight in 50 25 10 0; do
  echo "Setting canary weight to ${weight}%"
  kubectl patch ingress myapp-ingress \
    -p "{\"metadata\":{\"annotations\":{\"nginx.ingress.kubernetes.io/canary-weight\":\"${weight}\"}}}"

  sleep 30  # Let traffic stabilize

  # Check if rollback resolved issues
  ERROR_RATE=$(curl -s 'http://prometheus:9090/api/v1/query' \
    --data-urlencode 'query=rate(http_requests_total{status=~"5.."}[2m])' | \
    jq -r '.data.result[0].value[1]')

  echo "Current error rate: ${ERROR_RATE}"
done

# Scale down canary
kubectl scale deployment myapp-canary --replicas=0

echo "✅ Rollback complete"

apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: myapp-auto-rollback
spec:
  strategy:
    canary:
      steps:
      - setWeight: 20
      - pause: {duration: 5m}

      analysis:
        templates:
        - templateName: auto-rollback-analysis

        # Automatic rollback configuration
        startingStep: 1
        args:
        - name: service-name
          value: myapp-canary

      # Rollback on analysis failure
      abortScaleDownDelaySeconds: 30

---
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: auto-rollback-analysis
spec:
  metrics:
  - name: error-rate-check
    interval: 1m
    successCondition: result[0] < 0.01
    failureLimit: 3  # Rollback after 3 consecutive failures
    provider:
      prometheus:
        address: http://prometheus:9090
        query: |
          sum(rate(http_requests_total{
            service="{{args.service-name}}",
            status=~"5.."
          }[5m])) /
          sum(rate(http_requests_total{
            service="{{args.service-name}}"
          }[5m]))

-- Developer runs migration on Friday evening
ALTER TABLE users DROP COLUMN old_email;

-- Blue-Green switch happens
-- Old version (blue) still running, expects old_email column
-- Application crashes: ERROR column "old_email" does not exist
-- Weekend ruined, emergency rollback, angry customers

-- PHASE 1: EXPAND (Week 1)
-- Add new column, both versions can work
ALTER TABLE users ADD COLUMN email_verified BOOLEAN DEFAULT FALSE;

-- Backfill existing data
UPDATE users SET email_verified = (old_verified_flag = 1) WHERE email_verified IS NULL;

-- Deploy v2 that reads from BOTH columns (prefers new, falls back to old)

# Application code v2 (backward compatible)
def get_user_verification(user):
    # Try new column first
    if user.email_verified is not None:
        return user.email_verified
    # Fall back to old column
    return user.old_verified_flag == 1

-- PHASE 2: MIGRATE (Week 2)
-- Switch all writes to new column
-- Deploy v3 that writes to new column only

-- Ensure all data migrated
UPDATE users SET email_verified = (old_verified_flag = 1)
WHERE email_verified IS NULL;

-- PHASE 3: CONTRACT (Week 3+)
-- After old version completely terminated
-- Now safe to remove old column
ALTER TABLE users DROP COLUMN old_verified_flag;

# redis-session-store.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: redis-session
spec:
  replicas: 3
  selector:
    matchLabels:
      app: redis-session
  template:
    metadata:
      labels:
        app: redis-session
    spec:
      containers:
      - name: redis
        image: redis:7-alpine
        ports:
        - containerPort: 6379
        volumeMounts:
        - name: redis-data
          mountPath: /data
        resources:
          requests:
            memory: "256Mi"
            cpu: "250m"
      volumes:
      - name: redis-data
        persistentVolumeClaim:
          claimName: redis-pvc

# Application configuration
import redis
from flask_session import Session

app.config['SESSION_TYPE'] = 'redis'
app.config['SESSION_REDIS'] = redis.from_url('redis://redis-session:6379')
app.config['SESSION_PERMANENT'] = False
app.config['SESSION_USE_SIGNER'] = True
Session(app)

# No server-side session needed
from flask_jwt_extended import create_access_token, jwt_required

@app.route('/login', methods=['POST'])
def login():
    token = create_access_token(identity=user.id, expires_delta=timedelta(hours=2))
    return {'token': token}

@app.route('/protected', methods=['GET'])
@jwt_required()
def protected():
    current_user = get_jwt_identity()
    return {'user_id': current_user}

# Only if you can't externalize state
apiVersion: v1
kind: Service
metadata:
  name: myapp
spec:
  sessionAffinity: ClientIP
  sessionAffinityConfig:
    clientIP:
      timeoutSeconds: 10800  # 3 hours
  selector:
    app: myapp

# Proper monitoring windows
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: myapp-proper-monitoring
spec:
  strategy:
    canary:
      steps:
      # Phase 1: Initial canary
      - setWeight: 5
      - pause: {duration: 10m}  # Short window for crash bugs

      # Phase 2: Expand slowly
      - setWeight: 10
      - pause: {duration: 15m}  # Medium window for memory leaks

      # Phase 3: More confidence
      - setWeight: 25
      - pause: {duration: 20m}  # Longer window for connection leaks

      # Phase 4: Nearly there
      - setWeight: 50
      - pause: {duration: 30m}  # Full validation before 100%

      # Phase 5: Final rollout
      - setWeight: 100

      analysis:
        templates:
        - templateName: slow-leak-detection

apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: slow-leak-detection
spec:
  metrics:
  # Detect memory leaks
  - name: memory-growth-rate
    interval: 2m
    successCondition: result[0] < 5  # Less than 5MB/min growth
    failureLimit: 3
    provider:
      prometheus:
        address: http://prometheus:9090
        query: |
          rate(container_memory_usage_bytes{pod=~"myapp-canary.*"}[5m]) / 1024 / 1024

  # Detect connection pool exhaustion
  - name: connection-pool-usage
    interval: 2m
    successCondition: result[0] < 0.80  # Less than 80% pool usage
    failureLimit: 3
    provider:
      prometheus:
        address: http://prometheus:9090
        query: |
          sum(database_connection_pool_active{version="canary"}) /
          sum(database_connection_pool_max{version="canary"})

  # Detect goroutine/thread leaks
  - name: goroutine-count
    interval: 2m
    successCondition: result[0] < 10000
    failureLimit: 3
    provider:
      prometheus:
        address: http://prometheus:9090
        query: |
          go_goroutines{pod=~"myapp-canary.*"}

# Production is on fire, engineer panics
$ kubectl get deployments
# "Wait, which one is production?"

$ kubectl rollout undo deployment/myapp
error: no rollout history found

# Tries to remember the old image tag
$ kubectl set image deployment/myapp myapp=myapp:v1.2.3
# "Was it v1.2.3 or v1.2.4?"

# 15 minutes wasted while site is down

# Emergency Rollback Playbook

## 🚨 STOP AND READ THIS FIRST

**Before you rollback:**
1. Check #incidents Slack channel - is someone already handling this?
2. Announce in #engineering: "Rolling back myapp deployment"
3. Note the incident time and symptoms

## Quick Status Check

```bash
# What version is currently deployed?
kubectl get deployment myapp -o jsonpath='{.spec.template.spec.containers[0].image}'

# What's the error rate?
curl -s 'http://prometheus:9090/api/v1/query?query=rate(http_errors_total[5m])' | jq

# Abort current rollout immediately
kubectl argo rollouts abort myapp

# Verify rollback
kubectl argo rollouts status myapp
# Should show "Degraded" status, traffic back to stable

# Expected time: 10-30 seconds

# Get current active version
CURRENT=$(kubectl get service myapp-service -o jsonpath='{.spec.selector.version}')
echo "Current version: $CURRENT"

# Switch to other version
if [ "$CURRENT" = "blue" ]; then
  kubectl patch service myapp-service -p '{"spec":{"selector":{"version":"green"}}}'
else
  kubectl patch service myapp-service -p '{"spec":{"selector":{"version":"blue"}}}'
fi

# Verify traffic switched
kubectl get service myapp-service -o yaml | grep version

# Expected time: <10 seconds

# Show rollout history
kubectl rollout history deployment/myapp

# Rollback to previous version
kubectl rollout undo deployment/myapp

# Rollback to specific revision
kubectl rollout undo deployment/myapp --to-revision=3

# Watch rollback progress
kubectl rollout status deployment/myapp

# Expected time: 2-5 minutes

# Known good versions (update after each successful deploy)
# v2.1.0 - 2025-10-28 - Last known good
# v2.0.5 - 2025-10-25 - Stable
# v2.0.3 - 2025-10-20 - Stable

# Rollback to known good version
kubectl set image deployment/myapp myapp=myapp:v2.1.0

# Wait for rollout
kubectl rollout status deployment/myapp --timeout=5m

# Expected time: 3-7 minutes

# 1. Check error rate (should drop immediately)
watch -n 5 'curl -s "http://prometheus:9090/api/v1/query?query=rate(http_errors_total[2m])"'

# 2. Check pod status
kubectl get pods -l app=myapp

# 3. Sample health check
kubectl get pods -l app=myapp -o jsonpath='{.items[0].metadata.name}' | \
  xargs -I {} kubectl exec {} -- curl -s localhost:8080/health

# 4. Check recent logs for errors
kubectl logs -l app=myapp --tail=50 | grep ERROR

chmod +x scripts/emergency-rollback.sh

# Test in staging first!
./scripts/emergency-rollback.sh

# deployment-policy.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: deployment-policy
  namespace: production
data:
  policy.json: |
    {
      "allowed_windows": [
        {
          "days": ["Monday", "Tuesday", "Wednesday", "Thursday"],
          "hours": "02:00-06:00",
          "timezone": "America/New_York"
        },
        {
          "days": ["Friday"],
          "hours": "01:00-04:00",
          "timezone": "America/New_York",
          "approval_required": true
        }
      ],
      "blocked_dates": [
        "2025-11-24",  # Black Friday
        "2025-11-25",  # Black Friday weekend
        "2025-12-24",  # Christmas Eve
        "2025-12-25",  # Christmas
        "2025-12-31",  # New Year's Eve
        "2026-01-01"   # New Year's Day
      ],
      "traffic_threshold": {
        "max_requests_per_second": 1000,
        "action": "block_deployment"
      }
    }

#!/bin/bash
# scripts/validate-deployment-window.sh

set -e

CONFIG_FILE="/etc/deployment-policy/policy.json"
CURRENT_DAY=$(date +%A)
CURRENT_HOUR=$(date +%H)
CURRENT_DATE=$(date +%Y-%m-%d)

echo "🔍 Validating deployment window..."
echo "Current time: $(date)"

# Check if today is blocked
BLOCKED_DATES=$(jq -r '.blocked_dates[]' $CONFIG_FILE)
if echo "$BLOCKED_DATES" | grep -q "$CURRENT_DATE"; then
  echo "❌ DEPLOYMENT BLOCKED"
  echo "Reason: Today ($CURRENT_DATE) is a blocked date"
  echo "Blocked dates include major holidays and high-traffic events"
  echo ""
  echo "Override required from: engineering-lead"
  exit 1
fi

# Check allowed windows
ALLOWED=$(jq -r --arg day "$CURRENT_DAY" \
  '.allowed_windows[] | select(.days[] == $day) | .hours' \
  $CONFIG_FILE | head -1)

if [ -z "$ALLOWED" ]; then
  echo "❌ DEPLOYMENT BLOCKED"
  echo "Reason: No deployment window configured for $CURRENT_DAY"
  exit 1
fi

START_HOUR=$(echo $ALLOWED | cut -d'-' -f1 | cut -d':' -f1)
END_HOUR=$(echo $ALLOWED | cut -d'-' -f2 | cut -d':' -f1)

if [ $CURRENT_HOUR -lt $START_HOUR ] || [ $CURRENT_HOUR -ge $END_HOUR ]; then
  echo "❌ DEPLOYMENT BLOCKED"
  echo "Reason: Outside allowed deployment window"
  echo "Current hour: ${CURRENT_HOUR}:00"
  echo "Allowed window: ${ALLOWED}"
  echo ""
  echo "💡 Tip: Schedule deployment for tomorrow ${START_HOUR}:00"
  exit 1
fi

# Check current traffic
CURRENT_RPS=$(curl -s 'http://prometheus:9090/api/v1/query?query=rate(http_requests_total[5m])' | \
  jq -r '.data.result[0].value[1]' | cut -d'.' -f1)

MAX_RPS=$(jq -r '.traffic_threshold.max_requests_per_second' $CONFIG_FILE)

if [ "$CURRENT_RPS" -gt "$MAX_RPS" ]; then
  echo "⚠️  WARNING: High traffic detected"
  echo "Current: ${CURRENT_RPS} req/s"
  echo "Threshold: ${MAX_RPS} req/s"
  echo ""
  read -p "Continue anyway? (yes/no): " CONFIRM
  if [ "$CONFIRM" != "yes" ]; then
    echo "❌ Deployment cancelled"
    exit 1
  fi
fi

echo "✅ Deployment window validated"
echo "You are clear to deploy"
exit 0

# .github/workflows/deploy.yml
name: Production Deployment

on:
  push:
    branches: [main]

jobs:
  validate-window:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3

    - name: Check deployment window
      run: |
        # Download policy
        kubectl get configmap deployment-policy -n production \
          -o jsonpath='{.data.policy\.json}' > /tmp/policy.json

        # Run validation
        bash scripts/validate-deployment-window.sh

  deploy:
    needs: validate-window
    runs-on: ubuntu-latest
    steps:
    - name: Deploy to production
      run: |
        kubectl apply -f k8s/production/

#!/bin/bash
# scripts/emergency-override-deploy.sh

echo "🚨 EMERGENCY DEPLOYMENT OVERRIDE"
echo "================================"
echo ""
echo "This bypasses normal deployment windows."
echo "Only use for critical production issues."
echo ""

read -p "Incident ticket number: " TICKET
read -p "Approving manager: " MANAGER
read -p "Reason for override: " REASON

echo ""
echo "Override details:"
echo "  Ticket: $TICKET"
echo "  Approved by: $MANAGER"
echo "  Reason: $REASON"
echo ""

read -p "Confirm emergency deployment? (type EMERGENCY): " CONFIRM

if [ "$CONFIRM" != "EMERGENCY" ]; then
  echo "❌ Override cancelled"
  exit 1
fi

# Log override
echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) | EMERGENCY OVERRIDE | $TICKET | $MANAGER | $REASON" \
  >> /var/log/deployment-overrides.log

# Slack notification
curl -X POST $SLACK_WEBHOOK_URL \
  -H 'Content-Type: application/json' \
  -d "{
    \"text\": \"🚨 Emergency deployment override\",
    \"attachments\": [{
      \"color\": \"danger\",
      \"fields\": [
        {\"title\": \"Ticket\", \"value\": \"$TICKET\"},
        {\"title\": \"Approved by\", \"value\": \"$MANAGER\"},
        {\"title\": \"Reason\", \"value\": \"$REASON\"}
      ]
    }]
  }"

# Proceed with deployment
echo "✅ Override logged, proceeding with deployment..."
exec ./scripts/deploy.sh

# Example organization strategy matrix
Services:
  payment-service:
    strategy: blue-green
    reason: "Zero tolerance for errors, needs instant rollback"
    deploy_frequency: "Weekly"

  user-profile-api:
    strategy: canary
    reason: "High traffic, frequent changes, good monitoring"
    deploy_frequency: "10-15x per day"

  admin-dashboard:
    strategy: rolling
    reason: "Low risk, internal users, cost-sensitive"
    deploy_frequency: "2-3x per week"

  analytics-processor:
    strategy: rolling
    reason: "Background job, no user-facing impact"
    deploy_frequency: "Daily"

-- ❌ WRONG: Breaking change
ALTER TABLE orders DROP COLUMN old_status;
-- Canary v2 works, but stable v1 crashes!

-- ✅ RIGHT: Expand-contract pattern

-- Step 1: EXPAND (before canary)
ALTER TABLE orders ADD COLUMN status_v2 VARCHAR(50);
UPDATE orders SET status_v2 = old_status WHERE status_v2 IS NULL;

-- Step 2: Deploy v2 (reads from both, writes to new)
-- v2 application code:
-- status = row.status_v2 || row.old_status  -- Prefer new, fallback to old

-- Step 3: Migrate data (background job)
UPDATE orders SET status_v2 = old_status WHERE status_v2 IS NULL;

-- Step 4: CONTRACT (after v1 fully terminated)
ALTER TABLE orders DROP COLUMN old_status;

apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: datadog-analysis
spec:
  metrics:
  - name: error-rate
    provider:
      datadog:
        apiKey:
          secretKeyRef:
            name: datadog-api-key
            key: api-key
        query: |
          avg:error.rate{service:myapp,version:canary}

apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: newrelic-analysis
spec:
  metrics:
  - name: apdex-score
    provider:
      newRelic:
        apiKey:
          secretKeyRef:
            name: newrelic-api-key
            key: api-key
        query: |
          SELECT apdex(duration) FROM Transaction
          WHERE appName = 'myapp' AND version = 'canary'

apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: cloudwatch-analysis
spec:
  metrics:
  - name: latency
    provider:
      cloudWatch:
        region: us-east-1
        metricDataQueries:
        - id: rate
          expression: "SELECT AVG(Latency) FROM AWS/ApplicationELB WHERE TargetGroup = 'myapp-canary'"

apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: custom-metrics
spec:
  metrics:
  - name: business-metric
    provider:
      job:
        spec:
          template:
            spec:
              containers:
              - name: metric-check
                image: curlimages/curl:latest
                command:
                - sh
                - -c
                - |
                  METRIC=$(curl -s https://my-api.com/metrics?version=canary | jq -r '.error_rate')
                  if (( $(echo "$METRIC < 0.01" | bc -l) )); then
                    echo "success"
                    exit 0
                  else
                    echo "failure"
                    exit 1
                  fi
              restartPolicy: Never

# Calculate minimum sample size for statistical significance
def min_canary_traffic(daily_requests, baseline_error_rate=0.01):
    """
    Calculate minimum canary traffic for 95% confidence

    Args:
        daily_requests: Total daily request volume
        baseline_error_rate: Expected error rate (e.g., 0.01 = 1%)

    Returns:
        Minimum canary percentage
    """
    # Need ~15,000 requests to detect 0.5% error rate change
    MIN_REQUESTS = 15000

    # Requests per 10-minute window
    requests_per_10min = (daily_requests / 24 / 60) * 10

    # Calculate required percentage
    required_percentage = (MIN_REQUESTS / requests_per_10min) * 100

    return max(5, min(required_percentage, 25))  # Between 5% and 25%

# Examples:
print(min_canary_traffic(10_000_000))   # High traffic → 5% (minimum)
print(min_canary_traffic(1_000_000))    # Medium traffic → 10%
print(min_canary_traffic(100_000))      # Low traffic → 25% (maximum)

# High-traffic service (>10M req/day)
steps:
- setWeight: 1
- pause: {duration: 10m}
- setWeight: 5
- pause: {duration: 15m}
- setWeight: 25
- pause: {duration: 20m}
- setWeight: 50
- pause: {duration: 20m}

# Medium-traffic service (1M-10M req/day)
steps:
- setWeight: 10
- pause: {duration: 15m}
- setWeight: 25
- pause: {duration: 15m}
- setWeight: 50
- pause: {duration: 20m}

# Low-traffic service (<1M req/day)
steps:
- setWeight: 25
- pause: {duration: 20m}
- setWeight: 50
- pause: {duration: 20m}

strategy:
  canary:
    steps:
    - setWeight: 10
    - pause: {}  # Manual approval required
    - setWeight: 50
    - pause: {}  # Manual approval

strategy:
  canary:
    steps:
    - setWeight: 10
    - pause: {duration: 15m}

    analysis:
      templates:
      - templateName: basic-health

      # Alert but don't rollback
      failureLimit: 999  # Never auto-rollback

    # Manual promotion after analysis
    - pause: {}

strategy:
  canary:
    steps:
    - setWeight: 10
    - pause: {duration: 15m}

    analysis:
      templates:
      - templateName: comprehensive-health

      # Auto-rollback on failure
      failureLimit: 3

    - setWeight: 50
    - pause: {duration: 20m}

#!/bin/bash
# chaos-test-errors.sh

echo "🔥 Chaos Test: Injecting 5% error rate into canary"

# Deploy canary with intentional bug
kubectl set env deployment/myapp-canary ERROR_RATE=0.05

echo "⏳ Waiting 5 minutes for detection..."
sleep 300

# Check if rollback triggered
ROLLOUT_STATUS=$(kubectl argo rollouts status myapp)

if echo "$ROLLOUT_STATUS" | grep -q "Degraded"; then
  echo "✅ PASS: Automatic rollback triggered"
  exit 0
else
  echo "❌ FAIL: Rollback did not trigger"
  exit 1
fi

# latency-chaos.yaml
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: latency-test
spec:
  action: delay
  mode: one
  selector:
    labelSelectors:
      app: myapp
      track: canary
  delay:
    latency: "2s"  # Add 2-second latency
  duration: "10m"

# Apply chaos
kubectl apply -f latency-chaos.yaml

# Monitor for automatic rollback
kubectl argo rollouts get rollout myapp --watch

// Add to canary deployment
var leak [][]byte

func leakMemory() {
  // Allocate 10MB every minute
  ticker := time.NewTicker(1 * time.Minute)
  for range ticker.C {
    leak = append(leak, make([]byte, 10*1024*1024))
  }
}

# chaos_test.py
import requests
import threading

def exhaust_connections():
    """Open connections without closing them"""
    while True:
        try:
            # Open connection but never close
            requests.get('http://myapp-canary/api/test',
                        stream=True,
                        timeout=999999)
        except:
            pass

# Start 100 threads
for i in range(100):
    threading.Thread(target=exhaust_connections).start()

#!/bin/bash
# rollback-drill.sh

echo "🚨 ROLLBACK DRILL (This is a test)"
echo "=================================="

# 1. Deploy bad version to staging
kubectl apply -f staging/bad-deployment.yaml

# 2. Trigger alerts
sleep 120

# 3. Time the rollback
START=$(date +%s)

# Blue-Green rollback
kubectl patch service myapp-service \
  -p '{"spec":{"selector":{"version":"blue"}}}'

END=$(date +%s)
ROLLBACK_TIME=$((END - START))

echo "Rollback completed in: ${ROLLBACK_TIME} seconds"

# 4. Verify recovery
sleep 30
ERROR_RATE=$(curl -s 'http://staging-prometheus:9090/api/v1/query?query=rate(http_errors_total[2m])' | jq -r '.data.result[0].value[1]')

if (( $(echo "$ERROR_RATE < 0.01" | bc -l) )); then
  echo "✅ DRILL PASSED"
  echo "Rollback time: ${ROLLBACK_TIME}s (target: <10s)"
else
  echo "❌ DRILL FAILED"
  echo "Error rate still high after rollback"
fi

# multi-region-rollout.yaml
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: myapp-global
spec:
  strategy:
    canary:
      steps:
      # Phase 1: Single region canary
      - setWeight: 0
        setCanaryScale:
          matchTrafficWeight: false
          replicas: 2
        trafficRouting:
          istio:
            virtualService:
              routes:
              - primary
            destinationRule:
              canarySubsetName: canary-us-east-1

      - pause: {duration: 15m}

      # Phase 2: Expand to 10% in us-east-1
      - setWeight: 10
      - pause: {duration: 20m}

      # Phase 3: Full rollout in us-east-1
      - setWeight: 100
        experiment:
          templates:
          - name: deploy-eu-west-1
            replicas: 1

      - pause: {duration: 30m}

      # Phase 4: Begin eu-west-1 rollout
      # Similar pattern for other regions...

#!/bin/bash
# regional-rollout.sh

REGIONS=("us-east-1" "us-west-2" "eu-west-1" "ap-southeast-1")

for REGION in "${REGIONS[@]}"; do
  echo "🌍 Deploying to region: $REGION"

  # Switch kubectl context
  kubectl config use-context $REGION

  # Deploy canary
  kubectl apply -f k8s/canary/ --namespace=production

  # Monitor for 30 minutes
  echo "📊 Monitoring $REGION for 30 minutes..."

  for i in {1..30}; do
    ERROR_RATE=$(kubectl exec -n monitoring prometheus-0 -- \
      promtool query instant \
      'rate(http_errors_total{region="'$REGION'"}[5m])')

    echo "[$i/30] Error rate: $ERROR_RATE"

    if (( $(echo "$ERROR_RATE > 0.01" | bc -l) )); then
      echo "❌ High error rate in $REGION, aborting rollout"
      kubectl argo rollouts abort myapp
      exit 1
    fi

    sleep 60
  done

  echo "✅ $REGION deployment successful"

  # Promote canary
  kubectl argo rollouts promote myapp

  echo "⏸️  Waiting 1 hour before next region..."
  sleep 3600
done

echo "🎉 Global rollout complete!"