CI/CD Pipeline Troubleshooting -- Build Failures, Flaky Tests & Deployment Errors

# .github/workflows/build.yml -- Optimize caching
name: Build
on: [push]
jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      # Cache npm dependencies
      - name: Cache npm
        uses: actions/cache@v3
        with:
          path: ~/.npm
          key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-npm-

      # Cache Docker layers
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build and cache Docker image
        uses: docker/build-push-action@v5
        with:
          cache-from: type=gha
          cache-to: type=gha,mode=max
          push: false
          load: true
tags: myapp:latest

# Verify cache is being used locally
docker buildx build --cache-from=type=local,src=/tmp/docker-cache \
                    --cache-to=type=local,dest=/tmp/docker-cache \
                    -t myapp:latest .

# Check cache hit/miss in output
docker build . --progress=plain 2>&1 | grep "CACHED"

# test_flaky.py -- Example of a flaky async test
import pytest
import asyncio
import random

# Bad: race condition due to no synchronization
@pytest.mark.asyncio
async def test_async_race_condition():
    """This test fails intermittently."""
    results = []
    async def worker(i):
        await asyncio.sleep(random.random() * 0.1)
        results.append(i)  # Race: append is not atomic in some contexts
    await asyncio.gather(*[worker(i) for i in range(10)])
    assert len(results) == 10  # Fails sometimes

# Good: use proper synchronization
@pytest.mark.asyncio
async def test_async_fixed():
    results = []
    lock = asyncio.Lock()
    async def worker(i):
        await asyncio.sleep(random.random() * 0.1)
        async with lock:
            results.append(i)
    await asyncio.gather(*[worker(i) for i in range(10)])
    assert len(results) == 10  # Always passes

# Run a flaky test multiple times to reproduce
for i in $(seq 1 20); do
  echo "Run $i:"
  pytest test_flaky.py -x --timeout=30 2>&1 | tail -1
done

# Use pytest-flakefinder to run tests multiple times
pip install pytest-flakefinder
pytest --flake-finder --flake-runs=10 test_flaky.py

# Run tests with random order to detect order dependencies
pytest --random-order test_flaky.py

# .github/workflows/deploy.yml -- Canary deployment with rollback
name: Deploy
on:
  workflow_dispatch:
jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Deploy canary (10% traffic)
        run: |
          kubectl set image deployment/myapp-canary myapp=myapp:${{ github.sha }}
          kubectl rollout status deployment/myapp-canary --timeout=5m

      - name: Health check canary
        run: |
          for i in $(seq 1 12); do
            status=$(curl -s -o /dev/null -w "%{http_code}" https://canary.example.com/health)
            if [ "$status" == "200" ]; then
              echo "Canary healthy"
              exit 0
            fi
            echo "Attempt $i: status $status, waiting 5s..."
            sleep 5
          done
          echo "Canary health check failed, rolling back"
          kubectl rollout undo deployment/myapp-canary
          exit 1

      - name: Promote to production
        run: |
          kubectl set image deployment/myapp-production myapp=myapp:${{ github.sha }}
          kubectl rollout status deployment/myapp-production --timeout=5m

# Check deployment history
kubectl rollout history deployment/myapp-production

# Rollback to previous version
kubectl rollout undo deployment/myapp-production --to-revision=3

# Check pod logs for the failing revision
kubectl logs -l app=myapp --tail=100 --prefix | grep -i error

# .github/workflows/secret-check.yml -- Verify secrets before running
name: Deploy with Secrets
on: [push]
jobs:
  deploy:
    runs-on: ubuntu-latest
    env:
      # Check if required secrets are set
      HAS_TOKEN: ${{ secrets.DEPLOY_TOKEN != '' }}
    steps:
      - name: Validate secrets
        run: |
          if [ "$HAS_TOKEN" != "true" ]; then
            echo "::error::DEPLOY_TOKEN secret is not set. Add it in Settings -> Secrets."
            exit 1
          fi
          echo "All required secrets are configured"

      - name: Deploy
        env:
          DEPLOY_TOKEN: ${{ secrets.DEPLOY_TOKEN }}
          API_KEY: ${{ secrets.API_KEY }}
        run: |
          # Secrets are now available as environment variables
          ./deploy.sh

# Check if a secret exists in GitHub Actions (via API)
gh secret list --repo owner/repo | grep DEPLOY_TOKEN

# Update a secret
gh secret set DEPLOY_TOKEN --body "your-token-value"

# Verify the secret is available in a pipeline step
echo "Secret length: ${#DEPLOY_TOKEN}"  # Should print > 0

# Jenkinsfile -- Fix stage timeouts
pipeline {
    agent any
    options {
        timeout(time: 1, unit: 'HOURS')  // Global pipeline timeout
    }
    stages {
        stage('Build') {
            options {
                timeout(time: 10, unit: 'MINUTES')
            }
            steps {
                sh 'make build'
            }
        }
        stage('Test') {
            options {
                timeout(time: 30, unit: 'MINUTES')
                retry(2)  // Retry flaky steps
            }
            parallel {
                stage('Unit tests') {
                    steps { sh 'make test-unit' }
                }
                stage('Integration tests') {
                    steps { sh 'make test-integration' }
                }
            }
        }
    }
    post {
        failure {
            echo "Pipeline failed at ${env.STAGE_NAME}"
            currentBuild.result = 'FAILURE'
        }
    }
}

# Check the slowest stage in a finished pipeline
# Jenkins: Navigate to the build -> Blue Ocean -> Stage view
# GitHub Actions: Actions tab -> workflow run -> job -> step timing
# Identify the stage with the longest wall clock time

# Reduce build time by splitting a monolith into parallel jobs
# Check for unnecessary steps (e.g., npm install on every run)

permissions:
  contents: write
  packages: write
  id-token: write