Build Time Optimization Techniques: CI/CD Performance Tuning Guide

#!/bin/bash
# Comprehensive build time analysis

cat << 'EOF' > /usr/local/bin/build-analyzer.sh
#!/bin/bash

set -e

# Color codes
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Analyze build stages
analyze_build_stages() {
    local build_log=$1

    echo -e "${BLUE}=== Build Stage Analysis ===${NC}"
    echo

    if [ ! -f "$build_log" ]; then
        echo -e "${RED}Error: Build log not found: $build_log${NC}"
        return 1
    fi

    # Extract stage durations
    echo "Stage Durations:"
    echo "----------------"

    # GitLab CI format
    if grep -q "Job succeeded" "$build_log"; then
        awk '/Job.*started/ {stage=$2; start=$NF}
             /Job.*succeeded/ {if (stage) {end=$NF; print stage, end-start}}' \
             "$build_log" | \
        sort -k2 -rn | \
        while read stage duration; do
            printf "%-30s %10.2fs\n" "$stage" "$duration"
        done
    fi

    # GitHub Actions format
    if grep -q "##\[group\]" "$build_log"; then
        grep -E "##\[group\]|Duration:" "$build_log" | \
        paste - - | \
        sed 's/##\[group\]//' | \
        awk '{print $1, $NF}' | \
        sort -k2 -rn
    fi

    # Calculate total time
    TOTAL_TIME=$(grep -E "^Total.*time:" "$build_log" | awk '{print $NF}')
    if [ -n "$TOTAL_TIME" ]; then
        echo
        echo -e "${GREEN}Total build time: $TOTAL_TIME${NC}"
    fi
}

# Identify bottlenecks
identify_bottlenecks() {
    local build_log=$1

    echo
    echo -e "${BLUE}=== Build Bottleneck Analysis ===${NC}"
    echo

    # Find longest running steps
    echo "Top 5 Slowest Steps:"
    echo "-------------------"

    grep -E "^(Step|RUN|Building|Installing)" "$build_log" | \
    grep -E "[0-9]+\.[0-9]+s" | \
    sort -t'.' -k1 -rn | \
    head -5

    echo
    echo "Potential Bottlenecks:"
    echo "---------------------"

    # Check for serial execution
    if grep -q "waiting for" "$build_log"; then
        echo -e "${YELLOW}• Serial execution detected${NC}"
        grep "waiting for" "$build_log" | head -3
    fi

    # Check for large downloads
    if grep -qE "Downloading|Fetching" "$build_log"; then
        echo -e "${YELLOW}• Large dependency downloads detected${NC}"
        grep -E "Downloading|Fetching" "$build_log" | \
        grep -E "[0-9]+\s*(MB|GB)" | head -3
    fi

    # Check for uncached operations
    if grep -qE "cache miss|not cached" "$build_log"; then
        echo -e "${YELLOW}• Cache misses detected${NC}"
        grep -iE "cache miss|not cached" "$build_log" | wc -l | \
        awk '{print "  ", $1, "cache misses"}'
    fi

    # Check for compilation time
    if grep -qE "compiling|building" "$build_log"; then
        COMPILE_TIME=$(grep -E "compiling|building" "$build_log" | \
            grep -oE "[0-9]+\.[0-9]+s" | \
            awk '{sum+=$1} END {print sum}')
        if [ -n "$COMPILE_TIME" ]; then
            echo -e "${YELLOW}• Total compilation time: ${COMPILE_TIME}s${NC}"
        fi
    fi
}

# Compare builds
compare_builds() {
    local build1=$1
    local build2=$2

    echo
    echo -e "${BLUE}=== Build Comparison ===${NC}"
    echo

    TIME1=$(grep -E "^Total.*time:" "$build1" | awk '{print $NF}' | tr -d 's')
    TIME2=$(grep -E "^Total.*time:" "$build2" | awk '{print $NF}' | tr -d 's')

    if [ -z "$TIME1" ] || [ -z "$TIME2" ]; then
        echo -e "${RED}Error: Could not extract build times${NC}"
        return 1
    fi

    echo "Build 1: ${TIME1}s"
    echo "Build 2: ${TIME2}s"

    DIFF=$(echo "$TIME1 - $TIME2" | bc)
    PCT=$(echo "scale=2; ($DIFF * 100) / $TIME1" | bc)

    if (( $(echo "$DIFF > 0" | bc -l) )); then
        echo -e "${GREEN}Improvement: ${DIFF}s (${PCT}% faster)${NC}"
    else
        echo -e "${RED}Regression: ${DIFF}s (${PCT}% slower)${NC}"
    fi
}

# Generate optimization recommendations
generate_recommendations() {
    local build_log=$1

    echo
    echo -e "${BLUE}=== Optimization Recommendations ===${NC}"
    echo

    local recommendations=()

    # Check for parallelization opportunities
    if ! grep -qE "parallel|concurrent" "$build_log"; then
        recommendations+=("• Enable parallel job execution")
    fi

    # Check for caching
    if grep -qE "Downloading.*dependencies" "$build_log"; then
        recommendations+=("• Implement dependency caching")
    fi

    # Check for incremental builds
    if grep -qE "clean.*build" "$build_log"; then
        recommendations+=("• Enable incremental builds")
    fi

    # Check for test optimization
    if grep -qE "Running.*tests" "$build_log"; then
        TEST_TIME=$(grep "Running.*tests" "$build_log" | \
            grep -oE "[0-9]+\.[0-9]+s" | \
            awk '{sum+=$1} END {print sum}')
        if (( $(echo "$TEST_TIME > 60" | bc -l) )); then
            recommendations+=("• Parallelize test execution")
            recommendations+=("• Consider test splitting/sharding")
        fi
    fi

    # Check for artifact size
    if grep -qE "artifact.*size" "$build_log"; then
        recommendations+=("• Optimize artifact size")
        recommendations+=("• Remove unnecessary files from artifacts")
    fi

    # Print recommendations
    if [ ${#recommendations[@]} -gt 0 ]; then
        for rec in "${recommendations[@]}"; do
            echo "$rec"
        done
    else
        echo -e "${GREEN}No immediate optimization opportunities found${NC}"
    fi

    echo
    echo "General Best Practices:"
    echo "• Use matrix builds for parallel execution"
    echo "• Cache dependencies between builds"
    echo "• Optimize Docker layer caching"
    echo "• Use fast, SSD-backed runners"
    echo "• Split long-running tests"
    echo "• Minimize artifact uploads"
    echo "• Use build artifact caching"
}

# Build time breakdown
build_time_breakdown() {
    local build_log=$1

    echo
    echo -e "${BLUE}=== Build Time Breakdown ===${NC}"
    echo

    # Categories
    local setup_time=0
    local dependency_time=0
    local build_time=0
    local test_time=0
    local deploy_time=0

    # Extract times (format: stage_name duration)
    while IFS= read -r line; do
        stage=$(echo "$line" | awk '{print $1}')
        duration=$(echo "$line" | awk '{print $2}')

        case $stage in
            *setup*|*prepare*|*init*)
                setup_time=$(echo "$setup_time + $duration" | bc)
                ;;
            *depend*|*install*|*download*)
                dependency_time=$(echo "$dependency_time + $duration" | bc)
                ;;
            *build*|*compile*)
                build_time=$(echo "$build_time + $duration" | bc)
                ;;
            *test*|*spec*|*check*)
                test_time=$(echo "$test_time + $duration" | bc)
                ;;
            *deploy*|*publish*|*release*)
                deploy_time=$(echo "$deploy_time + $duration" | bc)
                ;;
        esac
    done < <(grep -oE "[a-z_-]+\s+[0-9]+\.[0-9]+" "$build_log")

    total=$(echo "$setup_time + $dependency_time + $build_time + $test_time + $deploy_time" | bc)

    if [ -n "$total" ] && (( $(echo "$total > 0" | bc -l) )); then
        echo "Category          Time(s)    Percentage"
        echo "--------          -------    ----------"
        printf "Setup           %8.2fs    %6.1f%%\n" \
            $setup_time $(echo "scale=1; $setup_time * 100 / $total" | bc)
        printf "Dependencies    %8.2fs    %6.1f%%\n" \
            $dependency_time $(echo "scale=1; $dependency_time * 100 / $total" | bc)
        printf "Build           %8.2fs    %6.1f%%\n" \
            $build_time $(echo "scale=1; $build_time * 100 / $total" | bc)
        printf "Tests           %8.2fs    %6.1f%%\n" \
            $test_time $(echo "scale=1; $test_time * 100 / $total" | bc)
        printf "Deploy          %8.2fs    %6.1f%%\n" \
            $deploy_time $(echo "scale=1; $deploy_time * 100 / $total" | bc)
        echo "                --------    ----------"
        printf "Total           %8.2fs    100.0%%\n" $total
    fi
}

# Main execution
case "${1:-help}" in
    analyze)
        if [ -z "$2" ]; then
            echo "Usage: $0 analyze <build-log>"
            exit 1
        fi
        analyze_build_stages "$2"
        identify_bottlenecks "$2"
        build_time_breakdown "$2"
        generate_recommendations "$2"
        ;;
    compare)
        if [ $# -lt 3 ]; then
            echo "Usage: $0 compare <build-log-1> <build-log-2>"
            exit 1
        fi
        compare_builds "$2" "$3"
        ;;
    breakdown)
        if [ -z "$2" ]; then
            echo "Usage: $0 breakdown <build-log>"
            exit 1
        fi
        build_time_breakdown "$2"
        ;;
    recommend)
        if [ -z "$2" ]; then
            echo "Usage: $0 recommend <build-log>"
            exit 1
        fi
        generate_recommendations "$2"
        ;;
    *)
        echo "Usage: $0 {analyze|compare|breakdown|recommend} [args]"
        echo
        echo "Commands:"
        echo "  analyze <log>        - Full build analysis"
        echo "  compare <log1> <log2>  - Compare two builds"
        echo "  breakdown <log>      - Time breakdown by category"
        echo "  recommend <log>      - Generate recommendations"
        exit 1
        ;;
esac
EOF

chmod +x /usr/local/bin/build-analyzer.sh

# .gitlab-ci.yml
# Optimized parallel build configuration

variables:
  DOCKER_DRIVER: overlay2
  DOCKER_BUILDKIT: 1
  GIT_DEPTH: 1  # Shallow clone for faster checkouts

# Define stages
stages:
  - setup
  - build
  - test
  - package
  - deploy

# Cache configuration
.cache_template: &cache_definition
  cache:
    key:
      files:
        - package-lock.json
        - go.sum
        - requirements.txt
      prefix: ${CI_COMMIT_REF_SLUG}
    paths:
      - .npm/
      - node_modules/
      - vendor/
      - .cache/
    policy: pull-push

# Setup stage (runs once)
setup:dependencies:
  stage: setup
  image: node:20-alpine
  <<: *cache_definition
  script:
    - npm ci --cache .npm --prefer-offline
  artifacts:
    paths:
      - node_modules/
    expire_in: 1 hour
  only:
    - merge_requests
    - main
    - develop

# Parallel build matrix
build:matrix:
  stage: build
  parallel:
    matrix:
      - TARGET: [linux/amd64, linux/arm64]
        VARIANT: [standard, alpine]
  image: docker:24-git
  services:
    - docker:24-dind
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
  script:
    # Build specific target
    - |
      docker buildx build \
        --platform $TARGET \
        --cache-from type=registry,ref=$CI_REGISTRY_IMAGE:cache-$VARIANT \
        --cache-to type=registry,ref=$CI_REGISTRY_IMAGE:cache-$VARIANT,mode=max \
        --build-arg VARIANT=$VARIANT \
        --push \
        --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-$(echo $TARGET | tr '/' '-')-$VARIANT \
        -f Dockerfile.$VARIANT \
        .
  dependencies: []
  needs: []

# Parallel test execution
test:unit:
  stage: test
  image: node:20-alpine
  parallel: 4
  <<: *cache_definition
  script:
    # Split tests across parallel jobs
    - |
      npm run test:unit -- \
        --shard=$(($CI_NODE_INDEX + 1))/$CI_NODE_TOTAL \
        --maxWorkers=4 \
        --coverage
  artifacts:
    reports:
      junit: test-results/junit.xml
      coverage_report:
        coverage_format: cobertura
        path: coverage/cobertura-coverage.xml
    paths:
      - coverage/
    expire_in: 1 week
  dependencies:
    - setup:dependencies

# Parallel integration tests
test:integration:
  stage: test
  parallel:
    matrix:
      - SERVICE: [api, worker, scheduler]
        DATABASE: [postgres, mysql]
  image: docker/compose:latest
  services:
    - docker:24-dind
  variables:
    COMPOSE_PROJECT_NAME: ${CI_PROJECT_NAME}_${SERVICE}_${DATABASE}
  script:
    # Run service-specific integration tests
    - docker-compose -f docker-compose.test.yml up -d ${DATABASE}
    - |
      docker-compose -f docker-compose.test.yml run --rm test-${SERVICE} \
        pytest tests/integration/${SERVICE}/ \
        --junit-xml=test-results/${SERVICE}-${DATABASE}.xml
  after_script:
    - docker-compose -f docker-compose.test.yml down -v
  artifacts:
    reports:
      junit: test-results/*.xml
  dependencies: []

# Conditional parallel jobs
build:conditional:
  stage: build
  rules:
    # Only run on specific file changes
    - if: '$CI_PIPELINE_SOURCE == "merge_request"'
      changes:
        - src/**/*
        - package.json
      when: always
    - when: never
  parallel:
    matrix:
      - COMPONENT: [frontend, backend, api]
  script:
    - cd ${COMPONENT}
    - npm run build
  artifacts:
    paths:
      - ${COMPONENT}/dist/
    expire_in: 1 hour

# Fast-fail strategy
.fast_fail: &fast_fail
  interruptible: true
  retry:
    max: 2
    when:
      - runner_system_failure
      - stuck_or_timeout_failure

# Quick validation (fail fast)
validate:quick:
  stage: .pre
  <<: *fast_fail
  parallel:
    matrix:
      - CHECK: [lint, format, type-check, security-scan]
  script:
    - npm run ${CHECK}
  dependencies: []

# .github/workflows/build-optimized.yml
# Optimized GitHub Actions workflow with matrix

name: Optimized Build Pipeline

on:
  push:
    branches: [ main, develop ]
  pull_request:
    branches: [ main ]

env:
  NODE_VERSION: '20'
  GO_VERSION: '1.21'

# Optimize checkout
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # Changes detection for conditional execution
  changes:
    runs-on: ubuntu-latest
    outputs:
      frontend: ${{ steps.filter.outputs.frontend }}
      backend: ${{ steps.filter.outputs.backend }}
      infra: ${{ steps.filter.outputs.infra }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            frontend:
              - 'frontend/**'
              - 'package.json'
            backend:
              - 'backend/**'
              - 'go.mod'
            infra:
              - 'infra/**'
              - '*.tf'

  # Parallel dependency installation
  setup:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        component: [frontend, backend]
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js (frontend)
        if: matrix.component == 'frontend'
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'
          cache-dependency-path: frontend/package-lock.json

      - name: Setup Go (backend)
        if: matrix.component == 'backend'
        uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: true
          cache-dependency-path: backend/go.sum

      - name: Install dependencies (frontend)
        if: matrix.component == 'frontend'
        working-directory: frontend
        run: npm ci --prefer-offline

      - name: Download Go modules (backend)
        if: matrix.component == 'backend'
        working-directory: backend
        run: go mod download

      - name: Cache dependencies
        uses: actions/cache/save@v3
        with:
          path: |
            ${{ matrix.component == 'frontend' && 'frontend/node_modules' || 'backend/vendor' }}
          key: ${{ runner.os }}-${{ matrix.component }}-${{ hashFiles(format('{0}/**/package-lock.json', matrix.component), format('{0}/go.sum', matrix.component)) }}

  # Matrix build strategy
  build:
    needs: [changes, setup]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - component: frontend
            if: needs.changes.outputs.frontend == 'true'
            build-command: npm run build
            artifact-path: frontend/dist
          - component: backend
            if: needs.changes.outputs.backend == 'true'
            build-command: go build -o bin/server ./cmd/server
            artifact-path: backend/bin
    steps:
      - uses: actions/checkout@v4

      - name: Restore dependencies
        uses: actions/cache/restore@v3
        with:
          path: |
            ${{ matrix.component }}/node_modules
            ${{ matrix.component }}/vendor
          key: ${{ runner.os }}-${{ matrix.component }}-${{ hashFiles(format('{0}/**/package-lock.json', matrix.component), format('{0}/go.sum', matrix.component)) }}

      - name: Build ${{ matrix.component }}
        working-directory: ${{ matrix.component }}
        run: ${{ matrix.build-command }}

      - name: Upload artifacts
        uses: actions/upload-artifact@v3
        with:
          name: ${{ matrix.component }}-build
          path: ${{ matrix.artifact-path }}
          retention-days: 1

  # Parallel test matrix
  test:
    needs: [changes, setup]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        component: [frontend, backend]
        test-type: [unit, integration]
        shard: [1, 2, 3, 4]
    steps:
      - uses: actions/checkout@v4

      - name: Restore dependencies
        uses: actions/cache/restore@v3
        with:
          path: |
            ${{ matrix.component }}/node_modules
            ${{ matrix.component }}/vendor
          key: ${{ runner.os }}-${{ matrix.component }}-*

      - name: Run tests (frontend)
        if: matrix.component == 'frontend'
        working-directory: frontend
        run: |
          npm run test:${{ matrix.test-type }} -- \
            --shard=${{ matrix.shard }}/4 \
            --maxWorkers=2

      - name: Run tests (backend)
        if: matrix.component == 'backend'
        working-directory: backend
        run: |
          go test -v -race -coverprofile=coverage.out \
            -run "Test.*$(printf '%02d' ${{ matrix.shard }})" \
            ./...

  # Multi-platform Docker builds
  docker:
    needs: build
    runs-on: ubuntu-latest
    strategy:
      matrix:
        platform: [linux/amd64, linux/arm64]
        component: [frontend, backend]
    steps:
      - uses: actions/checkout@v4

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Download build artifacts
        uses: actions/download-artifact@v3
        with:
          name: ${{ matrix.component }}-build
          path: ${{ matrix.component }}/dist

      - name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: ${{ matrix.component }}
          platforms: ${{ matrix.platform }}
          cache-from: type=gha,scope=${{ matrix.component }}-${{ matrix.platform }}
          cache-to: type=gha,mode=max,scope=${{ matrix.component }}-${{ matrix.platform }}
          push: true
          tags: |
            ghcr.io/${{ github.repository }}/${{ matrix.component }}:${{ github.sha }}

// Jenkinsfile
// Optimized parallel pipeline

pipeline {
    agent none

    options {
        buildDiscarder(logRotator(numToKeepStr: '10'))
        disableConcurrentBuilds()
        timeout(time: 30, unit: 'MINUTES')
        timestamps()
    }

    environment {
        DOCKER_REGISTRY = 'registry.example.com'
        IMAGE_NAME = 'myapp'
    }

    stages {
        stage('Setup') {
            agent { label 'docker' }
            steps {
                script {
                    // Parallel dependency installation
                    parallel(
                        'Frontend Dependencies': {
                            dir('frontend') {
                                sh 'npm ci --prefer-offline'
                            }
                        },
                        'Backend Dependencies': {
                            dir('backend') {
                                sh 'go mod download'
                            }
                        },
                        'Python Dependencies': {
                            dir('scripts') {
                                sh 'pip install -r requirements.txt --cache-dir .pip-cache'
                            }
                        }
                    )
                }
            }
        }

        stage('Parallel Build') {
            parallel {
                stage('Build Frontend') {
                    agent { label 'node' }
                    steps {
                        dir('frontend') {
                            sh 'npm run build'
                            stash includes: 'dist/**', name: 'frontend-dist'
                        }
                    }
                }

                stage('Build Backend') {
                    agent { label 'golang' }
                    steps {
                        dir('backend') {
                            sh 'go build -o bin/server ./cmd/server'
                            stash includes: 'bin/**', name: 'backend-bin'
                        }
                    }
                }

                stage('Build Docker Images') {
                    matrix {
                        axes {
                            axis {
                                name 'PLATFORM'
                                values 'linux/amd64', 'linux/arm64'
                            }
                            axis {
                                name 'COMPONENT'
                                values 'api', 'worker', 'frontend'
                            }
                        }
                        agent { label 'docker' }
                        stages {
                            stage('Build Image') {
                                steps {
                                    script {
                                        sh """
                                            docker buildx build \\
                                                --platform ${PLATFORM} \\
                                                --cache-from type=registry,ref=${DOCKER_REGISTRY}/${IMAGE_NAME}:cache-${COMPONENT} \\
                                                --cache-to type=registry,ref=${DOCKER_REGISTRY}/${IMAGE_NAME}:cache-${COMPONENT},mode=max \\
                                                --build-arg COMPONENT=${COMPONENT} \\
                                                --tag ${DOCKER_REGISTRY}/${IMAGE_NAME}:${BUILD_NUMBER}-${COMPONENT}-${PLATFORM.replace('/', '-')} \\
                                                --push \\
                                                -f Dockerfile.${COMPONENT} \\
                                                .
                                        """
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }

        stage('Parallel Tests') {
            parallel {
                stage('Unit Tests') {
                    matrix {
                        axes {
                            axis {
                                name 'COMPONENT'
                                values 'frontend', 'backend'
                            }
                            axis {
                                name 'SHARD'
                                values '1', '2', '3', '4'
                            }
                        }
                        agent { label 'test' }
                        stages {
                            stage('Run Tests') {
                                steps {
                                    script {
                                        dir(COMPONENT) {
                                            if (COMPONENT == 'frontend') {
                                                sh "npm run test -- --shard=${SHARD}/4"
                                            } else {
                                                sh "go test -v ./... -run=TestShard${SHARD}"
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                }

                stage('Integration Tests') {
                    agent { label 'docker' }
                    steps {
                        script {
                            // Parallel integration test suites
                            def integrationTests = [:]
                            ['api', 'worker', 'scheduler'].each { service ->
                                integrationTests["Integration: ${service}"] = {
                                    sh """
                                        docker-compose -f docker-compose.test.yml run --rm test-${service}
                                    """
                                }
                            }
                            parallel integrationTests
                        }
                    }
                }

                stage('E2E Tests') {
                    agent { label 'e2e' }
                    steps {
                        script {
                            // Parallel E2E test execution
                            def e2eTests = [:]
                            (1..4).each { shard ->
                                e2eTests["E2E Shard ${shard}"] = {
                                    sh "npm run test:e2e -- --shard=${shard}/4"
                                }
                            }
                            parallel e2eTests
                        }
                    }
                }
            }
        }

        stage('Package') {
            when {
                branch 'main'
            }
            parallel {
                stage('Create Helm Chart') {
                    agent { label 'kubectl' }
                    steps {
                        sh 'helm package charts/myapp --version ${BUILD_NUMBER}'
                        archiveArtifacts artifacts: '*.tgz'
                    }
                }

                stage('Create Release Notes') {
                    agent any
                    steps {
                        script {
                            sh '''
                                git log --pretty=format:"%h - %s" $(git describe --tags --abbrev=0)..HEAD > CHANGELOG.md
                            '''
                            archiveArtifacts artifacts: 'CHANGELOG.md'
                        }
                    }
                }
            }
        }
    }

    post {
        always {
            node('docker') {
                // Cleanup
                sh 'docker system prune -f --filter "until=24h"'
            }
        }
        success {
            echo 'Pipeline completed successfully!'
        }
        failure {
            emailext(
                subject: "Build Failed: ${env.JOB_NAME} - ${env.BUILD_NUMBER}",
                body: "Check console output at ${env.BUILD_URL}",
                to: "${env.CHANGE_AUTHOR_EMAIL}"
            )
        }
    }
}

# gitlab-cache-strategy.yml
# Advanced caching strategies for GitLab CI

variables:
  # Enable fallback cache keys
  CACHE_FALLBACK_KEY: "${CI_COMMIT_REF_SLUG}"

# Global cache configuration
.global_cache:
  cache:
    - key:
        files:
          - package-lock.json
        prefix: npm-${CI_COMMIT_REF_SLUG}
      paths:
        - node_modules/
        - .npm/
      policy: pull-push
      when: always

    - key:
        files:
          - go.sum
        prefix: go-${CI_COMMIT_REF_SLUG}
      paths:
        - vendor/
        - .go-cache/
      policy: pull-push

    - key:
        files:
          - requirements.txt
        prefix: python-${CI_COMMIT_REF_SLUG}
      paths:
        - .venv/
        - .pip-cache/
      policy: pull-push

    # Fallback to main branch cache if current branch cache doesn't exist
    - key: npm-main
      paths:
        - node_modules/
      policy: pull
      when: on_failure

# Build cache from main
build:cache:
  stage: build
  extends: .global_cache
  only:
    - main
  script:
    - npm ci --cache .npm
    - go mod download
    - pip install -r requirements.txt --cache-dir .pip-cache
  cache:
    policy: push

# Use cache in feature branches
build:feature:
  stage: build
  extends: .global_cache
  except:
    - main
  script:
    - npm ci --cache .npm --prefer-offline
  cache:
    policy: pull

# Selective cache invalidation
validate:dependencies:
  stage: .pre
  script:
    - |
      # Check if dependencies changed
      if git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA | grep -E "package-lock.json|go.sum|requirements.txt"; then
        echo "Dependencies changed, cache will be rebuilt"
        exit 0
      else
        echo "No dependency changes detected"
      fi

# github-artifact-cache.yml
# GitHub Actions artifact and cache strategy

name: Artifact Caching Strategy

on: [push, pull_request]

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      # Restore multiple caches with fallback
      - name: Cache node modules
        uses: actions/cache@v3
        with:
          path: |
            node_modules
            ~/.npm
          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-node-
            ${{ runner.os }}-

      # Cache build outputs
      - name: Cache build outputs
        uses: actions/cache@v3
        with:
          path: |
            dist
            .next/cache
            out
          key: ${{ runner.os }}-build-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-build-

      # Cache test results for incremental testing
      - name: Cache test results
        uses: actions/cache@v3
        with:
          path: |
            .jest-cache
            coverage
          key: ${{ runner.os }}-test-${{ hashFiles('**/*.spec.ts') }}
          restore-keys: |
            ${{ runner.os }}-test-

      - name: Build
        run: npm run build

      # Upload artifacts for use in subsequent jobs
      - name: Upload build artifacts
        uses: actions/upload-artifact@v3
        with:
          name: build-output
          path: dist/
          retention-days: 1
          if-no-files-found: error

  test:
    needs: build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      # Download build artifacts
      - name: Download build artifacts
        uses: actions/download-artifact@v3
        with:
          name: build-output
          path: dist/

      # Restore test cache
      - name: Restore test cache
        uses: actions/cache@v3
        with:
          path: .jest-cache
          key: ${{ runner.os }}-test-${{ hashFiles('**/*.spec.ts') }}

      - name: Run tests
        run: npm test -- --cacheDirectory=.jest-cache

# gitlab-runner-config.toml
# Optimized GitLab Runner configuration

concurrent = 10  # Number of concurrent jobs

[[runners]]
  name = "optimized-docker-runner"
  url = "https://gitlab.com/"
  token = "TOKEN"
  executor = "docker"

  # Resource limits
  [runners.docker]
    image = "alpine:latest"
    privileged = false
    disable_cache = false
    volumes = ["/cache", "/var/run/docker.sock:/var/run/docker.sock"]

    # CPU and memory limits
    cpus = "4"
    memory = "8g"
    memory_swap = "8g"
    memory_reservation = "4g"

    # SHM size for builds requiring shared memory
    shm_size = 2147483648  # 2GB

    # Use faster storage
    volume_driver = "local"

  # Cache configuration
  [runners.cache]
    Type = "s3"
    Shared = true
    [runners.cache.s3]
      ServerAddress = "s3.amazonaws.com"
      BucketName = "gitlab-runner-cache"
      BucketLocation = "us-east-1"

  # Pre-clone script for faster checkouts
  [runners.docker]
    pre_clone_script = """
      git config --global core.compression 0
      git config --global http.postBuffer 524288000
    """

#!/bin/bash
# Optimize build machine for CI/CD performance

cat << 'EOF' > /usr/local/bin/optimize-build-machine.sh
#!/bin/bash

set -e

echo "=== Optimizing Build Machine ==="

# Increase file descriptor limits
cat > /etc/security/limits.d/build-limits.conf << LIMITS
*    soft nofile 65536
*    hard nofile 65536
*    soft nproc  65536
*    hard nproc  65536
LIMITS

# Optimize kernel parameters for builds
cat > /etc/sysctl.d/99-build-optimization.conf << SYSCTL
# Increase inotify watches (for file monitoring)
fs.inotify.max_user_watches = 524288
fs.inotify.max_user_instances = 8192

# Optimize memory management
vm.swappiness = 1
vm.dirty_ratio = 10
vm.dirty_background_ratio = 5

# Network optimization
net.core.somaxconn = 1024
net.core.netdev_max_backlog = 5000
SYSCTL

sysctl -p /etc/sysctl.d/99-build-optimization.conf

# Configure Docker for optimal performance
mkdir -p /etc/docker
cat > /etc/docker/daemon.json << DOCKER
{
  "storage-driver": "overlay2",
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3"
  },
  "default-ulimits": {
    "nofile": {
      "Name": "nofile",
      "Hard": 65536,
      "Soft": 65536
    }
  },
  "max-concurrent-downloads": 10,
  "max-concurrent-uploads": 10
}
DOCKER

systemctl restart docker

# Optimize npm for CI
npm config set prefer-offline true
npm config set progress false
npm config set loglevel error

# Optimize git for large repos
git config --global core.preloadindex true
git config --global core.fscache true
git config --global gc.auto 256
git config --global pack.threads 4

# Set up ccache for C/C++ builds
if command -v ccache &> /dev/null; then
    ccache --max-size=10G
    ccache --set-config=compression=true
fi

echo "Build machine optimization complete"
EOF

chmod +x /usr/local/bin/optimize-build-machine.sh