#!/bin/bash
# tests/load/test-100-agent-sustained.sh
# Phase 6 Wave 5 :: Validate 100+ agent sustained load with <10% performance degradation (1 hour)

set -euo pipefail

PROJECT_ROOT=$(git rev-parse --show-toplevel)
source "$PROJECT_ROOT/tests/test-utils.sh"

# Test configuration
AGENT_COUNT=100
TEST_DURATION_SECONDS=3600  # 1 hour
SAMPLE_INTERVAL_SECONDS=60  # Sample every 60 seconds
MAX_DEGRADATION_PERCENT=10
METRICS_FILE="/tmp/load-test-metrics-$$.json"
AGENT_PIDS=()

cleanup() {
    log_info "Cleaning up load test resources..."

    # Terminate all spawned agents
    for pid in "${AGENT_PIDS[@]}"; do
        if kill -0 "$pid" 2>/dev/null; then
            kill -TERM "$pid" 2>/dev/null || true
        fi
    done

    # Clean up Docker containers
    docker ps -a --filter "label=load-test=100-agent" --format "{{.ID}}" | xargs -r docker rm -f 2>/dev/null || true

    # Clean up metrics file
    rm -f "$METRICS_FILE"

    log_info "Cleanup complete"
}
trap cleanup EXIT

# Collect system metrics
collect_metrics() {
    local timestamp=$1
    local sample_num=$2

    # CPU usage (average across all cores)
    local cpu_usage=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1)

    # Memory usage (percentage)
    local mem_usage=$(free | grep Mem | awk '{printf "%.2f", ($3/$2) * 100}')

    # Docker container count
    local container_count=$(docker ps --filter "label=load-test=100-agent" --format "{{.ID}}" | wc -l)

    # Network connections
    local network_conns=$(netstat -an 2>/dev/null | grep ESTABLISHED | wc -l)

    # Redis connections
    local redis_conns=0
    if redis-cli ping &>/dev/null; then
        redis_conns=$(redis-cli info clients 2>/dev/null | grep connected_clients | cut -d: -f2 | tr -d '\r' || echo 0)
    fi

    # Log metrics
    log_info "Sample $sample_num @ ${timestamp}s: CPU=${cpu_usage}% MEM=${mem_usage}% Containers=$container_count NetConns=$network_conns RedisConns=$redis_conns"

    # Store metrics in JSON format
    cat >> "$METRICS_FILE" <<EOF
{
  "timestamp": $timestamp,
  "sample": $sample_num,
  "cpu_percent": $cpu_usage,
  "memory_percent": $mem_usage,
  "container_count": $container_count,
  "network_connections": $network_conns,
  "redis_connections": $redis_conns
}
EOF
}

# Spawn a single agent container
spawn_agent() {
    local agent_id=$1
    local task_id="load-test-100-agent-${agent_id}"

    docker run -d \
        --rm \
        --name "cfn-load-agent-${agent_id}" \
        --label "load-test=100-agent" \
        --label "agent-id=${agent_id}" \
        --network cfn-network \
        -e AGENT_ID="${task_id}" \
        -e AGENT_TYPE="backend-developer" \
        -e REDIS_URL="redis://redis:6379" \
        -e CFN_MODE="production" \
        redis:7-alpine \
        sh -c "
            # Simulate agent workload
            while true; do
                # Simulate CPU work (light load)
                sleep 0.1
                # Simulate Redis operations
                redis-cli -h redis ping >/dev/null 2>&1 || true
                sleep 5
            done
        " &>/dev/null

    return $?
}

test_sustained_load_100_agents() {
    log_step "GIVEN clean environment with Redis and Docker available"

    # Verify prerequisites
    if ! command -v docker &>/dev/null; then
        log_error "Docker not available"
        return 1
    fi

    if ! docker network inspect cfn-network &>/dev/null; then
        log_info "Creating cfn-network"
        docker network create cfn-network
    fi

    # Ensure Redis is available
    if ! redis-cli ping &>/dev/null; then
        log_info "Starting Redis for load testing"
        docker run -d \
            --name redis-load-test \
            --network cfn-network \
            --label "load-test=100-agent" \
            -p 6379:6379 \
            redis:7-alpine
        sleep 3
    fi

    log_step "WHEN spawning $AGENT_COUNT agents simultaneously"

    # Initialize metrics file
    echo "[" > "$METRICS_FILE"

    # Collect baseline metrics
    collect_metrics 0 0
    echo "," >> "$METRICS_FILE"

    local spawn_start=$(date +%s)
    local spawned=0
    local failed=0

    # Spawn agents in batches of 10 to avoid overwhelming system
    for i in $(seq 1 $AGENT_COUNT); do
        if spawn_agent "$i"; then
            spawned=$((spawned + 1))

            # Progress indicator every 10 agents
            if [ $((i % 10)) -eq 0 ]; then
                log_info "Spawned $spawned/$AGENT_COUNT agents..."
            fi
        else
            failed=$((failed + 1))
            log_error "Failed to spawn agent $i"
        fi

        # Batch delay: wait 100ms every 10 agents
        if [ $((i % 10)) -eq 0 ]; then
            sleep 0.1
        fi
    done

    local spawn_end=$(date +%s)
    local spawn_duration=$((spawn_end - spawn_start))

    log_info "Spawned $spawned agents in ${spawn_duration}s (failed: $failed)"

    # Verify spawn success
    local actual_count=$(docker ps --filter "label=load-test=100-agent" --format "{{.ID}}" | wc -l)
    log_info "Active containers: $actual_count"

    if [ "$actual_count" -lt 90 ]; then
        log_error "Failed to spawn minimum agents (expected ≥90, got $actual_count)"
        return 1
    fi

    log_step "THEN monitoring performance for $TEST_DURATION_SECONDS seconds"

    local test_start=$(date +%s)
    local sample_num=1
    local baseline_cpu=0
    local baseline_mem=0

    while true; do
        local current=$(date +%s)
        local elapsed=$((current - test_start))

        # Check if test duration exceeded
        if [ "$elapsed" -ge "$TEST_DURATION_SECONDS" ]; then
            log_info "Test duration reached ($TEST_DURATION_SECONDS seconds)"
            break
        fi

        # Sample metrics at intervals
        if [ $((elapsed % SAMPLE_INTERVAL_SECONDS)) -eq 0 ] && [ "$elapsed" -gt 0 ]; then
            collect_metrics "$elapsed" "$sample_num"
            echo "," >> "$METRICS_FILE"

            # Capture baseline from first sample
            if [ "$sample_num" -eq 1 ]; then
                baseline_cpu=$(tail -2 "$METRICS_FILE" | head -1 | grep -o '"cpu_percent": [0-9.]*' | cut -d' ' -f2)
                baseline_mem=$(tail -2 "$METRICS_FILE" | head -1 | grep -o '"memory_percent": [0-9.]*' | cut -d' ' -f2)
                log_info "Baseline established: CPU=${baseline_cpu}% MEM=${baseline_mem}%"
            fi

            sample_num=$((sample_num + 1))
        fi

        # Check container health every 5 minutes
        if [ $((elapsed % 300)) -eq 0 ] && [ "$elapsed" -gt 0 ]; then
            local alive=$(docker ps --filter "label=load-test=100-agent" --format "{{.ID}}" | wc -l)
            log_info "Health check @ ${elapsed}s: $alive/$actual_count containers alive"

            if [ "$alive" -lt $((actual_count * 80 / 100)) ]; then
                log_error "Too many containers died ($alive/$actual_count remaining)"
                return 1
            fi
        fi

        sleep 1
    done

    # Collect final metrics
    collect_metrics "$TEST_DURATION_SECONDS" "$sample_num"
    echo "]" >> "$METRICS_FILE"

    log_step "THEN analyzing performance degradation"

    # Extract final metrics
    local final_cpu=$(tail -2 "$METRICS_FILE" | head -1 | grep -o '"cpu_percent": [0-9.]*' | cut -d' ' -f2)
    local final_mem=$(tail -2 "$METRICS_FILE" | head -1 | grep -o '"memory_percent": [0-9.]*' | cut -d' ' -f2)

    # Calculate degradation
    local cpu_degradation=$(echo "scale=2; (($final_cpu - $baseline_cpu) / $baseline_cpu) * 100" | bc -l 2>/dev/null || echo "0")
    local mem_degradation=$(echo "scale=2; (($final_mem - $baseline_mem) / $baseline_mem) * 100" | bc -l 2>/dev/null || echo "0")

    log_info "Performance Analysis:"
    log_info "  CPU: ${baseline_cpu}% → ${final_cpu}% (degradation: ${cpu_degradation}%)"
    log_info "  MEM: ${baseline_mem}% → ${final_mem}% (degradation: ${mem_degradation}%)"

    # Validate degradation within acceptable limits
    local cpu_exceeds=$(echo "$cpu_degradation > $MAX_DEGRADATION_PERCENT" | bc -l)
    local mem_exceeds=$(echo "$mem_degradation > $MAX_DEGRADATION_PERCENT" | bc -l)

    if [ "$cpu_exceeds" -eq 1 ]; then
        log_error "CPU degradation (${cpu_degradation}%) exceeds limit (${MAX_DEGRADATION_PERCENT}%)"
        return 1
    fi

    if [ "$mem_exceeds" -eq 1 ]; then
        log_error "Memory degradation (${mem_degradation}%) exceeds limit (${MAX_DEGRADATION_PERCENT}%)"
        return 1
    fi

    # Save metrics report
    local report_file="$PROJECT_ROOT/.artifacts/load-test-100-agent-$(date +%Y%m%d-%H%M%S).json"
    mkdir -p "$PROJECT_ROOT/.artifacts"
    cp "$METRICS_FILE" "$report_file"
    log_info "Metrics saved to: $report_file"

    log_success "100-agent sustained load test PASSED"
    log_info "  Duration: ${TEST_DURATION_SECONDS}s (1 hour)"
    log_info "  Agents: $actual_count spawned, $(docker ps --filter "label=load-test=100-agent" | wc -l) survived"
    log_info "  CPU degradation: ${cpu_degradation}% (limit: ${MAX_DEGRADATION_PERCENT}%)"
    log_info "  MEM degradation: ${mem_degradation}% (limit: ${MAX_DEGRADATION_PERCENT}%)"

    return 0
}

# Execute test
log_info "Starting 100-agent sustained load test (Phase 6 Wave 5)"
test_sustained_load_100_agents