pcbjam/scripts/test-gal-regression.sh

#!/bin/bash

# GAL Regression Test - Master Script
# ====================================
# Single script to build, run, and compare native OpenGL and WebGL GAL implementations.
#
# Two-level comparison:
# 1. native vs baseline - Catches if native code regressed
# 2. webgl vs native   - Verifies WebGL implementation matches native
#
# Usage:
#   ./scripts/test-gal-regression.sh           # Run all tests
#   ./scripts/test-gal-regression.sh native    # Run native only
#   ./scripts/test-gal-regression.sh webgl     # Run webgl only (requires native output)
#   ./scripts/test-gal-regression.sh compare   # Compare only (skip builds)
#   ./scripts/test-gal-regression.sh -v        # Verbose output

# Redirect all output to a log file (re-execs script with redirection)
source "$(dirname "$0")/common/logging.sh"

set -e

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"

# Directories
GAL_REGRESSION_DIR="$PROJECT_ROOT/tests/gal-regression"
BASELINE_DIR="$GAL_REGRESSION_DIR/baseline"
OUTPUT_DIR="$GAL_REGRESSION_DIR/output"
NATIVE_OUTPUT_DIR="$OUTPUT_DIR/native"
WEBGL_OUTPUT_DIR="$OUTPUT_DIR/webgl"
NATIVE_BUILD_DIR="$GAL_REGRESSION_DIR/native/build"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Parse arguments
VERBOSE=""
RUN_NATIVE=true
RUN_WEBGL=true
COMPARE_ONLY=false

for arg in "$@"; do
    case $arg in
        native)
            RUN_WEBGL=false
            ;;
        webgl)
            RUN_NATIVE=false
            ;;
        compare)
            COMPARE_ONLY=true
            ;;
        -v|--verbose)
            VERBOSE="-v"
            ;;
    esac
done

# ============================================================================
# Helper Functions
# ============================================================================

log_header() {
    echo ""
    echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}"
    echo -e "${BLUE}  $1${NC}"
    echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}"
}

log_step() {
    echo -e "${YELLOW}>>> $1${NC}"
}

log_success() {
    echo -e "${GREEN}✓ $1${NC}"
}

log_error() {
    echo -e "${RED}✗ $1${NC}"
}

# Compare two directories of screenshots
# Usage: compare_screenshots <dir1> <dir2> <label>
# Returns 0 if identical, 1 if different
compare_screenshots() {
    local dir1="$1"
    local dir2="$2"
    local label="$3"

    echo ""
    echo "Comparing: $label"
    echo "  Reference: $dir1"
    echo "  Current:   $dir2"
    echo ""

    if [ ! -d "$dir1" ]; then
        log_error "Reference directory not found: $dir1"
        return 1
    fi

    if [ ! -d "$dir2" ]; then
        log_error "Current directory not found: $dir2"
        return 1
    fi

    local total=0
    local identical=0
    local different=0
    local missing=0

    # Compare all reference screenshots
    for ref in "$dir1"/*.png; do
        [ -e "$ref" ] || continue  # Handle no matches

        local filename=$(basename "$ref")
        local current="$dir2/$filename"
        total=$((total + 1))

        if [ ! -f "$current" ]; then
            echo "  MISSING: $filename"
            missing=$((missing + 1))
            continue
        fi

        if cmp -s "$ref" "$current"; then
            identical=$((identical + 1))
            if [ -n "$VERBOSE" ]; then
                echo "  IDENTICAL: $filename"
            fi
        else
            different=$((different + 1))
            local ref_size=$(stat -f%z "$ref" 2>/dev/null || stat -c%s "$ref")
            local cur_size=$(stat -f%z "$current" 2>/dev/null || stat -c%s "$current")
            echo "  DIFFERENT: $filename (ref: ${ref_size}B, cur: ${cur_size}B)"
        fi
    done

    # Check for extra files
    local extra=0
    for current in "$dir2"/*.png; do
        [ -e "$current" ] || continue
        local filename=$(basename "$current")
        local ref="$dir1/$filename"
        if [ ! -f "$ref" ]; then
            extra=$((extra + 1))
            echo "  EXTRA: $filename (not in reference)"
        fi
    done

    echo ""
    echo "  Results: $identical/$total identical, $different different, $missing missing, $extra extra"

    if [ "$different" -gt 0 ] || [ "$missing" -gt 0 ]; then
        log_error "$label: FAILED"
        return 1
    else
        log_success "$label: PASSED"
        return 0
    fi
}

# ============================================================================
# Build Functions
# ============================================================================

build_native() {
    log_header "Building Native Test"
    log_step "Running scripts/build-gal-native-test.sh..."

    if "$SCRIPT_DIR/build-gal-native-test.sh"; then
        log_success "Native build succeeded"
    else
        log_error "Native build failed"
        exit 1
    fi
}

build_webgl() {
    log_header "Building WebGL Test"

    if [ ! -f "$SCRIPT_DIR/build-gal-webgl-test.sh" ]; then
        log_step "WebGL build script not found (Phase 2)"
        return 0
    fi

    # Check if we have the output already (built previously)
    local output_dir="$PROJECT_ROOT/tests/apps/gal-webgl"
    if [ -f "$output_dir/gal_webgl_test.js" ] && [ -f "$output_dir/gal_webgl_test.wasm" ]; then
        log_step "WebGL test already built, skipping rebuild"
        return 0
    fi

    # Check if Emscripten is available
    if ! command -v emcmake &> /dev/null; then
        log_step "Emscripten not available - WebGL build requires emsdk or Docker"
        log_step "Skipping WebGL build (run inside Docker or activate emsdk first)"
        return 0
    fi

    log_step "Running scripts/build-gal-webgl-test.sh..."

    if "$SCRIPT_DIR/build-gal-webgl-test.sh"; then
        log_success "WebGL build succeeded"
    else
        log_error "WebGL build failed (Emscripten may need Python 3.10+)"
        log_step "Try running inside Docker or with a compatible emsdk"
        return 0  # Don't fail the whole test, just skip WebGL
    fi
}

# ============================================================================
# Run Functions
# ============================================================================

run_native() {
    log_header "Running Native Test"

    local native_exe="$NATIVE_BUILD_DIR/gal_native_test"

    if [ ! -f "$native_exe" ]; then
        log_error "Native test executable not found: $native_exe"
        log_step "Run build first: ./scripts/test-gal-regression.sh"
        exit 1
    fi

    # Create output directory
    mkdir -p "$NATIVE_OUTPUT_DIR"

    log_step "Running gal_native_test --output $NATIVE_OUTPUT_DIR"

    if "$native_exe" --output "$NATIVE_OUTPUT_DIR"; then
        local count=$(ls -1 "$NATIVE_OUTPUT_DIR"/*.png 2>/dev/null | wc -l | tr -d ' ')
        log_success "Native test completed: $count screenshots generated"
    else
        log_error "Native test failed"
        exit 1
    fi
}

run_webgl() {
    log_header "Running WebGL Test"

    local spec_file="$PROJECT_ROOT/tests/e2e/gal-webgl.spec.ts"

    if [ ! -f "$spec_file" ]; then
        log_step "WebGL test spec not found (Phase 2)"
        return 0
    fi

    # Create output directory
    mkdir -p "$WEBGL_OUTPUT_DIR"

    log_step "Running Playwright test..."

    cd "$PROJECT_ROOT/tests"
    if npx playwright test gal-webgl.spec.ts; then
        local count=$(ls -1 "$WEBGL_OUTPUT_DIR"/*.png 2>/dev/null | wc -l | tr -d ' ')
        log_success "WebGL test completed: $count screenshots generated"
    else
        log_error "WebGL test failed"
        exit 1
    fi
    cd "$PROJECT_ROOT"
}

# ============================================================================
# Compare Functions
# ============================================================================

compare_native_vs_baseline() {
    log_header "Comparing Native vs Baseline"
    compare_screenshots "$BASELINE_DIR" "$NATIVE_OUTPUT_DIR" "native vs baseline"
}

compare_webgl_vs_native() {
    log_header "Comparing WebGL vs Native"

    if [ ! -d "$WEBGL_OUTPUT_DIR" ] || [ -z "$(ls -A "$WEBGL_OUTPUT_DIR" 2>/dev/null)" ]; then
        log_step "WebGL output not found (Phase 2)"
        return 0
    fi

    compare_screenshots "$NATIVE_OUTPUT_DIR" "$WEBGL_OUTPUT_DIR" "webgl vs native"
}

# ============================================================================
# Main
# ============================================================================

log_header "GAL Regression Test Suite"
echo "Project Root: $PROJECT_ROOT"
echo "Baseline Dir: $BASELINE_DIR"
echo "Output Dir:   $OUTPUT_DIR"

# Track overall status
NATIVE_COMPARE_STATUS=0
WEBGL_COMPARE_STATUS=0

if [ "$COMPARE_ONLY" = false ]; then
    # Build phase
    if [ "$RUN_NATIVE" = true ]; then
        build_native
    fi

    if [ "$RUN_WEBGL" = true ]; then
        build_webgl
    fi

    # Run phase
    if [ "$RUN_NATIVE" = true ]; then
        run_native
    fi

    if [ "$RUN_WEBGL" = true ]; then
        run_webgl
    fi
fi

# Compare phase
if [ "$RUN_NATIVE" = true ]; then
    if ! compare_native_vs_baseline; then
        NATIVE_COMPARE_STATUS=1
    fi
fi

if [ "$RUN_WEBGL" = true ]; then
    if ! compare_webgl_vs_native; then
        WEBGL_COMPARE_STATUS=1
    fi
fi

# Final summary
log_header "Final Results"

if [ "$NATIVE_COMPARE_STATUS" -eq 0 ]; then
    log_success "Native vs Baseline: PASSED"
else
    log_error "Native vs Baseline: FAILED"
fi

if [ "$RUN_WEBGL" = true ]; then
    if [ -d "$WEBGL_OUTPUT_DIR" ] && [ -n "$(ls -A "$WEBGL_OUTPUT_DIR" 2>/dev/null)" ]; then
        if [ "$WEBGL_COMPARE_STATUS" -eq 0 ]; then
            log_success "WebGL vs Native: PASSED"
        else
            log_error "WebGL vs Native: FAILED"
        fi
    else
        echo "WebGL vs Native: SKIPPED (Phase 2)"
    fi
fi

# Exit status
if [ "$NATIVE_COMPARE_STATUS" -ne 0 ] || [ "$WEBGL_COMPARE_STATUS" -ne 0 ]; then
    echo ""
    log_error "Some comparisons failed!"
    exit 1
else
    echo ""
    log_success "All comparisons passed!"
    exit 0
fi