1b1a9d1633
Changes: - Exclude gal-transform-api.png from native/WebGL comparison since Transform() is dead code in KiCad (never called, has no effect) - Add verbose diagnostic output showing pixel difference details, content bounds, and sample pixels for failing scenarios - Update README to document that Transform() is dead code in both native and WebGL implementations 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
496 lines
16 KiB
Shell
Executable file
496 lines
16 KiB
Shell
Executable file
#!/bin/bash
|
|
|
|
# GAL Regression Test - Master Script
|
|
# ====================================
|
|
# Single script to build, run, and compare native OpenGL and WebGL GAL implementations.
|
|
#
|
|
# Two-level comparison:
|
|
# 1. native vs baseline - Catches if native code regressed
|
|
# 2. webgl vs native - Verifies WebGL implementation matches native
|
|
#
|
|
# Usage:
|
|
# ./scripts/test-gal-regression.sh # Run all tests
|
|
# ./scripts/test-gal-regression.sh native # Run native only
|
|
# ./scripts/test-gal-regression.sh webgl # Run webgl only (requires native output)
|
|
# ./scripts/test-gal-regression.sh compare # Compare only (skip builds)
|
|
# ./scripts/test-gal-regression.sh -v # Verbose output
|
|
|
|
# Redirect all output to a log file (re-execs script with redirection)
|
|
source "$(dirname "$0")/common/logging.sh"
|
|
|
|
set -e
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
|
|
|
|
# Directories
|
|
GAL_REGRESSION_DIR="$PROJECT_ROOT/tests/gal-regression"
|
|
BASELINE_DIR="$GAL_REGRESSION_DIR/baseline"
|
|
OUTPUT_DIR="$GAL_REGRESSION_DIR/output"
|
|
NATIVE_OUTPUT_DIR="$OUTPUT_DIR/native"
|
|
WEBGL_OUTPUT_DIR="$OUTPUT_DIR/webgl"
|
|
NATIVE_BUILD_DIR="$GAL_REGRESSION_DIR/native/build"
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Parse arguments
|
|
VERBOSE=""
|
|
RUN_NATIVE=true
|
|
RUN_WEBGL=true
|
|
COMPARE_ONLY=false
|
|
|
|
for arg in "$@"; do
|
|
case $arg in
|
|
native)
|
|
RUN_WEBGL=false
|
|
;;
|
|
webgl)
|
|
RUN_NATIVE=false
|
|
;;
|
|
compare)
|
|
COMPARE_ONLY=true
|
|
;;
|
|
-v|--verbose)
|
|
VERBOSE="-v"
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# ============================================================================
|
|
# Helper Functions
|
|
# ============================================================================
|
|
|
|
log_header() {
|
|
echo ""
|
|
echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}"
|
|
echo -e "${BLUE} $1${NC}"
|
|
echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}"
|
|
}
|
|
|
|
log_step() {
|
|
echo -e "${YELLOW}>>> $1${NC}"
|
|
}
|
|
|
|
log_success() {
|
|
echo -e "${GREEN}✓ $1${NC}"
|
|
}
|
|
|
|
log_error() {
|
|
echo -e "${RED}✗ $1${NC}"
|
|
}
|
|
|
|
# Compare two directories of screenshots using ImageMagick
|
|
# Supports different dimensions (resizes to match) and allows small pixel differences
|
|
# Usage: compare_screenshots <dir1> <dir2> <label> [threshold]
|
|
# Returns 0 if match within threshold, 1 if different
|
|
compare_screenshots() {
|
|
local dir1="$1"
|
|
local dir2="$2"
|
|
local label="$3"
|
|
local threshold="${4:-1.0}" # Default 1% difference allowed
|
|
|
|
echo ""
|
|
echo "Comparing: $label"
|
|
echo " Reference: $dir1"
|
|
echo " Current: $dir2"
|
|
echo " Threshold: ${threshold}% pixel difference"
|
|
echo ""
|
|
|
|
if [ ! -d "$dir1" ]; then
|
|
log_error "Reference directory not found: $dir1"
|
|
return 1
|
|
fi
|
|
|
|
if [ ! -d "$dir2" ]; then
|
|
log_error "Current directory not found: $dir2"
|
|
return 1
|
|
fi
|
|
|
|
# Check for ImageMagick
|
|
if ! command -v compare &> /dev/null; then
|
|
log_error "ImageMagick 'compare' command not found. Install with: brew install imagemagick"
|
|
return 1
|
|
fi
|
|
|
|
local total=0
|
|
local matching=0
|
|
local different=0
|
|
local missing=0
|
|
|
|
# Create temp directory for resized images
|
|
local tmpdir=$(mktemp -d)
|
|
trap "rm -rf '$tmpdir'" EXIT
|
|
|
|
# Files to exclude from comparison (documented as broken/dead code)
|
|
local excluded_files="gal-transform-api.png"
|
|
|
|
# Compare all reference screenshots
|
|
for ref in "$dir1"/*.png; do
|
|
[ -e "$ref" ] || continue # Handle no matches
|
|
|
|
local filename=$(basename "$ref")
|
|
|
|
# Skip excluded files
|
|
if echo "$excluded_files" | grep -q "$filename"; then
|
|
if [ -n "$VERBOSE" ]; then
|
|
echo " SKIPPED: $filename (excluded - see README.md)"
|
|
fi
|
|
continue
|
|
fi
|
|
|
|
local current="$dir2/$filename"
|
|
total=$((total + 1))
|
|
|
|
if [ ! -f "$current" ]; then
|
|
echo " MISSING: $filename"
|
|
missing=$((missing + 1))
|
|
continue
|
|
fi
|
|
|
|
# Get dimensions
|
|
local ref_dims=$(identify -format "%wx%h" "$ref" 2>/dev/null)
|
|
local cur_dims=$(identify -format "%wx%h" "$current" 2>/dev/null)
|
|
|
|
# If dimensions differ, resize current to match reference
|
|
local compare_file="$current"
|
|
if [ "$ref_dims" != "$cur_dims" ]; then
|
|
compare_file="$tmpdir/resized_$filename"
|
|
convert "$current" -resize "$ref_dims!" "$compare_file" 2>/dev/null
|
|
if [ -n "$VERBOSE" ]; then
|
|
echo " RESIZED: $filename ($cur_dims -> $ref_dims)"
|
|
fi
|
|
fi
|
|
|
|
# Normalize both images to TrueColor RGB for consistent comparison
|
|
# This handles PNG format differences (RGBA vs RGB, palette vs truecolor)
|
|
local ref_normalized="$tmpdir/ref_$filename"
|
|
local cur_normalized="$tmpdir/cur_$filename"
|
|
convert "$ref" -flatten -colorspace sRGB -type TrueColor "$ref_normalized" 2>/dev/null
|
|
convert "$compare_file" -flatten -colorspace sRGB -type TrueColor "$cur_normalized" 2>/dev/null
|
|
|
|
# Compare with fuzz factor (allows small pixel differences from anti-aliasing)
|
|
# Use AE (Absolute Error) metric - counts differing pixels
|
|
# Output format: "123456 (0.123)" - extract just the first number
|
|
local compare_output=$(compare -metric AE -fuzz 2% "$ref_normalized" "$cur_normalized" null: 2>&1 || true)
|
|
local diff_pixels=$(echo "$compare_output" | awk '{print $1}')
|
|
|
|
# Handle scientific notation (e.g., 1.92e+06)
|
|
if [[ "$diff_pixels" =~ [eE] ]]; then
|
|
diff_pixels=$(printf "%.0f" "$diff_pixels")
|
|
fi
|
|
|
|
# Handle error cases (non-numeric output)
|
|
if ! [[ "$diff_pixels" =~ ^[0-9]+\.?[0-9]*$ ]]; then
|
|
echo " ERROR: $filename (comparison failed: $compare_output)"
|
|
different=$((different + 1))
|
|
continue
|
|
fi
|
|
|
|
# Calculate percentage
|
|
local total_pixels=$(identify -format "%[fx:w*h]" "$ref_normalized" 2>/dev/null)
|
|
# Handle scientific notation in total_pixels
|
|
if [[ "$total_pixels" =~ [eE] ]]; then
|
|
total_pixels=$(printf "%.0f" "$total_pixels")
|
|
fi
|
|
if [ -z "$total_pixels" ] || [ "$total_pixels" = "0" ]; then
|
|
total_pixels=1 # Avoid division by zero
|
|
fi
|
|
|
|
# Use awk for floating point arithmetic (more portable than bc)
|
|
local diff_pct=$(awk "BEGIN {printf \"%.4f\", ($diff_pixels * 100.0) / $total_pixels}")
|
|
|
|
# Compare against threshold
|
|
local is_match=$(awk "BEGIN {print ($diff_pct < $threshold) ? 1 : 0}")
|
|
|
|
if [ "$is_match" -eq 1 ]; then
|
|
matching=$((matching + 1))
|
|
if [ -n "$VERBOSE" ] || [ "$diff_pixels" -gt 0 ]; then
|
|
echo " MATCH: $filename (${diff_pct}% different)"
|
|
fi
|
|
else
|
|
different=$((different + 1))
|
|
echo " DIFFERENT: $filename (${diff_pct}% different, threshold: ${threshold}%)"
|
|
fi
|
|
done
|
|
|
|
# Check for extra files
|
|
local extra=0
|
|
for current in "$dir2"/*.png; do
|
|
[ -e "$current" ] || continue
|
|
local filename=$(basename "$current")
|
|
local ref="$dir1/$filename"
|
|
if [ ! -f "$ref" ]; then
|
|
extra=$((extra + 1))
|
|
echo " EXTRA: $filename (not in reference)"
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
echo " Results: $matching/$total matching, $different different, $missing missing, $extra extra"
|
|
|
|
# Show diagnostic details for failing scenarios
|
|
if [ "$different" -gt 0 ] && [ -n "$VERBOSE" ]; then
|
|
echo ""
|
|
echo " === DIAGNOSTIC DETAILS for DIFFERENT scenarios ==="
|
|
for ref in "$dir1"/*.png; do
|
|
[ -e "$ref" ] || continue
|
|
local filename=$(basename "$ref")
|
|
local current="$dir2/$filename"
|
|
[ -f "$current" ] || continue
|
|
|
|
# Recalculate diff to identify failing scenarios
|
|
local ref_normalized="$tmpdir/ref_$filename"
|
|
local cur_normalized="$tmpdir/cur_$filename"
|
|
[ -f "$ref_normalized" ] || convert "$ref" -flatten -colorspace sRGB -type TrueColor "$ref_normalized" 2>/dev/null
|
|
[ -f "$cur_normalized" ] || convert "$current" -flatten -colorspace sRGB -type TrueColor "$cur_normalized" 2>/dev/null
|
|
|
|
local compare_output=$(compare -metric AE -fuzz 2% "$ref_normalized" "$cur_normalized" null: 2>&1 || true)
|
|
local diff_pixels=$(echo "$compare_output" | awk '{print $1}')
|
|
[[ "$diff_pixels" =~ [eE] ]] && diff_pixels=$(printf "%.0f" "$diff_pixels")
|
|
[[ "$diff_pixels" =~ ^[0-9]+\.?[0-9]*$ ]] || continue
|
|
|
|
local total_pixels=$(identify -format "%[fx:w*h]" "$ref_normalized" 2>/dev/null)
|
|
[[ "$total_pixels" =~ [eE] ]] && total_pixels=$(printf "%.0f" "$total_pixels")
|
|
[ -z "$total_pixels" ] || [ "$total_pixels" = "0" ] && total_pixels=1
|
|
local diff_pct=$(awk "BEGIN {printf \"%.4f\", ($diff_pixels * 100.0) / $total_pixels}")
|
|
local is_match=$(awk "BEGIN {print ($diff_pct < $threshold) ? 1 : 0}")
|
|
|
|
if [ "$is_match" -eq 0 ]; then
|
|
echo ""
|
|
echo " --- $filename (${diff_pct}% different) ---"
|
|
|
|
# Content bounds
|
|
local ref_bounds=$(magick "$ref" -flatten -fuzz 1% -trim -format "%w x %h at %O" info: 2>/dev/null || echo "N/A")
|
|
local cur_bounds=$(magick "$current" -fuzz 1% -trim -format "%w x %h at %O" info: 2>/dev/null || echo "N/A")
|
|
echo " Content bounds:"
|
|
echo " Ref: $ref_bounds"
|
|
echo " Current: $cur_bounds"
|
|
|
|
# Sample pixel colors at center
|
|
local ref_dims=$(identify -format "%w %h" "$ref" 2>/dev/null)
|
|
local cx=$(echo "$ref_dims" | awk '{print int($1/2)}')
|
|
local cy=$(echo "$ref_dims" | awk '{print int($2/2)}')
|
|
local ref_pixel=$(magick "$ref" -format "%[pixel:p{$cx,$cy}]" info: 2>/dev/null || echo "N/A")
|
|
local cur_pixel=$(magick "$current" -format "%[pixel:p{$cx,$cy}]" info: 2>/dev/null || echo "N/A")
|
|
echo " Center pixel ($cx,$cy):"
|
|
echo " Ref: $ref_pixel"
|
|
echo " Current: $cur_pixel"
|
|
|
|
# Check for mostly-empty content
|
|
local cur_unique=$(magick "$current" -format %c histogram:info: 2>/dev/null | wc -l)
|
|
if [ "$cur_unique" -lt 10 ]; then
|
|
echo " WARNING: Current image has only $cur_unique unique colors (possible rendering issue)"
|
|
fi
|
|
fi
|
|
done
|
|
echo ""
|
|
fi
|
|
|
|
if [ "$different" -gt 0 ] || [ "$missing" -gt 0 ]; then
|
|
log_error "$label: FAILED"
|
|
return 1
|
|
else
|
|
log_success "$label: PASSED"
|
|
return 0
|
|
fi
|
|
}
|
|
|
|
# ============================================================================
|
|
# Build Functions
|
|
# ============================================================================
|
|
|
|
build_native() {
|
|
log_header "Building Native Test"
|
|
log_step "Running scripts/build-gal-native-test.sh..."
|
|
|
|
if "$SCRIPT_DIR/build-gal-native-test.sh"; then
|
|
log_success "Native build succeeded"
|
|
else
|
|
log_error "Native build failed"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
build_webgl() {
|
|
log_header "Building WebGL Test"
|
|
|
|
if [ ! -f "$SCRIPT_DIR/build-gal-webgl-test.sh" ]; then
|
|
log_step "WebGL build script not found (Phase 2)"
|
|
return 0
|
|
fi
|
|
|
|
# Check if Emscripten is available
|
|
if ! command -v emcmake &> /dev/null; then
|
|
log_step "Emscripten not available - WebGL build requires emsdk or Docker"
|
|
log_step "Skipping WebGL build (run inside Docker or activate emsdk first)"
|
|
return 0
|
|
fi
|
|
|
|
log_step "Running scripts/build-gal-webgl-test.sh..."
|
|
|
|
if "$SCRIPT_DIR/build-gal-webgl-test.sh"; then
|
|
log_success "WebGL build succeeded"
|
|
else
|
|
log_error "WebGL build failed (Emscripten may need Python 3.10+)"
|
|
log_step "Try running inside Docker or with a compatible emsdk"
|
|
return 0 # Don't fail the whole test, just skip WebGL
|
|
fi
|
|
}
|
|
|
|
# ============================================================================
|
|
# Run Functions
|
|
# ============================================================================
|
|
|
|
run_native() {
|
|
log_header "Running Native Test"
|
|
|
|
local native_exe="$NATIVE_BUILD_DIR/gal_native_test"
|
|
|
|
if [ ! -f "$native_exe" ]; then
|
|
log_error "Native test executable not found: $native_exe"
|
|
log_step "Run build first: ./scripts/test-gal-regression.sh"
|
|
exit 1
|
|
fi
|
|
|
|
# Create output directory
|
|
mkdir -p "$NATIVE_OUTPUT_DIR"
|
|
|
|
log_step "Running gal_native_test --output $NATIVE_OUTPUT_DIR"
|
|
|
|
if "$native_exe" --output "$NATIVE_OUTPUT_DIR"; then
|
|
local count=$(ls -1 "$NATIVE_OUTPUT_DIR"/*.png 2>/dev/null | wc -l | tr -d ' ')
|
|
log_success "Native test completed: $count screenshots generated"
|
|
else
|
|
log_error "Native test failed"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
run_webgl() {
|
|
log_header "Running WebGL Test"
|
|
|
|
local spec_file="$PROJECT_ROOT/tests/e2e/gal-webgl.spec.ts"
|
|
|
|
if [ ! -f "$spec_file" ]; then
|
|
log_step "WebGL test spec not found (Phase 2)"
|
|
return 0
|
|
fi
|
|
|
|
# Create output directory
|
|
mkdir -p "$WEBGL_OUTPUT_DIR"
|
|
|
|
log_step "Running Playwright test..."
|
|
|
|
cd "$PROJECT_ROOT/tests"
|
|
if npx playwright test gal-webgl.spec.ts; then
|
|
local count=$(ls -1 "$WEBGL_OUTPUT_DIR"/*.png 2>/dev/null | wc -l | tr -d ' ')
|
|
log_success "WebGL test completed: $count screenshots generated"
|
|
else
|
|
log_error "WebGL test failed"
|
|
exit 1
|
|
fi
|
|
cd "$PROJECT_ROOT"
|
|
}
|
|
|
|
# ============================================================================
|
|
# Compare Functions
|
|
# ============================================================================
|
|
|
|
compare_native_vs_baseline() {
|
|
log_header "Comparing Native vs Baseline"
|
|
compare_screenshots "$BASELINE_DIR" "$NATIVE_OUTPUT_DIR" "native vs baseline"
|
|
}
|
|
|
|
compare_webgl_vs_native() {
|
|
log_header "Comparing WebGL vs Native"
|
|
|
|
if [ ! -d "$WEBGL_OUTPUT_DIR" ] || [ -z "$(ls -A "$WEBGL_OUTPUT_DIR" 2>/dev/null)" ]; then
|
|
log_step "WebGL output not found (Phase 2)"
|
|
return 0
|
|
fi
|
|
|
|
compare_screenshots "$NATIVE_OUTPUT_DIR" "$WEBGL_OUTPUT_DIR" "webgl vs native"
|
|
}
|
|
|
|
# ============================================================================
|
|
# Main
|
|
# ============================================================================
|
|
|
|
log_header "GAL Regression Test Suite"
|
|
echo "Project Root: $PROJECT_ROOT"
|
|
echo "Baseline Dir: $BASELINE_DIR"
|
|
echo "Output Dir: $OUTPUT_DIR"
|
|
|
|
# Track overall status
|
|
NATIVE_COMPARE_STATUS=0
|
|
WEBGL_COMPARE_STATUS=0
|
|
|
|
if [ "$COMPARE_ONLY" = false ]; then
|
|
# Build phase
|
|
if [ "$RUN_NATIVE" = true ]; then
|
|
build_native
|
|
fi
|
|
|
|
if [ "$RUN_WEBGL" = true ]; then
|
|
build_webgl
|
|
fi
|
|
|
|
# Run phase
|
|
if [ "$RUN_NATIVE" = true ]; then
|
|
run_native
|
|
fi
|
|
|
|
if [ "$RUN_WEBGL" = true ]; then
|
|
run_webgl
|
|
fi
|
|
fi
|
|
|
|
# Compare phase
|
|
if [ "$RUN_NATIVE" = true ]; then
|
|
if ! compare_native_vs_baseline; then
|
|
NATIVE_COMPARE_STATUS=1
|
|
fi
|
|
fi
|
|
|
|
if [ "$RUN_WEBGL" = true ]; then
|
|
if ! compare_webgl_vs_native; then
|
|
WEBGL_COMPARE_STATUS=1
|
|
fi
|
|
fi
|
|
|
|
# Final summary
|
|
log_header "Final Results"
|
|
|
|
if [ "$NATIVE_COMPARE_STATUS" -eq 0 ]; then
|
|
log_success "Native vs Baseline: PASSED"
|
|
else
|
|
log_error "Native vs Baseline: FAILED"
|
|
fi
|
|
|
|
if [ "$RUN_WEBGL" = true ]; then
|
|
if [ -d "$WEBGL_OUTPUT_DIR" ] && [ -n "$(ls -A "$WEBGL_OUTPUT_DIR" 2>/dev/null)" ]; then
|
|
if [ "$WEBGL_COMPARE_STATUS" -eq 0 ]; then
|
|
log_success "WebGL vs Native: PASSED"
|
|
else
|
|
log_error "WebGL vs Native: FAILED"
|
|
fi
|
|
else
|
|
echo "WebGL vs Native: SKIPPED (Phase 2)"
|
|
fi
|
|
fi
|
|
|
|
# Exit status
|
|
if [ "$NATIVE_COMPARE_STATUS" -ne 0 ] || [ "$WEBGL_COMPARE_STATUS" -ne 0 ]; then
|
|
echo ""
|
|
log_error "Some comparisons failed!"
|
|
exit 1
|
|
else
|
|
echo ""
|
|
log_success "All comparisons passed!"
|
|
exit 0
|
|
fi
|