fix(tests): enforce screenshot gate; drop shadow dir, flaky retinascale, 12 stale

Finalize the screenshot review system after the first Linux re-baseline
(that run showed 355/356 stable — the placeholder floor is fine as-is):

- ENFORCE the gate: `screenshots:check --fail-on-change` now fails the build on any
  changed/added/removed vs baselines (it still posts the drift report first, so the
  failure is actionable; a real render change → re-promote). The generic "CI failed"
  notice is suppressed for gate failures so we don't double-post on drift.
- Exclude retinascale-01-loaded: a fullPage HiDPI test whose captured height + DPR
  scaling vary run-to-run (~60% inter-run diff) — a flaky test, not render noise.
  IGNORE_SCREENSHOTS in config.ts; compare/promote/gen-manifest skip it; baseline removed.
- Kill the baseline-dir shadowing: drop e2e/baseline-screenshots/ from BASELINE_DIRS +
  delete its 3 files (grid-tab-final, wxgrid-controls, wxgrid-dedicated-page) that
  duplicated names in baseline-screenshots/ with different bytes.
- Prune 12 stale baselines (renamed/removed specs: wizard-01..04, gerbview-wizard-01..04,
  zoom-pl_editor-*, popup-03-palette).
- Delete the dead compare-screenshots.sh / update-baseline-screenshots.sh.
- Regenerate screenshot-manifest.json (355 entries).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.github/workflows/wasm-build.yml

@@ -336,22 +336,30 @@ jobs:
       # seeded by `npm run screenshots:noise`). Posts ONLY on push to main and no-ops
       # without DISCORD_WEBHOOK_URL (inert on PRs/forks). No extra build — reads the
       # already-produced test-results (screenshots + perf-*.json).
-      - name: Screenshot + perf report (on success)
+      - name: Screenshot gate + perf report (on success)
+        id: screenshot_report
         if: success() && inputs.run_tests
-        continue-on-error: true
         working-directory: tests
         env:
           DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}
           GH_TOKEN: ${{ github.token }}
+        # ENFORCING gate: `screenshots:check --fail-on-change` writes report.json and exits
+        # nonzero on any changed/added/removed vs the committed baselines. We still post the
+        # report (drift triptychs + perf) so the failure is actionable, then exit with the
+        # gate's status so a drift fails the build. (A re-render change → re-promote baselines.)
         run: |
-          npm run screenshots:check
+          set +e
+          npm run screenshots:check -- --fail-on-change; GATE=$?
           npm run screenshots:report -- --e2e pass
+          exit $GATE
 
       # ON FAILURE (build or e2e): a minimal text-only "CI failed" notice, nothing
       # else (no images / no comparison). Uses curl, NOT the TS reporter, because on a
       # build failure the test deps (npm ci) never installed. Main-push only.
       - name: Discord CI-failure notice
-        if: failure() && github.ref == 'refs/heads/main' && github.event_name == 'push'
+        # Build/e2e failure only — NOT a screenshot-gate failure (that already posted the
+        # drift report), so we don't double-post "CI failed" on drift.
+        if: failure() && steps.screenshot_report.outcome != 'failure' && github.ref == 'refs/heads/main' && github.event_name == 'push'
         continue-on-error: true
         env:
           DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}

scripts/compare-screenshots.sh

@@ -1,108 +0,0 @@
-#!/bin/bash
-
-# Compare test screenshots with baseline screenshots
-# This script compares all PNG files in baseline-screenshots with test-results
-
-set -e
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
-
-BASELINE_DIR="$PROJECT_ROOT/tests/baseline-screenshots"
-TEST_RESULTS_DIR="$PROJECT_ROOT/tests/test-results"
-
-# Check if directories exist
-if [ ! -d "$BASELINE_DIR" ]; then
-    echo "ERROR: Baseline directory not found: $BASELINE_DIR"
-    exit 1
-fi
-
-if [ ! -d "$TEST_RESULTS_DIR" ]; then
-    echo "ERROR: Test results directory not found: $TEST_RESULTS_DIR"
-    exit 1
-fi
-
-echo "=== Screenshot Comparison ==="
-echo "Baseline:     $BASELINE_DIR"
-echo "Test Results: $TEST_RESULTS_DIR"
-echo ""
-
-# Counters
-total=0
-identical=0
-different=0
-missing_current=0
-extra_current=0
-
-# Compare all baseline screenshots
-echo "=== Comparing Baseline Screenshots ==="
-for baseline in "$BASELINE_DIR"/*.png; do
-    filename=$(basename "$baseline")
-    current="$TEST_RESULTS_DIR/$filename"
-    total=$((total + 1))
-
-    if [ ! -f "$current" ]; then
-        echo "MISSING: $filename (not in test results)"
-        missing_current=$((missing_current + 1))
-        continue
-    fi
-
-    # Compare using cmp (byte-by-byte comparison)
-    if cmp -s "$baseline" "$current"; then
-        identical=$((identical + 1))
-        # Only show identical files if verbose
-        if [ "$1" = "-v" ] || [ "$1" = "--verbose" ]; then
-            echo "IDENTICAL: $filename"
-        fi
-    else
-        different=$((different + 1))
-
-        # Get file sizes
-        baseline_size=$(stat -f%z "$baseline" 2>/dev/null || stat -c%s "$baseline")
-        current_size=$(stat -f%z "$current" 2>/dev/null || stat -c%s "$current")
-        diff_bytes=$((current_size - baseline_size))
-
-        # Calculate percent difference
-        if [ "$baseline_size" -gt 0 ]; then
-            diff_pct=$(echo "scale=2; ($diff_bytes * 100) / $baseline_size" | bc)
-        else
-            diff_pct="N/A"
-        fi
-
-        echo "DIFFERENT: $filename (baseline: ${baseline_size}B, current: ${current_size}B, diff: ${diff_bytes}B / ${diff_pct}%)"
-    fi
-done
-
-# Check for extra files in test results not in baseline
-echo ""
-echo "=== Checking for Extra Screenshots ==="
-for current in "$TEST_RESULTS_DIR"/*.png; do
-    filename=$(basename "$current")
-    baseline="$BASELINE_DIR/$filename"
-
-    if [ ! -f "$baseline" ]; then
-        extra_current=$((extra_current + 1))
-        current_size=$(stat -f%z "$current" 2>/dev/null || stat -c%s "$current")
-        echo "EXTRA: $filename (${current_size}B, not in baseline)"
-    fi
-done
-
-# Summary
-echo ""
-echo "=== Summary ==="
-echo "Total baseline screenshots: $total"
-echo "Identical:                  $identical"
-echo "Different:                  $different"
-echo "Missing from test results:  $missing_current"
-echo "Extra in test results:      $extra_current"
-
-# Exit with error if there are differences
-if [ "$different" -gt 0 ] || [ "$missing_current" -gt 0 ]; then
-    echo ""
-    echo "WARNING: There are differences between baseline and test results!"
-    exit 1
-else
-    echo ""
-    echo "SUCCESS: All screenshots match baseline!"
-    exit 0
-fi

scripts/update-baseline-screenshots.sh

@@ -1,82 +0,0 @@
-#!/bin/bash
-# Update baseline screenshots from test results
-# Only copies:
-#   - NEW screenshots (not in baseline)
-#   - Screenshots with SIGNIFICANT differences (>5% size change) when --all flag used
-#
-# Usage:
-#   ./update-baseline-screenshots.sh             # Only copy NEW screenshots
-#   ./update-baseline-screenshots.sh --all       # Copy new + significantly different
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-TESTS_DIR="$SCRIPT_DIR/../tests"
-
-COPY_ALL=0
-THRESHOLD=5  # Percent threshold for "significant" difference
-
-while [ $# -gt 0 ]; do
-    case "$1" in
-        --all) COPY_ALL=1; shift ;;
-        *) shift ;;
-    esac
-done
-
-SOURCE_DIR="$TESTS_DIR/test-results"
-DEST_DIR="$TESTS_DIR/baseline-screenshots"
-
-if [ $COPY_ALL -eq 1 ]; then
-    echo "Mode: Copy NEW + SIGNIFICANTLY DIFFERENT screenshots"
-else
-    echo "Mode: Copy NEW screenshots only (use --all to include significant changes)"
-fi
-
-if [ ! -d "$SOURCE_DIR" ]; then
-    echo "Error: test-results directory not found at $SOURCE_DIR"
-    echo "Run 'npm test' first to generate screenshots"
-    exit 1
-fi
-
-mkdir -p "$DEST_DIR"
-
-new_count=0
-updated_count=0
-
-for src_file in "$SOURCE_DIR"/*.png; do
-    [ -f "$src_file" ] || continue
-
-    filename=$(basename "$src_file")
-    dest_file="$DEST_DIR/$filename"
-
-    if [ ! -f "$dest_file" ]; then
-        # NEW: Screenshot doesn't exist in baseline
-        echo "NEW: $filename"
-        cp "$src_file" "$dest_file"
-        ((new_count++))
-    elif [ $COPY_ALL -eq 1 ]; then
-        # Check if significantly different
-        src_size=$(stat -f%z "$src_file" 2>/dev/null || stat -c%s "$src_file")
-        dest_size=$(stat -f%z "$dest_file" 2>/dev/null || stat -c%s "$dest_file")
-
-        if [ "$dest_size" -eq 0 ]; then
-            diff_pct=100
-        else
-            diff=$((src_size - dest_size))
-            diff=${diff#-}  # Absolute value
-            diff_pct=$((diff * 100 / dest_size))
-        fi
-
-        if [ "$diff_pct" -ge "$THRESHOLD" ]; then
-            echo "UPDATED ($diff_pct% diff): $filename"
-            cp "$src_file" "$dest_file"
-            ((updated_count++))
-        fi
-    fi
-done
-
-echo ""
-echo "=== Summary ==="
-echo "New screenshots added: $new_count"
-if [ $COPY_ALL -eq 1 ]; then
-    echo "Significantly changed: $updated_count"
-fi
-echo "Total in baseline: $(ls -1 "$DEST_DIR"/*.png 2>/dev/null | wc -l | tr -d ' ')"

tests/screenshot-manifest.json

@@ -53,6 +53,22 @@
       "name": "13-final.png",
       "engine": "chromium-swiftshader"
     },
+    {
+      "name": "3d-viewer-00-board-loaded.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "3d-viewer-pic_programmer-render.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "3d-viewer-pic_programmer.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "3d-viewer-titlebar.png",
+      "engine": "chromium-swiftshader"
+    },
     {
       "name": "appearance-00-layers.png",
       "engine": "chromium-swiftshader"
@@ -553,22 +569,6 @@
       "name": "gerbview-wizard-00-initial.png",
       "engine": "firefox-llvmpipe"
     },
-    {
-      "name": "gerbview-wizard-01.png",
-      "engine": "firefox-llvmpipe"
-    },
-    {
-      "name": "gerbview-wizard-02.png",
-      "engine": "firefox-llvmpipe"
-    },
-    {
-      "name": "gerbview-wizard-03.png",
-      "engine": "firefox-llvmpipe"
-    },
-    {
-      "name": "gerbview-wizard-04-finish.png",
-      "engine": "firefox-llvmpipe"
-    },
     {
       "name": "grid-tab-final.png",
       "engine": "chromium-swiftshader"
@@ -793,6 +793,34 @@
       "name": "menu-05-all-menus.png",
       "engine": "chromium-swiftshader"
     },
+    {
+      "name": "modal-01-border.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "modal-02-before-drag.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "modal-03-after-drag.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "modal-04-before-resize.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "modal-05-after-resize.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "notebook-01-scrolled.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "notebook-02-scrolled-again.png",
+      "engine": "chromium-swiftshader"
+    },
     {
       "name": "ownerdrawn-01-loaded.png",
       "engine": "chromium-swiftshader"
@@ -821,6 +849,10 @@
       "name": "pcbnew-context-submenu.png",
       "engine": "chromium-swiftshader"
     },
+    {
+      "name": "pcbnew-dark-mode-loaded.png",
+      "engine": "chromium-swiftshader"
+    },
     {
       "name": "pcbnew-draw-lines-00-before-tool-click.png",
       "engine": "chromium-swiftshader"
@@ -841,6 +873,14 @@
       "name": "pcbnew-loaded.png",
       "engine": "chromium-swiftshader"
     },
+    {
+      "name": "pcbnew-move-00-before.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "pcbnew-move-01-after.png",
+      "engine": "chromium-swiftshader"
+    },
     {
       "name": "pcbnew-sidebar-scrollbar-dragged.png",
       "engine": "chromium-swiftshader"
@@ -921,10 +961,6 @@
       "name": "popup-03-palette-open.png",
       "engine": "chromium-swiftshader"
     },
-    {
-      "name": "popup-03-palette.png",
-      "engine": "chromium-swiftshader"
-    },
     {
       "name": "popup-04-color.png",
       "engine": "chromium-swiftshader"
@@ -1033,10 +1069,6 @@
       "name": "regions.png",
       "engine": "chromium-swiftshader"
     },
-    {
-      "name": "retinascale-01-loaded.png",
-      "engine": "chromium-swiftshader"
-    },
     {
       "name": "scrollbar-01-loaded.png",
       "engine": "chromium-swiftshader"
@@ -1329,34 +1361,18 @@
       "name": "wizard-01-loaded.png",
       "engine": "chromium-swiftshader"
     },
-    {
-      "name": "wizard-01.png",
-      "engine": "chromium-swiftshader"
-    },
     {
       "name": "wizard-02-launch.png",
       "engine": "chromium-swiftshader"
     },
-    {
-      "name": "wizard-02.png",
-      "engine": "chromium-swiftshader"
-    },
     {
       "name": "wizard-03-next-page.png",
       "engine": "chromium-swiftshader"
     },
-    {
-      "name": "wizard-03.png",
-      "engine": "chromium-swiftshader"
-    },
     {
       "name": "wizard-04-back-page.png",
       "engine": "chromium-swiftshader"
     },
-    {
-      "name": "wizard-04-finish.png",
-      "engine": "chromium-swiftshader"
-    },
     {
       "name": "wizard-05-cancel.png",
       "engine": "chromium-swiftshader"
@@ -1404,18 +1420,6 @@
     {
       "name": "xml-06-results.png",
       "engine": "chromium-swiftshader"
-    },
-    {
-      "name": "zoom-pl_editor-00-baseline.png",
-      "engine": "firefox-llvmpipe"
-    },
-    {
-      "name": "zoom-pl_editor-01-zoomed-in-at-P.png",
-      "engine": "firefox-llvmpipe"
-    },
-    {
-      "name": "zoom-pl_editor-02-zoomed-out-back.png",
-      "engine": "firefox-llvmpipe"
     }
   ]
 }

tests/tools/screenshots/compare.ts

@@ -23,6 +23,7 @@ import {
     type EngineFloor,
     type Manifest,
     floorFor,
+    isIgnored,
 } from './config';
 import { diffImages, cluster, drawBoxes, composite, loadPng, savePng, type Box } from './image-ops';
 
@@ -123,6 +124,9 @@ export function classify(root: string, sha: string | null): Report {
     const baselines = baselineIndex(root);
     const resultsDir = path.join(root, RESULTS_DIR);
     const actuals = new Set(listPngs(resultsDir));
+    // Drop excluded screenshots from both sides so they're never compared or counted.
+    for (const name of [...baselines.keys()]) if (isIgnored(name)) baselines.delete(name);
+    for (const name of [...actuals]) if (isIgnored(name)) actuals.delete(name);
     const manifest = loadManifest(root);
     const outDir = path.join(root, DIFF_OUT_DIR);
     fs.mkdirSync(outDir, { recursive: true });

tests/tools/screenshots/config.ts

@@ -9,8 +9,12 @@
  * the npm scripts and CI steps run from).
  */
 
-/** Committed baseline directories, scanned in order. Filenames are the keys. */
+/**
-export const BASELINE_DIRS = ['baseline-screenshots', 'e2e/baseline-screenshots'] as const;
+ * Committed baseline directory. Single dir on purpose: `e2e/baseline-screenshots/`
+ * used to also hold a few names (grid-tab-final, wxgrid-*) that ALSO live here with
+ * different bytes — so they got silently shadowed. Consolidated to one dir.
+ */
+export const BASELINE_DIRS = ['baseline-screenshots'] as const;
 
 /** Where Playwright writes the current run's screenshots (gitignored). */
 export const RESULTS_DIR = 'test-results';
@@ -74,6 +78,20 @@ export const FLOORS: Record<string, EngineFloor> = {
  */
 export const IGNORE_REGIONS: Record<string, Array<{ x: number; y: number; width: number; height: number }>> = {};
 
+/**
+ * Screenshots excluded from comparison entirely (not compared, not counted as
+ * changed/added/removed, not put in the manifest). For nondeterministic captures
+ * that can't be a stable baseline — e.g. `retinascale-01-loaded` is a `fullPage`
+ * HiDPI test whose captured height + DPR scaling vary run-to-run (~60% inter-run
+ * diff observed), a flaky test rather than render noise.
+ */
+export const IGNORE_SCREENSHOTS = new Set<string>(['retinascale-01-loaded.png']);
+
+/** True if a screenshot is excluded from comparison. */
+export function isIgnored(name: string): boolean {
+    return IGNORE_SCREENSHOTS.has(name);
+}
+
 export type ManifestEntry = { name: string; engine: string };
 export type Manifest = { screenshots: ManifestEntry[] };
 

tests/tools/screenshots/gen-manifest.ts

@@ -19,7 +19,7 @@
  */
 import * as fs from 'fs';
 import * as path from 'path';
-import { BASELINE_DIRS, MANIFEST_PATH, type Manifest } from './config';
+import { BASELINE_DIRS, MANIFEST_PATH, isIgnored, type Manifest } from './config';
 
 const CHROMIUM = 'chromium-swiftshader';
 const FIREFOX = 'firefox-llvmpipe';
@@ -78,7 +78,7 @@ function listBaselines(root: string): string[] {
     for (const dir of BASELINE_DIRS) {
         const abs = path.join(root, dir);
         if (!fs.existsSync(abs)) continue;
-        for (const f of fs.readdirSync(abs)) if (f.toLowerCase().endsWith('.png')) names.add(f);
+        for (const f of fs.readdirSync(abs)) if (f.toLowerCase().endsWith('.png') && !isIgnored(f)) names.add(f);
     }
     return [...names].sort();
 }

tests/tools/screenshots/promote.ts

@@ -16,7 +16,7 @@ import * as fs from 'fs';
 import * as os from 'os';
 import * as path from 'path';
 import { execFileSync } from 'child_process';
-import { BASELINE_DIRS, MANIFEST_PATH, floorFor, type Manifest } from './config';
+import { BASELINE_DIRS, MANIFEST_PATH, floorFor, isIgnored, type Manifest } from './config';
 import { diffImages, loadPng } from './image-ops';
 
 function listPngs(dir: string): string[] {
@@ -67,6 +67,9 @@ type Plan = { updated: string[]; added: string[]; unchanged: string[]; removedCa
 function buildPlan(root: string, renderDir: string, manifest?: Manifest): { plan: Plan; apply: () => void } {
     const baselines = baselineIndex(root);
     const rendered = new Set(listPngs(renderDir));
+    // Never promote excluded screenshots (e.g. the flaky retinascale fullPage shot).
+    for (const name of [...rendered]) if (isIgnored(name)) rendered.delete(name);
+    for (const name of [...baselines.keys()]) if (isIgnored(name)) baselines.delete(name);
     const plan: Plan = { updated: [], added: [], unchanged: [], removedCandidates: [] };
     const actions: Array<() => void> = [];