fix(tests): enforce screenshot gate; drop shadow dir, flaky retinascale, 12 stale

Finalize the screenshot review system after the first Linux re-baseline
(that run showed 355/356 stable — the placeholder floor is fine as-is):

- ENFORCE the gate: `screenshots:check --fail-on-change` now fails the build on any
  changed/added/removed vs baselines (it still posts the drift report first, so the
  failure is actionable; a real render change → re-promote). The generic "CI failed"
  notice is suppressed for gate failures so we don't double-post on drift.
- Exclude retinascale-01-loaded: a fullPage HiDPI test whose captured height + DPR
  scaling vary run-to-run (~60% inter-run diff) — a flaky test, not render noise.
  IGNORE_SCREENSHOTS in config.ts; compare/promote/gen-manifest skip it; baseline removed.
- Kill the baseline-dir shadowing: drop e2e/baseline-screenshots/ from BASELINE_DIRS +
  delete its 3 files (grid-tab-final, wxgrid-controls, wxgrid-dedicated-page) that
  duplicated names in baseline-screenshots/ with different bytes.
- Prune 12 stale baselines (renamed/removed specs: wizard-01..04, gerbview-wizard-01..04,
  zoom-pl_editor-*, popup-03-palette).
- Delete the dead compare-screenshots.sh / update-baseline-screenshots.sh.
- Regenerate screenshot-manifest.json (355 entries).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/screenshot-manifest.json

@@ -53,6 +53,22 @@
       "name": "13-final.png",
       "engine": "chromium-swiftshader"
     },
+    {
+      "name": "3d-viewer-00-board-loaded.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "3d-viewer-pic_programmer-render.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "3d-viewer-pic_programmer.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "3d-viewer-titlebar.png",
+      "engine": "chromium-swiftshader"
+    },
     {
       "name": "appearance-00-layers.png",
       "engine": "chromium-swiftshader"
@@ -553,22 +569,6 @@
       "name": "gerbview-wizard-00-initial.png",
       "engine": "firefox-llvmpipe"
     },
-    {
-      "name": "gerbview-wizard-01.png",
-      "engine": "firefox-llvmpipe"
-    },
-    {
-      "name": "gerbview-wizard-02.png",
-      "engine": "firefox-llvmpipe"
-    },
-    {
-      "name": "gerbview-wizard-03.png",
-      "engine": "firefox-llvmpipe"
-    },
-    {
-      "name": "gerbview-wizard-04-finish.png",
-      "engine": "firefox-llvmpipe"
-    },
     {
       "name": "grid-tab-final.png",
       "engine": "chromium-swiftshader"
@@ -793,6 +793,34 @@
       "name": "menu-05-all-menus.png",
       "engine": "chromium-swiftshader"
     },
+    {
+      "name": "modal-01-border.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "modal-02-before-drag.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "modal-03-after-drag.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "modal-04-before-resize.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "modal-05-after-resize.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "notebook-01-scrolled.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "notebook-02-scrolled-again.png",
+      "engine": "chromium-swiftshader"
+    },
     {
       "name": "ownerdrawn-01-loaded.png",
       "engine": "chromium-swiftshader"
@@ -821,6 +849,10 @@
       "name": "pcbnew-context-submenu.png",
       "engine": "chromium-swiftshader"
     },
+    {
+      "name": "pcbnew-dark-mode-loaded.png",
+      "engine": "chromium-swiftshader"
+    },
     {
       "name": "pcbnew-draw-lines-00-before-tool-click.png",
       "engine": "chromium-swiftshader"
@@ -841,6 +873,14 @@
       "name": "pcbnew-loaded.png",
       "engine": "chromium-swiftshader"
     },
+    {
+      "name": "pcbnew-move-00-before.png",
+      "engine": "chromium-swiftshader"
+    },
+    {
+      "name": "pcbnew-move-01-after.png",
+      "engine": "chromium-swiftshader"
+    },
     {
       "name": "pcbnew-sidebar-scrollbar-dragged.png",
       "engine": "chromium-swiftshader"
@@ -921,10 +961,6 @@
       "name": "popup-03-palette-open.png",
       "engine": "chromium-swiftshader"
     },
-    {
-      "name": "popup-03-palette.png",
-      "engine": "chromium-swiftshader"
-    },
     {
       "name": "popup-04-color.png",
       "engine": "chromium-swiftshader"
@@ -1033,10 +1069,6 @@
       "name": "regions.png",
       "engine": "chromium-swiftshader"
     },
-    {
-      "name": "retinascale-01-loaded.png",
-      "engine": "chromium-swiftshader"
-    },
     {
       "name": "scrollbar-01-loaded.png",
       "engine": "chromium-swiftshader"
@@ -1329,34 +1361,18 @@
       "name": "wizard-01-loaded.png",
       "engine": "chromium-swiftshader"
     },
-    {
-      "name": "wizard-01.png",
-      "engine": "chromium-swiftshader"
-    },
     {
       "name": "wizard-02-launch.png",
       "engine": "chromium-swiftshader"
     },
-    {
-      "name": "wizard-02.png",
-      "engine": "chromium-swiftshader"
-    },
     {
       "name": "wizard-03-next-page.png",
       "engine": "chromium-swiftshader"
     },
-    {
-      "name": "wizard-03.png",
-      "engine": "chromium-swiftshader"
-    },
     {
       "name": "wizard-04-back-page.png",
       "engine": "chromium-swiftshader"
     },
-    {
-      "name": "wizard-04-finish.png",
-      "engine": "chromium-swiftshader"
-    },
     {
       "name": "wizard-05-cancel.png",
       "engine": "chromium-swiftshader"
@@ -1404,18 +1420,6 @@
     {
       "name": "xml-06-results.png",
       "engine": "chromium-swiftshader"
-    },
-    {
-      "name": "zoom-pl_editor-00-baseline.png",
-      "engine": "firefox-llvmpipe"
-    },
-    {
-      "name": "zoom-pl_editor-01-zoomed-in-at-P.png",
-      "engine": "firefox-llvmpipe"
-    },
-    {
-      "name": "zoom-pl_editor-02-zoomed-out-back.png",
-      "engine": "firefox-llvmpipe"
     }
   ]
 }

tests/tools/screenshots/compare.ts

@@ -23,6 +23,7 @@ import {
     type EngineFloor,
     type Manifest,
     floorFor,
+    isIgnored,
 } from './config';
 import { diffImages, cluster, drawBoxes, composite, loadPng, savePng, type Box } from './image-ops';
 
@@ -123,6 +124,9 @@ export function classify(root: string, sha: string | null): Report {
     const baselines = baselineIndex(root);
     const resultsDir = path.join(root, RESULTS_DIR);
     const actuals = new Set(listPngs(resultsDir));
+    // Drop excluded screenshots from both sides so they're never compared or counted.
+    for (const name of [...baselines.keys()]) if (isIgnored(name)) baselines.delete(name);
+    for (const name of [...actuals]) if (isIgnored(name)) actuals.delete(name);
     const manifest = loadManifest(root);
     const outDir = path.join(root, DIFF_OUT_DIR);
     fs.mkdirSync(outDir, { recursive: true });

tests/tools/screenshots/config.ts

@@ -9,8 +9,12 @@
  * the npm scripts and CI steps run from).
  */
 
-/** Committed baseline directories, scanned in order. Filenames are the keys. */
-export const BASELINE_DIRS = ['baseline-screenshots', 'e2e/baseline-screenshots'] as const;
+/**
+ * Committed baseline directory. Single dir on purpose: `e2e/baseline-screenshots/`
+ * used to also hold a few names (grid-tab-final, wxgrid-*) that ALSO live here with
+ * different bytes — so they got silently shadowed. Consolidated to one dir.
+ */
+export const BASELINE_DIRS = ['baseline-screenshots'] as const;
 
 /** Where Playwright writes the current run's screenshots (gitignored). */
 export const RESULTS_DIR = 'test-results';
@@ -74,6 +78,20 @@ export const FLOORS: Record<string, EngineFloor> = {
  */
 export const IGNORE_REGIONS: Record<string, Array<{ x: number; y: number; width: number; height: number }>> = {};
 
+/**
+ * Screenshots excluded from comparison entirely (not compared, not counted as
+ * changed/added/removed, not put in the manifest). For nondeterministic captures
+ * that can't be a stable baseline — e.g. `retinascale-01-loaded` is a `fullPage`
+ * HiDPI test whose captured height + DPR scaling vary run-to-run (~60% inter-run
+ * diff observed), a flaky test rather than render noise.
+ */
+export const IGNORE_SCREENSHOTS = new Set<string>(['retinascale-01-loaded.png']);
+
+/** True if a screenshot is excluded from comparison. */
+export function isIgnored(name: string): boolean {
+    return IGNORE_SCREENSHOTS.has(name);
+}
+
 export type ManifestEntry = { name: string; engine: string };
 export type Manifest = { screenshots: ManifestEntry[] };
 

tests/tools/screenshots/gen-manifest.ts

@@ -19,7 +19,7 @@
  */
 import * as fs from 'fs';
 import * as path from 'path';
-import { BASELINE_DIRS, MANIFEST_PATH, type Manifest } from './config';
+import { BASELINE_DIRS, MANIFEST_PATH, isIgnored, type Manifest } from './config';
 
 const CHROMIUM = 'chromium-swiftshader';
 const FIREFOX = 'firefox-llvmpipe';
@@ -78,7 +78,7 @@ function listBaselines(root: string): string[] {
     for (const dir of BASELINE_DIRS) {
         const abs = path.join(root, dir);
         if (!fs.existsSync(abs)) continue;
-        for (const f of fs.readdirSync(abs)) if (f.toLowerCase().endsWith('.png')) names.add(f);
+        for (const f of fs.readdirSync(abs)) if (f.toLowerCase().endsWith('.png') && !isIgnored(f)) names.add(f);
     }
     return [...names].sort();
 }

tests/tools/screenshots/promote.ts

@@ -16,7 +16,7 @@ import * as fs from 'fs';
 import * as os from 'os';
 import * as path from 'path';
 import { execFileSync } from 'child_process';
-import { BASELINE_DIRS, MANIFEST_PATH, floorFor, type Manifest } from './config';
+import { BASELINE_DIRS, MANIFEST_PATH, floorFor, isIgnored, type Manifest } from './config';
 import { diffImages, loadPng } from './image-ops';
 
 function listPngs(dir: string): string[] {
@@ -67,6 +67,9 @@ type Plan = { updated: string[]; added: string[]; unchanged: string[]; removedCa
 function buildPlan(root: string, renderDir: string, manifest?: Manifest): { plan: Plan; apply: () => void } {
     const baselines = baselineIndex(root);
     const rendered = new Set(listPngs(renderDir));
+    // Never promote excluded screenshots (e.g. the flaky retinascale fullPage shot).
+    for (const name of [...rendered]) if (isIgnored(name)) rendered.delete(name);
+    for (const name of [...baselines.keys()]) if (isIgnored(name)) baselines.delete(name);
     const plan: Plan = { updated: [], added: [], unchanged: [], removedCandidates: [] };
     const actions: Array<() => void> = [];