diff --git a/.github/workflows/wasm-build.yml b/.github/workflows/wasm-build.yml index 4619986..7f1c125 100644 --- a/.github/workflows/wasm-build.yml +++ b/.github/workflows/wasm-build.yml @@ -336,22 +336,30 @@ jobs: # seeded by `npm run screenshots:noise`). Posts ONLY on push to main and no-ops # without DISCORD_WEBHOOK_URL (inert on PRs/forks). No extra build — reads the # already-produced test-results (screenshots + perf-*.json). - - name: Screenshot + perf report (on success) + - name: Screenshot gate + perf report (on success) + id: screenshot_report if: success() && inputs.run_tests - continue-on-error: true working-directory: tests env: DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} GH_TOKEN: ${{ github.token }} + # ENFORCING gate: `screenshots:check --fail-on-change` writes report.json and exits + # nonzero on any changed/added/removed vs the committed baselines. We still post the + # report (drift triptychs + perf) so the failure is actionable, then exit with the + # gate's status so a drift fails the build. (A re-render change → re-promote baselines.) run: | - npm run screenshots:check + set +e + npm run screenshots:check -- --fail-on-change; GATE=$? npm run screenshots:report -- --e2e pass + exit $GATE # ON FAILURE (build or e2e): a minimal text-only "CI failed" notice, nothing # else (no images / no comparison). Uses curl, NOT the TS reporter, because on a # build failure the test deps (npm ci) never installed. Main-push only. - name: Discord CI-failure notice - if: failure() && github.ref == 'refs/heads/main' && github.event_name == 'push' + # Build/e2e failure only — NOT a screenshot-gate failure (that already posted the + # drift report), so we don't double-post "CI failed" on drift. + if: failure() && steps.screenshot_report.outcome != 'failure' && github.ref == 'refs/heads/main' && github.event_name == 'push' continue-on-error: true env: DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} diff --git a/scripts/compare-screenshots.sh b/scripts/compare-screenshots.sh deleted file mode 100755 index e0317f4..0000000 --- a/scripts/compare-screenshots.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -# Compare test screenshots with baseline screenshots -# This script compares all PNG files in baseline-screenshots with test-results - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" - -BASELINE_DIR="$PROJECT_ROOT/tests/baseline-screenshots" -TEST_RESULTS_DIR="$PROJECT_ROOT/tests/test-results" - -# Check if directories exist -if [ ! -d "$BASELINE_DIR" ]; then - echo "ERROR: Baseline directory not found: $BASELINE_DIR" - exit 1 -fi - -if [ ! -d "$TEST_RESULTS_DIR" ]; then - echo "ERROR: Test results directory not found: $TEST_RESULTS_DIR" - exit 1 -fi - -echo "=== Screenshot Comparison ===" -echo "Baseline: $BASELINE_DIR" -echo "Test Results: $TEST_RESULTS_DIR" -echo "" - -# Counters -total=0 -identical=0 -different=0 -missing_current=0 -extra_current=0 - -# Compare all baseline screenshots -echo "=== Comparing Baseline Screenshots ===" -for baseline in "$BASELINE_DIR"/*.png; do - filename=$(basename "$baseline") - current="$TEST_RESULTS_DIR/$filename" - total=$((total + 1)) - - if [ ! -f "$current" ]; then - echo "MISSING: $filename (not in test results)" - missing_current=$((missing_current + 1)) - continue - fi - - # Compare using cmp (byte-by-byte comparison) - if cmp -s "$baseline" "$current"; then - identical=$((identical + 1)) - # Only show identical files if verbose - if [ "$1" = "-v" ] || [ "$1" = "--verbose" ]; then - echo "IDENTICAL: $filename" - fi - else - different=$((different + 1)) - - # Get file sizes - baseline_size=$(stat -f%z "$baseline" 2>/dev/null || stat -c%s "$baseline") - current_size=$(stat -f%z "$current" 2>/dev/null || stat -c%s "$current") - diff_bytes=$((current_size - baseline_size)) - - # Calculate percent difference - if [ "$baseline_size" -gt 0 ]; then - diff_pct=$(echo "scale=2; ($diff_bytes * 100) / $baseline_size" | bc) - else - diff_pct="N/A" - fi - - echo "DIFFERENT: $filename (baseline: ${baseline_size}B, current: ${current_size}B, diff: ${diff_bytes}B / ${diff_pct}%)" - fi -done - -# Check for extra files in test results not in baseline -echo "" -echo "=== Checking for Extra Screenshots ===" -for current in "$TEST_RESULTS_DIR"/*.png; do - filename=$(basename "$current") - baseline="$BASELINE_DIR/$filename" - - if [ ! -f "$baseline" ]; then - extra_current=$((extra_current + 1)) - current_size=$(stat -f%z "$current" 2>/dev/null || stat -c%s "$current") - echo "EXTRA: $filename (${current_size}B, not in baseline)" - fi -done - -# Summary -echo "" -echo "=== Summary ===" -echo "Total baseline screenshots: $total" -echo "Identical: $identical" -echo "Different: $different" -echo "Missing from test results: $missing_current" -echo "Extra in test results: $extra_current" - -# Exit with error if there are differences -if [ "$different" -gt 0 ] || [ "$missing_current" -gt 0 ]; then - echo "" - echo "WARNING: There are differences between baseline and test results!" - exit 1 -else - echo "" - echo "SUCCESS: All screenshots match baseline!" - exit 0 -fi diff --git a/scripts/update-baseline-screenshots.sh b/scripts/update-baseline-screenshots.sh deleted file mode 100755 index c0946fa..0000000 --- a/scripts/update-baseline-screenshots.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -# Update baseline screenshots from test results -# Only copies: -# - NEW screenshots (not in baseline) -# - Screenshots with SIGNIFICANT differences (>5% size change) when --all flag used -# -# Usage: -# ./update-baseline-screenshots.sh # Only copy NEW screenshots -# ./update-baseline-screenshots.sh --all # Copy new + significantly different - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -TESTS_DIR="$SCRIPT_DIR/../tests" - -COPY_ALL=0 -THRESHOLD=5 # Percent threshold for "significant" difference - -while [ $# -gt 0 ]; do - case "$1" in - --all) COPY_ALL=1; shift ;; - *) shift ;; - esac -done - -SOURCE_DIR="$TESTS_DIR/test-results" -DEST_DIR="$TESTS_DIR/baseline-screenshots" - -if [ $COPY_ALL -eq 1 ]; then - echo "Mode: Copy NEW + SIGNIFICANTLY DIFFERENT screenshots" -else - echo "Mode: Copy NEW screenshots only (use --all to include significant changes)" -fi - -if [ ! -d "$SOURCE_DIR" ]; then - echo "Error: test-results directory not found at $SOURCE_DIR" - echo "Run 'npm test' first to generate screenshots" - exit 1 -fi - -mkdir -p "$DEST_DIR" - -new_count=0 -updated_count=0 - -for src_file in "$SOURCE_DIR"/*.png; do - [ -f "$src_file" ] || continue - - filename=$(basename "$src_file") - dest_file="$DEST_DIR/$filename" - - if [ ! -f "$dest_file" ]; then - # NEW: Screenshot doesn't exist in baseline - echo "NEW: $filename" - cp "$src_file" "$dest_file" - ((new_count++)) - elif [ $COPY_ALL -eq 1 ]; then - # Check if significantly different - src_size=$(stat -f%z "$src_file" 2>/dev/null || stat -c%s "$src_file") - dest_size=$(stat -f%z "$dest_file" 2>/dev/null || stat -c%s "$dest_file") - - if [ "$dest_size" -eq 0 ]; then - diff_pct=100 - else - diff=$((src_size - dest_size)) - diff=${diff#-} # Absolute value - diff_pct=$((diff * 100 / dest_size)) - fi - - if [ "$diff_pct" -ge "$THRESHOLD" ]; then - echo "UPDATED ($diff_pct% diff): $filename" - cp "$src_file" "$dest_file" - ((updated_count++)) - fi - fi -done - -echo "" -echo "=== Summary ===" -echo "New screenshots added: $new_count" -if [ $COPY_ALL -eq 1 ]; then - echo "Significantly changed: $updated_count" -fi -echo "Total in baseline: $(ls -1 "$DEST_DIR"/*.png 2>/dev/null | wc -l | tr -d ' ')" diff --git a/tests/baseline-screenshots/gerbview-wizard-01.png b/tests/baseline-screenshots/gerbview-wizard-01.png deleted file mode 100644 index df41467..0000000 Binary files a/tests/baseline-screenshots/gerbview-wizard-01.png and /dev/null differ diff --git a/tests/baseline-screenshots/gerbview-wizard-02.png b/tests/baseline-screenshots/gerbview-wizard-02.png deleted file mode 100644 index b980414..0000000 Binary files a/tests/baseline-screenshots/gerbview-wizard-02.png and /dev/null differ diff --git a/tests/baseline-screenshots/gerbview-wizard-03.png b/tests/baseline-screenshots/gerbview-wizard-03.png deleted file mode 100644 index 15d38a5..0000000 Binary files a/tests/baseline-screenshots/gerbview-wizard-03.png and /dev/null differ diff --git a/tests/baseline-screenshots/gerbview-wizard-04-finish.png b/tests/baseline-screenshots/gerbview-wizard-04-finish.png deleted file mode 100644 index c5decf7..0000000 Binary files a/tests/baseline-screenshots/gerbview-wizard-04-finish.png and /dev/null differ diff --git a/tests/baseline-screenshots/popup-03-palette.png b/tests/baseline-screenshots/popup-03-palette.png deleted file mode 100644 index b0cd8c9..0000000 Binary files a/tests/baseline-screenshots/popup-03-palette.png and /dev/null differ diff --git a/tests/baseline-screenshots/retinascale-01-loaded.png b/tests/baseline-screenshots/retinascale-01-loaded.png deleted file mode 100644 index 44ca43f..0000000 Binary files a/tests/baseline-screenshots/retinascale-01-loaded.png and /dev/null differ diff --git a/tests/baseline-screenshots/wizard-01.png b/tests/baseline-screenshots/wizard-01.png deleted file mode 100644 index a4db1e2..0000000 Binary files a/tests/baseline-screenshots/wizard-01.png and /dev/null differ diff --git a/tests/baseline-screenshots/wizard-02.png b/tests/baseline-screenshots/wizard-02.png deleted file mode 100644 index fd993de..0000000 Binary files a/tests/baseline-screenshots/wizard-02.png and /dev/null differ diff --git a/tests/baseline-screenshots/wizard-03.png b/tests/baseline-screenshots/wizard-03.png deleted file mode 100644 index 580794a..0000000 Binary files a/tests/baseline-screenshots/wizard-03.png and /dev/null differ diff --git a/tests/baseline-screenshots/wizard-04-finish.png b/tests/baseline-screenshots/wizard-04-finish.png deleted file mode 100644 index 7620a13..0000000 Binary files a/tests/baseline-screenshots/wizard-04-finish.png and /dev/null differ diff --git a/tests/baseline-screenshots/zoom-pl_editor-00-baseline.png b/tests/baseline-screenshots/zoom-pl_editor-00-baseline.png deleted file mode 100644 index 8aca343..0000000 Binary files a/tests/baseline-screenshots/zoom-pl_editor-00-baseline.png and /dev/null differ diff --git a/tests/baseline-screenshots/zoom-pl_editor-01-zoomed-in-at-P.png b/tests/baseline-screenshots/zoom-pl_editor-01-zoomed-in-at-P.png deleted file mode 100644 index d2a99b3..0000000 Binary files a/tests/baseline-screenshots/zoom-pl_editor-01-zoomed-in-at-P.png and /dev/null differ diff --git a/tests/baseline-screenshots/zoom-pl_editor-02-zoomed-out-back.png b/tests/baseline-screenshots/zoom-pl_editor-02-zoomed-out-back.png deleted file mode 100644 index 223847c..0000000 Binary files a/tests/baseline-screenshots/zoom-pl_editor-02-zoomed-out-back.png and /dev/null differ diff --git a/tests/e2e/baseline-screenshots/grid-tab-final.png b/tests/e2e/baseline-screenshots/grid-tab-final.png deleted file mode 100644 index fcf2c1e..0000000 Binary files a/tests/e2e/baseline-screenshots/grid-tab-final.png and /dev/null differ diff --git a/tests/e2e/baseline-screenshots/wxgrid-controls.png b/tests/e2e/baseline-screenshots/wxgrid-controls.png deleted file mode 100644 index 3421f48..0000000 Binary files a/tests/e2e/baseline-screenshots/wxgrid-controls.png and /dev/null differ diff --git a/tests/e2e/baseline-screenshots/wxgrid-dedicated-page.png b/tests/e2e/baseline-screenshots/wxgrid-dedicated-page.png deleted file mode 100644 index 3421f48..0000000 Binary files a/tests/e2e/baseline-screenshots/wxgrid-dedicated-page.png and /dev/null differ diff --git a/tests/screenshot-manifest.json b/tests/screenshot-manifest.json index 60b7ea3..9291ad4 100644 --- a/tests/screenshot-manifest.json +++ b/tests/screenshot-manifest.json @@ -53,6 +53,22 @@ "name": "13-final.png", "engine": "chromium-swiftshader" }, + { + "name": "3d-viewer-00-board-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "3d-viewer-pic_programmer-render.png", + "engine": "chromium-swiftshader" + }, + { + "name": "3d-viewer-pic_programmer.png", + "engine": "chromium-swiftshader" + }, + { + "name": "3d-viewer-titlebar.png", + "engine": "chromium-swiftshader" + }, { "name": "appearance-00-layers.png", "engine": "chromium-swiftshader" @@ -553,22 +569,6 @@ "name": "gerbview-wizard-00-initial.png", "engine": "firefox-llvmpipe" }, - { - "name": "gerbview-wizard-01.png", - "engine": "firefox-llvmpipe" - }, - { - "name": "gerbview-wizard-02.png", - "engine": "firefox-llvmpipe" - }, - { - "name": "gerbview-wizard-03.png", - "engine": "firefox-llvmpipe" - }, - { - "name": "gerbview-wizard-04-finish.png", - "engine": "firefox-llvmpipe" - }, { "name": "grid-tab-final.png", "engine": "chromium-swiftshader" @@ -793,6 +793,34 @@ "name": "menu-05-all-menus.png", "engine": "chromium-swiftshader" }, + { + "name": "modal-01-border.png", + "engine": "chromium-swiftshader" + }, + { + "name": "modal-02-before-drag.png", + "engine": "chromium-swiftshader" + }, + { + "name": "modal-03-after-drag.png", + "engine": "chromium-swiftshader" + }, + { + "name": "modal-04-before-resize.png", + "engine": "chromium-swiftshader" + }, + { + "name": "modal-05-after-resize.png", + "engine": "chromium-swiftshader" + }, + { + "name": "notebook-01-scrolled.png", + "engine": "chromium-swiftshader" + }, + { + "name": "notebook-02-scrolled-again.png", + "engine": "chromium-swiftshader" + }, { "name": "ownerdrawn-01-loaded.png", "engine": "chromium-swiftshader" @@ -821,6 +849,10 @@ "name": "pcbnew-context-submenu.png", "engine": "chromium-swiftshader" }, + { + "name": "pcbnew-dark-mode-loaded.png", + "engine": "chromium-swiftshader" + }, { "name": "pcbnew-draw-lines-00-before-tool-click.png", "engine": "chromium-swiftshader" @@ -841,6 +873,14 @@ "name": "pcbnew-loaded.png", "engine": "chromium-swiftshader" }, + { + "name": "pcbnew-move-00-before.png", + "engine": "chromium-swiftshader" + }, + { + "name": "pcbnew-move-01-after.png", + "engine": "chromium-swiftshader" + }, { "name": "pcbnew-sidebar-scrollbar-dragged.png", "engine": "chromium-swiftshader" @@ -921,10 +961,6 @@ "name": "popup-03-palette-open.png", "engine": "chromium-swiftshader" }, - { - "name": "popup-03-palette.png", - "engine": "chromium-swiftshader" - }, { "name": "popup-04-color.png", "engine": "chromium-swiftshader" @@ -1033,10 +1069,6 @@ "name": "regions.png", "engine": "chromium-swiftshader" }, - { - "name": "retinascale-01-loaded.png", - "engine": "chromium-swiftshader" - }, { "name": "scrollbar-01-loaded.png", "engine": "chromium-swiftshader" @@ -1329,34 +1361,18 @@ "name": "wizard-01-loaded.png", "engine": "chromium-swiftshader" }, - { - "name": "wizard-01.png", - "engine": "chromium-swiftshader" - }, { "name": "wizard-02-launch.png", "engine": "chromium-swiftshader" }, - { - "name": "wizard-02.png", - "engine": "chromium-swiftshader" - }, { "name": "wizard-03-next-page.png", "engine": "chromium-swiftshader" }, - { - "name": "wizard-03.png", - "engine": "chromium-swiftshader" - }, { "name": "wizard-04-back-page.png", "engine": "chromium-swiftshader" }, - { - "name": "wizard-04-finish.png", - "engine": "chromium-swiftshader" - }, { "name": "wizard-05-cancel.png", "engine": "chromium-swiftshader" @@ -1404,18 +1420,6 @@ { "name": "xml-06-results.png", "engine": "chromium-swiftshader" - }, - { - "name": "zoom-pl_editor-00-baseline.png", - "engine": "firefox-llvmpipe" - }, - { - "name": "zoom-pl_editor-01-zoomed-in-at-P.png", - "engine": "firefox-llvmpipe" - }, - { - "name": "zoom-pl_editor-02-zoomed-out-back.png", - "engine": "firefox-llvmpipe" } ] } diff --git a/tests/tools/screenshots/compare.ts b/tests/tools/screenshots/compare.ts index b77ca0a..1e3f1d4 100644 --- a/tests/tools/screenshots/compare.ts +++ b/tests/tools/screenshots/compare.ts @@ -23,6 +23,7 @@ import { type EngineFloor, type Manifest, floorFor, + isIgnored, } from './config'; import { diffImages, cluster, drawBoxes, composite, loadPng, savePng, type Box } from './image-ops'; @@ -123,6 +124,9 @@ export function classify(root: string, sha: string | null): Report { const baselines = baselineIndex(root); const resultsDir = path.join(root, RESULTS_DIR); const actuals = new Set(listPngs(resultsDir)); + // Drop excluded screenshots from both sides so they're never compared or counted. + for (const name of [...baselines.keys()]) if (isIgnored(name)) baselines.delete(name); + for (const name of [...actuals]) if (isIgnored(name)) actuals.delete(name); const manifest = loadManifest(root); const outDir = path.join(root, DIFF_OUT_DIR); fs.mkdirSync(outDir, { recursive: true }); diff --git a/tests/tools/screenshots/config.ts b/tests/tools/screenshots/config.ts index 2cd0b5d..d805485 100644 --- a/tests/tools/screenshots/config.ts +++ b/tests/tools/screenshots/config.ts @@ -9,8 +9,12 @@ * the npm scripts and CI steps run from). */ -/** Committed baseline directories, scanned in order. Filenames are the keys. */ -export const BASELINE_DIRS = ['baseline-screenshots', 'e2e/baseline-screenshots'] as const; +/** + * Committed baseline directory. Single dir on purpose: `e2e/baseline-screenshots/` + * used to also hold a few names (grid-tab-final, wxgrid-*) that ALSO live here with + * different bytes — so they got silently shadowed. Consolidated to one dir. + */ +export const BASELINE_DIRS = ['baseline-screenshots'] as const; /** Where Playwright writes the current run's screenshots (gitignored). */ export const RESULTS_DIR = 'test-results'; @@ -74,6 +78,20 @@ export const FLOORS: Record = { */ export const IGNORE_REGIONS: Record> = {}; +/** + * Screenshots excluded from comparison entirely (not compared, not counted as + * changed/added/removed, not put in the manifest). For nondeterministic captures + * that can't be a stable baseline — e.g. `retinascale-01-loaded` is a `fullPage` + * HiDPI test whose captured height + DPR scaling vary run-to-run (~60% inter-run + * diff observed), a flaky test rather than render noise. + */ +export const IGNORE_SCREENSHOTS = new Set(['retinascale-01-loaded.png']); + +/** True if a screenshot is excluded from comparison. */ +export function isIgnored(name: string): boolean { + return IGNORE_SCREENSHOTS.has(name); +} + export type ManifestEntry = { name: string; engine: string }; export type Manifest = { screenshots: ManifestEntry[] }; diff --git a/tests/tools/screenshots/gen-manifest.ts b/tests/tools/screenshots/gen-manifest.ts index 35597da..b319bf9 100644 --- a/tests/tools/screenshots/gen-manifest.ts +++ b/tests/tools/screenshots/gen-manifest.ts @@ -19,7 +19,7 @@ */ import * as fs from 'fs'; import * as path from 'path'; -import { BASELINE_DIRS, MANIFEST_PATH, type Manifest } from './config'; +import { BASELINE_DIRS, MANIFEST_PATH, isIgnored, type Manifest } from './config'; const CHROMIUM = 'chromium-swiftshader'; const FIREFOX = 'firefox-llvmpipe'; @@ -78,7 +78,7 @@ function listBaselines(root: string): string[] { for (const dir of BASELINE_DIRS) { const abs = path.join(root, dir); if (!fs.existsSync(abs)) continue; - for (const f of fs.readdirSync(abs)) if (f.toLowerCase().endsWith('.png')) names.add(f); + for (const f of fs.readdirSync(abs)) if (f.toLowerCase().endsWith('.png') && !isIgnored(f)) names.add(f); } return [...names].sort(); } diff --git a/tests/tools/screenshots/promote.ts b/tests/tools/screenshots/promote.ts index efa88c0..a4247ab 100644 --- a/tests/tools/screenshots/promote.ts +++ b/tests/tools/screenshots/promote.ts @@ -16,7 +16,7 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { execFileSync } from 'child_process'; -import { BASELINE_DIRS, MANIFEST_PATH, floorFor, type Manifest } from './config'; +import { BASELINE_DIRS, MANIFEST_PATH, floorFor, isIgnored, type Manifest } from './config'; import { diffImages, loadPng } from './image-ops'; function listPngs(dir: string): string[] { @@ -67,6 +67,9 @@ type Plan = { updated: string[]; added: string[]; unchanged: string[]; removedCa function buildPlan(root: string, renderDir: string, manifest?: Manifest): { plan: Plan; apply: () => void } { const baselines = baselineIndex(root); const rendered = new Set(listPngs(renderDir)); + // Never promote excluded screenshots (e.g. the flaky retinascale fullPage shot). + for (const name of [...rendered]) if (isIgnored(name)) rendered.delete(name); + for (const name of [...baselines.keys()]) if (isIgnored(name)) baselines.delete(name); const plan: Plan = { updated: [], added: [], unchanged: [], removedCandidates: [] }; const actions: Array<() => void> = [];