diff --git a/.github/workflows/ci-ubicloud.yml b/.github/workflows/ci-ubicloud.yml index 2b3ecf1..7e0aa1e 100644 --- a/.github/workflows/ci-ubicloud.yml +++ b/.github/workflows/ci-ubicloud.yml @@ -35,6 +35,9 @@ concurrency: jobs: build: uses: ./.github/workflows/wasm-build.yml + # Pass repo secrets (DISCORD_WEBHOOK_URL) through to the reusable build so the + # post-test step can post the screenshot + perf report on push to main. + secrets: inherit with: # -O1 asyncify shrink — the level we ship (release.yml uses the same -O1). # 3D viewer ON so 3d-viewer.spec.ts has a viewer. diff --git a/.github/workflows/screenshot-changelog.yml b/.github/workflows/screenshot-changelog.yml new file mode 100644 index 0000000..27396b7 --- /dev/null +++ b/.github/workflows/screenshot-changelog.yml @@ -0,0 +1,41 @@ +name: screenshot-changelog + +# Discord trigger B: when a push to main changes committed baseline PNGs, post an +# old | new+boxes | heatmap triptych per changed baseline (ADDED image / REMOVED +# title too). No build, no GPU — it only diffs two git revisions of PNGs that are +# already CI-rendered, so it runs in ~30s. Complements the re-render drift gate in +# wasm-build.yml (which catches un-blessed renders); this is the human-facing feed +# of intentional baseline updates as they land. + +on: + push: + branches: [main] + paths: + - 'tests/baseline-screenshots/**.png' + - 'tests/e2e/baseline-screenshots/**.png' + +concurrency: + group: screenshot-changelog-${{ github.ref }} + cancel-in-progress: false + +jobs: + changelog: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + # HEAD^ is needed to diff the tip commit's baseline changes. (A push of + # multiple commits reports only the tip commit's baseline diff — fine for + # a changelog; baseline refreshes are single commits in practice.) + fetch-depth: 2 + - uses: actions/setup-node@v4 + with: + node-version: 20 + - name: Install screenshot tooling deps + working-directory: tests + run: npm ci + - name: Post baseline changelog to Discord + working-directory: tests + env: + DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} + run: npm run screenshots:changelog diff --git a/.github/workflows/wasm-build.yml b/.github/workflows/wasm-build.yml index 46847a4..aa3a916 100644 --- a/.github/workflows/wasm-build.yml +++ b/.github/workflows/wasm-build.yml @@ -42,6 +42,13 @@ on: description: "Upload the publishable output/ subset as the 'wasm-output' artifact" type: boolean default: false + secrets: + # Declared so this reusable workflow may reference ${{ secrets.DISCORD_WEBHOOK_URL }} + # (an undeclared secret reference is a workflow startup failure). ci-ubicloud.yml + # passes it via `secrets: inherit`; release.yml doesn't (required: false) → the + # screenshot/perf report step just no-ops there. + DISCORD_WEBHOOK_URL: + required: false jobs: build-and-test: @@ -295,11 +302,13 @@ jobs: run: npm run setup:kicad - name: wxWidgets e2e (npm run test) + id: wx_e2e if: inputs.run_tests working-directory: tests run: npm run test - name: KiCad e2e (npm run test:kicad:ci) + id: kicad_e2e if: inputs.run_tests working-directory: tests run: xvfb-run -a npm run test:kicad:ci @@ -314,6 +323,24 @@ jobs: working-directory: tests run: xvfb-run -a npm run test:perf + # Screenshot drift gate + always-on Discord report (screenshots + perf). + # Report-only during rollout: compare.ts exits 0 without --fail-on-change and + # the step is continue-on-error, so it never blocks the build — flip to gating + # once the per-engine floors are calibrated (tests/tools/screenshots/config.ts, + # seeded by `npm run screenshots:noise`). post-discord posts ONLY on push to + # main and no-ops without DISCORD_WEBHOOK_URL (inert on PRs/forks). It reads the + # already-produced test-results (screenshots + perf-*.json) — no extra build. + - name: Screenshot compare + Discord report (screenshots + perf) + if: always() && inputs.run_tests + continue-on-error: true + working-directory: tests + env: + DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} + GH_TOKEN: ${{ github.token }} + run: | + npm run screenshots:check + npm run screenshots:report -- --e2e ${{ (steps.wx_e2e.outcome == 'success' && steps.kicad_e2e.outcome == 'success') && 'pass' || 'fail' }} + - name: Upload test logs & screenshots if: always() && inputs.run_tests uses: actions/upload-artifact@v4 diff --git a/CLAUDE.md b/CLAUDE.md index 8b0da0c..8425e29 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,8 +6,8 @@ A lot of native module have to be compiled to wasm, the most complex is wxwidget The e2e tests are in /tests, with a README and WHATWORKS md files The e2e tests are separated per feature Wxwidgets wasm port has hooks for finding positions of UI elements, tests use that -The test have screenshots that are tracked with git, use compare-screenshots.sh to see what changed -Update test images with /scripts/update-baseline-screenshots.sh when a new image is added +The test screenshots are tracked with git; CI's Linux render is the source of truth (tooling: tests/tools/screenshots/, see its README). +To update baselines, promote a CI run's render (churn-free — only meaningfully-changed images restage): `cd tests && npm run screenshots:promote -- --run `, then commit. `npm run screenshots:check` is the local gate; on each main push CI posts a screenshot-diff + runtime-perf report to Discord. The tests have log files in tests/logs/{wxwidgets/kicad}/{test-name} after each run where the js console and cpp logs are visible Always check screenshots for validating tests Run e2e tests from /tests folder: `npm run test:kicad` or `npm run test:e2e` (not playwright directly) diff --git a/README.md b/README.md index 251b759..690e12b 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,22 @@ npx playwright test menu # Menu tests only See [tests/README.md](tests/README.md) for test documentation. +### Screenshots + +CI's Linux render is the source of truth for baseline screenshots. On each `main` +push, CI compares its render against the committed baselines and posts the diff +(plus the runtime-perf numbers) to Discord. To update baselines after an intended +render change, promote a CI run's render — only meaningfully-changed images +restage, so it stays churn-free: + +```bash +cd tests +npm run screenshots:check # local gate: current vs baselines +npm run screenshots:promote -- --run # adopt a CI run's render, then commit +``` + +See [tests/tools/screenshots/README.md](tests/tools/screenshots/README.md). + ## Current Status - **wxWidgets WASM**: Core widgets working (menus, dialogs, grids, trees, OpenGL) diff --git a/tests/kicad/3d-viewer.spec.ts b/tests/kicad/3d-viewer.spec.ts index 7b12e9a..d206247 100644 --- a/tests/kicad/3d-viewer.spec.ts +++ b/tests/kicad/3d-viewer.spec.ts @@ -118,7 +118,7 @@ test.describe('3D viewer from pcbnew', () => { await waitForPcbnew(page); await loadBoard(page, testLogger); - await page.screenshot({ path: `test-results/3d-viewer-00-board-loaded.png`, scale: 'device' }); + await page.screenshot({ path: `test-results/3d-viewer-00-board-loaded.png`, scale: 'css' }); const glBefore = await countGlCanvases(page); console.log(`[TEST] glcanvas count before opening 3D viewer: ${glBefore}`); @@ -129,7 +129,7 @@ test.describe('3D viewer from pcbnew', () => { // the scene and render a few progressive passes. await page.waitForTimeout(5000); - await page.screenshot({ path: `test-results/3d-viewer-${DEMO.name}.png`, scale: 'device' }); + await page.screenshot({ path: `test-results/3d-viewer-${DEMO.name}.png`, scale: 'css' }); // Read the 3D viewer canvas (the newest glcanvas) directly from its backing // store: copy it onto a 2D canvas with drawImage and sample pixels. This is @@ -312,7 +312,7 @@ test.describe('3D viewer from pcbnew', () => { return all.find((id) => !before.includes(id)) ?? all[all.length - 1] ?? null; }, winsBefore); expect(winId, 'the 3D viewer should open a new top-level window').toBeTruthy(); - await page.screenshot({ path: 'test-results/3d-viewer-titlebar.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/3d-viewer-titlebar.png', scale: 'css' }); // It must have a real DOM title bar (the frames-only fix covers the viewer). const bar = page.locator(`#${winId} .window-titlebar`); diff --git a/tests/kicad/calculator.spec.ts b/tests/kicad/calculator.spec.ts index bf134b4..b51f83d 100644 --- a/tests/kicad/calculator.spec.ts +++ b/tests/kicad/calculator.spec.ts @@ -105,7 +105,7 @@ test.describe('PCB Calculator WASM', () => { const hasRegulatorPanel = labels.some(l => l === 'Calculate'); expect(hasRegulatorPanel, `expected the calculator's default Regulator panel to be live (Calculate button registered). Got: ${JSON.stringify(labels.slice(0, 30))}`).toBe(true); - await page.screenshot({ path: 'test-results/calculator-loaded.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/calculator-loaded.png', scale: 'css' }); }); test('treebook lists expected panels', async ({ page, testLogger }) => { @@ -132,7 +132,7 @@ test.describe('PCB Calculator WASM', () => { void testLogger; await completeFirstRunWizard(page); - await page.screenshot({ path: 'test-results/calculator-before-switch.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/calculator-before-switch.png', scale: 'css' }); const clicked = await clickTreeItem(page, 'Color Code'); expect(clicked, 'expected to find and click the Color Code tree item').toBe(true); @@ -145,6 +145,6 @@ test.describe('PCB Calculator WASM', () => { const onColorCodePanel = labelsAfter.some(l => /Tolerance/i.test(l)); expect(onColorCodePanel, `expected Color Code panel to be active after click; labels: ${JSON.stringify(labelsAfter.slice(0, 40))}`).toBe(true); - await page.screenshot({ path: 'test-results/calculator-color-code.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/calculator-color-code.png', scale: 'css' }); }); }); diff --git a/tests/kicad/eeschema-crosshair.spec.ts b/tests/kicad/eeschema-crosshair.spec.ts index 0786f0e..f0f3a4c 100644 --- a/tests/kicad/eeschema-crosshair.spec.ts +++ b/tests/kicad/eeschema-crosshair.spec.ts @@ -223,14 +223,14 @@ test.describe('Eeschema crosshair modes', () => { await page.mouse.move(probe.x, probe.y); await page.waitForTimeout(600); - const shotSmall = await page.screenshot({ path: 'test-results/eeschema-crosshair-00-small.png', scale: 'device' }); + const shotSmall = await page.screenshot({ path: 'test-results/eeschema-crosshair-00-small.png', scale: 'css' }); // click 1 -> full-window await clickAndSettle(); await expect.poll(tooltipNow, { message: 'one click should advance to Full-Window Crosshairs', timeout: 6000, }).toContain('Full-Window Crosshairs'); - const shotFull = await page.screenshot({ path: 'test-results/eeschema-crosshair-01-full.png', scale: 'device' }); + const shotFull = await page.screenshot({ path: 'test-results/eeschema-crosshair-01-full.png', scale: 'css' }); expect((await compareScreenshots(page, shotSmall, shotFull, diffRegion)).diffPixels, 'full-window crosshair should visibly differ from the small crosshair').toBeGreaterThan(200); @@ -239,7 +239,7 @@ test.describe('Eeschema crosshair modes', () => { await expect.poll(tooltipNow, { message: 'second click should advance to 45 Degree Crosshairs', timeout: 6000, }).toContain('45 Degree Crosshairs'); - const shot45 = await page.screenshot({ path: 'test-results/eeschema-crosshair-02-45.png', scale: 'device' }); + const shot45 = await page.screenshot({ path: 'test-results/eeschema-crosshair-02-45.png', scale: 'css' }); expect((await compareScreenshots(page, shotFull, shot45, diffRegion)).diffPixels, '45-degree crosshair should visibly differ from the full-window crosshair').toBeGreaterThan(200); @@ -248,7 +248,7 @@ test.describe('Eeschema crosshair modes', () => { await expect.poll(tooltipNow, { message: 'third click should cycle back to Small crosshairs', timeout: 6000, }).toContain('Small crosshairs'); - await page.screenshot({ path: 'test-results/eeschema-crosshair-03-small-again.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/eeschema-crosshair-03-small-again.png', scale: 'css' }); const realErrors = testLogger.errors.filter((error: string) => !error.includes('favicon')); expect(realErrors).toEqual([]); diff --git a/tests/kicad/eeschema-load.spec.ts b/tests/kicad/eeschema-load.spec.ts index 626a467..adfc8fc 100644 --- a/tests/kicad/eeschema-load.spec.ts +++ b/tests/kicad/eeschema-load.spec.ts @@ -132,7 +132,7 @@ test.describe('Eeschema schematic load', () => { await page.waitForTimeout(1000); await page.screenshot({ path: 'test-results/eeschema-load-rendered.png', - scale: 'device', + scale: 'css', }); }); }); diff --git a/tests/kicad/eeschema-url-regex.spec.ts b/tests/kicad/eeschema-url-regex.spec.ts index 4f968fb..168f580 100644 --- a/tests/kicad/eeschema-url-regex.spec.ts +++ b/tests/kicad/eeschema-url-regex.spec.ts @@ -110,7 +110,7 @@ test.describe('Eeschema URL-detection regex', () => { await page.screenshot({ path: 'test-results/eeschema-url-regex.png', - scale: 'device', + scale: 'css', }); // The wxRegEx compile failure surfaces two ways: a wxLogError logged to diff --git a/tests/kicad/eeschema.spec.ts b/tests/kicad/eeschema.spec.ts index 8b7d022..1c62ecd 100644 --- a/tests/kicad/eeschema.spec.ts +++ b/tests/kicad/eeschema.spec.ts @@ -229,7 +229,7 @@ async function completeWizard(page: Page): Promise { }, null, { timeout: 150000 }); await page.waitForTimeout(2000); - await page.screenshot({ path: 'test-results/eeschema-wizard-00-initial.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/eeschema-wizard-00-initial.png', scale: 'css' }); for (let i = 1; i <= 10; i++) { let clicked = await clickByLabel(page, 'Next >'); @@ -241,7 +241,7 @@ async function completeWizard(page: Page): Promise { await page.waitForTimeout(500); await page.screenshot({ path: `test-results/eeschema-wizard-${String(i).padStart(2, '0')}-finish.png`, - scale: 'device' + scale: 'css' }); } @@ -251,7 +251,7 @@ async function completeWizard(page: Page): Promise { await page.waitForTimeout(500); await page.screenshot({ path: `test-results/eeschema-wizard-${String(i).padStart(2, '0')}.png`, - scale: 'device' + scale: 'css' }); } @@ -313,7 +313,7 @@ test.describe('Eeschema WASM', () => { path: 'test-results/eeschema-loaded-css.png', scale: 'css' }); - await page.screenshot({ path: 'test-results/eeschema-loaded.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/eeschema-loaded.png', scale: 'css' }); const canvasCount = await page.locator('canvas').count(); expect(canvasCount).toBeGreaterThan(0); @@ -408,7 +408,7 @@ test.describe('Eeschema WASM', () => { await page.screenshot({ path: 'test-results/eeschema-draw-wires-00-before-tool-click.png', - scale: 'device' + scale: 'css' }); expect(await clickByTooltip(page, 'Draw Wires', { elementType: 'tool' })).toBe(true); @@ -429,7 +429,7 @@ test.describe('Eeschema WASM', () => { const afterToolClick = await page.screenshot({ path: 'test-results/eeschema-draw-wires-01-after-click.png', - scale: 'device' + scale: 'css' }); const glCanvasId = await page.evaluate(() => { @@ -475,7 +475,7 @@ test.describe('Eeschema WASM', () => { const afterDrawing = await page.screenshot({ path: 'test-results/eeschema-draw-wires-02-after-drawing.png', - scale: 'device' + scale: 'css' }); const diffRegion: DiffRegion = { diff --git a/tests/kicad/gerbview-print.spec.ts b/tests/kicad/gerbview-print.spec.ts index 303bc08..1beec5a 100644 --- a/tests/kicad/gerbview-print.spec.ts +++ b/tests/kicad/gerbview-print.spec.ts @@ -116,12 +116,12 @@ test.describe('gerbview Print dialog (WASM)', () => { test('no Print Preview button, and the dialog reopens (no wedge)', async ({ page, testLogger }) => { await waitForGerbview(page); - await page.screenshot({ path: 'test-results/gerbview-print-00-loaded.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/gerbview-print-00-loaded.png', scale: 'css' }); // --- Open the Print dialog --- await openPrintDialog(page); expect(await printDialogIsOpen(page), 'Print dialog should be open').toBe(true); - await page.screenshot({ path: 'test-results/gerbview-print-01-dialog.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/gerbview-print-01-dialog.png', scale: 'css' }); // --- The broken "Print Preview" button must be gone in the browser --- const preview = await findByLabel(page, 'Print Preview', { visible: true, exact: true }); @@ -139,7 +139,7 @@ test.describe('gerbview Print dialog (WASM)', () => { await openPrintDialog(page); expect(await printDialogIsOpen(page), 'Print dialog should reopen (no wedge)').toBe(true); await page.waitForTimeout(2500); // let the reopened dialog finish painting before capture - await page.screenshot({ path: 'test-results/gerbview-print-02-reopened.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/gerbview-print-02-reopened.png', scale: 'css' }); expect(hasAbort(testLogger), 'no WASM abort during the flow').toBe(false); }); diff --git a/tests/kicad/gerbview.spec.ts b/tests/kicad/gerbview.spec.ts index f8e0e83..459a96b 100644 --- a/tests/kicad/gerbview.spec.ts +++ b/tests/kicad/gerbview.spec.ts @@ -25,7 +25,7 @@ async function completeWizard(page: Page): Promise { }, null, { timeout: 90000 }); await page.waitForTimeout(2000); - await page.screenshot({ path: 'test-results/gerbview-wizard-00-initial.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/gerbview-wizard-00-initial.png', scale: 'css' }); for (let i = 1; i <= 10; i++) { let clicked = await clickByLabel(page, 'Next >'); @@ -37,7 +37,7 @@ async function completeWizard(page: Page): Promise { await page.waitForTimeout(500); await page.screenshot({ path: `test-results/gerbview-wizard-${String(i).padStart(2, '0')}-finish.png`, - scale: 'device' + scale: 'css' }); } @@ -47,7 +47,7 @@ async function completeWizard(page: Page): Promise { await page.waitForTimeout(500); await page.screenshot({ path: `test-results/gerbview-wizard-${String(i).padStart(2, '0')}.png`, - scale: 'device' + scale: 'css' }); } @@ -65,7 +65,7 @@ test.describe('gerbview WASM', () => { test('app loads, canvas visible, no WASM abort', async ({ page, testLogger }) => { await completeWizard(page); - await page.screenshot({ path: 'test-results/gerbview-01-loaded.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/gerbview-01-loaded.png', scale: 'css' }); expect(hasAbort(testLogger), 'no WASM abort during load').toBe(false); @@ -93,7 +93,7 @@ test.describe('gerbview WASM', () => { }; }); - await page.screenshot({ path: 'test-results/gerbview-02-metrics.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/gerbview-02-metrics.png', scale: 'css' }); expect(metrics.registryTotal, 'registry should be populated').toBeGreaterThan(10); expect(metrics.toolbarCount, 'at least one toolbar should be visible').toBeGreaterThanOrEqual(1); diff --git a/tests/kicad/load-pcb-probe.spec.ts b/tests/kicad/load-pcb-probe.spec.ts index c29dc9a..650e341 100644 --- a/tests/kicad/load-pcb-probe.spec.ts +++ b/tests/kicad/load-pcb-probe.spec.ts @@ -193,7 +193,7 @@ test.describe('PCB load probe', () => { test('inspect File→Open dialog state', async ({ page }) => { await completeWizard(page); - await page.screenshot({ path: 'test-results/probe-00-after-wizard.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/probe-00-after-wizard.png', scale: 'css' }); await dumpMemfs(page, [ '/', @@ -214,7 +214,7 @@ test.describe('PCB load probe', () => { console.log(`[PROBE] File menu clicked: ${fileClicked}`); await page.waitForTimeout(500); - await page.screenshot({ path: 'test-results/probe-01-file-menu-open.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/probe-01-file-menu-open.png', scale: 'css' }); await dumpRegistry(page, 'file-menu-open'); // Click Open menu item (try common label variants) @@ -228,12 +228,12 @@ test.describe('PCB load probe', () => { // is fast but goes through the Asyncify loop. await page.waitForTimeout(3000); - await page.screenshot({ path: 'test-results/probe-02-after-open-click.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/probe-02-after-open-click.png', scale: 'css' }); await dumpRegistry(page, 'after-open-click'); // Wait a bit longer and dump again, in case the dialog paints late await page.waitForTimeout(3000); - await page.screenshot({ path: 'test-results/probe-03-late.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/probe-03-late.png', scale: 'css' }); await dumpRegistry(page, 'late'); // Final check: re-dump MEMFS so we can confirm nothing changed under us diff --git a/tests/kicad/load-pcb.spec.ts b/tests/kicad/load-pcb.spec.ts index 2c62c34..a1c73a4 100644 --- a/tests/kicad/load-pcb.spec.ts +++ b/tests/kicad/load-pcb.spec.ts @@ -107,7 +107,7 @@ function runLoadPcbTest(demo: DemoCfg): void { await waitForPcbnew(page); await page.screenshot({ path: `test-results/load-pcb-${demo.name}-00-pcbnew-ready.png`, - scale: 'device', + scale: 'css', }); // ── Inject .kicad_pcb + .kicad_pro into the dialog's start dir. ── @@ -140,7 +140,7 @@ function runLoadPcbTest(demo: DemoCfg): void { await page.waitForTimeout(1000); await page.screenshot({ path: `test-results/load-pcb-${demo.name}-01-dialog-open.png`, - scale: 'device', + scale: 'css', }); // ── Focus the filename text field, type the name, accept. ────── @@ -189,7 +189,7 @@ function runLoadPcbTest(demo: DemoCfg): void { // fully painted by the time waitForBoardLoaded returns. await page.screenshot({ path: `test-results/load-pcb-${demo.name}.png`, - scale: 'device', + scale: 'css', }); const allLines = [...testLogger.consoleLogs, ...testLogger.errors]; diff --git a/tests/kicad/pcbnew-move.spec.ts b/tests/kicad/pcbnew-move.spec.ts index 8396d1e..91fb22a 100644 --- a/tests/kicad/pcbnew-move.spec.ts +++ b/tests/kicad/pcbnew-move.spec.ts @@ -135,7 +135,7 @@ test.describe('PCBnew move with "m" (#9)', () => { const drawnId = newItems[0].id; const pos0 = await getPos(page, drawnId); - const beforeMove = await page.screenshot({ path: 'test-results/pcbnew-move-00-before.png', scale: 'device' }); + const beforeMove = await page.screenshot({ path: 'test-results/pcbnew-move-00-before.png', scale: 'css' }); // Hover the cursor onto the line and select it, then move with the keyboard. await page.mouse.move(midPoint.x, midPoint.y); @@ -155,7 +155,7 @@ test.describe('PCBnew move with "m" (#9)', () => { await page.keyboard.press('Enter'); await page.waitForTimeout(500); - const afterMove = await page.screenshot({ path: 'test-results/pcbnew-move-01-after.png', scale: 'device' }); + const afterMove = await page.screenshot({ path: 'test-results/pcbnew-move-01-after.png', scale: 'css' }); const pos1 = await getPos(page, drawnId); const dx = pos1.x - pos0.x; diff --git a/tests/kicad/pcbnew.spec.ts b/tests/kicad/pcbnew.spec.ts index f988db6..92bca98 100644 --- a/tests/kicad/pcbnew.spec.ts +++ b/tests/kicad/pcbnew.spec.ts @@ -314,7 +314,7 @@ test.describe('PCBnew WASM', () => { expect(reference.meanChannelDiff, `${reference.name} mean channel diff`).toBeLessThan(region.maxMeanChannelDiff); } - await page.screenshot({ path: 'test-results/pcbnew-loaded.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/pcbnew-loaded.png', scale: 'css' }); const canvasCount = await page.locator('canvas').count(); expect(canvasCount).toBeGreaterThan(0); @@ -409,7 +409,7 @@ test.describe('PCBnew WASM', () => { const beforeToolClick = await page.screenshot({ path: 'test-results/pcbnew-draw-lines-00-before-tool-click.png', - scale: 'device' + scale: 'css' }); expect(await clickByTooltip(page, 'Draw Lines', { elementType: 'tool' })).toBe(true); @@ -430,7 +430,7 @@ test.describe('PCBnew WASM', () => { const afterToolClick = await page.screenshot({ path: 'test-results/pcbnew-draw-lines-01-after-click.png', - scale: 'device' + scale: 'css' }); const glCanvasId = await page.evaluate(() => { @@ -491,7 +491,7 @@ test.describe('PCBnew WASM', () => { const afterDrawing = await page.screenshot({ path: 'test-results/pcbnew-draw-lines-02-after-drawing.png', - scale: 'device' + scale: 'css' }); const diffRegion: DiffRegion = { diff --git a/tests/kicad/pl_editor-load.spec.ts b/tests/kicad/pl_editor-load.spec.ts index dfcfb7a..1964be9 100644 --- a/tests/kicad/pl_editor-load.spec.ts +++ b/tests/kicad/pl_editor-load.spec.ts @@ -98,7 +98,7 @@ test.describe('pl_editor drawing-sheet load', () => { .toMatch(/load-test/i); await page.waitForTimeout(1000); - await page.screenshot({ path: 'test-results/pl_editor-load-rendered.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/pl_editor-load-rendered.png', scale: 'css' }); expect(hasAbort(testLogger), 'no WASM abort during open').toBe(false); }); diff --git a/tests/kicad/pl_editor.spec.ts b/tests/kicad/pl_editor.spec.ts index adc1092..9cd3d6b 100644 --- a/tests/kicad/pl_editor.spec.ts +++ b/tests/kicad/pl_editor.spec.ts @@ -20,7 +20,7 @@ async function completeWizard(page: Page): Promise { await page.waitForFunction(() => !!window.wxElementRegistry, null, { timeout: 90000 }); await page.waitForTimeout(2000); - await page.screenshot({ path: 'test-results/pl_editor-wizard-00-initial.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/pl_editor-wizard-00-initial.png', scale: 'css' }); for (let i = 1; i <= 10; i++) { let clicked = await clickByLabel(page, 'Next >'); @@ -32,7 +32,7 @@ async function completeWizard(page: Page): Promise { await page.waitForTimeout(500); await page.screenshot({ path: `test-results/pl_editor-wizard-${String(i).padStart(2, '0')}-finish.png`, - scale: 'device' + scale: 'css' }); } @@ -42,7 +42,7 @@ async function completeWizard(page: Page): Promise { await page.waitForTimeout(500); await page.screenshot({ path: `test-results/pl_editor-wizard-${String(i).padStart(2, '0')}.png`, - scale: 'device' + scale: 'css' }); } @@ -62,7 +62,7 @@ test.describe('pl_editor WASM', () => { await expect(page.locator('#canvas')).toBeVisible({ timeout: 90000 }); await page.waitForFunction(() => !!window.wxElementRegistry, null, { timeout: 90000 }); await page.waitForTimeout(1500); - await page.screenshot({ path: 'test-results/pl_editor-01-loaded.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/pl_editor-01-loaded.png', scale: 'css' }); expect(hasAbort(testLogger), 'no WASM abort during load').toBe(false); @@ -89,7 +89,7 @@ test.describe('pl_editor WASM', () => { expect(blockingDialogs, 'no setup wizard/dialog should be visible (seed skipped it)').toBe(0); expect(hasAbort(testLogger), 'no WASM abort during launch').toBe(false); - await page.screenshot({ path: 'test-results/pl_editor-02-no-wizard.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/pl_editor-02-no-wizard.png', scale: 'css' }); }); test('File menu exposes Open... and Save As...', async ({ page, testLogger }) => { @@ -99,7 +99,7 @@ test.describe('pl_editor WASM', () => { expect(fileMenuClicked, 'File menubar item should be clickable').toBe(true); await page.waitForTimeout(400); - await page.screenshot({ path: 'test-results/pl_editor-03-file-menu.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/pl_editor-03-file-menu.png', scale: 'css' }); // Menu items are tracked in the "rendered" half of the registry (popup // widgets), not the regular findAll({visible:true}) set. Use findAllRendered @@ -150,7 +150,7 @@ test.describe('pl_editor WASM', () => { // screenshot catches the dialog as a black rectangle. await page.waitForTimeout(600); - await page.screenshot({ path: 'test-results/pl_editor-04-save-as-dialog.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/pl_editor-04-save-as-dialog.png', scale: 'css' }); // The bug: pressing Enter on a folder name treated it as a file and surfaced // "Unable to load /dev file". After the OnOk fix, the dialog should navigate @@ -160,7 +160,7 @@ test.describe('pl_editor WASM', () => { await page.keyboard.press('Enter'); await page.waitForTimeout(900); - await page.screenshot({ path: 'test-results/pl_editor-04b-after-enter.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/pl_editor-04b-after-enter.png', scale: 'css' }); // The wxFileDialog should still be visible — we navigated into /dev, didn't close it. const dialogStillOpen = await page.evaluate(() => { diff --git a/tests/kicad/symbol_editor.spec.ts b/tests/kicad/symbol_editor.spec.ts index 4d764cf..d094df0 100644 --- a/tests/kicad/symbol_editor.spec.ts +++ b/tests/kicad/symbol_editor.spec.ts @@ -26,7 +26,7 @@ async function completeWizard(page: Page): Promise { }, null, { timeout: 90000 }); await page.waitForTimeout(2000); - await page.screenshot({ path: 'test-results/symbol_editor-wizard-00-initial.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/symbol_editor-wizard-00-initial.png', scale: 'css' }); for (let i = 1; i <= 10; i++) { let clicked = await clickByLabel(page, 'Next >'); @@ -38,7 +38,7 @@ async function completeWizard(page: Page): Promise { await page.waitForTimeout(500); await page.screenshot({ path: `test-results/symbol_editor-wizard-${String(i).padStart(2, '0')}-finish.png`, - scale: 'device' + scale: 'css' }); } @@ -48,7 +48,7 @@ async function completeWizard(page: Page): Promise { await page.waitForTimeout(500); await page.screenshot({ path: `test-results/symbol_editor-wizard-${String(i).padStart(2, '0')}.png`, - scale: 'device' + scale: 'css' }); } @@ -66,7 +66,7 @@ test.describe('symbol_editor WASM', () => { test('app loads, canvas visible, no WASM abort', async ({ page, testLogger }) => { await completeWizard(page); - await page.screenshot({ path: 'test-results/symbol_editor-01-loaded.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/symbol_editor-01-loaded.png', scale: 'css' }); expect(hasAbort(testLogger), 'no WASM abort during load').toBe(false); @@ -94,7 +94,7 @@ test.describe('symbol_editor WASM', () => { }; }); - await page.screenshot({ path: 'test-results/symbol_editor-02-metrics.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/symbol_editor-02-metrics.png', scale: 'css' }); expect(metrics.registryTotal, 'registry should be populated').toBeGreaterThan(10); expect(metrics.toolbarCount, 'at least one toolbar should be visible').toBeGreaterThanOrEqual(1); diff --git a/tests/kicad/utils/screenshot-compare.ts b/tests/kicad/utils/screenshot-compare.ts index 7de658a..72482ee 100644 --- a/tests/kicad/utils/screenshot-compare.ts +++ b/tests/kicad/utils/screenshot-compare.ts @@ -151,7 +151,7 @@ export async function completeWizard(page: Page, opts: { screenshots?: boolean } await page.waitForTimeout(2000); if (opts.screenshots) { - await page.screenshot({ path: 'test-results/wizard-00-initial.png', scale: 'device' }); + await page.screenshot({ path: 'test-results/wizard-00-initial.png', scale: 'css' }); } for (let i = 1; i <= 10; i++) { @@ -164,7 +164,7 @@ export async function completeWizard(page: Page, opts: { screenshots?: boolean } await page.waitForTimeout(500); await page.screenshot({ path: `test-results/wizard-${String(i).padStart(2, '0')}-finish.png`, - scale: 'device' + scale: 'css' }); } @@ -176,7 +176,7 @@ export async function completeWizard(page: Page, opts: { screenshots?: boolean } if (opts.screenshots) { await page.screenshot({ path: `test-results/wizard-${String(i).padStart(2, '0')}.png`, - scale: 'device' + scale: 'css' }); } } diff --git a/tests/package-lock.json b/tests/package-lock.json index 9d9c932..2123e21 100644 --- a/tests/package-lock.json +++ b/tests/package-lock.json @@ -10,8 +10,13 @@ "devDependencies": { "@playwright/test": "^1.40.0", "@types/node": "^24.10.1", + "@types/pixelmatch": "^5.2.6", + "@types/pngjs": "^6.0.5", "esbuild": "^0.28.0", + "pixelmatch": "^5.3.0", + "pngjs": "^7.0.0", "serve": "^14.2.0", + "tsx": "^4.22.4", "typescript": "^5.9.3", "yjs": "^13.6.31" } @@ -484,6 +489,26 @@ "undici-types": "~7.16.0" } }, + "node_modules/@types/pixelmatch": { + "version": "5.2.6", + "resolved": "https://registry.npmjs.org/@types/pixelmatch/-/pixelmatch-5.2.6.tgz", + "integrity": "sha512-wC83uexE5KGuUODn6zkm9gMzTwdY5L0chiK+VrKcDfEjzxh1uadlWTvOmAbCpnM9zx/Ww3f8uKlYQVnO/TrqVg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/pngjs": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/@types/pngjs/-/pngjs-6.0.5.tgz", + "integrity": "sha512-0k5eKfrA83JOZPppLtS2C7OUtyNAl2wKNxfyYl9Q5g9lPkgBl/9hNyAu6HuEH2J4XmIv2znEpkDd0SaZVxW6iQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@zeit/schemas": { "version": "2.36.0", "resolved": "https://registry.npmjs.org/@zeit/schemas/-/schemas-2.36.0.tgz", @@ -1285,6 +1310,29 @@ "dev": true, "license": "MIT" }, + "node_modules/pixelmatch": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.3.0.tgz", + "integrity": "sha512-o8mkY4E/+LNUf6LzX96ht6k6CEDi65k9G2rjMtBe9Oo+VPKSvl+0GKHuH/AlG+GA5LPG/i5hrekkxUc3s2HU+Q==", + "dev": true, + "license": "ISC", + "dependencies": { + "pngjs": "^6.0.0" + }, + "bin": { + "pixelmatch": "bin/pixelmatch" + } + }, + "node_modules/pixelmatch/node_modules/pngjs": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-6.0.0.tgz", + "integrity": "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.13.0" + } + }, "node_modules/playwright": { "version": "1.57.0", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.57.0.tgz", @@ -1317,6 +1365,16 @@ "node": ">=18" } }, + "node_modules/pngjs": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz", + "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.19.0" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -1557,6 +1615,40 @@ "node": ">=8" } }, + "node_modules/tsx": { + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.22.4.tgz", + "integrity": "sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tsx/node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/type-fest": { "version": "2.19.0", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz", diff --git a/tests/package.json b/tests/package.json index 1638324..83aa729 100644 --- a/tests/package.json +++ b/tests/package.json @@ -42,13 +42,24 @@ "test:asyncify:firefox": "playwright test --config=playwright-asyncify.config.ts --project=firefox", "test:asyncify:chrome": "playwright test --config=playwright-asyncify.config.ts --project=chromium --headed", "test:asyncify:safari": "playwright test --config=playwright-asyncify.config.ts --project=webkit", - "test:asyncify:all": "npm run test:asyncify:firefox && npm run test:asyncify:safari && npm run test:asyncify:chrome" + "test:asyncify:all": "npm run test:asyncify:firefox && npm run test:asyncify:safari && npm run test:asyncify:chrome", + "screenshots:check": "tsx tools/screenshots/compare.ts", + "screenshots:promote": "tsx tools/screenshots/promote.ts", + "screenshots:noise": "tsx tools/screenshots/noise.ts", + "screenshots:report": "tsx tools/screenshots/post-discord.ts", + "screenshots:changelog": "tsx tools/screenshots/changelog.ts", + "screenshots:manifest": "tsx tools/screenshots/gen-manifest.ts" }, "devDependencies": { "@playwright/test": "^1.40.0", "@types/node": "^24.10.1", + "@types/pixelmatch": "^5.2.6", + "@types/pngjs": "^6.0.5", "esbuild": "^0.28.0", + "pixelmatch": "^5.3.0", + "pngjs": "^7.0.0", "serve": "^14.2.0", + "tsx": "^4.22.4", "typescript": "^5.9.3", "yjs": "^13.6.31" } diff --git a/tests/screenshot-manifest.json b/tests/screenshot-manifest.json new file mode 100644 index 0000000..60b7ea3 --- /dev/null +++ b/tests/screenshot-manifest.json @@ -0,0 +1,1421 @@ +{ + "_note": "engine tags are best-effort (gen-manifest.ts); the name list is authoritative. Refine engines after calibration.", + "screenshots": [ + { + "name": "01-loading.png", + "engine": "chromium-swiftshader" + }, + { + "name": "03-after-load.png", + "engine": "chromium-swiftshader" + }, + { + "name": "04-controls-tab.png", + "engine": "chromium-swiftshader" + }, + { + "name": "05-text-input-tab.png", + "engine": "chromium-swiftshader" + }, + { + "name": "06-text-input-typed.png", + "engine": "chromium-swiftshader" + }, + { + "name": "07-drawing-tab.png", + "engine": "chromium-swiftshader" + }, + { + "name": "08-drawing-done.png", + "engine": "chromium-swiftshader" + }, + { + "name": "09-lists-tab.png", + "engine": "chromium-swiftshader" + }, + { + "name": "10-lists-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "10b-choice-selected.png", + "engine": "chromium-swiftshader" + }, + { + "name": "11-file-menu.png", + "engine": "chromium-swiftshader" + }, + { + "name": "12-help-menu.png", + "engine": "chromium-swiftshader" + }, + { + "name": "13-final.png", + "engine": "chromium-swiftshader" + }, + { + "name": "appearance-00-layers.png", + "engine": "chromium-swiftshader" + }, + { + "name": "appearance-01-objects.png", + "engine": "chromium-swiftshader" + }, + { + "name": "appearance-02-nets.png", + "engine": "chromium-swiftshader" + }, + { + "name": "appearance-03-layers-again.png", + "engine": "chromium-swiftshader" + }, + { + "name": "appearance-10-layers-scrolled.png", + "engine": "chromium-swiftshader" + }, + { + "name": "appearance-11-layers-scrolled-back.png", + "engine": "chromium-swiftshader" + }, + { + "name": "appearance-20-objects-scrolled.png", + "engine": "chromium-swiftshader" + }, + { + "name": "aui-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "aui-02-panels.png", + "engine": "chromium-swiftshader" + }, + { + "name": "aui-03-close-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "aui-04-dragged.png", + "engine": "chromium-swiftshader" + }, + { + "name": "aui-05-multi-panel.png", + "engine": "chromium-swiftshader" + }, + { + "name": "aui-resize-01-hover.png", + "engine": "chromium-swiftshader" + }, + { + "name": "aui-resize-02-mid-drag-live.png", + "engine": "chromium-swiftshader" + }, + { + "name": "aui-resize-03-mid-drag.png", + "engine": "chromium-swiftshader" + }, + { + "name": "aui-resize-04-after.png", + "engine": "chromium-swiftshader" + }, + { + "name": "auinotebook-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "auinotebook-02-tab-switch.png", + "engine": "chromium-swiftshader" + }, + { + "name": "auinotebook-03-add-tab.png", + "engine": "chromium-swiftshader" + }, + { + "name": "auinotebook-04-remove-tab.png", + "engine": "chromium-swiftshader" + }, + { + "name": "auinotebook-05-tab-style.png", + "engine": "chromium-swiftshader" + }, + { + "name": "bitmapbuttons-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "bitmapbuttons-02-toolbar-click.png", + "engine": "chromium-swiftshader" + }, + { + "name": "bitmapbuttons-03-toggle.png", + "engine": "chromium-swiftshader" + }, + { + "name": "bitmapbuttons-04-multi-toggle.png", + "engine": "chromium-swiftshader" + }, + { + "name": "bitmapbuttons-05-enable-toggle.png", + "engine": "chromium-swiftshader" + }, + { + "name": "bitmapbuttons-06-shapes.png", + "engine": "chromium-swiftshader" + }, + { + "name": "bitmapbuttons-07-artprovider.png", + "engine": "chromium-swiftshader" + }, + { + "name": "bitmask.png", + "engine": "chromium-swiftshader" + }, + { + "name": "calculator-before-switch.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "calculator-color-code.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "calculator-loaded.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "calendar-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "calendar-02-select-date.png", + "engine": "chromium-swiftshader" + }, + { + "name": "calendar-03-next-month.png", + "engine": "chromium-swiftshader" + }, + { + "name": "calendar-04-prev-month.png", + "engine": "chromium-swiftshader" + }, + { + "name": "calendar-05-today.png", + "engine": "chromium-swiftshader" + }, + { + "name": "clipboard-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "clipboard-02-copy-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "clipboard-03-paste-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "clipboard-04-check-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "clipboard-05-clear-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "clipboard-06-full-flow.png", + "engine": "chromium-swiftshader" + }, + { + "name": "collapsible-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "collapsible-02-panes.png", + "engine": "chromium-swiftshader" + }, + { + "name": "collapsible-03-expanded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "collapsible-04-expand-all.png", + "engine": "chromium-swiftshader" + }, + { + "name": "collapsible-05-collapse-all.png", + "engine": "chromium-swiftshader" + }, + { + "name": "contextmenu-01-open.png", + "engine": "chromium-swiftshader" + }, + { + "name": "coroutine-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "coroutine-02-summary.png", + "engine": "chromium-swiftshader" + }, + { + "name": "coroutine-nested-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "coroutine-nested-02-summary.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dataview-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dataview-02-list-populated.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dataview-03-list-selected.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dataview-04-list-add.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dataview-05-tree-tab.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dataview-06-tree-selected.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dataview-07-tree-expanded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dataview-08-tree-collapsed.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dataview-09-column-click.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dataview-10-scrolled.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dataviewvirtual-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dataviewvirtual-02-large-dataset.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dataviewvirtual-03-scroll.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dataviewvirtual-04-selection.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dataviewvirtual-05-zone-manager.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialog-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialog-02-info-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialog-03-yesno-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialog-04-error-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialog-05-custom-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialogs-00-initial.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialogs-01-tab-selected.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialogs-custom-closed.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialogs-custom-open.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialogs-full-flow.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialogs-msgbox-error-closed.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialogs-msgbox-error-open.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialogs-msgbox-info-closed.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialogs-msgbox-info-open.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialogs-msgbox-yesno-closed.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialogs-msgbox-yesno-open.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialogs-timer-initial.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialogs-timer-multiple.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialogs-timer-running.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialogs-timer-started.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dialogs-timer-stopped.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dnd-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dnd-02-handlers.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dnd-03-dragenter.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dnd-04-dragleave.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dnd-05-drop.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dnd-06-file-written.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dnd-07-event-fired.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dnd-08-multiple-files.png", + "engine": "chromium-swiftshader" + }, + { + "name": "dnd-09-with-file.png", + "engine": "chromium-swiftshader" + }, + { + "name": "earlysize-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "earlysize-02-result.png", + "engine": "chromium-swiftshader" + }, + { + "name": "eeschema-crosshair-00-small.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "eeschema-crosshair-01-full.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "eeschema-crosshair-02-45.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "eeschema-crosshair-03-small-again.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "eeschema-draw-wires-00-before-tool-click.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "eeschema-draw-wires-01-after-click.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "eeschema-draw-wires-02-after-drawing.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "eeschema-load-rendered.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "eeschema-loaded-css.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "eeschema-loaded.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "eeschema-url-regex.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "eeschema-wizard-00-initial.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "filedialog-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "filedialog-02-open-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "filedialog-03-save-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "filedialog-04-multiple-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "filedialog-05-all-buttons.png", + "engine": "chromium-swiftshader" + }, + { + "name": "filedlg-folder-nav.png", + "engine": "chromium-swiftshader" + }, + { + "name": "fontenum.png", + "engine": "chromium-swiftshader" + }, + { + "name": "gerbview-01-loaded.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "gerbview-02-metrics.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "gerbview-print-00-loaded.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "gerbview-print-01-dialog.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "gerbview-print-02-reopened.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "gerbview-wizard-00-initial.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "gerbview-wizard-01.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "gerbview-wizard-02.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "gerbview-wizard-03.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "gerbview-wizard-04-finish.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "grid-tab-final.png", + "engine": "chromium-swiftshader" + }, + { + "name": "gridedit-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "gridedit-02-select-cell.png", + "engine": "chromium-swiftshader" + }, + { + "name": "gridedit-03-edit-cell.png", + "engine": "chromium-swiftshader" + }, + { + "name": "gridedit-04-add-row.png", + "engine": "chromium-swiftshader" + }, + { + "name": "gridedit-05-delete-row.png", + "engine": "chromium-swiftshader" + }, + { + "name": "gridrenderers-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "gridrenderers-02-color-cells.png", + "engine": "chromium-swiftshader" + }, + { + "name": "gridrenderers-03-icon-text.png", + "engine": "chromium-swiftshader" + }, + { + "name": "gridrenderers-04-striped.png", + "engine": "chromium-swiftshader" + }, + { + "name": "gridrenderers-05-checkbox-toggle.png", + "engine": "chromium-swiftshader" + }, + { + "name": "htmlwin-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "htmlwin-02-basic-content.png", + "engine": "chromium-swiftshader" + }, + { + "name": "htmlwin-03-tables.png", + "engine": "chromium-swiftshader" + }, + { + "name": "htmlwin-04-long-content.png", + "engine": "chromium-swiftshader" + }, + { + "name": "htmlwin-05-kicad-about.png", + "engine": "chromium-swiftshader" + }, + { + "name": "htmlwin-06-link-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "htmlwin-07-scrolled.png", + "engine": "chromium-swiftshader" + }, + { + "name": "htmlwin-08a-basic.png", + "engine": "chromium-swiftshader" + }, + { + "name": "htmlwin-08b-tables.png", + "engine": "chromium-swiftshader" + }, + { + "name": "htmlwin-08c-about.png", + "engine": "chromium-swiftshader" + }, + { + "name": "infobar-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "infobar-02-info.png", + "engine": "chromium-swiftshader" + }, + { + "name": "infobar-03-warning.png", + "engine": "chromium-swiftshader" + }, + { + "name": "infobar-04-error.png", + "engine": "chromium-swiftshader" + }, + { + "name": "infobar-05-dismiss.png", + "engine": "chromium-swiftshader" + }, + { + "name": "layout-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "layout-02-splitter.png", + "engine": "chromium-swiftshader" + }, + { + "name": "layout-03-sash-dragged.png", + "engine": "chromium-swiftshader" + }, + { + "name": "layout-04-scrolled-left.png", + "engine": "chromium-swiftshader" + }, + { + "name": "layout-05-scrolled-right.png", + "engine": "chromium-swiftshader" + }, + { + "name": "layout-06-combined.png", + "engine": "chromium-swiftshader" + }, + { + "name": "listctrl-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "listctrl-02-virtual.png", + "engine": "chromium-swiftshader" + }, + { + "name": "listctrl-03-columns.png", + "engine": "chromium-swiftshader" + }, + { + "name": "listctrl-04-selected.png", + "engine": "chromium-swiftshader" + }, + { + "name": "listctrl-05-scrolled.png", + "engine": "chromium-swiftshader" + }, + { + "name": "load-pcb-microwave-00-pcbnew-ready.png", + "engine": "chromium-swiftshader" + }, + { + "name": "load-pcb-microwave-01-dialog-open.png", + "engine": "chromium-swiftshader" + }, + { + "name": "load-pcb-microwave.png", + "engine": "chromium-swiftshader" + }, + { + "name": "load-pcb-pic_programmer-00-pcbnew-ready.png", + "engine": "chromium-swiftshader" + }, + { + "name": "load-pcb-pic_programmer-01-dialog-open.png", + "engine": "chromium-swiftshader" + }, + { + "name": "load-pcb-pic_programmer.png", + "engine": "chromium-swiftshader" + }, + { + "name": "logerror-01-initial.png", + "engine": "chromium-swiftshader" + }, + { + "name": "logerror-02-before-error.png", + "engine": "chromium-swiftshader" + }, + { + "name": "logerror-03-single-error-dialog.png", + "engine": "chromium-swiftshader" + }, + { + "name": "logerror-04-multiple-errors-dialog.png", + "engine": "chromium-swiftshader" + }, + { + "name": "logerror-05-mixed-levels.png", + "engine": "chromium-swiftshader" + }, + { + "name": "maximize-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "maximize-02-fullscreen.png", + "engine": "chromium-swiftshader" + }, + { + "name": "maximize-03-canvas.png", + "engine": "chromium-swiftshader" + }, + { + "name": "menu-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "menu-02-menubar.png", + "engine": "chromium-swiftshader" + }, + { + "name": "menu-03-file-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "menu-04-edit-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "menu-05-all-menus.png", + "engine": "chromium-swiftshader" + }, + { + "name": "ownerdrawn-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "ownerdrawn-02-layer.png", + "engine": "chromium-swiftshader" + }, + { + "name": "ownerdrawn-03-font.png", + "engine": "chromium-swiftshader" + }, + { + "name": "ownerdrawn-04-icon.png", + "engine": "chromium-swiftshader" + }, + { + "name": "ownerdrawn-05-log.png", + "engine": "chromium-swiftshader" + }, + { + "name": "pcbnew-context-menu.png", + "engine": "chromium-swiftshader" + }, + { + "name": "pcbnew-context-submenu.png", + "engine": "chromium-swiftshader" + }, + { + "name": "pcbnew-draw-lines-00-before-tool-click.png", + "engine": "chromium-swiftshader" + }, + { + "name": "pcbnew-draw-lines-01-after-click.png", + "engine": "chromium-swiftshader" + }, + { + "name": "pcbnew-draw-lines-02-after-drawing.png", + "engine": "chromium-swiftshader" + }, + { + "name": "pcbnew-loaded-css.png", + "engine": "chromium-swiftshader" + }, + { + "name": "pcbnew-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "pcbnew-sidebar-scrollbar-dragged.png", + "engine": "chromium-swiftshader" + }, + { + "name": "pcbnew-sidebar-scrollbar.png", + "engine": "chromium-swiftshader" + }, + { + "name": "pickers-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "pickers-02-colors.png", + "engine": "chromium-swiftshader" + }, + { + "name": "pickers-03-font.png", + "engine": "chromium-swiftshader" + }, + { + "name": "pickers-04-preview.png", + "engine": "chromium-swiftshader" + }, + { + "name": "pl_editor-01-loaded.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "pl_editor-02-no-wizard.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "pl_editor-03-file-menu.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "pl_editor-04-save-as-dialog.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "pl_editor-04b-after-enter.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "pl_editor-context-menu.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "pl_editor-load-rendered.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "pl_editor-scrollbar-dragged.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "pl_editor-scrollbar.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "pl_editor-wizard-00-initial.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "popup-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "popup-02-status.png", + "engine": "chromium-swiftshader" + }, + { + "name": "popup-03-palette-dismissed.png", + "engine": "chromium-swiftshader" + }, + { + "name": "popup-03-palette-open.png", + "engine": "chromium-swiftshader" + }, + { + "name": "popup-03-palette.png", + "engine": "chromium-swiftshader" + }, + { + "name": "popup-04-color.png", + "engine": "chromium-swiftshader" + }, + { + "name": "popup-05-positioning.png", + "engine": "chromium-swiftshader" + }, + { + "name": "popup-06-log.png", + "engine": "chromium-swiftshader" + }, + { + "name": "print-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "print-02-preview-panel.png", + "engine": "chromium-swiftshader" + }, + { + "name": "print-03-browser-print-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "print-04-preview-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "print-05-page-setup-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "print-06-print-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "print-07-callbacks.png", + "engine": "chromium-swiftshader" + }, + { + "name": "print-08-all-buttons.png", + "engine": "chromium-swiftshader" + }, + { + "name": "printpreview-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "printpreview-02-preview-area.png", + "engine": "chromium-swiftshader" + }, + { + "name": "printpreview-03-preview-frame.png", + "engine": "chromium-swiftshader" + }, + { + "name": "printpreview-04-page-setup.png", + "engine": "chromium-swiftshader" + }, + { + "name": "printpreview-05-settings.png", + "engine": "chromium-swiftshader" + }, + { + "name": "probe-00-after-wizard.png", + "engine": "chromium-swiftshader" + }, + { + "name": "probe-01-file-menu-open.png", + "engine": "chromium-swiftshader" + }, + { + "name": "probe-02-after-open-click.png", + "engine": "chromium-swiftshader" + }, + { + "name": "probe-03-late.png", + "engine": "chromium-swiftshader" + }, + { + "name": "propgrid-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "propgrid-02-categories.png", + "engine": "chromium-swiftshader" + }, + { + "name": "propgrid-03-selected.png", + "engine": "chromium-swiftshader" + }, + { + "name": "propgrid-04-manager.png", + "engine": "chromium-swiftshader" + }, + { + "name": "radiogroups-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "radiogroups-02-selections.png", + "engine": "chromium-swiftshader" + }, + { + "name": "regions.png", + "engine": "chromium-swiftshader" + }, + { + "name": "retinascale-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "scrollbar-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "scrollbar-02-dragged.png", + "engine": "chromium-swiftshader" + }, + { + "name": "searchctrl-01-visible.png", + "engine": "chromium-swiftshader" + }, + { + "name": "searchctrl-02-typing.png", + "engine": "chromium-swiftshader" + }, + { + "name": "searchctrl-03-after-enter.png", + "engine": "chromium-swiftshader" + }, + { + "name": "selectheight-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "selectheight-02-result.png", + "engine": "chromium-swiftshader" + }, + { + "name": "specialized-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "specialized-02-treebook-initial.png", + "engine": "chromium-swiftshader" + }, + { + "name": "specialized-03-treebook-subpage.png", + "engine": "chromium-swiftshader" + }, + { + "name": "specialized-04-treebook-expand.png", + "engine": "chromium-swiftshader" + }, + { + "name": "specialized-05-bitmapcombo-dropdown.png", + "engine": "chromium-swiftshader" + }, + { + "name": "specialized-06-bitmapcombo-select.png", + "engine": "chromium-swiftshader" + }, + { + "name": "specialized-07-rearrange-list.png", + "engine": "chromium-swiftshader" + }, + { + "name": "specialized-08-get-order.png", + "engine": "chromium-swiftshader" + }, + { + "name": "spinctrl-01-visible.png", + "engine": "chromium-swiftshader" + }, + { + "name": "spinctrl-02-after-click.png", + "engine": "chromium-swiftshader" + }, + { + "name": "stc-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "stc-02-python-default.png", + "engine": "chromium-swiftshader" + }, + { + "name": "stc-03-drc-mode.png", + "engine": "chromium-swiftshader" + }, + { + "name": "stc-04-plain-mode.png", + "engine": "chromium-swiftshader" + }, + { + "name": "stc-05-insert-sample.png", + "engine": "chromium-swiftshader" + }, + { + "name": "stc-06-cleared.png", + "engine": "chromium-swiftshader" + }, + { + "name": "stc-07-line-numbers-toggle.png", + "engine": "chromium-swiftshader" + }, + { + "name": "stc-08-folded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "stc-09-typed.png", + "engine": "chromium-swiftshader" + }, + { + "name": "stc-10a-python.png", + "engine": "chromium-swiftshader" + }, + { + "name": "stc-10b-drc.png", + "engine": "chromium-swiftshader" + }, + { + "name": "stc-10c-python-again.png", + "engine": "chromium-swiftshader" + }, + { + "name": "symbol_editor-01-loaded.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "symbol_editor-02-metrics.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "symbol_editor-wizard-00-initial.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "textdecor.png", + "engine": "chromium-swiftshader" + }, + { + "name": "threadpool-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "timer-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "timer-02-started.png", + "engine": "chromium-swiftshader" + }, + { + "name": "timer-03-ticked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "timer-04-stopped.png", + "engine": "chromium-swiftshader" + }, + { + "name": "timer-05-fast-started.png", + "engine": "chromium-swiftshader" + }, + { + "name": "timer-06-fast-running.png", + "engine": "chromium-swiftshader" + }, + { + "name": "timer-07-fast-stopped.png", + "engine": "chromium-swiftshader" + }, + { + "name": "timer-08-reset.png", + "engine": "chromium-swiftshader" + }, + { + "name": "toolbar-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "toolbar-02-buttons.png", + "engine": "chromium-swiftshader" + }, + { + "name": "toolbar-03-new-clicked.png", + "engine": "chromium-swiftshader" + }, + { + "name": "toolbar-04-zoom.png", + "engine": "chromium-swiftshader" + }, + { + "name": "toolbar-05-toggle-on.png", + "engine": "chromium-swiftshader" + }, + { + "name": "toolbar-06-toggle-off.png", + "engine": "chromium-swiftshader" + }, + { + "name": "toolbar-07-statusbar.png", + "engine": "chromium-swiftshader" + }, + { + "name": "toolbar-08-all-buttons.png", + "engine": "chromium-swiftshader" + }, + { + "name": "tree-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "tree-02-hierarchy.png", + "engine": "chromium-swiftshader" + }, + { + "name": "tree-03-selected.png", + "engine": "chromium-swiftshader" + }, + { + "name": "tree-04-expanded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "tree-05-collapsed.png", + "engine": "chromium-swiftshader" + }, + { + "name": "tree-06-added.png", + "engine": "chromium-swiftshader" + }, + { + "name": "tree-07-deleted.png", + "engine": "chromium-swiftshader" + }, + { + "name": "validators-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "validators-02-text.png", + "engine": "chromium-swiftshader" + }, + { + "name": "validators-03-integer.png", + "engine": "chromium-swiftshader" + }, + { + "name": "validators-04-float.png", + "engine": "chromium-swiftshader" + }, + { + "name": "validators-05-netname.png", + "engine": "chromium-swiftshader" + }, + { + "name": "validators-06-button.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wasmedge-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wasmedge-02-filesystem.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wasmedge-03-threading.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wasmedge-04-fonts.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wasmedge-05-clipboard.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wasmedge-06-memory.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wasmedge-07-runall.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wasmedge-08-log.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wizard-00-initial.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wizard-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wizard-01.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wizard-02-launch.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wizard-02.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wizard-03-next-page.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wizard-03.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wizard-04-back-page.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wizard-04-finish.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wizard-05-cancel.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wxgrid-01-tab.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wxgrid-02-selection.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wxgrid-03-editing.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wxgrid-controls.png", + "engine": "chromium-swiftshader" + }, + { + "name": "wxgrid-dedicated-page.png", + "engine": "chromium-swiftshader" + }, + { + "name": "xml-01-loaded.png", + "engine": "chromium-swiftshader" + }, + { + "name": "xml-02-input.png", + "engine": "chromium-swiftshader" + }, + { + "name": "xml-03-parse.png", + "engine": "chromium-swiftshader" + }, + { + "name": "xml-04-traverse.png", + "engine": "chromium-swiftshader" + }, + { + "name": "xml-05-create.png", + "engine": "chromium-swiftshader" + }, + { + "name": "xml-06-results.png", + "engine": "chromium-swiftshader" + }, + { + "name": "zoom-pl_editor-00-baseline.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "zoom-pl_editor-01-zoomed-in-at-P.png", + "engine": "firefox-llvmpipe" + }, + { + "name": "zoom-pl_editor-02-zoomed-out-back.png", + "engine": "firefox-llvmpipe" + } + ] +} diff --git a/tests/tools/screenshots/README.md b/tests/tools/screenshots/README.md new file mode 100644 index 0000000..35e40a0 --- /dev/null +++ b/tests/tools/screenshots/README.md @@ -0,0 +1,40 @@ +# Screenshot regression + Discord review tooling + +One comparison engine + a churn-free updater + a Discord reporter for the e2e +screenshots. Design and rationale: `~/.claude/plans/…snowglobe.md` (or ask). + +**Source of truth = CI's Linux render.** The dev never authors baselines on the +Mac (Mac fonts/GL ≠ CI). Instead, CI renders on every push; when a render change +is intentional you *promote* CI's artifact into the committed baselines. The +environment isn't pinned — if the host's Mesa/fonts drift, the gate lights up in +Discord and you just re-promote (broad + low-intensity change ⇒ likely drift). + +## Files +- `config.ts` — thresholds, baseline dirs, per-engine floors (calibrate!), clustering knobs. +- `image-ops.ts` — PNG load/save, pixelmatch diff (AA-excluded), connected-component boxes, triptych compositing, size-cap resize. +- `compare.ts` — the engine: classify baselines vs `test-results/` → `test-results/screenshot-diff/report.json` + triptych/heatmap PNGs. `--pair` diffs two files. +- `promote.ts` — churn-free updater: pull a CI run's shots (`gh run download`) or `--from DIR`; overwrite a baseline only when pixels differ beyond the floor (verbatim bytes, no re-encode) → no git churn. +- `perf-report.ts` — renders the track-only runtime-perf table (loadMs/openMs/FPS) with Δ vs the previous main run (fetched via `gh`). +- `post-discord.ts` — the always-on CI-on-main report: SHA + e2e status + perf table, then screenshot triptychs (batched, size-capped, flood-collapsed). +- `changelog.ts` — Discord trigger B: git-history diff of committed baseline PNGs (no build/GPU). +- `noise.ts` — calibration: diff two identical-input renders → per-engine noise floor. +- `gen-manifest.ts` — regenerate `screenshot-manifest.json` (canonical name list + best-effort engine tag) by scanning the committed baselines + specs. + +## npm scripts (run from `tests/`) +``` +npm run screenshots:check # gate: baselines vs test-results → report.json (exit 0; add --fail-on-change to gate) +npm run screenshots:promote -- --run # churn-free re-baseline from a CI run (or --from DIR) +npm run screenshots:report -- --e2e pass # post the CI report to Discord (main+push only; needs DISCORD_WEBHOOK_URL) +npm run screenshots:changelog # post the baseline changelog (main+push only) +npm run screenshots:noise -- run1/ run2/ # calibrate floors +npm run screenshots:manifest # regenerate the manifest (--check to verify it's fresh) +``` + +## Activation checklist +- [x] `screenshot-manifest.json` generated (name list authoritative; engine tags best-effort until calibration). +- [x] `scale:'device'`→`'css'` normalized (no-op at CI's DSF=1). +1. Add the `DISCORD_WEBHOOK_URL` repo secret — until then everything is inert. +2. Calibrate: run the suite twice in CI, `screenshots:noise` the two dirs, set `FLOORS` in `config.ts`. +3. First re-baseline: `promote` a clean CI run's render, commit (expect a big, one-time chrome-font diff vs the Mac baselines). +4. Delete the old `scripts/{compare,update-baseline}-screenshots.sh`. +5. Once floors are proven stable, flip the gate to `--fail-on-change`. diff --git a/tests/tools/screenshots/changelog.ts b/tests/tools/screenshots/changelog.ts new file mode 100644 index 0000000..175319d --- /dev/null +++ b/tests/tools/screenshots/changelog.ts @@ -0,0 +1,155 @@ +/** + * Baseline changelog (Discord trigger B) — no build, no GPU. + * + * On push to main, diff the committed baseline PNGs between two commits and post + * a "Baseline changelog" to Discord: a triptych (old | new+boxes | heatmap) for + * each CHANGED file, the image for each ADDED, and a titled line for each REMOVED. + * "added / removed" only mean anything as a git-history diff, which is why this is + * separate from the re-render drift gate. + * + * CLI (from tests/): + * tsx tools/screenshots/changelog.ts [--base REV] [--head REV] [--dry-run] [--force] + */ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { execFileSync } from 'child_process'; +import { BASELINE_DIRS, DIFF_OUT_DIR, floorFor } from './config'; +import { comparePair, type Report } from './compare'; +import { loadPng, savePng } from './image-ops'; +import { buildAttachments, paginate, postMessage } from './post-discord'; + +function git(root: string, args: string[]): string { + return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim(); +} + +function gitBlob(root: string, rev: string, repoPath: string): Buffer | null { + try { + return execFileSync('git', ['-C', root, 'show', `${rev}:${repoPath}`], { maxBuffer: 1 << 28 }); + } catch { + return null; + } +} + +function parseArgs(argv: string[]): Record { + const out: Record = {}; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--dry-run') out.dryRun = true; + else if (a === '--force') out.force = true; + else if (a === '--base') out.base = argv[++i]; + else if (a === '--head') out.head = argv[++i]; + } + return out; +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + const cwd = process.cwd(); + const repoRoot = git(cwd, ['rev-parse', '--show-toplevel']); + const head = (args.head as string) || 'HEAD'; + let base = (args.base as string) || ''; + if (!base) { + try { + base = git(cwd, ['rev-parse', `${head}^`]); + } catch { + console.log('[changelog] no parent commit — nothing to diff'); + return; + } + } + + // Repo-relative baseline dir prefixes (e.g. tests/baseline-screenshots). + const prefixes = BASELINE_DIRS.map((d) => path.relative(repoRoot, path.join(cwd, d))); + const diff = git(repoRoot, ['diff', '--name-status', base, head, '--', ...prefixes]); + if (!diff) { + console.log('[changelog] no baseline changes between', base.slice(0, 7), 'and', head); + return; + } + + const outDir = path.join(cwd, DIFF_OUT_DIR); + fs.mkdirSync(outDir, { recursive: true }); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'changelog-')); + const report: Report = { + generatedFor: process.env.GITHUB_SHA || head, + changed: [], + added: [], + removed: [], + unchangedCount: 0, + driftLikely: false, + }; + + for (const line of diff.split('\n')) { + const parts = line.split('\t'); + const status = parts[0][0]; // M/A/D/R + const repoPath = parts[status === 'R' ? 2 : 1]; + const oldPath = status === 'R' ? parts[1] : repoPath; + const name = path.basename(repoPath); + + if (status === 'A' || status === 'R') { + const buf = gitBlob(repoRoot, head, repoPath); + if (buf) { + const rel = path.join(DIFF_OUT_DIR, `${name}.added.png`); + fs.writeFileSync(path.join(cwd, rel), buf); + report.added.push({ name, image: rel }); + } + if (status === 'R') report.removed.push({ name: path.basename(oldPath) }); + } else if (status === 'D') { + report.removed.push({ name }); + } else { + // Modified: triptych old vs new. + const oldBuf = gitBlob(repoRoot, base, repoPath); + const newBuf = gitBlob(repoRoot, head, repoPath); + if (!oldBuf || !newBuf) continue; + const oldFile = path.join(tmp, `old-${name}`); + const newFile = path.join(tmp, `new-${name}`); + fs.writeFileSync(oldFile, oldBuf); + fs.writeFileSync(newFile, newBuf); + const { result, heatmap, triptych } = comparePair(loadPng(oldFile), loadPng(newFile), name, floorFor(name)); + const triptychRel = path.join(DIFF_OUT_DIR, `${name}.triptych.png`); + const heatmapRel = path.join(DIFF_OUT_DIR, `${name}.heatmap.png`); + savePng(path.join(cwd, triptychRel), triptych); + savePng(path.join(cwd, heatmapRel), heatmap); + report.changed.push({ ...result, triptych: triptychRel, heatmap: heatmapRel }); + } + } + report.changed.sort((a, b) => b.changedRatio - a.changedRatio); + + const sha7 = (process.env.GITHUB_SHA || head).slice(0, 7); + let subject = ''; + try { + subject = git(cwd, ['log', '-1', '--pretty=%s', head]); + } catch { /* ignore */ } + const header = + `🗂️ **Baseline changelog** · \`${sha7}\`` + + (subject ? `\n> ${subject}` : '') + + `\n${report.changed.length} changed, ${report.added.length} added, ${report.removed.length} removed` + + (report.removed.length ? '\n➖ REMOVED: ' + report.removed.map((r) => `\`${r.name}\``).join(', ') : ''); + + const { files, notes } = buildAttachments(cwd, report); + const messages = paginate(header + (notes.length ? '\n' + notes.join('\n') : ''), files); + + const isMainPush = process.env.GITHUB_REF === 'refs/heads/main' && process.env.GITHUB_EVENT_NAME === 'push'; + if (args.dryRun || (!args.force && !isMainPush)) { + for (const [i, m] of messages.entries()) { + console.log(`--- message ${i + 1}/${messages.length} (${m.files.length} files) ---`); + if (m.content) console.log(m.content); + for (const f of m.files) console.log(` [attach] ${f.name} (${f.buffer.length} bytes)`); + } + if (!args.dryRun) console.log('[changelog] not a push to main — not posting'); + return; + } + const webhook = process.env.DISCORD_WEBHOOK_URL; + if (!webhook) { + console.log('[changelog] DISCORD_WEBHOOK_URL unset — skipping'); + return; + } + for (const m of messages) await postMessage(webhook, m); + console.log(`[changelog] posted ${messages.length} message(s)`); +} + +if (require.main === module) { + main().catch((e) => { + console.error(`[changelog] ${e.message}`); + process.exitCode = 1; + }); +} diff --git a/tests/tools/screenshots/compare.ts b/tests/tools/screenshots/compare.ts new file mode 100644 index 0000000..b77ca0a --- /dev/null +++ b/tests/tools/screenshots/compare.ts @@ -0,0 +1,238 @@ +/** + * The one screenshot comparison engine + its CLI. + * + * Run modes (from the `tests/` directory): + * tsx tools/screenshots/compare.ts # gate: baselines vs test-results + * tsx tools/screenshots/compare.ts --fail-on-change # same, but exit 1 on any change + * tsx tools/screenshots/compare.ts --pair OLD NEW --name N --out DIR # diff two files + * + * The gate mode classifies every screenshot into changed / added / removed / + * unchanged, writes per-change triptych + heatmap PNGs and a machine-readable + * report.json into DIFF_OUT_DIR, and (unless --fail-on-change) exits 0 so it can + * run report-only first. post-discord.ts and the changelog workflow import the + * exported helpers rather than re-deriving the diff. + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { PNG } from 'pngjs'; +import { + BASELINE_DIRS, + RESULTS_DIR, + DIFF_OUT_DIR, + MANIFEST_PATH, + type EngineFloor, + type Manifest, + floorFor, +} from './config'; +import { diffImages, cluster, drawBoxes, composite, loadPng, savePng, type Box } from './image-ops'; + +export type PairVerdict = 'unchanged' | 'changed'; + +export type PairResult = { + name: string; + verdict: PairVerdict; + dimsMatch: boolean; + diffPixels: number; + changedRatio: number; + meanChannelDiff: number; + boxes: Box[]; + /** heuristic: a broad, low-intensity change smells like host env drift, not a code regression */ + driftHint: 'regression-like' | 'drift-like' | null; +}; + +/** Broad + low-intensity ⇒ likely environment drift rather than a localized regression. */ +function driftHint(changed: boolean, changedRatio: number, meanChannelDiff: number): PairResult['driftHint'] { + if (!changed) return null; + return changedRatio > 0.05 && meanChannelDiff < 4 ? 'drift-like' : 'regression-like'; +} + +/** Compare two decoded PNGs; returns the verdict/metrics plus the heatmap and old|new+boxes|heatmap triptych. */ +export function comparePair( + oldPng: PNG, + newPng: PNG, + name: string, + floor: EngineFloor +): { result: PairResult; heatmap: PNG; triptych: PNG } { + const d = diffImages(oldPng, newPng); + const changed = !d.dimsMatch || d.changedRatio > floor.changedRatio; + const boxes = changed ? cluster(d.mask, d.width, d.height) : []; + const triptych = composite([oldPng, drawBoxes(newPng, boxes), d.heatmap]); + return { + result: { + name, + verdict: changed ? 'changed' : 'unchanged', + dimsMatch: d.dimsMatch, + diffPixels: d.diffPixels, + changedRatio: d.changedRatio, + meanChannelDiff: d.meanChannelDiff, + boxes, + driftHint: driftHint(changed, d.changedRatio, d.meanChannelDiff), + }, + heatmap: d.heatmap, + triptych, + }; +} + +export type ChangedEntry = PairResult & { triptych: string; heatmap: string }; +export type Report = { + generatedFor: string | null; + changed: ChangedEntry[]; + added: Array<{ name: string; image: string }>; + removed: Array<{ name: string }>; + unchangedCount: number; + /** many changes, mostly drift-like ⇒ probably a host Mesa/font refresh; re-promote rather than debug */ + driftLikely: boolean; +}; + +const DRIFT_BULK = 20; + +function listPngs(dir: string): string[] { + if (!fs.existsSync(dir)) return []; + return fs.readdirSync(dir).filter((f) => f.toLowerCase().endsWith('.png')); +} + +/** basename → absolute baseline path (first BASELINE_DIRS entry wins; collisions warned). */ +function baselineIndex(root: string): Map { + const index = new Map(); + for (const dir of BASELINE_DIRS) { + const abs = path.join(root, dir); + for (const name of listPngs(abs)) { + if (index.has(name)) { + console.warn(`[compare] duplicate baseline name ${name} (${dir} shadowed by earlier dir)`); + continue; + } + index.set(name, path.join(abs, name)); + } + } + return index; +} + +function loadManifest(root: string): Manifest | undefined { + const p = path.join(root, MANIFEST_PATH); + if (!fs.existsSync(p)) return undefined; + try { + return JSON.parse(fs.readFileSync(p, 'utf8')) as Manifest; + } catch (e) { + console.warn(`[compare] could not parse ${MANIFEST_PATH}: ${(e as Error).message}`); + return undefined; + } +} + +/** Full gate run: classify baselines vs the current run's screenshots. */ +export function classify(root: string, sha: string | null): Report { + const baselines = baselineIndex(root); + const resultsDir = path.join(root, RESULTS_DIR); + const actuals = new Set(listPngs(resultsDir)); + const manifest = loadManifest(root); + const outDir = path.join(root, DIFF_OUT_DIR); + fs.mkdirSync(outDir, { recursive: true }); + + const report: Report = { + generatedFor: sha, + changed: [], + added: [], + removed: [], + unchangedCount: 0, + driftLikely: false, + }; + + // Changed / unchanged / removed: iterate the committed baselines. + for (const [name, baselinePath] of baselines) { + if (!actuals.has(name)) { + // Missing output. Only call it REMOVED when the manifest expects it — otherwise + // a flaky/OOM'd/skipped spec that simply didn't write a PNG would masquerade as + // an intentional removal. (The stronger "did the spec actually run" cross-check + // against the Playwright JSON report lands with the manifest work.) + if (manifest?.screenshots.some((e) => e.name === name)) { + report.removed.push({ name }); + } + continue; + } + const { result, heatmap, triptych } = comparePair( + loadPng(baselinePath), + loadPng(path.join(resultsDir, name)), + name, + floorFor(name, manifest) + ); + if (result.verdict === 'unchanged') { + report.unchangedCount++; + continue; + } + const triptychRel = path.join(DIFF_OUT_DIR, `${name}.triptych.png`); + const heatmapRel = path.join(DIFF_OUT_DIR, `${name}.heatmap.png`); + savePng(path.join(root, triptychRel), triptych); + savePng(path.join(root, heatmapRel), heatmap); + report.changed.push({ ...result, triptych: triptychRel, heatmap: heatmapRel }); + } + + // Added: an actual with no committed baseline. + for (const name of actuals) { + if (baselines.has(name)) continue; + const imageRel = path.join(DIFF_OUT_DIR, `${name}.added.png`); + fs.copyFileSync(path.join(resultsDir, name), path.join(root, imageRel)); + report.added.push({ name, image: imageRel }); + } + + const driftLike = report.changed.filter((c) => c.driftHint === 'drift-like').length; + report.driftLikely = report.changed.length >= DRIFT_BULK && driftLike * 2 >= report.changed.length; + + report.changed.sort((a, b) => b.changedRatio - a.changedRatio); + report.added.sort((a, b) => a.name.localeCompare(b.name)); + report.removed.sort((a, b) => a.name.localeCompare(b.name)); + return report; +} + +function parseArgs(argv: string[]): Record { + const out: Record = {}; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--fail-on-change') out.failOnChange = true; + else if (a === '--pair') { + out.oldPath = argv[++i]; + out.newPath = argv[++i]; + } else if (a === '--name') out.name = argv[++i]; + else if (a === '--out') out.out = argv[++i]; + else if (a === '--sha') out.sha = argv[++i]; + } + return out; +} + +function runPair(args: Record): void { + const name = (args.name as string) || 'pair'; + const outDir = (args.out as string) || path.join(RESULTS_DIR, 'screenshot-diff'); + fs.mkdirSync(outDir, { recursive: true }); + const oldPng = fs.existsSync(args.oldPath as string) + ? loadPng(args.oldPath as string) + : new PNG({ width: 1, height: 1 }); + const newPng = fs.existsSync(args.newPath as string) + ? loadPng(args.newPath as string) + : new PNG({ width: 1, height: 1 }); + const { result, heatmap, triptych } = comparePair(oldPng, newPng, name, floorFor(name)); + savePng(path.join(outDir, `${name}.triptych.png`), triptych); + savePng(path.join(outDir, `${name}.heatmap.png`), heatmap); + process.stdout.write(JSON.stringify(result, null, 2) + '\n'); +} + +function main(): void { + const args = parseArgs(process.argv.slice(2)); + if (args.oldPath && args.newPath) { + runPair(args); + return; + } + const root = process.cwd(); + const report = classify(root, (args.sha as string) || process.env.GITHUB_SHA || null); + fs.writeFileSync(path.join(root, DIFF_OUT_DIR, 'report.json'), JSON.stringify(report, null, 2)); + const { changed, added, removed, unchangedCount, driftLikely } = report; + console.log( + `[compare] changed=${changed.length} added=${added.length} removed=${removed.length} ` + + `unchanged=${unchangedCount}${driftLikely ? ' (looks like environment drift → re-promote)' : ''}` + ); + for (const c of changed.slice(0, 10)) { + console.log(` CHANGED ${c.name} ratio=${(c.changedRatio * 100).toFixed(3)}% ${c.driftHint}`); + } + if (args.failOnChange && (changed.length || added.length || removed.length)) { + process.exitCode = 1; + } +} + +if (require.main === module) main(); diff --git a/tests/tools/screenshots/config.ts b/tests/tools/screenshots/config.ts new file mode 100644 index 0000000..2cd0b5d --- /dev/null +++ b/tests/tools/screenshots/config.ts @@ -0,0 +1,84 @@ +/** + * Central config for the screenshot regression + review tooling. + * + * The comparison engine (compare.ts), the churn-free updater (promote.ts) and + * the Discord reporter (post-discord.ts) all read their knobs from here so + * there is exactly one place to tune thresholds and paths. + * + * Paths are relative to the `tests/` directory (that is the working directory + * the npm scripts and CI steps run from). + */ + +/** Committed baseline directories, scanned in order. Filenames are the keys. */ +export const BASELINE_DIRS = ['baseline-screenshots', 'e2e/baseline-screenshots'] as const; + +/** Where Playwright writes the current run's screenshots (gitignored). */ +export const RESULTS_DIR = 'test-results'; + +/** Where compare.ts writes diff/heatmap/triptych artifacts (gitignored). */ +export const DIFF_OUT_DIR = 'test-results/screenshot-diff'; + +/** The manifest that records every expected screenshot + which engine renders it. */ +export const MANIFEST_PATH = 'screenshot-manifest.json'; + +/** + * pixelmatch per-pixel settings. + * - `threshold` is the YIQ perceptual distance (0..1) below which two pixels + * are considered equal. 0.1 tolerates gamma/AA jitter but catches real colour + * change. + * - `includeAA: false` (the pixelmatch default) means anti-aliased edge pixels + * are DETECTED AND IGNORED — exactly the sub-pixel/AA noise the old + * `maxDiff>16` counter was dominated by (see screenshot-compare.ts:36-43). + */ +export const PIXELMATCH = { threshold: 0.1, includeAA: false } as const; + +/** Colour pixelmatch paints a real (non-AA) diff pixel with — the mask reads this back. */ +export const DIFF_COLOR: [number, number, number] = [255, 0, 0]; + +/** Connected-component clustering ("where to look") parameters. */ +export const CLUSTER = { + dilate: 2, // grow the mask so fragmented glyph pixels merge into one box + minBoxArea: 16, // drop specks smaller than this (px²) + maxBoxes: 6, // draw at most this many (largest-first) red boxes + boxColor: [255, 0, 0] as [number, number, number], +} as const; + +/** Horizontal montage layout for the old | new+boxes | heatmap triptych. */ +export const TRIPTYCH = { + gap: 8, + bg: [24, 24, 24, 255] as [number, number, number, number], + padFill: [40, 0, 40, 255] as [number, number, number, number], // magenta pad on dim-mismatch +} as const; + +/** + * Per-engine verdict floors. A screenshot is CHANGED when its AA-excluded + * changed-pixel ratio exceeds `changedRatio`. `meanChannelGuard` is recorded + * for the drift-vs-regression heuristic (broad + low-intensity ⇒ environment + * drift, not a localized regression), not for the primary verdict. + * + * NOTE: these are PLACEHOLDERS. `npm run screenshots:noise` renders the suite + * twice on the CI host and prints the real intra-CI floor per engine; set + * `changedRatio ≈ measured × 3` from that and commit the numbers here. + */ +export type EngineFloor = { changedRatio: number; meanChannelGuard: number }; + +export const FLOORS: Record = { + 'firefox-llvmpipe': { changedRatio: 0.002, meanChannelGuard: 2.0 }, + 'chromium-swiftshader': { changedRatio: 0.002, meanChannelGuard: 2.0 }, + default: { changedRatio: 0.002, meanChannelGuard: 2.0 }, +}; + +/** + * Optional per-file rectangles to ignore before diffing (e.g. a live clock). + * Keyed by screenshot filename. Empty for now. + */ +export const IGNORE_REGIONS: Record> = {}; + +export type ManifestEntry = { name: string; engine: string }; +export type Manifest = { screenshots: ManifestEntry[] }; + +/** Resolve the verdict floor for a screenshot via the manifest's engine tag. */ +export function floorFor(name: string, manifest?: Manifest): EngineFloor { + const engine = manifest?.screenshots.find((e) => e.name === name)?.engine; + return (engine && FLOORS[engine]) || FLOORS.default; +} diff --git a/tests/tools/screenshots/gen-manifest.ts b/tests/tools/screenshots/gen-manifest.ts new file mode 100644 index 0000000..35597da --- /dev/null +++ b/tests/tools/screenshots/gen-manifest.ts @@ -0,0 +1,124 @@ +/** + * Generate tests/screenshot-manifest.json — the canonical list of expected + * screenshots + the engine that renders each. + * + * The NAME list is authoritative (it's the committed baseline set) and is what + * lets compare/promote tell an intentional REMOVAL from a flaky/absent render. + * The ENGINE tag is best-effort (attributed by scanning which spec writes each + * `test-results/` and which project runs that spec) and only feeds the + * per-engine floors — refine after calibration. + * + * Engine routing (from the two playwright configs): + * e2e/*.spec.ts → chromium-swiftshader (npm run test — wx suite) + * kicad/*.spec.ts → firefox-llvmpipe, or chromium-swiftshader if the + * spec is in PCBNEW_FAMILY_SPECS (chromium-ci on CI) + * web/*.spec.ts → firefox-llvmpipe (web config, --project=firefox) + * + * CLI (from tests/): tsx tools/screenshots/gen-manifest.ts [--check] + * --check exits 1 if the committed manifest is stale (for CI hygiene). + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { BASELINE_DIRS, MANIFEST_PATH, type Manifest } from './config'; + +const CHROMIUM = 'chromium-swiftshader'; +const FIREFOX = 'firefox-llvmpipe'; +const DEFAULT_ENGINE = CHROMIUM; // baseline-screenshots is dominated by the wx suite + +function listSpecs(dir: string): string[] { + const out: string[] = []; + if (!fs.existsSync(dir)) return out; + for (const e of fs.readdirSync(dir, { withFileTypes: true })) { + const p = path.join(dir, e.name); + if (e.isDirectory()) out.push(...listSpecs(p)); + // Only spec files — a screenshot literal in a util (e.g. completeWizard's + // wizard-*) would be attributed to the util's dir, not its real caller. + // Leaving those unmatched lets them fall to the correct chromium default. + else if (e.name.endsWith('.spec.ts')) out.push(p); + } + return out; +} + +/** pcbnew-family spec basenames (routed to chromium-ci on CI), read from the config. */ +function pcbnewFamily(root: string): Set { + const cfg = fs.readFileSync(path.join(root, 'playwright-kicad.config.ts'), 'utf8'); + const block = cfg.match(/PCBNEW_FAMILY_SPECS\s*=\s*\[([\s\S]*?)\]/)?.[1] ?? ''; + return new Set([...block.matchAll(/'[^']*?([\w.-]+\.spec\.ts)'/g)].map((m) => m[1])); +} + +function engineForSpec(root: string, specPath: string, family: Set): string { + const rel = path.relative(root, specPath); + const base = path.basename(specPath); + if (rel.startsWith('e2e/')) return CHROMIUM; + if (rel.startsWith('web/')) return FIREFOX; + if (rel.startsWith('kicad/')) return family.has(base) ? CHROMIUM : FIREFOX; + return DEFAULT_ENGINE; +} + +/** Build prefix → engine from every `test-results/` literal in the specs. */ +function prefixEngineMap(root: string, family: Set): Array<{ prefix: string; engine: string }> { + const map = new Map(); + for (const dir of ['e2e', 'kicad', 'web']) { + for (const spec of listSpecs(path.join(root, dir))) { + const engine = engineForSpec(root, spec, family); + const content = fs.readFileSync(spec, 'utf8'); + for (const m of content.matchAll(/test-results\/([A-Za-z0-9_-]+)/g)) { + const prefix = m[1]; + // First writer wins; a chromium spec shouldn't be overridden by a later firefox one for the same literal. + if (!map.has(prefix)) map.set(prefix, engine); + } + } + } + // Longest prefix first so the most specific match wins. + return [...map.entries()].map(([prefix, engine]) => ({ prefix, engine })).sort((a, b) => b.prefix.length - a.prefix.length); +} + +function listBaselines(root: string): string[] { + const names = new Set(); + for (const dir of BASELINE_DIRS) { + const abs = path.join(root, dir); + if (!fs.existsSync(abs)) continue; + for (const f of fs.readdirSync(abs)) if (f.toLowerCase().endsWith('.png')) names.add(f); + } + return [...names].sort(); +} + +function main(): void { + const check = process.argv.includes('--check'); + const root = process.cwd(); + const family = pcbnewFamily(root); + const prefixes = prefixEngineMap(root, family); + + let unmatched = 0; + const screenshots = listBaselines(root).map((name) => { + const stem = name.replace(/\.png$/i, ''); + const hit = prefixes.find((p) => stem === p.prefix || stem.startsWith(p.prefix)); + if (!hit) unmatched++; + return { name, engine: hit?.engine ?? DEFAULT_ENGINE }; + }); + + const manifest: Manifest & { _note: string } = { + _note: 'engine tags are best-effort (gen-manifest.ts); the name list is authoritative. Refine engines after calibration.', + screenshots, + }; + const json = JSON.stringify(manifest, null, 2) + '\n'; + const outPath = path.join(root, MANIFEST_PATH); + + const dist = screenshots.reduce>((d, s) => ((d[s.engine] = (d[s.engine] ?? 0) + 1), d), {}); + console.log(`[manifest] ${screenshots.length} screenshots; engines=${JSON.stringify(dist)}; default-assigned=${unmatched}`); + + if (check) { + const current = fs.existsSync(outPath) ? fs.readFileSync(outPath, 'utf8') : ''; + if (current !== json) { + console.error('[manifest] STALE — run `npm run screenshots:manifest` and commit'); + process.exitCode = 1; + } else { + console.log('[manifest] up to date'); + } + return; + } + fs.writeFileSync(outPath, json); + console.log(`[manifest] wrote ${MANIFEST_PATH}`); +} + +if (require.main === module) main(); diff --git a/tests/tools/screenshots/image-ops.ts b/tests/tools/screenshots/image-ops.ts new file mode 100644 index 0000000..fa1c686 --- /dev/null +++ b/tests/tools/screenshots/image-ops.ts @@ -0,0 +1,256 @@ +/** + * Reusable pixel operations for the screenshot tooling: PNG load/save, padding, + * the pixelmatch-backed diff, connected-component clustering ("where to look"), + * box drawing, and horizontal compositing for the triptych. + * + * All images are handled as pngjs PNGs whose `.data` is a length `w*h*4` RGBA + * Buffer, regardless of the source PNG colour type (pngjs normalizes to RGBA). + */ +import * as fs from 'fs'; +import { PNG } from 'pngjs'; +import pixelmatch from 'pixelmatch'; +import { PIXELMATCH, DIFF_COLOR, CLUSTER, TRIPTYCH } from './config'; + +export type Box = { x: number; y: number; width: number; height: number; area: number }; + +export type DiffResult = { + width: number; + height: number; + dimsMatch: boolean; + /** AA-excluded changed-pixel count (from pixelmatch). */ + diffPixels: number; + /** diffPixels / (width*height). */ + changedRatio: number; + /** mean |Δ| over every RGBA channel sample of the whole frame (matches the legacy metric). */ + meanChannelDiff: number; + /** pixelmatch heatmap: dimmed base + red diffs / yellow AA. */ + heatmap: PNG; + /** boolean mask (1 = real, non-AA changed pixel) for clustering. */ + mask: Uint8Array; +}; + +export function loadPng(file: string): PNG { + return PNG.sync.read(fs.readFileSync(file)); +} + +export function savePng(file: string, png: PNG): void { + fs.writeFileSync(file, PNG.sync.write(png)); +} + +/** New PNG of `w`×`h` filled with `fill` (RGBA), with `src` blitted at top-left. */ +export function padTo(src: PNG, w: number, h: number, fill: [number, number, number, number]): PNG { + const out = new PNG({ width: w, height: h }); + for (let i = 0; i < out.data.length; i += 4) { + out.data[i] = fill[0]; + out.data[i + 1] = fill[1]; + out.data[i + 2] = fill[2]; + out.data[i + 3] = fill[3]; + } + for (let y = 0; y < Math.min(h, src.height); y++) { + const srcRow = y * src.width * 4; + const dstRow = y * w * 4; + const rowBytes = Math.min(w, src.width) * 4; + src.data.copy(out.data, dstRow, srcRow, srcRow + rowBytes); + } + return out; +} + +/** + * Diff two images. On a dimension mismatch both are padded (magenta) to the + * union size and `dimsMatch` is false (the caller treats that as CHANGED). + * The changed-pixel mask is read back from the heatmap's red diff pixels, so it + * inherits pixelmatch's anti-aliasing exclusion. + */ +export function diffImages(a: PNG, b: PNG): DiffResult { + const dimsMatch = a.width === b.width && a.height === b.height; + const width = Math.max(a.width, b.width); + const height = Math.max(a.height, b.height); + const pa = dimsMatch ? a : padTo(a, width, height, TRIPTYCH.padFill); + const pb = dimsMatch ? b : padTo(b, width, height, TRIPTYCH.padFill); + + const heatmap = new PNG({ width, height }); + const diffPixels = pixelmatch(pa.data, pb.data, heatmap.data, width, height, { + threshold: PIXELMATCH.threshold, + includeAA: PIXELMATCH.includeAA, + diffColor: DIFF_COLOR, + }); + + // Whole-frame mean channel delta (drift-vs-regression heuristic input). + let totalChannelDiff = 0; + for (let i = 0; i < pa.data.length; i++) { + totalChannelDiff += Math.abs(pa.data[i] - pb.data[i]); + } + const meanChannelDiff = totalChannelDiff / pa.data.length; + + // Mask = heatmap pixels painted with DIFF_COLOR (red). AA pixels are yellow, so excluded. + const mask = new Uint8Array(width * height); + for (let p = 0; p < width * height; p++) { + const o = p * 4; + if (heatmap.data[o] > 200 && heatmap.data[o + 1] < 80 && heatmap.data[o + 2] < 80) { + mask[p] = 1; + } + } + + return { + width, + height, + dimsMatch, + diffPixels, + changedRatio: diffPixels / (width * height), + meanChannelDiff, + heatmap, + mask, + }; +} + +/** Dilate a boolean mask by `r` (square structuring element), out of place. */ +function dilate(mask: Uint8Array, w: number, h: number, r: number): Uint8Array { + if (r <= 0) return mask; + const out = new Uint8Array(w * h); + for (let y = 0; y < h; y++) { + for (let x = 0; x < w; x++) { + if (!mask[y * w + x]) continue; + const y0 = Math.max(0, y - r); + const y1 = Math.min(h - 1, y + r); + const x0 = Math.max(0, x - r); + const x1 = Math.min(w - 1, x + r); + for (let yy = y0; yy <= y1; yy++) { + for (let xx = x0; xx <= x1; xx++) out[yy * w + xx] = 1; + } + } + } + return out; +} + +/** + * 8-connected connected-components over the (dilated) mask → bounding boxes, + * largest-area first, capped at `maxBoxes`, specks below `minBoxArea` dropped. + */ +export function cluster(mask: Uint8Array, w: number, h: number): Box[] { + const grown = dilate(mask, w, h, CLUSTER.dilate); + const seen = new Uint8Array(w * h); + const boxes: Box[] = []; + const stack: number[] = []; + + for (let start = 0; start < grown.length; start++) { + if (!grown[start] || seen[start]) continue; + let minX = w, minY = h, maxX = 0, maxY = 0, count = 0; + stack.push(start); + seen[start] = 1; + while (stack.length) { + const p = stack.pop()!; + const px = p % w; + const py = (p - px) / w; + count++; + if (px < minX) minX = px; + if (px > maxX) maxX = px; + if (py < minY) minY = py; + if (py > maxY) maxY = py; + for (let dy = -1; dy <= 1; dy++) { + for (let dx = -1; dx <= 1; dx++) { + if (!dx && !dy) continue; + const nx = px + dx; + const ny = py + dy; + if (nx < 0 || ny < 0 || nx >= w || ny >= h) continue; + const np = ny * w + nx; + if (grown[np] && !seen[np]) { + seen[np] = 1; + stack.push(np); + } + } + } + } + const bw = maxX - minX + 1; + const bh = maxY - minY + 1; + const area = bw * bh; + if (area >= CLUSTER.minBoxArea) { + boxes.push({ x: minX, y: minY, width: bw, height: bh, area }); + } + } + boxes.sort((p, q) => q.area - p.area); + return boxes.slice(0, CLUSTER.maxBoxes); +} + +/** Draw 2px rectangle outlines for each box onto a copy of `png`. */ +export function drawBoxes(png: PNG, boxes: Box[]): PNG { + const out = new PNG({ width: png.width, height: png.height }); + png.data.copy(out.data); + const [r, g, b] = CLUSTER.boxColor; + const set = (x: number, y: number) => { + if (x < 0 || y < 0 || x >= out.width || y >= out.height) return; + const o = (y * out.width + x) * 4; + out.data[o] = r; + out.data[o + 1] = g; + out.data[o + 2] = b; + out.data[o + 3] = 255; + }; + for (const box of boxes) { + for (let t = 0; t < 2; t++) { + for (let x = box.x; x < box.x + box.width; x++) { + set(x, box.y + t); + set(x, box.y + box.height - 1 - t); + } + for (let y = box.y; y < box.y + box.height; y++) { + set(box.x + t, y); + set(box.x + box.width - 1 - t, y); + } + } + } + return out; +} + +/** Nearest-neighbour downscale by `scale` (0 maxBytes && current.width > 320) { + current = resizeNearest(current, 0.5); + buf = PNG.sync.write(current); + } + return buf; +} + +/** Horizontally montage images (heights normalized to the tallest) with a gap + bg. */ +export function composite(panels: PNG[]): PNG { + const gap = TRIPTYCH.gap; + const bg = TRIPTYCH.bg; + const height = Math.max(...panels.map((p) => p.height)); + const width = panels.reduce((s, p) => s + p.width, 0) + gap * (panels.length - 1); + const out = new PNG({ width, height }); + for (let i = 0; i < out.data.length; i += 4) { + out.data[i] = bg[0]; + out.data[i + 1] = bg[1]; + out.data[i + 2] = bg[2]; + out.data[i + 3] = bg[3]; + } + let xOffset = 0; + for (const panel of panels) { + for (let y = 0; y < panel.height; y++) { + const srcRow = y * panel.width * 4; + const dstRow = (y * width + xOffset) * 4; + panel.data.copy(out.data, dstRow, srcRow, srcRow + panel.width * 4); + } + xOffset += panel.width + gap; + } + return out; +} diff --git a/tests/tools/screenshots/noise.ts b/tests/tools/screenshots/noise.ts new file mode 100644 index 0000000..7e67524 --- /dev/null +++ b/tests/tools/screenshots/noise.ts @@ -0,0 +1,76 @@ +/** + * Calibration: measure the intra-CI noise floor. + * + * Render the suite twice on the same host (into two dirs), then: + * tsx tools/screenshots/noise.ts + * prints, per engine (via the manifest) and globally, the max / p95 / mean + * changed-pixel ratio between the two identical-input renders. Set + * config.ts FLOORS..changedRatio ≈ (max × 3) from this. + * + * A near-zero floor means the render is deterministic enough to catch real + * changes tightly; a large floor exposes nondeterminism to chase before the + * gate can be trusted. + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { MANIFEST_PATH, type Manifest } from './config'; +import { diffImages, loadPng } from './image-ops'; + +function listPngs(dir: string): string[] { + return fs.existsSync(dir) ? fs.readdirSync(dir).filter((f) => f.toLowerCase().endsWith('.png')) : []; +} + +function engineOf(name: string, manifest?: Manifest): string { + return manifest?.screenshots.find((e) => e.name === name)?.engine ?? 'default'; +} + +function stats(values: number[]): { n: number; max: number; p95: number; mean: number } { + if (!values.length) return { n: 0, max: 0, p95: 0, mean: 0 }; + const sorted = [...values].sort((a, b) => a - b); + const p95 = sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * 0.95))]; + return { + n: values.length, + max: sorted[sorted.length - 1], + p95, + mean: values.reduce((s, v) => s + v, 0) / values.length, + }; +} + +function main(): void { + const [dir1, dir2] = process.argv.slice(2); + if (!dir1 || !dir2) { + console.error('usage: noise.ts '); + process.exitCode = 2; + return; + } + const root = process.cwd(); + const manifestPath = path.join(root, MANIFEST_PATH); + const manifest = fs.existsSync(manifestPath) + ? (JSON.parse(fs.readFileSync(manifestPath, 'utf8')) as Manifest) + : undefined; + + const common = listPngs(dir1).filter((n) => fs.existsSync(path.join(dir2, n))); + const byEngine = new Map(); + const all: number[] = []; + for (const name of common) { + const d = diffImages(loadPng(path.join(dir1, name)), loadPng(path.join(dir2, name))); + const ratio = d.dimsMatch ? d.changedRatio : 1; + all.push(ratio); + const eng = engineOf(name, manifest); + (byEngine.get(eng) ?? byEngine.set(eng, []).get(eng)!).push(ratio); + } + + const report = (label: string, values: number[]) => { + const s = stats(values); + console.log( + `${label.padEnd(24)} n=${String(s.n).padStart(4)} max=${(s.max * 100).toFixed(4)}% ` + + `p95=${(s.p95 * 100).toFixed(4)}% mean=${(s.mean * 100).toFixed(4)}% ` + + `→ suggest changedRatio=${(s.max * 3).toExponential(2)}` + ); + }; + console.log(`[noise] compared ${common.length} screenshots rendered twice`); + for (const [eng, values] of byEngine) report(eng, values); + report('ALL', all); +} + +if (require.main === module) main(); diff --git a/tests/tools/screenshots/perf-report.ts b/tests/tools/screenshots/perf-report.ts new file mode 100644 index 0000000..c04d0d0 --- /dev/null +++ b/tests/tools/screenshots/perf-report.ts @@ -0,0 +1,164 @@ +/** + * Renders the track-only runtime-perf block for the CI-on-main Discord comment. + * + * The perf e2e (tests/kicad/{eeschema,pcbnew}-perf.spec.ts) already writes + * test-results/perf-{app}.json — schema { app, when, loadMs, openMs, fps:[{throttle,fps}] }. + * We read those, fetch the PREVIOUS successful main run's perf via `gh run + * download` (so we can show a Δ without committing a baseline — stays + * no-write-back), and format an aligned monospace table (Discord doesn't render + * markdown tables, so it goes in a ``` code block). + * + * Track-only: nothing here gates the build. A regression past REGRESSION_PCT on + * the stable metrics (loadMs/openMs) is only flagged (a `*`), never failed. FPS + * is CPU-bound/noisy on CI's headless SwiftShader path, so it's shown but marked + * indicative. + * + * CLI (from tests/): + * tsx tools/screenshots/perf-report.ts [--results DIR] [--prev DIR] [--repo owner/repo] + */ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { execFileSync } from 'child_process'; +import { RESULTS_DIR } from './config'; + +export const PERF_APPS = ['eeschema', 'pcbnew'] as const; +const REGRESSION_PCT = 10; // stable-metric regression past this is flagged with `*` +const CI_WORKFLOW = 'ci-ubicloud.yml'; + +export type Fps = { throttle: number; fps: number }; +export type PerfData = { app: string; when?: string; loadMs: number; openMs: number; fps: Fps[] }; + +export function readPerf(dir: string): Map { + const out = new Map(); + for (const app of PERF_APPS) { + const p = path.join(dir, `perf-${app}.json`); + if (!fs.existsSync(p)) continue; + try { + out.set(app, JSON.parse(fs.readFileSync(p, 'utf8')) as PerfData); + } catch (e) { + console.warn(`[perf] could not parse ${p}: ${(e as Error).message}`); + } + } + return out; +} + +/** + * Best-effort fetch of the previous successful main CI run's perf JSONs into a + * temp dir. Returns the dir, or null if gh is unavailable / no prior run. + * `currentSha` is skipped so a re-run doesn't diff against itself. + */ +export function fetchPreviousPerf(repo: string | undefined, currentSha: string | undefined): string | null { + try { + const gh = (args: string[]) => + execFileSync('gh', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }); + const repoArgs = repo ? ['--repo', repo] : []; + const runs = JSON.parse( + gh([ + 'run', 'list', '--workflow', CI_WORKFLOW, '--branch', 'main', '--status', 'success', + '--limit', '15', '--json', 'databaseId,headSha', ...repoArgs, + ]) + ) as Array<{ databaseId: number; headSha: string }>; + const prev = runs.find((r) => r.headSha !== currentSha); + if (!prev) return null; + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'perf-prev-')); + // Artifact is named ubicloud-e2e-; -D extracts it under tmp//... + gh(['run', 'download', String(prev.databaseId), '-D', tmp, ...repoArgs]); + // Find the dir that actually holds the perf-*.json (artifact nests test-results/). + const hit = findPerfDir(tmp); + return hit; + } catch { + return null; + } +} + +function findPerfDir(root: string): string | null { + const stack = [root]; + while (stack.length) { + const dir = stack.pop()!; + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + continue; + } + if (entries.some((e) => e.isFile() && /^perf-\w+\.json$/.test(e.name))) return dir; + for (const e of entries) if (e.isDirectory()) stack.push(path.join(dir, e.name)); + } + return null; +} + +function pct(cur: number, prev: number): number { + return prev === 0 ? 0 : ((cur - prev) / prev) * 100; +} + +/** Format a stable-metric value with a Δ vs previous (lower is better). */ +function fmtMetric(cur: number, prev?: number): string { + if (prev === undefined) return `${cur}`; + const p = pct(cur, prev); + const arrow = p < 0 ? '▼' : p > 0 ? '▲' : '·'; + const flag = p > REGRESSION_PCT ? '*' : ''; // regression (slower) beyond threshold + return `${cur} ${arrow}${Math.abs(p).toFixed(0)}%${flag}`; +} + +function fmtFps(fps: Fps[]): string { + return [1, 4, 6].map((t) => { + const hit = fps.find((f) => f.throttle === t); + return hit ? Math.round(hit.fps) : '–'; + }).join('/'); +} + +function pad(s: string, n: number): string { + return s.length >= n ? s : s + ' '.repeat(n - s.length); +} + +export type PerfReport = { block: string; regressed: boolean }; + +/** Build the fenced code-block perf table for the Discord comment. */ +export function buildPerfReport(opts: { resultsDir?: string; prevDir?: string | null } = {}): PerfReport { + const cur = readPerf(opts.resultsDir ?? RESULTS_DIR); + if (cur.size === 0) return { block: '', regressed: false }; + const prev = opts.prevDir ? readPerf(opts.prevDir) : new Map(); + + const headers = ['app', 'loadMs (Δ)', 'openMs (Δ)', 'FPS 1/4/6']; + const rows: string[][] = []; + let regressed = false; + for (const app of PERF_APPS) { + const c = cur.get(app); + if (!c) continue; + const p = prev.get(app); + const loadCell = fmtMetric(c.loadMs, p?.loadMs); + const openCell = fmtMetric(c.openMs, p?.openMs); + if (loadCell.endsWith('*') || openCell.endsWith('*')) regressed = true; + rows.push([app, loadCell, openCell, fmtFps(c.fps)]); + } + if (rows.length === 0) return { block: '', regressed: false }; + + const widths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length))); + const line = (cells: string[]) => cells.map((c, i) => pad(c, widths[i])).join(' '); + const body = [line(headers), rows.map((r) => line(r)).join('\n')].join('\n'); + const footnote = `${prev.size ? 'Δ vs previous main run. ' : 'no prior main run for Δ. '}` + + `* = >${REGRESSION_PCT}% slower (track-only, non-gating). FPS is CI-headless — indicative only.`; + return { block: '**Runtime perf** (eeschema + pcbnew)\n```\n' + body + '\n```\n' + footnote, regressed }; +} + +function parseArgs(argv: string[]): Record { + const out: Record = {}; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--results') out.results = argv[++i]; + else if (a === '--prev') out.prev = argv[++i]; + else if (a === '--repo') out.repo = argv[++i]; + } + return out; +} + +function main(): void { + const args = parseArgs(process.argv.slice(2)); + const prevDir = args.prev ?? fetchPreviousPerf(args.repo || process.env.GITHUB_REPOSITORY, process.env.GITHUB_SHA); + const { block, regressed } = buildPerfReport({ resultsDir: args.results, prevDir }); + process.stdout.write((block || '(no perf-*.json found)') + '\n'); + if (regressed) console.error('[perf] a stable metric regressed >10% vs previous main (track-only, not failing)'); +} + +if (require.main === module) main(); diff --git a/tests/tools/screenshots/post-discord.ts b/tests/tools/screenshots/post-discord.ts new file mode 100644 index 0000000..1fb9271 --- /dev/null +++ b/tests/tools/screenshots/post-discord.ts @@ -0,0 +1,226 @@ +/** + * The always-on CI-on-main Discord report. + * + * One message leads with the commit SHA + e2e pass/fail + the runtime-perf table + * (perf-report.ts), then screenshot changes are attached as triptychs + * (old | new+boxes | heatmap), ADDED images, and a REMOVED list. Because perf is + * always present, the comment fires on every main commit even with no screenshot + * change — "see it in hindsight on every commit." + * + * Safety / robustness: + * - Posts only on push to main (unless --force); a missing DISCORD_WEBHOOK_URL is + * a silent no-op, so it's inert on PRs / forks (which can't see the secret). + * - Attachments batched ≤10/message and downscaled to fit Discord's size caps; + * 429 Retry-After honoured; first-run flood (hundreds of ADDED) collapsed. + * - --dry-run composes everything and prints it without POSTing (how it's tested). + * + * CLI (from tests/): + * tsx tools/screenshots/post-discord.ts [--dry-run] [--force] [--e2e pass|fail] [--subject S] + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { execFileSync } from 'child_process'; +import { DIFF_OUT_DIR } from './config'; +import { loadPng, encodeWithinCap } from './image-ops'; +import { buildPerfReport, fetchPreviousPerf } from './perf-report'; +import type { Report } from './compare'; + +const MAX_FILE_BYTES = 8 * 1024 * 1024; +const MAX_MSG_BYTES = 24 * 1024 * 1024; +const MAX_FILES_PER_MSG = 10; +const MAX_TOTAL_FILES = 30; // ~3 messages of images; excess is summarized, not posted +const FLOOD_N = 12; // more ADDED than this ⇒ show a few exemplars, not all + +type Attachment = { name: string; buffer: Buffer }; +type Message = { content: string; files: Attachment[] }; + +function sanitize(name: string): string { + return name.replace(/[^\w.-]+/g, '_'); +} + +function e2eBadge(status: string | undefined): string { + if (status === 'pass') return '✅ e2e passed'; + if (status === 'fail') return '❌ e2e failed'; + return 'ℹ️ e2e status unknown'; +} + +function commitSubject(sha: string | undefined): string { + if (!sha) return ''; + try { + return execFileSync('git', ['log', '-1', '--pretty=%s', sha], { encoding: 'utf8' }).trim(); + } catch { + return ''; + } +} + +function readReport(root: string): Report | null { + const p = path.join(root, DIFF_OUT_DIR, 'report.json'); + if (!fs.existsSync(p)) return null; + try { + return JSON.parse(fs.readFileSync(p, 'utf8')) as Report; + } catch { + return null; + } +} + +/** Encode a PNG path to a within-cap attachment, or null if missing/unreadable. */ +function attach(root: string, rel: string, name: string): Attachment | null { + const abs = path.isAbsolute(rel) ? rel : path.join(root, rel); + if (!fs.existsSync(abs)) return null; + try { + return { name: sanitize(name), buffer: encodeWithinCap(loadPng(abs), MAX_FILE_BYTES) }; + } catch { + return null; + } +} + +/** Build the header text: SHA, subject, e2e, perf table, screenshot summary, removed list. */ +function buildHeader(report: Report | null, perfBlock: string, meta: { sha?: string; subject: string; e2e?: string }): string { + const shortSha = meta.sha ? meta.sha.slice(0, 7) : 'local'; + const lines = [`📸 **CI screenshot + perf report** · \`${shortSha}\``]; + if (meta.subject) lines.push(`> ${meta.subject}`); + lines.push(e2eBadge(meta.e2e)); + if (perfBlock) lines.push('', perfBlock); + + lines.push(''); + if (!report || (!report.changed.length && !report.added.length && !report.removed.length)) { + lines.push('✅ no screenshot drift'); + } else if (report.driftLikely) { + lines.push(`⚠️ **${report.changed.length} screenshots changed broadly** — looks like host env drift → re-promote (\`npm run screenshots:promote\`)`); + } else { + lines.push(`⚠️ **screenshot drift**: ${report.changed.length} changed, ${report.added.length} added, ${report.removed.length} removed`); + } + if (report?.removed.length) { + lines.push('➖ REMOVED: ' + report.removed.map((r) => `\`${r.name}\``).join(', ')); + } + return lines.join('\n'); +} + +/** Collect the image attachments (changed triptychs + added images) with flood-collapse + total cap. */ +export function buildAttachments(root: string, report: Report | null): { files: Attachment[]; notes: string[] } { + const files: Attachment[] = []; + const notes: string[] = []; + if (!report) return { files, notes }; + + for (const c of report.changed) { + if (files.length >= MAX_TOTAL_FILES) break; + const a = attach(root, c.triptych, `CHANGED_${c.name}`); + if (a) files.push(a); + } + const addedToShow = report.added.length > FLOOD_N ? report.added.slice(0, 3) : report.added; + if (report.added.length > FLOOD_N) { + notes.push(`➕ ${report.added.length} added (showing ${addedToShow.length})`); + } + for (const ad of addedToShow) { + if (files.length >= MAX_TOTAL_FILES) break; + const a = attach(root, ad.image, `ADDED_${ad.name}`); + if (a) files.push(a); + } + const shown = files.length; + const wanted = report.changed.length + addedToShow.length; + if (wanted > shown) notes.push(`(${wanted - shown} more images omitted — see the CI artifact)`); + return { files, notes }; +} + +/** Split attachments into messages of ≤10 files and ≤MAX_MSG_BYTES; content only on the first. */ +export function paginate(header: string, files: Attachment[]): Message[] { + if (!files.length) return [{ content: header, files: [] }]; + const messages: Message[] = []; + let batch: Attachment[] = []; + let bytes = 0; + const flush = () => { + messages.push({ content: messages.length === 0 ? header : '', files: batch }); + batch = []; + bytes = 0; + }; + for (const f of files) { + if (batch.length >= MAX_FILES_PER_MSG || bytes + f.buffer.length > MAX_MSG_BYTES) flush(); + batch.push(f); + bytes += f.buffer.length; + } + if (batch.length || messages.length === 0) flush(); + return messages; +} + +async function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +export async function postMessage(webhook: string, msg: Message): Promise { + for (let attemptNo = 0; attemptNo < 6; attemptNo++) { + const form = new FormData(); + form.append('payload_json', JSON.stringify({ content: msg.content || '', allowed_mentions: { parse: [] } })); + msg.files.forEach((f, i) => form.append(`files[${i}]`, new Blob([new Uint8Array(f.buffer)], { type: 'image/png' }), f.name)); + const res = await fetch(webhook, { method: 'POST', body: form }); + if (res.ok) return; + if (res.status === 429) { + const retryAfter = Number(res.headers.get('retry-after')) || 2; + await sleep((retryAfter + 0.5) * 1000); + continue; + } + throw new Error(`Discord POST failed ${res.status}: ${(await res.text()).slice(0, 300)}`); + } + throw new Error('Discord POST failed after retries (429)'); +} + +function parseArgs(argv: string[]): Record { + const out: Record = {}; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--dry-run') out.dryRun = true; + else if (a === '--force') out.force = true; + else if (a === '--e2e') out.e2e = argv[++i]; + else if (a === '--subject') out.subject = argv[++i]; + else if (a === '--repo') out.repo = argv[++i]; + } + return out; +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + const root = process.cwd(); + const isMainPush = process.env.GITHUB_REF === 'refs/heads/main' && process.env.GITHUB_EVENT_NAME === 'push'; + if (!args.force && !args.dryRun && !isMainPush) { + console.log('[discord] not a push to main — skipping'); + return; + } + const webhook = process.env.DISCORD_WEBHOOK_URL; + if (!webhook && !args.dryRun) { + console.log('[discord] DISCORD_WEBHOOK_URL unset — skipping (inert on PRs/forks)'); + return; + } + + const sha = process.env.GITHUB_SHA; + const report = readReport(root); + const prevDir = fetchPreviousPerf((args.repo as string) || process.env.GITHUB_REPOSITORY, sha); + const { block: perfBlock } = buildPerfReport({ prevDir }); + + let header = buildHeader(report, perfBlock, { + sha, + subject: (args.subject as string) ?? commitSubject(sha), + e2e: args.e2e as string, + }); + const { files, notes } = buildAttachments(root, report); + if (notes.length) header += '\n' + notes.join('\n'); + + const messages = paginate(header, files); + + if (args.dryRun) { + for (const [i, m] of messages.entries()) { + console.log(`--- message ${i + 1}/${messages.length} (${m.files.length} files, ${m.files.reduce((s, f) => s + f.buffer.length, 0)} bytes) ---`); + if (m.content) console.log(m.content); + for (const f of m.files) console.log(` [attach] ${f.name} (${f.buffer.length} bytes)`); + } + return; + } + + for (const m of messages) await postMessage(webhook!, m); + console.log(`[discord] posted ${messages.length} message(s)`); +} + +if (require.main === module) { + main().catch((e) => { + console.error(`[discord] ${e.message}`); + process.exitCode = 1; + }); +} diff --git a/tests/tools/screenshots/promote.ts b/tests/tools/screenshots/promote.ts new file mode 100644 index 0000000..efa88c0 --- /dev/null +++ b/tests/tools/screenshots/promote.ts @@ -0,0 +1,149 @@ +/** + * Churn-free baseline updater — "promote CI's render". + * + * CI's x86 render is the source of truth. This pulls a CI run's screenshots + * (`gh run download`) — or a local dir via --from — and, for each one, overwrites + * the committed baseline ONLY when the decoded pixels differ beyond the per-engine + * floor. Unchanged baselines are left byte-identical (never re-encoded), so git + * sees no churn. New shots are added; baselines with no render are reported as + * removal candidates and only deleted with --prune. + * + * CLI (from tests/): + * tsx tools/screenshots/promote.ts --run [--repo owner/repo] [--prune] [--dry-run] + * tsx tools/screenshots/promote.ts --from [--prune] [--dry-run] + */ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { execFileSync } from 'child_process'; +import { BASELINE_DIRS, MANIFEST_PATH, floorFor, type Manifest } from './config'; +import { diffImages, loadPng } from './image-ops'; + +function listPngs(dir: string): string[] { + if (!fs.existsSync(dir)) return []; + return fs.readdirSync(dir).filter((f) => f.toLowerCase().endsWith('.png')); +} + +/** basename → absolute committed baseline path (first BASELINE_DIRS entry wins). */ +function baselineIndex(root: string): Map { + const index = new Map(); + for (const dir of BASELINE_DIRS) { + for (const name of listPngs(path.join(root, dir))) { + if (!index.has(name)) index.set(name, path.join(root, dir, name)); + } + } + return index; +} + +function loadManifest(root: string): Manifest | undefined { + const p = path.join(root, MANIFEST_PATH); + if (!fs.existsSync(p)) return undefined; + try { + return JSON.parse(fs.readFileSync(p, 'utf8')) as Manifest; + } catch { + return undefined; + } +} + +/** Download a CI run's artifact and return the top-level test-results dir holding the shots. */ +function downloadRun(runId: string, repo?: string): string { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'promote-')); + const repoArgs = repo ? ['--repo', repo] : []; + execFileSync('gh', ['run', 'download', runId, '-D', tmp, ...repoArgs], { stdio: 'inherit' }); + // The artifact stores test-results/** — find that dir; its immediate *.png are the shots + // (exclude the nested screenshot-diff/ triptychs). + const stack = [tmp]; + while (stack.length) { + const dir = stack.pop()!; + const entries = fs.readdirSync(dir, { withFileTypes: true }); + if (path.basename(dir) === 'test-results') return dir; + for (const e of entries) if (e.isDirectory()) stack.push(path.join(dir, e.name)); + } + throw new Error(`no test-results/ dir found in the downloaded artifact under ${tmp}`); +} + +type Plan = { updated: string[]; added: string[]; unchanged: string[]; removedCandidates: string[] }; + +function buildPlan(root: string, renderDir: string, manifest?: Manifest): { plan: Plan; apply: () => void } { + const baselines = baselineIndex(root); + const rendered = new Set(listPngs(renderDir)); + const plan: Plan = { updated: [], added: [], unchanged: [], removedCandidates: [] }; + const actions: Array<() => void> = []; + + for (const name of rendered) { + const src = path.join(renderDir, name); + const existing = baselines.get(name); + if (!existing) { + const dest = path.join(root, BASELINE_DIRS[0], name); + plan.added.push(name); + actions.push(() => fs.copyFileSync(src, dest)); // verbatim bytes + continue; + } + const d = diffImages(loadPng(existing), loadPng(src)); + const floor = floorFor(name, manifest); + if (!d.dimsMatch || d.changedRatio > floor.changedRatio) { + plan.updated.push(name); + actions.push(() => fs.copyFileSync(src, existing)); // verbatim bytes, no re-encode → no churn + } else { + plan.unchanged.push(name); // leave the committed file untouched + } + } + + // Removal candidates: a committed baseline the manifest expects but this render didn't produce. + for (const [name, abs] of baselines) { + if (rendered.has(name)) continue; + if (manifest && !manifest.screenshots.some((e) => e.name === name)) continue; + plan.removedCandidates.push(name); + actions.push(() => {}); // pruning is opt-in (see main) + void abs; + } + + return { plan, apply: () => actions.forEach((a) => a()) }; +} + +function parseArgs(argv: string[]): Record { + const out: Record = {}; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--run') out.run = argv[++i]; + else if (a === '--from') out.from = argv[++i]; + else if (a === '--repo') out.repo = argv[++i]; + else if (a === '--prune') out.prune = true; + else if (a === '--dry-run') out.dryRun = true; + } + return out; +} + +function main(): void { + const args = parseArgs(process.argv.slice(2)); + const root = process.cwd(); + if (!args.run && !args.from) { + console.error('usage: promote.ts --run [--repo owner/repo] | --from [--prune] [--dry-run]'); + process.exitCode = 2; + return; + } + const renderDir = args.from ? (args.from as string) : downloadRun(args.run as string, args.repo as string); + const manifest = loadManifest(root); + const { plan, apply } = buildPlan(root, renderDir, manifest); + + console.log( + `[promote] updated=${plan.updated.length} added=${plan.added.length} ` + + `unchanged=${plan.unchanged.length} removal-candidates=${plan.removedCandidates.length}` + ); + for (const n of plan.updated) console.log(` UPDATE ${n}`); + for (const n of plan.added) console.log(` ADD ${n}`); + for (const n of plan.removedCandidates) console.log(` REMOVE? ${n}${args.prune ? ' (pruning)' : ' (use --prune to delete)'}`); + + if (args.dryRun) { + console.log('[promote] dry-run — no files written'); + return; + } + apply(); + if (args.prune) { + const baselines = baselineIndex(root); + for (const n of plan.removedCandidates) fs.rmSync(baselines.get(n)!, { force: true }); + } + console.log('[promote] done — review `git status` and commit the changed baselines'); +} + +if (require.main === module) main(); diff --git a/tests/web/tool-switch.spec.ts b/tests/web/tool-switch.spec.ts index 84158d1..18bacef 100644 --- a/tests/web/tool-switch.spec.ts +++ b/tests/web/tool-switch.spec.ts @@ -62,7 +62,7 @@ test.describe('web app — tool switching', () => { await page.screenshot({ path: 'test-results/web-switch-sch-to-pcb.png', - scale: 'device', + scale: 'css', }); }); @@ -81,7 +81,7 @@ test.describe('web app — tool switching', () => { await page.screenshot({ path: 'test-results/web-switch-pcb-to-sch.png', - scale: 'device', + scale: 'css', }); }); }); diff --git a/tests/web/tools-open.spec.ts b/tests/web/tools-open.spec.ts index 272b5c3..20ebe86 100644 --- a/tests/web/tools-open.spec.ts +++ b/tests/web/tools-open.spec.ts @@ -81,7 +81,7 @@ test.describe('web app — tool open paths', () => { page, }) => { await bootAndAssert(page, tc); - await page.screenshot({ path: `test-results/web-${tool}.png`, scale: 'device' }); + await page.screenshot({ path: `test-results/web-${tool}.png`, scale: 'css' }); }); } });