pcbjam/tests/kicad/eeschema.spec.ts
Viktor Vaczi 13551f4f22 feat(tests): screenshot regression + Discord review, perf-tracked
New tooling in tests/tools/screenshots/ (TypeScript via tsx):
- compare.ts: one pixelmatch engine (AA-excluded), connected-component
  "where to look" boxes, old|new+boxes|heatmap triptych, per-engine floors.
- promote.ts: churn-free updater — overwrite a baseline only when decoded
  pixels differ beyond the floor, copying CI bytes verbatim (no re-encode
  churn); pulls a CI run via `gh run download` or a local --from dir.
- post-discord.ts: always-on CI-on-main report (SHA + e2e status + the
  track-only runtime-perf table), then screenshot triptychs, batched +
  size-capped + flood-collapsed + 429-aware.
- perf-report.ts: perf table with Δ vs the previous main run (via gh).
- changelog.ts: no-build git-history baseline differ (Discord trigger B).
- noise.ts / gen-manifest.ts: calibration + manifest generation.

CI wiring:
- wasm-build.yml: post-test step runs the gate + report on the already-
  produced test-results (no extra build); report-only (continue-on-error),
  posts only on push to main, inert without DISCORD_WEBHOOK_URL.
- ci-ubicloud.yml: secrets: inherit (pass the webhook through).
- screenshot-changelog.yml: ~30s no-build changelog on baseline changes.

screenshot-manifest.json: canonical 354-name set + best-effort engine tags
(313 chromium-swiftshader / 41 firefox-llvmpipe).

Normalize scale:'device'->'css' across 18 spec files (no-op at CI DSF=1)
so committed baselines are uniformly css-scaled.

Design: CI's Linux render is the single source of truth; no pinned
container (accept rare env drift -> re-promote); dev commits via promote.
Replaces the byte-cmp compare-screenshots.sh + file-size-proxy
update-baseline-screenshots.sh (kept for now until the first re-baseline).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-01 18:17:20 +02:00

499 lines
18 KiB
TypeScript

import type { Page } from '@playwright/test';
import { test, expect } from './fixtures';
import { clickByLabel, clickByTooltip, findByTooltip } from '../e2e/utils/element-tracker';
/**
* Eeschema (schematic editor) WASM E2E Tests
*
* Mirrors pcbnew.spec.ts. The wxWidgets setup wizard is shared infrastructure,
* so the wizard flow is identical. Editor-specific checks (Appearance pane,
* exact toolbar count, reference-image diff, etc.) are intentionally omitted
* here until the eeschema UI surface is empirically pinned down.
*/
type CanvasMetrics = {
dpr: number;
mainCanvas: null | {
width: number;
height: number;
rectWidth: number;
rectHeight: number;
};
glCanvas: null | {
id: string;
width: number;
height: number;
rectWidth: number;
rectHeight: number;
viewport: number[] | null;
};
};
type RegistryMetrics = {
elementStats: null | {
total: number;
byType: Record<string, number>;
};
renderedStats: null | {
total: number;
byType: Record<string, number>;
};
toolbars: Array<{
id: string;
typeName: string;
screenX: number;
screenY: number;
width: number;
height: number;
label: string;
name: string;
}>;
};
type DiffRegion = {
x: number;
y: number;
width: number;
height: number;
};
type ScreenshotDifference = {
actualWidth: number;
actualHeight: number;
diffPixels: number;
diffRatio: number;
meanChannelDiff: number;
};
async function compareScreenshots(
page: Page,
beforePng: Buffer,
afterPng: Buffer,
region: DiffRegion
): Promise<ScreenshotDifference> {
return page.evaluate(async ({ beforeBase64, afterBase64, crop }) => {
const loadImage = async (base64: string): Promise<HTMLImageElement> => {
const image = new Image();
image.src = `data:image/png;base64,${base64}`;
await image.decode();
return image;
};
const [before, after] = await Promise.all([
loadImage(beforeBase64),
loadImage(afterBase64),
]);
if (before.width !== after.width || before.height !== after.height) {
return {
actualWidth: after.width,
actualHeight: after.height,
diffPixels: Number.POSITIVE_INFINITY,
diffRatio: Number.POSITIVE_INFINITY,
meanChannelDiff: Number.POSITIVE_INFINITY,
};
}
const canvas = document.createElement('canvas');
canvas.width = crop.width;
canvas.height = crop.height;
const context = canvas.getContext('2d', { willReadFrequently: true });
if (!context) {
throw new Error('2D canvas context unavailable for screenshot comparison');
}
context.drawImage(before, crop.x, crop.y, crop.width, crop.height, 0, 0, crop.width, crop.height);
const beforeData = context.getImageData(0, 0, canvas.width, canvas.height).data;
context.clearRect(0, 0, canvas.width, canvas.height);
context.drawImage(after, crop.x, crop.y, crop.width, crop.height, 0, 0, crop.width, crop.height);
const afterData = context.getImageData(0, 0, canvas.width, canvas.height).data;
let diffPixels = 0;
let totalChannelDiff = 0;
for (let i = 0; i < beforeData.length; i += 4) {
const dr = Math.abs(beforeData[i] - afterData[i]);
const dg = Math.abs(beforeData[i + 1] - afterData[i + 1]);
const db = Math.abs(beforeData[i + 2] - afterData[i + 2]);
const da = Math.abs(beforeData[i + 3] - afterData[i + 3]);
const maxDiff = Math.max(dr, dg, db, da);
totalChannelDiff += dr + dg + db + da;
if (maxDiff > 16) {
diffPixels += 1;
}
}
return {
actualWidth: after.width,
actualHeight: after.height,
diffPixels,
diffRatio: diffPixels / (canvas.width * canvas.height),
meanChannelDiff: totalChannelDiff / beforeData.length,
};
}, {
beforeBase64: beforePng.toString('base64'),
afterBase64: afterPng.toString('base64'),
crop: region,
});
}
async function getCanvasMetrics(page: Page): Promise<CanvasMetrics> {
return page.evaluate(() => {
const dpr = window.devicePixelRatio || 1;
const mainCanvas = document.querySelector('#canvas') as HTMLCanvasElement | null;
const glCanvas =
Array.from(document.querySelectorAll('[id^="glcanvas-"]'))
.map((canvas) => canvas as HTMLCanvasElement)
.find((canvas) => {
const rect = canvas.getBoundingClientRect();
const style = window.getComputedStyle(canvas);
return style.display !== 'none' && rect.width > 0 && rect.height > 0;
}) ??
document.querySelector('[id^="glcanvas-"]') as HTMLCanvasElement | null;
const mainRect = mainCanvas?.getBoundingClientRect();
const glRect = glCanvas?.getBoundingClientRect();
const gl =
glCanvas?.getContext('webgl2') ||
glCanvas?.getContext('webgl');
const viewport = gl ? Array.from(gl.getParameter(gl.VIEWPORT) as Int32Array | number[]) : null;
return {
dpr,
mainCanvas: mainCanvas && mainRect ? {
width: mainCanvas.width,
height: mainCanvas.height,
rectWidth: mainRect.width,
rectHeight: mainRect.height,
} : null,
glCanvas: glCanvas && glRect ? {
id: glCanvas.id,
width: glCanvas.width,
height: glCanvas.height,
rectWidth: glRect.width,
rectHeight: glRect.height,
viewport,
} : null,
};
});
}
async function getRegistryMetrics(page: Page): Promise<RegistryMetrics> {
return page.evaluate(() => {
const registry = window.wxElementRegistry;
if (!registry) {
return {
elementStats: null,
renderedStats: null,
toolbars: [],
};
}
const allElements = registry.findAll({ visible: true });
const toolbars = allElements
.filter((element) => /ToolBar/.test(element.typeName))
.map((element) => ({
id: element.id,
typeName: element.typeName,
screenX: element.screenX,
screenY: element.screenY,
width: element.width,
height: element.height,
label: element.label,
name: element.name,
}));
return {
elementStats: registry.getStats(),
renderedStats: registry.getRenderedStats ? registry.getRenderedStats() : null,
toolbars,
};
});
}
async function completeWizard(page: Page): Promise<void> {
await expect(page.locator('#canvas')).toBeVisible({ timeout: 90000 });
await page.waitForFunction(() => !!window.wxElementRegistry, null, { timeout: 90000 });
// Registry object ≠ app booted: wait for real UI entries (the wizard is the
// first window) so the bounded click loop below doesn't start too early.
// CI boots slower (baseline-JIT wasm + software GL under xvfb).
await page.waitForFunction(() => {
const registry = window.wxElementRegistry;
return !!registry && registry.findAll({}).length > 0;
}, null, { timeout: 150000 });
await page.waitForTimeout(2000);
await page.screenshot({ path: 'test-results/eeschema-wizard-00-initial.png', scale: 'css' });
for (let i = 1; i <= 10; i++) {
let clicked = await clickByLabel(page, 'Next >');
if (!clicked) {
clicked = await clickByLabel(page, 'Finish');
if (clicked) {
await page.waitForTimeout(500);
await page.screenshot({
path: `test-results/eeschema-wizard-${String(i).padStart(2, '0')}-finish.png`,
scale: 'css'
});
}
break;
}
await page.waitForTimeout(500);
await page.screenshot({
path: `test-results/eeschema-wizard-${String(i).padStart(2, '0')}.png`,
scale: 'css'
});
}
await page.waitForTimeout(2000);
}
async function hideCursor(page: Page): Promise<void> {
await page.evaluate(() => {
document.documentElement.style.cursor = 'none';
document.body.style.cursor = 'none';
});
}
test.describe('Eeschema WASM', () => {
test.beforeEach(async ({ page }) => {
await page.goto('/kicad/eeschema.html');
});
test('click through setup wizard to load Eeschema', async ({ page }) => {
await completeWizard(page);
const metrics = await getCanvasMetrics(page);
const registryMetrics = await getRegistryMetrics(page);
// Headless Firefox runs at dpr=1; pcbnew's stricter `> 1` check assumes a
// Retina-aware run. The eeschema MVP just needs to verify dpr is sane.
expect(metrics.dpr).toBeGreaterThanOrEqual(1);
expect(metrics.mainCanvas).not.toBeNull();
expect(metrics.glCanvas).not.toBeNull();
expect(registryMetrics.toolbars.length).toBeGreaterThanOrEqual(2);
if (!metrics.mainCanvas || !metrics.glCanvas) {
throw new Error('KiCad canvases not initialized');
}
expect(Math.round(metrics.mainCanvas.rectWidth * metrics.dpr)).toBe(metrics.mainCanvas.width);
expect(Math.round(metrics.mainCanvas.rectHeight * metrics.dpr)).toBe(metrics.mainCanvas.height);
expect(metrics.glCanvas.rectWidth).toBeGreaterThan(800);
expect(metrics.glCanvas.rectHeight).toBeGreaterThan(500);
expect(Math.round(metrics.glCanvas.rectWidth * metrics.dpr)).toBe(metrics.glCanvas.width);
expect(Math.round(metrics.glCanvas.rectHeight * metrics.dpr)).toBe(metrics.glCanvas.height);
const viewport = metrics.glCanvas.viewport;
expect(viewport).not.toBeNull();
if (!viewport) {
throw new Error('WebGL viewport unavailable');
}
expect(viewport[2]).toBe(metrics.glCanvas.width);
expect(viewport[3]).toBe(metrics.glCanvas.height);
await hideCursor(page);
// Capture a CSS-scale screenshot for visual review; no reference image
// is wired up yet (eeschema's chrome differs enough from pcbnew that
// sharing pcbnew's baseline isn't viable). Add a dedicated baseline
// here once the layout is finalised.
await page.screenshot({
path: 'test-results/eeschema-loaded-css.png',
scale: 'css'
});
await page.screenshot({ path: 'test-results/eeschema-loaded.png', scale: 'css' });
const canvasCount = await page.locator('canvas').count();
expect(canvasCount).toBeGreaterThan(0);
});
test('select draw wires and draw on the schematic', async ({ page, testLogger }) => {
await completeWizard(page);
await hideCursor(page);
await page.evaluate(() => {
const canvases = Array.from(document.querySelectorAll('canvas')).map((canvas) => {
const rect = canvas.getBoundingClientRect();
const style = window.getComputedStyle(canvas);
return {
id: canvas.id,
className: canvas.className,
display: style.display,
visibility: style.visibility,
width: canvas.width,
height: canvas.height,
rectX: rect.x,
rectY: rect.y,
rectWidth: rect.width,
rectHeight: rect.height,
shouldBeVisible: (canvas as HTMLCanvasElement).dataset?.shouldBeVisible ?? null,
};
});
console.log(`[TEST] canvas summary ${JSON.stringify(canvases)}`);
const registry = window.wxElementRegistry;
const topLevels = (registry?.findAll?.({}) ?? [])
.filter((item) => /Frame|Dialog|Wizard/.test(item.typeName))
.slice(0, 20)
.map((item) => ({
id: item.id,
typeName: item.typeName,
label: item.label,
name: item.name,
visible: item.visible,
enabled: item.enabled,
screenX: item.screenX,
screenY: item.screenY,
width: item.width,
height: item.height,
}));
const rendered = registry?.findAllRendered?.({}) ?? [];
const byType = rendered.reduce<Record<string, number>>((acc, item) => {
acc[item.elementType] = (acc[item.elementType] ?? 0) + 1;
return acc;
}, {});
const tools = rendered
.filter((item) => item.elementType === 'tool')
.slice(0, 20)
.map((item) => ({
id: item.id,
label: item.label,
tooltip: item.tooltip,
checked: item.checked,
enabled: item.enabled,
}));
console.log(`[TEST] top-level summary ${JSON.stringify(topLevels)}`);
console.log(`[TEST] rendered summary ${JSON.stringify({ count: rendered.length, byType, tools })}`);
});
await page.waitForFunction(() => {
const registry = window.wxElementRegistry;
if (!registry?.findAllRendered) {
return false;
}
return registry.findAllRendered({ elementType: 'tool' })
.some((tool) => tool.tooltip?.includes('Draw Wires'));
}, null, { timeout: 15000 });
const drawWiresTool = await findByTooltip(page, 'Draw Wires', { elementType: 'tool' });
expect(drawWiresTool).not.toBeNull();
if (!drawWiresTool) {
throw new Error('Draw Wires tool not found in rendered element registry');
}
// The registry carries checked state via a " [checked]" label suffix
// appended by wxAuiToolBar::OnPaint on Emscripten — no schema change.
const isToolChecked = (t: { label?: string } | null | undefined) =>
(t?.label ?? '').includes('[checked]');
expect(drawWiresTool.enabled).toBe(true);
expect(isToolChecked(drawWiresTool)).toBe(false);
const baselineErrorCount = testLogger.errors.length;
await page.screenshot({
path: 'test-results/eeschema-draw-wires-00-before-tool-click.png',
scale: 'css'
});
expect(await clickByTooltip(page, 'Draw Wires', { elementType: 'tool' })).toBe(true);
await expect.poll(async () => {
const tool = await findByTooltip(page, 'Draw Wires', { elementType: 'tool' });
return isToolChecked(tool);
}, {
message: 'Draw Wires tool should stay selected after the click',
timeout: 5000,
}).toBe(true);
await page.mouse.move(640, 360);
await page.waitForTimeout(600);
const selectedDrawWiresTool = await findByTooltip(page, 'Draw Wires', { elementType: 'tool' });
expect(isToolChecked(selectedDrawWiresTool)).toBe(true);
const afterToolClick = await page.screenshot({
path: 'test-results/eeschema-draw-wires-01-after-click.png',
scale: 'css'
});
const glCanvasId = await page.evaluate(() => {
const glCanvas =
Array.from(document.querySelectorAll('[id^="glcanvas-"]'))
.map((canvas) => canvas as HTMLCanvasElement)
.find((canvas) => {
const rect = canvas.getBoundingClientRect();
const style = window.getComputedStyle(canvas);
return style.display !== 'none' && rect.width > 0 && rect.height > 0;
}) ??
document.querySelector('[id^="glcanvas-"]') as HTMLCanvasElement | null;
return glCanvas?.id ?? null;
});
expect(glCanvasId).not.toBeNull();
if (!glCanvasId) {
throw new Error('Visible GL canvas not found');
}
const glCanvasBox = await page.locator(`#${glCanvasId}`).boundingBox();
expect(glCanvasBox).not.toBeNull();
if (!glCanvasBox) {
throw new Error('GL canvas bounding box unavailable');
}
const startPoint = {
x: Math.round(glCanvasBox.x + glCanvasBox.width * 0.28),
y: Math.round(glCanvasBox.y + glCanvasBox.height * 0.36),
};
const endPoint = {
x: Math.round(glCanvasBox.x + glCanvasBox.width * 0.48),
y: Math.round(glCanvasBox.y + glCanvasBox.height * 0.47),
};
await page.mouse.click(startPoint.x, startPoint.y);
await page.waitForTimeout(250);
await page.mouse.click(endPoint.x, endPoint.y);
await page.waitForTimeout(750);
const afterDrawing = await page.screenshot({
path: 'test-results/eeschema-draw-wires-02-after-drawing.png',
scale: 'css'
});
const diffRegion: DiffRegion = {
x: Math.max(0, Math.min(startPoint.x, endPoint.x) - 24),
y: Math.max(0, Math.min(startPoint.y, endPoint.y) - 24),
width: Math.abs(endPoint.x - startPoint.x) + 48,
height: Math.abs(endPoint.y - startPoint.y) + 48,
};
const drawingDiff = await compareScreenshots(page, afterToolClick, afterDrawing, diffRegion);
expect(drawingDiff.diffPixels).toBeGreaterThan(120);
expect(drawingDiff.diffRatio).toBeGreaterThan(0.01);
expect(drawingDiff.meanChannelDiff).toBeGreaterThan(1);
const realErrors = testLogger.errors
.slice(baselineErrorCount)
.filter((error) => !error.includes('favicon'));
expect(realErrors).toEqual([]);
});
});