test(determinism): deterministic waits + stableShot screenshots; drop blind sleeps/ifs/retries
Make the Playwright e2e + kicad suites deterministic so screenshot flake stops tracing to timing races. - Blind page.waitForTimeout -> condition waits (expect.poll, web-first assertions, waitUntil) + readiness helpers (waitForWxApp, waitForCanvasApp). Remaining sleeps are documented interaction dwells (annotated). - Defensive "if element exists" branches -> loud asserts; label-fallback chains -> normalized clickMenuItemByText. First-run wizard for/if loops removed by seeding calculator/gerbview/pcbnew HTMLs. - Screenshots: new stableShot(page, name) settles the render in-page (canvas hash over rAF) then writes a raw PNG to test-results/ for the existing offline gate (tools/screenshots vs baseline-screenshots). Replaces toHaveScreenshot, which did inline compare + its own baselines and had decoupled the specs from the real gate. scale:'css' pinned. - retries: 0 in both configs. - Guard: tests/tools/lint-determinism.ts (npm run lint:determinism) bans blind sleeps / toHaveScreenshot / inline retries / swallowed catches in specs; documented exceptions carry a marker. Rules in tests/TESTING.md. Assertions, coverage, and renders unchanged (semantic-equivalence reviewed; captures pixel-identical modulo inherent timer/timestamp/3d-raytrace variance). Both suites green at retries:0 (e2e 340, kicad 92); ~35-61% faster. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BVX1pHMvRPYHdp6ZfEawrk
This commit is contained in:
Viktor Vaczi
parent
d9264549bf
commit
4c3a4cacd4
102 changed files with 2867 additions and 3407 deletions
|
|
@ -1,100 +1,110 @@
|
|||
// wxToolBar and wxStatusBar Tests - Toolbar and status bar KiCad uses
|
||||
import { test, expect, MAIN_CANVAS, tryLoadApp, getCanvasBox } from './utils/fixtures';
|
||||
import { clickToolbarTool, findRenderedByType } from './utils/element-tracker';
|
||||
import { test, expect, MAIN_CANVAS, getCanvasBox, waitForWxApp } from './utils/fixtures';
|
||||
import { clickToolbarTool, findRenderedByType, stableShot } from './utils/element-tracker';
|
||||
|
||||
test.describe('wxToolBar & wxStatusBar Tests', () => {
|
||||
|
||||
test('Toolbar test app loads successfully', async ({ page, testLogger }) => {
|
||||
await page.goto('/standalone/toolbar/toolbar_test.html');
|
||||
const loaded = await tryLoadApp(page);
|
||||
await waitForWxApp(page);
|
||||
|
||||
await page.screenshot({ path: 'test-results/toolbar-01-loaded.png', fullPage: true });
|
||||
await stableShot(page, 'toolbar-01-loaded.png', { fullPage: true });
|
||||
|
||||
const hasStartup = testLogger.consoleLogs.some(l => l.includes('Toolbar test app started'));
|
||||
|
||||
expect(loaded, 'Toolbar app should load').toBe(true);
|
||||
expect(testLogger.errors.filter(e => !e.includes('favicon'))).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('Toolbar buttons are visible', async ({ page, testLogger }) => {
|
||||
await page.goto('/standalone/toolbar/toolbar_test.html');
|
||||
const loaded = await tryLoadApp(page);
|
||||
expect(loaded, 'App should load').toBe(true);
|
||||
await waitForWxApp(page);
|
||||
|
||||
await page.screenshot({ path: 'test-results/toolbar-02-buttons.png', fullPage: true });
|
||||
await expect.poll(
|
||||
() => testLogger.consoleLogs.some(l => l.includes('Toolbar created')),
|
||||
{ message: 'Toolbar-created event should be logged' }
|
||||
).toBe(true);
|
||||
|
||||
const hasToolbarLog = testLogger.consoleLogs.some(l => l.includes('Toolbar created'));
|
||||
|
||||
expect(hasToolbarLog).toBe(true);
|
||||
await stableShot(page, 'toolbar-02-buttons.png', { fullPage: true });
|
||||
});
|
||||
|
||||
test('New tool button can be clicked', async ({ page, testLogger }) => {
|
||||
await page.goto('/standalone/toolbar/toolbar_test.html');
|
||||
const loaded = await tryLoadApp(page);
|
||||
expect(loaded, 'App should load').toBe(true);
|
||||
await waitForWxApp(page);
|
||||
|
||||
// Click New button using element registry
|
||||
const clicked = await clickToolbarTool(page, 'New');
|
||||
expect(clicked, 'New tool should be found and clicked').toBe(true);
|
||||
await page.waitForTimeout(500);
|
||||
await expect.poll(
|
||||
() => testLogger.consoleLogs.some(l => l.includes('Toolbar: New clicked')),
|
||||
{ message: 'New-clicked event should be logged' }
|
||||
).toBe(true);
|
||||
|
||||
await page.screenshot({ path: 'test-results/toolbar-03-new-clicked.png', fullPage: true });
|
||||
await stableShot(page, 'toolbar-03-new-clicked.png', { fullPage: true });
|
||||
});
|
||||
|
||||
test('Zoom tools can be clicked', async ({ page, testLogger }) => {
|
||||
await page.goto('/standalone/toolbar/toolbar_test.html');
|
||||
const loaded = await tryLoadApp(page);
|
||||
expect(loaded, 'App should load').toBe(true);
|
||||
await waitForWxApp(page);
|
||||
|
||||
// Click Zoom In using element registry
|
||||
const zoomInClicked = await clickToolbarTool(page, 'Zoom In');
|
||||
expect(zoomInClicked, 'Zoom In tool should be found and clicked').toBe(true);
|
||||
await page.waitForTimeout(300);
|
||||
await expect.poll(
|
||||
() => testLogger.consoleLogs.some(l => l.includes('Toolbar: Zoom In clicked')),
|
||||
{ message: 'Zoom-In-clicked event should be logged' }
|
||||
).toBe(true);
|
||||
|
||||
// Click Zoom Out using element registry
|
||||
const zoomOutClicked = await clickToolbarTool(page, 'Zoom Out');
|
||||
expect(zoomOutClicked, 'Zoom Out tool should be found and clicked').toBe(true);
|
||||
await page.waitForTimeout(300);
|
||||
await expect.poll(
|
||||
() => testLogger.consoleLogs.some(l => l.includes('Toolbar: Zoom Out clicked')),
|
||||
{ message: 'Zoom-Out-clicked event should be logged' }
|
||||
).toBe(true);
|
||||
|
||||
await page.screenshot({ path: 'test-results/toolbar-04-zoom.png', fullPage: true });
|
||||
await stableShot(page, 'toolbar-04-zoom.png', { fullPage: true });
|
||||
});
|
||||
|
||||
test('Toggle tool changes state', async ({ page, testLogger }) => {
|
||||
await page.goto('/standalone/toolbar/toolbar_test.html');
|
||||
const loaded = await tryLoadApp(page);
|
||||
expect(loaded, 'App should load').toBe(true);
|
||||
await waitForWxApp(page);
|
||||
|
||||
// Click Toggle button using element registry
|
||||
const toggleClicked1 = await clickToolbarTool(page, 'Toggle');
|
||||
expect(toggleClicked1, 'Toggle tool should be found and clicked').toBe(true);
|
||||
await page.waitForTimeout(300);
|
||||
await expect.poll(
|
||||
() => testLogger.consoleLogs.some(l => l.includes('Toolbar: Toggle ON')),
|
||||
{ message: 'Toggle-ON event should be logged' }
|
||||
).toBe(true);
|
||||
|
||||
await page.screenshot({ path: 'test-results/toolbar-05-toggle-on.png', fullPage: true });
|
||||
await stableShot(page, 'toolbar-05-toggle-on.png', { fullPage: true });
|
||||
|
||||
// Click again to toggle off
|
||||
const toggleClicked2 = await clickToolbarTool(page, 'Toggle');
|
||||
expect(toggleClicked2, 'Toggle tool should be found and clicked again').toBe(true);
|
||||
await page.waitForTimeout(300);
|
||||
await expect.poll(
|
||||
() => testLogger.consoleLogs.some(l => l.includes('Toolbar: Toggle OFF')),
|
||||
{ message: 'Toggle-OFF event should be logged' }
|
||||
).toBe(true);
|
||||
|
||||
await page.screenshot({ path: 'test-results/toolbar-06-toggle-off.png', fullPage: true });
|
||||
await stableShot(page, 'toolbar-06-toggle-off.png', { fullPage: true });
|
||||
});
|
||||
|
||||
test('Status bar shows messages', async ({ page, testLogger }) => {
|
||||
await page.goto('/standalone/toolbar/toolbar_test.html');
|
||||
const loaded = await tryLoadApp(page);
|
||||
expect(loaded, 'App should load').toBe(true);
|
||||
await waitForWxApp(page);
|
||||
|
||||
await page.screenshot({ path: 'test-results/toolbar-07-statusbar.png', fullPage: true });
|
||||
await expect.poll(
|
||||
() => testLogger.consoleLogs.some(l => l.includes('Status bar created')),
|
||||
{ message: 'Status-bar-created event should be logged' }
|
||||
).toBe(true);
|
||||
|
||||
const hasStatusBarLog = testLogger.consoleLogs.some(l => l.includes('Status bar created'));
|
||||
|
||||
expect(hasStatusBarLog).toBe(true);
|
||||
await stableShot(page, 'toolbar-07-statusbar.png', { fullPage: true });
|
||||
});
|
||||
|
||||
test('All toolbar buttons accessible', async ({ page, testLogger }) => {
|
||||
await page.goto('/standalone/toolbar/toolbar_test.html');
|
||||
const loaded = await tryLoadApp(page);
|
||||
expect(loaded, 'App should load').toBe(true);
|
||||
await waitForWxApp(page);
|
||||
|
||||
// Verify all tools are registered
|
||||
const tools = await findRenderedByType(page, 'tool');
|
||||
|
|
@ -102,13 +112,24 @@ test.describe('wxToolBar & wxStatusBar Tests', () => {
|
|||
|
||||
// Click all toolbar buttons by label
|
||||
const toolLabels = ['New', 'Open', 'Save', 'Zoom In', 'Zoom Out', 'Toggle'];
|
||||
const clickEvent: Record<string, string> = {
|
||||
'New': 'Toolbar: New clicked',
|
||||
'Open': 'Toolbar: Open clicked',
|
||||
'Save': 'Toolbar: Save clicked',
|
||||
'Zoom In': 'Toolbar: Zoom In clicked',
|
||||
'Zoom Out': 'Toolbar: Zoom Out clicked',
|
||||
'Toggle': 'Toolbar: Toggle',
|
||||
};
|
||||
for (const label of toolLabels) {
|
||||
const clicked = await clickToolbarTool(page, label);
|
||||
expect(clicked, `Tool "${label}" should be found and clicked`).toBe(true);
|
||||
await page.waitForTimeout(200);
|
||||
await expect.poll(
|
||||
() => testLogger.consoleLogs.some(l => l.includes(clickEvent[label])),
|
||||
{ message: `Tool "${label}" click event should be logged` }
|
||||
).toBe(true);
|
||||
}
|
||||
|
||||
await page.screenshot({ path: 'test-results/toolbar-08-all-buttons.png', fullPage: true });
|
||||
await stableShot(page, 'toolbar-08-all-buttons.png', { fullPage: true });
|
||||
|
||||
expect(testLogger.errors.filter(e => !e.includes('favicon'))).toHaveLength(0);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue