diff --git a/.github/workflows/wasm-build.yml b/.github/workflows/wasm-build.yml index 4de7685..46847a4 100644 --- a/.github/workflows/wasm-build.yml +++ b/.github/workflows/wasm-build.yml @@ -304,6 +304,16 @@ jobs: working-directory: tests run: xvfb-run -a npm run test:kicad:ci + # Runtime-perf E2E (eeschema + pcbnew): measures the current build's + # load / open+render / FPS and writes tests/test-results/perf-*.json. + # Track-only — never gates the build (continue-on-error). CI is + # headless/SwiftShader so FPS is CPU-bound + noisy; openMs is the stable number. + - name: KiCad runtime perf (track-only, non-gating) + if: inputs.run_tests + continue-on-error: true + working-directory: tests + run: xvfb-run -a npm run test:perf + - name: Upload test logs & screenshots if: always() && inputs.run_tests uses: actions/upload-artifact@v4 diff --git a/.gitignore b/.gitignore index e1511fb..3cc561f 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,11 @@ output/ /bench/ /scripts/bench/vm/ +# Temporary/disposable prebuilt WASM sets (native-EH + JS-EH × eeschema + pcbnew +# O1 builds) + the standalone perf harness, used to reproduce the runtime-perf +# comparison on demand. Large (~1 GB), never committed. +/benchmark-builds/ + /tests/.test-port-asyncify /memory/ diff --git a/tests/kicad/eeschema-perf.spec.ts b/tests/kicad/eeschema-perf.spec.ts new file mode 100644 index 0000000..acfd463 --- /dev/null +++ b/tests/kicad/eeschema-perf.spec.ts @@ -0,0 +1,47 @@ +import { test, expect } from './fixtures'; +import * as path from 'path'; +import { measureLoad, measureOpenRender, measureFps, setThrottle, recordPerf } from './utils/perf-utils'; + +/** + * eeschema runtime-perf (TRACK-ONLY, no gating). + * + * Measures the CURRENT build: cold load, open+render of the demo schematic, and + * sustained pan/zoom FPS across CPU-throttle rates. Numbers are logged and written + * to tests/test-results/perf-eeschema.json (CI uploads it). The only assertions are + * "the app booted and the doc opened" — never a perf threshold (would flake CI). + * Runs on the Chromium 'perf' project (CDP throttling); pass --headed for real-GPU FPS. + * Note: on headless/SwiftShader CI, FPS is CPU-bound and noisy — openMs is the stable metric. + */ + +const THROTTLES = (process.env.PERF_THROTTLES || '1,4,6').split(',').map(Number); +const FPS_SECS = parseInt(process.env.PERF_FPS_SECS || '6', 10); +const DEMO = path.join(__dirname, '..', 'fixtures', 'demo', 'demo.kicad_sch'); + +test.describe('eeschema perf', () => { + test('load + open+render + FPS (track-only)', async ({ page, testLogger }) => { + test.setTimeout(300000); + + const loadMs = await measureLoad(page, '/kicad/eeschema.html'); + console.log(`[perf] eeschema cold load = ${loadMs} ms`); + + const openMs = await measureOpenRender(page, DEMO, 'schematic', testLogger); + console.log(`[perf] eeschema open+render = ${openMs} ms`); + await page.keyboard.press('Escape').catch(() => {}); + + const cdp = await page.context().newCDPSession(page); + const fps: { throttle: number; fps: number }[] = []; + for (const rate of THROTTLES) { + await setThrottle(cdp, rate); + const f = await measureFps(page, FPS_SECS); + console.log(`[perf] eeschema FPS @ ${rate}x = ${f}`); + fps.push({ throttle: rate, fps: f }); + } + await setThrottle(cdp, 1); + + recordPerf('eeschema', { loadMs, openMs, fps }); + + // Track-only: assert only that it booted + opened. + expect(loadMs).toBeGreaterThan(0); + expect(openMs).toBeGreaterThan(0); + }); +}); diff --git a/tests/kicad/pcbnew-perf.spec.ts b/tests/kicad/pcbnew-perf.spec.ts new file mode 100644 index 0000000..d8f2649 --- /dev/null +++ b/tests/kicad/pcbnew-perf.spec.ts @@ -0,0 +1,44 @@ +import { test, expect } from './fixtures'; +import * as path from 'path'; +import { measureLoad, measureOpenRender, measureFps, setThrottle, recordPerf } from './utils/perf-utils'; + +/** + * pcbnew runtime-perf (TRACK-ONLY, no gating). Mirrors eeschema-perf for the board editor. + * + * Measures the CURRENT build: cold load, open+render of the demo board, and sustained + * pan/zoom FPS across CPU-throttle rates → tests/test-results/perf-pcbnew.json (CI uploads it). + * Runs on the Chromium 'perf' project (pcbnew's big module OOMs Firefox/SpiderMonkey anyway, + * and CDP throttling is Chromium-only). Only asserts booted + opened; never a perf threshold. + */ + +const THROTTLES = (process.env.PERF_THROTTLES || '1,4,6').split(',').map(Number); +const FPS_SECS = parseInt(process.env.PERF_FPS_SECS || '6', 10); +const DEMO = path.join(__dirname, '..', 'fixtures', 'demo', 'demo.kicad_pcb'); + +test.describe('pcbnew perf', () => { + test('load + open+render + FPS (track-only)', async ({ page, testLogger }) => { + test.setTimeout(300000); + + const loadMs = await measureLoad(page, '/kicad/pcbnew.html'); + console.log(`[perf] pcbnew cold load = ${loadMs} ms`); + + const openMs = await measureOpenRender(page, DEMO, 'board', testLogger); + console.log(`[perf] pcbnew open+render = ${openMs} ms`); + await page.keyboard.press('Escape').catch(() => {}); + + const cdp = await page.context().newCDPSession(page); + const fps: { throttle: number; fps: number }[] = []; + for (const rate of THROTTLES) { + await setThrottle(cdp, rate); + const f = await measureFps(page, FPS_SECS); + console.log(`[perf] pcbnew FPS @ ${rate}x = ${f}`); + fps.push({ throttle: rate, fps: f }); + } + await setThrottle(cdp, 1); + + recordPerf('pcbnew', { loadMs, openMs, fps }); + + expect(loadMs).toBeGreaterThan(0); + expect(openMs).toBeGreaterThan(0); + }); +}); diff --git a/tests/kicad/utils/perf-utils.ts b/tests/kicad/utils/perf-utils.ts new file mode 100644 index 0000000..1283dd3 --- /dev/null +++ b/tests/kicad/utils/perf-utils.ts @@ -0,0 +1,138 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import type { Page, CDPSession } from '@playwright/test'; +import { injectFileIntoMemfs } from './fs-inject'; +import { waitForBoardLoaded } from './board-ready'; + +/** + * Runtime-perf helpers for the track-only perf specs (eeschema-perf, pcbnew-perf). + * + * Measures the CURRENT build (whatever setup:kicad staged into tests/apps/kicad/): + * cold load, open+render time, and interaction FPS. No version A/B, no gating — + * results are logged and written to tests/test-results/perf-.json (gitignored, + * uploaded by CI). The measurement logic (the #canvas real-input FPS driver, the + * ready-signal, CDP throttling) is lifted from the validated native-vs-JS-EH + * benchmark harness — see docs/features/wasm-exceptions/12-native-vs-jseh-benchmark.md. + */ + +const MAIN_CANVAS = '#canvas'; +const RESULTS_DIR = path.join(__dirname, '..', '..', 'test-results'); + +type KicadModule = { kicadOpenFile(p: string): unknown }; + +/** Fully booted editor: visible canvas + wx registry + kicadOpenFile hook + a top-level *Frame. */ +export async function waitForReady(page: Page, timeout = 120000): Promise { + await page.locator(MAIN_CANVAS).waitFor({ state: 'visible', timeout }); + await page.waitForFunction(() => !!window.wxElementRegistry, null, { timeout }); + await page.waitForFunction( + () => + typeof (window as unknown as { Module?: KicadModule }).Module?.kicadOpenFile === 'function', + null, + { timeout }, + ); + await page.waitForFunction( + () => + !!window.wxElementRegistry && + window.wxElementRegistry + .findAll({ visible: true }) + .some((e) => /Frame$/.test(e.typeName) || (e.name || '').endsWith('Frame')), + null, + { timeout }, + ); +} + +/** Cold load: navigate then wait until fully ready. Returns ms. */ +export async function measureLoad(page: Page, url: string, timeout = 120000): Promise { + const t0 = Date.now(); + await page.goto(url, { waitUntil: 'commit', timeout }); + await waitForReady(page, timeout); + return Date.now() - t0; +} + +/** + * Open a document via Module.kicadOpenFile and wait until it's loaded+rendered. + * 'schematic' polls the editor title; 'board' uses the pcbnew progress-dialog signal. + * Returns ms. + */ +export async function measureOpenRender( + page: Page, + hostPath: string, + kind: 'schematic' | 'board', + logger: { consoleLogs: string[]; errors: string[] }, + timeout = 120000, +): Promise { + const ext = kind === 'board' ? 'kicad_pcb' : 'kicad_sch'; + const memfsPath = `/home/kicad/documents/perf-demo.${ext}`; + await injectFileIntoMemfs(page, hostPath, memfsPath); + + const t0 = Date.now(); + await page.evaluate((p) => { + (window as unknown as { Module: KicadModule }).Module.kicadOpenFile(p); + }, memfsPath); + + if (kind === 'board') { + await waitForBoardLoaded(page, logger, timeout); + } else { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + if (/perf-demo/i.test(await page.title())) break; + await page.waitForTimeout(200); + } + } + return Date.now() - t0; +} + +/** CDP CPU throttling (Chromium only): 1 = none, N = N× slower. */ +export async function setThrottle(cdp: CDPSession, rate: number): Promise { + await cdp.send('Emulation.setCPUThrottlingRate', { rate }); +} + +/** + * Sustained interaction FPS: drive real pan/zoom on #canvas (the emscripten input + * surface — glcanvas-* can be display:none) for `seconds`, counting main-thread rAF + * frames. Whatever throttle is currently set applies. + */ +export async function measureFps(page: Page, seconds: number): Promise { + const box = await page.locator(MAIN_CANVAS).boundingBox(); + if (!box) return 0; + type W = { __perfFrames: number; __perfRAF?: number }; + // Start ONE rAF frame counter, cancelling any loop left over from a prior call + // (otherwise loops accumulate across a throttle sweep and inflate the count). + await page.evaluate(() => { + const w = window as unknown as W; + if (w.__perfRAF !== undefined) cancelAnimationFrame(w.__perfRAF); + w.__perfFrames = 0; + const loop = () => { + w.__perfFrames++; + w.__perfRAF = requestAnimationFrame(loop); + }; + w.__perfRAF = requestAnimationFrame(loop); + }); + const cx = box.x + box.width / 2; + const cy = box.y + box.height / 2; + const start = Date.now(); + let k = 0; + await page.mouse.move(cx, cy); + while (Date.now() - start < seconds * 1000) { + await page.mouse.move(cx + Math.round(120 * Math.sin(k / 3)), cy + Math.round(80 * Math.cos(k / 4))); + if (k % 3 === 0) await page.mouse.wheel(0, k % 6 < 3 ? -120 : 120); + k++; + } + const elapsed = Date.now() - start; + const frames = await page.evaluate(() => { + const w = window as unknown as W; + if (w.__perfRAF !== undefined) cancelAnimationFrame(w.__perfRAF); + return w.__perfFrames; + }); + return +(frames / (elapsed / 1000)).toFixed(1); +} + +/** Write per-app results to tests/test-results/perf-.json (gitignored, CI-uploaded). */ +export function recordPerf(app: string, data: Record): void { + fs.mkdirSync(RESULTS_DIR, { recursive: true }); + const out = { app, when: new Date().toISOString(), ...data }; + fs.writeFileSync(path.join(RESULTS_DIR, `perf-${app}.json`), JSON.stringify(out, null, 2)); + // Also echo a compact line so it lands in the captured test log / CI output. + // eslint-disable-next-line no-console + console.log(`[perf] ${app}: ${JSON.stringify(data)}`); +} diff --git a/tests/package.json b/tests/package.json index 918fb5b..1638324 100644 --- a/tests/package.json +++ b/tests/package.json @@ -18,6 +18,7 @@ "test:kicad": "npm run test:kicad:firefox", "test:kicad:ci": "npm run setup:kicad && playwright test --config=playwright-kicad.config.ts --project=firefox --project=chromium-ci", "test:kicad:headed": "npm run test:kicad:chrome", + "test:perf": "npm run setup:kicad && playwright test --config=playwright-kicad.config.ts --project=perf --workers=1", "test:pcbnew:firefox": "npm run setup:kicad && playwright test --config=playwright-kicad.config.ts --project=firefox kicad/pcbnew.spec.ts", "test:pcbnew:chrome": "npm run setup:kicad && playwright test --config=playwright-kicad.config.ts --project=chromium --headed kicad/pcbnew.spec.ts", "test:eeschema:firefox": "npm run setup:kicad && playwright test --config=playwright-kicad.config.ts --project=firefox kicad/eeschema.spec.ts", diff --git a/tests/playwright-kicad.config.ts b/tests/playwright-kicad.config.ts index 058f3a3..7da34c7 100644 --- a/tests/playwright-kicad.config.ts +++ b/tests/playwright-kicad.config.ts @@ -92,6 +92,11 @@ const PCBNEW_FAMILY_SPECS = [ '**/footprint-3d-preview.spec.ts', ]; +// Runtime-perf specs run ONLY on the Chromium 'perf' project below: they need +// CDP CPU throttling (Chromium-only) and pcbnew needs V8. Excluded from the +// firefox/chromium projects so they don't double-run there. +const PERF_SPECS = ['**/*-perf.spec.ts']; + const appsDir = 'apps'; export default defineConfig({ @@ -120,9 +125,9 @@ export default defineConfig({ { // Firefox is the default for headless testing (works on ARM Mac) name: 'firefox', - // On CI the pcbnew-family specs run on chromium-ci instead (see - // PCBNEW_FAMILY_SPECS above for why). - ...(process.env.CI ? { testIgnore: PCBNEW_FAMILY_SPECS } : {}), + // Perf specs always run on the dedicated 'perf' project, never here. On CI + // the pcbnew-family specs also move to chromium-ci (see PCBNEW_FAMILY_SPECS). + testIgnore: [...PERF_SPECS, ...(process.env.CI ? PCBNEW_FAMILY_SPECS : [])], use: { ...devices['Desktop Firefox'], viewport: { width: 1280, height: 720 }, @@ -157,6 +162,7 @@ export default defineConfig({ // Chromium issues #1416283, #338414704 (SwiftShader WebGL bug). // Run via: npm run test:kicad:headed name: 'chromium', + testIgnore: PERF_SPECS, use: { channel: 'chrome', viewport: { width: 1280, height: 720 }, @@ -178,6 +184,20 @@ export default defineConfig({ }, }, }, + { + // Runtime-perf specs (*-perf.spec.ts): bundled Chromium for CDP CPU throttling. + // Bundled (not system Chrome) because system Chrome paces rAF oddly under CDP + // throttle (FPS rose with throttle). --enable-unsafe-swiftshader lets it use + // software WebGL headless on CI; harmless with a real GPU locally. Add --headed + // locally for real-GPU FPS numbers. + name: 'perf', + testMatch: PERF_SPECS, + use: { + ...devices['Desktop Chrome'], + viewport: { width: 1280, height: 720 }, + launchOptions: { args: ['--enable-unsafe-swiftshader'] }, + }, + }, ], webServer: {