bench: JSPI vs asyncify A/B — harness + results

Adds pcbnew-large-perf.spec.ts (PERF_LARGE-gated: repeated cold loads,
vme-wren/jetson opens, rAF + distinct-glcanvas-frame FPS under throttle,
wasm/JS heap checkpoints), fetchIntoMemfs + openAndWait/sampleMemory/
measureFpsDetailed perf-utils, dual 9.99+10.0 config seeding in pcbnew.html
so foreign-branch builds boot wizard-free, and the full benchmark report +
raw data under docs/features/async/migration-evidence/.

Headlines: wasm 94 vs 113 MB raw (18.6 vs 36.7 MB gzip), post-link tail
1.6 s/49 MB vs 63 s/6.1 GB per build, cold load −40 %, 27.7 MB board open
−45 %, real redraws +68 % at 4× throttle, boot heap −31 %.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016X9eh1s5sTx1o9Em9KBuwR
This commit is contained in:
Viktor Vaczi 2026-08-14 11:53:06 +02:00
commit da299ed6f9
12 changed files with 707 additions and 11 deletions

View file

@ -130,23 +130,28 @@
// Seed a minimal default config before main() so STARTWIZARD finds
// NeedsUserInput()==false and the first-run setup wizard never opens —
// same as eeschema.html / pl_editor.html.
// same as eeschema.html / pl_editor.html. Seeded for BOTH the current
// (10.0) and previous (9.99) settings versions: the perf-bench flow
// (pcbnew-large-perf.spec.ts) swaps in editor builds from other branches
// whose settings version differs, and an unseeded dir would send that
// binary into the wizard and skew every load-time number.
var seedKicadConfig = function() {
var cfgDir = '/home/kicad/.config/kicad/kicad/10.0';
FS.mkdirTree(cfgDir);
var writeIfAbsent = function(path, contents) {
try { FS.stat(path); return; } catch (e) { /* absent — seed it */ }
FS.writeFile(path, contents);
console.log('[KICAD] Seeded ' + path);
};
writeIfAbsent(cfgDir + '/kicad_common.json', JSON.stringify({
do_not_show_again: { update_check_prompt: true, data_collection_prompt: true }
}, null, 2));
writeIfAbsent(cfgDir + '/sym-lib-table', '(sym_lib_table\n (version 7)\n)\n');
writeIfAbsent(cfgDir + '/fp-lib-table', '(fp_lib_table\n (version 7)\n)\n');
writeIfAbsent(cfgDir + '/design-block-lib-table', '(design_block_lib_table\n (version 7)\n)\n');
['9.99', '10.0'].forEach(function(ver) {
var cfgDir = '/home/kicad/.config/kicad/kicad/' + ver;
FS.mkdirTree(cfgDir);
writeIfAbsent(cfgDir + '/kicad_common.json', JSON.stringify({
do_not_show_again: { update_check_prompt: true, data_collection_prompt: true }
}, null, 2));
writeIfAbsent(cfgDir + '/sym-lib-table', '(sym_lib_table\n (version 7)\n)\n');
writeIfAbsent(cfgDir + '/fp-lib-table', '(fp_lib_table\n (version 7)\n)\n');
writeIfAbsent(cfgDir + '/design-block-lib-table', '(design_block_lib_table\n (version 7)\n)\n');
});
};
var Module = {

View file

@ -0,0 +1,180 @@
import { test, expect } from './fixtures';
import * as fs from 'fs';
import * as path from 'path';
import * as crypto from 'crypto';
import {
measureLoad,
measureOpenRender,
openAndWait,
measureFpsDetailed,
setThrottle,
sampleMemory,
startHeapPeakSampler,
stopHeapPeakSampler,
getWasmResourceTiming,
} from './utils/perf-utils';
import { fetchIntoMemfs } from './utils/fs-inject';
import { waitForBoardLoaded } from './utils/board-ready';
/**
* Large-board perf battery (TRACK-ONLY, bench-driven). Extends pcbnew-perf with
* the measurements the JSPI-vs-asyncify comparison needs: repeated cold loads,
* open+render of a REAL large board (vme-wren: 1508 footprints / 24 858
* segments), rAF + distinct-frame FPS under CPU throttle, and wasm-heap /
* JS-heap checkpoints. Results append to
* tests/test-results/perf-bench-<arm>.ndjson (gitignored), one JSON row per
* measurement, each row carrying the sha256 of the wasm actually measured so
* A/B artifact swaps can't get misattributed.
*
* Gated behind PERF_LARGE=1 so the CI perf project (which matches
* *-perf.spec.ts) is unaffected. Fixtures are expected in
* tests/apps/kicad/board/ (gitignored) see docs/features/async/
* migration-evidence/jspi-vs-asyncify-bench-2026-08.md for the bench flow.
*/
const ARM = process.env.BENCH_ARM || 'current';
const LOAD_RUNS = parseInt(process.env.PERF_LOAD_RUNS || '5', 10);
const OPEN_RUNS = parseInt(process.env.PERF_OPEN_RUNS || '3', 10);
const THROTTLES = (process.env.PERF_THROTTLES || '1,4,6').split(',').map(Number);
const FPS_SECS = parseInt(process.env.PERF_FPS_SECS || '6', 10);
const FPS_REPS = parseInt(process.env.PERF_FPS_REPS || '2', 10);
const APPS_KICAD = path.join(__dirname, '..', 'apps', 'kicad');
const DEMO = path.join(__dirname, '..', 'fixtures', 'demo', 'demo.kicad_pcb');
const VME_URL = '/kicad/board/vme-wren.kicad_pcb';
const JETSON_URL = '/kicad/board/jetson-agx-thor-baseboard.kicad_pcb';
// NOT under test-results/ — Playwright clears that whole dir at session start,
// so an A/B pair of invocations would each wipe the other arm's rows.
const RESULTS = path.join(__dirname, '..', 'bench-results', `perf-bench-${ARM}.ndjson`);
let wasmSha = '';
function wasmSha256(): string {
if (!wasmSha) {
const bytes = fs.readFileSync(path.join(APPS_KICAD, 'kicad_editor.wasm'));
wasmSha = crypto.createHash('sha256').update(bytes).digest('hex').slice(0, 16);
}
return wasmSha;
}
function record(section: string, data: Record<string, unknown>): void {
fs.mkdirSync(path.dirname(RESULTS), { recursive: true });
const row = { arm: ARM, sha256: wasmSha256(), when: new Date().toISOString(), section, ...data };
fs.appendFileSync(RESULTS, JSON.stringify(row) + '\n');
// eslint-disable-next-line no-console
console.log(`[bench] ${section}: ${JSON.stringify(data)}`);
}
test.describe('pcbnew large-board bench', () => {
test.skip(!process.env.PERF_LARGE, 'bench battery — run with PERF_LARGE=1');
for (let run = 0; run < LOAD_RUNS; run++) {
test(`cold load #${run + 1}`, async ({ page }) => {
test.setTimeout(300000);
const loadMs = await measureLoad(page, '/kicad/pcbnew.html');
const wasmFetch = await getWasmResourceTiming(page);
const mem = await sampleMemory(page);
record('load', { run: run + 1, loadMs, wasmFetch, bootMem: mem });
expect(loadMs).toBeGreaterThan(0);
});
}
test('cold load with CDP pre-attached (tier-down sanity)', async ({ page }) => {
test.setTimeout(300000);
const cdp = await page.context().newCDPSession(page);
await setThrottle(cdp, 1); // attach + a no-op emulation command, like the FPS path
const loadMs = await measureLoad(page, '/kicad/pcbnew.html');
record('load-cdp-sanity', { loadMs });
expect(loadMs).toBeGreaterThan(0);
});
for (let run = 0; run < OPEN_RUNS; run++) {
test(`open demo board #${run + 1}`, async ({ page, testLogger }) => {
test.setTimeout(300000);
await measureLoad(page, '/kicad/pcbnew.html');
const openMs = await measureOpenRender(page, DEMO, 'board', testLogger);
const mem = await sampleMemory(page);
record('open-demo', { run: run + 1, openMs, postOpenMem: mem });
expect(openMs).toBeGreaterThan(0);
});
}
for (let run = 0; run < OPEN_RUNS; run++) {
test(`open vme-wren (27.7 MB) #${run + 1}`, async ({ page, testLogger }) => {
test.setTimeout(600000);
await measureLoad(page, '/kicad/pcbnew.html');
const bytes = await fetchIntoMemfs(page, VME_URL, '/home/kicad/documents/vme-wren.kicad_pcb');
await startHeapPeakSampler(page);
const openMs = await openAndWait(
page,
'/home/kicad/documents/vme-wren.kicad_pcb',
'board',
testLogger,
480000,
);
const peak = await stopHeapPeakSampler(page);
const mem = await sampleMemory(page);
record('open-vme', { run: run + 1, bytes, openMs, openPeakHeap: peak, postOpenMem: mem });
expect(openMs).toBeGreaterThan(0);
});
}
test('FPS on vme-wren across throttles', async ({ page, testLogger }) => {
test.setTimeout(900000);
await measureLoad(page, '/kicad/pcbnew.html');
await fetchIntoMemfs(page, VME_URL, '/home/kicad/documents/vme-wren.kicad_pcb');
await openAndWait(page, '/home/kicad/documents/vme-wren.kicad_pcb', 'board', testLogger, 480000);
await page.keyboard.press('Escape').catch(() => {}); // eslint-disable-line -- best-effort Escape
const cdp = await page.context().newCDPSession(page);
for (const rate of THROTTLES) {
await setThrottle(cdp, rate);
for (let rep = 0; rep < FPS_REPS; rep++) {
const f = await measureFpsDetailed(page, FPS_SECS);
record('fps-vme', { throttle: rate, rep: rep + 1, ...f });
expect(f.rafFps).toBeGreaterThan(0);
}
}
await setThrottle(cdp, 1);
const mem = await sampleMemory(page);
record('fps-vme-postmem', { postFpsMem: mem });
});
test('open jetson-agx-thor (80.9 MB) — outcome, OOM allowed', async ({ page, testLogger }) => {
test.setTimeout(900000);
await measureLoad(page, '/kicad/pcbnew.html');
const bytes = await fetchIntoMemfs(
page,
JETSON_URL,
'/home/kicad/documents/jetson-agx-thor-baseboard.kicad_pcb',
);
await startHeapPeakSampler(page);
const t0 = Date.now();
try {
await page.evaluate(() => {
(window as unknown as { Module: { kicadOpenFile(p: string): unknown } }).Module.kicadOpenFile(
'/home/kicad/documents/jetson-agx-thor-baseboard.kicad_pcb',
);
});
await waitForBoardLoaded(page, testLogger, 780000);
const peak = await stopHeapPeakSampler(page);
const mem = await sampleMemory(page);
record('open-jetson', {
bytes,
outcome: 'loaded',
openMs: Date.now() - t0,
openPeakHeap: peak,
postOpenMem: mem,
});
} catch (e) {
// A 4 GB-cap OOM / abort is a RESULT for this stress tier, not a harness error.
const peak = await stopHeapPeakSampler(page).catch(() => -1);
record('open-jetson', {
bytes,
outcome: 'failed',
afterMs: Date.now() - t0,
openPeakHeap: peak,
error: String(e).slice(0, 300),
});
}
});
});

View file

@ -42,6 +42,36 @@ export async function injectFileIntoMemfs(
);
}
/**
* Fetch a same-origin URL inside the page and write it into MEMFS.
*
* For large fixtures (tens of MB) the base64 round-trip through Playwright's
* JSON channel in injectFileIntoMemfs is the bottleneck an in-page fetch
* from the static server (tests/apps is the serve root) keeps the bytes in
* the browser. Returns the byte count written.
*/
export async function fetchIntoMemfs(
page: Page,
url: string, // e.g. "/kicad/board/vme-wren.kicad_pcb"
memfsPath: string,
): Promise<number> {
const memfsDir = memfsPath.replace(/\/[^/]+$/, '') || '/';
return await page.evaluate(
async ({ url, memfsDir, memfsPath }) => {
const res = await fetch(url);
if (!res.ok) throw new Error(`fetchIntoMemfs: ${url} -> HTTP ${res.status}`);
const data = new Uint8Array(await res.arrayBuffer());
// @ts-expect-error — Emscripten FS lives on window via Module
const FS = (window as any).FS;
FS.mkdirTree(memfsDir);
FS.writeFile(memfsPath, data);
console.log(`[KICAD] Fetched ${url} -> ${memfsPath} (${data.length} bytes)`);
return data.length;
},
{ url, memfsDir, memfsPath },
);
}
/**
* Convenience: read a file from the kicad/ submodule and inject it.
*/

View file

@ -64,7 +64,22 @@ export async function measureOpenRender(
const ext = kind === 'board' ? 'kicad_pcb' : 'kicad_sch';
const memfsPath = `/home/kicad/documents/perf-demo.${ext}`;
await injectFileIntoMemfs(page, hostPath, memfsPath);
return openAndWait(page, memfsPath, kind, logger, timeout);
}
/**
* Open an ALREADY-INJECTED memfs document and wait until loaded+rendered.
* Split out of measureOpenRender so large fixtures can arrive via
* fetchIntoMemfs (or any other route) and still get the same timed open.
*/
export async function openAndWait(
page: Page,
memfsPath: string,
kind: 'schematic' | 'board',
logger: { consoleLogs: string[]; errors: string[] },
timeout = 120000,
): Promise<number> {
const stem = memfsPath.replace(/^.*\//, '').replace(/\.[^.]+$/, '');
const t0 = Date.now();
await page.evaluate((p) => {
(window as unknown as { Module: KicadModule }).Module.kicadOpenFile(p);
@ -75,13 +90,77 @@ export async function measureOpenRender(
} else {
const deadline = Date.now() + timeout;
while (Date.now() < deadline) {
if (/perf-demo/i.test(await page.title())) break;
if (new RegExp(stem, 'i').test(await page.title())) break;
await page.waitForTimeout(200);
}
}
return Date.now() - t0;
}
export interface MemorySample {
/** WebAssembly linear memory size (Module.HEAPU8.byteLength). */
wasmHeapBytes: number;
/** Chromium-only usedJSHeapSize; 0 elsewhere. */
jsHeapBytes: number;
}
/** One-shot memory census — portable across builds (HEAPU8 verified reachable in both). */
export async function sampleMemory(page: Page): Promise<MemorySample> {
return await page.evaluate(() => {
const w = window as unknown as {
Module?: { HEAPU8?: { byteLength: number }; wasmMemory?: { buffer: ArrayBuffer } };
};
const perf = performance as unknown as { memory?: { usedJSHeapSize: number } };
return {
wasmHeapBytes:
w.Module?.wasmMemory?.buffer?.byteLength ?? w.Module?.HEAPU8?.byteLength ?? 0,
jsHeapBytes: perf.memory?.usedJSHeapSize ?? 0,
};
});
}
/**
* In-page wasm-heap peak sampler (250 ms). Start before a heavy operation
* (board open), stop after returns the max linear-memory size observed.
*/
export async function startHeapPeakSampler(page: Page): Promise<void> {
await page.evaluate(() => {
const w = window as unknown as {
Module?: { HEAPU8?: { byteLength: number }; wasmMemory?: { buffer: ArrayBuffer } };
__heapPeak?: number;
__heapPeakTimer?: number;
};
if (w.__heapPeakTimer !== undefined) clearInterval(w.__heapPeakTimer);
w.__heapPeak = 0;
w.__heapPeakTimer = setInterval(() => {
const b =
w.Module?.wasmMemory?.buffer?.byteLength ?? w.Module?.HEAPU8?.byteLength ?? 0;
if (b > (w.__heapPeak ?? 0)) w.__heapPeak = b;
}, 250) as unknown as number;
});
}
export async function stopHeapPeakSampler(page: Page): Promise<number> {
return await page.evaluate(() => {
const w = window as unknown as { __heapPeak?: number; __heapPeakTimer?: number };
if (w.__heapPeakTimer !== undefined) clearInterval(w.__heapPeakTimer);
w.__heapPeakTimer = undefined;
return w.__heapPeak ?? 0;
});
}
/** Resource-timing attribution for the main wasm fetch (download share of loadMs). */
export async function getWasmResourceTiming(
page: Page,
): Promise<{ durationMs: number; transferSize: number } | null> {
return await page.evaluate(() => {
const e = performance
.getEntriesByType('resource')
.find((r) => r.name.includes('kicad_editor.wasm')) as PerformanceResourceTiming | undefined;
return e ? { durationMs: Math.round(e.duration), transferSize: e.transferSize } : null;
});
}
/** CDP CPU throttling (Chromium only): 1 = none, N = N× slower. */
export async function setThrottle(cdp: CDPSession, rate: number): Promise<void> {
await cdp.send('Emulation.setCPUThrottlingRate', { rate });
@ -127,6 +206,79 @@ export async function measureFps(page: Page, seconds: number): Promise<number> {
return +(frames / (elapsed / 1000)).toFixed(1);
}
/**
* measureFps plus a DISTINCT-frame counter: rAF keeps ticking at vsync even
* when the GAL skips redraws (draw_panel_gal enforces a min redraw period and
* re-arms a timer when it can't keep up), so under load the rAF number can
* decouple from real render throughput. Alongside the rAF loop this samples
* the largest visible canvas at ~30 Hz (48×48 downscale hash the
* waitForCanvasStable technique; GAL sets preserveDrawingBuffer) and counts
* samples whose content changed. distinctFps is capped by the ~30 Hz sample
* rate; read it as "real redraws per second, up to 30".
*/
export async function measureFpsDetailed(
page: Page,
seconds: number,
): Promise<{ rafFps: number; distinctFps: number }> {
await page.evaluate(() => {
const w = window as unknown as {
__dfPrev?: string;
__dfCount?: number;
__dfSamples?: number;
__dfTimer?: number;
};
if (w.__dfTimer !== undefined) clearInterval(w.__dfTimer);
w.__dfPrev = undefined;
w.__dfCount = 0;
w.__dfSamples = 0;
const scratch = document.createElement('canvas');
scratch.width = 48;
scratch.height = 48;
const ctx = scratch.getContext('2d', { willReadFrequently: true })!;
w.__dfTimer = setInterval(() => {
const canvases = Array.from(document.querySelectorAll('canvas')).filter((c) => {
const r = c.getBoundingClientRect();
return r.width > 0 && r.height > 0 && c !== scratch;
});
if (!canvases.length) return;
// The GAL draws into a wxGLCanvas (id glcanvas-*), NOT the
// full-window emscripten #canvas — sample the GL surface where the
// board pixels actually change, falling back to the largest canvas.
const gl = canvases.filter((c) => /^glcanvas/.test(c.id));
const pool = gl.length ? gl : canvases;
const src = pool.reduce((a, b) => {
const ra = a.getBoundingClientRect();
const rb = b.getBoundingClientRect();
return ra.width * ra.height >= rb.width * rb.height ? a : b;
});
try {
ctx.drawImage(src, 0, 0, 48, 48);
const d = ctx.getImageData(0, 0, 48, 48).data;
let h = 0;
for (let i = 0; i < d.length; i += 16) h = ((h << 5) - h + d[i]) | 0;
const hs = String(h);
w.__dfSamples = (w.__dfSamples ?? 0) + 1;
if (w.__dfPrev !== undefined && hs !== w.__dfPrev) w.__dfCount = (w.__dfCount ?? 0) + 1;
w.__dfPrev = hs;
} catch {
/* tainted/zero-size canvas — skip the sample */
}
}, 33) as unknown as number;
});
const t0 = Date.now();
const rafFps = await measureFps(page, seconds);
const elapsed = (Date.now() - t0) / 1000;
const distinct = await page.evaluate(() => {
const w = window as unknown as { __dfCount?: number; __dfTimer?: number };
if (w.__dfTimer !== undefined) clearInterval(w.__dfTimer);
w.__dfTimer = undefined;
return w.__dfCount ?? 0;
});
return { rafFps, distinctFps: +(distinct / elapsed).toFixed(1) };
}
/** Write per-app results to tests/test-results/perf-<app>.json (gitignored, CI-uploaded). */
export function recordPerf(app: string, data: Record<string, unknown>): void {
fs.mkdirSync(RESULTS_DIR, { recursive: true });