bench: JSPI vs asyncify A/B — harness + results
Adds pcbnew-large-perf.spec.ts (PERF_LARGE-gated: repeated cold loads, vme-wren/jetson opens, rAF + distinct-glcanvas-frame FPS under throttle, wasm/JS heap checkpoints), fetchIntoMemfs + openAndWait/sampleMemory/ measureFpsDetailed perf-utils, dual 9.99+10.0 config seeding in pcbnew.html so foreign-branch builds boot wizard-free, and the full benchmark report + raw data under docs/features/async/migration-evidence/. Headlines: wasm 94 vs 113 MB raw (18.6 vs 36.7 MB gzip), post-link tail 1.6 s/49 MB vs 63 s/6.1 GB per build, cold load −40 %, 27.7 MB board open −45 %, real redraws +68 % at 4× throttle, boot heap −31 %. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016X9eh1s5sTx1o9Em9KBuwR
This commit is contained in:
Viktor Vaczi
parent
9c475a804e
commit
da299ed6f9
12 changed files with 707 additions and 11 deletions
180
tests/kicad/pcbnew-large-perf.spec.ts
Normal file
180
tests/kicad/pcbnew-large-perf.spec.ts
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
import { test, expect } from './fixtures';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as crypto from 'crypto';
|
||||
import {
|
||||
measureLoad,
|
||||
measureOpenRender,
|
||||
openAndWait,
|
||||
measureFpsDetailed,
|
||||
setThrottle,
|
||||
sampleMemory,
|
||||
startHeapPeakSampler,
|
||||
stopHeapPeakSampler,
|
||||
getWasmResourceTiming,
|
||||
} from './utils/perf-utils';
|
||||
import { fetchIntoMemfs } from './utils/fs-inject';
|
||||
import { waitForBoardLoaded } from './utils/board-ready';
|
||||
|
||||
/**
|
||||
* Large-board perf battery (TRACK-ONLY, bench-driven). Extends pcbnew-perf with
|
||||
* the measurements the JSPI-vs-asyncify comparison needs: repeated cold loads,
|
||||
* open+render of a REAL large board (vme-wren: 1508 footprints / 24 858
|
||||
* segments), rAF + distinct-frame FPS under CPU throttle, and wasm-heap /
|
||||
* JS-heap checkpoints. Results append to
|
||||
* tests/test-results/perf-bench-<arm>.ndjson (gitignored), one JSON row per
|
||||
* measurement, each row carrying the sha256 of the wasm actually measured so
|
||||
* A/B artifact swaps can't get misattributed.
|
||||
*
|
||||
* Gated behind PERF_LARGE=1 so the CI perf project (which matches
|
||||
* *-perf.spec.ts) is unaffected. Fixtures are expected in
|
||||
* tests/apps/kicad/board/ (gitignored) — see docs/features/async/
|
||||
* migration-evidence/jspi-vs-asyncify-bench-2026-08.md for the bench flow.
|
||||
*/
|
||||
|
||||
const ARM = process.env.BENCH_ARM || 'current';
|
||||
const LOAD_RUNS = parseInt(process.env.PERF_LOAD_RUNS || '5', 10);
|
||||
const OPEN_RUNS = parseInt(process.env.PERF_OPEN_RUNS || '3', 10);
|
||||
const THROTTLES = (process.env.PERF_THROTTLES || '1,4,6').split(',').map(Number);
|
||||
const FPS_SECS = parseInt(process.env.PERF_FPS_SECS || '6', 10);
|
||||
const FPS_REPS = parseInt(process.env.PERF_FPS_REPS || '2', 10);
|
||||
|
||||
const APPS_KICAD = path.join(__dirname, '..', 'apps', 'kicad');
|
||||
const DEMO = path.join(__dirname, '..', 'fixtures', 'demo', 'demo.kicad_pcb');
|
||||
const VME_URL = '/kicad/board/vme-wren.kicad_pcb';
|
||||
const JETSON_URL = '/kicad/board/jetson-agx-thor-baseboard.kicad_pcb';
|
||||
// NOT under test-results/ — Playwright clears that whole dir at session start,
|
||||
// so an A/B pair of invocations would each wipe the other arm's rows.
|
||||
const RESULTS = path.join(__dirname, '..', 'bench-results', `perf-bench-${ARM}.ndjson`);
|
||||
|
||||
let wasmSha = '';
|
||||
function wasmSha256(): string {
|
||||
if (!wasmSha) {
|
||||
const bytes = fs.readFileSync(path.join(APPS_KICAD, 'kicad_editor.wasm'));
|
||||
wasmSha = crypto.createHash('sha256').update(bytes).digest('hex').slice(0, 16);
|
||||
}
|
||||
return wasmSha;
|
||||
}
|
||||
|
||||
function record(section: string, data: Record<string, unknown>): void {
|
||||
fs.mkdirSync(path.dirname(RESULTS), { recursive: true });
|
||||
const row = { arm: ARM, sha256: wasmSha256(), when: new Date().toISOString(), section, ...data };
|
||||
fs.appendFileSync(RESULTS, JSON.stringify(row) + '\n');
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`[bench] ${section}: ${JSON.stringify(data)}`);
|
||||
}
|
||||
|
||||
test.describe('pcbnew large-board bench', () => {
|
||||
test.skip(!process.env.PERF_LARGE, 'bench battery — run with PERF_LARGE=1');
|
||||
|
||||
for (let run = 0; run < LOAD_RUNS; run++) {
|
||||
test(`cold load #${run + 1}`, async ({ page }) => {
|
||||
test.setTimeout(300000);
|
||||
const loadMs = await measureLoad(page, '/kicad/pcbnew.html');
|
||||
const wasmFetch = await getWasmResourceTiming(page);
|
||||
const mem = await sampleMemory(page);
|
||||
record('load', { run: run + 1, loadMs, wasmFetch, bootMem: mem });
|
||||
expect(loadMs).toBeGreaterThan(0);
|
||||
});
|
||||
}
|
||||
|
||||
test('cold load with CDP pre-attached (tier-down sanity)', async ({ page }) => {
|
||||
test.setTimeout(300000);
|
||||
const cdp = await page.context().newCDPSession(page);
|
||||
await setThrottle(cdp, 1); // attach + a no-op emulation command, like the FPS path
|
||||
const loadMs = await measureLoad(page, '/kicad/pcbnew.html');
|
||||
record('load-cdp-sanity', { loadMs });
|
||||
expect(loadMs).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
for (let run = 0; run < OPEN_RUNS; run++) {
|
||||
test(`open demo board #${run + 1}`, async ({ page, testLogger }) => {
|
||||
test.setTimeout(300000);
|
||||
await measureLoad(page, '/kicad/pcbnew.html');
|
||||
const openMs = await measureOpenRender(page, DEMO, 'board', testLogger);
|
||||
const mem = await sampleMemory(page);
|
||||
record('open-demo', { run: run + 1, openMs, postOpenMem: mem });
|
||||
expect(openMs).toBeGreaterThan(0);
|
||||
});
|
||||
}
|
||||
|
||||
for (let run = 0; run < OPEN_RUNS; run++) {
|
||||
test(`open vme-wren (27.7 MB) #${run + 1}`, async ({ page, testLogger }) => {
|
||||
test.setTimeout(600000);
|
||||
await measureLoad(page, '/kicad/pcbnew.html');
|
||||
const bytes = await fetchIntoMemfs(page, VME_URL, '/home/kicad/documents/vme-wren.kicad_pcb');
|
||||
await startHeapPeakSampler(page);
|
||||
const openMs = await openAndWait(
|
||||
page,
|
||||
'/home/kicad/documents/vme-wren.kicad_pcb',
|
||||
'board',
|
||||
testLogger,
|
||||
480000,
|
||||
);
|
||||
const peak = await stopHeapPeakSampler(page);
|
||||
const mem = await sampleMemory(page);
|
||||
record('open-vme', { run: run + 1, bytes, openMs, openPeakHeap: peak, postOpenMem: mem });
|
||||
expect(openMs).toBeGreaterThan(0);
|
||||
});
|
||||
}
|
||||
|
||||
test('FPS on vme-wren across throttles', async ({ page, testLogger }) => {
|
||||
test.setTimeout(900000);
|
||||
await measureLoad(page, '/kicad/pcbnew.html');
|
||||
await fetchIntoMemfs(page, VME_URL, '/home/kicad/documents/vme-wren.kicad_pcb');
|
||||
await openAndWait(page, '/home/kicad/documents/vme-wren.kicad_pcb', 'board', testLogger, 480000);
|
||||
await page.keyboard.press('Escape').catch(() => {}); // eslint-disable-line -- best-effort Escape
|
||||
|
||||
const cdp = await page.context().newCDPSession(page);
|
||||
for (const rate of THROTTLES) {
|
||||
await setThrottle(cdp, rate);
|
||||
for (let rep = 0; rep < FPS_REPS; rep++) {
|
||||
const f = await measureFpsDetailed(page, FPS_SECS);
|
||||
record('fps-vme', { throttle: rate, rep: rep + 1, ...f });
|
||||
expect(f.rafFps).toBeGreaterThan(0);
|
||||
}
|
||||
}
|
||||
await setThrottle(cdp, 1);
|
||||
const mem = await sampleMemory(page);
|
||||
record('fps-vme-postmem', { postFpsMem: mem });
|
||||
});
|
||||
|
||||
test('open jetson-agx-thor (80.9 MB) — outcome, OOM allowed', async ({ page, testLogger }) => {
|
||||
test.setTimeout(900000);
|
||||
await measureLoad(page, '/kicad/pcbnew.html');
|
||||
const bytes = await fetchIntoMemfs(
|
||||
page,
|
||||
JETSON_URL,
|
||||
'/home/kicad/documents/jetson-agx-thor-baseboard.kicad_pcb',
|
||||
);
|
||||
await startHeapPeakSampler(page);
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
await page.evaluate(() => {
|
||||
(window as unknown as { Module: { kicadOpenFile(p: string): unknown } }).Module.kicadOpenFile(
|
||||
'/home/kicad/documents/jetson-agx-thor-baseboard.kicad_pcb',
|
||||
);
|
||||
});
|
||||
await waitForBoardLoaded(page, testLogger, 780000);
|
||||
const peak = await stopHeapPeakSampler(page);
|
||||
const mem = await sampleMemory(page);
|
||||
record('open-jetson', {
|
||||
bytes,
|
||||
outcome: 'loaded',
|
||||
openMs: Date.now() - t0,
|
||||
openPeakHeap: peak,
|
||||
postOpenMem: mem,
|
||||
});
|
||||
} catch (e) {
|
||||
// A 4 GB-cap OOM / abort is a RESULT for this stress tier, not a harness error.
|
||||
const peak = await stopHeapPeakSampler(page).catch(() => -1);
|
||||
record('open-jetson', {
|
||||
bytes,
|
||||
outcome: 'failed',
|
||||
afterMs: Date.now() - t0,
|
||||
openPeakHeap: peak,
|
||||
error: String(e).slice(0, 300),
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -42,6 +42,36 @@ export async function injectFileIntoMemfs(
|
|||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a same-origin URL inside the page and write it into MEMFS.
|
||||
*
|
||||
* For large fixtures (tens of MB) the base64 round-trip through Playwright's
|
||||
* JSON channel in injectFileIntoMemfs is the bottleneck — an in-page fetch
|
||||
* from the static server (tests/apps is the serve root) keeps the bytes in
|
||||
* the browser. Returns the byte count written.
|
||||
*/
|
||||
export async function fetchIntoMemfs(
|
||||
page: Page,
|
||||
url: string, // e.g. "/kicad/board/vme-wren.kicad_pcb"
|
||||
memfsPath: string,
|
||||
): Promise<number> {
|
||||
const memfsDir = memfsPath.replace(/\/[^/]+$/, '') || '/';
|
||||
return await page.evaluate(
|
||||
async ({ url, memfsDir, memfsPath }) => {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) throw new Error(`fetchIntoMemfs: ${url} -> HTTP ${res.status}`);
|
||||
const data = new Uint8Array(await res.arrayBuffer());
|
||||
// @ts-expect-error — Emscripten FS lives on window via Module
|
||||
const FS = (window as any).FS;
|
||||
FS.mkdirTree(memfsDir);
|
||||
FS.writeFile(memfsPath, data);
|
||||
console.log(`[KICAD] Fetched ${url} -> ${memfsPath} (${data.length} bytes)`);
|
||||
return data.length;
|
||||
},
|
||||
{ url, memfsDir, memfsPath },
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience: read a file from the kicad/ submodule and inject it.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -64,7 +64,22 @@ export async function measureOpenRender(
|
|||
const ext = kind === 'board' ? 'kicad_pcb' : 'kicad_sch';
|
||||
const memfsPath = `/home/kicad/documents/perf-demo.${ext}`;
|
||||
await injectFileIntoMemfs(page, hostPath, memfsPath);
|
||||
return openAndWait(page, memfsPath, kind, logger, timeout);
|
||||
}
|
||||
|
||||
/**
|
||||
* Open an ALREADY-INJECTED memfs document and wait until loaded+rendered.
|
||||
* Split out of measureOpenRender so large fixtures can arrive via
|
||||
* fetchIntoMemfs (or any other route) and still get the same timed open.
|
||||
*/
|
||||
export async function openAndWait(
|
||||
page: Page,
|
||||
memfsPath: string,
|
||||
kind: 'schematic' | 'board',
|
||||
logger: { consoleLogs: string[]; errors: string[] },
|
||||
timeout = 120000,
|
||||
): Promise<number> {
|
||||
const stem = memfsPath.replace(/^.*\//, '').replace(/\.[^.]+$/, '');
|
||||
const t0 = Date.now();
|
||||
await page.evaluate((p) => {
|
||||
(window as unknown as { Module: KicadModule }).Module.kicadOpenFile(p);
|
||||
|
|
@ -75,13 +90,77 @@ export async function measureOpenRender(
|
|||
} else {
|
||||
const deadline = Date.now() + timeout;
|
||||
while (Date.now() < deadline) {
|
||||
if (/perf-demo/i.test(await page.title())) break;
|
||||
if (new RegExp(stem, 'i').test(await page.title())) break;
|
||||
await page.waitForTimeout(200);
|
||||
}
|
||||
}
|
||||
return Date.now() - t0;
|
||||
}
|
||||
|
||||
export interface MemorySample {
|
||||
/** WebAssembly linear memory size (Module.HEAPU8.byteLength). */
|
||||
wasmHeapBytes: number;
|
||||
/** Chromium-only usedJSHeapSize; 0 elsewhere. */
|
||||
jsHeapBytes: number;
|
||||
}
|
||||
|
||||
/** One-shot memory census — portable across builds (HEAPU8 verified reachable in both). */
|
||||
export async function sampleMemory(page: Page): Promise<MemorySample> {
|
||||
return await page.evaluate(() => {
|
||||
const w = window as unknown as {
|
||||
Module?: { HEAPU8?: { byteLength: number }; wasmMemory?: { buffer: ArrayBuffer } };
|
||||
};
|
||||
const perf = performance as unknown as { memory?: { usedJSHeapSize: number } };
|
||||
return {
|
||||
wasmHeapBytes:
|
||||
w.Module?.wasmMemory?.buffer?.byteLength ?? w.Module?.HEAPU8?.byteLength ?? 0,
|
||||
jsHeapBytes: perf.memory?.usedJSHeapSize ?? 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* In-page wasm-heap peak sampler (250 ms). Start before a heavy operation
|
||||
* (board open), stop after — returns the max linear-memory size observed.
|
||||
*/
|
||||
export async function startHeapPeakSampler(page: Page): Promise<void> {
|
||||
await page.evaluate(() => {
|
||||
const w = window as unknown as {
|
||||
Module?: { HEAPU8?: { byteLength: number }; wasmMemory?: { buffer: ArrayBuffer } };
|
||||
__heapPeak?: number;
|
||||
__heapPeakTimer?: number;
|
||||
};
|
||||
if (w.__heapPeakTimer !== undefined) clearInterval(w.__heapPeakTimer);
|
||||
w.__heapPeak = 0;
|
||||
w.__heapPeakTimer = setInterval(() => {
|
||||
const b =
|
||||
w.Module?.wasmMemory?.buffer?.byteLength ?? w.Module?.HEAPU8?.byteLength ?? 0;
|
||||
if (b > (w.__heapPeak ?? 0)) w.__heapPeak = b;
|
||||
}, 250) as unknown as number;
|
||||
});
|
||||
}
|
||||
|
||||
export async function stopHeapPeakSampler(page: Page): Promise<number> {
|
||||
return await page.evaluate(() => {
|
||||
const w = window as unknown as { __heapPeak?: number; __heapPeakTimer?: number };
|
||||
if (w.__heapPeakTimer !== undefined) clearInterval(w.__heapPeakTimer);
|
||||
w.__heapPeakTimer = undefined;
|
||||
return w.__heapPeak ?? 0;
|
||||
});
|
||||
}
|
||||
|
||||
/** Resource-timing attribution for the main wasm fetch (download share of loadMs). */
|
||||
export async function getWasmResourceTiming(
|
||||
page: Page,
|
||||
): Promise<{ durationMs: number; transferSize: number } | null> {
|
||||
return await page.evaluate(() => {
|
||||
const e = performance
|
||||
.getEntriesByType('resource')
|
||||
.find((r) => r.name.includes('kicad_editor.wasm')) as PerformanceResourceTiming | undefined;
|
||||
return e ? { durationMs: Math.round(e.duration), transferSize: e.transferSize } : null;
|
||||
});
|
||||
}
|
||||
|
||||
/** CDP CPU throttling (Chromium only): 1 = none, N = N× slower. */
|
||||
export async function setThrottle(cdp: CDPSession, rate: number): Promise<void> {
|
||||
await cdp.send('Emulation.setCPUThrottlingRate', { rate });
|
||||
|
|
@ -127,6 +206,79 @@ export async function measureFps(page: Page, seconds: number): Promise<number> {
|
|||
return +(frames / (elapsed / 1000)).toFixed(1);
|
||||
}
|
||||
|
||||
/**
|
||||
* measureFps plus a DISTINCT-frame counter: rAF keeps ticking at vsync even
|
||||
* when the GAL skips redraws (draw_panel_gal enforces a min redraw period and
|
||||
* re-arms a timer when it can't keep up), so under load the rAF number can
|
||||
* decouple from real render throughput. Alongside the rAF loop this samples
|
||||
* the largest visible canvas at ~30 Hz (48×48 downscale hash — the
|
||||
* waitForCanvasStable technique; GAL sets preserveDrawingBuffer) and counts
|
||||
* samples whose content changed. distinctFps is capped by the ~30 Hz sample
|
||||
* rate; read it as "real redraws per second, up to 30".
|
||||
*/
|
||||
export async function measureFpsDetailed(
|
||||
page: Page,
|
||||
seconds: number,
|
||||
): Promise<{ rafFps: number; distinctFps: number }> {
|
||||
await page.evaluate(() => {
|
||||
const w = window as unknown as {
|
||||
__dfPrev?: string;
|
||||
__dfCount?: number;
|
||||
__dfSamples?: number;
|
||||
__dfTimer?: number;
|
||||
};
|
||||
if (w.__dfTimer !== undefined) clearInterval(w.__dfTimer);
|
||||
w.__dfPrev = undefined;
|
||||
w.__dfCount = 0;
|
||||
w.__dfSamples = 0;
|
||||
const scratch = document.createElement('canvas');
|
||||
scratch.width = 48;
|
||||
scratch.height = 48;
|
||||
const ctx = scratch.getContext('2d', { willReadFrequently: true })!;
|
||||
w.__dfTimer = setInterval(() => {
|
||||
const canvases = Array.from(document.querySelectorAll('canvas')).filter((c) => {
|
||||
const r = c.getBoundingClientRect();
|
||||
return r.width > 0 && r.height > 0 && c !== scratch;
|
||||
});
|
||||
if (!canvases.length) return;
|
||||
// The GAL draws into a wxGLCanvas (id glcanvas-*), NOT the
|
||||
// full-window emscripten #canvas — sample the GL surface where the
|
||||
// board pixels actually change, falling back to the largest canvas.
|
||||
const gl = canvases.filter((c) => /^glcanvas/.test(c.id));
|
||||
const pool = gl.length ? gl : canvases;
|
||||
const src = pool.reduce((a, b) => {
|
||||
const ra = a.getBoundingClientRect();
|
||||
const rb = b.getBoundingClientRect();
|
||||
return ra.width * ra.height >= rb.width * rb.height ? a : b;
|
||||
});
|
||||
try {
|
||||
ctx.drawImage(src, 0, 0, 48, 48);
|
||||
const d = ctx.getImageData(0, 0, 48, 48).data;
|
||||
let h = 0;
|
||||
for (let i = 0; i < d.length; i += 16) h = ((h << 5) - h + d[i]) | 0;
|
||||
const hs = String(h);
|
||||
w.__dfSamples = (w.__dfSamples ?? 0) + 1;
|
||||
if (w.__dfPrev !== undefined && hs !== w.__dfPrev) w.__dfCount = (w.__dfCount ?? 0) + 1;
|
||||
w.__dfPrev = hs;
|
||||
} catch {
|
||||
/* tainted/zero-size canvas — skip the sample */
|
||||
}
|
||||
}, 33) as unknown as number;
|
||||
});
|
||||
|
||||
const t0 = Date.now();
|
||||
const rafFps = await measureFps(page, seconds);
|
||||
const elapsed = (Date.now() - t0) / 1000;
|
||||
|
||||
const distinct = await page.evaluate(() => {
|
||||
const w = window as unknown as { __dfCount?: number; __dfTimer?: number };
|
||||
if (w.__dfTimer !== undefined) clearInterval(w.__dfTimer);
|
||||
w.__dfTimer = undefined;
|
||||
return w.__dfCount ?? 0;
|
||||
});
|
||||
return { rafFps, distinctFps: +(distinct / elapsed).toFixed(1) };
|
||||
}
|
||||
|
||||
/** Write per-app results to tests/test-results/perf-<app>.json (gitignored, CI-uploaded). */
|
||||
export function recordPerf(app: string, data: Record<string, unknown>): void {
|
||||
fs.mkdirSync(RESULTS_DIR, { recursive: true });
|
||||
|
|
|
|||
Loading…
Reference in a new issue