From 19a713f4543fc748ac79d5c4df23322140d113d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20T=C3=B6rcsv=C3=A1ri?= Date: Wed, 29 Jul 2026 15:20:06 +0200 Subject: [PATCH] perf(editor): stage project files into MEMFS concurrently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The boot-time MEMFS staging fetched one file per round-trip, serially, so a many-file project (an uploaded repo) paid full request latency per file before the editor could open anything — the dominant cost of opening such a project. Fetch with a bounded pool (8, same as the lib presync) and write as each lands; the writes are synchronous FS calls on distinct paths, so completion order does not matter. A failed fetch still rejects the stage, after the in-flight siblings settle so none can write into MEMFS behind the caller. Tests cover all three properties: every file lands under reverse-staggered fetch delays, the overlap is >1 and <=8, and a failing fetch rejects. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01H8jo7zz1ZwzYpjJ64UZKN4 --- web/standalone/src/wasm/kicad-runner.test.ts | 111 +++++++++++++++++++ web/standalone/src/wasm/kicad-runner.ts | 53 ++++++--- 2 files changed, 150 insertions(+), 14 deletions(-) create mode 100644 web/standalone/src/wasm/kicad-runner.test.ts diff --git a/web/standalone/src/wasm/kicad-runner.test.ts b/web/standalone/src/wasm/kicad-runner.test.ts new file mode 100644 index 0000000..f6df7b5 --- /dev/null +++ b/web/standalone/src/wasm/kicad-runner.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from "vitest"; +import { driveProjectIntoTool, type DriveOptions } from "./kicad-runner"; +import { memfsFilePath } from "./constants"; + +/** + * MEMFS project staging (driveProjectIntoTool → syncProjectToMemfs). + * + * The staging fetch used to be serial — one full request round-trip per file, + * which dominated the open of a many-file project (an uploaded repo). These + * cover the concurrency contract: every file still lands, fetches overlap, the + * overlap stays bounded, and a failed fetch still rejects. + */ + +interface FakeFS { + files: Map; + dirs: Set; +} + +function fakeWin(): { win: Parameters[0]; fs: FakeFS } { + const fs: FakeFS = { files: new Map(), dirs: new Set() }; + const win = { + FS: { + writeFile: (path: string, data: Uint8Array) => { + fs.files.set(path, data); + }, + mkdirTree: (path: string) => { + fs.dirs.add(path); + }, + readFile: () => new Uint8Array(), + analyzePath: () => ({ exists: false }), + }, + } as unknown as Parameters[0]; + return { win, fs }; +} + +/** Options with no targetPath, so the run stops after staging (no open flow). */ +function opts( + files: string[], + fetchBytes: DriveOptions["fetchBytes"], +): DriveOptions { + return { + tool: "pcbnew", + slug: "proj", + files: files.map((path) => ({ path })), + fetchBytes, + log: () => {}, + onStatus: () => {}, + }; +} + +describe("MEMFS project staging", () => { + it("stages every file, whatever order the fetches settle in", async () => { + const { win, fs } = fakeWin(); + const files = Array.from({ length: 25 }, (_, i) => `dir${i % 4}/file${i}.kicad_sym`); + // Reverse-staggered delays: later files resolve FIRST, so a serial + // implementation and a parallel one produce different completion orders + // and any order-dependent bug shows up here. + await driveProjectIntoTool( + win, + opts(files, async (p) => { + const idx = files.indexOf(p); + await new Promise((r) => setTimeout(r, (files.length - idx) % 7)); + return new TextEncoder().encode(`body:${p}`); + }), + ); + + expect(fs.files.size).toBe(files.length); + for (const p of files) { + const written = fs.files.get(memfsFilePath("proj", p)); + expect(new TextDecoder().decode(written!)).toBe(`body:${p}`); + } + }); + + it("overlaps fetches without exceeding the concurrency cap", async () => { + const { win } = fakeWin(); + const files = Array.from({ length: 30 }, (_, i) => `f${i}.kicad_mod`); + let inFlight = 0; + let peak = 0; + + await driveProjectIntoTool( + win, + opts(files, async () => { + inFlight++; + peak = Math.max(peak, inFlight); + await new Promise((r) => setTimeout(r, 1)); + inFlight--; + return new Uint8Array([1]); + }), + ); + + // Parallel (the point of the change) but bounded — a serial staging peaks + // at 1, an unbounded one at files.length. + expect(peak).toBeGreaterThan(1); + expect(peak).toBeLessThanOrEqual(8); + }); + + it("rejects when a fetch fails", async () => { + const { win } = fakeWin(); + const files = ["ok1.kicad_sym", "bad.kicad_sym", "ok2.kicad_sym"]; + + await expect( + driveProjectIntoTool( + win, + opts(files, async (p) => { + if (p === "bad.kicad_sym") throw new Error("fetch exploded"); + return new Uint8Array([1]); + }), + ), + ).rejects.toThrow("fetch exploded"); + }); +}); diff --git a/web/standalone/src/wasm/kicad-runner.ts b/web/standalone/src/wasm/kicad-runner.ts index b389771..ba811d3 100644 --- a/web/standalone/src/wasm/kicad-runner.ts +++ b/web/standalone/src/wasm/kicad-runner.ts @@ -66,23 +66,48 @@ export function restageFile( log(`[memfs] wrote ${dest} (${bytes.length} bytes)`); } -/** Mirror the whole project tree into the tool's MEMFS (sync-whole-tree). */ +/** How many project files are fetched at once by the MEMFS staging below. + * Matches the lib presync's default: enough to hide per-request latency on a + * many-file project, low enough not to starve the parallel wasm download. */ +const STAGE_CONCURRENCY = 8; + +/** + * Mirror the whole project tree into the tool's MEMFS (sync-whole-tree). + * + * Fetches run CONCURRENTLY (bounded by STAGE_CONCURRENCY): a project with a + * few hundred files — an uploaded repo, say — spent one full request + * round-trip per file when this was serial, which dominated the open on any + * real-latency connection. The writes themselves are synchronous FS calls on + * distinct paths, so they can land in whatever order the fetches complete. + */ async function syncProjectToMemfs(win: ToolWindow, opts: DriveOptions): Promise { getFS(win).mkdirTree(memfsProjectDir(opts.slug)); - for (const file of opts.files) { - const bytes = await opts.fetchBytes(file.path); - restageFile(win, opts.slug, file.path, bytes, opts.log); - // 3D models: prefetch every model this board references (R2 → IDB → MEMFS) - // so the 3D viewer's first open resolves locally. Fire-and-forget — project - // open never waits on it; a ref that misses falls back to the C++ per-model - // ensure. No-op unless a model source is installed (bootKicadTool). - if (file.path.endsWith(".kicad_pcb")) { - const text = new TextDecoder().decode(bytes); - void prescanBoardModels(text).catch((e) => - opts.log(`[3d] prescan failed: ${String(e)}`), - ); + + const queue = [...opts.files]; + const worker = async (): Promise => { + for (let file = queue.shift(); file; file = queue.shift()) { + const bytes = await opts.fetchBytes(file.path); + restageFile(win, opts.slug, file.path, bytes, opts.log); + // 3D models: prefetch every model this board references (R2 → IDB → MEMFS) + // so the 3D viewer's first open resolves locally. Fire-and-forget — project + // open never waits on it; a ref that misses falls back to the C++ per-model + // ensure. No-op unless a model source is installed (bootKicadTool). + if (file.path.endsWith(".kicad_pcb")) { + const text = new TextDecoder().decode(bytes); + void prescanBoardModels(text).catch((e) => + opts.log(`[3d] prescan failed: ${String(e)}`), + ); + } } - } + }; + // One rejection fails the stage (same as the serial loop did), but let the + // in-flight siblings settle first so a failure can't leave a fetch writing + // into MEMFS after the caller has moved on. + const results = await Promise.allSettled( + Array.from({ length: Math.min(STAGE_CONCURRENCY, opts.files.length) }, worker), + ); + const failed = results.find((r) => r.status === "rejected"); + if (failed) throw (failed as PromiseRejectedResult).reason; } /**