perf(editor): stage project files into MEMFS concurrently

The boot-time MEMFS staging fetched one file per round-trip, serially, so a
many-file project (an uploaded repo) paid full request latency per file before
the editor could open anything — the dominant cost of opening such a project.

Fetch with a bounded pool (8, same as the lib presync) and write as each
lands; the writes are synchronous FS calls on distinct paths, so completion
order does not matter. A failed fetch still rejects the stage, after the
in-flight siblings settle so none can write into MEMFS behind the caller.

Tests cover all three properties: every file lands under reverse-staggered
fetch delays, the overlap is >1 and <=8, and a failing fetch rejects.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01H8jo7zz1ZwzYpjJ64UZKN4

web/standalone/src/wasm/kicad-runner.test.ts

@@ -0,0 +1,111 @@
+import { describe, expect, it } from "vitest";
+import { driveProjectIntoTool, type DriveOptions } from "./kicad-runner";
+import { memfsFilePath } from "./constants";
+
+/**
+ * MEMFS project staging (driveProjectIntoTool → syncProjectToMemfs).
+ *
+ * The staging fetch used to be serial — one full request round-trip per file,
+ * which dominated the open of a many-file project (an uploaded repo). These
+ * cover the concurrency contract: every file still lands, fetches overlap, the
+ * overlap stays bounded, and a failed fetch still rejects.
+ */
+
+interface FakeFS {
+  files: Map<string, Uint8Array>;
+  dirs: Set<string>;
+}
+
+function fakeWin(): { win: Parameters<typeof driveProjectIntoTool>[0]; fs: FakeFS } {
+  const fs: FakeFS = { files: new Map(), dirs: new Set() };
+  const win = {
+    FS: {
+      writeFile: (path: string, data: Uint8Array) => {
+        fs.files.set(path, data);
+      },
+      mkdirTree: (path: string) => {
+        fs.dirs.add(path);
+      },
+      readFile: () => new Uint8Array(),
+      analyzePath: () => ({ exists: false }),
+    },
+  } as unknown as Parameters<typeof driveProjectIntoTool>[0];
+  return { win, fs };
+}
+
+/** Options with no targetPath, so the run stops after staging (no open flow). */
+function opts(
+  files: string[],
+  fetchBytes: DriveOptions["fetchBytes"],
+): DriveOptions {
+  return {
+    tool: "pcbnew",
+    slug: "proj",
+    files: files.map((path) => ({ path })),
+    fetchBytes,
+    log: () => {},
+    onStatus: () => {},
+  };
+}
+
+describe("MEMFS project staging", () => {
+  it("stages every file, whatever order the fetches settle in", async () => {
+    const { win, fs } = fakeWin();
+    const files = Array.from({ length: 25 }, (_, i) => `dir${i % 4}/file${i}.kicad_sym`);
+    // Reverse-staggered delays: later files resolve FIRST, so a serial
+    // implementation and a parallel one produce different completion orders
+    // and any order-dependent bug shows up here.
+    await driveProjectIntoTool(
+      win,
+      opts(files, async (p) => {
+        const idx = files.indexOf(p);
+        await new Promise((r) => setTimeout(r, (files.length - idx) % 7));
+        return new TextEncoder().encode(`body:${p}`);
+      }),
+    );
+
+    expect(fs.files.size).toBe(files.length);
+    for (const p of files) {
+      const written = fs.files.get(memfsFilePath("proj", p));
+      expect(new TextDecoder().decode(written!)).toBe(`body:${p}`);
+    }
+  });
+
+  it("overlaps fetches without exceeding the concurrency cap", async () => {
+    const { win } = fakeWin();
+    const files = Array.from({ length: 30 }, (_, i) => `f${i}.kicad_mod`);
+    let inFlight = 0;
+    let peak = 0;
+
+    await driveProjectIntoTool(
+      win,
+      opts(files, async () => {
+        inFlight++;
+        peak = Math.max(peak, inFlight);
+        await new Promise((r) => setTimeout(r, 1));
+        inFlight--;
+        return new Uint8Array([1]);
+      }),
+    );
+
+    // Parallel (the point of the change) but bounded — a serial staging peaks
+    // at 1, an unbounded one at files.length.
+    expect(peak).toBeGreaterThan(1);
+    expect(peak).toBeLessThanOrEqual(8);
+  });
+
+  it("rejects when a fetch fails", async () => {
+    const { win } = fakeWin();
+    const files = ["ok1.kicad_sym", "bad.kicad_sym", "ok2.kicad_sym"];
+
+    await expect(
+      driveProjectIntoTool(
+        win,
+        opts(files, async (p) => {
+          if (p === "bad.kicad_sym") throw new Error("fetch exploded");
+          return new Uint8Array([1]);
+        }),
+      ),
+    ).rejects.toThrow("fetch exploded");
+  });
+});

web/standalone/src/wasm/kicad-runner.ts

@@ -66,23 +66,48 @@ export function restageFile(
   log(`[memfs] wrote ${dest} (${bytes.length} bytes)`);
 }
 
-/** Mirror the whole project tree into the tool's MEMFS (sync-whole-tree). */
+/** How many project files are fetched at once by the MEMFS staging below.
+ *  Matches the lib presync's default: enough to hide per-request latency on a
+ *  many-file project, low enough not to starve the parallel wasm download. */
+const STAGE_CONCURRENCY = 8;
+
+/**
+ * Mirror the whole project tree into the tool's MEMFS (sync-whole-tree).
+ *
+ * Fetches run CONCURRENTLY (bounded by STAGE_CONCURRENCY): a project with a
+ * few hundred files — an uploaded repo, say — spent one full request
+ * round-trip per file when this was serial, which dominated the open on any
+ * real-latency connection. The writes themselves are synchronous FS calls on
+ * distinct paths, so they can land in whatever order the fetches complete.
+ */
 async function syncProjectToMemfs(win: ToolWindow, opts: DriveOptions): Promise<void> {
   getFS(win).mkdirTree(memfsProjectDir(opts.slug));
-  for (const file of opts.files) {
-    const bytes = await opts.fetchBytes(file.path);
-    restageFile(win, opts.slug, file.path, bytes, opts.log);
-    // 3D models: prefetch every model this board references (R2 → IDB → MEMFS)
-    // so the 3D viewer's first open resolves locally. Fire-and-forget — project
-    // open never waits on it; a ref that misses falls back to the C++ per-model
-    // ensure. No-op unless a model source is installed (bootKicadTool).
-    if (file.path.endsWith(".kicad_pcb")) {
-      const text = new TextDecoder().decode(bytes);
-      void prescanBoardModels(text).catch((e) =>
-        opts.log(`[3d] prescan failed: ${String(e)}`),
-      );
+
+  const queue = [...opts.files];
+  const worker = async (): Promise<void> => {
+    for (let file = queue.shift(); file; file = queue.shift()) {
+      const bytes = await opts.fetchBytes(file.path);
+      restageFile(win, opts.slug, file.path, bytes, opts.log);
+      // 3D models: prefetch every model this board references (R2 → IDB → MEMFS)
+      // so the 3D viewer's first open resolves locally. Fire-and-forget — project
+      // open never waits on it; a ref that misses falls back to the C++ per-model
+      // ensure. No-op unless a model source is installed (bootKicadTool).
+      if (file.path.endsWith(".kicad_pcb")) {
+        const text = new TextDecoder().decode(bytes);
+        void prescanBoardModels(text).catch((e) =>
+          opts.log(`[3d] prescan failed: ${String(e)}`),
+        );
+      }
     }
-  }
+  };
+  // One rejection fails the stage (same as the serial loop did), but let the
+  // in-flight siblings settle first so a failure can't leave a fetch writing
+  // into MEMFS after the caller has moved on.
+  const results = await Promise.allSettled(
+    Array.from({ length: Math.min(STAGE_CONCURRENCY, opts.files.length) }, worker),
+  );
+  const failed = results.find((r) => r.status === "rejected");
+  if (failed) throw (failed as PromiseRejectedResult).reason;
 }
 
 /**