feat(libs): fat-list provider + getAllItems + parallel lib pre-sync; re-land onto KiCad 10 wasm-port

Replays the symbol-editor fast-lib-load work after the kicad fork was rebased
onto KiCad 10.0.4 and wasm-port was force-pushed. Carries the standalone libs
TS code (fat-list provider, getAllItems, parallel lib pre-sync) and bumps the
submodule pointers to the re-applied tips:

- kicad      -> c63009046 (lazy load on expand + chooser preview + fat-load enumerate)
- wxwidgets  -> 5d40228   (flush events + repaint after modal button click)
- pcbjam-shared -> c6a7e00 (bulk readAll for fat-load lib warm-up)

The 824e8cc build-kicad-target.sh sym-converter-contamination fix is omitted:
the KiCad 10 integration (ce344da) already pins KICAD_SYM_CONVERTER_WASM=OFF for
non-sym_convert targets. Verified: eeschema WASM kiface compiles + links; libs
unit tests 10/10.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

kicad

@@ -1 +1 @@
-Subproject commit 0a7d1baf0bc69b19b489bf52965f306a4d1624c3
+Subproject commit c63009046c06011a0220b0e3ada28e084ffe4e54

web/pcbjam-shared

@@ -1 +1 @@
-Subproject commit 3367a91b39060d6be51b5d30c5503a00933314f9
+Subproject commit c6a7e00bcaa39023be0be126a28f8f1acb9a2d67

web/standalone/src/components/WasmTool.tsx

@@ -51,6 +51,16 @@ import { SourceChip } from "@/components/SourceChip";
 
 // Tools with the v2 items bridge (kicadCollabSnapshotItems/ApplyItems embind exports).
 const COLLAB_TOOLS = new Set<Tool>(["pl_editor", "eeschema", "pcbnew"]);
+
+// Which library item kind each tool browses — drives the load-screen pre-sync
+// (warm the right bundles into IDB while the wasm downloads). Tools that don't
+// browse a library are omitted (no pre-sync).
+const LIB_KIND_FOR_TOOL: Partial<Record<Tool, "symbol" | "footprint">> = {
+  symbol_editor: "symbol",
+  eeschema: "symbol",
+  footprint_editor: "footprint",
+  pcbnew: "footprint",
+};
 const LEGACY_EXTENSION_TOOL: Record<string, Tool> = {
   ".sch": "eeschema",
   ".brd": "pcbnew",
@@ -560,6 +570,9 @@ export function WasmTool({
   const [ready, setReady] = React.useState(false);
   // A library item currently being fetched (open/save), for a transient spinner.
   const [libBusy, setLibBusy] = React.useState<string | null>(null);
+  // Load-screen pre-sync progress: warming the project's lib bundles into IDB in
+  // parallel with the wasm download. Null when idle/done.
+  const [libSync, setLibSync] = React.useState<string | null>(null);
   // Last lib error (e.g. a backend 404 on open), shown as a dismissible toast.
   const [libError, setLibError] = React.useState<string | null>(null);
 
@@ -653,6 +666,33 @@ export function WasmTool({
         // Resolve the per-tool asset base at runtime (CDN manifest → versioned
         // folder, or the flat local /wasm in dev). See wasm/wasm-assets.ts.
         const base = await resolveWasmBase(tool, assetBaseUrl);
+        // One source instance, shared by the wasm provider AND the pre-sync below
+        // (libsSourceConfig builds a fresh one each call — their SyncStack caches
+        // must be the same object for the warm-up to benefit the editor).
+        const source =
+          libsSource !== undefined ? libsSource : libsSourceConfig(projectId);
+        // Pre-warm the lib bundles into IDB in PARALLEL with the wasm download, so
+        // the editor's first enumerate reads a warm cache instead of freezing on N
+        // cold bundle fetches. Non-blocking + best-effort; the SyncStack dedups, so
+        // a lib the wasm reaches mid-presync just awaits the same in-flight fetch.
+        const libKind = LIB_KIND_FOR_TOOL[tool];
+        if (source?.presync && libKind) {
+          void source
+            .presync({
+              kind: libKind,
+              onProgress: ({ done, total, current }) =>
+                setLibSync(
+                  done >= total
+                    ? null
+                    : `Syncing ${libKind}s — ${current} (${done}/${total})`,
+                ),
+            })
+            .then(() => setLibSync(null))
+            .catch((e) => {
+              append(`[presync] ${String(e)}`);
+              setLibSync(null);
+            });
+        }
         await bootKicadTool({
           tool,
           base,
@@ -660,8 +700,7 @@ export function WasmTool({
           log: append,
           onStatus: setStatus,
           onAbort: oom.onAbort,
-          libsSource:
+          libsSource: source,
-            libsSource !== undefined ? libsSource : libsSourceConfig(projectId),
         });
         // Register the save sink before the file opens: from here on, every
         // editor File→Save (MEMFS write) is routed onward through saveBytes.
@@ -824,6 +863,9 @@ export function WasmTool({
               <p className="font-mono text-sm text-white/80">
                 {status || "Loading…"}
               </p>
+              {libSync && (
+                <p className="font-mono text-xs text-emerald-300/90">{libSync}</p>
+              )}
               <p className="font-mono text-xs text-white/40">
                 First load downloads the tool (large) — this can take a moment.
               </p>
@@ -846,6 +888,14 @@ export function WasmTool({
         </div>
       )}
 
+      {/* Lib pre-sync still warming IDB after the editor opened (big set) — small
+          unobtrusive indicator so the user knows browsing is still filling in. */}
+      {ready && libSync && (
+        <div className="pointer-events-none absolute bottom-9 left-3 z-20 flex items-center gap-2 rounded bg-black/80 px-3 py-1.5 text-xs text-emerald-200">
+          <Loader2 className="animate-spin" size={14} /> {libSync}
+        </div>
+      )}
+
       {/* A library item is being fetched (open/save). */}
       {ready && libBusy && (
         <div className="pointer-events-none absolute left-1/2 top-3 z-20 flex -translate-x-1/2 items-center gap-2 rounded bg-black/80 px-3 py-1.5 text-xs text-white">

web/standalone/src/wasm/libs/cdn-source.test.ts

@@ -90,6 +90,41 @@ describe("cdn libs source", () => {
     expect(await src.getItemBody("Device", "symbol", "Nope")).toBeNull();
   });
 
+  it("getAllItems returns every item with its body in one shot (fat list)", async () => {
+    const src = await fakeCdn();
+    const all = (await src.getAllItems!("Device")).sort((a, b) =>
+      a.name.localeCompare(b.name),
+    );
+    expect(all).toEqual([
+      { kind: "symbol", name: "C", body: "(kicad_symbol_lib (symbol C))" },
+      { kind: "symbol", name: "R", body: "(kicad_symbol_lib (symbol R))" },
+    ]);
+    // Bodies match the per-item getItemBody for the same kind/name.
+    for (const it of all) {
+      expect(it.body).toBe(await src.getItemBody("Device", it.kind, it.name));
+    }
+  });
+
+  it("presync warms a kind's lib bundles and reports per-lib progress", async () => {
+    const src = await fakeCdn();
+    const events: Array<{ done: number; total: number; current: string }> = [];
+    await src.presync!({ kind: "symbol", onProgress: (p) => events.push({ ...p }) });
+
+    const last = events[events.length - 1]!;
+    expect(last.total).toBe(1); // one symbol lib (Device)
+    expect(last.done).toBe(1); // completed
+    expect(events.some((e) => e.current === "Device")).toBe(true);
+    // Warmed: items are served from the opened stack.
+    expect((await src.listItems("Device")).length).toBe(2);
+  });
+
+  it("presync without a kind warms every lib", async () => {
+    const src = await fakeCdn();
+    let total = 0;
+    await src.presync!({ onProgress: (p) => (total = p.total) });
+    expect(total).toBe(2); // Device (symbol) + Resistor_SMD (footprint)
+  });
+
   it("is read-only (no save path)", async () => {
     const src = await fakeCdn();
     expect(src.saveItemBody).toBeUndefined();

web/standalone/src/wasm/libs/cdn-source.ts

@@ -40,12 +40,36 @@ export function cdnLibsSource(
   const fetchImpl = opts?.fetchImpl ?? fetch;
 
   let manifestP: Promise<CdnLibsManifest> | null = null;
-  const loadManifest = () =>
+  const fetchManifest = async (): Promise<CdnLibsManifest> => {
-    (manifestP ??= (async () => {
+    // Retry with backoff. Firefox can fail a cross-origin fetch issued in the
-      const r = await fetchImpl(manifestUrl, { cache: "no-store" });
+    // first moments after navigation under COEP (the lazy path runs seconds
-      if (!r.ok) throw new Error(`cdn libs manifest ${r.status}: ${manifestUrl}`);
+    // later and succeeds); a short retry rides past that window. The pre-sync
-      return (await r.json()) as CdnLibsManifest;
+    // warm-up, which fires this earliest, is what surfaced it.
-    })());
+    let lastErr: unknown;
+    for (let attempt = 0; attempt < 4; attempt++) {
+      if (attempt > 0) await new Promise((r) => setTimeout(r, 200 * attempt));
+      try {
+        const r = await fetchImpl(manifestUrl, { cache: "no-store" });
+        if (!r.ok)
+          throw new Error(`cdn libs manifest ${r.status}: ${manifestUrl}`);
+        return (await r.json()) as CdnLibsManifest;
+      } catch (e) {
+        lastErr = e;
+      }
+    }
+    throw lastErr;
+  };
+  const loadManifest = (): Promise<CdnLibsManifest> => {
+    if (!manifestP) {
+      // NEVER cache a rejection: a transient failure must not poison every later
+      // listLibs/getItemBody — the next call retries from scratch.
+      manifestP = fetchManifest().catch((e) => {
+        manifestP = null;
+        throw e;
+      });
+    }
+    return manifestP;
+  };
 
   // One lazily-opened SyncStack per lib (its IDB store is keyed by namespace, so
   // a lib is cached once and reused across opens).
@@ -67,7 +91,12 @@ export function cdnLibsSource(
         });
         await stack.open();
         return stack;
-      })();
+      })().catch((e) => {
+        // Don't cache a failed open (e.g. an early-boot fetch blip) — the lazy
+        // path must be able to retry this lib instead of inheriting the failure.
+        stacks.delete(libId);
+        throw e;
+      });
       stacks.set(libId, p);
     }
     return p;
@@ -89,6 +118,54 @@ export function cdnLibsSource(
       const stack = await openStack(libId);
       return (await stack.list()).map((e) => splitPath(e.path));
     },
+    async presync(opts): Promise<void> {
+      const { kind, concurrency = 6, onProgress, signal } = opts ?? {};
+      let m: CdnLibsManifest;
+      try {
+        m = await loadManifest();
+      } catch {
+        // Best-effort warm-up: a manifest hiccup here (e.g. the early-boot
+        // Firefox/COEP fetch blip) is non-fatal — skip quietly and let the lazy
+        // path fetch on demand (loadManifest no longer caches the failure).
+        return;
+      }
+      const libs = m.libs.filter((l) => !kind || l.kind === kind);
+      const total = libs.length;
+      let done = 0;
+      onProgress?.({ done, total, current: "" });
+      // Concurrency-limited pool: each openStack cold-fetches one bundle into IDB
+      // (warm ⇒ a cheap manifest diff). Tolerate per-lib failures so one bad lib
+      // doesn't abort the warm-up.
+      let idx = 0;
+      const worker = async (): Promise<void> => {
+        while (idx < libs.length) {
+          if (signal?.aborted) return;
+          const lib = libs[idx++]!;
+          onProgress?.({ done, total, current: lib.name });
+          try {
+            await openStack(lib.id);
+          } catch {
+            // best-effort: the lib still loads lazily on demand
+          }
+          onProgress?.({ done: ++done, total, current: lib.name });
+        }
+      };
+      await Promise.all(
+        Array.from({ length: Math.min(concurrency, total) }, () => worker()),
+      );
+    },
+    async getAllItems(
+      libId: string,
+    ): Promise<Array<{ kind: string; name: string; body: string }>> {
+      // One bulk merged read of the whole lib (the IDB cache after cold bundle),
+      // so the WASM plugin hydrates in a single crossing instead of N gets.
+      const stack = await openStack(libId);
+      const dec = new TextDecoder();
+      return [...(await stack.readAll())].map(([path, bytes]) => {
+        const { kind, name } = splitPath(path);
+        return { kind, name, body: dec.decode(bytes) };
+      });
+    },
     async getItemBody(
       libId: string,
       kind: string,

web/standalone/src/wasm/libs/scoped-source.ts

@@ -21,5 +21,30 @@ export function scopedLibsSource(base: LibsSource, libId: string): LibsSource {
     getItemBody(id: string, kind: string, name: string): Promise<string | null> {
       return base.getItemBody(id, kind, name);
     },
+    // Forward the bulk "fat list" only when the base supports it, so a scoped
+    // single-lib view keeps the one-crossing hydrate (else the provider's slow
+    // fallback applies).
+    ...(base.getAllItems
+      ? { getAllItems: (id: string) => base.getAllItems!(id) }
+      : {}),
+    // Pre-sync ONLY the scoped lib — NOT the base's whole catalog (that's what a
+    // bare `base.presync()` would do). Opening the one lib via listItems warms its
+    // bundle into IDB. Gated on the base being cache-capable (it exposes presync);
+    // per-item remote has no client cache to warm.
+    ...(base.presync
+      ? {
+          presync: async (
+            o?: Parameters<NonNullable<LibsSource["presync"]>>[0],
+          ): Promise<void> => {
+            o?.onProgress?.({ done: 0, total: 1, current: libId });
+            try {
+              await base.listItems(libId);
+            } catch {
+              // best-effort: the lib still loads lazily on demand
+            }
+            o?.onProgress?.({ done: 1, total: 1, current: libId });
+          },
+        }
+      : {}),
   };
 }

web/standalone/src/wasm/libs/source.test.ts

@@ -0,0 +1,98 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { installLibsProvider, type LibsSource } from "./source";
+import { libUri } from "./uri";
+
+/**
+ * The WASM lib plugins call `window.kicadLibs.request(op, lib, arg, kind)`. These
+ * tests pin the wire contract the C++ side parses — in particular the "fat list"
+ * (`arg === "bodies"`) shape `{symbols|footprints: [{name, body}]}` and its
+ * fallback for sources without a bulk `getAllItems`. Node env has no `window`, so
+ * we stub the minimal surface the provider touches.
+ */
+
+type Req = (
+  op: string,
+  lib: string,
+  arg: string,
+  kind?: string,
+) => Promise<string | null>;
+
+function installAndGetRequest(source: LibsSource): Req {
+  installLibsProvider(source, () => {});
+  return (globalThis as unknown as { window: { kicadLibs: { request: Req } } })
+    .window.kicadLibs.request;
+}
+
+const ITEMS = [
+  { kind: "symbol", name: "R", body: "(kicad_symbol_lib (symbol R))" },
+  { kind: "symbol", name: "C", body: "(kicad_symbol_lib (symbol C))" },
+  { kind: "footprint", name: "R_0402", body: "(footprint R_0402)" },
+];
+
+function baseSource(over: Partial<LibsSource> = {}): LibsSource {
+  return {
+    listLibs: async () => [],
+    listItems: async () => ITEMS.map((i) => ({ kind: i.kind, name: i.name })),
+    getItemBody: async (_id, kind, name) =>
+      ITEMS.find((i) => i.kind === kind && i.name === name)?.body ?? null,
+    ...over,
+  };
+}
+
+describe("installLibsProvider — fat list (arg=bodies)", () => {
+  beforeEach(() => {
+    (globalThis as unknown as { window: unknown }).window = {
+      location: { search: "" },
+      dispatchEvent: () => true,
+    };
+  });
+  afterEach(() => {
+    delete (globalThis as unknown as { window?: unknown }).window;
+  });
+
+  it("returns names+bodies for the requested kind via getAllItems", async () => {
+    const calls: string[] = [];
+    const request = installAndGetRequest(
+      baseSource({
+        getAllItems: async (id) => {
+          calls.push(id);
+          return ITEMS;
+        },
+      }),
+    );
+
+    const res = await request("list", libUri("Device"), "bodies", "symbol");
+    expect(JSON.parse(res!)).toEqual({
+      symbols: [
+        { name: "R", body: "(kicad_symbol_lib (symbol R))" },
+        { name: "C", body: "(kicad_symbol_lib (symbol C))" },
+      ],
+    });
+    // One bulk call, not per-item.
+    expect(calls).toEqual(["Device"]);
+
+    const fps = await request("list", libUri("Device"), "bodies", "footprint");
+    expect(JSON.parse(fps!)).toEqual({
+      footprints: [{ name: "R_0402", body: "(footprint R_0402)" }],
+    });
+  });
+
+  it("falls back to listItems + getItemBody when getAllItems is absent", async () => {
+    const request = installAndGetRequest(baseSource()); // no getAllItems
+    const res = await request("list", libUri("Device"), "bodies", "symbol");
+    expect(JSON.parse(res!)).toEqual({
+      symbols: [
+        { name: "R", body: "(kicad_symbol_lib (symbol R))" },
+        { name: "C", body: "(kicad_symbol_lib (symbol C))" },
+      ],
+    });
+  });
+
+  it("plain list (empty arg) still returns names only", async () => {
+    const request = installAndGetRequest(
+      baseSource({ getAllItems: async () => ITEMS }),
+    );
+    const res = await request("list", libUri("Device"), "", "symbol");
+    expect(JSON.parse(res!)).toEqual({ symbols: ["R", "C"] });
+  });
+});

web/standalone/src/wasm/libs/source.ts

@@ -20,6 +20,14 @@ export interface LibItemInfo {
   name: string;
 }
 
+/** Per-lib progress for {@link LibsSource.presync} (drives the load screen). */
+export interface LibPresyncProgress {
+  done: number;
+  total: number;
+  /** Display name of the lib currently being synced (one of the in-flight set). */
+  current: string;
+}
+
 export interface LibsSource {
   /**
    * Libraries to expose to the editor (one lib-table row each). `kind` (the
@@ -35,6 +43,34 @@ export interface LibsSource {
    * null if absent. `kind` is 'symbol' for now.
    */
   getItemBody(libId: string, kind: string, name: string): Promise<string | null>;
+  /**
+   * ALL items in a library with their bodies, in one shot — the "fat list" that
+   * lets the WASM plugin hydrate a whole library in a single bridge crossing
+   * instead of N per-item `get`s (see docs/features/libs/0011). Optional: sources
+   * that can't bulk-read omit it and the provider falls back to listItems + N
+   * getItemBody (the old slow path, kept working for the example backend).
+   */
+  getAllItems?(
+    libId: string,
+  ): Promise<Array<{ kind: string; name: string; body: string }>>;
+  /**
+   * Pre-warm this source's per-lib caches (IndexedDB bundles) WITHOUT touching the
+   * WASM runtime — call it in parallel with the wasm download so the editor's
+   * first enumerate reads a warm cache instead of freezing on N cold bundle
+   * fetches. Best-effort: a lib that fails to presync is skipped (it still loads
+   * lazily later); the SyncStack dedups, so a lib the wasm reaches mid-presync
+   * just awaits the same in-flight fetch. `onProgress` reports per-lib so the load
+   * screen can name what's syncing. Optional: sources without a client-side cache
+   * (per-item remote) omit it.
+   */
+  presync?(opts?: {
+    /** Limit to libs holding this item kind ("symbol" | "footprint"). */
+    kind?: string;
+    /** Max concurrent bundle fetches (default 6). */
+    concurrency?: number;
+    onProgress?: (p: LibPresyncProgress) => void;
+    signal?: AbortSignal;
+  }): Promise<void>;
   /**
    * Persist one item body into a writable (user) lib. Optional: read-only
    * sources omit it (a save into a non-writable source resolves false).
@@ -112,6 +148,25 @@ function artificialDelayMs(): number {
 
 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
 
+/**
+ * Slow-path "fat list" for sources without a bulk `getAllItems` (the example
+ * backend / per-item remote): just listItems + one getItemBody each. Same N
+ * round-trips as before — but it keeps the single WASM-side code path (the plugin
+ * always asks for bodies once) working against every source.
+ */
+async function fallbackGetAllItems(
+  source: LibsSource,
+  libId: string,
+): Promise<Array<{ kind: string; name: string; body: string }>> {
+  const items = await source.listItems(libId);
+  const out: Array<{ kind: string; name: string; body: string }> = [];
+  for (const it of items) {
+    const body = await source.getItemBody(libId, it.kind, it.name);
+    if (body != null) out.push({ kind: it.kind, name: it.name, body });
+  }
+  return out;
+}
+
 /** Escape a string for a KiCad s-expr quoted token. */
 function sexprEscape(s: string): string {
   return s.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
@@ -179,12 +234,24 @@ export function installLibsProvider(
     try {
       switch (op) {
         case "list": {
+          // Each plugin parses its own key: footprints / symbols.
+          const key = kind === "footprint" ? "footprints" : "symbols";
+          // "bodies" (arg) = the fat list: every item's body in one crossing, so
+          // the plugin pre-fills its cache and never per-item `get`s. Falls back
+          // to listItems + N getItemBody for sources without bulk read.
+          if (arg === "bodies") {
+            const all = source.getAllItems
+              ? await source.getAllItems(id)
+              : await fallbackGetAllItems(source, id);
+            const items = all
+              .filter((i) => i.kind === kind)
+              .map((i) => ({ name: i.name, body: i.body }));
+            return JSON.stringify({ [key]: items });
+          }
           const items = await source.listItems(id);
           const names = items
             .filter((i) => i.kind === kind)
             .map((i) => i.name);
-          // Each plugin parses its own key: footprints / symbols.
-          const key = kind === "footprint" ? "footprints" : "symbols";
           return JSON.stringify({ [key]: names });
         }
         case "get": {

web/standalone/src/wasm/libs/synced-source.ts

@@ -42,6 +42,29 @@ export function syncedLibsSource(
       const { stack } = await ensure();
       return (await stack.list()).map((e) => splitPath(e.path));
     },
+    async presync(opts): Promise<void> {
+      // One lib: resolving + opening its stack warms the IDB cache.
+      opts?.onProgress?.({ done: 0, total: 1, current: "library" });
+      try {
+        const { info } = await ensure();
+        opts?.onProgress?.({ done: 1, total: 1, current: info.name });
+      } catch {
+        opts?.onProgress?.({ done: 1, total: 1, current: "library" });
+      }
+    },
+    async getAllItems(): Promise<
+      Array<{ kind: string; name: string; body: string }>
+    > {
+      // Bulk merged read across the opaque layer stack (origin + mirror overlay),
+      // top-wins — the mirror invariant readAll() preserves. One crossing, no
+      // per-item gets.
+      const { stack } = await ensure();
+      const dec = new TextDecoder();
+      return [...(await stack.readAll())].map(([path, bytes]) => {
+        const { kind, name } = splitPath(path);
+        return { kind, name, body: dec.decode(bytes) };
+      });
+    },
     async getItemBody(_id, kind, name): Promise<string | null> {
       const { stack } = await ensure();
       const bytes = await stack.read(pathOf(kind, name));

wxwidgets

@@ -1 +1 @@
-Subproject commit 331cff5dba6289dcc9e016df4ed52f0383861f5c
+Subproject commit 5d40228ac4c446c0f57c2e8d2d2bea8db827d45d