diff --git a/kicad b/kicad index 0a7d1ba..c630090 160000 --- a/kicad +++ b/kicad @@ -1 +1 @@ -Subproject commit 0a7d1baf0bc69b19b489bf52965f306a4d1624c3 +Subproject commit c63009046c06011a0220b0e3ada28e084ffe4e54 diff --git a/web/pcbjam-shared b/web/pcbjam-shared index 3367a91..c6a7e00 160000 --- a/web/pcbjam-shared +++ b/web/pcbjam-shared @@ -1 +1 @@ -Subproject commit 3367a91b39060d6be51b5d30c5503a00933314f9 +Subproject commit c6a7e00bcaa39023be0be126a28f8f1acb9a2d67 diff --git a/web/standalone/src/components/WasmTool.tsx b/web/standalone/src/components/WasmTool.tsx index ad6f258..084e27b 100644 --- a/web/standalone/src/components/WasmTool.tsx +++ b/web/standalone/src/components/WasmTool.tsx @@ -51,6 +51,16 @@ import { SourceChip } from "@/components/SourceChip"; // Tools with the v2 items bridge (kicadCollabSnapshotItems/ApplyItems embind exports). const COLLAB_TOOLS = new Set(["pl_editor", "eeschema", "pcbnew"]); + +// Which library item kind each tool browses — drives the load-screen pre-sync +// (warm the right bundles into IDB while the wasm downloads). Tools that don't +// browse a library are omitted (no pre-sync). +const LIB_KIND_FOR_TOOL: Partial> = { + symbol_editor: "symbol", + eeschema: "symbol", + footprint_editor: "footprint", + pcbnew: "footprint", +}; const LEGACY_EXTENSION_TOOL: Record = { ".sch": "eeschema", ".brd": "pcbnew", @@ -560,6 +570,9 @@ export function WasmTool({ const [ready, setReady] = React.useState(false); // A library item currently being fetched (open/save), for a transient spinner. const [libBusy, setLibBusy] = React.useState(null); + // Load-screen pre-sync progress: warming the project's lib bundles into IDB in + // parallel with the wasm download. Null when idle/done. + const [libSync, setLibSync] = React.useState(null); // Last lib error (e.g. a backend 404 on open), shown as a dismissible toast. const [libError, setLibError] = React.useState(null); @@ -653,6 +666,33 @@ export function WasmTool({ // Resolve the per-tool asset base at runtime (CDN manifest → versioned // folder, or the flat local /wasm in dev). See wasm/wasm-assets.ts. const base = await resolveWasmBase(tool, assetBaseUrl); + // One source instance, shared by the wasm provider AND the pre-sync below + // (libsSourceConfig builds a fresh one each call — their SyncStack caches + // must be the same object for the warm-up to benefit the editor). + const source = + libsSource !== undefined ? libsSource : libsSourceConfig(projectId); + // Pre-warm the lib bundles into IDB in PARALLEL with the wasm download, so + // the editor's first enumerate reads a warm cache instead of freezing on N + // cold bundle fetches. Non-blocking + best-effort; the SyncStack dedups, so + // a lib the wasm reaches mid-presync just awaits the same in-flight fetch. + const libKind = LIB_KIND_FOR_TOOL[tool]; + if (source?.presync && libKind) { + void source + .presync({ + kind: libKind, + onProgress: ({ done, total, current }) => + setLibSync( + done >= total + ? null + : `Syncing ${libKind}s — ${current} (${done}/${total})`, + ), + }) + .then(() => setLibSync(null)) + .catch((e) => { + append(`[presync] ${String(e)}`); + setLibSync(null); + }); + } await bootKicadTool({ tool, base, @@ -660,8 +700,7 @@ export function WasmTool({ log: append, onStatus: setStatus, onAbort: oom.onAbort, - libsSource: - libsSource !== undefined ? libsSource : libsSourceConfig(projectId), + libsSource: source, }); // Register the save sink before the file opens: from here on, every // editor File→Save (MEMFS write) is routed onward through saveBytes. @@ -824,6 +863,9 @@ export function WasmTool({

{status || "Loading…"}

+ {libSync && ( +

{libSync}

+ )}

First load downloads the tool (large) — this can take a moment.

@@ -846,6 +888,14 @@ export function WasmTool({ )} + {/* Lib pre-sync still warming IDB after the editor opened (big set) — small + unobtrusive indicator so the user knows browsing is still filling in. */} + {ready && libSync && ( +
+ {libSync} +
+ )} + {/* A library item is being fetched (open/save). */} {ready && libBusy && (
diff --git a/web/standalone/src/wasm/libs/cdn-source.test.ts b/web/standalone/src/wasm/libs/cdn-source.test.ts index d29be57..a181f32 100644 --- a/web/standalone/src/wasm/libs/cdn-source.test.ts +++ b/web/standalone/src/wasm/libs/cdn-source.test.ts @@ -90,6 +90,41 @@ describe("cdn libs source", () => { expect(await src.getItemBody("Device", "symbol", "Nope")).toBeNull(); }); + it("getAllItems returns every item with its body in one shot (fat list)", async () => { + const src = await fakeCdn(); + const all = (await src.getAllItems!("Device")).sort((a, b) => + a.name.localeCompare(b.name), + ); + expect(all).toEqual([ + { kind: "symbol", name: "C", body: "(kicad_symbol_lib (symbol C))" }, + { kind: "symbol", name: "R", body: "(kicad_symbol_lib (symbol R))" }, + ]); + // Bodies match the per-item getItemBody for the same kind/name. + for (const it of all) { + expect(it.body).toBe(await src.getItemBody("Device", it.kind, it.name)); + } + }); + + it("presync warms a kind's lib bundles and reports per-lib progress", async () => { + const src = await fakeCdn(); + const events: Array<{ done: number; total: number; current: string }> = []; + await src.presync!({ kind: "symbol", onProgress: (p) => events.push({ ...p }) }); + + const last = events[events.length - 1]!; + expect(last.total).toBe(1); // one symbol lib (Device) + expect(last.done).toBe(1); // completed + expect(events.some((e) => e.current === "Device")).toBe(true); + // Warmed: items are served from the opened stack. + expect((await src.listItems("Device")).length).toBe(2); + }); + + it("presync without a kind warms every lib", async () => { + const src = await fakeCdn(); + let total = 0; + await src.presync!({ onProgress: (p) => (total = p.total) }); + expect(total).toBe(2); // Device (symbol) + Resistor_SMD (footprint) + }); + it("is read-only (no save path)", async () => { const src = await fakeCdn(); expect(src.saveItemBody).toBeUndefined(); diff --git a/web/standalone/src/wasm/libs/cdn-source.ts b/web/standalone/src/wasm/libs/cdn-source.ts index ea43f16..44222ab 100644 --- a/web/standalone/src/wasm/libs/cdn-source.ts +++ b/web/standalone/src/wasm/libs/cdn-source.ts @@ -40,12 +40,36 @@ export function cdnLibsSource( const fetchImpl = opts?.fetchImpl ?? fetch; let manifestP: Promise | null = null; - const loadManifest = () => - (manifestP ??= (async () => { - const r = await fetchImpl(manifestUrl, { cache: "no-store" }); - if (!r.ok) throw new Error(`cdn libs manifest ${r.status}: ${manifestUrl}`); - return (await r.json()) as CdnLibsManifest; - })()); + const fetchManifest = async (): Promise => { + // Retry with backoff. Firefox can fail a cross-origin fetch issued in the + // first moments after navigation under COEP (the lazy path runs seconds + // later and succeeds); a short retry rides past that window. The pre-sync + // warm-up, which fires this earliest, is what surfaced it. + let lastErr: unknown; + for (let attempt = 0; attempt < 4; attempt++) { + if (attempt > 0) await new Promise((r) => setTimeout(r, 200 * attempt)); + try { + const r = await fetchImpl(manifestUrl, { cache: "no-store" }); + if (!r.ok) + throw new Error(`cdn libs manifest ${r.status}: ${manifestUrl}`); + return (await r.json()) as CdnLibsManifest; + } catch (e) { + lastErr = e; + } + } + throw lastErr; + }; + const loadManifest = (): Promise => { + if (!manifestP) { + // NEVER cache a rejection: a transient failure must not poison every later + // listLibs/getItemBody — the next call retries from scratch. + manifestP = fetchManifest().catch((e) => { + manifestP = null; + throw e; + }); + } + return manifestP; + }; // One lazily-opened SyncStack per lib (its IDB store is keyed by namespace, so // a lib is cached once and reused across opens). @@ -67,7 +91,12 @@ export function cdnLibsSource( }); await stack.open(); return stack; - })(); + })().catch((e) => { + // Don't cache a failed open (e.g. an early-boot fetch blip) — the lazy + // path must be able to retry this lib instead of inheriting the failure. + stacks.delete(libId); + throw e; + }); stacks.set(libId, p); } return p; @@ -89,6 +118,54 @@ export function cdnLibsSource( const stack = await openStack(libId); return (await stack.list()).map((e) => splitPath(e.path)); }, + async presync(opts): Promise { + const { kind, concurrency = 6, onProgress, signal } = opts ?? {}; + let m: CdnLibsManifest; + try { + m = await loadManifest(); + } catch { + // Best-effort warm-up: a manifest hiccup here (e.g. the early-boot + // Firefox/COEP fetch blip) is non-fatal — skip quietly and let the lazy + // path fetch on demand (loadManifest no longer caches the failure). + return; + } + const libs = m.libs.filter((l) => !kind || l.kind === kind); + const total = libs.length; + let done = 0; + onProgress?.({ done, total, current: "" }); + // Concurrency-limited pool: each openStack cold-fetches one bundle into IDB + // (warm ⇒ a cheap manifest diff). Tolerate per-lib failures so one bad lib + // doesn't abort the warm-up. + let idx = 0; + const worker = async (): Promise => { + while (idx < libs.length) { + if (signal?.aborted) return; + const lib = libs[idx++]!; + onProgress?.({ done, total, current: lib.name }); + try { + await openStack(lib.id); + } catch { + // best-effort: the lib still loads lazily on demand + } + onProgress?.({ done: ++done, total, current: lib.name }); + } + }; + await Promise.all( + Array.from({ length: Math.min(concurrency, total) }, () => worker()), + ); + }, + async getAllItems( + libId: string, + ): Promise> { + // One bulk merged read of the whole lib (the IDB cache after cold bundle), + // so the WASM plugin hydrates in a single crossing instead of N gets. + const stack = await openStack(libId); + const dec = new TextDecoder(); + return [...(await stack.readAll())].map(([path, bytes]) => { + const { kind, name } = splitPath(path); + return { kind, name, body: dec.decode(bytes) }; + }); + }, async getItemBody( libId: string, kind: string, diff --git a/web/standalone/src/wasm/libs/scoped-source.ts b/web/standalone/src/wasm/libs/scoped-source.ts index 8240664..d95e411 100644 --- a/web/standalone/src/wasm/libs/scoped-source.ts +++ b/web/standalone/src/wasm/libs/scoped-source.ts @@ -21,5 +21,30 @@ export function scopedLibsSource(base: LibsSource, libId: string): LibsSource { getItemBody(id: string, kind: string, name: string): Promise { return base.getItemBody(id, kind, name); }, + // Forward the bulk "fat list" only when the base supports it, so a scoped + // single-lib view keeps the one-crossing hydrate (else the provider's slow + // fallback applies). + ...(base.getAllItems + ? { getAllItems: (id: string) => base.getAllItems!(id) } + : {}), + // Pre-sync ONLY the scoped lib — NOT the base's whole catalog (that's what a + // bare `base.presync()` would do). Opening the one lib via listItems warms its + // bundle into IDB. Gated on the base being cache-capable (it exposes presync); + // per-item remote has no client cache to warm. + ...(base.presync + ? { + presync: async ( + o?: Parameters>[0], + ): Promise => { + o?.onProgress?.({ done: 0, total: 1, current: libId }); + try { + await base.listItems(libId); + } catch { + // best-effort: the lib still loads lazily on demand + } + o?.onProgress?.({ done: 1, total: 1, current: libId }); + }, + } + : {}), }; } diff --git a/web/standalone/src/wasm/libs/source.test.ts b/web/standalone/src/wasm/libs/source.test.ts new file mode 100644 index 0000000..501d29f --- /dev/null +++ b/web/standalone/src/wasm/libs/source.test.ts @@ -0,0 +1,98 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { installLibsProvider, type LibsSource } from "./source"; +import { libUri } from "./uri"; + +/** + * The WASM lib plugins call `window.kicadLibs.request(op, lib, arg, kind)`. These + * tests pin the wire contract the C++ side parses — in particular the "fat list" + * (`arg === "bodies"`) shape `{symbols|footprints: [{name, body}]}` and its + * fallback for sources without a bulk `getAllItems`. Node env has no `window`, so + * we stub the minimal surface the provider touches. + */ + +type Req = ( + op: string, + lib: string, + arg: string, + kind?: string, +) => Promise; + +function installAndGetRequest(source: LibsSource): Req { + installLibsProvider(source, () => {}); + return (globalThis as unknown as { window: { kicadLibs: { request: Req } } }) + .window.kicadLibs.request; +} + +const ITEMS = [ + { kind: "symbol", name: "R", body: "(kicad_symbol_lib (symbol R))" }, + { kind: "symbol", name: "C", body: "(kicad_symbol_lib (symbol C))" }, + { kind: "footprint", name: "R_0402", body: "(footprint R_0402)" }, +]; + +function baseSource(over: Partial = {}): LibsSource { + return { + listLibs: async () => [], + listItems: async () => ITEMS.map((i) => ({ kind: i.kind, name: i.name })), + getItemBody: async (_id, kind, name) => + ITEMS.find((i) => i.kind === kind && i.name === name)?.body ?? null, + ...over, + }; +} + +describe("installLibsProvider — fat list (arg=bodies)", () => { + beforeEach(() => { + (globalThis as unknown as { window: unknown }).window = { + location: { search: "" }, + dispatchEvent: () => true, + }; + }); + afterEach(() => { + delete (globalThis as unknown as { window?: unknown }).window; + }); + + it("returns names+bodies for the requested kind via getAllItems", async () => { + const calls: string[] = []; + const request = installAndGetRequest( + baseSource({ + getAllItems: async (id) => { + calls.push(id); + return ITEMS; + }, + }), + ); + + const res = await request("list", libUri("Device"), "bodies", "symbol"); + expect(JSON.parse(res!)).toEqual({ + symbols: [ + { name: "R", body: "(kicad_symbol_lib (symbol R))" }, + { name: "C", body: "(kicad_symbol_lib (symbol C))" }, + ], + }); + // One bulk call, not per-item. + expect(calls).toEqual(["Device"]); + + const fps = await request("list", libUri("Device"), "bodies", "footprint"); + expect(JSON.parse(fps!)).toEqual({ + footprints: [{ name: "R_0402", body: "(footprint R_0402)" }], + }); + }); + + it("falls back to listItems + getItemBody when getAllItems is absent", async () => { + const request = installAndGetRequest(baseSource()); // no getAllItems + const res = await request("list", libUri("Device"), "bodies", "symbol"); + expect(JSON.parse(res!)).toEqual({ + symbols: [ + { name: "R", body: "(kicad_symbol_lib (symbol R))" }, + { name: "C", body: "(kicad_symbol_lib (symbol C))" }, + ], + }); + }); + + it("plain list (empty arg) still returns names only", async () => { + const request = installAndGetRequest( + baseSource({ getAllItems: async () => ITEMS }), + ); + const res = await request("list", libUri("Device"), "", "symbol"); + expect(JSON.parse(res!)).toEqual({ symbols: ["R", "C"] }); + }); +}); diff --git a/web/standalone/src/wasm/libs/source.ts b/web/standalone/src/wasm/libs/source.ts index f282cf0..61409bb 100644 --- a/web/standalone/src/wasm/libs/source.ts +++ b/web/standalone/src/wasm/libs/source.ts @@ -20,6 +20,14 @@ export interface LibItemInfo { name: string; } +/** Per-lib progress for {@link LibsSource.presync} (drives the load screen). */ +export interface LibPresyncProgress { + done: number; + total: number; + /** Display name of the lib currently being synced (one of the in-flight set). */ + current: string; +} + export interface LibsSource { /** * Libraries to expose to the editor (one lib-table row each). `kind` (the @@ -35,6 +43,34 @@ export interface LibsSource { * null if absent. `kind` is 'symbol' for now. */ getItemBody(libId: string, kind: string, name: string): Promise; + /** + * ALL items in a library with their bodies, in one shot — the "fat list" that + * lets the WASM plugin hydrate a whole library in a single bridge crossing + * instead of N per-item `get`s (see docs/features/libs/0011). Optional: sources + * that can't bulk-read omit it and the provider falls back to listItems + N + * getItemBody (the old slow path, kept working for the example backend). + */ + getAllItems?( + libId: string, + ): Promise>; + /** + * Pre-warm this source's per-lib caches (IndexedDB bundles) WITHOUT touching the + * WASM runtime — call it in parallel with the wasm download so the editor's + * first enumerate reads a warm cache instead of freezing on N cold bundle + * fetches. Best-effort: a lib that fails to presync is skipped (it still loads + * lazily later); the SyncStack dedups, so a lib the wasm reaches mid-presync + * just awaits the same in-flight fetch. `onProgress` reports per-lib so the load + * screen can name what's syncing. Optional: sources without a client-side cache + * (per-item remote) omit it. + */ + presync?(opts?: { + /** Limit to libs holding this item kind ("symbol" | "footprint"). */ + kind?: string; + /** Max concurrent bundle fetches (default 6). */ + concurrency?: number; + onProgress?: (p: LibPresyncProgress) => void; + signal?: AbortSignal; + }): Promise; /** * Persist one item body into a writable (user) lib. Optional: read-only * sources omit it (a save into a non-writable source resolves false). @@ -112,6 +148,25 @@ function artificialDelayMs(): number { const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +/** + * Slow-path "fat list" for sources without a bulk `getAllItems` (the example + * backend / per-item remote): just listItems + one getItemBody each. Same N + * round-trips as before — but it keeps the single WASM-side code path (the plugin + * always asks for bodies once) working against every source. + */ +async function fallbackGetAllItems( + source: LibsSource, + libId: string, +): Promise> { + const items = await source.listItems(libId); + const out: Array<{ kind: string; name: string; body: string }> = []; + for (const it of items) { + const body = await source.getItemBody(libId, it.kind, it.name); + if (body != null) out.push({ kind: it.kind, name: it.name, body }); + } + return out; +} + /** Escape a string for a KiCad s-expr quoted token. */ function sexprEscape(s: string): string { return s.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); @@ -179,12 +234,24 @@ export function installLibsProvider( try { switch (op) { case "list": { + // Each plugin parses its own key: footprints / symbols. + const key = kind === "footprint" ? "footprints" : "symbols"; + // "bodies" (arg) = the fat list: every item's body in one crossing, so + // the plugin pre-fills its cache and never per-item `get`s. Falls back + // to listItems + N getItemBody for sources without bulk read. + if (arg === "bodies") { + const all = source.getAllItems + ? await source.getAllItems(id) + : await fallbackGetAllItems(source, id); + const items = all + .filter((i) => i.kind === kind) + .map((i) => ({ name: i.name, body: i.body })); + return JSON.stringify({ [key]: items }); + } const items = await source.listItems(id); const names = items .filter((i) => i.kind === kind) .map((i) => i.name); - // Each plugin parses its own key: footprints / symbols. - const key = kind === "footprint" ? "footprints" : "symbols"; return JSON.stringify({ [key]: names }); } case "get": { diff --git a/web/standalone/src/wasm/libs/synced-source.ts b/web/standalone/src/wasm/libs/synced-source.ts index 3ee59f0..95e6d7b 100644 --- a/web/standalone/src/wasm/libs/synced-source.ts +++ b/web/standalone/src/wasm/libs/synced-source.ts @@ -42,6 +42,29 @@ export function syncedLibsSource( const { stack } = await ensure(); return (await stack.list()).map((e) => splitPath(e.path)); }, + async presync(opts): Promise { + // One lib: resolving + opening its stack warms the IDB cache. + opts?.onProgress?.({ done: 0, total: 1, current: "library" }); + try { + const { info } = await ensure(); + opts?.onProgress?.({ done: 1, total: 1, current: info.name }); + } catch { + opts?.onProgress?.({ done: 1, total: 1, current: "library" }); + } + }, + async getAllItems(): Promise< + Array<{ kind: string; name: string; body: string }> + > { + // Bulk merged read across the opaque layer stack (origin + mirror overlay), + // top-wins — the mirror invariant readAll() preserves. One crossing, no + // per-item gets. + const { stack } = await ensure(); + const dec = new TextDecoder(); + return [...(await stack.readAll())].map(([path, bytes]) => { + const { kind, name } = splitPath(path); + return { kind, name, body: dec.decode(bytes) }; + }); + }, async getItemBody(_id, kind, name): Promise { const { stack } = await ensure(); const bytes = await stack.read(pathOf(kind, name)); diff --git a/wxwidgets b/wxwidgets index 331cff5..5d40228 160000 --- a/wxwidgets +++ b/wxwidgets @@ -1 +1 @@ -Subproject commit 331cff5dba6289dcc9e016df4ed52f0383861f5c +Subproject commit 5d40228ac4c446c0f57c2e8d2d2bea8db827d45d