From 88aaba99c90d3be7d59659c596c3f63f5bb76aa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20T=C3=B6rcsv=C3=A1ri?= Date: Wed, 1 Jul 2026 09:56:32 +0200 Subject: [PATCH] feat(libs): react load overlay + progress bar; mimalloc mallinfo stub; framed tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - WasmTool/source.ts: LIB_LOADING_EVENT brackets the fat-load; full-cover overlay ('Loading libraries…') with a per-lib progress bar (done/total from listLibs), show-on-first / hide-after-last coalescing. - mallinfo stub (wasm/shims/mallinfo_stub.c + build-kicad-target.sh): -sMALLOC= mimalloc doesn't export glibc mallinfo() that OpenCASCADE OSD_MemInfo (pcbnew 3D) needs; zeroed no-op unblocks the pcbnew link. - source/cdn-source tests: updated to the framed Uint8Array 'bodies' contract. - bump kicad -> e8db3d3. Co-Authored-By: Claude Opus 4.8 (1M context) --- kicad | 2 +- scripts/kicad/build-kicad-target.sh | 9 +- wasm/shims/mallinfo_stub.c | 23 ++++ web/standalone/src/components/WasmTool.tsx | 72 ++++++++++++ .../src/wasm/libs/cdn-source.test.ts | 15 ++- web/standalone/src/wasm/libs/source.test.ts | 81 +++++++++---- web/standalone/src/wasm/libs/source.ts | 106 ++++++++++++++---- 7 files changed, 257 insertions(+), 51 deletions(-) create mode 100644 wasm/shims/mallinfo_stub.c diff --git a/kicad b/kicad index dc3219a..e8db3d3 160000 --- a/kicad +++ b/kicad @@ -1 +1 @@ -Subproject commit dc3219aa69ce98ca478800e5f09767d47f6b8370 +Subproject commit e8db3d35db635946b03ee0ae8698e1d3aad79a3c diff --git a/scripts/kicad/build-kicad-target.sh b/scripts/kicad/build-kicad-target.sh index de32977..dcddbf0 100755 --- a/scripts/kicad/build-kicad-target.sh +++ b/scripts/kicad/build-kicad-target.sh @@ -434,6 +434,13 @@ fi emcc -c -pthread "${PROJECT_ROOT}/wasm/shims/nanosleep_yield.c" -o "${STUBS_BUILD}/nanosleep_yield.o" NANOSLEEP_YIELD_LINK="${STUBS_BUILD}/nanosleep_yield.o" +# mallinfo() stub for the mimalloc build: -sMALLOC=mimalloc doesn't export the +# glibc mallinfo() that OpenCASCADE's OSD_MemInfo.cxx (libTKernel, pcbnew's 3D) +# references — without this the pcbnew link fails `undefined symbol: mallinfo`. +# Zeroed no-op (memory reporting only); harmless for apps that don't reference it. +emcc -c "${PROJECT_ROOT}/wasm/shims/mallinfo_stub.c" -o "${STUBS_BUILD}/mallinfo_stub.o" +MALLINFO_STUB_LINK="${STUBS_BUILD}/mallinfo_stub.o" + emcmake cmake "${KICAD_DIR}" \ ${CCACHE_OPTS} \ ${SYM_CONVERTER_CMAKE_FLAG} \ @@ -444,7 +451,7 @@ emcmake cmake "${KICAD_DIR}" \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS} -Xclang -fno-pch-timestamp -pthread -sUSE_ZLIB=1 -DKICAD_USE_PLATFORM_WASM=1${DIAG_DEFINES} -I${SYSROOT}/include -I${STUBS_DIR} -include ${STUBS_DIR}/char_traits_uint16_workaround.h" \ -DCMAKE_C_FLAGS="${EXTRA_FLAGS} -pthread -sUSE_ZLIB=1 -I${SYSROOT}/include -I${STUBS_DIR}" \ - -DCMAKE_EXE_LINKER_FLAGS="${LINKER_DEBUG_FLAGS} -pthread -sUSE_ZLIB=1 -sASYNCIFY=1 -sDYNCALLS=1 -sASYNCIFY_STACK_SIZE=65536 -sUSE_PTHREADS=1 -sMALLOC=mimalloc -sPTHREAD_POOL_SIZE='navigator.hardwareConcurrency' -sPTHREAD_POOL_SIZE_STRICT=0 -sALLOW_MEMORY_GROWTH=1 -sINITIAL_MEMORY=256MB -sMAXIMUM_MEMORY=4GB -sMAX_WEBGL_VERSION=2 ${GL3D_LINK_FLAGS} ${NANOSLEEP_YIELD_LINK} -sEXPORTED_RUNTIME_METHODS=['ccall','cwrap','UTF8ToString','stringToUTF8','lengthBytesUTF8','dynCall'] -sDEFAULT_LIBRARY_FUNCS_TO_INCLUDE=['\$dynCall'] --bind -L${SYSROOT}/lib ${STUBS_BUILD}/libgit2_stub.a ${STUBS_BUILD}/libcurl_stub.a${APP_STUB_LINK} ${STUBS_BUILD}/libnng_stub.a ${EMBIND_OBJ}" \ + -DCMAKE_EXE_LINKER_FLAGS="${LINKER_DEBUG_FLAGS} -pthread -sUSE_ZLIB=1 -sASYNCIFY=1 -sDYNCALLS=1 -sASYNCIFY_STACK_SIZE=65536 -sUSE_PTHREADS=1 -sMALLOC=mimalloc -sPTHREAD_POOL_SIZE='navigator.hardwareConcurrency' -sPTHREAD_POOL_SIZE_STRICT=0 -sALLOW_MEMORY_GROWTH=1 -sINITIAL_MEMORY=256MB -sMAXIMUM_MEMORY=4GB -sMAX_WEBGL_VERSION=2 ${GL3D_LINK_FLAGS} ${NANOSLEEP_YIELD_LINK} ${MALLINFO_STUB_LINK} -sEXPORTED_RUNTIME_METHODS=['ccall','cwrap','UTF8ToString','stringToUTF8','lengthBytesUTF8','dynCall'] -sDEFAULT_LIBRARY_FUNCS_TO_INCLUDE=['\$dynCall'] --bind -L${SYSROOT}/lib ${STUBS_BUILD}/libgit2_stub.a ${STUBS_BUILD}/libcurl_stub.a${APP_STUB_LINK} ${STUBS_BUILD}/libnng_stub.a ${EMBIND_OBJ}" \ -DCMAKE_PREFIX_PATH="${SYSROOT};${WX_BUILD}" \ -DwxWidgets_CONFIG_EXECUTABLE="${WX_BUILD}/wx-config" \ \ diff --git a/wasm/shims/mallinfo_stub.c b/wasm/shims/mallinfo_stub.c new file mode 100644 index 0000000..7a6997b --- /dev/null +++ b/wasm/shims/mallinfo_stub.c @@ -0,0 +1,23 @@ +/* + * mallinfo() stub for the mimalloc-allocator build. + * + * We link with -sMALLOC=mimalloc (per-thread heaps — needed so the parallel + * s-expr library parse doesn't serialize on dlmalloc's single global lock; see + * docs/features/libs/0013). Unlike emscripten's dlmalloc/emmalloc, mimalloc does + * NOT export the glibc memory-report API mallinfo()/mallinfo2(). OpenCASCADE's + * OSD_MemInfo.cxx (in libTKernel, linked only by pcbnew for 3D/STEP) references + * mallinfo(), so the pcbnew link fails with `undefined symbol: mallinfo` without + * this. (eeschema doesn't link OpenCASCADE, so it never needed it.) + * + * mallinfo() is used purely for optional memory reporting, so a zeroed result is + * harmless. Provided unconditionally (mirrors the nanosleep_yield / gl_ffp stub + * pattern); it's a no-op for apps that never reference it. + */ + +#include + +struct mallinfo mallinfo( void ) +{ + struct mallinfo info = { 0 }; + return info; +} diff --git a/web/standalone/src/components/WasmTool.tsx b/web/standalone/src/components/WasmTool.tsx index 644f095..0e783c6 100644 --- a/web/standalone/src/components/WasmTool.tsx +++ b/web/standalone/src/components/WasmTool.tsx @@ -25,8 +25,10 @@ import { resolveWasmBase } from "@/wasm/wasm-assets"; import { LIB_BUSY_EVENT, LIB_ERROR_EVENT, + LIB_LOADING_EVENT, type LibBusyDetail, type LibErrorDetail, + type LibLoadingDetail, type LibsSource, } from "@/wasm/libs/source"; import { memfsFilePath, memfsProjectDir } from "@/wasm/constants"; @@ -584,6 +586,15 @@ export function WasmTool({ const [libSync, setLibSync] = React.useState(null); // Last lib error (e.g. a backend 404 on open), shown as a dismissible toast. const [libError, setLibError] = React.useState(null); + // Eager whole-library idb→wasm load in flight (the ~tens-of-seconds fat-load on + // first chooser/editor open). Drives a full-cover overlay so the freeze reads as + // "loading, just slow" rather than a hang. Null when idle; `done/total` count the + // per-lib fat-load crossings so the overlay can show a progress bar. + const [libLoading, setLibLoading] = React.useState<{ + kind: string; + done: number; + total: number; + } | null>(null); const append = React.useCallback( (msg: string) => setLogs((prev) => [...prev.slice(-800), msg]), @@ -624,6 +635,31 @@ export function WasmTool({ return () => clearTimeout(t); }, [libError]); + // Full-library eager load overlay. The fat-load fires one loading:true/false + // pair PER library (222 on the full set), and between them the C++ side parses + // with the main thread blocked. Show immediately on `true`, and only hide after + // a short quiet gap on `false` (reset by the next lib's `true`) — so the overlay + // stays continuous across the whole run and drops shortly after the last lib, + // instead of flickering 222 times. + React.useEffect(() => { + let hideTimer: ReturnType | undefined; + const onLoading = (e: Event) => { + const d = (e as CustomEvent).detail; + clearTimeout(hideTimer); + // Update the bar on every event (true and false) so the count reflects the + // latest lib; arm the hide only when the run reports it's winding down. + setLibLoading({ kind: d.kind || "library", done: d.done, total: d.total }); + if (!d.loading) { + hideTimer = setTimeout(() => setLibLoading(null), 700); + } + }; + window.addEventListener(LIB_LOADING_EVENT, onLoading); + return () => { + clearTimeout(hideTimer); + window.removeEventListener(LIB_LOADING_EVENT, onLoading); + }; + }, []); + // "Taking too long": once the tool has been loading for a while without // becoming ready, surface a hint (slow link / something may be wrong) + a // reload, so a stalled boot doesn't look like a frozen blank screen. @@ -908,6 +944,42 @@ export function WasmTool({ )} + {/* Eager library load overlay — the first chooser/editor open hydrates the + whole library set from IDB into wasm (tens of seconds on the full CDN + set) with the main thread blocked. Cover the (frozen) editor so it reads + as "loading, just slow" rather than a hang. Shown post-boot; before + `ready` the boot overlay already covers it. */} + {ready && libLoading && ( +
+ +

+ {libLoading.kind === "library" + ? "Loading libraries…" + : `Loading ${libLoading.kind} libraries…`} +

+ {libLoading.total > 0 && ( +
+
+
+
+

+ {Math.min(libLoading.done, libLoading.total)} / {libLoading.total}{" "} + libraries +

+
+ )} +

+ Moving the library set into the editor. The first open can take a + moment — it's cached after this. +

+
+ )} + {/* Transient post-boot status (e.g. file open). */} {ready && status && (
diff --git a/web/standalone/src/wasm/libs/cdn-source.test.ts b/web/standalone/src/wasm/libs/cdn-source.test.ts index a181f32..752e5f7 100644 --- a/web/standalone/src/wasm/libs/cdn-source.test.ts +++ b/web/standalone/src/wasm/libs/cdn-source.test.ts @@ -7,6 +7,7 @@ const MANIFEST_URL = "https://cdn.test/libs/kicad/9.0.0/manifest.json"; const BASE = "https://cdn.test/libs/kicad/9.0.0"; const enc = new TextEncoder(); +const dec = new TextDecoder(); /** Build a static-origin snapshot (per-lib manifest + bundle) the way * publish-libs will, using the REAL wire codecs — so this pins the format. */ @@ -90,18 +91,24 @@ describe("cdn libs source", () => { expect(await src.getItemBody("Device", "symbol", "Nope")).toBeNull(); }); - it("getAllItems returns every item with its body in one shot (fat list)", async () => { + it("getAllItems returns every item with its (raw-bytes) body in one shot (fat list)", async () => { const src = await fakeCdn(); const all = (await src.getAllItems!("Device")).sort((a, b) => a.name.localeCompare(b.name), ); - expect(all).toEqual([ + // "Copy as-is": bodies come back as raw Uint8Array (no TextDecoder) so the + // provider can frame + memcpy them across the bridge unescaped. + expect( + all.map((i) => ({ kind: i.kind, name: i.name, body: dec.decode(i.body) })), + ).toEqual([ { kind: "symbol", name: "C", body: "(kicad_symbol_lib (symbol C))" }, { kind: "symbol", name: "R", body: "(kicad_symbol_lib (symbol R))" }, ]); - // Bodies match the per-item getItemBody for the same kind/name. + // Bodies (decoded) match the per-item getItemBody for the same kind/name. for (const it of all) { - expect(it.body).toBe(await src.getItemBody("Device", it.kind, it.name)); + expect(dec.decode(it.body)).toBe( + await src.getItemBody("Device", it.kind, it.name), + ); } }); diff --git a/web/standalone/src/wasm/libs/source.test.ts b/web/standalone/src/wasm/libs/source.test.ts index 501d29f..0e91eb6 100644 --- a/web/standalone/src/wasm/libs/source.test.ts +++ b/web/standalone/src/wasm/libs/source.test.ts @@ -5,9 +5,12 @@ import { libUri } from "./uri"; /** * The WASM lib plugins call `window.kicadLibs.request(op, lib, arg, kind)`. These * tests pin the wire contract the C++ side parses — in particular the "fat list" - * (`arg === "bodies"`) shape `{symbols|footprints: [{name, body}]}` and its - * fallback for sources without a bulk `getAllItems`. Node env has no `window`, so - * we stub the minimal surface the provider touches. + * (`arg === "bodies"`), which is the "copy as-is" FRAMED payload: a one-line JSON + * header `{symbols|footprints: [{name, len}]}`, a newline, then every body's raw + * bytes concatenated (no JSON escaping). The bodies cross as a `Uint8Array` the + * bridge memcpy's straight into the wasm heap. We also cover the fallback for + * sources without a bulk `getAllItems`. Node env has no `window`, so we stub the + * minimal surface the provider touches. */ type Req = ( @@ -15,7 +18,7 @@ type Req = ( lib: string, arg: string, kind?: string, -) => Promise; +) => Promise; function installAndGetRequest(source: LibsSource): Req { installLibsProvider(source, () => {}); @@ -29,6 +32,42 @@ const ITEMS = [ { kind: "footprint", name: "R_0402", body: "(footprint R_0402)" }, ]; +const enc = new TextEncoder(); +const dec = new TextDecoder(); + +/** getAllItems bodies now cross as raw bytes (copied as-is, no TextDecoder). */ +function itemsAsBytes(items = ITEMS) { + return items.map((i) => ({ + kind: i.kind, + name: i.name, + body: enc.encode(i.body), + })); +} + +/** + * Decode the framed fat-list payload back into `{name, body}` records so the + * assertions read the same as the underlying data. Mirrors what the C++ `fatLoad` + * does: split at the first `\n`, parse the header, slice bodies by byte length. + */ +function parseFramed( + res: string | Uint8Array | null, + key: "symbols" | "footprints", +): Array<{ name: string; body: string }> { + expect(res).toBeInstanceOf(Uint8Array); + const bytes = res as Uint8Array; + const nl = bytes.indexOf(0x0a); + expect(nl).toBeGreaterThan(0); + const header = JSON.parse(dec.decode(bytes.subarray(0, nl))) as { + [k: string]: Array<{ name: string; len: number }>; + }; + let off = nl + 1; + return (header[key] ?? []).map(({ name, len }) => { + const body = dec.decode(bytes.subarray(off, off + len)); + off += len; + return { name, body }; + }); +} + function baseSource(over: Partial = {}): LibsSource { return { listLibs: async () => [], @@ -50,49 +89,45 @@ describe("installLibsProvider — fat list (arg=bodies)", () => { delete (globalThis as unknown as { window?: unknown }).window; }); - it("returns names+bodies for the requested kind via getAllItems", async () => { + it("returns a framed names+bodies payload for the requested kind via getAllItems", async () => { const calls: string[] = []; const request = installAndGetRequest( baseSource({ getAllItems: async (id) => { calls.push(id); - return ITEMS; + return itemsAsBytes(); }, }), ); const res = await request("list", libUri("Device"), "bodies", "symbol"); - expect(JSON.parse(res!)).toEqual({ - symbols: [ - { name: "R", body: "(kicad_symbol_lib (symbol R))" }, - { name: "C", body: "(kicad_symbol_lib (symbol C))" }, - ], - }); + expect(parseFramed(res, "symbols")).toEqual([ + { name: "R", body: "(kicad_symbol_lib (symbol R))" }, + { name: "C", body: "(kicad_symbol_lib (symbol C))" }, + ]); // One bulk call, not per-item. expect(calls).toEqual(["Device"]); const fps = await request("list", libUri("Device"), "bodies", "footprint"); - expect(JSON.parse(fps!)).toEqual({ - footprints: [{ name: "R_0402", body: "(footprint R_0402)" }], - }); + expect(parseFramed(fps, "footprints")).toEqual([ + { name: "R_0402", body: "(footprint R_0402)" }, + ]); }); it("falls back to listItems + getItemBody when getAllItems is absent", async () => { const request = installAndGetRequest(baseSource()); // no getAllItems const res = await request("list", libUri("Device"), "bodies", "symbol"); - expect(JSON.parse(res!)).toEqual({ - symbols: [ - { name: "R", body: "(kicad_symbol_lib (symbol R))" }, - { name: "C", body: "(kicad_symbol_lib (symbol C))" }, - ], - }); + expect(parseFramed(res, "symbols")).toEqual([ + { name: "R", body: "(kicad_symbol_lib (symbol R))" }, + { name: "C", body: "(kicad_symbol_lib (symbol C))" }, + ]); }); it("plain list (empty arg) still returns names only", async () => { const request = installAndGetRequest( - baseSource({ getAllItems: async () => ITEMS }), + baseSource({ getAllItems: async () => itemsAsBytes() }), ); const res = await request("list", libUri("Device"), "", "symbol"); - expect(JSON.parse(res!)).toEqual({ symbols: ["R", "C"] }); + expect(JSON.parse(res as string)).toEqual({ symbols: ["R", "C"] }); }); }); diff --git a/web/standalone/src/wasm/libs/source.ts b/web/standalone/src/wasm/libs/source.ts index a381b77..f87fb63 100644 --- a/web/standalone/src/wasm/libs/source.ts +++ b/web/standalone/src/wasm/libs/source.ts @@ -122,6 +122,15 @@ declare global { */ export const LIB_BUSY_EVENT = "pcbjam:lib-busy"; export const LIB_ERROR_EVENT = "pcbjam:lib-error"; +/** + * Fired around the bulk "fat list" crossing (`list`/`bodies`) — the eager + * idb→wasm library load that can take tens of seconds on the full CDN set. Unlike + * LIB_BUSY (per-item open/save), this brackets the whole-library hydrate so the + * editor chrome can show a "loading libraries, just slow" overlay instead of a + * silent freeze. One `loading:true` per lib as its crossing starts, `loading:false` + * as the bytes are handed to the bridge; the consumer coalesces the per-lib run. + */ +export const LIB_LOADING_EVENT = "pcbjam:lib-loading"; export interface LibBusyDetail { busy: boolean; @@ -132,11 +141,23 @@ export interface LibBusyDetail { export interface LibErrorDetail { message: string; } +export interface LibLoadingDetail { + loading: boolean; + kind: string; + /** Libraries whose fat-load has started this burst (1-based, increasing). */ + done: number; + /** Total libs of this kind to load (from listLibs), or 0 if unknown. */ + total: number; +} function emitLibBusy(detail: LibBusyDetail): void { if (typeof window === "undefined") return; window.dispatchEvent(new CustomEvent(LIB_BUSY_EVENT, { detail })); } +function emitLibLoading(detail: LibLoadingDetail): void { + if (typeof window === "undefined") return; + window.dispatchEvent(new CustomEvent(LIB_LOADING_EVENT, { detail })); +} function emitLibError(message: string): void { if (typeof window === "undefined") return; window.dispatchEvent( @@ -228,6 +249,15 @@ export function installLibsProvider( if (window.kicadLibs) return; const delay = artificialDelayMs(); + // Per-burst fat-load progress. The plugin fat-loads every library of a kind + // one bridge crossing at a time, so counting `bodies` requests gives real + // per-lib progress; `total` comes from listLibs(kind) (cached). A trailing + // timer resets the counter once a burst goes quiet, so a later open starts + // fresh (mirrors the overlay's own hide debounce). + let fatDone = 0; + let fatTotal = 0; + let fatResetTimer: ReturnType | undefined; + const request: KicadLibsRequest = async (op, lib, arg, kind = "symbol") => { const id = libIdFromUri(lib); log(`[libs] request op=${op} kind=${kind} lib=${lib} (id=${id}) arg=${arg}`); @@ -247,29 +277,61 @@ export function installLibsProvider( // the plugin pre-fills its cache and never per-item `get`s. Falls back // to listItems + N getItemBody for sources without bulk read. if (arg === "bodies") { - const all = source.getAllItems - ? await source.getAllItems(id) - : await fallbackGetAllItems(source, id); - const items = all.filter((i) => i.kind === kind); - // "Copy as-is" framing: a one-line JSON header (names + UTF-8 byte - // lengths), a newline, then every body's RAW bytes concatenated — no - // JSON escaping. The C++ bridge memcpy's this straight into the wasm - // heap; the plugin parses the small header and slices the bodies, so - // none of the (hundreds of MB of) s-expr gets un-escaped. - const header = JSON.stringify({ - [key]: items.map((i) => ({ name: i.name, len: i.body.length })), - }); - const headerBytes = new TextEncoder().encode(header + "\n"); - const total = - headerBytes.length + items.reduce((n, i) => n + i.body.length, 0); - const out = new Uint8Array(total); - out.set(headerBytes, 0); - let off = headerBytes.length; - for (const i of items) { - out.set(i.body, off); - off += i.body.length; + // Bracket the whole-library hydrate so the editor can overlay a + // "loading libraries (slow, not hung)" state over the otherwise + // silent multi-second freeze. `true` before the (async) IDB read so + // the overlay can paint while the C++ side is Asyncify-suspended; + // `false` once the bytes are framed and about to cross the bridge. + clearTimeout(fatResetTimer); + if (fatTotal === 0) { + // First lib of the burst — learn the total for the progress bar. + try { + fatTotal = (await source.listLibs(kind)).length; + } catch { + fatTotal = 0; + } + } + fatDone++; + emitLibLoading({ loading: true, kind, done: fatDone, total: fatTotal }); + try { + const all = source.getAllItems + ? await source.getAllItems(id) + : await fallbackGetAllItems(source, id); + const items = all.filter((i) => i.kind === kind); + // "Copy as-is" framing: a one-line JSON header (names + UTF-8 byte + // lengths), a newline, then every body's RAW bytes concatenated — no + // JSON escaping. The C++ bridge memcpy's this straight into the wasm + // heap; the plugin parses the small header and slices the bodies, so + // none of the (hundreds of MB of) s-expr gets un-escaped. + const header = JSON.stringify({ + [key]: items.map((i) => ({ name: i.name, len: i.body.length })), + }); + const headerBytes = new TextEncoder().encode(header + "\n"); + const total = + headerBytes.length + + items.reduce((n, i) => n + i.body.length, 0); + const out = new Uint8Array(total); + out.set(headerBytes, 0); + let off = headerBytes.length; + for (const i of items) { + out.set(i.body, off); + off += i.body.length; + } + return out; + } finally { + emitLibLoading({ + loading: false, + kind, + done: fatDone, + total: fatTotal, + }); + // Reset the per-burst counter once the run goes quiet, so the next + // open starts from zero (the WASM drives these back-to-back). + fatResetTimer = setTimeout(() => { + fatDone = 0; + fatTotal = 0; + }, 1500); } - return out; } const items = await source.listItems(id); const names = items