From 4025b052183844e6b0dd96f79fbf68548a865b9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20T=C3=B6rcsv=C3=A1ri?= Date: Wed, 1 Jul 2026 07:58:30 +0200 Subject: [PATCH] =?UTF-8?q?feat(libs):=20parallel=20s-expr=20parse=20+=20m?= =?UTF-8?q?imalloc=20+=20copy-as-is=20transport=20=E2=80=94=20symbol=20cho?= =?UTF-8?q?oser=20cold=20274s=E2=86=9244s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- kicad | 2 +- scripts/kicad/build-kicad-target.sh | 2 +- web/standalone/src/wasm/libs/cdn-source.ts | 11 +++--- web/standalone/src/wasm/libs/source.ts | 38 ++++++++++++++----- web/standalone/src/wasm/libs/synced-source.ts | 7 ++-- 5 files changed, 40 insertions(+), 20 deletions(-) diff --git a/kicad b/kicad index a010915..dc3219a 160000 --- a/kicad +++ b/kicad @@ -1 +1 @@ -Subproject commit a010915df55ed19df227e9367bdb5fb93e287a15 +Subproject commit dc3219aa69ce98ca478800e5f09767d47f6b8370 diff --git a/scripts/kicad/build-kicad-target.sh b/scripts/kicad/build-kicad-target.sh index f54ec94..de32977 100755 --- a/scripts/kicad/build-kicad-target.sh +++ b/scripts/kicad/build-kicad-target.sh @@ -444,7 +444,7 @@ emcmake cmake "${KICAD_DIR}" \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS} -Xclang -fno-pch-timestamp -pthread -sUSE_ZLIB=1 -DKICAD_USE_PLATFORM_WASM=1${DIAG_DEFINES} -I${SYSROOT}/include -I${STUBS_DIR} -include ${STUBS_DIR}/char_traits_uint16_workaround.h" \ -DCMAKE_C_FLAGS="${EXTRA_FLAGS} -pthread -sUSE_ZLIB=1 -I${SYSROOT}/include -I${STUBS_DIR}" \ - -DCMAKE_EXE_LINKER_FLAGS="${LINKER_DEBUG_FLAGS} -pthread -sUSE_ZLIB=1 -sASYNCIFY=1 -sDYNCALLS=1 -sASYNCIFY_STACK_SIZE=65536 -sUSE_PTHREADS=1 -sPTHREAD_POOL_SIZE='navigator.hardwareConcurrency' -sPTHREAD_POOL_SIZE_STRICT=0 -sALLOW_MEMORY_GROWTH=1 -sINITIAL_MEMORY=256MB -sMAXIMUM_MEMORY=4GB -sMAX_WEBGL_VERSION=2 ${GL3D_LINK_FLAGS} ${NANOSLEEP_YIELD_LINK} -sEXPORTED_RUNTIME_METHODS=['ccall','cwrap','UTF8ToString','stringToUTF8','lengthBytesUTF8','dynCall'] -sDEFAULT_LIBRARY_FUNCS_TO_INCLUDE=['\$dynCall'] --bind -L${SYSROOT}/lib ${STUBS_BUILD}/libgit2_stub.a ${STUBS_BUILD}/libcurl_stub.a${APP_STUB_LINK} ${STUBS_BUILD}/libnng_stub.a ${EMBIND_OBJ}" \ + -DCMAKE_EXE_LINKER_FLAGS="${LINKER_DEBUG_FLAGS} -pthread -sUSE_ZLIB=1 -sASYNCIFY=1 -sDYNCALLS=1 -sASYNCIFY_STACK_SIZE=65536 -sUSE_PTHREADS=1 -sMALLOC=mimalloc -sPTHREAD_POOL_SIZE='navigator.hardwareConcurrency' -sPTHREAD_POOL_SIZE_STRICT=0 -sALLOW_MEMORY_GROWTH=1 -sINITIAL_MEMORY=256MB -sMAXIMUM_MEMORY=4GB -sMAX_WEBGL_VERSION=2 ${GL3D_LINK_FLAGS} ${NANOSLEEP_YIELD_LINK} -sEXPORTED_RUNTIME_METHODS=['ccall','cwrap','UTF8ToString','stringToUTF8','lengthBytesUTF8','dynCall'] -sDEFAULT_LIBRARY_FUNCS_TO_INCLUDE=['\$dynCall'] --bind -L${SYSROOT}/lib ${STUBS_BUILD}/libgit2_stub.a ${STUBS_BUILD}/libcurl_stub.a${APP_STUB_LINK} ${STUBS_BUILD}/libnng_stub.a ${EMBIND_OBJ}" \ -DCMAKE_PREFIX_PATH="${SYSROOT};${WX_BUILD}" \ -DwxWidgets_CONFIG_EXECUTABLE="${WX_BUILD}/wx-config" \ \ diff --git a/web/standalone/src/wasm/libs/cdn-source.ts b/web/standalone/src/wasm/libs/cdn-source.ts index 7d1c217..0f0527e 100644 --- a/web/standalone/src/wasm/libs/cdn-source.ts +++ b/web/standalone/src/wasm/libs/cdn-source.ts @@ -157,14 +157,15 @@ export function cdnLibsSource( }, async getAllItems( libId: string, - ): Promise> { - // One bulk merged read of the whole lib (the IDB cache after cold bundle), - // so the WASM plugin hydrates in a single crossing instead of N gets. + ): Promise> { + // One bulk merged read of the whole lib (the IDB cache after cold bundle). + // "Copy as-is": hand back the raw IDB bytes (no TextDecoder) so the provider + // frames them and the bridge memcpy's them into wasm with no string/JSON + // detour — the plugin slices them straight out of the wasm heap. const stack = await openStack(libId); - const dec = new TextDecoder(); return [...(await stack.readAll())].map(([path, bytes]) => { const { kind, name } = splitPath(path); - return { kind, name, body: dec.decode(bytes) }; + return { kind, name, body: bytes }; }); }, async getItemBody( diff --git a/web/standalone/src/wasm/libs/source.ts b/web/standalone/src/wasm/libs/source.ts index e8c30a7..a381b77 100644 --- a/web/standalone/src/wasm/libs/source.ts +++ b/web/standalone/src/wasm/libs/source.ts @@ -55,7 +55,7 @@ export interface LibsSource { */ getAllItems?( libId: string, - ): Promise>; + ): Promise>; /** * Pre-warm this source's per-lib caches (IndexedDB bundles) WITHOUT touching the * WASM runtime — call it in parallel with the wasm download so the editor's @@ -104,7 +104,9 @@ export type KicadLibsRequest = ( lib: string, arg: string, kind?: string, -) => Promise; + // "bodies" returns a framed Uint8Array (raw item bytes, copied as-is across the + // bridge); every other op returns a string (or null). +) => Promise; declare global { interface Window { @@ -160,12 +162,14 @@ const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); async function fallbackGetAllItems( source: LibsSource, libId: string, -): Promise> { +): Promise> { const items = await source.listItems(libId); - const out: Array<{ kind: string; name: string; body: string }> = []; + const enc = new TextEncoder(); + const out: Array<{ kind: string; name: string; body: Uint8Array }> = []; for (const it of items) { const body = await source.getItemBody(libId, it.kind, it.name); - if (body != null) out.push({ kind: it.kind, name: it.name, body }); + if (body != null) + out.push({ kind: it.kind, name: it.name, body: enc.encode(body) }); } return out; } @@ -246,10 +250,26 @@ export function installLibsProvider( const all = source.getAllItems ? await source.getAllItems(id) : await fallbackGetAllItems(source, id); - const items = all - .filter((i) => i.kind === kind) - .map((i) => ({ name: i.name, body: i.body })); - return JSON.stringify({ [key]: items }); + const items = all.filter((i) => i.kind === kind); + // "Copy as-is" framing: a one-line JSON header (names + UTF-8 byte + // lengths), a newline, then every body's RAW bytes concatenated — no + // JSON escaping. The C++ bridge memcpy's this straight into the wasm + // heap; the plugin parses the small header and slices the bodies, so + // none of the (hundreds of MB of) s-expr gets un-escaped. + const header = JSON.stringify({ + [key]: items.map((i) => ({ name: i.name, len: i.body.length })), + }); + const headerBytes = new TextEncoder().encode(header + "\n"); + const total = + headerBytes.length + items.reduce((n, i) => n + i.body.length, 0); + const out = new Uint8Array(total); + out.set(headerBytes, 0); + let off = headerBytes.length; + for (const i of items) { + out.set(i.body, off); + off += i.body.length; + } + return out; } const items = await source.listItems(id); const names = items diff --git a/web/standalone/src/wasm/libs/synced-source.ts b/web/standalone/src/wasm/libs/synced-source.ts index 8685dbb..fa123a1 100644 --- a/web/standalone/src/wasm/libs/synced-source.ts +++ b/web/standalone/src/wasm/libs/synced-source.ts @@ -54,16 +54,15 @@ export function syncedLibsSource( } }, async getAllItems(): Promise< - Array<{ kind: string; name: string; body: string }> + Array<{ kind: string; name: string; body: Uint8Array }> > { // Bulk merged read across the opaque layer stack (origin + mirror overlay), // top-wins — the mirror invariant readAll() preserves. One crossing, no - // per-item gets. + // per-item gets. "Copy as-is": raw bytes (no TextDecoder) — see cdn-source. const { stack } = await ensure(); - const dec = new TextDecoder(); return [...(await stack.readAll())].map(([path, bytes]) => { const { kind, name } = splitPath(path); - return { kind, name, body: dec.decode(bytes) }; + return { kind, name, body: bytes }; }); }, async getItemBody(_id, kind, name): Promise {