feat(libs): parallel s-expr parse + mimalloc + copy-as-is transport — symbol chooser cold 274s→44s

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

kicad

@@ -1 +1 @@
-Subproject commit a010915df55ed19df227e9367bdb5fb93e287a15
+Subproject commit dc3219aa69ce98ca478800e5f09767d47f6b8370

scripts/kicad/build-kicad-target.sh

@@ -444,7 +444,7 @@ emcmake cmake "${KICAD_DIR}" \
     -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
     -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS} -Xclang -fno-pch-timestamp -pthread -sUSE_ZLIB=1 -DKICAD_USE_PLATFORM_WASM=1${DIAG_DEFINES} -I${SYSROOT}/include -I${STUBS_DIR} -include ${STUBS_DIR}/char_traits_uint16_workaround.h" \
     -DCMAKE_C_FLAGS="${EXTRA_FLAGS} -pthread -sUSE_ZLIB=1 -I${SYSROOT}/include -I${STUBS_DIR}" \
-    -DCMAKE_EXE_LINKER_FLAGS="${LINKER_DEBUG_FLAGS} -pthread -sUSE_ZLIB=1 -sASYNCIFY=1 -sDYNCALLS=1 -sASYNCIFY_STACK_SIZE=65536 -sUSE_PTHREADS=1 -sPTHREAD_POOL_SIZE='navigator.hardwareConcurrency' -sPTHREAD_POOL_SIZE_STRICT=0 -sALLOW_MEMORY_GROWTH=1 -sINITIAL_MEMORY=256MB -sMAXIMUM_MEMORY=4GB -sMAX_WEBGL_VERSION=2 ${GL3D_LINK_FLAGS} ${NANOSLEEP_YIELD_LINK} -sEXPORTED_RUNTIME_METHODS=['ccall','cwrap','UTF8ToString','stringToUTF8','lengthBytesUTF8','dynCall'] -sDEFAULT_LIBRARY_FUNCS_TO_INCLUDE=['\$dynCall'] --bind -L${SYSROOT}/lib ${STUBS_BUILD}/libgit2_stub.a ${STUBS_BUILD}/libcurl_stub.a${APP_STUB_LINK} ${STUBS_BUILD}/libnng_stub.a ${EMBIND_OBJ}" \
+    -DCMAKE_EXE_LINKER_FLAGS="${LINKER_DEBUG_FLAGS} -pthread -sUSE_ZLIB=1 -sASYNCIFY=1 -sDYNCALLS=1 -sASYNCIFY_STACK_SIZE=65536 -sUSE_PTHREADS=1 -sMALLOC=mimalloc -sPTHREAD_POOL_SIZE='navigator.hardwareConcurrency' -sPTHREAD_POOL_SIZE_STRICT=0 -sALLOW_MEMORY_GROWTH=1 -sINITIAL_MEMORY=256MB -sMAXIMUM_MEMORY=4GB -sMAX_WEBGL_VERSION=2 ${GL3D_LINK_FLAGS} ${NANOSLEEP_YIELD_LINK} -sEXPORTED_RUNTIME_METHODS=['ccall','cwrap','UTF8ToString','stringToUTF8','lengthBytesUTF8','dynCall'] -sDEFAULT_LIBRARY_FUNCS_TO_INCLUDE=['\$dynCall'] --bind -L${SYSROOT}/lib ${STUBS_BUILD}/libgit2_stub.a ${STUBS_BUILD}/libcurl_stub.a${APP_STUB_LINK} ${STUBS_BUILD}/libnng_stub.a ${EMBIND_OBJ}" \
     -DCMAKE_PREFIX_PATH="${SYSROOT};${WX_BUILD}" \
     -DwxWidgets_CONFIG_EXECUTABLE="${WX_BUILD}/wx-config" \
     \

web/standalone/src/wasm/libs/cdn-source.ts

@@ -157,14 +157,15 @@ export function cdnLibsSource(
     },
     async getAllItems(
       libId: string,
-    ): Promise<Array<{ kind: string; name: string; body: string }>> {
+    ): Promise<Array<{ kind: string; name: string; body: Uint8Array }>> {
-      // One bulk merged read of the whole lib (the IDB cache after cold bundle),
+      // One bulk merged read of the whole lib (the IDB cache after cold bundle).
-      // so the WASM plugin hydrates in a single crossing instead of N gets.
+      // "Copy as-is": hand back the raw IDB bytes (no TextDecoder) so the provider
+      // frames them and the bridge memcpy's them into wasm with no string/JSON
+      // detour — the plugin slices them straight out of the wasm heap.
       const stack = await openStack(libId);
-      const dec = new TextDecoder();
       return [...(await stack.readAll())].map(([path, bytes]) => {
         const { kind, name } = splitPath(path);
-        return { kind, name, body: dec.decode(bytes) };
+        return { kind, name, body: bytes };
       });
     },
     async getItemBody(

web/standalone/src/wasm/libs/source.ts

@@ -55,7 +55,7 @@ export interface LibsSource {
    */
   getAllItems?(
     libId: string,
-  ): Promise<Array<{ kind: string; name: string; body: string }>>;
+  ): Promise<Array<{ kind: string; name: string; body: Uint8Array }>>;
   /**
    * Pre-warm this source's per-lib caches (IndexedDB bundles) WITHOUT touching the
    * WASM runtime — call it in parallel with the wasm download so the editor's
@@ -104,7 +104,9 @@ export type KicadLibsRequest = (
   lib: string,
   arg: string,
   kind?: string,
-) => Promise<string | null>;
+  // "bodies" returns a framed Uint8Array (raw item bytes, copied as-is across the
+  // bridge); every other op returns a string (or null).
+) => Promise<string | Uint8Array | null>;
 
 declare global {
   interface Window {
@@ -160,12 +162,14 @@ const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
 async function fallbackGetAllItems(
   source: LibsSource,
   libId: string,
-): Promise<Array<{ kind: string; name: string; body: string }>> {
+): Promise<Array<{ kind: string; name: string; body: Uint8Array }>> {
   const items = await source.listItems(libId);
-  const out: Array<{ kind: string; name: string; body: string }> = [];
+  const enc = new TextEncoder();
+  const out: Array<{ kind: string; name: string; body: Uint8Array }> = [];
   for (const it of items) {
     const body = await source.getItemBody(libId, it.kind, it.name);
-    if (body != null) out.push({ kind: it.kind, name: it.name, body });
+    if (body != null)
+      out.push({ kind: it.kind, name: it.name, body: enc.encode(body) });
   }
   return out;
 }
@@ -246,10 +250,26 @@ export function installLibsProvider(
             const all = source.getAllItems
               ? await source.getAllItems(id)
               : await fallbackGetAllItems(source, id);
-            const items = all
+            const items = all.filter((i) => i.kind === kind);
-              .filter((i) => i.kind === kind)
+            // "Copy as-is" framing: a one-line JSON header (names + UTF-8 byte
-              .map((i) => ({ name: i.name, body: i.body }));
+            // lengths), a newline, then every body's RAW bytes concatenated — no
-            return JSON.stringify({ [key]: items });
+            // JSON escaping. The C++ bridge memcpy's this straight into the wasm
+            // heap; the plugin parses the small header and slices the bodies, so
+            // none of the (hundreds of MB of) s-expr gets un-escaped.
+            const header = JSON.stringify({
+              [key]: items.map((i) => ({ name: i.name, len: i.body.length })),
+            });
+            const headerBytes = new TextEncoder().encode(header + "\n");
+            const total =
+              headerBytes.length + items.reduce((n, i) => n + i.body.length, 0);
+            const out = new Uint8Array(total);
+            out.set(headerBytes, 0);
+            let off = headerBytes.length;
+            for (const i of items) {
+              out.set(i.body, off);
+              off += i.body.length;
+            }
+            return out;
           }
           const items = await source.listItems(id);
           const names = items

web/standalone/src/wasm/libs/synced-source.ts

@@ -54,16 +54,15 @@ export function syncedLibsSource(
       }
     },
     async getAllItems(): Promise<
-      Array<{ kind: string; name: string; body: string }>
+      Array<{ kind: string; name: string; body: Uint8Array }>
     > {
       // Bulk merged read across the opaque layer stack (origin + mirror overlay),
       // top-wins — the mirror invariant readAll() preserves. One crossing, no
-      // per-item gets.
+      // per-item gets. "Copy as-is": raw bytes (no TextDecoder) — see cdn-source.
       const { stack } = await ensure();
-      const dec = new TextDecoder();
       return [...(await stack.readAll())].map(([path, bytes]) => {
         const { kind, name } = splitPath(path);
-        return { kind, name, body: dec.decode(bytes) };
+        return { kind, name, body: bytes };
       });
     },
     async getItemBody(_id, kind, name): Promise<string | null> {