From 157d60f6b8ee9bc6bfd54cead072d072b9ea132e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20T=C3=B6rcsv=C3=A1ri?= Date: Tue, 7 Jul 2026 17:39:52 +0200 Subject: [PATCH] feat(standalone): 8-way parallel lib presync + stable load counters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Library warm-up and the per-item fallback fat-load now run through a shared asyncMap helper (counting semaphore, 8 in flight): the CDN presync's hand-rolled 6-worker pool is replaced and the previously fully-serial listItems+getItemBody fallback is parallelized (order preserved). The load-screen sync line stops flashing: progress reports on lib COMPLETION only (no "current lib" — with 8 in flight there isn't one), the label is a fixed prefix ("Syncing symbol libraries — ") and the counter is space-padded to the total's width in font-mono + whitespace-pre, so 2→3 digit growth doesn't shift the text. Same padding for the fat-load overlay's "x / y libraries" line. Verified in demo mode against the live R2 CDN: 7-8 overlapping manifest/bundle requests, 222 cold symbol libs in a few seconds. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JgThWXtdvrYLK47EDFoGdq --- web/standalone/src/components/WasmTool.tsx | 48 +++++++++++++----- web/standalone/src/lib/async-map.ts | 58 ++++++++++++++++++++++ web/standalone/src/wasm/libs/cdn-source.ts | 23 ++++----- web/standalone/src/wasm/libs/source.ts | 24 +++++---- 4 files changed, 120 insertions(+), 33 deletions(-) create mode 100644 web/standalone/src/lib/async-map.ts diff --git a/web/standalone/src/components/WasmTool.tsx b/web/standalone/src/components/WasmTool.tsx index e81a3bd..aa3690b 100644 --- a/web/standalone/src/components/WasmTool.tsx +++ b/web/standalone/src/components/WasmTool.tsx @@ -619,8 +619,14 @@ export function WasmTool({ // A library item currently being fetched (open/save), for a transient spinner. const [libBusy, setLibBusy] = React.useState(null); // Load-screen pre-sync progress: warming the project's lib bundles into IDB in - // parallel with the wasm download. Null when idle/done. - const [libSync, setLibSync] = React.useState(null); + // parallel with the wasm download. Null when idle/done. Counts only (no + // "current lib") — the fetches run several-at-a-time, so there is no single + // current one, and a fixed label keeps the line still while it ticks. + const [libSync, setLibSync] = React.useState<{ + kind: string; + done: number; + total: number; + } | null>(null); // Last lib error (e.g. a backend 404 on open), shown as a dismissible toast. const [libError, setLibError] = React.useState(null); // Eager whole-library idb→wasm load in flight (the ~tens-of-seconds fat-load on @@ -876,11 +882,9 @@ export function WasmTool({ void source .presync({ kind: libKind, - onProgress: ({ done, total, current }) => + onProgress: ({ done, total }) => setLibSync( - done >= total - ? null - : `Syncing ${libKind}s — ${current} (${done}/${total})`, + done >= total ? null : { kind: libKind, done, total }, ), }) .then(() => setLibSync(null)) @@ -1095,7 +1099,9 @@ export function WasmTool({

{libSync && ( -

{libSync}

+

+ {libSyncLabel(libSync)} +

)}

First load downloads the tool (large) — this can take a moment. @@ -1142,9 +1148,14 @@ export function WasmTool({ }} /> -

- {Math.min(libLoading.done, libLoading.total)} / {libLoading.total}{" "} - libraries + {/* Space-pad `done` to `total`'s width so the centered line + doesn't shift as the count gains a digit. */} +

+ {String(Math.min(libLoading.done, libLoading.total)).padStart( + String(libLoading.total).length, + " ", + )}{" "} + / {libLoading.total} libraries

)} @@ -1189,8 +1200,9 @@ export function WasmTool({ {/* Lib pre-sync still warming IDB after the editor opened (big set) — small unobtrusive indicator so the user knows browsing is still filling in. */} {ready && libSync && ( -
- {libSync} +
+ {" "} + {libSyncLabel(libSync)}
)} @@ -1237,6 +1249,18 @@ export function WasmTool({ ); } +/** + * Fixed-width lib pre-sync line, e.g. "Syncing symbol libraries — 42/208". + * The prefix is constant and `done` is space-padded to `total`'s digit count, + * so the text stays still while the counter ticks (render it in a font-mono + + * whitespace-pre element so the pad spaces hold their width). + */ +function libSyncLabel(s: { kind: string; done: number; total: number }): string { + const total = String(s.total); + const done = String(Math.min(s.done, s.total)).padStart(total.length, " "); + return `Syncing ${s.kind} libraries — ${done}/${total}`; +} + /** * WASM download progress for the boot overlay. A determinate bar when the server * sent a Content-Length the decoded stream agrees with; otherwise just MB so far diff --git a/web/standalone/src/lib/async-map.ts b/web/standalone/src/lib/async-map.ts new file mode 100644 index 0000000..c47da05 --- /dev/null +++ b/web/standalone/src/lib/async-map.ts @@ -0,0 +1,58 @@ +/** + * A counting semaphore: `acquire()` resolves immediately while permits remain, + * otherwise queues (FIFO) until a `release()` hands the permit over. + */ +class Semaphore { + private waiters: (() => void)[] = []; + + private permits: number; + + constructor(max: number) { + this.permits = max; + } + + async acquire(): Promise { + if (this.permits > 0) { + this.permits--; + return Promise.resolve(); + } + return new Promise((resolve) => { + this.waiters.push(resolve); + }); + } + + release(): void { + const next = this.waiters.shift(); + if (next) next(); + else this.permits++; + } +} + +/** + * `Promise.all(array.map(callback))` with at most `concurrencyLimit` callbacks + * in flight at once. Results keep the input order. A callback rejection + * rejects the whole map (wrap the callback if per-item failures should be + * tolerated). + */ +export async function asyncMap( + array: T[], + callback: (item: T, index: number, array: T[]) => Promise, + concurrencyLimit: number, +): Promise { + const semaphore = new Semaphore(concurrencyLimit); + const results: U[] = []; + + const wrappedCallback = async (item: T, index: number, srcArray: T[]) => { + await semaphore.acquire(); + try { + const result = await callback(item, index, srcArray); + results[index] = result; + } finally { + semaphore.release(); + } + }; + + await Promise.all(array.map((item, index) => wrappedCallback(item, index, array))); + + return results; +} diff --git a/web/standalone/src/wasm/libs/cdn-source.ts b/web/standalone/src/wasm/libs/cdn-source.ts index 7bd45f2..b142a40 100644 --- a/web/standalone/src/wasm/libs/cdn-source.ts +++ b/web/standalone/src/wasm/libs/cdn-source.ts @@ -1,4 +1,5 @@ import { SyncStack, type SyncStackOptions } from "@pcbjam/sync-client"; +import { asyncMap } from "@/lib/async-map"; import type { LibInfo, LibItemInfo, LibsSource } from "./source"; /** @@ -122,7 +123,7 @@ export function cdnLibsSource( return (await stack.list()).map((e) => splitPath(e.path)); }, async presync(opts): Promise { - const { kind, concurrency = 6, onProgress, signal } = opts ?? {}; + const { kind, concurrency = 8, onProgress, signal } = opts ?? {}; let m: CdnLibsManifest; try { m = await loadManifest(); @@ -136,25 +137,23 @@ export function cdnLibsSource( const total = libs.length; let done = 0; onProgress?.({ done, total, current: "" }); - // Concurrency-limited pool: each openStack cold-fetches one bundle into IDB + // Concurrency-limited: each openStack cold-fetches one bundle into IDB // (warm ⇒ a cheap manifest diff). Tolerate per-lib failures so one bad lib - // doesn't abort the warm-up. - let idx = 0; - const worker = async (): Promise => { - while (idx < libs.length) { + // doesn't abort the warm-up. Progress is reported on COMPLETION only — + // with several libs in flight there is no single "current" one. + await asyncMap( + libs, + async (lib) => { if (signal?.aborted) return; - const lib = libs[idx++]!; - onProgress?.({ done, total, current: lib.name }); try { await openStack(lib.id); } catch { // best-effort: the lib still loads lazily on demand } + if (signal?.aborted) return; onProgress?.({ done: ++done, total, current: lib.name }); - } - }; - await Promise.all( - Array.from({ length: Math.min(concurrency, total) }, () => worker()), + }, + concurrency, ); }, async getAllItems( diff --git a/web/standalone/src/wasm/libs/source.ts b/web/standalone/src/wasm/libs/source.ts index e5d7b1d..6debba4 100644 --- a/web/standalone/src/wasm/libs/source.ts +++ b/web/standalone/src/wasm/libs/source.ts @@ -1,3 +1,4 @@ +import { asyncMap } from "@/lib/async-map"; import { handleModel3dRequest } from "./models-bridge"; import { libIdFromUri, libUri } from "./uri"; @@ -70,7 +71,7 @@ export interface LibsSource { presync?(opts?: { /** Limit to libs holding this item kind ("symbol" | "footprint"). */ kind?: string; - /** Max concurrent bundle fetches (default 6). */ + /** Max concurrent bundle fetches (default 8). */ concurrency?: number; onProgress?: (p: LibPresyncProgress) => void; signal?: AbortSignal; @@ -189,7 +190,8 @@ const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); * Slow-path "fat list" for sources without a bulk `getAllItems` (the example * backend / per-item remote): just listItems + one getItemBody each. Same N * round-trips as before — but it keeps the single WASM-side code path (the plugin - * always asks for bodies once) working against every source. + * always asks for bodies once) working against every source. The per-item gets + * run 8 at a time (they're independent HTTP round-trips); order is preserved. */ async function fallbackGetAllItems( source: LibsSource, @@ -197,13 +199,17 @@ async function fallbackGetAllItems( ): Promise> { const items = await source.listItems(libId); const enc = new TextEncoder(); - const out: Array<{ kind: string; name: string; body: Uint8Array }> = []; - for (const it of items) { - const body = await source.getItemBody(libId, it.kind, it.name); - if (body != null) - out.push({ kind: it.kind, name: it.name, body: enc.encode(body) }); - } - return out; + const bodies = await asyncMap( + items, + (it) => source.getItemBody(libId, it.kind, it.name).catch(() => null), + 8, + ); + return items.flatMap((it, i) => { + const body = bodies[i]; + return body != null + ? [{ kind: it.kind, name: it.name, body: enc.encode(body) }] + : []; + }); } /** Escape a string for a KiCad s-expr quoted token. */