feat(standalone): 8-way parallel lib presync + stable load counters

Library warm-up and the per-item fallback fat-load now run through a
shared asyncMap helper (counting semaphore, 8 in flight): the CDN
presync's hand-rolled 6-worker pool is replaced and the previously
fully-serial listItems+getItemBody fallback is parallelized (order
preserved).

The load-screen sync line stops flashing: progress reports on lib
COMPLETION only (no "current lib" — with 8 in flight there isn't one),
the label is a fixed prefix ("Syncing symbol libraries — ") and the
counter is space-padded to the total's width in font-mono +
whitespace-pre, so 2→3 digit growth doesn't shift the text. Same
padding for the fat-load overlay's "x / y libraries" line.

Verified in demo mode against the live R2 CDN: 7-8 overlapping
manifest/bundle requests, 222 cold symbol libs in a few seconds.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JgThWXtdvrYLK47EDFoGdq
This commit is contained in:
Gergő Törcsvári 2026-07-07 17:39:52 +02:00
commit 157d60f6b8
No known key found for this signature in database
GPG key ID: 8E75F2CDE64E5322
4 changed files with 120 additions and 33 deletions

View file

@ -619,8 +619,14 @@ export function WasmTool({
// A library item currently being fetched (open/save), for a transient spinner.
const [libBusy, setLibBusy] = React.useState<string | null>(null);
// Load-screen pre-sync progress: warming the project's lib bundles into IDB in
// parallel with the wasm download. Null when idle/done.
const [libSync, setLibSync] = React.useState<string | null>(null);
// parallel with the wasm download. Null when idle/done. Counts only (no
// "current lib") — the fetches run several-at-a-time, so there is no single
// current one, and a fixed label keeps the line still while it ticks.
const [libSync, setLibSync] = React.useState<{
kind: string;
done: number;
total: number;
} | null>(null);
// Last lib error (e.g. a backend 404 on open), shown as a dismissible toast.
const [libError, setLibError] = React.useState<string | null>(null);
// Eager whole-library idb→wasm load in flight (the ~tens-of-seconds fat-load on
@ -876,11 +882,9 @@ export function WasmTool({
void source
.presync({
kind: libKind,
onProgress: ({ done, total, current }) =>
onProgress: ({ done, total }) =>
setLibSync(
done >= total
? null
: `Syncing ${libKind}s — ${current} (${done}/${total})`,
done >= total ? null : { kind: libKind, done, total },
),
})
.then(() => setLibSync(null))
@ -1095,7 +1099,9 @@ export function WasmTool({
</p>
<DownloadProgress progress={progress} />
{libSync && (
<p className="font-mono text-xs text-emerald-300/90">{libSync}</p>
<p className="whitespace-pre font-mono text-xs text-emerald-300/90">
{libSyncLabel(libSync)}
</p>
)}
<p className="font-mono text-xs text-white/40">
First load downloads the tool (large) — this can take a moment.
@ -1142,9 +1148,14 @@ export function WasmTool({
}}
/>
</div>
<p className="mt-1 text-center font-mono text-[11px] text-white/50">
{Math.min(libLoading.done, libLoading.total)} / {libLoading.total}{" "}
libraries
{/* Space-pad `done` to `total`'s width so the centered line
doesn't shift as the count gains a digit. */}
<p className="mt-1 whitespace-pre text-center font-mono text-[11px] text-white/50">
{String(Math.min(libLoading.done, libLoading.total)).padStart(
String(libLoading.total).length,
" ",
)}{" "}
/ {libLoading.total} libraries
</p>
</div>
)}
@ -1189,8 +1200,9 @@ export function WasmTool({
{/* Lib pre-sync still warming IDB after the editor opened (big set) — small
unobtrusive indicator so the user knows browsing is still filling in. */}
{ready && libSync && (
<div className="pointer-events-none absolute bottom-9 left-3 z-20 flex items-center gap-2 rounded bg-black/80 px-3 py-1.5 text-xs text-emerald-200">
<Loader2 className="animate-spin" size={14} /> {libSync}
<div className="pointer-events-none absolute bottom-9 left-3 z-20 flex items-center gap-2 rounded bg-black/80 px-3 py-1.5 font-mono text-xs text-emerald-200">
<Loader2 className="animate-spin" size={14} />{" "}
<span className="whitespace-pre">{libSyncLabel(libSync)}</span>
</div>
)}
@ -1237,6 +1249,18 @@ export function WasmTool({
);
}
/**
* Fixed-width lib pre-sync line, e.g. "Syncing symbol libraries — 42/208".
* The prefix is constant and `done` is space-padded to `total`'s digit count,
* so the text stays still while the counter ticks (render it in a font-mono +
* whitespace-pre element so the pad spaces hold their width).
*/
function libSyncLabel(s: { kind: string; done: number; total: number }): string {
const total = String(s.total);
const done = String(Math.min(s.done, s.total)).padStart(total.length, " ");
return `Syncing ${s.kind} libraries — ${done}/${total}`;
}
/**
* WASM download progress for the boot overlay. A determinate bar when the server
* sent a Content-Length the decoded stream agrees with; otherwise just MB so far

View file

@ -0,0 +1,58 @@
/**
* A counting semaphore: `acquire()` resolves immediately while permits remain,
* otherwise queues (FIFO) until a `release()` hands the permit over.
*/
class Semaphore {
private waiters: (() => void)[] = [];
private permits: number;
constructor(max: number) {
this.permits = max;
}
async acquire(): Promise<void> {
if (this.permits > 0) {
this.permits--;
return Promise.resolve();
}
return new Promise((resolve) => {
this.waiters.push(resolve);
});
}
release(): void {
const next = this.waiters.shift();
if (next) next();
else this.permits++;
}
}
/**
* `Promise.all(array.map(callback))` with at most `concurrencyLimit` callbacks
* in flight at once. Results keep the input order. A callback rejection
* rejects the whole map (wrap the callback if per-item failures should be
* tolerated).
*/
export async function asyncMap<T, U>(
array: T[],
callback: (item: T, index: number, array: T[]) => Promise<U>,
concurrencyLimit: number,
): Promise<U[]> {
const semaphore = new Semaphore(concurrencyLimit);
const results: U[] = [];
const wrappedCallback = async (item: T, index: number, srcArray: T[]) => {
await semaphore.acquire();
try {
const result = await callback(item, index, srcArray);
results[index] = result;
} finally {
semaphore.release();
}
};
await Promise.all(array.map((item, index) => wrappedCallback(item, index, array)));
return results;
}

View file

@ -1,4 +1,5 @@
import { SyncStack, type SyncStackOptions } from "@pcbjam/sync-client";
import { asyncMap } from "@/lib/async-map";
import type { LibInfo, LibItemInfo, LibsSource } from "./source";
/**
@ -122,7 +123,7 @@ export function cdnLibsSource(
return (await stack.list()).map((e) => splitPath(e.path));
},
async presync(opts): Promise<void> {
const { kind, concurrency = 6, onProgress, signal } = opts ?? {};
const { kind, concurrency = 8, onProgress, signal } = opts ?? {};
let m: CdnLibsManifest;
try {
m = await loadManifest();
@ -136,25 +137,23 @@ export function cdnLibsSource(
const total = libs.length;
let done = 0;
onProgress?.({ done, total, current: "" });
// Concurrency-limited pool: each openStack cold-fetches one bundle into IDB
// Concurrency-limited: each openStack cold-fetches one bundle into IDB
// (warm ⇒ a cheap manifest diff). Tolerate per-lib failures so one bad lib
// doesn't abort the warm-up.
let idx = 0;
const worker = async (): Promise<void> => {
while (idx < libs.length) {
// doesn't abort the warm-up. Progress is reported on COMPLETION only —
// with several libs in flight there is no single "current" one.
await asyncMap(
libs,
async (lib) => {
if (signal?.aborted) return;
const lib = libs[idx++]!;
onProgress?.({ done, total, current: lib.name });
try {
await openStack(lib.id);
} catch {
// best-effort: the lib still loads lazily on demand
}
if (signal?.aborted) return;
onProgress?.({ done: ++done, total, current: lib.name });
}
};
await Promise.all(
Array.from({ length: Math.min(concurrency, total) }, () => worker()),
},
concurrency,
);
},
async getAllItems(

View file

@ -1,3 +1,4 @@
import { asyncMap } from "@/lib/async-map";
import { handleModel3dRequest } from "./models-bridge";
import { libIdFromUri, libUri } from "./uri";
@ -70,7 +71,7 @@ export interface LibsSource {
presync?(opts?: {
/** Limit to libs holding this item kind ("symbol" | "footprint"). */
kind?: string;
/** Max concurrent bundle fetches (default 6). */
/** Max concurrent bundle fetches (default 8). */
concurrency?: number;
onProgress?: (p: LibPresyncProgress) => void;
signal?: AbortSignal;
@ -189,7 +190,8 @@ const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
* Slow-path "fat list" for sources without a bulk `getAllItems` (the example
* backend / per-item remote): just listItems + one getItemBody each. Same N
* round-trips as before — but it keeps the single WASM-side code path (the plugin
* always asks for bodies once) working against every source.
* always asks for bodies once) working against every source. The per-item gets
* run 8 at a time (they're independent HTTP round-trips); order is preserved.
*/
async function fallbackGetAllItems(
source: LibsSource,
@ -197,13 +199,17 @@ async function fallbackGetAllItems(
): Promise<Array<{ kind: string; name: string; body: Uint8Array }>> {
const items = await source.listItems(libId);
const enc = new TextEncoder();
const out: Array<{ kind: string; name: string; body: Uint8Array }> = [];
for (const it of items) {
const body = await source.getItemBody(libId, it.kind, it.name);
if (body != null)
out.push({ kind: it.kind, name: it.name, body: enc.encode(body) });
}
return out;
const bodies = await asyncMap(
items,
(it) => source.getItemBody(libId, it.kind, it.name).catch(() => null),
8,
);
return items.flatMap((it, i) => {
const body = bodies[i];
return body != null
? [{ kind: it.kind, name: it.name, body: enc.encode(body) }]
: [];
});
}
/** Escape a string for a KiCad s-expr quoted token. */