feat(standalone): 8-way parallel lib presync + stable load counters
Library warm-up and the per-item fallback fat-load now run through a
shared asyncMap helper (counting semaphore, 8 in flight): the CDN
presync's hand-rolled 6-worker pool is replaced and the previously
fully-serial listItems+getItemBody fallback is parallelized (order
preserved).
The load-screen sync line stops flashing: progress reports on lib
COMPLETION only (no "current lib" — with 8 in flight there isn't one),
the label is a fixed prefix ("Syncing symbol libraries — ") and the
counter is space-padded to the total's width in font-mono +
whitespace-pre, so 2→3 digit growth doesn't shift the text. Same
padding for the fat-load overlay's "x / y libraries" line.
Verified in demo mode against the live R2 CDN: 7-8 overlapping
manifest/bundle requests, 222 cold symbol libs in a few seconds.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JgThWXtdvrYLK47EDFoGdq
This commit is contained in:
parent
93e73c590e
commit
157d60f6b8
4 changed files with 120 additions and 33 deletions
|
|
@ -619,8 +619,14 @@ export function WasmTool({
|
|||
// A library item currently being fetched (open/save), for a transient spinner.
|
||||
const [libBusy, setLibBusy] = React.useState<string | null>(null);
|
||||
// Load-screen pre-sync progress: warming the project's lib bundles into IDB in
|
||||
// parallel with the wasm download. Null when idle/done.
|
||||
const [libSync, setLibSync] = React.useState<string | null>(null);
|
||||
// parallel with the wasm download. Null when idle/done. Counts only (no
|
||||
// "current lib") — the fetches run several-at-a-time, so there is no single
|
||||
// current one, and a fixed label keeps the line still while it ticks.
|
||||
const [libSync, setLibSync] = React.useState<{
|
||||
kind: string;
|
||||
done: number;
|
||||
total: number;
|
||||
} | null>(null);
|
||||
// Last lib error (e.g. a backend 404 on open), shown as a dismissible toast.
|
||||
const [libError, setLibError] = React.useState<string | null>(null);
|
||||
// Eager whole-library idb→wasm load in flight (the ~tens-of-seconds fat-load on
|
||||
|
|
@ -876,11 +882,9 @@ export function WasmTool({
|
|||
void source
|
||||
.presync({
|
||||
kind: libKind,
|
||||
onProgress: ({ done, total, current }) =>
|
||||
onProgress: ({ done, total }) =>
|
||||
setLibSync(
|
||||
done >= total
|
||||
? null
|
||||
: `Syncing ${libKind}s — ${current} (${done}/${total})`,
|
||||
done >= total ? null : { kind: libKind, done, total },
|
||||
),
|
||||
})
|
||||
.then(() => setLibSync(null))
|
||||
|
|
@ -1095,7 +1099,9 @@ export function WasmTool({
|
|||
</p>
|
||||
<DownloadProgress progress={progress} />
|
||||
{libSync && (
|
||||
<p className="font-mono text-xs text-emerald-300/90">{libSync}</p>
|
||||
<p className="whitespace-pre font-mono text-xs text-emerald-300/90">
|
||||
{libSyncLabel(libSync)}
|
||||
</p>
|
||||
)}
|
||||
<p className="font-mono text-xs text-white/40">
|
||||
First load downloads the tool (large) — this can take a moment.
|
||||
|
|
@ -1142,9 +1148,14 @@ export function WasmTool({
|
|||
}}
|
||||
/>
|
||||
</div>
|
||||
<p className="mt-1 text-center font-mono text-[11px] text-white/50">
|
||||
{Math.min(libLoading.done, libLoading.total)} / {libLoading.total}{" "}
|
||||
libraries
|
||||
{/* Space-pad `done` to `total`'s width so the centered line
|
||||
doesn't shift as the count gains a digit. */}
|
||||
<p className="mt-1 whitespace-pre text-center font-mono text-[11px] text-white/50">
|
||||
{String(Math.min(libLoading.done, libLoading.total)).padStart(
|
||||
String(libLoading.total).length,
|
||||
" ",
|
||||
)}{" "}
|
||||
/ {libLoading.total} libraries
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
|
@ -1189,8 +1200,9 @@ export function WasmTool({
|
|||
{/* Lib pre-sync still warming IDB after the editor opened (big set) — small
|
||||
unobtrusive indicator so the user knows browsing is still filling in. */}
|
||||
{ready && libSync && (
|
||||
<div className="pointer-events-none absolute bottom-9 left-3 z-20 flex items-center gap-2 rounded bg-black/80 px-3 py-1.5 text-xs text-emerald-200">
|
||||
<Loader2 className="animate-spin" size={14} /> {libSync}
|
||||
<div className="pointer-events-none absolute bottom-9 left-3 z-20 flex items-center gap-2 rounded bg-black/80 px-3 py-1.5 font-mono text-xs text-emerald-200">
|
||||
<Loader2 className="animate-spin" size={14} />{" "}
|
||||
<span className="whitespace-pre">{libSyncLabel(libSync)}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
|
|
@ -1237,6 +1249,18 @@ export function WasmTool({
|
|||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fixed-width lib pre-sync line, e.g. "Syncing symbol libraries — 42/208".
|
||||
* The prefix is constant and `done` is space-padded to `total`'s digit count,
|
||||
* so the text stays still while the counter ticks (render it in a font-mono +
|
||||
* whitespace-pre element so the pad spaces hold their width).
|
||||
*/
|
||||
function libSyncLabel(s: { kind: string; done: number; total: number }): string {
|
||||
const total = String(s.total);
|
||||
const done = String(Math.min(s.done, s.total)).padStart(total.length, " ");
|
||||
return `Syncing ${s.kind} libraries — ${done}/${total}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* WASM download progress for the boot overlay. A determinate bar when the server
|
||||
* sent a Content-Length the decoded stream agrees with; otherwise just MB so far
|
||||
|
|
|
|||
58
web/standalone/src/lib/async-map.ts
Normal file
58
web/standalone/src/lib/async-map.ts
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
/**
|
||||
* A counting semaphore: `acquire()` resolves immediately while permits remain,
|
||||
* otherwise queues (FIFO) until a `release()` hands the permit over.
|
||||
*/
|
||||
class Semaphore {
|
||||
private waiters: (() => void)[] = [];
|
||||
|
||||
private permits: number;
|
||||
|
||||
constructor(max: number) {
|
||||
this.permits = max;
|
||||
}
|
||||
|
||||
async acquire(): Promise<void> {
|
||||
if (this.permits > 0) {
|
||||
this.permits--;
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
this.waiters.push(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
release(): void {
|
||||
const next = this.waiters.shift();
|
||||
if (next) next();
|
||||
else this.permits++;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* `Promise.all(array.map(callback))` with at most `concurrencyLimit` callbacks
|
||||
* in flight at once. Results keep the input order. A callback rejection
|
||||
* rejects the whole map (wrap the callback if per-item failures should be
|
||||
* tolerated).
|
||||
*/
|
||||
export async function asyncMap<T, U>(
|
||||
array: T[],
|
||||
callback: (item: T, index: number, array: T[]) => Promise<U>,
|
||||
concurrencyLimit: number,
|
||||
): Promise<U[]> {
|
||||
const semaphore = new Semaphore(concurrencyLimit);
|
||||
const results: U[] = [];
|
||||
|
||||
const wrappedCallback = async (item: T, index: number, srcArray: T[]) => {
|
||||
await semaphore.acquire();
|
||||
try {
|
||||
const result = await callback(item, index, srcArray);
|
||||
results[index] = result;
|
||||
} finally {
|
||||
semaphore.release();
|
||||
}
|
||||
};
|
||||
|
||||
await Promise.all(array.map((item, index) => wrappedCallback(item, index, array)));
|
||||
|
||||
return results;
|
||||
}
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
import { SyncStack, type SyncStackOptions } from "@pcbjam/sync-client";
|
||||
import { asyncMap } from "@/lib/async-map";
|
||||
import type { LibInfo, LibItemInfo, LibsSource } from "./source";
|
||||
|
||||
/**
|
||||
|
|
@ -122,7 +123,7 @@ export function cdnLibsSource(
|
|||
return (await stack.list()).map((e) => splitPath(e.path));
|
||||
},
|
||||
async presync(opts): Promise<void> {
|
||||
const { kind, concurrency = 6, onProgress, signal } = opts ?? {};
|
||||
const { kind, concurrency = 8, onProgress, signal } = opts ?? {};
|
||||
let m: CdnLibsManifest;
|
||||
try {
|
||||
m = await loadManifest();
|
||||
|
|
@ -136,25 +137,23 @@ export function cdnLibsSource(
|
|||
const total = libs.length;
|
||||
let done = 0;
|
||||
onProgress?.({ done, total, current: "" });
|
||||
// Concurrency-limited pool: each openStack cold-fetches one bundle into IDB
|
||||
// Concurrency-limited: each openStack cold-fetches one bundle into IDB
|
||||
// (warm ⇒ a cheap manifest diff). Tolerate per-lib failures so one bad lib
|
||||
// doesn't abort the warm-up.
|
||||
let idx = 0;
|
||||
const worker = async (): Promise<void> => {
|
||||
while (idx < libs.length) {
|
||||
// doesn't abort the warm-up. Progress is reported on COMPLETION only —
|
||||
// with several libs in flight there is no single "current" one.
|
||||
await asyncMap(
|
||||
libs,
|
||||
async (lib) => {
|
||||
if (signal?.aborted) return;
|
||||
const lib = libs[idx++]!;
|
||||
onProgress?.({ done, total, current: lib.name });
|
||||
try {
|
||||
await openStack(lib.id);
|
||||
} catch {
|
||||
// best-effort: the lib still loads lazily on demand
|
||||
}
|
||||
if (signal?.aborted) return;
|
||||
onProgress?.({ done: ++done, total, current: lib.name });
|
||||
}
|
||||
};
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(concurrency, total) }, () => worker()),
|
||||
},
|
||||
concurrency,
|
||||
);
|
||||
},
|
||||
async getAllItems(
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { asyncMap } from "@/lib/async-map";
|
||||
import { handleModel3dRequest } from "./models-bridge";
|
||||
import { libIdFromUri, libUri } from "./uri";
|
||||
|
||||
|
|
@ -70,7 +71,7 @@ export interface LibsSource {
|
|||
presync?(opts?: {
|
||||
/** Limit to libs holding this item kind ("symbol" | "footprint"). */
|
||||
kind?: string;
|
||||
/** Max concurrent bundle fetches (default 6). */
|
||||
/** Max concurrent bundle fetches (default 8). */
|
||||
concurrency?: number;
|
||||
onProgress?: (p: LibPresyncProgress) => void;
|
||||
signal?: AbortSignal;
|
||||
|
|
@ -189,7 +190,8 @@ const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|||
* Slow-path "fat list" for sources without a bulk `getAllItems` (the example
|
||||
* backend / per-item remote): just listItems + one getItemBody each. Same N
|
||||
* round-trips as before — but it keeps the single WASM-side code path (the plugin
|
||||
* always asks for bodies once) working against every source.
|
||||
* always asks for bodies once) working against every source. The per-item gets
|
||||
* run 8 at a time (they're independent HTTP round-trips); order is preserved.
|
||||
*/
|
||||
async function fallbackGetAllItems(
|
||||
source: LibsSource,
|
||||
|
|
@ -197,13 +199,17 @@ async function fallbackGetAllItems(
|
|||
): Promise<Array<{ kind: string; name: string; body: Uint8Array }>> {
|
||||
const items = await source.listItems(libId);
|
||||
const enc = new TextEncoder();
|
||||
const out: Array<{ kind: string; name: string; body: Uint8Array }> = [];
|
||||
for (const it of items) {
|
||||
const body = await source.getItemBody(libId, it.kind, it.name);
|
||||
if (body != null)
|
||||
out.push({ kind: it.kind, name: it.name, body: enc.encode(body) });
|
||||
}
|
||||
return out;
|
||||
const bodies = await asyncMap(
|
||||
items,
|
||||
(it) => source.getItemBody(libId, it.kind, it.name).catch(() => null),
|
||||
8,
|
||||
);
|
||||
return items.flatMap((it, i) => {
|
||||
const body = bodies[i];
|
||||
return body != null
|
||||
? [{ kind: it.kind, name: it.name, body: enc.encode(body) }]
|
||||
: [];
|
||||
});
|
||||
}
|
||||
|
||||
/** Escape a string for a KiCad s-expr quoted token. */
|
||||
|
|
|
|||
Loading…
Reference in a new issue