feat(standalone): 8-way parallel lib presync + stable load counters
Library warm-up and the per-item fallback fat-load now run through a
shared asyncMap helper (counting semaphore, 8 in flight): the CDN
presync's hand-rolled 6-worker pool is replaced and the previously
fully-serial listItems+getItemBody fallback is parallelized (order
preserved).
The load-screen sync line stops flashing: progress reports on lib
COMPLETION only (no "current lib" — with 8 in flight there isn't one),
the label is a fixed prefix ("Syncing symbol libraries — ") and the
counter is space-padded to the total's width in font-mono +
whitespace-pre, so 2→3 digit growth doesn't shift the text. Same
padding for the fat-load overlay's "x / y libraries" line.
Verified in demo mode against the live R2 CDN: 7-8 overlapping
manifest/bundle requests, 222 cold symbol libs in a few seconds.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JgThWXtdvrYLK47EDFoGdq
This commit is contained in:
parent
93e73c590e
commit
157d60f6b8
4 changed files with 120 additions and 33 deletions
|
|
@ -619,8 +619,14 @@ export function WasmTool({
|
||||||
// A library item currently being fetched (open/save), for a transient spinner.
|
// A library item currently being fetched (open/save), for a transient spinner.
|
||||||
const [libBusy, setLibBusy] = React.useState<string | null>(null);
|
const [libBusy, setLibBusy] = React.useState<string | null>(null);
|
||||||
// Load-screen pre-sync progress: warming the project's lib bundles into IDB in
|
// Load-screen pre-sync progress: warming the project's lib bundles into IDB in
|
||||||
// parallel with the wasm download. Null when idle/done.
|
// parallel with the wasm download. Null when idle/done. Counts only (no
|
||||||
const [libSync, setLibSync] = React.useState<string | null>(null);
|
// "current lib") — the fetches run several-at-a-time, so there is no single
|
||||||
|
// current one, and a fixed label keeps the line still while it ticks.
|
||||||
|
const [libSync, setLibSync] = React.useState<{
|
||||||
|
kind: string;
|
||||||
|
done: number;
|
||||||
|
total: number;
|
||||||
|
} | null>(null);
|
||||||
// Last lib error (e.g. a backend 404 on open), shown as a dismissible toast.
|
// Last lib error (e.g. a backend 404 on open), shown as a dismissible toast.
|
||||||
const [libError, setLibError] = React.useState<string | null>(null);
|
const [libError, setLibError] = React.useState<string | null>(null);
|
||||||
// Eager whole-library idb→wasm load in flight (the ~tens-of-seconds fat-load on
|
// Eager whole-library idb→wasm load in flight (the ~tens-of-seconds fat-load on
|
||||||
|
|
@ -876,11 +882,9 @@ export function WasmTool({
|
||||||
void source
|
void source
|
||||||
.presync({
|
.presync({
|
||||||
kind: libKind,
|
kind: libKind,
|
||||||
onProgress: ({ done, total, current }) =>
|
onProgress: ({ done, total }) =>
|
||||||
setLibSync(
|
setLibSync(
|
||||||
done >= total
|
done >= total ? null : { kind: libKind, done, total },
|
||||||
? null
|
|
||||||
: `Syncing ${libKind}s — ${current} (${done}/${total})`,
|
|
||||||
),
|
),
|
||||||
})
|
})
|
||||||
.then(() => setLibSync(null))
|
.then(() => setLibSync(null))
|
||||||
|
|
@ -1095,7 +1099,9 @@ export function WasmTool({
|
||||||
</p>
|
</p>
|
||||||
<DownloadProgress progress={progress} />
|
<DownloadProgress progress={progress} />
|
||||||
{libSync && (
|
{libSync && (
|
||||||
<p className="font-mono text-xs text-emerald-300/90">{libSync}</p>
|
<p className="whitespace-pre font-mono text-xs text-emerald-300/90">
|
||||||
|
{libSyncLabel(libSync)}
|
||||||
|
</p>
|
||||||
)}
|
)}
|
||||||
<p className="font-mono text-xs text-white/40">
|
<p className="font-mono text-xs text-white/40">
|
||||||
First load downloads the tool (large) — this can take a moment.
|
First load downloads the tool (large) — this can take a moment.
|
||||||
|
|
@ -1142,9 +1148,14 @@ export function WasmTool({
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
<p className="mt-1 text-center font-mono text-[11px] text-white/50">
|
{/* Space-pad `done` to `total`'s width so the centered line
|
||||||
{Math.min(libLoading.done, libLoading.total)} / {libLoading.total}{" "}
|
doesn't shift as the count gains a digit. */}
|
||||||
libraries
|
<p className="mt-1 whitespace-pre text-center font-mono text-[11px] text-white/50">
|
||||||
|
{String(Math.min(libLoading.done, libLoading.total)).padStart(
|
||||||
|
String(libLoading.total).length,
|
||||||
|
" ",
|
||||||
|
)}{" "}
|
||||||
|
/ {libLoading.total} libraries
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
@ -1189,8 +1200,9 @@ export function WasmTool({
|
||||||
{/* Lib pre-sync still warming IDB after the editor opened (big set) — small
|
{/* Lib pre-sync still warming IDB after the editor opened (big set) — small
|
||||||
unobtrusive indicator so the user knows browsing is still filling in. */}
|
unobtrusive indicator so the user knows browsing is still filling in. */}
|
||||||
{ready && libSync && (
|
{ready && libSync && (
|
||||||
<div className="pointer-events-none absolute bottom-9 left-3 z-20 flex items-center gap-2 rounded bg-black/80 px-3 py-1.5 text-xs text-emerald-200">
|
<div className="pointer-events-none absolute bottom-9 left-3 z-20 flex items-center gap-2 rounded bg-black/80 px-3 py-1.5 font-mono text-xs text-emerald-200">
|
||||||
<Loader2 className="animate-spin" size={14} /> {libSync}
|
<Loader2 className="animate-spin" size={14} />{" "}
|
||||||
|
<span className="whitespace-pre">{libSyncLabel(libSync)}</span>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
|
@ -1237,6 +1249,18 @@ export function WasmTool({
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fixed-width lib pre-sync line, e.g. "Syncing symbol libraries — 42/208".
|
||||||
|
* The prefix is constant and `done` is space-padded to `total`'s digit count,
|
||||||
|
* so the text stays still while the counter ticks (render it in a font-mono +
|
||||||
|
* whitespace-pre element so the pad spaces hold their width).
|
||||||
|
*/
|
||||||
|
function libSyncLabel(s: { kind: string; done: number; total: number }): string {
|
||||||
|
const total = String(s.total);
|
||||||
|
const done = String(Math.min(s.done, s.total)).padStart(total.length, " ");
|
||||||
|
return `Syncing ${s.kind} libraries — ${done}/${total}`;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* WASM download progress for the boot overlay. A determinate bar when the server
|
* WASM download progress for the boot overlay. A determinate bar when the server
|
||||||
* sent a Content-Length the decoded stream agrees with; otherwise just MB so far
|
* sent a Content-Length the decoded stream agrees with; otherwise just MB so far
|
||||||
|
|
|
||||||
58
web/standalone/src/lib/async-map.ts
Normal file
58
web/standalone/src/lib/async-map.ts
Normal file
|
|
@ -0,0 +1,58 @@
|
||||||
|
/**
|
||||||
|
* A counting semaphore: `acquire()` resolves immediately while permits remain,
|
||||||
|
* otherwise queues (FIFO) until a `release()` hands the permit over.
|
||||||
|
*/
|
||||||
|
class Semaphore {
|
||||||
|
private waiters: (() => void)[] = [];
|
||||||
|
|
||||||
|
private permits: number;
|
||||||
|
|
||||||
|
constructor(max: number) {
|
||||||
|
this.permits = max;
|
||||||
|
}
|
||||||
|
|
||||||
|
async acquire(): Promise<void> {
|
||||||
|
if (this.permits > 0) {
|
||||||
|
this.permits--;
|
||||||
|
return Promise.resolve();
|
||||||
|
}
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
this.waiters.push(resolve);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
release(): void {
|
||||||
|
const next = this.waiters.shift();
|
||||||
|
if (next) next();
|
||||||
|
else this.permits++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* `Promise.all(array.map(callback))` with at most `concurrencyLimit` callbacks
|
||||||
|
* in flight at once. Results keep the input order. A callback rejection
|
||||||
|
* rejects the whole map (wrap the callback if per-item failures should be
|
||||||
|
* tolerated).
|
||||||
|
*/
|
||||||
|
export async function asyncMap<T, U>(
|
||||||
|
array: T[],
|
||||||
|
callback: (item: T, index: number, array: T[]) => Promise<U>,
|
||||||
|
concurrencyLimit: number,
|
||||||
|
): Promise<U[]> {
|
||||||
|
const semaphore = new Semaphore(concurrencyLimit);
|
||||||
|
const results: U[] = [];
|
||||||
|
|
||||||
|
const wrappedCallback = async (item: T, index: number, srcArray: T[]) => {
|
||||||
|
await semaphore.acquire();
|
||||||
|
try {
|
||||||
|
const result = await callback(item, index, srcArray);
|
||||||
|
results[index] = result;
|
||||||
|
} finally {
|
||||||
|
semaphore.release();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
await Promise.all(array.map((item, index) => wrappedCallback(item, index, array)));
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import { SyncStack, type SyncStackOptions } from "@pcbjam/sync-client";
|
import { SyncStack, type SyncStackOptions } from "@pcbjam/sync-client";
|
||||||
|
import { asyncMap } from "@/lib/async-map";
|
||||||
import type { LibInfo, LibItemInfo, LibsSource } from "./source";
|
import type { LibInfo, LibItemInfo, LibsSource } from "./source";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -122,7 +123,7 @@ export function cdnLibsSource(
|
||||||
return (await stack.list()).map((e) => splitPath(e.path));
|
return (await stack.list()).map((e) => splitPath(e.path));
|
||||||
},
|
},
|
||||||
async presync(opts): Promise<void> {
|
async presync(opts): Promise<void> {
|
||||||
const { kind, concurrency = 6, onProgress, signal } = opts ?? {};
|
const { kind, concurrency = 8, onProgress, signal } = opts ?? {};
|
||||||
let m: CdnLibsManifest;
|
let m: CdnLibsManifest;
|
||||||
try {
|
try {
|
||||||
m = await loadManifest();
|
m = await loadManifest();
|
||||||
|
|
@ -136,25 +137,23 @@ export function cdnLibsSource(
|
||||||
const total = libs.length;
|
const total = libs.length;
|
||||||
let done = 0;
|
let done = 0;
|
||||||
onProgress?.({ done, total, current: "" });
|
onProgress?.({ done, total, current: "" });
|
||||||
// Concurrency-limited pool: each openStack cold-fetches one bundle into IDB
|
// Concurrency-limited: each openStack cold-fetches one bundle into IDB
|
||||||
// (warm ⇒ a cheap manifest diff). Tolerate per-lib failures so one bad lib
|
// (warm ⇒ a cheap manifest diff). Tolerate per-lib failures so one bad lib
|
||||||
// doesn't abort the warm-up.
|
// doesn't abort the warm-up. Progress is reported on COMPLETION only —
|
||||||
let idx = 0;
|
// with several libs in flight there is no single "current" one.
|
||||||
const worker = async (): Promise<void> => {
|
await asyncMap(
|
||||||
while (idx < libs.length) {
|
libs,
|
||||||
|
async (lib) => {
|
||||||
if (signal?.aborted) return;
|
if (signal?.aborted) return;
|
||||||
const lib = libs[idx++]!;
|
|
||||||
onProgress?.({ done, total, current: lib.name });
|
|
||||||
try {
|
try {
|
||||||
await openStack(lib.id);
|
await openStack(lib.id);
|
||||||
} catch {
|
} catch {
|
||||||
// best-effort: the lib still loads lazily on demand
|
// best-effort: the lib still loads lazily on demand
|
||||||
}
|
}
|
||||||
|
if (signal?.aborted) return;
|
||||||
onProgress?.({ done: ++done, total, current: lib.name });
|
onProgress?.({ done: ++done, total, current: lib.name });
|
||||||
}
|
},
|
||||||
};
|
concurrency,
|
||||||
await Promise.all(
|
|
||||||
Array.from({ length: Math.min(concurrency, total) }, () => worker()),
|
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
async getAllItems(
|
async getAllItems(
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
import { asyncMap } from "@/lib/async-map";
|
||||||
import { handleModel3dRequest } from "./models-bridge";
|
import { handleModel3dRequest } from "./models-bridge";
|
||||||
import { libIdFromUri, libUri } from "./uri";
|
import { libIdFromUri, libUri } from "./uri";
|
||||||
|
|
||||||
|
|
@ -70,7 +71,7 @@ export interface LibsSource {
|
||||||
presync?(opts?: {
|
presync?(opts?: {
|
||||||
/** Limit to libs holding this item kind ("symbol" | "footprint"). */
|
/** Limit to libs holding this item kind ("symbol" | "footprint"). */
|
||||||
kind?: string;
|
kind?: string;
|
||||||
/** Max concurrent bundle fetches (default 6). */
|
/** Max concurrent bundle fetches (default 8). */
|
||||||
concurrency?: number;
|
concurrency?: number;
|
||||||
onProgress?: (p: LibPresyncProgress) => void;
|
onProgress?: (p: LibPresyncProgress) => void;
|
||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
|
|
@ -189,7 +190,8 @@ const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||||
* Slow-path "fat list" for sources without a bulk `getAllItems` (the example
|
* Slow-path "fat list" for sources without a bulk `getAllItems` (the example
|
||||||
* backend / per-item remote): just listItems + one getItemBody each. Same N
|
* backend / per-item remote): just listItems + one getItemBody each. Same N
|
||||||
* round-trips as before — but it keeps the single WASM-side code path (the plugin
|
* round-trips as before — but it keeps the single WASM-side code path (the plugin
|
||||||
* always asks for bodies once) working against every source.
|
* always asks for bodies once) working against every source. The per-item gets
|
||||||
|
* run 8 at a time (they're independent HTTP round-trips); order is preserved.
|
||||||
*/
|
*/
|
||||||
async function fallbackGetAllItems(
|
async function fallbackGetAllItems(
|
||||||
source: LibsSource,
|
source: LibsSource,
|
||||||
|
|
@ -197,13 +199,17 @@ async function fallbackGetAllItems(
|
||||||
): Promise<Array<{ kind: string; name: string; body: Uint8Array }>> {
|
): Promise<Array<{ kind: string; name: string; body: Uint8Array }>> {
|
||||||
const items = await source.listItems(libId);
|
const items = await source.listItems(libId);
|
||||||
const enc = new TextEncoder();
|
const enc = new TextEncoder();
|
||||||
const out: Array<{ kind: string; name: string; body: Uint8Array }> = [];
|
const bodies = await asyncMap(
|
||||||
for (const it of items) {
|
items,
|
||||||
const body = await source.getItemBody(libId, it.kind, it.name);
|
(it) => source.getItemBody(libId, it.kind, it.name).catch(() => null),
|
||||||
if (body != null)
|
8,
|
||||||
out.push({ kind: it.kind, name: it.name, body: enc.encode(body) });
|
);
|
||||||
}
|
return items.flatMap((it, i) => {
|
||||||
return out;
|
const body = bodies[i];
|
||||||
|
return body != null
|
||||||
|
? [{ kind: it.kind, name: it.name, body: enc.encode(body) }]
|
||||||
|
: [];
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Escape a string for a KiCad s-expr quoted token. */
|
/** Escape a string for a KiCad s-expr quoted token. */
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue