AI_Assistant/pcbjam
Watch
1
0
Fork
You've already forked pcbjam
 0
Code Issues Pull requests Projects Releases 4 Packages Wiki Activity Actions 2

perf(editor): one batched lib resolve, and ydoc file downloads

Browse source 
Measured on a real Leonardo load against the local platform stack:

  per-lib sync-stack POSTs : 174-200  ->  0
  batch  sync-stacks POSTs :       0  ->  1   (216 stacks in one request)
  file GETs negotiating ydoc:      0  ->  107 of 133

Lib resolve: syncedScopeLibsSource now resolves every stack up front through
the shared paged batch client and feeds them to the per-lib sources, which skip
their own POST on a hit. Best-effort throughout — a failure (older backend
without the route, a network blip) leaves the map empty and every lib resolves
exactly as before, so this can only remove requests, never break a load. A
batched `null` is recorded as "backend says unresolvable" so a stale pin isn't
then retried one-by-one.

File downloads: fetchFileBytes sends Accept: application/x-pcbjam-ydoc and
converts client-side with the converters shared already exports, moving the
per-request materialize cost off the metered Worker. A backend that doesn't
negotiate answers with text and the branch never fires; a ydoc we fail to
convert re-fetches without negotiating rather than making the file
undownloadable.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0137pGo8W7asomGUTRMB7RzM
This commit is contained in:
Gergő Törcsvári 2026-07-31 11:31:54 +02:00
commit ad777bcbba
No known key found for this signature in database
GPG key ID: 8E75F2CDE64E5322
3 changed files with 110 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

27
web/standalone/src/lib/project-source.ts
View file

@ -1,8 +1,12 @@
import {
DEMO_SCOPE,
docToFile,
isYdocResponse,
type Project,
type ProjectFile,
type ProjectWithFiles,
YDOC_CONTENT_TYPE,
ydocUpdateToKicadDoc,
} from "@pcbjam/shared";
import {
API_BASE_URL,
@ -83,11 +87,32 @@ function remoteProjectSource(): ProjectSource {
// credentials: session-cookie auth (see contract-client.ts). The static
// gallery fetches below stay credential-less — a CDN's wildcard CORS
// rejects credentialed requests.
//
// Accept a ydoc (backend-wire.ts): for a file whose collab room has been
// opened, the backend would otherwise re-derive the KiCad text on EVERY
// download — a Y -> KicadDoc traversal plus an s-expr build that measured
// ~2.0 s on a ~2300-item board and exceeded the Workers CPU limit in
// production. We hold the same converters, so we ask for the raw update
// and do it here, where CPU is free. A backend that doesn't negotiate
// simply answers with text and the branch below never fires.
const res = await fetch(fileUrl(slug, relPath), {
credentials: "include",
headers: { accept: `${YDOC_CONTENT_TYPE}, */*` },
});
if (!res.ok) throw new Error(`download failed (${res.status}): ${relPath}`);
return new Uint8Array(await res.arrayBuffer());
const bytes = new Uint8Array(await res.arrayBuffer());
if (!isYdocResponse(res)) return bytes;
try {
return new TextEncoder().encode(docToFile(ydocUpdateToKicadDoc(bytes)));
} catch (err) {
// A ydoc we can't convert must not make the file undownloadable: retry
// without negotiating and let the backend materialize it as before.
const plain = await fetch(fileUrl(slug, relPath), { credentials: "include" });
if (!plain.ok) {
throw new Error(`download failed (${plain.status}): ${relPath} (${String(err)})`);
}
return new Uint8Array(await plain.arrayBuffer());
}
},
async uploadFileBytes(slug, relPath, bytes) {
const name = relPath.split("/").pop() ?? relPath;

97
web/standalone/src/wasm/libs/synced-source.ts
View file

@ -1,4 +1,11 @@
import { PROJECT_HEADER, SCOPE_HEADER, USER_HEADER } from "@pcbjam/shared";
import {
fetchSyncStacks,
PROJECT_HEADER,
SCOPE_HEADER,
SYNC_STACKS_BATCH_MAX,
type SyncStackDescriptor,
USER_HEADER,
} from "@pcbjam/shared";
import {
peekNamespaces,
SyncStack,
@ -35,6 +42,13 @@ export function syncedLibsSource(
user?: string;
project?: string;
log?: (msg: string) => void;
/**
* Already-resolved stack for this lib, from the scope-level batch resolve
* (see syncedScopeLibsSource). When it answers, the per-lib POST is skipped
* entirely — that request is the one a board load makes 156-200 times.
* Returning undefined falls back to resolving this lib on its own.
*/
stackFor?: (libId: string) => SyncStackDescriptor | null | undefined;
/** Test seams (default: global fetch / IDB stores / real WebSockets). */
fetchImpl?: typeof fetch;
storeFactory?: (namespace: string) => LayerStore;
@ -200,6 +214,7 @@ async function resolveAndOpen(
scope: string;
user?: string;
project?: string;
stackFor?: (libId: string) => SyncStackDescriptor | null | undefined;
fetchImpl?: typeof fetch;
storeFactory?: (namespace: string) => LayerStore;
channelFactory?: ChannelFactory;
@ -212,18 +227,26 @@ async function resolveAndOpen(
...(opts.user ? { [USER_HEADER]: opts.user } : {}),
...(opts.project ? { [PROJECT_HEADER]: opts.project } : {}),
};
const res = await baseFetch(
`${opts.apiBase}/api/scopes/${encodeURIComponent(opts.scope)}/libs/${encodeURIComponent(libId)}/sync-stack`,
// credentials: session-cookie auth, here and on every layer fetch below —
// live layers are membership-gated by the API worker per request.
{ method: "POST", headers, credentials: "include" },
);
if (!res.ok) throw new Error(`sync-stack resolve failed: HTTP ${res.status}`);
const body = (await res.json()) as {
lib: { id: string; name: string };
layers: LayerDescriptor[];
};
log(`[synced] resolved ${body.layers.length} layer(s) for lib ${libId}`);
// Batch-resolved already? Then this lib costs no request at all. `null` is a
// deliberate "the backend says this lib does not resolve" and must not be
// retried per-lib; only `undefined` (not in the batch) falls through.
const prefetched = opts.stackFor?.(libId);
if (prefetched === null) throw new Error(`sync-stack resolve failed: unknown lib ${libId}`);
let body: SyncStackDescriptor;
if (prefetched) {
body = prefetched;
log(`[synced] resolved ${body.layers.length} layer(s) for lib ${libId} (batched)`);
} else {
const res = await baseFetch(
`${opts.apiBase}/api/scopes/${encodeURIComponent(opts.scope)}/libs/${encodeURIComponent(libId)}/sync-stack`,
// credentials: session-cookie auth, here and on every layer fetch below —
// live layers are membership-gated by the API worker per request.
{ method: "POST", headers, credentials: "include" },
);
if (!res.ok) throw new Error(`sync-stack resolve failed: HTTP ${res.status}`);
body = (await res.json()) as SyncStackDescriptor;
log(`[synced] resolved ${body.layers.length} layer(s) for lib ${libId}`);
}
// The descriptors carry no bearer token — live-layer HTTP ops authenticate
// with the session cookie, so the stack's fetch must send credentials. The
@ -279,15 +302,58 @@ export function syncedScopeLibsSource(
},
): LibsSource {
const perLib = new Map<string, LibsSource>();
// Stacks resolved in bulk by `prefetchStacks`. A hit means the per-lib source
// makes NO resolve request; `null` records "backend says unresolvable" so a
// stale pin isn't retried one-by-one. Misses simply fall back per-lib, which
// is also what happens against a backend predating the batch route.
const batchedStacks = new Map<string, SyncStackDescriptor | null>();
const forLib = (libId: string): LibsSource => {
let src = perLib.get(libId);
if (!src) {
src = syncedLibsSource(libId, opts);
src = syncedLibsSource(libId, {
...opts,
stackFor: (id) => (batchedStacks.has(id) ? batchedStacks.get(id) : undefined),
});
perLib.set(libId, src);
}
return src;
};
/**
* Resolve every lib's stack up front, in a handful of paged requests instead
* of one per lib. A board load resolves 156-200 libraries, and each per-lib
* POST is a separate serverless invocation before any library CONTENT is
* fetched — the dominant cost of the whole phase.
*
* Best-effort by design: any failure (older backend without the route, a
* network blip) leaves the map empty and every lib resolves the old way, so
* this can only ever remove requests, never break the load.
*/
async function prefetchStacks(libs: LibInfo[]): Promise<void> {
const missing = libs.map((l) => l.id).filter((id) => !batchedStacks.has(id));
if (missing.length === 0) return;
const headers: Record<string, string> = {
[SCOPE_HEADER]: opts.scope,
...(opts.user ? { [USER_HEADER]: opts.user } : {}),
...(opts.project ? { [PROJECT_HEADER]: opts.project } : {}),
};
try {
const resolved = await fetchSyncStacks({
url: `${opts.apiBase}/api/scopes/${encodeURIComponent(opts.scope)}/libs/sync-stacks`,
libIds: missing,
...(opts.fetchImpl ? { fetchImpl: opts.fetchImpl } : {}),
headers,
});
for (const [id, stack] of resolved) batchedStacks.set(id, stack);
opts.log?.(
`[synced] batch-resolved ${resolved.size} lib stack(s) in ` +
`${Math.ceil(missing.length / SYNC_STACKS_BATCH_MAX)} request(s)`,
);
} catch (err) {
opts.log?.(`[synced] batch resolve unavailable (${String(err)}) — falling back per-lib`);
}
}
return {
listLibs: (kind) => remote.listLibs(kind),
createLib: remote.createLib?.bind(remote),
@ -341,6 +407,9 @@ export function syncedScopeLibsSource(
const total = libs.length;
let done = 0;
presyncOpts?.onProgress?.({ done, total, current: "libraries" });
// One batched resolve for the whole set before the per-lib fan-out, so
// the workers below open stacks without a request each.
if (!presyncOpts?.signal?.aborted) await prefetchStacks(libs);
const concurrency = presyncOpts?.concurrency ?? 8;
const queue = [...libs];
const worker = async (): Promise<void> => {