feat: load-path rework steps 1-2 — project-file IDB cache + immutable CDN lib layers

Step 1: warm platform loads stop re-downloading every project file. New
project-file-cache.ts (raw IDB, best-effort) keyed by the listing's
revision:updatedAt validator — the pair, not revision alone, because the resave
equivalent-body swap keeps revision but touches updatedAt. Listing row threaded
through fetchFileBytes(slug, path, meta?); ydoc-backed and revision-0 files are
never cached; prune runs on every fresh listing.

Step 2: cdnLibsSource marks its tag-pinned static layers immutable — warm demo
loads skip all ~155 per-lib manifest GETs (bumps pcbjam-shared for the
LayerDescriptor.immutable flag).

docs/features/load-path-rework/0001 steps 1-2.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VLSht9cadprtT2mhynawWu
This commit is contained in:
Gergő Törcsvári 2026-08-17 20:26:44 +02:00
commit 89b4a7b61c
No known key found for this signature in database
GPG key ID: 8E75F2CDE64E5322
8 changed files with 456 additions and 9 deletions

@ -1 +1 @@
Subproject commit e1df6fa1d633d26b7982a32fedaa01bd0d661324
Subproject commit c83c27be6cf1dcdc3554b9426fc4af06bbc9889f

View file

@ -1,4 +1,4 @@
import type { DriftReportBody, Project } from "@pcbjam/shared";
import type { DriftReportBody, Project, ProjectFile } from "@pcbjam/shared";
import { useQuery } from "@tanstack/react-query";
import { API_BASE_URL, currentScope, libsSourceConfig } from "./config";
import { client } from "./contract-client";
@ -72,12 +72,18 @@ export function useProject(slug: string) {
});
}
/** File bytes from the active source (backend stream, or the static CDN gallery). */
/**
* File bytes from the active source (backend stream, or the static CDN
* gallery). Pass the file's row from the current listing as `meta` when you
* have it it lets the remote source answer from its local body cache when
* the listed version vouches for the bytes (project-file-cache.ts).
*/
export function fetchFileBytes(
slug: string,
relPath: string,
meta?: ProjectFile,
): Promise<Uint8Array> {
return projectSource().fetchFileBytes(slug, relPath);
return projectSource().fetchFileBytes(slug, relPath, meta);
}
/**

View file

@ -0,0 +1,58 @@
import { describe, expect, it } from "vitest";
import type { ProjectFile } from "@pcbjam/shared";
import {
fileCacheValidator,
pruneProjectFileCache,
readCachedFileBytes,
writeCachedFileBytes,
} from "./project-file-cache";
const base: ProjectFile = {
id: "0b7a4bfa-0000-5000-8000-000000000001",
projectId: "0b7a4bfa-0000-5000-8000-000000000002",
path: "boards/main.kicad_pcb",
size: 10,
contentType: "text/plain",
revision: 3,
createdAt: "2026-08-01T00:00:00.000Z",
updatedAt: "2026-08-15T12:00:00.000Z",
};
describe("fileCacheValidator", () => {
it("derives revision:updatedAt for a published, non-collab file", () => {
expect(fileCacheValidator(base)).toBe("3:2026-08-15T12:00:00.000Z");
});
it("changes when only updatedAt moves (equivalent resave body swap)", () => {
// The backend's resave normalization swaps the stored body WITHOUT
// advancing revision — updatedAt is the only signal, so it must be
// part of the validator or the cache would serve pre-normalization bytes.
const swapped = { ...base, updatedAt: "2026-08-16T09:30:00.000Z" };
expect(fileCacheValidator(swapped)).not.toBe(fileCacheValidator(base));
});
it("rejects revision 0 and absent revision (no published CAS row)", () => {
expect(fileCacheValidator({ ...base, revision: 0 })).toBeNull();
expect(fileCacheValidator({ ...base, revision: undefined })).toBeNull();
});
it("rejects ydoc-backed files — their bytes move without a row change", () => {
expect(fileCacheValidator({ ...base, hasYdoc: true })).toBeNull();
expect(fileCacheValidator({ ...base, isLive: true })).toBeNull();
});
});
describe("without IndexedDB (node, private mode)", () => {
// The vitest environment is node: no indexedDB global. Every operation must
// degrade to a no-op — a broken cache may never break a load.
it("read resolves null, write and prune resolve without throwing", async () => {
expect(typeof indexedDB).toBe("undefined");
await expect(readCachedFileBytes("p", "a", "1:x")).resolves.toBeNull();
await expect(
writeCachedFileBytes("p", "a", "1:x", new Uint8Array([1])),
).resolves.toBeUndefined();
await expect(
pruneProjectFileCache("p", new Map([["a", "1:x"]])),
).resolves.toBeUndefined();
});
});

View file

@ -0,0 +1,165 @@
import type { ProjectFile } from "@pcbjam/shared";
/**
* Browser-local cache of REMOTE project-file bodies, keyed by the file's
* server-authoritative version so a warm load fetches only files that
* actually changed instead of re-downloading the whole project
* (docs/features/load-path-rework/0001, step 1).
*
* Validator, not TTL: an entry is served only when its stored validator equals
* the one derived from the CURRENT project listing (`fileCacheValidator`), so
* staleness is impossible as long as the listing is fresh and the listing is
* exactly what the editor just fetched to enumerate the files.
*
* The validator is `revision:updatedAt`, NOT revision alone: the backend's
* resave job can swap a file's stored body for a semantically-equivalent
* normalization WITHOUT advancing the client-visible revision
* (writeProjectFile's `equivalentToStorageKey` path) but every publish,
* equivalent or not, touches `updatedAt`, so the pair changes whenever the
* bytes can have.
*
* Ydoc-backed files (`hasYdoc`/`isLive`) are never cached: their served bytes
* come from room materialization, which moves without any file-row change.
* Revision 0 / absent means no published CAS row (collab-only docs, minimal
* backends) also uncacheable.
*
* Raw IndexedDB, mirroring idb-project-store.ts: one out-of-line-keyed store,
* key `projectId + NUL + path` so a project's entries form one contiguous key
* range that can't capture another project's. Every operation is best-effort
* a missing/broken IndexedDB (private mode, node tests) degrades to plain
* fetching, never to a failed load.
*/
const DB_NAME = "pcbjam-project-file-cache";
const DB_VERSION = 1;
const FILES = "files"; // key: projectId + SEP + path -> CacheRecord
const SEP = "\u0000"; // NUL: sorts below every id char (collision-proof ranges)
const MAX_CHAR = "\uffff"; // largest UTF-16 code unit -> key-range upper bound
interface CacheRecord {
validator: string;
bytes: Uint8Array;
cachedAt: string;
}
/**
* The cache validator for a listed file, or null when the file is uncacheable
* (no CAS revision, or ydoc-backed so bytes move independently of the row).
*/
export function fileCacheValidator(meta: ProjectFile): string | null {
if (!meta.revision || meta.revision <= 0) return null;
if (meta.hasYdoc || meta.isLive) return null;
return `${meta.revision}:${meta.updatedAt}`;
}
function hasIdb(): boolean {
return typeof indexedDB !== "undefined";
}
function reqDone<T>(req: IDBRequest<T>): Promise<T> {
return new Promise((resolve, reject) => {
req.onsuccess = () => resolve(req.result);
req.onerror = () => reject(req.error);
});
}
function txDone(tx: IDBTransaction): Promise<void> {
return new Promise((resolve, reject) => {
tx.oncomplete = () => resolve();
tx.onerror = () => reject(tx.error);
tx.onabort = () => reject(tx.error);
});
}
let dbP: Promise<IDBDatabase> | null = null;
function openDb(): Promise<IDBDatabase> {
return (dbP ??= new Promise((resolve, reject) => {
const req = indexedDB.open(DB_NAME, DB_VERSION);
req.onupgradeneeded = () => {
const db = req.result;
if (!db.objectStoreNames.contains(FILES)) db.createObjectStore(FILES);
};
req.onsuccess = () => resolve(req.result);
req.onerror = () => {
dbP = null; // a later call may retry (e.g. transient quota/open failure)
reject(req.error);
};
}));
}
function key(projectId: string, path: string): string {
return `${projectId}${SEP}${path}`;
}
function rangeFor(projectId: string): IDBKeyRange {
return IDBKeyRange.bound(`${projectId}${SEP}`, `${projectId}${SEP}${MAX_CHAR}`);
}
/** Cached bytes for (projectId, path) iff stored under exactly `validator`. */
export async function readCachedFileBytes(
projectId: string,
path: string,
validator: string,
): Promise<Uint8Array | null> {
if (!hasIdb()) return null;
try {
const db = await openDb();
const rec = await reqDone<CacheRecord | undefined>(
db.transaction(FILES, "readonly").objectStore(FILES).get(key(projectId, path)),
);
if (!rec || rec.validator !== validator) return null;
return rec.bytes;
} catch {
return null;
}
}
/** Best-effort store; failures (quota, private mode) are swallowed. */
export async function writeCachedFileBytes(
projectId: string,
path: string,
validator: string,
bytes: Uint8Array,
): Promise<void> {
if (!hasIdb()) return;
try {
const db = await openDb();
const tx = db.transaction(FILES, "readwrite");
const rec: CacheRecord = { validator, bytes, cachedAt: new Date().toISOString() };
tx.objectStore(FILES).put(rec, key(projectId, path));
await txDone(tx);
} catch {
// best-effort
}
}
/**
* Drop this project's entries that the current listing no longer vouches for:
* deleted paths and stale validators (superseded revisions, files that turned
* ydoc-backed). Called after every fresh project listing, fire-and-forget.
*/
export async function pruneProjectFileCache(
projectId: string,
valid: ReadonlyMap<string, string>,
): Promise<void> {
if (!hasIdb()) return;
try {
const db = await openDb();
const tx = db.transaction(FILES, "readwrite");
const store = tx.objectStore(FILES);
const prefixLen = projectId.length + SEP.length;
await new Promise<void>((resolve, reject) => {
const cur = store.openCursor(rangeFor(projectId));
cur.onsuccess = () => {
const c = cur.result;
if (!c) return resolve();
const path = String(c.key).slice(prefixLen);
const rec = c.value as CacheRecord;
if (valid.get(path) !== rec.validator) c.delete();
c.continue();
};
cur.onerror = () => reject(cur.error);
});
await txDone(tx);
} catch {
// best-effort
}
}

View file

@ -0,0 +1,165 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { ProjectFile } from "@pcbjam/shared";
// Real validator logic inside the mocked cache module — only the IDB-touching
// functions are replaced (the node test env has no IndexedDB anyway).
import { fileCacheValidator } from "./project-file-cache";
const PROJECT_ID = "0b7a4bfa-0000-5000-8000-0000000000aa";
const file = (over: Partial<ProjectFile> = {}): ProjectFile => ({
id: "0b7a4bfa-0000-5000-8000-0000000000ab",
projectId: PROJECT_ID,
path: "boards/main.kicad_pcb",
size: 3,
contentType: "text/plain",
revision: 2,
createdAt: "2026-08-01T00:00:00.000Z",
updatedAt: "2026-08-15T12:00:00.000Z",
...over,
});
function cacheMock() {
return {
fileCacheValidator,
readCachedFileBytes: vi.fn(async () => null as Uint8Array | null),
writeCachedFileBytes: vi.fn(async () => {}),
pruneProjectFileCache: vi.fn(async () => {}),
};
}
// project-source reads config at import time; mock it fresh then dynamic-import
// (same pattern as project-source.test.ts, but selecting the REMOTE source).
async function loadRemote(cache: ReturnType<typeof cacheMock>, client?: unknown) {
vi.resetModules();
vi.doMock("@/lib/config", () => ({
API_BASE_URL: "http://localhost:3050",
PROJECT_SOURCE_KIND: "remote",
PROJECT_MANIFEST_URL: undefined,
LOCAL_PROJECTS_ENABLED: false,
userSlug: () => "test-user",
currentScope: () => "team-a",
}));
vi.doMock("./project-file-cache", () => cache);
if (client) vi.doMock("./contract-client", () => ({ client }));
return (await import("./project-source")).projectSource;
}
afterEach(() => {
vi.unstubAllGlobals();
vi.restoreAllMocks();
vi.resetModules();
});
const plainResponse = (bytes: Uint8Array) => ({
ok: true,
headers: { get: (h: string) => (h === "content-type" ? "text/plain" : null) },
arrayBuffer: async () => bytes.buffer,
});
describe("remote source file-body cache", () => {
it("serves a cache hit without touching the network", async () => {
const cache = cacheMock();
const cached = new Uint8Array([9, 9, 9]);
cache.readCachedFileBytes.mockResolvedValue(cached);
const fetchMock = vi.fn();
vi.stubGlobal("fetch", fetchMock);
const src = (await loadRemote(cache))();
const meta = file();
const got = await src.fetchFileBytes("proj", meta.path, meta);
expect(got).toBe(cached);
expect(fetchMock).not.toHaveBeenCalled();
expect(cache.readCachedFileBytes).toHaveBeenCalledWith(
PROJECT_ID,
meta.path,
fileCacheValidator(meta),
);
});
it("on a miss, fetches and stores under the listing's validator", async () => {
const cache = cacheMock();
const bytes = new Uint8Array([1, 2, 3]);
vi.stubGlobal("fetch", vi.fn(async () => plainResponse(bytes)));
const src = (await loadRemote(cache))();
const meta = file();
const got = await src.fetchFileBytes("proj", meta.path, meta);
expect(Array.from(got)).toEqual([1, 2, 3]);
expect(cache.writeCachedFileBytes).toHaveBeenCalledWith(
PROJECT_ID,
meta.path,
fileCacheValidator(meta),
got,
);
});
it("never caches a ydoc-materialized response, even with cacheable meta", async () => {
// Listing said no ydoc, but a room appeared between listing and fetch: the
// server answers as a ydoc. The garbage update fails conversion → the
// plain-retry fallback serves the bytes, and NOTHING is written to the
// cache (those bytes move without a file-row change).
const cache = cacheMock();
const fetchMock = vi
.fn()
.mockResolvedValueOnce({
ok: true,
headers: {
get: (h: string) =>
h === "content-type" ? "application/x-pcbjam-ydoc" : null,
},
arrayBuffer: async () => new Uint8Array([0xde, 0xad]).buffer,
})
.mockResolvedValueOnce(plainResponse(new Uint8Array([7])));
vi.stubGlobal("fetch", fetchMock);
const src = (await loadRemote(cache))();
const got = await src.fetchFileBytes("proj", "a.kicad_sch", file());
expect(Array.from(got)).toEqual([7]);
expect(fetchMock).toHaveBeenCalledTimes(2);
expect(cache.writeCachedFileBytes).not.toHaveBeenCalled();
});
it("skips the cache entirely when no listing meta is passed", async () => {
const cache = cacheMock();
vi.stubGlobal(
"fetch",
vi.fn(async () => plainResponse(new Uint8Array([5]))),
);
const src = (await loadRemote(cache))();
await src.fetchFileBytes("proj", "x.txt");
expect(cache.readCachedFileBytes).not.toHaveBeenCalled();
expect(cache.writeCachedFileBytes).not.toHaveBeenCalled();
});
it("prunes to the fresh listing on getProject (cacheable rows only)", async () => {
const cache = cacheMock();
const cacheable = file();
const collabOnly = file({
path: "sheets/child.kicad_sch",
revision: 0,
hasYdoc: true,
});
const client = {
getProject: vi.fn(async () => ({
status: 200,
body: {
project: { id: PROJECT_ID, scopeId: "s", slug: "proj" },
files: [cacheable, collabOnly],
},
})),
};
const src = (await loadRemote(cache, client))();
await src.getProject("proj");
expect(cache.pruneProjectFileCache).toHaveBeenCalledWith(
PROJECT_ID,
new Map([[cacheable.path, fileCacheValidator(cacheable)!]]),
);
});
});

View file

@ -17,6 +17,12 @@ import {
} from "./config";
import { client } from "./contract-client";
import { idbProjectStore, type LocalProjectStore } from "./idb-project-store";
import {
fileCacheValidator,
pruneProjectFileCache,
readCachedFileBytes,
writeCachedFileBytes,
} from "./project-file-cache";
import {
SOURCE_DESCRIPTORS,
type SourceDescriptor,
@ -42,7 +48,17 @@ export interface ProjectSource {
readonly readOnly: boolean;
listProjects(): Promise<Project[]>;
getProject(slug: string): Promise<ProjectWithFiles>;
fetchFileBytes(slug: string, relPath: string): Promise<Uint8Array>;
/**
* `meta` is the file's row from the CURRENT project listing, when the caller
* has one. It lets a source serve the bytes from its local body cache when
* the listed version vouches for them (project-file-cache.ts); without it
* every call is a plain fetch. Purely an optimization callers may omit it.
*/
fetchFileBytes(
slug: string,
relPath: string,
meta?: ProjectFile,
): Promise<Uint8Array>;
/** Present only on writable sources; absent ⇒ read-only (download on save). */
uploadFileBytes?(
slug: string,
@ -81,9 +97,24 @@ function remoteProjectSource(): ProjectSource {
});
if (res.status === 404) throw new Error("project not found");
if (res.status !== 200) throw new Error("failed to load project");
// Fresh listing = fresh cache truth: drop cached bodies this listing no
// longer vouches for (deleted paths, superseded revisions). Best-effort.
const valid = new Map<string, string>();
for (const f of res.body.files) {
const v = fileCacheValidator(f);
if (v) valid.set(f.path, v);
}
void pruneProjectFileCache(res.body.project.id, valid);
return res.body;
},
async fetchFileBytes(slug, relPath) {
async fetchFileBytes(slug, relPath, meta) {
// Serve from the browser-local body cache when the listing's version
// vouches for it — a warm load then fetches only files that changed.
const validator = meta ? fileCacheValidator(meta) : null;
if (validator && meta) {
const hit = await readCachedFileBytes(meta.projectId, relPath, validator);
if (hit) return hit;
}
// credentials: session-cookie auth (see contract-client.ts). The static
// gallery fetches below stay credential-less — a CDN's wildcard CORS
// rejects credentialed requests.
@ -101,7 +132,15 @@ function remoteProjectSource(): ProjectSource {
});
if (!res.ok) throw new Error(`download failed (${res.status}): ${relPath}`);
const bytes = new Uint8Array(await res.arrayBuffer());
if (!isYdocResponse(res)) return bytes;
if (!isYdocResponse(res)) {
// The ydoc check re-guards the listing's hasYdoc: a room created
// between listing and fetch answers as ydoc, which must not be cached
// (its bytes move without a file-row change).
if (validator && meta) {
void writeCachedFileBytes(meta.projectId, relPath, validator, bytes);
}
return bytes;
}
try {
return new TextEncoder().encode(docToFile(ydocUpdateToKicadDoc(bytes)));
} catch (err) {
@ -246,7 +285,8 @@ function compositeProjectSource(
return [...a, ...b.filter((p) => !localSlugs.has(p.slug))];
},
getProject: (slug) => route(slug).then((s) => s.getProject(slug)),
fetchFileBytes: (slug, p) => route(slug).then((s) => s.fetchFileBytes(slug, p)),
fetchFileBytes: (slug, p, meta) =>
route(slug).then((s) => s.fetchFileBytes(slug, p, meta)),
uploadFileBytes: async (slug, p, bytes) => {
const s = await route(slug);
if (s.uploadFileBytes) return s.uploadFileBytes(slug, p, bytes);

View file

@ -1,3 +1,4 @@
import { useMemo } from "react";
import { useParams, useSearchParams } from "react-router-dom";
import { parseToolParam, toolForFile, type Tool } from "@pcbjam/shared";
import {
@ -31,6 +32,12 @@ export function ToolPage() {
const { data, isLoading, error } = useProject(slug);
const { data: sourceDescriptor } = useSourceDescriptor(slug);
// Listing rows by path, handed to fetchFileBytes so the remote source can
// serve unchanged files from the local body cache (project-file-cache.ts).
const filesByPath = useMemo(
() => new Map((data?.files ?? []).map((f) => [f.path, f])),
[data],
);
if (!tool) {
return (
@ -79,7 +86,9 @@ export function ToolPage() {
projectId={data.project.id}
files={data.files}
targetPath={targetPath}
fetchBytes={(relPath) => fetchFileBytes(slug, relPath)}
fetchBytes={(relPath) =>
fetchFileBytes(slug, relPath, filesByPath.get(relPath))
}
saveBytes={
readOnly
? undefined

View file

@ -111,6 +111,10 @@ export function cdnLibsSource(
namespace: `kicad:${m.tag}:${libId}`,
kind: "static",
url: `${baseDir}/${encodeURIComponent(libId)}`,
// The tag is in the URL and the namespace, and publish-libs
// never republishes an existing tag — a stored snapshot IS
// current, so warm opens skip the per-lib manifest GET.
immutable: true,
},
],
...opts,