diff --git a/web/pcbjam-shared b/web/pcbjam-shared index 140cef5..0230fde 160000 --- a/web/pcbjam-shared +++ b/web/pcbjam-shared @@ -1 +1 @@ -Subproject commit 140cef55eb106ceb34845a445f20d6970883ec8f +Subproject commit 0230fde538a4b6f2bf07b2e7856c3f8d440205fa diff --git a/web/standalone/src/lib/project-file-cache.test.ts b/web/standalone/src/lib/project-file-cache.test.ts index 2598337..98b64af 100644 --- a/web/standalone/src/lib/project-file-cache.test.ts +++ b/web/standalone/src/lib/project-file-cache.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import type { ProjectFile } from "@pcbjam/shared"; import { fileCacheValidator, + isYdocValidator, pruneProjectFileCache, readCachedFileBytes, writeCachedFileBytes, @@ -36,9 +37,25 @@ describe("fileCacheValidator", () => { expect(fileCacheValidator({ ...base, revision: undefined })).toBeNull(); }); - it("rejects ydoc-backed files — their bytes move without a row change", () => { + it("ydoc-backed + cold: the blob fingerprint is the validator", () => { + const v = fileCacheValidator({ ...base, hasYdoc: true, ydocTag: "etag-1" }); + expect(v).toBe("y1:etag-1"); + expect(isYdocValidator(v!)).toBe(true); + expect(isYdocValidator(fileCacheValidator(base)!)).toBe(false); + // Collab-only rows (revision 0 — never uploaded) are cacheable too: the + // blob IS their only source of truth. + expect( + fileCacheValidator({ ...base, revision: 0, hasYdoc: true, ydocTag: "e2" }), + ).toBe("y1:e2"); + }); + + it("ydoc-backed but LIVE or untagged stays uncacheable", () => { + // Live: bytes are moving under an open room, no fingerprint can vouch. + expect( + fileCacheValidator({ ...base, hasYdoc: true, ydocTag: "e", isLive: true }), + ).toBeNull(); + // Older backend without the ydocTag field: exactly the old behavior. expect(fileCacheValidator({ ...base, hasYdoc: true })).toBeNull(); - expect(fileCacheValidator({ ...base, isLive: true })).toBeNull(); }); }); diff --git a/web/standalone/src/lib/project-file-cache.ts b/web/standalone/src/lib/project-file-cache.ts index 69466b6..fe79db1 100644 --- a/web/standalone/src/lib/project-file-cache.ts +++ b/web/standalone/src/lib/project-file-cache.ts @@ -18,10 +18,15 @@ import type { ProjectFile } from "@pcbjam/shared"; * equivalent or not, touches `updatedAt`, so the pair changes whenever the * bytes can have. * - * Ydoc-backed files (`hasYdoc`/`isLive`) are never cached: their served bytes - * come from room materialization, which moves without any file-row change. - * Revision 0 / absent means no published CAS row (collab-only docs, minimal - * backends) — also uncacheable. + * Ydoc-backed files use the `.ydoc` blob's content fingerprint instead + * (`ydocTag`, load-path-rework: the ydoc-etag validator): while the room is + * COLD its served bytes are a pure function of that blob, and the room's + * last-close flush lands BEFORE the live marker is deleted, so a cold tag is + * trustworthy. What we cache is the CONVERTED KiCad text — a warm load skips + * the download AND the ydoc→s-expr conversion. The validator embeds + * {@link YDOC_CONVERT_EPOCH} so shipping a converter change invalidates + * every converted body at once. LIVE files stay uncacheable (bytes are + * moving under an open room), as do plain files without a CAS revision. * * Raw IndexedDB, mirroring idb-project-store.ts: one out-of-line-keyed store, * key `projectId + NUL + path` so a project's entries form one contiguous key @@ -43,15 +48,31 @@ interface CacheRecord { } /** - * The cache validator for a listed file, or null when the file is uncacheable - * (no CAS revision, or ydoc-backed so bytes move independently of the row). + * Bump when the client-side ydoc→KiCad-text conversion (`ydocUpdateToKicadDoc` + * + `docToFile`) can produce different output for the same blob — cached + * converted bodies are keyed under it. + */ +const YDOC_CONVERT_EPOCH = 1; + +/** + * The cache validator for a listed file, or null when the file is uncacheable: + * live (bytes moving under an open room), ydoc-backed without a blob + * fingerprint (older backend), or plain without a CAS revision. */ export function fileCacheValidator(meta: ProjectFile): string | null { + if (meta.isLive) return null; + if (meta.hasYdoc) { + return meta.ydocTag ? `y${YDOC_CONVERT_EPOCH}:${meta.ydocTag}` : null; + } if (!meta.revision || meta.revision <= 0) return null; - if (meta.hasYdoc || meta.isLive) return null; return `${meta.revision}:${meta.updatedAt}`; } +/** Is `validator` the ydoc-blob form (a converted-body entry is expected)? */ +export function isYdocValidator(validator: string): boolean { + return validator.startsWith(`y${YDOC_CONVERT_EPOCH}:`); +} + function hasIdb(): boolean { return typeof indexedDB !== "undefined"; } diff --git a/web/standalone/src/lib/project-source-cache.test.ts b/web/standalone/src/lib/project-source-cache.test.ts index 08729dc..57cba93 100644 --- a/web/standalone/src/lib/project-source-cache.test.ts +++ b/web/standalone/src/lib/project-source-cache.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { ProjectFile } from "@pcbjam/shared"; // Real validator logic inside the mocked cache module — only the IDB-touching // functions are replaced (the node test env has no IndexedDB anyway). -import { fileCacheValidator } from "./project-file-cache"; +import { fileCacheValidator, isYdocValidator } from "./project-file-cache"; const PROJECT_ID = "0b7a4bfa-0000-5000-8000-0000000000aa"; @@ -21,6 +21,7 @@ const file = (over: Partial = {}): ProjectFile => ({ function cacheMock() { return { fileCacheValidator, + isYdocValidator, readCachedFileBytes: vi.fn(async () => null as Uint8Array | null), writeCachedFileBytes: vi.fn(async () => {}), pruneProjectFileCache: vi.fn(async () => {}), @@ -136,6 +137,91 @@ describe("remote source file-body cache", () => { expect(cache.writeCachedFileBytes).not.toHaveBeenCalled(); }); + it("ydoc + cold: caches the CONVERTED text under the blob-tag validator", async () => { + const { fileToDoc, docToY, docToFile } = await import("@pcbjam/shared"); + const Y = await import("yjs"); + const kdoc = fileToDoc("(kicad_sch (version 20230121))"); + const ydoc = new Y.Doc(); + docToY(kdoc, ydoc); + const update = Y.encodeStateAsUpdate(ydoc); + ydoc.destroy(); + + const cache = cacheMock(); + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ + ok: true, + headers: { + get: (h: string) => + h === "content-type" ? "application/x-pcbjam-ydoc" : null, + }, + arrayBuffer: async () => update.buffer, + })), + ); + + const src = (await loadRemote(cache))(); + const meta = file({ hasYdoc: true, ydocTag: "etag-77" }); + const got = await src.fetchFileBytes("proj", meta.path, meta); + + // The returned bytes are the client-side conversion of the update… + expect(new TextDecoder().decode(got)).toBe(docToFile(kdoc)); + // …and exactly those bytes are cached under the y-form validator, so the + // next warm load skips the download AND the conversion. + const validator = fileCacheValidator(meta)!; + expect(isYdocValidator(validator)).toBe(true); + expect(cache.writeCachedFileBytes).toHaveBeenCalledWith( + PROJECT_ID, + meta.path, + validator, + got, + ); + }); + + it("ydoc + unconvertible: the plain fallback is cached once per blob tag", async () => { + // The stale-v1-ydoc double-fetch: negotiation returns garbage, the client + // re-fetches server-materialized text. Caching THAT under the blob tag + // turns two fetches per load into two fetches per blob generation. + const cache = cacheMock(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce({ + ok: true, + headers: { + get: (h: string) => + h === "content-type" ? "application/x-pcbjam-ydoc" : null, + }, + arrayBuffer: async () => new Uint8Array([0xde, 0xad]).buffer, + }) + .mockResolvedValueOnce(plainResponse(new Uint8Array([40, 41]))); + vi.stubGlobal("fetch", fetchMock); + + const src = (await loadRemote(cache))(); + const meta = file({ hasYdoc: true, ydocTag: "etag-stale" }); + const got = await src.fetchFileBytes("proj", meta.path, meta); + + expect(Array.from(got)).toEqual([40, 41]); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(cache.writeCachedFileBytes).toHaveBeenCalledWith( + PROJECT_ID, + meta.path, + fileCacheValidator(meta)!, + got, + ); + }); + + it("ydoc + LIVE: no cache read or write — bytes are moving", async () => { + const cache = cacheMock(); + vi.stubGlobal( + "fetch", + vi.fn(async () => plainResponse(new Uint8Array([1]))), + ); + const src = (await loadRemote(cache))(); + const meta = file({ hasYdoc: true, ydocTag: "e", isLive: true }); + await src.fetchFileBytes("proj", meta.path, meta); + expect(cache.readCachedFileBytes).not.toHaveBeenCalled(); + expect(cache.writeCachedFileBytes).not.toHaveBeenCalled(); + }); + it("prunes to the fresh listing on getProject (cacheable rows only)", async () => { const cache = cacheMock(); const cacheable = file(); diff --git a/web/standalone/src/lib/project-source.ts b/web/standalone/src/lib/project-source.ts index b522a38..0d3da9a 100644 --- a/web/standalone/src/lib/project-source.ts +++ b/web/standalone/src/lib/project-source.ts @@ -19,6 +19,7 @@ import { client } from "./contract-client"; import { idbProjectStore, type LocalProjectStore } from "./idb-project-store"; import { fileCacheValidator, + isYdocValidator, pruneProjectFileCache, readCachedFileBytes, writeCachedFileBytes, @@ -132,17 +133,31 @@ function remoteProjectSource(): ProjectSource { }); if (!res.ok) throw new Error(`download failed (${res.status}): ${relPath}`); const bytes = new Uint8Array(await res.arrayBuffer()); + // A converted ydoc body may be cached ONLY under the ydoc-form validator + // (blob fingerprint) — never under `revision:updatedAt`, whose row does + // not move with collab edits. This re-guards the listing's hasYdoc: a + // room created between listing and fetch answers as ydoc while the + // validator is still the revision form, and stays uncached. + const cacheYdocBody = + validator !== null && meta !== undefined && isYdocValidator(validator); if (!isYdocResponse(res)) { - // The ydoc check re-guards the listing's hasYdoc: a room created - // between listing and fetch answers as ydoc, which must not be cached - // (its bytes move without a file-row change). + // Plain body: correct for either validator form — for a ydoc-form one + // this is the server-materialized fallback of the same cold blob. if (validator && meta) { void writeCachedFileBytes(meta.projectId, relPath, validator, bytes); } return bytes; } try { - return new TextEncoder().encode(docToFile(ydocUpdateToKicadDoc(bytes))); + const text = new TextEncoder().encode( + docToFile(ydocUpdateToKicadDoc(bytes)), + ); + // Cache the CONVERTED text: a warm load skips the download and the + // (measured ~2s on big boards) ydoc→s-expr conversion both. + if (cacheYdocBody && validator && meta) { + void writeCachedFileBytes(meta.projectId, relPath, validator, text); + } + return text; } catch (err) { // A ydoc we can't convert must not make the file undownloadable: retry // without negotiating and let the backend materialize it as before. @@ -150,7 +165,13 @@ function remoteProjectSource(): ProjectSource { if (!plain.ok) { throw new Error(`download failed (${plain.status}): ${relPath} (${String(err)})`); } - return new Uint8Array(await plain.arrayBuffer()); + const materialized = new Uint8Array(await plain.arrayBuffer()); + // Caching the fallback under the blob tag ends the double-fetch for + // stale unconvertible ydocs — one per tag instead of two per load. + if (cacheYdocBody && validator && meta && !isYdocResponse(plain)) { + void writeCachedFileBytes(meta.projectId, relPath, validator, materialized); + } + return materialized; } }, async uploadFileBytes(slug, relPath, bytes) {