feat: cache ydoc-backed file bodies under the blob-etag validator

fileCacheValidator: hasYdoc && !isLive rows now validate as
y<YDOC_CONVERT_EPOCH>:<ydocTag> (revision-0 collab-only rows included); live
rows and untagged older backends stay uncacheable. The remote source caches
the CONVERTED KiCad text — a warm load skips the download and the measured
~2s-class ydoc→s-expr conversion — and the unconvertible-ydoc plain fallback
is cached under the same tag, ending the stale-ydoc double-fetch. A ydoc
response under a revision-form validator (room appeared mid-listing) stays
uncached, preserving the old race guard exactly.

Measured (Arduino Leonardo, dev stack): cold 52 file GETs → warm reload 0.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VLSht9cadprtT2mhynawWu

web/standalone/src/lib/project-file-cache.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
 import type { ProjectFile } from "@pcbjam/shared";
 import {
   fileCacheValidator,
+  isYdocValidator,
   pruneProjectFileCache,
   readCachedFileBytes,
   writeCachedFileBytes,
@@ -36,9 +37,25 @@ describe("fileCacheValidator", () => {
     expect(fileCacheValidator({ ...base, revision: undefined })).toBeNull();
   });
 
-  it("rejects ydoc-backed files — their bytes move without a row change", () => {
+  it("ydoc-backed + cold: the blob fingerprint is the validator", () => {
+    const v = fileCacheValidator({ ...base, hasYdoc: true, ydocTag: "etag-1" });
+    expect(v).toBe("y1:etag-1");
+    expect(isYdocValidator(v!)).toBe(true);
+    expect(isYdocValidator(fileCacheValidator(base)!)).toBe(false);
+    // Collab-only rows (revision 0 — never uploaded) are cacheable too: the
+    // blob IS their only source of truth.
+    expect(
+      fileCacheValidator({ ...base, revision: 0, hasYdoc: true, ydocTag: "e2" }),
+    ).toBe("y1:e2");
+  });
+
+  it("ydoc-backed but LIVE or untagged stays uncacheable", () => {
+    // Live: bytes are moving under an open room, no fingerprint can vouch.
+    expect(
+      fileCacheValidator({ ...base, hasYdoc: true, ydocTag: "e", isLive: true }),
+    ).toBeNull();
+    // Older backend without the ydocTag field: exactly the old behavior.
     expect(fileCacheValidator({ ...base, hasYdoc: true })).toBeNull();
-    expect(fileCacheValidator({ ...base, isLive: true })).toBeNull();
   });
 });
 

web/standalone/src/lib/project-file-cache.ts

@@ -18,10 +18,15 @@ import type { ProjectFile } from "@pcbjam/shared";
  * equivalent or not, touches `updatedAt`, so the pair changes whenever the
  * bytes can have.
  *
- * Ydoc-backed files (`hasYdoc`/`isLive`) are never cached: their served bytes
- * come from room materialization, which moves without any file-row change.
- * Revision 0 / absent means no published CAS row (collab-only docs, minimal
- * backends) — also uncacheable.
+ * Ydoc-backed files use the `.ydoc` blob's content fingerprint instead
+ * (`ydocTag`, load-path-rework: the ydoc-etag validator): while the room is
+ * COLD its served bytes are a pure function of that blob, and the room's
+ * last-close flush lands BEFORE the live marker is deleted, so a cold tag is
+ * trustworthy. What we cache is the CONVERTED KiCad text — a warm load skips
+ * the download AND the ydoc→s-expr conversion. The validator embeds
+ * {@link YDOC_CONVERT_EPOCH} so shipping a converter change invalidates
+ * every converted body at once. LIVE files stay uncacheable (bytes are
+ * moving under an open room), as do plain files without a CAS revision.
  *
  * Raw IndexedDB, mirroring idb-project-store.ts: one out-of-line-keyed store,
  * key `projectId + NUL + path` so a project's entries form one contiguous key
@@ -43,15 +48,31 @@ interface CacheRecord {
 }
 
 /**
- * The cache validator for a listed file, or null when the file is uncacheable
- * (no CAS revision, or ydoc-backed so bytes move independently of the row).
+ * Bump when the client-side ydoc→KiCad-text conversion (`ydocUpdateToKicadDoc`
+ * + `docToFile`) can produce different output for the same blob — cached
+ * converted bodies are keyed under it.
+ */
+const YDOC_CONVERT_EPOCH = 1;
+
+/**
+ * The cache validator for a listed file, or null when the file is uncacheable:
+ * live (bytes moving under an open room), ydoc-backed without a blob
+ * fingerprint (older backend), or plain without a CAS revision.
  */
 export function fileCacheValidator(meta: ProjectFile): string | null {
+  if (meta.isLive) return null;
+  if (meta.hasYdoc) {
+    return meta.ydocTag ? `y${YDOC_CONVERT_EPOCH}:${meta.ydocTag}` : null;
+  }
   if (!meta.revision || meta.revision <= 0) return null;
-  if (meta.hasYdoc || meta.isLive) return null;
   return `${meta.revision}:${meta.updatedAt}`;
 }
 
+/** Is `validator` the ydoc-blob form (a converted-body entry is expected)? */
+export function isYdocValidator(validator: string): boolean {
+  return validator.startsWith(`y${YDOC_CONVERT_EPOCH}:`);
+}
+
 function hasIdb(): boolean {
   return typeof indexedDB !== "undefined";
 }

web/standalone/src/lib/project-source-cache.test.ts

@@ -2,7 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
 import type { ProjectFile } from "@pcbjam/shared";
 // Real validator logic inside the mocked cache module — only the IDB-touching
 // functions are replaced (the node test env has no IndexedDB anyway).
-import { fileCacheValidator } from "./project-file-cache";
+import { fileCacheValidator, isYdocValidator } from "./project-file-cache";
 
 const PROJECT_ID = "0b7a4bfa-0000-5000-8000-0000000000aa";
 
@@ -21,6 +21,7 @@ const file = (over: Partial<ProjectFile> = {}): ProjectFile => ({
 function cacheMock() {
   return {
     fileCacheValidator,
+    isYdocValidator,
     readCachedFileBytes: vi.fn(async () => null as Uint8Array | null),
     writeCachedFileBytes: vi.fn(async () => {}),
     pruneProjectFileCache: vi.fn(async () => {}),
@@ -136,6 +137,91 @@ describe("remote source file-body cache", () => {
     expect(cache.writeCachedFileBytes).not.toHaveBeenCalled();
   });
 
+  it("ydoc + cold: caches the CONVERTED text under the blob-tag validator", async () => {
+    const { fileToDoc, docToY, docToFile } = await import("@pcbjam/shared");
+    const Y = await import("yjs");
+    const kdoc = fileToDoc("(kicad_sch (version 20230121))");
+    const ydoc = new Y.Doc();
+    docToY(kdoc, ydoc);
+    const update = Y.encodeStateAsUpdate(ydoc);
+    ydoc.destroy();
+
+    const cache = cacheMock();
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async () => ({
+        ok: true,
+        headers: {
+          get: (h: string) =>
+            h === "content-type" ? "application/x-pcbjam-ydoc" : null,
+        },
+        arrayBuffer: async () => update.buffer,
+      })),
+    );
+
+    const src = (await loadRemote(cache))();
+    const meta = file({ hasYdoc: true, ydocTag: "etag-77" });
+    const got = await src.fetchFileBytes("proj", meta.path, meta);
+
+    // The returned bytes are the client-side conversion of the update…
+    expect(new TextDecoder().decode(got)).toBe(docToFile(kdoc));
+    // …and exactly those bytes are cached under the y-form validator, so the
+    // next warm load skips the download AND the conversion.
+    const validator = fileCacheValidator(meta)!;
+    expect(isYdocValidator(validator)).toBe(true);
+    expect(cache.writeCachedFileBytes).toHaveBeenCalledWith(
+      PROJECT_ID,
+      meta.path,
+      validator,
+      got,
+    );
+  });
+
+  it("ydoc + unconvertible: the plain fallback is cached once per blob tag", async () => {
+    // The stale-v1-ydoc double-fetch: negotiation returns garbage, the client
+    // re-fetches server-materialized text. Caching THAT under the blob tag
+    // turns two fetches per load into two fetches per blob generation.
+    const cache = cacheMock();
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce({
+        ok: true,
+        headers: {
+          get: (h: string) =>
+            h === "content-type" ? "application/x-pcbjam-ydoc" : null,
+        },
+        arrayBuffer: async () => new Uint8Array([0xde, 0xad]).buffer,
+      })
+      .mockResolvedValueOnce(plainResponse(new Uint8Array([40, 41])));
+    vi.stubGlobal("fetch", fetchMock);
+
+    const src = (await loadRemote(cache))();
+    const meta = file({ hasYdoc: true, ydocTag: "etag-stale" });
+    const got = await src.fetchFileBytes("proj", meta.path, meta);
+
+    expect(Array.from(got)).toEqual([40, 41]);
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+    expect(cache.writeCachedFileBytes).toHaveBeenCalledWith(
+      PROJECT_ID,
+      meta.path,
+      fileCacheValidator(meta)!,
+      got,
+    );
+  });
+
+  it("ydoc + LIVE: no cache read or write — bytes are moving", async () => {
+    const cache = cacheMock();
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async () => plainResponse(new Uint8Array([1]))),
+    );
+    const src = (await loadRemote(cache))();
+    const meta = file({ hasYdoc: true, ydocTag: "e", isLive: true });
+    await src.fetchFileBytes("proj", meta.path, meta);
+    expect(cache.readCachedFileBytes).not.toHaveBeenCalled();
+    expect(cache.writeCachedFileBytes).not.toHaveBeenCalled();
+  });
+
   it("prunes to the fresh listing on getProject (cacheable rows only)", async () => {
     const cache = cacheMock();
     const cacheable = file();

web/standalone/src/lib/project-source.ts

@@ -19,6 +19,7 @@ import { client } from "./contract-client";
 import { idbProjectStore, type LocalProjectStore } from "./idb-project-store";
 import {
   fileCacheValidator,
+  isYdocValidator,
   pruneProjectFileCache,
   readCachedFileBytes,
   writeCachedFileBytes,
@@ -132,17 +133,31 @@ function remoteProjectSource(): ProjectSource {
       });
       if (!res.ok) throw new Error(`download failed (${res.status}): ${relPath}`);
       const bytes = new Uint8Array(await res.arrayBuffer());
+      // A converted ydoc body may be cached ONLY under the ydoc-form validator
+      // (blob fingerprint) — never under `revision:updatedAt`, whose row does
+      // not move with collab edits. This re-guards the listing's hasYdoc: a
+      // room created between listing and fetch answers as ydoc while the
+      // validator is still the revision form, and stays uncached.
+      const cacheYdocBody =
+        validator !== null && meta !== undefined && isYdocValidator(validator);
       if (!isYdocResponse(res)) {
-        // The ydoc check re-guards the listing's hasYdoc: a room created
-        // between listing and fetch answers as ydoc, which must not be cached
-        // (its bytes move without a file-row change).
+        // Plain body: correct for either validator form — for a ydoc-form one
+        // this is the server-materialized fallback of the same cold blob.
         if (validator && meta) {
           void writeCachedFileBytes(meta.projectId, relPath, validator, bytes);
         }
         return bytes;
       }
       try {
-        return new TextEncoder().encode(docToFile(ydocUpdateToKicadDoc(bytes)));
+        const text = new TextEncoder().encode(
+          docToFile(ydocUpdateToKicadDoc(bytes)),
+        );
+        // Cache the CONVERTED text: a warm load skips the download and the
+        // (measured ~2s on big boards) ydoc→s-expr conversion both.
+        if (cacheYdocBody && validator && meta) {
+          void writeCachedFileBytes(meta.projectId, relPath, validator, text);
+        }
+        return text;
       } catch (err) {
         // A ydoc we can't convert must not make the file undownloadable: retry
         // without negotiating and let the backend materialize it as before.
@@ -150,7 +165,13 @@ function remoteProjectSource(): ProjectSource {
         if (!plain.ok) {
           throw new Error(`download failed (${plain.status}): ${relPath} (${String(err)})`);
         }
-        return new Uint8Array(await plain.arrayBuffer());
+        const materialized = new Uint8Array(await plain.arrayBuffer());
+        // Caching the fallback under the blob tag ends the double-fetch for
+        // stale unconvertible ydocs — one per tag instead of two per load.
+        if (cacheYdocBody && validator && meta && !isYdocResponse(plain)) {
+          void writeCachedFileBytes(meta.projectId, relPath, validator, materialized);
+        }
+        return materialized;
       }
     },
     async uploadFileBytes(slug, relPath, bytes) {