feat(deploy): publish-libs (C1.2) — full KiCad set as r2-idb-sync static origins

extract-libs.ts gains extractAllLibs(): enumerate EVERY <Lib>.kicad_symdir / <Lib>.pretty under the source dirs into in-memory self-contained bodies (reuses the curated path's parse + extends resolution). At KiCad 10.0.x symbols already ship the one-symbol-per-file .kicad_symdir layout, so no split needed.

publish-libs.ts (tsx): per lib, build a SyncManifest (sha256 per item) + encodeBundle, write libs/kicad/<libTag>/<lib>/{manifest,bundle} (immutable) + a top manifest.json listing every lib. Keyed by upstream KiCad lib tag, skip-if-exists (HEAD the top manifest; --force overrides) so it is decoupled from the app deploy. Local + r2 drivers via cdn-store.

Validated over web/backend/.cache: 8 libs / 2154 items; Device bundle decodes to 537 bodies, its /manifest matches the bundle, and symbol/R body hash matches its manifest entry (warm-sync diff stays empty). Same wire format cdn-source.test reads, so publish and client are pinned to one format.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

scripts/deploy/publish-libs.ts

@@ -0,0 +1,124 @@
+// Publish the FULL default KiCad symbol + footprint set to the CDN as
+// version-pinned r2-idb-sync STATIC ORIGINS (one per lib), which the demo's
+// cdnLibsSource opens read-only + IDB-cached (1 bundle cold, 0 warm). See
+// docs/features/r2-idb-sync + wasm/libs/cdn-source.ts.
+//
+//   npx tsx scripts/deploy/publish-libs.ts --lib-tag 10.0.3 \
+//     --symbols-src <kicad-symbols checkout> --footprints-src <kicad-footprints> \
+//     --driver local --out /tmp/cdn-libs
+//   npx tsx scripts/deploy/publish-libs.ts --lib-tag 10.0.3 --symbols-src … \
+//     --footprints-src … --driver r2 --bucket pcbjam-cdn --remote
+//
+// Keyed by the upstream KiCad library tag (libs/kicad/<libTag>/), published ONCE
+// per tag: if <prefix>/<libTag>/manifest.json already exists it SKIPS the whole
+// run (override with --force) — so it's decoupled from the app/demo deploy.
+//
+// Per lib `<prefix>/<libTag>/<lib>/`:
+//   manifest   SyncManifest { version, entries: { "<kind>/<name>": {hash,size,mtime} } }
+//   bundle     encodeBundle(manifest, bodies)  — cold-init payload (all bodies)
+// + top `<prefix>/<libTag>/manifest.json` { schema, tag, libs:[{id,name,kind,itemCount}] }
+// All immutable (content is pinned by the tag).
+
+import { extractAllLibs } from "../../web/backend/src/extract/extract-libs.js";
+import { encodeBundle, type SyncManifest } from "../../web/pcbjam-shared/src/sync-wire.js";
+import { IMMUTABLE, makeStore, putJSON, sha256hex } from "./lib/cdn-store.mjs";
+
+interface Args {
+  libTag: string | null;
+  symbolsSrc: string | null;
+  footprintsSrc: string | null;
+  driver: string;
+  out: string | null;
+  bucket: string;
+  remote: boolean;
+  prefix: string;
+  force: boolean;
+}
+
+function parseArgs(argv: string[]): Args {
+  const a: Args = {
+    libTag: null,
+    symbolsSrc: null,
+    footprintsSrc: null,
+    driver: "local",
+    out: null,
+    bucket: "pcbjam-cdn",
+    remote: false,
+    prefix: "libs/kicad",
+    force: false,
+  };
+  for (let i = 2; i < argv.length; i++) {
+    const next = () => argv[++i]!;
+    switch (argv[i]) {
+      case "--lib-tag": a.libTag = next(); break;
+      case "--symbols-src": a.symbolsSrc = next(); break;
+      case "--footprints-src": a.footprintsSrc = next(); break;
+      case "--driver": a.driver = next(); break;
+      case "--out": a.out = next(); break;
+      case "--bucket": a.bucket = next(); break;
+      case "--remote": a.remote = true; break;
+      case "--prefix": a.prefix = next(); break;
+      case "--force": a.force = true; break;
+      default: throw new Error(`unknown arg: ${argv[i]}`);
+    }
+  }
+  if (!a.libTag) throw new Error("--lib-tag <kicad library tag> is required");
+  if (!a.symbolsSrc && !a.footprintsSrc)
+    throw new Error("at least one of --symbols-src / --footprints-src is required");
+  if (a.driver === "local" && !a.out) a.out = ".cdn-out";
+  return a;
+}
+
+async function main(): Promise<void> {
+  const a = parseArgs(process.argv);
+  const store = makeStore(a.driver, a);
+  const enc = new TextEncoder();
+  const topKey = `${a.prefix}/${a.libTag}/manifest.json`;
+
+  // Skip-if-exists: the snapshot is immutable + content-pinned by the tag.
+  if (!a.force && store.getJSON(topKey)) {
+    console.log(`publish-libs: ${topKey} already published — skipping (use --force)`);
+    return;
+  }
+
+  console.log(`publish-libs: tag=${a.libTag} driver=${store.kind} → ${a.prefix}/${a.libTag}/`);
+  const libs = await extractAllLibs({
+    symbolsSrc: a.symbolsSrc ?? undefined,
+    footprintsSrc: a.footprintsSrc ?? undefined,
+  });
+
+  const topLibs: Array<{ id: string; name: string; kind: string; itemCount: number }> = [];
+  let totalItems = 0;
+  for (const { lib, kind, items } of libs) {
+    const bodies = items.map(
+      (it): [string, Uint8Array] => [`${it.kind}/${it.name}`, enc.encode(it.body)],
+    );
+    const entries: SyncManifest["entries"] = {};
+    for (const [path, body] of bodies) {
+      entries[path] = { hash: sha256hex(body), size: body.length, mtime: 0 };
+    }
+    const manifest: SyncManifest = { version: 1, entries };
+    const base = `${a.prefix}/${a.libTag}/${lib}`;
+    putJSON(store, `${base}/manifest`, manifest, IMMUTABLE);
+    store.put(`${base}/bundle`, encodeBundle(manifest, bodies), {
+      contentType: "application/octet-stream",
+      contentEncoding: null,
+      cacheControl: IMMUTABLE,
+    });
+    topLibs.push({ id: lib, name: lib, kind, itemCount: items.length });
+    totalItems += items.length;
+  }
+
+  topLibs.sort((x, y) => x.id.localeCompare(y.id));
+  putJSON(store, topKey, { schema: 1, tag: a.libTag, libs: topLibs }, IMMUTABLE);
+
+  console.log(
+    `publish-libs: done — ${topLibs.length} libs, ${totalItems} items → ${topKey}`,
+  );
+  if (store.kind === "local") console.log(`local layout under: ${a.out}`);
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

web/backend/src/extract/extract-libs.ts

@@ -1,4 +1,4 @@
-import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
 import * as path from "node:path";
 import { pathToFileURL } from "node:url";
 import {
@@ -190,6 +190,99 @@ export async function extractAll(
   return { libs, symbols, footprints };
 }
 
+/* ----------------------------------------------------- full-set extraction --
+ * The CURATED extractAll above provisions a small example tree on disk. For the
+ * demo CDN we instead want EVERY lib, in memory, to publish as r2-idb-sync
+ * snapshots (see scripts/deploy/publish-libs.ts) — same per-item parse/extends
+ * resolution, no on-disk serve tree.
+ */
+
+export interface ExtractedItem {
+  kind: "symbol" | "footprint";
+  name: string;
+  /** Complete self-contained s-expr body (extends inlined for symbols). */
+  body: string;
+  description: string | null;
+  keywords: string | null;
+}
+export interface ExtractedLib {
+  lib: string;
+  kind: "symbol" | "footprint";
+  items: ExtractedItem[];
+}
+
+/** Extract EVERY lib present under the source dirs into in-memory bodies:
+ *  each `<Lib>.kicad_symdir/<name>.kicad_sym` (resolved + self-contained) and
+ *  each `<Lib>.pretty/<name>.kicad_mod`. Libs with no items are skipped. */
+export async function extractAllLibs(opts: {
+  symbolsSrc?: string;
+  footprintsSrc?: string;
+}): Promise<ExtractedLib[]> {
+  const out: ExtractedLib[] = [];
+
+  if (opts.symbolsSrc) {
+    const dirs = (await readdir(opts.symbolsSrc, { withFileTypes: true }))
+      .filter((e) => e.isDirectory() && e.name.endsWith(".kicad_symdir"))
+      .map((e) => e.name)
+      .sort();
+    for (const dirName of dirs) {
+      const lib = dirName.slice(0, -".kicad_symdir".length);
+      const symdir = path.join(opts.symbolsSrc, dirName);
+      const names = (await readdir(symdir))
+        .filter((f) => f.endsWith(".kicad_sym"))
+        .map((f) => f.slice(0, -".kicad_sym".length))
+        .sort();
+      const items: ExtractedItem[] = [];
+      for (const name of names) {
+        const { src: fileSrc, sym } = await readSymbol(symdir, name);
+        const parents = await resolveChain(symdir, sym);
+        items.push({
+          kind: "symbol",
+          name: sym.name,
+          body: buildSelfContainedLib(
+            libHeader(fileSrc),
+            parents.map((p) => p.block),
+            sym.block,
+          ),
+          description: sym.description,
+          keywords: sym.keywords,
+        });
+      }
+      if (items.length) out.push({ lib, kind: "symbol", items });
+    }
+  }
+
+  if (opts.footprintsSrc) {
+    const dirs = (await readdir(opts.footprintsSrc, { withFileTypes: true }))
+      .filter((e) => e.isDirectory() && e.name.endsWith(".pretty"))
+      .map((e) => e.name)
+      .sort();
+    for (const dirName of dirs) {
+      const lib = dirName.slice(0, -".pretty".length);
+      const pretty = path.join(opts.footprintsSrc, dirName);
+      const names = (await readdir(pretty))
+        .filter((f) => f.endsWith(".kicad_mod"))
+        .map((f) => f.slice(0, -".kicad_mod".length))
+        .sort();
+      const items: ExtractedItem[] = [];
+      for (const name of names) {
+        const fileSrc = await readFile(path.join(pretty, `${name}.kicad_mod`), "utf8");
+        const fp = parseFootprintFile(fileSrc, name);
+        items.push({
+          kind: "footprint",
+          name: fp.name,
+          body: fp.body,
+          description: fp.description,
+          keywords: fp.keywords,
+        });
+      }
+      if (items.length) out.push({ lib, kind: "footprint", items });
+    }
+  }
+
+  return out;
+}
+
 function arg(name: string): string | undefined {
   const i = process.argv.indexOf(`--${name}`);
   return i >= 0 ? process.argv[i + 1] : undefined;