// Publish KiCad 3D models (kicad-packages3D) to the CDN as r2-idb-sync SPARSE // origins: per-lib manifests keyed by the upstream tag, bodies content-addressed // under a shared blobs/ prefix (deduped across tags — models rarely change). // The standalone's cdnModelsSource opens each lib as a sparse layer: manifest // synced eagerly (small), bodies fetched exactly when a board references them. // See web/standalone/src/wasm/libs/models-source.ts + docs/features/3d-models. // // npx tsx scripts/deploy/publish-models.ts --model-tag 10.0.0 \ // --models-src --driver local --out /tmp/cdn-models // npx tsx scripts/deploy/publish-models.ts --model-tag 10.0.0 --models-src … \ // --driver r2 --bucket pcbjam-cdn --remote // # dev subset: only a few libs // … --libs Resistor_SMD,Capacitor_SMD,Package_QFP // // Layout under `` (default libs/kicad-models): // /manifest.json top index { schema, tag, libs:[{id,itemCount,bytes}] } // //manifest per-lib SyncManifest { "model3d/": {hash,size,mtime} } // blobs/sha256/ model bodies (brotli, content-addressed, shared) // blobs/registry.json published-blob index (hash → size) for cheap dedup // // Idempotent per tag: if //manifest.json exists the run SKIPS // (--force overrides). Blobs are skipped per-hash via the registry. import { execFileSync } from "node:child_process"; import { existsSync, mkdirSync, readdirSync, readFileSync, statSync } from "node:fs"; import { dirname, join } from "node:path"; import type { SyncManifest } from "../../web/pcbjam-shared/src/sync-wire.js"; import { compressBytes, IMMUTABLE, makeStore, NO_STORE, putJSON, sha256hex, } from "./lib/cdn-store.mjs"; const MODELS_URL = "https://gitlab.com/kicad/libraries/kicad-packages3D.git"; /** Model file extensions we publish (locked: WRL + STEP). */ const MODEL_EXTS = [".wrl", ".step", ".stp"]; interface Args { modelTag: string | null; modelsSrc: string | null; clone: string | null; driver: string; out: string | null; bucket: string; remote: boolean; prefix: string; force: boolean; libs: string[] | null; quality: number; compress: "br" | "none"; } function parseArgs(argv: string[]): Args { const a: Args = { modelTag: null, modelsSrc: null, clone: null, driver: "local", out: null, bucket: "pcbjam-cdn", remote: false, prefix: "libs/kicad-models", force: false, libs: null, quality: 5, compress: "br", }; for (let i = 2; i < argv.length; i++) { const next = () => argv[++i]!; switch (argv[i]) { case "--model-tag": a.modelTag = next(); break; case "--models-src": a.modelsSrc = next(); break; // Clone kicad-packages3D at --model-tag into /kicad-packages3D // (shallow; NOTE: multi-GB working tree) and use it as the source. case "--clone": a.clone = next(); break; case "--driver": a.driver = next(); break; case "--out": a.out = next(); break; case "--bucket": a.bucket = next(); break; case "--remote": a.remote = true; break; case "--prefix": a.prefix = next(); break; case "--force": a.force = true; break; // Dev subset: publish only these libs (names without .3dshapes). case "--libs": a.libs = next().split(",").map((s) => s.trim()).filter(Boolean); break; // Brotli quality for bodies (WRL/STEP are text-ish; 5 ≈ 4-5x, fast). case "--quality": a.quality = Number(next()); break; // "none" for --driver local when a plain static server (e.g. the vite dev // server) will serve the blobs — it can't send Content-Encoding: br. case "--compress": a.compress = next() as "br" | "none"; break; default: throw new Error(`unknown arg: ${argv[i]}`); } } if (!a.modelTag) throw new Error("--model-tag is required"); if (!a.modelsSrc && !a.clone) throw new Error("need --models-src or --clone "); if (a.driver === "local" && !a.out) a.out = ".cdn-models-out"; return a; } function cloneShallow(url: string, dest: string, ref: string): void { if (existsSync(dest)) { console.log(`clone: ${dest} present — reusing`); return; } mkdirSync(dirname(dest), { recursive: true }); console.log(`clone: ${url} @ ${ref} → ${dest} (multi-GB — this takes a while)`); execFileSync("git", ["clone", "--depth", "1", "--branch", ref, url, dest], { stdio: "inherit", }); } /** All `.3dshapes` dirs under the checkout root (non-recursive: the repo is flat). */ function listModelLibs(src: string): Array<{ id: string; dir: string }> { return readdirSync(src, { withFileTypes: true }) .filter((d) => d.isDirectory() && d.name.endsWith(".3dshapes")) .map((d) => ({ id: d.name.slice(0, -".3dshapes".length), dir: join(src, d.name) })) .sort((x, y) => x.id.localeCompare(y.id)); } async function main(): Promise { const a = parseArgs(process.argv); const store = makeStore(a.driver, a); const topKey = `${a.prefix}/${a.modelTag}/manifest.json`; const registryKey = `${a.prefix}/blobs/registry.json`; if (!a.force && store.getJSON(topKey)) { console.log(`publish-models: ${topKey} already published — skipping (use --force)`); return; } if (a.clone) { const dest = join(a.clone, "kicad-packages3D"); cloneShallow(MODELS_URL, dest, a.modelTag!); a.modelsSrc ??= dest; } console.log( `publish-models: tag=${a.modelTag} driver=${store.kind} → ${a.prefix}/${a.modelTag}/`, ); // Published-blob index: hash → original size. One GET up front, one PUT at the // end — the per-blob "does it exist" probe would otherwise be an R2 round-trip // per model (tens of thousands). const registry: Record = (store.getJSON(registryKey) as Record | null) ?? {}; let blobsPut = 0; let blobsSkipped = 0; let libs = listModelLibs(a.modelsSrc!); if (a.libs) { const want = new Set(a.libs); libs = libs.filter((l) => want.has(l.id)); const missing = a.libs.filter((id) => !libs.some((l) => l.id === id)); if (missing.length) console.warn(`publish-models: libs not found: ${missing.join(", ")}`); } if (!libs.length) throw new Error(`no .3dshapes libs under ${a.modelsSrc}`); const topLibs: Array<{ id: string; itemCount: number; bytes: number }> = []; let totalItems = 0; let totalBytes = 0; for (const lib of libs) { const files = readdirSync(lib.dir) .filter((f) => MODEL_EXTS.some((ext) => f.toLowerCase().endsWith(ext))) .sort(); if (!files.length) continue; const entries: SyncManifest["entries"] = {}; let libBytes = 0; for (const f of files) { const p = join(lib.dir, f); if (!statSync(p).isFile()) continue; const body = readFileSync(p); const hash = sha256hex(body); entries[`model3d/${f}`] = { hash, size: body.length, mtime: 0 }; libBytes += body.length; if (registry[hash] === undefined) { // WRL and STEP are text formats — brotli gets ~4-5x. The browser fetch // transparently decodes, so IDB caches (and hashes refer to) the // ORIGINAL bytes; `no-transform` keeps the edge from re-encoding. const { bytes, encoding } = compressBytes(body, a.compress, a.quality); store.put(`${a.prefix}/blobs/sha256/${hash}`, bytes, { contentType: "application/octet-stream", contentEncoding: encoding, cacheControl: IMMUTABLE, }); registry[hash] = body.length; blobsPut++; } else { blobsSkipped++; } } const manifest: SyncManifest = { version: 1, entries }; putJSON(store, `${a.prefix}/${a.modelTag}/${lib.id}/manifest`, manifest, IMMUTABLE); topLibs.push({ id: lib.id, itemCount: files.length, bytes: libBytes }); totalItems += files.length; totalBytes += libBytes; console.log(` ${lib.id}: ${files.length} models (${(libBytes / 1e6).toFixed(1)} MB)`); } putJSON(store, registryKey, registry, NO_STORE); putJSON(store, topKey, { schema: 1, tag: a.modelTag, libs: topLibs }, IMMUTABLE); console.log( `publish-models: done — ${topLibs.length} libs, ${totalItems} models ` + `(${(totalBytes / 1e6).toFixed(0)} MB raw), blobs put=${blobsPut} deduped=${blobsSkipped} → ${topKey}`, ); if (store.kind === "local") console.log(`local layout under: ${a.out}`); } main().catch((err) => { console.error(err); process.exit(1); });