feat(3d): lazy 3D model delivery R2→IDB→WASM
- cdnModelsSource: sparse per-lib stacks over the models CDN layout (libs/kicad-models/<tag>/<lib>/manifest + content-addressed blobs) - models-bridge: board prescan ((model …) scan → prefetch, 4/4 in 74ms on the demo board) + the kind=model3d 'ensure' op answering absolute MEMFS paths; only board-referenced bodies ever enter IDB - boot/runner/config/WasmTool wiring (VITE_MODELS_MANIFEST_URL, KICAD*_3DMODEL_DIR seeding, prefetch badge) - StorageUsageCard on HomePage: per-kind cached sizes + delete-3D-cache - publish-models.ts (brotli content-addressed publish, 606→93MB for the pic_programmer lib set) + dev-demo --models-tag/--models-local - e2e: 3d-viewer-models.spec.ts (bridge normalize/dedup + STEP/WRL render); submodule bumps: kicad (static 3D plugins + ensure hook), pcbjam-shared (sparse sync layer) Spec + findings: docs/features/3d-models (private repo). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014AT7gVHRktDYoQ68S4x6A4
This commit is contained in:
Gergő Törcsvári
parent
3efcdcf91c
commit
6f3f3bcf00
19 changed files with 1351 additions and 4 deletions
|
|
@ -72,6 +72,8 @@ function parseArgs(argv) {
|
|||
contentTag: null, // use the live CDN gallery for this tag (else build locally)
|
||||
galleryTag: "demo-local", // path tag for the locally-built gallery
|
||||
noGallery: false, // disable the example gallery (local-folder + IDB only)
|
||||
modelsTag: null, // 3D models snapshot tag (live CDN, or the local dir's tag)
|
||||
modelsLocal: null, // local publish-models --driver local output dir (serve same-origin)
|
||||
port: null,
|
||||
repo: "https://github.com/emergence-engineering/pcbjam",
|
||||
};
|
||||
|
|
@ -87,6 +89,8 @@ function parseArgs(argv) {
|
|||
case "--no-gallery": a.noGallery = true; break;
|
||||
case "--port": a.port = next(); break;
|
||||
case "--repo": a.repo = next(); break;
|
||||
case "--models-tag": a.modelsTag = next(); break;
|
||||
case "--models-local": a.modelsLocal = next(); break;
|
||||
case "-h": case "--help": a.help = true; break;
|
||||
default: throw new Error(`unknown arg: ${argv[i]}`);
|
||||
}
|
||||
|
|
@ -108,6 +112,9 @@ const HELP = `dev-demo.mjs — run the standalone locally in demo mode (R2-only
|
|||
--content-tag <tag> pin the LIVE CDN gallery for this release tag (default: build+serve the gallery locally)
|
||||
--gallery-tag <tag> path tag for the locally-built gallery (default demo-local)
|
||||
--no-gallery disable the example gallery (local-folder + IDB projects only)
|
||||
--models-tag <tag> enable lazy 3D models from the CDN snapshot at this tag
|
||||
--models-local <dir> serve a local publish-models layout (--driver local --compress none)
|
||||
same-origin instead of the CDN (requires --models-tag)
|
||||
--port <n> dev server port
|
||||
|
||||
By default the read-only example gallery (deploy/demo/gallery.json) is built
|
||||
|
|
@ -142,6 +149,26 @@ function main() {
|
|||
delete env.VITE_LIBS_MANIFEST_URL;
|
||||
}
|
||||
|
||||
// --- 3D models: lazy per-board bodies (docs/features/3d-models). Off unless a
|
||||
// tag is given. --models-local <publish-models --out dir> serves that
|
||||
// layout same-origin at /models-cdn via a public/ symlink (publish it with
|
||||
// --compress none — the dev server can't send Content-Encoding: br);
|
||||
// otherwise the live CDN snapshot for --models-tag is used.
|
||||
if (a.modelsTag && a.modelsLocal) {
|
||||
const link = join(repoRoot, "web/standalone/public/models-cdn");
|
||||
try {
|
||||
if (lstatSync(link)) rmSync(link, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* no existing link */
|
||||
}
|
||||
symlinkSync(resolve(a.modelsLocal, "libs/kicad-models"), link);
|
||||
env.VITE_MODELS_MANIFEST_URL = `/models-cdn/${a.modelsTag}/manifest.json`;
|
||||
} else if (a.modelsTag) {
|
||||
env.VITE_MODELS_MANIFEST_URL = `${a.cdn}/libs/kicad-models/${a.modelsTag}/manifest.json`;
|
||||
} else {
|
||||
delete env.VITE_MODELS_MANIFEST_URL;
|
||||
}
|
||||
|
||||
// --- No backend: collab is cross-tab only, document bytes are local (api path),
|
||||
// loaded folders persist to a browser-local IndexedDB project.
|
||||
env.VITE_YJS_PROVIDER = "broadcastchannel";
|
||||
|
|
@ -189,6 +216,7 @@ function main() {
|
|||
console.log(` VITE_LIBS_SOURCE=${env.VITE_LIBS_SOURCE}${env.VITE_LIBS_MANIFEST_URL ? ` (${env.VITE_LIBS_MANIFEST_URL})` : ""}`);
|
||||
console.log(` VITE_WASM_ROOT=${env.VITE_WASM_ROOT}${env.VITE_WASM_MANIFEST ? ` (${env.VITE_WASM_MANIFEST})` : " (local build)"}`);
|
||||
console.log(` VITE_PROJECT_SOURCE=${env.VITE_PROJECT_SOURCE}${env.VITE_PROJECT_MANIFEST_URL ? ` (${env.VITE_PROJECT_MANIFEST_URL})` : ""}`);
|
||||
console.log(` VITE_MODELS_MANIFEST_URL=${env.VITE_MODELS_MANIFEST_URL ?? "(unset — 3D models off)"}`);
|
||||
console.log(` VITE_YJS_PROVIDER=${env.VITE_YJS_PROVIDER} VITE_DOC_SOURCE=${env.VITE_DOC_SOURCE} VITE_LOCAL_PROJECTS=${env.VITE_LOCAL_PROJECTS}`);
|
||||
|
||||
const child = spawn("pnpm", viteArgs, { cwd: repoRoot, env, stdio: "inherit" });
|
||||
|
|
|
|||
219
scripts/deploy/publish-models.ts
Normal file
219
scripts/deploy/publish-models.ts
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
// Publish KiCad 3D models (kicad-packages3D) to the CDN as r2-idb-sync SPARSE
|
||||
// origins: per-lib manifests keyed by the upstream tag, bodies content-addressed
|
||||
// under a shared blobs/ prefix (deduped across tags — models rarely change).
|
||||
// The standalone's cdnModelsSource opens each lib as a sparse layer: manifest
|
||||
// synced eagerly (small), bodies fetched exactly when a board references them.
|
||||
// See web/standalone/src/wasm/libs/models-source.ts + docs/features/3d-models.
|
||||
//
|
||||
// npx tsx scripts/deploy/publish-models.ts --model-tag 10.0.0 \
|
||||
// --models-src <kicad-packages3D checkout> --driver local --out /tmp/cdn-models
|
||||
// npx tsx scripts/deploy/publish-models.ts --model-tag 10.0.0 --models-src … \
|
||||
// --driver r2 --bucket pcbjam-cdn --remote
|
||||
// # dev subset: only a few libs
|
||||
// … --libs Resistor_SMD,Capacitor_SMD,Package_QFP
|
||||
//
|
||||
// Layout under `<prefix>` (default libs/kicad-models):
|
||||
// <tag>/manifest.json top index { schema, tag, libs:[{id,itemCount,bytes}] }
|
||||
// <tag>/<lib>/manifest per-lib SyncManifest { "model3d/<name>": {hash,size,mtime} }
|
||||
// blobs/sha256/<hash> model bodies (brotli, content-addressed, shared)
|
||||
// blobs/registry.json published-blob index (hash → size) for cheap dedup
|
||||
//
|
||||
// Idempotent per tag: if <prefix>/<tag>/manifest.json exists the run SKIPS
|
||||
// (--force overrides). Blobs are skipped per-hash via the registry.
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, readdirSync, readFileSync, statSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import type { SyncManifest } from "../../web/pcbjam-shared/src/sync-wire.js";
|
||||
import {
|
||||
compressBytes,
|
||||
IMMUTABLE,
|
||||
makeStore,
|
||||
NO_STORE,
|
||||
putJSON,
|
||||
sha256hex,
|
||||
} from "./lib/cdn-store.mjs";
|
||||
|
||||
const MODELS_URL = "https://gitlab.com/kicad/libraries/kicad-packages3D.git";
|
||||
|
||||
/** Model file extensions we publish (locked: WRL + STEP). */
|
||||
const MODEL_EXTS = [".wrl", ".step", ".stp"];
|
||||
|
||||
interface Args {
|
||||
modelTag: string | null;
|
||||
modelsSrc: string | null;
|
||||
clone: string | null;
|
||||
driver: string;
|
||||
out: string | null;
|
||||
bucket: string;
|
||||
remote: boolean;
|
||||
prefix: string;
|
||||
force: boolean;
|
||||
libs: string[] | null;
|
||||
quality: number;
|
||||
compress: "br" | "none";
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): Args {
|
||||
const a: Args = {
|
||||
modelTag: null,
|
||||
modelsSrc: null,
|
||||
clone: null,
|
||||
driver: "local",
|
||||
out: null,
|
||||
bucket: "pcbjam-cdn",
|
||||
remote: false,
|
||||
prefix: "libs/kicad-models",
|
||||
force: false,
|
||||
libs: null,
|
||||
quality: 5,
|
||||
compress: "br",
|
||||
};
|
||||
for (let i = 2; i < argv.length; i++) {
|
||||
const next = () => argv[++i]!;
|
||||
switch (argv[i]) {
|
||||
case "--model-tag": a.modelTag = next(); break;
|
||||
case "--models-src": a.modelsSrc = next(); break;
|
||||
// Clone kicad-packages3D at --model-tag into <dir>/kicad-packages3D
|
||||
// (shallow; NOTE: multi-GB working tree) and use it as the source.
|
||||
case "--clone": a.clone = next(); break;
|
||||
case "--driver": a.driver = next(); break;
|
||||
case "--out": a.out = next(); break;
|
||||
case "--bucket": a.bucket = next(); break;
|
||||
case "--remote": a.remote = true; break;
|
||||
case "--prefix": a.prefix = next(); break;
|
||||
case "--force": a.force = true; break;
|
||||
// Dev subset: publish only these libs (names without .3dshapes).
|
||||
case "--libs": a.libs = next().split(",").map((s) => s.trim()).filter(Boolean); break;
|
||||
// Brotli quality for bodies (WRL/STEP are text-ish; 5 ≈ 4-5x, fast).
|
||||
case "--quality": a.quality = Number(next()); break;
|
||||
// "none" for --driver local when a plain static server (e.g. the vite dev
|
||||
// server) will serve the blobs — it can't send Content-Encoding: br.
|
||||
case "--compress": a.compress = next() as "br" | "none"; break;
|
||||
default: throw new Error(`unknown arg: ${argv[i]}`);
|
||||
}
|
||||
}
|
||||
if (!a.modelTag) throw new Error("--model-tag <kicad-packages3D tag> is required");
|
||||
if (!a.modelsSrc && !a.clone)
|
||||
throw new Error("need --models-src <kicad-packages3D checkout> or --clone <dir>");
|
||||
if (a.driver === "local" && !a.out) a.out = ".cdn-models-out";
|
||||
return a;
|
||||
}
|
||||
|
||||
function cloneShallow(url: string, dest: string, ref: string): void {
|
||||
if (existsSync(dest)) {
|
||||
console.log(`clone: ${dest} present — reusing`);
|
||||
return;
|
||||
}
|
||||
mkdirSync(dirname(dest), { recursive: true });
|
||||
console.log(`clone: ${url} @ ${ref} → ${dest} (multi-GB — this takes a while)`);
|
||||
execFileSync("git", ["clone", "--depth", "1", "--branch", ref, url, dest], {
|
||||
stdio: "inherit",
|
||||
});
|
||||
}
|
||||
|
||||
/** All `<lib>.3dshapes` dirs under the checkout root (non-recursive: the repo is flat). */
|
||||
function listModelLibs(src: string): Array<{ id: string; dir: string }> {
|
||||
return readdirSync(src, { withFileTypes: true })
|
||||
.filter((d) => d.isDirectory() && d.name.endsWith(".3dshapes"))
|
||||
.map((d) => ({ id: d.name.slice(0, -".3dshapes".length), dir: join(src, d.name) }))
|
||||
.sort((x, y) => x.id.localeCompare(y.id));
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const a = parseArgs(process.argv);
|
||||
const store = makeStore(a.driver, a);
|
||||
const topKey = `${a.prefix}/${a.modelTag}/manifest.json`;
|
||||
const registryKey = `${a.prefix}/blobs/registry.json`;
|
||||
|
||||
if (!a.force && store.getJSON(topKey)) {
|
||||
console.log(`publish-models: ${topKey} already published — skipping (use --force)`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (a.clone) {
|
||||
const dest = join(a.clone, "kicad-packages3D");
|
||||
cloneShallow(MODELS_URL, dest, a.modelTag!);
|
||||
a.modelsSrc ??= dest;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`publish-models: tag=${a.modelTag} driver=${store.kind} → ${a.prefix}/${a.modelTag}/`,
|
||||
);
|
||||
|
||||
// Published-blob index: hash → original size. One GET up front, one PUT at the
|
||||
// end — the per-blob "does it exist" probe would otherwise be an R2 round-trip
|
||||
// per model (tens of thousands).
|
||||
const registry: Record<string, number> =
|
||||
(store.getJSON(registryKey) as Record<string, number> | null) ?? {};
|
||||
let blobsPut = 0;
|
||||
let blobsSkipped = 0;
|
||||
|
||||
let libs = listModelLibs(a.modelsSrc!);
|
||||
if (a.libs) {
|
||||
const want = new Set(a.libs);
|
||||
libs = libs.filter((l) => want.has(l.id));
|
||||
const missing = a.libs.filter((id) => !libs.some((l) => l.id === id));
|
||||
if (missing.length) console.warn(`publish-models: libs not found: ${missing.join(", ")}`);
|
||||
}
|
||||
if (!libs.length) throw new Error(`no .3dshapes libs under ${a.modelsSrc}`);
|
||||
|
||||
const topLibs: Array<{ id: string; itemCount: number; bytes: number }> = [];
|
||||
let totalItems = 0;
|
||||
let totalBytes = 0;
|
||||
|
||||
for (const lib of libs) {
|
||||
const files = readdirSync(lib.dir)
|
||||
.filter((f) => MODEL_EXTS.some((ext) => f.toLowerCase().endsWith(ext)))
|
||||
.sort();
|
||||
if (!files.length) continue;
|
||||
|
||||
const entries: SyncManifest["entries"] = {};
|
||||
let libBytes = 0;
|
||||
for (const f of files) {
|
||||
const p = join(lib.dir, f);
|
||||
if (!statSync(p).isFile()) continue;
|
||||
const body = readFileSync(p);
|
||||
const hash = sha256hex(body);
|
||||
entries[`model3d/${f}`] = { hash, size: body.length, mtime: 0 };
|
||||
libBytes += body.length;
|
||||
|
||||
if (registry[hash] === undefined) {
|
||||
// WRL and STEP are text formats — brotli gets ~4-5x. The browser fetch
|
||||
// transparently decodes, so IDB caches (and hashes refer to) the
|
||||
// ORIGINAL bytes; `no-transform` keeps the edge from re-encoding.
|
||||
const { bytes, encoding } = compressBytes(body, a.compress, a.quality);
|
||||
store.put(`${a.prefix}/blobs/sha256/${hash}`, bytes, {
|
||||
contentType: "application/octet-stream",
|
||||
contentEncoding: encoding,
|
||||
cacheControl: IMMUTABLE,
|
||||
});
|
||||
registry[hash] = body.length;
|
||||
blobsPut++;
|
||||
} else {
|
||||
blobsSkipped++;
|
||||
}
|
||||
}
|
||||
|
||||
const manifest: SyncManifest = { version: 1, entries };
|
||||
putJSON(store, `${a.prefix}/${a.modelTag}/${lib.id}/manifest`, manifest, IMMUTABLE);
|
||||
topLibs.push({ id: lib.id, itemCount: files.length, bytes: libBytes });
|
||||
totalItems += files.length;
|
||||
totalBytes += libBytes;
|
||||
console.log(` ${lib.id}: ${files.length} models (${(libBytes / 1e6).toFixed(1)} MB)`);
|
||||
}
|
||||
|
||||
putJSON(store, registryKey, registry, NO_STORE);
|
||||
putJSON(store, topKey, { schema: 1, tag: a.modelTag, libs: topLibs }, IMMUTABLE);
|
||||
|
||||
console.log(
|
||||
`publish-models: done — ${topLibs.length} libs, ${totalItems} models ` +
|
||||
`(${(totalBytes / 1e6).toFixed(0)} MB raw), blobs put=${blobsPut} deduped=${blobsSkipped} → ${topKey}`,
|
||||
);
|
||||
if (store.kind === "local") console.log(`local layout under: ${a.out}`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
Loading…
Reference in a new issue