feat(3d): lazy 3D model delivery R2→IDB→WASM
- cdnModelsSource: sparse per-lib stacks over the models CDN layout (libs/kicad-models/<tag>/<lib>/manifest + content-addressed blobs) - models-bridge: board prescan ((model …) scan → prefetch, 4/4 in 74ms on the demo board) + the kind=model3d 'ensure' op answering absolute MEMFS paths; only board-referenced bodies ever enter IDB - boot/runner/config/WasmTool wiring (VITE_MODELS_MANIFEST_URL, KICAD*_3DMODEL_DIR seeding, prefetch badge) - StorageUsageCard on HomePage: per-kind cached sizes + delete-3D-cache - publish-models.ts (brotli content-addressed publish, 606→93MB for the pic_programmer lib set) + dev-demo --models-tag/--models-local - e2e: 3d-viewer-models.spec.ts (bridge normalize/dedup + STEP/WRL render); submodule bumps: kicad (static 3D plugins + ensure hook), pcbjam-shared (sparse sync layer) Spec + findings: docs/features/3d-models (private repo). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014AT7gVHRktDYoQ68S4x6A4
This commit is contained in:
Gergő Törcsvári
parent
3efcdcf91c
commit
6f3f3bcf00
19 changed files with 1351 additions and 4 deletions
2
kicad
2
kicad
|
|
@ -1 +1 @@
|
|||
Subproject commit e8db3d35db635946b03ee0ae8698e1d3aad79a3c
|
||||
Subproject commit 9d771391886ba3877dda90f72f57ad37d931fef9
|
||||
|
|
@ -72,6 +72,8 @@ function parseArgs(argv) {
|
|||
contentTag: null, // use the live CDN gallery for this tag (else build locally)
|
||||
galleryTag: "demo-local", // path tag for the locally-built gallery
|
||||
noGallery: false, // disable the example gallery (local-folder + IDB only)
|
||||
modelsTag: null, // 3D models snapshot tag (live CDN, or the local dir's tag)
|
||||
modelsLocal: null, // local publish-models --driver local output dir (serve same-origin)
|
||||
port: null,
|
||||
repo: "https://github.com/emergence-engineering/pcbjam",
|
||||
};
|
||||
|
|
@ -87,6 +89,8 @@ function parseArgs(argv) {
|
|||
case "--no-gallery": a.noGallery = true; break;
|
||||
case "--port": a.port = next(); break;
|
||||
case "--repo": a.repo = next(); break;
|
||||
case "--models-tag": a.modelsTag = next(); break;
|
||||
case "--models-local": a.modelsLocal = next(); break;
|
||||
case "-h": case "--help": a.help = true; break;
|
||||
default: throw new Error(`unknown arg: ${argv[i]}`);
|
||||
}
|
||||
|
|
@ -108,6 +112,9 @@ const HELP = `dev-demo.mjs — run the standalone locally in demo mode (R2-only
|
|||
--content-tag <tag> pin the LIVE CDN gallery for this release tag (default: build+serve the gallery locally)
|
||||
--gallery-tag <tag> path tag for the locally-built gallery (default demo-local)
|
||||
--no-gallery disable the example gallery (local-folder + IDB projects only)
|
||||
--models-tag <tag> enable lazy 3D models from the CDN snapshot at this tag
|
||||
--models-local <dir> serve a local publish-models layout (--driver local --compress none)
|
||||
same-origin instead of the CDN (requires --models-tag)
|
||||
--port <n> dev server port
|
||||
|
||||
By default the read-only example gallery (deploy/demo/gallery.json) is built
|
||||
|
|
@ -142,6 +149,26 @@ function main() {
|
|||
delete env.VITE_LIBS_MANIFEST_URL;
|
||||
}
|
||||
|
||||
// --- 3D models: lazy per-board bodies (docs/features/3d-models). Off unless a
|
||||
// tag is given. --models-local <publish-models --out dir> serves that
|
||||
// layout same-origin at /models-cdn via a public/ symlink (publish it with
|
||||
// --compress none — the dev server can't send Content-Encoding: br);
|
||||
// otherwise the live CDN snapshot for --models-tag is used.
|
||||
if (a.modelsTag && a.modelsLocal) {
|
||||
const link = join(repoRoot, "web/standalone/public/models-cdn");
|
||||
try {
|
||||
if (lstatSync(link)) rmSync(link, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* no existing link */
|
||||
}
|
||||
symlinkSync(resolve(a.modelsLocal, "libs/kicad-models"), link);
|
||||
env.VITE_MODELS_MANIFEST_URL = `/models-cdn/${a.modelsTag}/manifest.json`;
|
||||
} else if (a.modelsTag) {
|
||||
env.VITE_MODELS_MANIFEST_URL = `${a.cdn}/libs/kicad-models/${a.modelsTag}/manifest.json`;
|
||||
} else {
|
||||
delete env.VITE_MODELS_MANIFEST_URL;
|
||||
}
|
||||
|
||||
// --- No backend: collab is cross-tab only, document bytes are local (api path),
|
||||
// loaded folders persist to a browser-local IndexedDB project.
|
||||
env.VITE_YJS_PROVIDER = "broadcastchannel";
|
||||
|
|
@ -189,6 +216,7 @@ function main() {
|
|||
console.log(` VITE_LIBS_SOURCE=${env.VITE_LIBS_SOURCE}${env.VITE_LIBS_MANIFEST_URL ? ` (${env.VITE_LIBS_MANIFEST_URL})` : ""}`);
|
||||
console.log(` VITE_WASM_ROOT=${env.VITE_WASM_ROOT}${env.VITE_WASM_MANIFEST ? ` (${env.VITE_WASM_MANIFEST})` : " (local build)"}`);
|
||||
console.log(` VITE_PROJECT_SOURCE=${env.VITE_PROJECT_SOURCE}${env.VITE_PROJECT_MANIFEST_URL ? ` (${env.VITE_PROJECT_MANIFEST_URL})` : ""}`);
|
||||
console.log(` VITE_MODELS_MANIFEST_URL=${env.VITE_MODELS_MANIFEST_URL ?? "(unset — 3D models off)"}`);
|
||||
console.log(` VITE_YJS_PROVIDER=${env.VITE_YJS_PROVIDER} VITE_DOC_SOURCE=${env.VITE_DOC_SOURCE} VITE_LOCAL_PROJECTS=${env.VITE_LOCAL_PROJECTS}`);
|
||||
|
||||
const child = spawn("pnpm", viteArgs, { cwd: repoRoot, env, stdio: "inherit" });
|
||||
|
|
|
|||
219
scripts/deploy/publish-models.ts
Normal file
219
scripts/deploy/publish-models.ts
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
// Publish KiCad 3D models (kicad-packages3D) to the CDN as r2-idb-sync SPARSE
|
||||
// origins: per-lib manifests keyed by the upstream tag, bodies content-addressed
|
||||
// under a shared blobs/ prefix (deduped across tags — models rarely change).
|
||||
// The standalone's cdnModelsSource opens each lib as a sparse layer: manifest
|
||||
// synced eagerly (small), bodies fetched exactly when a board references them.
|
||||
// See web/standalone/src/wasm/libs/models-source.ts + docs/features/3d-models.
|
||||
//
|
||||
// npx tsx scripts/deploy/publish-models.ts --model-tag 10.0.0 \
|
||||
// --models-src <kicad-packages3D checkout> --driver local --out /tmp/cdn-models
|
||||
// npx tsx scripts/deploy/publish-models.ts --model-tag 10.0.0 --models-src … \
|
||||
// --driver r2 --bucket pcbjam-cdn --remote
|
||||
// # dev subset: only a few libs
|
||||
// … --libs Resistor_SMD,Capacitor_SMD,Package_QFP
|
||||
//
|
||||
// Layout under `<prefix>` (default libs/kicad-models):
|
||||
// <tag>/manifest.json top index { schema, tag, libs:[{id,itemCount,bytes}] }
|
||||
// <tag>/<lib>/manifest per-lib SyncManifest { "model3d/<name>": {hash,size,mtime} }
|
||||
// blobs/sha256/<hash> model bodies (brotli, content-addressed, shared)
|
||||
// blobs/registry.json published-blob index (hash → size) for cheap dedup
|
||||
//
|
||||
// Idempotent per tag: if <prefix>/<tag>/manifest.json exists the run SKIPS
|
||||
// (--force overrides). Blobs are skipped per-hash via the registry.
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, readdirSync, readFileSync, statSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import type { SyncManifest } from "../../web/pcbjam-shared/src/sync-wire.js";
|
||||
import {
|
||||
compressBytes,
|
||||
IMMUTABLE,
|
||||
makeStore,
|
||||
NO_STORE,
|
||||
putJSON,
|
||||
sha256hex,
|
||||
} from "./lib/cdn-store.mjs";
|
||||
|
||||
const MODELS_URL = "https://gitlab.com/kicad/libraries/kicad-packages3D.git";
|
||||
|
||||
/** Model file extensions we publish (locked: WRL + STEP). */
|
||||
const MODEL_EXTS = [".wrl", ".step", ".stp"];
|
||||
|
||||
interface Args {
|
||||
modelTag: string | null;
|
||||
modelsSrc: string | null;
|
||||
clone: string | null;
|
||||
driver: string;
|
||||
out: string | null;
|
||||
bucket: string;
|
||||
remote: boolean;
|
||||
prefix: string;
|
||||
force: boolean;
|
||||
libs: string[] | null;
|
||||
quality: number;
|
||||
compress: "br" | "none";
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): Args {
|
||||
const a: Args = {
|
||||
modelTag: null,
|
||||
modelsSrc: null,
|
||||
clone: null,
|
||||
driver: "local",
|
||||
out: null,
|
||||
bucket: "pcbjam-cdn",
|
||||
remote: false,
|
||||
prefix: "libs/kicad-models",
|
||||
force: false,
|
||||
libs: null,
|
||||
quality: 5,
|
||||
compress: "br",
|
||||
};
|
||||
for (let i = 2; i < argv.length; i++) {
|
||||
const next = () => argv[++i]!;
|
||||
switch (argv[i]) {
|
||||
case "--model-tag": a.modelTag = next(); break;
|
||||
case "--models-src": a.modelsSrc = next(); break;
|
||||
// Clone kicad-packages3D at --model-tag into <dir>/kicad-packages3D
|
||||
// (shallow; NOTE: multi-GB working tree) and use it as the source.
|
||||
case "--clone": a.clone = next(); break;
|
||||
case "--driver": a.driver = next(); break;
|
||||
case "--out": a.out = next(); break;
|
||||
case "--bucket": a.bucket = next(); break;
|
||||
case "--remote": a.remote = true; break;
|
||||
case "--prefix": a.prefix = next(); break;
|
||||
case "--force": a.force = true; break;
|
||||
// Dev subset: publish only these libs (names without .3dshapes).
|
||||
case "--libs": a.libs = next().split(",").map((s) => s.trim()).filter(Boolean); break;
|
||||
// Brotli quality for bodies (WRL/STEP are text-ish; 5 ≈ 4-5x, fast).
|
||||
case "--quality": a.quality = Number(next()); break;
|
||||
// "none" for --driver local when a plain static server (e.g. the vite dev
|
||||
// server) will serve the blobs — it can't send Content-Encoding: br.
|
||||
case "--compress": a.compress = next() as "br" | "none"; break;
|
||||
default: throw new Error(`unknown arg: ${argv[i]}`);
|
||||
}
|
||||
}
|
||||
if (!a.modelTag) throw new Error("--model-tag <kicad-packages3D tag> is required");
|
||||
if (!a.modelsSrc && !a.clone)
|
||||
throw new Error("need --models-src <kicad-packages3D checkout> or --clone <dir>");
|
||||
if (a.driver === "local" && !a.out) a.out = ".cdn-models-out";
|
||||
return a;
|
||||
}
|
||||
|
||||
function cloneShallow(url: string, dest: string, ref: string): void {
|
||||
if (existsSync(dest)) {
|
||||
console.log(`clone: ${dest} present — reusing`);
|
||||
return;
|
||||
}
|
||||
mkdirSync(dirname(dest), { recursive: true });
|
||||
console.log(`clone: ${url} @ ${ref} → ${dest} (multi-GB — this takes a while)`);
|
||||
execFileSync("git", ["clone", "--depth", "1", "--branch", ref, url, dest], {
|
||||
stdio: "inherit",
|
||||
});
|
||||
}
|
||||
|
||||
/** All `<lib>.3dshapes` dirs under the checkout root (non-recursive: the repo is flat). */
|
||||
function listModelLibs(src: string): Array<{ id: string; dir: string }> {
|
||||
return readdirSync(src, { withFileTypes: true })
|
||||
.filter((d) => d.isDirectory() && d.name.endsWith(".3dshapes"))
|
||||
.map((d) => ({ id: d.name.slice(0, -".3dshapes".length), dir: join(src, d.name) }))
|
||||
.sort((x, y) => x.id.localeCompare(y.id));
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const a = parseArgs(process.argv);
|
||||
const store = makeStore(a.driver, a);
|
||||
const topKey = `${a.prefix}/${a.modelTag}/manifest.json`;
|
||||
const registryKey = `${a.prefix}/blobs/registry.json`;
|
||||
|
||||
if (!a.force && store.getJSON(topKey)) {
|
||||
console.log(`publish-models: ${topKey} already published — skipping (use --force)`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (a.clone) {
|
||||
const dest = join(a.clone, "kicad-packages3D");
|
||||
cloneShallow(MODELS_URL, dest, a.modelTag!);
|
||||
a.modelsSrc ??= dest;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`publish-models: tag=${a.modelTag} driver=${store.kind} → ${a.prefix}/${a.modelTag}/`,
|
||||
);
|
||||
|
||||
// Published-blob index: hash → original size. One GET up front, one PUT at the
|
||||
// end — the per-blob "does it exist" probe would otherwise be an R2 round-trip
|
||||
// per model (tens of thousands).
|
||||
const registry: Record<string, number> =
|
||||
(store.getJSON(registryKey) as Record<string, number> | null) ?? {};
|
||||
let blobsPut = 0;
|
||||
let blobsSkipped = 0;
|
||||
|
||||
let libs = listModelLibs(a.modelsSrc!);
|
||||
if (a.libs) {
|
||||
const want = new Set(a.libs);
|
||||
libs = libs.filter((l) => want.has(l.id));
|
||||
const missing = a.libs.filter((id) => !libs.some((l) => l.id === id));
|
||||
if (missing.length) console.warn(`publish-models: libs not found: ${missing.join(", ")}`);
|
||||
}
|
||||
if (!libs.length) throw new Error(`no .3dshapes libs under ${a.modelsSrc}`);
|
||||
|
||||
const topLibs: Array<{ id: string; itemCount: number; bytes: number }> = [];
|
||||
let totalItems = 0;
|
||||
let totalBytes = 0;
|
||||
|
||||
for (const lib of libs) {
|
||||
const files = readdirSync(lib.dir)
|
||||
.filter((f) => MODEL_EXTS.some((ext) => f.toLowerCase().endsWith(ext)))
|
||||
.sort();
|
||||
if (!files.length) continue;
|
||||
|
||||
const entries: SyncManifest["entries"] = {};
|
||||
let libBytes = 0;
|
||||
for (const f of files) {
|
||||
const p = join(lib.dir, f);
|
||||
if (!statSync(p).isFile()) continue;
|
||||
const body = readFileSync(p);
|
||||
const hash = sha256hex(body);
|
||||
entries[`model3d/${f}`] = { hash, size: body.length, mtime: 0 };
|
||||
libBytes += body.length;
|
||||
|
||||
if (registry[hash] === undefined) {
|
||||
// WRL and STEP are text formats — brotli gets ~4-5x. The browser fetch
|
||||
// transparently decodes, so IDB caches (and hashes refer to) the
|
||||
// ORIGINAL bytes; `no-transform` keeps the edge from re-encoding.
|
||||
const { bytes, encoding } = compressBytes(body, a.compress, a.quality);
|
||||
store.put(`${a.prefix}/blobs/sha256/${hash}`, bytes, {
|
||||
contentType: "application/octet-stream",
|
||||
contentEncoding: encoding,
|
||||
cacheControl: IMMUTABLE,
|
||||
});
|
||||
registry[hash] = body.length;
|
||||
blobsPut++;
|
||||
} else {
|
||||
blobsSkipped++;
|
||||
}
|
||||
}
|
||||
|
||||
const manifest: SyncManifest = { version: 1, entries };
|
||||
putJSON(store, `${a.prefix}/${a.modelTag}/${lib.id}/manifest`, manifest, IMMUTABLE);
|
||||
topLibs.push({ id: lib.id, itemCount: files.length, bytes: libBytes });
|
||||
totalItems += files.length;
|
||||
totalBytes += libBytes;
|
||||
console.log(` ${lib.id}: ${files.length} models (${(libBytes / 1e6).toFixed(1)} MB)`);
|
||||
}
|
||||
|
||||
putJSON(store, registryKey, registry, NO_STORE);
|
||||
putJSON(store, topKey, { schema: 1, tag: a.modelTag, libs: topLibs }, IMMUTABLE);
|
||||
|
||||
console.log(
|
||||
`publish-models: done — ${topLibs.length} libs, ${totalItems} models ` +
|
||||
`(${(totalBytes / 1e6).toFixed(0)} MB raw), blobs put=${blobsPut} deduped=${blobsSkipped} → ${topKey}`,
|
||||
);
|
||||
if (store.kind === "local") console.log(`local layout under: ${a.out}`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
255
tests/kicad/3d-viewer-models.spec.ts
Normal file
255
tests/kicad/3d-viewer-models.spec.ts
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
import type { Page } from '@playwright/test';
|
||||
import { test, expect } from './fixtures';
|
||||
import { clickMenuBarItem, clickMenuItem } from '../e2e/utils/element-tracker';
|
||||
import { injectFromSubmodule } from './utils/fs-inject';
|
||||
import { waitForBoardLoaded } from './utils/board-ready';
|
||||
import { waitForPcbnew } from './utils/pcbnew-ready';
|
||||
|
||||
/**
|
||||
* 3D viewer COMPONENT MODELS e2e (docs/features/3d-models): load pic_programmer,
|
||||
* open the 3D viewer, and verify the model-delivery machinery end to end at the
|
||||
* KiCad/wasm level:
|
||||
*
|
||||
* 1. Statically linked format plugins (vrml + oce — upstream loads them via
|
||||
* dlopen, which wasm doesn't have) parse real model files.
|
||||
* 2. Project-local models resolve exactly as upstream: the board references
|
||||
* `${KIPRJMOD}/libs/3d_shapes/*.wrl`, injected with the project.
|
||||
* 3. The lazy-fetch fallback (S3D_CACHE::load → PCBJAM_3D::EnsureModelFile →
|
||||
* `kicadLibs.request("ensure", …, "model3d")`) asks JS for every
|
||||
* `${KICAD*_3DMODEL_DIR}` ref, with the ref NORMALIZED to
|
||||
* `<lib>.3dshapes/<name>.<ext>` — and a served ref (the stub writes the
|
||||
* bytes into MEMFS and answers "1") then resolves and renders.
|
||||
*
|
||||
* The stub provider stands in for the standalone's models-bridge (which fetches
|
||||
* from the CDN into IDB); here it serves ONE in-repo STEP fixture under a
|
||||
* board-referenced name — geometry is a USB-C connector where a DIP-8 socket
|
||||
* belongs, which is irrelevant: the assertion is parse+render, not fidelity.
|
||||
*/
|
||||
|
||||
const KICAD_VERSION_DIR = '10.0';
|
||||
const PROJECT_DIR_MEMFS = `/home/kicad/documents/kicad/${KICAD_VERSION_DIR}/projects`;
|
||||
|
||||
// The JS-owned MEMFS root the stub writes model bodies under — the same dir
|
||||
// the standalone's models-bridge uses (constants.ts MODELS_3D_ROOT). Its exact
|
||||
// location is immaterial: the ensure protocol answers with the ABSOLUTE path
|
||||
// and S3D_CACHE loads it directly (env-var expansion never resolves
|
||||
// ${KICAD*_3DMODEL_DIR} refs in the wasm runtime — see
|
||||
// docs/features/3d-models/0001).
|
||||
const MODELS_ROOT_MEMFS = '/pcbjam/3dmodels';
|
||||
|
||||
// The board ref the stub provider serves (normalized form the bridge must ask
|
||||
// for), and the in-repo STEP whose bytes stand in for it.
|
||||
const SERVED_REF = 'Package_DIP.3dshapes/DIP-8_W7.62mm.step';
|
||||
const STEP_FIXTURE = 'kicad/demos/openair-max/Libraries/HRO_TYPE-C-31-M-12.step';
|
||||
|
||||
const DEMO = { name: 'pic_programmer', dir: 'pic_programmer', stem: 'pic_programmer' } as const;
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
__modelEnsures?: Array<{ op: string; arg: string; kind: string }>;
|
||||
__stepFixtureB64?: string;
|
||||
}
|
||||
}
|
||||
|
||||
/** Record every model3d bridge request; serve SERVED_REF from the fixture. */
|
||||
async function installModelProviderStub(page: Page, serveAll = false): Promise<void> {
|
||||
await page.evaluate(
|
||||
({ stockDir, servedRef, serveAll }) => {
|
||||
window.__modelEnsures = [];
|
||||
(globalThis as any).kicadLibs = {
|
||||
request: async (op: string, _lib: string, arg: string, kind: string) => {
|
||||
if (kind !== 'model3d') return null;
|
||||
window.__modelEnsures!.push({ op, arg, kind });
|
||||
console.log(`[TEST-3D] ensure request: ${op} ${arg}`);
|
||||
if (op !== 'ensure' || (!serveAll && arg !== servedRef)) return null;
|
||||
|
||||
const b64 = window.__stepFixtureB64!;
|
||||
const binary = atob(b64);
|
||||
const data = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i++) data[i] = binary.charCodeAt(i);
|
||||
|
||||
// Mirror models-bridge.ts ensureModelInMemfs: write under the
|
||||
// JS-owned model root and answer with the ABSOLUTE path —
|
||||
// S3D_CACHE loads it directly (no env-var expansion needed).
|
||||
// @ts-expect-error — Emscripten FS lives on window
|
||||
const FS = (window as any).FS;
|
||||
const dest = `${stockDir}/${arg}`;
|
||||
FS.mkdirTree(dest.slice(0, dest.lastIndexOf('/')));
|
||||
FS.writeFile(dest, data);
|
||||
console.log(`[TEST-3D] served ${arg} → ${dest} (${data.length} bytes)`);
|
||||
return dest;
|
||||
},
|
||||
};
|
||||
},
|
||||
{ stockDir: MODELS_ROOT_MEMFS, servedRef: SERVED_REF, serveAll },
|
||||
);
|
||||
}
|
||||
|
||||
async function loadBoard(page: Page, testLogger: { consoleLogs: string[]; errors: string[] }): Promise<void> {
|
||||
const pcbFilename = `${DEMO.stem}.kicad_pcb`;
|
||||
const proFilename = `${DEMO.stem}.kicad_pro`;
|
||||
|
||||
await injectFromSubmodule(page, `kicad/demos/${DEMO.dir}/${pcbFilename}`,
|
||||
`${PROJECT_DIR_MEMFS}/${pcbFilename}`);
|
||||
await injectFromSubmodule(page, `kicad/demos/${DEMO.dir}/${proFilename}`,
|
||||
`${PROJECT_DIR_MEMFS}/${proFilename}`);
|
||||
// Project-local 3D models — the board references them as
|
||||
// ${KIPRJMOD}/libs/3d_shapes/<name>.wrl; resolved by the stock resolver, so
|
||||
// they must NOT go through the ensure bridge (asserted below).
|
||||
await injectFromSubmodule(page, `kicad/demos/${DEMO.dir}/libs/3d_shapes/textool_40.wrl`,
|
||||
`${PROJECT_DIR_MEMFS}/libs/3d_shapes/textool_40.wrl`);
|
||||
await injectFromSubmodule(page, `kicad/demos/${DEMO.dir}/libs/3d_shapes/adjustable_rx2v4.wrl`,
|
||||
`${PROJECT_DIR_MEMFS}/libs/3d_shapes/adjustable_rx2v4.wrl`);
|
||||
|
||||
expect(await clickMenuBarItem(page, 'File'), 'File menu should be findable').toBe(true);
|
||||
await page.waitForTimeout(400);
|
||||
expect(await clickMenuItem(page, 'Open...'), 'Open… menu item should be findable').toBe(true);
|
||||
|
||||
await page.waitForFunction(() => {
|
||||
const registry = window.wxElementRegistry;
|
||||
return !!registry && registry.findAll({ visible: true })
|
||||
.some((el) => el.typeName === 'wxFileDialog');
|
||||
}, null, { timeout: 15000 });
|
||||
await page.waitForTimeout(1000);
|
||||
|
||||
const filenameInput = await page.evaluate(() => {
|
||||
const registry = window.wxElementRegistry;
|
||||
if (!registry) return null;
|
||||
const text = registry.findAll({ visible: true })
|
||||
.find((el) => el.typeName === 'wxTextCtrl' && el.name === 'text');
|
||||
return text ? { x: text.centerX, y: text.centerY } : null;
|
||||
});
|
||||
expect(filenameInput, 'filename text input should be visible').not.toBeNull();
|
||||
if (!filenameInput) throw new Error('filename text input not found');
|
||||
|
||||
await page.mouse.click(filenameInput.x, filenameInput.y);
|
||||
await page.waitForTimeout(200);
|
||||
await page.keyboard.type(pcbFilename);
|
||||
await page.waitForTimeout(300);
|
||||
await page.keyboard.press('Enter');
|
||||
await page.waitForTimeout(1000);
|
||||
|
||||
const result = await waitForBoardLoaded(page, testLogger, 60000);
|
||||
console.log(`[TEST] ${DEMO.name} board-ready result: ${result}`);
|
||||
}
|
||||
|
||||
function countGlCanvases(page: Page): Promise<number> {
|
||||
return page.evaluate(() => document.querySelectorAll('canvas[id^="glcanvas-"]').length);
|
||||
}
|
||||
|
||||
async function openThreeDViewer(page: Page, glBefore: number): Promise<number> {
|
||||
let opened = false;
|
||||
if (await clickMenuBarItem(page, 'View')) {
|
||||
await page.waitForTimeout(400);
|
||||
opened = await clickMenuItem(page, '3D Viewer');
|
||||
}
|
||||
if (!opened) {
|
||||
console.log('[TEST] View → 3D Viewer not found via menu; trying Alt+3');
|
||||
await page.keyboard.press('Escape');
|
||||
await page.waitForTimeout(200);
|
||||
await page.keyboard.press('Alt+3');
|
||||
}
|
||||
|
||||
await page.waitForFunction(() => {
|
||||
return !!document.querySelector('#window-container [id^="window-"]')
|
||||
|| document.querySelectorAll('canvas[id^="glcanvas-"]').length > 0;
|
||||
}, null, { timeout: 60000 });
|
||||
|
||||
await page.waitForFunction((before: number) =>
|
||||
document.querySelectorAll('canvas[id^="glcanvas-"]').length > before,
|
||||
glBefore, { timeout: 60000 });
|
||||
|
||||
const glAfter = await countGlCanvases(page);
|
||||
console.log(`[TEST] glcanvas count after opening 3D viewer: ${glAfter}`);
|
||||
expect(glAfter, 'a new WebGL canvas should appear for the 3D viewer').toBeGreaterThan(glBefore);
|
||||
return glAfter;
|
||||
}
|
||||
|
||||
test.describe('3D viewer component models', () => {
|
||||
test.describe.configure({ mode: 'serial' });
|
||||
test.setTimeout(240000);
|
||||
|
||||
test('resolves project models, lazy-fetches lib models via the bridge, renders', async ({ page, testLogger }) => {
|
||||
await page.goto('/kicad/pcbnew.html');
|
||||
await waitForPcbnew(page);
|
||||
|
||||
// Stash the STEP fixture bytes + install the provider stub BEFORE the
|
||||
// viewer can issue any ensure request.
|
||||
const fs = require('fs') as typeof import('fs');
|
||||
const path = require('path') as typeof import('path');
|
||||
const fixtureAbs = path.resolve(__dirname, '..', '..', STEP_FIXTURE);
|
||||
await page.evaluate(
|
||||
(b64: string) => { window.__stepFixtureB64 = b64; },
|
||||
fs.readFileSync(fixtureAbs).toString('base64'),
|
||||
);
|
||||
await installModelProviderStub(page);
|
||||
|
||||
await loadBoard(page, testLogger);
|
||||
|
||||
const glBefore = await countGlCanvases(page);
|
||||
await openThreeDViewer(page, glBefore);
|
||||
|
||||
// Scene build + progressive raytrace passes.
|
||||
await page.waitForTimeout(8000);
|
||||
await page.screenshot({ path: `test-results/3d-viewer-models-${DEMO.name}.png`, scale: 'css' });
|
||||
|
||||
// --- bridge assertions -------------------------------------------------
|
||||
const ensures = await page.evaluate(() => window.__modelEnsures ?? []);
|
||||
console.log(`[TEST] ensure requests: ${ensures.length}`);
|
||||
for (const e of ensures.slice(0, 30)) console.log(`[TEST] ${e.op} ${e.arg}`);
|
||||
|
||||
// Every ${KICAD*_3DMODEL_DIR} ref crossed the bridge, normalized.
|
||||
const args = ensures.map((e) => e.arg);
|
||||
expect(args, 'the served lib ref must cross the bridge normalized')
|
||||
.toContain(SERVED_REF);
|
||||
expect(args.every((a) => /^[^/${]+\.3dshapes\//.test(a)),
|
||||
'every bridge ref is a normalized <lib>.3dshapes/<file> path').toBe(true);
|
||||
// Project-local (${KIPRJMOD}) models resolve natively — never bridged.
|
||||
expect(args.some((a) => a.includes('textool_40') || a.includes('adjustable_rx2v4')),
|
||||
'project-local models must not go through the ensure bridge').toBe(false);
|
||||
// Board refs are unique per model file — the C++ memo must not re-ask.
|
||||
expect(new Set(args).size, 'ensure requests are deduplicated').toBe(args.length);
|
||||
|
||||
// The served model landed in MEMFS where the resolver looks.
|
||||
const servedSize = await page.evaluate(
|
||||
({ stockDir, servedRef }) => {
|
||||
// @ts-expect-error — Emscripten FS lives on window
|
||||
const FS = (window as any).FS;
|
||||
try { return FS.stat(`${stockDir}/${servedRef}`).size as number; }
|
||||
catch { return -1; }
|
||||
},
|
||||
{ stockDir: MODELS_ROOT_MEMFS, servedRef: SERVED_REF },
|
||||
);
|
||||
expect(servedSize, 'served STEP written into the model root').toBeGreaterThan(1000);
|
||||
|
||||
// --- render assertion --------------------------------------------------
|
||||
const render = await page.evaluate(() => {
|
||||
const list = document.querySelectorAll('canvas[id^="glcanvas-"]');
|
||||
const el = list[list.length - 1] as HTMLCanvasElement;
|
||||
const tmp = document.createElement('canvas');
|
||||
tmp.width = el.width;
|
||||
tmp.height = el.height;
|
||||
const ctx = tmp.getContext('2d')!;
|
||||
ctx.drawImage(el, 0, 0);
|
||||
const colors = new Set<string>();
|
||||
for (let i = 0; i < 16; i++) {
|
||||
for (let j = 0; j < 16; j++) {
|
||||
const d = ctx.getImageData(Math.floor(el.width * i / 16),
|
||||
Math.floor(el.height * j / 16), 1, 1).data;
|
||||
colors.add(`${d[0]},${d[1]},${d[2]}`);
|
||||
}
|
||||
}
|
||||
return { id: el.id, w: el.width, h: el.height, distinctColors: colors.size,
|
||||
dataUrl: tmp.toDataURL('image/png') };
|
||||
});
|
||||
console.log(`[TEST] 3D canvas ${render.id} ${render.w}x${render.h}, distinct colours: ${render.distinctColors}`);
|
||||
const b64 = render.dataUrl.replace(/^data:image\/png;base64,/, '');
|
||||
fs.writeFileSync(`test-results/3d-viewer-models-${DEMO.name}-render.png`,
|
||||
Buffer.from(b64, 'base64'));
|
||||
expect(render.distinctColors,
|
||||
'the 3D viewer canvas should render the board + models, not a blank fill')
|
||||
.toBeGreaterThan(8);
|
||||
|
||||
expect(testLogger.errors, 'no page errors during the model flow').toEqual([]);
|
||||
});
|
||||
});
|
||||
|
|
@ -89,6 +89,7 @@ const PCBNEW_FAMILY_SPECS = [
|
|||
'**/pcbnew-move.spec.ts',
|
||||
// 3D viewer specs boot pcbnew.html (3D-enabled build) — same V8 routing.
|
||||
'**/3d-viewer.spec.ts',
|
||||
'**/3d-viewer-models.spec.ts',
|
||||
'**/footprint-3d-preview.spec.ts',
|
||||
];
|
||||
|
||||
|
|
|
|||
3
web/.gitignore
vendored
3
web/.gitignore
vendored
|
|
@ -13,3 +13,6 @@ dist/
|
|||
# .demo-cdn holds its generated bytes — both created by scripts/deploy/dev-demo.mjs.
|
||||
**/public/content
|
||||
**/standalone/.demo-cdn/
|
||||
# 3D models: public/models-cdn symlinks a local publish-models layout, created
|
||||
# by scripts/deploy/dev-demo.mjs --models-local — never committed.
|
||||
**/public/models-cdn
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit 9a1a269fdd9079921029ecf6e2dce89dece98fc8
|
||||
Subproject commit 067170e50f1c2ba19ffceb9815dcf5d34dc9e800
|
||||
235
web/standalone/src/components/StorageUsageCard.tsx
Normal file
235
web/standalone/src/components/StorageUsageCard.tsx
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
import * as React from "react";
|
||||
import { Database, Loader2, RefreshCw, Trash2 } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
|
||||
/**
|
||||
* Browser-storage usage for the editor's library caches, by kind, with a
|
||||
* "delete 3D model cache" action.
|
||||
*
|
||||
* All library data lives in r2-idb-sync IndexedDB databases named
|
||||
* `sync:<namespace>` (one per lib), each with a `bodies` store keyed
|
||||
* `"<kind>/<name>"`. Symbols/footprints are bulk-synced (whole-lib bundles);
|
||||
* 3D models are SPARSE — only the models boards actually rendered are stored
|
||||
* (namespaces `kicad-models:…`), which makes them the one cache that is both
|
||||
* potentially large and always safe to drop: manifests re-sync in one small
|
||||
* fetch and bodies lazily re-download exactly when a board needs them again.
|
||||
*/
|
||||
|
||||
const SYNC_PREFIX = "sync:";
|
||||
const MODELS_NS_PREFIX = `${SYNC_PREFIX}kicad-models:`;
|
||||
|
||||
interface KindUsage {
|
||||
bytes: number;
|
||||
items: number;
|
||||
}
|
||||
|
||||
interface StorageBreakdown {
|
||||
symbol: KindUsage;
|
||||
footprint: KindUsage;
|
||||
model3d: KindUsage;
|
||||
/** Whole-origin estimate (all IDB + caches), from navigator.storage. */
|
||||
originUsage: number | null;
|
||||
originQuota: number | null;
|
||||
/** Names of the model DBs (the delete target). */
|
||||
modelDbs: string[];
|
||||
}
|
||||
|
||||
function emptyUsage(): KindUsage {
|
||||
return { bytes: 0, items: 0 };
|
||||
}
|
||||
|
||||
/** Sum body sizes per kind in one `sync:*` DB via a cursor (no bulk getAll —
|
||||
* a model cache can be hundreds of MB and we only need the sizes). */
|
||||
function measureDb(name: string, into: StorageBreakdown): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
const req = indexedDB.open(name);
|
||||
// Never upgrade here (no version passed); a DB from a newer schema still
|
||||
// opens read-only for measuring.
|
||||
req.onerror = () => resolve();
|
||||
req.onsuccess = () => {
|
||||
const db = req.result;
|
||||
if (!db.objectStoreNames.contains("bodies")) {
|
||||
db.close();
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const tx = db.transaction("bodies", "readonly");
|
||||
const cursorReq = tx.objectStore("bodies").openCursor();
|
||||
cursorReq.onsuccess = () => {
|
||||
const cursor = cursorReq.result;
|
||||
if (!cursor) return; // tx completes → oncomplete below
|
||||
const key = String(cursor.key);
|
||||
const value = cursor.value as { byteLength?: number } | undefined;
|
||||
const size = value?.byteLength ?? 0;
|
||||
const kind = key.startsWith("symbol/")
|
||||
? "symbol"
|
||||
: key.startsWith("footprint/")
|
||||
? "footprint"
|
||||
: key.startsWith("model3d/")
|
||||
? "model3d"
|
||||
: null;
|
||||
if (kind) {
|
||||
into[kind].bytes += size;
|
||||
into[kind].items += 1;
|
||||
}
|
||||
cursor.continue();
|
||||
};
|
||||
tx.oncomplete = () => {
|
||||
db.close();
|
||||
resolve();
|
||||
};
|
||||
tx.onerror = () => {
|
||||
db.close();
|
||||
resolve();
|
||||
};
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function measureAll(): Promise<StorageBreakdown> {
|
||||
const breakdown: StorageBreakdown = {
|
||||
symbol: emptyUsage(),
|
||||
footprint: emptyUsage(),
|
||||
model3d: emptyUsage(),
|
||||
originUsage: null,
|
||||
originQuota: null,
|
||||
modelDbs: [],
|
||||
};
|
||||
// indexedDB.databases() is supported everywhere we run (Chromium, FF 126+,
|
||||
// Safari 14+); without it we just show the origin estimate.
|
||||
const dbs = (await indexedDB.databases?.()) ?? [];
|
||||
for (const db of dbs) {
|
||||
const name = db.name;
|
||||
if (!name || !name.startsWith(SYNC_PREFIX)) continue;
|
||||
if (name.startsWith(MODELS_NS_PREFIX)) breakdown.modelDbs.push(name);
|
||||
await measureDb(name, breakdown);
|
||||
}
|
||||
try {
|
||||
const est = await navigator.storage?.estimate?.();
|
||||
breakdown.originUsage = est?.usage ?? null;
|
||||
breakdown.originQuota = est?.quota ?? null;
|
||||
} catch {
|
||||
// estimate unavailable — the per-kind rows still stand on their own
|
||||
}
|
||||
return breakdown;
|
||||
}
|
||||
|
||||
function deleteDb(name: string): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
const req = indexedDB.deleteDatabase(name);
|
||||
req.onsuccess = req.onerror = req.onblocked = () => resolve();
|
||||
});
|
||||
}
|
||||
|
||||
export function formatBytes(n: number): string {
|
||||
if (n < 1024) return `${n} B`;
|
||||
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
|
||||
if (n < 1024 * 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)} MB`;
|
||||
return `${(n / 1024 / 1024 / 1024).toFixed(2)} GB`;
|
||||
}
|
||||
|
||||
export function StorageUsageCard() {
|
||||
const [data, setData] = React.useState<StorageBreakdown | null>(null);
|
||||
const [busy, setBusy] = React.useState(false);
|
||||
const [clearing, setClearing] = React.useState(false);
|
||||
|
||||
const refresh = React.useCallback(async () => {
|
||||
setBusy(true);
|
||||
try {
|
||||
setData(await measureAll());
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
React.useEffect(() => {
|
||||
void refresh();
|
||||
}, [refresh]);
|
||||
|
||||
const clearModels = React.useCallback(async () => {
|
||||
if (!data) return;
|
||||
setClearing(true);
|
||||
try {
|
||||
for (const name of data.modelDbs) await deleteDb(name);
|
||||
await refresh();
|
||||
} finally {
|
||||
setClearing(false);
|
||||
}
|
||||
}, [data, refresh]);
|
||||
|
||||
const rows = data
|
||||
? ([
|
||||
["Symbols", data.symbol],
|
||||
["Footprints", data.footprint],
|
||||
["3D models", data.model3d],
|
||||
] as const)
|
||||
: null;
|
||||
|
||||
return (
|
||||
<section className="mb-10 rounded-lg border p-5">
|
||||
<h2 className="mb-1 flex items-center gap-2 text-lg font-medium">
|
||||
<Database size={16} /> Storage
|
||||
</h2>
|
||||
<p className="mb-4 text-sm text-muted-foreground">
|
||||
Library data cached in this browser. Everything re-downloads on demand —
|
||||
clearing is always safe. 3D models are fetched per board, so their cache
|
||||
is the one worth reclaiming.
|
||||
</p>
|
||||
|
||||
{!data ? (
|
||||
<p className="flex items-center gap-2 text-sm text-muted-foreground">
|
||||
<Loader2 className="animate-spin" size={14} /> Measuring…
|
||||
</p>
|
||||
) : (
|
||||
<div className="space-y-2">
|
||||
<table className="w-full max-w-md text-sm">
|
||||
<tbody>
|
||||
{rows!.map(([label, usage]) => (
|
||||
<tr key={label} className="border-b last:border-b-0">
|
||||
<td className="py-1.5 text-muted-foreground">{label}</td>
|
||||
<td className="py-1.5 text-right tabular-nums">
|
||||
{usage.items.toLocaleString()} items
|
||||
</td>
|
||||
<td className="py-1.5 text-right font-medium tabular-nums">
|
||||
{formatBytes(usage.bytes)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
{data.originUsage !== null && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Site total (all caches): {formatBytes(data.originUsage)}
|
||||
{data.originQuota ? ` of ${formatBytes(data.originQuota)} available` : ""}
|
||||
</p>
|
||||
)}
|
||||
<div className="flex items-center gap-2 pt-1">
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => void clearModels()}
|
||||
disabled={clearing || data.model3d.items === 0}
|
||||
>
|
||||
{clearing ? (
|
||||
<Loader2 className="mr-1 animate-spin" size={14} />
|
||||
) : (
|
||||
<Trash2 className="mr-1" size={14} />
|
||||
)}
|
||||
Delete 3D model cache
|
||||
{data.model3d.bytes > 0 ? ` (${formatBytes(data.model3d.bytes)})` : ""}
|
||||
</Button>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => void refresh()}
|
||||
disabled={busy}
|
||||
aria-label="Refresh storage usage"
|
||||
>
|
||||
<RefreshCw className={busy ? "animate-spin" : ""} size={14} />
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
|
@ -17,6 +17,7 @@ import { ChevronDown, ChevronUp, Loader2 } from "lucide-react";
|
|||
import {
|
||||
currentScope,
|
||||
libsSourceConfig,
|
||||
modelsSourceConfig,
|
||||
yjsProviderConfig,
|
||||
type DocSource,
|
||||
} from "@/lib/config";
|
||||
|
|
@ -31,6 +32,10 @@ import {
|
|||
type LibLoadingDetail,
|
||||
type LibsSource,
|
||||
} from "@/wasm/libs/source";
|
||||
import {
|
||||
MODELS_LOADING_EVENT,
|
||||
type ModelsLoadingDetail,
|
||||
} from "@/wasm/libs/models-bridge";
|
||||
import { memfsFilePath, memfsProjectDir } from "@/wasm/constants";
|
||||
import { driveProjectIntoTool, type ToolFile } from "@/wasm/kicad-runner";
|
||||
import { registerSaveHook, type SaveBytes } from "@/wasm/save-flow";
|
||||
|
|
@ -595,6 +600,9 @@ export function WasmTool({
|
|||
done: number;
|
||||
total: number;
|
||||
} | null>(null);
|
||||
// Board 3D-model prefetch in flight (background; the viewer works without it —
|
||||
// anything still missing lazy-loads per model). Small badge, not an overlay.
|
||||
const [modelsSync, setModelsSync] = React.useState<string | null>(null);
|
||||
|
||||
const append = React.useCallback(
|
||||
(msg: string) => setLogs((prev) => [...prev.slice(-800), msg]),
|
||||
|
|
@ -660,6 +668,18 @@ export function WasmTool({
|
|||
};
|
||||
}, []);
|
||||
|
||||
// Board 3D-model prefetch progress (models-bridge prescan) — background badge.
|
||||
React.useEffect(() => {
|
||||
const onModels = (e: Event) => {
|
||||
const d = (e as CustomEvent<ModelsLoadingDetail>).detail;
|
||||
setModelsSync(
|
||||
d.loading ? `Fetching 3D models — ${d.done}/${d.total}` : null,
|
||||
);
|
||||
};
|
||||
window.addEventListener(MODELS_LOADING_EVENT, onModels);
|
||||
return () => window.removeEventListener(MODELS_LOADING_EVENT, onModels);
|
||||
}, []);
|
||||
|
||||
// "Taking too long": once the tool has been loading for a while without
|
||||
// becoming ready, surface a hint (slow link / something may be wrong) + a
|
||||
// reload, so a stalled boot doesn't look like a frozen blank screen.
|
||||
|
|
@ -756,6 +776,9 @@ export function WasmTool({
|
|||
onAbort: oom.onAbort,
|
||||
onProgress: (loaded, total) => setProgress({ loaded, total }),
|
||||
libsSource: source,
|
||||
// 3D models: lazy per-board source (null unless the CDN manifest is
|
||||
// configured) — feeds the board prescan + the viewer's ensure fallback.
|
||||
modelsSource: modelsSourceConfig(),
|
||||
});
|
||||
// Register the save sink before the file opens: from here on, every
|
||||
// editor File→Save (MEMFS write) is routed onward through saveBytes.
|
||||
|
|
@ -1002,6 +1025,13 @@ export function WasmTool({
|
|||
</div>
|
||||
)}
|
||||
|
||||
{/* Board 3D models still prefetching into the cache (background). */}
|
||||
{ready && modelsSync && (
|
||||
<div className="pointer-events-none absolute bottom-[4.25rem] left-3 z-20 flex items-center gap-2 rounded bg-black/80 px-3 py-1.5 text-xs text-sky-200">
|
||||
<Loader2 className="animate-spin" size={14} /> {modelsSync}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* A library item is being fetched (open/save). */}
|
||||
{ready && libBusy && (
|
||||
<div className="pointer-events-none absolute left-1/2 top-3 z-20 flex -translate-x-1/2 items-center gap-2 rounded bg-black/80 px-3 py-1.5 text-xs text-white">
|
||||
|
|
|
|||
|
|
@ -98,6 +98,7 @@ export const LOCAL_PROJECTS_ENABLED =
|
|||
|
||||
import type { ProviderConfig, ProviderKind } from "@/wasm/collab";
|
||||
import { cdnLibsSource } from "@/wasm/libs/cdn-source";
|
||||
import { cdnModelsSource, type Model3dSource } from "@/wasm/libs/models-source";
|
||||
import { remoteLibsSource } from "@/wasm/libs/remote-source";
|
||||
import { scopedLibsSource } from "@/wasm/libs/scoped-source";
|
||||
import type { LibsSource } from "@/wasm/libs/source";
|
||||
|
|
@ -192,6 +193,22 @@ export function currentScope(): string {
|
|||
export const CDN_LIBS_MANIFEST_URL =
|
||||
import.meta.env.VITE_LIBS_MANIFEST_URL || null;
|
||||
|
||||
/** Full URL of the CDN 3D-models top manifest, e.g.
|
||||
* https://cdn.pcbjam.com/libs/kicad-models/10.0.0/manifest.json. Bodies are
|
||||
* fetched lazily per board (sparse layers) and cached in IDB — never bulk
|
||||
* synced. Unset ⇒ the 3D viewer renders bare boards (no component models).
|
||||
* See wasm/libs/models-source.ts + docs/features/3d-models. */
|
||||
export const CDN_MODELS_MANIFEST_URL =
|
||||
import.meta.env.VITE_MODELS_MANIFEST_URL || null;
|
||||
|
||||
/** The 3D model source for a tool boot (null ⇒ models disabled). One instance
|
||||
* per call — WasmTool keeps a single instance per boot like the libs source. */
|
||||
export function modelsSourceConfig(): Model3dSource | null {
|
||||
return CDN_MODELS_MANIFEST_URL
|
||||
? cdnModelsSource(CDN_MODELS_MANIFEST_URL)
|
||||
: null;
|
||||
}
|
||||
|
||||
export function libsSourceConfig(projectId?: string): LibsSource | null {
|
||||
const kind = import.meta.env.VITE_LIBS_SOURCE ?? "remote";
|
||||
// "local" is the placeholder id for launches with no real backend project
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import { WaitlistForm } from "@/components/WaitlistForm";
|
|||
import { NewFileDialog } from "@/components/NewFileDialog";
|
||||
import type { SaveBytes } from "@/wasm/save-flow";
|
||||
import { LocalProjectView, type LocalFile } from "@/components/LocalProjectView";
|
||||
import { StorageUsageCard } from "@/components/StorageUsageCard";
|
||||
import { WasmTool } from "@/components/WasmTool";
|
||||
|
||||
/** A KiCad project picked from the local filesystem (no backend involved). */
|
||||
|
|
@ -322,6 +323,11 @@ export function HomePage() {
|
|||
/>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{/* --- Browser storage: per-kind cache sizes + delete-3D-cache. --- */}
|
||||
<div className="mt-10">
|
||||
<StorageUsageCard />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,15 @@
|
|||
import type { Tool } from "@pcbjam/shared";
|
||||
import {
|
||||
KICAD_CONFIG_DIR,
|
||||
MODELS_3D_ENV_VARS,
|
||||
MODELS_3D_ROOT,
|
||||
RESOURCE_PATH,
|
||||
TOOL_ARGV0,
|
||||
TOOL_LIB_KIND,
|
||||
TOOL_NEEDS_CONFIG_SEED,
|
||||
} from "./constants";
|
||||
import { installModel3dHandler } from "./libs/models-bridge";
|
||||
import type { Model3dSource } from "./libs/models-source";
|
||||
import {
|
||||
buildFpLibTable,
|
||||
buildSymLibTable,
|
||||
|
|
@ -60,6 +64,9 @@ export interface BootOptions {
|
|||
/** Library source backing `window.kicadLibs`. Null/omitted disables libs
|
||||
* (an empty sym-lib-table is seeded). Its libs become sym-lib-table rows. */
|
||||
libsSource?: LibsSource | null;
|
||||
/** 3D model source (lazy, per-board). Null/omitted ⇒ the viewer renders the
|
||||
* bare board only, exactly as before models existed. */
|
||||
modelsSource?: Model3dSource | null;
|
||||
}
|
||||
|
||||
let booted: { tool: Tool; promise: Promise<void> } | null = null;
|
||||
|
|
@ -177,8 +184,17 @@ async function fetchWasmWithProgress(
|
|||
}
|
||||
|
||||
async function doBoot(opts: BootOptions): Promise<void> {
|
||||
const { tool, base, container, log, onStatus, onAbort, onProgress, libsSource } =
|
||||
opts;
|
||||
const {
|
||||
tool,
|
||||
base,
|
||||
container,
|
||||
log,
|
||||
onStatus,
|
||||
onAbort,
|
||||
onProgress,
|
||||
libsSource,
|
||||
modelsSource,
|
||||
} = opts;
|
||||
const w = window as ToolWindow;
|
||||
|
||||
// The wasm reads the top-level frame geometry from a GLOBAL `mainWindow`
|
||||
|
|
@ -215,6 +231,12 @@ async function doBoot(opts: BootOptions): Promise<void> {
|
|||
const libKind = TOOL_LIB_KIND[tool];
|
||||
if (libsSource && libKind) {
|
||||
installLibsProvider(libsSource, log);
|
||||
// 3D models ride the same provider (kind "model3d"): the C++ ensure fallback
|
||||
// and the board prescan both resolve through this source.
|
||||
if (modelsSource) {
|
||||
installModel3dHandler(modelsSource, log);
|
||||
log("[3d] model source installed");
|
||||
}
|
||||
try {
|
||||
// Ensure the owner has at least one writable user lib to save items into.
|
||||
// Pass the tool's kind so origins are filtered to the right domain.
|
||||
|
|
@ -321,6 +343,9 @@ async function doBoot(opts: BootOptions): Promise<void> {
|
|||
FS.writeFile(path, contents);
|
||||
log(`[boot] seeded ${path}`);
|
||||
};
|
||||
// 3D models: the MEMFS root the prescan/ensure paths write into, plus the
|
||||
// env vars (every vintage) that make KiCad's resolver look there.
|
||||
FS.mkdirTree(MODELS_3D_ROOT);
|
||||
writeIfAbsent(
|
||||
`${KICAD_CONFIG_DIR}/kicad_common.json`,
|
||||
JSON.stringify(
|
||||
|
|
@ -329,6 +354,11 @@ async function doBoot(opts: BootOptions): Promise<void> {
|
|||
update_check_prompt: true,
|
||||
data_collection_prompt: true,
|
||||
},
|
||||
environment: {
|
||||
vars: Object.fromEntries(
|
||||
MODELS_3D_ENV_VARS.map((v) => [v, MODELS_3D_ROOT]),
|
||||
),
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
|
|
|
|||
|
|
@ -72,6 +72,24 @@ export const TOOL_LIB_KIND: Record<Tool, "symbol" | "footprint" | null> = {
|
|||
/** KiCad user settings dir for this build (PATHS::GetUserSettingsPath()). */
|
||||
export const KICAD_CONFIG_DIR = `/home/kicad/.config/kicad/kicad/${KICAD_VERSION_DIR}`;
|
||||
|
||||
/**
|
||||
* MEMFS root where 3D model bodies are materialized (JS prescan + the C++
|
||||
* lazy-ensure fallback both write `<root>/<lib>.3dshapes/<name>.<ext>` here).
|
||||
* Boot points every `KICAD*_3DMODEL_DIR` env var at this dir — official-lib
|
||||
* footprints reference models through vintage-specific vars (KICAD6..10 all
|
||||
* occur), and KiCad's stock FILENAME_RESOLVER picks up each var it finds — so
|
||||
* model paths resolve with zero resolver changes.
|
||||
*/
|
||||
export const MODELS_3D_ROOT = "/pcbjam/3dmodels";
|
||||
export const MODELS_3D_ENV_VARS = [
|
||||
"KISYS3DMOD", // pre-v6 legacy alias, still common in older boards
|
||||
"KICAD6_3DMODEL_DIR",
|
||||
"KICAD7_3DMODEL_DIR",
|
||||
"KICAD8_3DMODEL_DIR",
|
||||
"KICAD9_3DMODEL_DIR",
|
||||
"KICAD10_3DMODEL_DIR",
|
||||
] as const;
|
||||
|
||||
export function memfsProjectDir(slug: string): string {
|
||||
return `${MEMFS_PROJECTS_DIR}/${slug}`;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import type { Tool } from "@pcbjam/shared";
|
||||
import { FILELESS_TOOLS } from "@pcbjam/shared";
|
||||
import { memfsFilePath, memfsProjectDir } from "./constants";
|
||||
import { prescanBoardModels } from "./libs/models-bridge";
|
||||
import { openFileInTool } from "./open-flow";
|
||||
|
||||
/**
|
||||
|
|
@ -55,6 +56,16 @@ async function syncProjectToMemfs(win: ToolWindow, opts: DriveOptions): Promise<
|
|||
const bytes = await opts.fetchBytes(file.path);
|
||||
fs.writeFile(dest, bytes);
|
||||
opts.log(`[memfs] wrote ${dest} (${bytes.length} bytes)`);
|
||||
// 3D models: prefetch every model this board references (R2 → IDB → MEMFS)
|
||||
// so the 3D viewer's first open resolves locally. Fire-and-forget — project
|
||||
// open never waits on it; a ref that misses falls back to the C++ per-model
|
||||
// ensure. No-op unless a model source is installed (bootKicadTool).
|
||||
if (file.path.endsWith(".kicad_pcb")) {
|
||||
const text = new TextDecoder().decode(bytes);
|
||||
void prescanBoardModels(text).catch((e) =>
|
||||
opts.log(`[3d] prescan failed: ${String(e)}`),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
62
web/standalone/src/wasm/libs/models-bridge.test.ts
Normal file
62
web/standalone/src/wasm/libs/models-bridge.test.ts
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { normalizeModelRef, scanModelRefs } from "./models-bridge";
|
||||
|
||||
describe("normalizeModelRef", () => {
|
||||
it("strips any vintage of the model-dir var", () => {
|
||||
for (const v of ["KICAD6", "KICAD7", "KICAD8", "KICAD9", "KICAD10"]) {
|
||||
expect(
|
||||
normalizeModelRef(`\${${v}_3DMODEL_DIR}/Resistor_SMD.3dshapes/R_0201.wrl`),
|
||||
).toBe("Resistor_SMD.3dshapes/R_0201.wrl");
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts the paren syntax and the legacy KISYS3DMOD alias", () => {
|
||||
expect(normalizeModelRef("$(KICAD8_3DMODEL_DIR)/L.3dshapes/m.step")).toBe(
|
||||
"L.3dshapes/m.step",
|
||||
);
|
||||
expect(normalizeModelRef("${KISYS3DMOD}/L.3dshapes/m.wrl")).toBe(
|
||||
"L.3dshapes/m.wrl",
|
||||
);
|
||||
});
|
||||
|
||||
it("passes bare relative refs through", () => {
|
||||
expect(normalizeModelRef("L.3dshapes/m.wrl")).toBe("L.3dshapes/m.wrl");
|
||||
});
|
||||
|
||||
it("rejects refs it cannot serve", () => {
|
||||
expect(normalizeModelRef("${KIPRJMOD}/libs/3d/m.wrl")).toBeNull(); // project-local
|
||||
expect(normalizeModelRef("/abs/path/m.wrl")).toBeNull();
|
||||
expect(normalizeModelRef("kicad_embed://m.wrl")).toBeNull();
|
||||
expect(normalizeModelRef("")).toBeNull();
|
||||
expect(normalizeModelRef("${UNCLOSED/m.wrl")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("scanModelRefs", () => {
|
||||
it("finds, normalizes and dedupes board model refs", () => {
|
||||
const board = `
|
||||
(footprint "Resistor_THT:R_Axial"
|
||||
(model "\${KICAD10_3DMODEL_DIR}/Resistor_THT.3dshapes/R_Axial.step"
|
||||
(offset (xyz 0 0 0))))
|
||||
(footprint "Resistor_THT:R_Axial"
|
||||
(model "\${KICAD10_3DMODEL_DIR}/Resistor_THT.3dshapes/R_Axial.step"))
|
||||
(footprint "X:Y"
|
||||
(model "\${KIPRJMOD}/libs/3d_shapes/custom.wrl"))
|
||||
(footprint "L:M" (model "\${KICAD8_3DMODEL_DIR}/LED_THT.3dshapes/LED_D5.0mm.wrl"))
|
||||
`;
|
||||
expect(scanModelRefs(board).sort()).toEqual([
|
||||
"LED_THT.3dshapes/LED_D5.0mm.wrl",
|
||||
"Resistor_THT.3dshapes/R_Axial.step",
|
||||
]);
|
||||
});
|
||||
|
||||
it("handles escaped quotes inside the path", () => {
|
||||
expect(
|
||||
scanModelRefs('(model "${KICAD9_3DMODEL_DIR}/A.3dshapes/we\\"ird.wrl")'),
|
||||
).toEqual(['A.3dshapes/we"ird.wrl']);
|
||||
});
|
||||
|
||||
it("returns empty for a board with no models", () => {
|
||||
expect(scanModelRefs("(kicad_pcb (version 20240101))")).toEqual([]);
|
||||
});
|
||||
});
|
||||
181
web/standalone/src/wasm/libs/models-bridge.ts
Normal file
181
web/standalone/src/wasm/libs/models-bridge.ts
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
import { MODELS_3D_ROOT } from "../constants";
|
||||
import type { Model3dSource } from "./models-source";
|
||||
|
||||
/**
|
||||
* Glue between 3D model delivery and the running KiCad WASM tool:
|
||||
*
|
||||
* - `installModel3dHandler` backs the provider's `kind === "model3d"` requests
|
||||
* (the C++ lazy fallback in S3D_CACHE::load → PCBJAM_3D::EnsureModelFile asks
|
||||
* "ensure" for a ref the prescan missed).
|
||||
* - `prescanBoardModels` scans a board's `(model "…")` refs up front and
|
||||
* prefetches those bodies (R2 → IDB → MEMFS) so the 3D viewer's first open
|
||||
* resolves everything locally.
|
||||
*
|
||||
* Both paths converge on `ensureModelInMemfs`: fetch the body via the
|
||||
* `Model3dSource` (IDB-cached, sparse) and write it under `MODELS_3D_ROOT` —
|
||||
* where boot points every `KICAD*_3DMODEL_DIR` env var, so KiCad's stock
|
||||
* resolver finds the file with no C++ resolution changes.
|
||||
*/
|
||||
|
||||
/** Progress of a board-model prefetch burst (drives the 3D loading overlay). */
|
||||
export const MODELS_LOADING_EVENT = "pcbjam:models-loading";
|
||||
|
||||
export interface ModelsLoadingDetail {
|
||||
loading: boolean;
|
||||
done: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
function emitModelsLoading(detail: ModelsLoadingDetail): void {
|
||||
if (typeof window === "undefined") return;
|
||||
window.dispatchEvent(new CustomEvent(MODELS_LOADING_EVENT, { detail }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a footprint model reference to the source's relative form —
|
||||
* "${KICAD*_3DMODEL_DIR}/<lib>.3dshapes/<name>.<ext>" (any vintage, `${}` or
|
||||
* `$()`) → "<lib>.3dshapes/<name>.<ext>". Bare relative refs pass through;
|
||||
* absolute paths / ${KIPRJMOD} / kicad_embed:// are not ours → null. Mirrors
|
||||
* pcbjamNormalizeModelRef in kicad/3d-viewer/3d_cache/pcbjam_model_fetch.cpp.
|
||||
*/
|
||||
export function normalizeModelRef(raw: string): string | null {
|
||||
const ref = raw.trim();
|
||||
if (!ref) return null;
|
||||
if (ref.startsWith("${") || ref.startsWith("$(")) {
|
||||
const closing = ref[1] === "{" ? "}" : ")";
|
||||
const end = ref.indexOf(closing);
|
||||
if (end < 0) return null;
|
||||
const v = ref.slice(2, end);
|
||||
// Any vintage of the model-dir var, plus the pre-v6 legacy alias.
|
||||
if (!v.includes("3DMODEL_DIR") && v !== "KISYS3DMOD") return null;
|
||||
return ref.slice(end + 1).replace(/^[/\\]+/, "") || null;
|
||||
}
|
||||
if (ref.startsWith("/") || ref.includes("://")) return null;
|
||||
return ref;
|
||||
}
|
||||
|
||||
/** Every `(model "…")` ref in a KiCad board/footprint s-expr, normalized. */
|
||||
export function scanModelRefs(sexprText: string): string[] {
|
||||
const refs = new Set<string>();
|
||||
const re = /\(\s*model\s+"((?:[^"\\]|\\.)*)"/g;
|
||||
for (let m = re.exec(sexprText); m; m = re.exec(sexprText)) {
|
||||
const raw = m[1]!.replace(/\\(.)/g, "$1");
|
||||
const rel = normalizeModelRef(raw);
|
||||
if (rel) refs.add(rel);
|
||||
}
|
||||
return [...refs];
|
||||
}
|
||||
|
||||
type ModelFS = Pick<EmscriptenFS, "mkdirTree" | "writeFile" | "analyzePath">;
|
||||
|
||||
function toolFS(): ModelFS | null {
|
||||
const fs = (window as ToolWindow).FS;
|
||||
return fs && typeof fs.writeFile === "function" ? (fs as ModelFS) : null;
|
||||
}
|
||||
|
||||
let installedSource: Model3dSource | null = null;
|
||||
let installedLog: (msg: string) => void = () => {};
|
||||
/** Refs already materialized in MEMFS this session (bodies are immutable). */
|
||||
const written = new Set<string>();
|
||||
/** In-flight ensures, coalesced per ref (prescan and the C++ fallback race). */
|
||||
const ensuring = new Map<string, Promise<string | null>>();
|
||||
|
||||
/** Wire the model source used by the provider dispatch + prescan. */
|
||||
export function installModel3dHandler(
|
||||
source: Model3dSource,
|
||||
log: (msg: string) => void,
|
||||
): void {
|
||||
installedSource = source;
|
||||
installedLog = log;
|
||||
}
|
||||
|
||||
/** Fetch one model body and write it under MODELS_3D_ROOT. Resolves to the
|
||||
* absolute MEMFS path when present, null when the source can't serve it. */
|
||||
export async function ensureModelInMemfs(ref: string): Promise<string | null> {
|
||||
const dest = `${MODELS_3D_ROOT}/${ref}`;
|
||||
if (written.has(ref)) return dest;
|
||||
let p = ensuring.get(ref);
|
||||
if (!p) {
|
||||
p = doEnsure(ref, dest).finally(() => ensuring.delete(ref));
|
||||
ensuring.set(ref, p);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
async function doEnsure(ref: string, dest: string): Promise<string | null> {
|
||||
const source = installedSource;
|
||||
const fs = toolFS();
|
||||
if (!source || !fs) return null;
|
||||
if (fs.analyzePath(dest).exists) {
|
||||
written.add(ref);
|
||||
return dest;
|
||||
}
|
||||
const body = await source.getModelBody(ref);
|
||||
if (!body) return null;
|
||||
fs.mkdirTree(dest.slice(0, dest.lastIndexOf("/")));
|
||||
fs.writeFile(dest, body);
|
||||
written.add(ref);
|
||||
installedLog(`[3d] materialized ${ref} (${body.length} bytes)`);
|
||||
return dest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provider dispatch for `kind === "model3d"` (called by installLibsProvider's
|
||||
* request before any lib-id parsing — the C++ bridge passes an empty lib; the
|
||||
* ref itself carries the library). Answers with the ABSOLUTE MEMFS path of the
|
||||
* materialized file — S3D_CACHE loads that path directly, so model delivery
|
||||
* never depends on env-var expansion inside the wasm runtime.
|
||||
*/
|
||||
export async function handleModel3dRequest(
|
||||
op: string,
|
||||
arg: string,
|
||||
): Promise<string | null> {
|
||||
if (op !== "ensure") return null;
|
||||
const rel = normalizeModelRef(arg);
|
||||
if (!rel) return null;
|
||||
try {
|
||||
return await ensureModelInMemfs(rel);
|
||||
} catch (e) {
|
||||
installedLog(`[3d] ensure failed for ${arg}: ${String(e)}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prefetch every model a board references (fire-and-forget from the project
|
||||
* sync). Bodies land in IDB + MEMFS before the user opens the 3D viewer in the
|
||||
* common case; anything still missing falls back to the per-model C++ ensure.
|
||||
*/
|
||||
export async function prescanBoardModels(
|
||||
boardText: string,
|
||||
concurrency = 6,
|
||||
): Promise<void> {
|
||||
if (!installedSource) return;
|
||||
const refs = scanModelRefs(boardText);
|
||||
if (!refs.length) return;
|
||||
|
||||
const total = refs.length;
|
||||
let done = 0;
|
||||
emitModelsLoading({ loading: true, done, total });
|
||||
installedLog(`[3d] prescan: ${total} model ref(s) on board`);
|
||||
const started = performance.now();
|
||||
|
||||
let idx = 0;
|
||||
const worker = async (): Promise<void> => {
|
||||
while (idx < refs.length) {
|
||||
const ref = refs[idx++]!;
|
||||
try {
|
||||
await ensureModelInMemfs(ref);
|
||||
} catch {
|
||||
// best-effort: the C++ lazy path (or a later prescan) retries
|
||||
}
|
||||
emitModelsLoading({ loading: ++done < total, done, total });
|
||||
}
|
||||
};
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(concurrency, total) }, () => worker()),
|
||||
);
|
||||
installedLog(
|
||||
`[3d] prescan: ${done}/${total} in ${Math.round(performance.now() - started)}ms`,
|
||||
);
|
||||
}
|
||||
120
web/standalone/src/wasm/libs/models-source.test.ts
Normal file
120
web/standalone/src/wasm/libs/models-source.test.ts
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { sha256Hex, type SyncManifest } from "@pcbjam/shared";
|
||||
import { memStore, type LayerStore } from "@pcbjam/sync-client";
|
||||
import { cdnModelsSource } from "./models-source";
|
||||
|
||||
const MANIFEST_URL = "https://cdn.test/libs/kicad-models/9.0.9/manifest.json";
|
||||
const BASE = "https://cdn.test/libs/kicad-models/9.0.9";
|
||||
const BLOBS = "https://cdn.test/libs/kicad-models/blobs/sha256";
|
||||
|
||||
const enc = new TextEncoder();
|
||||
const dec = new TextDecoder();
|
||||
|
||||
/** Build the CDN layout the way publish-models.ts does: per-lib sparse manifest
|
||||
* + content-addressed blobs — pinning the format with the REAL hash codec. */
|
||||
async function fakeModelsCdn() {
|
||||
const bodies: Record<string, Uint8Array> = {
|
||||
"model3d/R_Axial.step": enc.encode("STEP R_Axial"),
|
||||
"model3d/R_Disc.wrl": enc.encode("#VRML V2.0 utf8 R_Disc"),
|
||||
};
|
||||
const entries: SyncManifest["entries"] = {};
|
||||
const blobs = new Map<string, Uint8Array>();
|
||||
for (const [path, body] of Object.entries(bodies)) {
|
||||
const hash = await sha256Hex(body);
|
||||
entries[path] = { hash, size: body.length, mtime: 0 };
|
||||
blobs.set(hash, body);
|
||||
}
|
||||
const libManifest: SyncManifest = { version: 1, entries };
|
||||
const top = {
|
||||
schema: 1,
|
||||
tag: "9.0.9",
|
||||
libs: [{ id: "Resistor_THT", itemCount: 2 }],
|
||||
};
|
||||
|
||||
let manifestFetches = 0;
|
||||
let blobFetches = 0;
|
||||
const json = (obj: unknown) => ({ ok: true, json: async () => obj });
|
||||
const bin = (bytes: Uint8Array) => ({
|
||||
ok: true,
|
||||
arrayBuffer: async () => bytes.buffer,
|
||||
});
|
||||
const fetchImpl = (async (url: string) => {
|
||||
if (url === MANIFEST_URL) return json(top);
|
||||
if (url === `${BASE}/Resistor_THT/manifest`) {
|
||||
manifestFetches += 1;
|
||||
return json(libManifest);
|
||||
}
|
||||
if (url.startsWith(`${BLOBS}/`)) {
|
||||
const blob = blobs.get(url.slice(`${BLOBS}/`.length));
|
||||
if (blob) {
|
||||
blobFetches += 1;
|
||||
return bin(blob);
|
||||
}
|
||||
}
|
||||
return { ok: false, status: 404 };
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
return {
|
||||
fetchImpl,
|
||||
counters: {
|
||||
get manifestFetches() {
|
||||
return manifestFetches;
|
||||
},
|
||||
get blobFetches() {
|
||||
return blobFetches;
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function storeMap() {
|
||||
const stores = new Map<string, LayerStore>();
|
||||
return (ns: string): LayerStore => {
|
||||
let s = stores.get(ns);
|
||||
if (!s) stores.set(ns, (s = memStore()));
|
||||
return s;
|
||||
};
|
||||
}
|
||||
|
||||
describe("cdnModelsSource", () => {
|
||||
it("fetches exactly the requested body (sparse), then serves from cache", async () => {
|
||||
const cdn = await fakeModelsCdn();
|
||||
const src = cdnModelsSource(MANIFEST_URL, {
|
||||
fetchImpl: cdn.fetchImpl,
|
||||
storeFactory: storeMap(),
|
||||
});
|
||||
|
||||
const body = await src.getModelBody("Resistor_THT.3dshapes/R_Axial.step");
|
||||
expect(dec.decode(body!)).toBe("STEP R_Axial");
|
||||
expect(cdn.counters.blobFetches).toBe(1); // only the asked-for model
|
||||
|
||||
await src.getModelBody("Resistor_THT.3dshapes/R_Axial.step");
|
||||
expect(cdn.counters.blobFetches).toBe(1); // cached
|
||||
expect(cdn.counters.manifestFetches).toBe(1); // lib opened once
|
||||
});
|
||||
|
||||
it("returns null for unknown models/libs/refs without throwing", async () => {
|
||||
const cdn = await fakeModelsCdn();
|
||||
const src = cdnModelsSource(MANIFEST_URL, {
|
||||
fetchImpl: cdn.fetchImpl,
|
||||
storeFactory: storeMap(),
|
||||
});
|
||||
|
||||
expect(await src.getModelBody("Resistor_THT.3dshapes/nope.step")).toBeNull();
|
||||
expect(await src.getModelBody("NoSuchLib.3dshapes/m.wrl")).toBeNull();
|
||||
expect(await src.getModelBody("not-a-model-ref")).toBeNull();
|
||||
expect(cdn.counters.blobFetches).toBe(0);
|
||||
});
|
||||
|
||||
it("hasModel answers from the manifest without fetching bodies", async () => {
|
||||
const cdn = await fakeModelsCdn();
|
||||
const src = cdnModelsSource(MANIFEST_URL, {
|
||||
fetchImpl: cdn.fetchImpl,
|
||||
storeFactory: storeMap(),
|
||||
});
|
||||
|
||||
expect(await src.hasModel("Resistor_THT.3dshapes/R_Disc.wrl")).toBe(true);
|
||||
expect(await src.hasModel("Resistor_THT.3dshapes/nope.wrl")).toBe(false);
|
||||
expect(cdn.counters.blobFetches).toBe(0);
|
||||
});
|
||||
});
|
||||
124
web/standalone/src/wasm/libs/models-source.ts
Normal file
124
web/standalone/src/wasm/libs/models-source.ts
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
import { SyncStack, type SyncStackOptions } from "@pcbjam/sync-client";
|
||||
|
||||
/**
|
||||
* Read-only source of 3D model bodies (`.wrl` / `.step`), addressed by the
|
||||
* KiCad-relative ref `"<lib>.3dshapes/<name>.<ext>"` (a footprint's `(model …)`
|
||||
* path with the `${KICAD*_3DMODEL_DIR}/` prefix stripped).
|
||||
*
|
||||
* Unlike symbols/footprints, models are never bulk-synced: each lib is an
|
||||
* r2-idb-sync **sparse** layer — only the (small) manifest syncs eagerly, and a
|
||||
* body is fetched exactly when a board references it, then cached in IDB. This
|
||||
* keeps the client cost proportional to what the user renders, not to the
|
||||
* ~GB-scale full set (docs/features/3d-models).
|
||||
*/
|
||||
export interface Model3dSource {
|
||||
/** One model body, IDB-cached; null when unknown/unavailable. */
|
||||
getModelBody(ref: string): Promise<Uint8Array | null>;
|
||||
/** Whether a ref exists in the set at all (manifest-only, no body fetch). */
|
||||
hasModel(ref: string): Promise<boolean>;
|
||||
}
|
||||
|
||||
interface CdnModelsManifest {
|
||||
schema: number;
|
||||
tag: string;
|
||||
libs: Array<{ id: string; itemCount?: number }>;
|
||||
}
|
||||
|
||||
/** Split "<lib>.3dshapes/<name>" → { lib, path: "model3d/<name>" }. */
|
||||
function splitRef(ref: string): { lib: string; path: string } | null {
|
||||
const i = ref.indexOf(".3dshapes/");
|
||||
if (i <= 0) return null;
|
||||
const lib = ref.slice(0, i);
|
||||
const name = ref.slice(i + ".3dshapes/".length);
|
||||
if (!name || name.includes("/")) return null;
|
||||
return { lib, path: `model3d/${name}` };
|
||||
}
|
||||
|
||||
/**
|
||||
* CDN-backed `Model3dSource`. Layout under the manifest's dir
|
||||
* (`<cdn>/libs/kicad-models/<tag>/`, see scripts/deploy/publish-models.ts):
|
||||
* manifest.json top index { schema, tag, libs:[{id,itemCount}] }
|
||||
* <lib>/manifest per-lib SyncManifest, entries "model3d/<name>.<ext>"
|
||||
* Bodies are content-addressed and shared across tags, one level up:
|
||||
* <cdn>/libs/kicad-models/blobs/sha256/<hash>
|
||||
*/
|
||||
export function cdnModelsSource(
|
||||
manifestUrl: string,
|
||||
opts?: Pick<SyncStackOptions, "fetchImpl" | "storeFactory">,
|
||||
): Model3dSource {
|
||||
const baseDir = manifestUrl.replace(/\/[^/]*$/, ""); // …/libs/kicad-models/<tag>
|
||||
const blobsBase = `${baseDir.replace(/\/[^/]*$/, "")}/blobs/sha256`;
|
||||
const fetchImpl = opts?.fetchImpl ?? fetch;
|
||||
|
||||
let manifestP: Promise<CdnModelsManifest> | null = null;
|
||||
const loadManifest = (): Promise<CdnModelsManifest> => {
|
||||
if (!manifestP) {
|
||||
// Never cache a rejection (mirrors cdn-source.ts): a transient failure
|
||||
// must not poison every later model read.
|
||||
manifestP = (async () => {
|
||||
const r = await fetchImpl(manifestUrl, { cache: "no-store" });
|
||||
if (!r.ok) throw new Error(`cdn models manifest ${r.status}: ${manifestUrl}`);
|
||||
return (await r.json()) as CdnModelsManifest;
|
||||
})().catch((e) => {
|
||||
manifestP = null;
|
||||
throw e;
|
||||
});
|
||||
}
|
||||
return manifestP;
|
||||
};
|
||||
|
||||
// One lazily-opened sparse stack per lib (IDB store keyed by namespace, so a
|
||||
// lib's cached models persist and dedupe across sessions).
|
||||
const stacks = new Map<string, Promise<SyncStack | null>>();
|
||||
const openStack = (libId: string): Promise<SyncStack | null> => {
|
||||
let p = stacks.get(libId);
|
||||
if (!p) {
|
||||
p = (async () => {
|
||||
const m = await loadManifest();
|
||||
if (!m.libs.some((l) => l.id === libId)) return null; // unknown lib
|
||||
const stack = new SyncStack({
|
||||
layers: [
|
||||
{
|
||||
namespace: `kicad-models:${m.tag}:${libId}`,
|
||||
kind: "sparse",
|
||||
url: `${baseDir}/${encodeURIComponent(libId)}`,
|
||||
bodyUrlTemplate: `${blobsBase}/{hash}`,
|
||||
},
|
||||
],
|
||||
...opts,
|
||||
});
|
||||
await stack.open();
|
||||
return stack;
|
||||
})().catch((e) => {
|
||||
stacks.delete(libId); // a failed open must stay retryable
|
||||
throw e;
|
||||
});
|
||||
stacks.set(libId, p);
|
||||
}
|
||||
return p;
|
||||
};
|
||||
|
||||
return {
|
||||
async getModelBody(ref: string): Promise<Uint8Array | null> {
|
||||
const split = splitRef(ref);
|
||||
if (!split) return null;
|
||||
try {
|
||||
const stack = await openStack(split.lib);
|
||||
return stack ? await stack.read(split.path) : null;
|
||||
} catch {
|
||||
return null; // missing models render as absent, never break the viewer
|
||||
}
|
||||
},
|
||||
async hasModel(ref: string): Promise<boolean> {
|
||||
const split = splitRef(ref);
|
||||
if (!split) return false;
|
||||
try {
|
||||
const stack = await openStack(split.lib);
|
||||
if (!stack) return false;
|
||||
return (await stack.list()).some((e) => e.path === split.path);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import { handleModel3dRequest } from "./models-bridge";
|
||||
import { libIdFromUri, libUri } from "./uri";
|
||||
|
||||
/**
|
||||
|
|
@ -259,6 +260,12 @@ export function installLibsProvider(
|
|||
let fatResetTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
|
||||
const request: KicadLibsRequest = async (op, lib, arg, kind = "symbol") => {
|
||||
// 3D models are addressed by ref, not lib-table URI (the C++ ensure bridge
|
||||
// passes an empty lib) — dispatch before the lib-id parse would null it out.
|
||||
if (kind === "model3d") {
|
||||
log(`[libs] request op=${op} kind=model3d arg=${arg}`);
|
||||
return handleModel3dRequest(op, arg);
|
||||
}
|
||||
const id = libIdFromUri(lib);
|
||||
log(`[libs] request op=${op} kind=${kind} lib=${lib} (id=${id}) arg=${arg}`);
|
||||
if (!id) return null;
|
||||
|
|
|
|||
Loading…
Reference in a new issue