AI_Assistant/pcbjam
Watch
1
0
Fork
You've already forked pcbjam
 0
Code Issues Pull requests Projects Releases 7 Packages Wiki Activity Actions

feat(3d): lazy 3D model delivery R2→IDB→WASM

Browse source 
- cdnModelsSource: sparse per-lib stacks over the models CDN layout
  (libs/kicad-models/<tag>/<lib>/manifest + content-addressed blobs)
- models-bridge: board prescan ((model …) scan → prefetch, 4/4 in 74ms
  on the demo board) + the kind=model3d 'ensure' op answering absolute
  MEMFS paths; only board-referenced bodies ever enter IDB
- boot/runner/config/WasmTool wiring (VITE_MODELS_MANIFEST_URL,
  KICAD*_3DMODEL_DIR seeding, prefetch badge)
- StorageUsageCard on HomePage: per-kind cached sizes + delete-3D-cache
- publish-models.ts (brotli content-addressed publish, 606→93MB for the
  pic_programmer lib set) + dev-demo --models-tag/--models-local
- e2e: 3d-viewer-models.spec.ts (bridge normalize/dedup + STEP/WRL
  render); submodule bumps: kicad (static 3D plugins + ensure hook),
  pcbjam-shared (sparse sync layer)

Spec + findings: docs/features/3d-models (private repo).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014AT7gVHRktDYoQ68S4x6A4
This commit is contained in:
Gergő Törcsvári 2026-07-02 11:00:51 +02:00
commit 6f3f3bcf00
No known key found for this signature in database
GPG key ID: 8E75F2CDE64E5322
19 changed files with 1351 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

2
kicad

@ -1 +1 @@
Subproject commit e8db3d35db635946b03ee0ae8698e1d3aad79a3c Subproject commit 9d771391886ba3877dda90f72f57ad37d931fef9

28
scripts/deploy/dev-demo.mjs
View file

@ -72,6 +72,8 @@ function parseArgs(argv) {
contentTag: null, // use the live CDN gallery for this tag (else build locally) contentTag: null, // use the live CDN gallery for this tag (else build locally)
galleryTag: "demo-local", // path tag for the locally-built gallery galleryTag: "demo-local", // path tag for the locally-built gallery
noGallery: false, // disable the example gallery (local-folder + IDB only) noGallery: false, // disable the example gallery (local-folder + IDB only)
modelsTag: null, // 3D models snapshot tag (live CDN, or the local dir's tag)
modelsLocal: null, // local publish-models --driver local output dir (serve same-origin)
port: null, port: null,
repo: "https://github.com/emergence-engineering/pcbjam", repo: "https://github.com/emergence-engineering/pcbjam",
}; };
@ -87,6 +89,8 @@ function parseArgs(argv) {
case "--no-gallery": a.noGallery = true; break; case "--no-gallery": a.noGallery = true; break;
case "--port": a.port = next(); break; case "--port": a.port = next(); break;
case "--repo": a.repo = next(); break; case "--repo": a.repo = next(); break;
case "--models-tag": a.modelsTag = next(); break;
case "--models-local": a.modelsLocal = next(); break;
case "-h": case "--help": a.help = true; break; case "-h": case "--help": a.help = true; break;
default: throw new Error(`unknown arg: ${argv[i]}`); default: throw new Error(`unknown arg: ${argv[i]}`);
} }
@ -108,6 +112,9 @@ const HELP = `dev-demo.mjs — run the standalone locally in demo mode (R2-only
--content-tag <tag> pin the LIVE CDN gallery for this release tag (default: build+serve the gallery locally) --content-tag <tag> pin the LIVE CDN gallery for this release tag (default: build+serve the gallery locally)
--gallery-tag <tag> path tag for the locally-built gallery (default demo-local) --gallery-tag <tag> path tag for the locally-built gallery (default demo-local)
--no-gallery disable the example gallery (local-folder + IDB projects only) --no-gallery disable the example gallery (local-folder + IDB projects only)
--models-tag <tag> enable lazy 3D models from the CDN snapshot at this tag
--models-local <dir> serve a local publish-models layout (--driver local --compress none)
same-origin instead of the CDN (requires --models-tag)
--port <n> dev server port --port <n> dev server port
By default the read-only example gallery (deploy/demo/gallery.json) is built By default the read-only example gallery (deploy/demo/gallery.json) is built
@ -142,6 +149,26 @@ function main() {
delete env.VITE_LIBS_MANIFEST_URL; delete env.VITE_LIBS_MANIFEST_URL;
} }
// --- 3D models: lazy per-board bodies (docs/features/3d-models). Off unless a
// tag is given. --models-local <publish-models --out dir> serves that
// layout same-origin at /models-cdn via a public/ symlink (publish it with
// --compress none — the dev server can't send Content-Encoding: br);
// otherwise the live CDN snapshot for --models-tag is used.
if (a.modelsTag && a.modelsLocal) {
const link = join(repoRoot, "web/standalone/public/models-cdn");
try {
if (lstatSync(link)) rmSync(link, { recursive: true, force: true });
} catch {
/* no existing link */
}
symlinkSync(resolve(a.modelsLocal, "libs/kicad-models"), link);
env.VITE_MODELS_MANIFEST_URL = `/models-cdn/${a.modelsTag}/manifest.json`;
} else if (a.modelsTag) {
env.VITE_MODELS_MANIFEST_URL = `${a.cdn}/libs/kicad-models/${a.modelsTag}/manifest.json`;
} else {
delete env.VITE_MODELS_MANIFEST_URL;
}
// --- No backend: collab is cross-tab only, document bytes are local (api path), // --- No backend: collab is cross-tab only, document bytes are local (api path),
// loaded folders persist to a browser-local IndexedDB project. // loaded folders persist to a browser-local IndexedDB project.
env.VITE_YJS_PROVIDER = "broadcastchannel"; env.VITE_YJS_PROVIDER = "broadcastchannel";
@ -189,6 +216,7 @@ function main() {
console.log(` VITE_LIBS_SOURCE=${env.VITE_LIBS_SOURCE}${env.VITE_LIBS_MANIFEST_URL ? ` (${env.VITE_LIBS_MANIFEST_URL})` : ""}`); console.log(` VITE_LIBS_SOURCE=${env.VITE_LIBS_SOURCE}${env.VITE_LIBS_MANIFEST_URL ? ` (${env.VITE_LIBS_MANIFEST_URL})` : ""}`);
console.log(` VITE_WASM_ROOT=${env.VITE_WASM_ROOT}${env.VITE_WASM_MANIFEST ? ` (${env.VITE_WASM_MANIFEST})` : " (local build)"}`); console.log(` VITE_WASM_ROOT=${env.VITE_WASM_ROOT}${env.VITE_WASM_MANIFEST ? ` (${env.VITE_WASM_MANIFEST})` : " (local build)"}`);
console.log(` VITE_PROJECT_SOURCE=${env.VITE_PROJECT_SOURCE}${env.VITE_PROJECT_MANIFEST_URL ? ` (${env.VITE_PROJECT_MANIFEST_URL})` : ""}`); console.log(` VITE_PROJECT_SOURCE=${env.VITE_PROJECT_SOURCE}${env.VITE_PROJECT_MANIFEST_URL ? ` (${env.VITE_PROJECT_MANIFEST_URL})` : ""}`);
console.log(` VITE_MODELS_MANIFEST_URL=${env.VITE_MODELS_MANIFEST_URL ?? "(unset — 3D models off)"}`);
console.log(` VITE_YJS_PROVIDER=${env.VITE_YJS_PROVIDER} VITE_DOC_SOURCE=${env.VITE_DOC_SOURCE} VITE_LOCAL_PROJECTS=${env.VITE_LOCAL_PROJECTS}`); console.log(` VITE_YJS_PROVIDER=${env.VITE_YJS_PROVIDER} VITE_DOC_SOURCE=${env.VITE_DOC_SOURCE} VITE_LOCAL_PROJECTS=${env.VITE_LOCAL_PROJECTS}`);
const child = spawn("pnpm", viteArgs, { cwd: repoRoot, env, stdio: "inherit" }); const child = spawn("pnpm", viteArgs, { cwd: repoRoot, env, stdio: "inherit" });

219
scripts/deploy/publish-models.ts Normal file
View file

@ -0,0 +1,219 @@
// Publish KiCad 3D models (kicad-packages3D) to the CDN as r2-idb-sync SPARSE
// origins: per-lib manifests keyed by the upstream tag, bodies content-addressed
// under a shared blobs/ prefix (deduped across tags — models rarely change).
// The standalone's cdnModelsSource opens each lib as a sparse layer: manifest
// synced eagerly (small), bodies fetched exactly when a board references them.
// See web/standalone/src/wasm/libs/models-source.ts + docs/features/3d-models.
//
// npx tsx scripts/deploy/publish-models.ts --model-tag 10.0.0 \
// --models-src <kicad-packages3D checkout> --driver local --out /tmp/cdn-models
// npx tsx scripts/deploy/publish-models.ts --model-tag 10.0.0 --models-src … \
// --driver r2 --bucket pcbjam-cdn --remote
// # dev subset: only a few libs
// … --libs Resistor_SMD,Capacitor_SMD,Package_QFP
//
// Layout under `<prefix>` (default libs/kicad-models):
// <tag>/manifest.json top index { schema, tag, libs:[{id,itemCount,bytes}] }
// <tag>/<lib>/manifest per-lib SyncManifest { "model3d/<name>": {hash,size,mtime} }
// blobs/sha256/<hash> model bodies (brotli, content-addressed, shared)
// blobs/registry.json published-blob index (hash → size) for cheap dedup
//
// Idempotent per tag: if <prefix>/<tag>/manifest.json exists the run SKIPS
// (--force overrides). Blobs are skipped per-hash via the registry.
import { execFileSync } from "node:child_process";
import { existsSync, mkdirSync, readdirSync, readFileSync, statSync } from "node:fs";
import { dirname, join } from "node:path";
import type { SyncManifest } from "../../web/pcbjam-shared/src/sync-wire.js";
import {
compressBytes,
IMMUTABLE,
makeStore,
NO_STORE,
putJSON,
sha256hex,
} from "./lib/cdn-store.mjs";
const MODELS_URL = "https://gitlab.com/kicad/libraries/kicad-packages3D.git";
/** Model file extensions we publish (locked: WRL + STEP). */
const MODEL_EXTS = [".wrl", ".step", ".stp"];
interface Args {
modelTag: string | null;
modelsSrc: string | null;
clone: string | null;
driver: string;
out: string | null;
bucket: string;
remote: boolean;
prefix: string;
force: boolean;
libs: string[] | null;
quality: number;
compress: "br" | "none";
}
function parseArgs(argv: string[]): Args {
const a: Args = {
modelTag: null,
modelsSrc: null,
clone: null,
driver: "local",
out: null,
bucket: "pcbjam-cdn",
remote: false,
prefix: "libs/kicad-models",
force: false,
libs: null,
quality: 5,
compress: "br",
};
for (let i = 2; i < argv.length; i++) {
const next = () => argv[++i]!;
switch (argv[i]) {
case "--model-tag": a.modelTag = next(); break;
case "--models-src": a.modelsSrc = next(); break;
// Clone kicad-packages3D at --model-tag into <dir>/kicad-packages3D
// (shallow; NOTE: multi-GB working tree) and use it as the source.
case "--clone": a.clone = next(); break;
case "--driver": a.driver = next(); break;
case "--out": a.out = next(); break;
case "--bucket": a.bucket = next(); break;
case "--remote": a.remote = true; break;
case "--prefix": a.prefix = next(); break;
case "--force": a.force = true; break;
// Dev subset: publish only these libs (names without .3dshapes).
case "--libs": a.libs = next().split(",").map((s) => s.trim()).filter(Boolean); break;
// Brotli quality for bodies (WRL/STEP are text-ish; 5 ≈ 4-5x, fast).
case "--quality": a.quality = Number(next()); break;
// "none" for --driver local when a plain static server (e.g. the vite dev
// server) will serve the blobs — it can't send Content-Encoding: br.
case "--compress": a.compress = next() as "br" | "none"; break;
default: throw new Error(`unknown arg: ${argv[i]}`);
}
}
if (!a.modelTag) throw new Error("--model-tag <kicad-packages3D tag> is required");
if (!a.modelsSrc && !a.clone)
throw new Error("need --models-src <kicad-packages3D checkout> or --clone <dir>");
if (a.driver === "local" && !a.out) a.out = ".cdn-models-out";
return a;
}
function cloneShallow(url: string, dest: string, ref: string): void {
if (existsSync(dest)) {
console.log(`clone: ${dest} present — reusing`);
return;
}
mkdirSync(dirname(dest), { recursive: true });
console.log(`clone: ${url} @ ${ref} → ${dest} (multi-GB — this takes a while)`);
execFileSync("git", ["clone", "--depth", "1", "--branch", ref, url, dest], {
stdio: "inherit",
});
}
/** All `<lib>.3dshapes` dirs under the checkout root (non-recursive: the repo is flat). */
function listModelLibs(src: string): Array<{ id: string; dir: string }> {
return readdirSync(src, { withFileTypes: true })
.filter((d) => d.isDirectory() && d.name.endsWith(".3dshapes"))
.map((d) => ({ id: d.name.slice(0, -".3dshapes".length), dir: join(src, d.name) }))
.sort((x, y) => x.id.localeCompare(y.id));
}
async function main(): Promise<void> {
const a = parseArgs(process.argv);
const store = makeStore(a.driver, a);
const topKey = `${a.prefix}/${a.modelTag}/manifest.json`;
const registryKey = `${a.prefix}/blobs/registry.json`;
if (!a.force && store.getJSON(topKey)) {
console.log(`publish-models: ${topKey} already published — skipping (use --force)`);
return;
}
if (a.clone) {
const dest = join(a.clone, "kicad-packages3D");
cloneShallow(MODELS_URL, dest, a.modelTag!);
a.modelsSrc ??= dest;
}
console.log(
`publish-models: tag=${a.modelTag} driver=${store.kind} → ${a.prefix}/${a.modelTag}/`,
);
// Published-blob index: hash → original size. One GET up front, one PUT at the
// end — the per-blob "does it exist" probe would otherwise be an R2 round-trip
// per model (tens of thousands).
const registry: Record<string, number> =
(store.getJSON(registryKey) as Record<string, number> | null) ?? {};
let blobsPut = 0;
let blobsSkipped = 0;
let libs = listModelLibs(a.modelsSrc!);
if (a.libs) {
const want = new Set(a.libs);
libs = libs.filter((l) => want.has(l.id));
const missing = a.libs.filter((id) => !libs.some((l) => l.id === id));
if (missing.length) console.warn(`publish-models: libs not found: ${missing.join(", ")}`);
}
if (!libs.length) throw new Error(`no .3dshapes libs under ${a.modelsSrc}`);
const topLibs: Array<{ id: string; itemCount: number; bytes: number }> = [];
let totalItems = 0;
let totalBytes = 0;
for (const lib of libs) {
const files = readdirSync(lib.dir)
.filter((f) => MODEL_EXTS.some((ext) => f.toLowerCase().endsWith(ext)))
.sort();
if (!files.length) continue;
const entries: SyncManifest["entries"] = {};
let libBytes = 0;
for (const f of files) {
const p = join(lib.dir, f);
if (!statSync(p).isFile()) continue;
const body = readFileSync(p);
const hash = sha256hex(body);
entries[`model3d/${f}`] = { hash, size: body.length, mtime: 0 };
libBytes += body.length;
if (registry[hash] === undefined) {
// WRL and STEP are text formats — brotli gets ~4-5x. The browser fetch
// transparently decodes, so IDB caches (and hashes refer to) the
// ORIGINAL bytes; `no-transform` keeps the edge from re-encoding.
const { bytes, encoding } = compressBytes(body, a.compress, a.quality);
store.put(`${a.prefix}/blobs/sha256/${hash}`, bytes, {
contentType: "application/octet-stream",
contentEncoding: encoding,
cacheControl: IMMUTABLE,
});
registry[hash] = body.length;
blobsPut++;
} else {
blobsSkipped++;
}
}
const manifest: SyncManifest = { version: 1, entries };
putJSON(store, `${a.prefix}/${a.modelTag}/${lib.id}/manifest`, manifest, IMMUTABLE);
topLibs.push({ id: lib.id, itemCount: files.length, bytes: libBytes });
totalItems += files.length;
totalBytes += libBytes;
console.log(` ${lib.id}: ${files.length} models (${(libBytes / 1e6).toFixed(1)} MB)`);
}
putJSON(store, registryKey, registry, NO_STORE);
putJSON(store, topKey, { schema: 1, tag: a.modelTag, libs: topLibs }, IMMUTABLE);
console.log(
`publish-models: done — ${topLibs.length} libs, ${totalItems} models ` +
`(${(totalBytes / 1e6).toFixed(0)} MB raw), blobs put=${blobsPut} deduped=${blobsSkipped} → ${topKey}`,
);
if (store.kind === "local") console.log(`local layout under: ${a.out}`);
}
main().catch((err) => {
console.error(err);
process.exit(1);
});

255
tests/kicad/3d-viewer-models.spec.ts Normal file
View file

@ -0,0 +1,255 @@
import type { Page } from '@playwright/test';
import { test, expect } from './fixtures';
import { clickMenuBarItem, clickMenuItem } from '../e2e/utils/element-tracker';
import { injectFromSubmodule } from './utils/fs-inject';
import { waitForBoardLoaded } from './utils/board-ready';
import { waitForPcbnew } from './utils/pcbnew-ready';
/**
* 3D viewer COMPONENT MODELS e2e (docs/features/3d-models): load pic_programmer,
* open the 3D viewer, and verify the model-delivery machinery end to end at the
* KiCad/wasm level:
*
* 1. Statically linked format plugins (vrml + oce — upstream loads them via
* dlopen, which wasm doesn't have) parse real model files.
* 2. Project-local models resolve exactly as upstream: the board references
* `${KIPRJMOD}/libs/3d_shapes/*.wrl`, injected with the project.
* 3. The lazy-fetch fallback (S3D_CACHE::load → PCBJAM_3D::EnsureModelFile →
* `kicadLibs.request("ensure", …, "model3d")`) asks JS for every
* `${KICAD*_3DMODEL_DIR}` ref, with the ref NORMALIZED to
* `<lib>.3dshapes/<name>.<ext>` — and a served ref (the stub writes the
* bytes into MEMFS and answers "1") then resolves and renders.
*
* The stub provider stands in for the standalone's models-bridge (which fetches
* from the CDN into IDB); here it serves ONE in-repo STEP fixture under a
* board-referenced name — geometry is a USB-C connector where a DIP-8 socket
* belongs, which is irrelevant: the assertion is parse+render, not fidelity.
*/
const KICAD_VERSION_DIR = '10.0';
const PROJECT_DIR_MEMFS = `/home/kicad/documents/kicad/${KICAD_VERSION_DIR}/projects`;
// The JS-owned MEMFS root the stub writes model bodies under — the same dir
// the standalone's models-bridge uses (constants.ts MODELS_3D_ROOT). Its exact
// location is immaterial: the ensure protocol answers with the ABSOLUTE path
// and S3D_CACHE loads it directly (env-var expansion never resolves
// ${KICAD*_3DMODEL_DIR} refs in the wasm runtime — see
// docs/features/3d-models/0001).
const MODELS_ROOT_MEMFS = '/pcbjam/3dmodels';
// The board ref the stub provider serves (normalized form the bridge must ask
// for), and the in-repo STEP whose bytes stand in for it.
const SERVED_REF = 'Package_DIP.3dshapes/DIP-8_W7.62mm.step';
const STEP_FIXTURE = 'kicad/demos/openair-max/Libraries/HRO_TYPE-C-31-M-12.step';
const DEMO = { name: 'pic_programmer', dir: 'pic_programmer', stem: 'pic_programmer' } as const;
declare global {
interface Window {
__modelEnsures?: Array<{ op: string; arg: string; kind: string }>;
__stepFixtureB64?: string;
}
}
/** Record every model3d bridge request; serve SERVED_REF from the fixture. */
async function installModelProviderStub(page: Page, serveAll = false): Promise<void> {
await page.evaluate(
({ stockDir, servedRef, serveAll }) => {
window.__modelEnsures = [];
(globalThis as any).kicadLibs = {
request: async (op: string, _lib: string, arg: string, kind: string) => {
if (kind !== 'model3d') return null;
window.__modelEnsures!.push({ op, arg, kind });
console.log(`[TEST-3D] ensure request: ${op} ${arg}`);
if (op !== 'ensure' || (!serveAll && arg !== servedRef)) return null;
const b64 = window.__stepFixtureB64!;
const binary = atob(b64);
const data = new Uint8Array(binary.length);
for (let i = 0; i < binary.length; i++) data[i] = binary.charCodeAt(i);
// Mirror models-bridge.ts ensureModelInMemfs: write under the
// JS-owned model root and answer with the ABSOLUTE path —
// S3D_CACHE loads it directly (no env-var expansion needed).
// @ts-expect-error — Emscripten FS lives on window
const FS = (window as any).FS;
const dest = `${stockDir}/${arg}`;
FS.mkdirTree(dest.slice(0, dest.lastIndexOf('/')));
FS.writeFile(dest, data);
console.log(`[TEST-3D] served ${arg} → ${dest} (${data.length} bytes)`);
return dest;
},
};
},
{ stockDir: MODELS_ROOT_MEMFS, servedRef: SERVED_REF, serveAll },
);
}
async function loadBoard(page: Page, testLogger: { consoleLogs: string[]; errors: string[] }): Promise<void> {
const pcbFilename = `${DEMO.stem}.kicad_pcb`;
const proFilename = `${DEMO.stem}.kicad_pro`;
await injectFromSubmodule(page, `kicad/demos/${DEMO.dir}/${pcbFilename}`,
`${PROJECT_DIR_MEMFS}/${pcbFilename}`);
await injectFromSubmodule(page, `kicad/demos/${DEMO.dir}/${proFilename}`,
`${PROJECT_DIR_MEMFS}/${proFilename}`);
// Project-local 3D models — the board references them as
// ${KIPRJMOD}/libs/3d_shapes/<name>.wrl; resolved by the stock resolver, so
// they must NOT go through the ensure bridge (asserted below).
await injectFromSubmodule(page, `kicad/demos/${DEMO.dir}/libs/3d_shapes/textool_40.wrl`,
`${PROJECT_DIR_MEMFS}/libs/3d_shapes/textool_40.wrl`);
await injectFromSubmodule(page, `kicad/demos/${DEMO.dir}/libs/3d_shapes/adjustable_rx2v4.wrl`,
`${PROJECT_DIR_MEMFS}/libs/3d_shapes/adjustable_rx2v4.wrl`);
expect(await clickMenuBarItem(page, 'File'), 'File menu should be findable').toBe(true);
await page.waitForTimeout(400);
expect(await clickMenuItem(page, 'Open...'), 'Open… menu item should be findable').toBe(true);
await page.waitForFunction(() => {
const registry = window.wxElementRegistry;
return !!registry && registry.findAll({ visible: true })
.some((el) => el.typeName === 'wxFileDialog');
}, null, { timeout: 15000 });
await page.waitForTimeout(1000);
const filenameInput = await page.evaluate(() => {
const registry = window.wxElementRegistry;
if (!registry) return null;
const text = registry.findAll({ visible: true })
.find((el) => el.typeName === 'wxTextCtrl' && el.name === 'text');
return text ? { x: text.centerX, y: text.centerY } : null;
});
expect(filenameInput, 'filename text input should be visible').not.toBeNull();
if (!filenameInput) throw new Error('filename text input not found');
await page.mouse.click(filenameInput.x, filenameInput.y);
await page.waitForTimeout(200);
await page.keyboard.type(pcbFilename);
await page.waitForTimeout(300);
await page.keyboard.press('Enter');
await page.waitForTimeout(1000);
const result = await waitForBoardLoaded(page, testLogger, 60000);
console.log(`[TEST] ${DEMO.name} board-ready result: ${result}`);
}
function countGlCanvases(page: Page): Promise<number> {
return page.evaluate(() => document.querySelectorAll('canvas[id^="glcanvas-"]').length);
}
async function openThreeDViewer(page: Page, glBefore: number): Promise<number> {
let opened = false;
if (await clickMenuBarItem(page, 'View')) {
await page.waitForTimeout(400);
opened = await clickMenuItem(page, '3D Viewer');
}
if (!opened) {
console.log('[TEST] View → 3D Viewer not found via menu; trying Alt+3');
await page.keyboard.press('Escape');
await page.waitForTimeout(200);
await page.keyboard.press('Alt+3');
}
await page.waitForFunction(() => {
return !!document.querySelector('#window-container [id^="window-"]')
|| document.querySelectorAll('canvas[id^="glcanvas-"]').length > 0;
}, null, { timeout: 60000 });
await page.waitForFunction((before: number) =>
document.querySelectorAll('canvas[id^="glcanvas-"]').length > before,
glBefore, { timeout: 60000 });
const glAfter = await countGlCanvases(page);
console.log(`[TEST] glcanvas count after opening 3D viewer: ${glAfter}`);
expect(glAfter, 'a new WebGL canvas should appear for the 3D viewer').toBeGreaterThan(glBefore);
return glAfter;
}
test.describe('3D viewer component models', () => {
test.describe.configure({ mode: 'serial' });
test.setTimeout(240000);
test('resolves project models, lazy-fetches lib models via the bridge, renders', async ({ page, testLogger }) => {
await page.goto('/kicad/pcbnew.html');
await waitForPcbnew(page);
// Stash the STEP fixture bytes + install the provider stub BEFORE the
// viewer can issue any ensure request.
const fs = require('fs') as typeof import('fs');
const path = require('path') as typeof import('path');
const fixtureAbs = path.resolve(__dirname, '..', '..', STEP_FIXTURE);
await page.evaluate(
(b64: string) => { window.__stepFixtureB64 = b64; },
fs.readFileSync(fixtureAbs).toString('base64'),
);
await installModelProviderStub(page);
await loadBoard(page, testLogger);
const glBefore = await countGlCanvases(page);
await openThreeDViewer(page, glBefore);
// Scene build + progressive raytrace passes.
await page.waitForTimeout(8000);
await page.screenshot({ path: `test-results/3d-viewer-models-${DEMO.name}.png`, scale: 'css' });
// --- bridge assertions -------------------------------------------------
const ensures = await page.evaluate(() => window.__modelEnsures ?? []);
console.log(`[TEST] ensure requests: ${ensures.length}`);
for (const e of ensures.slice(0, 30)) console.log(`[TEST] ${e.op} ${e.arg}`);
// Every ${KICAD*_3DMODEL_DIR} ref crossed the bridge, normalized.
const args = ensures.map((e) => e.arg);
expect(args, 'the served lib ref must cross the bridge normalized')
.toContain(SERVED_REF);
expect(args.every((a) => /^[^/${]+\.3dshapes\//.test(a)),
'every bridge ref is a normalized <lib>.3dshapes/<file> path').toBe(true);
// Project-local (${KIPRJMOD}) models resolve natively — never bridged.
expect(args.some((a) => a.includes('textool_40') || a.includes('adjustable_rx2v4')),
'project-local models must not go through the ensure bridge').toBe(false);
// Board refs are unique per model file — the C++ memo must not re-ask.
expect(new Set(args).size, 'ensure requests are deduplicated').toBe(args.length);
// The served model landed in MEMFS where the resolver looks.
const servedSize = await page.evaluate(
({ stockDir, servedRef }) => {
// @ts-expect-error — Emscripten FS lives on window
const FS = (window as any).FS;
try { return FS.stat(`${stockDir}/${servedRef}`).size as number; }
catch { return -1; }
},
{ stockDir: MODELS_ROOT_MEMFS, servedRef: SERVED_REF },
);
expect(servedSize, 'served STEP written into the model root').toBeGreaterThan(1000);
// --- render assertion --------------------------------------------------
const render = await page.evaluate(() => {
const list = document.querySelectorAll('canvas[id^="glcanvas-"]');
const el = list[list.length - 1] as HTMLCanvasElement;
const tmp = document.createElement('canvas');
tmp.width = el.width;
tmp.height = el.height;
const ctx = tmp.getContext('2d')!;
ctx.drawImage(el, 0, 0);
const colors = new Set<string>();
for (let i = 0; i < 16; i++) {
for (let j = 0; j < 16; j++) {
const d = ctx.getImageData(Math.floor(el.width * i / 16),
Math.floor(el.height * j / 16), 1, 1).data;
colors.add(`${d[0]},${d[1]},${d[2]}`);
}
}
return { id: el.id, w: el.width, h: el.height, distinctColors: colors.size,
dataUrl: tmp.toDataURL('image/png') };
});
console.log(`[TEST] 3D canvas ${render.id} ${render.w}x${render.h}, distinct colours: ${render.distinctColors}`);
const b64 = render.dataUrl.replace(/^data:image\/png;base64,/, '');
fs.writeFileSync(`test-results/3d-viewer-models-${DEMO.name}-render.png`,
Buffer.from(b64, 'base64'));
expect(render.distinctColors,
'the 3D viewer canvas should render the board + models, not a blank fill')
.toBeGreaterThan(8);
expect(testLogger.errors, 'no page errors during the model flow').toEqual([]);
});
});

1
tests/playwright-kicad.config.ts
View file

@ -89,6 +89,7 @@ const PCBNEW_FAMILY_SPECS = [
'**/pcbnew-move.spec.ts', '**/pcbnew-move.spec.ts',
// 3D viewer specs boot pcbnew.html (3D-enabled build) — same V8 routing. // 3D viewer specs boot pcbnew.html (3D-enabled build) — same V8 routing.
'**/3d-viewer.spec.ts', '**/3d-viewer.spec.ts',
'**/3d-viewer-models.spec.ts',
'**/footprint-3d-preview.spec.ts', '**/footprint-3d-preview.spec.ts',
]; ];

3
web/.gitignore vendored
View file

@ -13,3 +13,6 @@ dist/
# .demo-cdn holds its generated bytes — both created by scripts/deploy/dev-demo.mjs. # .demo-cdn holds its generated bytes — both created by scripts/deploy/dev-demo.mjs.
**/public/content **/public/content
**/standalone/.demo-cdn/ **/standalone/.demo-cdn/
# 3D models: public/models-cdn symlinks a local publish-models layout, created
# by scripts/deploy/dev-demo.mjs --models-local — never committed.
**/public/models-cdn

2
web/pcbjam-shared

@ -1 +1 @@
Subproject commit 9a1a269fdd9079921029ecf6e2dce89dece98fc8 Subproject commit 067170e50f1c2ba19ffceb9815dcf5d34dc9e800

235
web/standalone/src/components/StorageUsageCard.tsx Normal file
View file

@ -0,0 +1,235 @@
import * as React from "react";
import { Database, Loader2, RefreshCw, Trash2 } from "lucide-react";
import { Button } from "@/components/ui/button";
/**
* Browser-storage usage for the editor's library caches, by kind, with a
* "delete 3D model cache" action.
*
* All library data lives in r2-idb-sync IndexedDB databases named
* `sync:<namespace>` (one per lib), each with a `bodies` store keyed
* `"<kind>/<name>"`. Symbols/footprints are bulk-synced (whole-lib bundles);
* 3D models are SPARSE — only the models boards actually rendered are stored
* (namespaces `kicad-models:…`), which makes them the one cache that is both
* potentially large and always safe to drop: manifests re-sync in one small
* fetch and bodies lazily re-download exactly when a board needs them again.
*/
const SYNC_PREFIX = "sync:";
const MODELS_NS_PREFIX = `${SYNC_PREFIX}kicad-models:`;
interface KindUsage {
bytes: number;
items: number;
}
interface StorageBreakdown {
symbol: KindUsage;
footprint: KindUsage;
model3d: KindUsage;
/** Whole-origin estimate (all IDB + caches), from navigator.storage. */
originUsage: number | null;
originQuota: number | null;
/** Names of the model DBs (the delete target). */
modelDbs: string[];
}
function emptyUsage(): KindUsage {
return { bytes: 0, items: 0 };
}
/** Sum body sizes per kind in one `sync:*` DB via a cursor (no bulk getAll —
* a model cache can be hundreds of MB and we only need the sizes). */
function measureDb(name: string, into: StorageBreakdown): Promise<void> {
return new Promise((resolve) => {
const req = indexedDB.open(name);
// Never upgrade here (no version passed); a DB from a newer schema still
// opens read-only for measuring.
req.onerror = () => resolve();
req.onsuccess = () => {
const db = req.result;
if (!db.objectStoreNames.contains("bodies")) {
db.close();
resolve();
return;
}
const tx = db.transaction("bodies", "readonly");
const cursorReq = tx.objectStore("bodies").openCursor();
cursorReq.onsuccess = () => {
const cursor = cursorReq.result;
if (!cursor) return; // tx completes → oncomplete below
const key = String(cursor.key);
const value = cursor.value as { byteLength?: number } | undefined;
const size = value?.byteLength ?? 0;
const kind = key.startsWith("symbol/")
? "symbol"
: key.startsWith("footprint/")
? "footprint"
: key.startsWith("model3d/")
? "model3d"
: null;
if (kind) {
into[kind].bytes += size;
into[kind].items += 1;
}
cursor.continue();
};
tx.oncomplete = () => {
db.close();
resolve();
};
tx.onerror = () => {
db.close();
resolve();
};
};
});
}
async function measureAll(): Promise<StorageBreakdown> {
const breakdown: StorageBreakdown = {
symbol: emptyUsage(),
footprint: emptyUsage(),
model3d: emptyUsage(),
originUsage: null,
originQuota: null,
modelDbs: [],
};
// indexedDB.databases() is supported everywhere we run (Chromium, FF 126+,
// Safari 14+); without it we just show the origin estimate.
const dbs = (await indexedDB.databases?.()) ?? [];
for (const db of dbs) {
const name = db.name;
if (!name || !name.startsWith(SYNC_PREFIX)) continue;
if (name.startsWith(MODELS_NS_PREFIX)) breakdown.modelDbs.push(name);
await measureDb(name, breakdown);
}
try {
const est = await navigator.storage?.estimate?.();
breakdown.originUsage = est?.usage ?? null;
breakdown.originQuota = est?.quota ?? null;
} catch {
// estimate unavailable — the per-kind rows still stand on their own
}
return breakdown;
}
function deleteDb(name: string): Promise<void> {
return new Promise((resolve) => {
const req = indexedDB.deleteDatabase(name);
req.onsuccess = req.onerror = req.onblocked = () => resolve();
});
}
export function formatBytes(n: number): string {
if (n < 1024) return `${n} B`;
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
if (n < 1024 * 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)} MB`;
return `${(n / 1024 / 1024 / 1024).toFixed(2)} GB`;
}
export function StorageUsageCard() {
const [data, setData] = React.useState<StorageBreakdown | null>(null);
const [busy, setBusy] = React.useState(false);
const [clearing, setClearing] = React.useState(false);
const refresh = React.useCallback(async () => {
setBusy(true);
try {
setData(await measureAll());
} finally {
setBusy(false);
}
}, []);
React.useEffect(() => {
void refresh();
}, [refresh]);
const clearModels = React.useCallback(async () => {
if (!data) return;
setClearing(true);
try {
for (const name of data.modelDbs) await deleteDb(name);
await refresh();
} finally {
setClearing(false);
}
}, [data, refresh]);
const rows = data
? ([
["Symbols", data.symbol],
["Footprints", data.footprint],
["3D models", data.model3d],
] as const)
: null;
return (
<section className="mb-10 rounded-lg border p-5">
<h2 className="mb-1 flex items-center gap-2 text-lg font-medium">
<Database size={16} /> Storage
</h2>
<p className="mb-4 text-sm text-muted-foreground">
Library data cached in this browser. Everything re-downloads on demand —
clearing is always safe. 3D models are fetched per board, so their cache
is the one worth reclaiming.
</p>
{!data ? (
<p className="flex items-center gap-2 text-sm text-muted-foreground">
<Loader2 className="animate-spin" size={14} /> Measuring…
</p>
) : (
<div className="space-y-2">
<table className="w-full max-w-md text-sm">
<tbody>
{rows!.map(([label, usage]) => (
<tr key={label} className="border-b last:border-b-0">
<td className="py-1.5 text-muted-foreground">{label}</td>
<td className="py-1.5 text-right tabular-nums">
{usage.items.toLocaleString()} items
</td>
<td className="py-1.5 text-right font-medium tabular-nums">
{formatBytes(usage.bytes)}
</td>
</tr>
))}
</tbody>
</table>
{data.originUsage !== null && (
<p className="text-xs text-muted-foreground">
Site total (all caches): {formatBytes(data.originUsage)}
{data.originQuota ? ` of ${formatBytes(data.originQuota)} available` : ""}
</p>
)}
<div className="flex items-center gap-2 pt-1">
<Button
variant="outline"
size="sm"
onClick={() => void clearModels()}
disabled={clearing || data.model3d.items === 0}
>
{clearing ? (
<Loader2 className="mr-1 animate-spin" size={14} />
) : (
<Trash2 className="mr-1" size={14} />
)}
Delete 3D model cache
{data.model3d.bytes > 0 ? ` (${formatBytes(data.model3d.bytes)})` : ""}
</Button>
<Button
variant="ghost"
size="sm"
onClick={() => void refresh()}
disabled={busy}
aria-label="Refresh storage usage"
>
<RefreshCw className={busy ? "animate-spin" : ""} size={14} />
</Button>
</div>
</div>
)}
</section>
);
}

30
web/standalone/src/components/WasmTool.tsx
View file

@ -17,6 +17,7 @@ import { ChevronDown, ChevronUp, Loader2 } from "lucide-react";
import { import {
currentScope, currentScope,
libsSourceConfig, libsSourceConfig,
modelsSourceConfig,
yjsProviderConfig, yjsProviderConfig,
type DocSource, type DocSource,
} from "@/lib/config"; } from "@/lib/config";
@ -31,6 +32,10 @@ import {
type LibLoadingDetail, type LibLoadingDetail,
type LibsSource, type LibsSource,
} from "@/wasm/libs/source"; } from "@/wasm/libs/source";
import {
MODELS_LOADING_EVENT,
type ModelsLoadingDetail,
} from "@/wasm/libs/models-bridge";
import { memfsFilePath, memfsProjectDir } from "@/wasm/constants"; import { memfsFilePath, memfsProjectDir } from "@/wasm/constants";
import { driveProjectIntoTool, type ToolFile } from "@/wasm/kicad-runner"; import { driveProjectIntoTool, type ToolFile } from "@/wasm/kicad-runner";
import { registerSaveHook, type SaveBytes } from "@/wasm/save-flow"; import { registerSaveHook, type SaveBytes } from "@/wasm/save-flow";
@ -595,6 +600,9 @@ export function WasmTool({
done: number; done: number;
total: number; total: number;
} | null>(null); } | null>(null);
// Board 3D-model prefetch in flight (background; the viewer works without it —
// anything still missing lazy-loads per model). Small badge, not an overlay.
const [modelsSync, setModelsSync] = React.useState<string | null>(null);
const append = React.useCallback( const append = React.useCallback(
(msg: string) => setLogs((prev) => [...prev.slice(-800), msg]), (msg: string) => setLogs((prev) => [...prev.slice(-800), msg]),
@ -660,6 +668,18 @@ export function WasmTool({
}; };
}, []); }, []);
// Board 3D-model prefetch progress (models-bridge prescan) — background badge.
React.useEffect(() => {
const onModels = (e: Event) => {
const d = (e as CustomEvent<ModelsLoadingDetail>).detail;
setModelsSync(
d.loading ? `Fetching 3D models — ${d.done}/${d.total}` : null,
);
};
window.addEventListener(MODELS_LOADING_EVENT, onModels);
return () => window.removeEventListener(MODELS_LOADING_EVENT, onModels);
}, []);
// "Taking too long": once the tool has been loading for a while without // "Taking too long": once the tool has been loading for a while without
// becoming ready, surface a hint (slow link / something may be wrong) + a // becoming ready, surface a hint (slow link / something may be wrong) + a
// reload, so a stalled boot doesn't look like a frozen blank screen. // reload, so a stalled boot doesn't look like a frozen blank screen.
@ -756,6 +776,9 @@ export function WasmTool({
onAbort: oom.onAbort, onAbort: oom.onAbort,
onProgress: (loaded, total) => setProgress({ loaded, total }), onProgress: (loaded, total) => setProgress({ loaded, total }),
libsSource: source, libsSource: source,
// 3D models: lazy per-board source (null unless the CDN manifest is
// configured) — feeds the board prescan + the viewer's ensure fallback.
modelsSource: modelsSourceConfig(),
}); });
// Register the save sink before the file opens: from here on, every // Register the save sink before the file opens: from here on, every
// editor File→Save (MEMFS write) is routed onward through saveBytes. // editor File→Save (MEMFS write) is routed onward through saveBytes.
@ -1002,6 +1025,13 @@ export function WasmTool({
</div> </div>
)} )}
{/* Board 3D models still prefetching into the cache (background). */}
{ready && modelsSync && (
<div className="pointer-events-none absolute bottom-[4.25rem] left-3 z-20 flex items-center gap-2 rounded bg-black/80 px-3 py-1.5 text-xs text-sky-200">
<Loader2 className="animate-spin" size={14} /> {modelsSync}
</div>
)}
{/* A library item is being fetched (open/save). */} {/* A library item is being fetched (open/save). */}
{ready && libBusy && ( {ready && libBusy && (
<div className="pointer-events-none absolute left-1/2 top-3 z-20 flex -translate-x-1/2 items-center gap-2 rounded bg-black/80 px-3 py-1.5 text-xs text-white"> <div className="pointer-events-none absolute left-1/2 top-3 z-20 flex -translate-x-1/2 items-center gap-2 rounded bg-black/80 px-3 py-1.5 text-xs text-white">

17
web/standalone/src/lib/config.ts
View file

@ -98,6 +98,7 @@ export const LOCAL_PROJECTS_ENABLED =
import type { ProviderConfig, ProviderKind } from "@/wasm/collab"; import type { ProviderConfig, ProviderKind } from "@/wasm/collab";
import { cdnLibsSource } from "@/wasm/libs/cdn-source"; import { cdnLibsSource } from "@/wasm/libs/cdn-source";
import { cdnModelsSource, type Model3dSource } from "@/wasm/libs/models-source";
import { remoteLibsSource } from "@/wasm/libs/remote-source"; import { remoteLibsSource } from "@/wasm/libs/remote-source";
import { scopedLibsSource } from "@/wasm/libs/scoped-source"; import { scopedLibsSource } from "@/wasm/libs/scoped-source";
import type { LibsSource } from "@/wasm/libs/source"; import type { LibsSource } from "@/wasm/libs/source";
@ -192,6 +193,22 @@ export function currentScope(): string {
export const CDN_LIBS_MANIFEST_URL = export const CDN_LIBS_MANIFEST_URL =
import.meta.env.VITE_LIBS_MANIFEST_URL || null; import.meta.env.VITE_LIBS_MANIFEST_URL || null;
/** Full URL of the CDN 3D-models top manifest, e.g.
* https://cdn.pcbjam.com/libs/kicad-models/10.0.0/manifest.json. Bodies are
* fetched lazily per board (sparse layers) and cached in IDB — never bulk
* synced. Unset ⇒ the 3D viewer renders bare boards (no component models).
* See wasm/libs/models-source.ts + docs/features/3d-models. */
export const CDN_MODELS_MANIFEST_URL =
import.meta.env.VITE_MODELS_MANIFEST_URL || null;
/** The 3D model source for a tool boot (null ⇒ models disabled). One instance
* per call — WasmTool keeps a single instance per boot like the libs source. */
export function modelsSourceConfig(): Model3dSource | null {
return CDN_MODELS_MANIFEST_URL
? cdnModelsSource(CDN_MODELS_MANIFEST_URL)
: null;
}
export function libsSourceConfig(projectId?: string): LibsSource | null { export function libsSourceConfig(projectId?: string): LibsSource | null {
const kind = import.meta.env.VITE_LIBS_SOURCE ?? "remote"; const kind = import.meta.env.VITE_LIBS_SOURCE ?? "remote";
// "local" is the placeholder id for launches with no real backend project // "local" is the placeholder id for launches with no real backend project

6
web/standalone/src/pages/HomePage.tsx
View file

@ -23,6 +23,7 @@ import { WaitlistForm } from "@/components/WaitlistForm";
import { NewFileDialog } from "@/components/NewFileDialog"; import { NewFileDialog } from "@/components/NewFileDialog";
import type { SaveBytes } from "@/wasm/save-flow"; import type { SaveBytes } from "@/wasm/save-flow";
import { LocalProjectView, type LocalFile } from "@/components/LocalProjectView"; import { LocalProjectView, type LocalFile } from "@/components/LocalProjectView";
import { StorageUsageCard } from "@/components/StorageUsageCard";
import { WasmTool } from "@/components/WasmTool"; import { WasmTool } from "@/components/WasmTool";
/** A KiCad project picked from the local filesystem (no backend involved). */ /** A KiCad project picked from the local filesystem (no backend involved). */
@ -322,6 +323,11 @@ export function HomePage() {
/> />
</div> </div>
</section> </section>
{/* --- Browser storage: per-kind cache sizes + delete-3D-cache. --- */}
<div className="mt-10">
<StorageUsageCard />
</div>
</div> </div>
); );
} }

34
web/standalone/src/wasm/boot.ts
View file

@ -1,11 +1,15 @@
import type { Tool } from "@pcbjam/shared"; import type { Tool } from "@pcbjam/shared";
import { import {
KICAD_CONFIG_DIR, KICAD_CONFIG_DIR,
MODELS_3D_ENV_VARS,
MODELS_3D_ROOT,
RESOURCE_PATH, RESOURCE_PATH,
TOOL_ARGV0, TOOL_ARGV0,
TOOL_LIB_KIND, TOOL_LIB_KIND,
TOOL_NEEDS_CONFIG_SEED, TOOL_NEEDS_CONFIG_SEED,
} from "./constants"; } from "./constants";
import { installModel3dHandler } from "./libs/models-bridge";
import type { Model3dSource } from "./libs/models-source";
import { import {
buildFpLibTable, buildFpLibTable,
buildSymLibTable, buildSymLibTable,
@ -60,6 +64,9 @@ export interface BootOptions {
/** Library source backing `window.kicadLibs`. Null/omitted disables libs /** Library source backing `window.kicadLibs`. Null/omitted disables libs
* (an empty sym-lib-table is seeded). Its libs become sym-lib-table rows. */ * (an empty sym-lib-table is seeded). Its libs become sym-lib-table rows. */
libsSource?: LibsSource | null; libsSource?: LibsSource | null;
/** 3D model source (lazy, per-board). Null/omitted ⇒ the viewer renders the
* bare board only, exactly as before models existed. */
modelsSource?: Model3dSource | null;
} }
let booted: { tool: Tool; promise: Promise<void> } | null = null; let booted: { tool: Tool; promise: Promise<void> } | null = null;
@ -177,8 +184,17 @@ async function fetchWasmWithProgress(
} }
async function doBoot(opts: BootOptions): Promise<void> { async function doBoot(opts: BootOptions): Promise<void> {
const { tool, base, container, log, onStatus, onAbort, onProgress, libsSource } = const {
opts; tool,
base,
container,
log,
onStatus,
onAbort,
onProgress,
libsSource,
modelsSource,
} = opts;
const w = window as ToolWindow; const w = window as ToolWindow;
// The wasm reads the top-level frame geometry from a GLOBAL `mainWindow` // The wasm reads the top-level frame geometry from a GLOBAL `mainWindow`
@ -215,6 +231,12 @@ async function doBoot(opts: BootOptions): Promise<void> {
const libKind = TOOL_LIB_KIND[tool]; const libKind = TOOL_LIB_KIND[tool];
if (libsSource && libKind) { if (libsSource && libKind) {
installLibsProvider(libsSource, log); installLibsProvider(libsSource, log);
// 3D models ride the same provider (kind "model3d"): the C++ ensure fallback
// and the board prescan both resolve through this source.
if (modelsSource) {
installModel3dHandler(modelsSource, log);
log("[3d] model source installed");
}
try { try {
// Ensure the owner has at least one writable user lib to save items into. // Ensure the owner has at least one writable user lib to save items into.
// Pass the tool's kind so origins are filtered to the right domain. // Pass the tool's kind so origins are filtered to the right domain.
@ -321,6 +343,9 @@ async function doBoot(opts: BootOptions): Promise<void> {
FS.writeFile(path, contents); FS.writeFile(path, contents);
log(`[boot] seeded ${path}`); log(`[boot] seeded ${path}`);
}; };
// 3D models: the MEMFS root the prescan/ensure paths write into, plus the
// env vars (every vintage) that make KiCad's resolver look there.
FS.mkdirTree(MODELS_3D_ROOT);
writeIfAbsent( writeIfAbsent(
`${KICAD_CONFIG_DIR}/kicad_common.json`, `${KICAD_CONFIG_DIR}/kicad_common.json`,
JSON.stringify( JSON.stringify(
@ -329,6 +354,11 @@ async function doBoot(opts: BootOptions): Promise<void> {
update_check_prompt: true, update_check_prompt: true,
data_collection_prompt: true, data_collection_prompt: true,
}, },
environment: {
vars: Object.fromEntries(
MODELS_3D_ENV_VARS.map((v) => [v, MODELS_3D_ROOT]),
),
},
}, },
null, null,
2, 2,

18
web/standalone/src/wasm/constants.ts
View file

@ -72,6 +72,24 @@ export const TOOL_LIB_KIND: Record<Tool, "symbol" | "footprint" | null> = {
/** KiCad user settings dir for this build (PATHS::GetUserSettingsPath()). */ /** KiCad user settings dir for this build (PATHS::GetUserSettingsPath()). */
export const KICAD_CONFIG_DIR = `/home/kicad/.config/kicad/kicad/${KICAD_VERSION_DIR}`; export const KICAD_CONFIG_DIR = `/home/kicad/.config/kicad/kicad/${KICAD_VERSION_DIR}`;
/**
* MEMFS root where 3D model bodies are materialized (JS prescan + the C++
* lazy-ensure fallback both write `<root>/<lib>.3dshapes/<name>.<ext>` here).
* Boot points every `KICAD*_3DMODEL_DIR` env var at this dir — official-lib
* footprints reference models through vintage-specific vars (KICAD6..10 all
* occur), and KiCad's stock FILENAME_RESOLVER picks up each var it finds — so
* model paths resolve with zero resolver changes.
*/
export const MODELS_3D_ROOT = "/pcbjam/3dmodels";
export const MODELS_3D_ENV_VARS = [
"KISYS3DMOD", // pre-v6 legacy alias, still common in older boards
"KICAD6_3DMODEL_DIR",
"KICAD7_3DMODEL_DIR",
"KICAD8_3DMODEL_DIR",
"KICAD9_3DMODEL_DIR",
"KICAD10_3DMODEL_DIR",
] as const;
export function memfsProjectDir(slug: string): string { export function memfsProjectDir(slug: string): string {
return `${MEMFS_PROJECTS_DIR}/${slug}`; return `${MEMFS_PROJECTS_DIR}/${slug}`;
} }

11
web/standalone/src/wasm/kicad-runner.ts
View file

@ -1,6 +1,7 @@
import type { Tool } from "@pcbjam/shared"; import type { Tool } from "@pcbjam/shared";
import { FILELESS_TOOLS } from "@pcbjam/shared"; import { FILELESS_TOOLS } from "@pcbjam/shared";
import { memfsFilePath, memfsProjectDir } from "./constants"; import { memfsFilePath, memfsProjectDir } from "./constants";
import { prescanBoardModels } from "./libs/models-bridge";
import { openFileInTool } from "./open-flow"; import { openFileInTool } from "./open-flow";
/** /**
@ -55,6 +56,16 @@ async function syncProjectToMemfs(win: ToolWindow, opts: DriveOptions): Promise<
const bytes = await opts.fetchBytes(file.path); const bytes = await opts.fetchBytes(file.path);
fs.writeFile(dest, bytes); fs.writeFile(dest, bytes);
opts.log(`[memfs] wrote ${dest} (${bytes.length} bytes)`); opts.log(`[memfs] wrote ${dest} (${bytes.length} bytes)`);
// 3D models: prefetch every model this board references (R2 → IDB → MEMFS)
// so the 3D viewer's first open resolves locally. Fire-and-forget — project
// open never waits on it; a ref that misses falls back to the C++ per-model
// ensure. No-op unless a model source is installed (bootKicadTool).
if (file.path.endsWith(".kicad_pcb")) {
const text = new TextDecoder().decode(bytes);
void prescanBoardModels(text).catch((e) =>
opts.log(`[3d] prescan failed: ${String(e)}`),
);
}
} }
} }

62
web/standalone/src/wasm/libs/models-bridge.test.ts Normal file
View file

@ -0,0 +1,62 @@
import { describe, expect, it } from "vitest";
import { normalizeModelRef, scanModelRefs } from "./models-bridge";
describe("normalizeModelRef", () => {
it("strips any vintage of the model-dir var", () => {
for (const v of ["KICAD6", "KICAD7", "KICAD8", "KICAD9", "KICAD10"]) {
expect(
normalizeModelRef(`\${${v}_3DMODEL_DIR}/Resistor_SMD.3dshapes/R_0201.wrl`),
).toBe("Resistor_SMD.3dshapes/R_0201.wrl");
}
});
it("accepts the paren syntax and the legacy KISYS3DMOD alias", () => {
expect(normalizeModelRef("$(KICAD8_3DMODEL_DIR)/L.3dshapes/m.step")).toBe(
"L.3dshapes/m.step",
);
expect(normalizeModelRef("${KISYS3DMOD}/L.3dshapes/m.wrl")).toBe(
"L.3dshapes/m.wrl",
);
});
it("passes bare relative refs through", () => {
expect(normalizeModelRef("L.3dshapes/m.wrl")).toBe("L.3dshapes/m.wrl");
});
it("rejects refs it cannot serve", () => {
expect(normalizeModelRef("${KIPRJMOD}/libs/3d/m.wrl")).toBeNull(); // project-local
expect(normalizeModelRef("/abs/path/m.wrl")).toBeNull();
expect(normalizeModelRef("kicad_embed://m.wrl")).toBeNull();
expect(normalizeModelRef("")).toBeNull();
expect(normalizeModelRef("${UNCLOSED/m.wrl")).toBeNull();
});
});
describe("scanModelRefs", () => {
it("finds, normalizes and dedupes board model refs", () => {
const board = `
(footprint "Resistor_THT:R_Axial"
(model "\${KICAD10_3DMODEL_DIR}/Resistor_THT.3dshapes/R_Axial.step"
(offset (xyz 0 0 0))))
(footprint "Resistor_THT:R_Axial"
(model "\${KICAD10_3DMODEL_DIR}/Resistor_THT.3dshapes/R_Axial.step"))
(footprint "X:Y"
(model "\${KIPRJMOD}/libs/3d_shapes/custom.wrl"))
(footprint "L:M" (model "\${KICAD8_3DMODEL_DIR}/LED_THT.3dshapes/LED_D5.0mm.wrl"))
`;
expect(scanModelRefs(board).sort()).toEqual([
"LED_THT.3dshapes/LED_D5.0mm.wrl",
"Resistor_THT.3dshapes/R_Axial.step",
]);
});
it("handles escaped quotes inside the path", () => {
expect(
scanModelRefs('(model "${KICAD9_3DMODEL_DIR}/A.3dshapes/we\\"ird.wrl")'),
).toEqual(['A.3dshapes/we"ird.wrl']);
});
it("returns empty for a board with no models", () => {
expect(scanModelRefs("(kicad_pcb (version 20240101))")).toEqual([]);
});
});

181
web/standalone/src/wasm/libs/models-bridge.ts Normal file
View file

@ -0,0 +1,181 @@
import { MODELS_3D_ROOT } from "../constants";
import type { Model3dSource } from "./models-source";
/**
* Glue between 3D model delivery and the running KiCad WASM tool:
*
* - `installModel3dHandler` backs the provider's `kind === "model3d"` requests
* (the C++ lazy fallback in S3D_CACHE::load → PCBJAM_3D::EnsureModelFile asks
* "ensure" for a ref the prescan missed).
* - `prescanBoardModels` scans a board's `(model "…")` refs up front and
* prefetches those bodies (R2 → IDB → MEMFS) so the 3D viewer's first open
* resolves everything locally.
*
* Both paths converge on `ensureModelInMemfs`: fetch the body via the
* `Model3dSource` (IDB-cached, sparse) and write it under `MODELS_3D_ROOT` —
* where boot points every `KICAD*_3DMODEL_DIR` env var, so KiCad's stock
* resolver finds the file with no C++ resolution changes.
*/
/** Progress of a board-model prefetch burst (drives the 3D loading overlay). */
export const MODELS_LOADING_EVENT = "pcbjam:models-loading";
export interface ModelsLoadingDetail {
loading: boolean;
done: number;
total: number;
}
function emitModelsLoading(detail: ModelsLoadingDetail): void {
if (typeof window === "undefined") return;
window.dispatchEvent(new CustomEvent(MODELS_LOADING_EVENT, { detail }));
}
/**
* Normalize a footprint model reference to the source's relative form —
* "${KICAD*_3DMODEL_DIR}/<lib>.3dshapes/<name>.<ext>" (any vintage, `${}` or
* `$()`) → "<lib>.3dshapes/<name>.<ext>". Bare relative refs pass through;
* absolute paths / ${KIPRJMOD} / kicad_embed:// are not ours → null. Mirrors
* pcbjamNormalizeModelRef in kicad/3d-viewer/3d_cache/pcbjam_model_fetch.cpp.
*/
export function normalizeModelRef(raw: string): string | null {
const ref = raw.trim();
if (!ref) return null;
if (ref.startsWith("${") || ref.startsWith("$(")) {
const closing = ref[1] === "{" ? "}" : ")";
const end = ref.indexOf(closing);
if (end < 0) return null;
const v = ref.slice(2, end);
// Any vintage of the model-dir var, plus the pre-v6 legacy alias.
if (!v.includes("3DMODEL_DIR") && v !== "KISYS3DMOD") return null;
return ref.slice(end + 1).replace(/^[/\\]+/, "") || null;
}
if (ref.startsWith("/") || ref.includes("://")) return null;
return ref;
}
/** Every `(model "…")` ref in a KiCad board/footprint s-expr, normalized. */
export function scanModelRefs(sexprText: string): string[] {
const refs = new Set<string>();
const re = /\(\s*model\s+"((?:[^"\\]|\\.)*)"/g;
for (let m = re.exec(sexprText); m; m = re.exec(sexprText)) {
const raw = m[1]!.replace(/\\(.)/g, "$1");
const rel = normalizeModelRef(raw);
if (rel) refs.add(rel);
}
return [...refs];
}
type ModelFS = Pick<EmscriptenFS, "mkdirTree" | "writeFile" | "analyzePath">;
function toolFS(): ModelFS | null {
const fs = (window as ToolWindow).FS;
return fs && typeof fs.writeFile === "function" ? (fs as ModelFS) : null;
}
let installedSource: Model3dSource | null = null;
let installedLog: (msg: string) => void = () => {};
/** Refs already materialized in MEMFS this session (bodies are immutable). */
const written = new Set<string>();
/** In-flight ensures, coalesced per ref (prescan and the C++ fallback race). */
const ensuring = new Map<string, Promise<string | null>>();
/** Wire the model source used by the provider dispatch + prescan. */
export function installModel3dHandler(
source: Model3dSource,
log: (msg: string) => void,
): void {
installedSource = source;
installedLog = log;
}
/** Fetch one model body and write it under MODELS_3D_ROOT. Resolves to the
* absolute MEMFS path when present, null when the source can't serve it. */
export async function ensureModelInMemfs(ref: string): Promise<string | null> {
const dest = `${MODELS_3D_ROOT}/${ref}`;
if (written.has(ref)) return dest;
let p = ensuring.get(ref);
if (!p) {
p = doEnsure(ref, dest).finally(() => ensuring.delete(ref));
ensuring.set(ref, p);
}
return p;
}
async function doEnsure(ref: string, dest: string): Promise<string | null> {
const source = installedSource;
const fs = toolFS();
if (!source || !fs) return null;
if (fs.analyzePath(dest).exists) {
written.add(ref);
return dest;
}
const body = await source.getModelBody(ref);
if (!body) return null;
fs.mkdirTree(dest.slice(0, dest.lastIndexOf("/")));
fs.writeFile(dest, body);
written.add(ref);
installedLog(`[3d] materialized ${ref} (${body.length} bytes)`);
return dest;
}
/**
* Provider dispatch for `kind === "model3d"` (called by installLibsProvider's
* request before any lib-id parsing — the C++ bridge passes an empty lib; the
* ref itself carries the library). Answers with the ABSOLUTE MEMFS path of the
* materialized file — S3D_CACHE loads that path directly, so model delivery
* never depends on env-var expansion inside the wasm runtime.
*/
export async function handleModel3dRequest(
op: string,
arg: string,
): Promise<string | null> {
if (op !== "ensure") return null;
const rel = normalizeModelRef(arg);
if (!rel) return null;
try {
return await ensureModelInMemfs(rel);
} catch (e) {
installedLog(`[3d] ensure failed for ${arg}: ${String(e)}`);
return null;
}
}
/**
* Prefetch every model a board references (fire-and-forget from the project
* sync). Bodies land in IDB + MEMFS before the user opens the 3D viewer in the
* common case; anything still missing falls back to the per-model C++ ensure.
*/
export async function prescanBoardModels(
boardText: string,
concurrency = 6,
): Promise<void> {
if (!installedSource) return;
const refs = scanModelRefs(boardText);
if (!refs.length) return;
const total = refs.length;
let done = 0;
emitModelsLoading({ loading: true, done, total });
installedLog(`[3d] prescan: ${total} model ref(s) on board`);
const started = performance.now();
let idx = 0;
const worker = async (): Promise<void> => {
while (idx < refs.length) {
const ref = refs[idx++]!;
try {
await ensureModelInMemfs(ref);
} catch {
// best-effort: the C++ lazy path (or a later prescan) retries
}
emitModelsLoading({ loading: ++done < total, done, total });
}
};
await Promise.all(
Array.from({ length: Math.min(concurrency, total) }, () => worker()),
);
installedLog(
`[3d] prescan: ${done}/${total} in ${Math.round(performance.now() - started)}ms`,
);
}

120
web/standalone/src/wasm/libs/models-source.test.ts Normal file
View file

@ -0,0 +1,120 @@
import { describe, expect, it } from "vitest";
import { sha256Hex, type SyncManifest } from "@pcbjam/shared";
import { memStore, type LayerStore } from "@pcbjam/sync-client";
import { cdnModelsSource } from "./models-source";
const MANIFEST_URL = "https://cdn.test/libs/kicad-models/9.0.9/manifest.json";
const BASE = "https://cdn.test/libs/kicad-models/9.0.9";
const BLOBS = "https://cdn.test/libs/kicad-models/blobs/sha256";
const enc = new TextEncoder();
const dec = new TextDecoder();
/** Build the CDN layout the way publish-models.ts does: per-lib sparse manifest
* + content-addressed blobs — pinning the format with the REAL hash codec. */
async function fakeModelsCdn() {
const bodies: Record<string, Uint8Array> = {
"model3d/R_Axial.step": enc.encode("STEP R_Axial"),
"model3d/R_Disc.wrl": enc.encode("#VRML V2.0 utf8 R_Disc"),
};
const entries: SyncManifest["entries"] = {};
const blobs = new Map<string, Uint8Array>();
for (const [path, body] of Object.entries(bodies)) {
const hash = await sha256Hex(body);
entries[path] = { hash, size: body.length, mtime: 0 };
blobs.set(hash, body);
}
const libManifest: SyncManifest = { version: 1, entries };
const top = {
schema: 1,
tag: "9.0.9",
libs: [{ id: "Resistor_THT", itemCount: 2 }],
};
let manifestFetches = 0;
let blobFetches = 0;
const json = (obj: unknown) => ({ ok: true, json: async () => obj });
const bin = (bytes: Uint8Array) => ({
ok: true,
arrayBuffer: async () => bytes.buffer,
});
const fetchImpl = (async (url: string) => {
if (url === MANIFEST_URL) return json(top);
if (url === `${BASE}/Resistor_THT/manifest`) {
manifestFetches += 1;
return json(libManifest);
}
if (url.startsWith(`${BLOBS}/`)) {
const blob = blobs.get(url.slice(`${BLOBS}/`.length));
if (blob) {
blobFetches += 1;
return bin(blob);
}
}
return { ok: false, status: 404 };
}) as unknown as typeof fetch;
return {
fetchImpl,
counters: {
get manifestFetches() {
return manifestFetches;
},
get blobFetches() {
return blobFetches;
},
},
};
}
function storeMap() {
const stores = new Map<string, LayerStore>();
return (ns: string): LayerStore => {
let s = stores.get(ns);
if (!s) stores.set(ns, (s = memStore()));
return s;
};
}
describe("cdnModelsSource", () => {
it("fetches exactly the requested body (sparse), then serves from cache", async () => {
const cdn = await fakeModelsCdn();
const src = cdnModelsSource(MANIFEST_URL, {
fetchImpl: cdn.fetchImpl,
storeFactory: storeMap(),
});
const body = await src.getModelBody("Resistor_THT.3dshapes/R_Axial.step");
expect(dec.decode(body!)).toBe("STEP R_Axial");
expect(cdn.counters.blobFetches).toBe(1); // only the asked-for model
await src.getModelBody("Resistor_THT.3dshapes/R_Axial.step");
expect(cdn.counters.blobFetches).toBe(1); // cached
expect(cdn.counters.manifestFetches).toBe(1); // lib opened once
});
it("returns null for unknown models/libs/refs without throwing", async () => {
const cdn = await fakeModelsCdn();
const src = cdnModelsSource(MANIFEST_URL, {
fetchImpl: cdn.fetchImpl,
storeFactory: storeMap(),
});
expect(await src.getModelBody("Resistor_THT.3dshapes/nope.step")).toBeNull();
expect(await src.getModelBody("NoSuchLib.3dshapes/m.wrl")).toBeNull();
expect(await src.getModelBody("not-a-model-ref")).toBeNull();
expect(cdn.counters.blobFetches).toBe(0);
});
it("hasModel answers from the manifest without fetching bodies", async () => {
const cdn = await fakeModelsCdn();
const src = cdnModelsSource(MANIFEST_URL, {
fetchImpl: cdn.fetchImpl,
storeFactory: storeMap(),
});
expect(await src.hasModel("Resistor_THT.3dshapes/R_Disc.wrl")).toBe(true);
expect(await src.hasModel("Resistor_THT.3dshapes/nope.wrl")).toBe(false);
expect(cdn.counters.blobFetches).toBe(0);
});
});

124
web/standalone/src/wasm/libs/models-source.ts Normal file
View file

@ -0,0 +1,124 @@
import { SyncStack, type SyncStackOptions } from "@pcbjam/sync-client";
/**
* Read-only source of 3D model bodies (`.wrl` / `.step`), addressed by the
* KiCad-relative ref `"<lib>.3dshapes/<name>.<ext>"` (a footprint's `(model …)`
* path with the `${KICAD*_3DMODEL_DIR}/` prefix stripped).
*
* Unlike symbols/footprints, models are never bulk-synced: each lib is an
* r2-idb-sync **sparse** layer — only the (small) manifest syncs eagerly, and a
* body is fetched exactly when a board references it, then cached in IDB. This
* keeps the client cost proportional to what the user renders, not to the
* ~GB-scale full set (docs/features/3d-models).
*/
export interface Model3dSource {
/** One model body, IDB-cached; null when unknown/unavailable. */
getModelBody(ref: string): Promise<Uint8Array | null>;
/** Whether a ref exists in the set at all (manifest-only, no body fetch). */
hasModel(ref: string): Promise<boolean>;
}
interface CdnModelsManifest {
schema: number;
tag: string;
libs: Array<{ id: string; itemCount?: number }>;
}
/** Split "<lib>.3dshapes/<name>" → { lib, path: "model3d/<name>" }. */
function splitRef(ref: string): { lib: string; path: string } | null {
const i = ref.indexOf(".3dshapes/");
if (i <= 0) return null;
const lib = ref.slice(0, i);
const name = ref.slice(i + ".3dshapes/".length);
if (!name || name.includes("/")) return null;
return { lib, path: `model3d/${name}` };
}
/**
* CDN-backed `Model3dSource`. Layout under the manifest's dir
* (`<cdn>/libs/kicad-models/<tag>/`, see scripts/deploy/publish-models.ts):
* manifest.json top index { schema, tag, libs:[{id,itemCount}] }
* <lib>/manifest per-lib SyncManifest, entries "model3d/<name>.<ext>"
* Bodies are content-addressed and shared across tags, one level up:
* <cdn>/libs/kicad-models/blobs/sha256/<hash>
*/
export function cdnModelsSource(
manifestUrl: string,
opts?: Pick<SyncStackOptions, "fetchImpl" | "storeFactory">,
): Model3dSource {
const baseDir = manifestUrl.replace(/\/[^/]*$/, ""); // …/libs/kicad-models/<tag>
const blobsBase = `${baseDir.replace(/\/[^/]*$/, "")}/blobs/sha256`;
const fetchImpl = opts?.fetchImpl ?? fetch;
let manifestP: Promise<CdnModelsManifest> | null = null;
const loadManifest = (): Promise<CdnModelsManifest> => {
if (!manifestP) {
// Never cache a rejection (mirrors cdn-source.ts): a transient failure
// must not poison every later model read.
manifestP = (async () => {
const r = await fetchImpl(manifestUrl, { cache: "no-store" });
if (!r.ok) throw new Error(`cdn models manifest ${r.status}: ${manifestUrl}`);
return (await r.json()) as CdnModelsManifest;
})().catch((e) => {
manifestP = null;
throw e;
});
}
return manifestP;
};
// One lazily-opened sparse stack per lib (IDB store keyed by namespace, so a
// lib's cached models persist and dedupe across sessions).
const stacks = new Map<string, Promise<SyncStack | null>>();
const openStack = (libId: string): Promise<SyncStack | null> => {
let p = stacks.get(libId);
if (!p) {
p = (async () => {
const m = await loadManifest();
if (!m.libs.some((l) => l.id === libId)) return null; // unknown lib
const stack = new SyncStack({
layers: [
{
namespace: `kicad-models:${m.tag}:${libId}`,
kind: "sparse",
url: `${baseDir}/${encodeURIComponent(libId)}`,
bodyUrlTemplate: `${blobsBase}/{hash}`,
},
],
...opts,
});
await stack.open();
return stack;
})().catch((e) => {
stacks.delete(libId); // a failed open must stay retryable
throw e;
});
stacks.set(libId, p);
}
return p;
};
return {
async getModelBody(ref: string): Promise<Uint8Array | null> {
const split = splitRef(ref);
if (!split) return null;
try {
const stack = await openStack(split.lib);
return stack ? await stack.read(split.path) : null;
} catch {
return null; // missing models render as absent, never break the viewer
}
},
async hasModel(ref: string): Promise<boolean> {
const split = splitRef(ref);
if (!split) return false;
try {
const stack = await openStack(split.lib);
if (!stack) return false;
return (await stack.list()).some((e) => e.path === split.path);
} catch {
return false;
}
},
};
}

7
web/standalone/src/wasm/libs/source.ts
View file

@ -1,3 +1,4 @@
import { handleModel3dRequest } from "./models-bridge";
import { libIdFromUri, libUri } from "./uri"; import { libIdFromUri, libUri } from "./uri";
/** /**
@ -259,6 +260,12 @@ export function installLibsProvider(
let fatResetTimer: ReturnType<typeof setTimeout> | undefined; let fatResetTimer: ReturnType<typeof setTimeout> | undefined;
const request: KicadLibsRequest = async (op, lib, arg, kind = "symbol") => { const request: KicadLibsRequest = async (op, lib, arg, kind = "symbol") => {
// 3D models are addressed by ref, not lib-table URI (the C++ ensure bridge
// passes an empty lib) — dispatch before the lib-id parse would null it out.
if (kind === "model3d") {
log(`[libs] request op=${op} kind=model3d arg=${arg}`);
return handleModel3dRequest(op, arg);
}
const id = libIdFromUri(lib); const id = libIdFromUri(lib);
log(`[libs] request op=${op} kind=${kind} lib=${lib} (id=${id}) arg=${arg}`); log(`[libs] request op=${op} kind=${kind} lib=${lib} (id=${id}) arg=${arg}`);
if (!id) return null; if (!id) return null;