From 6f3f3bcf0084015b7d535b04d55cc52d2fb40f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20T=C3=B6rcsv=C3=A1ri?= Date: Thu, 2 Jul 2026 11:00:51 +0200 Subject: [PATCH] =?UTF-8?q?feat(3d):=20lazy=203D=20model=20delivery=20R2?= =?UTF-8?q?=E2=86=92IDB=E2=86=92WASM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cdnModelsSource: sparse per-lib stacks over the models CDN layout (libs/kicad-models///manifest + content-addressed blobs) - models-bridge: board prescan ((model …) scan → prefetch, 4/4 in 74ms on the demo board) + the kind=model3d 'ensure' op answering absolute MEMFS paths; only board-referenced bodies ever enter IDB - boot/runner/config/WasmTool wiring (VITE_MODELS_MANIFEST_URL, KICAD*_3DMODEL_DIR seeding, prefetch badge) - StorageUsageCard on HomePage: per-kind cached sizes + delete-3D-cache - publish-models.ts (brotli content-addressed publish, 606→93MB for the pic_programmer lib set) + dev-demo --models-tag/--models-local - e2e: 3d-viewer-models.spec.ts (bridge normalize/dedup + STEP/WRL render); submodule bumps: kicad (static 3D plugins + ensure hook), pcbjam-shared (sparse sync layer) Spec + findings: docs/features/3d-models (private repo). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014AT7gVHRktDYoQ68S4x6A4 --- kicad | 2 +- scripts/deploy/dev-demo.mjs | 28 ++ scripts/deploy/publish-models.ts | 219 +++++++++++++++ tests/kicad/3d-viewer-models.spec.ts | 255 ++++++++++++++++++ tests/playwright-kicad.config.ts | 1 + web/.gitignore | 3 + web/pcbjam-shared | 2 +- .../src/components/StorageUsageCard.tsx | 235 ++++++++++++++++ web/standalone/src/components/WasmTool.tsx | 30 +++ web/standalone/src/lib/config.ts | 17 ++ web/standalone/src/pages/HomePage.tsx | 6 + web/standalone/src/wasm/boot.ts | 34 ++- web/standalone/src/wasm/constants.ts | 18 ++ web/standalone/src/wasm/kicad-runner.ts | 11 + .../src/wasm/libs/models-bridge.test.ts | 62 +++++ web/standalone/src/wasm/libs/models-bridge.ts | 181 +++++++++++++ .../src/wasm/libs/models-source.test.ts | 120 +++++++++ web/standalone/src/wasm/libs/models-source.ts | 124 +++++++++ web/standalone/src/wasm/libs/source.ts | 7 + 19 files changed, 1351 insertions(+), 4 deletions(-) create mode 100644 scripts/deploy/publish-models.ts create mode 100644 tests/kicad/3d-viewer-models.spec.ts create mode 100644 web/standalone/src/components/StorageUsageCard.tsx create mode 100644 web/standalone/src/wasm/libs/models-bridge.test.ts create mode 100644 web/standalone/src/wasm/libs/models-bridge.ts create mode 100644 web/standalone/src/wasm/libs/models-source.test.ts create mode 100644 web/standalone/src/wasm/libs/models-source.ts diff --git a/kicad b/kicad index e8db3d3..9d77139 160000 --- a/kicad +++ b/kicad @@ -1 +1 @@ -Subproject commit e8db3d35db635946b03ee0ae8698e1d3aad79a3c +Subproject commit 9d771391886ba3877dda90f72f57ad37d931fef9 diff --git a/scripts/deploy/dev-demo.mjs b/scripts/deploy/dev-demo.mjs index 22f9ad5..26f334b 100644 --- a/scripts/deploy/dev-demo.mjs +++ b/scripts/deploy/dev-demo.mjs @@ -72,6 +72,8 @@ function parseArgs(argv) { contentTag: null, // use the live CDN gallery for this tag (else build locally) galleryTag: "demo-local", // path tag for the locally-built gallery noGallery: false, // disable the example gallery (local-folder + IDB only) + modelsTag: null, // 3D models snapshot tag (live CDN, or the local dir's tag) + modelsLocal: null, // local publish-models --driver local output dir (serve same-origin) port: null, repo: "https://github.com/emergence-engineering/pcbjam", }; @@ -87,6 +89,8 @@ function parseArgs(argv) { case "--no-gallery": a.noGallery = true; break; case "--port": a.port = next(); break; case "--repo": a.repo = next(); break; + case "--models-tag": a.modelsTag = next(); break; + case "--models-local": a.modelsLocal = next(); break; case "-h": case "--help": a.help = true; break; default: throw new Error(`unknown arg: ${argv[i]}`); } @@ -108,6 +112,9 @@ const HELP = `dev-demo.mjs — run the standalone locally in demo mode (R2-only --content-tag pin the LIVE CDN gallery for this release tag (default: build+serve the gallery locally) --gallery-tag path tag for the locally-built gallery (default demo-local) --no-gallery disable the example gallery (local-folder + IDB projects only) + --models-tag enable lazy 3D models from the CDN snapshot at this tag + --models-local serve a local publish-models layout (--driver local --compress none) + same-origin instead of the CDN (requires --models-tag) --port dev server port By default the read-only example gallery (deploy/demo/gallery.json) is built @@ -142,6 +149,26 @@ function main() { delete env.VITE_LIBS_MANIFEST_URL; } + // --- 3D models: lazy per-board bodies (docs/features/3d-models). Off unless a + // tag is given. --models-local serves that + // layout same-origin at /models-cdn via a public/ symlink (publish it with + // --compress none — the dev server can't send Content-Encoding: br); + // otherwise the live CDN snapshot for --models-tag is used. + if (a.modelsTag && a.modelsLocal) { + const link = join(repoRoot, "web/standalone/public/models-cdn"); + try { + if (lstatSync(link)) rmSync(link, { recursive: true, force: true }); + } catch { + /* no existing link */ + } + symlinkSync(resolve(a.modelsLocal, "libs/kicad-models"), link); + env.VITE_MODELS_MANIFEST_URL = `/models-cdn/${a.modelsTag}/manifest.json`; + } else if (a.modelsTag) { + env.VITE_MODELS_MANIFEST_URL = `${a.cdn}/libs/kicad-models/${a.modelsTag}/manifest.json`; + } else { + delete env.VITE_MODELS_MANIFEST_URL; + } + // --- No backend: collab is cross-tab only, document bytes are local (api path), // loaded folders persist to a browser-local IndexedDB project. env.VITE_YJS_PROVIDER = "broadcastchannel"; @@ -189,6 +216,7 @@ function main() { console.log(` VITE_LIBS_SOURCE=${env.VITE_LIBS_SOURCE}${env.VITE_LIBS_MANIFEST_URL ? ` (${env.VITE_LIBS_MANIFEST_URL})` : ""}`); console.log(` VITE_WASM_ROOT=${env.VITE_WASM_ROOT}${env.VITE_WASM_MANIFEST ? ` (${env.VITE_WASM_MANIFEST})` : " (local build)"}`); console.log(` VITE_PROJECT_SOURCE=${env.VITE_PROJECT_SOURCE}${env.VITE_PROJECT_MANIFEST_URL ? ` (${env.VITE_PROJECT_MANIFEST_URL})` : ""}`); + console.log(` VITE_MODELS_MANIFEST_URL=${env.VITE_MODELS_MANIFEST_URL ?? "(unset — 3D models off)"}`); console.log(` VITE_YJS_PROVIDER=${env.VITE_YJS_PROVIDER} VITE_DOC_SOURCE=${env.VITE_DOC_SOURCE} VITE_LOCAL_PROJECTS=${env.VITE_LOCAL_PROJECTS}`); const child = spawn("pnpm", viteArgs, { cwd: repoRoot, env, stdio: "inherit" }); diff --git a/scripts/deploy/publish-models.ts b/scripts/deploy/publish-models.ts new file mode 100644 index 0000000..0b33a41 --- /dev/null +++ b/scripts/deploy/publish-models.ts @@ -0,0 +1,219 @@ +// Publish KiCad 3D models (kicad-packages3D) to the CDN as r2-idb-sync SPARSE +// origins: per-lib manifests keyed by the upstream tag, bodies content-addressed +// under a shared blobs/ prefix (deduped across tags — models rarely change). +// The standalone's cdnModelsSource opens each lib as a sparse layer: manifest +// synced eagerly (small), bodies fetched exactly when a board references them. +// See web/standalone/src/wasm/libs/models-source.ts + docs/features/3d-models. +// +// npx tsx scripts/deploy/publish-models.ts --model-tag 10.0.0 \ +// --models-src --driver local --out /tmp/cdn-models +// npx tsx scripts/deploy/publish-models.ts --model-tag 10.0.0 --models-src … \ +// --driver r2 --bucket pcbjam-cdn --remote +// # dev subset: only a few libs +// … --libs Resistor_SMD,Capacitor_SMD,Package_QFP +// +// Layout under `` (default libs/kicad-models): +// /manifest.json top index { schema, tag, libs:[{id,itemCount,bytes}] } +// //manifest per-lib SyncManifest { "model3d/": {hash,size,mtime} } +// blobs/sha256/ model bodies (brotli, content-addressed, shared) +// blobs/registry.json published-blob index (hash → size) for cheap dedup +// +// Idempotent per tag: if //manifest.json exists the run SKIPS +// (--force overrides). Blobs are skipped per-hash via the registry. + +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { dirname, join } from "node:path"; +import type { SyncManifest } from "../../web/pcbjam-shared/src/sync-wire.js"; +import { + compressBytes, + IMMUTABLE, + makeStore, + NO_STORE, + putJSON, + sha256hex, +} from "./lib/cdn-store.mjs"; + +const MODELS_URL = "https://gitlab.com/kicad/libraries/kicad-packages3D.git"; + +/** Model file extensions we publish (locked: WRL + STEP). */ +const MODEL_EXTS = [".wrl", ".step", ".stp"]; + +interface Args { + modelTag: string | null; + modelsSrc: string | null; + clone: string | null; + driver: string; + out: string | null; + bucket: string; + remote: boolean; + prefix: string; + force: boolean; + libs: string[] | null; + quality: number; + compress: "br" | "none"; +} + +function parseArgs(argv: string[]): Args { + const a: Args = { + modelTag: null, + modelsSrc: null, + clone: null, + driver: "local", + out: null, + bucket: "pcbjam-cdn", + remote: false, + prefix: "libs/kicad-models", + force: false, + libs: null, + quality: 5, + compress: "br", + }; + for (let i = 2; i < argv.length; i++) { + const next = () => argv[++i]!; + switch (argv[i]) { + case "--model-tag": a.modelTag = next(); break; + case "--models-src": a.modelsSrc = next(); break; + // Clone kicad-packages3D at --model-tag into /kicad-packages3D + // (shallow; NOTE: multi-GB working tree) and use it as the source. + case "--clone": a.clone = next(); break; + case "--driver": a.driver = next(); break; + case "--out": a.out = next(); break; + case "--bucket": a.bucket = next(); break; + case "--remote": a.remote = true; break; + case "--prefix": a.prefix = next(); break; + case "--force": a.force = true; break; + // Dev subset: publish only these libs (names without .3dshapes). + case "--libs": a.libs = next().split(",").map((s) => s.trim()).filter(Boolean); break; + // Brotli quality for bodies (WRL/STEP are text-ish; 5 ≈ 4-5x, fast). + case "--quality": a.quality = Number(next()); break; + // "none" for --driver local when a plain static server (e.g. the vite dev + // server) will serve the blobs — it can't send Content-Encoding: br. + case "--compress": a.compress = next() as "br" | "none"; break; + default: throw new Error(`unknown arg: ${argv[i]}`); + } + } + if (!a.modelTag) throw new Error("--model-tag is required"); + if (!a.modelsSrc && !a.clone) + throw new Error("need --models-src or --clone "); + if (a.driver === "local" && !a.out) a.out = ".cdn-models-out"; + return a; +} + +function cloneShallow(url: string, dest: string, ref: string): void { + if (existsSync(dest)) { + console.log(`clone: ${dest} present — reusing`); + return; + } + mkdirSync(dirname(dest), { recursive: true }); + console.log(`clone: ${url} @ ${ref} → ${dest} (multi-GB — this takes a while)`); + execFileSync("git", ["clone", "--depth", "1", "--branch", ref, url, dest], { + stdio: "inherit", + }); +} + +/** All `.3dshapes` dirs under the checkout root (non-recursive: the repo is flat). */ +function listModelLibs(src: string): Array<{ id: string; dir: string }> { + return readdirSync(src, { withFileTypes: true }) + .filter((d) => d.isDirectory() && d.name.endsWith(".3dshapes")) + .map((d) => ({ id: d.name.slice(0, -".3dshapes".length), dir: join(src, d.name) })) + .sort((x, y) => x.id.localeCompare(y.id)); +} + +async function main(): Promise { + const a = parseArgs(process.argv); + const store = makeStore(a.driver, a); + const topKey = `${a.prefix}/${a.modelTag}/manifest.json`; + const registryKey = `${a.prefix}/blobs/registry.json`; + + if (!a.force && store.getJSON(topKey)) { + console.log(`publish-models: ${topKey} already published — skipping (use --force)`); + return; + } + + if (a.clone) { + const dest = join(a.clone, "kicad-packages3D"); + cloneShallow(MODELS_URL, dest, a.modelTag!); + a.modelsSrc ??= dest; + } + + console.log( + `publish-models: tag=${a.modelTag} driver=${store.kind} → ${a.prefix}/${a.modelTag}/`, + ); + + // Published-blob index: hash → original size. One GET up front, one PUT at the + // end — the per-blob "does it exist" probe would otherwise be an R2 round-trip + // per model (tens of thousands). + const registry: Record = + (store.getJSON(registryKey) as Record | null) ?? {}; + let blobsPut = 0; + let blobsSkipped = 0; + + let libs = listModelLibs(a.modelsSrc!); + if (a.libs) { + const want = new Set(a.libs); + libs = libs.filter((l) => want.has(l.id)); + const missing = a.libs.filter((id) => !libs.some((l) => l.id === id)); + if (missing.length) console.warn(`publish-models: libs not found: ${missing.join(", ")}`); + } + if (!libs.length) throw new Error(`no .3dshapes libs under ${a.modelsSrc}`); + + const topLibs: Array<{ id: string; itemCount: number; bytes: number }> = []; + let totalItems = 0; + let totalBytes = 0; + + for (const lib of libs) { + const files = readdirSync(lib.dir) + .filter((f) => MODEL_EXTS.some((ext) => f.toLowerCase().endsWith(ext))) + .sort(); + if (!files.length) continue; + + const entries: SyncManifest["entries"] = {}; + let libBytes = 0; + for (const f of files) { + const p = join(lib.dir, f); + if (!statSync(p).isFile()) continue; + const body = readFileSync(p); + const hash = sha256hex(body); + entries[`model3d/${f}`] = { hash, size: body.length, mtime: 0 }; + libBytes += body.length; + + if (registry[hash] === undefined) { + // WRL and STEP are text formats — brotli gets ~4-5x. The browser fetch + // transparently decodes, so IDB caches (and hashes refer to) the + // ORIGINAL bytes; `no-transform` keeps the edge from re-encoding. + const { bytes, encoding } = compressBytes(body, a.compress, a.quality); + store.put(`${a.prefix}/blobs/sha256/${hash}`, bytes, { + contentType: "application/octet-stream", + contentEncoding: encoding, + cacheControl: IMMUTABLE, + }); + registry[hash] = body.length; + blobsPut++; + } else { + blobsSkipped++; + } + } + + const manifest: SyncManifest = { version: 1, entries }; + putJSON(store, `${a.prefix}/${a.modelTag}/${lib.id}/manifest`, manifest, IMMUTABLE); + topLibs.push({ id: lib.id, itemCount: files.length, bytes: libBytes }); + totalItems += files.length; + totalBytes += libBytes; + console.log(` ${lib.id}: ${files.length} models (${(libBytes / 1e6).toFixed(1)} MB)`); + } + + putJSON(store, registryKey, registry, NO_STORE); + putJSON(store, topKey, { schema: 1, tag: a.modelTag, libs: topLibs }, IMMUTABLE); + + console.log( + `publish-models: done — ${topLibs.length} libs, ${totalItems} models ` + + `(${(totalBytes / 1e6).toFixed(0)} MB raw), blobs put=${blobsPut} deduped=${blobsSkipped} → ${topKey}`, + ); + if (store.kind === "local") console.log(`local layout under: ${a.out}`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/tests/kicad/3d-viewer-models.spec.ts b/tests/kicad/3d-viewer-models.spec.ts new file mode 100644 index 0000000..3838cab --- /dev/null +++ b/tests/kicad/3d-viewer-models.spec.ts @@ -0,0 +1,255 @@ +import type { Page } from '@playwright/test'; +import { test, expect } from './fixtures'; +import { clickMenuBarItem, clickMenuItem } from '../e2e/utils/element-tracker'; +import { injectFromSubmodule } from './utils/fs-inject'; +import { waitForBoardLoaded } from './utils/board-ready'; +import { waitForPcbnew } from './utils/pcbnew-ready'; + +/** + * 3D viewer COMPONENT MODELS e2e (docs/features/3d-models): load pic_programmer, + * open the 3D viewer, and verify the model-delivery machinery end to end at the + * KiCad/wasm level: + * + * 1. Statically linked format plugins (vrml + oce — upstream loads them via + * dlopen, which wasm doesn't have) parse real model files. + * 2. Project-local models resolve exactly as upstream: the board references + * `${KIPRJMOD}/libs/3d_shapes/*.wrl`, injected with the project. + * 3. The lazy-fetch fallback (S3D_CACHE::load → PCBJAM_3D::EnsureModelFile → + * `kicadLibs.request("ensure", …, "model3d")`) asks JS for every + * `${KICAD*_3DMODEL_DIR}` ref, with the ref NORMALIZED to + * `.3dshapes/.` — and a served ref (the stub writes the + * bytes into MEMFS and answers "1") then resolves and renders. + * + * The stub provider stands in for the standalone's models-bridge (which fetches + * from the CDN into IDB); here it serves ONE in-repo STEP fixture under a + * board-referenced name — geometry is a USB-C connector where a DIP-8 socket + * belongs, which is irrelevant: the assertion is parse+render, not fidelity. + */ + +const KICAD_VERSION_DIR = '10.0'; +const PROJECT_DIR_MEMFS = `/home/kicad/documents/kicad/${KICAD_VERSION_DIR}/projects`; + +// The JS-owned MEMFS root the stub writes model bodies under — the same dir +// the standalone's models-bridge uses (constants.ts MODELS_3D_ROOT). Its exact +// location is immaterial: the ensure protocol answers with the ABSOLUTE path +// and S3D_CACHE loads it directly (env-var expansion never resolves +// ${KICAD*_3DMODEL_DIR} refs in the wasm runtime — see +// docs/features/3d-models/0001). +const MODELS_ROOT_MEMFS = '/pcbjam/3dmodels'; + +// The board ref the stub provider serves (normalized form the bridge must ask +// for), and the in-repo STEP whose bytes stand in for it. +const SERVED_REF = 'Package_DIP.3dshapes/DIP-8_W7.62mm.step'; +const STEP_FIXTURE = 'kicad/demos/openair-max/Libraries/HRO_TYPE-C-31-M-12.step'; + +const DEMO = { name: 'pic_programmer', dir: 'pic_programmer', stem: 'pic_programmer' } as const; + +declare global { + interface Window { + __modelEnsures?: Array<{ op: string; arg: string; kind: string }>; + __stepFixtureB64?: string; + } +} + +/** Record every model3d bridge request; serve SERVED_REF from the fixture. */ +async function installModelProviderStub(page: Page, serveAll = false): Promise { + await page.evaluate( + ({ stockDir, servedRef, serveAll }) => { + window.__modelEnsures = []; + (globalThis as any).kicadLibs = { + request: async (op: string, _lib: string, arg: string, kind: string) => { + if (kind !== 'model3d') return null; + window.__modelEnsures!.push({ op, arg, kind }); + console.log(`[TEST-3D] ensure request: ${op} ${arg}`); + if (op !== 'ensure' || (!serveAll && arg !== servedRef)) return null; + + const b64 = window.__stepFixtureB64!; + const binary = atob(b64); + const data = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) data[i] = binary.charCodeAt(i); + + // Mirror models-bridge.ts ensureModelInMemfs: write under the + // JS-owned model root and answer with the ABSOLUTE path — + // S3D_CACHE loads it directly (no env-var expansion needed). + // @ts-expect-error — Emscripten FS lives on window + const FS = (window as any).FS; + const dest = `${stockDir}/${arg}`; + FS.mkdirTree(dest.slice(0, dest.lastIndexOf('/'))); + FS.writeFile(dest, data); + console.log(`[TEST-3D] served ${arg} → ${dest} (${data.length} bytes)`); + return dest; + }, + }; + }, + { stockDir: MODELS_ROOT_MEMFS, servedRef: SERVED_REF, serveAll }, + ); +} + +async function loadBoard(page: Page, testLogger: { consoleLogs: string[]; errors: string[] }): Promise { + const pcbFilename = `${DEMO.stem}.kicad_pcb`; + const proFilename = `${DEMO.stem}.kicad_pro`; + + await injectFromSubmodule(page, `kicad/demos/${DEMO.dir}/${pcbFilename}`, + `${PROJECT_DIR_MEMFS}/${pcbFilename}`); + await injectFromSubmodule(page, `kicad/demos/${DEMO.dir}/${proFilename}`, + `${PROJECT_DIR_MEMFS}/${proFilename}`); + // Project-local 3D models — the board references them as + // ${KIPRJMOD}/libs/3d_shapes/.wrl; resolved by the stock resolver, so + // they must NOT go through the ensure bridge (asserted below). + await injectFromSubmodule(page, `kicad/demos/${DEMO.dir}/libs/3d_shapes/textool_40.wrl`, + `${PROJECT_DIR_MEMFS}/libs/3d_shapes/textool_40.wrl`); + await injectFromSubmodule(page, `kicad/demos/${DEMO.dir}/libs/3d_shapes/adjustable_rx2v4.wrl`, + `${PROJECT_DIR_MEMFS}/libs/3d_shapes/adjustable_rx2v4.wrl`); + + expect(await clickMenuBarItem(page, 'File'), 'File menu should be findable').toBe(true); + await page.waitForTimeout(400); + expect(await clickMenuItem(page, 'Open...'), 'Open… menu item should be findable').toBe(true); + + await page.waitForFunction(() => { + const registry = window.wxElementRegistry; + return !!registry && registry.findAll({ visible: true }) + .some((el) => el.typeName === 'wxFileDialog'); + }, null, { timeout: 15000 }); + await page.waitForTimeout(1000); + + const filenameInput = await page.evaluate(() => { + const registry = window.wxElementRegistry; + if (!registry) return null; + const text = registry.findAll({ visible: true }) + .find((el) => el.typeName === 'wxTextCtrl' && el.name === 'text'); + return text ? { x: text.centerX, y: text.centerY } : null; + }); + expect(filenameInput, 'filename text input should be visible').not.toBeNull(); + if (!filenameInput) throw new Error('filename text input not found'); + + await page.mouse.click(filenameInput.x, filenameInput.y); + await page.waitForTimeout(200); + await page.keyboard.type(pcbFilename); + await page.waitForTimeout(300); + await page.keyboard.press('Enter'); + await page.waitForTimeout(1000); + + const result = await waitForBoardLoaded(page, testLogger, 60000); + console.log(`[TEST] ${DEMO.name} board-ready result: ${result}`); +} + +function countGlCanvases(page: Page): Promise { + return page.evaluate(() => document.querySelectorAll('canvas[id^="glcanvas-"]').length); +} + +async function openThreeDViewer(page: Page, glBefore: number): Promise { + let opened = false; + if (await clickMenuBarItem(page, 'View')) { + await page.waitForTimeout(400); + opened = await clickMenuItem(page, '3D Viewer'); + } + if (!opened) { + console.log('[TEST] View → 3D Viewer not found via menu; trying Alt+3'); + await page.keyboard.press('Escape'); + await page.waitForTimeout(200); + await page.keyboard.press('Alt+3'); + } + + await page.waitForFunction(() => { + return !!document.querySelector('#window-container [id^="window-"]') + || document.querySelectorAll('canvas[id^="glcanvas-"]').length > 0; + }, null, { timeout: 60000 }); + + await page.waitForFunction((before: number) => + document.querySelectorAll('canvas[id^="glcanvas-"]').length > before, + glBefore, { timeout: 60000 }); + + const glAfter = await countGlCanvases(page); + console.log(`[TEST] glcanvas count after opening 3D viewer: ${glAfter}`); + expect(glAfter, 'a new WebGL canvas should appear for the 3D viewer').toBeGreaterThan(glBefore); + return glAfter; +} + +test.describe('3D viewer component models', () => { + test.describe.configure({ mode: 'serial' }); + test.setTimeout(240000); + + test('resolves project models, lazy-fetches lib models via the bridge, renders', async ({ page, testLogger }) => { + await page.goto('/kicad/pcbnew.html'); + await waitForPcbnew(page); + + // Stash the STEP fixture bytes + install the provider stub BEFORE the + // viewer can issue any ensure request. + const fs = require('fs') as typeof import('fs'); + const path = require('path') as typeof import('path'); + const fixtureAbs = path.resolve(__dirname, '..', '..', STEP_FIXTURE); + await page.evaluate( + (b64: string) => { window.__stepFixtureB64 = b64; }, + fs.readFileSync(fixtureAbs).toString('base64'), + ); + await installModelProviderStub(page); + + await loadBoard(page, testLogger); + + const glBefore = await countGlCanvases(page); + await openThreeDViewer(page, glBefore); + + // Scene build + progressive raytrace passes. + await page.waitForTimeout(8000); + await page.screenshot({ path: `test-results/3d-viewer-models-${DEMO.name}.png`, scale: 'css' }); + + // --- bridge assertions ------------------------------------------------- + const ensures = await page.evaluate(() => window.__modelEnsures ?? []); + console.log(`[TEST] ensure requests: ${ensures.length}`); + for (const e of ensures.slice(0, 30)) console.log(`[TEST] ${e.op} ${e.arg}`); + + // Every ${KICAD*_3DMODEL_DIR} ref crossed the bridge, normalized. + const args = ensures.map((e) => e.arg); + expect(args, 'the served lib ref must cross the bridge normalized') + .toContain(SERVED_REF); + expect(args.every((a) => /^[^/${]+\.3dshapes\//.test(a)), + 'every bridge ref is a normalized .3dshapes/ path').toBe(true); + // Project-local (${KIPRJMOD}) models resolve natively — never bridged. + expect(args.some((a) => a.includes('textool_40') || a.includes('adjustable_rx2v4')), + 'project-local models must not go through the ensure bridge').toBe(false); + // Board refs are unique per model file — the C++ memo must not re-ask. + expect(new Set(args).size, 'ensure requests are deduplicated').toBe(args.length); + + // The served model landed in MEMFS where the resolver looks. + const servedSize = await page.evaluate( + ({ stockDir, servedRef }) => { + // @ts-expect-error — Emscripten FS lives on window + const FS = (window as any).FS; + try { return FS.stat(`${stockDir}/${servedRef}`).size as number; } + catch { return -1; } + }, + { stockDir: MODELS_ROOT_MEMFS, servedRef: SERVED_REF }, + ); + expect(servedSize, 'served STEP written into the model root').toBeGreaterThan(1000); + + // --- render assertion -------------------------------------------------- + const render = await page.evaluate(() => { + const list = document.querySelectorAll('canvas[id^="glcanvas-"]'); + const el = list[list.length - 1] as HTMLCanvasElement; + const tmp = document.createElement('canvas'); + tmp.width = el.width; + tmp.height = el.height; + const ctx = tmp.getContext('2d')!; + ctx.drawImage(el, 0, 0); + const colors = new Set(); + for (let i = 0; i < 16; i++) { + for (let j = 0; j < 16; j++) { + const d = ctx.getImageData(Math.floor(el.width * i / 16), + Math.floor(el.height * j / 16), 1, 1).data; + colors.add(`${d[0]},${d[1]},${d[2]}`); + } + } + return { id: el.id, w: el.width, h: el.height, distinctColors: colors.size, + dataUrl: tmp.toDataURL('image/png') }; + }); + console.log(`[TEST] 3D canvas ${render.id} ${render.w}x${render.h}, distinct colours: ${render.distinctColors}`); + const b64 = render.dataUrl.replace(/^data:image\/png;base64,/, ''); + fs.writeFileSync(`test-results/3d-viewer-models-${DEMO.name}-render.png`, + Buffer.from(b64, 'base64')); + expect(render.distinctColors, + 'the 3D viewer canvas should render the board + models, not a blank fill') + .toBeGreaterThan(8); + + expect(testLogger.errors, 'no page errors during the model flow').toEqual([]); + }); +}); diff --git a/tests/playwright-kicad.config.ts b/tests/playwright-kicad.config.ts index 37f7b90..b550faa 100644 --- a/tests/playwright-kicad.config.ts +++ b/tests/playwright-kicad.config.ts @@ -89,6 +89,7 @@ const PCBNEW_FAMILY_SPECS = [ '**/pcbnew-move.spec.ts', // 3D viewer specs boot pcbnew.html (3D-enabled build) — same V8 routing. '**/3d-viewer.spec.ts', + '**/3d-viewer-models.spec.ts', '**/footprint-3d-preview.spec.ts', ]; diff --git a/web/.gitignore b/web/.gitignore index e51f913..2039a46 100644 --- a/web/.gitignore +++ b/web/.gitignore @@ -13,3 +13,6 @@ dist/ # .demo-cdn holds its generated bytes — both created by scripts/deploy/dev-demo.mjs. **/public/content **/standalone/.demo-cdn/ +# 3D models: public/models-cdn symlinks a local publish-models layout, created +# by scripts/deploy/dev-demo.mjs --models-local — never committed. +**/public/models-cdn diff --git a/web/pcbjam-shared b/web/pcbjam-shared index 9a1a269..067170e 160000 --- a/web/pcbjam-shared +++ b/web/pcbjam-shared @@ -1 +1 @@ -Subproject commit 9a1a269fdd9079921029ecf6e2dce89dece98fc8 +Subproject commit 067170e50f1c2ba19ffceb9815dcf5d34dc9e800 diff --git a/web/standalone/src/components/StorageUsageCard.tsx b/web/standalone/src/components/StorageUsageCard.tsx new file mode 100644 index 0000000..82651ab --- /dev/null +++ b/web/standalone/src/components/StorageUsageCard.tsx @@ -0,0 +1,235 @@ +import * as React from "react"; +import { Database, Loader2, RefreshCw, Trash2 } from "lucide-react"; +import { Button } from "@/components/ui/button"; + +/** + * Browser-storage usage for the editor's library caches, by kind, with a + * "delete 3D model cache" action. + * + * All library data lives in r2-idb-sync IndexedDB databases named + * `sync:` (one per lib), each with a `bodies` store keyed + * `"/"`. Symbols/footprints are bulk-synced (whole-lib bundles); + * 3D models are SPARSE — only the models boards actually rendered are stored + * (namespaces `kicad-models:…`), which makes them the one cache that is both + * potentially large and always safe to drop: manifests re-sync in one small + * fetch and bodies lazily re-download exactly when a board needs them again. + */ + +const SYNC_PREFIX = "sync:"; +const MODELS_NS_PREFIX = `${SYNC_PREFIX}kicad-models:`; + +interface KindUsage { + bytes: number; + items: number; +} + +interface StorageBreakdown { + symbol: KindUsage; + footprint: KindUsage; + model3d: KindUsage; + /** Whole-origin estimate (all IDB + caches), from navigator.storage. */ + originUsage: number | null; + originQuota: number | null; + /** Names of the model DBs (the delete target). */ + modelDbs: string[]; +} + +function emptyUsage(): KindUsage { + return { bytes: 0, items: 0 }; +} + +/** Sum body sizes per kind in one `sync:*` DB via a cursor (no bulk getAll — + * a model cache can be hundreds of MB and we only need the sizes). */ +function measureDb(name: string, into: StorageBreakdown): Promise { + return new Promise((resolve) => { + const req = indexedDB.open(name); + // Never upgrade here (no version passed); a DB from a newer schema still + // opens read-only for measuring. + req.onerror = () => resolve(); + req.onsuccess = () => { + const db = req.result; + if (!db.objectStoreNames.contains("bodies")) { + db.close(); + resolve(); + return; + } + const tx = db.transaction("bodies", "readonly"); + const cursorReq = tx.objectStore("bodies").openCursor(); + cursorReq.onsuccess = () => { + const cursor = cursorReq.result; + if (!cursor) return; // tx completes → oncomplete below + const key = String(cursor.key); + const value = cursor.value as { byteLength?: number } | undefined; + const size = value?.byteLength ?? 0; + const kind = key.startsWith("symbol/") + ? "symbol" + : key.startsWith("footprint/") + ? "footprint" + : key.startsWith("model3d/") + ? "model3d" + : null; + if (kind) { + into[kind].bytes += size; + into[kind].items += 1; + } + cursor.continue(); + }; + tx.oncomplete = () => { + db.close(); + resolve(); + }; + tx.onerror = () => { + db.close(); + resolve(); + }; + }; + }); +} + +async function measureAll(): Promise { + const breakdown: StorageBreakdown = { + symbol: emptyUsage(), + footprint: emptyUsage(), + model3d: emptyUsage(), + originUsage: null, + originQuota: null, + modelDbs: [], + }; + // indexedDB.databases() is supported everywhere we run (Chromium, FF 126+, + // Safari 14+); without it we just show the origin estimate. + const dbs = (await indexedDB.databases?.()) ?? []; + for (const db of dbs) { + const name = db.name; + if (!name || !name.startsWith(SYNC_PREFIX)) continue; + if (name.startsWith(MODELS_NS_PREFIX)) breakdown.modelDbs.push(name); + await measureDb(name, breakdown); + } + try { + const est = await navigator.storage?.estimate?.(); + breakdown.originUsage = est?.usage ?? null; + breakdown.originQuota = est?.quota ?? null; + } catch { + // estimate unavailable — the per-kind rows still stand on their own + } + return breakdown; +} + +function deleteDb(name: string): Promise { + return new Promise((resolve) => { + const req = indexedDB.deleteDatabase(name); + req.onsuccess = req.onerror = req.onblocked = () => resolve(); + }); +} + +export function formatBytes(n: number): string { + if (n < 1024) return `${n} B`; + if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`; + if (n < 1024 * 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)} MB`; + return `${(n / 1024 / 1024 / 1024).toFixed(2)} GB`; +} + +export function StorageUsageCard() { + const [data, setData] = React.useState(null); + const [busy, setBusy] = React.useState(false); + const [clearing, setClearing] = React.useState(false); + + const refresh = React.useCallback(async () => { + setBusy(true); + try { + setData(await measureAll()); + } finally { + setBusy(false); + } + }, []); + + React.useEffect(() => { + void refresh(); + }, [refresh]); + + const clearModels = React.useCallback(async () => { + if (!data) return; + setClearing(true); + try { + for (const name of data.modelDbs) await deleteDb(name); + await refresh(); + } finally { + setClearing(false); + } + }, [data, refresh]); + + const rows = data + ? ([ + ["Symbols", data.symbol], + ["Footprints", data.footprint], + ["3D models", data.model3d], + ] as const) + : null; + + return ( +
+

+ Storage +

+

+ Library data cached in this browser. Everything re-downloads on demand — + clearing is always safe. 3D models are fetched per board, so their cache + is the one worth reclaiming. +

+ + {!data ? ( +

+ Measuring… +

+ ) : ( +
+ + + {rows!.map(([label, usage]) => ( + + + + + + ))} + +
{label} + {usage.items.toLocaleString()} items + + {formatBytes(usage.bytes)} +
+ {data.originUsage !== null && ( +

+ Site total (all caches): {formatBytes(data.originUsage)} + {data.originQuota ? ` of ${formatBytes(data.originQuota)} available` : ""} +

+ )} +
+ + +
+
+ )} +
+ ); +} diff --git a/web/standalone/src/components/WasmTool.tsx b/web/standalone/src/components/WasmTool.tsx index 0e783c6..4872b4d 100644 --- a/web/standalone/src/components/WasmTool.tsx +++ b/web/standalone/src/components/WasmTool.tsx @@ -17,6 +17,7 @@ import { ChevronDown, ChevronUp, Loader2 } from "lucide-react"; import { currentScope, libsSourceConfig, + modelsSourceConfig, yjsProviderConfig, type DocSource, } from "@/lib/config"; @@ -31,6 +32,10 @@ import { type LibLoadingDetail, type LibsSource, } from "@/wasm/libs/source"; +import { + MODELS_LOADING_EVENT, + type ModelsLoadingDetail, +} from "@/wasm/libs/models-bridge"; import { memfsFilePath, memfsProjectDir } from "@/wasm/constants"; import { driveProjectIntoTool, type ToolFile } from "@/wasm/kicad-runner"; import { registerSaveHook, type SaveBytes } from "@/wasm/save-flow"; @@ -595,6 +600,9 @@ export function WasmTool({ done: number; total: number; } | null>(null); + // Board 3D-model prefetch in flight (background; the viewer works without it — + // anything still missing lazy-loads per model). Small badge, not an overlay. + const [modelsSync, setModelsSync] = React.useState(null); const append = React.useCallback( (msg: string) => setLogs((prev) => [...prev.slice(-800), msg]), @@ -660,6 +668,18 @@ export function WasmTool({ }; }, []); + // Board 3D-model prefetch progress (models-bridge prescan) — background badge. + React.useEffect(() => { + const onModels = (e: Event) => { + const d = (e as CustomEvent).detail; + setModelsSync( + d.loading ? `Fetching 3D models — ${d.done}/${d.total}` : null, + ); + }; + window.addEventListener(MODELS_LOADING_EVENT, onModels); + return () => window.removeEventListener(MODELS_LOADING_EVENT, onModels); + }, []); + // "Taking too long": once the tool has been loading for a while without // becoming ready, surface a hint (slow link / something may be wrong) + a // reload, so a stalled boot doesn't look like a frozen blank screen. @@ -756,6 +776,9 @@ export function WasmTool({ onAbort: oom.onAbort, onProgress: (loaded, total) => setProgress({ loaded, total }), libsSource: source, + // 3D models: lazy per-board source (null unless the CDN manifest is + // configured) — feeds the board prescan + the viewer's ensure fallback. + modelsSource: modelsSourceConfig(), }); // Register the save sink before the file opens: from here on, every // editor File→Save (MEMFS write) is routed onward through saveBytes. @@ -1002,6 +1025,13 @@ export function WasmTool({ )} + {/* Board 3D models still prefetching into the cache (background). */} + {ready && modelsSync && ( +
+ {modelsSync} +
+ )} + {/* A library item is being fetched (open/save). */} {ready && libBusy && (
diff --git a/web/standalone/src/lib/config.ts b/web/standalone/src/lib/config.ts index 3ec5de9..32f938f 100644 --- a/web/standalone/src/lib/config.ts +++ b/web/standalone/src/lib/config.ts @@ -98,6 +98,7 @@ export const LOCAL_PROJECTS_ENABLED = import type { ProviderConfig, ProviderKind } from "@/wasm/collab"; import { cdnLibsSource } from "@/wasm/libs/cdn-source"; +import { cdnModelsSource, type Model3dSource } from "@/wasm/libs/models-source"; import { remoteLibsSource } from "@/wasm/libs/remote-source"; import { scopedLibsSource } from "@/wasm/libs/scoped-source"; import type { LibsSource } from "@/wasm/libs/source"; @@ -192,6 +193,22 @@ export function currentScope(): string { export const CDN_LIBS_MANIFEST_URL = import.meta.env.VITE_LIBS_MANIFEST_URL || null; +/** Full URL of the CDN 3D-models top manifest, e.g. + * https://cdn.pcbjam.com/libs/kicad-models/10.0.0/manifest.json. Bodies are + * fetched lazily per board (sparse layers) and cached in IDB — never bulk + * synced. Unset ⇒ the 3D viewer renders bare boards (no component models). + * See wasm/libs/models-source.ts + docs/features/3d-models. */ +export const CDN_MODELS_MANIFEST_URL = + import.meta.env.VITE_MODELS_MANIFEST_URL || null; + +/** The 3D model source for a tool boot (null ⇒ models disabled). One instance + * per call — WasmTool keeps a single instance per boot like the libs source. */ +export function modelsSourceConfig(): Model3dSource | null { + return CDN_MODELS_MANIFEST_URL + ? cdnModelsSource(CDN_MODELS_MANIFEST_URL) + : null; +} + export function libsSourceConfig(projectId?: string): LibsSource | null { const kind = import.meta.env.VITE_LIBS_SOURCE ?? "remote"; // "local" is the placeholder id for launches with no real backend project diff --git a/web/standalone/src/pages/HomePage.tsx b/web/standalone/src/pages/HomePage.tsx index a9af4f6..b3558f7 100644 --- a/web/standalone/src/pages/HomePage.tsx +++ b/web/standalone/src/pages/HomePage.tsx @@ -23,6 +23,7 @@ import { WaitlistForm } from "@/components/WaitlistForm"; import { NewFileDialog } from "@/components/NewFileDialog"; import type { SaveBytes } from "@/wasm/save-flow"; import { LocalProjectView, type LocalFile } from "@/components/LocalProjectView"; +import { StorageUsageCard } from "@/components/StorageUsageCard"; import { WasmTool } from "@/components/WasmTool"; /** A KiCad project picked from the local filesystem (no backend involved). */ @@ -322,6 +323,11 @@ export function HomePage() { />
+ + {/* --- Browser storage: per-kind cache sizes + delete-3D-cache. --- */} +
+ +
); } diff --git a/web/standalone/src/wasm/boot.ts b/web/standalone/src/wasm/boot.ts index 9d204ca..b0acc28 100644 --- a/web/standalone/src/wasm/boot.ts +++ b/web/standalone/src/wasm/boot.ts @@ -1,11 +1,15 @@ import type { Tool } from "@pcbjam/shared"; import { KICAD_CONFIG_DIR, + MODELS_3D_ENV_VARS, + MODELS_3D_ROOT, RESOURCE_PATH, TOOL_ARGV0, TOOL_LIB_KIND, TOOL_NEEDS_CONFIG_SEED, } from "./constants"; +import { installModel3dHandler } from "./libs/models-bridge"; +import type { Model3dSource } from "./libs/models-source"; import { buildFpLibTable, buildSymLibTable, @@ -60,6 +64,9 @@ export interface BootOptions { /** Library source backing `window.kicadLibs`. Null/omitted disables libs * (an empty sym-lib-table is seeded). Its libs become sym-lib-table rows. */ libsSource?: LibsSource | null; + /** 3D model source (lazy, per-board). Null/omitted ⇒ the viewer renders the + * bare board only, exactly as before models existed. */ + modelsSource?: Model3dSource | null; } let booted: { tool: Tool; promise: Promise } | null = null; @@ -177,8 +184,17 @@ async function fetchWasmWithProgress( } async function doBoot(opts: BootOptions): Promise { - const { tool, base, container, log, onStatus, onAbort, onProgress, libsSource } = - opts; + const { + tool, + base, + container, + log, + onStatus, + onAbort, + onProgress, + libsSource, + modelsSource, + } = opts; const w = window as ToolWindow; // The wasm reads the top-level frame geometry from a GLOBAL `mainWindow` @@ -215,6 +231,12 @@ async function doBoot(opts: BootOptions): Promise { const libKind = TOOL_LIB_KIND[tool]; if (libsSource && libKind) { installLibsProvider(libsSource, log); + // 3D models ride the same provider (kind "model3d"): the C++ ensure fallback + // and the board prescan both resolve through this source. + if (modelsSource) { + installModel3dHandler(modelsSource, log); + log("[3d] model source installed"); + } try { // Ensure the owner has at least one writable user lib to save items into. // Pass the tool's kind so origins are filtered to the right domain. @@ -321,6 +343,9 @@ async function doBoot(opts: BootOptions): Promise { FS.writeFile(path, contents); log(`[boot] seeded ${path}`); }; + // 3D models: the MEMFS root the prescan/ensure paths write into, plus the + // env vars (every vintage) that make KiCad's resolver look there. + FS.mkdirTree(MODELS_3D_ROOT); writeIfAbsent( `${KICAD_CONFIG_DIR}/kicad_common.json`, JSON.stringify( @@ -329,6 +354,11 @@ async function doBoot(opts: BootOptions): Promise { update_check_prompt: true, data_collection_prompt: true, }, + environment: { + vars: Object.fromEntries( + MODELS_3D_ENV_VARS.map((v) => [v, MODELS_3D_ROOT]), + ), + }, }, null, 2, diff --git a/web/standalone/src/wasm/constants.ts b/web/standalone/src/wasm/constants.ts index 2792c0c..d8f4fae 100644 --- a/web/standalone/src/wasm/constants.ts +++ b/web/standalone/src/wasm/constants.ts @@ -72,6 +72,24 @@ export const TOOL_LIB_KIND: Record = { /** KiCad user settings dir for this build (PATHS::GetUserSettingsPath()). */ export const KICAD_CONFIG_DIR = `/home/kicad/.config/kicad/kicad/${KICAD_VERSION_DIR}`; +/** + * MEMFS root where 3D model bodies are materialized (JS prescan + the C++ + * lazy-ensure fallback both write `/.3dshapes/.` here). + * Boot points every `KICAD*_3DMODEL_DIR` env var at this dir — official-lib + * footprints reference models through vintage-specific vars (KICAD6..10 all + * occur), and KiCad's stock FILENAME_RESOLVER picks up each var it finds — so + * model paths resolve with zero resolver changes. + */ +export const MODELS_3D_ROOT = "/pcbjam/3dmodels"; +export const MODELS_3D_ENV_VARS = [ + "KISYS3DMOD", // pre-v6 legacy alias, still common in older boards + "KICAD6_3DMODEL_DIR", + "KICAD7_3DMODEL_DIR", + "KICAD8_3DMODEL_DIR", + "KICAD9_3DMODEL_DIR", + "KICAD10_3DMODEL_DIR", +] as const; + export function memfsProjectDir(slug: string): string { return `${MEMFS_PROJECTS_DIR}/${slug}`; } diff --git a/web/standalone/src/wasm/kicad-runner.ts b/web/standalone/src/wasm/kicad-runner.ts index 8dfca97..c366752 100644 --- a/web/standalone/src/wasm/kicad-runner.ts +++ b/web/standalone/src/wasm/kicad-runner.ts @@ -1,6 +1,7 @@ import type { Tool } from "@pcbjam/shared"; import { FILELESS_TOOLS } from "@pcbjam/shared"; import { memfsFilePath, memfsProjectDir } from "./constants"; +import { prescanBoardModels } from "./libs/models-bridge"; import { openFileInTool } from "./open-flow"; /** @@ -55,6 +56,16 @@ async function syncProjectToMemfs(win: ToolWindow, opts: DriveOptions): Promise< const bytes = await opts.fetchBytes(file.path); fs.writeFile(dest, bytes); opts.log(`[memfs] wrote ${dest} (${bytes.length} bytes)`); + // 3D models: prefetch every model this board references (R2 → IDB → MEMFS) + // so the 3D viewer's first open resolves locally. Fire-and-forget — project + // open never waits on it; a ref that misses falls back to the C++ per-model + // ensure. No-op unless a model source is installed (bootKicadTool). + if (file.path.endsWith(".kicad_pcb")) { + const text = new TextDecoder().decode(bytes); + void prescanBoardModels(text).catch((e) => + opts.log(`[3d] prescan failed: ${String(e)}`), + ); + } } } diff --git a/web/standalone/src/wasm/libs/models-bridge.test.ts b/web/standalone/src/wasm/libs/models-bridge.test.ts new file mode 100644 index 0000000..9427233 --- /dev/null +++ b/web/standalone/src/wasm/libs/models-bridge.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; +import { normalizeModelRef, scanModelRefs } from "./models-bridge"; + +describe("normalizeModelRef", () => { + it("strips any vintage of the model-dir var", () => { + for (const v of ["KICAD6", "KICAD7", "KICAD8", "KICAD9", "KICAD10"]) { + expect( + normalizeModelRef(`\${${v}_3DMODEL_DIR}/Resistor_SMD.3dshapes/R_0201.wrl`), + ).toBe("Resistor_SMD.3dshapes/R_0201.wrl"); + } + }); + + it("accepts the paren syntax and the legacy KISYS3DMOD alias", () => { + expect(normalizeModelRef("$(KICAD8_3DMODEL_DIR)/L.3dshapes/m.step")).toBe( + "L.3dshapes/m.step", + ); + expect(normalizeModelRef("${KISYS3DMOD}/L.3dshapes/m.wrl")).toBe( + "L.3dshapes/m.wrl", + ); + }); + + it("passes bare relative refs through", () => { + expect(normalizeModelRef("L.3dshapes/m.wrl")).toBe("L.3dshapes/m.wrl"); + }); + + it("rejects refs it cannot serve", () => { + expect(normalizeModelRef("${KIPRJMOD}/libs/3d/m.wrl")).toBeNull(); // project-local + expect(normalizeModelRef("/abs/path/m.wrl")).toBeNull(); + expect(normalizeModelRef("kicad_embed://m.wrl")).toBeNull(); + expect(normalizeModelRef("")).toBeNull(); + expect(normalizeModelRef("${UNCLOSED/m.wrl")).toBeNull(); + }); +}); + +describe("scanModelRefs", () => { + it("finds, normalizes and dedupes board model refs", () => { + const board = ` + (footprint "Resistor_THT:R_Axial" + (model "\${KICAD10_3DMODEL_DIR}/Resistor_THT.3dshapes/R_Axial.step" + (offset (xyz 0 0 0)))) + (footprint "Resistor_THT:R_Axial" + (model "\${KICAD10_3DMODEL_DIR}/Resistor_THT.3dshapes/R_Axial.step")) + (footprint "X:Y" + (model "\${KIPRJMOD}/libs/3d_shapes/custom.wrl")) + (footprint "L:M" (model "\${KICAD8_3DMODEL_DIR}/LED_THT.3dshapes/LED_D5.0mm.wrl")) + `; + expect(scanModelRefs(board).sort()).toEqual([ + "LED_THT.3dshapes/LED_D5.0mm.wrl", + "Resistor_THT.3dshapes/R_Axial.step", + ]); + }); + + it("handles escaped quotes inside the path", () => { + expect( + scanModelRefs('(model "${KICAD9_3DMODEL_DIR}/A.3dshapes/we\\"ird.wrl")'), + ).toEqual(['A.3dshapes/we"ird.wrl']); + }); + + it("returns empty for a board with no models", () => { + expect(scanModelRefs("(kicad_pcb (version 20240101))")).toEqual([]); + }); +}); diff --git a/web/standalone/src/wasm/libs/models-bridge.ts b/web/standalone/src/wasm/libs/models-bridge.ts new file mode 100644 index 0000000..8bca5d0 --- /dev/null +++ b/web/standalone/src/wasm/libs/models-bridge.ts @@ -0,0 +1,181 @@ +import { MODELS_3D_ROOT } from "../constants"; +import type { Model3dSource } from "./models-source"; + +/** + * Glue between 3D model delivery and the running KiCad WASM tool: + * + * - `installModel3dHandler` backs the provider's `kind === "model3d"` requests + * (the C++ lazy fallback in S3D_CACHE::load → PCBJAM_3D::EnsureModelFile asks + * "ensure" for a ref the prescan missed). + * - `prescanBoardModels` scans a board's `(model "…")` refs up front and + * prefetches those bodies (R2 → IDB → MEMFS) so the 3D viewer's first open + * resolves everything locally. + * + * Both paths converge on `ensureModelInMemfs`: fetch the body via the + * `Model3dSource` (IDB-cached, sparse) and write it under `MODELS_3D_ROOT` — + * where boot points every `KICAD*_3DMODEL_DIR` env var, so KiCad's stock + * resolver finds the file with no C++ resolution changes. + */ + +/** Progress of a board-model prefetch burst (drives the 3D loading overlay). */ +export const MODELS_LOADING_EVENT = "pcbjam:models-loading"; + +export interface ModelsLoadingDetail { + loading: boolean; + done: number; + total: number; +} + +function emitModelsLoading(detail: ModelsLoadingDetail): void { + if (typeof window === "undefined") return; + window.dispatchEvent(new CustomEvent(MODELS_LOADING_EVENT, { detail })); +} + +/** + * Normalize a footprint model reference to the source's relative form — + * "${KICAD*_3DMODEL_DIR}/.3dshapes/." (any vintage, `${}` or + * `$()`) → ".3dshapes/.". Bare relative refs pass through; + * absolute paths / ${KIPRJMOD} / kicad_embed:// are not ours → null. Mirrors + * pcbjamNormalizeModelRef in kicad/3d-viewer/3d_cache/pcbjam_model_fetch.cpp. + */ +export function normalizeModelRef(raw: string): string | null { + const ref = raw.trim(); + if (!ref) return null; + if (ref.startsWith("${") || ref.startsWith("$(")) { + const closing = ref[1] === "{" ? "}" : ")"; + const end = ref.indexOf(closing); + if (end < 0) return null; + const v = ref.slice(2, end); + // Any vintage of the model-dir var, plus the pre-v6 legacy alias. + if (!v.includes("3DMODEL_DIR") && v !== "KISYS3DMOD") return null; + return ref.slice(end + 1).replace(/^[/\\]+/, "") || null; + } + if (ref.startsWith("/") || ref.includes("://")) return null; + return ref; +} + +/** Every `(model "…")` ref in a KiCad board/footprint s-expr, normalized. */ +export function scanModelRefs(sexprText: string): string[] { + const refs = new Set(); + const re = /\(\s*model\s+"((?:[^"\\]|\\.)*)"/g; + for (let m = re.exec(sexprText); m; m = re.exec(sexprText)) { + const raw = m[1]!.replace(/\\(.)/g, "$1"); + const rel = normalizeModelRef(raw); + if (rel) refs.add(rel); + } + return [...refs]; +} + +type ModelFS = Pick; + +function toolFS(): ModelFS | null { + const fs = (window as ToolWindow).FS; + return fs && typeof fs.writeFile === "function" ? (fs as ModelFS) : null; +} + +let installedSource: Model3dSource | null = null; +let installedLog: (msg: string) => void = () => {}; +/** Refs already materialized in MEMFS this session (bodies are immutable). */ +const written = new Set(); +/** In-flight ensures, coalesced per ref (prescan and the C++ fallback race). */ +const ensuring = new Map>(); + +/** Wire the model source used by the provider dispatch + prescan. */ +export function installModel3dHandler( + source: Model3dSource, + log: (msg: string) => void, +): void { + installedSource = source; + installedLog = log; +} + +/** Fetch one model body and write it under MODELS_3D_ROOT. Resolves to the + * absolute MEMFS path when present, null when the source can't serve it. */ +export async function ensureModelInMemfs(ref: string): Promise { + const dest = `${MODELS_3D_ROOT}/${ref}`; + if (written.has(ref)) return dest; + let p = ensuring.get(ref); + if (!p) { + p = doEnsure(ref, dest).finally(() => ensuring.delete(ref)); + ensuring.set(ref, p); + } + return p; +} + +async function doEnsure(ref: string, dest: string): Promise { + const source = installedSource; + const fs = toolFS(); + if (!source || !fs) return null; + if (fs.analyzePath(dest).exists) { + written.add(ref); + return dest; + } + const body = await source.getModelBody(ref); + if (!body) return null; + fs.mkdirTree(dest.slice(0, dest.lastIndexOf("/"))); + fs.writeFile(dest, body); + written.add(ref); + installedLog(`[3d] materialized ${ref} (${body.length} bytes)`); + return dest; +} + +/** + * Provider dispatch for `kind === "model3d"` (called by installLibsProvider's + * request before any lib-id parsing — the C++ bridge passes an empty lib; the + * ref itself carries the library). Answers with the ABSOLUTE MEMFS path of the + * materialized file — S3D_CACHE loads that path directly, so model delivery + * never depends on env-var expansion inside the wasm runtime. + */ +export async function handleModel3dRequest( + op: string, + arg: string, +): Promise { + if (op !== "ensure") return null; + const rel = normalizeModelRef(arg); + if (!rel) return null; + try { + return await ensureModelInMemfs(rel); + } catch (e) { + installedLog(`[3d] ensure failed for ${arg}: ${String(e)}`); + return null; + } +} + +/** + * Prefetch every model a board references (fire-and-forget from the project + * sync). Bodies land in IDB + MEMFS before the user opens the 3D viewer in the + * common case; anything still missing falls back to the per-model C++ ensure. + */ +export async function prescanBoardModels( + boardText: string, + concurrency = 6, +): Promise { + if (!installedSource) return; + const refs = scanModelRefs(boardText); + if (!refs.length) return; + + const total = refs.length; + let done = 0; + emitModelsLoading({ loading: true, done, total }); + installedLog(`[3d] prescan: ${total} model ref(s) on board`); + const started = performance.now(); + + let idx = 0; + const worker = async (): Promise => { + while (idx < refs.length) { + const ref = refs[idx++]!; + try { + await ensureModelInMemfs(ref); + } catch { + // best-effort: the C++ lazy path (or a later prescan) retries + } + emitModelsLoading({ loading: ++done < total, done, total }); + } + }; + await Promise.all( + Array.from({ length: Math.min(concurrency, total) }, () => worker()), + ); + installedLog( + `[3d] prescan: ${done}/${total} in ${Math.round(performance.now() - started)}ms`, + ); +} diff --git a/web/standalone/src/wasm/libs/models-source.test.ts b/web/standalone/src/wasm/libs/models-source.test.ts new file mode 100644 index 0000000..ef22b33 --- /dev/null +++ b/web/standalone/src/wasm/libs/models-source.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from "vitest"; +import { sha256Hex, type SyncManifest } from "@pcbjam/shared"; +import { memStore, type LayerStore } from "@pcbjam/sync-client"; +import { cdnModelsSource } from "./models-source"; + +const MANIFEST_URL = "https://cdn.test/libs/kicad-models/9.0.9/manifest.json"; +const BASE = "https://cdn.test/libs/kicad-models/9.0.9"; +const BLOBS = "https://cdn.test/libs/kicad-models/blobs/sha256"; + +const enc = new TextEncoder(); +const dec = new TextDecoder(); + +/** Build the CDN layout the way publish-models.ts does: per-lib sparse manifest + * + content-addressed blobs — pinning the format with the REAL hash codec. */ +async function fakeModelsCdn() { + const bodies: Record = { + "model3d/R_Axial.step": enc.encode("STEP R_Axial"), + "model3d/R_Disc.wrl": enc.encode("#VRML V2.0 utf8 R_Disc"), + }; + const entries: SyncManifest["entries"] = {}; + const blobs = new Map(); + for (const [path, body] of Object.entries(bodies)) { + const hash = await sha256Hex(body); + entries[path] = { hash, size: body.length, mtime: 0 }; + blobs.set(hash, body); + } + const libManifest: SyncManifest = { version: 1, entries }; + const top = { + schema: 1, + tag: "9.0.9", + libs: [{ id: "Resistor_THT", itemCount: 2 }], + }; + + let manifestFetches = 0; + let blobFetches = 0; + const json = (obj: unknown) => ({ ok: true, json: async () => obj }); + const bin = (bytes: Uint8Array) => ({ + ok: true, + arrayBuffer: async () => bytes.buffer, + }); + const fetchImpl = (async (url: string) => { + if (url === MANIFEST_URL) return json(top); + if (url === `${BASE}/Resistor_THT/manifest`) { + manifestFetches += 1; + return json(libManifest); + } + if (url.startsWith(`${BLOBS}/`)) { + const blob = blobs.get(url.slice(`${BLOBS}/`.length)); + if (blob) { + blobFetches += 1; + return bin(blob); + } + } + return { ok: false, status: 404 }; + }) as unknown as typeof fetch; + + return { + fetchImpl, + counters: { + get manifestFetches() { + return manifestFetches; + }, + get blobFetches() { + return blobFetches; + }, + }, + }; +} + +function storeMap() { + const stores = new Map(); + return (ns: string): LayerStore => { + let s = stores.get(ns); + if (!s) stores.set(ns, (s = memStore())); + return s; + }; +} + +describe("cdnModelsSource", () => { + it("fetches exactly the requested body (sparse), then serves from cache", async () => { + const cdn = await fakeModelsCdn(); + const src = cdnModelsSource(MANIFEST_URL, { + fetchImpl: cdn.fetchImpl, + storeFactory: storeMap(), + }); + + const body = await src.getModelBody("Resistor_THT.3dshapes/R_Axial.step"); + expect(dec.decode(body!)).toBe("STEP R_Axial"); + expect(cdn.counters.blobFetches).toBe(1); // only the asked-for model + + await src.getModelBody("Resistor_THT.3dshapes/R_Axial.step"); + expect(cdn.counters.blobFetches).toBe(1); // cached + expect(cdn.counters.manifestFetches).toBe(1); // lib opened once + }); + + it("returns null for unknown models/libs/refs without throwing", async () => { + const cdn = await fakeModelsCdn(); + const src = cdnModelsSource(MANIFEST_URL, { + fetchImpl: cdn.fetchImpl, + storeFactory: storeMap(), + }); + + expect(await src.getModelBody("Resistor_THT.3dshapes/nope.step")).toBeNull(); + expect(await src.getModelBody("NoSuchLib.3dshapes/m.wrl")).toBeNull(); + expect(await src.getModelBody("not-a-model-ref")).toBeNull(); + expect(cdn.counters.blobFetches).toBe(0); + }); + + it("hasModel answers from the manifest without fetching bodies", async () => { + const cdn = await fakeModelsCdn(); + const src = cdnModelsSource(MANIFEST_URL, { + fetchImpl: cdn.fetchImpl, + storeFactory: storeMap(), + }); + + expect(await src.hasModel("Resistor_THT.3dshapes/R_Disc.wrl")).toBe(true); + expect(await src.hasModel("Resistor_THT.3dshapes/nope.wrl")).toBe(false); + expect(cdn.counters.blobFetches).toBe(0); + }); +}); diff --git a/web/standalone/src/wasm/libs/models-source.ts b/web/standalone/src/wasm/libs/models-source.ts new file mode 100644 index 0000000..99a8db2 --- /dev/null +++ b/web/standalone/src/wasm/libs/models-source.ts @@ -0,0 +1,124 @@ +import { SyncStack, type SyncStackOptions } from "@pcbjam/sync-client"; + +/** + * Read-only source of 3D model bodies (`.wrl` / `.step`), addressed by the + * KiCad-relative ref `".3dshapes/."` (a footprint's `(model …)` + * path with the `${KICAD*_3DMODEL_DIR}/` prefix stripped). + * + * Unlike symbols/footprints, models are never bulk-synced: each lib is an + * r2-idb-sync **sparse** layer — only the (small) manifest syncs eagerly, and a + * body is fetched exactly when a board references it, then cached in IDB. This + * keeps the client cost proportional to what the user renders, not to the + * ~GB-scale full set (docs/features/3d-models). + */ +export interface Model3dSource { + /** One model body, IDB-cached; null when unknown/unavailable. */ + getModelBody(ref: string): Promise; + /** Whether a ref exists in the set at all (manifest-only, no body fetch). */ + hasModel(ref: string): Promise; +} + +interface CdnModelsManifest { + schema: number; + tag: string; + libs: Array<{ id: string; itemCount?: number }>; +} + +/** Split ".3dshapes/" → { lib, path: "model3d/" }. */ +function splitRef(ref: string): { lib: string; path: string } | null { + const i = ref.indexOf(".3dshapes/"); + if (i <= 0) return null; + const lib = ref.slice(0, i); + const name = ref.slice(i + ".3dshapes/".length); + if (!name || name.includes("/")) return null; + return { lib, path: `model3d/${name}` }; +} + +/** + * CDN-backed `Model3dSource`. Layout under the manifest's dir + * (`/libs/kicad-models//`, see scripts/deploy/publish-models.ts): + * manifest.json top index { schema, tag, libs:[{id,itemCount}] } + * /manifest per-lib SyncManifest, entries "model3d/." + * Bodies are content-addressed and shared across tags, one level up: + * /libs/kicad-models/blobs/sha256/ + */ +export function cdnModelsSource( + manifestUrl: string, + opts?: Pick, +): Model3dSource { + const baseDir = manifestUrl.replace(/\/[^/]*$/, ""); // …/libs/kicad-models/ + const blobsBase = `${baseDir.replace(/\/[^/]*$/, "")}/blobs/sha256`; + const fetchImpl = opts?.fetchImpl ?? fetch; + + let manifestP: Promise | null = null; + const loadManifest = (): Promise => { + if (!manifestP) { + // Never cache a rejection (mirrors cdn-source.ts): a transient failure + // must not poison every later model read. + manifestP = (async () => { + const r = await fetchImpl(manifestUrl, { cache: "no-store" }); + if (!r.ok) throw new Error(`cdn models manifest ${r.status}: ${manifestUrl}`); + return (await r.json()) as CdnModelsManifest; + })().catch((e) => { + manifestP = null; + throw e; + }); + } + return manifestP; + }; + + // One lazily-opened sparse stack per lib (IDB store keyed by namespace, so a + // lib's cached models persist and dedupe across sessions). + const stacks = new Map>(); + const openStack = (libId: string): Promise => { + let p = stacks.get(libId); + if (!p) { + p = (async () => { + const m = await loadManifest(); + if (!m.libs.some((l) => l.id === libId)) return null; // unknown lib + const stack = new SyncStack({ + layers: [ + { + namespace: `kicad-models:${m.tag}:${libId}`, + kind: "sparse", + url: `${baseDir}/${encodeURIComponent(libId)}`, + bodyUrlTemplate: `${blobsBase}/{hash}`, + }, + ], + ...opts, + }); + await stack.open(); + return stack; + })().catch((e) => { + stacks.delete(libId); // a failed open must stay retryable + throw e; + }); + stacks.set(libId, p); + } + return p; + }; + + return { + async getModelBody(ref: string): Promise { + const split = splitRef(ref); + if (!split) return null; + try { + const stack = await openStack(split.lib); + return stack ? await stack.read(split.path) : null; + } catch { + return null; // missing models render as absent, never break the viewer + } + }, + async hasModel(ref: string): Promise { + const split = splitRef(ref); + if (!split) return false; + try { + const stack = await openStack(split.lib); + if (!stack) return false; + return (await stack.list()).some((e) => e.path === split.path); + } catch { + return false; + } + }, + }; +} diff --git a/web/standalone/src/wasm/libs/source.ts b/web/standalone/src/wasm/libs/source.ts index f87fb63..e793dbf 100644 --- a/web/standalone/src/wasm/libs/source.ts +++ b/web/standalone/src/wasm/libs/source.ts @@ -1,3 +1,4 @@ +import { handleModel3dRequest } from "./models-bridge"; import { libIdFromUri, libUri } from "./uri"; /** @@ -259,6 +260,12 @@ export function installLibsProvider( let fatResetTimer: ReturnType | undefined; const request: KicadLibsRequest = async (op, lib, arg, kind = "symbol") => { + // 3D models are addressed by ref, not lib-table URI (the C++ ensure bridge + // passes an empty lib) — dispatch before the lib-id parse would null it out. + if (kind === "model3d") { + log(`[libs] request op=${op} kind=model3d arg=${arg}`); + return handleModel3dRequest(op, arg); + } const id = libIdFromUri(lib); log(`[libs] request op=${op} kind=${kind} lib=${lib} (id=${id}) arg=${arg}`); if (!id) return null;