feat(3d): lazy 3D model delivery R2→IDB→WASM

- cdnModelsSource: sparse per-lib stacks over the models CDN layout
  (libs/kicad-models/<tag>/<lib>/manifest + content-addressed blobs)
- models-bridge: board prescan ((model …) scan → prefetch, 4/4 in 74ms
  on the demo board) + the kind=model3d 'ensure' op answering absolute
  MEMFS paths; only board-referenced bodies ever enter IDB
- boot/runner/config/WasmTool wiring (VITE_MODELS_MANIFEST_URL,
  KICAD*_3DMODEL_DIR seeding, prefetch badge)
- StorageUsageCard on HomePage: per-kind cached sizes + delete-3D-cache
- publish-models.ts (brotli content-addressed publish, 606→93MB for the
  pic_programmer lib set) + dev-demo --models-tag/--models-local
- e2e: 3d-viewer-models.spec.ts (bridge normalize/dedup + STEP/WRL
  render); submodule bumps: kicad (static 3D plugins + ensure hook),
  pcbjam-shared (sparse sync layer)

Spec + findings: docs/features/3d-models (private repo).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014AT7gVHRktDYoQ68S4x6A4
This commit is contained in:
Gergő Törcsvári 2026-07-02 11:00:51 +02:00
commit 6f3f3bcf00
No known key found for this signature in database
GPG key ID: 8E75F2CDE64E5322
19 changed files with 1351 additions and 4 deletions

View file

@ -0,0 +1,235 @@
import * as React from "react";
import { Database, Loader2, RefreshCw, Trash2 } from "lucide-react";
import { Button } from "@/components/ui/button";
/**
* Browser-storage usage for the editor's library caches, by kind, with a
* "delete 3D model cache" action.
*
* All library data lives in r2-idb-sync IndexedDB databases named
* `sync:<namespace>` (one per lib), each with a `bodies` store keyed
* `"<kind>/<name>"`. Symbols/footprints are bulk-synced (whole-lib bundles);
* 3D models are SPARSE — only the models boards actually rendered are stored
* (namespaces `kicad-models:…`), which makes them the one cache that is both
* potentially large and always safe to drop: manifests re-sync in one small
* fetch and bodies lazily re-download exactly when a board needs them again.
*/
const SYNC_PREFIX = "sync:";
const MODELS_NS_PREFIX = `${SYNC_PREFIX}kicad-models:`;
interface KindUsage {
bytes: number;
items: number;
}
interface StorageBreakdown {
symbol: KindUsage;
footprint: KindUsage;
model3d: KindUsage;
/** Whole-origin estimate (all IDB + caches), from navigator.storage. */
originUsage: number | null;
originQuota: number | null;
/** Names of the model DBs (the delete target). */
modelDbs: string[];
}
function emptyUsage(): KindUsage {
return { bytes: 0, items: 0 };
}
/** Sum body sizes per kind in one `sync:*` DB via a cursor (no bulk getAll —
* a model cache can be hundreds of MB and we only need the sizes). */
function measureDb(name: string, into: StorageBreakdown): Promise<void> {
return new Promise((resolve) => {
const req = indexedDB.open(name);
// Never upgrade here (no version passed); a DB from a newer schema still
// opens read-only for measuring.
req.onerror = () => resolve();
req.onsuccess = () => {
const db = req.result;
if (!db.objectStoreNames.contains("bodies")) {
db.close();
resolve();
return;
}
const tx = db.transaction("bodies", "readonly");
const cursorReq = tx.objectStore("bodies").openCursor();
cursorReq.onsuccess = () => {
const cursor = cursorReq.result;
if (!cursor) return; // tx completes → oncomplete below
const key = String(cursor.key);
const value = cursor.value as { byteLength?: number } | undefined;
const size = value?.byteLength ?? 0;
const kind = key.startsWith("symbol/")
? "symbol"
: key.startsWith("footprint/")
? "footprint"
: key.startsWith("model3d/")
? "model3d"
: null;
if (kind) {
into[kind].bytes += size;
into[kind].items += 1;
}
cursor.continue();
};
tx.oncomplete = () => {
db.close();
resolve();
};
tx.onerror = () => {
db.close();
resolve();
};
};
});
}
async function measureAll(): Promise<StorageBreakdown> {
const breakdown: StorageBreakdown = {
symbol: emptyUsage(),
footprint: emptyUsage(),
model3d: emptyUsage(),
originUsage: null,
originQuota: null,
modelDbs: [],
};
// indexedDB.databases() is supported everywhere we run (Chromium, FF 126+,
// Safari 14+); without it we just show the origin estimate.
const dbs = (await indexedDB.databases?.()) ?? [];
for (const db of dbs) {
const name = db.name;
if (!name || !name.startsWith(SYNC_PREFIX)) continue;
if (name.startsWith(MODELS_NS_PREFIX)) breakdown.modelDbs.push(name);
await measureDb(name, breakdown);
}
try {
const est = await navigator.storage?.estimate?.();
breakdown.originUsage = est?.usage ?? null;
breakdown.originQuota = est?.quota ?? null;
} catch {
// estimate unavailable — the per-kind rows still stand on their own
}
return breakdown;
}
function deleteDb(name: string): Promise<void> {
return new Promise((resolve) => {
const req = indexedDB.deleteDatabase(name);
req.onsuccess = req.onerror = req.onblocked = () => resolve();
});
}
export function formatBytes(n: number): string {
if (n < 1024) return `${n} B`;
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
if (n < 1024 * 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)} MB`;
return `${(n / 1024 / 1024 / 1024).toFixed(2)} GB`;
}
export function StorageUsageCard() {
const [data, setData] = React.useState<StorageBreakdown | null>(null);
const [busy, setBusy] = React.useState(false);
const [clearing, setClearing] = React.useState(false);
const refresh = React.useCallback(async () => {
setBusy(true);
try {
setData(await measureAll());
} finally {
setBusy(false);
}
}, []);
React.useEffect(() => {
void refresh();
}, [refresh]);
const clearModels = React.useCallback(async () => {
if (!data) return;
setClearing(true);
try {
for (const name of data.modelDbs) await deleteDb(name);
await refresh();
} finally {
setClearing(false);
}
}, [data, refresh]);
const rows = data
? ([
["Symbols", data.symbol],
["Footprints", data.footprint],
["3D models", data.model3d],
] as const)
: null;
return (
<section className="mb-10 rounded-lg border p-5">
<h2 className="mb-1 flex items-center gap-2 text-lg font-medium">
<Database size={16} /> Storage
</h2>
<p className="mb-4 text-sm text-muted-foreground">
Library data cached in this browser. Everything re-downloads on demand —
clearing is always safe. 3D models are fetched per board, so their cache
is the one worth reclaiming.
</p>
{!data ? (
<p className="flex items-center gap-2 text-sm text-muted-foreground">
<Loader2 className="animate-spin" size={14} /> Measuring…
</p>
) : (
<div className="space-y-2">
<table className="w-full max-w-md text-sm">
<tbody>
{rows!.map(([label, usage]) => (
<tr key={label} className="border-b last:border-b-0">
<td className="py-1.5 text-muted-foreground">{label}</td>
<td className="py-1.5 text-right tabular-nums">
{usage.items.toLocaleString()} items
</td>
<td className="py-1.5 text-right font-medium tabular-nums">
{formatBytes(usage.bytes)}
</td>
</tr>
))}
</tbody>
</table>
{data.originUsage !== null && (
<p className="text-xs text-muted-foreground">
Site total (all caches): {formatBytes(data.originUsage)}
{data.originQuota ? ` of ${formatBytes(data.originQuota)} available` : ""}
</p>
)}
<div className="flex items-center gap-2 pt-1">
<Button
variant="outline"
size="sm"
onClick={() => void clearModels()}
disabled={clearing || data.model3d.items === 0}
>
{clearing ? (
<Loader2 className="mr-1 animate-spin" size={14} />
) : (
<Trash2 className="mr-1" size={14} />
)}
Delete 3D model cache
{data.model3d.bytes > 0 ? ` (${formatBytes(data.model3d.bytes)})` : ""}
</Button>
<Button
variant="ghost"
size="sm"
onClick={() => void refresh()}
disabled={busy}
aria-label="Refresh storage usage"
>
<RefreshCw className={busy ? "animate-spin" : ""} size={14} />
</Button>
</div>
</div>
)}
</section>
);
}

View file

@ -17,6 +17,7 @@ import { ChevronDown, ChevronUp, Loader2 } from "lucide-react";
import {
currentScope,
libsSourceConfig,
modelsSourceConfig,
yjsProviderConfig,
type DocSource,
} from "@/lib/config";
@ -31,6 +32,10 @@ import {
type LibLoadingDetail,
type LibsSource,
} from "@/wasm/libs/source";
import {
MODELS_LOADING_EVENT,
type ModelsLoadingDetail,
} from "@/wasm/libs/models-bridge";
import { memfsFilePath, memfsProjectDir } from "@/wasm/constants";
import { driveProjectIntoTool, type ToolFile } from "@/wasm/kicad-runner";
import { registerSaveHook, type SaveBytes } from "@/wasm/save-flow";
@ -595,6 +600,9 @@ export function WasmTool({
done: number;
total: number;
} | null>(null);
// Board 3D-model prefetch in flight (background; the viewer works without it —
// anything still missing lazy-loads per model). Small badge, not an overlay.
const [modelsSync, setModelsSync] = React.useState<string | null>(null);
const append = React.useCallback(
(msg: string) => setLogs((prev) => [...prev.slice(-800), msg]),
@ -660,6 +668,18 @@ export function WasmTool({
};
}, []);
// Board 3D-model prefetch progress (models-bridge prescan) — background badge.
React.useEffect(() => {
const onModels = (e: Event) => {
const d = (e as CustomEvent<ModelsLoadingDetail>).detail;
setModelsSync(
d.loading ? `Fetching 3D models — ${d.done}/${d.total}` : null,
);
};
window.addEventListener(MODELS_LOADING_EVENT, onModels);
return () => window.removeEventListener(MODELS_LOADING_EVENT, onModels);
}, []);
// "Taking too long": once the tool has been loading for a while without
// becoming ready, surface a hint (slow link / something may be wrong) + a
// reload, so a stalled boot doesn't look like a frozen blank screen.
@ -756,6 +776,9 @@ export function WasmTool({
onAbort: oom.onAbort,
onProgress: (loaded, total) => setProgress({ loaded, total }),
libsSource: source,
// 3D models: lazy per-board source (null unless the CDN manifest is
// configured) — feeds the board prescan + the viewer's ensure fallback.
modelsSource: modelsSourceConfig(),
});
// Register the save sink before the file opens: from here on, every
// editor File→Save (MEMFS write) is routed onward through saveBytes.
@ -1002,6 +1025,13 @@ export function WasmTool({
</div>
)}
{/* Board 3D models still prefetching into the cache (background). */}
{ready && modelsSync && (
<div className="pointer-events-none absolute bottom-[4.25rem] left-3 z-20 flex items-center gap-2 rounded bg-black/80 px-3 py-1.5 text-xs text-sky-200">
<Loader2 className="animate-spin" size={14} /> {modelsSync}
</div>
)}
{/* A library item is being fetched (open/save). */}
{ready && libBusy && (
<div className="pointer-events-none absolute left-1/2 top-3 z-20 flex -translate-x-1/2 items-center gap-2 rounded bg-black/80 px-3 py-1.5 text-xs text-white">

View file

@ -98,6 +98,7 @@ export const LOCAL_PROJECTS_ENABLED =
import type { ProviderConfig, ProviderKind } from "@/wasm/collab";
import { cdnLibsSource } from "@/wasm/libs/cdn-source";
import { cdnModelsSource, type Model3dSource } from "@/wasm/libs/models-source";
import { remoteLibsSource } from "@/wasm/libs/remote-source";
import { scopedLibsSource } from "@/wasm/libs/scoped-source";
import type { LibsSource } from "@/wasm/libs/source";
@ -192,6 +193,22 @@ export function currentScope(): string {
export const CDN_LIBS_MANIFEST_URL =
import.meta.env.VITE_LIBS_MANIFEST_URL || null;
/** Full URL of the CDN 3D-models top manifest, e.g.
* https://cdn.pcbjam.com/libs/kicad-models/10.0.0/manifest.json. Bodies are
* fetched lazily per board (sparse layers) and cached in IDB — never bulk
* synced. Unset ⇒ the 3D viewer renders bare boards (no component models).
* See wasm/libs/models-source.ts + docs/features/3d-models. */
export const CDN_MODELS_MANIFEST_URL =
import.meta.env.VITE_MODELS_MANIFEST_URL || null;
/** The 3D model source for a tool boot (null ⇒ models disabled). One instance
* per call — WasmTool keeps a single instance per boot like the libs source. */
export function modelsSourceConfig(): Model3dSource | null {
return CDN_MODELS_MANIFEST_URL
? cdnModelsSource(CDN_MODELS_MANIFEST_URL)
: null;
}
export function libsSourceConfig(projectId?: string): LibsSource | null {
const kind = import.meta.env.VITE_LIBS_SOURCE ?? "remote";
// "local" is the placeholder id for launches with no real backend project

View file

@ -23,6 +23,7 @@ import { WaitlistForm } from "@/components/WaitlistForm";
import { NewFileDialog } from "@/components/NewFileDialog";
import type { SaveBytes } from "@/wasm/save-flow";
import { LocalProjectView, type LocalFile } from "@/components/LocalProjectView";
import { StorageUsageCard } from "@/components/StorageUsageCard";
import { WasmTool } from "@/components/WasmTool";
/** A KiCad project picked from the local filesystem (no backend involved). */
@ -322,6 +323,11 @@ export function HomePage() {
/>
</div>
</section>
{/* --- Browser storage: per-kind cache sizes + delete-3D-cache. --- */}
<div className="mt-10">
<StorageUsageCard />
</div>
</div>
);
}

View file

@ -1,11 +1,15 @@
import type { Tool } from "@pcbjam/shared";
import {
KICAD_CONFIG_DIR,
MODELS_3D_ENV_VARS,
MODELS_3D_ROOT,
RESOURCE_PATH,
TOOL_ARGV0,
TOOL_LIB_KIND,
TOOL_NEEDS_CONFIG_SEED,
} from "./constants";
import { installModel3dHandler } from "./libs/models-bridge";
import type { Model3dSource } from "./libs/models-source";
import {
buildFpLibTable,
buildSymLibTable,
@ -60,6 +64,9 @@ export interface BootOptions {
/** Library source backing `window.kicadLibs`. Null/omitted disables libs
* (an empty sym-lib-table is seeded). Its libs become sym-lib-table rows. */
libsSource?: LibsSource | null;
/** 3D model source (lazy, per-board). Null/omitted ⇒ the viewer renders the
* bare board only, exactly as before models existed. */
modelsSource?: Model3dSource | null;
}
let booted: { tool: Tool; promise: Promise<void> } | null = null;
@ -177,8 +184,17 @@ async function fetchWasmWithProgress(
}
async function doBoot(opts: BootOptions): Promise<void> {
const { tool, base, container, log, onStatus, onAbort, onProgress, libsSource } =
opts;
const {
tool,
base,
container,
log,
onStatus,
onAbort,
onProgress,
libsSource,
modelsSource,
} = opts;
const w = window as ToolWindow;
// The wasm reads the top-level frame geometry from a GLOBAL `mainWindow`
@ -215,6 +231,12 @@ async function doBoot(opts: BootOptions): Promise<void> {
const libKind = TOOL_LIB_KIND[tool];
if (libsSource && libKind) {
installLibsProvider(libsSource, log);
// 3D models ride the same provider (kind "model3d"): the C++ ensure fallback
// and the board prescan both resolve through this source.
if (modelsSource) {
installModel3dHandler(modelsSource, log);
log("[3d] model source installed");
}
try {
// Ensure the owner has at least one writable user lib to save items into.
// Pass the tool's kind so origins are filtered to the right domain.
@ -321,6 +343,9 @@ async function doBoot(opts: BootOptions): Promise<void> {
FS.writeFile(path, contents);
log(`[boot] seeded ${path}`);
};
// 3D models: the MEMFS root the prescan/ensure paths write into, plus the
// env vars (every vintage) that make KiCad's resolver look there.
FS.mkdirTree(MODELS_3D_ROOT);
writeIfAbsent(
`${KICAD_CONFIG_DIR}/kicad_common.json`,
JSON.stringify(
@ -329,6 +354,11 @@ async function doBoot(opts: BootOptions): Promise<void> {
update_check_prompt: true,
data_collection_prompt: true,
},
environment: {
vars: Object.fromEntries(
MODELS_3D_ENV_VARS.map((v) => [v, MODELS_3D_ROOT]),
),
},
},
null,
2,

View file

@ -72,6 +72,24 @@ export const TOOL_LIB_KIND: Record<Tool, "symbol" | "footprint" | null> = {
/** KiCad user settings dir for this build (PATHS::GetUserSettingsPath()). */
export const KICAD_CONFIG_DIR = `/home/kicad/.config/kicad/kicad/${KICAD_VERSION_DIR}`;
/**
* MEMFS root where 3D model bodies are materialized (JS prescan + the C++
* lazy-ensure fallback both write `<root>/<lib>.3dshapes/<name>.<ext>` here).
* Boot points every `KICAD*_3DMODEL_DIR` env var at this dir — official-lib
* footprints reference models through vintage-specific vars (KICAD6..10 all
* occur), and KiCad's stock FILENAME_RESOLVER picks up each var it finds — so
* model paths resolve with zero resolver changes.
*/
export const MODELS_3D_ROOT = "/pcbjam/3dmodels";
export const MODELS_3D_ENV_VARS = [
"KISYS3DMOD", // pre-v6 legacy alias, still common in older boards
"KICAD6_3DMODEL_DIR",
"KICAD7_3DMODEL_DIR",
"KICAD8_3DMODEL_DIR",
"KICAD9_3DMODEL_DIR",
"KICAD10_3DMODEL_DIR",
] as const;
export function memfsProjectDir(slug: string): string {
return `${MEMFS_PROJECTS_DIR}/${slug}`;
}

View file

@ -1,6 +1,7 @@
import type { Tool } from "@pcbjam/shared";
import { FILELESS_TOOLS } from "@pcbjam/shared";
import { memfsFilePath, memfsProjectDir } from "./constants";
import { prescanBoardModels } from "./libs/models-bridge";
import { openFileInTool } from "./open-flow";
/**
@ -55,6 +56,16 @@ async function syncProjectToMemfs(win: ToolWindow, opts: DriveOptions): Promise<
const bytes = await opts.fetchBytes(file.path);
fs.writeFile(dest, bytes);
opts.log(`[memfs] wrote ${dest} (${bytes.length} bytes)`);
// 3D models: prefetch every model this board references (R2 → IDB → MEMFS)
// so the 3D viewer's first open resolves locally. Fire-and-forget — project
// open never waits on it; a ref that misses falls back to the C++ per-model
// ensure. No-op unless a model source is installed (bootKicadTool).
if (file.path.endsWith(".kicad_pcb")) {
const text = new TextDecoder().decode(bytes);
void prescanBoardModels(text).catch((e) =>
opts.log(`[3d] prescan failed: ${String(e)}`),
);
}
}
}

View file

@ -0,0 +1,62 @@
import { describe, expect, it } from "vitest";
import { normalizeModelRef, scanModelRefs } from "./models-bridge";
describe("normalizeModelRef", () => {
it("strips any vintage of the model-dir var", () => {
for (const v of ["KICAD6", "KICAD7", "KICAD8", "KICAD9", "KICAD10"]) {
expect(
normalizeModelRef(`\${${v}_3DMODEL_DIR}/Resistor_SMD.3dshapes/R_0201.wrl`),
).toBe("Resistor_SMD.3dshapes/R_0201.wrl");
}
});
it("accepts the paren syntax and the legacy KISYS3DMOD alias", () => {
expect(normalizeModelRef("$(KICAD8_3DMODEL_DIR)/L.3dshapes/m.step")).toBe(
"L.3dshapes/m.step",
);
expect(normalizeModelRef("${KISYS3DMOD}/L.3dshapes/m.wrl")).toBe(
"L.3dshapes/m.wrl",
);
});
it("passes bare relative refs through", () => {
expect(normalizeModelRef("L.3dshapes/m.wrl")).toBe("L.3dshapes/m.wrl");
});
it("rejects refs it cannot serve", () => {
expect(normalizeModelRef("${KIPRJMOD}/libs/3d/m.wrl")).toBeNull(); // project-local
expect(normalizeModelRef("/abs/path/m.wrl")).toBeNull();
expect(normalizeModelRef("kicad_embed://m.wrl")).toBeNull();
expect(normalizeModelRef("")).toBeNull();
expect(normalizeModelRef("${UNCLOSED/m.wrl")).toBeNull();
});
});
describe("scanModelRefs", () => {
it("finds, normalizes and dedupes board model refs", () => {
const board = `
(footprint "Resistor_THT:R_Axial"
(model "\${KICAD10_3DMODEL_DIR}/Resistor_THT.3dshapes/R_Axial.step"
(offset (xyz 0 0 0))))
(footprint "Resistor_THT:R_Axial"
(model "\${KICAD10_3DMODEL_DIR}/Resistor_THT.3dshapes/R_Axial.step"))
(footprint "X:Y"
(model "\${KIPRJMOD}/libs/3d_shapes/custom.wrl"))
(footprint "L:M" (model "\${KICAD8_3DMODEL_DIR}/LED_THT.3dshapes/LED_D5.0mm.wrl"))
`;
expect(scanModelRefs(board).sort()).toEqual([
"LED_THT.3dshapes/LED_D5.0mm.wrl",
"Resistor_THT.3dshapes/R_Axial.step",
]);
});
it("handles escaped quotes inside the path", () => {
expect(
scanModelRefs('(model "${KICAD9_3DMODEL_DIR}/A.3dshapes/we\\"ird.wrl")'),
).toEqual(['A.3dshapes/we"ird.wrl']);
});
it("returns empty for a board with no models", () => {
expect(scanModelRefs("(kicad_pcb (version 20240101))")).toEqual([]);
});
});

View file

@ -0,0 +1,181 @@
import { MODELS_3D_ROOT } from "../constants";
import type { Model3dSource } from "./models-source";
/**
* Glue between 3D model delivery and the running KiCad WASM tool:
*
* - `installModel3dHandler` backs the provider's `kind === "model3d"` requests
* (the C++ lazy fallback in S3D_CACHE::load → PCBJAM_3D::EnsureModelFile asks
* "ensure" for a ref the prescan missed).
* - `prescanBoardModels` scans a board's `(model "…")` refs up front and
* prefetches those bodies (R2 → IDB → MEMFS) so the 3D viewer's first open
* resolves everything locally.
*
* Both paths converge on `ensureModelInMemfs`: fetch the body via the
* `Model3dSource` (IDB-cached, sparse) and write it under `MODELS_3D_ROOT` —
* where boot points every `KICAD*_3DMODEL_DIR` env var, so KiCad's stock
* resolver finds the file with no C++ resolution changes.
*/
/** Progress of a board-model prefetch burst (drives the 3D loading overlay). */
export const MODELS_LOADING_EVENT = "pcbjam:models-loading";
export interface ModelsLoadingDetail {
loading: boolean;
done: number;
total: number;
}
function emitModelsLoading(detail: ModelsLoadingDetail): void {
if (typeof window === "undefined") return;
window.dispatchEvent(new CustomEvent(MODELS_LOADING_EVENT, { detail }));
}
/**
* Normalize a footprint model reference to the source's relative form —
* "${KICAD*_3DMODEL_DIR}/<lib>.3dshapes/<name>.<ext>" (any vintage, `${}` or
* `$()`) → "<lib>.3dshapes/<name>.<ext>". Bare relative refs pass through;
* absolute paths / ${KIPRJMOD} / kicad_embed:// are not ours → null. Mirrors
* pcbjamNormalizeModelRef in kicad/3d-viewer/3d_cache/pcbjam_model_fetch.cpp.
*/
export function normalizeModelRef(raw: string): string | null {
const ref = raw.trim();
if (!ref) return null;
if (ref.startsWith("${") || ref.startsWith("$(")) {
const closing = ref[1] === "{" ? "}" : ")";
const end = ref.indexOf(closing);
if (end < 0) return null;
const v = ref.slice(2, end);
// Any vintage of the model-dir var, plus the pre-v6 legacy alias.
if (!v.includes("3DMODEL_DIR") && v !== "KISYS3DMOD") return null;
return ref.slice(end + 1).replace(/^[/\\]+/, "") || null;
}
if (ref.startsWith("/") || ref.includes("://")) return null;
return ref;
}
/** Every `(model "…")` ref in a KiCad board/footprint s-expr, normalized. */
export function scanModelRefs(sexprText: string): string[] {
const refs = new Set<string>();
const re = /\(\s*model\s+"((?:[^"\\]|\\.)*)"/g;
for (let m = re.exec(sexprText); m; m = re.exec(sexprText)) {
const raw = m[1]!.replace(/\\(.)/g, "$1");
const rel = normalizeModelRef(raw);
if (rel) refs.add(rel);
}
return [...refs];
}
type ModelFS = Pick<EmscriptenFS, "mkdirTree" | "writeFile" | "analyzePath">;
function toolFS(): ModelFS | null {
const fs = (window as ToolWindow).FS;
return fs && typeof fs.writeFile === "function" ? (fs as ModelFS) : null;
}
let installedSource: Model3dSource | null = null;
let installedLog: (msg: string) => void = () => {};
/** Refs already materialized in MEMFS this session (bodies are immutable). */
const written = new Set<string>();
/** In-flight ensures, coalesced per ref (prescan and the C++ fallback race). */
const ensuring = new Map<string, Promise<string | null>>();
/** Wire the model source used by the provider dispatch + prescan. */
export function installModel3dHandler(
source: Model3dSource,
log: (msg: string) => void,
): void {
installedSource = source;
installedLog = log;
}
/** Fetch one model body and write it under MODELS_3D_ROOT. Resolves to the
* absolute MEMFS path when present, null when the source can't serve it. */
export async function ensureModelInMemfs(ref: string): Promise<string | null> {
const dest = `${MODELS_3D_ROOT}/${ref}`;
if (written.has(ref)) return dest;
let p = ensuring.get(ref);
if (!p) {
p = doEnsure(ref, dest).finally(() => ensuring.delete(ref));
ensuring.set(ref, p);
}
return p;
}
async function doEnsure(ref: string, dest: string): Promise<string | null> {
const source = installedSource;
const fs = toolFS();
if (!source || !fs) return null;
if (fs.analyzePath(dest).exists) {
written.add(ref);
return dest;
}
const body = await source.getModelBody(ref);
if (!body) return null;
fs.mkdirTree(dest.slice(0, dest.lastIndexOf("/")));
fs.writeFile(dest, body);
written.add(ref);
installedLog(`[3d] materialized ${ref} (${body.length} bytes)`);
return dest;
}
/**
* Provider dispatch for `kind === "model3d"` (called by installLibsProvider's
* request before any lib-id parsing — the C++ bridge passes an empty lib; the
* ref itself carries the library). Answers with the ABSOLUTE MEMFS path of the
* materialized file — S3D_CACHE loads that path directly, so model delivery
* never depends on env-var expansion inside the wasm runtime.
*/
export async function handleModel3dRequest(
op: string,
arg: string,
): Promise<string | null> {
if (op !== "ensure") return null;
const rel = normalizeModelRef(arg);
if (!rel) return null;
try {
return await ensureModelInMemfs(rel);
} catch (e) {
installedLog(`[3d] ensure failed for ${arg}: ${String(e)}`);
return null;
}
}
/**
* Prefetch every model a board references (fire-and-forget from the project
* sync). Bodies land in IDB + MEMFS before the user opens the 3D viewer in the
* common case; anything still missing falls back to the per-model C++ ensure.
*/
export async function prescanBoardModels(
boardText: string,
concurrency = 6,
): Promise<void> {
if (!installedSource) return;
const refs = scanModelRefs(boardText);
if (!refs.length) return;
const total = refs.length;
let done = 0;
emitModelsLoading({ loading: true, done, total });
installedLog(`[3d] prescan: ${total} model ref(s) on board`);
const started = performance.now();
let idx = 0;
const worker = async (): Promise<void> => {
while (idx < refs.length) {
const ref = refs[idx++]!;
try {
await ensureModelInMemfs(ref);
} catch {
// best-effort: the C++ lazy path (or a later prescan) retries
}
emitModelsLoading({ loading: ++done < total, done, total });
}
};
await Promise.all(
Array.from({ length: Math.min(concurrency, total) }, () => worker()),
);
installedLog(
`[3d] prescan: ${done}/${total} in ${Math.round(performance.now() - started)}ms`,
);
}

View file

@ -0,0 +1,120 @@
import { describe, expect, it } from "vitest";
import { sha256Hex, type SyncManifest } from "@pcbjam/shared";
import { memStore, type LayerStore } from "@pcbjam/sync-client";
import { cdnModelsSource } from "./models-source";
const MANIFEST_URL = "https://cdn.test/libs/kicad-models/9.0.9/manifest.json";
const BASE = "https://cdn.test/libs/kicad-models/9.0.9";
const BLOBS = "https://cdn.test/libs/kicad-models/blobs/sha256";
const enc = new TextEncoder();
const dec = new TextDecoder();
/** Build the CDN layout the way publish-models.ts does: per-lib sparse manifest
* + content-addressed blobs — pinning the format with the REAL hash codec. */
async function fakeModelsCdn() {
const bodies: Record<string, Uint8Array> = {
"model3d/R_Axial.step": enc.encode("STEP R_Axial"),
"model3d/R_Disc.wrl": enc.encode("#VRML V2.0 utf8 R_Disc"),
};
const entries: SyncManifest["entries"] = {};
const blobs = new Map<string, Uint8Array>();
for (const [path, body] of Object.entries(bodies)) {
const hash = await sha256Hex(body);
entries[path] = { hash, size: body.length, mtime: 0 };
blobs.set(hash, body);
}
const libManifest: SyncManifest = { version: 1, entries };
const top = {
schema: 1,
tag: "9.0.9",
libs: [{ id: "Resistor_THT", itemCount: 2 }],
};
let manifestFetches = 0;
let blobFetches = 0;
const json = (obj: unknown) => ({ ok: true, json: async () => obj });
const bin = (bytes: Uint8Array) => ({
ok: true,
arrayBuffer: async () => bytes.buffer,
});
const fetchImpl = (async (url: string) => {
if (url === MANIFEST_URL) return json(top);
if (url === `${BASE}/Resistor_THT/manifest`) {
manifestFetches += 1;
return json(libManifest);
}
if (url.startsWith(`${BLOBS}/`)) {
const blob = blobs.get(url.slice(`${BLOBS}/`.length));
if (blob) {
blobFetches += 1;
return bin(blob);
}
}
return { ok: false, status: 404 };
}) as unknown as typeof fetch;
return {
fetchImpl,
counters: {
get manifestFetches() {
return manifestFetches;
},
get blobFetches() {
return blobFetches;
},
},
};
}
function storeMap() {
const stores = new Map<string, LayerStore>();
return (ns: string): LayerStore => {
let s = stores.get(ns);
if (!s) stores.set(ns, (s = memStore()));
return s;
};
}
describe("cdnModelsSource", () => {
it("fetches exactly the requested body (sparse), then serves from cache", async () => {
const cdn = await fakeModelsCdn();
const src = cdnModelsSource(MANIFEST_URL, {
fetchImpl: cdn.fetchImpl,
storeFactory: storeMap(),
});
const body = await src.getModelBody("Resistor_THT.3dshapes/R_Axial.step");
expect(dec.decode(body!)).toBe("STEP R_Axial");
expect(cdn.counters.blobFetches).toBe(1); // only the asked-for model
await src.getModelBody("Resistor_THT.3dshapes/R_Axial.step");
expect(cdn.counters.blobFetches).toBe(1); // cached
expect(cdn.counters.manifestFetches).toBe(1); // lib opened once
});
it("returns null for unknown models/libs/refs without throwing", async () => {
const cdn = await fakeModelsCdn();
const src = cdnModelsSource(MANIFEST_URL, {
fetchImpl: cdn.fetchImpl,
storeFactory: storeMap(),
});
expect(await src.getModelBody("Resistor_THT.3dshapes/nope.step")).toBeNull();
expect(await src.getModelBody("NoSuchLib.3dshapes/m.wrl")).toBeNull();
expect(await src.getModelBody("not-a-model-ref")).toBeNull();
expect(cdn.counters.blobFetches).toBe(0);
});
it("hasModel answers from the manifest without fetching bodies", async () => {
const cdn = await fakeModelsCdn();
const src = cdnModelsSource(MANIFEST_URL, {
fetchImpl: cdn.fetchImpl,
storeFactory: storeMap(),
});
expect(await src.hasModel("Resistor_THT.3dshapes/R_Disc.wrl")).toBe(true);
expect(await src.hasModel("Resistor_THT.3dshapes/nope.wrl")).toBe(false);
expect(cdn.counters.blobFetches).toBe(0);
});
});

View file

@ -0,0 +1,124 @@
import { SyncStack, type SyncStackOptions } from "@pcbjam/sync-client";
/**
* Read-only source of 3D model bodies (`.wrl` / `.step`), addressed by the
* KiCad-relative ref `"<lib>.3dshapes/<name>.<ext>"` (a footprint's `(model …)`
* path with the `${KICAD*_3DMODEL_DIR}/` prefix stripped).
*
* Unlike symbols/footprints, models are never bulk-synced: each lib is an
* r2-idb-sync **sparse** layer — only the (small) manifest syncs eagerly, and a
* body is fetched exactly when a board references it, then cached in IDB. This
* keeps the client cost proportional to what the user renders, not to the
* ~GB-scale full set (docs/features/3d-models).
*/
export interface Model3dSource {
/** One model body, IDB-cached; null when unknown/unavailable. */
getModelBody(ref: string): Promise<Uint8Array | null>;
/** Whether a ref exists in the set at all (manifest-only, no body fetch). */
hasModel(ref: string): Promise<boolean>;
}
interface CdnModelsManifest {
schema: number;
tag: string;
libs: Array<{ id: string; itemCount?: number }>;
}
/** Split "<lib>.3dshapes/<name>" → { lib, path: "model3d/<name>" }. */
function splitRef(ref: string): { lib: string; path: string } | null {
const i = ref.indexOf(".3dshapes/");
if (i <= 0) return null;
const lib = ref.slice(0, i);
const name = ref.slice(i + ".3dshapes/".length);
if (!name || name.includes("/")) return null;
return { lib, path: `model3d/${name}` };
}
/**
* CDN-backed `Model3dSource`. Layout under the manifest's dir
* (`<cdn>/libs/kicad-models/<tag>/`, see scripts/deploy/publish-models.ts):
* manifest.json top index { schema, tag, libs:[{id,itemCount}] }
* <lib>/manifest per-lib SyncManifest, entries "model3d/<name>.<ext>"
* Bodies are content-addressed and shared across tags, one level up:
* <cdn>/libs/kicad-models/blobs/sha256/<hash>
*/
export function cdnModelsSource(
manifestUrl: string,
opts?: Pick<SyncStackOptions, "fetchImpl" | "storeFactory">,
): Model3dSource {
const baseDir = manifestUrl.replace(/\/[^/]*$/, ""); // …/libs/kicad-models/<tag>
const blobsBase = `${baseDir.replace(/\/[^/]*$/, "")}/blobs/sha256`;
const fetchImpl = opts?.fetchImpl ?? fetch;
let manifestP: Promise<CdnModelsManifest> | null = null;
const loadManifest = (): Promise<CdnModelsManifest> => {
if (!manifestP) {
// Never cache a rejection (mirrors cdn-source.ts): a transient failure
// must not poison every later model read.
manifestP = (async () => {
const r = await fetchImpl(manifestUrl, { cache: "no-store" });
if (!r.ok) throw new Error(`cdn models manifest ${r.status}: ${manifestUrl}`);
return (await r.json()) as CdnModelsManifest;
})().catch((e) => {
manifestP = null;
throw e;
});
}
return manifestP;
};
// One lazily-opened sparse stack per lib (IDB store keyed by namespace, so a
// lib's cached models persist and dedupe across sessions).
const stacks = new Map<string, Promise<SyncStack | null>>();
const openStack = (libId: string): Promise<SyncStack | null> => {
let p = stacks.get(libId);
if (!p) {
p = (async () => {
const m = await loadManifest();
if (!m.libs.some((l) => l.id === libId)) return null; // unknown lib
const stack = new SyncStack({
layers: [
{
namespace: `kicad-models:${m.tag}:${libId}`,
kind: "sparse",
url: `${baseDir}/${encodeURIComponent(libId)}`,
bodyUrlTemplate: `${blobsBase}/{hash}`,
},
],
...opts,
});
await stack.open();
return stack;
})().catch((e) => {
stacks.delete(libId); // a failed open must stay retryable
throw e;
});
stacks.set(libId, p);
}
return p;
};
return {
async getModelBody(ref: string): Promise<Uint8Array | null> {
const split = splitRef(ref);
if (!split) return null;
try {
const stack = await openStack(split.lib);
return stack ? await stack.read(split.path) : null;
} catch {
return null; // missing models render as absent, never break the viewer
}
},
async hasModel(ref: string): Promise<boolean> {
const split = splitRef(ref);
if (!split) return false;
try {
const stack = await openStack(split.lib);
if (!stack) return false;
return (await stack.list()).some((e) => e.path === split.path);
} catch {
return false;
}
},
};
}

View file

@ -1,3 +1,4 @@
import { handleModel3dRequest } from "./models-bridge";
import { libIdFromUri, libUri } from "./uri";
/**
@ -259,6 +260,12 @@ export function installLibsProvider(
let fatResetTimer: ReturnType<typeof setTimeout> | undefined;
const request: KicadLibsRequest = async (op, lib, arg, kind = "symbol") => {
// 3D models are addressed by ref, not lib-table URI (the C++ ensure bridge
// passes an empty lib) — dispatch before the lib-id parse would null it out.
if (kind === "model3d") {
log(`[libs] request op=${op} kind=model3d arg=${arg}`);
return handleModel3dRequest(op, arg);
}
const id = libIdFromUri(lib);
log(`[libs] request op=${op} kind=${kind} lib=${lib} (id=${id}) arg=${arg}`);
if (!id) return null;