feat(3d): live models CDN — 10.0.3 publish wiring + wrl→step fallback
- kicad-packages3D is STEP-only from the 10.x tags: ensureModelInMemfs falls back from a missing .wrl to the same-stem .step, written and answered under the .step path (the returned path's extension picks the parsing plugin — no C++ awareness). Verified against the live CDN with the gallery demo board (4/4 wrl refs served as step). - upload-models-r2.sh: rclone/S3 bulk upload of a local publish layout (wrangler-per-object can't move ~14k blobs); published 10.0.3 — 105 libs / 7,238 models, 3.4GB raw → 500MB brotli — to pcbjam-cdn. - deploy-demo.yml MODELS_TAG=10.0.3 → build-demo --models-tag → VITE_MODELS_MANIFEST_URL (matches LIB_TAG: model refs come from the footprints at that release). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014AT7gVHRktDYoQ68S4x6A4
This commit is contained in:
parent
6f3f3bcf00
commit
4330c9344e
5 changed files with 177 additions and 8 deletions
5
.github/workflows/deploy-demo.yml
vendored
5
.github/workflows/deploy-demo.yml
vendored
|
|
@ -28,6 +28,10 @@ env:
|
|||
# KiCad library snapshot the demo points at (published once by publish-libs.yml
|
||||
# to libs/kicad/<LIB_TAG>/). Bump when moving to a newer KiCad library release.
|
||||
LIB_TAG: "10.0.3"
|
||||
# kicad-packages3D snapshot the demo's lazy 3D models point at (published once
|
||||
# to libs/kicad-models/<MODELS_TAG>/ — see scripts/deploy/publish-models.ts +
|
||||
# upload-models-r2.sh; docs/features/3d-models). Empty ⇒ 3D models off.
|
||||
MODELS_TAG: "10.0.3"
|
||||
PAGES_PROJECT: pcbjam-demo
|
||||
# MUST be the Pages project's PRODUCTION branch — any other value makes
|
||||
# `wrangler pages deploy` a PREVIEW deploy and demo.pcbjam.com won't update.
|
||||
|
|
@ -94,6 +98,7 @@ jobs:
|
|||
run: >
|
||||
node scripts/deploy/build-demo.mjs --tag "${{ steps.tag.outputs.tag }}"
|
||||
--cdn "$CDN" --lib-tag "$LIB_TAG"
|
||||
${MODELS_TAG:+--models-tag "$MODELS_TAG"}
|
||||
--plausible "${{ vars.PLAUSIBLE_DOMAIN }}"
|
||||
|
||||
# 4) Ensure the Pages project exists (first deploy creates it; no-op after).
|
||||
|
|
|
|||
|
|
@ -44,6 +44,10 @@ function parseArgs(argv) {
|
|||
// KiCad library snapshot tag (published once to libs/kicad/<libTag>/).
|
||||
// Omitted ⇒ the offline built-in example symbols (back-compat).
|
||||
case "--lib-tag": a.libTag = next(); break;
|
||||
// kicad-packages3D snapshot tag (published once to
|
||||
// libs/kicad-models/<modelsTag>/). Omitted ⇒ 3D component models off
|
||||
// (the viewer renders bare boards, exactly as before the feature).
|
||||
case "--models-tag": a.modelsTag = next(); break;
|
||||
case "--landing": a.landing = next(); break;
|
||||
case "--waitlist": a.waitlist = next(); break;
|
||||
case "--plausible": a.plausible = next(); break;
|
||||
|
|
@ -98,6 +102,13 @@ function main() {
|
|||
VITE_LIBS_MANIFEST_URL: `${a.cdn}/libs/kicad/${a.libTag}/manifest.json`,
|
||||
}
|
||||
: { VITE_LIBS_SOURCE: "static" }),
|
||||
// 3D models: lazy per-board sparse fetch from the versioned models CDN
|
||||
// (docs/features/3d-models). Off unless a --models-tag is given.
|
||||
...(a.modelsTag
|
||||
? {
|
||||
VITE_MODELS_MANIFEST_URL: `${a.cdn}/libs/kicad-models/${a.modelsTag}/manifest.json`,
|
||||
}
|
||||
: {}),
|
||||
VITE_YJS_PROVIDER: "broadcastchannel",
|
||||
// Build identity for the version badge. The commit is the GPLv3
|
||||
// corresponding-source pointer (pins the kicad + wxwidgets submodules).
|
||||
|
|
@ -117,6 +128,7 @@ function main() {
|
|||
console.log(` VITE_PROJECT_MANIFEST_URL=${env.VITE_PROJECT_MANIFEST_URL}`);
|
||||
console.log(` VITE_APP_TAG=${env.VITE_APP_TAG} VITE_GIT_SHA=${env.VITE_GIT_SHA || "(none)"}`);
|
||||
console.log(` VITE_LIBS_SOURCE=${env.VITE_LIBS_SOURCE}${env.VITE_LIBS_MANIFEST_URL ? ` (${env.VITE_LIBS_MANIFEST_URL})` : ""}`);
|
||||
console.log(` VITE_MODELS_MANIFEST_URL=${env.VITE_MODELS_MANIFEST_URL ?? "(unset — 3D models off)"}`);
|
||||
console.log(` VITE_LANDING_URL=${env.VITE_LANDING_URL} VITE_WAITLIST_URL=${env.VITE_WAITLIST_URL}`);
|
||||
console.log(` VITE_PLAUSIBLE_DOMAIN=${env.VITE_PLAUSIBLE_DOMAIN || "(off)"}`);
|
||||
|
||||
|
|
|
|||
74
scripts/deploy/upload-models-r2.sh
Executable file
74
scripts/deploy/upload-models-r2.sh
Executable file
|
|
@ -0,0 +1,74 @@
|
|||
#!/usr/bin/env bash
|
||||
# Bulk-upload a publish-models.ts LOCAL layout to the R2 CDN via rclone.
|
||||
#
|
||||
# Why not the cdn-store r2 driver: it shells one `wrangler r2 object put` per
|
||||
# object, fine for the ~900 objects of publish-libs but hopeless for the ~47k
|
||||
# content-addressed model blobs. rclone over R2's S3 API moves them in minutes
|
||||
# with concurrent streams, and per-invocation --header-upload flags reproduce
|
||||
# the exact HTTP metadata the r2 driver would have set (all blobs share one
|
||||
# header set, all manifests another — that's what makes this split possible).
|
||||
#
|
||||
# Prereqs:
|
||||
# 1. `npx tsx scripts/deploy/publish-models.ts --model-tag <tag> \
|
||||
# --models-src <kicad-packages3D checkout> --driver local --out <dir>`
|
||||
# (default brotli — DO NOT pass --compress none for the live CDN)
|
||||
# 2. An R2 API token with S3 auth (dash → R2 → Manage API tokens → object
|
||||
# read+write on the bucket), exported as:
|
||||
# CLOUDFLARE_ACCOUNT_ID (the account hash in the R2 endpoint)
|
||||
# R2_S3_ACCESS_KEY_ID
|
||||
# R2_S3_SECRET_ACCESS_KEY
|
||||
#
|
||||
# Usage:
|
||||
# scripts/deploy/upload-models-r2.sh <local-out-dir> <tag> [bucket]
|
||||
#
|
||||
# Idempotent: rclone copy skips objects whose size already matches (blobs are
|
||||
# content-addressed, so same-key ⇒ same bytes); manifests are tiny re-puts.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
OUT_DIR="${1:?usage: upload-models-r2.sh <local-out-dir> <tag> [bucket]}"
|
||||
TAG="${2:?usage: upload-models-r2.sh <local-out-dir> <tag> [bucket]}"
|
||||
BUCKET="${3:-pcbjam-cdn}"
|
||||
PREFIX="libs/kicad-models"
|
||||
|
||||
: "${CLOUDFLARE_ACCOUNT_ID:?export CLOUDFLARE_ACCOUNT_ID (R2 endpoint account hash)}"
|
||||
: "${R2_S3_ACCESS_KEY_ID:?export R2_S3_ACCESS_KEY_ID (R2 API token, S3 auth)}"
|
||||
: "${R2_S3_SECRET_ACCESS_KEY:?export R2_S3_SECRET_ACCESS_KEY}"
|
||||
|
||||
[ -f "$OUT_DIR/$PREFIX/$TAG/manifest.json" ] \
|
||||
|| { echo "no $PREFIX/$TAG/manifest.json under $OUT_DIR — wrong dir or tag?" >&2; exit 1; }
|
||||
|
||||
# Config-file-less rclone: the :s3: backend picks these up from the environment.
|
||||
export RCLONE_S3_PROVIDER=Cloudflare
|
||||
export RCLONE_S3_ENDPOINT="https://${CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com"
|
||||
export RCLONE_S3_ACCESS_KEY_ID="$R2_S3_ACCESS_KEY_ID"
|
||||
export RCLONE_S3_SECRET_ACCESS_KEY="$R2_S3_SECRET_ACCESS_KEY"
|
||||
export RCLONE_S3_NO_CHECK_BUCKET=true # token may lack bucket-create rights
|
||||
|
||||
DEST=":s3:${BUCKET}/${PREFIX}"
|
||||
IMMUTABLE="public, max-age=31536000, immutable, no-transform"
|
||||
COMMON=(--transfers 32 --checkers 32 --s3-chunk-size 16M --stats 30s --stats-one-line)
|
||||
|
||||
echo "== blobs (brotli, content-addressed, immutable) =="
|
||||
rclone copy "${COMMON[@]}" \
|
||||
--header-upload "Content-Type: application/octet-stream" \
|
||||
--header-upload "Content-Encoding: br" \
|
||||
--header-upload "Cache-Control: ${IMMUTABLE}" \
|
||||
--exclude "registry.json" \
|
||||
"$OUT_DIR/$PREFIX/blobs" "$DEST/blobs"
|
||||
|
||||
echo "== per-lib + top manifests (json, immutable) =="
|
||||
rclone copy "${COMMON[@]}" \
|
||||
--header-upload "Content-Type: application/json" \
|
||||
--header-upload "Cache-Control: ${IMMUTABLE}" \
|
||||
"$OUT_DIR/$PREFIX/$TAG" "$DEST/$TAG"
|
||||
|
||||
echo "== blob registry (publish-time index, no-store) =="
|
||||
rclone copyto \
|
||||
--header-upload "Content-Type: application/json" \
|
||||
--header-upload "Cache-Control: no-store" \
|
||||
"$OUT_DIR/$PREFIX/blobs/registry.json" "$DEST/blobs/registry.json"
|
||||
|
||||
echo "== verify =="
|
||||
rclone lsl "$DEST/$TAG/manifest.json"
|
||||
echo "done: https://cdn.pcbjam.com/${PREFIX}/${TAG}/manifest.json"
|
||||
|
|
@ -1,5 +1,11 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { normalizeModelRef, scanModelRefs } from "./models-bridge";
|
||||
import {
|
||||
ensureModelInMemfs,
|
||||
installModel3dHandler,
|
||||
normalizeModelRef,
|
||||
scanModelRefs,
|
||||
} from "./models-bridge";
|
||||
import type { Model3dSource } from "./models-source";
|
||||
|
||||
describe("normalizeModelRef", () => {
|
||||
it("strips any vintage of the model-dir var", () => {
|
||||
|
|
@ -32,6 +38,49 @@ describe("normalizeModelRef", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("ensureModelInMemfs format fallback", () => {
|
||||
function installFakes(available: (ref: string) => boolean) {
|
||||
const files = new Map<string, Uint8Array>();
|
||||
const fs = {
|
||||
mkdirTree: () => {},
|
||||
writeFile: (p: string, b: Uint8Array) => void files.set(p, b),
|
||||
analyzePath: (p: string) => ({ exists: files.has(p) }),
|
||||
};
|
||||
(globalThis as unknown as { window: unknown }).window ??= globalThis;
|
||||
(globalThis as unknown as { FS: unknown }).FS = fs;
|
||||
const source: Model3dSource = {
|
||||
getModelBody: async (ref) =>
|
||||
available(ref) ? new TextEncoder().encode(`body:${ref}`) : null,
|
||||
hasModel: async (ref) => available(ref),
|
||||
};
|
||||
installModel3dHandler(source, () => {});
|
||||
return files;
|
||||
}
|
||||
|
||||
it("serves a .wrl ask from the same-stem .step, under the .step path", async () => {
|
||||
// kicad-packages3D is STEP-only from 10.x — old boards still ask for .wrl.
|
||||
const files = installFakes((r) => r.endsWith(".step"));
|
||||
const dest = await ensureModelInMemfs("FallbackLibA.3dshapes/M1.wrl");
|
||||
// Returned (and written) under the SUBSTITUTED extension: the path picks
|
||||
// the parsing plugin, so the .step body must dispatch to oce, not vrml.
|
||||
expect(dest).toBe("/pcbjam/3dmodels/FallbackLibA.3dshapes/M1.step");
|
||||
expect(files.has("/pcbjam/3dmodels/FallbackLibA.3dshapes/M1.step")).toBe(true);
|
||||
expect(files.has("/pcbjam/3dmodels/FallbackLibA.3dshapes/M1.wrl")).toBe(false);
|
||||
});
|
||||
|
||||
it("prefers the exact ref when it exists", async () => {
|
||||
const files = installFakes(() => true);
|
||||
const dest = await ensureModelInMemfs("FallbackLibB.3dshapes/M2.wrl");
|
||||
expect(dest).toBe("/pcbjam/3dmodels/FallbackLibB.3dshapes/M2.wrl");
|
||||
expect(files.has("/pcbjam/3dmodels/FallbackLibB.3dshapes/M2.wrl")).toBe(true);
|
||||
});
|
||||
|
||||
it("resolves null when no format of the model exists", async () => {
|
||||
installFakes(() => false);
|
||||
expect(await ensureModelInMemfs("FallbackLibC.3dshapes/M3.wrl")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("scanModelRefs", () => {
|
||||
it("finds, normalizes and dedupes board model refs", () => {
|
||||
const board = `
|
||||
|
|
|
|||
|
|
@ -102,6 +102,29 @@ export async function ensureModelInMemfs(ref: string): Promise<string | null> {
|
|||
return p;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format fallback: kicad-packages3D dropped `.wrl` at the 10.x generation
|
||||
* (STEP-only), but boards authored with KiCad ≤9 still reference `.wrl`. Try
|
||||
* the exact ref, then the same stem in the surviving formats. The substituted
|
||||
* file is written (and returned) under ITS OWN extension — the returned path's
|
||||
* extension is what picks the parsing plugin, so a `.wrl` ask served by a
|
||||
* `.step` body dispatches to oce, not vrml. No C++ involvement.
|
||||
*/
|
||||
const FALLBACK_EXTS: Record<string, string[]> = {
|
||||
".wrl": [".step", ".stp"],
|
||||
".wrz": [".step", ".stp"],
|
||||
".step": [".wrl"],
|
||||
".stp": [".wrl"],
|
||||
};
|
||||
|
||||
function refCandidates(ref: string): string[] {
|
||||
const dot = ref.lastIndexOf(".");
|
||||
if (dot < 0) return [ref];
|
||||
const ext = ref.slice(dot).toLowerCase();
|
||||
const stem = ref.slice(0, dot);
|
||||
return [ref, ...(FALLBACK_EXTS[ext] ?? []).map((e) => `${stem}${e}`)];
|
||||
}
|
||||
|
||||
async function doEnsure(ref: string, dest: string): Promise<string | null> {
|
||||
const source = installedSource;
|
||||
const fs = toolFS();
|
||||
|
|
@ -110,13 +133,19 @@ async function doEnsure(ref: string, dest: string): Promise<string | null> {
|
|||
written.add(ref);
|
||||
return dest;
|
||||
}
|
||||
const body = await source.getModelBody(ref);
|
||||
if (!body) return null;
|
||||
fs.mkdirTree(dest.slice(0, dest.lastIndexOf("/")));
|
||||
fs.writeFile(dest, body);
|
||||
for (const candidate of refCandidates(ref)) {
|
||||
const body = await source.getModelBody(candidate);
|
||||
if (!body) continue;
|
||||
const target = `${MODELS_3D_ROOT}/${candidate}`;
|
||||
fs.mkdirTree(target.slice(0, target.lastIndexOf("/")));
|
||||
fs.writeFile(target, body);
|
||||
written.add(ref);
|
||||
installedLog(`[3d] materialized ${ref} (${body.length} bytes)`);
|
||||
return dest;
|
||||
installedLog(
|
||||
`[3d] materialized ${candidate}${candidate === ref ? "" : ` (for ${ref})`} (${body.length} bytes)`,
|
||||
);
|
||||
return target;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Reference in a new issue