From e94f5b31be173a7b74f26a4e7b0aba8ac3f1580a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20T=C3=B6rcsv=C3=A1ri?= Date: Fri, 19 Jun 2026 14:49:17 +0200 Subject: [PATCH] ci: cache KiCad WASM build output to skip Docker on unchanged sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an output-cache layer to ci-ubicloud.yml so a build whose inputs are unchanged (typical when only tests/ or web/ change) skips the in-Docker compile + the 1-2h host asyncify/wasm-opt chain — the bulk of the ~1h52m run. On a hit, output/ and the GAL sysroot headers are restored from cache and the deps restore/seed, build.sh, and sysroot export are all gated on a miss, so Docker is never started. Key: kwasm--bin-k-wx-sc-e. The "sc" hash (scripts/deploy/wasm-cache-hash.mjs) folds in just the build-logic files that shape the wasm bytes (host post-processing, per-tool compile recipes, scripts/deps, docker/Dockerfile+build.sh) plus itself; it is content-based, order-independent, and identical on macOS/CI. *.wasm.debug.wasm (5+ GB, unused by tests) is excluded -> ~0.5 GB entry on Ubicloud's transparent 30 GB/repo/week cache. Invalidation: - bump .ci-cache-epoch in a commit for durable busts (inputs the sc-hash can't see: base-image/apt drift, a bad cache); - [no-cache] or [rebuild-wasm] in the commit message / PR title, or workflow_dispatch no_cache=true, for a one-off rebuild (split restore/save so the bypass still refreshes the entry). Co-Authored-By: Claude Opus 4.8 (1M context) --- .ci-cache-epoch | 1 + .github/workflows/ci-ubicloud.yml | 88 +++++++++++++++-- scripts/deploy/wasm-cache-hash.mjs | 149 +++++++++++++++++++++++++++++ 3 files changed, 231 insertions(+), 7 deletions(-) create mode 100644 .ci-cache-epoch create mode 100644 scripts/deploy/wasm-cache-hash.mjs diff --git a/.ci-cache-epoch b/.ci-cache-epoch new file mode 100644 index 0000000..573541a --- /dev/null +++ b/.ci-cache-epoch @@ -0,0 +1 @@ +0 diff --git a/.github/workflows/ci-ubicloud.yml b/.github/workflows/ci-ubicloud.yml index 458c1d4..37c974b 100644 --- a/.github/workflows/ci-ubicloud.yml +++ b/.github/workflows/ci-ubicloud.yml @@ -21,6 +21,7 @@ on: workflow_dispatch: inputs: binaryen_version: { description: "Binaryen version for the host asyncify step", required: false, default: "130" } + no_cache: { description: "Bypass the KiCad WASM output cache (force a full rebuild this run)", type: boolean, required: false, default: false } concurrency: # Per-ref: a new push to a PR (or to main) queues behind its own in-flight run @@ -72,6 +73,60 @@ jobs: - uses: actions/setup-node@v4 with: { node-version: 20 } + # --- KiCad WASM output cache ------------------------------------------- + # The expensive work is `./docker/build.sh all` below: the in-Docker compile + # of all tools + the 1-2h host-side asyncify/wasm-opt chain. Its output bytes + # are fixed by the kicad + wxwidgets submodule SHAs (sources), the Binaryen + # version + opt-level (job env), and the build-logic files hashed by + # scripts/deploy/wasm-cache-hash.mjs (the "sc" hash). When all match, the + # artifacts are bit-identical — so we cache output/ (+ the sysroot headers the + # host GAL build needs) and skip Docker entirely on a hit: build.sh, the deps + # restore/seed, and the sysroot export are all gated on a miss below. + # + # Routine edits under scripts/ do NOT auto-bust this — only the subset listed + # in wasm-cache-hash.mjs does. For inputs the sc-hash can't see (base-image / + # apt drift, a bad cache), bump .ci-cache-epoch in a commit; for a one-off + # rebuild, put [no-cache] or [rebuild-wasm] in the commit message / PR title. + # + # Backend: Ubicloud's transparent cache (NOT GitHub's — gh's cache API reads + # 0), 30 GB/repo/week, LRU + 7-day eviction. The *.wasm.debug.wasm files + # (5+ GB, unused by the tests) are excluded to keep the entry ~0.5 GB. + - name: Compute build inputs + id: keys + run: | + echo "kicad=$(git -C kicad rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "wx=$(git -C wxwidgets rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "sc=$(node scripts/deploy/wasm-cache-hash.mjs)" >> "$GITHUB_OUTPUT" + echo "epoch=$(cat .ci-cache-epoch 2>/dev/null || echo 0)" >> "$GITHUB_OUTPUT" + + - name: Cache control (commit message / dispatch) + id: cachectl + env: + HEAD_MSG: ${{ github.event.head_commit.message }} + PR_TITLE: ${{ github.event.pull_request.title }} + DISPATCH_NOCACHE: ${{ github.event.inputs.no_cache }} + run: | + SKIP=false + if printf '%s\n%s' "$HEAD_MSG" "$PR_TITLE" | grep -qiE '\[(no-cache|rebuild-wasm)\]'; then SKIP=true; fi + [ "$DISPATCH_NOCACHE" = "true" ] && SKIP=true + echo "skip=$SKIP" >> "$GITHUB_OUTPUT" + echo "KiCad WASM output-cache restore skip=$SKIP" + + - name: Restore KiCad WASM output cache + id: output-cache + if: steps.cachectl.outputs.skip != 'true' + uses: actions/cache/restore@v4 + with: + path: | + output/*.js + output/*.wasm + output/*.wasm.map + output/*.worker.js + output/images.tar.gz + build-wasm/sysroot/include + !output/*.wasm.debug.wasm + key: kwasm-${{ runner.os }}-bin${{ env.BINARYEN_VERSION }}${{ env.BINARYEN_OPT_LEVEL }}-k${{ steps.keys.outputs.kicad }}-wx${{ steps.keys.outputs.wx }}-sc${{ steps.keys.outputs.sc }}-e${{ steps.keys.outputs.epoch }} + # Cache the ~10-min --build-deps output (boost/cairo/occ/... sysroot + stamps, # which live in the kicad-build-cache docker volume). Key on the deps inputs # only — deps don't depend on the kicad/wx submodule SHAs, so it stays warm @@ -79,13 +134,14 @@ jobs: # check_stamp short-circuits; on a miss the deps build and we tar them out. - name: Restore deps cache id: deps-cache + if: steps.output-cache.outputs.cache-hit != 'true' uses: actions/cache@v4 with: path: deps-cache key: deps-${{ runner.os }}-${{ hashFiles('scripts/deps/**','scripts/common/versions.sh','scripts/common/functions.sh','scripts/common/env.sh','docker/Dockerfile','docker/docker-compose.yml') }} - name: Seed deps volume from cache - if: steps.deps-cache.outputs.cache-hit == 'true' + if: steps.output-cache.outputs.cache-hit != 'true' && steps.deps-cache.outputs.cache-hit == 'true' run: | docker volume create kicad-wasm-ci_kicad-build-cache docker run --rm -v kicad-wasm-ci_kicad-build-cache:/bw -v "$PWD/deps-cache":/cache \ @@ -97,6 +153,7 @@ jobs: # 'all' = pcbnew eeschema calculator pl_editor symbol_editor gerbview, # built in sequence. --build-deps short-circuits on a cache hit. - name: Build all KiCad tools WASM with Binaryen ${{ env.BINARYEN_VERSION }} + if: steps.output-cache.outputs.cache-hit != 'true' run: | # Lift the docker-compose dev-Mac caps and pipeline the host-side # asyncify pass with the next tool's container compile — same tuning @@ -121,7 +178,7 @@ jobs: # On a cache miss, tar the freshly-built deps (sysroot + stamps) out of the # volume so actions/cache saves them (post-job) under the key above. - name: Package deps for cache - if: steps.deps-cache.outputs.cache-hit != 'true' + if: steps.output-cache.outputs.cache-hit != 'true' && steps.deps-cache.outputs.cache-hit != 'true' run: | mkdir -p deps-cache docker run --rm -v kicad-wasm-ci_kicad-build-cache:/bw -v "$PWD/deps-cache":/cache \ @@ -131,16 +188,12 @@ jobs: # but uncached) keyed on the wx submodule SHA + the build script/versions. # On a hit, touch the tree so make treats the restored objects as current # (same SHA ⇒ identical sources), so build-wx-wasm.sh just relinks fast. - - name: wx submodule sha - id: wxsha - run: echo "sha=$(git -C wxwidgets rev-parse HEAD)" >> "$GITHUB_OUTPUT" - - name: Restore wx build cache id: wx-cache uses: actions/cache@v4 with: path: build-wasm/wxwidgets - key: wx-${{ runner.os }}-${{ steps.wxsha.outputs.sha }}-${{ hashFiles('scripts/build-wx-wasm.sh','scripts/common/versions.sh') }} + key: wx-${{ runner.os }}-${{ steps.keys.outputs.wx }}-${{ hashFiles('scripts/build-wx-wasm.sh','scripts/common/versions.sh') }} - name: Mark restored wx objects current if: steps.wx-cache.outputs.cache-hit == 'true' @@ -161,6 +214,7 @@ jobs: # "boost/ptr_container/ptr_vector.hpp not found"). Its Makefile only # needs $(SYSROOT)/include, so copy just the headers out of the volume. - name: Expose docker sysroot headers to host builds + if: steps.output-cache.outputs.cache-hit != 'true' run: | VOL=kicad-wasm-ci_kicad-build-cache echo "sysroot volume: $VOL" @@ -169,6 +223,26 @@ jobs: sh -c 'cp -r /bw/sysroot/include /host/' sudo chown -R "$(id -u):$(id -g)" build-wasm/sysroot + # Persist the freshly built KiCad WASM artifacts (+ the sysroot headers the + # GAL build needs) for future runs. Split save (not a combined cache action) + # so the [no-cache]/dispatch bypass can skip the restore yet still refresh + # here. Runs on a miss OR a bypass (cache-hit != 'true'); placed before the + # tests so artifacts are cached regardless of test outcome. No-op if the key + # already exists. + - name: Save KiCad WASM output cache + if: steps.output-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: | + output/*.js + output/*.wasm + output/*.wasm.map + output/*.worker.js + output/images.tar.gz + build-wasm/sysroot/include + !output/*.wasm.debug.wasm + key: kwasm-${{ runner.os }}-bin${{ env.BINARYEN_VERSION }}${{ env.BINARYEN_OPT_LEVEL }}-k${{ steps.keys.outputs.kicad }}-wx${{ steps.keys.outputs.wx }}-sc${{ steps.keys.outputs.sc }}-e${{ steps.keys.outputs.epoch }} + # gal_webgl_test.{js,wasm} are gitignored build artifacts with their own # build script — without this, all 29 gal-webgl scenarios time out on # galTest.isReady() because the page 404s the wasm (run 27359020746). diff --git a/scripts/deploy/wasm-cache-hash.mjs b/scripts/deploy/wasm-cache-hash.mjs new file mode 100644 index 0000000..4447ffc --- /dev/null +++ b/scripts/deploy/wasm-cache-hash.mjs @@ -0,0 +1,149 @@ +#!/usr/bin/env node +// Computes the "sc" (source-content) hash for the KiCad-WASM output cache key +// in .github/workflows/ci-ubicloud.yml. +// +// The cache key is: +// kwasm--bin-k-wx-sc-e +// +// The kicad/wx submodule SHAs already capture the *sources*. This hash captures +// the *build logic* that shapes the wasm bytes but lives outside those +// submodules — the asyncify/finalize/dyncall/wasm-opt host steps, the per-tool +// compile scripts, the dependency builds, and the Docker toolchain. Those were +// deliberately dropped from the key's hashFiles() (so routine script edits don't +// trigger a 1-2h rebuild); folding the *output-determining* subset back in here +// keeps the cache correct while leaving the rest under the manual .ci-cache-epoch +// / [no-cache] controls. +// +// The hash is content-based and order-independent: it builds a manifest of +// ` ` lines, sorts them, and hashes the +// manifest. Identical on macOS (dev) and Linux (CI) given a normal LF checkout, +// so you can predict cache hits locally. +// +// This script hashes ITSELF (and therefore the INPUTS list below) too, so +// editing it busts the cache — intended: a change here means the set of +// cache-invalidating inputs changed. +// +// Usage: +// node scripts/deploy/wasm-cache-hash.mjs # full hex sha256 -> stdout +// node scripts/deploy/wasm-cache-hash.mjs --short # 16-char prefix +// node scripts/deploy/wasm-cache-hash.mjs --short=12 # N-char prefix +// node scripts/deploy/wasm-cache-hash.mjs --manifest # per-file lines + total (stderr), hash on stdout +// +// In CI (the `keys` step, after checkout + setup-node): +// echo "sc=$(node scripts/deploy/wasm-cache-hash.mjs)" >> "$GITHUB_OUTPUT" +// +// MAINTENANCE: to add or remove inputs, edit INPUTS below — that is the only +// place the set is defined. + +import { createHash } from "node:crypto"; +import { readFileSync, readdirSync, existsSync, statSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { join, relative, sep } from "node:path"; + +// --- The inputs that determine the cached wasm bytes ------------------------- +// Each entry is one of: +// { file: "" } a single file (must exist) +// { dir: "" } every file under the dir, recursive +// { dir: "", match: RE } files under the dir whose BASENAME matches RE, recursive +// Paths are POSIX, relative to the repo root. This script always adds itself. +const INPUTS = [ + // Host-side post-processing — these directly shape the final wasm bytes. + { file: "scripts/common/apply-asyncify.sh" }, + { file: "scripts/common/apply-finalize.sh" }, + { file: "scripts/common/inject-dyncall-shims.sh" }, + { file: "scripts/common/get-wasm-opt.sh" }, + + // Per-tool compile recipes (compile flags / emcc link options). + { dir: "scripts/kicad", match: /^build-.*\.sh$/ }, + + // Dependency builds (boost/cairo/occ/... — the sysroot the wasm links against). + { dir: "scripts/deps" }, + + // Docker toolchain (base image, emsdk, build driver). + { file: "docker/Dockerfile" }, + { file: "docker/build.sh" }, +]; + +// --- helpers ----------------------------------------------------------------- +const ROOT = fileURLToPath(new URL("../..", import.meta.url)); // scripts/deploy -> repo root +const toPosix = (p) => p.split(sep).join("/"); +const sha256hex = (buf) => createHash("sha256").update(buf).digest("hex"); + +// Recursively collect repo-relative file paths under an absolute dir. Skips +// dotfiles/dot-dirs (no hidden files are intended inputs) and follows the +// optional basename matcher. +function walk(absDir, match) { + const out = []; + for (const ent of readdirSync(absDir, { withFileTypes: true })) { + if (ent.name.startsWith(".")) continue; + const abs = join(absDir, ent.name); + if (ent.isDirectory()) { + out.push(...walk(abs, match)); + } else if (ent.isFile()) { + if (match && !match.test(ent.name)) continue; + out.push(toPosix(relative(ROOT, abs))); + } + } + return out; +} + +// --- resolve the file set ---------------------------------------------------- +const files = new Set(); + +for (const entry of INPUTS) { + if (entry.file) { + const abs = join(ROOT, entry.file); + if (!existsSync(abs) || !statSync(abs).isFile()) { + // Hard error: a listed file vanished (renamed/moved). Silently dropping it + // would weaken the key and serve a stale cache. + console.error(`wasm-cache-hash: required input missing: ${entry.file}`); + process.exit(1); + } + files.add(toPosix(entry.file)); + } else if (entry.dir) { + const abs = join(ROOT, entry.dir); + if (!existsSync(abs) || !statSync(abs).isDirectory()) { + console.error(`wasm-cache-hash: required input dir missing: ${entry.dir}`); + process.exit(1); + } + const found = walk(abs, entry.match); + if (found.length === 0) { + // Non-fatal: a dir/matcher that yields nothing is suspicious but + // deterministic. Warn so a bad matcher doesn't go unnoticed. + console.error( + `wasm-cache-hash: warning: no files for ${entry.dir}` + + (entry.match ? ` matching ${entry.match}` : "") + ); + } + for (const f of found) files.add(f); + } +} + +// Always include this script (and thus the INPUTS list). +files.add(toPosix(relative(ROOT, fileURLToPath(import.meta.url)))); + +// --- build the manifest and hash it ------------------------------------------ +const manifest = [...files] + .sort() + .map((rel) => `${sha256hex(readFileSync(join(ROOT, rel)))} ${rel}`) + .join("\n"); + +const hash = sha256hex(Buffer.from(manifest, "utf8")); + +// --- output ------------------------------------------------------------------ +const args = process.argv.slice(2); +if (args.includes("--help") || args.includes("-h")) { + console.error(readFileSync(fileURLToPath(import.meta.url), "utf8").split("\n\n")[1]); + process.exit(0); +} +if (args.includes("--manifest")) { + // Manifest to stderr so stdout stays a clean, capturable hash. + console.error(manifest); + console.error(`-- ${files.size} files --`); +} +const shortArg = args.find((a) => a === "--short" || a.startsWith("--short=")); +const out = shortArg + ? hash.slice(0, Number(shortArg.split("=")[1]) || 16) + : hash; + +process.stdout.write(out + "\n");