perf(deploy): parallelize publish-wasm brotli (q11) across files

brotli q11 over the ~320MB wasm set ran serially on one core (~0.4MB/s),
dominating the release publish job. Compress every to-be-uploaded file
concurrently on the libuv threadpool (compressBytesAsync + UV_THREADPOOL_SIZE
sized to the machine), so wall time drops from sum-of-files to roughly the
largest single file (kicad_editor.wasm): 329s for the full set locally vs
~11.5min of CPU. Upload ordering invariants unchanged: meta.json still last
per tool, registry last overall; moved-tag guard and reuse path untouched.
Also adds BROTLI_PARAM_SIZE_HINT and moves the publish-wasm job to
ubicloud-standard-8 so there is a core per file.

Verified byte-identical CDN layout vs the old script (local driver, pinned
builtAt), blob roundtrip to source sha, reuse + --from-registry modes.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JQ9npuB6uLvkv6gzBV5GPh

.github/workflows/release.yml

@@ -75,10 +75,12 @@ jobs:
       upload_output: true
 
   # 2) Publish the build to the CDN (content-addressed; unchanged tools reuse)
-  #    and write manifest-<tag>.json. Cheap runner — just downloads + uploads.
+  #    and write manifest-<tag>.json. The slow step is brotli-q11 over ~300MB of
+  #    wasm; the script compresses all files in parallel (one core per file), so
+  #    give it enough cores that wall time ≈ the largest single file.
   publish-wasm:
     needs: [meta, build]
-    runs-on: ubuntu-latest
+    runs-on: ubicloud-standard-8
     steps:
       - uses: actions/checkout@v4
         with:

scripts/deploy/lib/cdn-store.mjs

@@ -13,7 +13,14 @@ import {
   writeFileSync,
 } from "node:fs";
 import { execFileSync } from "node:child_process";
-import { gzipSync, brotliCompressSync, constants as zc } from "node:zlib";
+import {
+  brotliCompress,
+  brotliCompressSync,
+  gzip,
+  gzipSync,
+  constants as zc,
+} from "node:zlib";
+import { promisify } from "node:util";
 import { tmpdir } from "node:os";
 import { join, dirname } from "node:path";
 
@@ -25,6 +32,13 @@ export const NO_STORE = "no-store";
 
 export const sha256hex = (buf) => createHash("sha256").update(buf).digest("hex");
 
+const brotliOpts = (bytes, quality) => ({
+  params: {
+    [zc.BROTLI_PARAM_QUALITY]: quality ?? 5,
+    [zc.BROTLI_PARAM_SIZE_HINT]: bytes.length,
+  },
+});
+
 /** Compress bytes; returns { bytes, encoding }. mode: "gzip" | "br" | "none". */
 export function compressBytes(bytes, mode, quality) {
   if (mode === "none") return { bytes, encoding: null };
@@ -32,9 +46,28 @@ export function compressBytes(bytes, mode, quality) {
     return { bytes: gzipSync(bytes, { level: quality ?? 6 }), encoding: "gzip" };
   // brotli — default quality 5 (good ratio, far faster than 11 on 100s of MB).
   return {
-    bytes: brotliCompressSync(bytes, {
+    bytes: brotliCompressSync(bytes, brotliOpts(bytes, quality)),
-      params: { [zc.BROTLI_PARAM_QUALITY]: quality ?? 5 },
+    encoding: "br",
-    }),
+  };
+}
+
+const gzipAsync = promisify(gzip);
+const brotliCompressAsync = promisify(brotliCompress);
+
+/**
+ * Async compressBytes: runs on the libuv threadpool, so N calls awaited
+ * together compress on N cores (set UV_THREADPOOL_SIZE before the first call).
+ * Byte-identical output to compressBytes.
+ */
+export async function compressBytesAsync(bytes, mode, quality) {
+  if (mode === "none") return { bytes, encoding: null };
+  if (mode === "gzip")
+    return {
+      bytes: await gzipAsync(bytes, { level: quality ?? 6 }),
+      encoding: "gzip",
+    };
+  return {
+    bytes: await brotliCompressAsync(bytes, brotliOpts(bytes, quality)),
     encoding: "br",
   };
 }

scripts/deploy/publish-wasm.mjs

@@ -17,9 +17,10 @@
 // needs only CLOUDFLARE_API_TOKEN (+ CLOUDFLARE_ACCOUNT_ID).
 
 import { existsSync, readFileSync } from "node:fs";
+import { availableParallelism } from "node:os";
 import { join } from "node:path";
 import {
-  compressBytes,
+  compressBytesAsync,
   IMMUTABLE,
   makeStore,
   NO_STORE,
@@ -27,6 +28,10 @@ import {
   sha256hex,
 } from "./lib/cdn-store.mjs";
 
+// Compression fans out over the libuv threadpool (compressBytesAsync); size it
+// to the machine BEFORE the first async zlib call or the default of 4 sticks.
+process.env.UV_THREADPOOL_SIZE ||= String(Math.max(4, availableParallelism()));
+
 // --- tools & per-file rules ---------------------------------------------------
 
 // Bundles served to the browser editor. The four editor tools (pcbnew, eeschema,
@@ -134,24 +139,33 @@ function gather(tool, srcDir) {
   });
 }
 
-function putFile(store, key, bytes, name, compress, quality) {
+// Compress a gathered file in place (adds .body/.encoding). Async so the whole
-  const rule = fileRule(name);
+// upload set compresses concurrently on the threadpool — brotli q11 over the
-  let body = bytes;
+// ~300MB wasm set is BY FAR the slow step of a publish, and serially it took
-  let encoding = null;
+// sum-of-files; in parallel it takes roughly the largest single file.
+async function compressFile(f, compress, quality) {
+  const rule = fileRule(f.name);
   if (rule.compress && compress !== "none") {
-    const c = compressBytes(bytes, compress, quality);
+    const c = await compressBytesAsync(f.bytes, compress, quality);
-    body = c.bytes;
+    f.body = c.bytes;
-    encoding = c.encoding;
+    f.encoding = c.encoding;
+  } else {
+    f.body = f.bytes;
+    f.encoding = null;
   }
-  store.put(key, body, {
-    contentType: rule.contentType,
-    contentEncoding: encoding,
-    cacheControl: rule.cacheControl,
-  });
-  return { name, raw: bytes.length, stored: body.length, encoding };
 }
 
-function main() {
+function putFile(store, key, f) {
+  const rule = fileRule(f.name);
+  store.put(key, f.body, {
+    contentType: rule.contentType,
+    contentEncoding: f.encoding,
+    cacheControl: rule.cacheControl,
+  });
+  return { name: f.name, raw: f.bytes.length, stored: f.body.length, encoding: f.encoding };
+}
+
+async function main() {
   const a = parseArgs(process.argv);
   const builtAt = a.builtAt || new Date().toISOString();
   const store = makeStore(a.driver, a);
@@ -191,9 +205,10 @@ function main() {
     `publish-wasm: tag=${a.tag} src=${a.src} driver=${store.kind} compress=${a.compress}`,
   );
 
-  let uploaded = 0;
   let reused = 0;
 
+  // Plan: hash every tool, decide reuse vs upload (moved-tag guard included).
+  const plan = [];
   for (const tool of a.tools) {
     const files = gather(tool, a.src);
     const hash = toolContentHash(files);
@@ -205,8 +220,7 @@ function main() {
       console.log(`  ${tool}: reuse ${ver} (${hash.slice(0, 19)}…)`);
     } else {
       ver = a.tag;
-      const metaKey = `${P}/${tool}/${ver}/meta.json`;
+      const existing = store.getJSON(`${P}/${tool}/${ver}/meta.json`)?.hash ?? null;
-      const existing = store.getJSON(metaKey)?.hash ?? null;
       if (existing && existing !== hash) {
         throw new Error(
           `moved-tag guard: ${P}/${tool}/${ver}/ already holds ${existing} ` +
@@ -214,15 +228,25 @@ function main() {
             `overwrite an immutable folder.`,
         );
       }
-      const sizes = [];
+      plan.push({ tool, files, hash, ver });
-      for (const f of files) {
+      idx[hash] = ver;
-        const key = `${P}/${tool}/${ver}/${f.name}`;
-        sizes.push(putFile(store, key, f.bytes, f.name, a.compress, a.quality));
     }
-      // meta.json LAST — its presence marks the bundle complete.
+    registry.tools[tool] = { version: ver, hash };
+    manifest.tools[tool] = ver;
+  }
+
+  // Compress every to-be-uploaded file concurrently, then upload. Uploads stay
+  // sequential per tool with meta.json LAST — its presence marks the bundle
+  // complete — and the registry is only written after every bundle is.
+  await Promise.all(
+    plan.flatMap(({ files }) => files.map((f) => compressFile(f, a.compress, a.quality))),
+  );
+
+  for (const { tool, files, hash, ver } of plan) {
+    const sizes = files.map((f) => putFile(store, `${P}/${tool}/${ver}/${f.name}`, f));
     putJSON(
       store,
-        metaKey,
+      `${P}/${tool}/${ver}/meta.json`,
       {
         tool,
         ver,
@@ -232,17 +256,13 @@ function main() {
       },
       IMMUTABLE,
     );
-      idx[hash] = ver;
-      uploaded++;
     const tot = sizes.reduce((s, x) => s + x.stored, 0);
     console.log(
       `  ${tool}: UPLOAD ${ver} (${hash.slice(0, 19)}…) ` +
         `${(tot / 1e6).toFixed(1)}MB stored`,
     );
   }
-    registry.tools[tool] = { version: ver, hash };
+  const uploaded = plan.length;
-    manifest.tools[tool] = ver;
-  }
 
   // Browser-facing manifest + convenience pointer, both uncached.
   putJSON(store, `${P}/manifest-${a.tag}.json`, manifest, NO_STORE);
@@ -256,4 +276,4 @@ function main() {
   if (store.kind === "local") console.log(`local layout under: ${a.out}`);
 }
 
-main();
+await main();