From f8ae378645dde17e94e7a9bea13a1b1e51f6b632 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20T=C3=B6rcsv=C3=A1ri?= Date: Tue, 28 Jul 2026 12:56:10 +0200 Subject: [PATCH] perf(deploy): parallelize publish-wasm brotli (q11) across files brotli q11 over the ~320MB wasm set ran serially on one core (~0.4MB/s), dominating the release publish job. Compress every to-be-uploaded file concurrently on the libuv threadpool (compressBytesAsync + UV_THREADPOOL_SIZE sized to the machine), so wall time drops from sum-of-files to roughly the largest single file (kicad_editor.wasm): 329s for the full set locally vs ~11.5min of CPU. Upload ordering invariants unchanged: meta.json still last per tool, registry last overall; moved-tag guard and reuse path untouched. Also adds BROTLI_PARAM_SIZE_HINT and moves the publish-wasm job to ubicloud-standard-8 so there is a core per file. Verified byte-identical CDN layout vs the old script (local driver, pinned builtAt), blob roundtrip to source sha, reuse + --from-registry modes. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JQ9npuB6uLvkv6gzBV5GPh --- .github/workflows/release.yml | 6 +- scripts/deploy/lib/cdn-store.mjs | 41 ++++++++++-- scripts/deploy/publish-wasm.mjs | 106 ++++++++++++++++++------------- 3 files changed, 104 insertions(+), 49 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eabd082..faccde4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -75,10 +75,12 @@ jobs: upload_output: true # 2) Publish the build to the CDN (content-addressed; unchanged tools reuse) - # and write manifest-.json. Cheap runner — just downloads + uploads. + # and write manifest-.json. The slow step is brotli-q11 over ~300MB of + # wasm; the script compresses all files in parallel (one core per file), so + # give it enough cores that wall time ≈ the largest single file. publish-wasm: needs: [meta, build] - runs-on: ubuntu-latest + runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 with: diff --git a/scripts/deploy/lib/cdn-store.mjs b/scripts/deploy/lib/cdn-store.mjs index bafb697..9f55998 100644 --- a/scripts/deploy/lib/cdn-store.mjs +++ b/scripts/deploy/lib/cdn-store.mjs @@ -13,7 +13,14 @@ import { writeFileSync, } from "node:fs"; import { execFileSync } from "node:child_process"; -import { gzipSync, brotliCompressSync, constants as zc } from "node:zlib"; +import { + brotliCompress, + brotliCompressSync, + gzip, + gzipSync, + constants as zc, +} from "node:zlib"; +import { promisify } from "node:util"; import { tmpdir } from "node:os"; import { join, dirname } from "node:path"; @@ -25,6 +32,13 @@ export const NO_STORE = "no-store"; export const sha256hex = (buf) => createHash("sha256").update(buf).digest("hex"); +const brotliOpts = (bytes, quality) => ({ + params: { + [zc.BROTLI_PARAM_QUALITY]: quality ?? 5, + [zc.BROTLI_PARAM_SIZE_HINT]: bytes.length, + }, +}); + /** Compress bytes; returns { bytes, encoding }. mode: "gzip" | "br" | "none". */ export function compressBytes(bytes, mode, quality) { if (mode === "none") return { bytes, encoding: null }; @@ -32,9 +46,28 @@ export function compressBytes(bytes, mode, quality) { return { bytes: gzipSync(bytes, { level: quality ?? 6 }), encoding: "gzip" }; // brotli — default quality 5 (good ratio, far faster than 11 on 100s of MB). return { - bytes: brotliCompressSync(bytes, { - params: { [zc.BROTLI_PARAM_QUALITY]: quality ?? 5 }, - }), + bytes: brotliCompressSync(bytes, brotliOpts(bytes, quality)), + encoding: "br", + }; +} + +const gzipAsync = promisify(gzip); +const brotliCompressAsync = promisify(brotliCompress); + +/** + * Async compressBytes: runs on the libuv threadpool, so N calls awaited + * together compress on N cores (set UV_THREADPOOL_SIZE before the first call). + * Byte-identical output to compressBytes. + */ +export async function compressBytesAsync(bytes, mode, quality) { + if (mode === "none") return { bytes, encoding: null }; + if (mode === "gzip") + return { + bytes: await gzipAsync(bytes, { level: quality ?? 6 }), + encoding: "gzip", + }; + return { + bytes: await brotliCompressAsync(bytes, brotliOpts(bytes, quality)), encoding: "br", }; } diff --git a/scripts/deploy/publish-wasm.mjs b/scripts/deploy/publish-wasm.mjs index 7b92424..b404980 100644 --- a/scripts/deploy/publish-wasm.mjs +++ b/scripts/deploy/publish-wasm.mjs @@ -17,9 +17,10 @@ // needs only CLOUDFLARE_API_TOKEN (+ CLOUDFLARE_ACCOUNT_ID). import { existsSync, readFileSync } from "node:fs"; +import { availableParallelism } from "node:os"; import { join } from "node:path"; import { - compressBytes, + compressBytesAsync, IMMUTABLE, makeStore, NO_STORE, @@ -27,6 +28,10 @@ import { sha256hex, } from "./lib/cdn-store.mjs"; +// Compression fans out over the libuv threadpool (compressBytesAsync); size it +// to the machine BEFORE the first async zlib call or the default of 4 sticks. +process.env.UV_THREADPOOL_SIZE ||= String(Math.max(4, availableParallelism())); + // --- tools & per-file rules --------------------------------------------------- // Bundles served to the browser editor. The four editor tools (pcbnew, eeschema, @@ -134,24 +139,33 @@ function gather(tool, srcDir) { }); } -function putFile(store, key, bytes, name, compress, quality) { - const rule = fileRule(name); - let body = bytes; - let encoding = null; +// Compress a gathered file in place (adds .body/.encoding). Async so the whole +// upload set compresses concurrently on the threadpool — brotli q11 over the +// ~300MB wasm set is BY FAR the slow step of a publish, and serially it took +// sum-of-files; in parallel it takes roughly the largest single file. +async function compressFile(f, compress, quality) { + const rule = fileRule(f.name); if (rule.compress && compress !== "none") { - const c = compressBytes(bytes, compress, quality); - body = c.bytes; - encoding = c.encoding; + const c = await compressBytesAsync(f.bytes, compress, quality); + f.body = c.bytes; + f.encoding = c.encoding; + } else { + f.body = f.bytes; + f.encoding = null; } - store.put(key, body, { - contentType: rule.contentType, - contentEncoding: encoding, - cacheControl: rule.cacheControl, - }); - return { name, raw: bytes.length, stored: body.length, encoding }; } -function main() { +function putFile(store, key, f) { + const rule = fileRule(f.name); + store.put(key, f.body, { + contentType: rule.contentType, + contentEncoding: f.encoding, + cacheControl: rule.cacheControl, + }); + return { name: f.name, raw: f.bytes.length, stored: f.body.length, encoding: f.encoding }; +} + +async function main() { const a = parseArgs(process.argv); const builtAt = a.builtAt || new Date().toISOString(); const store = makeStore(a.driver, a); @@ -191,9 +205,10 @@ function main() { `publish-wasm: tag=${a.tag} src=${a.src} driver=${store.kind} compress=${a.compress}`, ); - let uploaded = 0; let reused = 0; + // Plan: hash every tool, decide reuse vs upload (moved-tag guard included). + const plan = []; for (const tool of a.tools) { const files = gather(tool, a.src); const hash = toolContentHash(files); @@ -205,8 +220,7 @@ function main() { console.log(` ${tool}: reuse ${ver} (${hash.slice(0, 19)}…)`); } else { ver = a.tag; - const metaKey = `${P}/${tool}/${ver}/meta.json`; - const existing = store.getJSON(metaKey)?.hash ?? null; + const existing = store.getJSON(`${P}/${tool}/${ver}/meta.json`)?.hash ?? null; if (existing && existing !== hash) { throw new Error( `moved-tag guard: ${P}/${tool}/${ver}/ already holds ${existing} ` + @@ -214,36 +228,42 @@ function main() { `overwrite an immutable folder.`, ); } - const sizes = []; - for (const f of files) { - const key = `${P}/${tool}/${ver}/${f.name}`; - sizes.push(putFile(store, key, f.bytes, f.name, a.compress, a.quality)); - } - // meta.json LAST — its presence marks the bundle complete. - putJSON( - store, - metaKey, - { - tool, - ver, - hash, - builtAt, - files: Object.fromEntries(files.map((f) => [f.name, "sha256:" + sha256hex(f.bytes)])), - }, - IMMUTABLE, - ); + plan.push({ tool, files, hash, ver }); idx[hash] = ver; - uploaded++; - const tot = sizes.reduce((s, x) => s + x.stored, 0); - console.log( - ` ${tool}: UPLOAD ${ver} (${hash.slice(0, 19)}…) ` + - `${(tot / 1e6).toFixed(1)}MB stored`, - ); } registry.tools[tool] = { version: ver, hash }; manifest.tools[tool] = ver; } + // Compress every to-be-uploaded file concurrently, then upload. Uploads stay + // sequential per tool with meta.json LAST — its presence marks the bundle + // complete — and the registry is only written after every bundle is. + await Promise.all( + plan.flatMap(({ files }) => files.map((f) => compressFile(f, a.compress, a.quality))), + ); + + for (const { tool, files, hash, ver } of plan) { + const sizes = files.map((f) => putFile(store, `${P}/${tool}/${ver}/${f.name}`, f)); + putJSON( + store, + `${P}/${tool}/${ver}/meta.json`, + { + tool, + ver, + hash, + builtAt, + files: Object.fromEntries(files.map((f) => [f.name, "sha256:" + sha256hex(f.bytes)])), + }, + IMMUTABLE, + ); + const tot = sizes.reduce((s, x) => s + x.stored, 0); + console.log( + ` ${tool}: UPLOAD ${ver} (${hash.slice(0, 19)}…) ` + + `${(tot / 1e6).toFixed(1)}MB stored`, + ); + } + const uploaded = plan.length; + // Browser-facing manifest + convenience pointer, both uncached. putJSON(store, `${P}/manifest-${a.tag}.json`, manifest, NO_STORE); putJSON(store, `${P}/manifest-latest.json`, { tag: a.tag }, NO_STORE); @@ -256,4 +276,4 @@ function main() { if (store.kind === "local") console.log(`local layout under: ${a.out}`); } -main(); +await main();