From 348596c515d2fb8259f2e5fd950d6b280d8ab0dd Mon Sep 17 00:00:00 2001 From: Viktor Vaczi Date: Thu, 2 Jul 2026 10:54:14 +0200 Subject: [PATCH] =?UTF-8?q?perf(build):=20use=20all=20cores=20in=20host=20?= =?UTF-8?q?builds=20=E2=80=94=20wx=20lib,=20test=20apps,=20gal-webgl?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit env.sh exports a docker-safe JOBS=1 default, and every host build silently inherited it (the docker kicad path dodges it by passing -j explicitly): - build-wx-wasm.sh: the intended "${JOBS:-nproc}" fallback sat BELOW the env.sh source, so it was dead code — full wx builds ran make -j1 on the Mac and on CI. Compute the all-cores default before env.sh instead (explicit JOBS/PARALLEL_JOBS still wins); also -j the PCRE pre-build. - build-wasm-test.sh: same default fix, plus fan the post-link hoist+asyncify loop out across JOBS with xargs -P. Per-app wasm-opt can't feed many cores (small modules), so serial stays ~4min even with BINARYEN_CORES=16; fanning across the 74 independent apps is what scales. Safe: apply-asyncify is in-place per wasm, injector tmp is per-js, HOIST_WASMOPT resolved once up front. - build-gal-webgl-test.sh: make had no -j at all. Measured on a 16-core M4 Max, clean test-app build: 15m36s (-j1) → 2m08s, peak RAM 10.7 GB summed across all build processes (64 GB machine; CI runners have 120 GB). Core counts are always derived from nproc at runtime — nothing hardcoded, CI workers differ. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XeiSRRScdaox5jBueJNcyG --- scripts/build-gal-webgl-test.sh | 8 ++++++-- scripts/build-wasm-test.sh | 31 +++++++++++++++++++++++-------- scripts/build-wx-wasm.sh | 13 +++++++++---- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/scripts/build-gal-webgl-test.sh b/scripts/build-gal-webgl-test.sh index e0aca78..6c1f16c 100755 --- a/scripts/build-gal-webgl-test.sh +++ b/scripts/build-gal-webgl-test.sh @@ -21,6 +21,10 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Default to all cores BEFORE env.sh (its docker-safe JOBS=1 default would stick +# otherwise); an explicit JOBS/PARALLEL_JOBS from the caller still wins. +JOBS="${JOBS:-$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)}" + # Source common environment (sets up local emsdk) QUIET=1 source "$SCRIPT_DIR/common/env.sh" @@ -76,9 +80,9 @@ python3 generate_shaders.py echo "" echo "Building..." if [ "$DEBUG_BUILD" = "1" ]; then - make DEBUG=1 + make -j"${JOBS:-1}" DEBUG=1 else - make + make -j"${JOBS:-1}" fi echo "" diff --git a/scripts/build-wasm-test.sh b/scripts/build-wasm-test.sh index 0a9d0d9..501403b 100755 --- a/scripts/build-wasm-test.sh +++ b/scripts/build-wasm-test.sh @@ -3,6 +3,10 @@ # Redirect all output to a log file (re-execs script with redirection) source "$(dirname "$0")/common/logging.sh" +# Default to all cores BEFORE env.sh (its docker-safe JOBS=1 default would stick +# otherwise); an explicit JOBS/PARALLEL_JOBS from the caller still wins. Measured +# 2026-07-02 (16-core M4 Max): clean build 15m36s at -j1 → ~2min, peak RAM ~11 GB. +JOBS="${JOBS:-$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)}" source "$(dirname "$0")/common/env.sh" # This script creates library symlinks and builds the test apps # @@ -136,19 +140,30 @@ fi # so re-running it here is safe. The .wasm gets post-link hoist + asyncify first. _eh_restore_wasmopt; trap - EXIT echo "" -echo "=== Post-link --hoist-cpp-catches + --asyncify ===" -while IFS= read -r w; do - "$SCRIPT_DIR/common/apply-asyncify.sh" --no-removelist "$w" - js="${w%.wasm}.js" - if [ -f "$js" ]; then - ( cd "$(dirname "$js")" && "$SCRIPT_DIR/common/inject-dyncall-shims.sh" "$(basename "$js")" ) - fi +echo "=== Post-link --hoist-cpp-catches + --asyncify (${JOBS:-1}-wide) ===" +# The apps are independent here too (apply-asyncify rewrites each wasm in place; the shim +# injector's temp file is per-js), so fan the post-link out across JOBS like the make phase. +# This dominates the build: per-app wasm-opt can't feed many cores (small modules, serial +# parse/write), so parallelism must come from running apps side by side — serial 3m52s even +# with BINARYEN_CORES=16, vs 2m08s fanned out at ~11 GB peak RAM. HOIST_WASMOPT is resolved +# once above, so workers skip the binaryen ninja check. xargs fails the build (exit 123) if +# any app's post-link fails. +export SCRIPT_DIR # Match EVERY freshly-linked app wasm, not just standalone/*/*_test.wasm: the main demo # (apps/minimal_test.wasm) is at the apps/ root, and the coroutine-pthread repros + wxpt app # are *_repro*.wasm / *_wxpt.wasm. The old '*_test.wasm under standalone' filter silently # skipped all of those, so under native wasm-EH they never got hoist+asyncify and crashed at # runtime with "asyncify_start_unwind not found". -done < <(find "$WASM_APP_DIR" -name '*.wasm' -newer "$EH_MARKER") +find "$WASM_APP_DIR" -name '*.wasm' -newer "$EH_MARKER" -print0 \ + | xargs -0 -n1 -P "${JOBS:-1}" bash -c ' + set -eo pipefail + w="$1" + "$SCRIPT_DIR/common/apply-asyncify.sh" --no-removelist "$w" + js="${w%.wasm}.js" + if [ -f "$js" ]; then + ( cd "$(dirname "$js")" && "$SCRIPT_DIR/common/inject-dyncall-shims.sh" "$(basename "$js")" ) + fi +' _ rm -f "$EH_MARKER" echo "" diff --git a/scripts/build-wx-wasm.sh b/scripts/build-wx-wasm.sh index 153374e..0c438f0 100755 --- a/scripts/build-wx-wasm.sh +++ b/scripts/build-wx-wasm.sh @@ -5,6 +5,13 @@ # Redirect all output to a log file (re-execs script with redirection) source "$(dirname "$0")/common/logging.sh" +# Default to all cores BEFORE sourcing env.sh — env.sh exports a docker-safe +# JOBS=1 when unset, which would make the "${JOBS:-nproc}" fallback below it +# dead code (this build ran make -j1 everywhere for that reason). An explicit +# JOBS/PARALLEL_JOBS from the caller still wins (e.g. the docker pipeline's -j +# via build-kicad-target.sh). +JOBS="${JOBS:-$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)}" + # Source common environment (sets up local emsdk) source "$(dirname "$0")/common/env.sh" # Build-progress markers (parsed by scripts/build-monitor.sh). @@ -53,9 +60,6 @@ export CONFIG_SHELL="$SCRIPT_DIR/config/config-sub-wrapper.sh" # Disable autom4te cache to keep submodules clean export AUTOM4TE="$SCRIPT_DIR/config/autom4te-wrapper.sh" -# Use JOBS from env.sh if set, otherwise use all available cores -JOBS="${JOBS:-$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)}" - echo "=== Building wxWidgets for WASM ===" echo "Project root: $PROJECT_ROOT" echo "Build dir: $BUILD_DIR" @@ -187,7 +191,8 @@ if [ $NEEDS_CONFIGURE -eq 1 ]; then # PCRE headers (pcre2.h) must be generated before regex.cpp compiles echo "" echo "=== Building PCRE first (dependency) ===" - emmake make -C 3rdparty/pcre + # Serial relative to the main build (it fully completes first); parallel inside. + emmake make -j${JOBS} -C 3rdparty/pcre fi # Ensure the Emscripten zlib port exists before compiling. We configure with