perf(build): use all cores in host builds — wx lib, test apps, gal-webgl
env.sh exports a docker-safe JOBS=1 default, and every host build silently
inherited it (the docker kicad path dodges it by passing -j explicitly):
- build-wx-wasm.sh: the intended "${JOBS:-nproc}" fallback sat BELOW the
env.sh source, so it was dead code — full wx builds ran make -j1 on the
Mac and on CI. Compute the all-cores default before env.sh instead
(explicit JOBS/PARALLEL_JOBS still wins); also -j the PCRE pre-build.
- build-wasm-test.sh: same default fix, plus fan the post-link
hoist+asyncify loop out across JOBS with xargs -P. Per-app wasm-opt
can't feed many cores (small modules), so serial stays ~4min even with
BINARYEN_CORES=16; fanning across the 74 independent apps is what
scales. Safe: apply-asyncify is in-place per wasm, injector tmp is
per-js, HOIST_WASMOPT resolved once up front.
- build-gal-webgl-test.sh: make had no -j at all.
Measured on a 16-core M4 Max, clean test-app build: 15m36s (-j1) → 2m08s,
peak RAM 10.7 GB summed across all build processes (64 GB machine; CI
runners have 120 GB). Core counts are always derived from nproc at
runtime — nothing hardcoded, CI workers differ.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XeiSRRScdaox5jBueJNcyG
This commit is contained in:
parent
616bbbf67c
commit
348596c515
3 changed files with 38 additions and 14 deletions
|
|
@ -21,6 +21,10 @@ set -e
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
|
||||||
|
# Default to all cores BEFORE env.sh (its docker-safe JOBS=1 default would stick
|
||||||
|
# otherwise); an explicit JOBS/PARALLEL_JOBS from the caller still wins.
|
||||||
|
JOBS="${JOBS:-$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)}"
|
||||||
|
|
||||||
# Source common environment (sets up local emsdk)
|
# Source common environment (sets up local emsdk)
|
||||||
QUIET=1 source "$SCRIPT_DIR/common/env.sh"
|
QUIET=1 source "$SCRIPT_DIR/common/env.sh"
|
||||||
|
|
||||||
|
|
@ -76,9 +80,9 @@ python3 generate_shaders.py
|
||||||
echo ""
|
echo ""
|
||||||
echo "Building..."
|
echo "Building..."
|
||||||
if [ "$DEBUG_BUILD" = "1" ]; then
|
if [ "$DEBUG_BUILD" = "1" ]; then
|
||||||
make DEBUG=1
|
make -j"${JOBS:-1}" DEBUG=1
|
||||||
else
|
else
|
||||||
make
|
make -j"${JOBS:-1}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,10 @@
|
||||||
|
|
||||||
# Redirect all output to a log file (re-execs script with redirection)
|
# Redirect all output to a log file (re-execs script with redirection)
|
||||||
source "$(dirname "$0")/common/logging.sh"
|
source "$(dirname "$0")/common/logging.sh"
|
||||||
|
# Default to all cores BEFORE env.sh (its docker-safe JOBS=1 default would stick
|
||||||
|
# otherwise); an explicit JOBS/PARALLEL_JOBS from the caller still wins. Measured
|
||||||
|
# 2026-07-02 (16-core M4 Max): clean build 15m36s at -j1 → ~2min, peak RAM ~11 GB.
|
||||||
|
JOBS="${JOBS:-$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)}"
|
||||||
source "$(dirname "$0")/common/env.sh"
|
source "$(dirname "$0")/common/env.sh"
|
||||||
# This script creates library symlinks and builds the test apps
|
# This script creates library symlinks and builds the test apps
|
||||||
#
|
#
|
||||||
|
|
@ -136,19 +140,30 @@ fi
|
||||||
# so re-running it here is safe. The .wasm gets post-link hoist + asyncify first.
|
# so re-running it here is safe. The .wasm gets post-link hoist + asyncify first.
|
||||||
_eh_restore_wasmopt; trap - EXIT
|
_eh_restore_wasmopt; trap - EXIT
|
||||||
echo ""
|
echo ""
|
||||||
echo "=== Post-link --hoist-cpp-catches + --asyncify ==="
|
echo "=== Post-link --hoist-cpp-catches + --asyncify (${JOBS:-1}-wide) ==="
|
||||||
while IFS= read -r w; do
|
# The apps are independent here too (apply-asyncify rewrites each wasm in place; the shim
|
||||||
"$SCRIPT_DIR/common/apply-asyncify.sh" --no-removelist "$w"
|
# injector's temp file is per-js), so fan the post-link out across JOBS like the make phase.
|
||||||
js="${w%.wasm}.js"
|
# This dominates the build: per-app wasm-opt can't feed many cores (small modules, serial
|
||||||
if [ -f "$js" ]; then
|
# parse/write), so parallelism must come from running apps side by side — serial 3m52s even
|
||||||
( cd "$(dirname "$js")" && "$SCRIPT_DIR/common/inject-dyncall-shims.sh" "$(basename "$js")" )
|
# with BINARYEN_CORES=16, vs 2m08s fanned out at ~11 GB peak RAM. HOIST_WASMOPT is resolved
|
||||||
fi
|
# once above, so workers skip the binaryen ninja check. xargs fails the build (exit 123) if
|
||||||
|
# any app's post-link fails.
|
||||||
|
export SCRIPT_DIR
|
||||||
# Match EVERY freshly-linked app wasm, not just standalone/*/*_test.wasm: the main demo
|
# Match EVERY freshly-linked app wasm, not just standalone/*/*_test.wasm: the main demo
|
||||||
# (apps/minimal_test.wasm) is at the apps/ root, and the coroutine-pthread repros + wxpt app
|
# (apps/minimal_test.wasm) is at the apps/ root, and the coroutine-pthread repros + wxpt app
|
||||||
# are *_repro*.wasm / *_wxpt.wasm. The old '*_test.wasm under standalone' filter silently
|
# are *_repro*.wasm / *_wxpt.wasm. The old '*_test.wasm under standalone' filter silently
|
||||||
# skipped all of those, so under native wasm-EH they never got hoist+asyncify and crashed at
|
# skipped all of those, so under native wasm-EH they never got hoist+asyncify and crashed at
|
||||||
# runtime with "asyncify_start_unwind not found".
|
# runtime with "asyncify_start_unwind not found".
|
||||||
done < <(find "$WASM_APP_DIR" -name '*.wasm' -newer "$EH_MARKER")
|
find "$WASM_APP_DIR" -name '*.wasm' -newer "$EH_MARKER" -print0 \
|
||||||
|
| xargs -0 -n1 -P "${JOBS:-1}" bash -c '
|
||||||
|
set -eo pipefail
|
||||||
|
w="$1"
|
||||||
|
"$SCRIPT_DIR/common/apply-asyncify.sh" --no-removelist "$w"
|
||||||
|
js="${w%.wasm}.js"
|
||||||
|
if [ -f "$js" ]; then
|
||||||
|
( cd "$(dirname "$js")" && "$SCRIPT_DIR/common/inject-dyncall-shims.sh" "$(basename "$js")" )
|
||||||
|
fi
|
||||||
|
' _
|
||||||
rm -f "$EH_MARKER"
|
rm -f "$EH_MARKER"
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,13 @@
|
||||||
# Redirect all output to a log file (re-execs script with redirection)
|
# Redirect all output to a log file (re-execs script with redirection)
|
||||||
source "$(dirname "$0")/common/logging.sh"
|
source "$(dirname "$0")/common/logging.sh"
|
||||||
|
|
||||||
|
# Default to all cores BEFORE sourcing env.sh — env.sh exports a docker-safe
|
||||||
|
# JOBS=1 when unset, which would make the "${JOBS:-nproc}" fallback below it
|
||||||
|
# dead code (this build ran make -j1 everywhere for that reason). An explicit
|
||||||
|
# JOBS/PARALLEL_JOBS from the caller still wins (e.g. the docker pipeline's -j
|
||||||
|
# via build-kicad-target.sh).
|
||||||
|
JOBS="${JOBS:-$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)}"
|
||||||
|
|
||||||
# Source common environment (sets up local emsdk)
|
# Source common environment (sets up local emsdk)
|
||||||
source "$(dirname "$0")/common/env.sh"
|
source "$(dirname "$0")/common/env.sh"
|
||||||
# Build-progress markers (parsed by scripts/build-monitor.sh).
|
# Build-progress markers (parsed by scripts/build-monitor.sh).
|
||||||
|
|
@ -53,9 +60,6 @@ export CONFIG_SHELL="$SCRIPT_DIR/config/config-sub-wrapper.sh"
|
||||||
# Disable autom4te cache to keep submodules clean
|
# Disable autom4te cache to keep submodules clean
|
||||||
export AUTOM4TE="$SCRIPT_DIR/config/autom4te-wrapper.sh"
|
export AUTOM4TE="$SCRIPT_DIR/config/autom4te-wrapper.sh"
|
||||||
|
|
||||||
# Use JOBS from env.sh if set, otherwise use all available cores
|
|
||||||
JOBS="${JOBS:-$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)}"
|
|
||||||
|
|
||||||
echo "=== Building wxWidgets for WASM ==="
|
echo "=== Building wxWidgets for WASM ==="
|
||||||
echo "Project root: $PROJECT_ROOT"
|
echo "Project root: $PROJECT_ROOT"
|
||||||
echo "Build dir: $BUILD_DIR"
|
echo "Build dir: $BUILD_DIR"
|
||||||
|
|
@ -187,7 +191,8 @@ if [ $NEEDS_CONFIGURE -eq 1 ]; then
|
||||||
# PCRE headers (pcre2.h) must be generated before regex.cpp compiles
|
# PCRE headers (pcre2.h) must be generated before regex.cpp compiles
|
||||||
echo ""
|
echo ""
|
||||||
echo "=== Building PCRE first (dependency) ==="
|
echo "=== Building PCRE first (dependency) ==="
|
||||||
emmake make -C 3rdparty/pcre
|
# Serial relative to the main build (it fully completes first); parallel inside.
|
||||||
|
emmake make -j${JOBS} -C 3rdparty/pcre
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Ensure the Emscripten zlib port exists before compiling. We configure with
|
# Ensure the Emscripten zlib port exists before compiling. We configure with
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue