diff --git a/docker/build.sh b/docker/build.sh index 54a1fd7..268b03d 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -3,7 +3,7 @@ # asyncify and friends on the host. # # Usage: -# ./docker/build.sh [args...] +# ./docker/build.sh [,...] [args...] # # Apps: # pcbnew PCB editor @@ -12,14 +12,26 @@ # pl_editor drawing-sheet editor # symbol_editor symbol editor (eeschema kiface, FRAME_SCH_SYMBOL_EDITOR) # gerbview Gerber viewer -# all build all of the above sequentially +# all build all of the above +# +# A comma-separated list builds just those apps in order (e.g. +# "calculator,pl_editor" — used to exercise the multi-app pipeline cheaply). # # Any extra args are forwarded to scripts/kicad/build-.sh (e.g. -j 8, # --full, --release, --diag=gal). # # The build is split into two phases: # 1. Docker: Compile KiCad to WASM (without asyncify) -# 2. Host: Apply asyncify transformation (uses Binaryen v121) +# 2. Host: dyncall shims + finalize + asyncify + -O2 (Binaryen via get-wasm-opt.sh) +# +# KICAD_PIPELINE=1 (multi-app builds only): run phase 2 of each app in the +# background while the next app compiles in the container. wasm-opt is +# Amdahl-capped at ~4 effective cores, so on a many-core CI box the container +# would otherwise sit idle for the 1-2h of host-side wasm-opt (run 27226030304: +# 103 min of the 4h was tools serialized behind each other's wasm-opt). At most +# KICAD_PIPELINE_JOBS (default 2) postprocesses run concurrently — pcbnew's -O2 +# peaks ~34 GB RSS, so 2 fits the 128 GB CI box but NOT a dev Mac: leave +# KICAD_PIPELINE unset locally. # # Binaryen is downloaded automatically - no prerequisites needed. @@ -52,7 +64,7 @@ cd "$(dirname "$0")/.." VALID_APPS="pcbnew | eeschema | calculator | pl_editor | symbol_editor | gerbview | all" usage() { - echo "Usage: ./docker/build.sh [args...]" >&2 + echo "Usage: ./docker/build.sh [,...] [args...]" >&2 echo " : ${VALID_APPS}" >&2 echo " args: forwarded to scripts/kicad/build-.sh (e.g. -j 8, --release)" >&2 } @@ -71,14 +83,24 @@ fi APP_NAME="$1" shift -case "$APP_NAME" in - pcbnew|eeschema|calculator|pl_editor|symbol_editor|gerbview|all) ;; - *) - echo "Error: unknown app '$APP_NAME' (expected: ${VALID_APPS})" >&2 - usage - exit 1 - ;; -esac +# Expand the app argument into APPS[]: "all", a single app, or a comma list. +# pcbnew first in "all" — its 90-min host-side wasm-opt chain is the critical +# path, so it must start as early as possible (especially with KICAD_PIPELINE=1). +if [[ "$APP_NAME" == "all" ]]; then + APPS=(pcbnew eeschema calculator pl_editor symbol_editor gerbview) +else + IFS=',' read -r -a APPS <<< "$APP_NAME" + for app in "${APPS[@]}"; do + case "$app" in + pcbnew|eeschema|calculator|pl_editor|symbol_editor|gerbview) ;; + *) + echo "Error: unknown app '$app' (expected: ${VALID_APPS})" >&2 + usage + exit 1 + ;; + esac + done +fi # Use branch name as Docker Compose project name for isolated containers/volumes. # Honor a pre-set COMPOSE_PROJECT_NAME so a build can target an existing volume @@ -144,9 +166,9 @@ kicad_subdir_for() { esac } -# Build one app: compile in container, then run host-side post-processing. +# Phase 1 of one app: compile in the container and copy the output to ./output. # Args: [index] [total] — index/total drive the monitor's app counter. -build_app() { +compile_app() { local app="$1" local index="${2:-1}" local total="${3:-1}" @@ -174,6 +196,21 @@ build_app() { cp /workspace/build-wasm/kicad-${app}/resources/images.tar.gz /workspace/output/ 2>/dev/null || true; \ cp /workspace/build-wasm/wxwidgets/build/wasm/wx.js /workspace/output/ 2>/dev/null || true" + # The container runs as root, so files in the bind-mounted ./output land + # root-owned on the host. macOS Docker Desktop remaps ownership to the host + # user, but on a Linux CI runner the following host-side steps (dyncall, + # finalize, asyncify) can't write into ./output. Hand ownership back. + docker compose -f docker/docker-compose.yml exec kicad-wasm-builder \ + chown -R "$(id -u):$(id -g)" /workspace/output || true +} + +# Phase 2 of one app: host-side post-processing (dyncall shims, finalize, +# asyncify + -O2). Pure host work on output/${app}.* — independent of the +# container, which is what makes it safe to run in the background while the +# next app compiles. +postprocess_app() { + local app="$1" + # Inject dynCall shims (fixes "dynCall_* is not defined" errors in Emscripten 4.x) kw_stage dyncall-shims ./scripts/common/inject-dyncall-shims.sh "output/${app}.js" @@ -187,15 +224,88 @@ build_app() { ./scripts/common/apply-asyncify.sh "output/${app}.wasm" "output/${app}.wasm" } -if [[ "${APP_NAME}" == "all" ]]; then - build_app pcbnew 1 6 - build_app eeschema 2 6 - build_app calculator 3 6 - build_app pl_editor 4 6 - build_app symbol_editor 5 6 - build_app gerbview 6 6 +# --- Pipelined driver state (KICAD_PIPELINE=1) --- +# One background postprocess per app; logs + rc files land in logs/build/ so the +# interleaved output stays readable and failures survive until the final wait. +PIPELINE_PIDS=() +PIPELINE_APPS_BG=() +PIPELINE_LOG_DIR="logs/build" +PIPELINE_TS="$(date +%Y%m%d-%H%M%S)" + +pipeline_running_count() { + local n=0 pid + for pid in "${PIPELINE_PIDS[@]}"; do + kill -0 "$pid" 2>/dev/null && n=$((n + 1)) + done + echo "$n" +} + +# Launch postprocess_app in the background, capped at KICAD_PIPELINE_JOBS +# concurrent jobs (default 2: pcbnew's -O2 peaks ~34 GB RSS; two postprocesses +# plus the container compile fit the 128 GB CI box). Portable poll loop instead +# of `wait -n` (absent in macOS bash 3.2). +pipeline_postprocess() { + local app="$1" + local max_jobs="${KICAD_PIPELINE_JOBS:-2}" + while [ "$(pipeline_running_count)" -ge "$max_jobs" ]; do + sleep 10 + done + local log_file="${PIPELINE_LOG_DIR}/postprocess-${app}-${PIPELINE_TS}.log" + echo "Pipelining host-side postprocess of ${app} (log: ${log_file})" + ( + postprocess_app "$app" >"$log_file" 2>&1 + echo $? >"${log_file}.rc" + ) & + PIPELINE_PIDS+=($!) + PIPELINE_APPS_BG+=("$app") +} + +# Wait for all background postprocesses, replay their logs into the main log, +# and fail if any of them failed. +pipeline_wait_all() { + local failed=0 i pid app log_file rc + for i in "${!PIPELINE_PIDS[@]}"; do + pid="${PIPELINE_PIDS[$i]}" + app="${PIPELINE_APPS_BG[$i]}" + log_file="${PIPELINE_LOG_DIR}/postprocess-${app}-${PIPELINE_TS}.log" + wait "$pid" || true + rc="$(cat "${log_file}.rc" 2>/dev/null || echo 1)" + echo "" + echo "=== Postprocess ${app} (rc=${rc}) — ${log_file} ===" + cat "$log_file" 2>/dev/null || true + if [ "$rc" != "0" ]; then + echo "ERROR: postprocess of ${app} failed (rc=${rc})" + failed=1 + fi + done + return "$failed" +} + +TOTAL_APPS="${#APPS[@]}" +if [[ "${KICAD_PIPELINE:-0}" == "1" ]] && [ "$TOTAL_APPS" -gt 1 ]; then + mkdir -p "$PIPELINE_LOG_DIR" + # Pre-warm the Binaryen download once — two concurrent postprocesses racing + # the first download would collide on the extract/mv. + ./scripts/common/get-wasm-opt.sh >/dev/null + # If a compile fails, set -e aborts the script — don't leave orphaned + # wasm-opt jobs chewing 30 GB in the background. Keep the monitor's + # done/fail marker from the original EXIT trap (killing finished pids is a + # no-op, so this trap is safe on the success path too). + trap '_rc=$?; for p in "${PIPELINE_PIDS[@]}"; do kill "$p" 2>/dev/null || true; done; if [ $_rc -eq 0 ]; then kw_done; else kw_fail $_rc; fi' EXIT + idx=1 + for app in "${APPS[@]}"; do + compile_app "$app" "$idx" "$TOTAL_APPS" + pipeline_postprocess "$app" + idx=$((idx + 1)) + done + pipeline_wait_all else - build_app "${APP_NAME}" 1 1 + idx=1 + for app in "${APPS[@]}"; do + compile_app "$app" "$idx" "$TOTAL_APPS" + postprocess_app "$app" + idx=$((idx + 1)) + done fi echo "" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a853141..372b325 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -5,12 +5,15 @@ services: dockerfile: docker/Dockerfile # Container name is auto-generated with project prefix (set in build.sh) - # Resource limits (adjust based on your machine) + # Resource limits. Defaults are sized for a dev Mac (Docker Desktop VM). + # CI overrides via env: the 32-core Hetzner runner sets KICAD_DOCKER_CPUS=$(nproc) + # and KICAD_DOCKER_MEM=110G — with the 10-CPU default, run 27226030304 compiled + # everything on 10 of 32 cores while passing -j 32 (3x oversubscribed). deploy: resources: limits: - cpus: '10' - memory: 32G + cpus: '${KICAD_DOCKER_CPUS:-10}' + memory: ${KICAD_DOCKER_MEM:-32G} volumes: # Host source mounted read-only at staging location diff --git a/scripts/common/apply-asyncify.sh b/scripts/common/apply-asyncify.sh index 2007163..f43a6cd 100755 --- a/scripts/common/apply-asyncify.sh +++ b/scripts/common/apply-asyncify.sh @@ -14,6 +14,61 @@ PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" # Get wasm-opt path WASM_OPT=$("${SCRIPT_DIR}/get-wasm-opt.sh") +# Bound Binaryen's host thread pool. wasm-opt runs function-parallel passes, and +# each worker holds the optimization working-set of one function at a time — so +# peak RAM scales with thread count. Binaryen reads BINARYEN_CORES to size the +# pool; default to 8 for memory-constrained dev machines, overridable via the +# environment (CI sets it to $(nproc) on the 128 GB Hetzner runner). +export BINARYEN_CORES="${BINARYEN_CORES:-8}" + +# Preload a scalable allocator on Linux. wasm-opt churns a ~40 GB high-water mark +# of short-lived allocations across all worker threads; glibc malloc serializes +# concurrent alloc/free on per-arena locks, so under many threads ~half of every +# core's cycles collapse into futex lock-spin (strace: ~99% kernel time in futex) +# instead of optimization work — the more cores, the worse it gets. jemalloc and +# mimalloc are built for exactly this many-thread churn and eliminate the storm, +# roughly halving wall-clock. macOS already ships a scalable allocator +# (libmalloc/nano-zone), so only Linux needs this. Honor an externally-set +# WASM_OPT_PRELOAD; otherwise auto-detect a system jemalloc/mimalloc. +# +# WASM_OPT_PRELOAD=none (or 0) forces NO preload — a clean glibc baseline for +# benchmarking the allocator A/B (see scripts/bench/). +if [[ "${WASM_OPT_PRELOAD:-}" == "none" || "${WASM_OPT_PRELOAD:-}" == "0" ]]; then + WASM_OPT_PRELOAD="" + _PRELOAD_FORCED_OFF=1 +fi + +if [[ -z "${WASM_OPT_PRELOAD:-}" && -z "${_PRELOAD_FORCED_OFF:-}" && "$(uname -s)" == "Linux" ]]; then + for _alloc in \ + "/usr/lib/$(uname -m)-linux-gnu/libjemalloc.so.2" \ + "/usr/lib/$(uname -m)-linux-gnu/libmimalloc.so.2" \ + /usr/lib/libjemalloc.so.2 \ + /usr/lib/libmimalloc.so.2; do + if [[ -e "${_alloc}" ]]; then + WASM_OPT_PRELOAD="${_alloc}" + break + fi + done +fi + +# Build the command prefix that injects the allocator (preserving any existing +# LD_PRELOAD). Empty when no scalable allocator was found — wasm-opt then runs +# under the default allocator, just slower. +if [[ -n "${WASM_OPT_PRELOAD:-}" ]]; then + PRELOAD_CMD=(env "LD_PRELOAD=${WASM_OPT_PRELOAD}${LD_PRELOAD:+:${LD_PRELOAD}}") +else + PRELOAD_CMD=() +fi + +# Wrap wasm-opt in GNU `time -v` when available (Linux CI) so the log records +# peak RSS + wall-clock for each pass. macOS `time` lacks -v, so fall back to +# running wasm-opt directly there. +if /usr/bin/time -v true >/dev/null 2>&1; then + TIME_CMD=(/usr/bin/time -v) +else + TIME_CMD=() +fi + INPUT_WASM="${1:-output/pcbnew.wasm}" OUTPUT_WASM="${2:-${INPUT_WASM}}" @@ -54,13 +109,25 @@ ASYNCIFY_REMOVE_ARG=$(echo "${ASYNCIFY_REMOVE}" | tr '\n' ',' | sed 's/,$//') echo "" echo "Running wasm-opt --asyncify..." echo "This may take several minutes and use significant RAM..." +echo " BINARYEN_CORES=${BINARYEN_CORES}" +echo " LD_PRELOAD=${WASM_OPT_PRELOAD:-}" -"${WASM_OPT}" --asyncify \ +"${PRELOAD_CMD[@]}" "${TIME_CMD[@]}" "${WASM_OPT}" --asyncify \ "--pass-arg=asyncify-imports@${ASYNCIFY_IMPORTS}" \ "--pass-arg=asyncify-removelist@${ASYNCIFY_REMOVE_ARG}" \ --pass-arg=asyncify-propagate-addlist \ "${INPUT_WASM}" -o "${OUTPUT_WASM}" +# ASYNCIFY_ONLY=1 stops after the asyncify pass (skips -O2). Used by the +# benchmark harness (scripts/bench/) to time/compare just the asyncify pass, +# whose RAM fits where the -O2 pass on the bloated module would not. +if [[ "${ASYNCIFY_ONLY:-0}" == "1" ]]; then + echo "" + echo "ASYNCIFY_ONLY=1 → skipping -O2 pass (benchmark mode)." + ls -lh "${OUTPUT_WASM}" + exit 0 +fi + echo "" echo "Running wasm-opt -O2 on the asyncified wasm..." echo " Purpose: shrink asyncify-instrumented functions back under V8's" @@ -68,8 +135,10 @@ echo " per-function locals limit (otherwise large coroutine-entry and" echo " similar functions silently stall in Chrome's V8). See docs/debugging/DEBUG.md §7" echo " and memory/bundle-size-asyncify-optimization.md." echo " This pass also takes several minutes and ~10-15 GB RAM." +echo " BINARYEN_CORES=${BINARYEN_CORES}" +echo " LD_PRELOAD=${WASM_OPT_PRELOAD:-}" -"${WASM_OPT}" -O2 "${OUTPUT_WASM}" -o "${OUTPUT_WASM}" +"${PRELOAD_CMD[@]}" "${TIME_CMD[@]}" "${WASM_OPT}" -O2 "${OUTPUT_WASM}" -o "${OUTPUT_WASM}" echo "" echo "Asyncify + -O2 complete: ${OUTPUT_WASM}" diff --git a/scripts/common/functions.sh b/scripts/common/functions.sh index 82d2c09..c4edc63 100755 --- a/scripts/common/functions.sh +++ b/scripts/common/functions.sh @@ -100,12 +100,37 @@ download_file() { log_info "Downloading $(basename "$dest")..." mkdir -p "$(dirname "$dest")" - if ! curl -L -o "$dest" "$url"; then - log_error "Failed to download $url" + # -f: fail (non-zero exit) on HTTP >= 400 instead of silently saving the error + # page as the file — otherwise a transient GitHub 504 gets written as the + # "tarball" and only blows up later at `tar`/`unzip` ("not in gzip format"). + # --retry-all-errors + --retry: ride out transient 5xx from release CDNs + # (GitHub release assets intermittently 504) within a single call. + if ! curl -fL --retry 5 --retry-all-errors --retry-delay 5 \ + --connect-timeout 30 -o "$dest" "$url"; then + log_error "Failed to download $url after retries" rm -f "$dest" return 1 fi + # Defense in depth: validate archive integrity so a bad download fails here + # with a clear message rather than deep in a later build step. + case "$dest" in + *.tar.gz|*.tgz) + if ! gzip -t "$dest" 2>/dev/null; then + log_error "Downloaded file is not a valid gzip archive: $dest" + rm -f "$dest" + return 1 + fi + ;; + *.zip) + if command -v unzip >/dev/null 2>&1 && ! unzip -tqq "$dest" >/dev/null 2>&1; then + log_error "Downloaded file is not a valid zip archive: $dest" + rm -f "$dest" + return 1 + fi + ;; + esac + if [ -n "$expected_sha256" ]; then local actual_sha256 actual_sha256=$(shasum -a 256 "$dest" | cut -d' ' -f1) diff --git a/scripts/common/get-wasm-opt.sh b/scripts/common/get-wasm-opt.sh index a20b980..1e9e9f7 100755 --- a/scripts/common/get-wasm-opt.sh +++ b/scripts/common/get-wasm-opt.sh @@ -10,7 +10,7 @@ # (e.g. v121+72 dev) and a standalone release (v121) corrupts asyncify # metadata, causing "func is not a function" errors at runtime. # -# Falls back to downloading standalone Binaryen v121 if emsdk is not installed +# Falls back to downloading standalone Binaryen v130 if emsdk is not installed # locally (e.g. CI environments that only use Docker). set -e @@ -19,8 +19,13 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" # --- Prefer emsdk-bundled Binaryen (matches the Emscripten that compiled the WASM) --- +# Override: set BINARYEN_VERSION in the env to FORCE a specific standalone release +# (skips the emsdk preference). Used to A/B Binaryen versions — e.g. v121 carries a +# wasm::Type lock-contention bug that makes the host-side -O2 pass ~9x slower than +# v130 (see docs/ci-build-slowness-findings.md). Validate any bump with the e2e +# suite: a Binaryen/emsdk skew can corrupt asyncify metadata ("func is not a function"). EMSDK_WASM_OPT="${PROJECT_ROOT}/tools/emsdk/upstream/bin/wasm-opt" -if [ -x "${EMSDK_WASM_OPT}" ]; then +if [ -z "${BINARYEN_VERSION:-}" ] && [ -x "${EMSDK_WASM_OPT}" ]; then EMSDK_VERSION=$("${EMSDK_WASM_OPT}" --version 2>&1 || true) echo "Using emsdk-bundled Binaryen: ${EMSDK_VERSION}" >&2 echo "${EMSDK_WASM_OPT}" @@ -30,7 +35,50 @@ fi # --- Fallback: download standalone Binaryen (for CI or environments without local emsdk) --- echo "emsdk Binaryen not found at ${EMSDK_WASM_OPT}, falling back to standalone download..." >&2 -BINARYEN_VERSION="121" +# Default v130: v121 has a wasm::Type lock convoy that makes -O2 ~9x slower on +# many-core Linux (docs/ci-build-slowness-findings.md). v130 output validated by +# the full e2e suite locally (31/31) and Chromium-green on CI run 27226030304. +BINARYEN_VERSION="${BINARYEN_VERSION:-130}" + +# BINARYEN_BUILD_FROM_SOURCE=1: compile wasm-opt ourselves instead of using the +# official Linux release tarballs, which are badly built — measured on the +# calculator fixture, identical output sha256: asyncify 4x faster on x86 +# (CI run 27276830256: 3:50 -> 0:58) and 13x on aarch64; -O2 equal. The macOS +# tarball is well-built (self-build is ~12% SLOWER there), so this is only +# worth enabling on Linux CI. Needs cmake, ninja, g++, git. ~5 min on 32 cores, +# cached in build-wasm/tools after the first call. +if [[ "${BINARYEN_BUILD_FROM_SOURCE:-0}" == "1" ]]; then + SELF_DIR="${PROJECT_ROOT}/build-wasm/tools/binaryen-${BINARYEN_VERSION}-selfbuilt" + SELF_WASM_OPT="${SELF_DIR}/bin/wasm-opt" + if [ ! -x "${SELF_WASM_OPT}" ]; then + SRC_DIR="${PROJECT_ROOT}/build-wasm/tools/binaryen-src-${BINARYEN_VERSION}" + BUILD_DIR="${PROJECT_ROOT}/build-wasm/tools/binaryen-build-${BINARYEN_VERSION}" + echo "Building Binaryen v${BINARYEN_VERSION} from source (one-time, ~5 min)..." >&2 + if [ ! -d "${SRC_DIR}" ]; then + git clone -q --depth 1 --branch "version_${BINARYEN_VERSION}" \ + --recurse-submodules --shallow-submodules \ + https://github.com/WebAssembly/binaryen.git "${SRC_DIR}" >&2 + fi + cmake -S "${SRC_DIR}" -B "${BUILD_DIR}" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_FLAGS="-Wno-maybe-uninitialized" \ + -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON -DBUILD_TESTS=OFF >&2 + ninja -C "${BUILD_DIR}" wasm-opt wasm-emscripten-finalize >&2 + # wasm-opt links lib/libbinaryen.so via rpath $ORIGIN/../lib. + mkdir -p "${SELF_DIR}/bin" "${SELF_DIR}/lib" + cp "${BUILD_DIR}/bin/wasm-opt" "${BUILD_DIR}/bin/wasm-emscripten-finalize" "${SELF_DIR}/bin/" + cp "${BUILD_DIR}"/lib/libbinaryen.* "${SELF_DIR}/lib/" 2>/dev/null || true + echo "Self-built Binaryen installed to ${SELF_DIR}" >&2 + fi + SELF_VERSION=$("${SELF_WASM_OPT}" --version 2>&1 | grep -o '[0-9]\+' | head -1) + if [ "${SELF_VERSION}" != "${BINARYEN_VERSION}" ]; then + echo "ERROR: self-built wasm-opt version mismatch (got ${SELF_VERSION}, expected ${BINARYEN_VERSION})" >&2 + exit 1 + fi + echo "Using self-built Binaryen v${BINARYEN_VERSION} (${SELF_WASM_OPT})" >&2 + echo "${SELF_WASM_OPT}" + exit 0 +fi BINARYEN_DIR="${PROJECT_ROOT}/build-wasm/tools/binaryen-${BINARYEN_VERSION}" WASM_OPT="${BINARYEN_DIR}/bin/wasm-opt" diff --git a/scripts/common/inject-dyncall-shims.sh b/scripts/common/inject-dyncall-shims.sh index c0337a9..e1b1c8c 100755 --- a/scripts/common/inject-dyncall-shims.sh +++ b/scripts/common/inject-dyncall-shims.sh @@ -105,7 +105,8 @@ TOTAL_FIXED=0 apply_fix() { #