perf(build): 4h05m -> 1h15m full CI build — docker caps, pipelined wasm-opt, Binaryen 130
Three orchestration fixes, validated end-to-end on the Hetzner ccx53 (run 27280051992, 1h14m41s vs 4h05m baseline, e2e identical): - docker-compose CPU/memory caps were hardcoded to dev-Mac defaults (10 CPUs / 32G); now env-tunable via KICAD_DOCKER_CPUS/KICAD_DOCKER_MEM (CI exports nproc/110G — the 32-core runner was compiling on 10 cores). - docker/build.sh: split build_app into compile_app (container) + postprocess_app (host-side dyncall shims + finalize + asyncify + -O2) and added KICAD_PIPELINE=1 mode that overlaps each tool's host-side wasm-opt with the next tool's container compile (max 2 concurrent postprocesses — pcbnew -O2 peaks ~34G RSS). Also: comma-separated app lists for cheap pipeline repros. - get-wasm-opt.sh: Binaryen default 121 -> 130 (fixes the v121 -O2 lock convoy, ~9x) and BINARYEN_BUILD_FROM_SOURCE=1 support: the official x86_64-linux release tarballs (Alpine/musl, no LTO, assertions on) run asyncify 4x slower than a stock gcc -O3+LTO build with sha256-identical output; self-build takes ~5 min and is instantly repaid. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Istvan Matejcsok
parent
e6ff74a11e
commit
680eb9dc15
6 changed files with 289 additions and 33 deletions
154
docker/build.sh
154
docker/build.sh
|
|
@ -3,7 +3,7 @@
|
|||
# asyncify and friends on the host.
|
||||
#
|
||||
# Usage:
|
||||
# ./docker/build.sh <app> [args...]
|
||||
# ./docker/build.sh <app>[,<app>...] [args...]
|
||||
#
|
||||
# Apps:
|
||||
# pcbnew PCB editor
|
||||
|
|
@ -12,14 +12,26 @@
|
|||
# pl_editor drawing-sheet editor
|
||||
# symbol_editor symbol editor (eeschema kiface, FRAME_SCH_SYMBOL_EDITOR)
|
||||
# gerbview Gerber viewer
|
||||
# all build all of the above sequentially
|
||||
# all build all of the above
|
||||
#
|
||||
# A comma-separated list builds just those apps in order (e.g.
|
||||
# "calculator,pl_editor" — used to exercise the multi-app pipeline cheaply).
|
||||
#
|
||||
# Any extra args are forwarded to scripts/kicad/build-<app>.sh (e.g. -j 8,
|
||||
# --full, --release, --diag=gal).
|
||||
#
|
||||
# The build is split into two phases:
|
||||
# 1. Docker: Compile KiCad to WASM (without asyncify)
|
||||
# 2. Host: Apply asyncify transformation (uses Binaryen v121)
|
||||
# 2. Host: dyncall shims + finalize + asyncify + -O2 (Binaryen via get-wasm-opt.sh)
|
||||
#
|
||||
# KICAD_PIPELINE=1 (multi-app builds only): run phase 2 of each app in the
|
||||
# background while the next app compiles in the container. wasm-opt is
|
||||
# Amdahl-capped at ~4 effective cores, so on a many-core CI box the container
|
||||
# would otherwise sit idle for the 1-2h of host-side wasm-opt (run 27226030304:
|
||||
# 103 min of the 4h was tools serialized behind each other's wasm-opt). At most
|
||||
# KICAD_PIPELINE_JOBS (default 2) postprocesses run concurrently — pcbnew's -O2
|
||||
# peaks ~34 GB RSS, so 2 fits the 128 GB CI box but NOT a dev Mac: leave
|
||||
# KICAD_PIPELINE unset locally.
|
||||
#
|
||||
# Binaryen is downloaded automatically - no prerequisites needed.
|
||||
|
||||
|
|
@ -52,7 +64,7 @@ cd "$(dirname "$0")/.."
|
|||
VALID_APPS="pcbnew | eeschema | calculator | pl_editor | symbol_editor | gerbview | all"
|
||||
|
||||
usage() {
|
||||
echo "Usage: ./docker/build.sh <app> [args...]" >&2
|
||||
echo "Usage: ./docker/build.sh <app>[,<app>...] [args...]" >&2
|
||||
echo " <app>: ${VALID_APPS}" >&2
|
||||
echo " args: forwarded to scripts/kicad/build-<app>.sh (e.g. -j 8, --release)" >&2
|
||||
}
|
||||
|
|
@ -71,14 +83,24 @@ fi
|
|||
APP_NAME="$1"
|
||||
shift
|
||||
|
||||
case "$APP_NAME" in
|
||||
pcbnew|eeschema|calculator|pl_editor|symbol_editor|gerbview|all) ;;
|
||||
*)
|
||||
echo "Error: unknown app '$APP_NAME' (expected: ${VALID_APPS})" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
# Expand the app argument into APPS[]: "all", a single app, or a comma list.
|
||||
# pcbnew first in "all" — its 90-min host-side wasm-opt chain is the critical
|
||||
# path, so it must start as early as possible (especially with KICAD_PIPELINE=1).
|
||||
if [[ "$APP_NAME" == "all" ]]; then
|
||||
APPS=(pcbnew eeschema calculator pl_editor symbol_editor gerbview)
|
||||
else
|
||||
IFS=',' read -r -a APPS <<< "$APP_NAME"
|
||||
for app in "${APPS[@]}"; do
|
||||
case "$app" in
|
||||
pcbnew|eeschema|calculator|pl_editor|symbol_editor|gerbview) ;;
|
||||
*)
|
||||
echo "Error: unknown app '$app' (expected: ${VALID_APPS})" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
|
||||
# Use branch name as Docker Compose project name for isolated containers/volumes.
|
||||
# Honor a pre-set COMPOSE_PROJECT_NAME so a build can target an existing volume
|
||||
|
|
@ -144,9 +166,9 @@ kicad_subdir_for() {
|
|||
esac
|
||||
}
|
||||
|
||||
# Build one app: compile in container, then run host-side post-processing.
|
||||
# Phase 1 of one app: compile in the container and copy the output to ./output.
|
||||
# Args: <app> [index] [total] — index/total drive the monitor's app counter.
|
||||
build_app() {
|
||||
compile_app() {
|
||||
local app="$1"
|
||||
local index="${2:-1}"
|
||||
local total="${3:-1}"
|
||||
|
|
@ -174,6 +196,21 @@ build_app() {
|
|||
cp /workspace/build-wasm/kicad-${app}/resources/images.tar.gz /workspace/output/ 2>/dev/null || true; \
|
||||
cp /workspace/build-wasm/wxwidgets/build/wasm/wx.js /workspace/output/ 2>/dev/null || true"
|
||||
|
||||
# The container runs as root, so files in the bind-mounted ./output land
|
||||
# root-owned on the host. macOS Docker Desktop remaps ownership to the host
|
||||
# user, but on a Linux CI runner the following host-side steps (dyncall,
|
||||
# finalize, asyncify) can't write into ./output. Hand ownership back.
|
||||
docker compose -f docker/docker-compose.yml exec kicad-wasm-builder \
|
||||
chown -R "$(id -u):$(id -g)" /workspace/output || true
|
||||
}
|
||||
|
||||
# Phase 2 of one app: host-side post-processing (dyncall shims, finalize,
|
||||
# asyncify + -O2). Pure host work on output/${app}.* — independent of the
|
||||
# container, which is what makes it safe to run in the background while the
|
||||
# next app compiles.
|
||||
postprocess_app() {
|
||||
local app="$1"
|
||||
|
||||
# Inject dynCall shims (fixes "dynCall_* is not defined" errors in Emscripten 4.x)
|
||||
kw_stage dyncall-shims
|
||||
./scripts/common/inject-dyncall-shims.sh "output/${app}.js"
|
||||
|
|
@ -187,15 +224,88 @@ build_app() {
|
|||
./scripts/common/apply-asyncify.sh "output/${app}.wasm" "output/${app}.wasm"
|
||||
}
|
||||
|
||||
if [[ "${APP_NAME}" == "all" ]]; then
|
||||
build_app pcbnew 1 6
|
||||
build_app eeschema 2 6
|
||||
build_app calculator 3 6
|
||||
build_app pl_editor 4 6
|
||||
build_app symbol_editor 5 6
|
||||
build_app gerbview 6 6
|
||||
# --- Pipelined driver state (KICAD_PIPELINE=1) ---
|
||||
# One background postprocess per app; logs + rc files land in logs/build/ so the
|
||||
# interleaved output stays readable and failures survive until the final wait.
|
||||
PIPELINE_PIDS=()
|
||||
PIPELINE_APPS_BG=()
|
||||
PIPELINE_LOG_DIR="logs/build"
|
||||
PIPELINE_TS="$(date +%Y%m%d-%H%M%S)"
|
||||
|
||||
pipeline_running_count() {
|
||||
local n=0 pid
|
||||
for pid in "${PIPELINE_PIDS[@]}"; do
|
||||
kill -0 "$pid" 2>/dev/null && n=$((n + 1))
|
||||
done
|
||||
echo "$n"
|
||||
}
|
||||
|
||||
# Launch postprocess_app in the background, capped at KICAD_PIPELINE_JOBS
|
||||
# concurrent jobs (default 2: pcbnew's -O2 peaks ~34 GB RSS; two postprocesses
|
||||
# plus the container compile fit the 128 GB CI box). Portable poll loop instead
|
||||
# of `wait -n` (absent in macOS bash 3.2).
|
||||
pipeline_postprocess() {
|
||||
local app="$1"
|
||||
local max_jobs="${KICAD_PIPELINE_JOBS:-2}"
|
||||
while [ "$(pipeline_running_count)" -ge "$max_jobs" ]; do
|
||||
sleep 10
|
||||
done
|
||||
local log_file="${PIPELINE_LOG_DIR}/postprocess-${app}-${PIPELINE_TS}.log"
|
||||
echo "Pipelining host-side postprocess of ${app} (log: ${log_file})"
|
||||
(
|
||||
postprocess_app "$app" >"$log_file" 2>&1
|
||||
echo $? >"${log_file}.rc"
|
||||
) &
|
||||
PIPELINE_PIDS+=($!)
|
||||
PIPELINE_APPS_BG+=("$app")
|
||||
}
|
||||
|
||||
# Wait for all background postprocesses, replay their logs into the main log,
|
||||
# and fail if any of them failed.
|
||||
pipeline_wait_all() {
|
||||
local failed=0 i pid app log_file rc
|
||||
for i in "${!PIPELINE_PIDS[@]}"; do
|
||||
pid="${PIPELINE_PIDS[$i]}"
|
||||
app="${PIPELINE_APPS_BG[$i]}"
|
||||
log_file="${PIPELINE_LOG_DIR}/postprocess-${app}-${PIPELINE_TS}.log"
|
||||
wait "$pid" || true
|
||||
rc="$(cat "${log_file}.rc" 2>/dev/null || echo 1)"
|
||||
echo ""
|
||||
echo "=== Postprocess ${app} (rc=${rc}) — ${log_file} ==="
|
||||
cat "$log_file" 2>/dev/null || true
|
||||
if [ "$rc" != "0" ]; then
|
||||
echo "ERROR: postprocess of ${app} failed (rc=${rc})"
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
return "$failed"
|
||||
}
|
||||
|
||||
TOTAL_APPS="${#APPS[@]}"
|
||||
if [[ "${KICAD_PIPELINE:-0}" == "1" ]] && [ "$TOTAL_APPS" -gt 1 ]; then
|
||||
mkdir -p "$PIPELINE_LOG_DIR"
|
||||
# Pre-warm the Binaryen download once — two concurrent postprocesses racing
|
||||
# the first download would collide on the extract/mv.
|
||||
./scripts/common/get-wasm-opt.sh >/dev/null
|
||||
# If a compile fails, set -e aborts the script — don't leave orphaned
|
||||
# wasm-opt jobs chewing 30 GB in the background. Keep the monitor's
|
||||
# done/fail marker from the original EXIT trap (killing finished pids is a
|
||||
# no-op, so this trap is safe on the success path too).
|
||||
trap '_rc=$?; for p in "${PIPELINE_PIDS[@]}"; do kill "$p" 2>/dev/null || true; done; if [ $_rc -eq 0 ]; then kw_done; else kw_fail $_rc; fi' EXIT
|
||||
idx=1
|
||||
for app in "${APPS[@]}"; do
|
||||
compile_app "$app" "$idx" "$TOTAL_APPS"
|
||||
pipeline_postprocess "$app"
|
||||
idx=$((idx + 1))
|
||||
done
|
||||
pipeline_wait_all
|
||||
else
|
||||
build_app "${APP_NAME}" 1 1
|
||||
idx=1
|
||||
for app in "${APPS[@]}"; do
|
||||
compile_app "$app" "$idx" "$TOTAL_APPS"
|
||||
postprocess_app "$app"
|
||||
idx=$((idx + 1))
|
||||
done
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
|
|
|||
|
|
@ -5,12 +5,15 @@ services:
|
|||
dockerfile: docker/Dockerfile
|
||||
# Container name is auto-generated with project prefix (set in build.sh)
|
||||
|
||||
# Resource limits (adjust based on your machine)
|
||||
# Resource limits. Defaults are sized for a dev Mac (Docker Desktop VM).
|
||||
# CI overrides via env: the 32-core Hetzner runner sets KICAD_DOCKER_CPUS=$(nproc)
|
||||
# and KICAD_DOCKER_MEM=110G — with the 10-CPU default, run 27226030304 compiled
|
||||
# everything on 10 of 32 cores while passing -j 32 (3x oversubscribed).
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '10'
|
||||
memory: 32G
|
||||
cpus: '${KICAD_DOCKER_CPUS:-10}'
|
||||
memory: ${KICAD_DOCKER_MEM:-32G}
|
||||
|
||||
volumes:
|
||||
# Host source mounted read-only at staging location
|
||||
|
|
|
|||
|
|
@ -14,6 +14,61 @@ PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
|||
# Get wasm-opt path
|
||||
WASM_OPT=$("${SCRIPT_DIR}/get-wasm-opt.sh")
|
||||
|
||||
# Bound Binaryen's host thread pool. wasm-opt runs function-parallel passes, and
|
||||
# each worker holds the optimization working-set of one function at a time — so
|
||||
# peak RAM scales with thread count. Binaryen reads BINARYEN_CORES to size the
|
||||
# pool; default to 8 for memory-constrained dev machines, overridable via the
|
||||
# environment (CI sets it to $(nproc) on the 128 GB Hetzner runner).
|
||||
export BINARYEN_CORES="${BINARYEN_CORES:-8}"
|
||||
|
||||
# Preload a scalable allocator on Linux. wasm-opt churns a ~40 GB high-water mark
|
||||
# of short-lived allocations across all worker threads; glibc malloc serializes
|
||||
# concurrent alloc/free on per-arena locks, so under many threads ~half of every
|
||||
# core's cycles collapse into futex lock-spin (strace: ~99% kernel time in futex)
|
||||
# instead of optimization work — the more cores, the worse it gets. jemalloc and
|
||||
# mimalloc are built for exactly this many-thread churn and eliminate the storm,
|
||||
# roughly halving wall-clock. macOS already ships a scalable allocator
|
||||
# (libmalloc/nano-zone), so only Linux needs this. Honor an externally-set
|
||||
# WASM_OPT_PRELOAD; otherwise auto-detect a system jemalloc/mimalloc.
|
||||
#
|
||||
# WASM_OPT_PRELOAD=none (or 0) forces NO preload — a clean glibc baseline for
|
||||
# benchmarking the allocator A/B (see scripts/bench/).
|
||||
if [[ "${WASM_OPT_PRELOAD:-}" == "none" || "${WASM_OPT_PRELOAD:-}" == "0" ]]; then
|
||||
WASM_OPT_PRELOAD=""
|
||||
_PRELOAD_FORCED_OFF=1
|
||||
fi
|
||||
|
||||
if [[ -z "${WASM_OPT_PRELOAD:-}" && -z "${_PRELOAD_FORCED_OFF:-}" && "$(uname -s)" == "Linux" ]]; then
|
||||
for _alloc in \
|
||||
"/usr/lib/$(uname -m)-linux-gnu/libjemalloc.so.2" \
|
||||
"/usr/lib/$(uname -m)-linux-gnu/libmimalloc.so.2" \
|
||||
/usr/lib/libjemalloc.so.2 \
|
||||
/usr/lib/libmimalloc.so.2; do
|
||||
if [[ -e "${_alloc}" ]]; then
|
||||
WASM_OPT_PRELOAD="${_alloc}"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Build the command prefix that injects the allocator (preserving any existing
|
||||
# LD_PRELOAD). Empty when no scalable allocator was found — wasm-opt then runs
|
||||
# under the default allocator, just slower.
|
||||
if [[ -n "${WASM_OPT_PRELOAD:-}" ]]; then
|
||||
PRELOAD_CMD=(env "LD_PRELOAD=${WASM_OPT_PRELOAD}${LD_PRELOAD:+:${LD_PRELOAD}}")
|
||||
else
|
||||
PRELOAD_CMD=()
|
||||
fi
|
||||
|
||||
# Wrap wasm-opt in GNU `time -v` when available (Linux CI) so the log records
|
||||
# peak RSS + wall-clock for each pass. macOS `time` lacks -v, so fall back to
|
||||
# running wasm-opt directly there.
|
||||
if /usr/bin/time -v true >/dev/null 2>&1; then
|
||||
TIME_CMD=(/usr/bin/time -v)
|
||||
else
|
||||
TIME_CMD=()
|
||||
fi
|
||||
|
||||
INPUT_WASM="${1:-output/pcbnew.wasm}"
|
||||
OUTPUT_WASM="${2:-${INPUT_WASM}}"
|
||||
|
||||
|
|
@ -54,13 +109,25 @@ ASYNCIFY_REMOVE_ARG=$(echo "${ASYNCIFY_REMOVE}" | tr '\n' ',' | sed 's/,$//')
|
|||
echo ""
|
||||
echo "Running wasm-opt --asyncify..."
|
||||
echo "This may take several minutes and use significant RAM..."
|
||||
echo " BINARYEN_CORES=${BINARYEN_CORES}"
|
||||
echo " LD_PRELOAD=${WASM_OPT_PRELOAD:-<none>}"
|
||||
|
||||
"${WASM_OPT}" --asyncify \
|
||||
"${PRELOAD_CMD[@]}" "${TIME_CMD[@]}" "${WASM_OPT}" --asyncify \
|
||||
"--pass-arg=asyncify-imports@${ASYNCIFY_IMPORTS}" \
|
||||
"--pass-arg=asyncify-removelist@${ASYNCIFY_REMOVE_ARG}" \
|
||||
--pass-arg=asyncify-propagate-addlist \
|
||||
"${INPUT_WASM}" -o "${OUTPUT_WASM}"
|
||||
|
||||
# ASYNCIFY_ONLY=1 stops after the asyncify pass (skips -O2). Used by the
|
||||
# benchmark harness (scripts/bench/) to time/compare just the asyncify pass,
|
||||
# whose RAM fits where the -O2 pass on the bloated module would not.
|
||||
if [[ "${ASYNCIFY_ONLY:-0}" == "1" ]]; then
|
||||
echo ""
|
||||
echo "ASYNCIFY_ONLY=1 → skipping -O2 pass (benchmark mode)."
|
||||
ls -lh "${OUTPUT_WASM}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Running wasm-opt -O2 on the asyncified wasm..."
|
||||
echo " Purpose: shrink asyncify-instrumented functions back under V8's"
|
||||
|
|
@ -68,8 +135,10 @@ echo " per-function locals limit (otherwise large coroutine-entry and"
|
|||
echo " similar functions silently stall in Chrome's V8). See docs/debugging/DEBUG.md §7"
|
||||
echo " and memory/bundle-size-asyncify-optimization.md."
|
||||
echo " This pass also takes several minutes and ~10-15 GB RAM."
|
||||
echo " BINARYEN_CORES=${BINARYEN_CORES}"
|
||||
echo " LD_PRELOAD=${WASM_OPT_PRELOAD:-<none>}"
|
||||
|
||||
"${WASM_OPT}" -O2 "${OUTPUT_WASM}" -o "${OUTPUT_WASM}"
|
||||
"${PRELOAD_CMD[@]}" "${TIME_CMD[@]}" "${WASM_OPT}" -O2 "${OUTPUT_WASM}" -o "${OUTPUT_WASM}"
|
||||
|
||||
echo ""
|
||||
echo "Asyncify + -O2 complete: ${OUTPUT_WASM}"
|
||||
|
|
|
|||
|
|
@ -100,12 +100,37 @@ download_file() {
|
|||
log_info "Downloading $(basename "$dest")..."
|
||||
mkdir -p "$(dirname "$dest")"
|
||||
|
||||
if ! curl -L -o "$dest" "$url"; then
|
||||
log_error "Failed to download $url"
|
||||
# -f: fail (non-zero exit) on HTTP >= 400 instead of silently saving the error
|
||||
# page as the file — otherwise a transient GitHub 504 gets written as the
|
||||
# "tarball" and only blows up later at `tar`/`unzip` ("not in gzip format").
|
||||
# --retry-all-errors + --retry: ride out transient 5xx from release CDNs
|
||||
# (GitHub release assets intermittently 504) within a single call.
|
||||
if ! curl -fL --retry 5 --retry-all-errors --retry-delay 5 \
|
||||
--connect-timeout 30 -o "$dest" "$url"; then
|
||||
log_error "Failed to download $url after retries"
|
||||
rm -f "$dest"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Defense in depth: validate archive integrity so a bad download fails here
|
||||
# with a clear message rather than deep in a later build step.
|
||||
case "$dest" in
|
||||
*.tar.gz|*.tgz)
|
||||
if ! gzip -t "$dest" 2>/dev/null; then
|
||||
log_error "Downloaded file is not a valid gzip archive: $dest"
|
||||
rm -f "$dest"
|
||||
return 1
|
||||
fi
|
||||
;;
|
||||
*.zip)
|
||||
if command -v unzip >/dev/null 2>&1 && ! unzip -tqq "$dest" >/dev/null 2>&1; then
|
||||
log_error "Downloaded file is not a valid zip archive: $dest"
|
||||
rm -f "$dest"
|
||||
return 1
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ -n "$expected_sha256" ]; then
|
||||
local actual_sha256
|
||||
actual_sha256=$(shasum -a 256 "$dest" | cut -d' ' -f1)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@
|
|||
# (e.g. v121+72 dev) and a standalone release (v121) corrupts asyncify
|
||||
# metadata, causing "func is not a function" errors at runtime.
|
||||
#
|
||||
# Falls back to downloading standalone Binaryen v121 if emsdk is not installed
|
||||
# Falls back to downloading standalone Binaryen v130 if emsdk is not installed
|
||||
# locally (e.g. CI environments that only use Docker).
|
||||
|
||||
set -e
|
||||
|
|
@ -19,8 +19,13 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||
|
||||
# --- Prefer emsdk-bundled Binaryen (matches the Emscripten that compiled the WASM) ---
|
||||
# Override: set BINARYEN_VERSION in the env to FORCE a specific standalone release
|
||||
# (skips the emsdk preference). Used to A/B Binaryen versions — e.g. v121 carries a
|
||||
# wasm::Type lock-contention bug that makes the host-side -O2 pass ~9x slower than
|
||||
# v130 (see docs/ci-build-slowness-findings.md). Validate any bump with the e2e
|
||||
# suite: a Binaryen/emsdk skew can corrupt asyncify metadata ("func is not a function").
|
||||
EMSDK_WASM_OPT="${PROJECT_ROOT}/tools/emsdk/upstream/bin/wasm-opt"
|
||||
if [ -x "${EMSDK_WASM_OPT}" ]; then
|
||||
if [ -z "${BINARYEN_VERSION:-}" ] && [ -x "${EMSDK_WASM_OPT}" ]; then
|
||||
EMSDK_VERSION=$("${EMSDK_WASM_OPT}" --version 2>&1 || true)
|
||||
echo "Using emsdk-bundled Binaryen: ${EMSDK_VERSION}" >&2
|
||||
echo "${EMSDK_WASM_OPT}"
|
||||
|
|
@ -30,7 +35,50 @@ fi
|
|||
# --- Fallback: download standalone Binaryen (for CI or environments without local emsdk) ---
|
||||
echo "emsdk Binaryen not found at ${EMSDK_WASM_OPT}, falling back to standalone download..." >&2
|
||||
|
||||
BINARYEN_VERSION="121"
|
||||
# Default v130: v121 has a wasm::Type lock convoy that makes -O2 ~9x slower on
|
||||
# many-core Linux (docs/ci-build-slowness-findings.md). v130 output validated by
|
||||
# the full e2e suite locally (31/31) and Chromium-green on CI run 27226030304.
|
||||
BINARYEN_VERSION="${BINARYEN_VERSION:-130}"
|
||||
|
||||
# BINARYEN_BUILD_FROM_SOURCE=1: compile wasm-opt ourselves instead of using the
|
||||
# official Linux release tarballs, which are badly built — measured on the
|
||||
# calculator fixture, identical output sha256: asyncify 4x faster on x86
|
||||
# (CI run 27276830256: 3:50 -> 0:58) and 13x on aarch64; -O2 equal. The macOS
|
||||
# tarball is well-built (self-build is ~12% SLOWER there), so this is only
|
||||
# worth enabling on Linux CI. Needs cmake, ninja, g++, git. ~5 min on 32 cores,
|
||||
# cached in build-wasm/tools after the first call.
|
||||
if [[ "${BINARYEN_BUILD_FROM_SOURCE:-0}" == "1" ]]; then
|
||||
SELF_DIR="${PROJECT_ROOT}/build-wasm/tools/binaryen-${BINARYEN_VERSION}-selfbuilt"
|
||||
SELF_WASM_OPT="${SELF_DIR}/bin/wasm-opt"
|
||||
if [ ! -x "${SELF_WASM_OPT}" ]; then
|
||||
SRC_DIR="${PROJECT_ROOT}/build-wasm/tools/binaryen-src-${BINARYEN_VERSION}"
|
||||
BUILD_DIR="${PROJECT_ROOT}/build-wasm/tools/binaryen-build-${BINARYEN_VERSION}"
|
||||
echo "Building Binaryen v${BINARYEN_VERSION} from source (one-time, ~5 min)..." >&2
|
||||
if [ ! -d "${SRC_DIR}" ]; then
|
||||
git clone -q --depth 1 --branch "version_${BINARYEN_VERSION}" \
|
||||
--recurse-submodules --shallow-submodules \
|
||||
https://github.com/WebAssembly/binaryen.git "${SRC_DIR}" >&2
|
||||
fi
|
||||
cmake -S "${SRC_DIR}" -B "${BUILD_DIR}" -G Ninja \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_CXX_FLAGS="-Wno-maybe-uninitialized" \
|
||||
-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON -DBUILD_TESTS=OFF >&2
|
||||
ninja -C "${BUILD_DIR}" wasm-opt wasm-emscripten-finalize >&2
|
||||
# wasm-opt links lib/libbinaryen.so via rpath $ORIGIN/../lib.
|
||||
mkdir -p "${SELF_DIR}/bin" "${SELF_DIR}/lib"
|
||||
cp "${BUILD_DIR}/bin/wasm-opt" "${BUILD_DIR}/bin/wasm-emscripten-finalize" "${SELF_DIR}/bin/"
|
||||
cp "${BUILD_DIR}"/lib/libbinaryen.* "${SELF_DIR}/lib/" 2>/dev/null || true
|
||||
echo "Self-built Binaryen installed to ${SELF_DIR}" >&2
|
||||
fi
|
||||
SELF_VERSION=$("${SELF_WASM_OPT}" --version 2>&1 | grep -o '[0-9]\+' | head -1)
|
||||
if [ "${SELF_VERSION}" != "${BINARYEN_VERSION}" ]; then
|
||||
echo "ERROR: self-built wasm-opt version mismatch (got ${SELF_VERSION}, expected ${BINARYEN_VERSION})" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Using self-built Binaryen v${BINARYEN_VERSION} (${SELF_WASM_OPT})" >&2
|
||||
echo "${SELF_WASM_OPT}"
|
||||
exit 0
|
||||
fi
|
||||
BINARYEN_DIR="${PROJECT_ROOT}/build-wasm/tools/binaryen-${BINARYEN_VERSION}"
|
||||
WASM_OPT="${BINARYEN_DIR}/bin/wasm-opt"
|
||||
|
||||
|
|
|
|||
|
|
@ -105,7 +105,8 @@ TOTAL_FIXED=0
|
|||
apply_fix() { # <grep/sed pattern> <sed replacement> <label>
|
||||
local before; before=$(grep -c "$1" "$JS_FILE" || true)
|
||||
if [ "$before" -gt 0 ]; then
|
||||
sed -i '' "s/$1/$2/g" "$JS_FILE"
|
||||
# Portable in-place edit (BSD `sed -i ''` and GNU `sed -i` differ; temp+mv works on both).
|
||||
sed "s/$1/$2/g" "$JS_FILE" > "${JS_FILE}.sedtmp" && mv "${JS_FILE}.sedtmp" "$JS_FILE"
|
||||
local after; after=$(grep -c "$1" "$JS_FILE" || true)
|
||||
echo " Fixed $((before - after)) $3"
|
||||
TOTAL_FIXED=$((TOTAL_FIXED + before - after))
|
||||
|
|
|
|||
Loading…
Reference in a new issue