pcbjam/docker/build.sh

#!/bin/bash
# Build a KiCad app (pcbnew, eeschema, calculator) inside Docker, then run
# asyncify and friends on the host.
#
# Usage:
#   ./docker/build.sh <app>[,<app>...] [args...]
#
# Apps:
#   pcbnew         PCB editor
#   eeschema       schematic editor
#   calculator     PCB calculator
#   pl_editor      drawing-sheet editor
#   symbol_editor  symbol editor (eeschema kiface, FRAME_SCH_SYMBOL_EDITOR)
#   gerbview       Gerber viewer
#   all            build all of the above
#
# A comma-separated list builds just those apps in order (e.g.
# "calculator,pl_editor" — used to exercise the multi-app pipeline cheaply).
#
# Any extra args are forwarded to scripts/kicad/build-<app>.sh (e.g. -j 8,
# --full, --release, --diag=gal).
#
# The build is split into two phases:
# 1. Docker: Compile KiCad to WASM (without asyncify)
# 2. Host: dyncall shims + finalize + asyncify + -O2 (Binaryen via get-wasm-opt.sh)
#
# KICAD_PIPELINE=1 (multi-app builds only): run phase 2 of each app in the
# background while the next app compiles in the container. wasm-opt is
# Amdahl-capped at ~4 effective cores, so on a many-core CI box the container
# would otherwise sit idle for the 1-2h of host-side wasm-opt (run 27226030304:
# 103 min of the 4h was tools serialized behind each other's wasm-opt). At most
# KICAD_PIPELINE_JOBS (default 2) postprocesses run concurrently — pcbnew's -O2
# peaks ~34 GB RSS, so 2 fits the 128 GB CI box but NOT a dev Mac: leave
# KICAD_PIPELINE unset locally.
#
# Binaryen is downloaded automatically - no prerequisites needed.

# Auto-launch the live progress dashboard in this terminal (handled by logging.sh,
# which owns the TTY before it re-execs us with output redirected). Set KICAD_NO_MONITOR=1
# to disable. MUST be set before sourcing logging.sh — that's where the dashboard is
# launched, in the pre-re-exec process.
export KICAD_MONITOR=1

# Redirect all output to a log file (re-execs script with redirection).
# MUST be sourced before arg parsing — the re-exec relies on the original
# "$@", so any shifts before this point would strip args from the re-exec.
source "$(dirname "$0")/../scripts/common/logging.sh"

# Build-progress markers (parsed by scripts/build-monitor.sh).
source "$(dirname "$0")/../scripts/common/stages.sh"

set -e

# Emit a completion/failure marker no matter how the build ends, so the monitor
# can stop on a clean "done" or show an aborted state instead of hanging.
trap '_rc=$?; if [ $_rc -eq 0 ]; then kw_done; else kw_fail $_rc; fi' EXIT
# On Ctrl-C, mark the build aborted so the final dashboard frame shows failed
# (not a stale "running" state). The EXIT trap above also fires; the monitor reads
# the last marker, so the duplicate is harmless.
trap 'kw_fail 130; exit 130' INT TERM

cd "$(dirname "$0")/.."

VALID_APPS="pcbnew | eeschema | calculator | pl_editor | symbol_editor | footprint_editor | gerbview | sym_convert | all"

usage() {
    echo "Usage: ./docker/build.sh <app>[,<app>...] [args...]" >&2
    echo "  <app>: ${VALID_APPS}" >&2
    echo "  args:  forwarded to scripts/kicad/build-<app>.sh (e.g. -j 8, --release)" >&2
}

# First positional arg must be the app name. No default — picking one would
# silently build the wrong thing for someone who forgot the argument.
if [[ "${1:-}" == "-h" ]] || [[ "${1:-}" == "--help" ]]; then
    usage
    exit 0
fi
if [[ $# -lt 1 ]] || [[ "$1" == -* ]]; then
    echo "Error: missing <app> argument" >&2
    usage
    exit 1
fi
APP_NAME="$1"
shift

# Expand the app argument into APPS[]: "all", a single app, or a comma list.
# pcbnew first in "all" — its 90-min host-side wasm-opt chain is the critical
# path, so it must start as early as possible (especially with KICAD_PIPELINE=1).
if [[ "$APP_NAME" == "all" ]]; then
    APPS=(pcbnew eeschema calculator pl_editor symbol_editor footprint_editor gerbview)
else
    IFS=',' read -r -a APPS <<< "$APP_NAME"
    for app in "${APPS[@]}"; do
        case "$app" in
            pcbnew|eeschema|calculator|pl_editor|symbol_editor|footprint_editor|gerbview|sym_convert) ;;
            *)
                echo "Error: unknown app '$app' (expected: ${VALID_APPS})" >&2
                usage
                exit 1
                ;;
        esac
    done
fi

# Use branch name as Docker Compose project name for isolated containers/volumes.
# Honor a pre-set COMPOSE_PROJECT_NAME so a build can target an existing volume
# (e.g. reuse another branch's already-provisioned deps).
BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD | tr '/' '-' | tr '[:upper:]' '[:lower:]')
export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-kicad-wasm-${BRANCH_NAME}}"
echo "Using Docker project: ${COMPOSE_PROJECT_NAME}"
echo "Building app: ${APP_NAME}"

# Add -j 10 by default if no -j flag is given
ARGS=("$@")
if [[ ! " ${ARGS[*]} " =~ " -j " ]]; then
    ARGS+=("-j" "10")
fi

# Start container if not running
docker compose -f docker/docker-compose.yml up -d

# Sync source code to container volume (fixes macOS Docker VirtioFS issues)
# Use --checksum to only transfer files with different CONTENT, not timestamps.
# This avoids the timestamp mismatch cycle that caused full rebuilds every time.
# Transferred files get current container time, so make detects them correctly.
kw_stage container-sync
echo "Syncing source code to container..."
# rsync into the macOS-backed volume intermittently hits transient VirtioFS glitches:
# temp-file rename failures (exit 23) or vanished-source files (exit 24, harmless).
# --inplace avoids the temp-file+rename pattern that triggers exit 23; retry up to 3x
# for any residual flakiness (--checksum makes each retry skip already-synced files).
sync_rc=0
for sync_attempt in 1 2 3; do
    if docker compose -f docker/docker-compose.yml exec kicad-wasm-builder \
        rsync -r --delete --checksum --inplace \
            --exclude="build-wasm" \
            --exclude="output" \
            --exclude=".git" \
            --exclude="logs" \
            --exclude=".idea" \
            --exclude="node_modules" \
            --exclude="tools/emsdk" \
            /workspace-host/ /workspace/
    then
        sync_rc=0
    else
        sync_rc=$?
    fi
    { [ $sync_rc -eq 0 ] || [ $sync_rc -eq 24 ]; } && break
    echo "rsync attempt ${sync_attempt} failed (exit ${sync_rc}); retrying in 2s..."
    sleep 2
done
if [ $sync_rc -ne 0 ] && [ $sync_rc -ne 24 ]; then
    echo "ERROR: source sync failed after retries (exit ${sync_rc})"; exit 1
fi

# Map an app name to its inner CMake build subdirectory. Most apps share their
# subdir name with the app name; pcb_calculator emits OUTPUT_NAME=calculator
# but lives under the pcb_calculator/ subtree.
kicad_subdir_for() {
    case "$1" in
        calculator)       echo "pcb_calculator" ;;
        pl_editor)        echo "pagelayout_editor" ;;
        symbol_editor)    echo "eeschema" ;;
        footprint_editor) echo "pcbnew" ;;
        sym_convert)      echo "eeschema" ;;
        *)                echo "$1" ;;
    esac
}

# Phase 1 of one app: compile in the container and copy the output to ./output.
# Args: <app> [index] [total] — index/total drive the monitor's app counter.
compile_app() {
    local app="$1"
    local index="${2:-1}"
    local total="${3:-1}"
    local subdir
    subdir=$(kicad_subdir_for "$app")
    kw_app "$app" "$index" "$total"
    echo ""
    echo "=== Building ${app} (${index}/${total}) ==="

    local out_dir="output"
    local kicad_build="kicad-${app}"

    # Run build inside the container.
    # -e EMSDK=/emsdk: `docker compose exec` bypasses the entrypoint that sources
    # emsdk_env.sh, so the build shell would lack emcc/embuilder on PATH. Setting
    # EMSDK lets scripts/common/env.sh source /emsdk/emsdk_env.sh and activate the toolchain.
    docker compose -f docker/docker-compose.yml exec -e EMSDK=/emsdk \
        -e BUILD_3D_VIEWER="${BUILD_3D_VIEWER:-OFF}" \
        kicad-wasm-builder \
        "/workspace/scripts/kicad/build-${app}.sh" "${ARGS[@]}"

    # Copy output to host-accessible directory.
    # ${app}.wasm.debug.wasm contains DWARF debug info (when built with -gseparate-dwarf).
    kw_stage copy-output
    echo "Copying ${app} build output to ./${out_dir}/..."
    docker compose -f docker/docker-compose.yml exec kicad-wasm-builder \
        bash -c "mkdir -p /workspace/${out_dir} && \
            cp /workspace/build-wasm/${kicad_build}/${subdir}/${app}.{js,wasm,wasm.debug.wasm,wasm.map,worker.js} /workspace/${out_dir}/ 2>/dev/null || \
            cp /workspace/build-wasm/${kicad_build}/${subdir}/${app}.{js,wasm} /workspace/${out_dir}/; \
            cp /workspace/build-wasm/${kicad_build}/resources/images.tar.gz /workspace/${out_dir}/ 2>/dev/null || true; \
            cp /workspace/wxwidgets/build/wasm/wx.js /workspace/${out_dir}/ 2>/dev/null || true; \
            cp /workspace/wxwidgets/build/wasm/wx-dom.js /workspace/${out_dir}/ 2>/dev/null || true"

    # The container runs as root, so files in the bind-mounted ./output land
    # root-owned on the host. macOS Docker Desktop remaps ownership to the host
    # user, but on a Linux CI runner the following host-side steps (dyncall,
    # finalize, asyncify) can't write into ./output. Hand ownership back.
    docker compose -f docker/docker-compose.yml exec kicad-wasm-builder \
        chown -R "$(id -u):$(id -g)" /workspace/output || true
}

# Phase 2 of one app: host-side post-processing (dyncall shims, finalize,
# asyncify + -O2). Pure host work on output/${app}.* — independent of the
# container, which is what makes it safe to run in the background while the
# next app compiles.
postprocess_app() {
    local app="$1"
    local out_dir="output"

    # The converter is finalized in-container (real tools, small -g0 wasm) and is
    # a synchronous node CLI, so it needs no host post-processing.
    if [ "$app" = "sym_convert" ]; then
        echo "Skipping host post-processing for ${app} (finalized in-container)"
        return 0
    fi

    # Inject dynCall shims (fixes "dynCall_* is not defined" errors in Emscripten 4.x)
    kw_stage dyncall-shims
    ./scripts/common/inject-dyncall-shims.sh "${out_dir}/${app}.js"

    # Apply wasm-emscripten-finalize on host (skipped in Docker due to memory limits)
    kw_stage finalize
    ./scripts/common/apply-finalize.sh "${out_dir}/${app}.wasm" "${out_dir}/${app}.wasm"

    # Apply asyncify transformation on host. The converter is a synchronous node
    # CLI built with ASYNCIFY=0, so asyncify is unnecessary and would be wrong.
    if [ "$app" != "sym_convert" ]; then
        kw_stage asyncify
        ./scripts/common/apply-asyncify.sh "${out_dir}/${app}.wasm" "${out_dir}/${app}.wasm"
    fi
}

# --- Pipelined driver state (KICAD_PIPELINE=1) ---
# One background postprocess per app; logs + rc files land in logs/build/ so the
# interleaved output stays readable and failures survive until the final wait.
PIPELINE_PIDS=()
PIPELINE_APPS_BG=()
PIPELINE_LOG_DIR="logs/build"
PIPELINE_TS="$(date +%Y%m%d-%H%M%S)"

pipeline_running_count() {
    local n=0 pid
    for pid in "${PIPELINE_PIDS[@]}"; do
        kill -0 "$pid" 2>/dev/null && n=$((n + 1))
    done
    echo "$n"
}

# Launch postprocess_app in the background, capped at KICAD_PIPELINE_JOBS
# concurrent jobs (default 2: pcbnew's -O2 peaks ~34 GB RSS; two postprocesses
# plus the container compile fit the 128 GB CI box). Portable poll loop instead
# of `wait -n` (absent in macOS bash 3.2).
pipeline_postprocess() {
    local app="$1"
    local max_jobs="${KICAD_PIPELINE_JOBS:-2}"
    while [ "$(pipeline_running_count)" -ge "$max_jobs" ]; do
        sleep 10
    done
    local log_file="${PIPELINE_LOG_DIR}/postprocess-${app}-${PIPELINE_TS}.log"
    echo "Pipelining host-side postprocess of ${app} (log: ${log_file})"
    (
        postprocess_app "$app" >"$log_file" 2>&1
        echo $? >"${log_file}.rc"
    ) &
    PIPELINE_PIDS+=($!)
    PIPELINE_APPS_BG+=("$app")
}

# Wait for all background postprocesses, replay their logs into the main log,
# and fail if any of them failed.
pipeline_wait_all() {
    local failed=0 i pid app log_file rc
    for i in "${!PIPELINE_PIDS[@]}"; do
        pid="${PIPELINE_PIDS[$i]}"
        app="${PIPELINE_APPS_BG[$i]}"
        log_file="${PIPELINE_LOG_DIR}/postprocess-${app}-${PIPELINE_TS}.log"
        wait "$pid" || true
        rc="$(cat "${log_file}.rc" 2>/dev/null || echo 1)"
        echo ""
        echo "=== Postprocess ${app} (rc=${rc}) — ${log_file} ==="
        cat "$log_file" 2>/dev/null || true
        if [ "$rc" != "0" ]; then
            echo "ERROR: postprocess of ${app} failed (rc=${rc})"
            failed=1
        fi
    done
    return "$failed"
}

TOTAL_APPS="${#APPS[@]}"
if [[ "${KICAD_PIPELINE:-0}" == "1" ]] && [ "$TOTAL_APPS" -gt 1 ]; then
    mkdir -p "$PIPELINE_LOG_DIR"
    # Pre-warm the Binaryen download once — two concurrent postprocesses racing
    # the first download would collide on the extract/mv.
    ./scripts/common/get-wasm-opt.sh >/dev/null
    # If a compile fails, set -e aborts the script — don't leave orphaned
    # wasm-opt jobs chewing 30 GB in the background. Keep the monitor's
    # done/fail marker from the original EXIT trap (killing finished pids is a
    # no-op, so this trap is safe on the success path too).
    trap '_rc=$?; for p in "${PIPELINE_PIDS[@]}"; do kill "$p" 2>/dev/null || true; done; if [ $_rc -eq 0 ]; then kw_done; else kw_fail $_rc; fi' EXIT
    idx=1
    for app in "${APPS[@]}"; do
        compile_app "$app" "$idx" "$TOTAL_APPS"
        pipeline_postprocess "$app"
        idx=$((idx + 1))
    done
    pipeline_wait_all
else
    idx=1
    for app in "${APPS[@]}"; do
        compile_app "$app" "$idx" "$TOTAL_APPS"
        postprocess_app "$app"
        idx=$((idx + 1))
    done
fi

echo ""
echo "Build complete. Output files in ./output/"
ls -lh output/