From e14faeca8b6c93d2dc8cd4fd9d907603e6892342 Mon Sep 17 00:00:00 2001 From: Viktor Vaczi Date: Thu, 13 Aug 2026 08:39:12 +0200 Subject: [PATCH] =?UTF-8?q?jspi:=20retire=20the=20asyncify=20pipeline=20?= =?UTF-8?q?=E2=80=94=20knob,=20post-link=20tail,=20binaryen=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 8 in the parent repo. Deleted: asyncify-scheduler.js, apply-asyncify.sh, apply-finalize.sh, inject-dyncall-shims.sh, asyncify-imports/removelist.txt, the wasm-opt/finalize stub pair, scripts/binaryen-hoist-pass/ (the fork stays a dormant submodule; removal is a follow-up), bench/wasm-opt-bench.sh (README marked historical), wasm/shims/context_sleep.cpp, and the sched-context harness app + Makefile targets. PCBJAM_ASYNC_BACKEND is gone: build-wx-wasm.sh hardcodes the jspi stamp (still force-cleans pre-migration trees), build-kicad-target.sh gives editors the JSPI link surface and the CLIs nothing (they pin ASYNCIFY=0), the stub dance is replaced by an unconditional .real-restore, build-wasm-test.sh lost its whole post-link loop, docker/build.sh's postprocess is the ENV shim only, and Makefile.wasm links every app JSPI with the scheduler shim as a tracked prerequisite. pcbjam_async_policy.h keys on __EMSCRIPTEN__. jspi-scheduler.js: wxWasmMainLoopPump dropped from the wrap census (the export died with the D5 detach); inert [TRACE] instrumentation removed. CI: wasm-build.yml rewritten for the single-cache pipeline (one output cache keyed on compile inputs; post-processed bytes cached after the shim); opt_level input removed from both callers. wasm-cache-hash.mjs inputs now cover patch-env-shim.mjs + jspi-scheduler.js + jspi-exports.txt. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NDeBaKKhQztd8KiVtHuyXr --- .github/workflows/ci-ubicloud.yml | 7 +- .github/workflows/release.yml | 14 +- .github/workflows/wasm-build.yml | 196 ++-- docker/build.sh | 62 +- kicad | 2 +- scripts/bench/README.md | 6 + scripts/bench/wasm-opt-bench.sh | 114 -- scripts/binaryen-hoist-pass/build-wasm-opt.sh | 51 - .../tests/asyncify-harness.js | 36 - .../tests/delegate-orphan-suspend.wat | 32 - .../tests/delegate-orphan.wat | 60 - .../tests/nested-catchall-exit-block.wat | 49 - .../tests/nested-catchall-suspend.wat | 26 - scripts/binaryen-hoist-pass/tests/run.sh | 57 - .../tests/value-typed-cpp-catch.wat | 20 - .../tests/value-typed-suspend.wat | 10 - scripts/build-wasm-test.sh | 77 +- scripts/build-wx-wasm.sh | 22 +- scripts/common/apply-asyncify.sh | 151 --- scripts/common/apply-finalize.sh | 44 - scripts/common/asyncify-imports.txt | 20 - scripts/common/asyncify-removelist.txt | 32 - scripts/common/inject-dyncall-shims.sh | 176 --- scripts/common/jspi-exports.txt | 1 - scripts/common/shims/asyncify-scheduler.js | 801 ------------- scripts/common/shims/jspi-scheduler.js | 7 +- scripts/deploy/wasm-cache-hash.mjs | 20 +- scripts/kicad/build-kicad-target.sh | 121 +- tests/apps/Makefile.wasm | 76 +- tests/apps/standalone/jspi-coroutine/build.sh | 4 +- .../sched-context/sched_context_test.cpp | 1013 ----------------- wasm/bindings/pcbjam_async_policy.h | 14 +- wasm/shims/context_sleep.cpp | 129 --- wasm/stubs/wasm-emscripten-finalize-stub.sh | 34 - wasm/stubs/wasm-opt-stub.sh | 32 - wxwidgets | 2 +- 36 files changed, 170 insertions(+), 3348 deletions(-) delete mode 100755 scripts/bench/wasm-opt-bench.sh delete mode 100755 scripts/binaryen-hoist-pass/build-wasm-opt.sh delete mode 100644 scripts/binaryen-hoist-pass/tests/asyncify-harness.js delete mode 100644 scripts/binaryen-hoist-pass/tests/delegate-orphan-suspend.wat delete mode 100644 scripts/binaryen-hoist-pass/tests/delegate-orphan.wat delete mode 100644 scripts/binaryen-hoist-pass/tests/nested-catchall-exit-block.wat delete mode 100644 scripts/binaryen-hoist-pass/tests/nested-catchall-suspend.wat delete mode 100755 scripts/binaryen-hoist-pass/tests/run.sh delete mode 100644 scripts/binaryen-hoist-pass/tests/value-typed-cpp-catch.wat delete mode 100644 scripts/binaryen-hoist-pass/tests/value-typed-suspend.wat delete mode 100755 scripts/common/apply-asyncify.sh delete mode 100755 scripts/common/apply-finalize.sh delete mode 100644 scripts/common/asyncify-imports.txt delete mode 100644 scripts/common/asyncify-removelist.txt delete mode 100755 scripts/common/inject-dyncall-shims.sh delete mode 100644 scripts/common/shims/asyncify-scheduler.js delete mode 100644 tests/apps/standalone/sched-context/sched_context_test.cpp delete mode 100644 wasm/shims/context_sleep.cpp delete mode 100755 wasm/stubs/wasm-emscripten-finalize-stub.sh delete mode 100755 wasm/stubs/wasm-opt-stub.sh diff --git a/.github/workflows/ci-ubicloud.yml b/.github/workflows/ci-ubicloud.yml index 6ca47b5..5caf21a 100644 --- a/.github/workflows/ci-ubicloud.yml +++ b/.github/workflows/ci-ubicloud.yml @@ -3,9 +3,8 @@ name: CI full build + e2e (Ubicloud) # Main/PR gate: build all 6 KiCad WASM tools + run the wxWidgets + KiCad e2e # suites on a Ubicloud runner. The actual recipe lives in the reusable # wasm-build.yml (the SINGLE build definition) — this is just the main/PR caller. -# The asyncify wasm-opt tail is -O1 everywhere (main, release, local); the tag -# release (release.yml) calls the same recipe at the same -O1 and then publishes, -# so the build can never diverge from what ships. +# The tag release (release.yml) calls the same recipe with the same inputs and +# then publishes, so the build can never diverge from what ships. # # Routine edits under scripts/ do NOT bust the WASM output cache — only the subset # in scripts/deploy/wasm-cache-hash.mjs does. For inputs the sc-hash can't see @@ -44,9 +43,7 @@ jobs: # post-test step can post the screenshot + perf report on push to main. secrets: inherit with: - # -O1 asyncify shrink — the level we ship (release.yml uses the same -O1). # 3D viewer ON so 3d-viewer.spec.ts has a viewer. - opt_level: "-O1" build_3d_viewer: "ON" run_tests: true no_cache: ${{ inputs.no_cache || false }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8610c53..713c930 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,10 +1,9 @@ name: release # Release pipeline on a vX.Y.Z tag, in order: -# 1) build — the SAME wasm-build.yml recipe as CI, at the SAME -O1 asyncify tail -# + run the e2e gate. Because the opt level now matches main, this -# FINAL-cache-hits main's build and skips the asyncify tail rebuild. -# Uploads output/. +# 1) build — the SAME wasm-build.yml recipe as CI + the e2e gate. For an +# already-built commit this hits main's WASM output cache and +# skips the build entirely. Uploads output/. # 2) publish-wasm — push the build to the CDN (content-addressed, idempotent) # and write manifest-.json. The registry now reflects THIS tag. # 3) deploy-demo — build the standalone pinned to that manifest + deploy to @@ -69,13 +68,12 @@ jobs: - id: tag run: echo "tag=${{ github.event.inputs.tag || github.ref_name }}" >> "$GITHUB_OUTPUT" - # 1) Build (-O1, identical to main) + e2e gate. Reuses main's FINAL cache (same - # opt level) so the asyncify tail isn't rebuilt. Uploads the publishable - # output/ as the 'wasm-output' artifact for the publish job. + # 1) Build (identical to main) + e2e gate. Reuses main's WASM output cache for + # an already-built commit. Uploads the publishable output/ as the + # 'wasm-output' artifact for the publish job. build: uses: ./.github/workflows/wasm-build.yml with: - opt_level: "-O1" build_3d_viewer: "ON" run_tests: true upload_output: true diff --git a/.github/workflows/wasm-build.yml b/.github/workflows/wasm-build.yml index b139899..28d1f3c 100644 --- a/.github/workflows/wasm-build.yml +++ b/.github/workflows/wasm-build.yml @@ -1,31 +1,25 @@ name: wasm-build (reusable) # THE single KiCad WASM build+test recipe, called by ci-ubicloud.yml (main/PR) and -# release.yml (tag). Both build the SAME way — the asyncify `wasm-opt` tail is -O1 -# everywhere — so the build can never diverge from what ships (the bug that shipped -# a -O1 / 3D-off demo while CI built -O1 / 3D-on). The opt_level input remains as an -# escape hatch for a one-off -O2 build, but is -O1 for both real callers, so a tag -# release FINAL-cache-hits main's build and skips the asyncify tail entirely. +# release.yml (tag). Both build the SAME way — the only build knob is the +# 3D-viewer flag, ON for both real callers — so the build can never diverge from +# what ships (the bug that once shipped a 3D-off demo while CI built 3D-on). # -# Two-tier output cache around docker/build.sh's --compile-only / --postprocess-only -# split. The expensive container compile (→ base wasm) is opt- and binaryen- -# independent; only the host `asyncify + wasm-opt -O` tail depends on the binaryen -# fork + opt level. So: -# - BASE cache (compile-input key): the --compile-only output (base wasm + -# sysroot headers). Reused whenever only the binaryen fork / asyncify config -# changed — the compile is skipped and just the tail reruns. -# - FINAL cache (base + binaryen SHA + opt level): the post-processed output; -# fast-path for re-running the same SHA (a tag release reusing main, a re-deploy). -# Both keys include the 3D-viewer flag, so a 3D-on and 3D-off build can never -# poison each other's cache. +# ONE output cache around docker/build.sh's --compile-only / --postprocess-only +# split. Under JSPI the wasm links fully finalized in-container; the only host +# post-process left is the deterministic, seconds-cheap ENV merge shim +# (patch-env-shim.mjs). So the cache holds the post-processed output/, keyed on +# the compile inputs alone: the kicad/wx submodule SHAs, the sc source-content +# hash (scripts/deploy/wasm-cache-hash.mjs — build logic outside the +# submodules), the 3D-viewer flag (a 3D-on and 3D-off build can never poison +# each other's entry), and the manual .ci-cache-epoch escape hatch. On a hit +# the whole build is skipped (a tag release reuses main's entry for the same +# commit); on a miss the compile reruns and the shim is re-applied before the +# save. on: workflow_call: inputs: - opt_level: - description: "Binaryen wasm-opt shrink level for the asyncify tail (-O1 everywhere; escape hatch for a one-off -O2)" - type: string - default: "-O1" build_3d_viewer: description: "Build the WASM 3D viewer into kicad_editor (ON/OFF)" type: string @@ -35,7 +29,7 @@ on: type: boolean default: true no_cache: - description: "Bypass the WASM output caches (force a full rebuild this run)" + description: "Bypass the WASM output cache (force a full rebuild this run)" type: boolean default: false upload_output: @@ -60,19 +54,18 @@ jobs: timeout-minutes: 300 env: KICAD_LOG_NESTED: "1" - # Opt level for the asyncify shrink pass (the only opt-dependent step). - BINARYEN_OPT_LEVEL: ${{ inputs.opt_level }} BUILD_3D_VIEWER: ${{ inputs.build_3d_viewer }} # Stable docker-compose project → deterministic build-cache volume name. COMPOSE_PROJECT_NAME: kicad-wasm-ci steps: - - name: Install build toolchain (Binaryen from-source) + - name: Install build toolchain run: | export DEBIAN_FRONTEND=noninteractive sudo apt-get update # xvfb: kicad e2e runs headed Firefox under a virtual display. # autoconf/automake/make: host wxWidgets + wx test-app builds. - sudo apt-get install -y cmake ninja-build g++ libjemalloc2 xvfb autoconf automake make + # cmake/ninja/g++: host GAL/3D test-app builds. + sudo apt-get install -y cmake ninja-build g++ xvfb autoconf automake make - uses: actions/checkout@v4 with: { submodules: recursive } @@ -80,29 +73,27 @@ jobs: - uses: actions/setup-node@v4 with: { node-version: 20 } - # --- cache keys -------------------------------------------------------- - # base = opt-INDEPENDENT (no binaryen / opt level): docker compile out. - # final = opt-SPECIFIC: post-processed (asyncify + wasm-opt -O) out, keyed - # on the binaryen submodule SHA — the host post-process uses that fork's - # wasm-opt (--hoist-cpp-catches + --asyncify + -O), so bumping the fork - # must bust this cache — plus the opt level. - # Both include the 3D flag so 3D-on/off never share an entry. + # --- cache key --------------------------------------------------------- + # ONE key, from the compile inputs only: kicad/wx submodule SHAs, the sc + # source-content hash (wasm-cache-hash.mjs — build logic outside the + # submodules), the 3D flag (3D-on/off never share an entry), and the + # manual .ci-cache-epoch. The host post-process (ENV merge shim) is + # deterministic and cheap, so the cached bytes are the fully + # post-processed output — no separate post-process tier. - name: Compute build inputs id: keys run: | KICAD=$(git -C kicad rev-parse HEAD) WX=$(git -C wxwidgets rev-parse HEAD) - BIN=$(git -C binaryen rev-parse --short HEAD) SC=$(node scripts/deploy/wasm-cache-hash.mjs) EPOCH=$(cat .ci-cache-epoch 2>/dev/null || echo 0) EMV=$(. scripts/common/versions.sh && echo "$EMSCRIPTEN_VERSION") THREED='${{ inputs.build_3d_viewer }}' - BASE="kbase-${{ runner.os }}-k${KICAD}-wx${WX}-sc${SC}-3d${THREED}-e${EPOCH}" - FINAL="kwasm-${{ runner.os }}-bin${BIN}${{ inputs.opt_level }}-k${KICAD}-wx${WX}-sc${SC}-3d${THREED}-e${EPOCH}" + KEY="kwasm-${{ runner.os }}-k${KICAD}-wx${WX}-sc${SC}-3d${THREED}-e${EPOCH}" { echo "kicad=$KICAD"; echo "wx=$WX"; echo "sc=$SC"; echo "epoch=$EPOCH" - echo "bin=$BIN"; echo "emv=$EMV" - echo "base_key=$BASE"; echo "final_key=$FINAL" + echo "emv=$EMV" + echo "key=$KEY" } >> "$GITHUB_OUTPUT" - name: Cache control (commit message / dispatch) @@ -118,32 +109,10 @@ jobs: echo "skip=$SKIP" >> "$GITHUB_OUTPUT" echo "WASM output-cache restore skip=$SKIP" - # Binaryen post-process tools (submodule fork): otherwise built from source on - # every fresh VM (~46s on 30 cores, measured run 28577824366). Cache bin/ AND - # lib/ (the tools dynamically link lib/libbinaryen.so — bin/ alone is a loader - # error at first exec, run 28585074335), keyed on the exact submodule SHA; on a - # hit BINARYEN_TRUST_PREBUILT tells build-wasm-opt.sh to skip cmake+ninja and - # trust the restored binaries. v2: v1 entries hold a poisoned bin-only layout. - # Needed by BOTH the host post-process (final-miss path) and the test-app - # build, hence not gated on run_tests. - - name: Cache Binaryen post-process tools - id: binopt-cache - uses: actions/cache@v4 - with: - path: | - build-wasm/tools/binaryen-hoist-build/bin - build-wasm/tools/binaryen-hoist-build/lib - key: binopt-v2-${{ runner.os }}-${{ steps.keys.outputs.bin }} - - - name: Trust prebuilt Binaryen tools (cache hit) - if: steps.binopt-cache.outputs.cache-hit == 'true' - run: echo "BINARYEN_TRUST_PREBUILT=1" >> "$GITHUB_ENV" - - # The cached paths the e2e tests need: final wasms (or base, mid-build) + - # the sysroot headers the host GAL build compiles against. Same glob set for - # both tiers — only the bytes (base vs final) and the key differ. - - name: Restore FINAL WASM output cache - id: final-cache + # The cached paths the e2e tests need: the post-processed wasms + the + # sysroot headers the host GAL build compiles against. + - name: Restore WASM output cache + id: wasm-cache if: steps.cachectl.outputs.skip != 'true' uses: actions/cache/restore@v4 with: @@ -155,64 +124,46 @@ jobs: output/images.tar.gz build-wasm/sysroot/include !output/*.wasm.debug.wasm - key: ${{ steps.keys.outputs.final_key }} + key: ${{ steps.keys.outputs.key }} - # Only consulted when the final (opt-specific) cache missed: the - # opt-independent compile output, warm across -O1/-O2. - - name: Restore BASE compile cache - id: base-cache - if: steps.cachectl.outputs.skip != 'true' && steps.final-cache.outputs.cache-hit != 'true' - uses: actions/cache/restore@v4 - with: - path: | - output/*.js - output/*.wasm - output/*.wasm.map - output/*.worker.js - output/images.tar.gz - build-wasm/sysroot/include - !output/*.wasm.debug.wasm - key: ${{ steps.keys.outputs.base_key }} - - # deps (sysroot + stamps) are only needed when we must COMPILE (base miss). + # deps (sysroot + stamps) are only needed when we must COMPILE (cache miss). - name: Restore deps cache id: deps-cache - if: steps.final-cache.outputs.cache-hit != 'true' && steps.base-cache.outputs.cache-hit != 'true' + if: steps.wasm-cache.outputs.cache-hit != 'true' uses: actions/cache@v4 with: path: deps-cache key: deps-${{ runner.os }}-${{ hashFiles('scripts/deps/**','scripts/common/versions.sh','scripts/common/functions.sh','scripts/common/env.sh','docker/Dockerfile','docker/docker-compose.yml') }} - name: Seed deps volume from cache - if: steps.final-cache.outputs.cache-hit != 'true' && steps.base-cache.outputs.cache-hit != 'true' && steps.deps-cache.outputs.cache-hit == 'true' + if: steps.wasm-cache.outputs.cache-hit != 'true' && steps.deps-cache.outputs.cache-hit == 'true' run: | docker volume create kicad-wasm-ci_kicad-build-cache docker run --rm -v kicad-wasm-ci_kicad-build-cache:/bw -v "$PWD/deps-cache":/cache \ alpine sh -c 'tar xzf /cache/deps.tar.gz -C /bw' - # PHASE 1 (base miss only): container compile of all 4 bundles → output/ base - # wasm (opt-independent). 3D viewer per input. --compile-only skips the host - # asyncify/-O tail. KICAD_PIPELINE has no effect here (no post-process to - # overlap), but the deps short-circuit on a warm volume. - - name: Compile all KiCad tools (container, base wasm) - if: steps.final-cache.outputs.cache-hit != 'true' && steps.base-cache.outputs.cache-hit != 'true' + # PHASE 1 (cache miss only): container compile of all bundles → output/ + # wasm, fully finalized in-container (JSPI links with the real tools). + # 3D viewer per input. + - name: Compile all KiCad tools (container) + if: steps.wasm-cache.outputs.cache-hit != 'true' run: | export KICAD_DOCKER_CPUS="$(( $(nproc) - 1 ))" KICAD_DOCKER_MEM=110G - echo "Compiling ALL tools (base wasm), 3D viewer=${BUILD_3D_VIEWER}, -j $(nproc)" + echo "Compiling ALL tools, 3D viewer=${BUILD_3D_VIEWER}, -j $(nproc)" ./docker/build.sh all --compile-only --build-deps -j "$(nproc)" ls -lh output/*.wasm - name: Package deps for cache - if: steps.final-cache.outputs.cache-hit != 'true' && steps.base-cache.outputs.cache-hit != 'true' && steps.deps-cache.outputs.cache-hit != 'true' + if: steps.wasm-cache.outputs.cache-hit != 'true' && steps.deps-cache.outputs.cache-hit != 'true' run: | mkdir -p deps-cache docker run --rm -v kicad-wasm-ci_kicad-build-cache:/bw -v "$PWD/deps-cache":/cache \ alpine sh -c 'cd /bw && tar czf /cache/deps.tar.gz sysroot stamps' # GAL test compiles against kicad headers from the docker sysroot volume — - # export them to the host so they ride in the base cache (and thus any hit). + # export them to the host so they ride in the output cache (and thus any hit). - name: Expose docker sysroot headers to host builds - if: steps.final-cache.outputs.cache-hit != 'true' && steps.base-cache.outputs.cache-hit != 'true' + if: steps.wasm-cache.outputs.cache-hit != 'true' run: | VOL=kicad-wasm-ci_kicad-build-cache mkdir -p build-wasm/sysroot @@ -220,36 +171,18 @@ jobs: sh -c 'cp -r /bw/sysroot/include /host/' sudo chown -R "$(id -u):$(id -g)" build-wasm/sysroot - # Save the opt-independent base (compile output + headers) for cross-opt reuse. - - name: Save BASE compile cache - if: steps.final-cache.outputs.cache-hit != 'true' && steps.base-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: | - output/*.js - output/*.wasm - output/*.wasm.map - output/*.worker.js - output/images.tar.gz - build-wasm/sysroot/include - !output/*.wasm.debug.wasm - key: ${{ steps.keys.outputs.base_key }} - - # PHASE 2 (any final miss): pure-host post-process on the base wasm — - # dyncall + finalize + asyncify + `wasm-opt ${opt_level}`. The ONLY - # opt-dependent work. No container; the binaryen submodule fork's wasm-opt - # is built on demand via scripts/binaryen-hoist-pass/build-wasm-opt.sh. - - name: Host post-process (asyncify + wasm-opt ${{ inputs.opt_level }}) - if: steps.final-cache.outputs.cache-hit != 'true' + # PHASE 2 (cache miss only): the sole remaining host post-process — the + # deterministic ENV merge shim (patch-env-shim.mjs) on the editor glue. + # No container needed; the CLIs (kicad_tools/occ_service/ngspice_service) + # skip even this inside build.sh. + - name: Host post-process (ENV merge shim) + if: steps.wasm-cache.outputs.cache-hit != 'true' run: | - export KICAD_PIPELINE=1 BINARYEN_CORES=16 - echo "Post-processing ALL tools with ${BINARYEN_OPT_LEVEL}" ./docker/build.sh all --postprocess-only - echo "wasm-opt used:"; "$(./scripts/binaryen-hoist-pass/build-wasm-opt.sh 2>/dev/null)" --version || true ls -lh output/*.wasm - - name: Save FINAL WASM output cache - if: steps.final-cache.outputs.cache-hit != 'true' + - name: Save WASM output cache + if: steps.wasm-cache.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: path: | @@ -260,7 +193,7 @@ jobs: output/images.tar.gz build-wasm/sysroot/include !output/*.wasm.debug.wasm - key: ${{ steps.keys.outputs.final_key }} + key: ${{ steps.keys.outputs.key }} # --- publishable artifact (release path) ------------------------------- - name: Upload WASM output artifact @@ -313,15 +246,14 @@ jobs: if: inputs.run_tests run: ./scripts/build-wx-wasm.sh - # Built wx test apps (tests/apps): without this every run recompiles, relinks - # and — the expensive part — re-runs the hoist+asyncify post-link on all ~74 - # apps (~2m20s even 30-wide). Key = every build input: the wx lib identity - # (submodule SHA + the same script hashes as the wx cache key; the wx SHA also - # covers the Makefile's JS_FILES from wxwidgets/build/wasm), the KICAD - # submodule SHA (some apps compile real KiCad sources — thread_pool.cpp, - # libcontext, headers), the binaryen SHA (post-link wasm-opt), the app - # sources (tracked cpp/h/html + Makefile), and the build/post-link scripts + - # JS shims (wasm/** = shims + the wasm-opt stub). hashFiles runs at restore + # Built wx test apps (tests/apps): without this every run recompiles and + # relinks all ~74 apps (~2m20s even 30-wide). Key = every build input: the + # wx lib identity (submodule SHA + the same script hashes as the wx cache + # key; the wx SHA also covers the Makefile's JS_FILES from + # wxwidgets/build/wasm), the KICAD submodule SHA (some apps compile real + # KiCad sources — thread_pool.cpp, libcontext, headers), the app sources + # (tracked cpp/h/html + Makefile), and the build scripts + + # JS shims (wasm/**). hashFiles runs at restore # time, on a fresh checkout, so it sees only tracked sources — never build # outputs. On a hit the build step is skipped entirely. Excluded: # tests/apps/kicad (setup:kicad staging from output/) and gal-webgl (its own @@ -341,7 +273,7 @@ jobs: !tests/apps/kicad !tests/apps/gal-webgl !tests/apps/3d-webgl - key: testapps-${{ runner.os }}-wx${{ steps.keys.outputs.wx }}-k${{ steps.keys.outputs.kicad }}-bin${{ steps.keys.outputs.bin }}-${{ hashFiles('tests/apps/**/*.cpp', 'tests/apps/**/*.h', 'tests/apps/**/*.html', 'tests/apps/Makefile.wasm', 'scripts/build-wx-wasm.sh', 'scripts/build-wasm-test.sh', 'scripts/common/versions.sh', 'scripts/common/env.sh', 'scripts/common/functions.sh', 'scripts/common/apply-asyncify.sh', 'scripts/common/asyncify-imports.txt', 'scripts/common/inject-dyncall-shims.sh', 'scripts/common/shims/**', 'wasm/**') }} + key: testapps-${{ runner.os }}-wx${{ steps.keys.outputs.wx }}-k${{ steps.keys.outputs.kicad }}-${{ hashFiles('tests/apps/**/*.cpp', 'tests/apps/**/*.h', 'tests/apps/**/*.html', 'tests/apps/Makefile.wasm', 'scripts/build-wx-wasm.sh', 'scripts/build-wasm-test.sh', 'scripts/common/versions.sh', 'scripts/common/env.sh', 'scripts/common/functions.sh', 'scripts/common/shims/**', 'wasm/**') }} - name: Build wxWidgets test apps if: inputs.run_tests && steps.testapps-cache.outputs.cache-hit != 'true' @@ -429,7 +361,7 @@ jobs: run: npm run setup:kicad # ONE merged Playwright invocation for every apps-server suite (wx, - # kicad×2 engines, asyncify, coroutine — see playwright.config.ts). One + # kicad×2 engines, jspi, coroutine — see playwright.config.ts). One # invocation = one start-of-run outputDir wipe BEFORE anything rendered, # so the engine-scoped screenshots in test-results/{chromium,firefox}/ # accumulate naturally for the offline compare. A failure fails the JOB diff --git a/docker/build.sh b/docker/build.sh index 4e8f48b..04a14e6 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -1,6 +1,6 @@ #!/bin/bash # Build a KiCad app (pcbnew, eeschema, calculator) inside Docker, then run -# asyncify and friends on the host. +# the ENV shim on the host. # # Usage: # ./docker/build.sh [,...] [args...] @@ -22,8 +22,8 @@ # --full, --release, --diag=gal). # # The build is split into two phases: -# 1. Docker: Compile KiCad to WASM (without asyncify) -# 2. Host: dyncall shims + finalize + asyncify + wasm-opt -O1 (Binaryen submodule via build-wasm-opt.sh) +# 1. Docker: Compile KiCad to WASM (fully finalized; JSPI links in-container) +# 2. Host: ENV merge shim on the glue (patch-env-shim.mjs) # # KICAD_PIPELINE=1 (multi-app builds only): run phase 2 of each app in the # background while the next app compiles in the container. wasm-opt is @@ -137,13 +137,11 @@ echo "Building app: ${APP_NAME}" # Build phase (cache split). The container compile produces an OPT-INDEPENDENT # base wasm; the only opt-DEPENDENT work is the final `wasm-opt -O$LEVEL` shrink -# inside the host post-process (apply-asyncify.sh). Splitting them lets CI cache -# the expensive compile once (shared regardless of the asyncify tail) and re-run just the -# asyncify/-O tail per opt level — see .github/workflows/. +# Splitting compile and postprocess lets CI cache the expensive compile once +# and re-run just the host tail — see .github/workflows/. # (default) both — compile in-container, then host post-process. # --compile-only — only the in-container compile → base wasm in output/. -# --postprocess-only — only the host post-process (dyncall + finalize + -# asyncify + wasm-opt -O$BINARYEN_OPT_LEVEL) on the +# --postprocess-only — only the host post-process (ENV merge shim) on the # existing output/ base wasm; NO container needed # (build-wasm-opt.sh self-provisions the Binaryen submodule). # Extracted here so they are NOT forwarded to the inner build-.sh scripts. @@ -272,14 +270,14 @@ compile_app() { # The container runs as root, so files in the bind-mounted ./output land # root-owned on the host. macOS Docker Desktop remaps ownership to the host - # user, but on a Linux CI runner the following host-side steps (dyncall, - # finalize, asyncify) can't write into ./output. Hand ownership back. + # user, but on a Linux CI runner the host-side ENV-shim step can't write + # into ./output. Hand ownership back. docker compose -f docker/docker-compose.yml exec kicad-wasm-builder \ chown -R "$(id -u):$(id -g)" /workspace/output || true } -# Phase 2 of one app: host-side post-processing (dyncall shims, finalize, -# asyncify + wasm-opt -O1). Pure host work on output/${app}.* — independent of the +# Phase 2 of one app: host-side post-processing (ENV merge shim). Pure host +# work on output/${app}.* — independent of the # container, which is what makes it safe to run in the background while the # next app compiles. postprocess_app() { @@ -294,36 +292,12 @@ postprocess_app() { return 0 fi - # JSPI backend: the app links with the real in-container tools and needs no - # dyncall shims (no -sDYNCALLS), no host finalize, and no asyncify pass — - # the scheduler ships as a --pre-js at link. Only the ENV merge shim remains. - if [ "${PCBJAM_ASYNC_BACKEND:-asyncify}" = "jspi" ]; then - kw_stage env-shim - node ./scripts/common/patch-env-shim.mjs "${out_dir}/${app}.js" - return 0 - fi - - # Inject dynCall shims (fixes "dynCall_* is not defined" errors in Emscripten 4.x) - kw_stage dyncall-shims - ./scripts/common/inject-dyncall-shims.sh "${out_dir}/${app}.js" - - # Merge Module.ENV into the runtime ENV: the emscripten glue never merges it, - # so ?trace= (boot.ts sets Module.ENV.KICAD_TRACE) was a silent no-op — - # environ_get on the app pthread proxies to the main thread, whose ENV stayed - # empty. Replaces a manual per-build glue edit. Idempotent; runtime no-op when - # Module.ENV is unset. See docs/features/libs/0013. + # JSPI: the app links fully finalized with the real in-container tools — + # no dyncall shims, no host finalize, no asyncify pass. Only the ENV merge + # shim remains: the emscripten glue never merges Module.ENV into the + # runtime ENV (?trace= would be a silent no-op — see docs/features/libs/0013). + kw_stage env-shim node ./scripts/common/patch-env-shim.mjs "${out_dir}/${app}.js" - - # Apply wasm-emscripten-finalize on host (skipped in Docker due to memory limits) - kw_stage finalize - ./scripts/common/apply-finalize.sh "${out_dir}/${app}.wasm" "${out_dir}/${app}.wasm" - - # Apply asyncify transformation on host (the ASYNCIFY=0 CLIs returned - # early above and never reach this). - # apply-asyncify always runs the --hoist-cpp-catches pass FIRST (native wasm-EH is the only build - # mode) so Asyncify can suspend from inside C++ catch arms, then asyncify + removelist + wasm-opt -O1. - kw_stage asyncify - ./scripts/common/apply-asyncify.sh "${out_dir}/${app}.wasm" "${out_dir}/${app}.wasm" } # --- Pipelined driver state (KICAD_PIPELINE=1) --- @@ -403,8 +377,6 @@ elif [[ "$PHASE" == "postprocess" ]]; then # wasm (no container). Parallelize across apps when pipelining. if [[ "${KICAD_PIPELINE:-0}" == "1" ]] && [ "$TOTAL_APPS" -gt 1 ]; then mkdir -p "$PIPELINE_LOG_DIR" - kw_stage binaryen - ./scripts/binaryen-hoist-pass/build-wasm-opt.sh >/dev/null # pre-warm Binaryen (submodule) once _install_pipeline_trap for app in "${APPS[@]}"; do pipeline_postprocess "$app" @@ -419,10 +391,6 @@ elif [[ "${KICAD_PIPELINE:-0}" == "1" ]] && [ "$TOTAL_APPS" -gt 1 ]; then # both, pipelined: overlap app[i+1]'s container compile with app[i]'s host # post-process (KICAD_PIPELINE=1). mkdir -p "$PIPELINE_LOG_DIR" - # Pre-build the Binaryen submodule once — two concurrent postprocesses racing - # the first from-source build would collide. - kw_stage binaryen - ./scripts/binaryen-hoist-pass/build-wasm-opt.sh >/dev/null _install_pipeline_trap idx=1 for app in "${APPS[@]}"; do diff --git a/kicad b/kicad index a261e9f..09163c0 160000 --- a/kicad +++ b/kicad @@ -1 +1 @@ -Subproject commit a261e9f0d0913b20dde3b0ac7b1271ee00578c79 +Subproject commit 09163c0a1a0323045def89bbaca86ae967e08c8f diff --git a/scripts/bench/README.md b/scripts/bench/README.md index 28c03fa..493ae74 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -1,3 +1,9 @@ +> **Historical (asyncify era).** These benches timed the Binaryen +> `apply-asyncify` post-link tail, which the JSPI migration deleted — the +> scripts below that reference it are gone. The VM provisioning pieces +> (setup-vm.sh, vm-build.sh, cloud-init) remain useful for any host-side +> build benching. + # wasm-opt allocator/core benchmark Fast, local feedback loop for the CI perf issue: the host-side `wasm-opt`/asyncify diff --git a/scripts/bench/wasm-opt-bench.sh b/scripts/bench/wasm-opt-bench.sh deleted file mode 100755 index 0e5a53e..0000000 --- a/scripts/bench/wasm-opt-bench.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash -# wasm-opt allocator/core benchmark — RUNS INSIDE THE LINUX VM. -# -# Times the host-side wasm-opt/asyncify pass (scripts/common/apply-asyncify.sh) -# over a prebuilt eeschema .wasm across a matrix of {glibc, jemalloc} x core -# counts, to find why the step is slow on glibc CI and what BINARYEN_CORES helps. -# -# Why this isolates the right thing: wasm-opt/asyncify is a standalone pass over -# an already-compiled .wasm (see docker/build.sh:194). We never compile KiCad -# here — we just replay the optimizer over a fixture built once on the host. -# -# Usage (in the VM, from the repo root): -# ./scripts/bench/wasm-opt-bench.sh [fixture.wasm] -# Env: -# CORES="1 4 8 10" core counts to sweep (BINARYEN_CORES) -# ALLOCS="glibc jemalloc" -# STRACE=1 also run a syscall-count pass per allocator at max cores -# -# Output: a CSV table on stdout (also tee'd to bench/results.csv) plus per-cell -# logs under bench/results/ (each holds apply-asyncify's own per-pass `time -v`). - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO="$(cd "${SCRIPT_DIR}/../.." && pwd)" - -FIXTURE="${1:-${REPO}/bench/eeschema.finalized.wasm}" -CORES="${CORES:-1 4 8 10}" -ALLOCS="${ALLOCS:-glibc jemalloc}" -OUTDIR="${REPO}/bench/results" -CSV="${REPO}/bench/results.csv" - -if [[ ! -f "${FIXTURE}" ]]; then - echo "ERROR: fixture not found: ${FIXTURE}" >&2 - echo "Create it on the host (see scripts/bench/README.md) and scp it in." >&2 - exit 1 -fi -if [[ "$(uname -s)" != "Linux" ]]; then - echo "ERROR: run this inside the Linux VM (glibc is the point); host is $(uname -s)." >&2 - exit 1 -fi -command -v /usr/bin/time >/dev/null || { echo "ERROR: install GNU time (apt-get install -y time)" >&2; exit 1; } - -mkdir -p "${OUTDIR}" -echo "cores,alloc,wall_clock,wall_s,peak_rss_kb,preload" > "${CSV}" - -# Convert GNU time's "Elapsed (wall clock)" field ([h:]m:ss[.ss]) to seconds. -to_seconds() { - awk -F: '{ if (NF==3) print $1*3600+$2*60+$3; else if (NF==2) print $1*60+$2; else print $1 }' -} - -run_cell() { - local cores="$1" alloc="$2" - local logf="${OUTDIR}/${alloc}-c${cores}.log" - local timef="${OUTDIR}/${alloc}-c${cores}.time" - - cp "${FIXTURE}" /tmp/bench-in.wasm - - # glibc baseline forces no preload; jemalloc leaves WASM_OPT_PRELOAD unset so - # apply-asyncify.sh auto-detects the system libjemalloc. - local -a env_prefix=(BINARYEN_CORES="${cores}") - if [[ "${alloc}" == "glibc" ]]; then - env_prefix+=(WASM_OPT_PRELOAD=none) - fi - - echo ">>> ${alloc} BINARYEN_CORES=${cores}" >&2 - if ! env "${env_prefix[@]}" /usr/bin/time -v -o "${timef}" \ - "${REPO}/scripts/common/apply-asyncify.sh" /tmp/bench-in.wasm /tmp/bench-out.wasm \ - >"${logf}" 2>&1; then - echo " FAILED (see ${logf})" >&2 - echo "${cores},${alloc},FAILED,,," >> "${CSV}" - return 0 - fi - - local wall maxrss preload wall_s - wall=$(grep -F "Elapsed (wall clock)" "${timef}" | awk '{print $NF}') - maxrss=$(grep -F "Maximum resident set size" "${timef}" | awk '{print $NF}') - preload=$(grep -m1 -F "LD_PRELOAD=" "${logf}" | sed 's/.*LD_PRELOAD=//' | tr -d ' ') - wall_s=$(printf '%s' "${wall}" | to_seconds) - echo " wall=${wall} (${wall_s}s) peakRSS=${maxrss}KB preload=${preload}" >&2 - echo "${cores},${alloc},${wall},${wall_s},${maxrss},${preload}" >> "${CSV}" -} - -for c in ${CORES}; do - for a in ${ALLOCS}; do - run_cell "${c}" "${a}" - done -done - -# Optional: confirm the futex storm collapses with jemalloc. strace -c adds heavy -# overhead, so this is a separate, single-pass-per-allocator measurement at the -# highest core count, not part of the timing matrix above. -if [[ "${STRACE:-0}" == "1" ]]; then - command -v strace >/dev/null || { echo "strace not installed; skipping" >&2; STRACE=0; } -fi -if [[ "${STRACE:-0}" == "1" ]]; then - maxc="$(echo ${CORES} | tr ' ' '\n' | sort -n | tail -1)" - WASM_OPT="$("${REPO}/scripts/common/get-wasm-opt.sh" 2>/dev/null)" - for a in ${ALLOCS}; do - cp "${FIXTURE}" /tmp/bench-in.wasm - local_preload="" - [[ "${a}" == "jemalloc" ]] && local_preload="$(ls /usr/lib/$(uname -m)-linux-gnu/libjemalloc.so.2 2>/dev/null || true)" - echo ">>> strace ${a} (asyncify pass, BINARYEN_CORES=${maxc})" >&2 - env BINARYEN_CORES="${maxc}" ${local_preload:+LD_PRELOAD=${local_preload}} \ - strace -f -c -e trace=futex,mmap,munmap -o "${OUTDIR}/strace-${a}.txt" \ - "${WASM_OPT}" --asyncify /tmp/bench-in.wasm -o /tmp/bench-out.wasm \ - >"${OUTDIR}/strace-${a}.log" 2>&1 || echo " strace ${a} failed (see log)" >&2 - done - echo "strace summaries: ${OUTDIR}/strace-*.txt" >&2 -fi - -echo "" -echo "=== results (${CSV}) ===" -column -t -s, "${CSV}" diff --git a/scripts/binaryen-hoist-pass/build-wasm-opt.sh b/scripts/binaryen-hoist-pass/build-wasm-opt.sh deleted file mode 100755 index 97e4bcc..0000000 --- a/scripts/binaryen-hoist-pass/build-wasm-opt.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# Build the host post-process Binaryen tools from our submodule: wasm-opt (with the -# catch-arm-hoisting pass, --hoist-cpp-catches) AND wasm-emscripten-finalize. -# -# Source of truth is the tracked Binaryen submodule (binaryen/, branch wasm-port = -# upstream version_130 + src/passes/HoistCppCatches.cpp). This configures an out-of-source -# build into the gitignored build-wasm/ tree and prints the wasm-opt path on stdout -# (build progress to stderr); wasm-emscripten-finalize lands next to it in the same bin/ -# (apply-finalize.sh derives it from the wasm-opt dir). Building both from one submodule -# keeps finalize and asyncify on a single Binaryen version and removes the host emsdk -# dependency from the post-process. See docs/features/wasm-exceptions/06-spike-plan.md (Phase 1.5). -set -eo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -SRC="${PROJECT_ROOT}/binaryen" -BUILD="${PROJECT_ROOT}/build-wasm/tools/binaryen-hoist-build" - -if [ ! -f "${SRC}/src/passes/HoistCppCatches.cpp" ]; then - echo "ERROR: binaryen submodule is missing the hoist pass." >&2 - echo " Run: git submodule update --init binaryen" >&2 - exit 1 -fi - -# CI fast-path: the workflow cache-restores bin/ + lib/ keyed on the exact submodule -# SHA and sets this var on a hit, so the restored binaries are authoritative — skip -# cmake+ninja. Trust only if both tools actually RUN: the binaries dynamically link -# lib/libbinaryen.so, so an existence check alone passes on an incomplete restore -# (bin/ without lib/ shipped a red main, run 28585074335) while --version proves the -# loader resolves everything. Never set the var locally when iterating on the pass: -# uncommitted source edits would be silently ignored (the SHA key can't see them). -if [ "${BINARYEN_TRUST_PREBUILT:-0}" = "1" ] \ - && "${BUILD}/bin/wasm-opt" --version >/dev/null 2>&1 \ - && "${BUILD}/bin/wasm-emscripten-finalize" --version >/dev/null 2>&1; then - echo "Using prebuilt Binaryen tools (BINARYEN_TRUST_PREBUILT=1): ${BUILD}/bin" >&2 - echo "${BUILD}/bin/wasm-opt" - exit 0 -fi - -# Configure once (mirrors scripts/common/get-wasm-opt.sh's from-source flags). -if [ ! -f "${BUILD}/build.ninja" ]; then - echo "Configuring Binaryen submodule build (one-time, ~5 min to build)..." >&2 - cmake -S "${SRC}" -B "${BUILD}" -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_FLAGS="-Wno-maybe-uninitialized" \ - -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON -DBUILD_TESTS=OFF >&2 -fi - -ninja -C "${BUILD}" wasm-opt wasm-emscripten-finalize >&2 - -echo "${BUILD}/bin/wasm-opt" diff --git a/scripts/binaryen-hoist-pass/tests/asyncify-harness.js b/scripts/binaryen-hoist-pass/tests/asyncify-harness.js deleted file mode 100644 index 133acec..0000000 --- a/scripts/binaryen-hoist-pass/tests/asyncify-harness.js +++ /dev/null @@ -1,36 +0,0 @@ -// Minimal Asyncify harness: drive one unwind/rewind through $vt and check it yields 50. -const fs = require('fs'); -const path = process.argv[2]; -const BUF = 16, STACK = 1024, STACK_END = 16384; -let inst, rewinding = false, pending = 0; - -const imports = { - env: { - sleep: (ms) => { - if (!rewinding) { - inst.exports.asyncify_start_unwind(BUF); - pending = ms; - return 0; // dummy; value is discarded as we unwind - } else { - inst.exports.asyncify_stop_rewind(); - rewinding = false; - return pending; // real value supplied on rewind - } - }, - }, -}; - -const mod = new WebAssembly.Module(fs.readFileSync(path)); -inst = new WebAssembly.Instance(mod, imports); -// asyncify buffer struct: [current, end] -const mem = new Int32Array(inst.exports.memory.buffer); -mem[BUF >> 2] = STACK; -mem[(BUF + 4) >> 2] = STACK_END; - -inst.exports.vt(); // runs, throws, catch calls sleep -> starts unwind, returns -inst.exports.asyncify_stop_unwind(); -inst.exports.asyncify_start_rewind(BUF); -rewinding = true; -const r = inst.exports.vt(); // rewinds into the catch handler; sleep returns 50; vt completes -console.log('vt result =', r); -process.exit(r === 50 ? 0 : 1); diff --git a/scripts/binaryen-hoist-pass/tests/delegate-orphan-suspend.wat b/scripts/binaryen-hoist-pass/tests/delegate-orphan-suspend.wat deleted file mode 100644 index 101a052..0000000 --- a/scripts/binaryen-hoist-pass/tests/delegate-orphan-suspend.wat +++ /dev/null @@ -1,32 +0,0 @@ -;; The DuplicateSymbol delegate-orphan shape, but the hoisted cpp arm SUSPENDS (calls an async import) -;; before its __cxa_end_catch cleanup (try (do ..) (delegate $M)). Drives a real Asyncify unwind/rewind -;; through the hoisted arm AND its retargeted delegate ($M -> caller), proving the fix doesn't break -;; the suspend/rewind the hoist exists for. Same harness contract as the other -suspend tests: $vt -;; yields 50. -(module - (import "env" "sleep" (func $sleep (param i32) (result i32))) - (memory (export "memory") 1) - (tag $cpp (param i32)) - (func $vt (export "vt") (result i32) - (local $r i32) - (try $A - (do - (throw $cpp (i32.const 0))) - (catch_all - (try $M - (do - (try $inner - (do - (rethrow $A)) - (catch $cpp - (drop (pop i32)) - (local.set $r (call $sleep (i32.const 50))) - (try - (do (nop)) - (delegate $M))) - (catch_all - (rethrow $A)))) - (catch_all - (rethrow $A))))) - (local.get $r)) -) diff --git a/scripts/binaryen-hoist-pass/tests/delegate-orphan.wat b/scripts/binaryen-hoist-pass/tests/delegate-orphan.wat deleted file mode 100644 index 7fae585..0000000 --- a/scripts/binaryen-hoist-pass/tests/delegate-orphan.wat +++ /dev/null @@ -1,60 +0,0 @@ -;; Regression repro of SYMBOL_EDIT_FRAME::DuplicateSymbol (KiCad eeschema). A cpp catch arm that is -;; hoisted PAST an ancestor catch_all (the case-6 deferral) carries a nested __cxa_end_catch cleanup -;; (try (do ..) (delegate $M)) whose delegate target $M is a mid try sitting INSIDE that ancestor -;; catch_all. Hoisting the arm out (into the dispatch section, outside every try) orphaned the -;; delegate -> wasm-validator "all delegate targets must be valid, on (delegate $M)". The fix -;; retargets it to DELEGATE_CALLER_TARGET (a delegate can only target a try or the caller, not the -;; $done block); re-throwing a cleanup exception to the caller is the C++ throw-during-cleanup -;; (std::terminate) path, never taken in normal flow. cpp tag = single i32. -(module - (tag $cpp (param i32)) - - ;; Exception path: $A throws cpp(1); its catch_all (re)throws into $inner whose cpp arm sets r:=42, - ;; its __cxa_end_catch cleanup delegates to the enclosing $M. expect 42. - (func $caught (export "caught") (result i32) - (local $r i32) - (try $A - (do - (throw $cpp (i32.const 1))) - (catch_all - (try $M - (do - (try $inner - (do - (rethrow $A)) - (catch $cpp - (drop (pop i32)) - (local.set $r (i32.const 42)) - (try - (do (nop)) - (delegate $M))) - (catch_all - (rethrow $A)))) - (catch_all - (rethrow $A))))) - (local.get $r)) - - ;; No-exception path: $A body falls through with r:=7. expect 7. - (func $normal (export "normal") (result i32) - (local $r i32) - (try $A - (do - (local.set $r (i32.const 7))) - (catch_all - (try $M - (do - (try $inner - (do - (rethrow $A)) - (catch $cpp - (drop (pop i32)) - (local.set $r (i32.const 42)) - (try - (do (nop)) - (delegate $M))) - (catch_all - (rethrow $A)))) - (catch_all - (rethrow $A))))) - (local.get $r)) -) diff --git a/scripts/binaryen-hoist-pass/tests/nested-catchall-exit-block.wat b/scripts/binaryen-hoist-pass/tests/nested-catchall-exit-block.wat deleted file mode 100644 index 7c16f2e..0000000 --- a/scripts/binaryen-hoist-pass/tests/nested-catchall-exit-block.wat +++ /dev/null @@ -1,49 +0,0 @@ -;; Regression repro of PGM_BASE::HandleException (KiCad) — the case-6 shape (a cpp catch nested in an -;; outer try's catch_all cleanup pad) WITH an intervening block in that cleanup that the cpp arm -;; br's to. LLVM emits this for `try{} catch(A&) catch(B&) catch(...)`: the outer try has only a -;; catch_all (destructor cleanup), inside which it (rethrow)s into a nested try whose cpp catch does -;; the __cxa type dispatch and, when done, (br)s OUT to a block ($blk) sitting between the escape -;; target and the arm. Hoisting that arm to the dispatch section orphaned the br — "all break targets -;; must be valid, on (br $blk)". cpp tag = single i32. -(module - (tag $cpp (param i32)) - - ;; Exception path: outer body throws cpp(1); the catch_all reclassifies via (rethrow $outer) inside - ;; (block $blk); the nested cpp arm handles it (r := 42) then (br $blk) to finish. expect 42. - (func $caught (export "caught") (result i32) - (local $r i32) - (try $outer - (do - (throw $cpp (i32.const 1))) - (catch_all - (block $blk - (try - (do - (rethrow $outer)) - (catch $cpp - (drop (pop i32)) - (local.set $r (i32.const 42)) - (br $blk)) - (catch_all - (rethrow $outer)))))) - (local.get $r)) - - ;; No-exception path: body falls through with r := 7, neither catch runs. expect 7. - (func $normal (export "normal") (result i32) - (local $r i32) - (try $outer - (do - (local.set $r (i32.const 7))) - (catch_all - (block $blk - (try - (do - (rethrow $outer)) - (catch $cpp - (drop (pop i32)) - (local.set $r (i32.const 42)) - (br $blk)) - (catch_all - (rethrow $outer)))))) - (local.get $r)) -) diff --git a/scripts/binaryen-hoist-pass/tests/nested-catchall-suspend.wat b/scripts/binaryen-hoist-pass/tests/nested-catchall-suspend.wat deleted file mode 100644 index a015509..0000000 --- a/scripts/binaryen-hoist-pass/tests/nested-catchall-suspend.wat +++ /dev/null @@ -1,26 +0,0 @@ -;; The HandleException nested-catchall-exit-block shape, but the nested cpp arm SUSPENDS (calls an -;; async import) before it br's the intervening block. This drives a real Asyncify unwind/rewind -;; through the hoisted arm AND its retargeted br ($blk -> $done), proving the fix doesn't break the -;; suspend/rewind the hoist exists for. Same harness contract as value-typed-suspend.wat: $vt yields 50. -(module - (import "env" "sleep" (func $sleep (param i32) (result i32))) - (memory (export "memory") 1) - (tag $cpp (param i32)) - (func $vt (export "vt") (result i32) - (local $r i32) - (try $outer - (do - (throw $cpp (i32.const 0))) - (catch_all - (block $blk - (try - (do - (rethrow $outer)) - (catch $cpp - (drop (pop i32)) - (local.set $r (call $sleep (i32.const 50))) - (br $blk)) - (catch_all - (rethrow $outer)))))) - (local.get $r)) -) diff --git a/scripts/binaryen-hoist-pass/tests/run.sh b/scripts/binaryen-hoist-pass/tests/run.sh deleted file mode 100755 index 2dc74ec..0000000 --- a/scripts/binaryen-hoist-pass/tests/run.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -# Regression test for the value-typed (concrete-result) path of --hoist-cpp-catches. -# -# Value-typed cpp-catch tries do not arise from normal C++ EH lowering (LLVM keeps catch values in -# locals → void/unreachable tries), so this case can't live in a C++ EH toy. These -# hand-written modules exercise it directly: -# (1) fuzz-exec — the pass must preserve the result value of value-typed cpp-catch tries -# (exception path, no-exception path, and exception-payload routing). -# (2) a real asyncify unwind/rewind through a value-typed catch that SUSPENDS (must yield 50). -# -# Requires wasm-opt built from the binaryen submodule (scripts/binaryen-hoist-pass/build-wasm-opt.sh); -# that one binary (version_130 + our hoist pass) does both --hoist-cpp-catches and --asyncify. Run from anywhere. -set -euo pipefail -ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" -DIR="$ROOT/scripts/binaryen-hoist-pass/tests" -WASMOPT="$ROOT/build-wasm/tools/binaryen-hoist-build/bin/wasm-opt" -V130="$WASMOPT" # the submodule fork IS version_130 (asyncify unchanged), so it does --asyncify too -[ -x "$WASMOPT" ] || { echo "build wasm-opt first: scripts/binaryen-hoist-pass/build-wasm-opt.sh"; exit 1; } - -echo "== (1) value semantics preserved (fuzz-exec) ==" -"$WASMOPT" --hoist-cpp-catches -all -all --fuzz-exec "$DIR/value-typed-cpp-catch.wat" -o /dev/null 2>&1 \ - | grep -E 'comparing|=>' - -echo "== (2) asyncify unwind/rewind through a value-typed suspending catch ==" -"$WASMOPT" --hoist-cpp-catches -all -all "$DIR/value-typed-suspend.wat" -o /tmp/vt_s.hoisted.wasm -"$V130" --asyncify -all --pass-arg=asyncify-imports@env.sleep /tmp/vt_s.hoisted.wasm -o /tmp/vt_s.async.wasm -node "$DIR/asyncify-harness.js" /tmp/vt_s.async.wasm -echo "OK — value-typed path verified" - -# Regression for the PGM_BASE::HandleException shape: a cpp catch nested in an outer try's catch_all -# cleanup pad whose arm br's to an intervening block ($blk). Before the fix the hoisted arm's br was -# orphaned -> "all break targets must be valid". (3) validates + checks value semantics; (4) drives a -# real unwind/rewind through the hoisted arm with the retargeted br. -echo "== (3) nested-catchall exit-block hoists + validates (fuzz-exec: caught=>42, normal=>7) ==" -"$WASMOPT" --hoist-cpp-catches -all -all --fuzz-exec "$DIR/nested-catchall-exit-block.wat" -o /dev/null 2>&1 \ - | grep -E 'comparing|=>' - -echo "== (4) asyncify unwind/rewind through a suspending nested-catchall arm ==" -"$WASMOPT" --hoist-cpp-catches -all -all "$DIR/nested-catchall-suspend.wat" -o /tmp/ncb_s.hoisted.wasm -"$V130" --asyncify -all --pass-arg=asyncify-imports@env.sleep /tmp/ncb_s.hoisted.wasm -o /tmp/ncb_s.async.wasm -node "$DIR/asyncify-harness.js" /tmp/ncb_s.async.wasm -echo "OK — nested-catchall path verified" - -# Regression for the SYMBOL_EDIT_FRAME::DuplicateSymbol shape: a cpp catch arm hoisted PAST an ancestor -# catch_all carries a nested __cxa_end_catch cleanup (try (do ..) (delegate $M)) whose delegate target -# sits inside that catch_all -> orphaned by hoisting ("all delegate targets must be valid"). The fix -# retargets it to DELEGATE_CALLER_TARGET. (5) validates + checks value semantics; (6) drives a real -# unwind/rewind through the hoisted arm with the retargeted delegate. -echo "== (5) delegate-orphan hoists + validates (fuzz-exec: caught=>42, normal=>7) ==" -"$WASMOPT" --hoist-cpp-catches -all -all --fuzz-exec "$DIR/delegate-orphan.wat" -o /dev/null 2>&1 \ - | grep -E 'comparing|=>' - -echo "== (6) asyncify unwind/rewind through a suspending delegate-orphan arm ==" -"$WASMOPT" --hoist-cpp-catches -all -all "$DIR/delegate-orphan-suspend.wat" -o /tmp/dlg_s.hoisted.wasm -"$V130" --asyncify -all --pass-arg=asyncify-imports@env.sleep /tmp/dlg_s.hoisted.wasm -o /tmp/dlg_s.async.wasm -node "$DIR/asyncify-harness.js" /tmp/dlg_s.async.wasm -echo "OK — delegate-orphan path verified" diff --git a/scripts/binaryen-hoist-pass/tests/value-typed-cpp-catch.wat b/scripts/binaryen-hoist-pass/tests/value-typed-cpp-catch.wat deleted file mode 100644 index 539b064..0000000 --- a/scripts/binaryen-hoist-pass/tests/value-typed-cpp-catch.wat +++ /dev/null @@ -1,20 +0,0 @@ -;; Hand-written value-typed cpp-catch tries to exercise the $result routing in --hoist-cpp-catches. -;; cpp tag = single i32 param. Each function returns an i32 via a (try (result i32) ...). -(module - (tag $cpp (param i32)) - ;; exception path: body throws, catch yields 42 -> expect 42 - (func $vt_throw (export "vt_throw") (result i32) - (try (result i32) - (do (throw $cpp (i32.const 99))) - (catch $cpp (drop (pop i32)) (i32.const 42)))) - ;; normal path: body yields 7, catch never runs -> expect 7 - (func $vt_normal (export "vt_normal") (result i32) - (try (result i32) - (do (i32.const 7)) - (catch $cpp (drop (pop i32)) (i32.const 42)))) - ;; payload routing: catch returns the exception payload it caught (123) -> expect 123 - (func $vt_payload (export "vt_payload") (result i32) - (try (result i32) - (do (throw $cpp (i32.const 123))) - (catch $cpp (pop i32)))) -) diff --git a/scripts/binaryen-hoist-pass/tests/value-typed-suspend.wat b/scripts/binaryen-hoist-pass/tests/value-typed-suspend.wat deleted file mode 100644 index 06116e8..0000000 --- a/scripts/binaryen-hoist-pass/tests/value-typed-suspend.wat +++ /dev/null @@ -1,10 +0,0 @@ -;; Value-typed try whose catch SUSPENDS: the catch calls an async import and yields its result. -;; Drives the $result routing across a real asyncify unwind/rewind. -(module - (import "env" "sleep" (func $sleep (param i32) (result i32))) - (memory (export "memory") 1) - (tag $cpp (param i32)) - (func $vt (export "vt") (result i32) - (try (result i32) - (do (throw $cpp (i32.const 0))) - (catch $cpp (drop (pop i32)) (call $sleep (i32.const 50)))))) diff --git a/scripts/build-wasm-test.sh b/scripts/build-wasm-test.sh index 093fd9b..e951b0a 100755 --- a/scripts/build-wasm-test.sh +++ b/scripts/build-wasm-test.sh @@ -96,36 +96,12 @@ if [ "$CLEAN_BUILD" = "1" ]; then make -f Makefile.wasm clean 2>/dev/null || true fi -# Native wasm-EH is the only build mode. The emsdk-bundled Binaryen v121 crashes asyncifying wasm-EH, -# so we stub the in-link Asyncify and run --hoist-cpp-catches + --asyncify post-link on the Binaryen -# submodule (version_130 + hoist pass) via apply-asyncify.sh. -EMSDK_WASM_OPT="$PROJECT_ROOT/tools/emsdk/upstream/bin/wasm-opt" -WASMOPT_STUB="$PROJECT_ROOT/wasm/stubs/wasm-opt-stub.sh" -_eh_restore_wasmopt() { if [ -f "${EMSDK_WASM_OPT}.ehbak" ]; then mv -f "${EMSDK_WASM_OPT}.ehbak" "${EMSDK_WASM_OPT}"; fi; } -EH_MARKER="$(mktemp)" # created before the build so 'find -newer' below selects freshly-linked apps -if [ "${PCBJAM_ASYNC_BACKEND:-asyncify}" = "jspi" ]; then - # JSPI: no binaryen instrumentation exists — no fork build, no stub dance, - # no post-link pass, no dyncall/scheduler injection (the jspi-scheduler - # ships as a --pre-js from Makefile.wasm). emcc's real wasm-opt runs - # in-link like any normal build. - echo "" - echo "=== JSPI backend: in-link build, no post-link instrumentation ===" -else -echo "" -echo "=== Building the Binaryen submodule (version_130 + hoist pass) ===" -# One binaryen everywhere: the submodule fork is version_130 (asyncify unchanged) + our hoist -# pass, so the same binary does --hoist-cpp-catches AND --asyncify/-O2. No separate v130 clone. -export HOIST_WASMOPT="$("$SCRIPT_DIR/binaryen-hoist-pass/build-wasm-opt.sh")" -export V130_WASMOPT="$HOIST_WASMOPT" -echo " submodule wasm-opt: $HOIST_WASMOPT" -echo "Stubbing in-link Asyncify (will run post-link instead)..." -cp "$EMSDK_WASM_OPT" "${EMSDK_WASM_OPT}.ehbak" -cp "$WASMOPT_STUB" "$EMSDK_WASM_OPT"; chmod +x "$EMSDK_WASM_OPT" -trap _eh_restore_wasmopt EXIT -fi +# JSPI: no binaryen instrumentation exists — no fork build, no stub dance, no +# post-link pass, no shim injection (the jspi-scheduler ships as a --pre-js +# from Makefile.wasm). emcc's real wasm-opt runs in-link like any normal build. # Build (pass DEBUG flag if requested). App links are independent, so honor -# JOBS/PARALLEL_JOBS from env.sh (each emcc link is slow due to Asyncify). +# JOBS/PARALLEL_JOBS from env.sh. if [ "$DEBUG_BUILD" = "1" ]; then make -j"${JOBS:-1}" -f Makefile.wasm DEBUG=1 "$MAKE_TARGET" else @@ -133,54 +109,11 @@ else fi make_rc=$? if [ "$make_rc" -ne 0 ]; then - # Fail loudly. Silently continuing to the post-link leaves the freshly-linked apps - # asyncify-stubbed / un-injected, which looks like mass test failures rather than a build - # error. (The EXIT trap restores the stubbed emsdk wasm-opt in the native-EH build.) echo "" >&2 - echo "ERROR: make failed (exit $make_rc); aborting before the post-link step." >&2 + echo "ERROR: make failed (exit $make_rc)." >&2 exit "$make_rc" fi -# Inject the dyncall + asyncify-scheduler shims into every freshly-linked app. The -# scheduler's currData save/restore (Emscripten #9153) is needed: without it a rewind that -# resumes through a fresh wasm re-entry hits _asyncify_start_rewind(null) -> "memory access out -# of bounds" — e.g. a context-menu pick while the main loop is parked. The Makefile only injects -# it for the coroutine apps; inject-dyncall-shims.sh is idempotent (skips an already-shimmed glue), -# so re-running it here is safe. The .wasm gets post-link hoist + asyncify first. -_eh_restore_wasmopt; trap - EXIT -if [ "${PCBJAM_ASYNC_BACKEND:-asyncify}" = "jspi" ]; then - rm -f "$EH_MARKER" - echo "" - echo "=== Build complete (jspi backend) ===" - exit 0 -fi -echo "" -echo "=== Post-link --hoist-cpp-catches + --asyncify (${JOBS:-1}-wide) ===" -# The apps are independent here too (apply-asyncify rewrites each wasm in place; the shim -# injector's temp file is per-js), so fan the post-link out across JOBS like the make phase. -# This dominates the build: per-app wasm-opt can't feed many cores (small modules, serial -# parse/write), so parallelism must come from running apps side by side — serial 3m52s even -# with BINARYEN_CORES=16, vs 2m08s fanned out at ~11 GB peak RAM. HOIST_WASMOPT is resolved -# once above, so workers skip the binaryen ninja check. xargs fails the build (exit 123) if -# any app's post-link fails. -export SCRIPT_DIR -# Match EVERY freshly-linked app wasm, not just standalone/*/*_test.wasm: the main demo -# (apps/minimal_test.wasm) is at the apps/ root, and the coroutine-pthread repros + wxpt app -# are *_repro*.wasm / *_wxpt.wasm. The old '*_test.wasm under standalone' filter silently -# skipped all of those, so under native wasm-EH they never got hoist+asyncify and crashed at -# runtime with "asyncify_start_unwind not found". -find "$WASM_APP_DIR" -name '*.wasm' -newer "$EH_MARKER" -print0 \ - | xargs -0 -n1 -P "${JOBS:-1}" bash -c ' - set -eo pipefail - w="$1" - "$SCRIPT_DIR/common/apply-asyncify.sh" --no-removelist "$w" - js="${w%.wasm}.js" - if [ -f "$js" ]; then - ( cd "$(dirname "$js")" && "$SCRIPT_DIR/common/inject-dyncall-shims.sh" "$(basename "$js")" ) - fi -' _ -rm -f "$EH_MARKER" - echo "" echo "=== Build complete ===" diff --git a/scripts/build-wx-wasm.sh b/scripts/build-wx-wasm.sh index 370820e..4a9729e 100755 --- a/scripts/build-wx-wasm.sh +++ b/scripts/build-wx-wasm.sh @@ -60,13 +60,14 @@ done BUILD_DIR="$PROJECT_ROOT/build-wasm/wxwidgets" WXLIB_PREFIX="libwx_wasmu" -# Async-backend stamp: PCBJAM_ASYNC_BACKEND changes COMPILE flags -# (-DPCBJAM_JSPI selects whole other halves of the wasm port), which the -# configure-cached incremental build cannot see — a knob flip against a stale -# tree silently links the WRONG backend into every consumer. Force a clean -# build whenever the stamp disagrees. +# Async-backend stamp: the JSPI migration changed COMPILE flags in ways the +# configure-cached incremental build cannot see — building over a stale +# asyncify-era tree silently links a MIXED library. Force a clean build +# whenever the stamp disagrees. BACKEND_STAMP="$BUILD_DIR/.pcbjam-async-backend" -CURRENT_BACKEND="${PCBJAM_ASYNC_BACKEND:-asyncify}" +# Single backend since the JSPI migration: the stamp still force-cleans any +# pre-migration tree (asyncify objects would silently mix into the library). +CURRENT_BACKEND="jspi" # ABSENT stamp = unknown provenance = same as a mismatch: an incremental build # over objects of unknown backend produced a MIXED library once (jspi evtloop # EM_JS in the glue next to live fiber dispatch — --allow-multiple-definition @@ -193,15 +194,6 @@ if [ $NEEDS_CONFIGURE -eq 1 ]; then WX_EH_FLAGS="$DEPS_EH_FLAGS" echo "wx EH model flags: ${WX_EH_FLAGS}" - # Async backend (experiment/jspi): PCBJAM_ASYNC_BACKEND=jspi compiles the - # wasm port's JSPI lanes (evtloop/app/window PCBJAM_JSPI blocks) instead of - # the Asyncify/fiber lanes. ONE wx build output — flipping backends means - # rebuilding (use --clean). Default stays asyncify until Phase 4. - if [ "${PCBJAM_ASYNC_BACKEND:-asyncify}" = "jspi" ]; then - WX_EH_FLAGS="$WX_EH_FLAGS -DPCBJAM_JSPI=1" - echo "async backend: jspi (-DPCBJAM_JSPI)" - fi - # Include emscripten cache sysroot for zlib headers export CFLAGS="-DZ_HAVE_UNISTD_H=1 -I$EM_CACHE_SYSROOT/include ${WX_DEBUG_FLAGS} ${WX_EH_FLAGS} -pthread -matomics -mbulk-memory" export CXXFLAGS="-DZ_HAVE_UNISTD_H=1 -I$EM_CACHE_SYSROOT/include -I$PCRE2_INCLUDE ${WX_DEBUG_FLAGS} ${WX_EH_FLAGS} -pthread -matomics -mbulk-memory" diff --git a/scripts/common/apply-asyncify.sh b/scripts/common/apply-asyncify.sh deleted file mode 100755 index 7f1b609..0000000 --- a/scripts/common/apply-asyncify.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash -# Unified post-link Asyncify pass for KiCad AND the wx test apps. -# -# Usage: apply-asyncify.sh [--no-removelist] [output.wasm] -# -# Always: run our --hoist-cpp-catches fork pass FIRST (lets Asyncify suspend from inside C++ catch -# blocks under native wasm-EH) with all wasm features enabled (-all, so binaryen parses the -# EH instructions), then --asyncify + remove-list + -O2. Native wasm-EH is the only build mode. -# --no-removelist skip the KiCad big-function remove-list (the small wx test apps don't contain -# those symbols, and one bare entry — "match" — could collide). -# -# WHY post-link (not emcc's in-link Asyncify): the emsdk-bundled Binaryen crashes asyncifying -# wasm-EH and a compiler/standalone version skew corrupts asyncify metadata. So the in-link pass is -# stubbed (build-kicad-target.sh / build-wasm-test.sh) and the real transform runs here, on the host -# (more RAM), with a pinned Binaryen. The cost: emcc's automatic asyncify-imports generation is -# bypassed, so the BOUNDARY import list lives in asyncify-imports.txt (see that file). -# -# Binaryen selection (env overrides win, set by build-wasm-test.sh): one binaryen everywhere — the -# submodule fork (version_130 + our --hoist-cpp-catches). Override the path via HOIST_WASMOPT / V130_WASMOPT. - -set -eo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" - -# --- flags --- -# Native wasm-EH is the only build mode, so the catch-arm hoist pass always runs. -DO_HOIST=1 -USE_REMOVELIST=1 -while [[ "${1:-}" == --* ]]; do - case "$1" in - --no-removelist) USE_REMOVELIST=0; shift ;; - *) echo "apply-asyncify: unknown flag: $1" >&2; exit 1 ;; - esac -done - -INPUT_WASM="${1:-output/pcbnew.wasm}" -OUTPUT_WASM="${2:-${INPUT_WASM}}" -[ -f "${INPUT_WASM}" ] || { echo "ERROR: Input file not found: ${INPUT_WASM}" >&2; exit 1; } - -# --- Binaryen tool: ONE binaryen everywhere (incl. docker/CI) — the submodule fork. It IS Binaryen -# version_130 (its Asyncify.cpp is unmodified upstream) + our HoistCppCatches pass, so the SAME binary -# does --hoist-cpp-catches AND --asyncify/-O2, for both native-EH and legacy JS-EH wasm. Built once via -# build-wasm-opt.sh; no separate binaryen downloads (the emsdk-bundled v121 can't even asyncify wasm-EH). -SUBMODULE_WASMOPT="${HOIST_WASMOPT:-$("${SCRIPT_DIR}/../binaryen-hoist-pass/build-wasm-opt.sh")}" -WASM_OPT="${V130_WASMOPT:-$SUBMODULE_WASMOPT}" -[ "$DO_HOIST" = 1 ] && HOIST_OPT="$SUBMODULE_WASMOPT" - -# native wasm-EH needs -all so binaryen parses the EH instructions; HOIST_KEEP_NAMES keeps the -# names section through -O2 for callstack debugging. KiCad/JS-EH uses neither (matches old behavior). -FEAT=() -[ "$DO_HOIST" = 1 ] && FEAT=(-all) -G="${HOIST_KEEP_NAMES:+-g}" - -# The asyncify removelist matches functions by NAME. wasm-opt strips the names section by default, so -# the hoist pass (run before asyncify) would otherwise hand asyncify nameless functions — the -# removelist then matches NOTHING and the giant try-dense functions it is meant to exclude -# (BuildBitmapInfo: ~4986 native tries, etc.) get instrumented, which is the dominant driver of the -# multi-GB asyncify RAM blowup on native-EH. So force the hoist pass to keep names whenever a -# removelist is in play, so asyncify can see + exclude them. asyncify/-O2 keep their own G, so the -# FINAL wasm is unchanged unless HOIST_KEEP_NAMES is set (asyncify matches on its INPUT names, which -# the hoist output now carries). -HOIST_G="${G}" -[ "$USE_REMOVELIST" = 1 ] && HOIST_G="-g" - -# --- the import boundary (shared) + the KiCad remove-list (opt-out), read from sibling files --- -_join_list() { grep -vE '^[[:space:]]*#|^[[:space:]]*$' "$1" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | tr '\n' ',' | sed 's/,$//'; } -ASYNCIFY_IMPORTS="${ASYNCIFY_IMPORTS_PASS:-$(_join_list "${SCRIPT_DIR}/asyncify-imports.txt")}" -REMOVE_ARG=() -if [ "$USE_REMOVELIST" = 1 ]; then - REMOVE_ARG=("--pass-arg=asyncify-removelist@$(_join_list "${SCRIPT_DIR}/asyncify-removelist.txt")") -fi - -# --- memory machinery (identical to before; matters for the host-side KiCad pass) --- -# Bound Binaryen's host thread pool: peak RAM scales with thread count. -export BINARYEN_CORES="${BINARYEN_CORES:-8}" - -# Preload a scalable allocator on Linux — glibc malloc collapses into futex lock-spin under -# wasm-opt's many-thread allocation churn; jemalloc/mimalloc roughly halve wall-clock. macOS -# already ships a scalable allocator. WASM_OPT_PRELOAD=none|0 forces a clean glibc baseline. -if [[ "${WASM_OPT_PRELOAD:-}" == "none" || "${WASM_OPT_PRELOAD:-}" == "0" ]]; then - WASM_OPT_PRELOAD=""; _PRELOAD_FORCED_OFF=1 -fi -if [[ -z "${WASM_OPT_PRELOAD:-}" && -z "${_PRELOAD_FORCED_OFF:-}" && "$(uname -s)" == "Linux" ]]; then - for _alloc in \ - "/usr/lib/$(uname -m)-linux-gnu/libjemalloc.so.2" \ - "/usr/lib/$(uname -m)-linux-gnu/libmimalloc.so.2" \ - /usr/lib/libjemalloc.so.2 /usr/lib/libmimalloc.so.2; do - [[ -e "${_alloc}" ]] && { WASM_OPT_PRELOAD="${_alloc}"; break; } - done -fi -if [[ -n "${WASM_OPT_PRELOAD:-}" ]]; then - PRELOAD_CMD=(env "LD_PRELOAD=${WASM_OPT_PRELOAD}${LD_PRELOAD:+:${LD_PRELOAD}}") -else - PRELOAD_CMD=() -fi -# GNU `time -v` on Linux CI records peak RSS + wall-clock per pass; macOS `time` lacks -v. -if /usr/bin/time -v true >/dev/null 2>&1; then TIME_CMD=(/usr/bin/time -v); else TIME_CMD=(); fi - -echo "Applying Asyncify${DO_HOIST:+ (+hoist-cpp-catches)}..." -echo " Input: ${INPUT_WASM}" -echo " Output: ${OUTPUT_WASM}" -echo " asyncify wasm-opt: ${WASM_OPT}" -[ "$DO_HOIST" = 1 ] && echo " hoist wasm-opt: ${HOIST_OPT}" -echo " BINARYEN_CORES=${BINARYEN_CORES} LD_PRELOAD=${WASM_OPT_PRELOAD:-}" - -SRC="${INPUT_WASM}" - -# Refuse to double-instrument. A postprocess re-run on an artifact that a -# previous (killed/partial) run already asyncified re-instruments the -# instrumented module: the pass balloons to OOM/jetsam death, and the output -# would be broken anyway. The asyncify export names only exist in a module -# the pass already touched. Recover by re-copying the pristine post-link -# artifact from the docker volume (docker/build.sh compile copy step). -if LC_ALL=C grep -aq "asyncify_start_unwind" "${INPUT_WASM}"; then - echo "ERROR: ${INPUT_WASM} already contains asyncify exports - refusing to" >&2 - echo "double-instrument. Restore the pristine post-link wasm first." >&2 - exit 1 -fi - -# 1. (native wasm-EH only) hoist C++ catch arms so Asyncify can suspend from inside them. -if [ "$DO_HOIST" = 1 ]; then - echo "Running --hoist-cpp-catches${HOIST_G:+ (keeping names for removelist matching)}..." - "${PRELOAD_CMD[@]}" "${TIME_CMD[@]}" "${HOIST_OPT}" --hoist-cpp-catches "${FEAT[@]}" ${HOIST_G} "${SRC}" -o "${OUTPUT_WASM}" - SRC="${OUTPUT_WASM}" -fi - -# 2. The real Asyncify transform. ASYNCIFY_EXTRA_OPTS: optional extra wasm-opt flags (origin/main hook). -echo "Running wasm-opt --asyncify (several minutes + significant RAM)..." -"${PRELOAD_CMD[@]}" "${TIME_CMD[@]}" "${WASM_OPT}" --asyncify ${ASYNCIFY_EXTRA_OPTS:-} "${FEAT[@]}" ${G} \ - "--pass-arg=asyncify-imports@${ASYNCIFY_IMPORTS}" \ - "${REMOVE_ARG[@]}" \ - --pass-arg=asyncify-propagate-addlist \ - "${SRC}" -o "${OUTPUT_WASM}" - -# ASYNCIFY_ONLY=1 stops before -O2 (benchmark harness in scripts/bench/ times just the transform). -if [[ "${ASYNCIFY_ONLY:-0}" == "1" ]]; then - echo "ASYNCIFY_ONLY=1 → skipping -O2 (benchmark mode)."; ls -lh "${OUTPUT_WASM}"; exit 0 -fi - -# 3. Post-asyncify shrink. Asyncify spills every live local; without coalescing, large -# coroutine-entry functions exceed V8's per-function locals limit and stall/crash the renderer. -# -O1 runs CoalesceLocals (enough to keep instrumented functions under V8's local limit) and is the -# level we ship EVERYWHERE — main CI, tag releases, and local builds all use it. NOT -Os/-Oz (they -# break the asyncify runtime — Binaryen #4484). See docs/debugging/DEBUG.md §6-7. -BINARYEN_OPT_LEVEL="${BINARYEN_OPT_LEVEL:--O1}" -echo "Running wasm-opt ${BINARYEN_OPT_LEVEL} (shrink instrumented functions under V8's local limit)..." -"${PRELOAD_CMD[@]}" "${TIME_CMD[@]}" "${WASM_OPT}" "${BINARYEN_OPT_LEVEL}" "${FEAT[@]}" ${G} "${OUTPUT_WASM}" -o "${OUTPUT_WASM}" - -echo "Asyncify + ${BINARYEN_OPT_LEVEL} complete: ${OUTPUT_WASM}" -ls -lh "${OUTPUT_WASM}" diff --git a/scripts/common/apply-finalize.sh b/scripts/common/apply-finalize.sh deleted file mode 100755 index aaaea1a..0000000 --- a/scripts/common/apply-finalize.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# Apply wasm-emscripten-finalize transformation on host. -# -# Usage: ./scripts/common/apply-finalize.sh - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" - -# wasm-emscripten-finalize comes from our Binaryen submodule (version_130 + --hoist-cpp-catches), -# built alongside wasm-opt by build-wasm-opt.sh. Pinning finalize and the asyncify wasm-opt to ONE -# Binaryen version removes the host's emsdk dependency from the post-process entirely (it is now -# dyncall=node + finalize/asyncify=submodule), which is what was breaking on the ephemeral CI host. -# HOIST_WASMOPT, when the build driver pre-warms the submodule build, points at the already-built -# wasm-opt so we don't re-invoke the build per app; finalize sits next to it in the same bin/. -WASM_OPT="${HOIST_WASMOPT:-$("${SCRIPT_DIR}/../binaryen-hoist-pass/build-wasm-opt.sh")}" -FINALIZE="$(dirname "${WASM_OPT}")/wasm-emscripten-finalize" - -INPUT_WASM="${1:-output/pcbnew.wasm}" -OUTPUT_WASM="${2:-${INPUT_WASM}}" - -if [ ! -f "${INPUT_WASM}" ]; then - echo "ERROR: Input file not found: ${INPUT_WASM}" - exit 1 -fi - -echo "Applying wasm-emscripten-finalize..." -echo " Input: ${INPUT_WASM}" -echo " Output: ${OUTPUT_WASM}" -echo " Tool: ${FINALIZE}" - -# Run finalize with the same flags Emscripten would use -# NOTE: --dwarf removed because debug info is in separate .debug.wasm file -"${FINALIZE}" \ - -g \ - --bigint \ - --no-legalize-javascript-ffi \ - --detect-features \ - "${INPUT_WASM}" \ - -o "${OUTPUT_WASM}" - -echo "Finalize complete: ${OUTPUT_WASM}" -ls -lh "${OUTPUT_WASM}" diff --git a/scripts/common/asyncify-imports.txt b/scripts/common/asyncify-imports.txt deleted file mode 100644 index bc387e9..0000000 --- a/scripts/common/asyncify-imports.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Suspending imports for the post-link Asyncify pass (consumed by apply-asyncify.sh). -# Binaryen's --asyncify instruments every function that can transitively REACH one of these -# imports. This is the BOUNDARY list, not a function allowlist — the callers are auto-discovered. -# -# We run Asyncify post-link (outside emcc), so emcc's automatic import generation never runs; -# this file replicates emcc's default async built-ins + our own suspending imports. A superset -# is safe — binaryen ignores any import the wasm doesn't actually contain. -# -# --- Emscripten async built-ins (what emcc auto-adds for the in-link Asyncify) --- -env.emscripten_sleep -env.emscripten_scan_registers -env.emscripten_lazy_load_code -env.emscripten_wget -env.emscripten_wget_data -env.emscripten_idb_* -# -# --- Project suspending imports --- -env.__asyncjs__* -env.emscripten_fiber_swap -env.js_* diff --git a/scripts/common/asyncify-removelist.txt b/scripts/common/asyncify-removelist.txt deleted file mode 100644 index fef07cd..0000000 --- a/scripts/common/asyncify-removelist.txt +++ /dev/null @@ -1,32 +0,0 @@ -# Functions EXCLUDED from Asyncify instrumentation (consumed by apply-asyncify.sh unless -# --no-removelist). These are large, NON-suspending KiCad/OpenCASCADE functions (generated resource -# tables, wxFormBuilder UI constructors, OCC geometry) that are dense with native wasm try/catch. -# Asyncify's per-function cost is superlinear in try-count (it builds a CFG + liveness over every -# try), so instrumenting these few giants is what drives the multi-GB RAM blowup of `wasm-opt -# --asyncify` on the native-EH build. They never call a suspending import, so excluding them is safe. -# -# MATCHING: Binaryen escapes each entry, then treats entries containing `*` as wildcard patterns -# (String::wildcardMatch) and entries without `*` as exact escaped function names. Full demangled -# signatures don't exact-match (subtle spacing) and a `*` inside a pointer type (e.g. wxWindow*) turns -# the whole entry into a pattern that still misses — so use a PREFIX wildcard per symbol, which matches -# the function's escaped demangled name regardless of argument formatting. Patterns that match nothing -# in a given app (e.g. the OCC entries when 3D is off) just emit a harmless "non-matching" warning. -# Only the KiCad build passes the remove-list; the small wx test apps opt out via --no-removelist -# (and the bare name "match" could collide there). -COLOR_SETTINGS::COLOR_SETTINGS* -BuildBitmapInfo* -match -DIALOG_PAD_PROPERTIES_BASE::DIALOG_PAD_PROPERTIES_BASE* -buildKicadAboutBanner* -IGESToBRep_CurveAndSurface::TransferGeometry* -StepAP214_Protocol::StepAP214_Protocol* -BRepCheck_ParallelAnalyzer::operator* -ShapeFix_Wire::FixGap3d* -ShapeFix_Wire::FixGap2d* -PCB_EDIT_FRAME::setupUIConditions* -# The ngspice model-parameter table initializers (sim_model_ngspice_data_*.cpp, -# restored for the simulator split): thousands of straight-line emplace_backs, -# nothing that can suspend. Uninstrumented they stay under the JS engines' -# per-function locals limit and shave the post-asyncify module size -# (bsim4/b3soi/b4soi/hsim alone are the four largest functions in eeschema). -NGSPICE_MODEL_INFO_MAP::add* diff --git a/scripts/common/inject-dyncall-shims.sh b/scripts/common/inject-dyncall-shims.sh deleted file mode 100755 index ac26b27..0000000 --- a/scripts/common/inject-dyncall-shims.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/bin/bash -# Post-process the Emscripten-generated .js for KiCad WASM (pcbnew, eeschema, -# pl_editor, calculator, …). -# -# The actual JavaScript that gets injected lives in readable, standalone files in -# scripts/common/shims/ (not inline heredocs): -# - asyncify-scheduler.js the mailbox/scheduler (docs/features/async/17) — -# the ONLY asyncify runtime (the legacy handlesleep.js -# opt-out was deleted at doc 20 D-1) -# - diagnostics.js optional logging-only instrumentation (see SHIM_DIAGNOSTICS) -# -# Native wasm-EH is the only build mode, so the .js has no invoke_* wrappers / dynCall_ call -# sites to bind. The build still links -sDYNCALLS=1, so asyncify-INSTRUMENTED dynCall_* trampolines -# exist as wasm EXPORTS; the empty-callback fixes below route function-pointer stubs through -# wasmExports["dynCall_"]. This MUST be the wasm trampoline, NOT getWasmTableEntry — the latter -# bypasses the instrumentation and breaks unwind/rewind through indirect calls ("indirect call -# signature mismatch" — caught every frame in Firefox; a hard renderer crash in Chrome/V8). -# -# Usage: -# inject-dyncall-shims.sh -# SHIM_DIAGNOSTICS=1 inject-dyncall-shims.sh # also inject diagnostics.js - -set -e - -JS_FILE="$1" -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SHIM_DIR="$SCRIPT_DIR/shims" - -# One-line toggle for the diagnostics module (default OFF). -SHIM_DIAGNOSTICS="${SHIM_DIAGNOSTICS:-0}" - -if [ -z "$JS_FILE" ] || [ ! -f "$JS_FILE" ]; then - echo "Error: JS file not found: $JS_FILE" - echo "Usage: $0 " - exit 1 -fi -for f in asyncify-scheduler.js diagnostics.js; do - if [ ! -f "$SHIM_DIR/$f" ]; then - echo "Error: missing shim source $SHIM_DIR/$f" - exit 1 - fi -done - -# --- 1. Empty-callback fixes --------------------------------------------------- -# Emscripten+pthreads emits some direct-call paths as no-op ((a1)=>{}) stubs that ARE used. Native -# wasm-EH eliminates the invoke_* wrappers, so the .js has no dynCall_ call sites to bind — but -# the DYNCALLS=1 trampolines are still EXPORTED on the wasm, so route each function-pointer stub -# through wasmExports["dynCall_"]. This MUST be the wasm trampoline, NOT getWasmTableEntry: the -# fiber entry runs a coroutine that suspends+rewinds via Asyncify, and an Asyncify rewind cannot -# resume through getWasmTableEntry's JS wrapper — the fiber would re-enter from the top and the tool's -# Wait() re-runs (tool_manager ScheduleWait "!pendingWait" assert + busy-loop). The instrumented -# dynCall_ export rewinds correctly. (Without these fixes the libcontext fiber entry stays the -# empty (a1=>{}) stub, so tool coroutines never start and every GAL app stalls at InvokeTool.) -echo "Fixing empty callback arrow functions..." -TOTAL_FIXED=0 -apply_fix() { #