diff --git a/.github/workflows/ci-full-build.yml b/.github/workflows/ci-full-build.yml new file mode 100644 index 0000000..677d6bc --- /dev/null +++ b/.github/workflows/ci-full-build.yml @@ -0,0 +1,135 @@ +# Full KiCad WASM build (all 6 tools) + e2e on an ephemeral Hetzner ccx53. +# This is the main-branch CI — the ONLY push-triggered Hetzner workflow; the +# bench (wasm-opt-bench.yml) and feature-branch (ci.yml) workflows are +# workflow_dispatch-only to keep paid VM spend deliberate. Validated at ~1h15m +# (run 27280051992, was 4h05m): lifted docker CPU caps, pipelined host-side +# wasm-opt (KICAD_PIPELINE=1), self-built Binaryen v130. History and +# measurements in docs/ci-build-slowness-findings.md. + +name: CI full build + e2e (Hetzner) + +on: + push: + branches: ["main"] + # docs-only commits must not burn a ~1h15m Hetzner slot + paths-ignore: ["docs/**", "**.md"] + workflow_dispatch: + inputs: + binaryen_version: { description: "Binaryen version for the host asyncify+-O2 step", required: false, default: "130" } + +concurrency: + group: wasm-opt-bench # share the one-Hetzner-slot lock with the sweeps + cancel-in-progress: false + +jobs: + create-runner: + name: Create Hetzner runner + runs-on: ubuntu-latest + outputs: + label: ${{ steps.create.outputs.label }} + server_id: ${{ steps.create.outputs.server_id }} + steps: + - uses: Cyclenerd/hcloud-github-runner@v1 + id: create + with: + mode: create + github_token: ${{ secrets.HETZNER_RUNNER_PAT }} + hcloud_token: ${{ secrets.HCLOUD_TOKEN }} + server_type: ccx53 + location: nbg1 + image: ubuntu-24.04 + + validate: + name: Build all tools + KiCad e2e + needs: create-runner + runs-on: ${{ needs.create-runner.outputs.label }} + timeout-minutes: 300 + env: + KICAD_LOG_NESTED: "1" + BINARYEN_VERSION: ${{ inputs.binaryen_version || '130' }} + steps: + - name: Install system deps (Docker, git) + run: | + export DEBIAN_FRONTEND=noninteractive + sudo apt-get update + sudo apt-get install -y ca-certificates curl git libjemalloc2 + sudo install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + sudo chmod a+r /etc/apt/keyrings/docker.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo $VERSION_CODENAME) stable" \ + | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + # toolchain for BINARYEN_BUILD_FROM_SOURCE (get-wasm-opt.sh) + sudo apt-get install -y cmake ninja-build g++ + + - uses: actions/checkout@v4 + with: { submodules: recursive } + + - uses: actions/setup-node@v4 + with: { node-version: 20 } + + # Full KiCad WASM build of ALL 6 tools, but the host-side asyncify + -O2 use + # Binaryen v130 (BINARYEN_VERSION in env → get-wasm-opt.sh forces that + # standalone release). 'all' = pcbnew eeschema calculator pl_editor + # symbol_editor gerbview, built in sequence. + - name: Build all KiCad tools WASM with Binaryen ${{ env.BINARYEN_VERSION }} + run: | + # Lift the docker-compose dev-Mac caps (10 CPUs / 32G): run 27226030304 + # compiled on 10 of 32 cores with -j 32. Pipeline the host-side + # asyncify+-O2 with the next tool's container compile (KICAD_PIPELINE=1; + # wasm-opt only reaches ~350% CPU, the container is idle meanwhile). + # BINARYEN_CORES=16: -O2 is Amdahl-capped ~4 effective cores, and up to + # two postprocesses run concurrently next to the compile. + export KICAD_DOCKER_CPUS="$(nproc)" KICAD_DOCKER_MEM=110G + export KICAD_PIPELINE=1 BINARYEN_CORES=16 + # Self-built wasm-opt: the official x86_64-linux tarball runs asyncify + # 4x slower than a stock gcc -O3+LTO build, identical output + # (A/B on this runner, run 27276830256). ~5 min build, instantly repaid. + export BINARYEN_BUILD_FROM_SOURCE=1 + echo "Building ALL tools with BINARYEN_VERSION=${BINARYEN_VERSION}, compile -j $(nproc) on ${KICAD_DOCKER_CPUS} CPUs, pipelined wasm-opt" + ./docker/build.sh all --build-deps -j "$(nproc)" + echo "wasm-opt used:"; ./scripts/common/get-wasm-opt.sh --version 2>/dev/null || true + ls -lh output/*.wasm + + - name: Install test deps + working-directory: tests + run: npm ci + + - name: Install Playwright browsers + working-directory: tests + run: npx playwright install --with-deps firefox chromium + + - name: Stage KiCad WASM for tests + working-directory: tests + run: npm run setup:kicad + + - name: KiCad e2e (npm run test:kicad) + working-directory: tests + run: npm run test:kicad + + - name: Upload test logs & screenshots + if: always() + uses: actions/upload-artifact@v4 + with: + name: full-build-e2e-${{ github.run_id }} + path: | + tests/logs/** + tests/test-results/** + tests/playwright-report/** + if-no-files-found: ignore + + delete-runner: + name: Delete Hetzner runner + needs: [create-runner, validate] + runs-on: ubuntu-latest + if: always() + steps: + - uses: Cyclenerd/hcloud-github-runner@v1 + with: + mode: delete + github_token: ${{ secrets.HETZNER_RUNNER_PAT }} + hcloud_token: ${{ secrets.HCLOUD_TOKEN }} + name: ${{ needs.create-runner.outputs.label }} + server_id: ${{ needs.create-runner.outputs.server_id }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6209afe..4ed4152 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,43 +1,99 @@ -# Cold build of all KiCad WASM apps (with deps) + full e2e suites, on Ubicloud. +# Ephemeral Hetzner Cloud CI: a fresh ccx53 VM is created per run, the full +# build + e2e suites run on it, then it is destroyed. No persistent infra. # -# Runs on Ubicloud's managed GitHub Actions runners. Premium runners (AMD Ryzen 9 -# 7950X3D, ~2x faster) are enabled at the ACCOUNT level in the Ubicloud Console -# (GitHub Runners -> Settings -> Enable Premium Runners); jobs on the -# `ubicloud-standard-*` labels below are then auto-routed to the premium fleet with -# no change needed here. +# Lifecycle (Cyclenerd/hcloud-github-runner): create-runner spins up a Hetzner +# VM registered as an ephemeral self-hosted runner -> build-and-test runs on it +# -> delete-runner tears it down (always, even on failure -> no lingering cost). # -# One-time setup required: -# The kicad/wxwidgets submodules are private SSH repos in the VV-EE org, which the -# default GITHUB_TOKEN cannot read. Add a repo secret `SUBMODULES_TOKEN` (a PAT with -# read access to VV-EE/kicad-source-mirror and VV-EE/wxWidgets); actions/checkout -# rewrites the SSH submodule URLs to token-authenticated HTTPS. +# Required repo secrets (emergence-engineering/pcbjam -> Settings -> Secrets): +# HCLOUD_TOKEN Hetzner Cloud API token, Read & Write +# HETZNER_RUNNER_PAT GitHub PAT with Administration: read & write (registers +# and unregisters the self-hosted runner) +# +# Submodules: kicad/wxwidgets use SSH URLs (.gitmodules) but point at PUBLIC repos, +# so actions/checkout rewrites them to HTTPS and the default GITHUB_TOKEN reads them. -name: CI +name: CI (Hetzner) +# Manual-only (workflow_dispatch): every run costs a paid ephemeral Hetzner VM. +# Main-branch CI is ci-full-build.yml. on: - push: - branches: [main] workflow_dispatch: + inputs: + binaryen_cores: + description: "BINARYEN_CORES for the wasm-opt/asyncify step (blank = all cores / nproc). Use to A/B core counts." + required: false + default: "" concurrency: - group: ci-${{ github.ref }} + group: ci-hetzner-${{ github.ref }} cancel-in-progress: true jobs: + create-runner: + name: Create Hetzner runner + runs-on: ubuntu-latest # or ubicloud-standard-2 if GitHub-hosted is disabled + outputs: + label: ${{ steps.create.outputs.label }} + server_id: ${{ steps.create.outputs.server_id }} + steps: + - name: Create ephemeral ccx53 runner + id: create + uses: Cyclenerd/hcloud-github-runner@v1 + with: + mode: create + github_token: ${{ secrets.HETZNER_RUNNER_PAT }} + hcloud_token: ${{ secrets.HCLOUD_TOKEN }} + server_type: ccx53 + location: nbg1 + image: ubuntu-24.04 + build-and-test: - runs-on: ubicloud-standard-16 - timeout-minutes: 240 # cold --build-deps dominates; + wx build + both suites + name: Build & test (Hetzner ccx53) + needs: create-runner + runs-on: ${{ needs.create-runner.outputs.label }} + timeout-minutes: 160 # cold --build-deps dominates; + wx build + both suites env: - # Stream build-script output straight to the Actions log instead of redirecting - # it to logs/