feat(standalone): OOM tab recovery (0002)

Automatically recover from an out-of-memory by respawning a fresh browsing
context (fresh wasm heap) and discarding the dead one — the manual "duplicate
tab, close the OOM'd one" fix, automated and capped at 2 retries so a too-small
machine stops instead of looping.

- recovery/oom-watch.ts: the recover() state machine with the retry counter in
  the URL (?oomRetry=N, cap 2). Mode A soft detection — Module.onAbort plus
  window error/unhandledrejection matched against an OOM signature, behind an
  idempotent latch. Mode B hard-kill detection — a per-tab localStorage
  heartbeat sentinel; a stale sentinel on the next load continues the retry
  chain (heartbeat keys avoid mistaking a second live tab for a crash). Healthy
  session (alive HEALTHY_RESET_MS) resets the chain. Default strategy is
  location.replace; window.open+close is behind ?oomStrategy=newtab. Platform
  access goes through injectable win/storage seams for testability.
- recovery/MemoryExhaustedDialog.tsx: terminal "out of memory" UI, reusing
  0001's BlockingDialog; copy is honest about lost unsaved edits.
- wasm/boot.ts: forward emscripten Module.onAbort to the watcher (optional
  BootOptions.onAbort).
- WasmTool: install the watcher, gate boot on its proceed flag (skip when the
  chain is already exhausted), and render the terminal dialog.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Gergő Törcsvári 2026-06-10 15:02:47 +02:00
commit 0d2f88b437
No known key found for this signature in database
GPG key ID: 8E75F2CDE64E5322
4 changed files with 391 additions and 2 deletions

View file

@ -13,6 +13,8 @@ import { memfsProjectDir } from "@/wasm/constants";
import { driveProjectIntoTool, type ToolFile } from "@/wasm/kicad-runner";
import type { CollabWindow } from "@/wasm/collab";
import { clog, cwarn } from "@/wasm/collab/debug";
import { createOomWatch } from "@/recovery/oom-watch";
import { MemoryExhaustedDialog } from "@/recovery/MemoryExhaustedDialog";
// Tools with a working collab bridge (kicadCollabSnapshot/Apply embind exports).
const COLLAB_TOOLS = new Set<Tool>(["pl_editor", "eeschema", "pcbnew"]);
@ -259,6 +261,7 @@ export function WasmTool({
const [status, setStatus] = React.useState("Loading tool…");
const [logs, setLogs] = React.useState<string[]>([]);
const [showLog, setShowLog] = React.useState(false);
const [oomExhausted, setOomExhausted] = React.useState(false);
const base = (assetBaseUrl ?? WASM_ASSET_BASE_URL).replace(/\/$/, "");
const append = React.useCallback(
@ -291,9 +294,27 @@ export function WasmTool({
const win = window as ToolWindow;
// OOM recovery (feature 0002): watch for soft aborts + a stale hard-kill
// sentinel, respawning a fresh tab (capped). If the chain is already
// exhausted, skip boot and show the terminal dialog.
const oom = createOomWatch({
channelKey: `${slug}:${targetPath ?? tool}`,
showExhaustedDialog: () => setOomExhausted(true),
log: append,
});
const { proceed } = oom.start();
if (!proceed) return;
void (async () => {
try {
await bootKicadTool({ tool, base, container, log: append, onStatus: setStatus });
await bootKicadTool({
tool,
base,
container,
log: append,
onStatus: setStatus,
onAbort: oom.onAbort,
});
await driveProjectIntoTool(win, {
tool,
slug,
@ -309,6 +330,8 @@ export function WasmTool({
setStatus(`Error: ${String(err)}`);
}
})();
return () => oom.stop();
// Boot is one-shot per mount; deps intentionally exclude files/targetPath so
// they don't retrigger a (rejected) second boot.
// eslint-disable-next-line react-hooks/exhaustive-deps
@ -325,6 +348,10 @@ export function WasmTool({
<div ref={containerRef} id="main-window" className="absolute inset-0 h-full w-full" />
<div id="window-container" />
{oomExhausted && (
<MemoryExhaustedDialog onReload={() => window.location.reload()} />
)}
{status && (
<div className="pointer-events-none absolute left-3 top-3 z-20 rounded bg-black/70 px-3 py-2 font-mono text-xs text-white">
{status}

View file

@ -0,0 +1,35 @@
import { BlockingDialog } from "@/preflight/BlockingDialog";
/**
* Terminal "not enough memory" UI for feature 0002 shown after the OOM retry
* chain hits MAX_RETRIES. Reuses 0001's blocking dialog for visual consistency.
*
* The copy is honest about the two limitations the spec calls out: a respawn
* re-opens the same file but loses in-tab edits not yet pushed to the backend /
* Y.Doc, and `window.close()` may not close the very first (user-opened) tab.
*/
export function MemoryExhaustedDialog({ onReload }: { onReload?: () => void }) {
return (
<BlockingDialog
title="Your device ran out of memory"
description="The editor tried to recover a few times but kept running out of memory, so it stopped to avoid looping. This design is likely too large for this device's available memory."
reasons={[
{
title: "What you can try",
detail:
"Close other tabs and applications to free memory, use a desktop machine with more RAM, or open a smaller design.",
},
{
title: "Unsaved changes",
detail:
"Edits that weren't yet synced to the server may be lost. Any older tab left open can still be closed manually.",
},
]}
primary={
onReload
? { label: "Reload and try again", onClick: onReload }
: undefined
}
/>
);
}

View file

@ -0,0 +1,317 @@
/**
* OOM tab recovery feature 0002.
*
* The KiCad WASM runtime is process-global and cannot be torn down (see boot.ts:
* "Reload the page to open another tool"). After an out-of-memory the wasm heap
* and `Module` are unrecoverable in-context, so the only clean reset is a brand
* new browsing context exactly the manual "duplicate the tab, close the dead
* one" fix. This module automates it, capped at MAX_RETRIES so a genuinely
* too-small machine stops instead of looping forever.
*
* Two crash modes:
* - Mode A (soft abort): JS is still alive emscripten `abort()` /
* `Module.onAbort`, or a `RangeError`/`RuntimeError`/"Aborted(" surfacing as
* a window error/unhandledrejection. The tab self-recovers.
* - Mode B (hard kill): the renderer is killed ("Aw, Snap"); no in-tab code
* runs. We detect it on the NEXT load via a stale `localStorage` heartbeat
* sentinel and continue the retry chain.
*
* The retry counter rides in the URL query (`?oomRetry=N`) so a fresh context
* knows its place in the chain. A session that stays healthy for HEALTHY_RESET_MS
* resets the chain.
*
* Pure-ish and unit-testable: all platform access goes through injectable
* `win`/`storage` seams (default to the real `window`/`localStorage`).
*/
export const MAX_RETRIES = 2;
const RETRY_PARAM = "oomRetry";
const STRATEGY_PARAM = "oomStrategy";
const SENTINEL_PREFIX = "oom:running:";
/** A session alive this long is "healthy" → clear the retry chain. */
export const HEALTHY_RESET_MS = 30_000;
/** How often a live tab refreshes its sentinel heartbeat. */
const HEARTBEAT_MS = 10_000;
/** A sentinel whose heartbeat is older than this means a hard-killed tab, not a
* live one (the false-positive guard for two tabs open on the same file). */
export const SENTINEL_STALE_MS = 60_000;
const OOM_SIGNATURE =
/out of memory|RangeError|Aborted\(|Cannot enlarge memory|memory access out of bounds/i;
/** Whether a console/error string looks like an OOM. Exported for tests. */
export function looksLikeOom(text: string | null | undefined): boolean {
return typeof text === "string" && OOM_SIGNATURE.test(text);
}
interface SentinelValue {
startedAt: number;
/** Last heartbeat — staleness is measured from this. */
beat: number;
/** Retry-chain position this instance was running at. */
retry: number;
}
export interface OomWatchOptions {
/** Project slug + target file — same keying as the collab channel. */
channelKey: string;
/** Show the terminal "not enough memory" UI when the cap is reached. */
showExhaustedDialog: () => void;
log?: (msg: string) => void;
/** Injectable for tests; defaults to the real globals. */
win?: Window;
storage?: Storage;
}
export interface OomWatch {
/**
* Install listeners + the sentinel, and reconcile any stale hard-kill sentinel.
* Returns `proceed: false` when the chain is already exhausted (boot should be
* skipped the terminal dialog is shown instead).
*/
start: () => { proceed: boolean };
/** Remove listeners + timers and clear our own sentinel (clean unload). */
stop: () => void;
/** Soft-abort entry point — wired to `Module.onAbort` in boot.ts. */
onAbort: (what?: unknown) => void;
}
function now(): number {
// App code (not a workflow script): Date.now is available.
return Date.now();
}
function newTabId(): string {
try {
if (typeof crypto !== "undefined" && crypto.randomUUID) return crypto.randomUUID();
} catch {
/* fall through */
}
return `${now().toString(36)}-${Math.floor(Math.random() * 1e9).toString(36)}`;
}
export function createOomWatch(opts: OomWatchOptions): OomWatch {
const win = opts.win ?? window;
const storage: Storage | null = opts.storage ?? safeLocalStorage(win);
const log = opts.log ?? (() => {});
const tabId = newTabId();
const sentinelKey = `${SENTINEL_PREFIX}${opts.channelKey}:${tabId}`;
let recovering = false; // idempotent latch — one OOM fires recovery once
let started = false;
let healthyTimer: ReturnType<typeof setTimeout> | undefined;
let heartbeatTimer: ReturnType<typeof setInterval> | undefined;
const currentRetry = (): number => {
const raw = new URLSearchParams(win.location.search).get(RETRY_PARAM);
const n = Number(raw ?? "0");
return Number.isFinite(n) && n > 0 ? Math.floor(n) : 0;
};
const strategy = (): "replace" | "newtab" =>
new URLSearchParams(win.location.search).get(STRATEGY_PARAM) === "newtab"
? "newtab"
: "replace";
// --- sentinel helpers ------------------------------------------------------
const writeSentinel = (retry: number): void => {
if (!storage) return;
const t = now();
const value: SentinelValue = { startedAt: t, beat: t, retry };
try {
storage.setItem(sentinelKey, JSON.stringify(value));
} catch {
/* storage disabled — Mode B simply won't be available */
}
};
const beat = (): void => {
if (!storage) return;
try {
const raw = storage.getItem(sentinelKey);
if (!raw) return;
const value = JSON.parse(raw) as SentinelValue;
value.beat = now();
storage.setItem(sentinelKey, JSON.stringify(value));
} catch {
/* ignore */
}
};
const clearOwnSentinel = (): void => {
if (!storage) return;
try {
storage.removeItem(sentinelKey);
} catch {
/* ignore */
}
};
/** Scan for stale sentinels of THIS channel from other (crashed) tabs; remove
* them and return the highest retry seen, or null if none. */
const reapStaleSentinels = (): number | null => {
if (!storage) return null;
const prefix = `${SENTINEL_PREFIX}${opts.channelKey}:`;
const stale: string[] = [];
let maxRetry: number | null = null;
const t = now();
try {
for (let i = 0; i < storage.length; i++) {
const k = storage.key(i);
if (!k || k === sentinelKey || !k.startsWith(prefix)) continue;
let value: SentinelValue | null = null;
try {
value = JSON.parse(storage.getItem(k) ?? "") as SentinelValue;
} catch {
value = null;
}
// Malformed or heartbeat older than the staleness window ⇒ dead tab.
if (!value || typeof value.beat !== "number" || t - value.beat > SENTINEL_STALE_MS) {
stale.push(k);
const r = value && typeof value.retry === "number" ? value.retry : 0;
maxRetry = Math.max(maxRetry ?? 0, r);
}
}
for (const k of stale) storage.removeItem(k);
} catch {
return maxRetry;
}
return maxRetry;
};
// --- recovery state machine ------------------------------------------------
const recover = (reason: string, what?: string): void => {
if (recovering) return;
recovering = true;
const n = currentRetry();
log(`[oom] recover (${reason}${what ? `: ${what}` : ""}) at retry ${n}`);
if (n >= MAX_RETRIES) {
log(`[oom] retry cap (${MAX_RETRIES}) reached — giving up`);
clearOwnSentinel();
opts.showExhaustedDialog();
return;
}
const url = new URL(win.location.href);
url.searchParams.set(RETRY_PARAM, String(n + 1));
clearOwnSentinel(); // the new context writes its own sentinel
if (strategy() === "newtab") {
try {
const w = win.open(url.toString(), "_blank");
if (w) {
// We own the new tab → close ourselves; the new tab is the survivor.
// (window.close only reliably works on script-opened windows; on the
// very first user-opened tab it may be ignored — acceptable, the new
// tab still works.)
win.close();
return;
}
} catch {
/* popup blocked → fall through to in-place replace */
}
}
// Default + fallback: in-place reload with the bumped counter (fresh heap,
// no popup risk).
win.location.replace(url.toString());
};
// --- Mode A listeners ------------------------------------------------------
const onError = (e: ErrorEvent): void => {
const msg = e?.message || (e?.error ? String(e.error) : "");
if (looksLikeOom(msg)) recover("error", msg);
};
const onRejection = (e: PromiseRejectionEvent): void => {
const reason = e?.reason;
const msg = typeof reason === "string" ? reason : reason ? String(reason) : "";
if (looksLikeOom(msg)) recover("unhandledrejection", msg);
};
const onBeforeUnload = (): void => {
// Clean unload → drop our sentinel so it isn't mistaken for a hard kill.
clearOwnSentinel();
};
// --- public API ------------------------------------------------------------
const onAbort = (what?: unknown): void => {
recover("onabort", what === undefined ? undefined : String(what));
};
const start = (): { proceed: boolean } => {
if (started) return { proceed: true };
started = true;
// Mode B: a stale sentinel from a prior hard-killed instance ⇒ continue its
// retry chain rather than resetting (so the cap still applies).
const staleRetry = reapStaleSentinels();
if (staleRetry !== null) {
const n = Math.max(currentRetry(), staleRetry);
if (n >= MAX_RETRIES) {
log(`[oom] hard-kill chain already exhausted at retry ${n}`);
opts.showExhaustedDialog();
return { proceed: false };
}
// Adopt the chain position so a further OOM in THIS instance caps correctly.
const url = new URL(win.location.href);
url.searchParams.set(RETRY_PARAM, String(n));
try {
win.history.replaceState(null, "", url.toString());
} catch {
/* ignore — currentRetry will still read the original */
}
log(`[oom] continuing hard-kill chain at retry ${n}`);
}
writeSentinel(currentRetry());
heartbeatTimer = setInterval(beat, HEARTBEAT_MS);
win.addEventListener("error", onError);
win.addEventListener("unhandledrejection", onRejection);
win.addEventListener("beforeunload", onBeforeUnload);
// Healthy-session reset: after surviving HEALTHY_RESET_MS, clear the chain so
// a later, unrelated OOM starts counting fresh.
if (currentRetry() > 0) {
healthyTimer = setTimeout(() => {
const url = new URL(win.location.href);
if (url.searchParams.has(RETRY_PARAM)) {
url.searchParams.delete(RETRY_PARAM);
try {
win.history.replaceState(null, "", url.toString());
} catch {
/* ignore */
}
log("[oom] healthy session — retry chain reset");
}
writeSentinel(0);
}, HEALTHY_RESET_MS);
}
return { proceed: true };
};
const stop = (): void => {
if (healthyTimer) clearTimeout(healthyTimer);
if (heartbeatTimer) clearInterval(heartbeatTimer);
win.removeEventListener("error", onError);
win.removeEventListener("unhandledrejection", onRejection);
win.removeEventListener("beforeunload", onBeforeUnload);
clearOwnSentinel();
};
return { start, stop, onAbort };
}
function safeLocalStorage(win: Window): Storage | null {
try {
return win.localStorage;
} catch {
return null;
}
}

View file

@ -37,6 +37,9 @@ export interface BootOptions {
container: HTMLElement;
log: (msg: string) => void;
onStatus: (text: string) => void;
/** OOM recovery hook (feature 0002): emscripten `abort()` routes here so a
* soft OOM can respawn a fresh tab. Optional boot works without it. */
onAbort?: (what: string) => void;
}
let booted: { tool: Tool; promise: Promise<void> } | null = null;
@ -75,7 +78,7 @@ function loadScript(src: string): Promise<void> {
}
async function doBoot(opts: BootOptions): Promise<void> {
const { tool, base, container, log, onStatus } = opts;
const { tool, base, container, log, onStatus, onAbort } = opts;
const w = window as ToolWindow;
// The wasm reads the top-level frame geometry from a GLOBAL `mainWindow`
@ -197,6 +200,13 @@ async function doBoot(opts: BootOptions): Promise<void> {
setStatus: (text: string) => {
if (text) onStatus(text);
},
// OOM recovery (feature 0002): emscripten calls onAbort on abort() — commonly
// how an out-of-memory surfaces. Forward it so the watcher can respawn.
onAbort: (what: unknown) => {
const msg = what === undefined ? "" : String(what);
log(`[boot] abort: ${msg}`);
onAbort?.(msg);
},
monitorRunDependencies: () => {},
onRuntimeInitialized: () => {
log("[boot] runtime initialized");