feat(standalone): OOM tab recovery (0002)
Automatically recover from an out-of-memory by respawning a fresh browsing context (fresh wasm heap) and discarding the dead one — the manual "duplicate tab, close the OOM'd one" fix, automated and capped at 2 retries so a too-small machine stops instead of looping. - recovery/oom-watch.ts: the recover() state machine with the retry counter in the URL (?oomRetry=N, cap 2). Mode A soft detection — Module.onAbort plus window error/unhandledrejection matched against an OOM signature, behind an idempotent latch. Mode B hard-kill detection — a per-tab localStorage heartbeat sentinel; a stale sentinel on the next load continues the retry chain (heartbeat keys avoid mistaking a second live tab for a crash). Healthy session (alive HEALTHY_RESET_MS) resets the chain. Default strategy is location.replace; window.open+close is behind ?oomStrategy=newtab. Platform access goes through injectable win/storage seams for testability. - recovery/MemoryExhaustedDialog.tsx: terminal "out of memory" UI, reusing 0001's BlockingDialog; copy is honest about lost unsaved edits. - wasm/boot.ts: forward emscripten Module.onAbort to the watcher (optional BootOptions.onAbort). - WasmTool: install the watcher, gate boot on its proceed flag (skip when the chain is already exhausted), and render the terminal dialog. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
ae20d49241
commit
0d2f88b437
4 changed files with 391 additions and 2 deletions
|
|
@ -13,6 +13,8 @@ import { memfsProjectDir } from "@/wasm/constants";
|
|||
import { driveProjectIntoTool, type ToolFile } from "@/wasm/kicad-runner";
|
||||
import type { CollabWindow } from "@/wasm/collab";
|
||||
import { clog, cwarn } from "@/wasm/collab/debug";
|
||||
import { createOomWatch } from "@/recovery/oom-watch";
|
||||
import { MemoryExhaustedDialog } from "@/recovery/MemoryExhaustedDialog";
|
||||
|
||||
// Tools with a working collab bridge (kicadCollabSnapshot/Apply embind exports).
|
||||
const COLLAB_TOOLS = new Set<Tool>(["pl_editor", "eeschema", "pcbnew"]);
|
||||
|
|
@ -259,6 +261,7 @@ export function WasmTool({
|
|||
const [status, setStatus] = React.useState("Loading tool…");
|
||||
const [logs, setLogs] = React.useState<string[]>([]);
|
||||
const [showLog, setShowLog] = React.useState(false);
|
||||
const [oomExhausted, setOomExhausted] = React.useState(false);
|
||||
|
||||
const base = (assetBaseUrl ?? WASM_ASSET_BASE_URL).replace(/\/$/, "");
|
||||
const append = React.useCallback(
|
||||
|
|
@ -291,9 +294,27 @@ export function WasmTool({
|
|||
|
||||
const win = window as ToolWindow;
|
||||
|
||||
// OOM recovery (feature 0002): watch for soft aborts + a stale hard-kill
|
||||
// sentinel, respawning a fresh tab (capped). If the chain is already
|
||||
// exhausted, skip boot and show the terminal dialog.
|
||||
const oom = createOomWatch({
|
||||
channelKey: `${slug}:${targetPath ?? tool}`,
|
||||
showExhaustedDialog: () => setOomExhausted(true),
|
||||
log: append,
|
||||
});
|
||||
const { proceed } = oom.start();
|
||||
if (!proceed) return;
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
await bootKicadTool({ tool, base, container, log: append, onStatus: setStatus });
|
||||
await bootKicadTool({
|
||||
tool,
|
||||
base,
|
||||
container,
|
||||
log: append,
|
||||
onStatus: setStatus,
|
||||
onAbort: oom.onAbort,
|
||||
});
|
||||
await driveProjectIntoTool(win, {
|
||||
tool,
|
||||
slug,
|
||||
|
|
@ -309,6 +330,8 @@ export function WasmTool({
|
|||
setStatus(`Error: ${String(err)}`);
|
||||
}
|
||||
})();
|
||||
|
||||
return () => oom.stop();
|
||||
// Boot is one-shot per mount; deps intentionally exclude files/targetPath so
|
||||
// they don't retrigger a (rejected) second boot.
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
|
|
@ -325,6 +348,10 @@ export function WasmTool({
|
|||
<div ref={containerRef} id="main-window" className="absolute inset-0 h-full w-full" />
|
||||
<div id="window-container" />
|
||||
|
||||
{oomExhausted && (
|
||||
<MemoryExhaustedDialog onReload={() => window.location.reload()} />
|
||||
)}
|
||||
|
||||
{status && (
|
||||
<div className="pointer-events-none absolute left-3 top-3 z-20 rounded bg-black/70 px-3 py-2 font-mono text-xs text-white">
|
||||
{status}
|
||||
|
|
|
|||
35
web/standalone/src/recovery/MemoryExhaustedDialog.tsx
Normal file
35
web/standalone/src/recovery/MemoryExhaustedDialog.tsx
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
import { BlockingDialog } from "@/preflight/BlockingDialog";
|
||||
|
||||
/**
|
||||
* Terminal "not enough memory" UI for feature 0002 — shown after the OOM retry
|
||||
* chain hits MAX_RETRIES. Reuses 0001's blocking dialog for visual consistency.
|
||||
*
|
||||
* The copy is honest about the two limitations the spec calls out: a respawn
|
||||
* re-opens the same file but loses in-tab edits not yet pushed to the backend /
|
||||
* Y.Doc, and `window.close()` may not close the very first (user-opened) tab.
|
||||
*/
|
||||
export function MemoryExhaustedDialog({ onReload }: { onReload?: () => void }) {
|
||||
return (
|
||||
<BlockingDialog
|
||||
title="Your device ran out of memory"
|
||||
description="The editor tried to recover a few times but kept running out of memory, so it stopped to avoid looping. This design is likely too large for this device's available memory."
|
||||
reasons={[
|
||||
{
|
||||
title: "What you can try",
|
||||
detail:
|
||||
"Close other tabs and applications to free memory, use a desktop machine with more RAM, or open a smaller design.",
|
||||
},
|
||||
{
|
||||
title: "Unsaved changes",
|
||||
detail:
|
||||
"Edits that weren't yet synced to the server may be lost. Any older tab left open can still be closed manually.",
|
||||
},
|
||||
]}
|
||||
primary={
|
||||
onReload
|
||||
? { label: "Reload and try again", onClick: onReload }
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
);
|
||||
}
|
||||
317
web/standalone/src/recovery/oom-watch.ts
Normal file
317
web/standalone/src/recovery/oom-watch.ts
Normal file
|
|
@ -0,0 +1,317 @@
|
|||
/**
|
||||
* OOM tab recovery — feature 0002.
|
||||
*
|
||||
* The KiCad WASM runtime is process-global and cannot be torn down (see boot.ts:
|
||||
* "Reload the page to open another tool"). After an out-of-memory the wasm heap
|
||||
* and `Module` are unrecoverable in-context, so the only clean reset is a brand
|
||||
* new browsing context — exactly the manual "duplicate the tab, close the dead
|
||||
* one" fix. This module automates it, capped at MAX_RETRIES so a genuinely
|
||||
* too-small machine stops instead of looping forever.
|
||||
*
|
||||
* Two crash modes:
|
||||
* - Mode A (soft abort): JS is still alive — emscripten `abort()` /
|
||||
* `Module.onAbort`, or a `RangeError`/`RuntimeError`/"Aborted(" surfacing as
|
||||
* a window error/unhandledrejection. The tab self-recovers.
|
||||
* - Mode B (hard kill): the renderer is killed ("Aw, Snap"); no in-tab code
|
||||
* runs. We detect it on the NEXT load via a stale `localStorage` heartbeat
|
||||
* sentinel and continue the retry chain.
|
||||
*
|
||||
* The retry counter rides in the URL query (`?oomRetry=N`) so a fresh context
|
||||
* knows its place in the chain. A session that stays healthy for HEALTHY_RESET_MS
|
||||
* resets the chain.
|
||||
*
|
||||
* Pure-ish and unit-testable: all platform access goes through injectable
|
||||
* `win`/`storage` seams (default to the real `window`/`localStorage`).
|
||||
*/
|
||||
|
||||
export const MAX_RETRIES = 2;
|
||||
|
||||
const RETRY_PARAM = "oomRetry";
|
||||
const STRATEGY_PARAM = "oomStrategy";
|
||||
const SENTINEL_PREFIX = "oom:running:";
|
||||
|
||||
/** A session alive this long is "healthy" → clear the retry chain. */
|
||||
export const HEALTHY_RESET_MS = 30_000;
|
||||
/** How often a live tab refreshes its sentinel heartbeat. */
|
||||
const HEARTBEAT_MS = 10_000;
|
||||
/** A sentinel whose heartbeat is older than this means a hard-killed tab, not a
|
||||
* live one (the false-positive guard for two tabs open on the same file). */
|
||||
export const SENTINEL_STALE_MS = 60_000;
|
||||
|
||||
const OOM_SIGNATURE =
|
||||
/out of memory|RangeError|Aborted\(|Cannot enlarge memory|memory access out of bounds/i;
|
||||
|
||||
/** Whether a console/error string looks like an OOM. Exported for tests. */
|
||||
export function looksLikeOom(text: string | null | undefined): boolean {
|
||||
return typeof text === "string" && OOM_SIGNATURE.test(text);
|
||||
}
|
||||
|
||||
interface SentinelValue {
|
||||
startedAt: number;
|
||||
/** Last heartbeat — staleness is measured from this. */
|
||||
beat: number;
|
||||
/** Retry-chain position this instance was running at. */
|
||||
retry: number;
|
||||
}
|
||||
|
||||
export interface OomWatchOptions {
|
||||
/** Project slug + target file — same keying as the collab channel. */
|
||||
channelKey: string;
|
||||
/** Show the terminal "not enough memory" UI when the cap is reached. */
|
||||
showExhaustedDialog: () => void;
|
||||
log?: (msg: string) => void;
|
||||
/** Injectable for tests; defaults to the real globals. */
|
||||
win?: Window;
|
||||
storage?: Storage;
|
||||
}
|
||||
|
||||
export interface OomWatch {
|
||||
/**
|
||||
* Install listeners + the sentinel, and reconcile any stale hard-kill sentinel.
|
||||
* Returns `proceed: false` when the chain is already exhausted (boot should be
|
||||
* skipped — the terminal dialog is shown instead).
|
||||
*/
|
||||
start: () => { proceed: boolean };
|
||||
/** Remove listeners + timers and clear our own sentinel (clean unload). */
|
||||
stop: () => void;
|
||||
/** Soft-abort entry point — wired to `Module.onAbort` in boot.ts. */
|
||||
onAbort: (what?: unknown) => void;
|
||||
}
|
||||
|
||||
function now(): number {
|
||||
// App code (not a workflow script): Date.now is available.
|
||||
return Date.now();
|
||||
}
|
||||
|
||||
function newTabId(): string {
|
||||
try {
|
||||
if (typeof crypto !== "undefined" && crypto.randomUUID) return crypto.randomUUID();
|
||||
} catch {
|
||||
/* fall through */
|
||||
}
|
||||
return `${now().toString(36)}-${Math.floor(Math.random() * 1e9).toString(36)}`;
|
||||
}
|
||||
|
||||
export function createOomWatch(opts: OomWatchOptions): OomWatch {
|
||||
const win = opts.win ?? window;
|
||||
const storage: Storage | null = opts.storage ?? safeLocalStorage(win);
|
||||
const log = opts.log ?? (() => {});
|
||||
const tabId = newTabId();
|
||||
const sentinelKey = `${SENTINEL_PREFIX}${opts.channelKey}:${tabId}`;
|
||||
|
||||
let recovering = false; // idempotent latch — one OOM fires recovery once
|
||||
let started = false;
|
||||
let healthyTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let heartbeatTimer: ReturnType<typeof setInterval> | undefined;
|
||||
|
||||
const currentRetry = (): number => {
|
||||
const raw = new URLSearchParams(win.location.search).get(RETRY_PARAM);
|
||||
const n = Number(raw ?? "0");
|
||||
return Number.isFinite(n) && n > 0 ? Math.floor(n) : 0;
|
||||
};
|
||||
|
||||
const strategy = (): "replace" | "newtab" =>
|
||||
new URLSearchParams(win.location.search).get(STRATEGY_PARAM) === "newtab"
|
||||
? "newtab"
|
||||
: "replace";
|
||||
|
||||
// --- sentinel helpers ------------------------------------------------------
|
||||
|
||||
const writeSentinel = (retry: number): void => {
|
||||
if (!storage) return;
|
||||
const t = now();
|
||||
const value: SentinelValue = { startedAt: t, beat: t, retry };
|
||||
try {
|
||||
storage.setItem(sentinelKey, JSON.stringify(value));
|
||||
} catch {
|
||||
/* storage disabled — Mode B simply won't be available */
|
||||
}
|
||||
};
|
||||
|
||||
const beat = (): void => {
|
||||
if (!storage) return;
|
||||
try {
|
||||
const raw = storage.getItem(sentinelKey);
|
||||
if (!raw) return;
|
||||
const value = JSON.parse(raw) as SentinelValue;
|
||||
value.beat = now();
|
||||
storage.setItem(sentinelKey, JSON.stringify(value));
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
};
|
||||
|
||||
const clearOwnSentinel = (): void => {
|
||||
if (!storage) return;
|
||||
try {
|
||||
storage.removeItem(sentinelKey);
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
};
|
||||
|
||||
/** Scan for stale sentinels of THIS channel from other (crashed) tabs; remove
|
||||
* them and return the highest retry seen, or null if none. */
|
||||
const reapStaleSentinels = (): number | null => {
|
||||
if (!storage) return null;
|
||||
const prefix = `${SENTINEL_PREFIX}${opts.channelKey}:`;
|
||||
const stale: string[] = [];
|
||||
let maxRetry: number | null = null;
|
||||
const t = now();
|
||||
try {
|
||||
for (let i = 0; i < storage.length; i++) {
|
||||
const k = storage.key(i);
|
||||
if (!k || k === sentinelKey || !k.startsWith(prefix)) continue;
|
||||
let value: SentinelValue | null = null;
|
||||
try {
|
||||
value = JSON.parse(storage.getItem(k) ?? "") as SentinelValue;
|
||||
} catch {
|
||||
value = null;
|
||||
}
|
||||
// Malformed or heartbeat older than the staleness window ⇒ dead tab.
|
||||
if (!value || typeof value.beat !== "number" || t - value.beat > SENTINEL_STALE_MS) {
|
||||
stale.push(k);
|
||||
const r = value && typeof value.retry === "number" ? value.retry : 0;
|
||||
maxRetry = Math.max(maxRetry ?? 0, r);
|
||||
}
|
||||
}
|
||||
for (const k of stale) storage.removeItem(k);
|
||||
} catch {
|
||||
return maxRetry;
|
||||
}
|
||||
return maxRetry;
|
||||
};
|
||||
|
||||
// --- recovery state machine ------------------------------------------------
|
||||
|
||||
const recover = (reason: string, what?: string): void => {
|
||||
if (recovering) return;
|
||||
recovering = true;
|
||||
const n = currentRetry();
|
||||
log(`[oom] recover (${reason}${what ? `: ${what}` : ""}) at retry ${n}`);
|
||||
|
||||
if (n >= MAX_RETRIES) {
|
||||
log(`[oom] retry cap (${MAX_RETRIES}) reached — giving up`);
|
||||
clearOwnSentinel();
|
||||
opts.showExhaustedDialog();
|
||||
return;
|
||||
}
|
||||
|
||||
const url = new URL(win.location.href);
|
||||
url.searchParams.set(RETRY_PARAM, String(n + 1));
|
||||
clearOwnSentinel(); // the new context writes its own sentinel
|
||||
|
||||
if (strategy() === "newtab") {
|
||||
try {
|
||||
const w = win.open(url.toString(), "_blank");
|
||||
if (w) {
|
||||
// We own the new tab → close ourselves; the new tab is the survivor.
|
||||
// (window.close only reliably works on script-opened windows; on the
|
||||
// very first user-opened tab it may be ignored — acceptable, the new
|
||||
// tab still works.)
|
||||
win.close();
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
/* popup blocked → fall through to in-place replace */
|
||||
}
|
||||
}
|
||||
// Default + fallback: in-place reload with the bumped counter (fresh heap,
|
||||
// no popup risk).
|
||||
win.location.replace(url.toString());
|
||||
};
|
||||
|
||||
// --- Mode A listeners ------------------------------------------------------
|
||||
|
||||
const onError = (e: ErrorEvent): void => {
|
||||
const msg = e?.message || (e?.error ? String(e.error) : "");
|
||||
if (looksLikeOom(msg)) recover("error", msg);
|
||||
};
|
||||
const onRejection = (e: PromiseRejectionEvent): void => {
|
||||
const reason = e?.reason;
|
||||
const msg = typeof reason === "string" ? reason : reason ? String(reason) : "";
|
||||
if (looksLikeOom(msg)) recover("unhandledrejection", msg);
|
||||
};
|
||||
const onBeforeUnload = (): void => {
|
||||
// Clean unload → drop our sentinel so it isn't mistaken for a hard kill.
|
||||
clearOwnSentinel();
|
||||
};
|
||||
|
||||
// --- public API ------------------------------------------------------------
|
||||
|
||||
const onAbort = (what?: unknown): void => {
|
||||
recover("onabort", what === undefined ? undefined : String(what));
|
||||
};
|
||||
|
||||
const start = (): { proceed: boolean } => {
|
||||
if (started) return { proceed: true };
|
||||
started = true;
|
||||
|
||||
// Mode B: a stale sentinel from a prior hard-killed instance ⇒ continue its
|
||||
// retry chain rather than resetting (so the cap still applies).
|
||||
const staleRetry = reapStaleSentinels();
|
||||
if (staleRetry !== null) {
|
||||
const n = Math.max(currentRetry(), staleRetry);
|
||||
if (n >= MAX_RETRIES) {
|
||||
log(`[oom] hard-kill chain already exhausted at retry ${n}`);
|
||||
opts.showExhaustedDialog();
|
||||
return { proceed: false };
|
||||
}
|
||||
// Adopt the chain position so a further OOM in THIS instance caps correctly.
|
||||
const url = new URL(win.location.href);
|
||||
url.searchParams.set(RETRY_PARAM, String(n));
|
||||
try {
|
||||
win.history.replaceState(null, "", url.toString());
|
||||
} catch {
|
||||
/* ignore — currentRetry will still read the original */
|
||||
}
|
||||
log(`[oom] continuing hard-kill chain at retry ${n}`);
|
||||
}
|
||||
|
||||
writeSentinel(currentRetry());
|
||||
heartbeatTimer = setInterval(beat, HEARTBEAT_MS);
|
||||
|
||||
win.addEventListener("error", onError);
|
||||
win.addEventListener("unhandledrejection", onRejection);
|
||||
win.addEventListener("beforeunload", onBeforeUnload);
|
||||
|
||||
// Healthy-session reset: after surviving HEALTHY_RESET_MS, clear the chain so
|
||||
// a later, unrelated OOM starts counting fresh.
|
||||
if (currentRetry() > 0) {
|
||||
healthyTimer = setTimeout(() => {
|
||||
const url = new URL(win.location.href);
|
||||
if (url.searchParams.has(RETRY_PARAM)) {
|
||||
url.searchParams.delete(RETRY_PARAM);
|
||||
try {
|
||||
win.history.replaceState(null, "", url.toString());
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
log("[oom] healthy session — retry chain reset");
|
||||
}
|
||||
writeSentinel(0);
|
||||
}, HEALTHY_RESET_MS);
|
||||
}
|
||||
|
||||
return { proceed: true };
|
||||
};
|
||||
|
||||
const stop = (): void => {
|
||||
if (healthyTimer) clearTimeout(healthyTimer);
|
||||
if (heartbeatTimer) clearInterval(heartbeatTimer);
|
||||
win.removeEventListener("error", onError);
|
||||
win.removeEventListener("unhandledrejection", onRejection);
|
||||
win.removeEventListener("beforeunload", onBeforeUnload);
|
||||
clearOwnSentinel();
|
||||
};
|
||||
|
||||
return { start, stop, onAbort };
|
||||
}
|
||||
|
||||
function safeLocalStorage(win: Window): Storage | null {
|
||||
try {
|
||||
return win.localStorage;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -37,6 +37,9 @@ export interface BootOptions {
|
|||
container: HTMLElement;
|
||||
log: (msg: string) => void;
|
||||
onStatus: (text: string) => void;
|
||||
/** OOM recovery hook (feature 0002): emscripten `abort()` routes here so a
|
||||
* soft OOM can respawn a fresh tab. Optional — boot works without it. */
|
||||
onAbort?: (what: string) => void;
|
||||
}
|
||||
|
||||
let booted: { tool: Tool; promise: Promise<void> } | null = null;
|
||||
|
|
@ -75,7 +78,7 @@ function loadScript(src: string): Promise<void> {
|
|||
}
|
||||
|
||||
async function doBoot(opts: BootOptions): Promise<void> {
|
||||
const { tool, base, container, log, onStatus } = opts;
|
||||
const { tool, base, container, log, onStatus, onAbort } = opts;
|
||||
const w = window as ToolWindow;
|
||||
|
||||
// The wasm reads the top-level frame geometry from a GLOBAL `mainWindow`
|
||||
|
|
@ -197,6 +200,13 @@ async function doBoot(opts: BootOptions): Promise<void> {
|
|||
setStatus: (text: string) => {
|
||||
if (text) onStatus(text);
|
||||
},
|
||||
// OOM recovery (feature 0002): emscripten calls onAbort on abort() — commonly
|
||||
// how an out-of-memory surfaces. Forward it so the watcher can respawn.
|
||||
onAbort: (what: unknown) => {
|
||||
const msg = what === undefined ? "" : String(what);
|
||||
log(`[boot] abort: ${msg}`);
|
||||
onAbort?.(msg);
|
||||
},
|
||||
monitorRunDependencies: () => {},
|
||||
onRuntimeInitialized: () => {
|
||||
log("[boot] runtime initialized");
|
||||
|
|
|
|||
Loading…
Reference in a new issue