tooling(sourcetrail): code-graph setup + asyncify allocator-suspend plan

sourcetrail/: Sourcetrail 2021.4.19 indexing pipeline for the wasm port —
compile-db transform (rsp expansion, PCH strip, libc++-11 pinning, path
rewrites), removelist-candidate analysis over the indexed call graph, README
with regen steps and the hard-won tricks (relative --project-file hang,
SDK-header poisoning). Heavy artifacts (.srctrldb, compile dbs, libcxx
headers) stay untracked via the folder's .gitignore.

docs/asyncify-allocator-suspend/plan.md: verified plan for the nanosleep-shim
zero-duration guard (mimalloc mi_atomic_yield=sleep(0) can suspend malloc on
the main thread), the red/green contention test, and the adversarially
verified removelist additions with measured payoff (80.9k -> 60.0k
instrumented functions). Planned, not yet executed.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01P2NHhbzuEHqP2JmcrSD96D

sourcetrail/transform_compile_db.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""Transform the in-container emscripten compile_commands.json into one
+Sourcetrail 2021.4.19 (bundled clang ~11) can index on the host.
+
+- expands @CMakeFiles/....rsp response files inline
+- strips emscripten-only and PCH flags (old clang can't load clang-20 .pch)
+- replaces the PCH with `-include cmake_pch.hxx` so those headers still parse
+- injects libc++ 11 headers (-nostdinc++) + emscripten sysroot includes
+- rewrites /workspace -> host pcbjam, /emsdk -> host tools/emsdk mirror
+
+Usage: transform_compile_db.py [--no-wasm-target] <in.json> <out.json>
+"""
+import json
+import os
+import re
+import shlex
+import sys
+
+PCBJAM = "/Users/V/IdeaProjects/pcbjam-private/pcbjam"
+ST_DIR = f"{PCBJAM}/sourcetrail"
+EMSDK_HOST = f"{PCBJAM}/tools/emsdk"
+
+PATH_MAP = [("/workspace/", PCBJAM + "/"), ("/emsdk/", EMSDK_HOST + "/")]
+
+DROP_EXACT = {
+    "-fwasm-exceptions",
+    "-Winvalid-pch",
+    "--emit-symbol-map",
+}
+DROP_PREFIX_RE = re.compile(r"^-s[A-Z][A-Z_0-9]*(=.*)?$|^-gseparate-dwarf")
+
+
+def host_path(p: str) -> str:
+    for src, dst in PATH_MAP:
+        if p.startswith(src):
+            return dst + p[len(src):]
+    return p
+
+
+def rewrite_tok(tok: str) -> str:
+    for src, dst in PATH_MAP:
+        tok = tok.replace(src, dst)
+    return tok
+
+
+def expand_rsp(tokens, directory):
+    out = []
+    for tok in tokens:
+        if tok.startswith("@"):
+            rsp = tok[1:]
+            if not os.path.isabs(rsp):
+                rsp = os.path.join(directory, rsp)
+            rsp_host = host_path(rsp)
+            with open(rsp_host) as f:
+                out.extend(shlex.split(f.read()))
+        else:
+            out.append(tok)
+    return out
+
+
+def transform(tokens, directory, wasm_target=True):
+    tokens = expand_rsp(tokens, directory)
+
+    argv0 = tokens[0]
+    lang_cxx = argv0.endswith("++")
+    out = []
+
+    i = 1
+    pch_headers = []
+    while i < len(tokens):
+        tok = tokens[i]
+        if tok == "-Xclang" and i + 1 < len(tokens):
+            nxt = tokens[i + 1]
+            if nxt == "-fno-pch-timestamp":
+                i += 2
+                continue
+            if nxt == "-include-pch":
+                # -Xclang -include-pch -Xclang <path.pch>
+                if i + 3 < len(tokens) and tokens[i + 2] == "-Xclang":
+                    pch = tokens[i + 3]
+                    hdr = pch[:-4] if pch.endswith(".pch") else pch
+                    if os.path.exists(host_path(hdr)):
+                        pch_headers.append(hdr)
+                    i += 4
+                    continue
+                i += 2
+                continue
+            out.extend([tok, nxt])
+            i += 2
+            continue
+        if tok in DROP_EXACT or DROP_PREFIX_RE.match(tok):
+            i += 1
+            continue
+        if not wasm_target and tok in ("-matomics", "-mbulk-memory", "-pthread"):
+            i += 1
+            continue
+        out.append(tok)
+        i += 1
+
+    for hdr in pch_headers:
+        out.extend(["-include", hdr])
+
+    # Rewrite container paths in the ORIGINAL tokens only, then prepend the
+    # host-path injections — rewriting after injection would re-fire on the
+    # /emsdk/ substring inside the host tools/emsdk mirror path.
+    out = [rewrite_tok(t) for t in out]
+
+    inject = ["-isystem", f"{EMSDK_HOST}/upstream/emscripten/cache/sysroot/include",
+              "-fexceptions"]
+    if lang_cxx:
+        inject = ["-nostdinc++", "-isystem", f"{ST_DIR}/libcxx-11/include"] + inject
+    if wasm_target:
+        inject = ["--target=wasm32-unknown-emscripten"] + inject
+
+    return ["clang++" if lang_cxx else "clang"] + inject + out
+
+
+def main():
+    args = sys.argv[1:]
+    wasm_target = True
+    if args and args[0] == "--no-wasm-target":
+        wasm_target = False
+        args = args[1:]
+    src, dst = args
+
+    with open(src) as f:
+        db = json.load(f)
+
+    out_db = []
+    missing = 0
+    for e in db:
+        tokens = shlex.split(e["command"])
+        directory = e["directory"]
+        new_tokens = transform(tokens, directory, wasm_target)
+        file_host = host_path(e["file"])
+        if not os.path.exists(file_host):
+            missing += 1
+            continue
+        out_db.append({
+            "directory": host_path(directory),
+            "command": " ".join(shlex.quote(t) for t in new_tokens),
+            "file": file_host,
+        })
+
+    with open(dst, "w") as f:
+        json.dump(out_db, f, indent=1)
+    print(f"wrote {len(out_db)} entries to {dst} ({missing} skipped: file missing on host)")
+
+
+if __name__ == "__main__":
+    main()