#!/usr/bin/env python3 """Transform the in-container emscripten compile_commands.json into one Sourcetrail 2021.4.19 (bundled clang ~11) can index on the host. - expands @CMakeFiles/....rsp response files inline - strips emscripten-only and PCH flags (old clang can't load clang-20 .pch) - replaces the PCH with `-include cmake_pch.hxx` so those headers still parse - injects libc++ 11 headers (-nostdinc++) + emscripten sysroot includes - rewrites /workspace -> host pcbjam, /emsdk -> host tools/emsdk mirror Usage: transform_compile_db.py [--no-wasm-target] """ import json import os import re import shlex import sys PCBJAM = "/Users/V/IdeaProjects/pcbjam-private/pcbjam" ST_DIR = f"{PCBJAM}/sourcetrail" EMSDK_HOST = f"{PCBJAM}/tools/emsdk" PATH_MAP = [("/workspace/", PCBJAM + "/"), ("/emsdk/", EMSDK_HOST + "/")] DROP_EXACT = { "-fwasm-exceptions", "-Winvalid-pch", "--emit-symbol-map", } DROP_PREFIX_RE = re.compile(r"^-s[A-Z][A-Z_0-9]*(=.*)?$|^-gseparate-dwarf") def host_path(p: str) -> str: for src, dst in PATH_MAP: if p.startswith(src): return dst + p[len(src):] return p def rewrite_tok(tok: str) -> str: for src, dst in PATH_MAP: tok = tok.replace(src, dst) return tok def expand_rsp(tokens, directory): out = [] for tok in tokens: if tok.startswith("@"): rsp = tok[1:] if not os.path.isabs(rsp): rsp = os.path.join(directory, rsp) rsp_host = host_path(rsp) with open(rsp_host) as f: out.extend(shlex.split(f.read())) else: out.append(tok) return out def transform(tokens, directory, wasm_target=True): tokens = expand_rsp(tokens, directory) argv0 = tokens[0] lang_cxx = argv0.endswith("++") out = [] i = 1 pch_headers = [] while i < len(tokens): tok = tokens[i] if tok == "-Xclang" and i + 1 < len(tokens): nxt = tokens[i + 1] if nxt == "-fno-pch-timestamp": i += 2 continue if nxt == "-include-pch": # -Xclang -include-pch -Xclang if i + 3 < len(tokens) and tokens[i + 2] == "-Xclang": pch = tokens[i + 3] hdr = pch[:-4] if pch.endswith(".pch") else pch if os.path.exists(host_path(hdr)): pch_headers.append(hdr) i += 4 continue i += 2 continue out.extend([tok, nxt]) i += 2 continue if tok in DROP_EXACT or DROP_PREFIX_RE.match(tok): i += 1 continue if not wasm_target and tok in ("-matomics", "-mbulk-memory", "-pthread"): i += 1 continue out.append(tok) i += 1 for hdr in pch_headers: out.extend(["-include", hdr]) # Rewrite container paths in the ORIGINAL tokens only, then prepend the # host-path injections — rewriting after injection would re-fire on the # /emsdk/ substring inside the host tools/emsdk mirror path. out = [rewrite_tok(t) for t in out] inject = ["-isystem", f"{EMSDK_HOST}/upstream/emscripten/cache/sysroot/include", "-fexceptions"] if lang_cxx: inject = ["-nostdinc++", "-isystem", f"{ST_DIR}/libcxx-11/include"] + inject if wasm_target: inject = ["--target=wasm32-unknown-emscripten"] + inject return ["clang++" if lang_cxx else "clang"] + inject + out def main(): args = sys.argv[1:] wasm_target = True if args and args[0] == "--no-wasm-target": wasm_target = False args = args[1:] src, dst = args with open(src) as f: db = json.load(f) out_db = [] missing = 0 for e in db: tokens = shlex.split(e["command"]) directory = e["directory"] new_tokens = transform(tokens, directory, wasm_target) file_host = host_path(e["file"]) if not os.path.exists(file_host): missing += 1 continue out_db.append({ "directory": host_path(directory), "command": " ".join(shlex.quote(t) for t in new_tokens), "file": file_host, }) with open(dst, "w") as f: json.dump(out_db, f, indent=1) print(f"wrote {len(out_db)} entries to {dst} ({missing} skipped: file missing on host)") if __name__ == "__main__": main()