From bff6cf3f1251e146bd846a98150f1ec79ed43b5a Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 13:15:03 -0500 Subject: [PATCH 01/28] feat(retools): add consolidated kb.h parser (retools.kb) Create new kb.py module with single source of truth for parsing kb.h knowledge-base files. Provides: - parse_kb(text_or_path) -> Kb with functions, globals, typedefs - KbFunction, KbGlobal, Kb dataclasses - extract_function_name() for signature parsing - read_existing_addresses() for bootstrap dedup Co-Authored-By: Claude Opus 4.8 --- retools/kb.py | 104 +++++++++++++++++++++++++++++++++++++++++++++++ tests/test_kb.py | 92 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 retools/kb.py create mode 100644 tests/test_kb.py diff --git a/retools/kb.py b/retools/kb.py new file mode 100644 index 00000000..197e2f9b --- /dev/null +++ b/retools/kb.py @@ -0,0 +1,104 @@ +"""Single source of truth for the kb.h grammar. + +kb.h lines take three shapes: + @ 0xADDR ; -- function at an address + $ 0xADDR -- global variable at an address + -- bare typedef / struct / enum +Lines that are blank or begin with ``//`` are ignored. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass(frozen=True) +class KbFunction: + address: int + signature: str # no leading "@ 0xADDR", no trailing ";" + name: str + + +@dataclass(frozen=True) +class KbGlobal: + address: int + type: str # "" when the line is "$ 0xADDR name" + name: str + + +@dataclass +class Kb: + functions: list[KbFunction] = field(default_factory=list) + globals: list[KbGlobal] = field(default_factory=list) + typedefs: list[str] = field(default_factory=list) + + +def extract_function_name(sig: str) -> str: + """Extract the function name from a signature (no address, no ';'). + + Name is the last whitespace-separated token before '(' (or the whole + pre-paren text), with leading pointer/reference decorators stripped. + """ + paren = sig.find("(") + pre = sig[:paren] if paren != -1 else sig + pre = pre.strip() + if not pre: + return "" + return pre.rsplit(None, 1)[-1].lstrip("*&") + + +def _resolve_text(text_or_path: str | Path) -> str: + if isinstance(text_or_path, Path): + return text_or_path.read_text(encoding="utf-8", errors="replace") + if os.path.isfile(text_or_path): + return Path(text_or_path).read_text(encoding="utf-8", errors="replace") + return text_or_path + + +def parse_kb(text_or_path: str | Path) -> Kb: + """Parse kb.h content (a string) or a path to a kb.h file.""" + kb = Kb() + for raw in _resolve_text(text_or_path).splitlines(): + line = raw.strip() + if not line or line.startswith("//"): + continue + + if line.startswith("@ "): + parts = line[2:].split(None, 1) + if len(parts) < 2: + continue + try: + addr = int(parts[0], 16) + except ValueError: + continue + sig = parts[1].rstrip(";").strip() + kb.functions.append( + KbFunction(address=addr, signature=sig, name=extract_function_name(sig)) + ) + elif line.startswith("$ "): + parts = line[2:].split() + if len(parts) < 2: + continue + try: + addr = int(parts[0], 16) + except ValueError: + continue + name = parts[-1] + type_ = " ".join(parts[1:-1]) + kb.globals.append(KbGlobal(address=addr, type=type_, name=name)) + else: + kb.typedefs.append(line) + return kb + + +def read_existing_addresses(path: str | Path) -> set[int]: + """Return the addresses of ``@`` function entries in a kb.h file. + + Matches the dedup semantics bootstrap relied on (function entries only). + Returns an empty set if the file does not exist. + """ + if not os.path.isfile(path): + return set() + return {f.address for f in parse_kb(Path(path)).functions} diff --git a/tests/test_kb.py b/tests/test_kb.py new file mode 100644 index 00000000..41c707ed --- /dev/null +++ b/tests/test_kb.py @@ -0,0 +1,92 @@ +"""Tests for retools/kb.py -- consolidated kb.h grammar.""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) + + +class TestParseKb: + def test_function_line(self): + from kb import parse_kb + kb = parse_kb("@ 0x401000 void __cdecl ProcessInput(int key);") + assert len(kb.functions) == 1 + f = kb.functions[0] + assert f.address == 0x401000 + assert f.name == "ProcessInput" + assert f.signature == "void __cdecl ProcessInput(int key)" + + def test_function_pointer_return_strips_decorator(self): + from kb import parse_kb + kb = parse_kb("@ 0x401000 Foo* GetFoo(void);") + assert kb.functions[0].name == "GetFoo" + + def test_function_no_parens(self): + from kb import parse_kb + kb = parse_kb("@ 0xDEAD _malloc;") + assert kb.functions[0].address == 0xDEAD + assert kb.functions[0].name == "_malloc" + + def test_global_with_type(self): + from kb import parse_kb + kb = parse_kb("$ 0x7C5548 Object* g_mainObject") + assert len(kb.globals) == 1 + g = kb.globals[0] + assert g.address == 0x7C5548 + assert g.name == "g_mainObject" + assert g.type == "Object*" + + def test_global_no_type(self): + from kb import parse_kb + kb = parse_kb("$ 0x7C554C g_flag") + assert kb.globals[0].name == "g_flag" + assert kb.globals[0].type == "" + + def test_typedef_line(self): + from kb import parse_kb + kb = parse_kb("struct Foo { int x; float y; };") + assert kb.typedefs == ["struct Foo { int x; float y; };"] + + def test_comments_and_blanks_skipped(self): + from kb import parse_kb + kb = parse_kb("// a comment\n\n// another\n") + assert kb.functions == [] + assert kb.globals == [] + assert kb.typedefs == [] + + def test_malformed_address_skipped(self): + from kb import parse_kb + kb = parse_kb("@ 0xZZZZ notahex(void);") + assert kb.functions == [] + + def test_parses_from_path(self, tmp_path): + from kb import parse_kb + p = tmp_path / "kb.h" + p.write_text("@ 0x401000 void Foo(void);\n") + kb = parse_kb(p) + assert kb.functions[0].name == "Foo" + + def test_existing_meccha_kb_all_comments(self): + # The real freeform kb.h is entirely // comments + typedefs at its head; + # it must parse without error and yield zero function/global entries there. + from kb import parse_kb + meccha = Path(__file__).resolve().parent.parent / "patches" / "MecchaChameleon" / "kb.h" + if not meccha.is_file(): + pytest.skip("MecchaChameleon kb.h not present") + kb = parse_kb(meccha) # must not raise + assert isinstance(kb.functions, list) + + +class TestReadExistingAddresses: + def test_collects_function_addresses(self, tmp_path): + from kb import read_existing_addresses + p = tmp_path / "kb.h" + p.write_text("@ 0x401000 void Foo(void);\n$ 0x7C5548 int g_x\n") + addrs = read_existing_addresses(p) + assert addrs == {0x401000} + + def test_missing_file_empty_set(self, tmp_path): + from kb import read_existing_addresses + assert read_existing_addresses(tmp_path / "nope.h") == set() From b8d73999c7c4291f999042dfcd50d67a1b1bf2df Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 13:25:49 -0500 Subject: [PATCH 02/28] refactor(retools): route the three kb.h parsers through retools.kb --- retools/bootstrap.py | 21 ++------------- retools/context.py | 62 +++++-------------------------------------- retools/decompiler.py | 38 ++++++++++---------------- 3 files changed, 22 insertions(+), 99 deletions(-) diff --git a/retools/bootstrap.py b/retools/bootstrap.py index 039ede24..53e37b88 100644 --- a/retools/bootstrap.py +++ b/retools/bootstrap.py @@ -96,24 +96,6 @@ def _is_packed(pe: pefile.PE) -> bool: # KB file I/O # --------------------------------------------------------------------------- -def _read_existing_addresses(kb_path: str) -> set[int]: - """Parse existing kb.h and return the set of known addresses.""" - addresses: set[int] = set() - if not os.path.isfile(kb_path): - return addresses - with open(kb_path) as f: - for line in f: - line = line.strip() - if line.startswith("@ 0x"): - parts = line.split() - if len(parts) >= 2: - try: - addresses.add(int(parts[1], 16)) - except ValueError: - pass - return addresses - - def _write_kb_entries(kb_path: str, entries: list[str], known: set[int]) -> int: """Append new entries to kb.h, skipping addresses already present. @@ -388,7 +370,8 @@ def bootstrap( return {"packed": True, "functions_identified": 0} b = Binary(binary_path) - known_addresses = _read_existing_addresses(kb_path) + from kb import read_existing_addresses + known_addresses = read_existing_addresses(kb_path) stats: dict = { "packed": False, "compiler": "unknown", diff --git a/retools/context.py b/retools/context.py index 3faee36a..9edb1b7c 100644 --- a/retools/context.py +++ b/retools/context.py @@ -40,69 +40,19 @@ # --------------------------------------------------------------------------- def _parse_kb_names(kb_path: Path) -> dict[int, str]: - """Parse ``@ 0xADDR sig;`` lines and extract the function name. - - Handles signatures like: - @ 0x401000 void __cdecl ProcessInput(int key); - @ 0xDEAD _malloc; - """ + """Map function address -> name from a kb.h file.""" if not kb_path.is_file(): return {} - names: dict[int, str] = {} - for line in kb_path.read_text(encoding="utf-8", errors="replace").splitlines(): - line = line.strip() - if not line.startswith("@ "): - continue - # Split: "@", "0xADDR", rest... - parts = line.split(None, 2) - if len(parts) < 3: - continue - try: - va = int(parts[1], 16) - except ValueError: - continue - sig = parts[2].rstrip(";").strip() - # Extract name: last identifier before '(' or the whole token - paren = sig.find("(") - if paren != -1: - pre = sig[:paren].strip() - else: - pre = sig - # Name is the last whitespace-separated token - name = pre.rsplit(None, 1)[-1] if pre else "" - # Strip pointer/ref decorators - name = name.lstrip("*&") - if name: - names[va] = name - return names + from kb import parse_kb + return {f.address: f.name for f in parse_kb(kb_path).functions if f.name} def _parse_kb_globals(kb_path: Path) -> dict[int, str]: - """Parse ``$ 0xADDR type name`` lines and extract the global name. - - Handles lines like: - $ 0x7C5548 Object* g_mainObject - $ 0x7C554C Flags g_renderFlags - """ + """Map global address -> name from a kb.h file.""" if not kb_path.is_file(): return {} - globals_: dict[int, str] = {} - for line in kb_path.read_text(encoding="utf-8", errors="replace").splitlines(): - line = line.strip() - if not line.startswith("$ "): - continue - parts = line.split() - if len(parts) < 3: - continue - try: - va = int(parts[1], 16) - except ValueError: - continue - # Name is the last token (type may have pointer decorators) - name = parts[-1] - if name: - globals_[va] = name - return globals_ + from kb import parse_kb + return {g.address: g.name for g in parse_kb(kb_path).globals if g.name} # --------------------------------------------------------------------------- diff --git a/retools/decompiler.py b/retools/decompiler.py index c390165f..229522fa 100644 --- a/retools/decompiler.py +++ b/retools/decompiler.py @@ -93,7 +93,9 @@ def _ensure_r2_in_path(r2_bin: str) -> None: def _load_types(r2, types_arg: str) -> None: - """Parse a knowledge-base string and send type/function/global commands to r2.""" + """Parse a knowledge base and send type/function/global commands to r2.""" + from retools.kb import parse_kb + if types_arg == "-": text = sys.stdin.read() elif os.path.isfile(types_arg): @@ -101,29 +103,17 @@ def _load_types(r2, types_arg: str) -> None: else: text = types_arg - for raw_line in text.splitlines(): - line = raw_line.strip() - if not line or line.startswith("//"): - continue - - if line.startswith("@ "): - rest = line[2:] - addr_str, sig = rest.split(None, 1) - addr = int(addr_str, 16) - name = sig.rstrip(";").split("(")[0].split()[-1] - r2.cmd(f"af @ {addr:#x}") - r2.cmd(f"afn {name} @ {addr:#x}") - r2.cmd(f"afs {sig.rstrip(';')} @ {addr:#x}") - elif line.startswith("$ "): - parts = line[2:].split() - addr = int(parts[0], 16) - name = parts[-1] - r2.cmd(f"f {name} @ {addr:#x}") - if len(parts) > 2: - type_name = " ".join(parts[1:-1]) - r2.cmd(f"tl {type_name} @ {addr:#x}") - else: - r2.cmd(f"td {line}") + kb = parse_kb(text) + for fn in kb.functions: + r2.cmd(f"af @ {fn.address:#x}") + r2.cmd(f"afn {fn.name} @ {fn.address:#x}") + r2.cmd(f"afs {fn.signature} @ {fn.address:#x}") + for g in kb.globals: + r2.cmd(f"f {g.name} @ {g.address:#x}") + if g.type: + r2.cmd(f"tl {g.type} @ {g.address:#x}") + for td in kb.typedefs: + r2.cmd(f"td {td}") def decompile(binary: str, va: int, *, backend: str = "auto", From b477fca60f392b2720eb305f1447a8079a8e877d Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 13:33:06 -0500 Subject: [PATCH 03/28] fix(decompiler): emit kb typedefs before type references; dedupe context import --- retools/context.py | 3 +-- retools/decompiler.py | 4 ++-- tests/test_decompiler.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/retools/context.py b/retools/context.py index 9edb1b7c..6781d92a 100644 --- a/retools/context.py +++ b/retools/context.py @@ -20,6 +20,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from common import Binary +from kb import parse_kb from funcinfo import find_start, analyze from structrefs import aggregate_struct from search import find_strings @@ -43,7 +44,6 @@ def _parse_kb_names(kb_path: Path) -> dict[int, str]: """Map function address -> name from a kb.h file.""" if not kb_path.is_file(): return {} - from kb import parse_kb return {f.address: f.name for f in parse_kb(kb_path).functions if f.name} @@ -51,7 +51,6 @@ def _parse_kb_globals(kb_path: Path) -> dict[int, str]: """Map global address -> name from a kb.h file.""" if not kb_path.is_file(): return {} - from kb import parse_kb return {g.address: g.name for g in parse_kb(kb_path).globals if g.name} diff --git a/retools/decompiler.py b/retools/decompiler.py index 229522fa..53496525 100644 --- a/retools/decompiler.py +++ b/retools/decompiler.py @@ -104,6 +104,8 @@ def _load_types(r2, types_arg: str) -> None: text = types_arg kb = parse_kb(text) + for td in kb.typedefs: + r2.cmd(f"td {td}") for fn in kb.functions: r2.cmd(f"af @ {fn.address:#x}") r2.cmd(f"afn {fn.name} @ {fn.address:#x}") @@ -112,8 +114,6 @@ def _load_types(r2, types_arg: str) -> None: r2.cmd(f"f {g.name} @ {g.address:#x}") if g.type: r2.cmd(f"tl {g.type} @ {g.address:#x}") - for td in kb.typedefs: - r2.cmd(f"td {td}") def decompile(binary: str, va: int, *, backend: str = "auto", diff --git a/tests/test_decompiler.py b/tests/test_decompiler.py index f7df7902..8b8ba4d5 100644 --- a/tests/test_decompiler.py +++ b/tests/test_decompiler.py @@ -51,3 +51,34 @@ def test_ghidra_backend_routes_to_pyghidra(self, tmp_path): mock_backend.decompile.assert_called_once() assert result == "void func() {}" + + +class FakeR2: + def __init__(self): + self.cmds = [] + + def cmd(self, c): + self.cmds.append(c) + return "" + + +class TestLoadTypesOrdering: + def test_typedefs_emitted_before_references(self, tmp_path): + """td must be sent to r2 before any tl/afs that could reference the type.""" + kb_path = tmp_path / "kb.h" + kb_path.write_text( + "struct Foo { int x; };\n" + "$ 0x7C5548 Foo* g_mainObject\n" + "@ 0x401000 void __cdecl ProcessInput(int key);\n" + ) + + from decompiler import _load_types + r2 = FakeR2() + _load_types(r2, str(kb_path)) + + td_idx = next(i for i, c in enumerate(r2.cmds) if c.startswith("td ")) + tl_idx = next(i for i, c in enumerate(r2.cmds) if c.startswith("tl ")) + afs_idx = next(i for i, c in enumerate(r2.cmds) if c.startswith("afs ")) + + assert td_idx < tl_idx + assert td_idx < afs_idx From 656ae1c33102b136a065b86feebad462e149399b Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 13:35:45 -0500 Subject: [PATCH 04/28] feat(retools): add per-game SQLite index (retools.index) Implement GameIndex writer with schema for per-game analysis results. Provides transactional replace-by-source, WAL with DELETE fallback, schema-version guard, and read-only connections. Skips addresses with bit 63 set (out of signed-64 range) with stderr warning. Co-Authored-By: Claude Haiku 4.5 --- retools/index.py | 198 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_index.py | 78 +++++++++++++++++ 2 files changed, 276 insertions(+) create mode 100644 retools/index.py create mode 100644 tests/test_index.py diff --git a/retools/index.py b/retools/index.py new file mode 100644 index 00000000..91b62140 --- /dev/null +++ b/retools/index.py @@ -0,0 +1,198 @@ +"""Per-game SQLite analysis index (idasql-aligned). + +Owns the schema and the GameIndex writer. Two producers populate it, +distinguished by a ``source`` column so authoritative Ghidra rows replace +provisional bootstrap rows (delete-by-source then insert, transactionally). + +Addresses are stored raw as signed-64 INTEGER (idasql convention). User-mode +PE image bases are positive and fit; rows whose address has bit 63 set are +skipped with a warning rather than corrupting ORDER BY. Render addresses with +SQLite's built-in printf('0x%x', address) -- no UDF, so the read-only +connection stays callback-free. + +CLI: + python -m retools.index status [--db PATH] +""" + +from __future__ import annotations + +import argparse +import sqlite3 +import sys +from pathlib import Path + +_PROJECT = Path(__file__).resolve().parent.parent + +SCHEMA_VERSION = 1 + +_SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL); +CREATE TABLE IF NOT EXISTS funcs (address INTEGER PRIMARY KEY, end_ea INTEGER, name TEXT, size INTEGER, + flags INTEGER, prototype TEXT, comment TEXT, source TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS names (address INTEGER, name TEXT, source TEXT NOT NULL, PRIMARY KEY(address, name)); +CREATE TABLE IF NOT EXISTS xrefs (from_ea INTEGER, to_ea INTEGER, type TEXT, is_code INTEGER, + from_func INTEGER, source TEXT NOT NULL); +CREATE INDEX IF NOT EXISTS ix_xrefs_to ON xrefs(to_ea); +CREATE INDEX IF NOT EXISTS ix_xrefs_from ON xrefs(from_ea); +CREATE TABLE IF NOT EXISTS strings (address INTEGER PRIMARY KEY, length INTEGER, type TEXT, encoding TEXT, + content TEXT, source TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS imports (address INTEGER, name TEXT, module TEXT, ordinal INTEGER, source TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS entries (ordinal INTEGER, address INTEGER, name TEXT, source TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS segments (start_ea INTEGER, end_ea INTEGER, name TEXT, class TEXT, perm INTEGER, + source TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS blocks (func_ea INTEGER, start_ea INTEGER, end_ea INTEGER, size INTEGER, source TEXT NOT NULL); + +CREATE VIEW IF NOT EXISTS callers AS SELECT x.from_func AS caller, x.to_ea AS callee_addr, f.name AS callee + FROM xrefs x LEFT JOIN funcs f ON f.address = x.to_ea WHERE x.is_code=1 AND x.type='call'; +CREATE VIEW IF NOT EXISTS callees AS SELECT x.from_func AS caller, x.to_ea AS callee_addr + FROM xrefs x WHERE x.is_code=1 AND x.type='call'; +CREATE VIEW IF NOT EXISTS grep AS + SELECT 'func' AS entity, address, name FROM funcs WHERE name IS NOT NULL + UNION ALL SELECT 'name', address, name FROM names + UNION ALL SELECT 'import', address, name FROM imports + UNION ALL SELECT 'export', address, name FROM entries + UNION ALL SELECT 'string', address, content FROM strings; +""" + +# Column order per table (source is always last; it is injected by replace()). +_TABLE_COLUMNS = { + "funcs": ["address", "end_ea", "name", "size", "flags", "prototype", "comment", "source"], + "names": ["address", "name", "source"], + "xrefs": ["from_ea", "to_ea", "type", "is_code", "from_func", "source"], + "strings": ["address", "length", "type", "encoding", "content", "source"], + "imports": ["address", "name", "module", "ordinal", "source"], + "entries": ["ordinal", "address", "name", "source"], + "segments": ["start_ea", "end_ea", "name", "class", "perm", "source"], + "blocks": ["func_ea", "start_ea", "end_ea", "size", "source"], +} + +_MAX_ADDR = (1 << 63) - 1 + + +def _is_addr_col(col: str) -> bool: + return col == "address" or col.endswith("_ea") + + +class GameIndex: + """SQLite-backed per-game analysis index. + + Args: + path: Filesystem path to the index database file. + """ + + def __init__(self, path: str = ":memory:"): + self._conn = sqlite3.connect(path) + self._conn.execute("PRAGMA journal_mode=WAL") + mode = self._conn.execute("PRAGMA journal_mode").fetchone()[0] + if str(mode).lower() != "wal": + self._conn.execute("PRAGMA journal_mode=DELETE") + self._init_schema() + + def _init_schema(self) -> None: + cur = self._conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='schema_version'" + ) + if cur.fetchone() is not None: + row = self._conn.execute("SELECT version FROM schema_version").fetchone() + if row and row[0] > SCHEMA_VERSION: + self._conn.close() + self._conn = None + raise RuntimeError( + f"Index schema version {row[0]} is newer than code version " + f"{SCHEMA_VERSION}. Update the code." + ) + return + self._conn.executescript(_SCHEMA_SQL) + self._conn.execute("INSERT INTO schema_version (version) VALUES (?)", (SCHEMA_VERSION,)) + self._conn.commit() + + def replace(self, table: str, rows: list[dict], source: str) -> int: + """Replace all *source* rows in *table* with *rows*, transactionally. + + Rows are dicts keyed by column name (``source`` is injected). Any row + whose address column exceeds signed-64 range is skipped with a warning. + + Returns: + Number of rows inserted. + """ + if table not in _TABLE_COLUMNS: + raise KeyError(f"unknown table: {table}") + cols = _TABLE_COLUMNS[table] + data_cols = [c for c in cols if c != "source"] + addr_cols = [c for c in data_cols if _is_addr_col(c)] + + tuples: list[tuple] = [] + skipped = 0 + for r in rows: + bad = False + for c in addr_cols: + v = r.get(c) + if v is not None and (v < 0 or v > _MAX_ADDR): + bad = True + break + if bad: + skipped += 1 + continue + tuples.append(tuple(r.get(c) for c in data_cols) + (source,)) + + if skipped: + print(f"[index] skipped {skipped} {table} row(s) with out-of-range address", + file=sys.stderr) + + placeholders = ",".join("?" * len(cols)) + insert_sql = f"INSERT INTO {table} ({','.join(cols)}) VALUES ({placeholders})" + with self._conn: # single transaction + self._conn.execute(f"DELETE FROM {table} WHERE source=?", (source,)) + self._conn.executemany(insert_sql, tuples) + return len(tuples) + + def counts(self) -> dict[str, int]: + return { + t: self._conn.execute(f"SELECT count(*) FROM {t}").fetchone()[0] + for t in _TABLE_COLUMNS + } + + def close(self) -> None: + if self._conn is not None: + try: + self._conn.close() + except Exception: + pass + self._conn = None + + @classmethod + def open_ro(cls, path: str) -> sqlite3.Connection: + """Open *path* read-only via a file: URI (never mutates the index).""" + uri = f"file:{Path(path).as_posix()}?mode=ro" + return sqlite3.connect(uri, uri=True) + + @staticmethod + def default_db_path(game: str) -> str: + return str(_PROJECT / "patches" / game / "index.db") + + +def main(argv: list[str] | None = None) -> None: + p = argparse.ArgumentParser(prog="retools.index", description="Per-game SQLite index") + sub = p.add_subparsers(dest="command", required=True) + s = sub.add_parser("status", help="Show per-table counts + schema version") + s.add_argument("game", help="Game/project name (patches//index.db)") + s.add_argument("--db", default=None, help="Explicit index.db path") + args = p.parse_args(argv) + + db_path = args.db or GameIndex.default_db_path(args.game) + if not Path(db_path).is_file(): + print(f"[error] no index at {db_path}. Run bootstrap or 'pyghidra_backend export' first.", + file=sys.stderr) + raise SystemExit(1) + + gi = GameIndex(db_path) + ver = gi._conn.execute("SELECT version FROM schema_version").fetchone()[0] + counts = gi.counts() + gi.close() + print(f"index: {db_path} (schema_version={ver})") + for table, n in counts.items(): + print(f" {table:10s} {n}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_index.py b/tests/test_index.py new file mode 100644 index 00000000..a8e1e03a --- /dev/null +++ b/tests/test_index.py @@ -0,0 +1,78 @@ +"""Tests for retools/index.py -- per-game SQLite index.""" + +import sqlite3 +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) + + +class TestGameIndex: + def test_creates_schema_and_version(self, tmp_path): + from index import GameIndex, SCHEMA_VERSION + gi = GameIndex(str(tmp_path / "index.db")) + ver = gi._conn.execute("SELECT version FROM schema_version").fetchone()[0] + assert ver == SCHEMA_VERSION + gi.close() + + def test_replace_inserts_rows(self, tmp_path): + from index import GameIndex + gi = GameIndex(str(tmp_path / "index.db")) + n = gi.replace("imports", [ + {"address": 0x1000, "name": "malloc", "module": "msvcrt.dll", "ordinal": None}, + ], source="bootstrap") + assert n == 1 + assert gi.counts()["imports"] == 1 + gi.close() + + def test_replace_by_source_is_isolated(self, tmp_path): + from index import GameIndex + gi = GameIndex(str(tmp_path / "index.db")) + gi.replace("funcs", [{"address": 0x1000, "name": "a"}], source="bootstrap") + gi.replace("funcs", [{"address": 0x2000, "name": "b"}], source="ghidra") + # Re-running bootstrap replaces only bootstrap rows, leaves ghidra intact. + gi.replace("funcs", [{"address": 0x1500, "name": "a2"}], source="bootstrap") + rows = gi._conn.execute("SELECT address FROM funcs ORDER BY address").fetchall() + assert [r[0] for r in rows] == [0x1500, 0x2000] + gi.close() + + def test_skips_high_bit_address(self, tmp_path): + from index import GameIndex + gi = GameIndex(str(tmp_path / "index.db")) + n = gi.replace("funcs", [ + {"address": 0x148000000, "name": "ok"}, # positive, fits + {"address": 0x8000000000000000, "name": "bad"}, # bit 63 set + ], source="bootstrap") + assert n == 1 + assert gi.counts()["funcs"] == 1 + gi.close() + + def test_open_ro_is_read_only(self, tmp_path): + from index import GameIndex + db = str(tmp_path / "index.db") + gi = GameIndex(db) + gi.replace("funcs", [{"address": 0x1000, "name": "a"}], source="bootstrap") + gi.close() + conn = GameIndex.open_ro(db) + assert conn.execute("SELECT count(*) FROM funcs").fetchone()[0] == 1 + with pytest.raises(sqlite3.OperationalError): + conn.execute("INSERT INTO funcs (address, source) VALUES (1, 'x')") + conn.close() + + def test_default_db_path(self): + from index import GameIndex + p = GameIndex.default_db_path("MyGame") + assert p.replace("\\", "/").endswith("patches/MyGame/index.db") + + def test_newer_schema_raises(self, tmp_path): + from index import GameIndex, SCHEMA_VERSION + db = str(tmp_path / "index.db") + GameIndex(db).close() + conn = sqlite3.connect(db) + conn.execute("UPDATE schema_version SET version = ?", (SCHEMA_VERSION + 1,)) + conn.commit() + conn.close() + with pytest.raises(RuntimeError): + GameIndex(db) From 2cbffbee571092887ec0c1faa25c2d829cfd6a9b Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 13:41:35 -0500 Subject: [PATCH 05/28] fix(index): guard from_func address; INSERT OR REPLACE for cross-source overwrite 1. Extend _is_addr_col to include from_func column (stores function-start addresses in xrefs that must be range-checked like other address columns). 2. Change INSERT to INSERT OR REPLACE to allow authoritative Ghidra rows (source='ghidra') to overwrite provisional bootstrap rows (source='bootstrap') at overlapping addresses. 3. Update replace() docstring to document last-writer-wins cross-source overwrite. Add two new tests: - test_cross_source_overwrite: verify bootstrap row replaced by ghidra at same address - test_from_func_high_bit_skipped: verify from_func column bit-63 guard works Co-Authored-By: Claude Haiku 4.5 --- retools/index.py | 7 +++++-- tests/test_index.py | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/retools/index.py b/retools/index.py index 91b62140..0b5ac669 100644 --- a/retools/index.py +++ b/retools/index.py @@ -70,7 +70,7 @@ def _is_addr_col(col: str) -> bool: - return col == "address" or col.endswith("_ea") + return col == "address" or col.endswith("_ea") or col == "from_func" class GameIndex: @@ -111,6 +111,9 @@ def replace(self, table: str, rows: list[dict], source: str) -> int: Rows are dicts keyed by column name (``source`` is injected). Any row whose address column exceeds signed-64 range is skipped with a warning. + If a row's address collides with a row from another source, it overwrites it + (last writer wins), allowing authoritative sources (e.g. Ghidra) to replace + provisional data (e.g. bootstrap). Returns: Number of rows inserted. @@ -140,7 +143,7 @@ def replace(self, table: str, rows: list[dict], source: str) -> int: file=sys.stderr) placeholders = ",".join("?" * len(cols)) - insert_sql = f"INSERT INTO {table} ({','.join(cols)}) VALUES ({placeholders})" + insert_sql = f"INSERT OR REPLACE INTO {table} ({','.join(cols)}) VALUES ({placeholders})" with self._conn: # single transaction self._conn.execute(f"DELETE FROM {table} WHERE source=?", (source,)) self._conn.executemany(insert_sql, tuples) diff --git a/tests/test_index.py b/tests/test_index.py index a8e1e03a..eff0009b 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -76,3 +76,27 @@ def test_newer_schema_raises(self, tmp_path): conn.close() with pytest.raises(RuntimeError): GameIndex(db) + + def test_cross_source_overwrite(self, tmp_path): + from index import GameIndex + gi = GameIndex(str(tmp_path / "index.db")) + # Bootstrap writes a func with None name + gi.replace("funcs", [{"address": 0x1000, "name": None}], source="bootstrap") + # Ghidra overwrites the same address with a real name + gi.replace("funcs", [{"address": 0x1000, "name": "Foo"}], source="ghidra") + # Should have 1 row, with name='Foo' from ghidra source + assert gi.counts()["funcs"] == 1 + row = gi._conn.execute("SELECT name, source FROM funcs").fetchone() + assert row[0] == "Foo" + assert row[1] == "ghidra" + gi.close() + + def test_from_func_high_bit_skipped(self, tmp_path): + from index import GameIndex + gi = GameIndex(str(tmp_path / "index.db")) + n = gi.replace("xrefs", [ + {"from_ea": 0x1000, "to_ea": 0x2000, "type": "call", "is_code": 1, "from_func": 0x8000000000000000} + ], source="ghidra") + assert n == 0 + assert gi.counts()["xrefs"] == 0 + gi.close() From 3b603d8edee3bcf05953d850677865d09fb73c95 Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 13:44:38 -0500 Subject: [PATCH 06/28] feat(bootstrap): seed per-game index.db from pefile data --- retools/bootstrap.py | 81 +++++++++++++++++++++++++++++++++++++++++ tests/test_bootstrap.py | 25 +++++++++++++ 2 files changed, 106 insertions(+) diff --git a/retools/bootstrap.py b/retools/bootstrap.py index 53e37b88..67f47098 100644 --- a/retools/bootstrap.py +++ b/retools/bootstrap.py @@ -316,6 +316,81 @@ def _propagate_labels( return kb_entries +def _seed_index(b: Binary, db_path: str) -> None: + """Populate index.db from already-computed pefile data (source='bootstrap'). + + Never raises out to the caller; the index is a convenience, not a + prerequisite for a successful bootstrap. + """ + from index import GameIndex + from search import find_imports, find_strings + + gi = GameIndex(db_path) + try: + # segments + seg_rows = [] + for s in b.pe.sections: + name = s.Name.rstrip(b"\x00").decode("ascii", errors="ignore") + start = b.base + s.VirtualAddress + seg_rows.append({ + "start_ea": start, + "end_ea": start + s.Misc_VirtualSize, + "name": name, + "class": None, + "perm": int(s.Characteristics), + }) + gi.replace("segments", seg_rows, source="bootstrap") + + # imports + imp_rows = [] + if hasattr(b.pe, "DIRECTORY_ENTRY_IMPORT"): + for entry in b.pe.DIRECTORY_ENTRY_IMPORT: + module = entry.dll.decode("ascii", errors="ignore") + for imp in entry.imports: + nm = (imp.name.decode("ascii", errors="ignore") + if imp.name else f"ordinal_{imp.ordinal}") + imp_rows.append({ + "address": imp.address, + "name": nm, + "module": module, + "ordinal": imp.ordinal, + }) + gi.replace("imports", imp_rows, source="bootstrap") + + # entries (exports) + exp_rows = [] + if hasattr(b.pe, "DIRECTORY_ENTRY_EXPORT"): + for exp in b.pe.DIRECTORY_ENTRY_EXPORT.symbols: + if not exp.address: + continue + exp_rows.append({ + "ordinal": exp.ordinal, + "address": b.base + exp.address, + "name": exp.name.decode("ascii", errors="ignore") if exp.name else None, + }) + gi.replace("entries", exp_rows, source="bootstrap") + + # strings + str_rows = [] + for sref in find_strings(b, min_len=4): + if sref.va is None: + continue + str_rows.append({ + "address": sref.va, + "length": len(sref.value), + "type": "ascii", + "encoding": "ascii", + "content": sref.value, + }) + gi.replace("strings", str_rows, source="bootstrap") + + # provisional funcs + names from the entry-point table + func_rows = [{"address": va, "name": None} for va in b.func_table] + gi.replace("funcs", func_rows, source="bootstrap") + finally: + gi.close() + + # --------------------------------------------------------------------------- # bootstrap -- orchestrator # --------------------------------------------------------------------------- @@ -463,6 +538,12 @@ def bootstrap( report_lines.append("") Path(report_path).write_text("\n".join(report_lines)) + # -- Seed the per-game index (best-effort; never breaks bootstrap) ------ + try: + _seed_index(b, os.path.join(project_dir, "index.db")) + except Exception as e: # index is a convenience, not a prerequisite + print(f"index seeding skipped: {e}", file=sys.stderr) + return stats diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 2d06714a..d85d9fb3 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -326,3 +326,28 @@ def test_main_no_args_exits(self): from bootstrap import main with pytest.raises(SystemExit): main([]) + + +# --------------------------------------------------------------------------- +# Index seeding +# --------------------------------------------------------------------------- + +class TestSeedIndex: + def test_seed_index_populates_tables(self, sample_binary, tmp_path): + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) + from bootstrap import _seed_index + from common import Binary + from index import GameIndex + + db = str(tmp_path / "index.db") + b = Binary(sample_binary) + _seed_index(b, db) # must not raise + + gi = GameIndex(db) + counts = gi.counts() + gi.close() + # A real system DLL always has imports and segments. + assert counts["segments"] > 0 + assert counts["imports"] >= 0 # kernel32 imports from other DLLs From 5cd0e0aa8c82386695909bf4abe70ad6a68c9d7d Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 13:47:54 -0500 Subject: [PATCH 07/28] chore(bootstrap): drop unused import; strengthen seed-index test assertion --- retools/bootstrap.py | 2 +- tests/test_bootstrap.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/retools/bootstrap.py b/retools/bootstrap.py index 67f47098..e495c33d 100644 --- a/retools/bootstrap.py +++ b/retools/bootstrap.py @@ -323,7 +323,7 @@ def _seed_index(b: Binary, db_path: str) -> None: prerequisite for a successful bootstrap. """ from index import GameIndex - from search import find_imports, find_strings + from search import find_strings gi = GameIndex(db_path) try: diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index d85d9fb3..e1c7d2a8 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -350,4 +350,4 @@ def test_seed_index_populates_tables(self, sample_binary, tmp_path): gi.close() # A real system DLL always has imports and segments. assert counts["segments"] > 0 - assert counts["imports"] >= 0 # kernel32 imports from other DLLs + assert counts["imports"] > 0 # kernel32 imports from ntdll and others From 44eb05f73dc2911684c9c7128c0c9bcceb35286a Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 13:49:26 -0500 Subject: [PATCH 08/28] feat(retools): add read-only SQL query front-end (retools.query) --- retools/query.py | 113 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_query.py | 83 ++++++++++++++++++++++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 retools/query.py create mode 100644 tests/test_query.py diff --git a/retools/query.py b/retools/query.py new file mode 100644 index 00000000..1d09a8f6 --- /dev/null +++ b/retools/query.py @@ -0,0 +1,113 @@ +"""Read-only SQL front-end for the per-game index. + +Opens index.db read-only (file: URI) so a query can never mutate the index. +Text mode column-aligns; --json emits the idasql envelope. + +Usage: + python -m retools.query "SELECT ..." [--db PATH] [--json] + python -m retools.query --list-tables + python -m retools.query --schema funcs +""" + +from __future__ import annotations + +import argparse +import json +import sqlite3 +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from index import GameIndex + + +def run_query(conn: sqlite3.Connection, sql: str) -> dict: + """Execute *sql* on a read-only connection, returning an envelope result.""" + t0 = time.perf_counter() + try: + cur = conn.execute(sql) + rows = cur.fetchall() + columns = [d[0] for d in cur.description] if cur.description else [] + elapsed = (time.perf_counter() - t0) * 1000.0 + return { + "columns": columns, + "rows": [list(r) for r in rows], + "row_count": len(rows), + "elapsed_ms": round(elapsed, 3), + "error": None, + } + except sqlite3.OperationalError as e: + elapsed = (time.perf_counter() - t0) * 1000.0 + return { + "columns": [], + "rows": [], + "row_count": 0, + "elapsed_ms": round(elapsed, 3), + "error": str(e), + } + + +def _format_text(res: dict) -> str: + if res["error"] is not None: + return f"[error] {res['error']}" + if not res["columns"]: + return "(no columns)" + cols = res["columns"] + widths = [len(c) for c in cols] + str_rows = [[("" if v is None else str(v)) for v in row] for row in res["rows"]] + for row in str_rows: + for i, cell in enumerate(row): + widths[i] = max(widths[i], len(cell)) + lines = [" ".join(c.ljust(widths[i]) for i, c in enumerate(cols))] + lines.append(" ".join("-" * widths[i] for i in range(len(cols)))) + for row in str_rows: + lines.append(" ".join(c.ljust(widths[i]) for i, c in enumerate(row))) + lines.append(f"({res['row_count']} rows, {res['elapsed_ms']} ms)") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> None: + p = argparse.ArgumentParser(prog="retools.query", description="Read-only SQL over index.db") + p.add_argument("game", help="Game/project name (patches//index.db)") + p.add_argument("sql", nargs="?", default="", help="SQL query (SELECT ...)") + p.add_argument("--db", default=None, help="Explicit index.db path") + p.add_argument("--json", action="store_true", help="Emit the idasql JSON envelope") + p.add_argument("--list-tables", action="store_true", help="List tables/views and exit") + p.add_argument("--schema", metavar="TABLE", help="Print PRAGMA table_info for TABLE and exit") + args = p.parse_args(argv) + + db_path = args.db or GameIndex.default_db_path(args.game) + if not Path(db_path).is_file(): + print(f"[error] no index at {db_path}. Run bootstrap or 'pyghidra_backend export' first.", + file=sys.stderr) + raise SystemExit(1) + + conn = GameIndex.open_ro(db_path) + try: + if args.list_tables: + sql = ("SELECT name, type FROM sqlite_master " + "WHERE type IN ('table','view') ORDER BY type, name") + elif args.schema: + sql = f"PRAGMA table_info({args.schema})" + else: + if not args.sql: + print("[error] provide a SQL query, --list-tables, or --schema TABLE", + file=sys.stderr) + raise SystemExit(1) + sql = args.sql + + res = run_query(conn, sql) + finally: + conn.close() + + if args.json: + print(json.dumps({"success": res["error"] is None, "results": [res]})) + else: + print(_format_text(res)) + if res["error"] is not None: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/test_query.py b/tests/test_query.py new file mode 100644 index 00000000..cdda1297 --- /dev/null +++ b/tests/test_query.py @@ -0,0 +1,83 @@ +"""Tests for retools/query.py -- read-only SQL front-end.""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) + + +def _make_db(tmp_path): + from index import GameIndex + db = str(tmp_path / "index.db") + gi = GameIndex(db) + gi.replace("imports", [ + {"address": 0x1000, "name": "malloc", "module": "msvcrt.dll", "ordinal": None}, + {"address": 0x1008, "name": "free", "module": "msvcrt.dll", "ordinal": None}, + ], source="bootstrap") + gi.close() + return db + + +class TestRunQuery: + def test_select_returns_rows(self, tmp_path): + from index import GameIndex + from query import run_query + db = _make_db(tmp_path) + conn = GameIndex.open_ro(db) + res = run_query(conn, "SELECT count(*) AS n FROM imports") + conn.close() + assert res["error"] is None + assert res["columns"] == ["n"] + assert res["rows"][0][0] == 2 + assert res["row_count"] == 1 + assert isinstance(res["elapsed_ms"], (int, float)) + + def test_syntax_error_captured(self, tmp_path): + from index import GameIndex + from query import run_query + db = _make_db(tmp_path) + conn = GameIndex.open_ro(db) + res = run_query(conn, "SELCT bogus") + conn.close() + assert res["error"] is not None + assert res["rows"] == [] + + def test_write_is_rejected(self, tmp_path): + from index import GameIndex + from query import run_query + db = _make_db(tmp_path) + conn = GameIndex.open_ro(db) + res = run_query(conn, "DELETE FROM imports") + conn.close() + assert res["error"] is not None # read-only connection + + +class TestCli: + def test_missing_db_directs_user(self, tmp_path, capsys): + from query import main + with pytest.raises(SystemExit): + main(["NoSuchGame", "SELECT 1", "--db", str(tmp_path / "absent.db")]) + err = capsys.readouterr().err + assert "bootstrap" in err.lower() or "export" in err.lower() + + def test_json_envelope_shape(self, tmp_path, capsys): + import json + from query import main + db = _make_db(tmp_path) + main(["Game", "SELECT name FROM imports ORDER BY name", "--db", db, "--json"]) + out = json.loads(capsys.readouterr().out) + assert out["success"] is True + assert len(out["results"]) == 1 + r = out["results"][0] + assert r["columns"] == ["name"] + assert r["row_count"] == 2 + assert r["error"] is None + + def test_list_tables(self, tmp_path, capsys): + from query import main + db = _make_db(tmp_path) + main(["Game", "", "--db", db, "--list-tables"]) + out = capsys.readouterr().out + assert "imports" in out and "funcs" in out From bc068fc500802d7dd24dd42035e6e67cf1a87b66 Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 13:54:18 -0500 Subject: [PATCH 09/28] feat(pyghidra): add export subcommand seeding index.db (source=ghidra) --- retools/pyghidra_backend.py | 127 +++++++++++++++++++++++++++++++++ tests/test_pyghidra_backend.py | 55 ++++++++++++++ 2 files changed, 182 insertions(+) diff --git a/retools/pyghidra_backend.py b/retools/pyghidra_backend.py index 78cb4f43..29e21bf2 100644 --- a/retools/pyghidra_backend.py +++ b/retools/pyghidra_backend.py @@ -47,6 +47,7 @@ def is_analyzed(project_dir: str, binary_name: str) -> bool: _HERE = Path(__file__).resolve().parent _TOOLS = _HERE.parent / "tools" +sys.path.insert(0, str(_HERE)) # so `from index import GameIndex` resolves as a sibling module def _ensure_java_env(): @@ -183,6 +184,120 @@ def decompile(project_dir: str, binary: str, va: int) -> str: return result.getDecompiledFunction().getC() +# --------------------------------------------------------------------------- +# export +# --------------------------------------------------------------------------- + +def _iter_xrefs(program, func_entries): + """Yield xref row dicts for references originating inside known functions. + + Streams the reference iterator; never materializes the full ref set. + """ + ref_mgr = program.getReferenceManager() + func_mgr = program.getFunctionManager() + it = ref_mgr.getReferenceIterator(program.getMinAddress()) + while it.hasNext(): + ref = it.next() + from_addr = ref.getFromAddress() + containing = func_mgr.getFunctionContaining(from_addr) + if containing is None: + continue + rt = ref.getReferenceType() + if rt.isCall(): + kind = "call" + elif rt.isJump(): + kind = "jump" + else: + kind = "data" + yield { + "from_ea": from_addr.getOffset(), + "to_ea": ref.getToAddress().getOffset(), + "type": kind, + "is_code": 0 if rt.isData() else 1, + "from_func": containing.getEntryPoint().getOffset(), + } + + +def _iter_blocks(program): + """Yield basic-block row dicts.""" + from ghidra.program.model.block import BasicBlockModel + from ghidra.util.task import ConsoleTaskMonitor + + model = BasicBlockModel(program) + monitor = ConsoleTaskMonitor() + it = model.getCodeBlocks(monitor) + while it.hasNext(): + blk = it.next() + start = blk.getFirstStartAddress().getOffset() + end = blk.getMaxAddress().getOffset() + 1 + yield {"func_ea": start, "start_ea": start, "end_ea": end, "size": end - start} + + +def _export_program(program, gi, xrefs=None, blocks=None) -> dict: + """Extract funcs/names/xrefs/blocks from an open program into *gi*. + + *xrefs*/*blocks* default to streaming iterators over the program; tests + pass explicit lists to avoid Ghidra-only classes. + """ + func_rows = [] + name_rows = [] + entries = set() + for func in program.getFunctionManager().getFunctions(True): + ep = func.getEntryPoint().getOffset() + entries.add(ep) + body = func.getBody() + name = func.getName() + func_rows.append({ + "address": ep, + "end_ea": body.getMaxAddress().getOffset() + 1, + "name": name, + "size": body.getNumAddresses(), + "flags": None, + "prototype": str(func.getSignature().getPrototypeString()), + "comment": None, + }) + name_rows.append({"address": ep, "name": name}) + + xref_rows = list(_iter_xrefs(program, entries)) if xrefs is None else xrefs + block_rows = list(_iter_blocks(program)) if blocks is None else blocks + + counts = { + "funcs": gi.replace("funcs", func_rows, source="ghidra"), + "names": gi.replace("names", name_rows, source="ghidra"), + "xrefs": gi.replace("xrefs", xref_rows, source="ghidra"), + "blocks": gi.replace("blocks", block_rows, source="ghidra"), + } + return counts + + +def export(project_dir: str, binary: str, db_path: str) -> str: + """Export analyzed Ghidra facts into index.db with source='ghidra'.""" + from index import GameIndex + + binary_path = Path(binary) + if not is_analyzed(project_dir, binary_path.name): + return f"[error] no analyzed project for {binary_path.name} in {project_dir}" + + pyghidra = _import_pyghidra() + if pyghidra is None: + return "[error] pyghidra is not installed" + if not os.environ.get("GHIDRA_INSTALL_DIR"): + return "[error] GHIDRA_INSTALL_DIR environment variable not set" + + pyghidra.start() + gi = GameIndex(db_path) + try: + with pyghidra.open_program( + binary, project_location=str(project_dir), + project_name=binary_path.stem, analyze=False, + ) as flat_api: + counts = _export_program(flat_api.getCurrentProgram(), gi) + finally: + gi.close() + summary = ", ".join(f"{k}={v}" for k, v in counts.items()) + return f"Export complete: {summary} -> {db_path}" + + # --------------------------------------------------------------------------- # CLI # --------------------------------------------------------------------------- @@ -211,6 +326,12 @@ def main(): p_status.add_argument("binary", help="Path to PE binary") p_status.add_argument("--project", required=True, help="Project directory") + # --- export --- + p_export = sub.add_parser("export", help="Export analyzed facts into index.db") + p_export.add_argument("binary", help="Path to PE binary") + p_export.add_argument("--project", required=True, help="Project directory") + p_export.add_argument("--db", default=None, help="index.db path (default patches//index.db)") + args = parser.parse_args() ghidra_dir = str(Path(args.project) / "ghidra") binary_name = Path(args.binary).name @@ -233,6 +354,12 @@ def main(): print(result) raise SystemExit(0) + if args.command == "export": + from index import GameIndex + db_path = args.db or GameIndex.default_db_path(Path(args.binary).stem) + print(export(ghidra_dir, args.binary, db_path)) + raise SystemExit(0) + if __name__ == "__main__": main() diff --git a/tests/test_pyghidra_backend.py b/tests/test_pyghidra_backend.py index c23f9aaf..72535d34 100644 --- a/tests/test_pyghidra_backend.py +++ b/tests/test_pyghidra_backend.py @@ -322,3 +322,58 @@ def test_decompile_subcommand(self, tmp_path, capsys, monkeypatch): main() captured = capsys.readouterr() assert "void bar(void)" in captured.out + + +# --------------------------------------------------------------------------- +# export / _export_program +# --------------------------------------------------------------------------- + +class TestExportProgram: + def test_export_program_writes_ghidra_rows(self, tmp_path): + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) + from pyghidra_backend import _export_program + from index import GameIndex + + # --- minimal fakes mimicking the Ghidra API surface used by _export_program --- + class FakeAddr: + def __init__(self, off): self._off = off + def getOffset(self): return self._off + + class FakeBody: + def __init__(self, start, end, n): + self._max = FakeAddr(end) + self._n = n + def getMaxAddress(self): return self._max + def getNumAddresses(self): return self._n + + class FakeSig: + def getPrototypeString(self): return "void Foo(void)" + + class FakeFunc: + def __init__(self, ep, name): + self._ep = FakeAddr(ep) + self._name = name + def getEntryPoint(self): return self._ep + def getName(self): return self._name + def getBody(self): return FakeBody(self._ep.getOffset(), self._ep.getOffset() + 0x40, 0x40) + def getSignature(self): return FakeSig() + + class FakeFuncMgr: + def getFunctions(self, forward): return [FakeFunc(0x148001000, "Foo")] + + class FakeProgram: + def getFunctionManager(self): return FakeFuncMgr() + + gi = GameIndex(str(tmp_path / "index.db")) + # _export_program must accept an optional iterables override so xrefs/blocks + # (which need Ghidra-only classes) can be supplied empty in the fake path. + counts = _export_program(FakeProgram(), gi, xrefs=[], blocks=[]) + assert counts["funcs"] == 1 + assert gi.counts()["funcs"] == 1 + row = gi._conn.execute("SELECT name, prototype, source FROM funcs").fetchone() + gi.close() + assert row[0] == "Foo" + assert row[1] == "void Foo(void)" + assert row[2] == "ghidra" From d3d9757f08e176523a099111c3713183b401b762 Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 13:59:58 -0500 Subject: [PATCH 10/28] chore(pyghidra): drop unused func_entries param and entries set in export --- retools/pyghidra_backend.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/retools/pyghidra_backend.py b/retools/pyghidra_backend.py index 29e21bf2..63514c98 100644 --- a/retools/pyghidra_backend.py +++ b/retools/pyghidra_backend.py @@ -188,7 +188,7 @@ def decompile(project_dir: str, binary: str, va: int) -> str: # export # --------------------------------------------------------------------------- -def _iter_xrefs(program, func_entries): +def _iter_xrefs(program): """Yield xref row dicts for references originating inside known functions. Streams the reference iterator; never materializes the full ref set. @@ -241,10 +241,8 @@ def _export_program(program, gi, xrefs=None, blocks=None) -> dict: """ func_rows = [] name_rows = [] - entries = set() for func in program.getFunctionManager().getFunctions(True): ep = func.getEntryPoint().getOffset() - entries.add(ep) body = func.getBody() name = func.getName() func_rows.append({ @@ -258,7 +256,7 @@ def _export_program(program, gi, xrefs=None, blocks=None) -> dict: }) name_rows.append({"address": ep, "name": name}) - xref_rows = list(_iter_xrefs(program, entries)) if xrefs is None else xrefs + xref_rows = list(_iter_xrefs(program)) if xrefs is None else xrefs block_rows = list(_iter_blocks(program)) if blocks is None else blocks counts = { From 2c93613bde761b0c1930f2967e1d6c3cbc79b3a0 Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 14:03:25 -0500 Subject: [PATCH 11/28] feat(pyghidra): add kb-apply subcommand pushing kb.h into Ghidra Applies kb.h function names, prototypes, global labels, and typedefs into an analyzed Ghidra program in one transaction via _kb_apply_program. --- retools/pyghidra_backend.py | 109 +++++++++++++++++++++++++++++++++ tests/test_pyghidra_backend.py | 48 +++++++++++++++ 2 files changed, 157 insertions(+) diff --git a/retools/pyghidra_backend.py b/retools/pyghidra_backend.py index 63514c98..241911bf 100644 --- a/retools/pyghidra_backend.py +++ b/retools/pyghidra_backend.py @@ -296,6 +296,105 @@ def export(project_dir: str, binary: str, db_path: str) -> str: return f"Export complete: {summary} -> {db_path}" +# --------------------------------------------------------------------------- +# kb-apply +# --------------------------------------------------------------------------- + +def _kb_apply_program(program, kb, flat_api, *, apply_prototypes=True, apply_types=True) -> dict: + """Apply a parsed Kb to an open program. Idempotent (USER_DEFINED upserts). + + apply_prototypes/apply_types gate the Ghidra-only signature/DTM code so a + fake program (tests) can exercise the name/label path without those classes. + """ + try: + from ghidra.program.model.symbol import SourceType + except ImportError: + # No Ghidra JVM bridge (e.g. under test with a fake program); the + # name/label path only needs a stand-in value to pass through. + class SourceType: + USER_DEFINED = None + + space = program.getAddressFactory().getDefaultAddressSpace() + listing = program.getListing() + symtab = program.getSymbolTable() + + counts = {"functions": 0, "globals": 0, "typedefs": 0} + + sig_parser = None + if apply_prototypes: + from ghidra.app.util.parser import FunctionSignatureParser + from ghidra.util.task import ConsoleTaskMonitor + sig_parser = FunctionSignatureParser(program.getDataTypeManager(), None) + _monitor = ConsoleTaskMonitor() + + for fn in kb.functions: + addr = space.getAddress(fn.address) + func = listing.getFunctionContaining(addr) + if func is None and flat_api is not None: + func = flat_api.createFunction(addr, fn.name) + if func is None: + continue + func.setName(fn.name, SourceType.USER_DEFINED) + counts["functions"] += 1 + if apply_prototypes and sig_parser is not None: + from ghidra.app.cmd.function import ApplyFunctionSignatureCmd + try: + definition = sig_parser.parse(func.getSignature(), fn.signature + ";") + if definition is not None: + cmd = ApplyFunctionSignatureCmd(addr, definition, SourceType.USER_DEFINED) + cmd.applyTo(program, _monitor) + except Exception: + pass # prototype text may be unparseable; name is already applied + + for g in kb.globals: + addr = space.getAddress(g.address) + symtab.createLabel(addr, g.name, SourceType.USER_DEFINED) + counts["globals"] += 1 + + if apply_types: + from ghidra.app.util.cparser.C import CParser + dtm = program.getDataTypeManager() + parser = CParser(dtm) + for td in kb.typedefs: + try: + parser.parse(td if td.endswith(";") else td + ";") + counts["typedefs"] += 1 + except Exception: + pass # non-type bare lines are skipped + + return counts + + +def kb_apply(project_dir: str, binary: str, kb_path: str) -> str: + """Apply kb.h to the analyzed Ghidra program (one transaction).""" + from kb import parse_kb + + binary_path = Path(binary) + if not is_analyzed(project_dir, binary_path.name): + return f"[error] no analyzed project for {binary_path.name} in {project_dir}" + pyghidra = _import_pyghidra() + if pyghidra is None: + return "[error] pyghidra is not installed" + if not os.environ.get("GHIDRA_INSTALL_DIR"): + return "[error] GHIDRA_INSTALL_DIR environment variable not set" + + kb = parse_kb(Path(kb_path)) + pyghidra.start() + with pyghidra.open_program( + binary, project_location=str(project_dir), + project_name=binary_path.stem, analyze=False, + ) as flat_api: + program = flat_api.getCurrentProgram() + txn = program.startTransaction("kb_apply") + try: + counts = _kb_apply_program(program, kb, flat_api) + finally: + program.endTransaction(txn, True) + program.save("kb_apply", None) + summary = ", ".join(f"{k}={v}" for k, v in counts.items()) + return f"kb-apply complete: {summary}" + + # --------------------------------------------------------------------------- # CLI # --------------------------------------------------------------------------- @@ -330,6 +429,12 @@ def main(): p_export.add_argument("--project", required=True, help="Project directory") p_export.add_argument("--db", default=None, help="index.db path (default patches//index.db)") + # --- kb-apply --- + p_kb = sub.add_parser("kb-apply", help="Apply kb.h names/types into the Ghidra project") + p_kb.add_argument("binary", help="Path to PE binary") + p_kb.add_argument("--project", required=True, help="Project directory") + p_kb.add_argument("--kb", required=True, help="Path to kb.h") + args = parser.parse_args() ghidra_dir = str(Path(args.project) / "ghidra") binary_name = Path(args.binary).name @@ -358,6 +463,10 @@ def main(): print(export(ghidra_dir, args.binary, db_path)) raise SystemExit(0) + if args.command == "kb-apply": + print(kb_apply(ghidra_dir, args.binary, args.kb)) + raise SystemExit(0) + if __name__ == "__main__": main() diff --git a/tests/test_pyghidra_backend.py b/tests/test_pyghidra_backend.py index 72535d34..6a9b24ec 100644 --- a/tests/test_pyghidra_backend.py +++ b/tests/test_pyghidra_backend.py @@ -377,3 +377,51 @@ def getFunctionManager(self): return FakeFuncMgr() assert row[0] == "Foo" assert row[1] == "void Foo(void)" assert row[2] == "ghidra" + + +# --------------------------------------------------------------------------- +# kb-apply / _kb_apply_program +# --------------------------------------------------------------------------- + +class TestKbApplyProgram: + def test_applies_function_name_and_global(self, tmp_path): + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) + from pyghidra_backend import _kb_apply_program + from kb import parse_kb + + applied = {"names": [], "labels": []} + + class FakeAddr: + def __init__(self, off): self._off = off + + class FakeAddrSpace: + def getAddress(self, off): return FakeAddr(off) + + class FakeAddrFactory: + def getDefaultAddressSpace(self): return FakeAddrSpace() + + class FakeFunc: + def setName(self, name, src): applied["names"].append(name) + + class FakeListing: + def getFunctionContaining(self, addr): return FakeFunc() + + class FakeSymbolTable: + def createLabel(self, addr, name, src): + applied["labels"].append(name) + return object() + + class FakeProgram: + def getAddressFactory(self): return FakeAddrFactory() + def getListing(self): return FakeListing() + def getSymbolTable(self): return FakeSymbolTable() + + kb = parse_kb("@ 0x401000 void Foo(void);\n$ 0x7C5548 int g_x\n") + counts = _kb_apply_program(FakeProgram(), kb, flat_api=None, + apply_prototypes=False, apply_types=False) + assert "Foo" in applied["names"] + assert "g_x" in applied["labels"] + assert counts["functions"] == 1 + assert counts["globals"] == 1 From 88eadfdf0aad079dcc5e3e6c235368d5856f35d5 Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 14:06:18 -0500 Subject: [PATCH 12/28] refactor(pyghidra): move Ghidra SourceType test shim from production into the test --- retools/pyghidra_backend.py | 8 +------- tests/test_pyghidra_backend.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/retools/pyghidra_backend.py b/retools/pyghidra_backend.py index 241911bf..6358906d 100644 --- a/retools/pyghidra_backend.py +++ b/retools/pyghidra_backend.py @@ -306,13 +306,7 @@ def _kb_apply_program(program, kb, flat_api, *, apply_prototypes=True, apply_typ apply_prototypes/apply_types gate the Ghidra-only signature/DTM code so a fake program (tests) can exercise the name/label path without those classes. """ - try: - from ghidra.program.model.symbol import SourceType - except ImportError: - # No Ghidra JVM bridge (e.g. under test with a fake program); the - # name/label path only needs a stand-in value to pass through. - class SourceType: - USER_DEFINED = None + from ghidra.program.model.symbol import SourceType space = program.getAddressFactory().getDefaultAddressSpace() listing = program.getListing() diff --git a/tests/test_pyghidra_backend.py b/tests/test_pyghidra_backend.py index 6a9b24ec..53a4775e 100644 --- a/tests/test_pyghidra_backend.py +++ b/tests/test_pyghidra_backend.py @@ -384,13 +384,22 @@ def getFunctionManager(self): return FakeFuncMgr() # --------------------------------------------------------------------------- class TestKbApplyProgram: - def test_applies_function_name_and_global(self, tmp_path): + def test_applies_function_name_and_global(self, monkeypatch): import sys + import types from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) from pyghidra_backend import _kb_apply_program from kb import parse_kb + for name in ("ghidra", "ghidra.program", "ghidra.program.model", "ghidra.program.model.symbol"): + monkeypatch.setitem(sys.modules, name, types.ModuleType(name)) + + class _SourceType: + USER_DEFINED = object() + + monkeypatch.setattr(sys.modules["ghidra.program.model.symbol"], "SourceType", _SourceType, raising=False) + applied = {"names": [], "labels": []} class FakeAddr: From d3396a31a5b4cd9496237ad5759fa8a47432ffa8 Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 14:11:33 -0500 Subject: [PATCH 13/28] feat(retools): add per-project Ghidra daemon + transparent routing Adds ghidra_client.py/ghidra_server.py cloning the livetools daemon pattern on port 27043 for a warm per-project Ghidra program, so repeat decompiles skip the ~3s JVM cold start. pyghidra_backend's decompile/ export/kb_apply now route through a live daemon transparently when one is running for the project, falling back to the existing cold in-process path otherwise (RETOOLS_GHIDRA_COLD=1 / --cold forces cold). --- retools/ghidra_client.py | 100 ++++++++++++++++ retools/ghidra_server.py | 228 ++++++++++++++++++++++++++++++++++++ retools/pyghidra_backend.py | 90 +++++++++++--- tests/test_ghidra_client.py | 66 +++++++++++ 4 files changed, 467 insertions(+), 17 deletions(-) create mode 100644 retools/ghidra_client.py create mode 100644 retools/ghidra_server.py create mode 100644 tests/test_ghidra_client.py diff --git a/retools/ghidra_client.py b/retools/ghidra_client.py new file mode 100644 index 00000000..08f7e28d --- /dev/null +++ b/retools/ghidra_client.py @@ -0,0 +1,100 @@ +"""TCP client + state-file helpers for the per-project Ghidra daemon. + +Mirrors livetools/client.py, but the daemon is per-project: the state file +lives under the project's ghidra dir, not next to this module. Port 27043 +(livetools owns 27042). +""" + +from __future__ import annotations + +import json +import os +import socket +import struct +from pathlib import Path + +HOST = "127.0.0.1" +PORT = 27043 +RECV_BUF = 1 << 20 + + +def state_path(project_dir: str) -> Path: + return Path(project_dir) / ".state.json" + + +def read_state(project_dir: str) -> dict | None: + p = state_path(project_dir) + if not p.exists(): + return None + try: + return json.loads(p.read_text()) + except Exception: + return None + + +def _pid_alive(pid: int | None) -> bool: + if not pid: + return False + try: + import ctypes + kernel32 = ctypes.windll.kernel32 + h = kernel32.OpenProcess(0x00100000, False, pid) # SYNCHRONIZE + if h: + kernel32.CloseHandle(h) + return True + return False + except Exception: + try: + os.kill(pid, 0) + return True + except (OSError, ProcessLookupError): + return False + + +def is_daemon_alive(project_dir: str) -> bool: + state = read_state(project_dir) + if state is None: + return False + try: + s = socket.create_connection((HOST, state.get("port", PORT)), timeout=2) + s.close() + return True + except OSError: + if not _pid_alive(state.get("pid")): + state_path(project_dir).unlink(missing_ok=True) + return False + + +def _send_raw(sock: socket.socket, data: bytes) -> None: + sock.sendall(struct.pack("!I", len(data)) + data) + + +def _recv_raw(sock: socket.socket) -> bytes: + hdr = b"" + while len(hdr) < 4: + chunk = sock.recv(4 - len(hdr)) + if not chunk: + raise ConnectionError("daemon closed connection") + hdr += chunk + length = struct.unpack("!I", hdr)[0] + parts, remaining = [], length + while remaining > 0: + chunk = sock.recv(min(remaining, RECV_BUF)) + if not chunk: + raise ConnectionError("daemon closed connection") + parts.append(chunk) + remaining -= len(chunk) + return b"".join(parts) + + +def send_command(project_dir: str, cmd: dict, timeout: float | None = None) -> dict: + state = read_state(project_dir) + port = state.get("port", PORT) if state else PORT + sock = socket.create_connection((HOST, port), timeout=5) + if timeout is not None: + sock.settimeout(timeout + 10) + try: + _send_raw(sock, json.dumps(cmd).encode()) + return json.loads(_recv_raw(sock)) + finally: + sock.close() diff --git a/retools/ghidra_server.py b/retools/ghidra_server.py new file mode 100644 index 00000000..fe9e69d5 --- /dev/null +++ b/retools/ghidra_server.py @@ -0,0 +1,228 @@ +"""Per-project Ghidra daemon holding one warm open_program handle. + +Cloned from livetools/server.py. Binds 127.0.0.1:27043, 4-byte big-endian +length-prefixed JSON, dict-dispatched _cmd_*, idle-timeout thread. Keeps the +program open so repeat decompiles skip the ~3s JVM cold start. + +Usage: + python -m retools.ghidra_server [--idle 600] +""" + +from __future__ import annotations + +import argparse +import json +import os +import signal +import socket +import struct +import sys +import threading +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from ghidra_client import HOST, PORT, state_path +import pyghidra_backend as pb + +_PROJECT = Path(__file__).resolve().parent.parent + + +class GhidraDaemon: + def __init__(self, game: str, idle: float = 600.0): + self.game = game + self.project_dir = str(_PROJECT / "patches" / game / "ghidra") + self.binary = None + self.idle = idle + self._pyghidra = None + self._ctx = None # open_program context manager + self._flat_api = None + self._program = None + self._running = True + self._last_activity = time.monotonic() + self._lock = threading.Lock() + + # -- program lifecycle --------------------------------------------------- + + def _open(self, binary: str) -> dict: + pg = pb._import_pyghidra() + if pg is None: + return {"ok": False, "error": "pyghidra not installed"} + pg.start() + self._pyghidra = pg + self.binary = binary + stem = Path(binary).stem + self._ctx = pg.open_program( + binary, project_location=self.project_dir, project_name=stem, analyze=False) + self._flat_api = self._ctx.__enter__() + self._program = self._flat_api.getCurrentProgram() + return {"ok": True, "binary": binary} + + def _close_program(self) -> None: + if self._ctx is not None: + try: + self._ctx.__exit__(None, None, None) + except Exception: + pass + self._ctx = self._flat_api = self._program = None + + # -- commands ------------------------------------------------------------ + + def _cmd_status(self, cmd): + return {"ok": True, "game": self.game, "binary": self.binary, + "open": self._program is not None} + + def _cmd_open(self, cmd): + if self._program is not None: + return {"ok": True, "binary": self.binary, "already": True} + return self._open(cmd["binary"]) + + def _cmd_decompile(self, cmd): + if self._program is None: + r = self._open(cmd["binary"]) + if not r.get("ok"): + return r + va = int(cmd["va"]) + return {"ok": True, "text": pb._decompile_open(self._program, va)} + + def _cmd_export(self, cmd): + if self._program is None: + r = self._open(cmd["binary"]) + if not r.get("ok"): + return r + from index import GameIndex + gi = GameIndex(cmd["db"]) + try: + counts = pb._export_program(self._program, gi) + finally: + gi.close() + return {"ok": True, "counts": counts} + + def _cmd_kb_apply(self, cmd): + if self._program is None: + r = self._open(cmd["binary"]) + if not r.get("ok"): + return r + from kb import parse_kb + kb = parse_kb(Path(cmd["kb"])) + txn = self._program.startTransaction("kb_apply") + try: + counts = pb._kb_apply_program(self._program, kb, self._flat_api) + finally: + self._program.endTransaction(txn, True) + self._program.save("kb_apply", None) + return {"ok": True, "counts": counts} + + def _cmd_close(self, cmd): + self._close_program() + return {"ok": True} + + def _cmd_shutdown(self, cmd): + self._close_program() + self._running = False + return {"ok": True} + + # -- dispatch + serve ---------------------------------------------------- + + def handle(self, cmd: dict) -> dict: + self._last_activity = time.monotonic() + op = cmd.get("cmd", "") + handler = getattr(self, f"_cmd_{op}", None) + if handler is None: + return {"ok": False, "error": f"unknown command: {op}"} + with self._lock: + try: + return handler(cmd) + except Exception as exc: + return {"ok": False, "error": str(exc)} + + def _idle_watch(self): + while self._running: + time.sleep(5) + if time.monotonic() - self._last_activity > self.idle: + self._running = False + try: + socket.create_connection((HOST, PORT), timeout=1).close() # unblock accept + except OSError: + pass + return + + def serve(self): + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srv.bind((HOST, PORT)) + srv.listen(4) + srv.settimeout(1.0) + + sp = state_path(self.project_dir) + sp.parent.mkdir(parents=True, exist_ok=True) + sp.write_text(json.dumps({ + "pid": os.getpid(), "port": PORT, "project": self.game, + "binary": self.binary, "started": int(time.time()), + })) + print(f"[ghidra daemon] listening on {HOST}:{PORT}, project={self.game}") + threading.Thread(target=self._idle_watch, daemon=True).start() + + while self._running: + try: + conn, _ = srv.accept() + except socket.timeout: + continue + except OSError: + break + threading.Thread(target=self._handle_conn, args=(conn,), daemon=True).start() + + srv.close() + self._cleanup() + + def _handle_conn(self, conn): + try: + conn.settimeout(300) + hdr = b"" + while len(hdr) < 4: + chunk = conn.recv(4 - len(hdr)) + if not chunk: + return + hdr += chunk + length = struct.unpack("!I", hdr)[0] + buf = b"" + while len(buf) < length: + chunk = conn.recv(min(length - len(buf), 1 << 20)) + if not chunk: + return + buf += chunk + resp = self.handle(json.loads(buf)) + data = json.dumps(resp).encode() + conn.sendall(struct.pack("!I", len(data)) + data) + except Exception: + pass + finally: + conn.close() + + def _cleanup(self): + self._close_program() + state_path(self.project_dir).unlink(missing_ok=True) + print("[ghidra daemon] stopped") + + +def main(): + p = argparse.ArgumentParser(prog="retools.ghidra_server") + p.add_argument("game", help="Game/project name (patches//ghidra)") + p.add_argument("--idle", type=float, default=600.0, help="Idle timeout seconds") + args = p.parse_args() + + daemon = GhidraDaemon(args.game, idle=args.idle) + + def _shutdown(sig, frame): + daemon._running = False + + signal.signal(signal.SIGINT, _shutdown) + signal.signal(signal.SIGTERM, _shutdown) + try: + daemon.serve() + finally: + daemon._cleanup() + + +if __name__ == "__main__": + main() diff --git a/retools/pyghidra_backend.py b/retools/pyghidra_backend.py index 6358906d..85b171ca 100644 --- a/retools/pyghidra_backend.py +++ b/retools/pyghidra_backend.py @@ -125,18 +125,61 @@ def analyze(binary: str, project_dir: str) -> str: return f"Analysis complete: {binary_path.name} ({elapsed:.1f}s), project saved to {proj_dir}" +# --------------------------------------------------------------------------- +# daemon routing +# --------------------------------------------------------------------------- + +def _route_daemon(game: str, cmd: dict): + """Return the daemon response dict if a live daemon serves *game*, else None. + + A pure no-op (returns None) whenever no live daemon is reachable, so + callers fall through to the cold in-process path unchanged. + """ + if os.environ.get("RETOOLS_GHIDRA_COLD") == "1": + return None + try: + import ghidra_client + except ImportError: + return None + project_dir = str(Path("patches") / game / "ghidra") + if not ghidra_client.is_daemon_alive(project_dir): + return None + try: + return ghidra_client.send_command(project_dir, cmd, timeout=120) + except Exception: + return None + + # --------------------------------------------------------------------------- # decompile # --------------------------------------------------------------------------- +def _decompile_open(program, va: int) -> str: + """Decompile the function containing *va* in an already-open program.""" + from ghidra.app.decompiler import DecompInterface, DecompileOptions + from ghidra.util.task import ConsoleTaskMonitor + + ifc = DecompInterface() + ifc.setOptions(DecompileOptions()) + ifc.openProgram(program) + addr = program.getAddressFactory().getDefaultAddressSpace().getAddress(va) + func = program.getListing().getFunctionContaining(addr) + if func is None: + return f"[error] no function found at 0x{va:X}" + result = ifc.decompileFunction(func, 60, ConsoleTaskMonitor()) + return result.getDecompiledFunction().getC() + + def decompile(project_dir: str, binary: str, va: int) -> str: """Decompile a function at the given virtual address. Opens a previously-analyzed Ghidra project and uses DecompInterface - to produce C output for the function containing ``va``. + to produce C output for the function containing ``va``. Transparently + routes to a live per-project daemon (see ``ghidra_client``) when one is + running, to skip the ~3s JVM cold start on repeat calls. Args: - project_dir: Path to the Ghidra project directory. + project_dir: Path to the Ghidra project directory (patches//ghidra). binary: Path to the original PE binary. va: Virtual address inside the target function. @@ -149,6 +192,12 @@ def decompile(project_dir: str, binary: str, va: int) -> str: if not is_analyzed(project_dir, binary_name): return f"[error] no analyzed project for {binary_name} in {project_dir}" + game = Path(project_dir).parent.name # patches//ghidra -> + routed = _route_daemon(game, {"cmd": "decompile", "binary": binary, "va": va}) + if routed is not None: + return routed.get("text", f"[error] {routed.get('error')}") if routed.get("ok") \ + else f"[error] {routed.get('error')}" + pyghidra = _import_pyghidra() if pyghidra is None: return "[error] pyghidra is not installed" @@ -167,21 +216,7 @@ def decompile(project_dir: str, binary: str, va: int) -> str: analyze=False, ) as flat_api: program = flat_api.getCurrentProgram() - from ghidra.app.decompiler import DecompInterface, DecompileOptions - from ghidra.util.task import ConsoleTaskMonitor - - ifc = DecompInterface() - ifc.setOptions(DecompileOptions()) - ifc.openProgram(program) - - addr = program.getAddressFactory().getDefaultAddressSpace().getAddress(va) - func = program.getListing().getFunctionContaining(addr) - if func is None: - return f"[error] no function found at 0x{va:X}" - - monitor = ConsoleTaskMonitor() - result = ifc.decompileFunction(func, 60, monitor) - return result.getDecompiledFunction().getC() + return _decompile_open(program, va) # --------------------------------------------------------------------------- @@ -276,6 +311,14 @@ def export(project_dir: str, binary: str, db_path: str) -> str: if not is_analyzed(project_dir, binary_path.name): return f"[error] no analyzed project for {binary_path.name} in {project_dir}" + game = Path(project_dir).parent.name + routed = _route_daemon(game, {"cmd": "export", "binary": binary, "db": db_path}) + if routed is not None: + if not routed.get("ok"): + return f"[error] {routed.get('error')}" + summary = ", ".join(f"{k}={v}" for k, v in routed["counts"].items()) + return f"Export complete: {summary} -> {db_path}" + pyghidra = _import_pyghidra() if pyghidra is None: return "[error] pyghidra is not installed" @@ -366,6 +409,15 @@ def kb_apply(project_dir: str, binary: str, kb_path: str) -> str: binary_path = Path(binary) if not is_analyzed(project_dir, binary_path.name): return f"[error] no analyzed project for {binary_path.name} in {project_dir}" + + game = Path(project_dir).parent.name + routed = _route_daemon(game, {"cmd": "kb_apply", "binary": binary, "kb": kb_path}) + if routed is not None: + if not routed.get("ok"): + return f"[error] {routed.get('error')}" + summary = ", ".join(f"{k}={v}" for k, v in routed["counts"].items()) + return f"kb-apply complete: {summary}" + pyghidra = _import_pyghidra() if pyghidra is None: return "[error] pyghidra is not installed" @@ -399,6 +451,8 @@ def main(): prog="pyghidra_backend", description="Pyghidra headless Ghidra backend", ) + parser.add_argument("--cold", action="store_true", + help="Force cold in-process start, bypassing any live daemon") sub = parser.add_subparsers(dest="command", required=True) # --- analyze --- @@ -430,6 +484,8 @@ def main(): p_kb.add_argument("--kb", required=True, help="Path to kb.h") args = parser.parse_args() + if args.cold: + os.environ["RETOOLS_GHIDRA_COLD"] = "1" ghidra_dir = str(Path(args.project) / "ghidra") binary_name = Path(args.binary).name diff --git a/tests/test_ghidra_client.py b/tests/test_ghidra_client.py new file mode 100644 index 00000000..b8f5f01c --- /dev/null +++ b/tests/test_ghidra_client.py @@ -0,0 +1,66 @@ +"""Tests for retools/ghidra_client.py -- state file + TCP protocol helpers.""" + +import json +import socket +import struct +import sys +import threading +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) + + +class TestPort: + def test_port_is_27043(self): + import ghidra_client + assert ghidra_client.PORT == 27043 # livetools owns 27042 + + +class TestState: + def test_read_state_missing(self, tmp_path): + import ghidra_client + assert ghidra_client.read_state(str(tmp_path)) is None + + def test_read_state_roundtrip(self, tmp_path): + import ghidra_client + payload = {"pid": 1, "port": 27043, "project": "G", "binary": "g.exe", "started": 0} + ghidra_client.state_path(str(tmp_path)).write_text(json.dumps(payload)) + assert ghidra_client.read_state(str(tmp_path))["project"] == "G" + + +class TestSendCommand: + def test_roundtrip_against_echo_server(self, tmp_path): + import ghidra_client + + def _recv(sock): + hdr = b"" + while len(hdr) < 4: + hdr += sock.recv(4 - len(hdr)) + length = struct.unpack("!I", hdr)[0] + buf = b"" + while len(buf) < length: + buf += sock.recv(length - len(buf)) + return buf + + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srv.bind(("127.0.0.1", 0)) + port = srv.getsockname()[1] + srv.listen(1) + + def serve(): + conn, _ = srv.accept() + req = json.loads(_recv(conn)) + resp = json.dumps({"ok": True, "echo": req["cmd"]}).encode() + conn.sendall(struct.pack("!I", len(resp)) + resp) + conn.close() + + t = threading.Thread(target=serve, daemon=True) + t.start() + + payload = {"pid": 1, "port": port, "project": "G", "binary": "g.exe", "started": 0} + ghidra_client.state_path(str(tmp_path)).write_text(json.dumps(payload)) + resp = ghidra_client.send_command(str(tmp_path), {"cmd": "status"}) + srv.close() + assert resp["ok"] is True + assert resp["echo"] == "status" From 8ddd25811035b2c456290157b7c86be1b5760ee1 Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 14:21:17 -0500 Subject: [PATCH 14/28] fix(ghidra-daemon): lock cleanup against in-flight handlers, guard wrong-binary reuse, cover routing branch --- retools/ghidra_server.py | 95 ++++++++++++++++++++++------------ retools/pyghidra_backend.py | 16 +++--- tests/test_pyghidra_backend.py | 74 ++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 40 deletions(-) diff --git a/retools/ghidra_server.py b/retools/ghidra_server.py index fe9e69d5..a0c34f28 100644 --- a/retools/ghidra_server.py +++ b/retools/ghidra_server.py @@ -41,6 +41,7 @@ def __init__(self, game: str, idle: float = 600.0): self._running = True self._last_activity = time.monotonic() self._lock = threading.Lock() + self._conn_threads = [] # -- program lifecycle --------------------------------------------------- @@ -58,7 +59,21 @@ def _open(self, binary: str) -> dict: self._program = self._flat_api.getCurrentProgram() return {"ok": True, "binary": binary} - def _close_program(self) -> None: + def _ensure_open(self, binary: str) -> dict: + """Open *binary*, swapping out a warm program for a different one. + + Callers run inside ``handle()``'s lock, so a binary switch closes the + stale program via the no-lock ``_close_program_locked`` (the + lock-acquiring ``_close_program`` would deadlock here). + """ + if self._program is not None and self.binary != binary: + self._close_program_locked() + if self._program is None: + return self._open(binary) + return {"ok": True, "binary": self.binary, "already": True} + + def _close_program_locked(self) -> None: + """Tear down the open program. Caller must already hold ``self._lock``.""" if self._ctx is not None: try: self._ctx.__exit__(None, None, None) @@ -66,6 +81,19 @@ def _close_program(self) -> None: pass self._ctx = self._flat_api = self._program = None + def _close_program(self) -> None: + """Tear down the open program, waiting for any in-flight command first. + + Acquires ``self._lock`` so this cannot run concurrently with a + command still executing inside ``handle()`` -- otherwise a + long-running decompile could be closed out from under it (crash / + ``.rep`` corruption). Only call this from outside ``handle()`` + (e.g. ``_cleanup``); command handlers already hold the lock and + must use ``_close_program_locked`` directly. + """ + with self._lock: + self._close_program_locked() + # -- commands ------------------------------------------------------------ def _cmd_status(self, cmd): @@ -73,23 +101,19 @@ def _cmd_status(self, cmd): "open": self._program is not None} def _cmd_open(self, cmd): - if self._program is not None: - return {"ok": True, "binary": self.binary, "already": True} - return self._open(cmd["binary"]) + return self._ensure_open(cmd["binary"]) def _cmd_decompile(self, cmd): - if self._program is None: - r = self._open(cmd["binary"]) - if not r.get("ok"): - return r + r = self._ensure_open(cmd["binary"]) + if not r.get("ok"): + return r va = int(cmd["va"]) return {"ok": True, "text": pb._decompile_open(self._program, va)} def _cmd_export(self, cmd): - if self._program is None: - r = self._open(cmd["binary"]) - if not r.get("ok"): - return r + r = self._ensure_open(cmd["binary"]) + if not r.get("ok"): + return r from index import GameIndex gi = GameIndex(cmd["db"]) try: @@ -99,10 +123,9 @@ def _cmd_export(self, cmd): return {"ok": True, "counts": counts} def _cmd_kb_apply(self, cmd): - if self._program is None: - r = self._open(cmd["binary"]) - if not r.get("ok"): - return r + r = self._ensure_open(cmd["binary"]) + if not r.get("ok"): + return r from kb import parse_kb kb = parse_kb(Path(cmd["kb"])) txn = self._program.startTransaction("kb_apply") @@ -114,11 +137,11 @@ def _cmd_kb_apply(self, cmd): return {"ok": True, "counts": counts} def _cmd_close(self, cmd): - self._close_program() + self._close_program_locked() return {"ok": True} def _cmd_shutdown(self, cmd): - self._close_program() + self._close_program_locked() self._running = False return {"ok": True} @@ -163,17 +186,26 @@ def serve(self): print(f"[ghidra daemon] listening on {HOST}:{PORT}, project={self.game}") threading.Thread(target=self._idle_watch, daemon=True).start() - while self._running: - try: - conn, _ = srv.accept() - except socket.timeout: - continue - except OSError: - break - threading.Thread(target=self._handle_conn, args=(conn,), daemon=True).start() - - srv.close() - self._cleanup() + try: + while self._running: + try: + conn, _ = srv.accept() + except socket.timeout: + continue + except OSError: + break + t = threading.Thread(target=self._handle_conn, args=(conn,), daemon=True) + t.start() + self._conn_threads.append(t) + finally: + srv.close() + # Drain in-flight connection handlers before tearing down the + # program; _close_program's lock acquisition is the actual + # correctness guarantee below, this just avoids leaving threads + # dangling on a closed socket. + for t in self._conn_threads: + t.join(timeout=5.0) + self._cleanup() def _handle_conn(self, conn): try: @@ -218,10 +250,7 @@ def _shutdown(sig, frame): signal.signal(signal.SIGINT, _shutdown) signal.signal(signal.SIGTERM, _shutdown) - try: - daemon.serve() - finally: - daemon._cleanup() + daemon.serve() # serve() guarantees cleanup in its own try/finally if __name__ == "__main__": diff --git a/retools/pyghidra_backend.py b/retools/pyghidra_backend.py index 85b171ca..db393744 100644 --- a/retools/pyghidra_backend.py +++ b/retools/pyghidra_backend.py @@ -133,18 +133,20 @@ def _route_daemon(game: str, cmd: dict): """Return the daemon response dict if a live daemon serves *game*, else None. A pure no-op (returns None) whenever no live daemon is reachable, so - callers fall through to the cold in-process path unchanged. + callers fall through to the cold in-process path unchanged. The probe + and send are wrapped in one broad except so a missing ghidra_client + module, a dead/absent daemon, a corrupted state file (e.g. a + wrong-typed "port" raising inside is_daemon_alive), or a transport + error during send_command all degrade to the cold path instead of + raising. """ if os.environ.get("RETOOLS_GHIDRA_COLD") == "1": return None try: import ghidra_client - except ImportError: - return None - project_dir = str(Path("patches") / game / "ghidra") - if not ghidra_client.is_daemon_alive(project_dir): - return None - try: + project_dir = str(Path("patches") / game / "ghidra") + if not ghidra_client.is_daemon_alive(project_dir): + return None return ghidra_client.send_command(project_dir, cmd, timeout=120) except Exception: return None diff --git a/tests/test_pyghidra_backend.py b/tests/test_pyghidra_backend.py index 53a4775e..c61b3ea5 100644 --- a/tests/test_pyghidra_backend.py +++ b/tests/test_pyghidra_backend.py @@ -434,3 +434,77 @@ def getSymbolTable(self): return FakeSymbolTable() assert "g_x" in applied["labels"] assert counts["functions"] == 1 assert counts["globals"] == 1 + + +# --------------------------------------------------------------------------- +# _route_daemon +# --------------------------------------------------------------------------- + +class TestRouteDaemon: + """Covers all four branches of _route_daemon without Ghidra or sockets.""" + + def test_daemon_present_returns_response(self, monkeypatch): + from pyghidra_backend import _route_daemon + import ghidra_client + + monkeypatch.setattr(ghidra_client, "is_daemon_alive", lambda project_dir: True) + monkeypatch.setattr( + ghidra_client, "send_command", + lambda project_dir, cmd, timeout=None: {"ok": True, "text": "ROUTED"}, + ) + result = _route_daemon("TestGame", {"cmd": "decompile", "binary": "b.exe", "va": 1}) + assert result == {"ok": True, "text": "ROUTED"} + + def test_cold_env_returns_none(self, monkeypatch): + from pyghidra_backend import _route_daemon + import ghidra_client + + monkeypatch.setenv("RETOOLS_GHIDRA_COLD", "1") + # Even a daemon that would answer "alive" must not be reached. + monkeypatch.setattr(ghidra_client, "is_daemon_alive", lambda project_dir: True) + assert _route_daemon("TestGame", {"cmd": "decompile"}) is None + + def test_dead_daemon_returns_none(self, monkeypatch): + from pyghidra_backend import _route_daemon + import ghidra_client + + monkeypatch.setattr(ghidra_client, "is_daemon_alive", lambda project_dir: False) + assert _route_daemon("TestGame", {"cmd": "decompile"}) is None + + def test_send_error_returns_none(self, monkeypatch): + from pyghidra_backend import _route_daemon + import ghidra_client + + monkeypatch.setattr(ghidra_client, "is_daemon_alive", lambda project_dir: True) + + def _raise(project_dir, cmd, timeout=None): + raise ConnectionError("boom") + + monkeypatch.setattr(ghidra_client, "send_command", _raise) + assert _route_daemon("TestGame", {"cmd": "decompile"}) is None + + def test_corrupted_state_degrades_to_none(self, monkeypatch): + """A malformed state file raising inside is_daemon_alive still degrades cleanly.""" + from pyghidra_backend import _route_daemon + import ghidra_client + + def _raise(project_dir): + raise TypeError("'>' not supported between instances of 'str' and 'int'") + + monkeypatch.setattr(ghidra_client, "is_daemon_alive", _raise) + assert _route_daemon("TestGame", {"cmd": "decompile"}) is None + + +class TestDecompileRouting: + """Proves decompile() actually uses _route_daemon's result when present.""" + + def test_decompile_uses_daemon_when_routed(self, tmp_path, monkeypatch): + import pyghidra_backend + + monkeypatch.setattr(pyghidra_backend, "is_analyzed", lambda project_dir, binary_name: True) + monkeypatch.setattr( + pyghidra_backend, "_route_daemon", + lambda game, cmd: {"ok": True, "text": "ROUTED"}, + ) + result = pyghidra_backend.decompile(str(tmp_path / "ghidra"), "test.exe", 0x401000) + assert result == "ROUTED" From 8a4bead5936d7e3b11a6140e9fdb88813a0f5447 Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 14:24:41 -0500 Subject: [PATCH 15/28] feat(context): resolve callees from index.db when present --- retools/context.py | 50 +++++++++++++++++++++++++++++++++---------- tests/test_context.py | 30 ++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 11 deletions(-) diff --git a/retools/context.py b/retools/context.py index 6781d92a..250905cc 100644 --- a/retools/context.py +++ b/retools/context.py @@ -26,6 +26,7 @@ from search import find_strings from sigdb import SignatureDB, extract_structural_sig from dataflow import propagate_cfg, Const, Unknown +from index import GameIndex # Pattern: fcn.XXXXXXXX or FUN_XXXXXXXX (r2ghidra or Ghidra naming) _FCN_RE = re.compile(r"(?:fcn\.|FUN_)([0-9a-fA-F]{8,16})") @@ -91,6 +92,27 @@ def postprocess(raw_output: str, kb_names: dict[int, str], # assemble # --------------------------------------------------------------------------- +def _callees_from_index(db_path: str, func_ea: int) -> list[tuple[int, str]] | None: + """Resolve (callee_addr, name) pairs for a function from index.db. + + Returns None when the index is absent so the caller falls back to scanning. + """ + if not Path(db_path).is_file(): + return None + conn = GameIndex.open_ro(db_path) + try: + rows = conn.execute( + "SELECT x.to_ea, COALESCE(f.name, '') FROM xrefs x " + "LEFT JOIN funcs f ON f.address = x.to_ea " + "WHERE x.from_func = ? AND x.is_code = 1 AND x.type = 'call' " + "ORDER BY x.to_ea", + (func_ea,), + ).fetchall() + finally: + conn.close() + return [(int(a), n) for a, n in rows] + + def _find_kb_path(project_dir: str, project_dir_for_kb: str | None = None) -> Path: """Locate kb.h: try explicit override, then patches//kb.h.""" if project_dir_for_kb: @@ -138,19 +160,25 @@ def assemble(b: Binary, va: int, project_dir: str, db_path: str | None = None, else: lines.append(f"[identity] unknown function at 0x{start:0{w}X}") - # -- Callees -- + # -- Callees (index fast-path, else scan) -- rets, calls, end_va = analyze(b, start, max_size=0x2000) lines.append("[callees]") - seen_targets: set[int | str] = set() - for _, target in calls: - if target in seen_targets: - continue - seen_targets.add(target) - if isinstance(target, int): - name = kb_names.get(target, "unknown") - lines.append(f" 0x{target:0{w}X}: {name}") - else: - lines.append(f" {target}: indirect call") + db_index = GameIndex.default_db_path(Path(project_dir).name) + indexed = _callees_from_index(db_index, start) + if indexed: + for target, name in indexed: + lines.append(f" 0x{target:0{w}X}: {name or kb_names.get(target, 'unknown')}") + else: + seen_targets: set[int | str] = set() + for _, target in calls: + if target in seen_targets: + continue + seen_targets.add(target) + if isinstance(target, int): + name = kb_names.get(target, "unknown") + lines.append(f" 0x{target:0{w}X}: {name}") + else: + lines.append(f" {target}: indirect call") # -- Struct fields (best-effort) -- try: diff --git a/tests/test_context.py b/tests/test_context.py index 358d491c..91b72285 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -320,3 +320,33 @@ def test_no_dataflow_flag(self, tmp_path): result = assemble(b, 0x401500, str(proj), no_dataflow=True) assert "[dataflow]" not in result + + +# --------------------------------------------------------------------------- +# _callees_from_index +# --------------------------------------------------------------------------- + +class TestIndexFastPath: + def test_callees_from_index(self, tmp_path): + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) + from index import GameIndex + from context import _callees_from_index + + db = str(tmp_path / "index.db") + gi = GameIndex(db) + gi.replace("funcs", [{"address": 0x2000, "name": "Target"}], source="ghidra") + gi.replace("xrefs", [{"from_ea": 0x1010, "to_ea": 0x2000, "type": "call", + "is_code": 1, "from_func": 0x1000}], source="ghidra") + gi.close() + + callees = _callees_from_index(db, 0x1000) + assert callees == [(0x2000, "Target")] + + def test_missing_index_returns_none(self, tmp_path): + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) + from context import _callees_from_index + assert _callees_from_index(str(tmp_path / "absent.db"), 0x1000) is None From 644edc30cbfb2ef3f7ad61ab114156f0e672cce9 Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 14:27:40 -0500 Subject: [PATCH 16/28] fix(context): dedup index callees and trust authoritative empty result --- retools/context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/retools/context.py b/retools/context.py index 250905cc..23f7eca4 100644 --- a/retools/context.py +++ b/retools/context.py @@ -105,7 +105,7 @@ def _callees_from_index(db_path: str, func_ea: int) -> list[tuple[int, str]] | N "SELECT x.to_ea, COALESCE(f.name, '') FROM xrefs x " "LEFT JOIN funcs f ON f.address = x.to_ea " "WHERE x.from_func = ? AND x.is_code = 1 AND x.type = 'call' " - "ORDER BY x.to_ea", + "GROUP BY x.to_ea ORDER BY x.to_ea", (func_ea,), ).fetchall() finally: @@ -165,7 +165,7 @@ def assemble(b: Binary, va: int, project_dir: str, db_path: str | None = None, lines.append("[callees]") db_index = GameIndex.default_db_path(Path(project_dir).name) indexed = _callees_from_index(db_index, start) - if indexed: + if indexed is not None: for target, name in indexed: lines.append(f" 0x{target:0{w}X}: {name or kb_names.get(target, 'unknown')}") else: From 8adcc9482fcd628a1baa3e96debf8501756a9709 Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 14:28:47 -0500 Subject: [PATCH 17/28] chore(verify_install): register kb/index/query/ghidra modules Add retools.kb, retools.index, retools.query, retools.ghidra_server, and retools.ghidra_client to the import check module list. All five modules import cleanly without Ghidra installed. Co-Authored-By: Claude Opus 4.8 --- verify_install.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/verify_install.py b/verify_install.py index 857bdcaf..f458fc58 100644 --- a/verify_install.py +++ b/verify_install.py @@ -175,6 +175,8 @@ def check_retools_import(): "retools.vtable", "retools.rtti", "retools.search", "retools.readmem", "retools.dumpinfo", "retools.throwmap", "retools.asi_patcher", + "retools.kb", "retools.index", "retools.query", + "retools.ghidra_server", "retools.ghidra_client", ] ok, bad = 0, [] for mod in modules: From 3b09dd5719aee9c13e034a78fe7265c65bafcbed Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 14:39:29 -0500 Subject: [PATCH 18/28] docs: document index/query/ghidra-server; reframe Ghidra-primary backends Documents the Task 1-10 additions (retools.index, retools.query, pyghidra_backend export/kb-apply, ghidra_server/ghidra_client daemon) across all four IDE trees, and reframes decompiler backend guidance from "two peer backends" to "Ghidra primary (indexed, daemon-backed, kb-applied); r2ghidra zero-setup fallback and second opinion." Co-Authored-By: Claude Opus 4.8 --- .claude/agents/static-analyzer.md | 54 ++++++++++++++-- .claude/references/tool-catalog.md | 33 +++++++--- .claude/rules/subagent-workflow.md | 12 ++-- .claude/rules/tool-dispatch.md | 7 ++- .cursor/agents/static-analyzer.md | 53 ++++++++++++++-- .cursor/rules/subagent-workflow.mdc | 12 ++-- .cursor/rules/tool-catalog.mdc | 63 ++++++++----------- .github/agents/static-analyzer.agent.md | 29 ++++++++- .github/copilot-instructions.md | 15 +++-- .../instructions/tool-catalog.instructions.md | 63 ++++++++----------- .kiro/agents/static-analyzer.md | 53 ++++++++++++++-- .kiro/steering/subagent-workflow.md | 12 ++-- .kiro/steering/tool-catalog.md | 63 ++++++++----------- README.md | 4 ++ 14 files changed, 316 insertions(+), 157 deletions(-) diff --git a/.claude/agents/static-analyzer.md b/.claude/agents/static-analyzer.md index 6c855591..f2430bff 100644 --- a/.claude/agents/static-analyzer.md +++ b/.claude/agents/static-analyzer.md @@ -35,30 +35,58 @@ python retools/pyghidra_backend.py status --project patches/ ``` If "Not analyzed", run `python retools/pyghidra_backend.py analyze --project patches/`. Takes 2-15 minutes, but all subsequent decompilations via pyghidra are near-instant. +**5. Index**: Check whether the project has an index.db and what's in it before scanning the binary yourself: +```bash +python -m retools.index status +``` +If `funcs`/`xrefs` show `source='bootstrap'` only (or the table is empty), and a Ghidra project exists, run `pyghidra_backend.py export` to seed authoritative facts — see "Query-first workflow" below. + ## Running Tools Run all tools from the repo root. Use `python -m retools.` or `python retools/.py` syntax: -### Decompilation (two backends) +### Decompilation -- Ghidra primary, r2ghidra fallback -**pyghidra (preferred when Ghidra project exists)** — better MSVC type propagation, library call resolution, larger function scope detection: +**pyghidra is the primary backend** once a Ghidra project exists — better MSVC type propagation, library call resolution, larger function scope detection, and its facts can be exported into `index.db` for instant SQL lookups later: ``` python retools/pyghidra_backend.py decompile binary.exe 0x401000 --project patches/proj ``` -**r2ghidra (fast fallback)** — better `__thiscall` on small functions, no JVM startup: +**r2ghidra is the zero-setup fallback and second opinion** — no Ghidra install required, better `__thiscall` recovery on small functions, no JVM startup, and useful to cross-check a pyghidra decompile that looks wrong: ``` python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h --backend pdg ``` -**Auto mode (tries pyghidra first, falls back to r2ghidra)**: +**Auto mode (tries pyghidra first, falls back to r2ghidra)** — routing unchanged: ``` python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h --project patches/proj ``` When told to use a specific backend, use it. Otherwise prefer auto mode with both `--types` and `--project`. +**Ghidra daemon**: if `python -m retools.ghidra_server ` is running (port 27043; livetools owns 27042), `decompile`/`export`/`kb-apply` route through it automatically and repeat calls become sub-second. Warm the server yourself before a batch of decompiles on the same project: `python -m retools.ghidra_server --idle 600` (background it). `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold in-process run when you need to bypass the daemon. + +### Query-first workflow + +Before re-scanning a binary with xrefs/datarefs/search/funcinfo, check whether `index.db` already has the answer — a SQL query against a local file is cheaper than re-disassembling: + +```bash +python -m retools.index status # per-table counts + schema_version +python -m retools.query --list-tables # confirm what's queryable +python -m retools.query --schema funcs # PRAGMA table_info before writing joins +python -m retools.query "SELECT * FROM callers WHERE callee_addr=0x401000" +python -m retools.query "SELECT * FROM grep WHERE name LIKE '%Ground%'" --json +``` + +Only fall back to `xrefs.py`/`datarefs.py`/`search.py`/`funcinfo.py` for facts `index.db` doesn't have yet (e.g. no `export` has run, or the question needs a live disassembly detail not captured by the schema). + +**Hard pushdown rule**: `decompile` and `export` require a specific function address (or, for `export`, an analyzed program) — never invoke them without one, or you decompile/scan the whole binary instead of the function you actually need. If you don't have an address yet, get one from `query`, `search`, or `xrefs` first. + +**Read-First mutation discipline**: `kb-apply` mutates the Ghidra project. Always decompile or `query` the target function first to confirm the current name/prototype, run `kb-apply`, then **re-decompile the same function** to verify the change landed before reporting it as done. `kb-apply` is idempotent — re-running it should produce stable counts and no errors, so if a second run changes anything, treat that as a bug, not expected behavior. + +**Cost guard**: run `export` once per analysis pass (after `kb-apply`, so exported names reflect it), not once per query — repeated `export` calls re-walk the whole program for no benefit once `index.db` is current. + ### Other tools ``` python -m retools.search binary.exe strings -f "error" --xrefs @@ -74,6 +102,11 @@ python -m retools.sigdb fingerprint binary.exe python -m retools.context assemble binary.exe 0x401000 --project MyGame python retools/pyghidra_backend.py analyze binary.exe --project patches/MyGame python retools/pyghidra_backend.py status binary.exe --project patches/MyGame +python retools/pyghidra_backend.py export binary.exe --project patches/MyGame +python retools/pyghidra_backend.py kb-apply binary.exe --project patches/MyGame --kb patches/MyGame/kb.h +python -m retools.index status MyGame +python -m retools.query MyGame "SELECT * FROM funcs WHERE name LIKE '%Update%'" +python -m retools.ghidra_server MyGame --idle 600 ``` If `retools/data/signatures.db` is missing, run `python -m retools.sigdb pull` to download it. @@ -140,3 +173,16 @@ Also update `patches//kb.h` with any new function signatures, structs, In your return message, state the file path you wrote to and give a brief summary. The main agent will read the file for full details. Update your agent memory with significant architectural discoveries, identified subsystems, and class hierarchies that will be useful in future sessions. + +## Routing to Adjacent Skills/Docs + +This agent owns offline static analysis. Hand off to the right reference/skill instead of improvising: + +| Need | Go to | +|------|-------| +| Full tool syntax, flags, caveats for any retools/DX-script/dumpinfo tool | `.claude/references/tool-catalog.md` | +| Whether a task should run inline vs be delegated | `.claude/rules/tool-dispatch.md` | +| Bootstrap ordering, parallel dual-backend runs, delegation table | `.claude/rules/subagent-workflow.md` | +| Attaching to a live process, breakpoints, tracing, memory patching | `/dynamic-analysis` skill (main agent only — this agent must not use livetools) | +| Porting a DX9 game to FFP for RTX Remix (renderer.cpp, ffp_state, vertex decls, skinning) | `dx9-ffp-port` skill | +| D3D9-specific static questions (VS/PS constants, render states, vertex formats) | DX analysis scripts (`rtx_remix_tools/dx/scripts/`) before general retools | diff --git a/.claude/references/tool-catalog.md b/.claude/references/tool-catalog.md index ca27fde5..0695b829 100644 --- a/.claude/references/tool-catalog.md +++ b/.claude/references/tool-catalog.md @@ -27,6 +27,8 @@ These are fast (<5s) and allowed inline: - "Trace where this value comes from" → `python -m retools.dataflow $B $VA --slice TARGET_VA:REG` - "Build an ASI patch DLL" → `python -m retools.asi_patcher build spec.json` - "Does a Ghidra project exist for this binary?" → `python retools/pyghidra_backend.py status $B --project $P` +- "What's in this game's index?" → `python -m retools.index status [--db PATH]` +- "Query facts already indexed (funcs, names, xrefs, strings, imports, callers/callees)" → `python -m retools.query "SQL" [--db PATH] [--json] [--list-tables] [--schema TABLE]` ### Delegate to `static-analyzer` subagent @@ -35,13 +37,13 @@ Everything else. Tell the subagent WHAT you need, not HOW to run it — it has t **D3D9-specific questions?** Check the DX analysis scripts section below first — they're faster and more targeted than general retools for D3D API usage, device calls, shader constants, and vertex formats. - "What does this function do?" → decompile + callgraph + xrefs + dataflow --constants -- "Who calls this function?" → xrefs or callgraph --up -- "What does this function call?" → callgraph --down (add --indirect for vtable calls) +- "Who calls this function?" → xrefs or callgraph --up (prefer `retools.query "SELECT * FROM callers WHERE callee_addr=..."` when index.db exists) +- "What does this function call?" → callgraph --down (add --indirect for vtable calls) (prefer `retools.query "SELECT * FROM callees WHERE caller=..."` when index.db exists) - "Who calls this virtual method?" → xrefs --indirect + filter by vtable slot offset - "What constant reaches this call?" → dataflow --constants or --slice VA:REG - "Resolve a switch/jump table" → cfg (auto-resolves MSVC switch patterns) -- "Find a string and who uses it" → string search with xrefs -- "Where is this global read/written?" → datarefs +- "Find a string and who uses it" → string search with xrefs (prefer `retools.query "SELECT * FROM grep WHERE name LIKE '%...%'"` when index.db exists) +- "Where is this global read/written?" → datarefs (prefer `retools.query` against `names`/`xrefs` when index.db exists) - "Where is struct field +0x54 used?" → structrefs - "What does this struct look like?" → structrefs --aggregate - "What C++ class is this vtable?" → RTTI resolution @@ -51,6 +53,8 @@ Everything else. Tell the subagent WHAT you need, not HOW to run it — it has t - "Map all throw sites to error strings" → throwmap list - "First time analyzing a binary?" → bootstrap (2-5 min) + pyghidra analyze (5-15 min) in parallel - "Bulk signature scan" → sigdb scan (1-3 min) +- "Seed/refresh the index from a Ghidra project" → `pyghidra_backend.py export` (funcs/names/xrefs/blocks, source='ghidra', authoritative over provisional bootstrap rows) +- "Push kb.h names/prototypes into the Ghidra project" → `pyghidra_backend.py kb-apply` (idempotent) - Any combination of the above ### Live tools (main agent, requires attached process) @@ -102,19 +106,24 @@ These are fast first-pass scanners — they surface candidate addresses. Follow | `pyghidra_backend.py analyze $B --project $P` | **Full Ghidra analysis** -- one-time, saves reusable project | `pyghidra_backend.py analyze game.exe --project patches/MyGame` | | `pyghidra_backend.py decompile $B $VA --project $P` | Decompile via saved Ghidra project | `pyghidra_backend.py decompile game.exe 0x401000 --project patches/MyGame` | | `pyghidra_backend.py status $B --project $P` | Check if Ghidra project exists | `pyghidra_backend.py status game.exe --project patches/MyGame` | -| `funcinfo.py $B $VA` | Find function start/end, rets, calling convention, callees | `funcinfo.py binary.exe 0x401000` | +| `pyghidra_backend.py export $B --project $P [--db]` | Seed funcs/names/xrefs/blocks from an analyzed Ghidra program into index.db (`source='ghidra'`, overwrites provisional bootstrap rows at the same address) | `pyghidra_backend.py export game.exe --project patches/MyGame` | +| `pyghidra_backend.py kb-apply $B --project $P --kb $KB` | Push kb.h names/prototypes/globals/typedefs into the Ghidra project. Idempotent — safe to re-run | `pyghidra_backend.py kb-apply game.exe --project patches/MyGame --kb patches/MyGame/kb.h` | +| `index.py status $GAME [--db]` | Per-table row counts + schema_version for `patches//index.db` | `python -m retools.index status MyGame` | +| `query.py $GAME "SQL" [--db] [--json] [--list-tables] [--schema TABLE]` | Read-only SQL over index.db. Views: `callers`, `callees`, `grep`. Addresses render as hex via `printf('0x%x', address)`. Connection is opened read-only — cannot mutate | `python -m retools.query MyGame "SELECT * FROM grep WHERE name LIKE '%Ground%'"` | +| `ghidra_server.py $GAME [--idle SECS]` | Run a per-project Ghidra daemon (port 27043; livetools owns 27042) holding one warm program so repeat `decompile`/`export`/`kb-apply` calls are sub-second instead of paying JVM+analysis startup each time | `python -m retools.ghidra_server MyGame --idle 600` | +| `funcinfo.py $B $VA` | Find function start/end, rets, calling convention, callees. Prefer `retools.query` on `funcs`/`blocks` when index.db exists | `funcinfo.py binary.exe 0x401000` | | `cfg.py $B $VA` | Control flow graph (basic blocks + edges, text or mermaid). Resolves MSVC switch/jump tables automatically. `--switch-details` shows table info | `cfg.py binary.exe 0x401000 --format mermaid` | | `callgraph.py $B $VA` | Caller/callee tree (multi-level, --up/--down N). `--indirect` adds vtable/fptr calls to --down trees | `callgraph.py binary.exe 0x401000 --down 2 --indirect` | -| `xrefs.py $B $VA` | Find all calls/jumps TO an address. `--indirect` also scans for `call [reg+offset]`, `call [reg]`, `call [addr]` | `xrefs.py binary.exe 0x401000 --indirect` | +| `xrefs.py $B $VA` | Find all calls/jumps TO an address. `--indirect` also scans for `call [reg+offset]`, `call [reg]`, `call [addr]`. Prefer `retools.query "SELECT * FROM callers WHERE callee_addr=..."` when index.db exists | `xrefs.py binary.exe 0x401000 --indirect` | | `dataflow.py $B $VA` | Forward constant propagation (`--constants`) or backward register slice (`--slice VA:REG`) within a function | `dataflow.py binary.exe 0x401000 --constants` | -| `datarefs.py $B $VA` | Find instructions that reference a global address (mem deref + `--imm` for push/mov constants) | `datarefs.py binary.exe 0x7A0000 --imm` | +| `datarefs.py $B $VA` | Find instructions that reference a global address (mem deref + `--imm` for push/mov constants). Prefer `retools.query` against `xrefs`/`names` when index.db exists | `datarefs.py binary.exe 0x7A0000 --imm` | | `structrefs.py $B $OFF` | Find all `[reg+offset]` accesses (struct field usage) | `structrefs.py binary.exe 0x54 --base esi` | | `structrefs.py $B --aggregate` | Reconstruct C struct from all field accesses in a function | `structrefs.py binary.exe --aggregate --fn 0x401000 --base esi` | | `vtable.py $B dump $VA` | Dump C++ vtable slots with instruction preview | `vtable.py binary.exe dump 0x6A0000` | | `vtable.py $B calls $OFF` | Find all indirect `call [reg+offset]` (vtable call sites) | `vtable.py binary.exe calls 0xB0` | | `rtti.py $B vtable $VA` | Resolve C++ class name + inheritance chain from vtable (MSVC RTTI) | `rtti.py binary.dll vtable 0x6A0000` | | `rtti.py $B throwinfo $RVA` | Resolve exception type from `_ThrowInfo` (MSVC RTTI) | `rtti.py binary.dll throwinfo 0x5040CF8` | -| `search.py $B strings` | Extract strings with keyword filter | `search.py binary.exe strings -f render,draw` | +| `search.py $B strings` | Extract strings with keyword filter. Prefer `retools.query "SELECT * FROM grep WHERE name LIKE '%...%'"` when index.db exists | `search.py binary.exe strings -f render,draw` | | `search.py $B strings --xrefs` | Find strings AND code locations that reference them | `search.py binary.exe strings -f "error" --xrefs` | | `search.py $B pattern` | Find exact byte pattern | `search.py binary.exe pattern "D9 56 54 D8 1D"` | | `search.py $B imports` | List PE imports, filter by DLL | `search.py binary.exe imports -d kernel32` | @@ -289,13 +298,17 @@ Minidumps vary in how much data they capture depending on `MiniDumpWriteDump` fl These tools find references via absolute memory operands, immediate values (with `--imm` flag), and RIP-relative addressing. If you suspect a reference exists but the tool doesn't find it, the address might be computed at runtime. Try `search.py pattern` with the address bytes directly, or use `livetools memwatch`. -### `pyghidra_backend.py` -- requires Ghidra installation +### `pyghidra_backend.py` -- Ghidra primary, r2ghidra fallback + +Ghidra (via `pyghidra_backend.py`, indexed into `index.db`, daemon-backed, kb-applied) is the **primary** decompilation backend once a project exists — it gives better type propagation, library call resolution, and lets `retools.query` answer structural questions without re-scanning the binary. r2ghidra (`decompiler.py --backend pdg`) is the **zero-setup fallback and second opinion**: no Ghidra install required, faster on small functions, and useful to cross-check a pyghidra result that looks wrong. `decompiler.py --backend auto` tries pyghidra first and falls back to r2ghidra automatically — this routing is unchanged. Requires Ghidra 11.x+ installed and `GHIDRA_INSTALL_DIR` environment variable set. **Optional** -- the toolkit works without it (r2ghidra remains the fallback). **Disk usage**: Ghidra projects are ~10-20x the binary size. A 30MB game exe produces a ~300-600MB `.rep/` directory under `patches//ghidra/`. This directory is already covered by `.gitignore` (the `patches/` exclusion). -**First-time analysis** takes 5-15 minutes depending on binary size. Subsequent decompilation from the saved project is instant (<1s plus ~3s JVM startup per process). +**First-time analysis** takes 5-15 minutes depending on binary size. Subsequent decompilation from the saved project is near-instant (<1s plus ~3s JVM startup per cold process) -- or truly sub-second when a `ghidra_server.py` daemon is warm for that project. `export` and `kb-apply` route through the same live daemon when one is running; `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold in-process run for either. + +**Index-first**: after `export`, prefer `retools.query` over datarefs/xrefs/search/funcinfo for anything already captured in `index.db` (funcs, names, xrefs, strings, imports, blocks) -- it's a local SQL query instead of a fresh binary scan. Fall back to the scanners only for facts index.db doesn't have yet. ### `livetools` -- static vs runtime addresses diff --git a/.claude/rules/subagent-workflow.md b/.claude/rules/subagent-workflow.md index 0900bf61..1b3de392 100644 --- a/.claude/rules/subagent-workflow.md +++ b/.claude/rules/subagent-workflow.md @@ -29,10 +29,12 @@ CLAUDE.md lists allowlisted fast commands (run directly) and the general delegat |------|-------|-------| | Web research (docs, API refs, specs) | `web-researcher` subagent | | | dx9tracer offline analysis | `static-analyzer` subagent | | -| Subsequent Ghidra decompile | `static-analyzer` subagent | Fast: JVM ~3s + decompile <1s | +| Subsequent Ghidra decompile | `static-analyzer` subagent | Fast: JVM ~3s + decompile <1s, sub-second with a warm `ghidra_server.py` daemon | | sigdb scan / build | `static-analyzer` subagent | scan 1-3 min, build 1-5 min | | Dataflow: constants + backward slice (`dataflow.py`) | Main agent | fast (<5s) | -| KB updates from findings | `static-analyzer` writes kb.h | main agent may refine | +| KB updates from findings | `static-analyzer` writes kb.h, then `kb-apply` to push into Ghidra | main agent may refine | +| `index status` / `query` (SQL over index.db) | Main agent | fast (<5s); prefer over xrefs/datarefs/search/funcinfo when index.db already has the answer | +| `pyghidra_backend.py export` (seed index.db from Ghidra) | `static-analyzer` subagent | run once per analysis pass, after `kb-apply` | ## Subagent Output @@ -49,12 +51,14 @@ Multiple `static-analyzer` instances can run in parallel for independent questio ## Dual-Backend Deep Analysis -For complex exploratory tasks (finding subsystems, mapping pipelines), spawn **two parallel agents**: +Ghidra (indexed, daemon-backed, kb-applied) is the primary backend once a project exists — prefer it plus `retools.query` over spawning two agents. Reserve the dual-agent pattern below for two specific cases: **no Ghidra project exists yet** (so there's no `index.db` or warm daemon to lean on), or **pyghidra output on a specific function looks wrong** and you need an independent r2ghidra read to cross-check it. + +When one of those applies, spawn **two parallel agents**: 1. **r2ghidra**: `--backend pdg --types kb.h` → writes `findings_r2.md` 2. **pyghidra**: `pyghidra_backend.py decompile` → writes `findings.md` -r2ghidra: better `__thiscall` recovery, low-level D3D. pyghidra: better library call resolution, type propagation. Merge both for complete picture. Not needed for single-function decompilation — use `--backend auto`. +r2ghidra: better `__thiscall` recovery, low-level D3D, no JVM/project dependency. pyghidra: better library call resolution, type propagation, and its output is exportable into `index.db` for future queries. Merge both for complete picture. Not needed for single-function decompilation once a Ghidra project exists — use `--backend auto` (Ghidra primary, r2ghidra fallback). ## Main Agent During Analysis diff --git a/.claude/rules/tool-dispatch.md b/.claude/rules/tool-dispatch.md index 72f8521e..17e0ce66 100644 --- a/.claude/rules/tool-dispatch.md +++ b/.claude/rules/tool-dispatch.md @@ -19,15 +19,20 @@ Run all tools from repo root via `python -m `. **ALWAYS pass `--types pa - `python -m retools.dataflow $B $VA --slice TARGET_VA:REG` — backward register slice - `python -m retools.asi_patcher build spec.json` — build ASI patch DLL - `python retools/pyghidra_backend.py status $B --project $P` — Ghidra project existence check +- `python -m retools.index status [--db PATH]` — per-table row counts + schema_version for the game's index.db +- `python -m retools.query "SQL" [--db PATH] [--json] [--list-tables] [--schema TABLE]` — read-only SQL over index.db (`callers`/`callees`/`grep` views); prefer this over a fresh xrefs/datarefs/search scan whenever index.db already has the data ## Delegate to `static-analyzer` Everything else in `retools`. Tell it WHAT you need, not HOW. D3D9-specific questions — try DX scripts first (faster). -- Decompile / callgraph / xrefs / string search / datarefs / structrefs / RTTI / throwmap / dumpinfo +- Decompile / callgraph / xrefs / string search / datarefs / structrefs / RTTI / throwmap / dumpinfo — check `index status` / `query` first; only fall back to these scanners when index.db lacks the answer - Bootstrap new binary (2-5 min) / pyghidra analyze (5-15 min) / bulk sigdb scan (1-3 min) +- `pyghidra_backend.py export` (seed index.db from a Ghidra project) / `kb-apply` (push kb.h into the Ghidra project) - dx9tracer offline analysis (summary, render-passes, shader-map, etc.) +**Ghidra daemon**: `python -m retools.ghidra_server [--idle 600]` keeps one warm Ghidra program per project on port 27043 (livetools owns 27042). `decompile`/`export`/`kb-apply` route through a live daemon automatically when one is running for that project — repeat decompiles become sub-second instead of paying JVM startup each time. `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold in-process run. + ## Live tools (main agent, attached process) Full syntax and recipes: the `/dynamic-analysis` skill (canonical livetools reference). diff --git a/.cursor/agents/static-analyzer.md b/.cursor/agents/static-analyzer.md index 8f2acbc3..884e91e2 100644 --- a/.cursor/agents/static-analyzer.md +++ b/.cursor/agents/static-analyzer.md @@ -34,30 +34,58 @@ python retools/pyghidra_backend.py status --project patches/ ``` If "Not analyzed", run `python retools/pyghidra_backend.py analyze --project patches/`. Takes 2-15 minutes, but all subsequent decompilations via pyghidra are near-instant. +**5. Index**: Check whether the project has an index.db and what's in it before scanning the binary yourself: +```bash +python -m retools.index status +``` +If `funcs`/`xrefs` show `source='bootstrap'` only (or the table is empty), and a Ghidra project exists, run `pyghidra_backend.py export` to seed authoritative facts — see "Query-first workflow" below. + ## Running Tools Run all tools from the repo root. Use `python -m retools.` or `python retools/.py` syntax: -### Decompilation (two backends) +### Decompilation -- Ghidra primary, r2ghidra fallback -**pyghidra (preferred when Ghidra project exists)** — better MSVC type propagation, library call resolution, larger function scope detection: +**pyghidra is the primary backend** once a Ghidra project exists — better MSVC type propagation, library call resolution, larger function scope detection, and its facts can be exported into `index.db` for instant SQL lookups later: ``` python retools/pyghidra_backend.py decompile binary.exe 0x401000 --project patches/proj ``` -**r2ghidra (fast fallback)** — better `__thiscall` on small functions, no JVM startup: +**r2ghidra is the zero-setup fallback and second opinion** — no Ghidra install required, better `__thiscall` recovery on small functions, no JVM startup, and useful to cross-check a pyghidra decompile that looks wrong: ``` python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h --backend pdg ``` -**Auto mode (tries pyghidra first, falls back to r2ghidra)**: +**Auto mode (tries pyghidra first, falls back to r2ghidra)** — routing unchanged: ``` python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h --project patches/proj ``` When told to use a specific backend, use it. Otherwise prefer auto mode with both `--types` and `--project`. +**Ghidra daemon**: if `python -m retools.ghidra_server ` is running (port 27043; livetools owns 27042), `decompile`/`export`/`kb-apply` route through it automatically and repeat calls become sub-second. Warm the server yourself before a batch of decompiles on the same project: `python -m retools.ghidra_server --idle 600` (background it). `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold in-process run when you need to bypass the daemon. + +### Query-first workflow + +Before re-scanning a binary with xrefs/datarefs/search/funcinfo, check whether `index.db` already has the answer — a SQL query against a local file is cheaper than re-disassembling: + +```bash +python -m retools.index status # per-table counts + schema_version +python -m retools.query --list-tables # confirm what's queryable +python -m retools.query --schema funcs # PRAGMA table_info before writing joins +python -m retools.query "SELECT * FROM callers WHERE callee_addr=0x401000" +python -m retools.query "SELECT * FROM grep WHERE name LIKE '%Ground%'" --json +``` + +Only fall back to `xrefs.py`/`datarefs.py`/`search.py`/`funcinfo.py` for facts `index.db` doesn't have yet (e.g. no `export` has run, or the question needs a live disassembly detail not captured by the schema). + +**Hard pushdown rule**: `decompile` and `export` require a specific function address (or, for `export`, an analyzed program) — never invoke them without one, or you decompile/scan the whole binary instead of the function you actually need. If you don't have an address yet, get one from `query`, `search`, or `xrefs` first. + +**Read-First mutation discipline**: `kb-apply` mutates the Ghidra project. Always decompile or `query` the target function first to confirm the current name/prototype, run `kb-apply`, then **re-decompile the same function** to verify the change landed before reporting it as done. `kb-apply` is idempotent — re-running it should produce stable counts and no errors, so if a second run changes anything, treat that as a bug, not expected behavior. + +**Cost guard**: run `export` once per analysis pass (after `kb-apply`, so exported names reflect it), not once per query — repeated `export` calls re-walk the whole program for no benefit once `index.db` is current. + ### Other tools ``` python -m retools.search binary.exe strings -f "error" --xrefs @@ -73,6 +101,11 @@ python -m retools.sigdb fingerprint binary.exe python -m retools.context assemble binary.exe 0x401000 --project MyGame python retools/pyghidra_backend.py analyze binary.exe --project patches/MyGame python retools/pyghidra_backend.py status binary.exe --project patches/MyGame +python retools/pyghidra_backend.py export binary.exe --project patches/MyGame +python retools/pyghidra_backend.py kb-apply binary.exe --project patches/MyGame --kb patches/MyGame/kb.h +python -m retools.index status MyGame +python -m retools.query MyGame "SELECT * FROM funcs WHERE name LIKE '%Update%'" +python -m retools.ghidra_server MyGame --idle 600 ``` If `retools/data/signatures.db` is missing, run `python -m retools.sigdb pull` to download it. @@ -139,3 +172,15 @@ Also update `patches//kb.h` with any new function signatures, structs, In your return message, state the file path you wrote to and give a brief summary. The main agent will read the file for full details. Update your agent memory with significant architectural discoveries, identified subsystems, and class hierarchies that will be useful in future sessions. + +## Routing to Adjacent Skills/Docs + +This agent owns offline static analysis. Hand off to the right reference/skill instead of improvising: + +| Need | Go to | +|------|-------| +| Full tool syntax, flags, caveats for any retools/DX-script/dumpinfo tool, and run-directly vs delegate guidance | `.cursor/rules/tool-catalog.mdc` | +| Bootstrap ordering, parallel dual-backend runs, delegation table | `.cursor/rules/subagent-workflow.mdc` | +| Attaching to a live process, breakpoints, tracing, memory patching | `/dynamic-analysis` skill (main agent only — this agent must not use livetools) | +| Porting a DX9 game to FFP for RTX Remix (renderer.cpp, ffp_state, vertex decls, skinning) | `dx9-ffp-port` skill | +| D3D9-specific static questions (VS/PS constants, render states, vertex formats) | DX analysis scripts (`rtx_remix_tools/dx/scripts/`) before general retools | diff --git a/.cursor/rules/subagent-workflow.mdc b/.cursor/rules/subagent-workflow.mdc index 1ae2e646..cb37d4bf 100644 --- a/.cursor/rules/subagent-workflow.mdc +++ b/.cursor/rules/subagent-workflow.mdc @@ -44,7 +44,9 @@ When analyzing a binary for the first time (no existing or sparsely populated `p | Decompiler postprocess (`context.py postprocess`) | Main agent -- instant | | Dataflow: constants + backward slice (`dataflow.py`) | Main agent -- fast (<5s) | | File editing, patch specs, builds | Main agent — directly | -| KB updates from subagent findings | `static-analyzer` writes to `kb.h`; main agent may refine | +| KB updates from subagent findings | `static-analyzer` writes to `kb.h`, then `kb-apply` pushes it into Ghidra; main agent may refine | +| `index status` / `query` (SQL over `index.db`) | Main agent -- fast (<5s); prefer over xrefs/datarefs/search/funcinfo when index.db already has the answer | +| `pyghidra_backend.py export` (seed `index.db` from Ghidra) | `static-analyzer` subagent -- run once per analysis pass, after `kb-apply` | ## Subagent Output Files @@ -62,14 +64,16 @@ Multiple `static-analyzer` instances can run in parallel for independent questio ## Dual-Backend Deep Analysis -For deep analysis tasks (finding subsystems, mapping call chains, understanding large code areas), spawn **two parallel static-analyzer agents using different decompiler backends**: +Ghidra (indexed into `index.db`, daemon-backed via `ghidra_server.py`, kb-applied) is the **primary** backend once a project exists — prefer it plus `retools.query` over spawning two agents for most exploratory work. Reserve the dual-agent pattern below for two specific cases: **no Ghidra project exists yet** for this binary, or **pyghidra output on a specific function looks wrong** and you need an independent r2ghidra read to cross-check it. + +When one of those applies, spawn **two parallel static-analyzer agents using different decompiler backends**: 1. **r2ghidra agent** — uses `--backend pdg` (with `--types kb.h`), writes to `patches//findings_r2.md` 2. **pyghidra agent** — uses `pyghidra_backend.py decompile` (requires Ghidra project), writes to `patches//findings.md` -**Why both:** Each backend has different strengths. r2ghidra is better at `__thiscall` recovery on small functions and low-level D3D details. pyghidra resolves more library calls, finds larger function scopes, and propagates types better. Neither finds everything alone — merging both gives the most complete picture. +**Why both:** Each backend has different strengths. r2ghidra is better at `__thiscall` recovery on small functions, low-level D3D details, and needs no JVM/project setup. pyghidra resolves more library calls, finds larger function scopes, propagates types better, and its output is exportable into `index.db` for future queries. Neither finds everything alone — merging both gives the most complete picture. -**When to use dual-backend:** Complex exploratory tasks ("find the culling system", "map the rendering pipeline", "understand the network protocol"). Not needed for single-function decompilation — use `--backend auto` for that. +**When to use dual-backend:** Only when no Ghidra project exists yet, or when pyghidra output on a specific function looks wrong. Not needed for single-function decompilation once a Ghidra project exists — use `--backend auto` (Ghidra primary, r2ghidra fallback). **Synthesis:** When both agents return, the main agent reads both findings files and merges them into a unified analysis. Conflicting information is resolved by checking which backend's output is more complete for that specific function. diff --git a/.cursor/rules/tool-catalog.mdc b/.cursor/rules/tool-catalog.mdc index 650f0828..aa7b5cd9 100644 --- a/.cursor/rules/tool-catalog.mdc +++ b/.cursor/rules/tool-catalog.mdc @@ -28,6 +28,8 @@ These are fast (<5s) and allowed inline: - "Trace where this value comes from" → `python -m retools.dataflow $B $VA --slice TARGET_VA:REG` - "Build an ASI patch DLL" → `python -m retools.asi_patcher build spec.json` - "Does a Ghidra project exist for this binary?" → `python retools/pyghidra_backend.py status $B --project $P` +- "What's in this game's index?" → `python -m retools.index status [--db PATH]` +- "Query facts already indexed (funcs, names, xrefs, strings, imports, callers/callees)" → `python -m retools.query "SQL" [--db PATH] [--json] [--list-tables] [--schema TABLE]` ### Delegate to `static-analyzer` subagent @@ -36,13 +38,13 @@ Everything else. Tell the subagent WHAT you need, not HOW to run it — it has t **D3D9-specific questions?** Check the DX analysis scripts section below first — they're faster and more targeted than general retools for D3D API usage, device calls, shader constants, and vertex formats. - "What does this function do?" → decompile + callgraph + xrefs + dataflow --constants -- "Who calls this function?" → xrefs or callgraph --up -- "What does this function call?" → callgraph --down (add --indirect for vtable calls) +- "Who calls this function?" → xrefs or callgraph --up (prefer `retools.query "SELECT * FROM callers WHERE callee_addr=..."` when index.db exists) +- "What does this function call?" → callgraph --down (add --indirect for vtable calls) (prefer `retools.query "SELECT * FROM callees WHERE caller=..."` when index.db exists) - "Who calls this virtual method?" → xrefs --indirect + filter by vtable slot offset - "What constant reaches this call?" → dataflow --constants or --slice VA:REG - "Resolve a switch/jump table" → cfg (auto-resolves MSVC switch patterns) -- "Find a string and who uses it" → string search with xrefs -- "Where is this global read/written?" → datarefs +- "Find a string and who uses it" → string search with xrefs (prefer `retools.query "SELECT * FROM grep WHERE name LIKE '%...%'"` when index.db exists) +- "Where is this global read/written?" → datarefs (prefer `retools.query` against `names`/`xrefs` when index.db exists) - "Where is struct field +0x54 used?" → structrefs - "What does this struct look like?" → structrefs --aggregate - "What C++ class is this vtable?" → RTTI resolution @@ -52,6 +54,8 @@ Everything else. Tell the subagent WHAT you need, not HOW to run it — it has t - "Map all throw sites to error strings" → throwmap list - "First time analyzing a binary?" → bootstrap (2-5 min) + pyghidra analyze (5-15 min) in parallel - "Bulk signature scan" → sigdb scan (1-3 min) +- "Seed/refresh the index from a Ghidra project" → `pyghidra_backend.py export` (funcs/names/xrefs/blocks, source='ghidra', authoritative over provisional bootstrap rows) +- "Push kb.h names/prototypes into the Ghidra project" → `pyghidra_backend.py kb-apply` (idempotent) - Any combination of the above ### Live tools (main agent, requires attached process) @@ -103,19 +107,24 @@ These are fast first-pass scanners — they surface candidate addresses. Follow | `pyghidra_backend.py analyze $B --project $P` | **Full Ghidra analysis** -- one-time, saves reusable project | `pyghidra_backend.py analyze game.exe --project patches/MyGame` | | `pyghidra_backend.py decompile $B $VA --project $P` | Decompile via saved Ghidra project | `pyghidra_backend.py decompile game.exe 0x401000 --project patches/MyGame` | | `pyghidra_backend.py status $B --project $P` | Check if Ghidra project exists | `pyghidra_backend.py status game.exe --project patches/MyGame` | -| `funcinfo.py $B $VA` | Find function start/end, rets, calling convention, callees | `funcinfo.py binary.exe 0x401000` | +| `pyghidra_backend.py export $B --project $P [--db]` | Seed funcs/names/xrefs/blocks from an analyzed Ghidra program into index.db (`source='ghidra'`, overwrites provisional bootstrap rows at the same address) | `pyghidra_backend.py export game.exe --project patches/MyGame` | +| `pyghidra_backend.py kb-apply $B --project $P --kb $KB` | Push kb.h names/prototypes/globals/typedefs into the Ghidra project. Idempotent — safe to re-run | `pyghidra_backend.py kb-apply game.exe --project patches/MyGame --kb patches/MyGame/kb.h` | +| `index.py status $GAME [--db]` | Per-table row counts + schema_version for `patches//index.db` | `python -m retools.index status MyGame` | +| `query.py $GAME "SQL" [--db] [--json] [--list-tables] [--schema TABLE]` | Read-only SQL over index.db. Views: `callers`, `callees`, `grep`. Addresses render as hex via `printf('0x%x', address)`. Connection is opened read-only — cannot mutate | `python -m retools.query MyGame "SELECT * FROM grep WHERE name LIKE '%Ground%'"` | +| `ghidra_server.py $GAME [--idle SECS]` | Run a per-project Ghidra daemon (port 27043; livetools owns 27042) holding one warm program so repeat `decompile`/`export`/`kb-apply` calls are sub-second instead of paying JVM+analysis startup each time | `python -m retools.ghidra_server MyGame --idle 600` | +| `funcinfo.py $B $VA` | Find function start/end, rets, calling convention, callees. Prefer `retools.query` on `funcs`/`blocks` when index.db exists | `funcinfo.py binary.exe 0x401000` | | `cfg.py $B $VA` | Control flow graph (basic blocks + edges, text or mermaid). Resolves MSVC switch/jump tables automatically. `--switch-details` shows table info | `cfg.py binary.exe 0x401000 --format mermaid` | | `callgraph.py $B $VA` | Caller/callee tree (multi-level, --up/--down N). `--indirect` adds vtable/fptr calls to --down trees | `callgraph.py binary.exe 0x401000 --down 2 --indirect` | -| `xrefs.py $B $VA` | Find all calls/jumps TO an address. `--indirect` also scans for `call [reg+offset]`, `call [reg]`, `call [addr]` | `xrefs.py binary.exe 0x401000 --indirect` | +| `xrefs.py $B $VA` | Find all calls/jumps TO an address. `--indirect` also scans for `call [reg+offset]`, `call [reg]`, `call [addr]`. Prefer `retools.query "SELECT * FROM callers WHERE callee_addr=..."` when index.db exists | `xrefs.py binary.exe 0x401000 --indirect` | | `dataflow.py $B $VA` | Forward constant propagation (`--constants`) or backward register slice (`--slice VA:REG`) within a function | `dataflow.py binary.exe 0x401000 --constants` | -| `datarefs.py $B $VA` | Find instructions that reference a global address (mem deref + `--imm` for push/mov constants) | `datarefs.py binary.exe 0x7A0000 --imm` | +| `datarefs.py $B $VA` | Find instructions that reference a global address (mem deref + `--imm` for push/mov constants). Prefer `retools.query` against `xrefs`/`names` when index.db exists | `datarefs.py binary.exe 0x7A0000 --imm` | | `structrefs.py $B $OFF` | Find all `[reg+offset]` accesses (struct field usage) | `structrefs.py binary.exe 0x54 --base esi` | | `structrefs.py $B --aggregate` | Reconstruct C struct from all field accesses in a function | `structrefs.py binary.exe --aggregate --fn 0x401000 --base esi` | | `vtable.py $B dump $VA` | Dump C++ vtable slots with instruction preview | `vtable.py binary.exe dump 0x6A0000` | | `vtable.py $B calls $OFF` | Find all indirect `call [reg+offset]` (vtable call sites) | `vtable.py binary.exe calls 0xB0` | | `rtti.py $B vtable $VA` | Resolve C++ class name + inheritance chain from vtable (MSVC RTTI) | `rtti.py binary.dll vtable 0x6A0000` | | `rtti.py $B throwinfo $RVA` | Resolve exception type from `_ThrowInfo` (MSVC RTTI) | `rtti.py binary.dll throwinfo 0x5040CF8` | -| `search.py $B strings` | Extract strings with keyword filter | `search.py binary.exe strings -f render,draw` | +| `search.py $B strings` | Extract strings with keyword filter. Prefer `retools.query "SELECT * FROM grep WHERE name LIKE '%...%'"` when index.db exists | `search.py binary.exe strings -f render,draw` | | `search.py $B strings --xrefs` | Find strings AND code locations that reference them | `search.py binary.exe strings -f "error" --xrefs` | | `search.py $B pattern` | Find exact byte pattern | `search.py binary.exe pattern "D9 56 54 D8 1D"` | | `search.py $B imports` | List PE imports, filter by DLL | `search.py binary.exe imports -d kernel32` | @@ -162,33 +171,7 @@ These are fast first-pass scanners — they surface candidate addresses. Follow ## Dynamic Analysis (`livetools/`) -- Frida-based, attaches to running process -``` -python -m livetools attach # attach to running process by name or PID -python -m livetools attach "C:/Games/game.exe" --spawn # launch + instrument before init code runs -python -m livetools detach # end session -python -m livetools status # check connection -``` - -| Command | Purpose | -|---------|---------| -| `trace $VA` | Non-blocking: log N hits with register/memory reads | -| `steptrace $VA` | Instruction-level trace (Stalker) with call depth control | -| `collect $VA [$VA2...]` | Multi-address hit counting over duration | -| `bp add/del/list $VA` | Breakpoints (stops target) | -| `watch` | Wait for breakpoint hit | -| `regs` / `stack` / `bt` | Inspect registers, stack, backtrace at break | -| `mem read $VA $SIZE` | Read live process memory (supports --as float32) | -| `mem write $VA $HEX` | Write live process memory | -| `mem alloc $SIZE` | Allocate rwx memory in target (for code caves) | -| `disasm [$VA]` | Disassemble from live process | -| `scan $PATTERN` | Search process memory for byte pattern | -| `modules` | List loaded modules with base addresses | -| `dipcnt on/off/read` | D3D9 DrawIndexedPrimitive call counter | -| `dipcnt callers [N]` | Sample N DIP calls and histogram return addresses | -| `memwatch start/stop/read` | Memory write watchpoint with backtrace | -| `vishook on/off/stats` | Selective visibility override via code cave (forces visible above caller threshold) | -| `gamectl key/keys/click/macro` | Send keys/clicks to game window (no Frida, no focus steal) | -| `analyze $FILE` | Offline analysis of collected .jsonl trace data | +Main-agent only (requires a live process; static-analyzer subagents must not use these). Canonical command reference with syntax, read-spec format, and recipes: the `/dynamic-analysis` skill (`.claude/skills/dynamic-analysis/SKILL.md`). Covers attach/spawn, breakpoints, trace/steptrace/collect, mem read/write/alloc, scan, disasm, modules, dipcnt, memwatch, vishook, gamectl, and offline `analyze`. **NOTE**: Some processes require their window to be focused for traces to capture data. @@ -202,7 +185,7 @@ A proxy DLL that intercepts all 119 `IDirect3DDevice9` methods, capturing every ``` python -m graphics.directx.dx9.tracer codegen -o d3d9_trace_hooks.inc # C hooks (standalone proxy) -python -m graphics.directx.dx9.tracer codegen -f cpp -o tracer_dispatch.inc # C++ dispatch (remix-comp-proxy module) +python -m graphics.directx.dx9.tracer codegen -f cpp -o tracer_dispatch.inc # C++ dispatch (remix-comp module) cd graphics/directx/dx9/tracer/src && build.bat # build standalone proxy DLL # Deploy d3d9.dll + proxy.ini to game directory python -m graphics.directx.dx9.tracer trigger --game-dir # trigger capture (3s countdown) @@ -316,13 +299,17 @@ Minidumps vary in how much data they capture depending on `MiniDumpWriteDump` fl These tools find references via absolute memory operands, immediate values (with `--imm` flag), and RIP-relative addressing. If you suspect a reference exists but the tool doesn't find it, the address might be computed at runtime. Try `search.py pattern` with the address bytes directly, or use `livetools memwatch`. -### `pyghidra_backend.py` -- requires Ghidra installation +### `pyghidra_backend.py` -- Ghidra primary, r2ghidra fallback + +Ghidra (via `pyghidra_backend.py`, indexed into `index.db`, daemon-backed, kb-applied) is the **primary** decompilation backend once a project exists — it gives better type propagation, library call resolution, and lets `retools.query` answer structural questions without re-scanning the binary. r2ghidra (`decompiler.py --backend pdg`) is the **zero-setup fallback and second opinion**: no Ghidra install required, faster on small functions, and useful to cross-check a pyghidra result that looks wrong. `decompiler.py --backend auto` tries pyghidra first and falls back to r2ghidra automatically — this routing is unchanged. Requires Ghidra 11.x+ installed and `GHIDRA_INSTALL_DIR` environment variable set. **Optional** -- the toolkit works without it (r2ghidra remains the fallback). **Disk usage**: Ghidra projects are ~10-20x the binary size. A 30MB game exe produces a ~300-600MB `.rep/` directory under `patches//ghidra/`. This directory is already covered by `.gitignore` (the `patches/` exclusion). -**First-time analysis** takes 5-15 minutes depending on binary size. Subsequent decompilation from the saved project is instant (<1s plus ~3s JVM startup per process). +**First-time analysis** takes 5-15 minutes depending on binary size. Subsequent decompilation from the saved project is near-instant (<1s plus ~3s JVM startup per cold process) -- or truly sub-second when a `ghidra_server.py` daemon is warm for that project. `export` and `kb-apply` route through the same live daemon when one is running; `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold in-process run for either. + +**Index-first**: after `export`, prefer `retools.query` over datarefs/xrefs/search/funcinfo for anything already captured in `index.db` (funcs, names, xrefs, strings, imports, blocks) -- it's a local SQL query instead of a fresh binary scan. Fall back to the scanners only for facts index.db doesn't have yet. ### `livetools` -- static vs runtime addresses diff --git a/.github/agents/static-analyzer.agent.md b/.github/agents/static-analyzer.agent.md index 9fed1660..a2a51331 100644 --- a/.github/agents/static-analyzer.agent.md +++ b/.github/agents/static-analyzer.agent.md @@ -26,13 +26,15 @@ grep -cE '^[@$]|^struct |^enum ' patches//kb.h 2>/dev/null || echo 0 ``` If the count is under 50 (or the file doesn't exist), run `python -m retools.bootstrap --project ` first. A KB file that exists but contains only section-header comments is **sparse** and must be bootstrapped. Do not skip bootstrap just because the file exists. +**3. Index**: Check `python -m retools.index status ` before scanning the binary yourself — if `index.db` already has `funcs`/`xrefs`/`names`/`strings`, prefer `retools.query` over a fresh xrefs/datarefs/search/funcinfo pass. + ## Running Tools -Run all tools from the repo root using `python -m retools.` syntax: +Run all tools from the repo root using `python -m retools.` syntax. Decompilation has two backends: **pyghidra is primary** once a Ghidra project exists (better type propagation, library call resolution, and its output feeds `index.db`); **r2ghidra is the zero-setup fallback and second opinion** (no Ghidra install needed, faster on small functions). `--backend auto` tries pyghidra first, falls back to r2ghidra. ``` python -m retools.decompiler binary.exe 0x401000 -python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h +python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h --project patches/proj python -m retools.search binary.exe strings -f "error" --xrefs python -m retools.xrefs binary.exe 0x401000 -t call python -m retools.callgraph binary.exe 0x401000 --up 3 @@ -45,6 +47,11 @@ python -m retools.sigdb scan binary.exe --db retools/data/signatures.db python -m retools.sigdb identify binary.exe 0x401000 --db retools/data/signatures.db python -m retools.sigdb fingerprint binary.exe python -m retools.context assemble binary.exe 0x401000 --project MyGame +python retools/pyghidra_backend.py export binary.exe --project patches/MyGame +python retools/pyghidra_backend.py kb-apply binary.exe --project patches/MyGame --kb patches/MyGame/kb.h +python -m retools.index status MyGame +python -m retools.query MyGame "SELECT * FROM funcs WHERE name LIKE '%Update%'" +python -m retools.ghidra_server MyGame --idle 600 ``` If `retools/data/signatures.db` is missing, run `python -m retools.sigdb pull` to download it before using `sigdb scan` or `sigdb build`. @@ -53,6 +60,14 @@ Collect MORE information per command run. Prefer wide queries over narrow ones Always pass `--types ` to `decompiler.py` when a KB file exists for the project. +**Query-first**: before `xrefs`/`datarefs`/`search`/`funcinfo`, check `index status`/`query` for the answer — it's a local SQL query instead of a fresh binary scan. + +**Hard pushdown rule**: `decompile` and `export` require a specific function address (or, for `export`, an analyzed program) — never invoke them without one. + +**Read-First mutation discipline**: `kb-apply` mutates the Ghidra project. Decompile or `query` the target function first, run `kb-apply`, then re-decompile to verify the change landed. `kb-apply` is idempotent — re-running it must produce stable counts. + +**Cost guard**: run `export` once per analysis pass (after `kb-apply`), not once per query. Warm `ghidra_server.py` for a project before a batch of decompiles. + ## Knowledge Base When you discover something significant, update the project KB file (`patches//kb.h`). @@ -104,3 +119,13 @@ Format: Also update `patches//kb.h` with any new function signatures, structs, or globals discovered. In your return message, state the file path you wrote to and give a brief summary. The main agent will read the file for full details. + +## Routing to Adjacent Skills/Docs + +| Need | Go to | +|------|-------| +| Full tool syntax, flags, caveats, run-directly vs delegate guidance | `.github/instructions/tool-catalog.instructions.md` | +| Bootstrap ordering, Ghidra-primary/r2ghidra-fallback framing, delegation rules | `.github/copilot-instructions.md` ("Delegation Rule" / "Ghidra-Primary Decompilation") | +| Attaching to a live process, breakpoints, tracing, memory patching | `/dynamic-analysis` skill (main agent only — this agent must not use livetools) | +| Porting a DX9 game to FFP for RTX Remix (renderer.cpp, ffp_state, vertex decls, skinning) | `dx9-ffp-port` skill | +| D3D9-specific static questions (VS/PS constants, render states, vertex formats) | DX analysis scripts (`rtx_remix_tools/dx/scripts/`) before general retools | diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 165cfed1..ab1b9d6c 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -23,6 +23,8 @@ Run `python verify_install.py` from the repo root before first use. If pyghidra/ - `readmem.py` — single typed read from a PE file - `asi_patcher.py build` — build step, not analysis - `pyghidra_backend.py status` — project existence check (<1s) +- `index.py status [--db PATH]` — per-table row counts + schema_version for the game's index.db +- `query.py "SQL" [--db PATH] [--json] [--list-tables] [--schema TABLE]` — read-only SQL over index.db (`callers`/`callees`/`grep` views); prefer this over a fresh xrefs/datarefs/search/funcinfo scan whenever index.db already has the answer If you're about to run a second `retools` command in the same turn, stop and delegate everything to a subagent. @@ -32,16 +34,17 @@ Run all tools from the repo root using `python -m ` syntax (e.g. `python The main agent owns `livetools` — always use them to verify static findings and act on leads from subagents. Use `attach ` for running processes, or `attach --spawn` to launch + instrument before init code runs. When a subagent returns addresses or candidates, immediately follow up with live tools (trace, breakpoint, mem read/write) rather than spawning more static analysis. Static analysis finds clues; live tools confirm and act on them. Do not wait idle for subagents — use live tools to explore independently while static analysis runs in the background. -## Dual-Backend Decompilation +## Ghidra-Primary Decompilation -The decompiler supports two backends with different strengths: +Ghidra, via `pyghidra_backend.py`, is the **primary** decompilation backend once a project exists — indexed into `index.db`, daemon-backed (`ghidra_server.py`, port 27043; livetools owns 27042), and kb-applied. It gives better MSVC type propagation, library call resolution, and larger function scope detection, and its facts can be queried with `retools.query` instead of re-scanning the binary. Requires a Ghidra project (`pyghidra_backend.py analyze` creates one). Use when `patches//ghidra/.gpr` exists. -- **pyghidra (preferred)** — better MSVC type propagation, library call resolution, larger function scope detection. Requires a Ghidra project (`pyghidra_backend.py analyze` creates one). Use when `patches//ghidra/.gpr` exists. -- **r2ghidra (fallback)** — better `__thiscall` on small functions, no JVM startup. Always available. +**r2ghidra (zero-setup fallback / second opinion)** — no Ghidra install required, better `__thiscall` on small functions, no JVM startup, and useful to cross-check a pyghidra result that looks wrong. -**Auto mode**: `python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h --project patches/proj` tries pyghidra first, falls back to r2ghidra. Use `--project` alongside `--types` for auto selection. +**Auto mode**: `python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h --project patches/proj` tries pyghidra first, falls back to r2ghidra. Use `--project` alongside `--types` for auto selection. This routing is unchanged. -**Dual-backend deep analysis**: For complex exploratory tasks (finding subsystems, mapping call chains), run both backends in parallel on the same functions and merge findings. r2ghidra results go to `findings_r2.md`, pyghidra to `findings.md`. Neither backend finds everything alone -- merging both gives the most complete picture. +**Index and daemon**: `pyghidra_backend.py export` seeds `funcs`/`names`/`xrefs`/`blocks` into `index.db` (`source='ghidra'`, overwrites provisional bootstrap rows at the same address). `pyghidra_backend.py kb-apply` pushes kb.h names/prototypes/globals into the Ghidra project (idempotent). `python -m retools.ghidra_server [--idle 600]` keeps one warm Ghidra program per project so repeat `decompile`/`export`/`kb-apply` calls become sub-second; `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold run. + +**Dual-backend deep analysis**: reserve this for two cases — **no Ghidra project exists yet**, or **pyghidra output on a specific function looks wrong** and needs an independent r2ghidra cross-check. Run both backends in parallel on the same functions and merge findings: r2ghidra results go to `findings_r2.md`, pyghidra to `findings.md`. Not needed once a Ghidra project exists and `--backend auto` is available. ## Engineering Standards diff --git a/.github/instructions/tool-catalog.instructions.md b/.github/instructions/tool-catalog.instructions.md index 182d1d97..dd386b8b 100644 --- a/.github/instructions/tool-catalog.instructions.md +++ b/.github/instructions/tool-catalog.instructions.md @@ -27,6 +27,8 @@ These are fast (<5s) and allowed inline: - "Trace where this value comes from" → `python -m retools.dataflow $B $VA --slice TARGET_VA:REG` - "Build an ASI patch DLL" → `python -m retools.asi_patcher build spec.json` - "Does a Ghidra project exist for this binary?" → `python retools/pyghidra_backend.py status $B --project $P` +- "What's in this game's index?" → `python -m retools.index status [--db PATH]` +- "Query facts already indexed (funcs, names, xrefs, strings, imports, callers/callees)" → `python -m retools.query "SQL" [--db PATH] [--json] [--list-tables] [--schema TABLE]` ### Delegate to `static-analyzer` subagent @@ -35,13 +37,13 @@ Everything else. Tell the subagent WHAT you need, not HOW to run it — it has t **D3D9-specific questions?** Check the DX analysis scripts section below first — they're faster and more targeted than general retools for D3D API usage, device calls, shader constants, and vertex formats. - "What does this function do?" → decompile + callgraph + xrefs + dataflow --constants -- "Who calls this function?" → xrefs or callgraph --up -- "What does this function call?" → callgraph --down (add --indirect for vtable calls) +- "Who calls this function?" → xrefs or callgraph --up (prefer `retools.query "SELECT * FROM callers WHERE callee_addr=..."` when index.db exists) +- "What does this function call?" → callgraph --down (add --indirect for vtable calls) (prefer `retools.query "SELECT * FROM callees WHERE caller=..."` when index.db exists) - "Who calls this virtual method?" → xrefs --indirect + filter by vtable slot offset - "What constant reaches this call?" → dataflow --constants or --slice VA:REG - "Resolve a switch/jump table" → cfg (auto-resolves MSVC switch patterns) -- "Find a string and who uses it" → string search with xrefs -- "Where is this global read/written?" → datarefs +- "Find a string and who uses it" → string search with xrefs (prefer `retools.query "SELECT * FROM grep WHERE name LIKE '%...%'"` when index.db exists) +- "Where is this global read/written?" → datarefs (prefer `retools.query` against `names`/`xrefs` when index.db exists) - "Where is struct field +0x54 used?" → structrefs - "What does this struct look like?" → structrefs --aggregate - "What C++ class is this vtable?" → RTTI resolution @@ -51,6 +53,8 @@ Everything else. Tell the subagent WHAT you need, not HOW to run it — it has t - "Map all throw sites to error strings" → throwmap list - "First time analyzing a binary?" → bootstrap (2-5 min) + pyghidra analyze (5-15 min) in parallel - "Bulk signature scan" → sigdb scan (1-3 min) +- "Seed/refresh the index from a Ghidra project" → `pyghidra_backend.py export` (funcs/names/xrefs/blocks, source='ghidra', authoritative over provisional bootstrap rows) +- "Push kb.h names/prototypes into the Ghidra project" → `pyghidra_backend.py kb-apply` (idempotent) - Any combination of the above ### Live tools (main agent, requires attached process) @@ -102,19 +106,24 @@ These are fast first-pass scanners — they surface candidate addresses. Follow | `pyghidra_backend.py analyze $B --project $P` | **Full Ghidra analysis** -- one-time, saves reusable project | `pyghidra_backend.py analyze game.exe --project patches/MyGame` | | `pyghidra_backend.py decompile $B $VA --project $P` | Decompile via saved Ghidra project | `pyghidra_backend.py decompile game.exe 0x401000 --project patches/MyGame` | | `pyghidra_backend.py status $B --project $P` | Check if Ghidra project exists | `pyghidra_backend.py status game.exe --project patches/MyGame` | -| `funcinfo.py $B $VA` | Find function start/end, rets, calling convention, callees | `funcinfo.py binary.exe 0x401000` | +| `pyghidra_backend.py export $B --project $P [--db]` | Seed funcs/names/xrefs/blocks from an analyzed Ghidra program into index.db (`source='ghidra'`, overwrites provisional bootstrap rows at the same address) | `pyghidra_backend.py export game.exe --project patches/MyGame` | +| `pyghidra_backend.py kb-apply $B --project $P --kb $KB` | Push kb.h names/prototypes/globals/typedefs into the Ghidra project. Idempotent — safe to re-run | `pyghidra_backend.py kb-apply game.exe --project patches/MyGame --kb patches/MyGame/kb.h` | +| `index.py status $GAME [--db]` | Per-table row counts + schema_version for `patches//index.db` | `python -m retools.index status MyGame` | +| `query.py $GAME "SQL" [--db] [--json] [--list-tables] [--schema TABLE]` | Read-only SQL over index.db. Views: `callers`, `callees`, `grep`. Addresses render as hex via `printf('0x%x', address)`. Connection is opened read-only — cannot mutate | `python -m retools.query MyGame "SELECT * FROM grep WHERE name LIKE '%Ground%'"` | +| `ghidra_server.py $GAME [--idle SECS]` | Run a per-project Ghidra daemon (port 27043; livetools owns 27042) holding one warm program so repeat `decompile`/`export`/`kb-apply` calls are sub-second instead of paying JVM+analysis startup each time | `python -m retools.ghidra_server MyGame --idle 600` | +| `funcinfo.py $B $VA` | Find function start/end, rets, calling convention, callees. Prefer `retools.query` on `funcs`/`blocks` when index.db exists | `funcinfo.py binary.exe 0x401000` | | `cfg.py $B $VA` | Control flow graph (basic blocks + edges, text or mermaid). Resolves MSVC switch/jump tables automatically. `--switch-details` shows table info | `cfg.py binary.exe 0x401000 --format mermaid` | | `callgraph.py $B $VA` | Caller/callee tree (multi-level, --up/--down N). `--indirect` adds vtable/fptr calls to --down trees | `callgraph.py binary.exe 0x401000 --down 2 --indirect` | -| `xrefs.py $B $VA` | Find all calls/jumps TO an address. `--indirect` also scans for `call [reg+offset]`, `call [reg]`, `call [addr]` | `xrefs.py binary.exe 0x401000 --indirect` | +| `xrefs.py $B $VA` | Find all calls/jumps TO an address. `--indirect` also scans for `call [reg+offset]`, `call [reg]`, `call [addr]`. Prefer `retools.query "SELECT * FROM callers WHERE callee_addr=..."` when index.db exists | `xrefs.py binary.exe 0x401000 --indirect` | | `dataflow.py $B $VA` | Forward constant propagation (`--constants`) or backward register slice (`--slice VA:REG`) within a function | `dataflow.py binary.exe 0x401000 --constants` | -| `datarefs.py $B $VA` | Find instructions that reference a global address (mem deref + `--imm` for push/mov constants) | `datarefs.py binary.exe 0x7A0000 --imm` | +| `datarefs.py $B $VA` | Find instructions that reference a global address (mem deref + `--imm` for push/mov constants). Prefer `retools.query` against `xrefs`/`names` when index.db exists | `datarefs.py binary.exe 0x7A0000 --imm` | | `structrefs.py $B $OFF` | Find all `[reg+offset]` accesses (struct field usage) | `structrefs.py binary.exe 0x54 --base esi` | | `structrefs.py $B --aggregate` | Reconstruct C struct from all field accesses in a function | `structrefs.py binary.exe --aggregate --fn 0x401000 --base esi` | | `vtable.py $B dump $VA` | Dump C++ vtable slots with instruction preview | `vtable.py binary.exe dump 0x6A0000` | | `vtable.py $B calls $OFF` | Find all indirect `call [reg+offset]` (vtable call sites) | `vtable.py binary.exe calls 0xB0` | | `rtti.py $B vtable $VA` | Resolve C++ class name + inheritance chain from vtable (MSVC RTTI) | `rtti.py binary.dll vtable 0x6A0000` | | `rtti.py $B throwinfo $RVA` | Resolve exception type from `_ThrowInfo` (MSVC RTTI) | `rtti.py binary.dll throwinfo 0x5040CF8` | -| `search.py $B strings` | Extract strings with keyword filter | `search.py binary.exe strings -f render,draw` | +| `search.py $B strings` | Extract strings with keyword filter. Prefer `retools.query "SELECT * FROM grep WHERE name LIKE '%...%'"` when index.db exists | `search.py binary.exe strings -f render,draw` | | `search.py $B strings --xrefs` | Find strings AND code locations that reference them | `search.py binary.exe strings -f "error" --xrefs` | | `search.py $B pattern` | Find exact byte pattern | `search.py binary.exe pattern "D9 56 54 D8 1D"` | | `search.py $B imports` | List PE imports, filter by DLL | `search.py binary.exe imports -d kernel32` | @@ -161,33 +170,7 @@ These are fast first-pass scanners — they surface candidate addresses. Follow ## Dynamic Analysis (`livetools/`) -- Frida-based, attaches to running process -``` -python -m livetools attach # attach to running process by name or PID -python -m livetools attach "C:/Games/game.exe" --spawn # launch + instrument before init code runs -python -m livetools detach # end session -python -m livetools status # check connection -``` - -| Command | Purpose | -|---------|---------| -| `trace $VA` | Non-blocking: log N hits with register/memory reads | -| `steptrace $VA` | Instruction-level trace (Stalker) with call depth control | -| `collect $VA [$VA2...]` | Multi-address hit counting over duration | -| `bp add/del/list $VA` | Breakpoints (stops target) | -| `watch` | Wait for breakpoint hit | -| `regs` / `stack` / `bt` | Inspect registers, stack, backtrace at break | -| `mem read $VA $SIZE` | Read live process memory (supports --as float32) | -| `mem write $VA $HEX` | Write live process memory | -| `mem alloc $SIZE` | Allocate rwx memory in target (for code caves) | -| `disasm [$VA]` | Disassemble from live process | -| `scan $PATTERN` | Search process memory for byte pattern | -| `modules` | List loaded modules with base addresses | -| `dipcnt on/off/read` | D3D9 DrawIndexedPrimitive call counter | -| `dipcnt callers [N]` | Sample N DIP calls and histogram return addresses | -| `memwatch start/stop/read` | Memory write watchpoint with backtrace | -| `vishook on/off/stats` | Selective visibility override via code cave (forces visible above caller threshold) | -| `gamectl key/keys/click/macro` | Send keys/clicks to game window (no Frida, no focus steal) | -| `analyze $FILE` | Offline analysis of collected .jsonl trace data | +Main-agent only (requires a live process; static-analyzer subagents must not use these). Canonical command reference with syntax, read-spec format, and recipes: the `/dynamic-analysis` skill (`.claude/skills/dynamic-analysis/SKILL.md`). Covers attach/spawn, breakpoints, trace/steptrace/collect, mem read/write/alloc, scan, disasm, modules, dipcnt, memwatch, vishook, gamectl, and offline `analyze`. **NOTE**: Some processes require their window to be focused for traces to capture data. @@ -201,7 +184,7 @@ A proxy DLL that intercepts all 119 `IDirect3DDevice9` methods, capturing every ``` python -m graphics.directx.dx9.tracer codegen -o d3d9_trace_hooks.inc # C hooks (standalone proxy) -python -m graphics.directx.dx9.tracer codegen -f cpp -o tracer_dispatch.inc # C++ dispatch (remix-comp-proxy module) +python -m graphics.directx.dx9.tracer codegen -f cpp -o tracer_dispatch.inc # C++ dispatch (remix-comp module) cd graphics/directx/dx9/tracer/src && build.bat # build standalone proxy DLL # Deploy d3d9.dll + proxy.ini to game directory python -m graphics.directx.dx9.tracer trigger --game-dir # trigger capture (3s countdown) @@ -315,13 +298,17 @@ Minidumps vary in how much data they capture depending on `MiniDumpWriteDump` fl These tools find references via absolute memory operands, immediate values (with `--imm` flag), and RIP-relative addressing. If you suspect a reference exists but the tool doesn't find it, the address might be computed at runtime. Try `search.py pattern` with the address bytes directly, or use `livetools memwatch`. -### `pyghidra_backend.py` -- requires Ghidra installation +### `pyghidra_backend.py` -- Ghidra primary, r2ghidra fallback + +Ghidra (via `pyghidra_backend.py`, indexed into `index.db`, daemon-backed, kb-applied) is the **primary** decompilation backend once a project exists — it gives better type propagation, library call resolution, and lets `retools.query` answer structural questions without re-scanning the binary. r2ghidra (`decompiler.py --backend pdg`) is the **zero-setup fallback and second opinion**: no Ghidra install required, faster on small functions, and useful to cross-check a pyghidra result that looks wrong. `decompiler.py --backend auto` tries pyghidra first and falls back to r2ghidra automatically — this routing is unchanged. Requires Ghidra 11.x+ installed and `GHIDRA_INSTALL_DIR` environment variable set. **Optional** -- the toolkit works without it (r2ghidra remains the fallback). **Disk usage**: Ghidra projects are ~10-20x the binary size. A 30MB game exe produces a ~300-600MB `.rep/` directory under `patches//ghidra/`. This directory is already covered by `.gitignore` (the `patches/` exclusion). -**First-time analysis** takes 5-15 minutes depending on binary size. Subsequent decompilation from the saved project is instant (<1s plus ~3s JVM startup per process). +**First-time analysis** takes 5-15 minutes depending on binary size. Subsequent decompilation from the saved project is near-instant (<1s plus ~3s JVM startup per cold process) -- or truly sub-second when a `ghidra_server.py` daemon is warm for that project. `export` and `kb-apply` route through the same live daemon when one is running; `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold in-process run for either. + +**Index-first**: after `export`, prefer `retools.query` over datarefs/xrefs/search/funcinfo for anything already captured in `index.db` (funcs, names, xrefs, strings, imports, blocks) -- it's a local SQL query instead of a fresh binary scan. Fall back to the scanners only for facts index.db doesn't have yet. ### `livetools` -- static vs runtime addresses diff --git a/.kiro/agents/static-analyzer.md b/.kiro/agents/static-analyzer.md index ab56370d..dd8573db 100644 --- a/.kiro/agents/static-analyzer.md +++ b/.kiro/agents/static-analyzer.md @@ -34,30 +34,58 @@ python retools/pyghidra_backend.py status --project patches/ ``` If "Not analyzed", run `python retools/pyghidra_backend.py analyze --project patches/`. Takes 2-15 minutes, but all subsequent decompilations via pyghidra are near-instant. +**5. Index**: Check whether the project has an index.db and what's in it before scanning the binary yourself: +```bash +python -m retools.index status +``` +If `funcs`/`xrefs` show `source='bootstrap'` only (or the table is empty), and a Ghidra project exists, run `pyghidra_backend.py export` to seed authoritative facts — see "Query-first workflow" below. + ## Running Tools Run all tools from the repo root. Use `python -m retools.` or `python retools/.py` syntax: -### Decompilation (two backends) +### Decompilation -- Ghidra primary, r2ghidra fallback -**pyghidra (preferred when Ghidra project exists)** — better MSVC type propagation, library call resolution, larger function scope detection: +**pyghidra is the primary backend** once a Ghidra project exists — better MSVC type propagation, library call resolution, larger function scope detection, and its facts can be exported into `index.db` for instant SQL lookups later: ``` python retools/pyghidra_backend.py decompile binary.exe 0x401000 --project patches/proj ``` -**r2ghidra (fast fallback)** — better `__thiscall` on small functions, no JVM startup: +**r2ghidra is the zero-setup fallback and second opinion** — no Ghidra install required, better `__thiscall` recovery on small functions, no JVM startup, and useful to cross-check a pyghidra decompile that looks wrong: ``` python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h --backend pdg ``` -**Auto mode (tries pyghidra first, falls back to r2ghidra)**: +**Auto mode (tries pyghidra first, falls back to r2ghidra)** — routing unchanged: ``` python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h --project patches/proj ``` When told to use a specific backend, use it. Otherwise prefer auto mode with both `--types` and `--project`. +**Ghidra daemon**: if `python -m retools.ghidra_server ` is running (port 27043; livetools owns 27042), `decompile`/`export`/`kb-apply` route through it automatically and repeat calls become sub-second. Warm the server yourself before a batch of decompiles on the same project: `python -m retools.ghidra_server --idle 600` (background it). `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold in-process run when you need to bypass the daemon. + +### Query-first workflow + +Before re-scanning a binary with xrefs/datarefs/search/funcinfo, check whether `index.db` already has the answer — a SQL query against a local file is cheaper than re-disassembling: + +```bash +python -m retools.index status # per-table counts + schema_version +python -m retools.query --list-tables # confirm what's queryable +python -m retools.query --schema funcs # PRAGMA table_info before writing joins +python -m retools.query "SELECT * FROM callers WHERE callee_addr=0x401000" +python -m retools.query "SELECT * FROM grep WHERE name LIKE '%Ground%'" --json +``` + +Only fall back to `xrefs.py`/`datarefs.py`/`search.py`/`funcinfo.py` for facts `index.db` doesn't have yet (e.g. no `export` has run, or the question needs a live disassembly detail not captured by the schema). + +**Hard pushdown rule**: `decompile` and `export` require a specific function address (or, for `export`, an analyzed program) — never invoke them without one, or you decompile/scan the whole binary instead of the function you actually need. If you don't have an address yet, get one from `query`, `search`, or `xrefs` first. + +**Read-First mutation discipline**: `kb-apply` mutates the Ghidra project. Always decompile or `query` the target function first to confirm the current name/prototype, run `kb-apply`, then **re-decompile the same function** to verify the change landed before reporting it as done. `kb-apply` is idempotent — re-running it should produce stable counts and no errors, so if a second run changes anything, treat that as a bug, not expected behavior. + +**Cost guard**: run `export` once per analysis pass (after `kb-apply`, so exported names reflect it), not once per query — repeated `export` calls re-walk the whole program for no benefit once `index.db` is current. + ### Other tools ``` python -m retools.search binary.exe strings -f "error" --xrefs @@ -73,6 +101,11 @@ python -m retools.sigdb fingerprint binary.exe python -m retools.context assemble binary.exe 0x401000 --project MyGame python retools/pyghidra_backend.py analyze binary.exe --project patches/MyGame python retools/pyghidra_backend.py status binary.exe --project patches/MyGame +python retools/pyghidra_backend.py export binary.exe --project patches/MyGame +python retools/pyghidra_backend.py kb-apply binary.exe --project patches/MyGame --kb patches/MyGame/kb.h +python -m retools.index status MyGame +python -m retools.query MyGame "SELECT * FROM funcs WHERE name LIKE '%Update%'" +python -m retools.ghidra_server MyGame --idle 600 ``` If `retools/data/signatures.db` is missing, run `python -m retools.sigdb pull` to download it. @@ -139,3 +172,15 @@ Also update `patches//kb.h` with any new function signatures, structs, In your return message, state the file path you wrote to and give a brief summary. The main agent will read the file for full details. Update your agent memory with significant architectural discoveries, identified subsystems, and class hierarchies that will be useful in future sessions. + +## Routing to Adjacent Skills/Docs + +This agent owns offline static analysis. Hand off to the right reference/skill instead of improvising: + +| Need | Go to | +|------|-------| +| Full tool syntax, flags, caveats for any retools/DX-script/dumpinfo tool, and run-directly vs delegate guidance | `.kiro/steering/tool-catalog.md` | +| Bootstrap ordering, parallel dual-backend runs, delegation table | `.kiro/steering/subagent-workflow.md` | +| Attaching to a live process, breakpoints, tracing, memory patching | `/dynamic-analysis` skill (main agent only — this agent must not use livetools) | +| Porting a DX9 game to FFP for RTX Remix (renderer.cpp, ffp_state, vertex decls, skinning) | `dx9-ffp-port` skill | +| D3D9-specific static questions (VS/PS constants, render states, vertex formats) | DX analysis scripts (`rtx_remix_tools/dx/scripts/`) before general retools | diff --git a/.kiro/steering/subagent-workflow.md b/.kiro/steering/subagent-workflow.md index 2a8a2a65..55230c66 100644 --- a/.kiro/steering/subagent-workflow.md +++ b/.kiro/steering/subagent-workflow.md @@ -44,7 +44,9 @@ When analyzing a binary for the first time (no existing or sparsely populated `p | Decompiler postprocess (`context.py postprocess`) | Main agent -- instant | | Dataflow: constants + backward slice (`dataflow.py`) | Main agent -- fast (<5s) | | File editing, patch specs, builds | Main agent — directly | -| KB updates from subagent findings | `static-analyzer` writes to `kb.h`; main agent may refine | +| KB updates from subagent findings | `static-analyzer` writes to `kb.h`, then `kb-apply` pushes it into Ghidra; main agent may refine | +| `index status` / `query` (SQL over `index.db`) | Main agent -- fast (<5s); prefer over xrefs/datarefs/search/funcinfo when index.db already has the answer | +| `pyghidra_backend.py export` (seed `index.db` from Ghidra) | `static-analyzer` subagent -- run once per analysis pass, after `kb-apply` | ## Subagent Output Files @@ -62,14 +64,16 @@ Multiple `static-analyzer` instances can run in parallel for independent questio ## Dual-Backend Deep Analysis -For deep analysis tasks (finding subsystems, mapping call chains, understanding large code areas), spawn **two parallel static-analyzer agents using different decompiler backends**: +Ghidra (indexed into `index.db`, daemon-backed via `ghidra_server.py`, kb-applied) is the **primary** backend once a project exists — prefer it plus `retools.query` over spawning two agents for most exploratory work. Reserve the dual-agent pattern below for two specific cases: **no Ghidra project exists yet** for this binary, or **pyghidra output on a specific function looks wrong** and you need an independent r2ghidra read to cross-check it. + +When one of those applies, spawn **two parallel static-analyzer agents using different decompiler backends**: 1. **r2ghidra agent** — uses `--backend pdg` (with `--types kb.h`), writes to `patches//findings_r2.md` 2. **pyghidra agent** — uses `pyghidra_backend.py decompile` (requires Ghidra project), writes to `patches//findings.md` -**Why both:** Each backend has different strengths. r2ghidra is better at `__thiscall` recovery on small functions and low-level D3D details. pyghidra resolves more library calls, finds larger function scopes, and propagates types better. Neither finds everything alone — merging both gives the most complete picture. +**Why both:** Each backend has different strengths. r2ghidra is better at `__thiscall` recovery on small functions, low-level D3D details, and needs no JVM/project setup. pyghidra resolves more library calls, finds larger function scopes, propagates types better, and its output is exportable into `index.db` for future queries. Neither finds everything alone — merging both gives the most complete picture. -**When to use dual-backend:** Complex exploratory tasks ("find the culling system", "map the rendering pipeline", "understand the network protocol"). Not needed for single-function decompilation — use `--backend auto` for that. +**When to use dual-backend:** Only when no Ghidra project exists yet, or when pyghidra output on a specific function looks wrong. Not needed for single-function decompilation once a Ghidra project exists — use `--backend auto` (Ghidra primary, r2ghidra fallback). **Synthesis:** When both agents return, the main agent reads both findings files and merges them into a unified analysis. Conflicting information is resolved by checking which backend's output is more complete for that specific function. diff --git a/.kiro/steering/tool-catalog.md b/.kiro/steering/tool-catalog.md index 1882aa1c..aab90078 100644 --- a/.kiro/steering/tool-catalog.md +++ b/.kiro/steering/tool-catalog.md @@ -28,6 +28,8 @@ These are fast (<5s) and allowed inline: - "Trace where this value comes from" → `python -m retools.dataflow $B $VA --slice TARGET_VA:REG` - "Build an ASI patch DLL" → `python -m retools.asi_patcher build spec.json` - "Does a Ghidra project exist for this binary?" → `python retools/pyghidra_backend.py status $B --project $P` +- "What's in this game's index?" → `python -m retools.index status [--db PATH]` +- "Query facts already indexed (funcs, names, xrefs, strings, imports, callers/callees)" → `python -m retools.query "SQL" [--db PATH] [--json] [--list-tables] [--schema TABLE]` ### Delegate to `static-analyzer` subagent @@ -36,13 +38,13 @@ Everything else. Tell the subagent WHAT you need, not HOW to run it — it has t **D3D9-specific questions?** Check the DX analysis scripts section below first — they're faster and more targeted than general retools for D3D API usage, device calls, shader constants, and vertex formats. - "What does this function do?" → decompile + callgraph + xrefs + dataflow --constants -- "Who calls this function?" → xrefs or callgraph --up -- "What does this function call?" → callgraph --down (add --indirect for vtable calls) +- "Who calls this function?" → xrefs or callgraph --up (prefer `retools.query "SELECT * FROM callers WHERE callee_addr=..."` when index.db exists) +- "What does this function call?" → callgraph --down (add --indirect for vtable calls) (prefer `retools.query "SELECT * FROM callees WHERE caller=..."` when index.db exists) - "Who calls this virtual method?" → xrefs --indirect + filter by vtable slot offset - "What constant reaches this call?" → dataflow --constants or --slice VA:REG - "Resolve a switch/jump table" → cfg (auto-resolves MSVC switch patterns) -- "Find a string and who uses it" → string search with xrefs -- "Where is this global read/written?" → datarefs +- "Find a string and who uses it" → string search with xrefs (prefer `retools.query "SELECT * FROM grep WHERE name LIKE '%...%'"` when index.db exists) +- "Where is this global read/written?" → datarefs (prefer `retools.query` against `names`/`xrefs` when index.db exists) - "Where is struct field +0x54 used?" → structrefs - "What does this struct look like?" → structrefs --aggregate - "What C++ class is this vtable?" → RTTI resolution @@ -52,6 +54,8 @@ Everything else. Tell the subagent WHAT you need, not HOW to run it — it has t - "Map all throw sites to error strings" → throwmap list - "First time analyzing a binary?" → bootstrap (2-5 min) + pyghidra analyze (5-15 min) in parallel - "Bulk signature scan" → sigdb scan (1-3 min) +- "Seed/refresh the index from a Ghidra project" → `pyghidra_backend.py export` (funcs/names/xrefs/blocks, source='ghidra', authoritative over provisional bootstrap rows) +- "Push kb.h names/prototypes into the Ghidra project" → `pyghidra_backend.py kb-apply` (idempotent) - Any combination of the above ### Live tools (main agent, requires attached process) @@ -103,19 +107,24 @@ These are fast first-pass scanners — they surface candidate addresses. Follow | `pyghidra_backend.py analyze $B --project $P` | **Full Ghidra analysis** -- one-time, saves reusable project | `pyghidra_backend.py analyze game.exe --project patches/MyGame` | | `pyghidra_backend.py decompile $B $VA --project $P` | Decompile via saved Ghidra project | `pyghidra_backend.py decompile game.exe 0x401000 --project patches/MyGame` | | `pyghidra_backend.py status $B --project $P` | Check if Ghidra project exists | `pyghidra_backend.py status game.exe --project patches/MyGame` | -| `funcinfo.py $B $VA` | Find function start/end, rets, calling convention, callees | `funcinfo.py binary.exe 0x401000` | +| `pyghidra_backend.py export $B --project $P [--db]` | Seed funcs/names/xrefs/blocks from an analyzed Ghidra program into index.db (`source='ghidra'`, overwrites provisional bootstrap rows at the same address) | `pyghidra_backend.py export game.exe --project patches/MyGame` | +| `pyghidra_backend.py kb-apply $B --project $P --kb $KB` | Push kb.h names/prototypes/globals/typedefs into the Ghidra project. Idempotent — safe to re-run | `pyghidra_backend.py kb-apply game.exe --project patches/MyGame --kb patches/MyGame/kb.h` | +| `index.py status $GAME [--db]` | Per-table row counts + schema_version for `patches//index.db` | `python -m retools.index status MyGame` | +| `query.py $GAME "SQL" [--db] [--json] [--list-tables] [--schema TABLE]` | Read-only SQL over index.db. Views: `callers`, `callees`, `grep`. Addresses render as hex via `printf('0x%x', address)`. Connection is opened read-only — cannot mutate | `python -m retools.query MyGame "SELECT * FROM grep WHERE name LIKE '%Ground%'"` | +| `ghidra_server.py $GAME [--idle SECS]` | Run a per-project Ghidra daemon (port 27043; livetools owns 27042) holding one warm program so repeat `decompile`/`export`/`kb-apply` calls are sub-second instead of paying JVM+analysis startup each time | `python -m retools.ghidra_server MyGame --idle 600` | +| `funcinfo.py $B $VA` | Find function start/end, rets, calling convention, callees. Prefer `retools.query` on `funcs`/`blocks` when index.db exists | `funcinfo.py binary.exe 0x401000` | | `cfg.py $B $VA` | Control flow graph (basic blocks + edges, text or mermaid). Resolves MSVC switch/jump tables automatically. `--switch-details` shows table info | `cfg.py binary.exe 0x401000 --format mermaid` | | `callgraph.py $B $VA` | Caller/callee tree (multi-level, --up/--down N). `--indirect` adds vtable/fptr calls to --down trees | `callgraph.py binary.exe 0x401000 --down 2 --indirect` | -| `xrefs.py $B $VA` | Find all calls/jumps TO an address. `--indirect` also scans for `call [reg+offset]`, `call [reg]`, `call [addr]` | `xrefs.py binary.exe 0x401000 --indirect` | +| `xrefs.py $B $VA` | Find all calls/jumps TO an address. `--indirect` also scans for `call [reg+offset]`, `call [reg]`, `call [addr]`. Prefer `retools.query "SELECT * FROM callers WHERE callee_addr=..."` when index.db exists | `xrefs.py binary.exe 0x401000 --indirect` | | `dataflow.py $B $VA` | Forward constant propagation (`--constants`) or backward register slice (`--slice VA:REG`) within a function | `dataflow.py binary.exe 0x401000 --constants` | -| `datarefs.py $B $VA` | Find instructions that reference a global address (mem deref + `--imm` for push/mov constants) | `datarefs.py binary.exe 0x7A0000 --imm` | +| `datarefs.py $B $VA` | Find instructions that reference a global address (mem deref + `--imm` for push/mov constants). Prefer `retools.query` against `xrefs`/`names` when index.db exists | `datarefs.py binary.exe 0x7A0000 --imm` | | `structrefs.py $B $OFF` | Find all `[reg+offset]` accesses (struct field usage) | `structrefs.py binary.exe 0x54 --base esi` | | `structrefs.py $B --aggregate` | Reconstruct C struct from all field accesses in a function | `structrefs.py binary.exe --aggregate --fn 0x401000 --base esi` | | `vtable.py $B dump $VA` | Dump C++ vtable slots with instruction preview | `vtable.py binary.exe dump 0x6A0000` | | `vtable.py $B calls $OFF` | Find all indirect `call [reg+offset]` (vtable call sites) | `vtable.py binary.exe calls 0xB0` | | `rtti.py $B vtable $VA` | Resolve C++ class name + inheritance chain from vtable (MSVC RTTI) | `rtti.py binary.dll vtable 0x6A0000` | | `rtti.py $B throwinfo $RVA` | Resolve exception type from `_ThrowInfo` (MSVC RTTI) | `rtti.py binary.dll throwinfo 0x5040CF8` | -| `search.py $B strings` | Extract strings with keyword filter | `search.py binary.exe strings -f render,draw` | +| `search.py $B strings` | Extract strings with keyword filter. Prefer `retools.query "SELECT * FROM grep WHERE name LIKE '%...%'"` when index.db exists | `search.py binary.exe strings -f render,draw` | | `search.py $B strings --xrefs` | Find strings AND code locations that reference them | `search.py binary.exe strings -f "error" --xrefs` | | `search.py $B pattern` | Find exact byte pattern | `search.py binary.exe pattern "D9 56 54 D8 1D"` | | `search.py $B imports` | List PE imports, filter by DLL | `search.py binary.exe imports -d kernel32` | @@ -162,33 +171,7 @@ These are fast first-pass scanners — they surface candidate addresses. Follow ## Dynamic Analysis (`livetools/`) -- Frida-based, attaches to running process -``` -python -m livetools attach # attach to running process by name or PID -python -m livetools attach "C:/Games/game.exe" --spawn # launch + instrument before init code runs -python -m livetools detach # end session -python -m livetools status # check connection -``` - -| Command | Purpose | -|---------|---------| -| `trace $VA` | Non-blocking: log N hits with register/memory reads | -| `steptrace $VA` | Instruction-level trace (Stalker) with call depth control | -| `collect $VA [$VA2...]` | Multi-address hit counting over duration | -| `bp add/del/list $VA` | Breakpoints (stops target) | -| `watch` | Wait for breakpoint hit | -| `regs` / `stack` / `bt` | Inspect registers, stack, backtrace at break | -| `mem read $VA $SIZE` | Read live process memory (supports --as float32) | -| `mem write $VA $HEX` | Write live process memory | -| `mem alloc $SIZE` | Allocate rwx memory in target (for code caves) | -| `disasm [$VA]` | Disassemble from live process | -| `scan $PATTERN` | Search process memory for byte pattern | -| `modules` | List loaded modules with base addresses | -| `dipcnt on/off/read` | D3D9 DrawIndexedPrimitive call counter | -| `dipcnt callers [N]` | Sample N DIP calls and histogram return addresses | -| `memwatch start/stop/read` | Memory write watchpoint with backtrace | -| `vishook on/off/stats` | Selective visibility override via code cave (forces visible above caller threshold) | -| `gamectl key/keys/click/macro` | Send keys/clicks to game window (no Frida, no focus steal) | -| `analyze $FILE` | Offline analysis of collected .jsonl trace data | +Main-agent only (requires a live process; static-analyzer subagents must not use these). Canonical command reference with syntax, read-spec format, and recipes: the `/dynamic-analysis` skill (`.claude/skills/dynamic-analysis/SKILL.md`). Covers attach/spawn, breakpoints, trace/steptrace/collect, mem read/write/alloc, scan, disasm, modules, dipcnt, memwatch, vishook, gamectl, and offline `analyze`. **NOTE**: Some processes require their window to be focused for traces to capture data. @@ -202,7 +185,7 @@ A proxy DLL that intercepts all 119 `IDirect3DDevice9` methods, capturing every ``` python -m graphics.directx.dx9.tracer codegen -o d3d9_trace_hooks.inc # C hooks (standalone proxy) -python -m graphics.directx.dx9.tracer codegen -f cpp -o tracer_dispatch.inc # C++ dispatch (remix-comp-proxy module) +python -m graphics.directx.dx9.tracer codegen -f cpp -o tracer_dispatch.inc # C++ dispatch (remix-comp module) cd graphics/directx/dx9/tracer/src && build.bat # build standalone proxy DLL # Deploy d3d9.dll + proxy.ini to game directory python -m graphics.directx.dx9.tracer trigger --game-dir # trigger capture (3s countdown) @@ -316,13 +299,17 @@ Minidumps vary in how much data they capture depending on `MiniDumpWriteDump` fl These tools find references via absolute memory operands, immediate values (with `--imm` flag), and RIP-relative addressing. If you suspect a reference exists but the tool doesn't find it, the address might be computed at runtime. Try `search.py pattern` with the address bytes directly, or use `livetools memwatch`. -### `pyghidra_backend.py` -- requires Ghidra installation +### `pyghidra_backend.py` -- Ghidra primary, r2ghidra fallback + +Ghidra (via `pyghidra_backend.py`, indexed into `index.db`, daemon-backed, kb-applied) is the **primary** decompilation backend once a project exists — it gives better type propagation, library call resolution, and lets `retools.query` answer structural questions without re-scanning the binary. r2ghidra (`decompiler.py --backend pdg`) is the **zero-setup fallback and second opinion**: no Ghidra install required, faster on small functions, and useful to cross-check a pyghidra result that looks wrong. `decompiler.py --backend auto` tries pyghidra first and falls back to r2ghidra automatically — this routing is unchanged. Requires Ghidra 11.x+ installed and `GHIDRA_INSTALL_DIR` environment variable set. **Optional** -- the toolkit works without it (r2ghidra remains the fallback). **Disk usage**: Ghidra projects are ~10-20x the binary size. A 30MB game exe produces a ~300-600MB `.rep/` directory under `patches//ghidra/`. This directory is already covered by `.gitignore` (the `patches/` exclusion). -**First-time analysis** takes 5-15 minutes depending on binary size. Subsequent decompilation from the saved project is instant (<1s plus ~3s JVM startup per process). +**First-time analysis** takes 5-15 minutes depending on binary size. Subsequent decompilation from the saved project is near-instant (<1s plus ~3s JVM startup per cold process) -- or truly sub-second when a `ghidra_server.py` daemon is warm for that project. `export` and `kb-apply` route through the same live daemon when one is running; `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold in-process run for either. + +**Index-first**: after `export`, prefer `retools.query` over datarefs/xrefs/search/funcinfo for anything already captured in `index.db` (funcs, names, xrefs, strings, imports, blocks) -- it's a local SQL query instead of a fresh binary scan. Fall back to the scanners only for facts index.db doesn't have yet. ### `livetools` -- static vs runtime addresses diff --git a/README.md b/README.md index 374786d5..0828a045 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,10 @@ pip install -r requirements.txt **Static analysis** (`retools/`) works directly on PE files on disk: disassembly, decompilation, cross-references, call graphs, vtable analysis, byte pattern search, and more. +**Indexed queries** (`retools/index.py` + `retools/query.py`) cache analyzed facts (functions, names, cross-references, strings, imports) into a per-game SQLite file (`patches//index.db`), so the agent can answer "who calls this" or "find all strings matching X" with a local SQL query instead of re-scanning the binary. Bootstrap and Ghidra analysis both feed it; Ghidra-sourced facts win over provisional ones. + +**Ghidra server** (`retools/ghidra_server.py`) keeps one Ghidra program warm per game project so repeat decompilations return in well under a second instead of paying Ghidra's analysis cost on every call. + **Dynamic analysis** (`livetools/`) attaches to a running process via Frida: breakpoints, register/memory inspection, function tracing, instruction-level stepping, and live memory patching. **Game window automation** (`livetools/gamectl.py`) sends keystrokes and mouse clicks to a game window without Frida. Uses `SendInput` with `AttachThreadInput` focus management — works with DirectInput/RawInput games that ignore `PostMessage`. Target by process exe name: From 922271a3bc6b65197fb4828c0190f6e8ff701185 Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 14:45:24 -0500 Subject: [PATCH 19/28] docs: correct remix-comp-proxy module name across trees; document ghidra daemon state file --- .claude/agents/static-analyzer.md | 2 +- .claude/references/tool-catalog.md | 4 ++-- .claude/rules/tool-dispatch.md | 2 +- .cursor/agents/static-analyzer.md | 2 +- .cursor/rules/tool-catalog.mdc | 4 ++-- .github/agents/static-analyzer.agent.md | 2 +- .github/copilot-instructions.md | 2 +- .github/instructions/tool-catalog.instructions.md | 4 ++-- .kiro/agents/static-analyzer.md | 2 +- .kiro/steering/tool-catalog.md | 4 ++-- 10 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.claude/agents/static-analyzer.md b/.claude/agents/static-analyzer.md index f2430bff..05360bc6 100644 --- a/.claude/agents/static-analyzer.md +++ b/.claude/agents/static-analyzer.md @@ -65,7 +65,7 @@ python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h --pro When told to use a specific backend, use it. Otherwise prefer auto mode with both `--types` and `--project`. -**Ghidra daemon**: if `python -m retools.ghidra_server ` is running (port 27043; livetools owns 27042), `decompile`/`export`/`kb-apply` route through it automatically and repeat calls become sub-second. Warm the server yourself before a batch of decompiles on the same project: `python -m retools.ghidra_server --idle 600` (background it). `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold in-process run when you need to bypass the daemon. +**Ghidra daemon**: if `python -m retools.ghidra_server ` is running (port 27043; livetools owns 27042), `decompile`/`export`/`kb-apply` route through it automatically and repeat calls become sub-second. Warm the server yourself before a batch of decompiles on the same project: `python -m retools.ghidra_server --idle 600` (background it). `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold in-process run when you need to bypass the daemon. The daemon records its pid/port/project/binary in `patches//ghidra/.state.json`, deleted on shutdown once the Ghidra program is closed and the Windows `.rep` lock is released. ### Query-first workflow diff --git a/.claude/references/tool-catalog.md b/.claude/references/tool-catalog.md index 0695b829..1822a62e 100644 --- a/.claude/references/tool-catalog.md +++ b/.claude/references/tool-catalog.md @@ -110,7 +110,7 @@ These are fast first-pass scanners — they surface candidate addresses. Follow | `pyghidra_backend.py kb-apply $B --project $P --kb $KB` | Push kb.h names/prototypes/globals/typedefs into the Ghidra project. Idempotent — safe to re-run | `pyghidra_backend.py kb-apply game.exe --project patches/MyGame --kb patches/MyGame/kb.h` | | `index.py status $GAME [--db]` | Per-table row counts + schema_version for `patches//index.db` | `python -m retools.index status MyGame` | | `query.py $GAME "SQL" [--db] [--json] [--list-tables] [--schema TABLE]` | Read-only SQL over index.db. Views: `callers`, `callees`, `grep`. Addresses render as hex via `printf('0x%x', address)`. Connection is opened read-only — cannot mutate | `python -m retools.query MyGame "SELECT * FROM grep WHERE name LIKE '%Ground%'"` | -| `ghidra_server.py $GAME [--idle SECS]` | Run a per-project Ghidra daemon (port 27043; livetools owns 27042) holding one warm program so repeat `decompile`/`export`/`kb-apply` calls are sub-second instead of paying JVM+analysis startup each time | `python -m retools.ghidra_server MyGame --idle 600` | +| `ghidra_server.py $GAME [--idle SECS]` | Run a per-project Ghidra daemon (port 27043; livetools owns 27042) holding one warm program so repeat `decompile`/`export`/`kb-apply` calls are sub-second instead of paying JVM+analysis startup each time. Tracks itself in `patches//ghidra/.state.json` (pid/port/project/binary), deleted on shutdown once the Ghidra program is closed | `python -m retools.ghidra_server MyGame --idle 600` | | `funcinfo.py $B $VA` | Find function start/end, rets, calling convention, callees. Prefer `retools.query` on `funcs`/`blocks` when index.db exists | `funcinfo.py binary.exe 0x401000` | | `cfg.py $B $VA` | Control flow graph (basic blocks + edges, text or mermaid). Resolves MSVC switch/jump tables automatically. `--switch-details` shows table info | `cfg.py binary.exe 0x401000 --format mermaid` | | `callgraph.py $B $VA` | Caller/callee tree (multi-level, --up/--down N). `--indirect` adds vtable/fptr calls to --down trees | `callgraph.py binary.exe 0x401000 --down 2 --indirect` | @@ -184,7 +184,7 @@ A proxy DLL that intercepts all 119 `IDirect3DDevice9` methods, capturing every ``` python -m graphics.directx.dx9.tracer codegen -o d3d9_trace_hooks.inc # C hooks (standalone proxy) -python -m graphics.directx.dx9.tracer codegen -f cpp -o tracer_dispatch.inc # C++ dispatch (remix-comp module) +python -m graphics.directx.dx9.tracer codegen -f cpp -o tracer_dispatch.inc # C++ dispatch (remix-comp-proxy module) cd graphics/directx/dx9/tracer/src && build.bat # build standalone proxy DLL # Deploy d3d9.dll + proxy.ini to game directory python -m graphics.directx.dx9.tracer trigger --game-dir # trigger capture (3s countdown) diff --git a/.claude/rules/tool-dispatch.md b/.claude/rules/tool-dispatch.md index 17e0ce66..74016d8f 100644 --- a/.claude/rules/tool-dispatch.md +++ b/.claude/rules/tool-dispatch.md @@ -31,7 +31,7 @@ Everything else in `retools`. Tell it WHAT you need, not HOW. D3D9-specific ques - `pyghidra_backend.py export` (seed index.db from a Ghidra project) / `kb-apply` (push kb.h into the Ghidra project) - dx9tracer offline analysis (summary, render-passes, shader-map, etc.) -**Ghidra daemon**: `python -m retools.ghidra_server [--idle 600]` keeps one warm Ghidra program per project on port 27043 (livetools owns 27042). `decompile`/`export`/`kb-apply` route through a live daemon automatically when one is running for that project — repeat decompiles become sub-second instead of paying JVM startup each time. `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold in-process run. +**Ghidra daemon**: `python -m retools.ghidra_server [--idle 600]` keeps one warm Ghidra program per project on port 27043 (livetools owns 27042). `decompile`/`export`/`kb-apply` route through a live daemon automatically when one is running for that project — repeat decompiles become sub-second instead of paying JVM startup each time. `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold in-process run. The daemon tracks itself in `patches//ghidra/.state.json` (pid/port/project/binary), deleted on shutdown once the Ghidra program is closed. ## Live tools (main agent, attached process) diff --git a/.cursor/agents/static-analyzer.md b/.cursor/agents/static-analyzer.md index 884e91e2..b256746b 100644 --- a/.cursor/agents/static-analyzer.md +++ b/.cursor/agents/static-analyzer.md @@ -64,7 +64,7 @@ python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h --pro When told to use a specific backend, use it. Otherwise prefer auto mode with both `--types` and `--project`. -**Ghidra daemon**: if `python -m retools.ghidra_server ` is running (port 27043; livetools owns 27042), `decompile`/`export`/`kb-apply` route through it automatically and repeat calls become sub-second. Warm the server yourself before a batch of decompiles on the same project: `python -m retools.ghidra_server --idle 600` (background it). `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold in-process run when you need to bypass the daemon. +**Ghidra daemon**: if `python -m retools.ghidra_server ` is running (port 27043; livetools owns 27042), `decompile`/`export`/`kb-apply` route through it automatically and repeat calls become sub-second. Warm the server yourself before a batch of decompiles on the same project: `python -m retools.ghidra_server --idle 600` (background it). `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold in-process run when you need to bypass the daemon. The daemon records its pid/port/project/binary in `patches//ghidra/.state.json`, deleted on shutdown once the Ghidra program is closed and the Windows `.rep` lock is released. ### Query-first workflow diff --git a/.cursor/rules/tool-catalog.mdc b/.cursor/rules/tool-catalog.mdc index aa7b5cd9..cb94e388 100644 --- a/.cursor/rules/tool-catalog.mdc +++ b/.cursor/rules/tool-catalog.mdc @@ -111,7 +111,7 @@ These are fast first-pass scanners — they surface candidate addresses. Follow | `pyghidra_backend.py kb-apply $B --project $P --kb $KB` | Push kb.h names/prototypes/globals/typedefs into the Ghidra project. Idempotent — safe to re-run | `pyghidra_backend.py kb-apply game.exe --project patches/MyGame --kb patches/MyGame/kb.h` | | `index.py status $GAME [--db]` | Per-table row counts + schema_version for `patches//index.db` | `python -m retools.index status MyGame` | | `query.py $GAME "SQL" [--db] [--json] [--list-tables] [--schema TABLE]` | Read-only SQL over index.db. Views: `callers`, `callees`, `grep`. Addresses render as hex via `printf('0x%x', address)`. Connection is opened read-only — cannot mutate | `python -m retools.query MyGame "SELECT * FROM grep WHERE name LIKE '%Ground%'"` | -| `ghidra_server.py $GAME [--idle SECS]` | Run a per-project Ghidra daemon (port 27043; livetools owns 27042) holding one warm program so repeat `decompile`/`export`/`kb-apply` calls are sub-second instead of paying JVM+analysis startup each time | `python -m retools.ghidra_server MyGame --idle 600` | +| `ghidra_server.py $GAME [--idle SECS]` | Run a per-project Ghidra daemon (port 27043; livetools owns 27042) holding one warm program so repeat `decompile`/`export`/`kb-apply` calls are sub-second instead of paying JVM+analysis startup each time. Tracks itself in `patches//ghidra/.state.json` (pid/port/project/binary), deleted on shutdown once the Ghidra program is closed | `python -m retools.ghidra_server MyGame --idle 600` | | `funcinfo.py $B $VA` | Find function start/end, rets, calling convention, callees. Prefer `retools.query` on `funcs`/`blocks` when index.db exists | `funcinfo.py binary.exe 0x401000` | | `cfg.py $B $VA` | Control flow graph (basic blocks + edges, text or mermaid). Resolves MSVC switch/jump tables automatically. `--switch-details` shows table info | `cfg.py binary.exe 0x401000 --format mermaid` | | `callgraph.py $B $VA` | Caller/callee tree (multi-level, --up/--down N). `--indirect` adds vtable/fptr calls to --down trees | `callgraph.py binary.exe 0x401000 --down 2 --indirect` | @@ -185,7 +185,7 @@ A proxy DLL that intercepts all 119 `IDirect3DDevice9` methods, capturing every ``` python -m graphics.directx.dx9.tracer codegen -o d3d9_trace_hooks.inc # C hooks (standalone proxy) -python -m graphics.directx.dx9.tracer codegen -f cpp -o tracer_dispatch.inc # C++ dispatch (remix-comp module) +python -m graphics.directx.dx9.tracer codegen -f cpp -o tracer_dispatch.inc # C++ dispatch (remix-comp-proxy module) cd graphics/directx/dx9/tracer/src && build.bat # build standalone proxy DLL # Deploy d3d9.dll + proxy.ini to game directory python -m graphics.directx.dx9.tracer trigger --game-dir # trigger capture (3s countdown) diff --git a/.github/agents/static-analyzer.agent.md b/.github/agents/static-analyzer.agent.md index a2a51331..ec2a6069 100644 --- a/.github/agents/static-analyzer.agent.md +++ b/.github/agents/static-analyzer.agent.md @@ -66,7 +66,7 @@ Always pass `--types ` to `decompiler.py` when a KB file exists for the **Read-First mutation discipline**: `kb-apply` mutates the Ghidra project. Decompile or `query` the target function first, run `kb-apply`, then re-decompile to verify the change landed. `kb-apply` is idempotent — re-running it must produce stable counts. -**Cost guard**: run `export` once per analysis pass (after `kb-apply`), not once per query. Warm `ghidra_server.py` for a project before a batch of decompiles. +**Cost guard**: run `export` once per analysis pass (after `kb-apply`), not once per query. Warm `ghidra_server.py` for a project before a batch of decompiles — it tracks itself in `patches//ghidra/.state.json`, deleted on shutdown once the Ghidra program is closed. ## Knowledge Base diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index ab1b9d6c..88b70b8a 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -42,7 +42,7 @@ Ghidra, via `pyghidra_backend.py`, is the **primary** decompilation backend once **Auto mode**: `python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h --project patches/proj` tries pyghidra first, falls back to r2ghidra. Use `--project` alongside `--types` for auto selection. This routing is unchanged. -**Index and daemon**: `pyghidra_backend.py export` seeds `funcs`/`names`/`xrefs`/`blocks` into `index.db` (`source='ghidra'`, overwrites provisional bootstrap rows at the same address). `pyghidra_backend.py kb-apply` pushes kb.h names/prototypes/globals into the Ghidra project (idempotent). `python -m retools.ghidra_server [--idle 600]` keeps one warm Ghidra program per project so repeat `decompile`/`export`/`kb-apply` calls become sub-second; `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold run. +**Index and daemon**: `pyghidra_backend.py export` seeds `funcs`/`names`/`xrefs`/`blocks` into `index.db` (`source='ghidra'`, overwrites provisional bootstrap rows at the same address). `pyghidra_backend.py kb-apply` pushes kb.h names/prototypes/globals into the Ghidra project (idempotent). `python -m retools.ghidra_server [--idle 600]` keeps one warm Ghidra program per project so repeat `decompile`/`export`/`kb-apply` calls become sub-second; `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold run. The daemon tracks itself in `patches//ghidra/.state.json` (pid/port/project/binary), deleted on shutdown once the Ghidra program is closed. **Dual-backend deep analysis**: reserve this for two cases — **no Ghidra project exists yet**, or **pyghidra output on a specific function looks wrong** and needs an independent r2ghidra cross-check. Run both backends in parallel on the same functions and merge findings: r2ghidra results go to `findings_r2.md`, pyghidra to `findings.md`. Not needed once a Ghidra project exists and `--backend auto` is available. diff --git a/.github/instructions/tool-catalog.instructions.md b/.github/instructions/tool-catalog.instructions.md index dd386b8b..d3d7ee09 100644 --- a/.github/instructions/tool-catalog.instructions.md +++ b/.github/instructions/tool-catalog.instructions.md @@ -110,7 +110,7 @@ These are fast first-pass scanners — they surface candidate addresses. Follow | `pyghidra_backend.py kb-apply $B --project $P --kb $KB` | Push kb.h names/prototypes/globals/typedefs into the Ghidra project. Idempotent — safe to re-run | `pyghidra_backend.py kb-apply game.exe --project patches/MyGame --kb patches/MyGame/kb.h` | | `index.py status $GAME [--db]` | Per-table row counts + schema_version for `patches//index.db` | `python -m retools.index status MyGame` | | `query.py $GAME "SQL" [--db] [--json] [--list-tables] [--schema TABLE]` | Read-only SQL over index.db. Views: `callers`, `callees`, `grep`. Addresses render as hex via `printf('0x%x', address)`. Connection is opened read-only — cannot mutate | `python -m retools.query MyGame "SELECT * FROM grep WHERE name LIKE '%Ground%'"` | -| `ghidra_server.py $GAME [--idle SECS]` | Run a per-project Ghidra daemon (port 27043; livetools owns 27042) holding one warm program so repeat `decompile`/`export`/`kb-apply` calls are sub-second instead of paying JVM+analysis startup each time | `python -m retools.ghidra_server MyGame --idle 600` | +| `ghidra_server.py $GAME [--idle SECS]` | Run a per-project Ghidra daemon (port 27043; livetools owns 27042) holding one warm program so repeat `decompile`/`export`/`kb-apply` calls are sub-second instead of paying JVM+analysis startup each time. Tracks itself in `patches//ghidra/.state.json` (pid/port/project/binary), deleted on shutdown once the Ghidra program is closed | `python -m retools.ghidra_server MyGame --idle 600` | | `funcinfo.py $B $VA` | Find function start/end, rets, calling convention, callees. Prefer `retools.query` on `funcs`/`blocks` when index.db exists | `funcinfo.py binary.exe 0x401000` | | `cfg.py $B $VA` | Control flow graph (basic blocks + edges, text or mermaid). Resolves MSVC switch/jump tables automatically. `--switch-details` shows table info | `cfg.py binary.exe 0x401000 --format mermaid` | | `callgraph.py $B $VA` | Caller/callee tree (multi-level, --up/--down N). `--indirect` adds vtable/fptr calls to --down trees | `callgraph.py binary.exe 0x401000 --down 2 --indirect` | @@ -184,7 +184,7 @@ A proxy DLL that intercepts all 119 `IDirect3DDevice9` methods, capturing every ``` python -m graphics.directx.dx9.tracer codegen -o d3d9_trace_hooks.inc # C hooks (standalone proxy) -python -m graphics.directx.dx9.tracer codegen -f cpp -o tracer_dispatch.inc # C++ dispatch (remix-comp module) +python -m graphics.directx.dx9.tracer codegen -f cpp -o tracer_dispatch.inc # C++ dispatch (remix-comp-proxy module) cd graphics/directx/dx9/tracer/src && build.bat # build standalone proxy DLL # Deploy d3d9.dll + proxy.ini to game directory python -m graphics.directx.dx9.tracer trigger --game-dir # trigger capture (3s countdown) diff --git a/.kiro/agents/static-analyzer.md b/.kiro/agents/static-analyzer.md index dd8573db..9aa6015e 100644 --- a/.kiro/agents/static-analyzer.md +++ b/.kiro/agents/static-analyzer.md @@ -64,7 +64,7 @@ python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h --pro When told to use a specific backend, use it. Otherwise prefer auto mode with both `--types` and `--project`. -**Ghidra daemon**: if `python -m retools.ghidra_server ` is running (port 27043; livetools owns 27042), `decompile`/`export`/`kb-apply` route through it automatically and repeat calls become sub-second. Warm the server yourself before a batch of decompiles on the same project: `python -m retools.ghidra_server --idle 600` (background it). `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold in-process run when you need to bypass the daemon. +**Ghidra daemon**: if `python -m retools.ghidra_server ` is running (port 27043; livetools owns 27042), `decompile`/`export`/`kb-apply` route through it automatically and repeat calls become sub-second. Warm the server yourself before a batch of decompiles on the same project: `python -m retools.ghidra_server --idle 600` (background it). `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold in-process run when you need to bypass the daemon. The daemon records its pid/port/project/binary in `patches//ghidra/.state.json`, deleted on shutdown once the Ghidra program is closed and the Windows `.rep` lock is released. ### Query-first workflow diff --git a/.kiro/steering/tool-catalog.md b/.kiro/steering/tool-catalog.md index aab90078..ad4e5a2c 100644 --- a/.kiro/steering/tool-catalog.md +++ b/.kiro/steering/tool-catalog.md @@ -111,7 +111,7 @@ These are fast first-pass scanners — they surface candidate addresses. Follow | `pyghidra_backend.py kb-apply $B --project $P --kb $KB` | Push kb.h names/prototypes/globals/typedefs into the Ghidra project. Idempotent — safe to re-run | `pyghidra_backend.py kb-apply game.exe --project patches/MyGame --kb patches/MyGame/kb.h` | | `index.py status $GAME [--db]` | Per-table row counts + schema_version for `patches//index.db` | `python -m retools.index status MyGame` | | `query.py $GAME "SQL" [--db] [--json] [--list-tables] [--schema TABLE]` | Read-only SQL over index.db. Views: `callers`, `callees`, `grep`. Addresses render as hex via `printf('0x%x', address)`. Connection is opened read-only — cannot mutate | `python -m retools.query MyGame "SELECT * FROM grep WHERE name LIKE '%Ground%'"` | -| `ghidra_server.py $GAME [--idle SECS]` | Run a per-project Ghidra daemon (port 27043; livetools owns 27042) holding one warm program so repeat `decompile`/`export`/`kb-apply` calls are sub-second instead of paying JVM+analysis startup each time | `python -m retools.ghidra_server MyGame --idle 600` | +| `ghidra_server.py $GAME [--idle SECS]` | Run a per-project Ghidra daemon (port 27043; livetools owns 27042) holding one warm program so repeat `decompile`/`export`/`kb-apply` calls are sub-second instead of paying JVM+analysis startup each time. Tracks itself in `patches//ghidra/.state.json` (pid/port/project/binary), deleted on shutdown once the Ghidra program is closed | `python -m retools.ghidra_server MyGame --idle 600` | | `funcinfo.py $B $VA` | Find function start/end, rets, calling convention, callees. Prefer `retools.query` on `funcs`/`blocks` when index.db exists | `funcinfo.py binary.exe 0x401000` | | `cfg.py $B $VA` | Control flow graph (basic blocks + edges, text or mermaid). Resolves MSVC switch/jump tables automatically. `--switch-details` shows table info | `cfg.py binary.exe 0x401000 --format mermaid` | | `callgraph.py $B $VA` | Caller/callee tree (multi-level, --up/--down N). `--indirect` adds vtable/fptr calls to --down trees | `callgraph.py binary.exe 0x401000 --down 2 --indirect` | @@ -185,7 +185,7 @@ A proxy DLL that intercepts all 119 `IDirect3DDevice9` methods, capturing every ``` python -m graphics.directx.dx9.tracer codegen -o d3d9_trace_hooks.inc # C hooks (standalone proxy) -python -m graphics.directx.dx9.tracer codegen -f cpp -o tracer_dispatch.inc # C++ dispatch (remix-comp module) +python -m graphics.directx.dx9.tracer codegen -f cpp -o tracer_dispatch.inc # C++ dispatch (remix-comp-proxy module) cd graphics/directx/dx9/tracer/src && build.bat # build standalone proxy DLL # Deploy d3d9.dll + proxy.ini to game directory python -m graphics.directx.dx9.tracer trigger --game-dir # trigger capture (3s countdown) From 1eec9337712419ba10937bb28efaf838808d5227 Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 14:56:10 -0500 Subject: [PATCH 20/28] fix: scan fallback for empty-xrefs index, guard kb-apply globals, broaden query errors --- retools/context.py | 5 +++++ retools/index.py | 6 +++++- retools/pyghidra_backend.py | 10 ++++++++-- retools/query.py | 2 +- tests/test_context.py | 19 +++++++++++++++++++ 5 files changed, 38 insertions(+), 4 deletions(-) diff --git a/retools/context.py b/retools/context.py index 23f7eca4..5e043299 100644 --- a/retools/context.py +++ b/retools/context.py @@ -101,6 +101,11 @@ def _callees_from_index(db_path: str, func_ea: int) -> list[tuple[int, str]] | N return None conn = GameIndex.open_ro(db_path) try: + # xrefs is exclusively ghidra-sourced (bootstrap never writes it), so a + # bootstrap-only index has an empty xrefs table -- fall back to scanning + # rather than trusting an authoritative-looking empty callees result. + if conn.execute("SELECT 1 FROM xrefs LIMIT 1").fetchone() is None: + return None rows = conn.execute( "SELECT x.to_ea, COALESCE(f.name, '') FROM xrefs x " "LEFT JOIN funcs f ON f.address = x.to_ea " diff --git a/retools/index.py b/retools/index.py index 0b5ac669..6c6e6d02 100644 --- a/retools/index.py +++ b/retools/index.py @@ -113,7 +113,11 @@ def replace(self, table: str, rows: list[dict], source: str) -> int: whose address column exceeds signed-64 range is skipped with a warning. If a row's address collides with a row from another source, it overwrites it (last writer wins), allowing authoritative sources (e.g. Ghidra) to replace - provisional data (e.g. bootstrap). + provisional data (e.g. bootstrap). This is a plain address-keyed upsert with + no source-priority check, so re-running bootstrap AFTER a ghidra export + overwrites the authoritative source='ghidra' funcs row with a provisional + source='bootstrap' row (name=NULL) at the same address; recovery is to + re-run export. Returns: Number of rows inserted. diff --git a/retools/pyghidra_backend.py b/retools/pyghidra_backend.py index db393744..7134b53d 100644 --- a/retools/pyghidra_backend.py +++ b/retools/pyghidra_backend.py @@ -144,6 +144,9 @@ def _route_daemon(game: str, cmd: dict): return None try: import ghidra_client + # Relative path: daemon routing only engages under the canonical + # patches//ghidra layout with cwd=repo root. A different cwd + # or layout silently falls back to the cold path (correct-but-slow). project_dir = str(Path("patches") / game / "ghidra") if not ghidra_client.is_daemon_alive(project_dir): return None @@ -387,8 +390,11 @@ def _kb_apply_program(program, kb, flat_api, *, apply_prototypes=True, apply_typ for g in kb.globals: addr = space.getAddress(g.address) - symtab.createLabel(addr, g.name, SourceType.USER_DEFINED) - counts["globals"] += 1 + try: + symtab.createLabel(addr, g.name, SourceType.USER_DEFINED) + counts["globals"] += 1 + except Exception: + pass # global name may be invalid; others still apply if apply_types: from ghidra.app.util.cparser.C import CParser diff --git a/retools/query.py b/retools/query.py index 1d09a8f6..a1966cd7 100644 --- a/retools/query.py +++ b/retools/query.py @@ -37,7 +37,7 @@ def run_query(conn: sqlite3.Connection, sql: str) -> dict: "elapsed_ms": round(elapsed, 3), "error": None, } - except sqlite3.OperationalError as e: + except (sqlite3.Error, sqlite3.Warning) as e: elapsed = (time.perf_counter() - t0) * 1000.0 return { "columns": [], diff --git a/tests/test_context.py b/tests/test_context.py index 91b72285..2c2a6a6d 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -350,3 +350,22 @@ def test_missing_index_returns_none(self, tmp_path): sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) from context import _callees_from_index assert _callees_from_index(str(tmp_path / "absent.db"), 0x1000) is None + + def test_empty_xrefs_returns_none(self, tmp_path): + """Bootstrap-only projects write funcs but never xrefs (ghidra-only). + + _callees_from_index must fall back to None (scan) rather than + trusting an empty callees query result as authoritative. + """ + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) + from index import GameIndex + from context import _callees_from_index + + db = str(tmp_path / "index.db") + gi = GameIndex(db) + gi.replace("funcs", [{"address": 0x1000, "name": "SomeFunc"}], source="bootstrap") + gi.close() + + assert _callees_from_index(db, 0x1000) is None From 90faf6a1de004c6082f6ae177c1bd276256b9ede Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Thu, 16 Jul 2026 18:28:59 -0500 Subject: [PATCH 21/28] fix: resolve whole-branch review findings + kb-apply transaction bug Addresses the findings from the branch-wide review and a real bug found running the full Ghidra path on a real binary (Warband mb_warband.exe). Correctness - ghidra daemon: reject cross-project routing. is_daemon_alive checks the recorded pid before the shared port, and the server validates the game per command (wrong_project -> client cold-falls-back); dropped SO_REUSEADDR so a second daemon can't silently bind an occupied port. - pyghidra export: default --db derives from --project, not the binary stem (was writing patches//index.db); _iter_blocks keys blocks by the containing function entry, not the block's own start. - _route_daemon: a socket.timeout now propagates instead of silently cold- retrying into a daemon mid-operation (no double-execution); sends absolute binary/db/kb paths so the daemon can't resolve them against a different cwd. - context.assemble: verify the queried start is a known function before trusting an empty callee result, fall back to the scan on a schemaless index.db (was crashing), recover indirect-call markers, and let a fresh kb.h name outrank the exported Ghidra name. - index.db path anchoring: bootstrap (writer) and context (reader) now resolve the same /index.db from the same --project. - index schema-version: repair a missing version row on open instead of leaving the DB permanently versionless after an interrupted first open. - decompiler.py: put repo root on sys.path so `python retools/decompiler.py` resolves its package imports (script-mode invocation was broken). - kb-apply (found on real Ghidra): remove the manual program.save() that fails with "Unable to lock due to active transaction" inside pyghidra's open_program context; persist via pyghidra's project.save on context exit (cold) / by closing the program (daemon). @ entries at data addresses (RTTI vtables, string refs) now become labels, not bogus functions. Cleanup / conventions - index: add ix_xrefs_from_func; source-priority upsert so re-running bootstrap can't downgrade authoritative Ghidra funcs; narrow catch-alls in GameIndex.close and ghidra_client.read_state. - daemon: single-threaded accept loop (commands already serialize), removing the lock, the close-variant split, the unbounded thread list, and the idle self-connect; reuse ghidra_client's framing; answer malformed frames with an error instead of a silent pass; cache the DecompInterface for warm repeats. - context: parse kb.h once per assemble. - query/index: share resolve_db so both CLIs print the same guidance. - bootstrap: one string sweep feeds both the error-string KB seed and the index seed. - extract _kb_apply_txn shared by cold + daemon paths. Tests: 123 passed / 1 skipped across all changed modules, including new coverage for the daemon lifecycle, identity handshake, source-priority, empty-xrefs fallback, and the kb-apply label path. Co-Authored-By: Claude Opus 4.8 --- retools/bootstrap.py | 54 ++++++--- retools/context.py | 57 ++++++--- retools/decompiler.py | 6 + retools/ghidra_client.py | 16 ++- retools/ghidra_server.py | 147 +++++++++++------------- retools/index.py | 111 ++++++++++++------ retools/kb.py | 19 ++- retools/pyghidra_backend.py | 141 ++++++++++++++++------- retools/query.py | 6 +- tests/test_bootstrap.py | 23 ++++ tests/test_context.py | 81 ++++++++++++- tests/test_decompiler.py | 27 +++++ tests/test_ghidra_server.py | 132 +++++++++++++++++++++ tests/test_index.py | 58 ++++++++++ tests/test_kb.py | 21 ++++ tests/test_pyghidra_backend.py | 204 ++++++++++++++++++++++++++++++++- 16 files changed, 887 insertions(+), 216 deletions(-) create mode 100644 tests/test_ghidra_server.py diff --git a/retools/bootstrap.py b/retools/bootstrap.py index e495c33d..2e11d810 100644 --- a/retools/bootstrap.py +++ b/retools/bootstrap.py @@ -226,27 +226,33 @@ def _analyze_imports(b: Binary) -> list: return [] -def _seed_strings(b: Binary) -> tuple[int, list[str]]: +_ERROR_KEYWORDS = [ + "error", "fail", "assert", "fatal", "exception", + "invalid", "corrupt", "abort", "panic", "warning", +] + + +def _seed_strings(all_strings: list) -> tuple[int, list[str]]: """Seed KB with error/diagnostic string references. + Filters a single precomputed string sweep rather than re-scanning: an + error string is any string of length >= 6 containing a keyword, which is + exactly what ``find_strings(filter_keywords=..., min_len=6)`` would return + because the sweep yields maximal printable runs. + Args: - b: Loaded Binary instance. + all_strings: Result of one ``find_strings(b, min_len=4)`` sweep. Returns: - (string count, list of KB entry strings). + (matched string count, list of KB entry strings). """ - error_keywords = [ - "error", "fail", "assert", "fatal", "exception", - "invalid", "corrupt", "abort", "panic", "warning", + matched = [ + s for s in all_strings + if len(s.value) >= 6 and any(kw in s.value.lower() for kw in _ERROR_KEYWORDS) ] - try: - from search import find_strings - strings = find_strings(b, filter_keywords=error_keywords, min_len=6) - except (ImportError, ValueError): - return 0, [] kb_entries = [] - for sref in strings: + for sref in matched: if sref.va is None: continue safe_str = sref.value[:80].replace("*/", "* /") @@ -254,7 +260,7 @@ def _seed_strings(b: Binary) -> tuple[int, list[str]]: label = re.sub(r"[^A-Za-z0-9_]", "_", sref.value[:40]).strip("_") if label: kb_entries.append(f"{comment}\n@ 0x{sref.va:X} str_{label};") - return len(strings), kb_entries + return len(matched), kb_entries def _propagate_labels( @@ -316,14 +322,18 @@ def _propagate_labels( return kb_entries -def _seed_index(b: Binary, db_path: str) -> None: +def _seed_index(b: Binary, db_path: str, strings: list | None = None) -> None: """Populate index.db from already-computed pefile data (source='bootstrap'). Never raises out to the caller; the index is a convenience, not a - prerequisite for a successful bootstrap. + prerequisite for a successful bootstrap. *strings* reuses the caller's + single ``find_strings`` sweep; when None it sweeps once itself. """ from index import GameIndex - from search import find_strings + + if strings is None: + from search import find_strings + strings = find_strings(b, min_len=4) gi = GameIndex(db_path) try: @@ -372,7 +382,7 @@ def _seed_index(b: Binary, db_path: str) -> None: # strings str_rows = [] - for sref in find_strings(b, min_len=4): + for sref in strings: if sref.va is None: continue str_rows.append({ @@ -472,7 +482,13 @@ def bootstrap( imports = _analyze_imports(b) stats["imports"] = len(imports) - string_count, string_entries = _seed_strings(b) + # One string sweep feeds both the error-string KB seed and the index seed. + try: + from search import find_strings + all_strings = find_strings(b, min_len=4) + except (ImportError, ValueError): + all_strings = [] + string_count, string_entries = _seed_strings(all_strings) stats["strings_seeded"] = string_count all_entries = sig_entries + rtti_entries + string_entries @@ -540,7 +556,7 @@ def bootstrap( # -- Seed the per-game index (best-effort; never breaks bootstrap) ------ try: - _seed_index(b, os.path.join(project_dir, "index.db")) + _seed_index(b, os.path.join(project_dir, "index.db"), all_strings) except Exception as e: # index is a convenience, not a prerequisite print(f"index seeding skipped: {e}", file=sys.stderr) diff --git a/retools/context.py b/retools/context.py index 5e043299..ea8b4e1d 100644 --- a/retools/context.py +++ b/retools/context.py @@ -15,12 +15,13 @@ import argparse import re +import sqlite3 import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from common import Binary -from kb import parse_kb +from kb import parse_kb, Kb from funcinfo import find_start, analyze from structrefs import aggregate_struct from search import find_strings @@ -95,24 +96,32 @@ def postprocess(raw_output: str, kb_names: dict[int, str], def _callees_from_index(db_path: str, func_ea: int) -> list[tuple[int, str]] | None: """Resolve (callee_addr, name) pairs for a function from index.db. - Returns None when the index is absent so the caller falls back to scanning. + Returns None (so the caller falls back to a disassembly scan) when the index + is absent, has no Ghidra xrefs, is missing/foreign-schema'd, or does not know + *func_ea* as a function entry -- in the last case an empty result would mean + "find_start disagreed with Ghidra", not "this function calls nothing", and + must not be trusted. """ if not Path(db_path).is_file(): return None conn = GameIndex.open_ro(db_path) try: - # xrefs is exclusively ghidra-sourced (bootstrap never writes it), so a - # bootstrap-only index has an empty xrefs table -- fall back to scanning - # rather than trusting an authoritative-looking empty callees result. - if conn.execute("SELECT 1 FROM xrefs LIMIT 1").fetchone() is None: - return None - rows = conn.execute( - "SELECT x.to_ea, COALESCE(f.name, '') FROM xrefs x " - "LEFT JOIN funcs f ON f.address = x.to_ea " - "WHERE x.from_func = ? AND x.is_code = 1 AND x.type = 'call' " - "GROUP BY x.to_ea ORDER BY x.to_ea", - (func_ea,), - ).fetchall() + try: + has_xrefs = conn.execute("SELECT 1 FROM xrefs LIMIT 1").fetchone() is not None + known = conn.execute( + "SELECT 1 FROM funcs WHERE address = ? LIMIT 1", (func_ea,) + ).fetchone() is not None + if not has_xrefs or not known: + return None + rows = conn.execute( + "SELECT x.to_ea, COALESCE(f.name, '') FROM xrefs x " + "LEFT JOIN funcs f ON f.address = x.to_ea " + "WHERE x.from_func = ? AND x.is_code = 1 AND x.type = 'call' " + "GROUP BY x.to_ea ORDER BY x.to_ea", + (func_ea,), + ).fetchall() + except sqlite3.OperationalError: + return None # half-initialised or foreign schema -> scan instead finally: conn.close() return [(int(a), n) for a, n in rows] @@ -154,10 +163,11 @@ def assemble(b: Binary, va: int, project_dir: str, db_path: str | None = None, start = find_start(b, va) or va lines: list[str] = [f"=== CONTEXT FOR 0x{start:0{w}X} ==="] - # -- KB lookup -- + # -- KB lookup (parse once, derive both maps) -- kb_path = _find_kb_path(project_dir, project_dir_for_kb) - kb_names = _parse_kb_names(kb_path) - kb_globals = _parse_kb_globals(kb_path) + kb = parse_kb(kb_path) if kb_path.is_file() else Kb() + kb_names = {f.address: f.name for f in kb.functions if f.name} + kb_globals = {g.address: g.name for g in kb.globals if g.name} # -- Identity -- if start in kb_names: @@ -168,11 +178,20 @@ def assemble(b: Binary, va: int, project_dir: str, db_path: str | None = None, # -- Callees (index fast-path, else scan) -- rets, calls, end_va = analyze(b, start, max_size=0x2000) lines.append("[callees]") - db_index = GameIndex.default_db_path(Path(project_dir).name) + db_index = GameIndex.project_db_path(project_dir) indexed = _callees_from_index(db_index, start) if indexed is not None: + # kb.h is the hand-edited source of truth, so a kb name outranks the + # exported Ghidra name (which may be a stale FUN_ auto-name). for target, name in indexed: - lines.append(f" 0x{target:0{w}X}: {name or kb_names.get(target, 'unknown')}") + lines.append(f" 0x{target:0{w}X}: {kb_names.get(target) or name or 'unknown'}") + # Ghidra emits no xref row for unresolved indirect calls, so recover + # those markers from the scan to keep virtual dispatch visible. + seen_indirect: set[str] = set() + for _, target in calls: + if isinstance(target, str) and target not in seen_indirect: + seen_indirect.add(target) + lines.append(f" {target}: indirect call") else: seen_targets: set[int | str] = set() for _, target in calls: diff --git a/retools/decompiler.py b/retools/decompiler.py index 53496525..13c6be41 100644 --- a/retools/decompiler.py +++ b/retools/decompiler.py @@ -49,6 +49,12 @@ _HERE = Path(__file__).resolve().parent _PROJECT = _HERE.parent +# Keep the repo root importable so `python retools/decompiler.py` (which puts +# only retools/ on sys.path) can still resolve the `retools.*` package imports +# used below, matching the `-m retools.decompiler` invocation. +if str(_PROJECT) not in sys.path: + sys.path.insert(0, str(_PROJECT)) + _BACKEND_CMDS = { "pdg": "pdg", # r2ghidra – best quality "pdc": "pdc", # r2 built-in pseudo-C diff --git a/retools/ghidra_client.py b/retools/ghidra_client.py index 08f7e28d..054fe502 100644 --- a/retools/ghidra_client.py +++ b/retools/ghidra_client.py @@ -11,6 +11,7 @@ import os import socket import struct +from json import JSONDecodeError from pathlib import Path HOST = "127.0.0.1" @@ -28,7 +29,7 @@ def read_state(project_dir: str) -> dict | None: return None try: return json.loads(p.read_text()) - except Exception: + except (OSError, JSONDecodeError): return None @@ -52,16 +53,25 @@ def _pid_alive(pid: int | None) -> bool: def is_daemon_alive(project_dir: str) -> bool: + """Whether this project's own daemon is running. + + The recorded pid is checked first: all projects share one port, so a hard + kill can leave a stale state file while a *different* project's daemon holds + the port. Verifying the pid before connecting stops this project's commands + from being routed into that foreign daemon; a dead pid also prunes the + stale state file. + """ state = read_state(project_dir) if state is None: return False + if not _pid_alive(state.get("pid")): + state_path(project_dir).unlink(missing_ok=True) + return False try: s = socket.create_connection((HOST, state.get("port", PORT)), timeout=2) s.close() return True except OSError: - if not _pid_alive(state.get("pid")): - state_path(project_dir).unlink(missing_ok=True) return False diff --git a/retools/ghidra_server.py b/retools/ghidra_server.py index a0c34f28..a10b84be 100644 --- a/retools/ghidra_server.py +++ b/retools/ghidra_server.py @@ -1,9 +1,13 @@ """Per-project Ghidra daemon holding one warm open_program handle. -Cloned from livetools/server.py. Binds 127.0.0.1:27043, 4-byte big-endian -length-prefixed JSON, dict-dispatched _cmd_*, idle-timeout thread. Keeps the +Binds 127.0.0.1:27043, 4-byte big-endian length-prefixed JSON (framing shared +with ghidra_client), dict-dispatched _cmd_*, idle-timeout thread. Keeps the program open so repeat decompiles skip the ~3s JVM cold start. +Connections are served one at a time on the accept loop -- Ghidra's program and +decompiler are single-threaded and every command mutates or reads that one warm +handle, so there is nothing to gain from concurrency and no lock to reason about. + Usage: python -m retools.ghidra_server [--idle 600] """ @@ -15,14 +19,13 @@ import os import signal import socket -import struct import sys -import threading import time +from json import JSONDecodeError from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) -from ghidra_client import HOST, PORT, state_path +from ghidra_client import HOST, PORT, state_path, _recv_raw, _send_raw import pyghidra_backend as pb _PROJECT = Path(__file__).resolve().parent.parent @@ -33,15 +36,15 @@ def __init__(self, game: str, idle: float = 600.0): self.game = game self.project_dir = str(_PROJECT / "patches" / game / "ghidra") self.binary = None + self.port = PORT self.idle = idle self._pyghidra = None self._ctx = None # open_program context manager self._flat_api = None self._program = None + self._decomp = None # warm DecompInterface for the open program self._running = True self._last_activity = time.monotonic() - self._lock = threading.Lock() - self._conn_threads = [] # -- program lifecycle --------------------------------------------------- @@ -60,20 +63,30 @@ def _open(self, binary: str) -> dict: return {"ok": True, "binary": binary} def _ensure_open(self, binary: str) -> dict: - """Open *binary*, swapping out a warm program for a different one. - - Callers run inside ``handle()``'s lock, so a binary switch closes the - stale program via the no-lock ``_close_program_locked`` (the - lock-acquiring ``_close_program`` would deadlock here). - """ + """Open *binary*, swapping out a warm program for a different one.""" if self._program is not None and self.binary != binary: - self._close_program_locked() + self._close_program() if self._program is None: return self._open(binary) return {"ok": True, "binary": self.binary, "already": True} - def _close_program_locked(self) -> None: - """Tear down the open program. Caller must already hold ``self._lock``.""" + def _decompiler(self): + """Return the warm DecompInterface, opening it against the program once.""" + if self._decomp is None: + from ghidra.app.decompiler import DecompInterface, DecompileOptions + ifc = DecompInterface() + ifc.setOptions(DecompileOptions()) + ifc.openProgram(self._program) + self._decomp = ifc + return self._decomp + + def _close_program(self) -> None: + if self._decomp is not None: + try: + self._decomp.dispose() + except Exception: + pass + self._decomp = None if self._ctx is not None: try: self._ctx.__exit__(None, None, None) @@ -81,19 +94,6 @@ def _close_program_locked(self) -> None: pass self._ctx = self._flat_api = self._program = None - def _close_program(self) -> None: - """Tear down the open program, waiting for any in-flight command first. - - Acquires ``self._lock`` so this cannot run concurrently with a - command still executing inside ``handle()`` -- otherwise a - long-running decompile could be closed out from under it (crash / - ``.rep`` corruption). Only call this from outside ``handle()`` - (e.g. ``_cleanup``); command handlers already hold the lock and - must use ``_close_program_locked`` directly. - """ - with self._lock: - self._close_program_locked() - # -- commands ------------------------------------------------------------ def _cmd_status(self, cmd): @@ -108,7 +108,7 @@ def _cmd_decompile(self, cmd): if not r.get("ok"): return r va = int(cmd["va"]) - return {"ok": True, "text": pb._decompile_open(self._program, va)} + return {"ok": True, "text": pb._decompile_open(self._program, va, self._decompiler())} def _cmd_export(self, cmd): r = self._ensure_open(cmd["binary"]) @@ -128,20 +128,19 @@ def _cmd_kb_apply(self, cmd): return r from kb import parse_kb kb = parse_kb(Path(cmd["kb"])) - txn = self._program.startTransaction("kb_apply") - try: - counts = pb._kb_apply_program(self._program, kb, self._flat_api) - finally: - self._program.endTransaction(txn, True) - self._program.save("kb_apply", None) + counts = pb._kb_apply_txn(self._program, kb) + # Persist by closing the program: pyghidra's open_program runs + # project.save(program) on exit (an explicit save mid-context is + # rejected). The next command reopens the saved program. + self._close_program() return {"ok": True, "counts": counts} def _cmd_close(self, cmd): - self._close_program_locked() + self._close_program() return {"ok": True} def _cmd_shutdown(self, cmd): - self._close_program_locked() + self._close_program() self._running = False return {"ok": True} @@ -149,41 +148,44 @@ def _cmd_shutdown(self, cmd): def handle(self, cmd: dict) -> dict: self._last_activity = time.monotonic() + game = cmd.get("game") + if game is not None and game != self.game: + return {"ok": False, "wrong_project": True, + "error": f"daemon serves '{self.game}', not '{game}'"} op = cmd.get("cmd", "") handler = getattr(self, f"_cmd_{op}", None) if handler is None: return {"ok": False, "error": f"unknown command: {op}"} - with self._lock: - try: - return handler(cmd) - except Exception as exc: - return {"ok": False, "error": str(exc)} + try: + return handler(cmd) + except Exception as exc: + return {"ok": False, "error": str(exc)} def _idle_watch(self): while self._running: time.sleep(5) if time.monotonic() - self._last_activity > self.idle: - self._running = False - try: - socket.create_connection((HOST, PORT), timeout=1).close() # unblock accept - except OSError: - pass + self._running = False # accept() polls _running every second return def serve(self): srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # No SO_REUSEADDR: a second daemon must fail to bind an occupied port + # rather than silently sharing it and corrupting the other's project. srv.bind((HOST, PORT)) srv.listen(4) srv.settimeout(1.0) + self.port = srv.getsockname()[1] sp = state_path(self.project_dir) sp.parent.mkdir(parents=True, exist_ok=True) sp.write_text(json.dumps({ - "pid": os.getpid(), "port": PORT, "project": self.game, + "pid": os.getpid(), "port": self.port, "project": self.game, "binary": self.binary, "started": int(time.time()), })) - print(f"[ghidra daemon] listening on {HOST}:{PORT}, project={self.game}") + print(f"[ghidra daemon] listening on {HOST}:{self.port}, project={self.game}") + + import threading threading.Thread(target=self._idle_watch, daemon=True).start() try: @@ -194,43 +196,32 @@ def serve(self): continue except OSError: break - t = threading.Thread(target=self._handle_conn, args=(conn,), daemon=True) - t.start() - self._conn_threads.append(t) + self._handle_conn(conn) finally: srv.close() - # Drain in-flight connection handlers before tearing down the - # program; _close_program's lock acquisition is the actual - # correctness guarantee below, this just avoids leaving threads - # dangling on a closed socket. - for t in self._conn_threads: - t.join(timeout=5.0) self._cleanup() def _handle_conn(self, conn): try: conn.settimeout(300) - hdr = b"" - while len(hdr) < 4: - chunk = conn.recv(4 - len(hdr)) - if not chunk: - return - hdr += chunk - length = struct.unpack("!I", hdr)[0] - buf = b"" - while len(buf) < length: - chunk = conn.recv(min(length - len(buf), 1 << 20)) - if not chunk: - return - buf += chunk - resp = self.handle(json.loads(buf)) - data = json.dumps(resp).encode() - conn.sendall(struct.pack("!I", len(data)) + data) - except Exception: - pass + try: + cmd = json.loads(_recv_raw(conn)) + except (ConnectionError, OSError, JSONDecodeError, ValueError) as exc: + # Malformed/short frame: answer with an error so the client sees + # a failure instead of hanging until its own timeout. + self._try_send(conn, {"ok": False, "error": f"bad request: {exc}"}) + return + self._try_send(conn, self.handle(cmd)) finally: conn.close() + @staticmethod + def _try_send(conn, resp): + try: + _send_raw(conn, json.dumps(resp).encode()) + except OSError: + pass # client already gone; nothing to deliver + def _cleanup(self): self._close_program() state_path(self.project_dir).unlink(missing_ok=True) diff --git a/retools/index.py b/retools/index.py index 6c6e6d02..32852f66 100644 --- a/retools/index.py +++ b/retools/index.py @@ -34,6 +34,7 @@ from_func INTEGER, source TEXT NOT NULL); CREATE INDEX IF NOT EXISTS ix_xrefs_to ON xrefs(to_ea); CREATE INDEX IF NOT EXISTS ix_xrefs_from ON xrefs(from_ea); +CREATE INDEX IF NOT EXISTS ix_xrefs_from_func ON xrefs(from_func); CREATE TABLE IF NOT EXISTS strings (address INTEGER PRIMARY KEY, length INTEGER, type TEXT, encoding TEXT, content TEXT, source TEXT NOT NULL); CREATE TABLE IF NOT EXISTS imports (address INTEGER, name TEXT, module TEXT, ordinal INTEGER, source TEXT NOT NULL); @@ -68,11 +69,27 @@ _MAX_ADDR = (1 << 63) - 1 +# Authoritative producers outrank provisional ones at the same address. A row +# never overwrites a row written by a higher-priority source, so re-running +# bootstrap after a Ghidra export cannot downgrade a named Ghidra function back +# to a provisional (name=NULL) bootstrap row. Unlisted sources rank 0. +_SOURCE_PRIORITY = {"bootstrap": 0, "ghidra": 10} + +# Tables whose single-address primary key can collide across sources, so their +# inserts must respect _SOURCE_PRIORITY. Append-only tables (xrefs, imports, +# entries, segments, blocks) never collide destructively and are excluded. +_PRIORITY_KEY = {"funcs": "address"} + def _is_addr_col(col: str) -> bool: return col == "address" or col.endswith("_ea") or col == "from_func" +def _priority_case(ref: str) -> str: + whens = " ".join(f"WHEN '{s}' THEN {p}" for s, p in _SOURCE_PRIORITY.items()) + return f"CASE {ref}.source {whens} ELSE 0 END" + + class GameIndex: """SQLite-backed per-game analysis index. @@ -89,35 +106,34 @@ def __init__(self, path: str = ":memory:"): self._init_schema() def _init_schema(self) -> None: - cur = self._conn.execute( - "SELECT name FROM sqlite_master WHERE type='table' AND name='schema_version'" - ) - if cur.fetchone() is not None: - row = self._conn.execute("SELECT version FROM schema_version").fetchone() - if row and row[0] > SCHEMA_VERSION: - self._conn.close() - self._conn = None - raise RuntimeError( - f"Index schema version {row[0]} is newer than code version " - f"{SCHEMA_VERSION}. Update the code." - ) - return + # The DDL is fully idempotent (CREATE ... IF NOT EXISTS), so running it + # unconditionally also repairs a schema left partial by an interrupted + # first open. The version row is written separately and may be missing if + # a crash landed between the DDL and its INSERT -- treat missing as fresh. self._conn.executescript(_SCHEMA_SQL) - self._conn.execute("INSERT INTO schema_version (version) VALUES (?)", (SCHEMA_VERSION,)) - self._conn.commit() + row = self._conn.execute("SELECT version FROM schema_version").fetchone() + if row is None: + self._conn.execute("INSERT INTO schema_version (version) VALUES (?)", (SCHEMA_VERSION,)) + self._conn.commit() + elif row[0] > SCHEMA_VERSION: + self._conn.close() + self._conn = None + raise RuntimeError( + f"Index schema version {row[0]} is newer than code version " + f"{SCHEMA_VERSION}. Update the code." + ) def replace(self, table: str, rows: list[dict], source: str) -> int: """Replace all *source* rows in *table* with *rows*, transactionally. Rows are dicts keyed by column name (``source`` is injected). Any row whose address column exceeds signed-64 range is skipped with a warning. - If a row's address collides with a row from another source, it overwrites it - (last writer wins), allowing authoritative sources (e.g. Ghidra) to replace - provisional data (e.g. bootstrap). This is a plain address-keyed upsert with - no source-priority check, so re-running bootstrap AFTER a ghidra export - overwrites the authoritative source='ghidra' funcs row with a provisional - source='bootstrap' row (name=NULL) at the same address; recovery is to - re-run export. + + Cross-source collisions on a single-address key (see ``_PRIORITY_KEY``) + respect ``_SOURCE_PRIORITY``: a row never overwrites one from a + higher-priority source. So Ghidra's export replaces provisional bootstrap + rows, but a later bootstrap re-run leaves the authoritative Ghidra rows + intact. Returns: Number of rows inserted. @@ -147,7 +163,16 @@ def replace(self, table: str, rows: list[dict], source: str) -> int: file=sys.stderr) placeholders = ",".join("?" * len(cols)) - insert_sql = f"INSERT OR REPLACE INTO {table} ({','.join(cols)}) VALUES ({placeholders})" + key = _PRIORITY_KEY.get(table) + if key is not None: + set_clause = ", ".join(f"{c}=excluded.{c}" for c in cols if c != key) + insert_sql = ( + f"INSERT INTO {table} ({','.join(cols)}) VALUES ({placeholders}) " + f"ON CONFLICT({key}) DO UPDATE SET {set_clause} " + f"WHERE {_priority_case('excluded')} >= {_priority_case(table)}" + ) + else: + insert_sql = f"INSERT OR REPLACE INTO {table} ({','.join(cols)}) VALUES ({placeholders})" with self._conn: # single transaction self._conn.execute(f"DELETE FROM {table} WHERE source=?", (source,)) self._conn.executemany(insert_sql, tuples) @@ -161,10 +186,7 @@ def counts(self) -> dict[str, int]: def close(self) -> None: if self._conn is not None: - try: - self._conn.close() - except Exception: - pass + self._conn.close() self._conn = None @classmethod @@ -175,8 +197,35 @@ def open_ro(cls, path: str) -> sqlite3.Connection: @staticmethod def default_db_path(game: str) -> str: + """Canonical index.db for a game *name* (used by the name-only CLIs).""" return str(_PROJECT / "patches" / game / "index.db") + @staticmethod + def resolve_db(game: str, db: str | None = None) -> str: + """Resolve a game's index.db for the read CLIs, or exit with guidance. + + Shared by the ``index status`` and ``query`` front-ends so both print the + same message and honour ``--db`` identically. + """ + db_path = db or GameIndex.default_db_path(game) + if not Path(db_path).is_file(): + print(f"[error] no index at {db_path}. Run bootstrap or 'pyghidra_backend export' first.", + file=sys.stderr) + raise SystemExit(1) + return db_path + + @staticmethod + def project_db_path(project_dir: str) -> str: + """index.db location for a project *directory*. + + The index always lives beside kb.h in the project dir, so bootstrap + (writer), export (writer), and context (reader) resolve the same file + from the same ``--project`` value regardless of cwd. Equivalent to + ``default_db_path`` when the project dir is the canonical + ``patches/``. + """ + return str(Path(project_dir) / "index.db") + def main(argv: list[str] | None = None) -> None: p = argparse.ArgumentParser(prog="retools.index", description="Per-game SQLite index") @@ -186,13 +235,9 @@ def main(argv: list[str] | None = None) -> None: s.add_argument("--db", default=None, help="Explicit index.db path") args = p.parse_args(argv) - db_path = args.db or GameIndex.default_db_path(args.game) - if not Path(db_path).is_file(): - print(f"[error] no index at {db_path}. Run bootstrap or 'pyghidra_backend export' first.", - file=sys.stderr) - raise SystemExit(1) + db_path = GameIndex.resolve_db(args.game, args.db) - gi = GameIndex(db_path) + gi = GameIndex(db_path) # repairs a missing version row on open ver = gi._conn.execute("SELECT version FROM schema_version").fetchone()[0] counts = gi.counts() gi.close() diff --git a/retools/kb.py b/retools/kb.py index 197e2f9b..60f29458 100644 --- a/retools/kb.py +++ b/retools/kb.py @@ -49,18 +49,17 @@ def extract_function_name(sig: str) -> str: return pre.rsplit(None, 1)[-1].lstrip("*&") -def _resolve_text(text_or_path: str | Path) -> str: - if isinstance(text_or_path, Path): - return text_or_path.read_text(encoding="utf-8", errors="replace") - if os.path.isfile(text_or_path): - return Path(text_or_path).read_text(encoding="utf-8", errors="replace") - return text_or_path - - def parse_kb(text_or_path: str | Path) -> Kb: - """Parse kb.h content (a string) or a path to a kb.h file.""" + """Parse kb.h content or a file. + + A ``Path`` is read from disk; a ``str`` is always treated as content. Callers + that hold a filesystem path pass a ``Path`` so a one-line kb string is never + mistaken for a filename. + """ + text = (text_or_path.read_text(encoding="utf-8", errors="replace") + if isinstance(text_or_path, Path) else text_or_path) kb = Kb() - for raw in _resolve_text(text_or_path).splitlines(): + for raw in text.splitlines(): line = raw.strip() if not line or line.startswith("//"): continue diff --git a/retools/pyghidra_backend.py b/retools/pyghidra_backend.py index 7134b53d..33aac1d7 100644 --- a/retools/pyghidra_backend.py +++ b/retools/pyghidra_backend.py @@ -7,6 +7,7 @@ import argparse import os import shutil +import socket import sys import time from pathlib import Path @@ -129,44 +130,69 @@ def analyze(binary: str, project_dir: str) -> str: # daemon routing # --------------------------------------------------------------------------- -def _route_daemon(game: str, cmd: dict): - """Return the daemon response dict if a live daemon serves *game*, else None. +def _route_daemon(project_dir: str, cmd: dict): + """Return the daemon response if a live daemon serves *project_dir*, else None. - A pure no-op (returns None) whenever no live daemon is reachable, so - callers fall through to the cold in-process path unchanged. The probe - and send are wrapped in one broad except so a missing ghidra_client - module, a dead/absent daemon, a corrupted state file (e.g. a - wrong-typed "port" raising inside is_daemon_alive), or a transport - error during send_command all degrade to the cold path instead of - raising. + *project_dir* is the caller's own Ghidra project directory + (``patches//ghidra``), so routing works from any cwd and honours an + absolute ``--project`` -- the daemon's state file lives there. The command + is tagged with the game name so the daemon can reject a stale state file + that points at another project's daemon (returns None -> cold path). + + Failure handling is deliberately split: a missing client module, a corrupt + state file, or a daemon that is not reachable soft-fails to None (cold + path, nothing ran). But once the command has been sent, a ``socket.timeout`` + may mean the daemon is still executing it, so that error propagates rather + than triggering a cold re-run that would double-execute a mutating command. """ if os.environ.get("RETOOLS_GHIDRA_COLD") == "1": return None try: import ghidra_client - # Relative path: daemon routing only engages under the canonical - # patches//ghidra layout with cwd=repo root. A different cwd - # or layout silently falls back to the cold path (correct-but-slow). - project_dir = str(Path("patches") / game / "ghidra") - if not ghidra_client.is_daemon_alive(project_dir): - return None - return ghidra_client.send_command(project_dir, cmd, timeout=120) + except ImportError: + return None + try: + alive = ghidra_client.is_daemon_alive(project_dir) except Exception: + return None # unreadable/corrupt state file -> no reachable daemon + if not alive: return None + game = Path(project_dir).parent.name # patches//ghidra -> + # The daemon resolves paths against its own cwd, which need not match the + # client's, so send absolute paths (resolved here, where the user invoked). + cmd = {**cmd, "game": game} + for k in ("binary", "db", "kb"): + if cmd.get(k) is not None: + cmd[k] = str(Path(cmd[k]).resolve()) + try: + resp = ghidra_client.send_command(project_dir, cmd, timeout=120) + except socket.timeout: + raise + except OSError: + return None # daemon vanished before doing the work -> safe cold path + if resp.get("wrong_project"): + return None # stale state pointed at another project's daemon + return resp # --------------------------------------------------------------------------- # decompile # --------------------------------------------------------------------------- -def _decompile_open(program, va: int) -> str: - """Decompile the function containing *va* in an already-open program.""" +def _decompile_open(program, va: int, ifc=None) -> str: + """Decompile the function containing *va* in an already-open program. + + *ifc* lets a caller (the daemon) pass a warm DecompInterface so repeat + decompiles reuse one native decompiler process; when None a throwaway one + is built for this single call. + """ from ghidra.app.decompiler import DecompInterface, DecompileOptions from ghidra.util.task import ConsoleTaskMonitor - ifc = DecompInterface() - ifc.setOptions(DecompileOptions()) - ifc.openProgram(program) + if ifc is None: + ifc = DecompInterface() + ifc.setOptions(DecompileOptions()) + ifc.openProgram(program) addr = program.getAddressFactory().getDefaultAddressSpace().getAddress(va) func = program.getListing().getFunctionContaining(addr) if func is None: @@ -197,8 +223,7 @@ def decompile(project_dir: str, binary: str, va: int) -> str: if not is_analyzed(project_dir, binary_name): return f"[error] no analyzed project for {binary_name} in {project_dir}" - game = Path(project_dir).parent.name # patches//ghidra -> - routed = _route_daemon(game, {"cmd": "decompile", "binary": binary, "va": va}) + routed = _route_daemon(project_dir, {"cmd": "decompile", "binary": binary, "va": va}) if routed is not None: return routed.get("text", f"[error] {routed.get('error')}") if routed.get("ok") \ else f"[error] {routed.get('error')}" @@ -236,10 +261,18 @@ def _iter_xrefs(program): ref_mgr = program.getReferenceManager() func_mgr = program.getFunctionManager() it = ref_mgr.getReferenceIterator(program.getMinAddress()) + # References arrive in ascending address order and consecutive ones almost + # always share a containing function, so cache the last one and re-test its + # body before paying another cross-JVM manager lookup. + cached = None while it.hasNext(): ref = it.next() from_addr = ref.getFromAddress() - containing = func_mgr.getFunctionContaining(from_addr) + if cached is not None and cached.getBody().contains(from_addr): + containing = cached + else: + containing = func_mgr.getFunctionContaining(from_addr) + cached = containing if containing is None: continue rt = ref.getReferenceType() @@ -263,14 +296,18 @@ def _iter_blocks(program): from ghidra.program.model.block import BasicBlockModel from ghidra.util.task import ConsoleTaskMonitor + func_mgr = program.getFunctionManager() model = BasicBlockModel(program) monitor = ConsoleTaskMonitor() it = model.getCodeBlocks(monitor) while it.hasNext(): blk = it.next() - start = blk.getFirstStartAddress().getOffset() + start_addr = blk.getFirstStartAddress() + start = start_addr.getOffset() end = blk.getMaxAddress().getOffset() + 1 - yield {"func_ea": start, "start_ea": start, "end_ea": end, "size": end - start} + func = func_mgr.getFunctionContaining(start_addr) + func_ea = func.getEntryPoint().getOffset() if func is not None else start + yield {"func_ea": func_ea, "start_ea": start, "end_ea": end, "size": end - start} def _export_program(program, gi, xrefs=None, blocks=None) -> dict: @@ -316,8 +353,7 @@ def export(project_dir: str, binary: str, db_path: str) -> str: if not is_analyzed(project_dir, binary_path.name): return f"[error] no analyzed project for {binary_path.name} in {project_dir}" - game = Path(project_dir).parent.name - routed = _route_daemon(game, {"cmd": "export", "binary": binary, "db": db_path}) + routed = _route_daemon(project_dir, {"cmd": "export", "binary": binary, "db": db_path}) if routed is not None: if not routed.get("ok"): return f"[error] {routed.get('error')}" @@ -348,9 +384,16 @@ def export(project_dir: str, binary: str, db_path: str) -> str: # kb-apply # --------------------------------------------------------------------------- -def _kb_apply_program(program, kb, flat_api, *, apply_prototypes=True, apply_types=True) -> dict: +def _kb_apply_program(program, kb, *, apply_prototypes=True, apply_types=True) -> dict: """Apply a parsed Kb to an open program. Idempotent (USER_DEFINED upserts). + An ``@`` entry renames the function that contains its address; when no + function is there (bootstrap emits ``@`` lines for RTTI vtables and string + refs, which are data) it becomes a label instead of a bogus function. All + mutation goes through Program-level APIs so it stays inside the caller's + single transaction -- FlatProgramAPI.createFunction would open its own + nested transaction and leave it dangling, which then fails program.save(). + apply_prototypes/apply_types gate the Ghidra-only signature/DTM code so a fake program (tests) can exercise the name/label path without those classes. """ @@ -360,7 +403,7 @@ def _kb_apply_program(program, kb, flat_api, *, apply_prototypes=True, apply_typ listing = program.getListing() symtab = program.getSymbolTable() - counts = {"functions": 0, "globals": 0, "typedefs": 0} + counts = {"functions": 0, "labels": 0, "globals": 0, "typedefs": 0} sig_parser = None if apply_prototypes: @@ -372,9 +415,13 @@ def _kb_apply_program(program, kb, flat_api, *, apply_prototypes=True, apply_typ for fn in kb.functions: addr = space.getAddress(fn.address) func = listing.getFunctionContaining(addr) - if func is None and flat_api is not None: - func = flat_api.createFunction(addr, fn.name) if func is None: + # No function here -> label the address (vtable / string / data). + try: + symtab.createLabel(addr, fn.name, SourceType.USER_DEFINED) + counts["labels"] += 1 + except Exception: + pass # name may be invalid; others still apply continue func.setName(fn.name, SourceType.USER_DEFINED) counts["functions"] += 1 @@ -410,6 +457,22 @@ def _kb_apply_program(program, kb, flat_api, *, apply_prototypes=True, apply_typ return counts +def _kb_apply_txn(program, kb) -> dict: + """Apply *kb* to *program* inside one transaction and return the counts. + + Persistence is the caller's job, because an explicit ``program.save`` is + rejected ("Unable to lock due to active transaction") while the program is + live inside pyghidra's ``open_program`` context. The supported path is + pyghidra's own ``project.save(program)`` on context exit: the cold path gets + it from leaving the ``with`` block, the daemon by closing the program. + """ + txn = program.startTransaction("kb_apply") + try: + return _kb_apply_program(program, kb) + finally: + program.endTransaction(txn, True) + + def kb_apply(project_dir: str, binary: str, kb_path: str) -> str: """Apply kb.h to the analyzed Ghidra program (one transaction).""" from kb import parse_kb @@ -418,8 +481,7 @@ def kb_apply(project_dir: str, binary: str, kb_path: str) -> str: if not is_analyzed(project_dir, binary_path.name): return f"[error] no analyzed project for {binary_path.name} in {project_dir}" - game = Path(project_dir).parent.name - routed = _route_daemon(game, {"cmd": "kb_apply", "binary": binary, "kb": kb_path}) + routed = _route_daemon(project_dir, {"cmd": "kb_apply", "binary": binary, "kb": kb_path}) if routed is not None: if not routed.get("ok"): return f"[error] {routed.get('error')}" @@ -439,12 +501,7 @@ def kb_apply(project_dir: str, binary: str, kb_path: str) -> str: project_name=binary_path.stem, analyze=False, ) as flat_api: program = flat_api.getCurrentProgram() - txn = program.startTransaction("kb_apply") - try: - counts = _kb_apply_program(program, kb, flat_api) - finally: - program.endTransaction(txn, True) - program.save("kb_apply", None) + counts = _kb_apply_txn(program, kb) summary = ", ".join(f"{k}={v}" for k, v in counts.items()) return f"kb-apply complete: {summary}" @@ -483,7 +540,7 @@ def main(): p_export = sub.add_parser("export", help="Export analyzed facts into index.db") p_export.add_argument("binary", help="Path to PE binary") p_export.add_argument("--project", required=True, help="Project directory") - p_export.add_argument("--db", default=None, help="index.db path (default patches//index.db)") + p_export.add_argument("--db", default=None, help="index.db path (default /index.db)") # --- kb-apply --- p_kb = sub.add_parser("kb-apply", help="Apply kb.h names/types into the Ghidra project") @@ -517,7 +574,7 @@ def main(): if args.command == "export": from index import GameIndex - db_path = args.db or GameIndex.default_db_path(Path(args.binary).stem) + db_path = args.db or GameIndex.project_db_path(args.project) print(export(ghidra_dir, args.binary, db_path)) raise SystemExit(0) diff --git a/retools/query.py b/retools/query.py index a1966cd7..843b9ef6 100644 --- a/retools/query.py +++ b/retools/query.py @@ -77,11 +77,7 @@ def main(argv: list[str] | None = None) -> None: p.add_argument("--schema", metavar="TABLE", help="Print PRAGMA table_info for TABLE and exit") args = p.parse_args(argv) - db_path = args.db or GameIndex.default_db_path(args.game) - if not Path(db_path).is_file(): - print(f"[error] no index at {db_path}. Run bootstrap or 'pyghidra_backend export' first.", - file=sys.stderr) - raise SystemExit(1) + db_path = GameIndex.resolve_db(args.game, args.db) conn = GameIndex.open_ro(db_path) try: diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index e1c7d2a8..63c1c908 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -332,6 +332,29 @@ def test_main_no_args_exits(self): # Index seeding # --------------------------------------------------------------------------- +class TestStringSweepDedup: + def test_bootstrap_scans_strings_once(self, tmp_path): + """bootstrap must sweep the binary for strings a single time and reuse + the result for both the error-string KB seed and the index seed.""" + import search + from bootstrap import bootstrap + + pe_path = _make_minimal_pe(str(tmp_path)) + project_dir = str(tmp_path / "project") + + calls = {"n": 0} + orig = search.find_strings + + def counting(b, **kw): + calls["n"] += 1 + return orig(b, **kw) + + with patch("search.find_strings", counting): + bootstrap(pe_path, project_dir) + + assert calls["n"] == 1 + + class TestSeedIndex: def test_seed_index_populates_tables(self, sample_binary, tmp_path): import sys diff --git a/tests/test_context.py b/tests/test_context.py index 2c2a6a6d..05f09adf 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -336,7 +336,10 @@ def test_callees_from_index(self, tmp_path): db = str(tmp_path / "index.db") gi = GameIndex(db) - gi.replace("funcs", [{"address": 0x2000, "name": "Target"}], source="ghidra") + # Both the caller (0x1000) and callee (0x2000) are known functions, as in + # a real Ghidra export. + gi.replace("funcs", [{"address": 0x1000, "name": "Caller"}, + {"address": 0x2000, "name": "Target"}], source="ghidra") gi.replace("xrefs", [{"from_ea": 0x1010, "to_ea": 0x2000, "type": "call", "is_code": 1, "from_func": 0x1000}], source="ghidra") gi.close() @@ -369,3 +372,79 @@ def test_empty_xrefs_returns_none(self, tmp_path): gi.close() assert _callees_from_index(db, 0x1000) is None + + def test_unknown_start_returns_none(self, tmp_path): + """When the queried start isn't a known function entry (find_start + disagreed with Ghidra), an empty result must not be trusted.""" + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) + from index import GameIndex + from context import _callees_from_index + + db = str(tmp_path / "index.db") + gi = GameIndex(db) + gi.replace("funcs", [{"address": 0x2000, "name": "Known"}], source="ghidra") + gi.replace("xrefs", [{"from_ea": 0x2010, "to_ea": 0x3000, "type": "call", + "is_code": 1, "from_func": 0x2000}], source="ghidra") + gi.close() + # 0x9999 is not a function entry -> caller must scan, not trust [] + assert _callees_from_index(db, 0x9999) is None + + def test_schemaless_db_returns_none(self, tmp_path): + """A present but foreign/half-initialised index.db (no xrefs table) must + fall back to None rather than raising OperationalError.""" + import sqlite3 + from context import _callees_from_index + + db = str(tmp_path / "index.db") + conn = sqlite3.connect(db) + conn.execute("CREATE TABLE unrelated (x INTEGER)") + conn.commit() + conn.close() + assert _callees_from_index(db, 0x1000) is None + + +class TestAssembleIndexBranch: + def _mock_binary(self): + b = MagicMock() + b.is_64 = False + b.base = 0x400000 + b.disasm.return_value = [] + b.abs_mem_refs.return_value = [] + b.abs_imm_refs.return_value = [] + return b + + def test_kb_name_beats_index_name_and_indirect_recovered(self, tmp_path): + """On the index fast-path, a fresh kb.h name outranks the exported Ghidra + name, and indirect-call markers from the scan are still emitted.""" + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) + from index import GameIndex + from context import assemble + + proj = tmp_path / "proj" + proj.mkdir() + gi = GameIndex(str(proj / "index.db")) + gi.replace("funcs", [{"address": 0x401500, "name": "FUN_00401500"}, + {"address": 0x401200, "name": "FUN_00401200"}], source="ghidra") + gi.replace("xrefs", [{"from_ea": 0x401510, "to_ea": 0x401200, "type": "call", + "is_code": 1, "from_func": 0x401500}], source="ghidra") + gi.close() + + kb = proj / "kb.h" + kb.write_text("@ 0x401200 void __cdecl RealCallee();\n") + + b = self._mock_binary() + with patch("context.find_start", return_value=0x401500), \ + patch("context.analyze", return_value=([], [(0x401510, 0x401200), + (0x401520, "dword ptr [eax + 0x14]")], 0x401600)), \ + patch("context.aggregate_struct", return_value=[]), \ + patch("context.find_strings", return_value=[]), \ + patch("context.propagate_cfg", return_value={}): + result = assemble(b, 0x401500, str(proj), project_dir_for_kb=str(proj)) + + assert "RealCallee" in result # kb.h name wins over FUN_00401200 + assert "FUN_00401200" not in result + assert "indirect call" in result # indirect marker recovered diff --git a/tests/test_decompiler.py b/tests/test_decompiler.py index 8b8ba4d5..88af9802 100644 --- a/tests/test_decompiler.py +++ b/tests/test_decompiler.py @@ -9,6 +9,33 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) +class TestScriptModeImports: + def test_load_types_import_resolves_in_script_mode(self, tmp_path): + """`python retools/decompiler.py` puts only retools/ on sys.path[0]; + _load_types must still be able to import the kb parser.""" + import subprocess + import textwrap + + retools = Path(__file__).resolve().parent.parent / "retools" + driver = tmp_path / "driver.py" + driver.write_text(textwrap.dedent(f""" + import sys + sys.path.insert(0, r"{retools}") # simulate script-dir-only sys.path + import decompiler + + class FakeR2: + def cmd(self, c): pass + + decompiler._load_types(FakeR2(), "@ 0x401000 void Foo(void);") + print("OK") + """)) + proc = subprocess.run( + [sys.executable, str(driver)], + cwd=str(tmp_path), capture_output=True, text=True, + ) + assert "OK" in proc.stdout, proc.stderr + + class TestBackendRouting: def test_auto_no_project_uses_r2(self): """--backend auto without --project should use r2ghidra.""" diff --git a/tests/test_ghidra_server.py b/tests/test_ghidra_server.py new file mode 100644 index 00000000..3ba1d5e7 --- /dev/null +++ b/tests/test_ghidra_server.py @@ -0,0 +1,132 @@ +"""Tests for retools/ghidra_server.py -- daemon dispatch, identity, lifecycle. + +These exercise the pure socket/dispatch logic; no Ghidra program is opened +(status/shutdown need none), so they run without a JDK. +""" + +import json +import sys +import threading +import time +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) + + +class TestIdentityHandshake: + def test_rejects_foreign_game(self): + import ghidra_server + d = ghidra_server.GhidraDaemon("GameA") + resp = d.handle({"cmd": "status", "game": "GameB"}) + assert resp["ok"] is False + assert resp.get("wrong_project") is True + + def test_accepts_matching_game(self): + import ghidra_server + d = ghidra_server.GhidraDaemon("GameA") + resp = d.handle({"cmd": "status", "game": "GameA"}) + assert resp["ok"] is True + + def test_accepts_missing_game(self): + import ghidra_server + d = ghidra_server.GhidraDaemon("GameA") + resp = d.handle({"cmd": "status"}) + assert resp["ok"] is True + + def test_unknown_command(self): + import ghidra_server + d = ghidra_server.GhidraDaemon("GameA") + resp = d.handle({"cmd": "nope", "game": "GameA"}) + assert resp["ok"] is False + + +class TestServeLifecycle: + def test_status_then_shutdown_over_socket(self, tmp_path, monkeypatch): + import ghidra_server + import ghidra_client + + # Ephemeral port so the test never collides with a real daemon. + monkeypatch.setattr(ghidra_server, "PORT", 0) + + d = ghidra_server.GhidraDaemon("GameA", idle=30.0) + d.project_dir = str(tmp_path / "ghidra") + + t = threading.Thread(target=d.serve, daemon=True) + t.start() + + # Wait for the state file (written once the socket is bound). + sp = ghidra_client.state_path(d.project_dir) + for _ in range(200): + if sp.exists(): + break + time.sleep(0.01) + assert sp.exists(), "daemon never wrote its state file" + state = json.loads(sp.read_text()) + assert state["port"] != 0 # real bound port recorded, not the sentinel + + resp = ghidra_client.send_command(d.project_dir, {"cmd": "status", "game": "GameA"}) + assert resp["ok"] is True + assert resp["game"] == "GameA" + + ghidra_client.send_command(d.project_dir, {"cmd": "shutdown", "game": "GameA"}) + t.join(timeout=5) + assert not t.is_alive() + assert not sp.exists() # cleanup removed the state file + + def test_bad_frame_gets_error_response(self, tmp_path, monkeypatch): + import socket + import struct + import ghidra_server + import ghidra_client + + monkeypatch.setattr(ghidra_server, "PORT", 0) + d = ghidra_server.GhidraDaemon("GameA", idle=30.0) + d.project_dir = str(tmp_path / "ghidra") + t = threading.Thread(target=d.serve, daemon=True) + t.start() + + sp = ghidra_client.state_path(d.project_dir) + for _ in range(200): + if sp.exists(): + break + time.sleep(0.01) + port = json.loads(sp.read_text())["port"] + + # Send a length-prefixed frame whose body is not valid JSON. + s = socket.create_connection((ghidra_client.HOST, port), timeout=5) + junk = b"not json" + s.sendall(struct.pack("!I", len(junk)) + junk) + reply = ghidra_client._recv_raw(s) + s.close() + resp = json.loads(reply) + assert resp["ok"] is False # error response, not a silent hang + + ghidra_client.send_command(d.project_dir, {"cmd": "shutdown", "game": "GameA"}) + t.join(timeout=5) + + +class TestClientStalePid: + def test_dead_pid_marks_not_alive_even_when_port_is_open(self, tmp_path, monkeypatch): + """The cross-project hazard: a foreign daemon holds the port while a + stale state file points at a dead pid. Checking the pid first rejects it + instead of routing this project's commands into the foreign daemon.""" + import socket + import ghidra_client + + # A live listener on some port (stands in for a foreign daemon). + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.bind(("127.0.0.1", 0)) + srv.listen(1) + open_port = srv.getsockname()[1] + + payload = {"pid": 999999, "port": open_port, "project": "G", "binary": "g.exe", "started": 0} + sp = ghidra_client.state_path(str(tmp_path)) + sp.write_text(json.dumps(payload)) + monkeypatch.setattr(ghidra_client, "_pid_alive", lambda pid: False) + try: + assert ghidra_client.is_daemon_alive(str(tmp_path)) is False + assert not sp.exists() # stale state file cleaned up + finally: + srv.close() diff --git a/tests/test_index.py b/tests/test_index.py index eff0009b..79c5b978 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -100,3 +100,61 @@ def test_from_func_high_bit_skipped(self, tmp_path): assert n == 0 assert gi.counts()["xrefs"] == 0 gi.close() + + def test_empty_schema_version_recovers(self, tmp_path): + """A crash between schema DDL and the version INSERT leaves schema_version + created-but-empty; reopening must repair it, not silently skip the insert.""" + from index import GameIndex, SCHEMA_VERSION + db = str(tmp_path / "index.db") + GameIndex(db).close() + # Simulate the crash window: drop the version row, keep the table. + conn = sqlite3.connect(db) + conn.execute("DELETE FROM schema_version") + conn.commit() + conn.close() + gi = GameIndex(db) # must not leave the DB versionless + ver = gi._conn.execute("SELECT version FROM schema_version").fetchone() + gi.close() + assert ver is not None and ver[0] == SCHEMA_VERSION + + def test_bootstrap_rerun_does_not_downgrade_ghidra(self, tmp_path): + """Re-running bootstrap after a ghidra export must not overwrite the + authoritative ghidra funcs row with a provisional bootstrap row.""" + from index import GameIndex + gi = GameIndex(str(tmp_path / "index.db")) + gi.replace("funcs", [{"address": 0x1000, "name": None}], source="bootstrap") + gi.replace("funcs", [{"address": 0x1000, "name": "RealName"}], source="ghidra") + # Second bootstrap pass re-seeds the same provisional address. + gi.replace("funcs", [{"address": 0x1000, "name": None}], source="bootstrap") + row = gi._conn.execute("SELECT name, source FROM funcs WHERE address=0x1000").fetchone() + gi.close() + assert row == ("RealName", "ghidra") + + def test_xrefs_from_func_is_indexed(self, tmp_path): + """The callees view and context lookups filter on xrefs.from_func; + it must have a supporting index so the lookup is not a full scan.""" + from index import GameIndex + db = str(tmp_path / "index.db") + GameIndex(db).close() + conn = sqlite3.connect(db) + plan = conn.execute( + "EXPLAIN QUERY PLAN SELECT * FROM xrefs WHERE from_func = 0x1000" + ).fetchall() + conn.close() + assert any("ix_xrefs_from_func" in " ".join(str(c) for c in row) for row in plan) + + def test_project_db_path(self, tmp_path): + from index import GameIndex + p = GameIndex.project_db_path(str(tmp_path / "patches" / "MyGame")) + assert p.replace("\\", "/").endswith("patches/MyGame/index.db") + + def test_resolve_db_returns_existing(self, tmp_path): + from index import GameIndex + db = str(tmp_path / "index.db") + GameIndex(db).close() + assert GameIndex.resolve_db("ignored", db) == db + + def test_resolve_db_exits_when_absent(self, tmp_path): + from index import GameIndex + with pytest.raises(SystemExit): + GameIndex.resolve_db("NoSuchGame", str(tmp_path / "absent.db")) diff --git a/tests/test_kb.py b/tests/test_kb.py index 41c707ed..fc5155b1 100644 --- a/tests/test_kb.py +++ b/tests/test_kb.py @@ -79,6 +79,27 @@ def test_existing_meccha_kb_all_comments(self): assert isinstance(kb.functions, list) +class TestParseKbInputHandling: + def test_content_that_names_a_file_is_parsed_as_content(self, tmp_path, monkeypatch): + """A one-line kb string is parsed as content even if it happens to match + an existing filename -- parse_kb must not sniff strings as paths.""" + from kb import parse_kb + monkeypatch.chdir(tmp_path) + # Create a file whose name equals the content we pass. + content = "@ 0x401000 void Foo(void);" + (tmp_path / content.replace("/", "_")).write_text("@ 0xDEAD void Other(void);\n") + kb = parse_kb(content) # str -> treated as content, never read as a path + assert kb.functions[0].name == "Foo" + assert kb.functions[0].address == 0x401000 + + def test_path_is_read(self, tmp_path): + from kb import parse_kb + p = tmp_path / "kb.h" + p.write_text("@ 0x402000 void Bar(void);\n") + kb = parse_kb(p) + assert kb.functions[0].name == "Bar" + + class TestReadExistingAddresses: def test_collects_function_addresses(self, tmp_path): from kb import read_existing_addresses diff --git a/tests/test_pyghidra_backend.py b/tests/test_pyghidra_backend.py index c61b3ea5..cc3e8cd1 100644 --- a/tests/test_pyghidra_backend.py +++ b/tests/test_pyghidra_backend.py @@ -415,7 +415,10 @@ class FakeFunc: def setName(self, name, src): applied["names"].append(name) class FakeListing: - def getFunctionContaining(self, addr): return FakeFunc() + # A real function lives at 0x401000; 0x402000 is a data address + # (e.g. an RTTI vtable) with no containing function. + def getFunctionContaining(self, addr): + return FakeFunc() if addr._off == 0x401000 else None class FakeSymbolTable: def createLabel(self, addr, name, src): @@ -427,12 +430,16 @@ def getAddressFactory(self): return FakeAddrFactory() def getListing(self): return FakeListing() def getSymbolTable(self): return FakeSymbolTable() - kb = parse_kb("@ 0x401000 void Foo(void);\n$ 0x7C5548 int g_x\n") - counts = _kb_apply_program(FakeProgram(), kb, flat_api=None, + kb = parse_kb("@ 0x401000 void Foo(void);\n" + "@ 0x402000 SomeClass_vtable;\n" + "$ 0x7C5548 int g_x\n") + counts = _kb_apply_program(FakeProgram(), kb, apply_prototypes=False, apply_types=False) - assert "Foo" in applied["names"] - assert "g_x" in applied["labels"] + assert "Foo" in applied["names"] # existing function renamed + assert "SomeClass_vtable" in applied["labels"] # data @ -> label, not a bogus function + assert "g_x" in applied["labels"] # $ global -> label assert counts["functions"] == 1 + assert counts["labels"] == 1 assert counts["globals"] == 1 @@ -504,7 +511,192 @@ def test_decompile_uses_daemon_when_routed(self, tmp_path, monkeypatch): monkeypatch.setattr(pyghidra_backend, "is_analyzed", lambda project_dir, binary_name: True) monkeypatch.setattr( pyghidra_backend, "_route_daemon", - lambda game, cmd: {"ok": True, "text": "ROUTED"}, + lambda project_dir, cmd: {"ok": True, "text": "ROUTED"}, ) result = pyghidra_backend.decompile(str(tmp_path / "ghidra"), "test.exe", 0x401000) assert result == "ROUTED" + + +class TestRouteDaemonSafety: + def test_wrong_project_response_falls_back_to_cold(self, monkeypatch): + """A daemon that reports it serves a different project must not answer; + _route_daemon returns None so the caller takes the cold path.""" + from pyghidra_backend import _route_daemon + import ghidra_client + + monkeypatch.setattr(ghidra_client, "is_daemon_alive", lambda project_dir: True) + monkeypatch.setattr( + ghidra_client, "send_command", + lambda project_dir, cmd, timeout=None: {"ok": False, "wrong_project": True}, + ) + assert _route_daemon(str(Path("patches") / "G" / "ghidra"), {"cmd": "decompile"}) is None + + def test_timeout_propagates_not_silent_cold_retry(self, monkeypatch): + """A timeout on a live daemon may have already run the command; falling + through to a cold re-run would double-execute, so it must raise.""" + import socket + from pyghidra_backend import _route_daemon + import ghidra_client + + monkeypatch.setattr(ghidra_client, "is_daemon_alive", lambda project_dir: True) + + def _timeout(project_dir, cmd, timeout=None): + raise socket.timeout("timed out") + + monkeypatch.setattr(ghidra_client, "send_command", _timeout) + with pytest.raises(Exception): + _route_daemon(str(Path("patches") / "G" / "ghidra"), {"cmd": "export"}) + + def test_injects_game_identity(self, monkeypatch): + """_route_daemon tags the command with the game derived from project_dir + so the daemon can reject cross-project routing.""" + from pyghidra_backend import _route_daemon + import ghidra_client + + seen = {} + monkeypatch.setattr(ghidra_client, "is_daemon_alive", lambda project_dir: True) + monkeypatch.setattr( + ghidra_client, "send_command", + lambda project_dir, cmd, timeout=None: seen.update(cmd) or {"ok": True}, + ) + _route_daemon(str(Path("patches") / "MyGame" / "ghidra"), + {"cmd": "export", "binary": "game.exe", "db": "patches/MyGame/index.db"}) + assert seen.get("game") == "MyGame" + # Paths are absolutised so the daemon can't resolve them against a + # different cwd than the client's. + assert Path(seen["binary"]).is_absolute() + assert Path(seen["db"]).is_absolute() + + +class TestIterBlocks: + def test_func_ea_is_containing_function_entry(self, monkeypatch): + """Every basic block must be keyed by its owning function's entry point, + not by the block's own start address.""" + import sys + import types + from pathlib import Path + sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) + + for name in ("ghidra", "ghidra.program", "ghidra.program.model", + "ghidra.program.model.block", "ghidra.util", "ghidra.util.task"): + monkeypatch.setitem(sys.modules, name, types.ModuleType(name)) + + class FakeAddr: + def __init__(self, off): self._off = off + def getOffset(self): return self._off + + class FakeBlock: + def __init__(self, start, end): + self._start = FakeAddr(start) + self._end = FakeAddr(end - 1) + def getFirstStartAddress(self): return self._start + def getMaxAddress(self): return self._end + + class FakeIter: + def __init__(self, blocks): self._b = list(blocks) + def hasNext(self): return bool(self._b) + def next(self): return self._b.pop(0) + + class FakeModel: + def __init__(self, program): pass + def getCodeBlocks(self, monitor): + return FakeIter([FakeBlock(0x401000, 0x401020), + FakeBlock(0x401020, 0x401055)]) + + class FakeFunc: + def getEntryPoint(self): return FakeAddr(0x401000) + + class FakeFuncMgr: + def getFunctionContaining(self, addr): return FakeFunc() + + class FakeProgram: + def getFunctionManager(self): return FakeFuncMgr() + + monkeypatch.setattr(sys.modules["ghidra.program.model.block"], + "BasicBlockModel", FakeModel, raising=False) + monkeypatch.setattr(sys.modules["ghidra.util.task"], + "ConsoleTaskMonitor", lambda: object(), raising=False) + + from pyghidra_backend import _iter_blocks + rows = list(_iter_blocks(FakeProgram())) + assert [r["func_ea"] for r in rows] == [0x401000, 0x401000] + assert [r["start_ea"] for r in rows] == [0x401000, 0x401020] + + +class TestIterXrefs: + def test_containing_function_is_cached_across_consecutive_refs(self): + """Consecutive refs in the same function must not each pay a manager + lookup; the containing function is cached and re-tested by body.""" + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "retools")) + from pyghidra_backend import _iter_xrefs + + class FakeAddr: + def __init__(self, off): self._off = off + def getOffset(self): return self._off + + class FakeType: + def isCall(self): return True + def isJump(self): return False + def isData(self): return False + + class FakeRef: + def __init__(self, frm, to): + self._f = FakeAddr(frm) + self._t = FakeAddr(to) + def getFromAddress(self): return self._f + def getToAddress(self): return self._t + def getReferenceType(self): return FakeType() + + class FakeBody: + def contains(self, addr): return 0x401000 <= addr.getOffset() < 0x402000 + + class FakeFunc: + def getBody(self): return FakeBody() + def getEntryPoint(self): return FakeAddr(0x401000) + + class FakeRefIter: + def __init__(self, refs): self._r = list(refs) + def hasNext(self): return bool(self._r) + def next(self): return self._r.pop(0) + + class FakeRefMgr: + def getReferenceIterator(self, addr): + return FakeRefIter([FakeRef(0x401010, 0x500000), FakeRef(0x401030, 0x500004)]) + + calls = {"n": 0} + + class FakeFuncMgr: + def getFunctionContaining(self, addr): + calls["n"] += 1 + return FakeFunc() + + class FakeProgram: + def getReferenceManager(self): return FakeRefMgr() + def getFunctionManager(self): return FakeFuncMgr() + def getMinAddress(self): return FakeAddr(0) + + rows = list(_iter_xrefs(FakeProgram())) + assert len(rows) == 2 + assert all(r["from_func"] == 0x401000 for r in rows) + assert calls["n"] == 1 # second ref served from cache + + +class TestExportCLIDbPath: + def test_export_db_defaults_to_project_dir(self, tmp_path, monkeypatch): + """`export --project patches/MyGame` (no --db) must write to + patches/MyGame/index.db, not patches//index.db.""" + import pyghidra_backend + captured = {} + monkeypatch.setattr( + pyghidra_backend, "export", + lambda ghidra_dir, binary, db_path: captured.update(db=db_path) or "ok", + ) + binary = tmp_path / "game.exe" + binary.write_bytes(b"MZ") + proj = tmp_path / "patches" / "MyGame" + sys.argv = ["pyghidra_backend", "export", str(binary), "--project", str(proj)] + with pytest.raises(SystemExit): + pyghidra_backend.main() + assert captured["db"] == str(proj / "index.db") From 64bd46413b9133d5486647dc029a646eb751de98 Mon Sep 17 00:00:00 2001 From: Ben Gregg Date: Fri, 17 Jul 2026 14:20:03 -0500 Subject: [PATCH 22/28] Make agent instructions single source of truth --- .claude/CLAUDE.md | 66 +-- .cursor/agents/static-analyzer.md | 186 ------- .cursor/agents/web-researcher.md | 32 -- .cursor/rules/code-comments.mdc | 32 -- .cursor/rules/dx9-ffp-port.mdc | 281 ---------- .cursor/rules/no-copium.mdc | 39 -- .cursor/rules/project-workspace.mdc | 44 -- .cursor/rules/subagent-workflow.mdc | 147 ------ .cursor/rules/tool-catalog.mdc | 320 ------------ .cursor/skills/dx9-ffp-port/SKILL.md | 325 ------------ .../references/remix-comp-context.md | 58 --- .cursor/skills/dynamic-analysis/SKILL.md | 485 ----------------- .github/agents/static-analyzer.agent.md | 131 ----- .github/agents/web-researcher.agent.md | 30 -- .github/copilot-instructions.md | 72 +-- .../instructions/ffp-proxy.instructions.md | 156 ------ .../instructions/kb-format.instructions.md | 43 -- .../instructions/tool-catalog.instructions.md | 319 ------------ .github/prompts/assemble-context.prompt.md | 22 - .github/prompts/dx9-ffp-port.prompt.md | 152 ------ .github/prompts/identify-function.prompt.md | 22 - .github/skills/dx9-ffp-port/SKILL.md | 325 ------------ .../references/remix-comp-context.md | 58 --- .github/skills/dynamic-analysis/SKILL.md | 486 ----------------- .gitignore | 10 +- .kiro/agents/static-analyzer.md | 186 ------- .kiro/agents/web-researcher.md | 32 -- .kiro/hooks/update-docs-after-task.json | 4 +- .kiro/hooks/verify-and-update-docs.json | 2 +- .kiro/powers/dx9-ffp-port/POWER.md | 327 ------------ .../references/remix-comp-context.md | 58 --- .kiro/powers/dynamic-analysis/POWER.md | 488 ------------------ .kiro/steering/dx9-ffp-port.md | 158 ------ .kiro/steering/engineering-standards.md | 61 --- .kiro/steering/project-workspace.md | 45 -- .kiro/steering/subagent-workflow.md | 139 ----- .kiro/steering/tool-catalog.md | 320 ------------ .vscode/settings.json | 4 + AGENTS.md | 108 ++++ README.md | 24 +- 40 files changed, 146 insertions(+), 5651 deletions(-) delete mode 100644 .cursor/agents/static-analyzer.md delete mode 100644 .cursor/agents/web-researcher.md delete mode 100644 .cursor/rules/code-comments.mdc delete mode 100644 .cursor/rules/dx9-ffp-port.mdc delete mode 100644 .cursor/rules/no-copium.mdc delete mode 100644 .cursor/rules/project-workspace.mdc delete mode 100644 .cursor/rules/subagent-workflow.mdc delete mode 100644 .cursor/rules/tool-catalog.mdc delete mode 100644 .cursor/skills/dx9-ffp-port/SKILL.md delete mode 100644 .cursor/skills/dx9-ffp-port/references/remix-comp-context.md delete mode 100644 .cursor/skills/dynamic-analysis/SKILL.md delete mode 100644 .github/agents/static-analyzer.agent.md delete mode 100644 .github/agents/web-researcher.agent.md delete mode 100644 .github/instructions/ffp-proxy.instructions.md delete mode 100644 .github/instructions/kb-format.instructions.md delete mode 100644 .github/instructions/tool-catalog.instructions.md delete mode 100644 .github/prompts/assemble-context.prompt.md delete mode 100644 .github/prompts/dx9-ffp-port.prompt.md delete mode 100644 .github/prompts/identify-function.prompt.md delete mode 100644 .github/skills/dx9-ffp-port/SKILL.md delete mode 100644 .github/skills/dx9-ffp-port/references/remix-comp-context.md delete mode 100644 .github/skills/dynamic-analysis/SKILL.md delete mode 100644 .kiro/agents/static-analyzer.md delete mode 100644 .kiro/agents/web-researcher.md delete mode 100644 .kiro/powers/dx9-ffp-port/POWER.md delete mode 100644 .kiro/powers/dx9-ffp-port/references/remix-comp-context.md delete mode 100644 .kiro/powers/dynamic-analysis/POWER.md delete mode 100644 .kiro/steering/dx9-ffp-port.md delete mode 100644 .kiro/steering/engineering-standards.md delete mode 100644 .kiro/steering/project-workspace.md delete mode 100644 .kiro/steering/subagent-workflow.md delete mode 100644 .kiro/steering/tool-catalog.md create mode 100644 .vscode/settings.json create mode 100644 AGENTS.md diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index a929532f..80421f6b 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -1,18 +1,10 @@ # Vibe Reverse Engineering -- Claude Code Instructions -## Read-Only Templates +Shared conventions (project overview, read-only templates, workspace/backup/KB rules, engineering standards, code comments) are canonical in the root file, auto-loaded here: -These directories are **shared tooling and templates**. Do not modify them for game-specific work — per-game changes go in `patches//`. +@../AGENTS.md -- `rtx_remix_tools/dx/remix-comp-proxy/` — proxy framework **template** (copied per-game) -- `rtx_remix_tools/dx/scripts/` — DX9 analysis scripts (shared tooling) -- `retools/` — static analysis toolkit (shared tooling) -- `livetools/` — Frida-based dynamic analysis (shared tooling) -- `graphics/` — DX9 tracer framework (shared tooling) - -**Per-game work goes in `patches//`.** When starting a new game, copy `rtx_remix_tools/dx/remix-comp-proxy/` (excluding `build/`) to `patches//` and edit the copy. If the user says "edit remix-comp-proxy code" without specifying, ask whether they mean the template or a game copy. - -Shared tooling can be modified to improve the tools themselves — just not for game-specific customization. +The sections below are Claude Code-specific. --- @@ -30,58 +22,6 @@ The main agent owns `livetools` — always use them to verify static findings, p --- -## Engineering Standards - -Every change should make the codebase better, not just make the problem go away. If a solution needs a paragraph to justify why it's not a hack, it's a hack. - -### Remove -- **Fixes in the wrong layer**: a guard on a canvas to suppress commits that a model should own. Put the fix where the problem originates. -- **Tolerance inflation**: widening deltas or adding retries to hide flaky behavior. If the value is wrong, find out why. -- **Catch-all exception swallowing**: `try/except Exception: pass` to hide symptoms. -- **Excessive error/null handling**: adding too many error/None "if" checks. If the error is expected, handle it. If unexpected, raise it. -- **God methods**: 200+ line functions doing multiple things. Break into named steps. Focus on cognitive load. Design for fewer indentation levels. -- **Leaky abstractions**: implementation details leaking into layers/modules that should be agnostic of one another. - -### Design For -- **Single responsibility**: one component, one job. If you need "and" to describe it, split it. -- **Ownership**: the component that creates the problem owns the fix. -- **Minimal public surface**: expose what consumers need, nothing more. - -### Commit to the New Code -- **No legacy fallbacks**: if you replace a system, remove the old one. -- **No dead code**: commented-out blocks, unused imports, orphan functions "just in case". Version control is the safety net. -- **No multiple paths to the same result**: one way to do each thing. If two paths exist, one is wrong. -- **No half-migrations**: finish the job -- update every reference, remove old APIs. - -### Smell Tests -- "It works if I add a sleep" -- broken data flow. -- "It works if I read from widget instead of storage" -- the two are out of sync. -- "It passes alone but fails with other tests" -- shared mutable state leaking. -- "I added a flag to skip this code path" -- why does that path run in the first place? - -## Code Comments - -Each file reads as if it was always designed this way. Comments guide the next developer, not narrate the development journey. - -### Remove -- **Implementation backstories**: "We do this because the other day X happened" -- **Obvious narration**: "Create the attribute", "Loop through keys", "Check if valid" -- if the code says it, the comment is noise -- **Debugging breadcrumbs**: "Without this, subsequent tests may see the modifier key as still held" -- **Trial-and-error reasoning**: "We tried X but it caused Y so we do Z instead" - -### Keep -- **Non-obvious design decisions**: stated as *what* and *why this design*, not *what happened to us* -- **Tricky invariants**: conditions that would be easy to accidentally break -- **API contracts**: docstrings on public methods with Args, Returns, Raises - -### Prefer Instead -- **Rename** a variable or function to be self-explanatory rather than adding a comment -- **Docstrings** on classes and public methods (Google style: `Args:`, `Returns:`, `Raises:`) -- **Type hints** over comments about expected types -- **Short inline comments** on the *why*, never the *what* - ---- - ## DX9 FFP Porting Invoke the **`dx9-ffp-port` skill** before editing `renderer.cpp`, `ffp_state.cpp`, `remix-comp-proxy.ini`, or draw routing; porting a game for RTX Remix; diagnosing VS constants, vertex declarations, matrix mapping, or skinning; or building/deploying a remix-comp-proxy patch. diff --git a/.cursor/agents/static-analyzer.md b/.cursor/agents/static-analyzer.md deleted file mode 100644 index b256746b..00000000 --- a/.cursor/agents/static-analyzer.md +++ /dev/null @@ -1,186 +0,0 @@ ---- -name: static-analyzer -description: Offline PE binary analysis using retools. Dispatch this subagent for decompilation, disassembly, xrefs, string/pattern search, struct reconstruction, callgraphs, vtable/RTTI resolution, crash dump analysis, bootstrapping new binaries, signature DB operations, context assembly, and any static analysis task. Use proactively whenever static analysis is needed. -model: inherit -readonly: false ---- - -You are a reverse engineering analyst specializing in static analysis of PE binaries (.exe and .dll). You run offline analysis tools and return structured findings to the orchestrating agent. - -## Setup - -On first invocation, read the full tool catalog at `.cursor/rules/tool-catalog.mdc` in the working directory. It contains exact syntax, flags, and caveats for every tool. - -## Pre-flight Checks - -Before any analysis, run these checks in order: - -**1. Verify install**: Run `python verify_install.py` on first invocation. If pyghidra/Ghidra/Java show as WARN, run `python verify_install.py --setup` to auto-download JDK 21 + Ghidra + pyghidra. One-time ~600MB download. - -**2. Signature DB**: If `retools/data/signatures.db` does not exist, pull it first: -```bash -test -f retools/data/signatures.db || python retools/sigdb.py pull -``` - -**3. Bootstrap**: Check if the project KB needs bootstrapping: -```bash -grep -cE '^[@$]|^struct |^enum ' patches//kb.h 2>/dev/null || echo 0 -``` -If the count is under 50 (or the file doesn't exist), run `python -m retools.bootstrap --project ` first. A KB file that exists but contains only section-header comments is **sparse** and must be bootstrapped. Do not skip bootstrap just because the file exists. - -**4. Ghidra project**: Check if a Ghidra project exists for the binary: -```bash -python retools/pyghidra_backend.py status --project patches/ -``` -If "Not analyzed", run `python retools/pyghidra_backend.py analyze --project patches/`. Takes 2-15 minutes, but all subsequent decompilations via pyghidra are near-instant. - -**5. Index**: Check whether the project has an index.db and what's in it before scanning the binary yourself: -```bash -python -m retools.index status -``` -If `funcs`/`xrefs` show `source='bootstrap'` only (or the table is empty), and a Ghidra project exists, run `pyghidra_backend.py export` to seed authoritative facts — see "Query-first workflow" below. - -## Running Tools - -Run all tools from the repo root. Use `python -m retools.` or `python retools/.py` syntax: - -### Decompilation -- Ghidra primary, r2ghidra fallback - -**pyghidra is the primary backend** once a Ghidra project exists — better MSVC type propagation, library call resolution, larger function scope detection, and its facts can be exported into `index.db` for instant SQL lookups later: -``` -python retools/pyghidra_backend.py decompile binary.exe 0x401000 --project patches/proj -``` - -**r2ghidra is the zero-setup fallback and second opinion** — no Ghidra install required, better `__thiscall` recovery on small functions, no JVM startup, and useful to cross-check a pyghidra decompile that looks wrong: -``` -python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h -python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h --backend pdg -``` - -**Auto mode (tries pyghidra first, falls back to r2ghidra)** — routing unchanged: -``` -python -m retools.decompiler binary.exe 0x401000 --types patches/proj/kb.h --project patches/proj -``` - -When told to use a specific backend, use it. Otherwise prefer auto mode with both `--types` and `--project`. - -**Ghidra daemon**: if `python -m retools.ghidra_server ` is running (port 27043; livetools owns 27042), `decompile`/`export`/`kb-apply` route through it automatically and repeat calls become sub-second. Warm the server yourself before a batch of decompiles on the same project: `python -m retools.ghidra_server --idle 600` (background it). `RETOOLS_GHIDRA_COLD=1` or `--cold` forces a cold in-process run when you need to bypass the daemon. The daemon records its pid/port/project/binary in `patches//ghidra/.state.json`, deleted on shutdown once the Ghidra program is closed and the Windows `.rep` lock is released. - -### Query-first workflow - -Before re-scanning a binary with xrefs/datarefs/search/funcinfo, check whether `index.db` already has the answer — a SQL query against a local file is cheaper than re-disassembling: - -```bash -python -m retools.index status # per-table counts + schema_version -python -m retools.query --list-tables # confirm what's queryable -python -m retools.query --schema funcs # PRAGMA table_info before writing joins -python -m retools.query "SELECT * FROM callers WHERE callee_addr=0x401000" -python -m retools.query "SELECT * FROM grep WHERE name LIKE '%Ground%'" --json -``` - -Only fall back to `xrefs.py`/`datarefs.py`/`search.py`/`funcinfo.py` for facts `index.db` doesn't have yet (e.g. no `export` has run, or the question needs a live disassembly detail not captured by the schema). - -**Hard pushdown rule**: `decompile` and `export` require a specific function address (or, for `export`, an analyzed program) — never invoke them without one, or you decompile/scan the whole binary instead of the function you actually need. If you don't have an address yet, get one from `query`, `search`, or `xrefs` first. - -**Read-First mutation discipline**: `kb-apply` mutates the Ghidra project. Always decompile or `query` the target function first to confirm the current name/prototype, run `kb-apply`, then **re-decompile the same function** to verify the change landed before reporting it as done. `kb-apply` is idempotent — re-running it should produce stable counts and no errors, so if a second run changes anything, treat that as a bug, not expected behavior. - -**Cost guard**: run `export` once per analysis pass (after `kb-apply`, so exported names reflect it), not once per query — repeated `export` calls re-walk the whole program for no benefit once `index.db` is current. - -### Other tools -``` -python -m retools.search binary.exe strings -f "error" --xrefs -python -m retools.xrefs binary.exe 0x401000 -t call -python -m retools.callgraph binary.exe 0x401000 --up 3 -python -m retools.structrefs binary.exe --aggregate --fn 0x401000 --base esi -python -m retools.dumpinfo crash.dmp diagnose --binary d3d9.dll -python -m retools.throwmap d3d9.dll match --dump crash.dmp -python -m retools.bootstrap binary.exe --project MyGame -python -m retools.sigdb scan binary.exe --db retools/data/signatures.db -python -m retools.sigdb identify binary.exe 0x401000 --db retools/data/signatures.db -python -m retools.sigdb fingerprint binary.exe -python -m retools.context assemble binary.exe 0x401000 --project MyGame -python retools/pyghidra_backend.py analyze binary.exe --project patches/MyGame -python retools/pyghidra_backend.py status binary.exe --project patches/MyGame -python retools/pyghidra_backend.py export binary.exe --project patches/MyGame -python retools/pyghidra_backend.py kb-apply binary.exe --project patches/MyGame --kb patches/MyGame/kb.h -python -m retools.index status MyGame -python -m retools.query MyGame "SELECT * FROM funcs WHERE name LIKE '%Update%'" -python -m retools.ghidra_server MyGame --idle 600 -``` - -If `retools/data/signatures.db` is missing, run `python -m retools.sigdb pull` to download it. - -Collect MORE information per command run. Prefer wide queries over narrow ones — a single decompilation with `--types` is better than five disassembly snippets. - -Always pass `--types ` to `decompiler.py` when a KB file exists for the project. - -## Knowledge Base - -When you discover something significant, update the project KB file (`patches//kb.h`). - -Format: -```c -// Structs, enums, typedefs — no prefix -struct Foo { int x; float y; }; -enum Mode { MODE_A=0, MODE_B=1 }; - -// Function signatures — @ prefix -@ 0x401000 void __cdecl ProcessInput(int key); - -// Global variables — $ prefix -$ 0x7C5548 Object* g_mainObject -``` - -Update KB when you: identify a function's purpose, reconstruct a struct, identify a global, find magic constants, or resolve RTTI class names. - -## What NOT to Do - -- Do NOT use `livetools` commands — those require a live process and are handled by the main agent -- Do NOT use `graphics.directx.dx9.tracer` — capture and trigger are handled by the main agent -- Do NOT edit source code files — only update KB files and write analysis notes to `patches/` - -## Output - -Write findings to the appropriate file, creating it if needed. Append — do not overwrite previous findings. - -- **Default**: `patches//findings.md` -- **If told to use r2ghidra for a dual-backend comparison**: `patches//findings_r2.md` - -Use clear headings per analysis task so the main agent can read specific sections. - -Format: -```markdown -## — - -### Summary - - -### Key Addresses -| Address | Description | -|---------|-------------| -| 0x401000 | FunctionName — what it does | - -### Details - - -### Suggested Live Verification - -``` - -Also update `patches//kb.h` with any new function signatures, structs, or globals discovered. - -In your return message, state the file path you wrote to and give a brief summary. The main agent will read the file for full details. - -Update your agent memory with significant architectural discoveries, identified subsystems, and class hierarchies that will be useful in future sessions. - -## Routing to Adjacent Skills/Docs - -This agent owns offline static analysis. Hand off to the right reference/skill instead of improvising: - -| Need | Go to | -|------|-------| -| Full tool syntax, flags, caveats for any retools/DX-script/dumpinfo tool, and run-directly vs delegate guidance | `.cursor/rules/tool-catalog.mdc` | -| Bootstrap ordering, parallel dual-backend runs, delegation table | `.cursor/rules/subagent-workflow.mdc` | -| Attaching to a live process, breakpoints, tracing, memory patching | `/dynamic-analysis` skill (main agent only — this agent must not use livetools) | -| Porting a DX9 game to FFP for RTX Remix (renderer.cpp, ffp_state, vertex decls, skinning) | `dx9-ffp-port` skill | -| D3D9-specific static questions (VS/PS constants, render states, vertex formats) | DX analysis scripts (`rtx_remix_tools/dx/scripts/`) before general retools | diff --git a/.cursor/agents/web-researcher.md b/.cursor/agents/web-researcher.md deleted file mode 100644 index a40c7c8e..00000000 --- a/.cursor/agents/web-researcher.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -name: web-researcher -description: Web research and documentation lookups. Dispatch this subagent for API references, library documentation, SDK docs, file format specs, protocol details, or any question requiring external knowledge. Use proactively when external docs are needed. -model: inherit -readonly: true ---- - -You are a technical research assistant supporting a reverse engineering workflow. You fetch documentation, API references, and technical specs from the web and return concise, actionable findings. - -## Tools - -- **WebFetch**: Fetch and extract content from a specific URL -- **WebSearch**: Search the web for technical information -- **Context7 MCP**: Use `resolve-library-id` then `query-docs` for library-specific documentation (DirectX, Win32 API, game engine docs, etc.) -- **Read**: Read local files for context about what's being researched - -## How to Work - -1. Understand what the caller needs — a specific API signature, a file format layout, a protocol detail, etc. -2. Search or fetch the most authoritative source (MSDN, official docs, specs) -3. Extract the specific information needed — don't return entire pages -4. Format findings for direct use in reverse engineering or code writing - -## Output - -Return concise, structured results: -- The specific answer or data requested -- Key details (function signatures, struct layouts, enum values, constants) -- Source URL for reference -- Any caveats or version-specific differences - -Do NOT return long summaries or background context unless specifically asked. The caller already knows the domain — they need the specific data point. diff --git a/.cursor/rules/code-comments.mdc b/.cursor/rules/code-comments.mdc deleted file mode 100644 index 02315387..00000000 --- a/.cursor/rules/code-comments.mdc +++ /dev/null @@ -1,32 +0,0 @@ ---- -description: Code commenting principles -- what to write, what to remove, when code should speak for itself -alwaysApply: true ---- - -# Code Comments - -## Principle - -Each file reads as if it was always designed this way. Comments guide the next developer, not narrate the development journey. - -Note: These rules are not exhaustive. Extrapolate from the principles and examples to the specific context you are working in. - -## Remove - -- **Implementation backstories**: "We do this because the other day X happened" -- **Obvious narration**: "Create the attribute", "Loop through keys", "Check if valid" -- if the code says it, the comment is noise -- **Debugging breadcrumbs**: "Without this, subsequent tests may see the modifier key as still held" -- **Trial-and-error reasoning**: "We tried X but it caused Y so we do Z instead" - -## Keep - -- **Non-obvious design decisions**: stated as *what* and *why this design*, not *what happened to us* -- **Tricky invariants**: conditions that would be easy to accidentally break -- **API contracts**: docstrings on public methods with Args, Returns, Raises - -## Prefer Instead - -- **Rename** a variable or function to be self-explanatory rather than adding a comment -- **Docstrings** on classes and public methods (Google style: `Args:`, `Returns:`, `Raises:`) -- **Type hints** over comments about expected types -- **Short inline comments** on the *why*, never the *what* diff --git a/.cursor/rules/dx9-ffp-port.mdc b/.cursor/rules/dx9-ffp-port.mdc deleted file mode 100644 index 1c006da1..00000000 --- a/.cursor/rules/dx9-ffp-port.mdc +++ /dev/null @@ -1,281 +0,0 @@ ---- -description: DX9 FFP Proxy porting guide for RTX Remix compatibility. Use when porting a DX9 shader-based game to the fixed-function pipeline. -alwaysApply: false ---- - -# DX9 FFP Proxy — Game Porting Guide - -You are helping a user port a DX9 shader-based game to the fixed-function pipeline. Each game folder under `patches//` is a self-contained remix-comp-proxy project (copied from the template at `rtx_remix_tools/dx/remix-comp-proxy/`). The goal is RTX Remix compatibility: Remix requires FFP geometry to inject path-traced lighting and replaceable assets. Also use the Vibe RE tools (retools, livetools) for static and dynamic analysis to assist with developing this wrapper. They are meant to be used together. - -**SKINNING IS OFF BY DEFAULT.** Do NOT enable skinning, modify skinning code, or discuss skinning infrastructure unless the user explicitly asks for character model / bone / skeletal animation support. Until then, treat skinning as non-existent. When the user does request it, read `src/comp/modules/skinning.hpp` and `src/comp/modules/skinning.cpp` for the full implementation. - -**SKINNING APPROACH: FFP indexed vertex blending, NOT CPU matrix math.** When skinning is enabled, keep BLENDINDICES and BLENDWEIGHT in the vertex declaration and buffer, upload bone matrices via `SetTransform(D3DTS_WORLDMATRIX(n), &boneMatrix[n])`, enable `D3DRS_INDEXEDVERTEXBLENDENABLE = TRUE`, and set `D3DRS_VERTEXBLEND` to the weight count. CPU-side vertex skinning is a **last resort** -- it is extremely expensive and tanks frame rate. Always prefer the hardware path. - ---- - -## What remix-comp-proxy Does - -Each game's remix-comp-proxy folder is a C++20 compatibility mod based on remix-comp-base that intercepts `IDirect3DDevice9` and: - -1. Captures vertex shader constants (View, Projection, World matrices) from `SetVertexShaderConstantF` -2. Parses `SetVertexDeclaration` to detect per-element attributes: BLENDWEIGHT+BLENDINDICES (skinned), POSITIONT (screen-space), NORMAL presence, and per-element byte offsets and types -3. Routes `DrawIndexedPrimitive` by vertex layout: - - No NORMAL -> HUD/UI pass-through (uses different VS constant layout than world geometry) - - Skinned with skinning module enabled -> FFP indexed vertex blending - - Rigid 3D (has NORMAL) -> NULLs shaders, applies FFP transforms, draws -4. Routes `DrawPrimitive` by declaration state: world-space draws (have decl, no POSITIONT, not skinned) engage FFP; screen-space and no-decl draws pass through -5. Applies captured matrices via `SetTransform` (FFP) -6. Sets up texture stages and lighting for FFP rendering (stages 1-7 disabled to prevent stale auxiliary textures reaching Remix) -7. Chain-loads RTX Remix (`d3d9_remix.dll`) - -## Source File Map - -| File | Role | -|------|------| -| `src/comp/main.cpp` | DLL entry, module loading, initialization | -| `src/comp/modules/renderer.cpp` | Draw call routing -- `on_draw_indexed_prim()` and `on_draw_primitive()` | -| `src/comp/modules/renderer.hpp` | Renderer class, `drawcall_mod_context` for save/restore state | -| `src/comp/modules/d3d9ex.cpp` | `IDirect3DDevice9` hook layer -- intercepts all 119 methods | -| `src/comp/modules/d3d9ex.hpp` | D3D9 hook declarations | -| `src/comp/modules/skinning.cpp` | Skinning module (vertex expansion, bone upload, FFP blending) | -| `src/comp/modules/skinning.hpp` | Skinning class interface | -| `src/comp/modules/diagnostics.cpp` | Diagnostic logging to `rtx_comp/diagnostics.log` | -| `src/comp/modules/diagnostics.hpp` | Diagnostics class interface | -| `src/comp/modules/imgui.cpp` | ImGui debug overlay (F4 toggle) | -| `src/shared/common/ffp_state.cpp` | FFP state tracker -- engage/disengage, matrix transforms, texture stages | -| `src/shared/common/ffp_state.hpp` | `ffp_state` class with all state accessors | -| `src/shared/common/config.cpp` | INI config parser for `remix-comp-proxy.ini` | -| `src/shared/common/config.hpp` | Config structures: `ffp_settings`, `skinning_settings`, etc. | -| `remix-comp-proxy.ini` (in `assets/`) | Runtime config: `[FFP]`, `[Skinning]`, `[Diagnostics]`, `[Remix]`, `[Chain]` | -| `build.bat` | Build script: outputs d3d9.dll proxy | - -The codebase is C++20 with a `build.bat` build script, component module system for extensibility. - -## What Needs to Change Per Game - -The VS constant register layout is defined in `src/shared/common/ffp_state.hpp` as member defaults. Edit these when porting, then rebuild: - -```cpp -int vs_reg_view_start_ = 0; int vs_reg_view_end_ = 4; -int vs_reg_proj_start_ = 4; int vs_reg_proj_end_ = 8; -int vs_reg_world_start_ = 16; int vs_reg_world_end_ = 20; -int vs_reg_bone_threshold_ = 20; // first register treated as bone palette -int vs_regs_per_bone_ = 3; // 3 = 4x3 packed, 4 = full 4x4 -int vs_bone_min_regs_ = 3; // min count to qualify as bone upload -``` - -**Bone config:** Run `find_skinning.py` to determine bone start register and upload pattern. Some games upload all bones at once; others upload in groups until hitting a max (e.g., groups of 15, max 75). If grouped, lower `vs_bone_min_regs_`. If bone uploads overlap with non-bone constants, raise `vs_reg_bone_threshold_`. - -Beyond the INI config, users may need to modify: -- `renderer.cpp` `on_draw_indexed_prim()` -- draw call routing (which draws get FFP vs shader pass-through) -- `renderer.cpp` `on_draw_primitive()` -- UI/particle handling -- `ffp_state.cpp` `setup_lighting()`, `setup_texture_stages()`, `apply_transforms()` -- FFP render state and matrix configuration -- `AlbedoStage` in `remix-comp-proxy.ini` `[FFP]` section -- which texture stage holds the diffuse/albedo - -## Porting Workflow - -Follow these steps in order for ideal results. Each step depends on the previous. Be sure to use the Vibe Reverse Engineering tools (retools, livetools) for static and dynamic analysis as well. You do not need to strictly follow the order laid out here. - -### Step 1a: Static Analysis - -Run the analysis scripts to understand the game's D3D9 usage: - -```bash -python rtx_remix_tools/dx/scripts/find_d3d_calls.py "" -python rtx_remix_tools/dx/scripts/find_vs_constants.py "" -python rtx_remix_tools/dx/scripts/decode_vtx_decls.py "" --scan -python rtx_remix_tools/dx/scripts/find_device_calls.py "" -python rtx_remix_tools/dx/scripts/find_skinning.py "" -python rtx_remix_tools/dx/scripts/find_blend_states.py "" -``` - -Key things to find: -- How the game obtains its D3D device (Direct3DCreate9 call site -> CreateDevice call) -- Which functions call `SetVertexShaderConstantF` and with what register/count patterns -- What vertex declaration formats the game uses (BLENDWEIGHT/BLENDINDICES = skinning) -- Where the main rendering loop/draw calls live - -### Step 1b: D3D9 Frame Trace (recommended -- fastest path to answers) - -Deploy the D3D9 tracer (`graphics/directx/dx9/tracer/bin/`) to the game directory, capture 2 frames, then run analysis. This is the fastest way to answer all three porting questions without manual RE: - -```bash -python -m graphics.directx.dx9.tracer trigger --game-dir -python -m graphics.directx.dx9.tracer analyze --shader-map -python -m graphics.directx.dx9.tracer analyze --const-provenance -python -m graphics.directx.dx9.tracer analyze --vtx-formats -python -m graphics.directx.dx9.tracer analyze --render-passes --pipeline-diagram -``` - -- `--shader-map` -- CTAB disassembly shows named parameters and register mappings (e.g. `WorldViewProj c0 4`, `WorldView c4 3`, `FogValue c8 1`). Directly reveals which constant registers hold View, Projection, and World matrices. -- `--const-provenance` -- shows which `SetVertexShaderConstantF` call set each register at each draw -- `--vtx-formats` -- groups draws by vertex declaration with full element breakdown (POSITION, NORMAL, BLENDWEIGHT, etc.) -- `--render-passes` + `--pipeline-diagram` -- shows the render pipeline structure and pass types -- `--classify-draws` -- auto-tags draws by render state (alpha, ztest, fog, etc.) - -### Step 2: Discover VS Constant Layout - -This is the **most critical** step. You must determine which VS constant registers hold View, Projection, and World matrices. - -**Static approach:** Decompile functions that call `SetVertexShaderConstantF`: -```bash -python -m retools.decompiler --types patches//kb.h -``` - -**Dynamic approach:** Trace `SetVertexShaderConstantF` calls live: -```bash -python -m livetools trace --count 50 \ - --read "[esp+8]:4:uint32; [esp+10]:4:uint32; *[esp+c]:64:float32" -``` -This captures: startRegister, Vector4fCount, and the actual float data (first 4 vec4 constants, dereferenced from `pConstantData`). - -**How to identify matrices:** -- View matrix: changes with camera movement, contains camera orientation -- Projection matrix: contains aspect ratio and FOV, rarely changes -- World matrix: changes per object, contains position/rotation/scale -- Look for 4x4 matrices (16 floats = 4 registers). Row 3 often has `[0, 0, 0, 1]` for affine transforms. - -### Step 3: Set Up Per-Game Project - -Copy the entire `rtx_remix_tools/dx/remix-comp-proxy/` folder to `patches//` (excluding `build/`). The game folder is now self-contained. Edit files directly: - -1. Edit register layout defaults in `src/shared/common/ffp_state.hpp` -2. Edit `src/comp/main.cpp`: set `WINDOW_CLASS_NAME` to the game's window class -3. Customize `src/comp/modules/renderer.cpp` draw routing if needed -4. Customize `src/comp/game/game.cpp` with game-specific hooks -5. Update `kb.h` with discovered function signatures, structs, and globals - -### Step 4: Build and Deploy - -```bash -cd patches/ -build.bat release --name -``` - -Deploy to game directory: `d3d9.dll` + `remix-comp-proxy.ini`. If using Remix, also place `d3d9_remix.dll` there. - -### Step 5: Diagnose with Log and ImGui - -The proxy writes `rtx_comp/diagnostics.log` in the game directory. After a configurable delay (default 50 seconds via `[Diagnostics] DelayMs`), it logs frames of detailed draw call data: - -- **VS regs written**: shows which constant registers the game actually fills -- **Vertex declarations**: what vertex elements each draw uses (POSITION, NORMAL, TEXCOORD, BLENDWEIGHT, etc.) -- **Draw calls**: primitive type, vertex count, index count, textures bound per stage -- **Matrices**: actual View/Proj/World values being applied - -Press **F4** to open the ImGui debug overlay, which shows the FFP debug tab with live draw call stats and state information. - -Use this to iterate: wrong matrices -> re-check register mapping. Missing textures -> adjust AlbedoStage. Objects at wrong positions -> world matrix register is wrong. - -## Architecture Details for Editing - -### Code Map: Edit vs Do-Not-Touch - -**Only edit sections marked YES or MAYBE:** - -| File / Section | Edit Per-Game? | -|----------------|----------------| -| `ffp_state.hpp` register layout defaults | **YES** -- set register layout | -| `remix-comp-proxy.ini` `[Skinning] Enabled=` | **YES** -- only after rigid FFP works | -| `remix-comp-proxy.ini` `[FFP] AlbedoStage=` | **YES** -- set albedo texture stage | -| `renderer.cpp` `on_draw_indexed_prim()` | **YES** -- main draw routing | -| `renderer.cpp` `on_draw_primitive()` | **YES** -- draw routing for non-indexed draws | -| `ffp_state.cpp` `setup_lighting()`, `setup_texture_stages()`, `apply_transforms()` | MAYBE -- tweak if game needs different FFP state | -| `ffp_state.cpp` `on_set_vertex_declaration()` | MAYBE -- element parsing; add extra usages if needed | -| `ffp_state.cpp` `on_set_vs_const_f()` | MAYBE -- dirty tracking | -| `d3d9ex.cpp` hook implementations | NO -- infrastructure | -| `ffp_state.cpp` `engage()` / `disengage()` | NO -- enter/leave FFP mode | -| `skinning.cpp` | NO -- infrastructure (no per-game edits) | -| `diagnostics.cpp` | NO -- logging infrastructure | -| `imgui.cpp` | NO -- debug overlay | - -### DrawIndexedPrimitive Decision Tree - -This is the routing logic in `renderer.cpp` `on_draw_indexed_prim()`: - -``` -viewProjValid? -+-- NO -> shader passthrough (transforms not captured yet) -+-- YES - +-- curDeclIsSkinned? - | +-- YES + skinning module -> skinning::draw_skinned_dip() - | +-- YES + no skinning -> shader passthrough - +-- NOT skinned - +-- !curDeclHasNormal -> shader passthrough (HUD/UI) - +-- hasNormal -> ffp_state::engage + rigid FFP draw -``` - -**Common per-game changes to this tree:** -- Game's world geometry omits NORMAL -> remove or change the `!cur_decl_has_normal()` filter -- Game has special passes (shadow, reflection) -> filter by shader pointer, render target, or vertex count -- Game draws UI with DrawIndexedPrimitive + NORMAL -> add a filter (e.g. check stride or texture) - -### DrawPrimitive Decision Tree - -``` -viewProjValid AND lastDecl AND !curDeclHasPosT AND !curDeclIsSkinned? -+-- YES -> ffp_state::engage (world-space particles, non-indexed geometry) -+-- NO -> shader passthrough (screen-space UI, POSITIONT, no decl, skinned) -``` - -### Skinning Data Flow - -When skinning is enabled via `[Skinning] Enabled=1` in `remix-comp-proxy.ini`: - -1. **`ffp_state::on_set_vertex_declaration()`** -- Parses `D3DVERTEXELEMENT9` array. If both BLENDWEIGHT and BLENDINDICES are present, sets `cur_decl_is_skinned_` and captures per-element byte offsets and types. - -2. **`ffp_state::on_set_vs_const_f()`** -- When a write hits registers >= `BoneThreshold` with count >= `BoneMinRegs` and divisible by `RegsPerBone`, stores `bone_start_reg_` and `num_bones_`. - -3. **`skinning::draw_skinned_dip()`** -- Locks the game's source vertex buffer, calls `expand_skin_vertex()` per vertex, caches results by hash key. - -4. **`skinning::upload_bones()`** -- Reads bone matrices from VS constants, transposes, uploads via `SetTransform(WORLDMATRIX(i))`. Sets `D3DRS_VERTEXBLEND` and `D3DRS_INDEXEDVERTEXBLENDENABLE`. - -5. **Draw** -- The expanded VB + shared declaration are bound, draw executes with FFP indexed vertex blending. After the draw, original VB/decl/textures are restored. - -### Key Component Notes - -- **`ffp_state::engage()` / `disengage()`**: `engage()` NULLs shaders, applies transforms, sets up texture stages. `disengage()` restores the game's shaders. Avoids redundant state changes between consecutive FFP draw calls. -- **`ffp_state::apply_transforms()`**: Reads from the VS constant array using the INI register settings and calls `SetTransform` with transposed matrices (D3D9 FFP expects row-major). -- **ImGui overlay (F4)**: Shows live draw call stats, FFP conversion counts, and shader pass-through counts for real-time debugging. - -## Analysis Scripts -- Entry Points, Not Endpoints - -The scripts below are fast first-pass scanners. They surface candidate addresses and call sites to give you a starting point. They do **not** replace deep analysis -- always follow up with `retools` and `livetools` to understand what is actually happening. - -| Script | What it surfaces | -|--------|------------------| -| `scripts/find_d3d_calls.py ` | D3D9/D3DX imports and call sites | -| `scripts/find_vs_constants.py ` | `SetVertexShaderConstantF` call sites and register/count args | -| `scripts/find_ps_constants.py ` | `SetPixelShaderConstantF/I/B` call sites and register/count args | -| `scripts/find_device_calls.py ` | Device vtable call patterns and device pointer refs | -| `scripts/find_render_states.py ` | SetRenderState args decoded by category (culling, blending, depth, fog) | -| `scripts/find_texture_ops.py ` | Texture pipeline: SetTexture stages, TSS ops, sampler states | -| `scripts/find_transforms.py ` | SetTransform/MultiplyTransform types (World, View, Projection, Texture) | -| `scripts/find_surface_formats.py ` | CreateTexture/RenderTarget/DepthStencil format extraction | -| `scripts/find_stateblocks.py ` | State block creation, recording, and apply patterns | -| `scripts/decode_fvf.py ` | FVF bitfield decode from SetFVF calls | -| `scripts/find_vtable_calls.py ` | D3DX constant table usage and D3D9 vtable calls | -| `scripts/decode_vtx_decls.py --scan` | Vertex declaration formats (BLENDWEIGHT/BLENDINDICES -> skinning) | -| `scripts/find_shader_bytecode.py ` | Embedded shader bytecode extraction (version, size) | -| `scripts/classify_draws.py ` | Draw call classification by state context (FFP/shader/hybrid) | -| `scripts/find_matrix_registers.py ` | Identify View/Proj/World registers (CTAB + frequency + layout suggestion) | -| `scripts/find_skinning.py ` | Consolidated skinning analysis: skinned decls, bone palettes, blend states, suggested INI | -| `scripts/find_blend_states.py ` | D3DRS_VERTEXBLEND + INDEXEDVERTEXBLENDENABLE + WORLDMATRIX transforms | -| `scripts/scan_d3d_region.py 0xSTART 0xEND` | Map all D3D9 vtable calls in a code region | - -Scripts are at `rtx_remix_tools/dx/scripts/`. - -## Common Pitfalls - -- **Concatenated WVP/VP instead of separate matrices**: This is the **#1 Remix porting mistake**. Remix requires separate World, View, and Projection matrices passed via `SetTransform`. If the game uploads a pre-multiplied WorldViewProj or ViewProj to a single register range, the proxy gets a combined matrix it can't decompose. **Fix**: find where the game multiplies W*V*P and hook to capture individual matrices *before* concatenation. Use `find_matrix_registers.py` to detect this. -- **Matrices look wrong**: D3D9 FFP `SetTransform` expects row-major matrices. The proxy transposes them. If the game stores matrices column-major in VS constants (the common case), the transpose is correct. If the game is already row-major, remove the transpose in `ffp_state::apply_transforms()`. -- **Everything is white/black**: The game's albedo texture might be on stage 1+ instead of stage 0. Set `AlbedoStage` in `remix-comp-proxy.ini` `[FFP]` section, or trace `SetTexture` calls to find the pattern. -- **Some objects render, others don't**: `on_draw_primitive()` routes by vertex declaration -- world-space draws (have decl, no POSITIONT, not skinned) engage FFP; screen-space/no-decl pass through. `on_draw_indexed_prim()` additionally filters out draws without NORMAL as likely HUD/UI. If world geometry is missing, check whether its vertex decl has NORMAL and whether `view_proj_valid()` is true when those draws happen. -- **Skinned meshes are invisible**: Enable skinning with `[Skinning] Enabled=1` in `remix-comp-proxy.ini`. Check the log for bone count and declaration issues. -- **Game crashes on startup**: The chain-loaded Remix DLL might not be present. Set `Enabled=0` in `remix-comp-proxy.ini` `[Remix]` section to test without Remix first. -- **Geometry at origin / piled up**: World matrix register mapping is wrong. Every object gets identity world transform. Re-examine VS constant writes. -- **Characters' world geometry shifts after a skinned draw**: After uploading bone matrices, WORLDMATRIX(0) is clobbered by bone[0]. The proxy sets world dirty so `apply_transforms()` re-applies the world matrix on the next rigid draw. If this still causes issues, the bone threshold register may overlap with the world matrix register range. - -## Notes -- Do not change the diagnostic logging delay (unless specified by the user). The delay is important to ensure the user is able to get into the game with actual geometry being drawn before the logs start, otherwise they may get lost in the initial burst of draw calls during loading. -- Tell the user when you want to launch a game and have them interact with it for logging or hooking purposes. They MUST interact with the game to have this task be useful. diff --git a/.cursor/rules/no-copium.mdc b/.cursor/rules/no-copium.mdc deleted file mode 100644 index a6db1600..00000000 --- a/.cursor/rules/no-copium.mdc +++ /dev/null @@ -1,39 +0,0 @@ ---- -description: Engineering standards -- no workarounds, no duct tape, no copium -alwaysApply: true ---- - -# No Copium - -## Principle - -Every change should make the codebase better, not just make the problem go away. If a solution needs a paragraph to justify why it's not a hack, it's a hack. - -## Remove - -- **Fixes in the wrong layer**: a guard on a canvas to suppress commits that a model should own. Put the fix where the problem originates. -- **Tolerance inflation**: widening deltas or adding retries to hide flaky behavior. If the value is wrong, find out why. -- **Catch-all exception swallowing**: `try/except Exception: pass` to hide symptoms. -- **Excessive error/null handling**: adding too many error/None "if" checks. If the error is expected, handle it. If unexpected, raise it. -- **God methods**: 200+ line functions doing multiple things. Break into named steps. Focus on cognitive load. Design for fewer indentation levels. -- **Leaky abstractions**: implementation details leaking into layers/modules that should be agnostic of one another. - -## Design For - -- **Single responsibility**: one component, one job. If you need "and" to describe it, split it. -- **Ownership**: the component that creates the problem owns the fix. -- **Minimal public surface**: expose what consumers need, nothing more. - -## Commit to the New Code - -- **No legacy fallbacks**: if you replace a system, remove the old one. -- **No dead code**: commented-out blocks, unused imports, orphan functions "just in case". Version control is the safety net. -- **No multiple paths to the same result**: one way to do each thing. If two paths exist, one is wrong. -- **No half-migrations**: finish the job -- update every reference, remove old APIs. - -## Smell Tests - -- "It works if I add a sleep" -- broken data flow. -- "It works if I read from widget instead of storage" -- the two are out of sync. -- "It passes alone but fails with other tests" -- shared mutable state leaking. -- "I added a flag to skip this code path" -- why does that path run in the first place? diff --git a/.cursor/rules/project-workspace.mdc b/.cursor/rules/project-workspace.mdc deleted file mode 100644 index 9e38b27e..00000000 --- a/.cursor/rules/project-workspace.mdc +++ /dev/null @@ -1,44 +0,0 @@ ---- -description: Project workspace conventions — patches/ directory, backups, and knowledge base format -alwaysApply: true ---- - -# Project Workspace - -Use `patches//` (git-ignored) for all project-specific artifacts: -- Knowledge base files (`kb.h`) -- One-off analysis scripts -- ASI patch specs and builds -- Notes, logs, collected trace data - -Create the project subfolder on first use. - -# Backups - -Before modifying project files (proxy source, kb.h, proxy.ini, build scripts, ASI specs), create a timestamped backup in `patches//backups/`: - -``` -patches//backups/YYYY-MM-DD_HHMM_/ -``` - -Copy ALL files being modified into the backup folder. The description should be a short slug of what the update does (e.g. `added-world-matrix-regs`, `fixed-albedo-stage`, `enabled-skinning`). - -Create the backup BEFORE making changes so it captures the last known-good state. This applies to all development work — FFP proxy edits, ASI patch specs, build config changes, and any other project file modifications. - -# Knowledge Base - -When reverse engineering a binary, maintain a knowledge base file (`.h`) that accumulates discoveries. Store in `patches//kb.h`. - -**Format:** C types (no prefix), functions (`@` prefix), globals (`$` prefix): -```c -struct Foo { int x; float y; }; -@ 0x401000 void __cdecl ProcessInput(int key); -$ 0x7C5548 Object* g_mainObject -``` - -**When to update the KB:** -- When you identify a function's purpose, add `@ 0xADDR` with a descriptive name and signature -- When you reconstruct a struct (e.g., from `structrefs.py --aggregate`), add the struct definition -- When you identify a global variable via `datarefs.py`, add `$ 0xADDR` with its name and type -- When you identify magic constants, define an enum with named values -- When `rtti.py` reveals a class name, use it in struct/function names diff --git a/.cursor/rules/subagent-workflow.mdc b/.cursor/rules/subagent-workflow.mdc deleted file mode 100644 index cb37d4bf..00000000 --- a/.cursor/rules/subagent-workflow.mdc +++ /dev/null @@ -1,147 +0,0 @@ ---- -description: Subagent delegation rules — when to delegate static analysis vs run livetools directly, parallel work patterns -alwaysApply: true ---- - -# Subagent Workflow - -The main agent orchestrates and focuses on **live tools**, **dx9tracer capture**, **user interaction**, and **synthesis**. Heavy static analysis and web research are delegated to subagents so the user isn't blocked. - -## Pre-flight: Ensure Ghidra Backend - -Before first use of pyghidra, the `static-analyzer` subagent should check if Ghidra is available. Run `python verify_install.py` — if pyghidra/Ghidra shows WARN, run `python verify_install.py --setup` to auto-download JDK 21 + Ghidra 11.4.3 + pyghidra. This is a one-time setup (~600MB download). Skip if pyghidra already shows PASS. - -## Bootstrap First — New Binaries - -When analyzing a binary for the first time (no existing or sparsely populated `patches//kb.h`), **always bootstrap before other static analysis**: - -1. The `static-analyzer` subagent auto-pulls `signatures.db` if missing (pre-flight check). Spawn it to run `bootstrap.py --project ` — this seeds `patches//kb.h` with RTTI classes, CRT/library function IDs, compiler info, and propagated labels. **Bootstrap takes 2-5 minutes.** Tell the user it's running and do other work while it completes. The output goes to `patches//kb.h` — verify this file exists and has content after bootstrap returns. **Bootstrap speeds up all subsequent decompilation**: when `--types kb.h` is passed to the decompiler, it pre-analyzes every known function (`af` per KB entry) so cross-references resolve to named functions, callees get inlined signatures, and you avoid the expensive full-binary `aaa` analysis pass. -2. **In parallel**, spawn a second `static-analyzer` to run `pyghidra_backend.py analyze --project patches/`. This runs Ghidra's full analysis (PE loader, MSVC calling convention detection, type propagation, RTTI parsing) and saves a reusable project. **Takes 5-15 minutes.** Once complete, all subsequent decompilations via `--backend auto --project patches/` will use Ghidra's higher-quality output. -3. Any other static analysis subagents should run in parallel, but their decompilation output will be richer if bootstrap finishes first -4. After bootstrap, all subsequent `decompiler.py` calls **must** use `--types patches//kb.h` -5. After pyghidra analyze, all subsequent `decompiler.py` calls should also use `--project patches/` so `--backend auto` prefers Ghidra when available - -**How to detect "needs bootstrap":** Check if `patches//kb.h` exists AND has real content (function signatures `@`, globals `$`, or struct definitions beyond section headers). An empty or stub KB with only comment headers counts as sparse — bootstrap it. Quick check: `grep -cE '^[@$]|^struct |^enum ' patches//kb.h` — if the count is under 50, bootstrap. - -**How to detect "needs pyghidra analyze":** Check if `patches//ghidra/.gpr` exists. If not, spawn `pyghidra_backend.py analyze`. If kb.h also needs bootstrap, spawn both in parallel. - -## Delegation Rules - -| Task | Where | -|------|-------| -| Static analysis (`retools`: decompiler, disasm, xrefs, search, structrefs, callgraph, rtti, datarefs, dumpinfo, throwmap) | `static-analyzer` subagent | -| Web research (docs, API refs, format specs, SDK docs) | `web-researcher` subagent | -| Live tools (`livetools`: attach, trace, bp, memwatch, dipcnt, mem read/write) | Main agent — directly | -| dx9tracer trigger/capture | Main agent — directly | -| dx9tracer analyze (offline JSONL analysis) | `static-analyzer` subagent | -| Bootstrap new binary (`bootstrap.py`) | `static-analyzer` subagent -- takes 2-5 min | -| pyghidra analyze (first-time Ghidra analysis) | `static-analyzer` subagent -- takes 5-15 min | -| Decompiler with `--backend ghidra` (subsequent) | `static-analyzer` subagent -- fast (JVM ~3s + decompile <1s) | -| Bulk signature scan (`sigdb.py scan`) | `static-analyzer` subagent -- takes 1-3 min | -| Signature DB build (`sigdb.py build`) | `static-analyzer` subagent -- takes 1-5 min | -| Single function ID (`sigdb.py identify`, `fingerprint`) | Main agent -- fast (<5s) | -| Context assembly (`context.py assemble`) | Main agent -- fast (<5s) | -| Decompiler postprocess (`context.py postprocess`) | Main agent -- instant | -| Dataflow: constants + backward slice (`dataflow.py`) | Main agent -- fast (<5s) | -| File editing, patch specs, builds | Main agent — directly | -| KB updates from subagent findings | `static-analyzer` writes to `kb.h`, then `kb-apply` pushes it into Ghidra; main agent may refine | -| `index status` / `query` (SQL over `index.db`) | Main agent -- fast (<5s); prefer over xrefs/datarefs/search/funcinfo when index.db already has the answer | -| `pyghidra_backend.py export` (seed `index.db` from Ghidra) | `static-analyzer` subagent -- run once per analysis pass, after `kb-apply` | - -## Subagent Output Files - -Subagents write detailed findings to `patches//findings.md` (appended, not overwritten). When a subagent returns, it states the file path — **read the file** for full details including decompilation output, address tables, and suggested livetools commands. The return message is just a summary. - -## Parallel Work - -When both static and dynamic analysis are needed: -1. Spawn `static-analyzer` **in background** for the static questions -2. **Immediately ask the user** if the game/process is running or ask them to launch it — don't wait for static results -3. While the subagent works, prepare livetools (attach, set up traces) or discuss the approach with the user -4. Synthesize findings when the subagent returns - -Multiple `static-analyzer` instances can run in parallel for independent questions (e.g., decompiling two unrelated functions, analyzing different modules). When a subagent returns findings with multiple leads (e.g., "5 candidate functions found"), spawn parallel subagents to chase independent leads simultaneously — don't serialize them or try to analyze them yourself. - -## Dual-Backend Deep Analysis - -Ghidra (indexed into `index.db`, daemon-backed via `ghidra_server.py`, kb-applied) is the **primary** backend once a project exists — prefer it plus `retools.query` over spawning two agents for most exploratory work. Reserve the dual-agent pattern below for two specific cases: **no Ghidra project exists yet** for this binary, or **pyghidra output on a specific function looks wrong** and you need an independent r2ghidra read to cross-check it. - -When one of those applies, spawn **two parallel static-analyzer agents using different decompiler backends**: - -1. **r2ghidra agent** — uses `--backend pdg` (with `--types kb.h`), writes to `patches//findings_r2.md` -2. **pyghidra agent** — uses `pyghidra_backend.py decompile` (requires Ghidra project), writes to `patches//findings.md` - -**Why both:** Each backend has different strengths. r2ghidra is better at `__thiscall` recovery on small functions, low-level D3D details, and needs no JVM/project setup. pyghidra resolves more library calls, finds larger function scopes, propagates types better, and its output is exportable into `index.db` for future queries. Neither finds everything alone — merging both gives the most complete picture. - -**When to use dual-backend:** Only when no Ghidra project exists yet, or when pyghidra output on a specific function looks wrong. Not needed for single-function decompilation once a Ghidra project exists — use `--backend auto` (Ghidra primary, r2ghidra fallback). - -**Synthesis:** When both agents return, the main agent reads both findings files and merges them into a unified analysis. Conflicting information is resolved by checking which backend's output is more complete for that specific function. - -## Main Agent Responsibilities During Analysis - -**Do not silently wait for subagents.** While static analysis runs: -- Ask the user to launch the game/process if live verification or patching will be needed -- Discuss the approach, explain what the subagent is looking for -- Prepare livetools commands based on what you already know -- If the task involves runtime patching (disabling culling, skipping checks, etc.), assume live tools WILL be needed and prompt the user early - -## Examples - -**"Disable culling in game.exe"** -1. Spawn `static-analyzer` #1 (r2ghidra): find `SetRenderState` calls with `D3DRS_CULLMODE`, string search for "cull", xrefs --indirect to find vtable call sites. Uses `--backend pdg --types kb.h`. Writes to `findings_r2.md`. -2. Spawn `static-analyzer` #2 (pyghidra): same search strategy but decompile with `pyghidra_backend.py decompile`. Writes to `findings.md`. -3. Immediately tell the user: "Please launch the game — I'll need to attach with livetools to patch culling at runtime once I find the addresses" -4. While waiting, run `dataflow.py --constants` on any known render functions to see what cull mode constants flow in (e.g., `eax = 0x2` = D3DCULL_CW) -5. When both return, merge findings and use `livetools` to verify and patch: `mem write` to NOP the cull-enable instruction or force `D3DRS_CULLMODE` to `D3DCULL_NONE` - -**"What does function 0x401000 do?"** -1. Spawn `static-analyzer`: decompile with `--types kb.h`, get callgraph --indirect, xrefs -2. Run `dataflow.py 0x401000 --constants` inline — see what constants flow through -3. Tell the user: "Static analysis is running. Want me to also trace this function live to see actual register values and call frequency?" -4. If yes, attach with `livetools trace 0x401000 --count 20 --read` - -**"Find who writes to address 0x7A0000"** -1. Spawn `static-analyzer`: `datarefs.py` for static references -2. Ask user: "Is the game running? I can also set a `livetools memwatch` to catch runtime writes that static analysis might miss" -3. Combine static xrefs with live write traces for complete picture - -**"Why does the game crash in d3d9.dll?"** -1. Spawn `static-analyzer`: `dumpinfo.py diagnose`, `throwmap.py match` -2. Tell the user: "Analyzing the crash dump. If you can reproduce the crash, launch the game and I'll attach to catch it live" - -**"Analyze game.exe for the first time"** -1. Spawn `static-analyzer` #1 in background: `bootstrap.py game.exe --project MyGame` -2. Spawn `static-analyzer` #2 in background: `pyghidra_backend.py analyze game.exe --project patches/MyGame` -3. Tell the user: "Bootstrapping the binary and running Ghidra analysis in parallel. Bootstrap ~3 min, Ghidra ~10 min." -4. While both run, use `sigdb.py fingerprint` (fast) to tell the user the compiler version -5. When bootstrap returns, read the report and summarize coverage to the user -6. When pyghidra returns, tell the user: "Ghidra analysis complete. Subsequent decompilations will use Ghidra's higher-quality output." -7. All subsequent decompilations use `--types patches/MyGame/kb.h --project patches/MyGame` - -## Anti-Patterns - -**The Cascade Trap.** The main agent runs "one quick xref" -> sees an interesting caller -> decompiles it -> follows another xref -> now it's doing a full static analysis session while the user waits. If you catch yourself about to run a second retools command, stop and delegate everything to a subagent. - -**Duplicating subagent work.** After spawning a static-analyzer, don't also grep/search for the same thing yourself. Trust the subagent. Use the wait time for livetools or user interaction. - -**Silent waiting.** Spawning a subagent and then producing no output until it returns. Always talk to the user or do livetools work while subagents run. - -## When NOT to Delegate - -- Allowlisted fast commands (see CLAUDE.md Delegation Rule): `sigdb identify`, `sigdb fingerprint`, `context assemble`, `context postprocess`, `readmem.py`, `asi_patcher.py build` -- Anything requiring a live attached process — always main agent -- Iterative debugging loops where each step depends on the last live result — main agent - -Everything else in `retools.*` goes to a `static-analyzer` subagent. No exceptions. - -## Cursor Subagent Setup - -Cursor supports parallel subagent dispatch via the `Task` tool. To make it work: - -1. **Select a specific model** in the model dropdown (e.g. `claude-sonnet-4`, `gpt-4o`). Do NOT use "Auto" or "Composer" — these break the Task tool binding. -2. The `static-analyzer` and `web-researcher` agents in `.cursor/agents/` are loaded automatically. The parent agent reads their `description` fields to decide when to delegate. -3. **Parallel dispatch**: Send multiple `Task` calls in a single message to run subagents simultaneously. Each gets its own context window. -4. Subagents inherit all tools from the parent (including MCP tools). There is no `tools` field in Cursor agents — use `readonly: true` to restrict write access for read-only agents like `web-researcher`. -5. **`is_background: true`** makes a subagent non-blocking (parent continues while subagent works). Works at level 1 only — nested subagents block synchronously. - -**If subagent dispatch is unavailable** (wrong model selected, CLI mode, or Task tool not bound): follow the delegation rules yourself — do not run multiple retools commands in sequence. Collect all static analysis questions and run them in a single comprehensive pass. The principle is the same — avoid the Cascade Trap where "one quick xref" turns into a full analysis session. diff --git a/.cursor/rules/tool-catalog.mdc b/.cursor/rules/tool-catalog.mdc deleted file mode 100644 index cb94e388..00000000 --- a/.cursor/rules/tool-catalog.mdc +++ /dev/null @@ -1,320 +0,0 @@ ---- -description: Catalog of all RE tools -- pick the right tool for the job -alwaysApply: true ---- - -# Tool Catalog - -**BEFORE FIRST USE**: Run `python verify_install.py` from the repo root. Do NOT proceed with any tool until every required check passes. If pyghidra/Ghidra shows as WARN, run `python verify_install.py --setup` to auto-download JDK 21 + Ghidra + pyghidra. Common failures: missing `git lfs pull` (LFS pointer stubs instead of binaries), missing `pip install -r requirements.txt`. - -All tools work on PE binaries (`.exe` and `.dll`). `$B` = path to binary, `$VA` = hex address, `$D` = path to minidump `.dmp` file. Check tools help command for more info on usage. -Always consult this catalog before making any move to take the best decision on what to use with best bang for your buck. -Run all tools from the repo root directory using `python -m ` syntax (e.g. `python -m retools.search`). Do NOT modify files inside `retools/`, `livetools/`, or `graphics/` unless working on the tools themselves. - -IMPORTANT: Collecting MORE INFORMATION per command run is encouraged over minor snippets of data/output that don't reveal the whole picture. - -## Decision Guide - -### Run Directly (main agent) - -These are fast (<5s) and allowed inline: - -- "What compiler built this?" → `python -m retools.sigdb fingerprint $B` -- "Is this a known library function?" → `python -m retools.sigdb identify $B $VA` -- "Get full context before reasoning about a function" → `python -m retools.context assemble $B $VA --project $P` -- "Clean up decompiler output with known names" → pipe through `python -m retools.context postprocess` -- "Read a typed value from the PE file" → `python -m retools.readmem $B $VA $TYPE` -- "What constant flows into this register?" → `python -m retools.dataflow $B $VA --constants` -- "Trace where this value comes from" → `python -m retools.dataflow $B $VA --slice TARGET_VA:REG` -- "Build an ASI patch DLL" → `python -m retools.asi_patcher build spec.json` -- "Does a Ghidra project exist for this binary?" → `python retools/pyghidra_backend.py status $B --project $P` -- "What's in this game's index?" → `python -m retools.index status [--db PATH]` -- "Query facts already indexed (funcs, names, xrefs, strings, imports, callers/callees)" → `python -m retools.query "SQL" [--db PATH] [--json] [--list-tables] [--schema TABLE]` - -### Delegate to `static-analyzer` subagent - -Everything else. Tell the subagent WHAT you need, not HOW to run it — it has the full tool catalog. - -**D3D9-specific questions?** Check the DX analysis scripts section below first — they're faster and more targeted than general retools for D3D API usage, device calls, shader constants, and vertex formats. - -- "What does this function do?" → decompile + callgraph + xrefs + dataflow --constants -- "Who calls this function?" → xrefs or callgraph --up (prefer `retools.query "SELECT * FROM callers WHERE callee_addr=..."` when index.db exists) -- "What does this function call?" → callgraph --down (add --indirect for vtable calls) (prefer `retools.query "SELECT * FROM callees WHERE caller=..."` when index.db exists) -- "Who calls this virtual method?" → xrefs --indirect + filter by vtable slot offset -- "What constant reaches this call?" → dataflow --constants or --slice VA:REG -- "Resolve a switch/jump table" → cfg (auto-resolves MSVC switch patterns) -- "Find a string and who uses it" → string search with xrefs (prefer `retools.query "SELECT * FROM grep WHERE name LIKE '%...%'"` when index.db exists) -- "Where is this global read/written?" → datarefs (prefer `retools.query` against `names`/`xrefs` when index.db exists) -- "Where is struct field +0x54 used?" → structrefs -- "What does this struct look like?" → structrefs --aggregate -- "What C++ class is this vtable?" → RTTI resolution -- "What type was a caught/thrown exception?" → RTTI throwinfo -- "Find instructions using a specific constant" → instruction search -- "What crashed and what was the error message?" → dump diagnosis + throwmap -- "Map all throw sites to error strings" → throwmap list -- "First time analyzing a binary?" → bootstrap (2-5 min) + pyghidra analyze (5-15 min) in parallel -- "Bulk signature scan" → sigdb scan (1-3 min) -- "Seed/refresh the index from a Ghidra project" → `pyghidra_backend.py export` (funcs/names/xrefs/blocks, source='ghidra', authoritative over provisional bootstrap rows) -- "Push kb.h names/prototypes into the Ghidra project" → `pyghidra_backend.py kb-apply` (idempotent) -- Any combination of the above - -### Live tools (main agent, requires attached process) - -- "Is this function reached at runtime?" → `livetools trace` or `collect` -- "What are the actual register values?" → `livetools trace --read` or `bp` + `regs` -- "How many draw calls happen?" → `livetools dipcnt` -- "Who writes to this memory address?" → `livetools memwatch` -- "Send keys/clicks to the game window?" → `livetools gamectl` - -### DX analysis scripts (main agent, fast first-pass) - -These are targeted D3D9 scanners under `rtx_remix_tools/dx/scripts/`. They run in seconds and surface D3D-specific patterns that general-purpose retools would take longer to find. **Use these BEFORE retools** when the question is about D3D9 API usage, device calls, shaders, or vertex formats. Run as `python rtx_remix_tools/dx/scripts/