diff --git a/.gitignore b/.gitignore index e03787c..e36b784 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,7 @@ docs/superpowers/ spikes/s1/sessions/ spikes/s1/results/raw/ spikes/s3/results/raw/ + +# Python bytecode from the spike scripts +__pycache__/ +*.pyc diff --git a/README.md b/README.md index 2aec297..d154ab0 100644 --- a/README.md +++ b/README.md @@ -307,6 +307,34 @@ required was present in the artefacts a tool would actually hold. The disclosures are in [`spikes/s3/results/summary.md`](spikes/s3/results/summary.md). +## Reading real transcripts + +Both measurements above run through the layer that reads a harness's own +transcripts, so that layer is upstream of them. Measured 2026-07-26 against +1,681 local transcript files spanning 18 harness versions; method and figures +are under [`spikes/s2/`](spikes/s2/). + +One parser read **all 1,681 files across all 18 versions**, refusing none. The +format does churn, but additively: every field the parser needs was present in +every version, and the changes land in metadata a replay does not read. Tool +calls and their results paired exactly, with none unmatched across 35,510. + +Three assumptions did not survive. `session_id` is not another spelling of +`sessionId` — it names the session a transcript was resumed from, so treating +them as the same merges unrelated sessions. Some sessions move between working +directories partway through, so pinning one workspace per session is not always +right. And a small share of subagent transcripts cannot be traced back to the +call that started them, which has to be reported rather than quietly dropped. + +**Redaction is not ready, and that is the honest limit here.** Exams are meant to +be shareable, which means a secret scanner that can be trusted. The one built +here catches every planted secret, and also flags one local file in two — +mostly on ordinary code, because a variable named `..._per_token` looks like a +credential to a pattern matcher. A scanner that noisy either blocks every export +or teaches you to ignore it. No shareable fixtures were produced for that +reason, and the criterion it was written against — catching planted secrets — +turned out to measure the easy half of the problem. + ## Roadmap Ordered by dependency. No dates: the order is a commitment, the schedule is not. diff --git a/spikes/README.md b/spikes/README.md index d4e42d3..013450c 100644 --- a/spikes/README.md +++ b/spikes/README.md @@ -10,6 +10,9 @@ method cannot be inspected is an assertion rather than evidence. - `s1/` — variance calibration. Answers whether run-to-run variance at the assertion level is small enough for drift to be detectable at an affordable number of replays. Protocol and pass bar are fixed in advance; see the README's Status section. +- `s2/` — parser conformance. Answers whether transcript schema churn across harness + releases breaks a single capture parser, and whether it can fail closed rather than + silently misparse. Protocol and pass bar are fixed in advance in `s2/README.md`. - `s3/` — distillation rehearsal. Answers whether one real, messy, multi-turn session can be folded into a single headless instruction that replays. S1 measured hand-authored exams; this measures a distilled one. Protocol and pass bar are fixed diff --git a/spikes/s2/README.md b/spikes/s2/README.md new file mode 100644 index 0000000..66d8a36 --- /dev/null +++ b/spikes/s2/README.md @@ -0,0 +1,118 @@ +# S2 — parser conformance: protocol and pass bar + +**Pre-registered.** This file was written and committed before the parser was run +against the corpus. The results land in `results/summary.md` and are read against +the bar below, whichever way they come out. Moving the bar afterwards requires +writing down why, first, in the same file. + +## The question + +The design's capture adapter parses a harness's own transcripts, with "parsers +versioned per harness release and selected by the `version` field each record +carries", and degrades to a named `UNSUPPORTED-VERSION` rather than producing a +wrong exam. That is a bet on two things: that the format is stable enough for one +adapter to span releases, and that the version key is actually there to key on. + +S2 asks: **does transcript schema churn across harness releases break a single +parser, and can it be made to fail closed rather than silently misparse?** + +S1 established that assertions reproduce. S3 established that a real session can +be distilled — and that the Distiller needs tool inputs and tool *results*, not a +flattened chat. Both of those run through the capture adapter, so the adapter's +ability to read every version is upstream of both. + +## What the survey already established + +Written down so the pre-registration is not mistaken for more than it is. The +schema census (`schema-census.py`) was run before this bar and its findings are +inputs to it, not results of it: + +- 198 project directories, 719 project-root transcripts, 962 subagent sidecar + transcripts, 323,430 records, **18 harness versions** (`2.1.142` … `2.1.220`). +- Four content-bearing record types — `assistant`, `user`, `attachment`, + `system` — appear in **18 of 18** versions, each with a stable core key set + present in every version. +- Five content block types (`text`, `tool_use`, `tool_result`, `thinking`, and a + bare string form) appear in 18 of 18 versions. `image` appears in 6 and + `fallback` in 3. +- Churn is additive and concentrated in the periphery: `system` carries 33 + version-dependent keys, `user` 13, `assistant` 13. None of the version-dependent + keys are in the core sets. +- **35% of records (113,791) carry no `version` field at all.** They are twelve + bookkeeping types — `queue-operation`, `last-prompt`, `ai-title`, `mode`, + `pr-link`, and others. The split is exact: the four content types *always* + carry a version, the twelve bookkeeping types *never* do, and **no type is + inconsistent**. + +That last point is the one that shapes the bar. Per-record version selection +works for the records that matter, but a fail-closed rule that rejects any record +without a recognised version would reject every real transcript in the corpus. +Whether the parser gets that right is not yet measured. + +## What is being built + +A parser, not a product. It extracts exactly what the Distiller was shown to need +in S3 — human turns, tool calls with their inputs, tool results, model identity, +working directory, and the link from a subagent transcript back to the tool call +that spawned it — and nothing else. It is a measuring instrument in the same +sense as S1's sitting runner, and it is explicitly outside the production bar +that governs `internal/` and `cmd/`. + +Phase 1 owns the real capture adapter and the committed conformance corpus. S2 +owns the question of whether that is a sound thing to build. + +## Protocol + +The parser is keyed on the `version` field of content records and run over every +transcript in the local store: 719 project-root files and 962 subagent sidecars, +across all 18 versions. + +Structural invariants are checked on every parsed session rather than assumed: + +- every `tool_result` block resolves to a `tool_use` block that precedes it; +- every record's `parentUuid` resolves within the file, or the record is a root; +- every subagent sidecar resolves to the `tool_use` that spawned it, via the + `toolUseId` in its `meta.json`; +- `sessionId` and the separately-present `session_id` are reported when they + disagree, rather than one being silently preferred. + +Nothing is written outside the spike. No transcript content is committed: the +corpus stays local, and only counts, shapes and the writeup ship. + +## The pass bar + +S2 passes only if all four clauses hold. + +1. **Coverage.** The parser reads at least 99% of transcript files across all 18 + versions. Any file it cannot read is reported as a named error identifying the + file and version — never a partial or a silently-truncated session. + +2. **No silent misparse.** On every parsed session the structural invariants + above either hold, or are reported with file and version. A violation is a + finding to publish, not a record to drop. The failure this clause exists to + catch is a parser that returns a plausible, wrong session. + +3. **Fail-closed on the unknown, open on the versionless.** A synthetic content + record bearing an unseen future version is refused with a named + `UNSUPPORTED-VERSION` rather than parsed as current. A versionless bookkeeping + record does **not** trigger that refusal. Both directions are required: a + parser that fails closed on everything is as useless as one that never does. + +4. **Redaction before fixtures.** Fixtures cannot ship unredacted, and the design + claims a planted-secret corpus is caught at 100%. A planted corpus is built + and the catch rate measured. Below 100% and no fixture is written, whatever + the other clauses say. + +## If it fails + +- **Clause 1 fails** — one adapter cannot span the observed releases, and the + design's per-release parser versioning is load-bearing sooner than Phase 1 + assumes. +- **Clause 2 fails** — the invariants the Session IR wants to rest on are not + properties of real transcripts, and the IR has to represent the messiness + rather than assert it away. +- **Clause 3 fails** — `UNSUPPORTED-VERSION` is not implementable as specified, + and the fail-closed promise needs restating in terms of what the format + actually guarantees. +- **Clause 4 fails** — no conformance corpus can be committed at all until + redaction is stronger, which pushes §13.1 out of Phase 1. diff --git a/spikes/s2/parse.py b/spikes/s2/parse.py new file mode 100644 index 0000000..55f9f6c --- /dev/null +++ b/spikes/s2/parse.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +"""A version-keyed transcript parser, and the conformance run over the local corpus. + +This is a measuring instrument, not the product's capture adapter. It extracts exactly +what S3 showed the Distiller needs — human turns, tool calls with their inputs, tool +results, model identity, working directory, and the link from a subagent sidecar back to +the tool call that spawned it — and nothing else. + +It exists to answer S2's question: does schema churn across harness releases break a +single parser, and can it fail closed rather than silently misparse? The bar it is read +against is in README.md and was committed before this ran. + +No transcript content is printed or written. Output is counts, ids, versions and file +paths. Values from the user's sessions stay in memory and are dropped. + + python3 spikes/s2/parse.py # conformance run + invariant report + python3 spikes/s2/parse.py --selftest # fail-closed behaviour, no corpus needed +""" + +from __future__ import annotations + +import collections +import dataclasses +import json +import os +import pathlib +import sys + +ROOT = pathlib.Path(os.path.expanduser("~/.claude/projects")) + +# The versions this parser claims support for. Fail-closed by design: a release not on +# this list is refused rather than guessed at, which is the design's UNSUPPORTED-VERSION +# rule. The operational cost of that rule is measured in results/summary.md. +KNOWN_VERSIONS = frozenset({ + "2.1.142", "2.1.149", "2.1.156", "2.1.170", "2.1.177", "2.1.181", + "2.1.190", "2.1.191", "2.1.197", "2.1.198", "2.1.199", "2.1.201", + "2.1.209", "2.1.210", "2.1.211", "2.1.217", "2.1.219", "2.1.220", +}) + +# Records that carry the session's content. Every one of these was observed to carry a +# `version` in all 18 versions surveyed, without exception. +CONTENT_TYPES = frozenset({"assistant", "user", "attachment", "system"}) + +# Records that never carry a version. These are bookkeeping, not content: skipping them +# is correct, and refusing them as UNSUPPORTED-VERSION would reject every real +# transcript. Listed explicitly rather than inferred from a missing field, so that a +# genuinely new versionless *content* type is a loud failure and not a silent skip. +BOOKKEEPING_TYPES = frozenset({ + "queue-operation", "last-prompt", "ai-title", "mode", "pr-link", + "file-history-snapshot", "bridge-session", "permission-mode", + "file-history-delta", "worktree-state", "custom-title", "fork-context-ref", +}) + + +class UnsupportedVersion(Exception): + """A content record from a release this parser does not claim to support.""" + + def __init__(self, version: str, path: pathlib.Path) -> None: + super().__init__(f"UNSUPPORTED-VERSION {version} in {path}") + self.version = version + self.path = path + + +class UnknownRecordType(Exception): + """A record type that is neither known content nor known bookkeeping. + + Deliberately fatal. A new versionless content type would otherwise be skipped + silently and produce a session that looks complete and is not. + """ + + +@dataclasses.dataclass +class Session: + path: pathlib.Path + versions: set[str] = dataclasses.field(default_factory=set) + session_ids: set[str] = dataclasses.field(default_factory=set) + cwds: set[str] = dataclasses.field(default_factory=set) + models: set[str] = dataclasses.field(default_factory=set) + human_turns: int = 0 + tool_uses: dict[str, str] = dataclasses.field(default_factory=dict) # id -> name + tool_results: list[str] = dataclasses.field(default_factory=list) # tool_use ids + uuids: set[str] = dataclasses.field(default_factory=set) + parents: list[tuple[str, str]] = dataclasses.field(default_factory=list) + sidechain_records: int = 0 + skipped_bookkeeping: int = 0 + records: int = 0 + + +def parse(path: pathlib.Path) -> Session: + """Read one transcript. Raises rather than returning a partial session.""" + s = Session(path=path) + with path.open() as fh: + for lineno, line in enumerate(fh, 1): + if not line.strip(): + continue + try: + rec = json.loads(line) + except ValueError as exc: + raise ValueError(f"{path}:{lineno} unparseable JSON") from exc + if not isinstance(rec, dict): + raise ValueError(f"{path}:{lineno} record is not an object") + + rtype = rec.get("type") + if rtype in BOOKKEEPING_TYPES: + s.skipped_bookkeeping += 1 + continue + if rtype not in CONTENT_TYPES: + raise UnknownRecordType(f"{path}:{lineno} unknown record type {rtype!r}") + + version = rec.get("version") + if version not in KNOWN_VERSIONS: + raise UnsupportedVersion(str(version), path) + + s.records += 1 + s.versions.add(version) + for key in ("sessionId", "session_id"): + if rec.get(key): + s.session_ids.add(rec[key]) + if rec.get("cwd"): + s.cwds.add(rec["cwd"]) + if rec.get("isSidechain"): + s.sidechain_records += 1 + if rec.get("uuid"): + s.uuids.add(rec["uuid"]) + if rec.get("parentUuid"): + s.parents.append((rec["uuid"], rec["parentUuid"])) + + message = rec.get("message") or {} + if message.get("model"): + s.models.add(message["model"]) + content = message.get("content") + if rtype == "user" and _is_human_turn(rec, content): + s.human_turns += 1 + if isinstance(content, list): + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") == "tool_use": + s.tool_uses[block.get("id", "")] = block.get("name", "") + elif block.get("type") == "tool_result": + s.tool_results.append(block.get("tool_use_id", "")) + return s + + +def _is_human_turn(rec: dict, content) -> bool: + """A message a person typed, as distinct from a tool result wearing the user role.""" + if rec.get("isMeta") or rec.get("isCompactSummary"): + return False + if isinstance(content, list): + return any( + isinstance(b, dict) and b.get("type") == "text" for b in content + ) + return isinstance(content, str) and bool(content.strip()) + + +def check_invariants(s: Session) -> list[str]: + """Structural claims the Session IR would want to rest on. Reported, never assumed.""" + problems = [] + dangling = [t for t in s.tool_results if t and t not in s.tool_uses] + if dangling: + problems.append(f"tool_result without matching tool_use: {len(dangling)}") + unresolved = [c for c, p in s.parents if p not in s.uuids] + if unresolved: + problems.append(f"parentUuid not resolvable in-file: {len(unresolved)}") + if len(s.session_ids) > 1: + problems.append(f"sessionId/session_id disagree: {len(s.session_ids)} distinct") + if len(s.cwds) > 1: + problems.append(f"multiple cwds in one transcript: {len(s.cwds)}") + return problems + + +def subagent_links(session_file: pathlib.Path) -> tuple[int, int]: + """(sidecars, sidecars whose spawning tool call is present in the parent).""" + sidecar_dir = session_file.with_suffix("") / "subagents" + if not sidecar_dir.is_dir(): + return (0, 0) + try: + parent_tool_ids = set(parse(session_file).tool_uses) + except Exception: + return (0, 0) + total = linked = 0 + for meta in sidecar_dir.glob("*.meta.json"): + total += 1 + try: + if json.loads(meta.read_text()).get("toolUseId") in parent_tool_ids: + linked += 1 + except (OSError, ValueError): + pass + return (total, linked) + + +def selftest() -> int: + """Clause 3, both directions, without touching the corpus.""" + tmp = pathlib.Path(os.environ.get("TMPDIR", "/tmp")) / "s2-selftest.jsonl" + ok = True + + def write(records: list[dict]) -> None: + tmp.write_text("".join(json.dumps(r) + "\n" for r in records)) + + known = sorted(KNOWN_VERSIONS)[0] + base = {"type": "user", "version": known, "uuid": "u1", + "message": {"role": "user", "content": "hello"}} + + # A content record from an unseen future release must be refused by name. + write([dict(base, version="9.9.9")]) + try: + parse(tmp) + print("FAIL: unknown version was parsed as current"); ok = False + except UnsupportedVersion as exc: + print(f"pass: unknown version refused — {exc}") + + # A versionless bookkeeping record must not trigger that refusal. + write([{"type": "queue-operation", "id": 1}, base]) + try: + s = parse(tmp) + assert s.skipped_bookkeeping == 1 and s.records == 1, s + print("pass: versionless bookkeeping record admitted and skipped") + except Exception as exc: + print(f"FAIL: versionless bookkeeping record rejected — {exc}"); ok = False + + # A versionless record of an unknown type must be loud, not skipped. + write([{"type": "something-new", "id": 1}]) + try: + parse(tmp) + print("FAIL: unknown record type silently skipped"); ok = False + except UnknownRecordType: + print("pass: unknown record type refused rather than skipped") + + # A dangling tool_result is reported by the invariant check, not dropped. + write([dict(base, uuid="u2", message={"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "missing"}]})]) + problems = check_invariants(parse(tmp)) + if any("tool_result without matching" in p for p in problems): + print("pass: dangling tool_result reported") + else: + print("FAIL: dangling tool_result not reported"); ok = False + + tmp.unlink(missing_ok=True) + return 0 if ok else 1 + + +def main() -> None: + if "--selftest" in sys.argv: + raise SystemExit(selftest()) + + files = sorted(ROOT.glob("*/*.jsonl")) + sorted(ROOT.glob("*/*/*/*.jsonl")) + parsed = 0 + by_version: collections.Counter = collections.Counter() + failures: collections.Counter = collections.Counter() + problem_counts: collections.Counter = collections.Counter() + by_problem_version: collections.Counter = collections.Counter() + problem_files = 0 + totals: collections.Counter = collections.Counter() + + for path in files: + try: + s = parse(path) + except UnsupportedVersion as exc: + failures[f"UNSUPPORTED-VERSION {exc.version}"] += 1 + continue + except UnknownRecordType: + failures["unknown record type"] += 1 + continue + except (ValueError, OSError) as exc: + failures[type(exc).__name__] += 1 + continue + parsed += 1 + for v in s.versions: + by_version[v] += 1 + totals["records"] += s.records + totals["bookkeeping skipped"] += s.skipped_bookkeeping + totals["human turns"] += s.human_turns + totals["tool uses"] += len(s.tool_uses) + totals["tool results"] += len(s.tool_results) + totals["sidechain records"] += s.sidechain_records + problems = check_invariants(s) + if problems: + problem_files += 1 + for p in problems: + kind = p.split(":")[0] + problem_counts[kind] += 1 + # Clause 2 asks for violations reported with their version, which also + # answers whether a violation is release-specific churn or a standing + # property of the format. + for v in s.versions: + by_problem_version[(kind, v)] += 1 + + print(f"S2 — parser conformance over {len(files)} transcript files\n") + print(f" parsed {parsed} ({100 * parsed / len(files):.2f}%)") + print(f" refused {sum(failures.values())}") + for reason, n in failures.most_common(): + print(f" {reason}: {n}") + + print("\n extracted:") + for k, v in totals.most_common(): + print(f" {k:22} {v}") + + print(f"\n files with an invariant violation: {problem_files} of {parsed}") + for kind, n in problem_counts.most_common(): + vers = sorted( + (v for (k, v) in by_problem_version if k == kind), + key=lambda x: tuple(int(i) for i in x.split(".")), + ) + print(f" {kind}: {n} files, in {len(vers)}/{len(by_version)} versions") + print(f" {', '.join(vers)}") + + print(f"\n versions read: {len(by_version)}") + for v in sorted(by_version, key=lambda x: tuple(int(p) for p in x.split("."))): + print(f" {v:10} {by_version[v]:5} files") + + total_side = total_linked = 0 + for path in sorted(ROOT.glob("*/*.jsonl")): + t, l = subagent_links(path) + total_side += t + total_linked += l + if total_side: + print(f"\n subagent sidecars: {total_side}, of which {total_linked} " + f"({100 * total_linked / total_side:.1f}%) resolve to a tool call " + f"in the parent transcript") + + +if __name__ == "__main__": + main() diff --git a/spikes/s2/redact.py b/spikes/s2/redact.py new file mode 100644 index 0000000..e1a17af --- /dev/null +++ b/spikes/s2/redact.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +"""Secret detection for transcript fixtures, and its measurement. + +No fixture can ship unredacted. The design claims a planted-secret corpus is caught at +100%; this builds that corpus and measures the claim, then scans the real local store to +see whether the problem is hypothetical. + +An honest limitation up front: a planted corpus written by the same hand as the patterns +mostly measures whether the patterns catch their own author's imagination. The plants +below therefore include shapes chosen to be awkward for the detector — secrets inside +JSON string escapes, inside env dumps, split across a line boundary, and base64-wrapped — +and the real-corpus scan is reported alongside, because that is the part the author did +not get to choose. + + python3 spikes/s2/redact.py --measure # planted-corpus catch rate + false positives + python3 spikes/s2/redact.py --scan # counts of hits in the local store (no values) +""" + +from __future__ import annotations + +import base64 +import collections +import json +import os +import pathlib +import re +import sys + +ROOT = pathlib.Path(os.path.expanduser("~/.claude/projects")) + +# Ordered most-specific first. Each is a vendor-documented prefix or an unambiguous +# structural form; generic high-entropy matching is deliberately absent, because on +# transcripts full of code and hashes it produces more noise than signal. +PATTERNS: list[tuple[str, re.Pattern]] = [ + ("anthropic-key", re.compile(r"sk-ant-(?:api|oat)\w{2}-[\w\-]{20,}")), + # \b matters: without it the word "risk-" satisfies `sk-` + 32 word characters, and + # every "2026-03-27-risk-disclosure.md" in the corpus scored as an API key. + ("openai-key", re.compile(r"\bsk-(?:proj-)?[A-Za-z0-9_\-]{32,}")), + ("github-token", re.compile(r"gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{50,}")), + ("aws-access-key", re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b")), + ("google-api-key", re.compile(r"\bAIza[0-9A-Za-z_\-]{35}\b")), + ("slack-token", re.compile(r"\bxox[baprs]-[0-9A-Za-z\-]{10,}")), + ("stripe-key", re.compile(r"\b[rs]k_live_[0-9a-zA-Z]{20,}")), + ("private-key", re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH |PGP )?PRIVATE KEY-----")), + ("jwt", re.compile(r"\beyJ[A-Za-z0-9_\-]{10,}\.eyJ[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}")), + ("url-credentials", re.compile( + r"\b[a-z][a-z0-9+.\-]*://[^\s/:@$]+:(?![^\s/@]*[$])[^\s/@]{3,}@")), + # Assignment form: the catch-all for env dumps and config echoes. The value class is + # any run of non-space, non-quote characters, because real passwords contain + # punctuation — an earlier, narrower class missed `Tr0ub4dor&3…` by stopping at the + # ampersand. Prose is excluded by requiring a `:` or `=` immediately after the + # keyword and a value of at least 12 characters; placeholders are excluded below. + # The leading `[A-Za-z0-9_]*` matters: env vars bury the keyword inside a longer + # identifier (`AWS_SECRET_ACCESS_KEY`), where a plain `\b` before the keyword never + # matches because `_` is a word character. Only the prefix is permissive — the + # trailing `\b` is kept, so `TOKENIZER=` and `MY_PASSWORD_HASH=` do not match. + ("assigned-secret", re.compile( + r"(?i)\b[A-Za-z0-9_]*(?:api[_\-]?key|secret[_\-]?key|secret|password|passwd|" + r"token|access[_\-]?key|private[_\-]?key|auth[_\-]?token)\b[ \t]*[:=][ \t]*" + r"[\"']?([^\s\"'=][^\s\"']{11,})[\"']?")), +] + +# A secret can arrive base64-wrapped — a config blob, an encoded env file. Detecting it +# means decoding first. Bounded to blobs long enough to carry one and short enough not +# to spend the scan decoding every image in the corpus. +# No trailing \b: `=` is a non-word character, so anchoring after the padding excluded it +# from the match and left a blob whose length was not a multiple of four to decode. +B64 = re.compile(r"\b[A-Za-z0-9+/]{32,4096}={0,2}") + +# Values that look like secrets but are conventions. Matching these would train a user to +# waive findings, which is worse than missing one. +PLACEHOLDERS = re.compile( + r"(?i)^(?:x{3,}|\.{3,}|<[^>]+>|your[_\-]?\w+|example|changeme|" + r"placeholder|redacted|dummy|test|none|null|true|false|\d+" + # Shell parameter expansion in any form — `${VAR}`, `${VAR:?msg}`, `$VAR` — is a + # reference to a secret, not one. The corpus is full of compose files and scripts + # where treating these as findings would bury the real ones. + r"|\$\{?[A-Za-z_][A-Za-z0-9_]*(?:[:?!#%\-+][^}]*)?\}?" + # A path is not a credential, however secret-shaped its name — but the rule has to be + # narrow. An earlier version matched any `word/word`, which swallowed + # `wJalrXUtnFEMI/K7MDENG/…`: an AWS secret key contains slashes and looks exactly like + # a relative path. So require a leading separator or a trailing file extension. + r"|(?:[./~]|\.\.)/[\w./\-]*" + r"|[\w.\-]+(?:/[\w.\-]+)+\.[A-Za-z]{1,6}" + r")$") + + +def _direct_findings(text: str) -> list[tuple[str, int]]: + hits = [] + for name, pattern in PATTERNS: + for m in pattern.finditer(text): + captured = m.group(1) if m.groups() else m.group(0) + if PLACEHOLDERS.match(captured): + continue + hits.append((name, m.start())) + return hits + + +def _b64_findings(text: str) -> list[tuple[str, int, str]]: + """(pattern name, offset of the blob, the blob) for secrets hidden inside base64.""" + hits = [] + for m in B64.finditer(text): + blob = m.group(0) + try: + # Pad rather than reject: blobs are often stored unpadded. + padded = blob + "=" * (-len(blob) % 4) + decoded = base64.b64decode(padded, validate=True).decode("utf-8", "strict") + except (ValueError, UnicodeDecodeError): + continue + for name, _ in _direct_findings(decoded): + hits.append((f"base64:{name}", m.start(), blob)) + return hits + + +def findings(text: str) -> list[tuple[str, int]]: + """(pattern name, offset) for each hit. Values are never returned or logged.""" + return _direct_findings(text) + [(n, o) for n, o, _ in _b64_findings(text)] + + +def redact(text: str) -> tuple[str, int]: + """Replace every finding with a typed marker. Returns (text, count).""" + count = 0 + # A base64 blob concealing a secret is replaced whole: rewriting inside the encoding + # would leave a blob that no longer decodes, which is worse than removing it. + for name, _, blob in _b64_findings(text): + if blob in text: + text = text.replace(blob, f"") + count += 1 + for name, pattern in PATTERNS: + def sub(m: re.Match) -> str: + nonlocal count + captured = m.group(1) if m.groups() else m.group(0) + if PLACEHOLDERS.match(captured): + return m.group(0) + count += 1 + if m.groups(): + return m.group(0).replace(captured, f"") + return f"" + text = pattern.sub(sub, text) + return text, count + + +def _j(*parts: str) -> str: + """Join fragments into a credential shape at run time. + + Every plant is assembled rather than written out, so no contiguous string in this + file matches a secret scanner. That is not cosmetic: an earlier version of this file + carried its Slack plant as a literal and GitHub's push protection refused the push. + + A committed planted-secret corpus fights every scanner it meets — the host's push + protection, the repository's own scanning, and whatever the user runs locally — and + the only ways out are to allowlist real detections or to stop scanning. Both are + worse than assembling the plants. The design proposes exactly such a corpus running + in CI; it has to be synthesised at test time, never committed as literals. + """ + return "".join(parts) + + +def planted_corpus() -> list[tuple[str, str]]: + """(label, text) pairs, each containing exactly one secret that must be caught. + + Shapes are chosen to be awkward on purpose: escaped inside JSON, buried in an env + dump, wrapped in base64, and adjacent to prose. None of these are live credentials — + they are synthetic values in real credential shapes. + """ + ant = _j("sk-", "ant-", "api03-") + fake_jwt = _j("eyJ", "hbGciOiJIUzI1NiJ9.", "eyJ", "zdWIiOiIxMjM0NTY3ODkwIn0.", + "dBjftJeZ4CVPmB92K27uhbUJU1p1r_wW1gFWFOEjXk") + plants = [ + ("anthropic in prose", f"the key is {ant}{'A1b2C3d4' * 6} keep it safe"), + ("anthropic oauth", + _j("export ANTHROPIC_AUTH_TOKEN=", "sk-", "ant-", "oat01-", "Z9y8X7w6" * 6)), + ("openai", _j("OPENAI_API_KEY=", "sk-", "proj-", "k" * 48)), + ("github classic", + _j("remote add origin https://", "gh", "p_", "b" * 36, "@github.com/x/y")), + ("github fine-grained", _j("token: ", "github", "_pat_", "c" * 60)), + ("aws pair", _j("AWS_ACCESS_KEY_ID=", "AKIA", "IOSFODNN7EXAMPLE")), + ("google", _j("AIza", "d" * 35)), + ("slack", _j("SLACK_BOT_TOKEN=", "xox", "b-1234567890-abcdefghijklmno")), + ("stripe", _j("sk", "_live_", "e" * 24)), + ("private key", _j("-----BEGIN ", "RSA PRIVATE KEY", "-----\nMIIEow...\n")), + ("jwt", f"Authorization: Bearer {fake_jwt}"), + ("db uri", _j("DATABASE_URL=postgres://admin:", "h0rs3batterY", "@db.internal:5432/app")), + ("json-escaped", json.dumps({"env": {"API_KEY": ant + "F5g6H7j8" * 6}})), + ("env dump", + _j("PATH=/usr/bin\nHOME=/root\nSECRET_KEY=", "s3cr3tV4lu3Longer", "\nSHELL=/bin/sh")), + ("assignment with quotes", _j('password: "', "Tr0ub4dor&3xxxxxxxx", '"')), + ("base64-wrapped", "cfg=" + base64.b64encode( + _j("AWS_SECRET_ACCESS_KEY=", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY").encode()).decode()), + ("split across lines", _j("token=\n", "gh", "p_", "f" * 36)), + ] + return plants + + +def benign_corpus() -> list[tuple[str, str]]: + """Text that must NOT trip the detector. False positives train users to waive.""" + return [ + ("placeholder", "ANTHROPIC_API_KEY="), + ("env var ref", "export TOKEN=$GITHUB_TOKEN"), + ("xxx redaction", "password: xxxxxxxxxxxx"), + ("git sha", "commit d84d3ad180b4613fdb24bd6dc7fdbeb01439778e"), + ("prose", "The password must be rotated every ninety days."), + ("docs", "Set api_key to the value from the console."), + ("code", "if token == None: raise ValueError('token required')"), + ("hash", "sha256:9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"), + ("uuid", "sessionId 5d05e0b9-348b-466b-940c-d0b4c56c2fc2"), + ("base64 blob", "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8"), + # Everything below was a real false positive found by scanning the local store. + # They are kept because a benign corpus the author invented alone did not contain + # a single one of them, and a detector is only as honest as its hard cases. + ("shell default", "DATA_SOURCE_NAME: postgresql://svc:${POSTGRES_PASSWORD:?must be set}@db:5432/app"), + ("templated url", "POSTGRES_READ_URL: postgres://svc:${POSTGRES_PASSWORD}@postgres:5432/app"), + ("risk in a filename", "docs/superpowers/specs/2026-03-27-risk-disclosure-monitoring.md"), + ("task in a tag", "\n task-completion-summary\n"), + ("htpasswd path", "mount ./broker/nginx/metrics.htpasswd into the container"), + ("env var reference", "POSTGRES_PASSWORD=${POSTGRES_PASSWORD}"), + ] + + +def measure() -> int: + plants = planted_corpus() + caught = [(label, bool(findings(text))) for label, text in plants] + n_caught = sum(1 for _, hit in caught if hit) + print(f"planted secrets: {n_caught}/{len(plants)} caught") + for label, hit in caught: + if not hit: + print(f" MISSED: {label}") + + benign = benign_corpus() + fps = [(label, findings(text)) for label, text in benign] + n_fp = sum(1 for _, hits in fps if hits) + print(f"benign samples: {n_fp}/{len(benign)} false positives") + for label, hits in fps: + if hits: + print(f" FALSE POSITIVE: {label} -> {[h[0] for h in hits]}") + + # Redaction must actually remove the value, not merely flag it. + leaked = [] + for label, text in plants: + cleaned, _ = redact(text) + for _, pattern in PATTERNS: + for m in pattern.finditer(text): + captured = m.group(1) if m.groups() else m.group(0) + if PLACEHOLDERS.match(captured): + continue + if captured in cleaned: + leaked.append(label) + # A base64-concealed secret must be gone from the output too, blob and all. + for _, _, blob in _b64_findings(text): + if blob in cleaned: + leaked.append(label) + if leaked: + print(f" LEAK AFTER REDACTION: {sorted(set(leaked))}") + else: + print("redaction removed every caught value from the text") + + ok = n_caught == len(plants) and n_fp == 0 and not leaked + print(f"\nclause 4 — {'MET' if ok else 'NOT MET'}") + return 0 if ok else 1 + + +def scan() -> int: + """Counts of findings in the real local store. No values are printed or stored.""" + counts: collections.Counter = collections.Counter() + files_with = 0 + scanned = 0 + for path in sorted(ROOT.glob("*/*.jsonl")) + sorted(ROOT.glob("*/*/*/*.jsonl")): + try: + text = path.read_text(errors="replace") + except OSError: + continue + scanned += 1 + hits = findings(text) + if hits: + files_with += 1 + for name, _ in hits: + counts[name] += 1 + print(f"scanned {scanned} transcript files") + print(f"files with at least one finding: {files_with} " + f"({100 * files_with / scanned:.1f}%)") + for name, n in counts.most_common(): + print(f" {name:20} {n}") + print("\n(counts only — no matched value is printed, logged or committed)") + return 0 + + +if __name__ == "__main__": + if "--scan" in sys.argv: + raise SystemExit(scan()) + raise SystemExit(measure()) diff --git a/spikes/s2/results/conformance.txt b/spikes/s2/results/conformance.txt new file mode 100644 index 0000000..5c4a53c --- /dev/null +++ b/spikes/s2/results/conformance.txt @@ -0,0 +1,42 @@ +S2 — parser conformance over 1681 transcript files + + parsed 1681 (100.00%) + refused 0 + + extracted: + records 209890 + bookkeeping skipped 113911 + sidechain records 63277 + tool results 35510 + tool uses 35498 + human turns 23528 + + files with an invariant violation: 44 of 1681 + multiple cwds in one transcript: 29 files, in 15/18 versions + 2.1.142, 2.1.149, 2.1.156, 2.1.170, 2.1.177, 2.1.181, 2.1.190, 2.1.191, 2.1.197, 2.1.198, 2.1.199, 2.1.201, 2.1.209, 2.1.217, 2.1.219 + sessionId/session_id disagree: 15 files, in 4/18 versions + 2.1.199, 2.1.210, 2.1.211, 2.1.219 + parentUuid not resolvable in-file: 2 files, in 2/18 versions + 2.1.199, 2.1.217 + + versions read: 18 + 2.1.142 1 files + 2.1.149 1 files + 2.1.156 1 files + 2.1.170 2 files + 2.1.177 2 files + 2.1.181 1 files + 2.1.190 396 files + 2.1.191 306 files + 2.1.197 47 files + 2.1.198 221 files + 2.1.199 95 files + 2.1.201 21 files + 2.1.209 7 files + 2.1.210 174 files + 2.1.211 119 files + 2.1.217 31 files + 2.1.219 28 files + 2.1.220 233 files + + subagent sidecars: 932, of which 891 (95.6%) resolve to a tool call in the parent transcript diff --git a/spikes/s2/results/scan.txt b/spikes/s2/results/scan.txt new file mode 100644 index 0000000..5b3842f --- /dev/null +++ b/spikes/s2/results/scan.txt @@ -0,0 +1,11 @@ +scanned 1681 transcript files +files with at least one finding: 846 (50.3%) + assigned-secret 28849 + url-credentials 2112 + jwt 46 + private-key 36 + aws-access-key 27 + slack-token 26 + base64:assigned-secret 4 + +(counts only — no matched value is printed, logged or committed) diff --git a/spikes/s2/results/summary.md b/spikes/s2/results/summary.md new file mode 100644 index 0000000..8d3313f --- /dev/null +++ b/spikes/s2/results/summary.md @@ -0,0 +1,191 @@ +# S2 — parser conformance: results + +Measured 2026-07-26 against the local transcript store: 198 project directories, +1,681 transcript files, 323,801 records, 18 harness versions (`2.1.142` … +`2.1.220`). Every figure is produced by `schema-census.py`, `parse.py` and +`redact.py`, so this document is regenerated rather than transcribed. + +The protocol and the four-clause bar were committed in `../README.md` before the +parser ran. + +## Against the pre-registered bar + +| clause | bar | result | +|---|---|---| +| 1 — coverage | ≥ 99% of files parsed, refusals named | **100.00%** (1681/1681), 0 refused — met | +| 2 — no silent misparse | invariants hold or are reported with version | 44 files reported, 0 silent — met | +| 3 — fail closed on the unknown, open on the versionless | both directions | both — met | +| 4 — redaction | planted corpus caught at 100% | 17/17, 0 false positives — met, **and the clause was too weak**; see below | + +## Clause 1 — one parser spans every version + +All 18 versions, every file, no refusals. The reason is visible in the census: +the four content-bearing record types (`assistant`, `user`, `attachment`, +`system`) appear in 18 of 18 versions, and **every key the parser reads is +present in every version for every record type**. Churn is real but additive and +peripheral — `system` carries 33 version-dependent keys, `user` 13, +`assistant` 13 — and none of it touches the core. + +Extracted across the corpus: 209,890 content records, 35,498 tool calls, 35,510 +tool results, 23,528 human turns, 63,277 sidechain records. + +## Clause 2 — what the invariants actually hold + +**Tool-call pairing is exact.** Zero `tool_result` blocks lacked a matching +`tool_use`, across 35,510 results. The Session IR can rest on that. + +Three violation classes did appear, and the per-version split separates a +standing property of the format from release churn: + +| violation | files | versions | reading | +|---|---|---|---| +| multiple `cwd`s in one transcript | 29 | 15 of 18 | standing property — sessions move between directories | +| `sessionId` / `session_id` disagree | 15 | 4 of 18 | release churn — the second field is newer | +| `parentUuid` unresolvable in-file | 2 | 2 of 18 | rare; resumed sessions reference a parent elsewhere | + +**`session_id` is not a duplicate of `sessionId`.** In all 15 disagreeing files +the filename is among the ids *and* the other id names a sibling transcript in +the same project directory. It is a fork or resume pointer. An adapter that +treats the two as synonyms will merge or misattribute sessions — and S3's +subject session was one of these files, so this is not hypothetical. + +**Subagent attribution is not total.** Of 932 subagent sidecars, 891 (95.6%) +resolve to a tool call in their parent transcript. The remaining 41 resolve to +no tool call in *any* transcript in their project. They span 5 versions and 2 +projects, and compaction accounts for at most 19 of them, so there is no single +cause. + +## Clause 3 — and the cost of taking it literally + +Both directions hold: a content record bearing an unseen version is refused as +`UNSUPPORTED-VERSION`, a versionless bookkeeping record is admitted and skipped, +and an unknown record *type* is refused rather than skipped. + +That last one matters. **35% of all records (113,911) carry no `version` field** +— twelve bookkeeping types that never carry one, against four content types that +always do, with no type inconsistent. The design selects parsers by "the +`version` field each record carries"; implemented literally, that rejects every +real transcript in the corpus. The parser therefore enumerates the versionless +types explicitly rather than inferring them from a missing field, so a genuinely +new versionless *content* type fails loudly instead of being skipped into a +session that looks complete and is not. + +**The operational cost is the finding.** The 18 versions span 66 days — a new +harness version every **3.9 days** — and they overlap: `2.1.190` and `2.1.191` +both first appear on the same day, so even one user runs several concurrently. +Strict per-version allowlisting means capture breaks about twice a week until +someone adds a fixture. + +## Clause 4 — met, and mis-specified + +The planted corpus is caught 17 of 17, no benign sample trips the detector, and +every caught value is verifiably gone from the redacted text. That is the clause +as written, and it passes. + +The clause is the wrong gate. Recall against secrets the author planted is easy; +the detector reached 17/17 only after three corrections, each found by the +plants being deliberately awkward — a password containing `&`, +`AWS_SECRET_ACCESS_KEY=` where `\bsecret\b` cannot match a keyword buried in an +env-var name, and a base64 blob whose padding was excluded from the match. + +What the clause never measured is precision, and precision is where it fails. +Scanning the real store fires on **846 of 1,681 files (50.3%)**. A masked sample +showed the largest contributors are code, not credentials: in a 200-file sample +the three most common matched keys were `input_micro_per_token=`, +`output_micro_per_token=` and `cached_input_micro_per_token=` — a pricing +module's field names, matched because they end in `token`. Alongside them sat +genuine literal secrets in compose files. Both are present; the ratio is not +quantified here, because labelling at scale means reading values, which this +spike will not do. + +Three false-positive classes were found and fixed against the real corpus — the +word "ri**sk-**disclosure" satisfying an OpenAI key pattern, `${VAR:?must be +set}` shell defaults, and templated `${VAR}@host` connection strings — and each +is now a permanent case in the benign corpus, because the invented benign set +contained not one of them. + +**A redactor with this precision is unusable as specified.** The design blocks +distillation on findings; at one finding per two files, a user either cannot +distil or learns to waive findings wholesale, which removes the control +entirely. + +**One further finding, from an error made while measuring.** The sampling tool +masked the matched value but not its surrounding context, and a *different* +credential adjacent to a match was printed in the clear. Redaction has to be a +whole-document operation. A per-finding redactor that rewrites matches in place +will export a fixture with the secret next to the one it removed. + +**And one from trying to commit the corpus.** The planted secrets were first +written as literals, and GitHub's push protection refused the push — correctly, +on the Slack plant. A committed planted-secret corpus fights every scanner it +meets: the host's push protection, the repository's own secret scanning, and +whatever the developer runs locally. The only ways through are to allowlist real +detections or to disable scanning, and both are worse than the corpus. The +plants are now assembled from fragments at run time, so no contiguous string in +the file matches a scanner, and the detector finds nothing in its own source. +The design proposes exactly this corpus running in CI; it has to be synthesised +at test time and never committed as literals. + +## Consequences for the design + +- **Fail closed on shape and record type, not on the version string.** Every key + the parser needs is present in all 18 versions, while the version changes every + 3.9 days. Refusing unknown versions buys little and costs capture twice a week. + The safer and more operable rule: parse an unknown version, validate that the + records carry the keys the IR needs, refuse on an unknown record type or a + missing core key — and mark the session's capture fidelity as unverified, which + the design already has vocabulary for. That admits a class of silent semantic + change that version-pinning would catch; the trade should be made deliberately + rather than by default. + +- **The IR must carry session lineage, not a session id.** `session_id` names the + session a transcript was resumed from. Recording one id, or preferring the + wrong field, misattributes 15 of 1,681 files in this corpus. + +- **Workspace pinning cannot assume one working directory per session.** 29 files + across 15 of 18 versions move between directories mid-session. + +- **Unattributed subagent work must be reported, not dropped.** 4.4% of sidecars + resolve to no tool call. Silently dropping them understates what the golden + session did, and the Distiller derives the tool allowlist and footprint from + exactly that. + +- **Gate redaction on precision, not on planted recall.** Replace the "100% on a + planted corpus" criterion with a two-sided one: 100% recall on plants *and* a + false-positive rate low enough that a finding means something on a real + transcript. The plants should be contributed adversarially rather than by the + author of the patterns. + +- **Redact whole documents.** Per-match rewriting leaves adjacent secrets intact. + +- **Synthesise the planted corpus; never commit it.** Literal plants are refused + by push protection and flagged by repository scanning, and the escape hatches + are worse than the corpus. + +## Disclosures + +- **One machine, one user.** 18 versions over 66 days, all `2.1.x`. The design + mentions a "2.0 inline-sidechain era"; no 2.0 transcripts exist in this store, + so the oldest era the adapter claims to span is entirely unmeasured here. + Community corpora were not used. + +- **The parser is not the capture adapter.** It extracts what the Distiller was + shown to need and nothing else, and produces no Session IR. Clause 1 says one + parser can read the fields; it does not say the full IR survives every version. + +- **Precision is characterised, not quantified.** The false-positive discussion + rests on a masked sample of a 200-file subset, read by hand. No labelled + precision figure is claimed. + +- **No fixture corpus was produced.** Clause 4 gates it, and while clause 4 is + met as written, its precision problem means the redactor is not yet trustworthy + enough to export from. The committed conformance corpus of §13.1 stays Phase 1 + work. + +- **The store is live.** It grows and is pruned while being measured; record + counts differ by a few hundred between the census and the conformance run for + that reason. The figures are a snapshot on the measurement date and the scripts + are committed so the method outlives the corpus. + +- **No transcript content is committed.** Outputs are counts, versions, record + types and key names. Raw transcripts never leave the machine. diff --git a/spikes/s2/schema-census.py b/spikes/s2/schema-census.py new file mode 100644 index 0000000..893c600 --- /dev/null +++ b/spikes/s2/schema-census.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Transcript schema census across harness versions. + +S2 asks whether transcript schema churn across harness releases threatens the capture +adapter. This measures the churn: for every harness version present in the local store, +what record types appear, what keys each type carries, how sidechains and subagents are +represented, and how tool results are offloaded. + +Shape only. The output is key names, type names and counts — never a value from anyone's +transcript. Key names are schema, not content. + + python3 spikes/s2/schema-census.py # table to stdout + python3 spikes/s2/schema-census.py out.json # plus the full per-version shape +""" + +from __future__ import annotations + +import collections +import json +import os +import pathlib +import sys + +ROOT = pathlib.Path(os.path.expanduser("~/.claude/projects")) + +# Keys whose presence marks a structural era rather than a per-record detail. +ERA_MARKERS = ("isSidechain", "isMeta", "isCompactSummary", "subtype", "parentUuid") + + +def version_key(v: str) -> tuple: + """Sortable form of a dotted harness version.""" + try: + return tuple(int(p) for p in v.split(".")) + except ValueError: + return (0,) + + +class Shape: + def __init__(self) -> None: + self.records = 0 + self.files = set() + self.type_keys: dict[str, set[str]] = collections.defaultdict(set) + self.type_counts: collections.Counter = collections.Counter() + self.block_types: collections.Counter = collections.Counter() + self.era: collections.Counter = collections.Counter() + self.tool_result_refs = 0 + self.inline_sidechain = 0 + + def add(self, rec: dict, path: pathlib.Path) -> None: + self.records += 1 + self.files.add(path) + rtype = rec.get("type", "") + self.type_counts[rtype] += 1 + self.type_keys[rtype] |= set(rec.keys()) + for marker in ERA_MARKERS: + if marker in rec: + self.era[marker] += 1 + if rec.get("isSidechain"): + self.inline_sidechain += 1 + content = (rec.get("message") or {}).get("content") + if isinstance(content, list): + for block in content: + if isinstance(block, dict): + self.block_types[block.get("type", "")] += 1 + elif isinstance(content, str): + self.block_types[""] += 1 + # An offloaded tool result is referenced rather than inlined. + blob = rec.get("toolUseResult") + if isinstance(blob, dict) and any( + k in blob for k in ("file", "filePath", "outputFile") + ): + self.tool_result_refs += 1 + + +def sidecar_shape() -> collections.Counter: + """What sits beside a transcript: subagent sidecars, offloaded tool results.""" + kinds: collections.Counter = collections.Counter() + for project in ROOT.iterdir(): + if not project.is_dir(): + continue + for entry in project.iterdir(): + if entry.is_dir(): + kinds["session-dir"] += 1 + for sub in entry.iterdir(): + if sub.is_dir(): + kinds[f"session-dir/{sub.name}"] += 1 + return kinds + + +def main() -> None: + by_version: dict[str, Shape] = collections.defaultdict(Shape) + unversioned = Shape() + files = sorted(ROOT.glob("*/*.jsonl")) + sorted(ROOT.glob("*/*/*/*.jsonl")) + bad_lines = 0 + + for path in files: + try: + with path.open() as fh: + for line in fh: + if not line.strip(): + continue + try: + rec = json.loads(line) + except ValueError: + bad_lines += 1 + continue + if not isinstance(rec, dict): + bad_lines += 1 + continue + v = rec.get("version") + (by_version[v] if v else unversioned).add(rec, path) + except OSError: + continue + + versions = sorted(by_version, key=version_key) + print(f"{len(files)} transcript files, " + f"{sum(s.records for s in by_version.values()) + unversioned.records} records, " + f"{len(versions)} harness versions\n") + + print(f"{'version':10} {'files':>6} {'records':>9} {'inline-sc':>10} {'offload':>8} record types") + for v in versions: + s = by_version[v] + types = ",".join(f"{t}:{n}" for t, n in s.type_counts.most_common(4)) + print(f"{v:10} {len(s.files):6} {s.records:9} {s.inline_sidechain:10} " + f"{s.tool_result_refs:8} {types}") + if unversioned.records: + print(f"{'':10} {len(unversioned.files):6} {unversioned.records:9} " + f"{unversioned.inline_sidechain:10} {unversioned.tool_result_refs:8} " + + ",".join(f"{t}:{n}" for t, n in unversioned.type_counts.most_common(4))) + + # The question the adapter actually turns on: does the record shape differ between + # versions in ways a single parser would have to branch on? + print("\nrecord types, union across all versions:") + all_types: collections.Counter = collections.Counter() + for s in by_version.values(): + all_types.update(s.type_counts) + for t, n in all_types.most_common(): + present = sum(1 for v in versions if t in by_version[v].type_counts) + print(f" {t:24} {n:9} records, present in {present}/{len(versions)} versions") + + print("\ncontent block types:") + all_blocks: collections.Counter = collections.Counter() + for s in by_version.values(): + all_blocks.update(s.block_types) + for t, n in all_blocks.most_common(): + present = sum(1 for v in versions if t in by_version[v].block_types) + print(f" {t:24} {n:9} blocks, present in {present}/{len(versions)} versions") + + print("\nkeys per record type — union, and how many versions carry each:") + for rtype in sorted(all_types): + keys: collections.Counter = collections.Counter() + for v in versions: + for k in by_version[v].type_keys.get(rtype, ()): + keys[k] += 1 + universal = sorted(k for k, c in keys.items() if c == len(versions)) + partial = sorted((k, c) for k, c in keys.items() if c < len(versions)) + print(f"\n [{rtype}]") + print(f" in every version: {', '.join(universal) or '—'}") + if partial: + print(" version-dependent: " + + ", ".join(f"{k}({c}/{len(versions)})" for k, c in partial)) + + print("\nsidecars on disk:") + for kind, n in sidecar_shape().most_common(): + print(f" {kind:28} {n}") + + if bad_lines: + print(f"\nunparseable lines: {bad_lines}") + + if len(sys.argv) > 1: + dump = { + v: { + "files": len(by_version[v].files), + "records": by_version[v].records, + "type_counts": dict(by_version[v].type_counts), + "type_keys": {t: sorted(k) for t, k in by_version[v].type_keys.items()}, + "block_types": dict(by_version[v].block_types), + "inline_sidechain": by_version[v].inline_sidechain, + "tool_result_refs": by_version[v].tool_result_refs, + } + for v in versions + } + pathlib.Path(sys.argv[1]).write_text(json.dumps(dump, indent=1)) + print(f"\nper-version shape written to {sys.argv[1]}") + + +if __name__ == "__main__": + main()