diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..c8c4beb --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,11 @@ +{ + "name": "tfa-rca", + "description": "Drive collaborative root-cause analysis over all failed tests of a build, generic across product and infra.", + "version": "0.1.0", + "author": { + "name": "BrowserStack", + "url": "https://www.browserstack.com" + }, + "homepage": "https://github.com/browserstack/browserstack-ai-tfa-demo", + "license": "MIT" +} diff --git a/.cursor-mcp.json b/.cursor-mcp.json new file mode 100644 index 0000000..eed3690 --- /dev/null +++ b/.cursor-mcp.json @@ -0,0 +1,13 @@ +{ + "mcpServers": { + "bstack": { + "command": "npx", + "args": ["-y", "@browserstack/mcp-server@1.2.27-beta.1"], + "env": { + "BROWSERSTACK_USERNAME": "${BROWSERSTACK_USERNAME}", + "BROWSERSTACK_ACCESS_KEY": "${BROWSERSTACK_ACCESS_KEY}", + "O11Y_TFA_RCA_BASE_URL": "${O11Y_TFA_RCA_BASE_URL}" + } + } + } +} diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json new file mode 100644 index 0000000..e7998b7 --- /dev/null +++ b/.cursor-plugin/plugin.json @@ -0,0 +1,8 @@ +{ + "name": "tfa-rca", + "description": "Collaborative root-cause analysis over all failed tests of a BrowserStack build, generic across product and infra.", + "version": "0.1.0", + "mcpServers": "../.cursor-mcp.json", + "skills": "./skills/", + "author": { "name": "BrowserStack" } +} diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d86819e --- /dev/null +++ b/.env.example @@ -0,0 +1,8 @@ +# BrowserStack credentials — used by the bundled bstack MCP server for +# listTestIds + tfaRcaTurn. Per-user; never commit real values. +BROWSERSTACK_USERNAME= +BROWSERSTACK_ACCESS_KEY= + +# Observability base URL the TFA RCA chat runs against. Optional — +# the bstack MCP server defaults to its rengg-tfa staging URL when unset. +# O11Y_TFA_RCA_BASE_URL=https://api-observability-rengg-tfa.bsstag.com diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..433d1ce --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +node_modules/ +.env +.DS_Store +*.code-workspace +# Per-run RCA batch state (the CSV/WAL spine + report) is workspace-local. +.rca/ +# Planning docs (brainstorm/ideation/plan) stay local — not pushed. +docs/ diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..7fca468 --- /dev/null +++ b/.mcp.json @@ -0,0 +1,14 @@ +{ + "mcpServers": { + "bstack": { + "type": "stdio", + "command": "npx", + "args": ["-y", "@browserstack/mcp-server@1.2.27-beta.1"], + "env": { + "BROWSERSTACK_USERNAME": "${BROWSERSTACK_USERNAME}", + "BROWSERSTACK_ACCESS_KEY": "${BROWSERSTACK_ACCESS_KEY}", + "O11Y_TFA_RCA_BASE_URL": "${O11Y_TFA_RCA_BASE_URL}" + } + } + } +} diff --git a/INTEGRATION.md b/INTEGRATION.md new file mode 100644 index 0000000..f7bedad --- /dev/null +++ b/INTEGRATION.md @@ -0,0 +1,102 @@ +# Multi-client integration (Claude Code · Cursor · Codex) + +This plugin is built so the **MCP core is truly cross-client** and the **harness +layer ports via the cross-vendor Agent Skills standard**. Only one piece is +genuinely Claude-Code-specific (the batch *dynamic workflow*); on Cursor and +Codex that role is filled by the sequential harness or subagents. Every path is +autonomous after the single `/rca-build` gate — no host ever prompts mid-run. + +## What transfers, what doesn't + +| Layer | Claude Code | Cursor | Codex | +|---|---|---|---| +| `bstack` MCP server (`listTestIds` + `tfaRcaTurn` + `triggerRcaReport`) | `.mcp.json` (auto-discovered) | `.cursor-mcp.json` / `.cursor/mcp.json` | `~/.codex/config.toml` `[mcp_servers.bstack]` | +| `rca-build` skill (`SKILL.md`) | plugin `skills/` | Agent Skills (`.cursor/skills/` or cursor-plugin `"skills":"./skills/"`) | Agent Skills (`.agents/skills/`) | +| `ai-tfa-coordinator` agent | plugin `agents/` | `.cursor/agents/` (also reads `.claude/agents/`) | `.codex/agents/` | +| Per-test RCA **loop** | `agents/ai-tfa-coordinator.md` | same skill/agent | same skill/agent | +| Batch orchestration | dynamic workflow `workflows/rca-batch.mjs` (or subagents) | subagents, or **sequential** `lib/loop.mjs` | subagents, or **sequential** `lib/loop.mjs` | + +The dynamic workflow (`workflows/rca-batch.mjs`) uses Claude Code's Workflow +runtime, which Cursor/Codex don't have. The same batch still runs there via +**subagents** (both hosts support subagents) or the **sequential thin-client +harness** `lib/loop.mjs` (`runRcaLoop`) — the conformance-tested loop that +drives `tfaRcaTurn` over the same contract without any host-specific +orchestration. On every host the run finishes the same way: glimpse table → +`triggerRcaReport(buildUuid)` → "Full report on the Test Observability UI: +". No local report file is ever written. + +## Claude Code + +```bash +cp .env.example .env # BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY +claude --plugin-dir ./ +/rca-build +``` + +`.claude-plugin/plugin.json` + root `.mcp.json` + `skills/` + `agents/` are +auto-discovered. (No `commands/rca-build.md` on purpose — a command and skill +with the same name collide and the skill body fails to load.) + +## Cursor + +The repo ships Cursor parity files mirroring `slack-mcp-plugin`: +`.cursor-plugin/plugin.json` (points at `../.cursor-mcp.json` and `./skills/`) +and `.cursor-mcp.json` (the stdio `bstack` server). + +**Wire the MCP server** — either: +- copy `.cursor-mcp.json`'s `bstack` entry into your project `.cursor/mcp.json` + (top-level `mcpServers`), or +- Cursor → Settings → Cursor Settings → **MCP** → paste the same JSON, or +- use an **Add to Cursor** deeplink: + `cursor://anysphere.cursor-deeplink/mcp/install?name=bstack&config=` + +Set `BROWSERSTACK_USERNAME` / `BROWSERSTACK_ACCESS_KEY` / `O11Y_TFA_RCA_BASE_URL` +in your environment (or replace the `${…}` placeholders with literals). + +**Skill + agent discovery** — Cursor reads `.cursor/skills/` and `.cursor/agents/` +(and also `.claude/agents/`). The simplest no-duplication setup is to symlink the +shared trees: + +```bash +mkdir -p .cursor +ln -s ../skills .cursor/skills +ln -s ../agents .cursor/agents +``` + +Then drive it from Agent chat: invoke the `rca-build` skill with a build id. + +## Codex + +Codex reads the global `~/.codex/config.toml` (no per-project MCP file). + +**Wire the MCP server** — either copy the block from `codex-mcp.example.toml` +into `~/.codex/config.toml`, or: + +```bash +codex mcp add bstack \ + --env BROWSERSTACK_USERNAME=… --env BROWSERSTACK_ACCESS_KEY=… \ + --env O11Y_TFA_RCA_BASE_URL=https://api-observability-rengg-tfa.bsstag.com \ + -- npx -y @browserstack/mcp-server@1.2.27-beta.1 +``` + +**Skill + agent discovery** — Codex reads `.agents/skills/` (skills) and +`.codex/agents/` (subagents). Symlink the shared trees: + +```bash +mkdir -p .agents .codex +ln -s ../skills .agents/skills +ln -s ../agents .codex/agents +``` + +Then run the `rca-build` skill; the coordinator + `tfaRcaTurn` loop are identical. + +## Notes + +- The `bstack` server is **stdio** (`npx @browserstack/mcp-server@1.2.27-beta.1`), not a remote + OAuth server — so the configs use `command`/`args`/`env`, unlike Slack's + `url`+`oauth`/`auth` shape. +- Env-var interpolation (`${VAR}`) is honored by Claude Code's `.mcp.json`; on + Cursor/Codex, replace the placeholders with literals if your client doesn't + expand them. +- Everything in `lib/` and the `SKILL.md`/agent prose is host-agnostic — only the + MCP wiring file and the dynamic workflow are host-specific. diff --git a/README.md b/README.md index 423d780..ca64af1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,118 @@ -# browserstack-ai-tfa-demo -AI TFA Demo +# tfa-rca — generic multi-client RCA agent plugin + +Drive BrowserStack's collaborative root-cause-analysis loop over **all failed +tests of a build**, generic across product and infra, from inside an agentic +MCP client (Claude Code / Cursor / Codex). + +The plugin wraps three stable MCP tools — `listTestIds`, `tfaRcaTurn`, and +`triggerRcaReport` (from the `bstack` MCP server) — and adds the harness that +batches RCA over a whole build, clusters failures by signature, routes evidence +requests to whatever skills/tools the client already has, and lands a per-test +RCA in the TRA (Test Observability) dashboard. + +> **The full RCA report lives on the Test Observability UI, not in Claude.** +> The plugin surfaces a terse glimpse, triggers the dashboard report +> (`triggerRcaReport`), and prints the link. It **discovers and delegates** to +> the infra skills/tools already in your client (GitHub, whatever runtime you have — k8s/ECS/docker/… — kibana/other +> logs, metrics). It does **not** install or own those connectors, and it never +> writes a local report file. + +## Install + +```bash +git clone https://github.com/browserstack/browserstack-ai-tfa-demo.git +cd browserstack-ai-tfa-demo +cp .env.example .env # fill in BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY +claude --plugin-dir ./ +``` + +The plugin auto-configures on load: the `bstack` MCP server (from `.mcp.json`), +the `rca-build` skill, and the `ai-tfa-coordinator` agent are all discovered by +convention. (There is deliberately **no** command file named `rca-build` — a +command and skill sharing a name collide and the skill body fails to load.) + +### Cursor & Codex + +The MCP core (`listTestIds` + `tfaRcaTurn` + `triggerRcaReport`) and the +skill/agent layer port to both — Cursor uses `.cursor-plugin/plugin.json` + +`.cursor-mcp.json`, Codex uses `~/.codex/config.toml` (see +`codex-mcp.example.toml`). The only Claude-specific piece is the batch *dynamic +workflow*; on Cursor/Codex the same batch runs via subagents or the sequential +harness (`lib/loop.mjs`). Full per-host wiring (MCP config, skill/agent +discovery, deeplink) is in **[INTEGRATION.md](INTEGRATION.md)**. + +## Usage + +``` +/rca-build +/rca-build build_id= https://github.com/org/repo/pull/123 +``` + +Args: a build id (bare, `build_id=`, or a dashboard link) plus optional PR URLs +/ repo hints. + +## The single gate + +The run has exactly **one gate** before execution, with two parts: + +1. **Connector discovery + validation** — every connector relevant to test RCA + (github, infra, logs, metrics, …) is enumerated and probe-validated (`gh auth + status`, an infra probe matching whatever runtime exists — kubectl/docker/ecs/… — MCP tools listed). The result is a validated + capability manifest: `connector → valid | invalid | absent`. A gap is + recorded and declared to the TFA agent ("I don't have logs/metrics access") — + never a blocker. +2. **Requirements** — intake fields (product repo, automation repo, branches, + PRs in play, build id) are resolved **by assumption** wherever possible + (invocation args, `gh repo view`, current branch). At most **one** + consolidated question may be asked at gate close, and only for genuinely + non-assumable, load-bearing fields. Headless (`claude -p`) never asks: a + missing build id fails fast; everything else is a recorded gap. + +**After the gate closes, the run never asks you anything again** — RCA +execution is fully autonomous. Evidence gaps degrade to "unavailable" back to +the TFA agent, which finalizes best-effort. + +## Output + +When every test is terminal, the run prints a terse **glimpse table** +(`testRunId → cluster → status → confidence one-liner`), calls +`triggerRcaReport(buildUuid)`, and prints: + +``` +Full report on the Test Observability UI: +``` + +That dashboard report — populated per-test by the BrowserStack agent, with +mandatory culprit-PR links on application bugs — is the real deliverable. + +## Requirements + +- The `bstack` MCP server (bundled via `.mcp.json`). +- Credentials in `.env` (or your client's MCP env). +- For full evidence coverage: whatever GitHub / infra / logging / metrics + skills your client already has. Missing ones degrade gracefully (the RCA's + confidence band reflects what evidence was actually available). + +## Run + +Point it at any red BrowserStack build — the harness discovers what it needs: + +``` +# BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY exported; default base is +# production, override O11Y_TFA_RCA_BASE_URL only for a staging tenant. +/rca-build +``` + +The single gate validates connectors (github via `gh`, infra via whatever +runtime connector exists — kubectl/docker/ecs/…) and resolves the intake +**by inference** (product/automation repo from the cwd's git remote, branches, +any PRs you pass). It never assumes a product repo from unrelated workspace +docs — if it can't infer one that matches the failures, it records the gap and +proceeds RCA-only rather than blaming the wrong repo. Then it clusters the +failures, drives `tfaRcaTurn` per cluster, and lands per-test RCAs on the +dashboard, printing the glimpse + the Test Observability link. + +## Layout + +Implementation plan + requirements live under `docs/` (local, gitignored). +Cross-client wiring is in `INTEGRATION.md`. diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md new file mode 100644 index 0000000..6c3deec --- /dev/null +++ b/agents/ai-tfa-coordinator.md @@ -0,0 +1,425 @@ +--- +name: ai-tfa-coordinator +description: 'Per-test collaborative-RCA coordinator (autonomous — never prompts a user). Given ONE testRunId, drives the tfaRcaTurn MCP loop to a terminal root cause: TFA reads the run logs; this coordinator supplies every non-log evidence ask (product code, infra/runtime, logs, metrics, deploy, ci) using whatever skills/tools the client has, routed through the validated capability manifest. Skips every test_logs ask (TFA owns logs). For application bugs it MUST hunt the culprit PR via the github connector. Emits a structured RCA_OUTPUT block. Generic over product and infra — no hardcoded tools. Examples: +- orchestrator: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=39 — error: empty buildName rejected on POST /builds") → drives the loop, returns RCA_OUTPUT +- sibling confirm: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=40 — pre-seed: cause=, suspect PR=#7421") → one-turn confirm against this test logs +- user: "run collaborative RCA on test run 39" → single-test loop to RESOLVED/PENDING' +tools: [Bash, Read, Grep, Glob, Task, mcp__*__tfaRcaTurn, mcp__*__getTfaTurnResult, mcp__github__*] +model: sonnet +--- + +# Per-Test Collaborative RCA Coordinator (`ai-tfa-coordinator`) + +Drives the `tfaRcaTurn` MCP loop for a **single** failed test to a terminal RCA. +The collaboration contract is fixed: **TFA owns logs; this coordinator owns +everything else.** TFA (server-side, via the tool) reads the run's logs from its +own access and emits typed evidence asks; this coordinator fulfills every +**non-log** ask using whatever skills/tools the client has — routed through the +validated capability manifest — digests the findings, and feeds them back on the +same thread until TFA converges. TFA authors the RCA into the TRA dashboard; +this coordinator only ever sees the **trimmed glimpse** of it. The full report +lives on the Test Observability UI. + +This coordinator is **fully autonomous**: the `/rca-build` gate closed before it +was dispatched, so it **never prompts a user** — an evidence gap degrades to an +`unavailable` block back to TFA, always. + +This coordinator is the **reusable unit**: it takes one `testRunId` and runs +standalone, driven by the batch workflow, a subagent dispatch, or the thin +sequential harness (`lib/loop.mjs`). It is **generic over product and infra** — +it names no `kubectl` / `chitragupta` / `bifrost`; it routes by *capability*. + +## Inputs + +- `testRunId` — **required**, the integer test-run ID. Maps to the tool's `testRunId` arg. +- `error_digest` — optional short error title + endpoint (NOT logs) for the first-turn message. +- `pre_seed` — optional. For a **cluster sibling**: the representative's + `root_cause` + suspect `related_prs`. When present, the first-turn message + states the hypothesis and asks TFA to **confirm it against this test's own logs**. +- `resume` — optional `{ threadId, turnId }` from a prior PENDING run. +- `manifest` — the validated capability manifest `{ capability: { available, via } }` + (built once at the `/rca-build` gate — Part A). +- `evidenceFile` — optional. Absolute path to the build-level pre-fetch + artifact (`lib/evidence-file.mjs`, `/rca-build` Step 4). Holds pre-digested + `github` (PR window, deploy state) and `logs`/`infra` (app-side sweep) + evidence, keyed by repo and by workload — gathered ONCE by the orchestrator + for every repo/workload this build's failures implicate. `Read` it before + any live gather call (see Operating Principle 0) — and treat it as + read-WRITE: a live gather that fills a gap or goes deeper is written back + via `contributeGithubEvidence`/`contributeLogsEvidence` (writing your own + per-writer shard, keyed by your `testRunId`) so later dispatches — this + test's own siblings, or another cluster sharing the same repo/workload — + benefit too. + +If `testRunId` is missing or not parseable as an integer, emit a `failed` +`RCA_OUTPUT` block with `root_cause: "no testRunId provided"` and stop — do not +call the tool. + +## What the tool returns (trimmed shapes) + +`tfaRcaTurn` returns **trimmed** terminal turns — never the full RCA payload: + +- `RESOLVED` → `{ status, confidence, threadId, glimpse: { root_cause (≤220 + chars), failure_type, related_prs }, viewRca }`. The `viewRca` link points at + the Test Observability UI — pass it through to the output. +- `PENDING` → `{ status, turnId, threadId }`. **Not an agent verdict** — the tool + abandoned its own in-call poll at 90s while TFA kept working. Drain it with + `getTfaTurnResult` (below); never treat it as an answer. +- `NEEDS_INFO` → `questions` / `asks` / `suggestions` **verbatim** — this loop + consumes them exactly as sent. +- `BLOCKED` → terminal: TFA cannot proceed. No asks; stop the loop. + +`getTfaTurnResult(testRunId, turnId)` reads a submitted turn **once**, returning +the same four shapes — still `PENDING` if the agent is mid-flight. It is +read-only and has no side effects, so a read is always safe to repeat. + +## Operating principles + +0. **Read the pre-fetch first.** If `evidenceFile` is present, `Read` it + before considering any live github/infra/logs call. It holds build-level + evidence (PR window, deploy state, log sweeps) already gathered once by the + orchestrator for the repos/workloads this build's failures implicate. Use + what it covers directly — its entries are already digest-shaped (an + `evidence-block.md`-style `block`); paste, don't re-digest. Only make a live + call for what it does NOT cover: a repo/workload it doesn't name, an entry + marked with a `gap` (a `gap` is never coverage — treat it exactly as if the + file didn't have that entry), or evidence genuinely specific to this one + test that a build-wide sweep window could plausibly have missed. For a + sibling (`pre_seed` present): the file's data about YOUR OWN test's + workload/repo is real evidence, not inheritance — reading it is fine; the + CONFIRMATION judgment against it must still be independently yours (see + principle 1 and the sibling note in "The loop"). + + **Write back what you gather live.** A live call that fills a gap, or goes + deeper than the file already had (a full diff instead of a summary, a PR + the pre-fetch never named, a log sweep that succeeded where the file + recorded one as gapped) is exactly the kind of build-level fact this file + exists to share — not just this test's own answer. Persist it via + `contributeGithubEvidence(evidenceFilePath, writerId, repo, patch, nowMs)` + or `contributeLogsEvidence(evidenceFilePath, writerId, workload, patch, + nowMs)` (`lib/evidence-file.mjs`), where **`writerId` is your own + `testRunId`** — that is what keeps writes safe. Each coordinator writes only + its own shard file under `.contrib/`, so + concurrent coordinators can never clobber each other or the orchestrator's + base pre-fetch; readers fold base + every shard back into one view + automatically. Write back before finishing this test, so a sibling + dispatched after you (or any other cluster sharing the same repo/workload) + reads the enriched entry instead of re-fetching what you just fetched. + Only write back genuinely new/deeper findings — never a no-op re-write of + an already-covered entry. It's a best-effort optimization, not a + correctness dependency: never block or retry on it. + + **Route read-only lookups through the tool cache.** The evidence file + shares *digested findings*; the cache below shares *raw call results*, which + is where most duplicate work actually hides (measured on one real build: + `gh` was 37% of all coordinator tool calls, 46 of them byte-identical + commands re-run by different coordinators — one spec file fetched 12 + times). Given `buildId` and your own `testRunId` as `writerId`: + + - **Shell (`gh`/`kubectl`/`curl`/`git`)** — prefix the fetch with the + wrapper; it behaves exactly like the raw command (same stdout, same exit + code) but only executes on a miss: + `node /bin/cached-exec.mjs ''` + Wrap ONLY the fetch and pipe *outside* it, so different downstream + filters share one cached fetch: + `node .../cached-exec.mjs "$B" 3895 'gh api repos/o/r/contents/f' | jq -r .content | head -40` + One fetch per call — the wrapper refuses `;`/`&&`/backticks/redirects. + - **MCP data queries** (grafana/VictoriaLogs, `listTestIds`, + `getFailureLogs`) — check first, and store your digest on a miss: + `node /bin/cached-mcp.mjs get ''` + (exit 0 = hit, use it and skip the MCP call; exit 1 = miss, make the call + then `... put '' ` with the digest on stdin). + Worth it for expensive build-level queries several coordinators would + each re-run; skip it for a one-off only this test needs, since a miss + costs two extra calls. + - **NEVER cache `tfaRcaTurn` / `getTfaTurnResult` / `triggerRcaReport`** — + they are stateful, and the cache refuses them outright. + - Don't re-probe a connector the gate already validated (`gh auth status`, + `kubectl version`); the manifest above is the answer. + - Two wrapper gotchas, both hit in real use: **(i)** hit/miss banners go to + stderr so `| jq` works, but `2>&1 | jq` merges the banner into the pipe + and jq dies on it — don't redirect stderr into a pipe. **(ii)** a command + containing its own single quotes (e.g. `--jq '.[] | "\(.number)"'`) can't + be nested inside a single-quoted argument; pipe it in on stdin instead: + `printf '%s' '' | node .../cached-exec.mjs -`. + Metacharacters *inside* a quoted argument are fine — only a standalone + shell operator is refused, and a pipe belongs outside the wrapper anyway. + + **Never read an empty `prsInWindow` as "no PRs in the window."** An empty + list means "no PRs" ONLY when the entry also has `prsSearched: true`; + otherwise it was never populated and the two are indistinguishable in the + data. Check `coverage.reposWithUntrustedPrList` (or call + `hasTrustworthyPrList(doc, repo)`) before concluding anything from an empty + list — and when it is untrusted, run the PR search live. This is not + hypothetical: a pre-fetch once asserted 0 PRs for a repo that had 21, + which would have produced a confident "no culprit PR identified." When you + do run the search, contribute the result back — that records + `prsSearched` and spares everyone else the same trap. +1. **Logs by TFA — the core contract.** Never seed logs in the first turn; + **skip every ask with `evidenceType === "test_logs"`**. Never fetch, paste, + or digest log content. Logs are TFA's job. +2. **Read-only.** Every gather mechanism is read-only. Never write to a repo, + cluster, ticket, or the run. Produce a block and stop. +3. **Turn-cap** = `turnCap` from `config/rca.config.json` (default 6). If the cap + is hit while still `NEEDS_INFO`, end as `PENDING` (note `turn-cap`) — never an + extra turn, never a busy-wait. +4. **One thread per test.** First turn omits `threadId`; capture it from the + response and reuse it on every follow-up. Never start a second thread. +4b. **A drain ERROR kills the TURN, not the THREAD — resubmit, don't give up.** + `getTfaTurnResult` returning `TFA agent run failed` (or the submit itself + throwing it) is a dead turn, not a dead thread: observed repeatedly, a + fresh submit on the SAME `threadId` succeeds immediately and resolves at + high confidence. So when the drain fast-fails on consecutive hard errors, + the next move is to resubmit on that same thread (counting it as a turn) — + NOT to mint a new thread and not to end the run `PENDING`. Ending PENDING + here throws away a resolvable test. Only stop once the turn cap is spent. + +4b-i. **Two DIFFERENT TFA failures, don't confuse them.** + - `TFA agent run failed` — the wedge. Unrelated to message size (a + 240-char message wedged like a 1500-char one). Fix: resubmit on the same + thread, per 4b. + - `turn expired or not found` — observed on an over-cap (~2000-char) + submit. The text names a thread/turn problem, which reads as a wedge and + sends you down the wrong path; it is really a size rejection. If you see + this, shorten and resend before assuming the thread is broken. + +4b-ii. **Size-check any large fetch before trusting a negative result.** A + truncated payload turns "grep found nothing" into a false negative, and it + is silent. A coordinator nearly concluded a manifest didn't contain an + entry when the file had simply been cut at ~64KB — its own `wc -l` check + is what caught it (1042 lines vs 1518 real). The tool cache does not do + this (it truncates only past 256KB, and marks it), but the surrounding + tool plumbing can. So on any fetch of a big file: verify size or line + count first, and only then treat an absent match as evidence of absence. + +4c. **Keep every turn message under `turnMessageMaxChars` (1000)** — for + digest discipline, NOT as a wedge cure. An early correlation suggested + oversized messages caused the turn wedge (~1400/~1350-char submits failed + where a ~940-char retry landed, twice), but a later run refuted it + outright: a 240-char message wedged exactly as a 1500-char one did. So + respect the cap because a tight digest is the contract (link, don't paste) + — but do not expect trimming to prevent a wedge, and do not read a wedge + as evidence your message was too long. The wedge is a TFA-side fault whose + trigger is still unidentified; the reliable response is 4b (resubmit on the + same thread), not shrinking the payload. + +5. **Soft-PENDING is DRAINED, not reported.** `status: "PENDING"` means the tool's + 90s in-call poll expired, not that TFA has nothing to say — turns landing past + 90s are routine (a first turn finalizing `NEEDS_INFO` at 104s is a real, + observed case). So on `PENDING`, **call `getTfaTurnResult(testRunId, turnId)` + FIRST** and keep reading on the `softPendingDrain` budget + (`config/rca.config.json`: every 5s, ≤40 reads / ≤10min) until the status is + `RESOLVED` / `NEEDS_INFO` / `BLOCKED`. Only then route asks and submit the next + message. **Reads never count against the turn cap** — a drain re-reads the + *same* turn. Never submit a new message onto a turn still in flight: that + stacks two turns on one thread. Only when the drain budget is fully spent does + the run end `PENDING` (note `soft-pending`), resumable via `threadId`+`turnId`. + If the client has no `getTfaTurnResult` tool, end `PENDING` immediately as + before — never busy-wait through `tfaRcaTurn` resubmits instead. +6. **Digest, don't dump.** Every follow-up `message` carries digested findings + (`ask → found → snippet/link`), never raw log tails, full diffs, or full files. + Size caps + block shape live in `references/evidence-routing.md` — read it + before fulfilling any ask. The plugin config caps `message` at 1000 chars + (`turnMessageMaxChars` in `config/rca.config.json`); the `tfaRcaTurn` tool + itself would allow up to 5000, but the plugin self-limits to 1000. +7. **Report gaps, don't drop them.** An ask the coordinator cannot fulfill becomes + a `not-found` / `unreachable` / `unavailable` block, never a silent omission — + and **never a user prompt**. TFA finalizes best-effort with lower confidence. +8. **Never editorialize.** Report findings (suspect PR, server-side error line), + not verdicts. The root cause is TFA's to state on `RESOLVED`; pass its + `glimpse` through verbatim. +9. **Field-filter every gather call, always.** Before running any + capability-provided command (`gh`, `kubectl`, or whatever the manifest + resolved to for `github`/`infra`), project down to only the field(s) this + ask needs — `--jq`, `-o custom-columns`, `-o jsonpath`, or a `grep`/`head` + immediately piped. Never run the unfiltered form "just to see the shape" — + an exploratory call costs the same context whether or not its output ends + up in the digest, and a raw repo/commit/pod object typically carries + orders of magnitude more noise (license/URL metadata, multi-hundred-char + signature blocks, unrequested columns) than any evidence ask ever uses. + This governs what enters *your own* context via the tool result — distinct + from principle 6, which governs the digest you send back to TFA. Exact + command templates: `references/github-evidence.md` § Field-filtering. + +## Application bugs — the culprit-PR mandate (MANDATORY) + +Whenever TFA's classification (in an ask, a suggestion, or the resolving +`glimpse.failure_type`) is **PRODUCT_BUG / application bug**, the github +connector is the deliverable, not optional evidence: + +- **Hunt the culprit PR**: deploy timeline vs the last-pass window, changed + paths vs the failure signature (`references/github-evidence.md`), run the + falsification protocol on each candidate. +- **Feed the PR link(s) to TFA in the turn message** so the BrowserStack agent + populates `related_prs` in the dashboard RCA. +- **An application-bug RCA with no GitHub PR link is INCOMPLETE.** Keep digging + on subsequent turns until the turn cap. If still none, the turn message must + explicitly state `no culprit PR identified after ` — and the orchestrator records the gap on the CSV row. +- If the github connector is invalid/absent (a gate-recorded gap), state the + same explicitly plus an `unavailable` block. Never fabricate a PR. + +## Suspect-PR falsification (github asks) + +For `product_code` / `deploy` / `ci` asks, follow `references/github-evidence.md`: +gather the **exact** evidence (diff-since-baseline, PRs-in-window touching the +failing path, blame, deploy timing) via **GitHub MCP → `gh` → degrade**, and for +each candidate suspect **try to disprove it** (path overlap? shipped before the +failure window? behind an OFF flag?). Feed both supporting *and* disconfirming +evidence back as a structured suspect packet; only `verdict: supported` suspects +belong in `related_prs`. Reuse the pre-computed build-level evidence — do not +re-fetch per test (the `evidenceFile`'s `github` section, if present and not +`gap`-marked for this repo; otherwise the live github connector). A culprit +hunt often needs to go deeper than the file's summary — a full diff, a +downstream consumer of a changed flag — write that depth back via +`contributeGithubEvidence` once found, so a sibling confirming the same +suspect PR doesn't re-run the same diff/search. Never fabricate a PR when the github +capability is unavailable — emit an +`unavailable` block. + +## The loop + +``` +0. Parse inputs → testRunId (int). Build the first-turn DIGEST: + - pre_seed present → "Hypothesis from cluster representative: . + Suspect PR(s): . Confirm against THIS test's logs." (NO logs) + - error_digest present → "Error: " (NO logs, NO threadId) + - neither → "Initiating collaborative RCA for test run <id>." +1. SUBMIT turn 1: tfaRcaTurn(testRunId=<id>, message=<digest>). Capture threadId. turns_used = 1. + (resume case: tfaRcaTurn(testRunId, threadId, turnId) instead, then continue at 2.) +2. CLASSIFY result.status: + PENDING → DRAIN FIRST, do not resubmit and do not end here: + capture threadId + turnId, then loop on + getTfaTurnResult(testRunId, turnId) every softPendingDrain.intervalMs + until status != PENDING, or the budget (maxReads / maxWaitMs) is spent. + landed → replace `result` with it and re-CLASSIFY (turns_used UNCHANGED — + a read is not a turn; drop the spent turnId). + spent → END (PENDING, note "soft-pending"), row stays resumable. + no getTfaTurnResult tool → END (PENDING, note "soft-pending"). + RESOLVED → capture glimpse + viewRca; END (RESOLVED). + BLOCKED → END (PENDING, note "blocked") — terminal, no asks to route. + NEEDS_INFO → go to 3. +3. ROUTE the asks (read references/evidence-routing.md; route via lib/routing.mjs): + For each ask, high → medium → low: + skip → record in asks_skipped, emit nothing. + gather → FIRST check `evidenceFile` (if present) for this ask's scope — + repo for a github ask, workload for an infra/logs ask. Covered + (present, `gap` falsy) → paste its `block` straight in, no + re-digesting, no live call. Not named in the file, or its + entry has a `gap`, or no `evidenceFile` at all → run the + discovered skill/tool live, exactly as before — THEN write the + result back via `contributeGithubEvidence`/ + `contributeLogsEvidence` with your own testRunId as writerId + (Operating Principle 0) so this fills the gap for whoever + reads the file next. + Digest into one block. Record evidenceType in asks_fulfilled (dedupe). + gap → emit an `unavailable` block (record in asks_unavailable). NEVER prompt. + PRODUCT_BUG in play + no supported PR yet → widen the github hunt this turn. + Concatenate per-ask blocks into the next-turn MESSAGE (respect size caps). +4. SUBMIT follow-up on the SAME thread: tfaRcaTurn(testRunId, message, threadId). turns_used += 1. +5. TURN-CAP CHECK: if turns_used >= turnCap and still NEEDS_INFO → END (PENDING, "turn-cap"). + else → go to 2 with the new result. +6. EMIT the RCA_OUTPUT block from the captured terminal state. +``` + +> The loop mechanics above have an **executable mirror** in `lib/loop.mjs` +> (`runRcaLoop`) — conformance-tested against recorded `tfaRcaTurn` transcripts +> (`tests/conformance.test.mjs`). It also serves as the **sequential thin-client +> harness**: MCP clients without workflows/subagents drive the same contract +> by calling `runRcaLoop` with a real `submit` bound to `tfaRcaTurn`. + +**Sibling confirm (cluster member).** When `pre_seed` is present the first turn +states the representative's hypothesis and asks TFA to confirm against this +test's own logs. If TFA `RESOLVED`s in one turn → a logs-grounded per-test RCA at +minimal cost. If TFA instead returns `NEEDS_INFO` (the hypothesis does not hold +for this test), **fall back to the normal loop** — never blindly inherit the +representative's cause. + +## Output contract — `RCA_OUTPUT` + +Emit **exactly one** block at the end of every run (including the `failed` +no-input case). The orchestrator parses it into one CSV row / glimpse line. + +``` +RCA_OUTPUT_START + +## testRunId +<integer> + +## status +<RESOLVED | PENDING | failed> + +## confidence +<high | medium | low | unknown> # from the terminal turn; unknown for PENDING/failed + +## root_cause +<RESOLVED → glimpse.root_cause verbatim (already ≤220 chars) · PENDING/failed → "not available" or the note> + +## failure_type +<RESOLVED → glimpse.failure_type verbatim · else "not available"> + +## related_prs +- <each PR in glimpse.related_prs; "none" if empty — for PRODUCT_BUG, "none" only after the mandated hunt + explicit statement> + +## view_rca +<viewRca link from the RESOLVED turn (Test Observability UI) · "not available" if none> + +## suspect_signals +- <each non-log signal surfaced: suspect PR / deploy / server-side error line; "none" if empty> + +## thread_id +<threadId from the first turn · "not available" if none> + +## turn_id +<turnId — present for PENDING (resume handle); else "not available"> + +## turns_used +<integer 1..turnCap> + +## asks_fulfilled +- <evidenceType> # every non-test_logs type fulfilled; "none" if empty + +## asks_skipped +- test_logs # present once a test_logs ask appeared + +## asks_unavailable +- <evidenceType> # gate-recorded gaps (drives the coverage stamp); "none" if empty + +RCA_OUTPUT_END +``` + +Notes: +- `status` is one of exactly three values. `turn-cap`, `soft-pending` (drain + budget spent) and `blocked` all report as `PENDING`; note which in `root_cause`. + A `PENDING` from a *drained* turn should never appear — a drain that lands + re-classifies instead. +- `asks_skipped` always includes `test_logs` whenever TFA asked for logs. + `asks_fulfilled` **never** includes `test_logs`. +- `asks_unavailable` is the evidence-coverage signal the coverage stamp turns + into a confidence band. +- `failed` is the no-parseable-result / no-input case; the orchestrator + synthesizes a `failed` row if this coordinator dies — keep the block valid. + +## Hard limits + +- **Never** treat a `gap`-marked `evidenceFile` entry as coverage — a `gap` + means attempt a live call exactly as if the file didn't have that entry. +- **Never** prompt, ask, or wait on a user — the gate is closed; gaps degrade to `unavailable`. +- **Never** fulfill or seed a `test_logs` ask — TFA owns logs. +- **Never** exceed `turnCap` `tfaRcaTurn` calls in one run. +- **Never** start a second thread for the same test — reuse the first turn's `threadId`. +- **Never** submit a new `tfaRcaTurn` message while a turn is soft-`PENDING` — + drain it with `getTfaTurnResult` first; resubmitting stacks two turns on one thread. +- **Never** let drain reads consume the turn cap, and never drain past the + `softPendingDrain` budget — a wedged turn must not hang the batch. +- **Never** dump raw logs, full diffs, or full file contents into a turn message — digest only. +- **Never** run an unfiltered gather call (a bare `gh api ...` with no `--jq`, + `kubectl get ... -o wide`/`-o yaml` when a narrower `-o custom-columns` + answers the ask) — project to the needed field(s) before the call runs, not + by reading past the noise after. +- **Never** write to any repo / cluster / ticket / the run — every action is read-only. +- **Never** editorialize a cause — pass TFA's `glimpse` through verbatim. +- **Never** blindly inherit a representative's cause for a sibling — confirm against its own logs. +- **Never** resolve an application bug silently without a PR link — hunt until the + turn cap, else state "no culprit PR identified after <searched>" explicitly. +- **Always** emit exactly one valid `RCA_OUTPUT` block, even on the `failed` path. diff --git a/bin/cached-exec.mjs b/bin/cached-exec.mjs new file mode 100644 index 0000000..b6ab9fd --- /dev/null +++ b/bin/cached-exec.mjs @@ -0,0 +1,163 @@ +#!/usr/bin/env node +// Run a READ-ONLY command through the build's tool cache, in ONE tool call. +// +// Why a wrapper: a "check cache / run / store" sequence done by hand costs +// three tool calls to save one, which is worse than not caching. This collapses +// it to a single call that behaves exactly like the underlying command — +// same stdout, same exit code — but only actually executes on a miss. +// +// Usage (command is ONE argument, so the caller's own quoting survives): +// node bin/cached-exec.mjs <buildId> <writerId> '<command>' +// node bin/cached-exec.mjs <buildId> <writerId> - # command on STDIN +// node bin/cached-exec.mjs <buildId> --stats +// +// Wrap only the expensive fetch and leave filtering to the outer shell: +// node bin/cached-exec.mjs "$B" 3895581484 'gh api repos/o/r/contents/f' | jq -r .content | head -40 +// Two coordinators piping the same fetch through different greps then share +// one cache entry, instead of each paying for the fetch. +// +// TWO GOTCHAS, both hit in real use: +// +// 1. Hit/miss banners go to STDERR, so stdout stays byte-identical to the raw +// command and `| jq` works. But `2>&1 | jq` merges the banner back into +// the pipe and jq dies on it ("Invalid literal at line 1, column 12"). +// Don't redirect stderr into a pipe. If you silence it with `2>/dev/null` +// you also lose the hit/miss signal — so set `TOOLCACHE_LOG=<path>` and +// the banners are teed there too: `grep -c HIT <path>` still works. +// +// 2. Nested single quotes. A command containing its own `'…'` (typically +// `--jq '.[] | "\(.number)"'`) cannot be passed inside a single-quoted +// argument — the outer shell terminates the string early and the argument +// arrives mangled. Use `-` and pipe the command in on stdin instead: +// printf '%s' 'gh pr list -R o/r --json number --jq ".[].number"' \ +// | node bin/cached-exec.mjs "$B" 3895 - + +import { execFileSync } from "node:child_process"; +import { readFileSync, appendFileSync } from "node:fs"; +import { + toolCacheDirFor, cacheKey, cacheGet, cachePut, cacheStats, isRunnable, tokenize, +} from "../lib/tool-cache.mjs"; + +const [, , buildId, writerOrFlag, commandArg] = process.argv; + +// `-` means the command arrives on stdin, which sidesteps the nested-quoting +// problem entirely (see gotcha 2 above). +let command = commandArg; +if (command === "-") { + try { + command = readFileSync(0, "utf8").trim(); + } catch { + command = ""; + } + if (!command) { + console.error("[tool-cache] '-' given but stdin was empty"); + process.exit(2); + } +} + +if (!buildId || (writerOrFlag !== "--stats" && !command)) { + console.error("usage: cached-exec.mjs <buildId> <writerId> '<command>'"); + console.error(" cached-exec.mjs <buildId> --stats"); + process.exit(2); +} + +const dir = toolCacheDirFor(buildId, process.env.RCA_STATE_DIR ?? ""); + +// Where hit/miss banners go. Default stderr keeps stdout byte-identical to the +// wrapped command. But callers pipe stdout into jq/sed and silence stderr with +// `2>/dev/null` to keep the tool chatter out — which also throws away the +// banner, so the run's own hit-rate becomes unmeasurable. Setting +// TOOLCACHE_LOG=<path> tees banners to a file, letting a caller suppress +// stderr and still count hits afterwards (`grep -c HIT <path>`). +const logPath = process.env.TOOLCACHE_LOG ?? ""; +function banner(line) { + console.error(line); + if (logPath) { + try { + appendFileSync(logPath, line + "\n", { encoding: "utf8", mode: 0o600 }); + } catch { + /* logging must never break the fetch */ + } + } +} + +if (writerOrFlag === "--stats") { + const s = cacheStats(dir); + console.log(JSON.stringify({ cacheDir: dir, ...s }, null, 2)); + process.exit(0); +} + +// Parse into a fetch + filter chain before anything runs. +const gate = isRunnable(command); +if (!gate.ok) { + console.error(`[tool-cache REFUSED] ${gate.reason}`); + console.error(` command: ${command}`); + process.exit(2); +} + +// Key on the FETCH ONLY. Downstream filters are pure text transforms, so two +// agents filtering the same fetch differently share one cached network call. +const key = cacheKey(gate.fetchText); + +// Run one argv with `input` on stdin, no shell. Returns { stdout, exitCode }. +function run(argv, input) { + try { + return { + stdout: execFileSync(argv[0], argv.slice(1), { + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + // Capture stderr rather than let it inherit: execFileSync otherwise + // BOTH inherits and captures, so relaying it ourselves printed + // failures three times. + stdio: [input === undefined ? "ignore" : "pipe", "pipe", "pipe"], + ...(input === undefined ? {} : { input }), + }), + exitCode: 0, + }; + } catch (err) { + if (err.stderr) process.stderr.write(err.stderr.toString()); // the only copy + return { + stdout: (err.stdout ?? "").toString(), + exitCode: typeof err.status === "number" ? err.status : 1, + }; + } +} + +let fetched; +const hit = cacheGet(dir, key); +if (hit) { + banner(`[tool-cache HIT ${key} — captured by ${hit.writerId ?? "?"}, ${hit.bytes}B]`); + fetched = hit.stdout; +} else { + const res = run(gate.fetch, undefined); + fetched = res.stdout; + if (res.exitCode !== 0) { + // Preserve the real behaviour. Deliberately NOT cached — a transient + // failure (rate limit, expired token) must not become a permanent answer. + banner(`[tool-cache MISS ${key} — fetch exited ${res.exitCode}, NOT cached]`); + process.stdout.write(fetched); + process.exit(res.exitCode); + } + if (fetched.trim() === "") { + // An empty result is usually a wrong selector or a silently failed lookup; + // caching it creates a sticky, invisible negative for every later reader. + banner(`[tool-cache MISS ${key} — empty result, NOT cached]`); + } else { + // nowMs is read here, at the process edge — lib/ keeps its no-clock + // discipline so it stays sandbox-safe. + cachePut(dir, key, { command: gate.fetchText, writerId: writerOrFlag, stdout: fetched, exitCode: 0 }, Date.now()); + banner(`[tool-cache MISS ${key} — stored ${fetched.length}B]`); + } +} + +// Apply the filter chain to whatever the fetch produced (cached or fresh). +let out = fetched; +let finalExit = 0; +for (const f of gate.filters) { + const res = run(f, out); + out = res.stdout; + if (res.exitCode !== 0) { finalExit = res.exitCode; break; } +} + +process.stdout.write(out); +process.exit(finalExit); diff --git a/bin/cached-mcp.mjs b/bin/cached-mcp.mjs new file mode 100644 index 0000000..a5074ee --- /dev/null +++ b/bin/cached-mcp.mjs @@ -0,0 +1,133 @@ +#!/usr/bin/env node +// Memo cache for READ-ONLY **MCP** tool calls, sharing the same per-build +// store as `cached-exec.mjs`. +// +// Shell calls can be wrapped transparently (`cached-exec.mjs` runs the command +// for you). MCP calls cannot — only the agent can invoke an MCP tool — so the +// contract here is check-then-call: +// +// 1. get → node bin/cached-mcp.mjs <buildId> get <tool> '<argsJson>' +// exit 0 + result on stdout = HIT, skip the MCP call entirely +// exit 1, empty stdout = MISS, make the MCP call yourself +// 2. put → node bin/cached-mcp.mjs <buildId> put <tool> '<argsJson>' <writerId> +// (payload on STDIN — pipe the digest you want shared) +// +// WHEN THIS PAYS OFF, and when it does not. A hit replaces one MCP call with +// one cheap local read, so it wins on latency and on tokens whenever the +// cached payload is a digest smaller than the raw response. A miss costs two +// extra calls (the probe + the store), so this is worth it for **expensive, +// broadly-reusable, build-level queries** — a VictoriaLogs sweep, a +// `listTestIds`, a `getFailureLogs` several coordinators would each re-run — +// and NOT worth it for a one-off lookup only this test will ever need. +// +// Never cacheable (refused): `tfaRcaTurn`, `getTfaTurnResult`, +// `triggerRcaReport`. Those are stateful — a turn's status is *expected* to +// change between reads, so serving one from cache is wrong, not just stale. +// Prefer storing a DIGEST rather than a raw payload: the point is to spare the +// next reader the raw rows, not to relay them. + +import { readFileSync, readdirSync, existsSync, appendFileSync } from "node:fs"; +import { join } from "node:path"; +import { + toolCacheDirFor, mcpCacheKey, cacheGet, cachePut, cacheStats, isCacheableMcp, +} from "../lib/tool-cache.mjs"; + +// Same TOOLCACHE_LOG tee as cached-exec, so shell and MCP hits can be counted +// from one file. Previously only shell banners were logged, which made a run's +// combined hit rate impossible to total. +const logPath = process.env.TOOLCACHE_LOG ?? ""; +function banner(line) { + console.error(line); + if (logPath) { + try { appendFileSync(logPath, line + "\n", { encoding: "utf8", mode: 0o600 }); } catch { /* never break the call */ } + } +} + +const [, , buildId, verb, tool, argsJson, writerId] = process.argv; + +if (!buildId || !verb) { + console.error("usage: cached-mcp.mjs <buildId> get <tool> '<argsJson>'"); + console.error(" cached-mcp.mjs <buildId> put <tool> '<argsJson>' <writerId> # payload on stdin"); + console.error(" cached-mcp.mjs <buildId> list # what is cached, with exact args to copy"); + console.error(" cached-mcp.mjs <buildId> stats"); + process.exit(2); +} + +const dir = toolCacheDirFor(buildId, process.env.RCA_STATE_DIR ?? ""); + +if (verb === "stats") { + console.log(JSON.stringify({ cacheDir: dir, ...cacheStats(dir) }, null, 2)); + process.exit(0); +} + +// `list` exists because a HIT requires reproducing the args EXACTLY, and +// canonicalization only normalizes key ORDER, not content. A coordinator that +// guesses the logql/window/limit triple misses — one real run burned four +// probe calls guessing, to save two. Listing what is actually cached turns +// that into a single call: read the available queries, then `get` the one you +// want with its args copied verbatim. +if (verb === "list") { + if (!existsSync(dir)) { console.log("(no cache yet)"); process.exit(0); } + let n = 0; + for (const f of readdirSync(dir).filter((x) => x.endsWith(".json"))) { + let e; try { e = JSON.parse(readFileSync(join(dir, f), "utf8")); } catch { continue; } + if (!/^mcp__/.test(e.command ?? "")) continue; // shell entries live here too + n++; + const sp = e.command.indexOf(" "); + console.log(`\n[${e.key}] ${e.command.slice(0, sp)} (by ${e.writerId ?? "?"}, ${e.bytes}B)`); + console.log(` args: ${e.command.slice(sp + 1)}`); + console.log(` digest: ${String(e.stdout).replace(/\s+/g, " ").slice(0, 150)}…`); + } + if (!n) console.log("(no MCP entries cached — the orchestrator should pre-seed Step 4's queries)"); + process.exit(0); +} + +if (!tool || argsJson === undefined) { + console.error("both <tool> and '<argsJson>' are required"); + process.exit(2); +} + +if (!isCacheableMcp(tool)) { + console.error(`[mcp-cache REFUSED] ${tool} is stateful — never cache it; call it directly.`); + process.exit(2); +} + +let args; +try { + args = JSON.parse(argsJson); +} catch (err) { + console.error(`[mcp-cache] argsJson is not valid JSON: ${err.message}`); + process.exit(2); +} + +const key = mcpCacheKey(tool, args); + +if (verb === "get") { + const hit = cacheGet(dir, key); + if (!hit) { + banner(`[mcp-cache MISS ${key} ${tool}] — make the MCP call, then 'put' the digest`); + process.exit(1); + } + banner(`[mcp-cache HIT ${key} ${tool} — captured by ${hit.writerId ?? "?"}, ${hit.bytes}B]`); + process.stdout.write(hit.stdout); + process.exit(0); +} + +if (verb === "put") { + let payload = ""; + try { + payload = readFileSync(0, "utf8"); // stdin + } catch { + payload = ""; + } + if (!payload.trim()) { + console.error("[mcp-cache] refusing to store an empty payload"); + process.exit(2); + } + const rec = cachePut(dir, key, { command: `${tool} ${argsJson}`, writerId, stdout: payload }, Date.now()); + banner(`[mcp-cache STORED ${key} ${tool} — ${rec.bytes}B]`); + process.exit(0); +} + +console.error(`unknown verb: ${verb}`); +process.exit(2); diff --git a/bin/evidence-show.mjs b/bin/evidence-show.mjs new file mode 100644 index 0000000..7aa5ab8 --- /dev/null +++ b/bin/evidence-show.mjs @@ -0,0 +1,90 @@ +#!/usr/bin/env node +// Print the FOLDED evidence view: the orchestrator's base pre-fetch with every +// coordinator's contribution shard merged on top. +// +// Why this exists: coordinators are handed one path — the base file — and +// naturally read it with `cat`/`jq`. That shows base ONLY, so every +// contribution written by a sibling is invisible. A real run hit this: an +// agent reported "the file has 2 repos" when the folded view had 5, including +// the 11-PR observability-api entry a prior coordinator had contributed. The +// shard layout is what makes concurrent write-back safe, so the fix is to give +// the merged view its own command rather than to abandon shards. +// +// Usage: +// node bin/evidence-show.mjs <evidenceFilePath> # full folded JSON +// node bin/evidence-show.mjs <evidenceFilePath> --summary # one line per repo/workload +// node bin/evidence-show.mjs <evidenceFilePath> --repo <name> + +import { readEvidenceFile, readBaseFile, contribDirFor, hasTrustworthyPrList } from "../lib/evidence-file.mjs"; +import { existsSync, readdirSync } from "node:fs"; + +const [, , filePath, mode, arg] = process.argv; +if (!filePath) { + console.error("usage: evidence-show.mjs <evidenceFilePath> [--summary | --repo <name>]"); + process.exit(2); +} + +const folded = readEvidenceFile(filePath); + +if (mode === "--repo") { + console.log(JSON.stringify(folded.github?.[arg] ?? null, null, 2)); + process.exit(0); +} + +// `--prs` prints the one table that does the most falsification work per byte: +// mergedAt | #num | title. A coordinator compares mergedAt against the build's +// start_at and disqualifies everything merged after it — no diffs fetched. On +// one real run that removed 11 of 22 candidates before a single `gh pr view`, +// and getting there previously required piping --repo's raw JSON through an +// ad-hoc node one-liner. +if (mode === "--prs") { + const repos = arg ? [arg] : Object.keys(folded.github ?? {}); + for (const repo of repos) { + const e = folded.github?.[repo]; + if (!e) { console.log(`${repo}: (not in evidence file)`); continue; } + const prs = e.prsInWindow ?? []; + const trust = e.prsSearched === true || prs.length > 0 ? "" : " [LIST NOT TRUSTWORTHY — never searched]"; + console.log(`\n${repo} (${prs.length} PR(s))${trust}`); + for (const p of prs.sort((a, b) => String(a.mergedAt).localeCompare(String(b.mergedAt)))) { + console.log(` ${p.mergedAt ?? "?".padEnd(24)} ${String(p.pr).padEnd(7)} ${String(p.title ?? "").slice(0, 88)}`); + } + } + const w = folded.suspectWindow; + if (w?.startedAt) { + console.log(`\nbuild started_at: ${w.startedAt}`); + console.log(" → anything merged AFTER that could not have shipped in this build (window guard)."); + } + process.exit(0); +} + +if (mode !== "--summary") { + console.log(JSON.stringify(folded, null, 2)); + process.exit(0); +} + +const base = readBaseFile(filePath); +const dir = contribDirFor(filePath); +const shards = existsSync(dir) ? readdirSync(dir).filter((f) => f.endsWith(".json")) : []; + +console.log(`build : ${folded.buildId}`); +console.log(`base repos : ${Object.keys(base.github ?? {}).length}`); +console.log(`contribution shards: ${shards.length} (${shards.map((s) => s.replace(".json", "")).join(", ") || "none"})`); +console.log(""); +console.log("github (folded):"); +for (const [repo, e] of Object.entries(folded.github ?? {})) { + const prs = (e.prsInWindow ?? []).length; + const trust = hasTrustworthyPrList(folded, repo) ? "trustworthy" : "PR LIST NOT TRUSTWORTHY (never searched)"; + console.log(` ${repo}: ${prs} PR(s), deployState=${e.deployState ? "yes" : "no"}, gap=${e.gap ?? "none"} — ${trust}`); +} +console.log(""); +console.log("logs (folded):"); +for (const [wl, e] of Object.entries(folded.logs ?? {})) { + const k = e.kubectlSweep?.gap ? "gapped" : e.kubectlSweep ? "present" : "absent"; + const v = e.victorialogs?.gap ? "gapped" : e.victorialogs ? "present" : "absent"; + console.log(` ${wl}: kubectl=${k}, victorialogs=${v}, gap=${e.gap ?? "none"}`); +} +if (folded.coverage?.reposWithUntrustedPrList?.length) { + console.log(""); + console.log(`WARNING untrusted PR lists: ${folded.coverage.reposWithUntrustedPrList.join(", ")}`); + console.log(" an empty prsInWindow here does NOT mean 'no PRs' — search live before concluding."); +} diff --git a/codex-mcp.example.toml b/codex-mcp.example.toml new file mode 100644 index 0000000..e3dda80 --- /dev/null +++ b/codex-mcp.example.toml @@ -0,0 +1,11 @@ +# Codex MCP wiring for the bstack server. +# Codex reads ~/.codex/config.toml (no per-project MCP file), so copy this block +# into your global config — or use the `codex mcp add` one-liner in INTEGRATION.md. +# Replace the env values with your BrowserStack credentials. + +[mcp_servers.bstack] +command = "npx" +args = ["-y", "@browserstack/mcp-server@1.2.27-beta.1"] +env = { "BROWSERSTACK_USERNAME" = "your-username", "BROWSERSTACK_ACCESS_KEY" = "your-access-key", "O11Y_TFA_RCA_BASE_URL" = "" # optional: set only to target a staging tenant (default is production) } +startup_timeout_sec = 15 +tool_timeout_sec = 120 diff --git a/config/rca.config.json b/config/rca.config.json new file mode 100644 index 0000000..aea7ace --- /dev/null +++ b/config/rca.config.json @@ -0,0 +1,70 @@ +{ + "$comment": "Central config for the /rca-build RCA harness. All formerly-hardcoded product/infra values live here. No kubectl/chitragupta/bifrost literals — connectors are discovered and probe-validated at the gate into the capability manifest (see skills/rca-build/references/evidence-routing.md). No reportFile: the plugin never writes a local RCA report — the full report lives on the Test Observability UI (triggerRcaReport).", + "mcpServerName": "bstack", + "$concurrencyComment": "Max coordinator subagents run in parallel during Step 5 fan-out. HONORED LITERALLY on the default path (direct Agent-tool dispatch of ai-tfa-coordinator subagents — one message, up to `concurrency` tool-use blocks per batch) and by the sequential harness (lib/loop.mjs). Only a SOFT target on the opt-in Workflow-tool path (workflows/rca-batch.mjs), which the Workflow runtime hard-caps at min(16, cpu cores - 2) regardless of this value — that cap is architectural and cannot be raised from this repo. See SKILL.md Step 5.", + "concurrency": 20, + "turnCap": 6, + "turnMessageMaxChars": 1000, + "pollSoftPendingMs": 90000, + "$softPendingDrainComment": "tfaRcaTurn abandons its in-call poll at pollSoftPendingMs (90s) and returns a soft PENDING while the TFA agent keeps working — turns finalizing past 90s are routine. On a soft PENDING the loop READS the same turnId via getTfaTurnResult on this budget before it routes asks or submits anything further; reads do not consume turnCap. Only when the budget is spent does the run end PENDING (pending-resume row).", + "$maxErrorReadsComment": "A soft PENDING is drained on the full budget below, but a HARD read failure (a thrown error, or a result whose status/message says the TFA agent run failed) is a different signal: it will not resolve by asking again. After this many CONSECUTIVE failed reads the drain stops early and the row ends PENDING with a `tfa-error` note, still resumable. A single good read clears the streak. Measured motivation: on one real build, drain reads plus their sleeps were 23% of all coordinator tool calls, and the four tests that wedged this way were the four slowest in the batch.", + "softPendingDrain": { + "maxWaitMs": 600000, + "intervalMs": 5000, + "maxReads": 40, + "maxErrorReads": 3 + }, + "reaperHeartbeatTtlSec": 600, + "errorSummaryMaxChars": 200, + "paths": { + "$comment": "State CSV path is ALWAYS derived per build via lib/csv-state.mjs csvPathFor(buildId, stateDir): the build id is in the filename (no cross-build collisions) and the default directory is OS temp (<tmpdir>/bstack-rca/), so the harness never pollutes the invoking workspace. Set stateDir only to retain the CSV somewhere specific (e.g. a CI artifact dir).", + "stateDir": "" + }, + "evidenceRouting": { + "test_logs": { + "owner": "tfa", + "skip": true + }, + "product_code": { + "capability": "github", + "discoveryHints": [ + "github-mcp", + "gh" + ] + }, + "deploy": { + "capability": "github", + "discoveryHints": [ + "github-mcp", + "gh" + ] + }, + "ci": { + "capability": "github", + "discoveryHints": [ + "github-mcp", + "gh" + ] + }, + "infra": { + "capability": "infra", + "discoveryHints": [] + }, + "k8s": { + "capability": "infra", + "discoveryHints": [] + }, + "kibana": { + "capability": "logs", + "discoveryHints": [] + }, + "metrics": { + "capability": "metrics", + "discoveryHints": [] + }, + "other": { + "capability": "other", + "discoveryHints": [] + } + } +} diff --git a/lib/coverage.mjs b/lib/coverage.mjs new file mode 100644 index 0000000..0266ceb --- /dev/null +++ b/lib/coverage.mjs @@ -0,0 +1,39 @@ +// Evidence-coverage stamp (ideation #6, v1 — the per-row coverage band; the +// build-level blast-radius digest is deferred). A RESOLVED RCA built with +// infra+logs+metrics all "unavailable" must not read like one with full +// evidence. The client (which routed every ask) stamps each row with a coverage +// vector and derives a coverage-capped confidence band the reviewer sees: +// "low confidence BECAUSE kibana was unavailable", not "low confidence, trust me". + +const BAND_ORDER = ["low", "medium", "high"]; + +// coverage classification from what was fulfilled vs. left unavailable. +export function classifyCoverage(asksFulfilled = [], asksUnavailable = []) { + const unavailable = [...new Set(asksUnavailable.filter(Boolean))]; + const fulfilled = [...new Set(asksFulfilled.filter(Boolean))]; + if (unavailable.length === 0) return "full"; + if (fulfilled.length > 0) return "partial"; + return "thin"; +} + +// Cap the band: full coverage keeps TFA's confidence; partial caps at medium; +// thin caps at low. Unknown/absent TFA confidence floors to low. +function capBand(tfaConfidence, coverage) { + const base = BAND_ORDER.includes(tfaConfidence) ? tfaConfidence : "low"; + const cap = coverage === "full" ? "high" : coverage === "partial" ? "medium" : "low"; + return BAND_ORDER[Math.min(BAND_ORDER.indexOf(base), BAND_ORDER.indexOf(cap))]; +} + +// The stamp written to a row at flip time. Returns { coverage, band, unavailable }. +export function coverageStamp({ + asksFulfilled = [], + asksUnavailable = [], + tfaConfidence = "unknown", +} = {}) { + const coverage = classifyCoverage(asksFulfilled, asksUnavailable); + return { + coverage, + band: capBand(tfaConfidence, coverage), + unavailable: [...new Set(asksUnavailable.filter(Boolean))], + }; +} diff --git a/lib/csv-state.mjs b/lib/csv-state.mjs new file mode 100644 index 0000000..ee970c5 --- /dev/null +++ b/lib/csv-state.mjs @@ -0,0 +1,333 @@ +// CSV write-ahead-log spine for the batch (D4 + ideation #7). The CSV is the +// single durable, resumable source of truth for "RCA over ALL failed tests": +// every test is a row, seeded `pending`, claimed by a worker, heartbeated while +// in flight, and flipped to a terminal state with its RCA. A reaper reclaims +// rows stranded by a crashed worker. +// +// Timestamps are passed in as `nowMs` (never read from the clock here) so this +// module is deterministic in tests AND usable from the auto-mode dynamic +// workflow, whose sandbox forbids Date.now(). +// +// In-session / in-workspace only — cross-session durability is deferred. Writes +// are synchronous read-modify-write; Node's single thread serializes them, which +// is sufficient for the in-process 5-concurrent workflow (true multi-process +// locking is out of scope). + +import { readFileSync, writeFileSync, existsSync, mkdirSync, chmodSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { tmpdir } from "node:os"; + +/** + * Canonical state-file path for one build's run. Two invariants (D-temp): + * 1. The BUILD ID IS IN THE FILENAME — runs over different builds can never + * collide/"resume" into each other's state. + * 2. Default location is OS TEMP (`<tmpdir>/bstack-rca/`), not the invoking + * workspace — a background harness must not pollute the repo it runs from. + * `stateDir` (config `paths.stateDir`) overrides the directory only — e.g. a CI + * job that wants the CSV as a retained artifact. Resume-safety is per build: + * same buildId → same path. + */ +export function csvPathFor(buildId, stateDir = "") { + const safe = String(buildId ?? "").replace(/[^A-Za-z0-9._-]/g, "_") || "unknown-build"; + const dir = stateDir && String(stateDir).trim() !== "" ? String(stateDir) : join(tmpdir(), "bstack-rca"); + return join(dir, `rca-state.${safe}.csv`); +} + +export const COLUMNS = [ + "buildId", + "testRunId", + "testName", + "failure_category", + "error_summary", + "file_path", + "cluster_id", + "rca_done", + "in_flight_worker", + "heartbeat_ts", + "threadId", + "turnId", + "last_evidence_digest", + "root_cause", + "failure_type", + "possible_fix", + "related_prs", + "coverage", + // Both are part of the RCA_OUTPUT contract but had no column, so `flip` + // silently discarded them — `view_rca` in particular is the dashboard link + // the whole run exists to produce. + "view_rca", + "turns_used", + "confidence", + "timestamp", +]; + +export const PENDING = "pending"; +export const RESUMABLE = "pending-resume"; +// Truly done — never re-claimed, listed, or reaped. +const TERMINAL_STATES = new Set(["resolved", "blocked", "failed"]); +// Valid outcomes flip() may write. `pending-resume` is a *soft* terminal: this +// attempt ended (claim cleared) but the row stays resumable — it keeps its +// threadId/turnId and is picked back up by the next fan-out / resume pass. +const FLIP_STATES = new Set(["resolved", "blocked", "failed", RESUMABLE]); + +// ---- minimal RFC4180-ish CSV codec ---------------------------------------- + +function encodeField(value) { + const s = value == null ? "" : String(value); + if (/[",\r\n]/.test(s)) { + return `"${s.replace(/"/g, '""')}"`; + } + return s; +} + +function encodeRows(rows) { + const lines = [COLUMNS.join(",")]; + for (const row of rows) { + lines.push(COLUMNS.map((c) => encodeField(row[c])).join(",")); + } + return lines.join("\n") + "\n"; +} + +function parseCsv(text) { + const rows = []; + let field = ""; + let record = []; + let inQuotes = false; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (inQuotes) { + if (ch === '"') { + if (text[i + 1] === '"') { + field += '"'; + i++; + } else { + inQuotes = false; + } + } else { + field += ch; + } + } else if (ch === '"') { + inQuotes = true; + } else if (ch === ",") { + record.push(field); + field = ""; + } else if (ch === "\n" || ch === "\r") { + if (ch === "\r" && text[i + 1] === "\n") i++; + record.push(field); + rows.push(record); + field = ""; + record = []; + } else { + field += ch; + } + } + if (field.length > 0 || record.length > 0) { + record.push(field); + rows.push(record); + } + return rows; +} + +// ---- read / write ---------------------------------------------------------- + +export function readRows(csvPath) { + if (!existsSync(csvPath)) return []; + const text = readFileSync(csvPath, "utf8"); + const raw = parseCsv(text).filter((r) => r.some((c) => c.length > 0)); + if (raw.length === 0) return []; + const header = raw[0]; + return raw.slice(1).map((cells) => { + const row = {}; + header.forEach((col, idx) => { + row[col] = cells[idx] ?? ""; + }); + return row; + }); +} + +// Owner-only (0700 dir / 0600 file): the state CSV lives in a world-readable +// OS temp dir and records root causes, culprit PRs and evidence digests. +export function writeRows(csvPath, rows) { + const dir = dirname(csvPath); + if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true, mode: 0o700 }); + const existed = existsSync(csvPath); + writeFileSync(csvPath, encodeRows(rows), { encoding: "utf8", mode: 0o600 }); + // `mode` applies on create only — tighten a pre-hardening leftover too. + if (existed) chmodSync(csvPath, 0o600); +} + +function emptyRow() { + return Object.fromEntries(COLUMNS.map((c) => [c, ""])); +} + +// ---- operations ------------------------------------------------------------- + +// Seed the CSV from a listTestIds(failed, includeFailureDetail) payload. Every +// row starts `pending`. Idempotent: existing rows are preserved (terminal rows +// are never reset; signature columns are refreshed on still-pending rows). New +// tests are appended. Returns the full row set. +export function seed(csvPath, buildId, tests) { + const existing = readRows(csvPath); + const byId = new Map(existing.map((r) => [String(r.testRunId), r])); + + for (const t of tests) { + const id = String(t.test_id ?? t.testRunId); + const sig = t.failure ?? {}; + const prior = byId.get(id); + if (prior) { + // Keep terminal results; only refresh signature on still-pending rows. + if (prior.rca_done === PENDING) { + prior.failure_category = sig.category ?? prior.failure_category; + prior.error_summary = sig.error_summary ?? prior.error_summary; + prior.file_path = sig.file_path ?? prior.file_path; + } + continue; + } + const row = emptyRow(); + row.buildId = buildId; + row.testRunId = id; + row.testName = t.test_name ?? t.testName ?? `Test ${id}`; + row.failure_category = sig.category ?? ""; + row.error_summary = sig.error_summary ?? ""; + row.file_path = sig.file_path ?? ""; + row.rca_done = PENDING; + byId.set(id, row); + existing.push(row); + } + + writeRows(csvPath, existing); + return existing; +} + +// Claim a pending row for `worker`. Refuses (returns false) if another worker +// already owns it. Returns true on success. +export function claim(csvPath, testRunId, worker, nowMs) { + const rows = readRows(csvPath); + const row = rows.find((r) => String(r.testRunId) === String(testRunId)); + if (!row) return false; + if (row.in_flight_worker && row.in_flight_worker !== worker) return false; + if (TERMINAL_STATES.has(row.rca_done)) return false; + row.in_flight_worker = worker; + row.heartbeat_ts = String(nowMs); + writeRows(csvPath, rows); + return true; +} + +export function heartbeat(csvPath, testRunId, worker, nowMs) { + const rows = readRows(csvPath); + const row = rows.find((r) => String(r.testRunId) === String(testRunId)); + if (!row || row.in_flight_worker !== worker) return false; + row.heartbeat_ts = String(nowMs); + writeRows(csvPath, rows); + return true; +} + +// Flip a row to a terminal state, recording the RCA fields and clearing the +// in-flight claim. `fields` carries any of: rca_done, root_cause, failure_type, +// possible_fix, related_prs, threadId, turnId, coverage, confidence, +// last_evidence_digest, cluster_id. +// The RCA_OUTPUT contract speaks `RESOLVED | PENDING | failed`, while the CSV +// stores `resolved | blocked | failed | pending-resume`. Callers naturally pass +// the vocabulary their own output block mandates, so accept it and translate +// rather than silently rejecting — a silent `false` here cost a whole batch of +// results, since the row simply stayed `pending` and looked un-run. +const FLIP_ALIASES = new Map([ + ["resolved", "resolved"], + ["pending", RESUMABLE], + ["pending-resume", RESUMABLE], + ["blocked", "blocked"], + ["failed", "failed"], + ["done", "resolved"], +]); + +// Column aliases for the same reason: the output block says `thread_id` and +// `status`, the CSV says `threadId` and `rca_done`. +const COLUMN_ALIASES = new Map([ + ["thread_id", "threadId"], + ["turn_id", "turnId"], + ["status", "rca_done"], + ["test_run_id", "testRunId"], +]); + +export function flip(csvPath, testRunId, fields, nowMs) { + // Arity guard. `flip` is positional with csvPath FIRST, and a caller that + // drops it — `flip(testRunId, fields)` — otherwise binds an object to + // testRunId, reads a nonexistent CSV, and gets a bare `false` that is easy + // to mistake for success. Name the mistake precisely instead. + if (typeof csvPath !== "string" || (testRunId !== null && typeof testRunId === "object")) { + console.warn( + "[csv-state] flip called with the wrong arguments. Signature is " + + "flip(csvPath, testRunId, fields, nowMs) — csvPath FIRST, e.g. " + + "flip(csvPathFor(buildId), '3904695279', { status: 'RESOLVED', ... }, Date.now()). " + + `Got csvPath=${JSON.stringify(csvPath)?.slice(0, 60)}, testRunId=${JSON.stringify(testRunId)?.slice(0, 60)}. Row NOT written.`, + ); + return false; + } + // Enforce the contract: a flip must name a valid outcome. A partial flip with + // a missing/non-terminal rca_done would otherwise clear the claim yet leave the + // row `pending` — re-exposing it for a duplicate RCA that clobbers this result. + // Reject without mutating so the worker keeps its claim and the bug surfaces. + const raw = fields?.rca_done ?? fields?.status; + const state = FLIP_ALIASES.get(String(raw ?? "").trim().toLowerCase()); + if (!state) { + // Loud, not silent: the previous bare `false` was indistinguishable from + // success to a caller that didn't check, and results were lost that way. + console.warn( + `[csv-state] flip REJECTED for testRunId=${testRunId}: rca_done=${JSON.stringify(raw)} ` + + `is not one of ${[...new Set(FLIP_ALIASES.values())].join(" | ")} (case-insensitive). Row NOT written.`, + ); + return false; + } + const rows = readRows(csvPath); + const row = rows.find((r) => String(r.testRunId) === String(testRunId)); + if (!row) { + console.warn(`[csv-state] flip REJECTED: no row for testRunId=${testRunId} in ${csvPath}`); + return false; + } + const dropped = []; + for (const [k0, v] of Object.entries(fields)) { + const k = COLUMN_ALIASES.get(k0) ?? k0; + if (COLUMNS.includes(k)) { + row[k] = Array.isArray(v) ? v.join("; ") : (v ?? ""); + } else { + dropped.push(k0); + } + } + row.rca_done = state; // normalized, whatever spelling arrived + if (dropped.length) { + console.warn(`[csv-state] flip ignored unknown field(s) for ${testRunId}: ${dropped.join(", ")}`); + } + row.in_flight_worker = ""; + row.timestamp = String(nowMs); + writeRows(csvPath, rows); + return true; +} + +// Reclaim rows stranded in flight (heartbeat older than ttlSec) back to pending. +// Returns the testRunIds reclaimed. Run on startup before resuming a batch. +export function reaper(csvPath, ttlSec, nowMs) { + const rows = readRows(csvPath); + const reclaimed = []; + for (const row of rows) { + if (!row.in_flight_worker) continue; + if (TERMINAL_STATES.has(row.rca_done)) continue; + const hb = Number(row.heartbeat_ts); + const stale = !row.heartbeat_ts || nowMs - hb > ttlSec * 1000; + if (stale) { + row.in_flight_worker = ""; + row.rca_done = PENDING; + reclaimed.push(String(row.testRunId)); + } + } + if (reclaimed.length > 0) writeRows(csvPath, rows); + return reclaimed; +} + +// Rows still needing work: fresh/reclaimed `pending` AND `pending-resume` rows +// (soft-PENDING attempts that retain a threadId/turnId to resume). The fan-out +// work-list. Truly terminal rows (resolved/blocked/failed) are excluded. +export function pendingRows(csvPath) { + return readRows(csvPath).filter( + (r) => r.rca_done === PENDING || r.rca_done === RESUMABLE, + ); +} diff --git a/lib/evidence-cache.mjs b/lib/evidence-cache.mjs new file mode 100644 index 0000000..b9c9523 --- /dev/null +++ b/lib/evidence-cache.mjs @@ -0,0 +1,47 @@ +// Build-level evidence cache (ideation #2). "Diff since last green", "deploy +// timeline", "PRs in the suspect window" are properties of the BUILD, not the +// test — yet a naive loop re-fetches them per test. Compute once, cache by +// (repo, commit-range, evidenceType), and pre-seed every coordinator with the +// same grounded suspect window. Collapses N×M redundant git/infra calls to ~M. +// +// The cache is created fresh per run (function-scoped Map — never a module-level +// global), so it holds no cross-run/cross-user state: in-workspace, single +// session, multi-tenant-safe by construction. + +export function makeEvidenceCache() { + const store = new Map(); + const keyOf = (repo, range, evidenceType) => + `${repo ?? ""}@@${range ?? ""}@@${evidenceType ?? ""}`; + + return { + has(repo, range, evidenceType) { + return store.has(keyOf(repo, range, evidenceType)); + }, + get(repo, range, evidenceType) { + return store.get(keyOf(repo, range, evidenceType)); + }, + set(repo, range, evidenceType, value) { + store.set(keyOf(repo, range, evidenceType), value); + return value; + }, + // Compute-once: run `fn` only on a cache miss; reuse on every later call. + async compute(repo, range, evidenceType, fn) { + const k = keyOf(repo, range, evidenceType); + if (store.has(k)) return store.get(k); + const value = await fn(); + store.set(k, value); + return value; + }, + size() { + return store.size; + }, + }; +} + +// Resolve the baseline ref for the last-green→this-build delta. When there is no +// "last green" (e.g. a never-green flaky suite) fall back to a configured ref and +// flag it so the report can note the weaker grounding. +export function resolveBaseline(lastGreenRef, fallbackRef) { + if (lastGreenRef) return { ref: lastGreenRef, isFallback: false }; + return { ref: fallbackRef ?? null, isFallback: true }; +} diff --git a/lib/evidence-file.mjs b/lib/evidence-file.mjs new file mode 100644 index 0000000..2e76215 --- /dev/null +++ b/lib/evidence-file.mjs @@ -0,0 +1,396 @@ +// Build-level evidence pre-fetch artifact (see docs/plan: evidence-file). PR +// windows, deploy state, and app-log sweeps are properties of the BUILD, not +// of any one test — a naive batch re-fetches them once per dispatched +// coordinator. This module persists them to a file ONCE so every +// representative and sibling `ai-tfa-coordinator` dispatch can `Read` the same +// artifact instead of re-running the same `gh`/`kubectl`/`grafana` calls. +// +// Layered under `lib/evidence-cache.mjs`, not merged with it: the cache is an +// in-process, function-scoped Map that dedups compute *within* the +// orchestrator's own Step 4 pass; this module is what makes that result +// visible to OTHER processes (the independently-dispatched coordinator +// subagents, which share no memory with the orchestrator or each other). +// +// Path convention mirrors `lib/csv-state.mjs`'s `csvPathFor` exactly: the +// build id is in the filename (no cross-build collisions) and the default +// directory is OS temp (`<tmpdir>/bstack-rca/`), so a build's evidence file +// sits right next to its state CSV. `stateDir` overrides the directory only. +// +// Invariant: this file NEVER carries `test_logs` content. `logs` is keyed by +// *workload* (an infra/pod concept), populated only via the `infra`/`logs` +// capability — TFA remains the sole owner of test-side SDK/driver/session +// logs, which structurally cannot land here. +// +// Timestamps are passed in as `nowMs` (never read from the clock here), same +// discipline as `csv-state.mjs`, so this stays usable from the Workflow-tool +// sandbox (which forbids `Date.now()`). +// +// Write-back, WITHOUT a lock and WITHOUT lost updates — single-writer shards. +// The orchestrator's Step 4 pass is not the only writer: a coordinator that +// had to gather live (a repo/PR/workload the pre-fetch didn't cover, or +// covered only with a summary) should persist what it found so a sibling +// dispatched after it — or another cluster sharing the same repo/workload — +// reads the enriched result instead of re-fetching it. +// +// Naively that means N concurrent coordinators read-modify-writing ONE JSON +// file, which drops updates whenever two writes interleave. Instead of a lock +// (fragile, and `csv-state.mjs` already declares multi-process locking out of +// scope) the layout makes contention structurally impossible: +// +// <tmpdir>/bstack-rca/ +// rca-evidence.<buildId>.json <- BASE: only the orchestrator writes it +// rca-evidence.<buildId>.contrib/ +// <writerId>.json <- one file per coordinator; sole writer +// <writerId>.json +// +// Every file has exactly ONE writer, so no write can ever clobber another's. +// Reads (`readEvidenceFile`) fold base + every shard into a single view, +// deterministically (shards applied in sorted filename order). This is the +// same "the temp dir is ours, use more of it" trick the CSV path convention +// already leans on. + +// Fold precedence, applied per leaf when base and shards disagree: +// 1. Real evidence beats a recorded gap — a coordinator that actually got +// the data overrides the pre-fetch's "couldn't reach this". +// 2. Among two real values, the later shard wins (sorted order), on the +// assumption a coordinator only writes back something deeper than what +// it read. +// 3. `prsInWindow` is unioned by PR number rather than replaced, so two +// coordinators finding different PRs in the same repo both survive. + +import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, chmodSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { tmpdir } from "node:os"; + +const safeName = (v, fallback) => + String(v ?? "").replace(/[^A-Za-z0-9._-]/g, "_") || fallback; + +/** Canonical evidence-file path for one build's run. Same two invariants as + * `csvPathFor`: build id in the filename; OS temp by default; `stateDir` + * overrides the directory only. */ +export function evidencePathFor(buildId, stateDir = "") { + const safe = safeName(buildId, "unknown-build"); + const dir = stateDir && String(stateDir).trim() !== "" ? String(stateDir) : join(tmpdir(), "bstack-rca"); + return join(dir, `rca-evidence.${safe}.json`); +} + +/** Directory holding this build's per-coordinator contribution shards. Derived + * from the base path so callers only ever have to pass `evidenceFilePath` + * around — one input, no second path to thread through every dispatch. */ +export function contribDirFor(basePath) { + return String(basePath).replace(/\.json$/, "") + ".contrib"; +} + +/** The one file a given writer owns. Exactly one writer per path is the whole + * point — never call this for a writerId that isn't yours. */ +export function contribPathFor(basePath, writerId) { + return join(contribDirFor(basePath), `${safeName(writerId, "unknown-writer")}.json`); +} + +/** Shard docs in deterministic (sorted-filename) order. Missing dir → []. A + * corrupt/half-written shard is skipped rather than throwing: a coordinator + * killed mid-write must not break every subsequent read. */ +function readContribs(basePath) { + const dir = contribDirFor(basePath); + if (!existsSync(dir)) return []; + const out = []; + for (const name of readdirSync(dir).filter((n) => n.endsWith(".json")).sort()) { + try { + out.push(JSON.parse(readFileSync(join(dir, name), "utf8"))); + } catch { + // skip unreadable/partial shard + } + } + return out; +} + +export function emptyEvidenceFile(buildId, nowMs) { + return { + buildId: String(buildId ?? ""), + generatedAtMs: nowMs, + baseline: null, + suspectWindow: null, + github: {}, + logs: {}, + coverage: { reposCovered: [], reposGapped: [], workloadsCovered: [], workloadsGapped: [] }, + }; +} + +/** The BASE file alone, no shards folded in. Internal to the orchestrator's + * write path: `set*`/`merge*` must read-modify-write base only, or they would + * absorb shard content into base and duplicate it on the next fold. */ +export function readBaseFile(filePath) { + if (!existsSync(filePath)) return emptyEvidenceFile("unknown-build", 0); + try { + return JSON.parse(readFileSync(filePath, "utf8")); + } catch { + return emptyEvidenceFile("unknown-build", 0); + } +} + +// A {block, gap} leaf: real evidence beats a gap; between two real values the +// later (shard) one wins. `undefined`/`null` incoming never overwrites. +function pickLeaf(base, incoming) { + if (incoming == null) return base ?? null; + if (base == null) return incoming; + if (!incoming.gap) return incoming; + if (!base.gap) return base; + return incoming; +} + +function foldGithub(target, repo, entry) { + const cur = target[repo] ?? { deployState: null, prsInWindow: [], gap: null }; + const next = { + deployState: pickLeaf(cur.deployState, entry.deployState), + prsInWindow: cur.prsInWindow ?? [], + // Sticky: once ANY writer has genuinely run the PR search, the entry stays + // trustworthy — a later contributor that didn't search must not silently + // downgrade it back to "unknown". + prsSearched: cur.prsSearched === true || entry.prsSearched === true, + gap: cur.gap ?? null, + }; + if (Array.isArray(entry.prsInWindow)) { + const byPr = new Map((next.prsInWindow ?? []).map((p) => [String(p.pr), p])); + for (const pr of entry.prsInWindow) byPr.set(String(pr.pr), pr); + next.prsInWindow = [...byPr.values()]; + } + // A contributor supplying real content clears the pre-fetch's gap. + if (entry.gap === null || entry.gap === undefined) { + if (entry.deployState || Array.isArray(entry.prsInWindow)) next.gap = null; + } else if (!next.deployState && (next.prsInWindow ?? []).length === 0) { + next.gap = entry.gap; + } + target[repo] = next; +} + +function foldLogs(target, workload, entry) { + const cur = target[workload] ?? { clusterIds: [], kubectlSweep: null, victorialogs: null, gap: null }; + const next = { + clusterIds: [...new Set([...(cur.clusterIds ?? []), ...(entry.clusterIds ?? [])])], + kubectlSweep: pickLeaf(cur.kubectlSweep, entry.kubectlSweep), + victorialogs: pickLeaf(cur.victorialogs, entry.victorialogs), + gap: cur.gap ?? null, + }; + if (entry.gap === null || entry.gap === undefined) { + if (entry.kubectlSweep || entry.victorialogs) next.gap = null; + } else if (!next.kubectlSweep && !next.victorialogs) { + next.gap = entry.gap; + } + target[workload] = next; +} + +/** The full view every CONSUMER should read: the orchestrator's base pre-fetch + * with every coordinator's contribution shard folded on top, deterministically. + * Never throws on a missing base or a corrupt shard — an absent/partial result + * just means those asks fall back to a live gather, which is the whole + * degradation contract. */ +export function readEvidenceFile(filePath) { + const doc = readBaseFile(filePath); + for (const shard of readContribs(filePath)) { + for (const [repo, entry] of Object.entries(shard.github ?? {})) foldGithub(doc.github, repo, entry); + for (const [wl, entry] of Object.entries(shard.logs ?? {})) foldLogs(doc.logs, wl, entry); + if (shard.generatedAtMs > (doc.generatedAtMs ?? 0)) doc.generatedAtMs = shard.generatedAtMs; + } + return doc; +} + +// Owner-only (0700 dir / 0600 file): this sits in a world-readable OS temp dir +// and carries private-repo PR detail and app-log digests. +export function writeEvidenceFile(filePath, doc) { + const dir = dirname(filePath); + if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true, mode: 0o700 }); + const existed = existsSync(filePath); + writeFileSync(filePath, JSON.stringify(doc, null, 2), { encoding: "utf8", mode: 0o600 }); + // `mode` is only honoured when the file is CREATED. A file left over from a + // run that predates this hardening would otherwise keep its old 0644 + // forever, so tighten it explicitly on overwrite too. + if (existed) chmodSync(filePath, 0o600); +} + +function loadOrInit(filePath, nowMs) { + if (!existsSync(filePath)) return emptyEvidenceFile("unknown-build", nowMs); + return readBaseFile(filePath); +} + +/** Idempotent: creates the file with the given `buildId` if it doesn't exist + * yet, otherwise leaves an existing file untouched (never clobbers prior + * writes on a resume). Call this FIRST, before any `set*` call, so `buildId` + * is recorded correctly — the `set*` functions below fall back to + * `"unknown-build"` only as a safety net if called without this. */ +export function initEvidenceFile(filePath, buildId, nowMs) { + if (existsSync(filePath)) return readBaseFile(filePath); + const doc = emptyEvidenceFile(buildId, nowMs); + writeEvidenceFile(filePath, doc); + return doc; +} + +/** Records the diff/PR-window baseline once, at the start of the Step 4 pass. + * `baseline` is `resolveBaseline(...)`'s return value from `evidence-cache.mjs` + * (`{ref, isFallback}`); `suspectWindow` is whatever shape the active connector + * skill uses to describe the window (e.g. `{reposRequested, startedAt}`). */ +export function setBaseline(filePath, baseline, suspectWindow, nowMs) { + const doc = loadOrInit(filePath, nowMs); + doc.baseline = baseline; + doc.suspectWindow = suspectWindow; + doc.generatedAtMs = nowMs; + writeEvidenceFile(filePath, doc); + return doc; +} + +/** Read-modify-write merge into `doc.github[repo]`. `entry` shape: + * `{ deployState: {block, gap}, prsInWindow: [{pr, files, block, verdict}], + * gap }` — `gap` (top-level, on the repo entry) is what `recomputeCoverage` + * checks; a repo present with a non-null `gap` is NOT counted as covered. + * Only ever touches this one repo's key — every other repo/workload already + * in the file is untouched. */ +export function setGithubEvidence(filePath, repo, entry, nowMs) { + const doc = loadOrInit(filePath, nowMs); + doc.github[repo] = entry; + doc.generatedAtMs = nowMs; + writeEvidenceFile(filePath, doc); + return doc; +} + +/** Read-modify-write merge into `doc.logs[workload]`. `entry` shape: + * `{ clusterIds, kubectlSweep: {block, gap}, victorialogs: {block, gap}, gap }`. + * Same no-clobber guarantee as `setGithubEvidence`, keyed by workload instead + * of repo. */ +export function setLogsEvidence(filePath, workload, entry, nowMs) { + const doc = loadOrInit(filePath, nowMs); + doc.logs[workload] = entry; + doc.generatedAtMs = nowMs; + writeEvidenceFile(filePath, doc); + return doc; +} + +// ---- coordinator write-back: own-shard only, never the base file ---------- +// +// `writerId` must be unique per concurrent writer — the dispatched +// coordinator's `testRunId` is the natural choice (one coordinator per test). +// Because a writer only ever opens its OWN shard, two coordinators writing at +// the same instant touch different files and neither can lose the other's +// update. Reads fold every shard back together (`readEvidenceFile`). + +function loadOwnShard(basePath, writerId, nowMs) { + const p = contribPathFor(basePath, writerId); + if (!existsSync(p)) { + return { path: p, doc: { writerId: String(writerId), generatedAtMs: nowMs, github: {}, logs: {} } }; + } + try { + return { path: p, doc: JSON.parse(readFileSync(p, "utf8")) }; + } catch { + return { path: p, doc: { writerId: String(writerId), generatedAtMs: nowMs, github: {}, logs: {} } }; + } +} + +function writeShard(path, doc) { + const dir = dirname(path); + if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true, mode: 0o700 }); + writeFileSync(path, JSON.stringify(doc, null, 2), { encoding: "utf8", mode: 0o600 }); +} + +/** Contribute what THIS coordinator gathered live for a repo — a deeper + * `deployState` (e.g. the full diff, not just a summary), and/or PRs to fold + * into `prsInWindow` (deduped by `pr`). `patch = { deployState?, prsInWindow?, + * gap? }`; omit a field to leave it untouched. Writes only this writer's + * shard, so it can never clobber another coordinator's contribution or the + * orchestrator's base pre-fetch. */ +export function contributeGithubEvidence(basePath, writerId, repo, patch, nowMs) { + const { path, doc } = loadOwnShard(basePath, writerId, nowMs); + const entry = doc.github[repo] ?? { deployState: null, prsInWindow: [], gap: null }; + if (patch.deployState !== undefined) entry.deployState = patch.deployState; + if (Array.isArray(patch.prsInWindow)) { + const byPr = new Map((entry.prsInWindow ?? []).map((p) => [String(p.pr), p])); + for (const pr of patch.prsInWindow) byPr.set(String(pr.pr), pr); + entry.prsInWindow = [...byPr.values()]; + // Contributing a list — even an empty one — means you actually ran the + // search, so record that. Pass `prsSearched: false` explicitly to opt out. + entry.prsSearched = patch.prsSearched !== false; + } + if (patch.prsSearched !== undefined) entry.prsSearched = patch.prsSearched; + if (patch.gap !== undefined) entry.gap = patch.gap; + doc.github[repo] = entry; + doc.generatedAtMs = nowMs; + writeShard(path, doc); + return entry; +} + +/** Contribute what THIS coordinator gathered live for a workload's app-logs. + * Same single-writer-shard discipline as `contributeGithubEvidence`; + * `clusterIds` is unioned rather than replaced. */ +export function contributeLogsEvidence(basePath, writerId, workload, patch, nowMs) { + const { path, doc } = loadOwnShard(basePath, writerId, nowMs); + const entry = doc.logs[workload] ?? { clusterIds: [], kubectlSweep: null, victorialogs: null, gap: null }; + if (patch.kubectlSweep !== undefined) entry.kubectlSweep = patch.kubectlSweep; + if (patch.victorialogs !== undefined) entry.victorialogs = patch.victorialogs; + if (Array.isArray(patch.clusterIds)) { + entry.clusterIds = [...new Set([...(entry.clusterIds ?? []), ...patch.clusterIds])]; + } + if (patch.gap !== undefined) entry.gap = patch.gap; + doc.logs[workload] = entry; + doc.generatedAtMs = nowMs; + writeShard(path, doc); + return entry; +} + +// A requested item is "covered" only if it is present AND its own `gap` field +// is falsy. Presence with a `gap` is a recorded, deliberate miss — not +// coverage — so a coordinator (or this function) never mistakes "we looked +// and couldn't get it" for "we have it." +// +// For a github entry there is a further trap, hit for real in testing: an +// empty `prsInWindow` is byte-identical whether the PR search RAN and found +// nothing, or was never populated at all. A coordinator trusting the former +// reads "no PRs in window" and concludes "no culprit PR" — confidently wrong. +// (Observed: a file asserting 0 PRs for a repo that actually had 21, because +// the loader's search silently returned empty.) So an empty `prsInWindow` +// only counts as coverage when `prsSearched === true` explicitly records that +// the search was really performed. +// Kept deliberately at the REPO level: an entry with deploy state but no PR +// search is still real coverage of that repo. PR-list trustworthiness is a +// narrower question, reported separately as `reposWithUntrustedPrList` so it +// is visible without distorting covered/gapped. +function isCovered(doc, section, key) { + const entry = doc[section]?.[key]; + return Boolean(entry) && !entry.gap; +} + +/** True when this repo entry can be trusted to answer "which PRs were in the + * window" — i.e. it either lists PRs, or explicitly records that the search + * ran and legitimately found none. Coordinators should call this before + * concluding "no culprit PR" from the pre-fetch. */ +export function hasTrustworthyPrList(doc, repo) { + const entry = doc?.github?.[repo]; + if (!entry || entry.gap) return false; + return (entry.prsInWindow ?? []).length > 0 || entry.prsSearched === true; +} + +/** Derives `doc.coverage` from exactly which requested repos/workloads have a + * gap-free entry, and persists it. `requested = {repos:[...], workloads:[...]}` + * — normally the Gate Part A scope-probe-validated repo list and the union of + * workloads every cluster's representative implicates (see SKILL.md Step 4). */ +export function recomputeCoverage(filePath, requested, nowMs) { + // Coverage is judged against the FOLDED view — a gap the orchestrator + // recorded but a coordinator later filled is genuinely covered now — while + // the result is persisted to base, which the orchestrator solely owns. + const folded = readEvidenceFile(filePath); + const doc = loadOrInit(filePath, nowMs); + const repos = requested?.repos ?? []; + const workloads = requested?.workloads ?? []; + doc.coverage = { + reposCovered: repos.filter((r) => isCovered(folded, "github", r)), + reposGapped: repos.filter((r) => !isCovered(folded, "github", r)), + workloadsCovered: workloads.filter((w) => isCovered(folded, "logs", w)), + workloadsGapped: workloads.filter((w) => !isCovered(folded, "logs", w)), + // Repos whose PR list must NOT be read as "no PRs in window": the list is + // empty and nothing recorded that a search actually ran. Surfacing this + // separately keeps a coordinator from concluding "no culprit PR" off an + // array that was simply never filled in. + reposWithUntrustedPrList: repos.filter( + (r) => isCovered(folded, "github", r) && !hasTrustworthyPrList(folded, r), + ), + }; + doc.generatedAtMs = nowMs; + writeEvidenceFile(filePath, doc); + return doc.coverage; +} diff --git a/lib/glimpse.mjs b/lib/glimpse.mjs new file mode 100644 index 0000000..38d7fb0 --- /dev/null +++ b/lib/glimpse.mjs @@ -0,0 +1,43 @@ +// Terse end-of-run summary — the ONLY in-client output of a /rca-build run. +// Deliberately a COMPLETION NOTICE, not a report: status counts + the UI link, +// nothing else. Root causes, culprit PRs, per-test analysis, cluster breakdowns +// live ONLY on the Test Observability dashboard (triggerRcaReport → viewReport). +// Do not reintroduce per-test cause/PR lines here — that is the whole point of +// the trim. Degrade, don't crash: missing fields are treated as absent. + +import { readRows } from "./csv-state.mjs"; + +// Map raw CSV rca_done states → the three buckets a human cares about. +function bucket(state) { + const s = (state || "").toLowerCase(); + if (s === "resolved") return "resolved"; + if (s === "pending" || s === "pending-resume") return "pending"; + return "failed"; +} + +// Render the completion summary. Returns a plain-text block: +// RCA analysis complete — build <id> +// <N> tests · <R> resolved · <P> pending · <F> failed +// (the caller appends the "Full report on the Test Observability UI: <link>" +// line from triggerRcaReport's viewReport — see SKILL.md Step 6). +export function renderGlimpse(rows, { buildId } = {}) { + const head = `RCA analysis complete${buildId ? ` — build ${buildId}` : ""}`; + if (!rows || rows.length === 0) { + return `${head}\nNo failed tests analyzed.\n`; + } + const counts = rows.reduce( + (acc, r) => { + acc[bucket(r.rca_done)] += 1; + return acc; + }, + { resolved: 0, pending: 0, failed: 0 }, + ); + const parts = [`${rows.length} test(s)`, `${counts.resolved} resolved`]; + if (counts.pending) parts.push(`${counts.pending} pending`); + if (counts.failed) parts.push(`${counts.failed} failed`); + return `${head}\n${parts.join(" · ")}\n`; +} + +export function renderGlimpseFromCsv(csvPath, opts = {}) { + return renderGlimpse(readRows(csvPath), opts); +} diff --git a/lib/loop.mjs b/lib/loop.mjs new file mode 100644 index 0000000..d9535fa --- /dev/null +++ b/lib/loop.mjs @@ -0,0 +1,262 @@ +// Executable mirror of the ai-tfa-coordinator loop (agents/ai-tfa-coordinator.md). +// It drives the collaborative loop against an injected `submit` (real = the +// tfaRcaTurn MCP tool; tests = a recorded-turn replayer), so the loop mechanics — +// status branching, ask routing, gap degradation, turn-cap, one-thread, +// soft-PENDING — are tested rather than assumed. +// +// Double duty: this is ALSO the **sequential thin-client harness** — the third +// caller of the same contract, for MCP clients without workflows/subagents. +// Pure + dependency-light (imports only the routing registry). +// +// The loop is fully autonomous: an evidence gap ALWAYS degrades to an +// `unavailable` block back to TFA. There is no user-prompt path — the /rca-build +// gate closed before this loop ever runs. +// +// tfaRcaTurn returns TRIMMED terminal shapes: +// RESOLVED → { status, confidence, threadId, +// glimpse: { root_cause (≤220 chars), failure_type, related_prs }, +// viewRca } +// PENDING → { status, turnId, threadId } (soft-pending, resumable) +// NEEDS_INFO → { status, questions/asks/suggestions } (verbatim — the loop needs them) +// +// Soft-PENDING is NOT an agent verdict. It is the tfaRcaTurn util abandoning its +// own in-call poll at POLL_MAX_WAIT_MS (90s) while the agent keeps working +// server-side — observed routinely, e.g. a first turn that finalized NEEDS_INFO +// at 104s. So a PENDING is DRAINED here: read that same turnId via +// `readTurn` (the getTfaTurnResult MCP tool) until it lands a real agent status, +// and only then route asks and submit the next message. Re-submitting on a +// PENDING instead would stack a second turn on one still in flight. + +import { routeAsks } from "./routing.mjs"; + +// Drain budget for one soft-PENDING. Bounded so a wedged turn can never hang the +// batch — on exhaustion the loop still ends `PENDING` (resumable via the CSV's +// `pending-resume` row), which is the old behaviour as a floor, not a default. +const DEFAULT_DRAIN = { maxWaitMs: 600_000, intervalMs: 5_000, maxReads: 40, maxErrorReads: 3 }; + +const defaultSleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// A real agent verdict, i.e. anything the drain is allowed to stop on. +const isAgentStatus = (s) => + s === "RESOLVED" || s === "NEEDS_INFO" || s === "BLOCKED"; + +/** + * Distinguish "TFA is still thinking" from "this read HARD-FAILED". + * + * These deserve opposite responses and the original drain conflated them: + * a `PENDING` should be waited out on the full budget, but a server-side + * `TFA agent run failed` will keep failing, and patiently re-reading it burns + * the entire 40-read / 10-minute budget to learn nothing. Measured on one real + * build: drain reads + their sleeps were 23% of ALL coordinator tool calls, + * and the four agents that wedged this way were the four slowest in the batch. + * + * Only explicit failure signals count — an unrecognised-but-parseable turn is + * treated as "still working", so a new upstream status can never be + * misclassified as an error and cut the drain short. + */ +function isErrorRead(turn, threw) { + if (threw) return true; + if (turn == null) return true; + if (typeof turn === "string") return /\b(fail(ed|ure)?|error)\b/i.test(turn); + if (turn.error) return true; + if (typeof turn.status === "string" && /^(ERROR|FAILED)$/i.test(turn.status)) return true; + if (typeof turn.message === "string" && /\b(fail(ed|ure)?|error)\b/i.test(turn.message)) return true; + return false; +} + +function unavailableBlock(gap) { + const what = gap?.ask?.what ?? ""; + return [ + `ASK: ${what}`, + `TYPE: ${gap.evidenceType}`, + `FOUND: no`, + `SUMMARY: unavailable — no ${gap.capability} connector for this client.`, + ].join("\n"); +} + +// drainSoftPending reads one in-flight turn to a real agent status. +// +// Reads do NOT consume the turn cap — a drain is the SAME turn being read again, +// not a new turn. `readTurn` is read-only and side-effect free, so the only +// budget that applies is wall clock / read count. +// +// Returns { turn, reads, reason }: `turn` is the landed agent turn, or null if +// the drain gave up — `reason` is `landed` | `tfa-error` | `budget-spent` | +// `not-drainable`, which the caller surfaces in the RCA_OUTPUT note so a human +// can tell "TFA was slow" apart from "TFA broke". +async function drainSoftPending({ testRunId, pending, readTurn, sleep, drain }) { + const { maxWaitMs, intervalMs, maxReads, maxErrorReads } = { ...DEFAULT_DRAIN, ...(drain ?? {}) }; + const turnId = pending.turnId; + let reads = 0; + + // No turnId → nothing addressable to read; no readTurn → client lacks the + // getTfaTurnResult tool. Either way fall back to reporting it resumable. + if (!turnId || typeof readTurn !== "function") return { turn: null, reads, reason: "not-drainable" }; + + const started = Date.now(); + let consecutiveErrors = 0; + while (reads < maxReads && Date.now() - started < maxWaitMs) { + await sleep(intervalMs); + reads++; + let turn; + let threw = false; + try { + turn = await readTurn({ testRunId, turnId }); + } catch { + threw = true; + } + + if (isErrorRead(turn, threw)) { + // Hard failure. Allow a couple of retries for a genuine blip, then stop: + // a wedged turn will not un-wedge by being asked the same question 37 + // more times, and the row stays resumable either way. + if (++consecutiveErrors >= maxErrorReads) { + return { turn: null, reads, reason: "tfa-error" }; + } + continue; + } + + consecutiveErrors = 0; // a good read clears the streak + if (isAgentStatus(turn?.status)) return { turn, reads, reason: "landed" }; + // still PENDING → the agent is working; read again. + } + return { turn: null, reads, reason: "budget-spent" }; +} + +// runRcaLoop drives one test to a terminal RCA_OUTPUT object. +// +// submit({ testRunId, message, threadId, turnId }) → Promise<turn> (tfaRcaTurn shape) +// readTurn({ testRunId, turnId }) → Promise<turn> (getTfaTurnResult shape) +// gather(routedGatherEntry) → Promise<string> (one digest block) +export async function runRcaLoop({ + testRunId, + firstMessage = "", + submit, + readTurn, + config = {}, + manifest = {}, + gather = async () => "", + turnCap = config?.turnCap ?? 6, + drain = config?.softPendingDrain, + sleep = defaultSleep, +}) { + if (testRunId == null || Number.isNaN(Number(testRunId))) { + return { + testRunId: String(testRunId), + status: "failed", + root_cause: "no testRunId provided", + turns_used: 0, + asks_fulfilled: [], + asks_skipped: [], + asks_unavailable: [], + }; + } + + let threadId; + let turnId; + let turns = 0; + let message = firstMessage; + const fulfilled = new Set(); + const skipped = new Set(); + const unavailable = new Set(); + + const out = (status, turn, note) => { + const glimpse = turn?.glimpse ?? {}; + return { + testRunId: String(testRunId), + status, + confidence: turn?.confidence ?? "unknown", + root_cause: status === "RESOLVED" ? (glimpse.root_cause ?? "") : (note ?? ""), + failure_type: glimpse.failure_type ?? "", + related_prs: glimpse.related_prs ?? [], + view_rca: turn?.viewRca ?? "", + threadId: threadId ?? null, + turnId: turnId ?? null, + turns_used: turns, + asks_fulfilled: [...fulfilled], + asks_skipped: [...skipped], + asks_unavailable: [...unavailable], + }; + }; + + while (true) { + turns++; + let turn = await submit({ testRunId, message, threadId, turnId }); + threadId = turn.threadId ?? threadId; + + // Soft-PENDING → the in-call poll capped out, not a verdict. Read the SAME + // turnId to a real status BEFORE routing asks or submitting anything else. + if (turn.status === "PENDING") { + turnId = turn.turnId ?? turnId; + const drained = await drainSoftPending({ + testRunId, + pending: turn, + readTurn, + sleep, + drain, + }); + if (!drained.turn) { + const note = + drained.reason === "tfa-error" + ? `tfa-error: read failed ${drained.reads} time(s) — stopped early, row stays resumable` + : drained.reason === "not-drainable" + ? `soft-pending: no turnId or no getTfaTurnResult tool` + : `soft-pending: still working after ${drained.reads} read(s)`; + return out("PENDING", turn, note); + } + turn = drained.turn; + threadId = turn.threadId ?? threadId; + // Landed. This was never a new turn, so `turns` is unchanged and the + // resume handle is spent — later submits go by threadId alone. + turnId = undefined; + } + + if (turn.status === "RESOLVED") return out("RESOLVED", turn); + // BLOCKED is a terminal agent verdict: TFA cannot proceed. It carries no + // asks, so treating it as NEEDS_INFO would resubmit empty messages to the + // cap. Reported as PENDING (the output contract's non-resolved value) with + // the reason in the note. + if (turn.status === "BLOCKED") return out("PENDING", turn, "blocked"); + + // NEEDS_INFO. Check the turn-cap BEFORE gathering — evidence assembled on a + // turn we will never submit is wasted work (and a side-effecting gather() + // would run for nothing). + if (turns >= turnCap) return out("PENDING", turn, "turn-cap"); + + // Route + fulfill. Gaps degrade to `unavailable` — never a user prompt. + const buckets = routeAsks(turn.asks ?? [], config, manifest); + const blocks = []; + for (const s of buckets.skip) skipped.add(s.evidenceType); + for (const g of buckets.gather) { + blocks.push(await gather(g)); + fulfilled.add(g.evidenceType); + } + for (const gap of buckets.gap) { + unavailable.add(gap.evidenceType); + blocks.push(unavailableBlock(gap)); + } + + message = blocks.join("\n\n"); + } +} + +// Replay helper for tests: returns a submit() that yields recorded turns in order. +export function replaySubmit(turns) { + let i = 0; + return async () => { + const turn = turns[Math.min(i, turns.length - 1)]; + i++; + return turn; + }; +} + +// Replay helper for tests: a readTurn() that yields recorded getTfaTurnResult +// reads in order (typically N × PENDING then the landed agent turn). +export function replayRead(reads) { + let i = 0; + return async () => { + const read = reads[Math.min(i, reads.length - 1)]; + i++; + return read; + }; +} diff --git a/lib/routing.mjs b/lib/routing.mjs new file mode 100644 index 0000000..ba3d699 --- /dev/null +++ b/lib/routing.mjs @@ -0,0 +1,107 @@ +// Evidence-routing registry (D3). Maps a TFA `ask.evidenceType` onto an +// action, given the run's validated capability manifest. Pure + dependency-free +// so it is testable and reusable by the batch workflow, subagents, and the +// sequential harness alike. +// +// `test_logs` is the TFA agent's own evidence and is always skipped. Every +// other type routes to a capability; whether that capability is *available* is +// decided by the manifest (built once per run — see U6 / buildManifest). + +import { readFileSync } from "node:fs"; + +export const TEST_LOGS = "test_logs"; + +const PRIORITY_RANK = { high: 0, medium: 1, low: 2 }; + +// Load and parse config/rca.config.json from an absolute or cwd-relative path. +export function loadConfig(configPath) { + return JSON.parse(readFileSync(configPath, "utf8")); +} + +// Order a turn's asks high → medium → low (unknown priority sorts last). +export function orderAsks(asks = []) { + return [...asks].sort( + (a, b) => + (PRIORITY_RANK[a?.priority] ?? 99) - (PRIORITY_RANK[b?.priority] ?? 99), + ); +} + +// Classify one ask. Returns one of: +// { action: "skip", ... } — test_logs / TFA-owned; the coordinator emits nothing +// { action: "gather", ... } — a capability is available; gather + digest +// { action: "gap", ... } — no valid connector; the caller emits an +// "unavailable" block back to TFA (never a prompt) +// +// `manifest` shape: { [capability]: { available: boolean, via?: string } }. +export function routeAsk(ask, config, manifest = {}) { + const evidenceType = ask?.evidenceType ?? "other"; + const routing = config?.evidenceRouting ?? {}; + const entry = routing[evidenceType] ?? routing.other ?? { capability: "other" }; + + if (entry.skip || entry.owner === "tfa") { + return { evidenceType, action: "skip", reason: "tfa-owned" }; + } + + const capability = entry.capability ?? "other"; + const cap = manifest[capability]; + if (cap && cap.available) { + return { + evidenceType, + action: "gather", + capability, + via: cap.via ?? null, + }; + } + + return { + evidenceType, + action: "gap", + capability, + discoveryHints: entry.discoveryHints ?? [], + reason: "no-capability", + }; +} + +// Split a turn's asks into the three buckets, in priority order. The +// coordinator gathers `gather`, emits an "unavailable" block for each `gap`, +// and records `skip` (test_logs) without emitting anything. +export function routeAsks(asks, config, manifest = {}) { + const ordered = orderAsks(asks); + const buckets = { skip: [], gather: [], gap: [] }; + for (const ask of ordered) { + const routed = routeAsk(ask, config, manifest); + buckets[routed.action].push({ ask, ...routed }); + } + return buckets; +} + +// ---- capability manifest (ideation #3) ------------------------------------- + +// Build the capability manifest ONCE per run from the capabilities the client +// agent actually discovered. `discovered` is a list of +// { capability, via } the orchestrator collected by asking "what skills/tools +// are available?". Every capability the routing registry references (except the +// TFA-owned test_logs) appears in the manifest, marked available iff discovered. +// Declaring this to TFA lets it avoid asking for evidence the client can't get. +export function buildManifest(config, discovered = []) { + const byCap = new Map(discovered.map((d) => [d.capability, d])); + const manifest = {}; + for (const entry of Object.values(config?.evidenceRouting ?? {})) { + if (entry.skip || entry.owner === "tfa") continue; + const cap = entry.capability; + if (!cap || cap in manifest) continue; + const found = byCap.get(cap); + manifest[cap] = found + ? { available: true, via: found.via ?? null } + : { available: false, via: null }; + } + return manifest; +} + +// Capabilities that will be unavailable this run — declared to the user up front +// ("infra + metrics not available") and to TFA so it plans asks around them. +export function unavailableCapabilities(manifest) { + return Object.entries(manifest) + .filter(([, v]) => !v.available) + .map(([cap]) => cap); +} diff --git a/lib/signature.mjs b/lib/signature.mjs new file mode 100644 index 0000000..42dc0ae --- /dev/null +++ b/lib/signature.mjs @@ -0,0 +1,78 @@ +// Failure-signature clustering (ideation #1). A red build's N failures usually +// trace to a handful of causes; clustering collapses the expensive evidence hunt +// to O(distinct causes). The signature is computed from the trimmed failure +// detail U1 surfaces on each listTestIds row (category + first error line + file +// path) — no extra probe turns. +// +// Dependency-free + deterministic (no crypto, no clock, no random) so it is +// usable from the auto-mode workflow sandbox and trivially testable. + +// Normalize a string for signature comparison: lowercase and fold the volatile +// tokens that make two instances of the SAME failure look different (ids, +// timestamps, hex/uuids, line:col, bare numbers). +export function normalize(value) { + return String(value ?? "") + .toLowerCase() + .replace(/\b\d{4}-\d{2}-\d{2}[t ]\d{2}:\d{2}:\d{2}\S*/g, "<ts>") // ISO timestamps + .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g, "<uuid>") + .replace(/0x[0-9a-f]+/g, "<hex>") // memory addresses + .replace(/:\d+(:\d+)?\b/g, ":<line>") // file:line(:col) + .replace(/\d+/g, "<n>") // remaining numbers (incl. unit-suffixed, e.g. 3000ms) + .replace(/\s+/g, " ") + .trim(); +} + +// The signature triple: normalized category | error summary | file path. +export function computeSignature(row) { + const category = normalize(row.failure_category); + const error = normalize(row.error_summary); + const file = normalize(row.file_path); + const sig = `${category}|${error}|${file}`; + return sig.replace(/\|/g, "").trim().length === 0 ? "" : sig; +} + +// Deterministic short id for a signature string (FNV-1a → base36). +function hashId(s) { + let h = 0x811c9dc5; + for (let i = 0; i < s.length; i++) { + h ^= s.charCodeAt(i); + h = Math.imul(h, 0x01000193); + } + return (h >>> 0).toString(36); +} + +// A stable representative for a cluster: prefer a non-flaky member (a flaky test +// is a poor exemplar), then the smallest testRunId. Deterministic. +export function selectRepresentative(members) { + return [...members].sort((a, b) => { + const aFlaky = a.is_flaky === "true" || a.is_flaky === true ? 1 : 0; + const bFlaky = b.is_flaky === "true" || b.is_flaky === true ? 1 : 0; + if (aFlaky !== bFlaky) return aFlaky - bFlaky; + return Number(a.testRunId) - Number(b.testRunId); + })[0]; +} + +// Cluster rows by signature. Mutates each row's `cluster_id`. Rows with no +// signal (empty signature) become their own singleton (never merged into a +// catch-all). Returns { rows, clusters } where each cluster carries its +// representative + siblings. +export function clusterRows(rows) { + const groups = new Map(); + + for (const row of rows) { + const sig = computeSignature(row); + const id = sig === "" ? `solo-${row.testRunId}` : `c-${hashId(sig)}`; + row.cluster_id = id; + if (!groups.has(id)) groups.set(id, { cluster_id: id, signature: sig, members: [] }); + groups.get(id).members.push(row); + } + + const clusters = []; + for (const group of groups.values()) { + const representative = selectRepresentative(group.members); + const siblings = group.members.filter((m) => m !== representative); + clusters.push({ ...group, representative, siblings }); + } + + return { rows, clusters }; +} diff --git a/lib/tool-cache.mjs b/lib/tool-cache.mjs new file mode 100644 index 0000000..e77f71f --- /dev/null +++ b/lib/tool-cache.mjs @@ -0,0 +1,391 @@ +// Build-scoped memo cache for READ-ONLY tool calls (`gh`, `kubectl`, curl, …). +// +// The evidence file (`evidence-file.mjs`) shares *digested findings* whose +// shape the schema knows about — deploy state, PRs, log sweeps. But a +// coordinator's real tool traffic is mostly raw lookups that fit no schema +// slot: fetching a spec file's contents at a ref, listing a git tree, running +// a code search. Measured on one real 10-test build: `gh` was 37% of all +// coordinator tool calls, and 46 of them were byte-identical commands re-run +// by different coordinators — the same BStackAutomation spec fetched 12 +// times, a frontend component 5 times. +// +// This module memoizes at the CALL level instead, so duplication is caught +// regardless of what the call was for. Any coordinator about to run a +// read-only command checks here first; on a miss it runs the command and +// stores the result for everyone else. +// +// CONCURRENCY: one file per cache KEY (`<sha>.json`), not one per writer. +// Distinct calls write distinct files; two agents racing on the *same* call +// write byte-identical content, so the race is benign. Writes go through a +// temp file + `rename`, which is atomic on POSIX, so a reader never observes +// a half-written entry. No locking, no lost updates, no torn reads. +// +// WHAT IS NOT CACHED (deliberate): +// - Any command that fails (non-zero exit). A transient `gh` rate-limit or +// an expired kube token must never be memoized into a persistent "answer" +// that poisons every later reader. +// - Anything matching the mutation denylist below. The plugin is read-only +// by contract, but caching is a correctness-sensitive place to trust that +// contract blindly, so mutations are refused defensively. +// - Secrets: values that look like tokens/passwords are redacted from the +// stored payload before it ever touches disk. + +import { + readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, renameSync, chmodSync, +} from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { createHash } from "node:crypto"; + +/** Per-build cache directory, sitting alongside the state CSV and evidence + * file under the same OS-temp convention. */ +export function toolCacheDirFor(buildId, stateDir = "") { + const safe = String(buildId ?? "").replace(/[^A-Za-z0-9._-]/g, "_") || "unknown-build"; + const dir = stateDir && String(stateDir).trim() !== "" ? String(stateDir) : join(tmpdir(), "bstack-rca"); + return join(dir, `rca-toolcache.${safe}`); +} + +/** Stable key for one call. Whitespace is normalized so trivially-different + * formatting of the same command still hits, but nothing else is rewritten — + * `| head -20` vs `| head -200` genuinely return different output and must + * stay distinct keys. */ +export function cacheKey(command) { + const norm = String(command ?? "").replace(/\s+/g, " ").trim(); + return createHash("sha256").update(norm).digest("hex").slice(0, 24); +} + +// Commands that must never be memoized, even if someone wires this into a +// non-read-only context by mistake. +// Note: output redirection is deliberately NOT in this list. A `>/dev/null` +// is a redirect, not a mutation, and lumping the two together produced the +// misleading refusal "command looks mutating" for ordinary read-only calls. +// Redirects are handled separately, with an accurate message. +const MUTATING = /\b(rm|mv|cp|dd|truncate|tee)\b|\bgit\s+(push|commit|merge|rebase|reset|checkout|clean)\b|\bgh\s+(pr\s+(create|merge|close|edit|comment|review)|issue\s+(create|close|edit|comment)|release\s+create|repo\s+(create|delete)|api\s+(-X\s*)?(POST|PUT|PATCH|DELETE))|\bkubectl\s+(apply|delete|edit|patch|scale|create|replace|annotate|label|cordon|drain|exec|cp|port-forward|rollout\s+(undo|restart|pause|resume))\b|\bcurl\b[^|]*\s-(X|-request)\s*(POST|PUT|PATCH|DELETE)/i; + +/** + * Strip stderr-plumbing that the wrapper already handles. + * + * `2>&1` and `2>/dev/null` appear on the majority of real recorded calls — + * agents add them reflexively because `gh` is chatty. They say nothing about + * WHAT to fetch, only where stderr should go, and the wrapper captures stderr + * separately regardless. Refusing them rejected 134 of 223 recorded calls and + * drove the effective hit rate to zero, so they are normalized away instead. + * + * Genuine FILE redirects (`> out.json`, `>> log`, `< in`) are left in place so + * the check below still refuses them: those change where data goes, which the + * wrapper cannot honour while also returning stdout to the caller. + */ +function stripStderrPlumbing(seg) { + return String(seg ?? "") + .replace(/\s*2>&1\s*/g, " ") + .replace(/\s*2>\s*\/dev\/null\s*/g, " ") + .trim(); +} + +/** True if the segment still contains a real redirect outside quotes, after + * stderr plumbing has been normalized away. */ +function hasUnquotedRedirect(seg) { + let quote = null; + for (const ch of String(seg ?? "")) { + if (quote) { if (ch === quote) quote = null; continue; } + if (ch === "'" || ch === '"') { quote = ch; continue; } + if (ch === ">" || ch === "<") return true; + } + return false; +} + +export function isCacheable(command) { + return !MUTATING.test(String(command ?? "")); +} + +// Read-only data-fetching binaries this wrapper will run. Anything else is +// refused outright. +const ALLOWED_LEADER = /^\s*(gh|kubectl|curl|git)\s/; + +// Pure text filters allowed AFTER the fetch in a pipeline. They transform the +// fetch's output and never reach the network, so they are deliberately outside +// the cache key: `gh api X | jq .a` and `gh api X | jq .b` share ONE cached +// fetch. Measured motivation — replaying real recorded traffic, 89% of calls +// embedded the fetch in a pipeline, so refusing pipelines outright meant the +// cache applied to almost nothing in practice. +const ALLOWED_FILTER = new Set([ + "jq", "grep", "egrep", "head", "tail", "sort", "uniq", "wc", + "cut", "tr", "sed", "awk", "base64", "python3", "rev", "column", +]); + +/** Split on top-level `|` only — a pipe inside quotes (a jq expression) stays + * part of its segment. Returns trimmed segment strings. */ +export function splitPipeline(command) { + const segs = []; + let cur = ""; + let quote = null; + const s = String(command ?? ""); + for (let i = 0; i < s.length; i++) { + const ch = s[i]; + // A backslash escapes the next character. Without this, `\"` inside a + // double-quoted jq filter reads as "close quote", the parser thinks it is + // back outside quotes, and a `|` in a regex alternation like + // `test("vite|env";"i")` gets split as a shell pipe. + if (ch === "\\" && i + 1 < s.length && quote !== "'") { + cur += ch + s[i + 1]; + i++; + continue; + } + if (quote) { + cur += ch; + if (ch === quote) quote = null; + continue; + } + if (ch === "'" || ch === '"') { quote = ch; cur += ch; continue; } + if (ch === "|") { segs.push(cur.trim()); cur = ""; continue; } + cur += ch; + } + segs.push(cur.trim()); + return segs.filter((s2) => s2.length > 0); +} + +// Shell operators, checked as whole ARGV TOKENS rather than by scanning the +// raw string. Scanning the string was too blunt and refused legitimate reads: +// `--jq 'test("rcaThree";"i")'` was rejected for the `;` inside a quoted jq +// expression, and `search/code?q=X&per_page=20` for the `&` inside a URL. +// +// Post-tokenization this distinction is exact: quoted metacharacters end up +// *inside* an argument (harmless — we execFile, so no shell ever interprets +// them), while a real operator survives as its own standalone token. +const OPERATOR_TOKENS = new Set([";", "|", "||", "&&", "&", ">", ">>", "<", "<<"]); + +/** + * Gate for `bin/cached-exec.mjs`. Returns `{ ok, reason }`. + * + * Scope note, stated plainly: this is defense-in-depth, NOT a security + * boundary. The only caller is a coordinator agent that already has direct + * shell access via its Bash tool, so the wrapper grants no capability the + * caller lacks and cannot meaningfully contain a caller that wants to misuse + * it. What it does buy: a fat-fingered or model-hallucinated command can't + * quietly run something mutating *through the cache path* and get memoized, + * and refusing `;`-chained loops nudges callers toward one-fetch-per-call, + * which caches far better anyway. + */ +/** + * Validate a command and return its execution plan. + * + * On success: `{ ok, fetch: string[], filters: string[][] }` — the fetch is + * executed (or served from cache) and its output is piped through the filters, + * each run via execFile with NO shell anywhere in the chain. + * + * Pipelines are accepted rather than refused because refusing them is what + * made the cache useless on real traffic. Only the FETCH is keyed, so several + * agents filtering one fetch differently all share a single cached result. + */ +export function isRunnable(command) { + const c = String(command ?? ""); + if (!isCacheable(c)) return { ok: false, reason: "command looks mutating" }; + + const segments = splitPipeline(c).map(stripStderrPlumbing).filter((s) => s.length > 0); + if (segments.length === 0) return { ok: false, reason: "empty command" }; + + if (!ALLOWED_LEADER.test(segments[0])) { + return { ok: false, reason: "command must start with gh, kubectl, curl, or git" }; + } + + const parsed = []; + for (const seg of segments) { + if (hasUnquotedRedirect(seg)) { + return { + ok: false, + reason: "file redirects (>, >>, <) are not supported — the wrapper returns stdout to you directly, so drop the redirect. (`2>&1` and `2>/dev/null` are fine; they're stripped automatically.)", + }; + } + let argv; + try { + argv = tokenize(seg); + } catch (err) { + return { ok: false, reason: err.message }; + } + if (argv.length === 0) return { ok: false, reason: "empty pipeline segment" }; + const op = argv.find((t) => OPERATOR_TOKENS.has(t)); + if (op) { + return { + ok: false, + reason: `'${op}' is a shell operator. Pipes are supported, but ';', '&&', redirects and substitution are not — issue one fetch per call.`, + }; + } + parsed.push(argv); + } + + for (const argv of parsed.slice(1)) { + if (!ALLOWED_FILTER.has(argv[0])) { + return { + ok: false, + reason: `'${argv[0]}' is not an allowed filter after the fetch (allowed: ${[...ALLOWED_FILTER].join(", ")})`, + }; + } + } + + return { ok: true, fetch: parsed[0], filters: parsed.slice(1), fetchText: segments[0] }; +} + +/** + * Split a command string into argv the way a shell would for the simple cases + * we allow — honouring single/double quotes — WITHOUT invoking a shell. The + * caller then runs `execFile(argv[0], argv.slice(1))`, so no shell ever + * interprets metacharacters and command injection has no surface. Throws on + * an unterminated quote rather than guessing. + */ +export function tokenize(command) { + const out = []; + let cur = ""; + let quote = null; + let started = false; + const s = String(command ?? ""); + for (let i = 0; i < s.length; i++) { + const ch = s[i]; + // Backslash escape, POSIX-style: literal everywhere except inside single + // quotes. Missing this mangled jq's most common idiom — `\"` was treated + // as a quote delimiter, so `select(.filename==\"x\")` reached the binary + // as `select(.filename==\x\)` with the quotes eaten. + if (ch === "\\" && i + 1 < s.length && quote !== "'") { + cur += s[i + 1]; + i++; + started = true; + continue; + } + if (quote) { + if (ch === quote) quote = null; + else cur += ch; + continue; + } + if (ch === "'" || ch === '"') { quote = ch; started = true; continue; } + if (/\s/.test(ch)) { + if (started) { out.push(cur); cur = ""; started = false; } + continue; + } + cur += ch; + started = true; + } + if (quote) throw new Error("unterminated quote in command"); + if (started) out.push(cur); + return out; +} + +// ---- MCP calls ------------------------------------------------------------ + +// Stateful MCP tools that must NEVER be memoized. `tfaRcaTurn` advances a +// conversation and `getTfaTurnResult` reads a turn whose status is *expected* +// to change between reads — serving either from cache would be actively +// wrong, not merely stale. +const MCP_NEVER = /tfaRcaTurn|getTfaTurnResult|triggerRcaReport/i; + +export function isCacheableMcp(toolName) { + return !MCP_NEVER.test(String(toolName ?? "")); +} + +/** Key an MCP call by tool name + canonicalized args (object keys sorted), so + * the same query written with its arguments in a different order still hits. */ +export function mcpCacheKey(toolName, args) { + const canon = (v) => { + if (Array.isArray(v)) return v.map(canon); + if (v && typeof v === "object") { + return Object.keys(v).sort().reduce((a, k) => { a[k] = canon(v[k]); return a; }, {}); + } + return v; + }; + const payload = JSON.stringify({ tool: String(toolName ?? ""), args: canon(args ?? {}) }); + return createHash("sha256").update(payload).digest("hex").slice(0, 24); +} + +// Redact the secret VALUE, bounded by the first structural delimiter — never +// "the rest of the line". +// +// The rest-of-line version silently destroyed data. GitHub's file API returns +// SINGLE-LINE JSON whose `download_url` always carries `?token=…`, so matching +// `token=` and consuming `[^\r\n]*` swallowed the entire remaining payload: +// a 214KB response cached as 816 bytes, content field gone, no warning. Every +// private-repo file fetch was affected. +// +// A value therefore stops at whitespace, quote, comma, semicolon, brace, +// bracket or `&` — enough to cover a real credential, never enough to eat the +// surrounding document. +const SECRET_KV = + /((?:token|authorization|api[_-]?key|secret|password|passwd|access[_-]?key)"?\s*[=:]\s*"?)((?:bearer|basic|token)\s+)?([^\s"'`,;}\]&\r\n]{4,})/gi; + +// A bare `Bearer <token>` / `Basic <token>` with no key= in front of it. +const SECRET_SCHEME = /\b(bearer|basic)\s+([A-Za-z0-9._~+/=-]{8,})/gi; + +/** Redact anything token-shaped before it is persisted. The cache lives in + * temp, but a cached `gh api` response or log line could still carry a + * credential, and "it's only temp" is not a reason to write one to disk. */ +export function redact(text) { + return String(text ?? "") + .replace(SECRET_KV, (_m, key) => `${key}<redacted>`) + .replace(SECRET_SCHEME, (_m, scheme) => `${scheme} <redacted>`); +} + +const MAX_BYTES = 256 * 1024; + +let tmpSeq = 0; + +export function cacheGet(cacheDir, key) { + const p = join(cacheDir, `${key}.json`); + if (!existsSync(p)) return null; + try { + return JSON.parse(readFileSync(p, "utf8")); + } catch { + return null; // half-written or corrupt -> treat as a miss, never throw + } +} + +/** Atomic write: temp file + rename, so concurrent readers only ever see a + * complete entry. `nowMs` is passed in (same clock discipline as the rest of + * lib/). Returns the stored entry. */ +export function cachePut(cacheDir, key, entry, nowMs) { + // Owner-only (0700 dir / 0600 files). The cache lives under a world-readable + // OS temp dir and holds raw `gh`/`kubectl` output — private repo source, + // internal hostnames, log bodies. `redact()` below is best-effort pattern + // matching and will not catch everything, so the filesystem permission is + // the actual control, not a backstop. + // + // The FILE mode is the load-bearing part: a pre-existing directory (e.g. a + // `stateDir` the user already created, or one left by an earlier run) keeps + // its own permissions, since silently chmod'ing a path we were handed would + // be presumptuous. Entries stay 0600 regardless, and a traversable directory + // only exposes opaque hash filenames, not their contents. + if (!existsSync(cacheDir)) mkdirSync(cacheDir, { recursive: true, mode: 0o700 }); + const raw = redact(entry.stdout ?? ""); + const truncated = raw.length > MAX_BYTES; + const rec = { + key, + command: entry.command, + writerId: entry.writerId ?? null, + capturedAtMs: nowMs, + exitCode: entry.exitCode ?? 0, + truncated, + bytes: raw.length, + stdout: truncated ? raw.slice(0, MAX_BYTES) + "\n… [truncated by tool-cache]" : raw, + }; + const finalPath = join(cacheDir, `${key}.json`); + // pid + counter keeps the temp name unique per writer, so `mode` genuinely + // applies (it is honoured on create, not on truncate of an existing file). + const tmpPath = join(cacheDir, `.${key}.${process.pid}.${tmpSeq++}.tmp`); + writeFileSync(tmpPath, JSON.stringify(rec, null, 2), { encoding: "utf8", mode: 0o600 }); + renameSync(tmpPath, finalPath); // atomic on POSIX; preserves the 0600 mode + return rec; +} + +/** Cache-wide counters for the run's summary — how much duplicate work this + * actually saved, rather than assuming it saved any. */ +export function cacheStats(cacheDir) { + if (!existsSync(cacheDir)) return { entries: 0, bytes: 0 }; + let entries = 0; + let bytes = 0; + for (const f of readdirSync(cacheDir)) { + if (!f.endsWith(".json") || f.startsWith(".")) continue; + entries++; + try { + bytes += JSON.parse(readFileSync(join(cacheDir, f), "utf8")).bytes ?? 0; + } catch { + /* skip */ + } + } + return { entries, bytes }; +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..27344a7 --- /dev/null +++ b/package.json @@ -0,0 +1,10 @@ +{ + "name": "tfa-rca-plugin", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Generic multi-client RCA agent plugin harness", + "scripts": { + "test": "node --test" + } +} diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md new file mode 100644 index 0000000..27b856d --- /dev/null +++ b/skills/rca-build/SKILL.md @@ -0,0 +1,470 @@ +--- +name: rca-build +description: Single-gate autonomous batch RCA over every failed test of a BrowserStack build via tfaRcaTurn. One gate (connector validation + assumed intake), then fully autonomous — clusters failures, routes evidence, triggers the dashboard report. Args: build id, optional PR URLs / repo hints. +--- + +# rca-build — single-gate autonomous RCA over a build + +Drives the `tfaRcaTurn` collaborative loop over **every failed test** of a build +and lands a per-test RCA in the TRA (Test Observability) dashboard. **TFA owns +logs; the client agent owns everything else** (product code, infra/runtime, logs, +metrics, deploy, ci) — routed by capability, generic over product and infra. + +This skill is the **build-level orchestrator** (`ai-tfa-orchestrator` role). It +never calls `tfaRcaTurn` itself — it dispatches the `ai-tfa-coordinator` +(test-level) per test/cluster member, which drives the loop and lets TFA author +the dashboard RCA. **The full RCA report lives on the Test Observability UI, not +in Claude** — this run's job is to feed it, then surface a terse glimpse and the +link. + +There is exactly **one mode**: autonomous. There is exactly **one gate** (Step +1) before execution. After the gate closes, **the run never asks the user +anything again.** + +Config (concurrency, turn-cap, paths, evidence registry) lives in +`config/rca.config.json`. State lives in the CSV/WAL spine (`lib/csv-state.mjs`). + +## Step 0 — input + +Parse the build id from the invocation args. Accepted forms: a bare build id, a +`build_id=<id>` token, or a build dashboard link (extract the id). Also accept +any **PR URLs** and **repo hints** (product/automation repo names or paths) the +user supplies — carry them into Gate Part B as pre-answered intake. + +- No build id present: + - interactive session → fold "which build?" into the gate's single + consolidated question (Step 1, Part B). It is the only genuinely + load-bearing field. + - **headless (`claude -p`) → end immediately (fail fast).** + +## Step 1 — THE GATE (one gate, two parts, closes once) + +Everything the run could possibly need from the user is settled here, in one +pass. The gate has two parts; both run before any RCA work starts. + +### Part A — connector discovery + validation + +**Step 0 — enumerate connector-shaped skills FIRST (before probing raw MCP tools).** +Run: + +```bash +ls .claude/skills/ ~/.claude/skills/ 2>/dev/null +``` + +For each `SKILL.md` found, open it and look for a **Capability declaration** +block (or a `capability: <name>` line in the frontmatter/body). Any skill that +declares `capability: github | infra | logs | metrics | other` **IS** the +connector for that capability and MUST be added to the manifest — it +**SUPERSEDES** the raw MCP tool for that capability because it carries +product-specific routing (repo map, cluster/namespace, branch conventions, +falsification protocol) the raw tool does not. Record the skill name in the +manifest entry (e.g. `github: valid, via: gh (skill=<product>-github)`). Skipping +this step is the failure mode where the orchestrator dispatches coordinators +that grep the wrong repos on the wrong branch. + +**Disambiguating by product (nudge / one-question rule).** Connector skills are +product-scoped — a workspace may hold none, one, or several product families +(e.g. `<product-a>-*`, `<product-b>-*`, whatever the user has). After the `ls`, +pick the *product family* whose connector skills apply to THIS build: + +- **Zero families found** → **nudge the user in the gate summary**: + "No connector-shaped skills found under `.claude/skills/` — proceeding with + raw MCP tools only; culprit-PR attribution will be best-effort against + workspace `git remote` guesses. Add a `<product>-github` / `<product>-infra` + skill for higher-fidelity routing." Then proceed with raw connectors. **Do + NOT block.** +- **Exactly one family** → use it. No question. +- **Multiple families** (e.g. `<product-a>-*` AND `<product-b>-*` …) → try to + disambiguate WITHOUT asking: + 1. Match the build's project / build name (from `getBuildId` metadata or + the invocation args) against each family's SKILL.md description / product + hints — if one family matches unambiguously, use it. + 2. Match the discovered failure signatures (from Step 2's `listTestIds` if it + has already run, else defer this to a re-visit after discovery) against + each family's declared file paths / error patterns — if one family owns + the failure surface, use it. + If both signals leave the choice ambiguous, this earns the **one + consolidated gate question** (Part B rules apply): fold it into the same + question as any other non-assumable field, e.g. *"Multiple product families + found (`<product-a>`, `<product-b>`); build/failure signatures don't + uniquely pick one — which family owns this build's failures?"* Headless: + pick the first alphabetically and record the ambiguity as a gap. + +Then enumerate every connector relevant to test RCA: + +- from `config/rca.config.json` → `evidenceRouting`: **github** + (product_code/deploy/ci), **infra** (whatever runtime the user has — k8s, + ECS, docker, Nomad, plain VMs, PM2, …), **logs** (kibana or any log store), + **metrics**, **other**; +- plus any connector-shaped skills / MCP servers present in the session + (a log-search MCP, a metrics MCP, an infra skill, …). + +**Validate** each with a cheap probe — discovery alone is not enough: + +| Connector | Probe | +|---|---| +| github | `gh auth status` (or a GitHub MCP tool listed) | +| infra | ANY runtime connector the user has — probe what exists, never assume one: `kubectl version --request-timeout=5s`, `docker ps`, `aws ecs list-clusters`, `nomad status`, `pm2 ls`, or an infra-shaped skill/MCP tool. Record the KIND in the manifest (`via: kubectl \| docker \| ecs \| …`) | +| logs | a log-search skill/MCP tool actually listed in the session | +| metrics | a metrics skill/MCP tool actually listed in the session | +| other | best-effort; default `absent` | + +**Scope validation — run the SCOPE PROBES declared by each connector skill.** +The base probe above (`gh auth status`, `kubectl version`, …) only confirms the +raw tool works. It does not confirm the *concrete targets* a coordinator will +touch — specific repos, branches, clusters, namespaces, indices — are actually +reachable. That is the connector SKILL's job: each connector skill MUST +declare, in its `Capability declaration` section, a `Scope probes:` list +naming what to check and how. This orchestrator's contract is generic: + +1. For every connector skill added to the manifest in Step 0, read its + `Scope probes:` list. +2. Run each probe verbatim. +3. Record every target's result in the manifest entry — passes go into a + resolved-scope field (e.g. `repos_validated: [...]`, `namespace: ok`), + failures go into a per-target gap (e.g. `<target>: 404 not_accessible`). +4. A per-target failure is a scoped gap, not a connector-wide failure — the + connector stays `valid` for the targets that did pass. + +Coordinators can then act freely inside the resolved scope and must fail +closed outside it. This closes the failure mode where a coordinator degrades +to `unavailable` because the orchestrator didn't confirm the specific target. + +Skills that don't declare `Scope probes:` degrade to a manifest-time warning +("scope probes missing — coordinator may over-degrade"). Do not invent +product-specific probes here. + +Output the **validated capability manifest**: `connector → valid | invalid | +absent` (`lib/routing.mjs` → `buildManifest`; `valid` maps to +`available: true`). An `invalid` or `absent` connector is a **recorded gap** — +declared to the user in the gate summary and to TFA on the first turn ("I don't +have logs/metrics access") — **never a blocker**. The run always proceeds. + +### Part B — requirements (assume first, ask once, at most) + +Intake fields: product repo, automation (test) repo, working branch, default +branch, the PRs in play, and the build id. **Resolve every field by ASSUMPTION +wherever possible** — this is an assumption-OK workflow; less user interaction +is the point: + +- invocation args (build id, PR URLs, repo hints from Step 0), +- `gh repo view` / git remotes for the repos, +- the current branch for the working branch, +- cheap inference (e.g. the automation repo is the cwd if it holds the tests). + +**Product-repo corroboration (do NOT skip).** The product repo must plausibly +be the *system under test for THIS build's failures* — not merely a repo name +found lying around. A repo mentioned only in workspace docs/READMEs is a **weak +hint, never an assumption**: cross-check it against the failure signatures +(discovery runs first if needed) — do the failing area, files, or error strings +relate to that repo's domain? If they don't (e.g. the failures are self-healing +`healedElement is null` cases but the only named repo is an observability API), +the doc-sourced repo is discarded — never carry it (or its PRs) into the +manifest as a settled product repo. + +When corroboration leaves **no** product repo, decide by whether a human can help: +- **PRs were supplied** → treat those as the suspect surface; product repo is + derived from them. No question needed. +- **No PRs, interactive session** → the product repo is now **non-assumable AND + load-bearing** (without it the mandatory culprit-PR hunt is dead), so it earns + the single consolidated gate question below — ask it; don't silently degrade. +- **No PRs, headless** → record the gap ("product repo: unknown") and proceed + RCA-only; every culprit-PR hunt reports "no culprit PR identified". + +Record each assumption in the gate summary (format: +`templates/gate-summary.md`; worked example: `examples/sample-run.md`) +("assumed product repo = +`org/obs-api` from git remote"). A field that cannot be assumed is recorded as +"none" and the run proceeds RCA-only for it — **unless** it is both genuinely +non-assumable AND load-bearing. In practice that set is: the build id; **the +product repo when it could not be corroborated and no PRs were supplied** (see +above — without it the culprit-PR hunt cannot run); and rarely an ambiguous repo +when PRs were supplied. Those, and only those, may be asked **ONCE, in a single +consolidated question at gate close** — e.g. *"Failures look like `<domain>`; +which repo owns that code? (reply 'none' → I'll RCA without culprit-PR +attribution)."* Never a second question. **Headless: skip asking entirely; +record the gaps.** + +### Gate close + +Print a one-screen summary: resolved intake (with assumptions marked) + the +validated capability manifest (with gaps named). Then the gate closes. + +**AFTER THE GATE CLOSES, THE RUN NEVER ASKS THE USER ANYTHING AGAIN.** RCA +execution is fully autonomous: every downstream evidence gap becomes an +`unavailable` block back to TFA (best-effort finalize), never a prompt. + +## Step 2 — discovery + +Call the bundled MCP tool: + +``` +listTestIds(buildId=<id>, status="failed", includeFailureDetail=true) +``` + +`includeFailureDetail=true` returns each row's trimmed failure signature +(`failure.{category, error_summary, file_path, …}`) — the seed for clustering, +so no per-test probe turns are needed. + +Resolve the state file with `lib/csv-state.mjs` → `csvPathFor(buildId, +config.paths.stateDir)` — the **build id is in the filename** and the default +directory is **OS temp** (`<tmpdir>/bstack-rca/rca-state.<buildId>.csv`), so +different builds can never collide and the invoking workspace stays clean. Pass +this exact path to the fan-out workflow as `csvPath`. + +Seed the CSV/WAL spine from the payload (`lib/csv-state.mjs` → `seed`): one row +per failed test, every row `rca_done=pending`, signature columns populated. +Re-running `seed` on an existing CSV is idempotent and preserves terminal rows +(resume-safe — same build id → same path). If `listTestIds` returns empty → +write an empty CSV, report "no failed tests", stop. + +## Step 3 — failure-signature clustering (see references/clustering.md) + +Compute a failure signature per row and assign `cluster_id` (`lib/signature.mjs`). +Each cluster gets one **representative** (full multi-turn loop) and `N−1` +**siblings** (pre-seeded one-turn confirm against their own logs). This collapses +the expensive evidence hunt to O(distinct causes) while every test still lands a +per-test RCA. Singleton clusters are just plain per-test loops. + +## Step 4 — build-evidence pre-fetch (see references/evidence-routing.md and lib/evidence-file.mjs) + +Once, after clustering (Step 3) and before fan-out — the capability manifest +already exists from Gate Part A, reuse it, do not re-discover. This step +replaces each coordinator's own turn-1 evidence sweep with ONE pre-fetch: +it does not remove the requirement that turn-1 evidence exists, only *who +gathers it*. + +1. Resolve the evidence-file path: `lib/evidence-file.mjs` → + `evidencePathFor(buildId, config.paths.stateDir)` — + `<tmpdir>/bstack-rca/rca-evidence.<buildId>.json`, alongside the state CSV. + `initEvidenceFile(path, buildId, nowMs)`. +2. **Scope the pre-fetch to the full union, never a single guess:** + - **Repos** — every repo in Gate Part A's scope-probe-validated + `repos_validated` list (e.g. a VRT-lane build validates `frontend` + + `railsApp`; an nl2steps build validates `misc-services` + `ai-sdk-node`). + - **Workloads** — the union of workloads every cluster's **representative** + implicates, via the active connector skill's failure-signature→workload + routing table (never one workload guessed from the first failing test). +3. For each repo: run the connector skill's PR-window-search + deploy-state + recipes **once**, using `lib/evidence-cache.mjs`'s `compute(repo, range, + evidenceType, fn)` to dedupe if two steps need the same `(repo, range)`. + Digest the result into the `evidence-block.md` shape, then persist via + `setGithubEvidence(path, repo, {deployState, prsInWindow, gap}, nowMs)`. + A repo the connector can't reach records `{gap: "<reason>"}` — never blocks + the rest of the pre-fetch. +4. For each workload: run the connector skill's compulsory kubectl + + VictoriaLogs sweep **once**, anchored to the build's own clock — never + "now". **PAD the window: `started_at − 2m` .. `finished_at + 10m`.** + `finished_at` is when the build was *marked* finished, which is not when + the failing behaviour stopped: on one real build an upstream outage began + at 06:12:00 and ran to 06:15:51, while `finished_at` was 06:12:21 — a + sweep scoped strictly to `started_at..finished_at` saw 21 seconds of a + 4-minute outage and would have missed the cause entirely. Label every + finding with whether it falls inside or outside the strict window so a + coordinator can weigh it; do NOT silently widen to an arbitrary window + (that is the separate, opposite failure of matching a coincidence from + unrelated traffic). Persist via `setLogsEvidence(path, workload, + {clusterIds, kubectlSweep, victorialogs, gap}, nowMs)`. + + Two query mechanics that cost real calls when missed: + - **`direction` defaults to newest-first**, so a limited query always + returns the END of the window. To find when something *started* — the + first request after a gap, the onset of an error burst — pass + `direction: "forward"`. A gap "confirmed" from a backward query is not + confirmed at all; it is just the tail of the range. + - **Absence needs a control.** A zero-result query is indistinguishable + from a wrong selector. Before reporting "no traffic", prove the logger + was alive in the same window with a query you expect to be non-empty + (e.g. readiness probes from a named pod). Only then is silence evidence. +5. `resolveBaseline(lastGreenRef, fallbackRef)` (from `lib/evidence-cache.mjs`) + → `setBaseline(path, baseline, suspectWindow, nowMs)`. No "last green" + baseline (never-green suite) → fall back to a configured baseline ref and + note the weaker grounding — this note travels into the file, not just a + spoken log line, so every coordinator sees it. +6. `recomputeCoverage(path, {repos, workloads}, nowMs)` and declare the + resulting path in the gate summary alongside the capability manifest, so + a human re-reading the run can find it. + +**Size discipline is enforced at write time, not just at submit time.** Every +leaf (`deployState`, each PR, each log sweep) must already be a digested +`block` per `evidence-routing.md`'s caps (`SUMMARY≤80`, `SNIPPET≤4/8 lines`, +link over diff) — never a raw dump. Cap `prsInWindow` to the top ~30 candidates +by path-overlap relevance, not every PR in the window. + +Pass `evidencePathFor(...)`'s path to Step 5's fan-out as `evidenceFilePath` — +every dispatch (representative and sibling) must be told to read it first. + +## Step 5 — fan-out (fully autonomous) + +Drive the cluster work-list, **`concurrency` (default 20) at a time**: +representatives deep, siblings one-turn-confirm. Eagerly persist to the CSV/WAL +(claim → heartbeat → flip) so the run is resumable. + +> **Concurrency comes from `config/rca.config.json` — always read it from +> there, never hardcode.** The default path (direct Agent-tool dispatch) honors +> the JSON value literally: fan out coordinator subagents in batches of +> `concurrency` (one message, up to `concurrency` tool-use blocks per batch). +> The opt-in `workflows/rca-batch.mjs` path is subject to the Workflow tool's +> architectural cap of `min(16, cpu cores - 2)` — on that path `concurrency` +> is a soft upper bound and excess work queues rather than running N-wide. +> If you need literal fan-out, use the default direct-dispatch path. + +- **Default (all hosts, including Claude Code) → direct Agent-tool dispatch.** + Read `concurrency` from `config/rca.config.json` and dispatch + `tfa-rca:ai-tfa-coordinator` subagents in batches of that size (one message, + up to `concurrency` tool-use blocks per batch). This path is **outside the + Workflow runtime**, so the `min(16, cores-2)` ceiling does not apply and the + JSON value is honored literally. Prefer this path whenever the machine's + Workflow cap (`min(16, cores-2)`) would be smaller than the configured + `concurrency` — e.g. an 8-core Mac caps Workflow at 6 while the JSON asks + for 20. +- Opt-in `workflows/rca-batch.mjs` (Claude Code only) → use only when the + Workflow tool's structured `pipeline()`/`parallel()` orchestration, + `resumeFromRunId` resumability, or progress UI is worth the concurrency + trade. On this path `concurrency` is a soft target only — the runtime hard- + caps at `min(16, cores-2)` regardless of the JSON value. +- Hosts without the Workflow runtime and without Agent-tool fan-out → drive + the sequential harness `lib/loop.mjs` (`runRcaLoop`) one test at a time. + Same contract, same no-prompt rule. + +Subagents/coordinators return compact `RCA_OUTPUT` blocks, never transcripts. A +coordinator that dies becomes a recorded `failed` row — one stuck test never +sinks the batch (partial-first). No path ever prompts the user (the gate is +closed). + +**Coordinator prompts MUST name every connector-shaped skill on the manifest.** +Each dispatch prompt lists, per capability, the resolved connector skill from +Gate Part A Step 0 — e.g. *"Use `<resolved-github-skill>` for every +product_code / deploy / ci ask (canonical repos + branch live in the skill; do +NOT grep other repos). Use `<resolved-infra-skill>` for every infra ask."* A +coordinator prompt that omits a manifest-listed connector skill — and that +therefore lets the +coordinator infer repos from workspace `git remote` or cwd — is a bug: the +coordinator will land plausible-but-wrong PR attributions on adjacent repos. + +**Coordinator prompts MUST also name the Step 4 evidence file.** Every +dispatch prompt (representative and sibling alike) includes the absolute +`evidenceFilePath` from Step 4 with the instruction: *"Read `<path>` (via the +Read tool) before making any live github/infra/logs gather call. It's a +pre-fetch, not a hard dependency — a repo/workload it doesn't name, or marks +with a `gap`, is a genuine gap: fall back to the capability manifest above +exactly as if no file existed."* For a sibling, add: *"The file's data about +your OWN test's workload is real evidence, not inheritance — reading it is +fine. What must stay independent is the CONFIRMATION judgment: never adopt the +representative's verdict just because the file already has the answer in +it."* A dispatch prompt that omits this path forces its coordinator back into +a full independent sweep — exactly the redundancy Step 4 exists to remove. + +**The file is read-write, not just read-only.** When a coordinator has to +gather live (a genuine gap), tell it to write the result back — +`contributeGithubEvidence`/`contributeLogsEvidence` (`lib/evidence-file.mjs`), +passing its own `testRunId` as `writerId` — before finishing, not just answer +TFA and move on. A representative's deep dive (a full diff, a downstream +trace, a PR the pre-fetch never named) then benefits its own siblings and any +other cluster sharing the same repo/workload, instead of every one of them +re-running the same live search. This is already baked into +`agents/ai-tfa-coordinator.md`'s Operating Principle 0 for any dispatch of +that agent type — no need to repeat the mechanics in the prompt, just don't +omit `evidenceFilePath` (above), since write-back has nothing to write to +without it. + +**Pre-seed the MCP cache with the queries you just ran.** Step 4's log sweeps +are MCP calls, and a coordinator will often want the same ones. Deposit each +result under the key it would compute — `mcpCacheKey(tool, args)` then +`cachePut(toolCacheDirFor(buildId), key, {…, writerId: "orchestrator"}, nowMs)` +from `lib/tool-cache.mjs` — storing the DIGEST, not the raw rows. + +This is not optional polish; without it the MCP cache goes unused. Measured +across every live run before this was added: **zero MCP entries ever stored**, +because an agent's check-then-call-then-store costs three calls on a miss to +save one later, so skipping it is the rational choice for a one-off query. +Pre-seeding inverts that — the agent's `get` is a single call that usually +hits. Store the same digest you put in the evidence file; the two are +complementary (the file is read wholesale at turn 1, the cache answers a +specific repeat query later). + +**Also hand every dispatch the tool cache.** The evidence file shares digested +*findings*; `bin/cached-exec.mjs` / `bin/cached-mcp.mjs` share raw *call +results*, which is where most duplicate work actually hides — on one measured +build `gh` was 37% of all coordinator tool calls and 46 were byte-identical +commands re-run by different coordinators. Include the plugin root in each +dispatch prompt so coordinators can invoke the wrappers, and tell them to pass +their own `testRunId` as `writerId`. The cache lives at +`<tmpdir>/bstack-rca/rca-toolcache.<buildId>/`, one file per call key, shared +by shell and MCP alike. Read `node bin/cached-exec.mjs <buildId> --stats` at +the end of the run to report how much it actually saved rather than assuming. + +**Concurrency is handled by layout, not by locking.** Base +(`rca-evidence.<buildId>.json`) has exactly one writer — this orchestrator, in +Step 4. Every coordinator writes only its own shard under +`rca-evidence.<buildId>.contrib/<testRunId>.json`. Since no two processes ever +open the same file for writing, concurrent write-back cannot lose an update; +`readEvidenceFile` folds base + all shards into one view, applying shards in +sorted order, with real evidence taking precedence over a recorded `gap`. A +measured comparison: 8 concurrent writers with a realistic read→work→write +window lost **28 of 40 updates** against a single shared file, and **0 of 40** +under this layout. + +**Application bugs need a culprit PR.** Whenever a test's RCA classifies as +PRODUCT_BUG / application bug, the coordinator MUST hunt the culprit PR via the +github connector (deploy timeline vs last-pass window, changed paths vs failure +signature — `references/github-evidence.md`) and feed the PR link(s) to TFA in +the turn message so the dashboard RCA's `related_prs` populates. An +application-bug RCA with no GitHub PR link is **incomplete**: keep digging until +the turn cap; if still none, the turn must explicitly state "no culprit PR +identified after <what was searched>" and the CSV row records the gap. + +## Step 6 — finish: glimpse + dashboard report (NO local report) + +This plugin **never renders or writes a local RCA report, and never surfaces RCA +detail in Claude.** The in-Claude output is a two-line completion notice plus the +link — that is all. When every row is terminal: + +1. Print the **completion summary** from the CSV (`lib/glimpse.mjs` → + `renderGlimpse`): `RCA analysis complete — build <id>` + a status count line + (`<N> tests · <R> resolved · <P> pending · <F> failed`). **Nothing per-test.** +2. Call **`triggerRcaReport(buildUuid=<build id>)`** (add `force=true` only to + re-run over an existing completed report). +3. Print the link line, verbatim shape: + + ``` + Full report on the Test Observability UI: <viewReport> + ``` + +**Do NOT print** root causes, culprit/related PRs, cluster breakdowns, per-test +analysis, confidence rationales, or a per-test table — root_cause, related_prs, +suspect_signals and the like are for the CSV + the dashboard ONLY. If a human +wants the "why", they open the link. Claude's job here is "analysis complete → +report is at <link>", not to re-narrate the RCA the BrowserStack agent authored. + +## Resume + +On startup, run the reaper (`lib/csv-state.mjs` → `reaper`) to reclaim rows +stranded `in_flight` by a crashed worker (heartbeat older than +`reaperHeartbeatTtlSec`) back to `pending`, then re-point fan-out at the CSV. +Live `threadId`/`turnId` resume the prior thread; dead threads re-run from +pending. Resuming a run does **not** reopen the gate — no new questions. +(In-session only — cross-session durability is deferred.) + +A `pending-resume` row now means the coordinator's **soft-PENDING drain budget +was spent** (`softPendingDrain`), not merely that a turn ran past 90s — the +common case is drained in-flight and never reaches the CSV. Resume reads such a +row's `turnId` with `getTfaTurnResult` **before** submitting anything new on the +thread. + +## Hard rules + +- Exactly one gate. At most one consolidated question, at gate close. **After + the gate closes, never ask the user anything.** +- An invalid/absent connector is a recorded gap, never a blocker. +- Headless + missing build id → end immediately. Headless never asks. +- Never call `tfaRcaTurn` from this skill — always via the `ai-tfa-coordinator`. +- A soft-`PENDING` is never an answer: it must be drained with + `getTfaTurnResult(testRunId, turnId)` before any further submit on that thread. + Only a spent drain budget may end a test `PENDING`. +- Every failed test must end terminal in the CSV — partial-first, no abort-on-one-failure. +- Never gather `test_logs` — TFA owns logs. +- Never render/write a local RCA report — glimpse table + `triggerRcaReport` + + the Test Observability UI link only. +- A PRODUCT_BUG RCA without a GitHub PR link is incomplete — dig until the turn + cap, else state what was searched and record the gap. diff --git a/skills/rca-build/examples/sample-run.md b/skills/rca-build/examples/sample-run.md new file mode 100644 index 0000000..5efa6e2 --- /dev/null +++ b/skills/rca-build/examples/sample-run.md @@ -0,0 +1,77 @@ +# Example — one full run (fictional data, matches the recorded-turn fixtures) + +Invocation: `/rca-build awswx…fw2` (build id given; nothing else passed). + +## 1. Gate closes (the only user-visible checkpoint) + +``` +GATE CLOSED — capability manifest: + github ✅ valid (gh, authed) · infra ✅ valid (via kubectl, ctx staging-euc1) · logs ❌ absent · metrics ❌ absent + +Intake: + build id: awswx…fw2 (given) + product repo: acme/obs-api (assumed — from git remote) + automation repo: acme/obs-e2e (assumed — cwd holds the tests) + working branch: main (assumed — current branch) + default branch: main (assumed — origin HEAD) + PRs in play: none (gap) + +Gaps declared to TFA: logs, metrics +Proceeding autonomously: discovery → clustering → fan-out (concurrency 5, turn-cap 6). +``` + +## 2. A NEEDS_INFO turn answered (what the coordinator sends back) + +TFA asked: *"Did request-validation on POST /builds change since last green?"* +(`evidenceType: product_code`, priority high). The coordinator replies on the +same `threadId`: + +``` +ASK: Did request-validation on POST /builds change since last green? +TYPE: product_code +FOUND: yes +SUMMARY: Yes — the buildName validator was tightened to reject empty strings in the +suspect window. One PR touches the failing path; falsification below. +LINK: https://github.com/acme/obs-api/pull/7421 + +SUSPECT: + pr: #7421 + files: src/validators/build.ts + hunks: `- allowEmpty: true` → `+ allowEmpty: false` (validator schema) + author: jdoe + merged_at: 2026-07-01T09:14Z vs last_green: 2026-07-01T02:10Z vs started_at: 2026-07-01T21:40Z + verdict: supported + link: https://github.com/acme/obs-api/pull/7421 + +SUSPECT: + pr: #7418 + files: src/routes/builds.ts + hunks: logging middleware reorder only + author: asmith + merged_at: 2026-06-30T18:02Z vs last_green: 2026-07-01T02:10Z vs started_at: 2026-07-01T21:40Z + verdict: ruled-out (shipped-after check passed but no-path-overlap — hunks never touch the validator) + link: https://github.com/acme/obs-api/pull/7418 + +ASK: Full run logs for test 39 +TYPE: test_logs +FOUND: no +SUMMARY: out-of-scope — TFA owns test logs; skipped by contract. +``` + +## 3. Terminal output (glimpse only — NO local report) + +``` +RCA analysis complete — build awswx…fw2 +7 test(s) · 6 resolved · 1 pending + +Full report on the Test Observability UI: +https://automation.browserstack.com/builds/awswx…fw2?tab=ai_report&subTab=aitfa +``` + +That is the **entire** in-Claude output. No root causes, no culprit PRs, no +per-test table — those are on the dashboard, authored by the BrowserStack agent. +The RESOLVED turn shown earlier is the coordinator↔TFA exchange (internal to the +loop), not something re-printed to the user at the end. + +State file: `<tmpdir>/bstack-rca/rca-state.awswx…fw2.csv` (resume-safe; re-run +the same build id to pick up the PENDING row). diff --git a/skills/rca-build/references/clustering.md b/skills/rca-build/references/clustering.md new file mode 100644 index 0000000..1ac5866 --- /dev/null +++ b/skills/rca-build/references/clustering.md @@ -0,0 +1,60 @@ +# Failure-signature clustering + +Why: a red build's N failures usually trace to a handful of causes (one bad +PR/deploy/shared helper). Running the full collaborative loop once per *cause* +instead of once per *test* turns the dominant cost from **O(tests) → O(distinct +causes)** — the only thing that makes "RCA for ALL failed tests, even thousands" +feasible. But **every failed test must still show a per-test RCA in the TRA +dashboard**, so clustering collapses the *evidence hunt*, not the *output*. + +The logic lives in `lib/signature.mjs`; this file is the protocol. + +## The signature + +Computed from the trimmed failure detail `listTestIds(includeFailureDetail=true)` +already returns on each row — **no extra probe turns**: + +``` +signature = normalize(failure_category) | normalize(error_summary) | normalize(file_path) +``` + +`normalize` folds the volatile tokens that make two instances of the *same* +failure look different: ISO timestamps, UUIDs, hex/memory addresses, `file:line:col`, +and bare numbers. So `timeout after 3000ms on node-7` and `timeout after 5000ms +on node-2` share a signature. + +A row with **no signal** (empty category, error, and path) is **not** merged into +a catch-all — it becomes its own singleton (`solo-<testRunId>`). Better an +un-clustered test than a wrong cluster. + +## Representative + siblings + +Each cluster gets: + +- **Representative** — a stable exemplar (non-flaky preferred, then smallest + `testRunId`). Runs the **full multi-turn `ai-tfa-coordinator` loop** → + confirmed root cause + culprit `related_prs`. +- **Siblings** (`N−1`) — each runs its **own** coordinator, **pre-seeded** with + the representative's `root_cause` + suspect PRs. TFA confirms the hypothesis + **against that sibling's own logs in a single turn** → a logs-grounded per-test + RCA in the dashboard at minimal cost. + +Net cost per cluster: **1 deep investigation + (N−1) one-turn confirms.** + +## The safeguard — never blindly inherit + +Distinct failures can share an error string. A sibling's pre-seed turn is a +*hypothesis to confirm*, not a verdict to copy: + +- TFA `RESOLVED`s the sibling in one turn → logs-grounded inheritance, cheap. +- TFA returns `NEEDS_INFO` (the hypothesis does not hold for this + test's logs) → the sibling **falls back to its own full loop**. The + representative's cause is never stamped onto a sibling without log confirmation. + +This keeps correctness independent of the cost optimization: worst case, every +sibling runs its own full loop (same as no clustering); best case, one deep run +covers the whole cluster. + +## Singletons + +A cluster of one is just a plain per-test loop — no pre-seed, no confirm step. diff --git a/skills/rca-build/references/evidence-routing.md b/skills/rca-build/references/evidence-routing.md new file mode 100644 index 0000000..9132fef --- /dev/null +++ b/skills/rca-build/references/evidence-routing.md @@ -0,0 +1,165 @@ +# Evidence Routing + +Load this file **before fulfilling any `NEEDS_INFO` ask** in the per-test RCA +loop (`agents/ai-tfa-coordinator`). It maps each TFA `evidenceType` to a +**capability** (not a hardcoded tool), and defines the **digest** the coordinator +submits on the next turn. + +The core contract: **TFA owns logs; the client agent owns everything else.** The +coordinator never seeds logs and never fulfills a `test_logs` ask. Every other +`evidenceType` routes to a capability that is gathered via **whatever skill/tool +the client actually has** for it (discovered **and validated** once into the +capability manifest — see `SKILL.md` § Gate Part A). There are **no `kubectl` / +`chitragupta` / +`bifrost` literals here** — that is the whole point of going generic. + +The registry logic lives in `lib/routing.mjs` (`routeAsk` / `routeAsks`); this +file is the human/agent-facing contract for the digest and the size caps. + +--- + +## How a turn's asks are processed + +A `NEEDS_INFO` turn returns `asks: TfaAsk[]`, each `{ what, why, evidenceType, +priority }`. For each ask, in descending `priority` (`high` → `medium` → `low`): + +1. Route the `evidenceType` (via `lib/routing.mjs` → the config registry + + capability manifest). The result is one of three actions: + - **skip** — `test_logs` (TFA-owned). Gather nothing; record in `asks_skipped`. + - **gather** — a capability is available. Run its discovered skill/tool scoped + by `what` / `why`, then digest the result into one ask block. + - **gap** — no valid connector for that `evidenceType` (the gate recorded it + as `invalid`/`absent`). Emit an `unavailable` block back to TFA — **never + prompt the user** (the gate is closed; the run is autonomous). +2. Concatenate the per-ask blocks into the next-turn `message` and resubmit on + the same `threadId`. + +An ask that cannot be fulfilled is **never silently dropped** — it becomes a +`not-found` / `unreachable` / `unavailable` block so TFA can reason about the gap. + +--- + +## Routing table (capability, not tool) + +`evidenceType` literals are exactly those `tfaRcaTurn` emits: `test_logs`, +`product_code`, `infra` (TFA may still spell it `k8s` — both route the same), +`kibana`, `metrics`, `deploy`, `ci`, `other`. + +| `evidenceType` | Capability | Gathered via (discovered at runtime) | +|---|---|---| +| `test_logs` | — (TFA, skip) | never gathered; TFA self-serves from its own log access | +| `product_code` | `github` | the client's GitHub capability — **GitHub MCP if present, else `gh`** (see `references/github-evidence.md`) | +| `deploy` | `github` | deploy timeline via the GitHub capability (releases/tags + deploy record) | +| `ci` | `github` | CI config + run history via the GitHub capability | +| `infra` / `k8s` | `infra` | **whatever runtime connector the user has** — k8s/EKS, ECS, docker, Nomad, plain VMs, PM2, … Discovered and probed at the gate, NEVER assumed to be Kubernetes; the manifest records the kind (`via`) | +| `kibana` | `logs` | whatever log-search skill the client has (kibana or other) | +| `metrics` | `metrics` | whatever metrics skill the client has | +| `other` | `other` | best-effort by ask text; else a `not-found` block | + +The mapping is data in `config/rca.config.json` (`evidenceRouting`), so a +different deployment can remap `evidenceType → capability` without code changes. + +**Deployment-state guard:** a suspect PR only matters if its code was actually +live in the run's env at the failure window. If you can cheaply confirm it was +not deployed / behind an OFF flag, say so in the digest rather than feeding TFA a +suspect that could not have caused the failure. (Full protocol: U9 / +`references/github-evidence.md`.) + +--- + +## Digest format + +The single most important discipline: **digested input, not raw dumps.** Every +turn's `message` loads into the agent's context *and* is sent to TFA; a raw log +tail or full PR diff blows both budgets and degrades TFA's reasoning. Supply the +*findings*, not the *haystack*. + +### Per-ask block shape — `ask → found → snippet/link` + +**The canonical fillable format lives in +[`../templates/evidence-block.md`](../templates/evidence-block.md)** (fulfilled +and unfulfillable variants) — copy it, don't retype it. Shape: +`ASK / TYPE / FOUND: yes|no|partial / SUMMARY ≤400 / SNIPPET (caps below) / LINK`. + +- `SUMMARY` is the answer. `SNIPPET` is the *minimum* evidence backing it. `LINK` + lets TFA (or a human) verify without the bytes living in the message. +- Prefer **LINK over SNIPPET** whenever a permalink fully carries the evidence. + +### Size caps (hard ceilings — truncate, never exceed) + +| Field / scope | Soft target | Hard ceiling | On exceed | +|---|---|---|---| +| `SUMMARY` | ≤ 60 chars | 80 chars | Tighten to the finding; drop restatement of the ask | +| `SNIPPET` per ask | ≤ 4 lines | 8 lines | Keep the load-bearing lines; replace the rest with `… (N lines elided — see LINK)` | +| Code diff in a `product_code` snippet | ≤ 1 hunk | 2 hunks | Show changed lines only, no context lines; link the full PR | +| Whole next-turn `message` | ≤ 40 lines | 80 lines (and ≤ `turnMessageMaxChars`) | Drop `low`-priority asks first; keep every `high` ask's block | +| Asks fulfilled per turn | all `high` + `medium` | — | Defer `low` asks to a later turn rather than truncating a `high` ask | + +Truncation rule of thumb: **never truncate a `high`-priority ask's block to fit a +`low`-priority one.** Drop the low block whole; keep the high block intact. The +whole-message ceiling honors `turnMessageMaxChars` from +`config/rca.config.json`, now set to **1000 chars** — a plugin-configured +self-limit, tighter than the underlying tool's actual hard cap (the +`tfaRcaTurn` MCP tool itself allows up to 5000 chars per `message`; the plugin +just chooses not to use all of it). At this budget, expect at most 2-3 ask +blocks per turn before hitting the ceiling — defer lower-priority asks to a +follow-up turn rather than cramming everything into one. + +### What never goes in a digest + +- Raw log tails, full log output, full file contents, full PR diffs — link or excerpt. +- `test_logs` content of any kind (TFA owns it). +- Credentials, tokens, internal hostnames, or any secret surfaced by an env/secret dump. +- Speculation dressed as a finding. If `FOUND: no`, say what was checked; do not invent a cause. + +--- + +## Unfulfillable asks — report, don't drop + +``` +ASK: <verbatim what> +TYPE: <evidenceType> +FOUND: no +SUMMARY: not-found | unreachable | unavailable | out-of-scope — <one line: what was checked or why blocked> +``` + +- `not-found` — the skill/tool ran but the signal isn't there. State the search performed. +- `unreachable` — the surface was not reachable from this agent context. State which. +- `unavailable` — no valid connector exists for this `evidenceType` (a gate-recorded gap). +- `out-of-scope` — the ask is `test_logs` or otherwise not the agent's to fulfill. + +An all-`unavailable` / all-`not-found` turn still resubmits — TFA decides how to +converge (best-effort, lower confidence) or what else to ask. The coordinator +does not pre-empt that decision. + +--- + +## Capability manifest (built once, at the gate) + +Rather than re-discover "is there a kibana skill?" on every ask across every +test, Gate Part A enumerates **and probe-validates** the client's connectors +**once** up front into a manifest (`lib/routing.mjs` → `buildManifest`). +`valid` maps to `available: true`; `invalid`/`absent` map to `available: false` +(a recorded gap): + +``` +{ github: {available: true, via: "gh"}, infra: {available: true, via: "kubectl"}, logs: {available: false}, ... } +``` + +- Every ask routes against this manifest — reproducible, no per-ask discovery. +- The gate summary **declares the gaps to the user** ("infra + metrics not + available") and the first turn declares them to TFA so it plans asks around + what's obtainable. +- Frozen at gate close. A skill appearing mid-run is not picked up until the next run. + +## Build-level evidence cache (compute once) + +"Diff since last green", "deploy timeline", and "PRs in the suspect window" are +properties of the **build**, not the test. The orchestrator computes the +last-green→this-build delta **once** (`lib/evidence-cache.mjs`), caches it by +`(repo, commit-range, evidenceType)`, and pre-seeds every coordinator with the +same grounded suspect window — collapsing N×M redundant git/infra calls to ~M and +front-loading the highest-signal evidence so many tests RESOLVE before any infra +ask fires. No "last green" (never-green suite) → fall back to a configured +baseline ref and note the weaker grounding in the turn digest (it lands in the +dashboard RCA). diff --git a/skills/rca-build/references/github-evidence.md b/skills/rca-build/references/github-evidence.md new file mode 100644 index 0000000..a3d6b3f --- /dev/null +++ b/skills/rca-build/references/github-evidence.md @@ -0,0 +1,122 @@ +# GitHub evidence — what to gather, and how to rule a suspect OUT + +The worst automated-RCA outcome is **confidently blaming an innocent PR**. This +file is the contract for `product_code` / `deploy` / `ci` asks (the `github` +capability): the **exact** evidence to gather, and a **falsification protocol** +that tries to *disprove* each suspect before it enters `related_prs`. + +> We do **not** ship a GitHub forensics harness or MCP tool. We specify what's +> needed and use whatever the client already has — **GitHub MCP if available, +> else `gh`, else degrade** to an `unavailable` block. + +## Capability discovery (in order) + +1. **GitHub MCP** (`mcp__github__*`) — preferred for structured PR/diff/blame queries. +2. **`gh` CLI** — fall back for git-graph operations (`gh pr list --search`, + `gh api`, `merge-base`, ancestry) and anything the MCP doesn't cover. +3. **Neither** → emit an `unavailable` block for the ask (do not fabricate a PR). + +The gate records which is present **and probe-validated** (`gh auth status` / +GitHub MCP tools listed) in the capability manifest +(`capability: github → { available, via }`); route every github ask against it. + +## Application bugs REQUIRE a culprit-PR hunt (mandatory) + +Whenever TFA's working classification is **PRODUCT_BUG / application bug**, the +github connector is not optional evidence — it is the deliverable. The +coordinator MUST hunt the culprit PR: + +1. **Deploy timeline vs last-pass window** — what shipped to the run's env + between the last passing run and this failure. +2. **Changed paths vs failure signature** — intersect the window's PRs' changed + files with the failing file/function from the signature. +3. Run the falsification protocol below on each candidate. + +Feed the surviving PR **link(s)** to TFA in the turn message so the BrowserStack +agent populates `related_prs` in the dashboard RCA. **An application-bug RCA +with no GitHub PR link is INCOMPLETE**: keep digging on subsequent turns until +the turn cap. If still none, the turn must explicitly state +`no culprit PR identified after <what was searched: window, repos, paths>` and +the CSV row records the gap. Never fabricate a PR; if the github connector is +invalid/absent, the same explicit statement plus an `unavailable` block goes to +TFA (a gate-recorded gap). + +## Evidence each ask needs (be specific — no fishing) + +| Ask intent | Gather exactly | +|---|---| +| "Did `<X>` change since the last passing run?" | the diff of `<X>`'s file/function between the **baseline ref** (last-green, or the configured fallback) and the build's commit — not the whole repo diff | +| "Which PRs are suspect?" | PRs **merged in the window** `(baselineRef, build commit]` that **touch the failing code path** — intersect changed files with the failing file/function | +| "Who/what last changed the failing line?" | `blame` on the specific failing lines (from the test's `file_path` + the error) | +| "What shipped to the run's env before the failure?" | deploy timeline (`gh` releases/tags + the env's deploy record); compare deploy time vs. the run's `started_at` | +| "Did CI change?" | the workflow-file diff + recent `gh run` history for the failing job | + +Scope everything by the failing test's `file_path` + the error summary. The +build-level evidence (diff-since-last-green, PR window) is **pre-computed once** +and passed in — reuse it; do not re-fetch per test. + +## Field-filtering — project before you pull, every call + +The single most common way a gather call wastes context: pulling a full +object when the ask only needs one or two fields from it. This applies to +whichever connector resolved for `github` (most commonly the `gh` CLI today, +or a GitHub MCP tool) — every call should already be filtered to the field(s) +the ask needs, not filtered after the fact by reading past the noise. The +same discipline applies to `infra` gather calls (`kubectl` or whatever the +manifest resolved to), since the failure mode is identical. + +| Need | Don't — pulls the whole object | Do — projects to the field(s) the ask needs | +|---|---|---| +| Repo exists / default branch | `gh api repos/OWNER/REPO` | `gh api repos/OWNER/REPO --jq '.default_branch'` | +| Branch exists on the shipping branch | `gh api repos/OWNER/REPO/branches/BRANCH` | `gh api repos/OWNER/REPO/branches/BRANCH --jq '.name'` | +| Commit history / PR-window search | `gh api "repos/OWNER/REPO/commits?sha=BRANCH&per_page=100"` | add `--jq '[.[] | {sha: .sha[0:8], date: .commit.committer.date, msg: (.commit.message | split("\n")[0])}]'` | +| PR metadata | `gh pr view N --repo OWNER/REPO` (full payload) | `gh pr view N --repo OWNER/REPO --json state,mergedAt,baseRefName,headRefOid,files,author` — `--json` is itself a field allowlist; list only the fields this ask uses | +| Pod / workload listing | `kubectl get pods -n NS -o wide` | `kubectl get pods -n NS -o custom-columns='NAME:.metadata.name,STATUS:.status.phase'` | +| Deploy / image state | `kubectl get deploy -n NS -o yaml` | `kubectl get deploy -n NS -o custom-columns='NAME:.metadata.name,IMAGE:.spec.template.spec.containers[0].image'` | +| Log sweep | a raw `--tail` dump | `kubectl logs POD --since=<window> --tail=2000 \| grep -E '<correlation token>\|ERROR\|Exception'` — filter by the correlation token, never a raw tail | + +**Never run the unfiltered form "to see the shape first."** An exploratory +raw call costs the same context whether or not its output ends up in the +digest — a bare repo or commit object routinely carries license/URL metadata +and a multi-hundred-character signature block that no evidence ask ever +consults. If the exact field path is genuinely unknown, learn the shape from +one throwaway call against a cheap target, then filter every real call from +that point on — never repeat the unfiltered form per repo, per PR, or per +test. + +## Falsification protocol — rule out, don't just rule in + +For **each** candidate suspect PR, try to **break** the hypothesis: + +1. **Path overlap.** Do the PR's changed hunks actually touch the failing code + path (the function/line in the stack)? No overlap → **ruled out**. +2. **Deployment-state guard.** Was the PR's code actually **live** in the run's + env at `started_at`? If it shipped *after* the failure window, or sits behind + an **OFF** flag, it could not have caused this failure → **ruled out**. +3. **Direction.** Does the change plausibly produce *this* error (e.g. a validator + tightened to reject the input the test sends)? If the change is unrelated to + the symptom → **weak**, mark accordingly. + +Feed **both supporting and disconfirming** evidence back to TFA. A suspect that +survives 1–3 is a real candidate; one that fails any is reported as ruled-out +(with the reason), **not** dropped silently. + +## The suspect packet (structured, not free text) + +Each surviving/ruled-out suspect is one structured block so `related_prs` +populates deterministically. **The canonical fillable format lives in +[`../templates/suspect-packet.md`](../templates/suspect-packet.md)** (fields: +pr, files, hunks, author, merged_at vs last_green vs started_at, verdict with +rule-out reason, link) — copy it, don't retype it. A worked example (supported ++ ruled-out side by side) is in +[`../examples/sample-run.md`](../examples/sample-run.md). + +Only `verdict: supported` suspects should end up in TFA's `related_prs`. Ruled-out +suspects stay in the thread as disconfirming evidence so TFA (and a human) can see +the elimination, not just the conclusion. + +## Digest discipline + +Same caps as `references/evidence-routing.md`: prefer a PR **link** over pasting a +diff; at most 1 hunk (3 hard) per `product_code` snippet; never paste a full diff. +The packet is *findings*, not the haystack. diff --git a/skills/rca-build/templates/evidence-block.md b/skills/rca-build/templates/evidence-block.md new file mode 100644 index 0000000..97b41f7 --- /dev/null +++ b/skills/rca-build/templates/evidence-block.md @@ -0,0 +1,26 @@ +# Template — evidence block (one per fulfilled/unfulfilled ask) + +The unit of evidence sent back to TFA in a turn message. Rules, size caps and +the forbidden list: `../references/evidence-routing.md`. + +Fulfilled ask: + +``` +ASK: <verbatim `what` from the TfaAsk, ≤ 80 chars> +TYPE: <evidenceType> +FOUND: <yes | no | partial> +SUMMARY: <1 sentence — the finding, in the agent's words. ≤ 80 chars> +SNIPPET: + <the load-bearing excerpt only, ≤ 4 lines — see size caps. Omit if a LINK fully carries it.> +LINK: <permalink to the source — PR/commit/log-search/metrics panel/deploy record. Omit if N/A.> +``` + +Unfulfillable ask (report, don't drop — machine-generated for absent connectors +by `lib/loop.mjs` `unavailableBlock`): + +``` +ASK: <verbatim what> +TYPE: <evidenceType> +FOUND: no +SUMMARY: not-found | unreachable | unavailable | out-of-scope — <one line: what was checked or why blocked> +``` diff --git a/skills/rca-build/templates/gate-summary.md b/skills/rca-build/templates/gate-summary.md new file mode 100644 index 0000000..48ac6bf --- /dev/null +++ b/skills/rca-build/templates/gate-summary.md @@ -0,0 +1,25 @@ +# Template — gate summary (printed once, when THE GATE closes) + +One terse FYI before autonomous execution starts. Every intake field is tagged +`given | assumed | gap`; every connector `valid | invalid | absent`. After this +prints, the run never asks the user anything. + +If the product repo could not be corroborated against the failures AND no PRs +were supplied, the gate asks ONE question (interactive only) before printing +this summary. Headless skips it and prints `product repo: unknown (gap)`. + +``` +GATE CLOSED — capability manifest: + github ✅ valid (gh, authed) · infra ✅ valid (via <kubectl ctx …, docker, ecs, …>) · logs ❌ absent · metrics ❌ absent + +Intake: + build id: <id> (given) + product repo: <org/repo> (assumed — corroborated vs failures | asked — human answered | unknown (gap)) + automation repo: <org/repo> (assumed — cwd holds the tests) + working branch: <branch> (assumed — current branch) + default branch: <branch> (assumed — origin HEAD) + PRs in play: <#123, #456 | none> (given | gap) + +Gaps declared to TFA: <logs, metrics | none> +Proceeding autonomously: discovery → clustering → fan-out (concurrency <N>, turn-cap <M>). +``` diff --git a/skills/rca-build/templates/suspect-packet.md b/skills/rca-build/templates/suspect-packet.md new file mode 100644 index 0000000..676fcf4 --- /dev/null +++ b/skills/rca-build/templates/suspect-packet.md @@ -0,0 +1,22 @@ +# Template — SUSPECT packet (one block per candidate PR) + +Fill one block per suspect, supported **and** ruled-out (elimination is evidence +too). Only `verdict: supported` suspects may feed `related_prs`. Guidance + +falsification protocol: `../references/github-evidence.md`. + +``` +SUSPECT: + pr: <#number> + files: <changed files overlapping the failing path> + hunks: <the 1-3 load-bearing changed hunks — see digest size caps> + author: <login> + merged_at: <ts> vs last_green: <ts> vs started_at: <ts> + verdict: supported | ruled-out (<no-path-overlap | shipped-after | behind-off-flag | unrelated>) + link: <PR permalink> +``` + +If the hunt ends empty after a real search (never fabricate): + +``` +no culprit PR identified after <what was searched: window, repos, paths> +``` diff --git a/tests/conformance.test.mjs b/tests/conformance.test.mjs new file mode 100644 index 0000000..d9af539 --- /dev/null +++ b/tests/conformance.test.mjs @@ -0,0 +1,318 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import { runRcaLoop, replaySubmit, replayRead } from "../lib/loop.mjs"; + +const here = dirname(fileURLToPath(import.meta.url)); +const load = (name) => + JSON.parse(readFileSync(join(here, "fixtures", "recorded-turns", name), "utf8")); + +const CONFIG = { + turnCap: 6, + evidenceRouting: { + test_logs: { owner: "tfa", skip: true }, + product_code: { capability: "github" }, + other: { capability: "other" }, + }, +}; +const GITHUB_AVAILABLE = { github: { available: true, via: "gh" } }; + +// A coordinator gather() stub: returns a one-line digest block. +const gather = async (g) => `ASK: ${g.ask.what}\nTYPE: ${g.evidenceType}\nFOUND: yes\nSUMMARY: stub`; + +// Drain tests inject a no-op sleep so the 5s read interval costs no wall clock. +const noSleep = async () => {}; + +test("resolved fixture: NEEDS_INFO → evidence → RESOLVED, trimmed glimpse captured, test_logs skipped", async () => { + const fx = load("resolved.json"); + const result = await runRcaLoop({ + testRunId: fx.testRunId, + firstMessage: "Error: empty buildName", + submit: replaySubmit(fx.turns), + config: CONFIG, + manifest: GITHUB_AVAILABLE, + gather, + }); + assert.equal(result.status, "RESOLVED"); + assert.match(result.root_cause, /#7421/); + assert.ok(result.root_cause.length <= 220); // glimpse root_cause is trimmed server-side + assert.equal(result.failure_type, "product_regression"); + assert.deepEqual(result.related_prs, ["#7421"]); + assert.match(result.view_rca, /^https:\/\/automation\.browserstack\.com/); + assert.deepEqual(result.asks_fulfilled, ["product_code"]); + assert.deepEqual(result.asks_skipped, ["test_logs"]); // TFA-owned, never gathered + assert.equal(result.turns_used, 2); + assert.equal(result.threadId, "thr-39"); +}); + +test("pending fixture: soft-PENDING with NO getTfaTurnResult tool → ends resumable, never resubmits", async () => { + // Client lacks readTurn: the drain is impossible, so the old floor applies — + // report it resumable rather than busy-waiting through tfaRcaTurn resubmits. + const fx = load("pending.json"); + let calls = 0; + const counting = async (args) => { + calls++; + return replaySubmit(fx.turns)(args); + }; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: counting, + config: CONFIG, + sleep: noSleep, + }); + assert.equal(result.status, "PENDING"); + assert.equal(result.turnId, "turn-81-1"); + assert.equal(result.threadId, "thr-81"); + assert.equal(calls, 1); // one submit, no resubmit +}); + +test("soft-PENDING is DRAINED via getTfaTurnResult before the next submit, not reported", async () => { + // The real 3840238857 case: turn 1 finalized NEEDS_INFO at 104s, past the tool's + // 90s in-call cap, so submit() handed back a soft PENDING. Reading the same + // turnId lands the NEEDS_INFO the agent had already committed to. + const fx = load("soft-pending-drain.json"); + const submits = []; + const submit = replaySubmit(fx.turns); + const read = replayRead(fx.reads); + let reads = 0; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + firstMessage: "Initiating collaborative RCA", + submit: async (args) => { + submits.push(args); + return submit(args); + }, + readTurn: async (args) => { + reads++; + return read(args); + }, + config: CONFIG, + manifest: GITHUB_AVAILABLE, + gather, + sleep: noSleep, + }); + + assert.equal(result.status, "RESOLVED"); + assert.equal(reads, 3); // PENDING, PENDING, then the landed NEEDS_INFO + assert.equal(submits.length, 2); // turn 1, then the evidence turn — no submit mid-flight + assert.equal(result.turns_used, 2); // 3 reads did NOT consume the turn cap + assert.deepEqual(result.asks_fulfilled, ["product_code"]); + assert.deepEqual(result.asks_skipped, ["test_logs"]); // TFA owns logs, even post-drain + assert.match(result.root_cause, /#current-url/); +}); + +test("drain reads the SAME turnId, and stops resubmitting it once landed", async () => { + const fx = load("soft-pending-drain.json"); + const readArgs = []; + const submits = []; + const submit = replaySubmit(fx.turns); + const read = replayRead(fx.reads); + await runRcaLoop({ + testRunId: fx.testRunId, + submit: async (args) => { + submits.push(args); + return submit(args); + }, + readTurn: async (args) => { + readArgs.push(args); + return read(args); + }, + config: CONFIG, + manifest: GITHUB_AVAILABLE, + gather, + sleep: noSleep, + }); + // Every read targets the turnId the soft-PENDING handed back. + for (const a of readArgs) { + assert.equal(a.turnId, "c2e1a6fd-2243-4f93-bc69-62f298db062c"); + assert.equal(String(a.testRunId), "3840238857"); + } + // The spent resume handle is dropped: the follow-up submit rides threadId only. + assert.equal(submits[1].turnId, undefined); + assert.equal(submits[1].threadId, "chat:3840238857"); +}); + +test("drain budget is bounded: a wedged turn ends PENDING instead of hanging the batch", async () => { + const fx = load("soft-pending-drain.json"); + let reads = 0; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit([fx.turns[0]]), // always soft-PENDING + readTurn: async () => { + reads++; + return { status: "PENDING", turnId: "c2e1a6fd-2243-4f93-bc69-62f298db062c" }; + }, + config: { ...CONFIG, softPendingDrain: { maxWaitMs: 60_000, intervalMs: 1, maxReads: 4 } }, + sleep: noSleep, + }); + assert.equal(result.status, "PENDING"); + assert.equal(reads, 4); // capped by maxReads, never unbounded + assert.match(result.root_cause, /soft-pending: still working after 4 read\(s\)/); + assert.equal(result.turnId, "c2e1a6fd-2243-4f93-bc69-62f298db062c"); // still resumable +}); + +test("a failed read is not a verdict — the drain keeps reading and still lands", async () => { + const fx = load("soft-pending-drain.json"); + const landed = fx.reads[2]; + let reads = 0; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit(fx.turns), + readTurn: async () => { + reads++; + if (reads === 1) throw new Error("transient 502 from o11y"); + return landed; + }, + config: CONFIG, + manifest: GITHUB_AVAILABLE, + gather, + sleep: noSleep, + }); + assert.equal(result.status, "RESOLVED"); + assert.equal(reads, 2); +}); + +test("a PERSISTENT hard error stops the drain early instead of burning the budget", async () => { + const fx = load("soft-pending-drain.json"); + let reads = 0; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit([fx.turns[0]]), // always soft-PENDING + readTurn: async () => { + reads++; + throw new Error("Failed to get tfa turn result: TFA agent run failed"); + }, + // Budget allows 40 reads; the error cap must cut it off long before that. + config: { ...CONFIG, softPendingDrain: { maxWaitMs: 600_000, intervalMs: 1, maxReads: 40, maxErrorReads: 3 } }, + sleep: noSleep, + }); + assert.equal(result.status, "PENDING"); + assert.equal(reads, 3, "stopped at maxErrorReads, not the 40-read budget"); + assert.match(result.root_cause, /tfa-error/); + assert.equal(result.turnId, "c2e1a6fd-2243-4f93-bc69-62f298db062c"); // still resumable +}); + +test("an error-shaped RESULT (not thrown) also trips the fast-fail", async () => { + const fx = load("soft-pending-drain.json"); + let reads = 0; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit([fx.turns[0]]), + // The MCP tool reports the wedge as a returned payload, not an exception. + readTurn: async () => { + reads++; + return { status: "ERROR", message: "TFA agent run failed" }; + }, + config: { ...CONFIG, softPendingDrain: { maxWaitMs: 600_000, intervalMs: 1, maxReads: 40, maxErrorReads: 2 } }, + sleep: noSleep, + }); + assert.equal(result.status, "PENDING"); + assert.equal(reads, 2); + assert.match(result.root_cause, /tfa-error/); +}); + +test("INTERMITTENT errors do not trip the fast-fail — a good read clears the streak", async () => { + const fx = load("soft-pending-drain.json"); + const landed = fx.reads[2]; + let reads = 0; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit(fx.turns), + readTurn: async () => { + reads++; + // fail, ok, fail, ok, ... never 2 consecutive failures + if (reads % 2 === 1) throw new Error("transient 502"); + return reads < 6 ? { status: "PENDING" } : landed; + }, + config: { ...CONFIG, softPendingDrain: { maxWaitMs: 600_000, intervalMs: 1, maxReads: 40, maxErrorReads: 2 } }, + manifest: GITHUB_AVAILABLE, + gather, + sleep: noSleep, + }); + assert.equal(result.status, "RESOLVED", "flaky-but-recovering reads must still land"); + assert.equal(reads, 6); +}); + +test("BLOCKED surfaced by a drain is terminal — no empty resubmits to the turn cap", async () => { + const fx = load("soft-pending-drain.json"); + let submits = 0; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: async (args) => { + submits++; + return replaySubmit(fx.turns)(args); + }, + readTurn: async () => ({ status: "BLOCKED", threadId: "chat:3840238857" }), + config: CONFIG, + sleep: noSleep, + }); + assert.equal(result.status, "PENDING"); + assert.equal(result.root_cause, "blocked"); + assert.equal(submits, 1); // BLOCKED carries no asks; never resubmitted +}); + +test("turn-cap fixture: ends PENDING(turn-cap) at the cap, never a 7th submit", async () => { + const fx = load("turn-cap.json"); + let submits = 0; + const counting = async (args) => { + submits++; + return replaySubmit(fx.turns)(args); + }; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: counting, + config: CONFIG, + manifest: GITHUB_AVAILABLE, + gather, + }); + assert.equal(result.status, "PENDING"); + assert.equal(result.root_cause, "turn-cap"); + assert.equal(submits, 6); // capped at turnCap, never 7 +}); + +test("degraded path: no connector → gap degrades to unavailable (never a prompt), still terminal", async () => { + // Same resolved fixture, but the client has NO github connector. The loop is + // autonomous: the gap becomes an `unavailable` block and it still RESOLVEs. + const fx = load("resolved.json"); + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit(fx.turns), + config: CONFIG, + manifest: {}, // nothing valid at the gate + }); + assert.equal(result.status, "RESOLVED"); + assert.deepEqual(result.asks_unavailable, ["product_code"]); + assert.deepEqual(result.asks_fulfilled, []); +}); + +test("unavailable block names the missing connector in the resubmitted message", async () => { + const fx = load("resolved.json"); + const messages = []; + const recording = (inner) => async (args) => { + messages.push(args.message); + return inner(args); + }; + await runRcaLoop({ + testRunId: fx.testRunId, + submit: recording(replaySubmit(fx.turns)), + config: CONFIG, + manifest: {}, + }); + assert.match(messages[1], /unavailable — no github connector/); +}); + +test("no testRunId → failed block, tool never called", async () => { + let called = false; + const result = await runRcaLoop({ + testRunId: undefined, + submit: async () => { + called = true; + return {}; + }, + config: CONFIG, + }); + assert.equal(result.status, "failed"); + assert.equal(called, false); +}); diff --git a/tests/coverage-glimpse.test.mjs b/tests/coverage-glimpse.test.mjs new file mode 100644 index 0000000..3bc0e48 --- /dev/null +++ b/tests/coverage-glimpse.test.mjs @@ -0,0 +1,93 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { coverageStamp, classifyCoverage } from "../lib/coverage.mjs"; +import { renderGlimpse } from "../lib/glimpse.mjs"; + +// ---- coverage stamp -------------------------------------------------------- + +test("full coverage keeps TFA confidence", () => { + const s = coverageStamp({ + asksFulfilled: ["product_code"], + asksUnavailable: [], + tfaConfidence: "high", + }); + assert.equal(s.coverage, "full"); + assert.equal(s.band, "high"); +}); + +test("partial coverage caps a high TFA confidence at medium", () => { + const s = coverageStamp({ + asksFulfilled: ["product_code"], + asksUnavailable: ["kibana"], + tfaConfidence: "high", + }); + assert.equal(s.coverage, "partial"); + assert.equal(s.band, "medium"); + assert.deepEqual(s.unavailable, ["kibana"]); +}); + +test("thin coverage (nothing fulfilled, gaps) caps at low", () => { + const s = coverageStamp({ + asksFulfilled: [], + asksUnavailable: ["infra", "metrics"], + tfaConfidence: "high", + }); + assert.equal(s.coverage, "thin"); + assert.equal(s.band, "low"); +}); + +test("unknown TFA confidence floors to low even at full coverage", () => { + const s = coverageStamp({ asksFulfilled: [], asksUnavailable: [], tfaConfidence: "unknown" }); + assert.equal(s.coverage, "full"); + assert.equal(s.band, "low"); +}); + +test("classifyCoverage dedupes and handles empties", () => { + assert.equal(classifyCoverage(["a", "a"], []), "full"); + assert.equal(classifyCoverage([], ["x"]), "thin"); +}); + +// ---- glimpse (the ONLY in-client output — no local report) ------------------ + +test("empty batch renders a valid glimpse, no crash", () => { + const txt = renderGlimpse([], { buildId: "b1" }); + assert.match(txt, /No failed tests analyzed/); +}); + +test("glimpse is a completion notice with counts — NO per-test detail", () => { + const rows = [ + { + testRunId: "101", + cluster_id: "c1", + rca_done: "resolved", + confidence: "high", + root_cause: "PR #7421 tightened validator", + related_prs: "#7421", + }, + { testRunId: "102", cluster_id: "c1", rca_done: "pending-resume" }, + { testRunId: "103", cluster_id: "c2", rca_done: "failed" }, + ]; + const txt = renderGlimpse(rows, { buildId: "b1" }); + assert.match(txt, /RCA analysis complete — build b1/); + assert.match(txt, /3 test\(s\)/); + assert.match(txt, /1 resolved/); + assert.match(txt, /1 pending/); // pending-resume buckets to "pending" + assert.match(txt, /1 failed/); + // The trim: no per-test lines, no root cause, no PRs, no testRunIds leak. + assert.doesNotMatch(txt, /7421|PR #|→|101|102|103|c1|c2/); +}); + +test("glimpse never leaks a verbose root_cause, however long", () => { + const rows = [ + { + testRunId: "1", + cluster_id: "solo-1", + rca_done: "resolved", + confidence: "medium", + root_cause: `line one\nline two ${"x".repeat(200)}`, + }, + ]; + const txt = renderGlimpse(rows); + assert.doesNotMatch(txt, /line one|line two|xxxx/); // cause stays out of Claude + assert.match(txt, /1 test\(s\) · 1 resolved/); +}); diff --git a/tests/csv-state.test.mjs b/tests/csv-state.test.mjs new file mode 100644 index 0000000..b5caeb3 --- /dev/null +++ b/tests/csv-state.test.mjs @@ -0,0 +1,209 @@ +import { test, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + csvPathFor, + seed, + readRows, + claim, + heartbeat, + flip, + reaper, + pendingRows, + PENDING, +} from "../lib/csv-state.mjs"; + +let dir; +let csv; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "rca-csv-")); + csv = join(dir, "state.csv"); +}); +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +const TESTS = [ + { + test_id: 101, + test_name: "login", + failure: { category: "Assertion", error_summary: "expected 200", file_path: "a.rb" }, + }, + { test_id: 102, test_name: "checkout", failure: { category: "Timeout" } }, +]; + +test("seed writes one pending row per test with signature columns", () => { + const rows = seed(csv, "build-1", TESTS); + assert.equal(rows.length, 2); + assert.ok(rows.every((r) => r.rca_done === PENDING)); + const login = rows.find((r) => r.testRunId === "101"); + assert.equal(login.failure_category, "Assertion"); + assert.equal(login.error_summary, "expected 200"); + assert.equal(login.buildId, "build-1"); +}); + +test("seed is idempotent — no duplicate rows on re-seed", () => { + seed(csv, "build-1", TESTS); + const rows = seed(csv, "build-1", TESTS); + assert.equal(rows.length, 2); +}); + +test("seed preserves a terminal row on re-seed", () => { + seed(csv, "build-1", TESTS); + flip(csv, 101, { rca_done: "resolved", root_cause: "bad PR" }, 1000); + seed(csv, "build-1", TESTS); + const login = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(login.rca_done, "resolved"); + assert.equal(login.root_cause, "bad PR"); +}); + +test("claim sets the worker; a second worker is refused", () => { + seed(csv, "build-1", TESTS); + assert.equal(claim(csv, 101, "w1", 1000), true); + assert.equal(claim(csv, 101, "w2", 1000), false); + const row = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(row.in_flight_worker, "w1"); +}); + +test("heartbeat updates ts only for the owning worker", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + assert.equal(heartbeat(csv, 101, "w1", 2000), true); + assert.equal(heartbeat(csv, 101, "w2", 3000), false); + assert.equal(readRows(csv).find((r) => r.testRunId === "101").heartbeat_ts, "2000"); +}); + +test("flip records terminal fields, joins related_prs, clears the claim", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + flip( + csv, + 101, + { rca_done: "resolved", root_cause: "PR #7421", related_prs: ["#7421", "#7430"], confidence: "high" }, + 5000, + ); + const row = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(row.rca_done, "resolved"); + assert.equal(row.related_prs, "#7421; #7430"); + assert.equal(row.confidence, "high"); + assert.equal(row.in_flight_worker, ""); + assert.equal(row.timestamp, "5000"); +}); + +test("reaper reclaims only stale in-flight rows", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); // stale + claim(csv, 102, "w2", 9000); // fresh + const ttl = 600; // seconds + const now = 1000 + ttl * 1000 + 1; // just past TTL for w1, fresh for w2 + const reclaimed = reaper(csv, ttl, now); + assert.deepEqual(reclaimed, ["101"]); + const rows = readRows(csv); + assert.equal(rows.find((r) => r.testRunId === "101").in_flight_worker, ""); + assert.equal(rows.find((r) => r.testRunId === "101").rca_done, PENDING); + assert.equal(rows.find((r) => r.testRunId === "102").in_flight_worker, "w2"); +}); + +test("reaper leaves terminal rows alone even if in_flight lingered", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + flip(csv, 101, { rca_done: "resolved" }, 2000); // flip clears in_flight + const reclaimed = reaper(csv, 600, 10_000_000); + assert.deepEqual(reclaimed, []); +}); + +test("pendingRows returns only pending work", () => { + seed(csv, "build-1", TESTS); + flip(csv, 101, { rca_done: "resolved" }, 1000); + const pend = pendingRows(csv); + assert.equal(pend.length, 1); + assert.equal(pend[0].testRunId, "102"); +}); + +// Regression: `flip` used to accept ONLY the lowercase CSV vocabulary and +// return a bare `false` for anything else — including `RESOLVED`, the exact +// value the RCA_OUTPUT contract mandates. A whole batch of coordinator results +// was lost that way: they called flip, got a silent no-op, and the rows stayed +// `pending` looking un-run. +test("flip accepts the RCA_OUTPUT vocabulary and normalizes it", () => { + seed(csv, "build-1", TESTS); + assert.equal(flip(csv, 101, { rca_done: "RESOLVED", root_cause: "x" }, 1000), true); + assert.equal(readRows(csv).find((r) => r.testRunId === "101").rca_done, "resolved"); + + assert.equal(flip(csv, 102, { status: "PENDING" }, 1000), true); + assert.equal(readRows(csv).find((r) => r.testRunId === "102").rca_done, "pending-resume"); +}); + +test("flip maps the output block's field names onto real columns", () => { + seed(csv, "build-1", TESTS); + flip(csv, 101, { rca_done: "resolved", thread_id: "chat:101", turn_id: "t-7" }, 1000); + const row = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(row.threadId, "chat:101"); + assert.equal(row.turnId, "t-7"); +}); + +test("flip rejects a missing/non-terminal rca_done without mutating the row", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + // missing rca_done + assert.equal(flip(csv, 101, { root_cause: "x" }, 2000), false); + // invalid rca_done + assert.equal(flip(csv, 101, { rca_done: "weird" }, 2000), false); + const row = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(row.rca_done, PENDING); // not reverted to claimable-pending silently + assert.equal(row.in_flight_worker, "w1"); // claim intact — bug surfaces, no clobber + assert.equal(row.root_cause, ""); // nothing written +}); + +test("pending-resume is resumable: not terminal, listed, and re-claimable", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + flip(csv, 101, { rca_done: "pending-resume", threadId: "thr-1", turnId: "t-1" }, 2000); + const row = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(row.in_flight_worker, ""); // this attempt released the claim + assert.equal(row.threadId, "thr-1"); // resume handles retained + assert.equal(row.turnId, "t-1"); + // appears in the fan-out work-list and can be claimed by the resume pass + assert.ok(pendingRows(csv).some((r) => r.testRunId === "101")); + assert.equal(claim(csv, 101, "w2", 3000), true); +}); + +test("reaper ignores pending-resume rows (not in flight)", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + flip(csv, 101, { rca_done: "pending-resume" }, 2000); + assert.deepEqual(reaper(csv, 600, 10_000_000), []); +}); + +test("CSV codec round-trips fields with commas, quotes, newlines", () => { + seed(csv, "build-1", [{ test_id: 200, test_name: "weird" }]); + flip( + csv, + 200, + { rca_done: "resolved", root_cause: 'Failed: "x", got <y>\nsecond line' }, + 1000, + ); + const row = readRows(csv).find((r) => r.testRunId === "200"); + assert.equal(row.root_cause, 'Failed: "x", got <y>\nsecond line'); +}); + +test("csvPathFor: build id is in the filename, default dir is OS temp", () => { + const p = csvPathFor("abc123XYZ"); + assert.ok(p.startsWith(join(tmpdir(), "bstack-rca"))); + assert.ok(p.endsWith("rca-state.abc123XYZ.csv")); +}); + +test("csvPathFor: different builds never share a path", () => { + assert.notEqual(csvPathFor("build-A"), csvPathFor("build-B")); +}); + +test("csvPathFor: sanitizes hostile ids and handles empty", () => { + assert.ok(csvPathFor("../../etc/passwd").endsWith("rca-state..._.._etc_passwd.csv")); + assert.ok(csvPathFor("").endsWith("rca-state.unknown-build.csv")); +}); + +test("csvPathFor: stateDir override wins over temp", () => { + const p = csvPathFor("b1", "/ci/artifacts"); + assert.equal(p, join("/ci/artifacts", "rca-state.b1.csv")); +}); diff --git a/tests/evidence-file.test.mjs b/tests/evidence-file.test.mjs new file mode 100644 index 0000000..e00cc2c --- /dev/null +++ b/tests/evidence-file.test.mjs @@ -0,0 +1,298 @@ +import { test, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, writeFileSync, statSync, chmodSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + evidencePathFor, + emptyEvidenceFile, + initEvidenceFile, + readEvidenceFile, + writeEvidenceFile, + setBaseline, + setGithubEvidence, + setLogsEvidence, + contributeGithubEvidence, + contributeLogsEvidence, + contribDirFor, + contribPathFor, + readBaseFile, + hasTrustworthyPrList, + recomputeCoverage, +} from "../lib/evidence-file.mjs"; + +let dir; +let file; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "rca-evidence-")); + file = join(dir, "evidence.json"); +}); +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +test("evidencePathFor: build id is in the filename, default dir is OS temp", () => { + const p = evidencePathFor("abc123XYZ"); + assert.ok(p.startsWith(join(tmpdir(), "bstack-rca"))); + assert.ok(p.endsWith("rca-evidence.abc123XYZ.json")); +}); + +test("evidencePathFor: different builds never share a path", () => { + assert.notEqual(evidencePathFor("build-A"), evidencePathFor("build-B")); +}); + +test("evidencePathFor: sanitizes hostile ids and handles empty", () => { + assert.ok(evidencePathFor("../../etc/passwd").endsWith("rca-evidence..._.._etc_passwd.json")); + assert.ok(evidencePathFor("").endsWith("rca-evidence.unknown-build.json")); +}); + +test("evidencePathFor: stateDir override wins over temp", () => { + const p = evidencePathFor("b1", "/ci/artifacts"); + assert.equal(p, join("/ci/artifacts", "rca-evidence.b1.json")); +}); + +test("readEvidenceFile on a missing path returns the empty shape, never throws", () => { + const doc = readEvidenceFile(file); + assert.deepEqual(doc, emptyEvidenceFile("unknown-build", 0)); +}); + +test("initEvidenceFile creates the file with the given buildId", () => { + const doc = initEvidenceFile(file, "build-1", 1000); + assert.equal(doc.buildId, "build-1"); + assert.equal(doc.generatedAtMs, 1000); + assert.deepEqual(readEvidenceFile(file), doc); +}); + +test("initEvidenceFile is idempotent — does not clobber an existing file", () => { + initEvidenceFile(file, "build-1", 1000); + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "x" } }, 2000); + const before = readEvidenceFile(file); + const again = initEvidenceFile(file, "build-1", 9999); + assert.deepEqual(again, before); +}); + +test("setGithubEvidence and setLogsEvidence coexist without clobbering each other", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "a-deploy" } }, 1000); + setLogsEvidence(file, "workload-1", { gap: null, kubectlSweep: { block: "w1-logs" } }, 1000); + const doc = readEvidenceFile(file); + assert.equal(doc.github["org/a"].deployState.block, "a-deploy"); + assert.equal(doc.logs["workload-1"].kubectlSweep.block, "w1-logs"); +}); + +test("setGithubEvidence for a second repo does not disturb the first", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "a" } }, 1000); + setGithubEvidence(file, "org/b", { gap: null, deployState: { block: "b" } }, 1000); + const doc = readEvidenceFile(file); + assert.equal(doc.github["org/a"].deployState.block, "a"); + assert.equal(doc.github["org/b"].deployState.block, "b"); +}); + +test("setGithubEvidence twice for the SAME repo overwrites only that repo", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "old" } }, 1000); + setGithubEvidence(file, "org/b", { gap: null, deployState: { block: "b" } }, 1000); + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "new" } }, 2000); + const doc = readEvidenceFile(file); + assert.equal(doc.github["org/a"].deployState.block, "new"); + assert.equal(doc.github["org/b"].deployState.block, "b"); // untouched +}); + +test("setBaseline records baseline and suspectWindow without touching github/logs", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "a" } }, 1000); + setBaseline(file, { ref: "sha123", isFallback: false }, { reposRequested: ["org/a"] }, 2000); + const doc = readEvidenceFile(file); + assert.deepEqual(doc.baseline, { ref: "sha123", isFallback: false }); + assert.deepEqual(doc.suspectWindow, { reposRequested: ["org/a"] }); + assert.equal(doc.github["org/a"].deployState.block, "a"); // untouched +}); + +test("recomputeCoverage: a covered repo/workload has no gap; a missing one is gapped", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "a" } }, 1000); + setLogsEvidence(file, "w1", { gap: null, kubectlSweep: { block: "w1" } }, 1000); + const coverage = recomputeCoverage( + file, + { repos: ["org/a", "org/b"], workloads: ["w1", "w2"] }, + 2000, + ); + assert.deepEqual(coverage.reposCovered, ["org/a"]); + assert.deepEqual(coverage.reposGapped, ["org/b"]); + assert.deepEqual(coverage.workloadsCovered, ["w1"]); + assert.deepEqual(coverage.workloadsGapped, ["w2"]); +}); + +test("recomputeCoverage: a present entry with a non-null gap is NOT covered", () => { + setGithubEvidence(file, "org/a", { gap: "gh auth failed for this repo" }, 1000); + const coverage = recomputeCoverage(file, { repos: ["org/a"], workloads: [] }, 2000); + assert.deepEqual(coverage.reposCovered, []); + assert.deepEqual(coverage.reposGapped, ["org/a"]); +}); + +test("recomputeCoverage persists onto the file (readable afterwards)", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "a" } }, 1000); + recomputeCoverage(file, { repos: ["org/a"], workloads: [] }, 2000); + const doc = readEvidenceFile(file); + assert.deepEqual(doc.coverage.reposCovered, ["org/a"]); +}); + +test("a block string with newlines and quotes round-trips through JSON unchanged", () => { + const block = 'ASK: did X change?\nTYPE: product_code\nFOUND: yes\nSUMMARY: "quoted" finding\nSNIPPET: line1\nline2'; + setGithubEvidence(file, "org/a", { gap: null, deployState: { block } }, 1000); + const doc = readEvidenceFile(file); + assert.equal(doc.github["org/a"].deployState.block, block); +}); + +test("contribute writes a shard, never the base file", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "base" } }, 1000); + contributeGithubEvidence(file, "3895581484", "org/a", { + deployState: { block: "coordinator's full diff" }, + }, 2000); + // base is untouched... + assert.equal(readBaseFile(file).github["org/a"].deployState.block, "base"); + // ...but the folded view shows the contribution + assert.equal(readEvidenceFile(file).github["org/a"].deployState.block, "coordinator's full diff"); +}); + +test("contribPathFor: one file per writer, under the build's contrib dir", () => { + const p = contribPathFor(file, "3895581484"); + assert.ok(p.startsWith(contribDirFor(file))); + assert.ok(p.endsWith("3895581484.json")); + assert.notEqual(contribPathFor(file, "w1"), contribPathFor(file, "w2")); +}); + +test("contribPathFor sanitizes a hostile writerId", () => { + assert.ok(contribPathFor(file, "../../etc/passwd").endsWith("_.._etc_passwd.json")); +}); + +test("CONCURRENCY: two writers on the same repo both survive (no lost update)", () => { + setGithubEvidence(file, "org/a", { + gap: null, deployState: { block: "base" }, prsInWindow: [{ pr: "#1" }], + }, 1000); + // Interleave the two writers the way real concurrent coordinators would: + // each reads, then each writes — under a single shared file this is exactly + // the sequence that drops the first writer's update. + contributeGithubEvidence(file, "writerA", "org/a", { prsInWindow: [{ pr: "#2", by: "A" }] }, 2000); + contributeGithubEvidence(file, "writerB", "org/a", { prsInWindow: [{ pr: "#3", by: "B" }] }, 2000); + const prs = readEvidenceFile(file).github["org/a"].prsInWindow.map((p) => p.pr).sort(); + assert.deepEqual(prs, ["#1", "#2", "#3"]); // base + BOTH contributions +}); + +test("CONCURRENCY: two writers on the same workload both survive", () => { + contributeLogsEvidence(file, "writerA", "w1", { kubectlSweep: { block: "A found 3 lines" } }, 1000); + contributeLogsEvidence(file, "writerB", "w1", { victorialogs: { block: "B found 5xx" } }, 1000); + const w = readEvidenceFile(file).logs["w1"]; + assert.equal(w.kubectlSweep.block, "A found 3 lines"); + assert.equal(w.victorialogs.block, "B found 5xx"); +}); + +test("fold: real contributed evidence beats a base-recorded gap", () => { + setGithubEvidence(file, "org/a", { gap: "gh auth failed" }, 1000); + contributeGithubEvidence(file, "w1", "org/a", { + gap: null, deployState: { block: "reachable after all" }, + }, 2000); + const entry = readEvidenceFile(file).github["org/a"]; + assert.equal(entry.gap, null); + assert.equal(entry.deployState.block, "reachable after all"); +}); + +test("fold: a contributed gap does NOT overwrite real base evidence", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "real base evidence" } }, 1000); + contributeGithubEvidence(file, "w1", "org/a", { deployState: { gap: "my call failed" } }, 2000); + assert.equal(readEvidenceFile(file).github["org/a"].deployState.block, "real base evidence"); +}); + +test("fold: same PR number contributed later wins (deeper finding replaces placeholder)", () => { + setGithubEvidence(file, "org/a", { + gap: null, prsInWindow: [{ pr: "#9011", verdict: "unassessed", files: null }], + }, 1000); + contributeGithubEvidence(file, "w1", "org/a", { + prsInWindow: [{ pr: "#9011", verdict: "supported", files: ["Foo.java"] }], + }, 2000); + const prs = readEvidenceFile(file).github["org/a"].prsInWindow; + assert.equal(prs.length, 1); + assert.equal(prs[0].verdict, "supported"); +}); + +test("fold: contributing a repo the pre-fetch never named", () => { + contributeGithubEvidence(file, "w1", "org/brand-new", { + prsInWindow: [{ pr: "#8912", verdict: "supported" }], + }, 1000); + assert.equal(readEvidenceFile(file).github["org/brand-new"].prsInWindow[0].pr, "#8912"); +}); + +test("fold: clusterIds union across base and multiple shards", () => { + setLogsEvidence(file, "w1", { gap: null, clusterIds: ["c-A"], kubectlSweep: { block: "x" } }, 1000); + contributeLogsEvidence(file, "w1writer", "w1", { clusterIds: ["c-B"] }, 2000); + contributeLogsEvidence(file, "w2writer", "w1", { clusterIds: ["c-C"] }, 2000); + assert.deepEqual(readEvidenceFile(file).logs["w1"].clusterIds.sort(), ["c-A", "c-B", "c-C"]); +}); + +test("fold: a corrupt shard is skipped, not fatal", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "base" } }, 1000); + contributeGithubEvidence(file, "good", "org/a", { prsInWindow: [{ pr: "#2" }] }, 2000); + writeFileSync(contribPathFor(file, "corrupt"), "{not json", "utf8"); + const doc = readEvidenceFile(file); // must not throw + assert.equal(doc.github["org/a"].prsInWindow[0].pr, "#2"); +}); + +test("recomputeCoverage counts a coordinator-filled gap as covered", () => { + setGithubEvidence(file, "org/a", { gap: "unreachable at pre-fetch time" }, 1000); + let cov = recomputeCoverage(file, { repos: ["org/a"], workloads: [] }, 2000); + assert.deepEqual(cov.reposGapped, ["org/a"]); + contributeGithubEvidence(file, "w1", "org/a", { gap: null, deployState: { block: "got it" } }, 3000); + cov = recomputeCoverage(file, { repos: ["org/a"], workloads: [] }, 4000); + assert.deepEqual(cov.reposCovered, ["org/a"]); + assert.deepEqual(cov.reposGapped, []); +}); + +// Regression: an empty prsInWindow with gap:null used to read as "searched, +// found none" when it may simply never have been populated. Observed live — +// a file asserted 0 PRs for a repo that actually had 21, which would have let +// a coordinator conclude "no culprit PR" with false confidence. +test("empty prsInWindow is NOT coverage unless the search is recorded", () => { + setGithubEvidence(file, "org/never-searched", { gap: null, deployState: { block: "d" }, prsInWindow: [] }, 1000); + setGithubEvidence(file, "org/searched-empty", { gap: null, deployState: { block: "d" }, prsInWindow: [], prsSearched: true }, 1000); + const cov = recomputeCoverage(file, { repos: ["org/never-searched", "org/searched-empty"], workloads: [] }, 2000); + // Both repos ARE covered (each has deploy state) — but only one has a PR + // list safe to read as "no PRs in window". + assert.deepEqual(cov.reposCovered.sort(), ["org/never-searched", "org/searched-empty"]); + assert.deepEqual(cov.reposWithUntrustedPrList, ["org/never-searched"]); +}); + +test("hasTrustworthyPrList distinguishes searched-empty from never-populated", () => { + setGithubEvidence(file, "org/a", { gap: null, prsInWindow: [] }, 1000); + setGithubEvidence(file, "org/b", { gap: null, prsInWindow: [], prsSearched: true }, 1000); + setGithubEvidence(file, "org/c", { gap: null, prsInWindow: [{ pr: "#1" }] }, 1000); + const doc = readEvidenceFile(file); + assert.equal(hasTrustworthyPrList(doc, "org/a"), false); + assert.equal(hasTrustworthyPrList(doc, "org/b"), true); + assert.equal(hasTrustworthyPrList(doc, "org/c"), true); +}); + +test("contributing a PR list records that the search actually ran", () => { + contributeGithubEvidence(file, "w1", "org/a", { prsInWindow: [] }, 1000); + assert.equal(hasTrustworthyPrList(readEvidenceFile(file), "org/a"), true); +}); + +test("prsSearched is sticky — a later non-searching contributor cannot downgrade it", () => { + setGithubEvidence(file, "org/a", { gap: null, prsInWindow: [{ pr: "#1" }], prsSearched: true }, 1000); + contributeGithubEvidence(file, "w1", "org/a", { deployState: { block: "just deploy info" } }, 2000); + assert.equal(readEvidenceFile(file).github["org/a"].prsSearched, true); +}); + +test("a pre-existing loose-mode file is tightened to 0600 on the next write", () => { + writeEvidenceFile(file, emptyEvidenceFile("b", 0)); + chmodSync(file, 0o644); // simulate a file left by a pre-hardening run + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "x" } }, 1000); + assert.equal(statSync(file).mode & 0o777, 0o600); +}); + +test("evidence file and contribution shards are owner-only (0600)", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "private PR detail" } }, 1000); + contributeGithubEvidence(file, "w1", "org/a", { prsInWindow: [{ pr: "#1" }] }, 2000); + assert.equal(statSync(file).mode & 0o777, 0o600); + assert.equal(statSync(contribPathFor(file, "w1")).mode & 0o777, 0o600); +}); + +test("writeEvidenceFile creates the parent directory if missing", () => { + const nested = join(dir, "nested", "sub", "evidence.json"); + writeEvidenceFile(nested, emptyEvidenceFile("build-1", 0)); + assert.deepEqual(readEvidenceFile(nested).buildId, "build-1"); +}); diff --git a/tests/evidence.test.mjs b/tests/evidence.test.mjs new file mode 100644 index 0000000..041c694 --- /dev/null +++ b/tests/evidence.test.mjs @@ -0,0 +1,77 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { buildManifest, unavailableCapabilities } from "../lib/routing.mjs"; +import { makeEvidenceCache, resolveBaseline } from "../lib/evidence-cache.mjs"; + +const CONFIG = { + evidenceRouting: { + test_logs: { owner: "tfa", skip: true }, + product_code: { capability: "github" }, + deploy: { capability: "github" }, + infra: { capability: "infra" }, + k8s: { capability: "infra" }, + metrics: { capability: "metrics" }, + other: { capability: "other" }, + }, +}; + +test("buildManifest marks discovered capabilities available with via", () => { + const manifest = buildManifest(CONFIG, [ + { capability: "github", via: "github-mcp" }, + ]); + assert.equal(manifest.github.available, true); + assert.equal(manifest.github.via, "github-mcp"); + assert.equal(manifest.infra.available, false); +}); + +test("buildManifest excludes the TFA-owned test_logs capability", () => { + const manifest = buildManifest(CONFIG, []); + assert.ok(!("undefined" in manifest)); + assert.ok(!Object.keys(manifest).includes("test_logs")); +}); + +test("buildManifest dedupes capabilities shared by multiple evidence types", () => { + // product_code + deploy both map to github → one manifest entry + const manifest = buildManifest(CONFIG, [{ capability: "github" }]); + assert.equal(Object.keys(manifest).filter((k) => k === "github").length, 1); +}); + +test("unavailableCapabilities lists what the client can't get", () => { + const manifest = buildManifest(CONFIG, [{ capability: "github" }]); + const unavailable = unavailableCapabilities(manifest).sort(); + assert.deepEqual(unavailable, ["infra", "metrics", "other"]); +}); + +test("evidence cache computes once and reuses across calls", async () => { + const cache = makeEvidenceCache(); + let calls = 0; + const fn = async () => { + calls++; + return { prs: ["#1"] }; + }; + const a = await cache.compute("repo", "abc..def", "deploy", fn); + const b = await cache.compute("repo", "abc..def", "deploy", fn); + assert.equal(calls, 1); + assert.deepEqual(a, b); + assert.equal(cache.size(), 1); +}); + +test("evidence cache key distinguishes commit ranges", async () => { + const cache = makeEvidenceCache(); + let calls = 0; + const fn = async () => ++calls; + await cache.compute("repo", "r1", "deploy", fn); + await cache.compute("repo", "r2", "deploy", fn); + assert.equal(calls, 2); +}); + +test("resolveBaseline uses last-green when present, else flags fallback", () => { + assert.deepEqual(resolveBaseline("v1.2.3", "main"), { + ref: "v1.2.3", + isFallback: false, + }); + assert.deepEqual(resolveBaseline(null, "main"), { + ref: "main", + isFallback: true, + }); +}); diff --git a/tests/fixtures/recorded-turns/pending.json b/tests/fixtures/recorded-turns/pending.json new file mode 100644 index 0000000..2e73ce7 --- /dev/null +++ b/tests/fixtures/recorded-turns/pending.json @@ -0,0 +1,11 @@ +{ + "name": "soft-pending — resumable (trimmed shape: status/threadId/turnId only)", + "testRunId": 81, + "turns": [ + { + "status": "PENDING", + "threadId": "thr-81", + "turnId": "turn-81-1" + } + ] +} diff --git a/tests/fixtures/recorded-turns/resolved.json b/tests/fixtures/recorded-turns/resolved.json new file mode 100644 index 0000000..a7964b8 --- /dev/null +++ b/tests/fixtures/recorded-turns/resolved.json @@ -0,0 +1,41 @@ +{ + "name": "needs_info → evidence → resolved (trimmed terminal glimpse)", + "testRunId": 39, + "turns": [ + { + "status": "NEEDS_INFO", + "confidence": "low", + "threadId": "thr-39", + "questions": [ + "Did the buildName validator change?" + ], + "asks": [ + { + "what": "Did request-validation on POST /builds change since last green?", + "why": "the failing test posts an empty buildName", + "evidenceType": "product_code", + "priority": "high" + }, + { + "what": "Full run logs for test 39", + "why": "to read the failure", + "evidenceType": "test_logs", + "priority": "high" + } + ] + }, + { + "status": "RESOLVED", + "confidence": "high", + "threadId": "thr-39", + "glimpse": { + "root_cause": "PR #7421 tightened the buildName validator to reject empty strings", + "failure_type": "product_regression", + "related_prs": [ + "#7421" + ] + }, + "viewRca": "https://automation.browserstack.com — open the build's AI report (tab=ai_report, subTab=aitfa) to view the full RCA" + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/recorded-turns/soft-pending-drain.json b/tests/fixtures/recorded-turns/soft-pending-drain.json new file mode 100644 index 0000000..35a7b41 --- /dev/null +++ b/tests/fixtures/recorded-turns/soft-pending-drain.json @@ -0,0 +1,50 @@ +{ + "name": "soft-PENDING drained via getTfaTurnResult — recorded from test run 3840238857, whose first turn finalized NEEDS_INFO at 104s (past the tool's 90s in-call poll cap), so tfaRcaTurn returned a soft PENDING while the agent was already committed", + "testRunId": 3840238857, + "turns": [ + { + "status": "PENDING", + "threadId": "chat:3840238857", + "turnId": "c2e1a6fd-2243-4f93-bc69-62f298db062c" + }, + { + "status": "RESOLVED", + "confidence": "medium", + "threadId": "chat:3840238857", + "glimpse": { + "root_cause": "#current-url was removed from html-tags.html, so the url-normalisation script threw before healing ran", + "failure_type": "product_regression", + "related_prs": ["#8814"] + }, + "viewRca": "https://automation.browserstack.com/dashboard/v2/builds/x/tests/3840238857" + } + ], + "reads": [ + { "status": "PENDING", "threadId": "chat:3840238857", "turnId": "c2e1a6fd-2243-4f93-bc69-62f298db062c" }, + { "status": "PENDING", "threadId": "chat:3840238857", "turnId": "c2e1a6fd-2243-4f93-bc69-62f298db062c" }, + { + "status": "NEEDS_INFO", + "confidence": "medium", + "threadId": "chat:3840238857", + "questions": [ + "Was the `#current-url` element recently removed or renamed on the `htmlforms` / `html-tags.html` page?" + ], + "asks": [ + { + "what": "The source code for the page html-tags.html and any recent changes to it.", + "why": "Logs show repeated \"Cannot set properties of null (setting 'textContent')\" on #current-url.", + "evidenceType": "product_code", + "priority": "high" + }, + { + "what": "The run's terminal logs.", + "why": "To confirm the JS error ordering.", + "evidenceType": "test_logs", + "priority": "low" + } + ], + "suggestions": [], + "hypotheses": [] + } + ] +} diff --git a/tests/fixtures/recorded-turns/turn-cap.json b/tests/fixtures/recorded-turns/turn-cap.json new file mode 100644 index 0000000..4638b61 --- /dev/null +++ b/tests/fixtures/recorded-turns/turn-cap.json @@ -0,0 +1,12 @@ +{ + "name": "turn-cap — never resolves", + "testRunId": 99, + "turns": [ + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] } + ] +} diff --git a/tests/routing.test.mjs b/tests/routing.test.mjs new file mode 100644 index 0000000..6ebdb9b --- /dev/null +++ b/tests/routing.test.mjs @@ -0,0 +1,81 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { routeAsk, routeAsks, orderAsks, TEST_LOGS } from "../lib/routing.mjs"; + +const CONFIG = { + evidenceRouting: { + test_logs: { owner: "tfa", skip: true }, + product_code: { capability: "github", discoveryHints: ["github-mcp", "gh"] }, + infra: { capability: "infra", discoveryHints: [] }, + k8s: { capability: "infra", discoveryHints: [] }, + other: { capability: "other", discoveryHints: [] }, + }, +}; + +test("test_logs is always skipped (TFA-owned)", () => { + const r = routeAsk({ evidenceType: TEST_LOGS, priority: "high" }, CONFIG, { + github: { available: true }, + }); + assert.equal(r.action, "skip"); + assert.equal(r.reason, "tfa-owned"); +}); + +test("available capability → gather, carrying via", () => { + const r = routeAsk({ evidenceType: "product_code", priority: "high" }, CONFIG, { + github: { available: true, via: "github-mcp" }, + }); + assert.equal(r.action, "gather"); + assert.equal(r.capability, "github"); + assert.equal(r.via, "github-mcp"); +}); + +test("unavailable capability → gap, carrying discovery hints", () => { + const r = routeAsk({ evidenceType: "k8s", priority: "medium" }, CONFIG, { + infra: { available: false }, + }); + assert.equal(r.action, "gap"); + assert.equal(r.capability, "infra"); + assert.equal(r.reason, "no-capability"); +}); + +test("capability absent from manifest entirely → gap", () => { + const r = routeAsk({ evidenceType: "k8s", priority: "low" }, CONFIG, {}); + assert.equal(r.action, "gap"); +}); + +test("unknown evidenceType falls back to the 'other' entry", () => { + const r = routeAsk({ evidenceType: "weird", priority: "low" }, CONFIG, { + other: { available: true, via: "best-effort" }, + }); + assert.equal(r.action, "gather"); + assert.equal(r.capability, "other"); +}); + +test("orderAsks sorts high → medium → low, unknown last", () => { + const ordered = orderAsks([ + { what: "c", priority: "low" }, + { what: "a", priority: "high" }, + { what: "d", priority: undefined }, + { what: "b", priority: "medium" }, + ]); + assert.deepEqual( + ordered.map((a) => a.what), + ["a", "b", "c", "d"], + ); +}); + +test("routeAsks buckets a mixed turn in priority order", () => { + const buckets = routeAsks( + [ + { evidenceType: "k8s", priority: "low" }, + { evidenceType: "test_logs", priority: "high" }, + { evidenceType: "product_code", priority: "high" }, + ], + CONFIG, + { github: { available: true, via: "gh" } }, + ); + assert.equal(buckets.skip.length, 1); + assert.equal(buckets.gather.length, 1); + assert.equal(buckets.gap.length, 1); + assert.equal(buckets.gather[0].evidenceType, "product_code"); +}); diff --git a/tests/signature.test.mjs b/tests/signature.test.mjs new file mode 100644 index 0000000..f721167 --- /dev/null +++ b/tests/signature.test.mjs @@ -0,0 +1,79 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + normalize, + computeSignature, + selectRepresentative, + clusterRows, +} from "../lib/signature.mjs"; + +function row(id, extra = {}) { + return { + testRunId: String(id), + failure_category: "Assertion", + error_summary: "expected 200 but got 500", + file_path: "spec/login.rb", + is_flaky: "false", + ...extra, + }; +} + +test("normalize folds timestamps, uuids, hex, line:col, and numbers", () => { + assert.equal(normalize("Error at line :42:7"), "error at line :<line>"); + assert.equal(normalize("got 500 at 0xAF3"), "got <n> at <hex>"); + assert.equal( + normalize("failed 2026-06-23T10:00:00Z"), + "failed <ts>", + ); +}); + +test("identical category+error+path → same cluster", () => { + const { clusters } = clusterRows([row(1), row(2)]); + assert.equal(clusters.length, 1); + assert.equal(clusters[0].members.length, 2); +}); + +test("numbers in the error are folded so siblings still cluster", () => { + const a = row(1, { error_summary: "timeout after 3000ms on node-7" }); + const b = row(2, { error_summary: "timeout after 5000ms on node-2" }); + assert.equal(computeSignature(a), computeSignature(b)); + const { clusters } = clusterRows([a, b]); + assert.equal(clusters.length, 1); +}); + +test("distinct failures → distinct clusters", () => { + const a = row(1, { error_summary: "null pointer in Foo" }); + const b = row(2, { error_summary: "connection refused" }); + const { clusters } = clusterRows([a, b]); + assert.equal(clusters.length, 2); +}); + +test("rows with no signal become their own singletons (no catch-all merge)", () => { + const a = { testRunId: "1", failure_category: "", error_summary: "", file_path: "" }; + const b = { testRunId: "2", failure_category: "", error_summary: "", file_path: "" }; + const { clusters } = clusterRows([a, b]); + assert.equal(clusters.length, 2); + assert.ok(clusters.every((c) => c.cluster_id.startsWith("solo-"))); +}); + +test("singleton cluster has a representative and no siblings", () => { + const { clusters } = clusterRows([row(1)]); + assert.equal(clusters[0].siblings.length, 0); + assert.equal(clusters[0].representative.testRunId, "1"); +}); + +test("representative is deterministic: non-flaky, then smallest testRunId", () => { + const members = [ + row(5, { is_flaky: "true" }), + row(9, { is_flaky: "false" }), + row(7, { is_flaky: "false" }), + ]; + assert.equal(selectRepresentative(members).testRunId, "7"); +}); + +test("clusterRows stamps cluster_id onto every row", () => { + const rows = [row(1), row(2, { error_summary: "different" })]; + clusterRows(rows); + assert.ok(rows.every((r) => r.cluster_id)); + assert.notEqual(rows[0].cluster_id, rows[1].cluster_id); +}); diff --git a/tests/tool-cache.test.mjs b/tests/tool-cache.test.mjs new file mode 100644 index 0000000..56e22da --- /dev/null +++ b/tests/tool-cache.test.mjs @@ -0,0 +1,272 @@ +import { test, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, writeFileSync, statSync, readdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + toolCacheDirFor, cacheKey, mcpCacheKey, cacheGet, cachePut, cacheStats, + isCacheable, isCacheableMcp, isRunnable, redact, tokenize, splitPipeline, +} from "../lib/tool-cache.mjs"; + +let dir; +beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "rca-toolcache-")); }); +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +test("toolCacheDirFor: build id in the path, OS temp default, stateDir override", () => { + assert.ok(toolCacheDirFor("b1").startsWith(join(tmpdir(), "bstack-rca"))); + assert.ok(toolCacheDirFor("b1").endsWith("rca-toolcache.b1")); + assert.equal(toolCacheDirFor("b1", "/ci/art"), join("/ci/art", "rca-toolcache.b1")); + assert.ok(toolCacheDirFor("../../etc").endsWith("rca-toolcache..._.._etc")); +}); + +test("cacheKey: whitespace-insensitive, but content-sensitive", () => { + assert.equal(cacheKey("gh api repos/a"), cacheKey("gh api repos/a")); + assert.notEqual(cacheKey("gh api repos/a | head -20"), cacheKey("gh api repos/a | head -200")); +}); + +test("put then get round-trips", () => { + const k = cacheKey("gh api repos/a"); + cachePut(dir, k, { command: "gh api repos/a", writerId: "w1", stdout: "hello" }, 1000); + const hit = cacheGet(dir, k); + assert.equal(hit.stdout, "hello"); + assert.equal(hit.writerId, "w1"); + assert.equal(hit.capturedAtMs, 1000); +}); + +test("get on a miss returns null, never throws", () => { + assert.equal(cacheGet(dir, cacheKey("never run")), null); +}); + +test("a corrupt entry reads as a miss rather than throwing", () => { + const k = cacheKey("gh api repos/a"); + writeFileSync(join(dir, `${k}.json`), "{not json", "utf8"); + assert.equal(cacheGet(dir, k), null); +}); + +test("secrets are redacted before anything is written to disk", () => { + const k = cacheKey("gh api repos/a"); + cachePut(dir, k, { + command: "gh api repos/a", + stdout: 'ok\nAuthorization: Bearer abc123SECRET\napi_key=zzz999\ndone', + }, 1000); + const raw = cacheGet(dir, k).stdout; + assert.ok(!raw.includes("abc123SECRET"), "bearer token must not persist"); + assert.ok(!raw.includes("zzz999"), "api_key must not persist"); + assert.ok(raw.includes("<redacted>")); +}); + +test("redact leaves ordinary output untouched", () => { + assert.equal(redact("just some log output"), "just some log output"); +}); + +// Regression, found by a live coordinator. Redaction used to consume the REST +// OF THE LINE after a secret-ish key. GitHub's file API returns SINGLE-LINE +// JSON whose download_url always carries `?token=…`, so a 214KB response was +// silently cached as 816 bytes with the content field gone — every +// private-repo file fetch was corrupted, with no warning. +test("redact bounds the value and does NOT eat the rest of a single-line JSON", () => { + const json = '{"name":"F.java","download_url":"https://raw.example/F.java?token=BRFIJBHPIG5IILHZ",' + + '"type":"file","content":"' + "A".repeat(5000) + '"}'; + const out = redact(json); + assert.ok(!out.includes("BRFIJBHPIG5IILHZ"), "the token itself must be redacted"); + assert.ok(out.includes('"type":"file"'), "structure after the token must survive"); + assert.ok(out.includes("A".repeat(5000)), "the content payload must survive"); + assert.ok(out.length > 5000, `expected full payload, got ${out.length} bytes`); +}); + +test("redact still catches a bare Bearer token and a key=value secret", () => { + assert.equal(redact("Authorization: Bearer abc123SECRET"), "Authorization: <redacted>"); + assert.equal(redact("api_key=zzz999"), "api_key=<redacted>"); + assert.ok(!redact("Bearer eyJhbGciOiJIUzI1NiJ9").includes("eyJhbGciOiJIUzI1NiJ9")); +}); + +// Regression, found by two live coordinators. Neither tokenize nor +// splitPipeline handled backslash escapes, so `\"` read as a closing quote. +// That mangled jq's two most common idioms: string equality and, because the +// parser then believed it was outside quotes, regex alternation got split as +// a shell pipe. +test("escaped double quotes survive tokenization for jq", () => { + const argv = tokenize(String.raw`gh api x --jq .[]|select(.filename==\"a/b.json\")`); + assert.equal(argv[argv.length - 1], '.[]|select(.filename=="a/b.json")'); +}); + +test("a pipe inside an escaped-quote jq regex is NOT a shell pipe", () => { + const cmd = String.raw`gh pr view 51044 --json files | jq -c "select(test(\"vite|env|s3\";\"i\"))"`; + assert.deepEqual(splitPipeline(cmd).length, 2, "must split into fetch + one filter only"); + const g = isRunnable(cmd); + assert.equal(g.ok, true, g.reason); + assert.deepEqual(g.fetch, ["gh", "pr", "view", "51044", "--json", "files"]); + assert.equal(g.filters[0][2], 'select(test("vite|env|s3";"i"))'); +}); + +test("single quotes suppress escape processing, POSIX-style", () => { + assert.deepEqual(tokenize(String.raw`gh api 'a\nb'`), ["gh", "api", String.raw`a\nb`]); +}); + +test("oversized payloads are truncated and flagged", () => { + const k = cacheKey("gh api big"); + const rec = cachePut(dir, k, { command: "gh api big", stdout: "x".repeat(400 * 1024) }, 1000); + assert.equal(rec.truncated, true); + assert.ok(rec.stdout.includes("[truncated by tool-cache]")); +}); + +test("cacheStats counts entries", () => { + cachePut(dir, "k1", { command: "a", stdout: "12345" }, 1); + cachePut(dir, "k2", { command: "b", stdout: "123" }, 1); + const s = cacheStats(dir); + assert.equal(s.entries, 2); + assert.equal(s.bytes, 8); +}); + +test("isCacheable rejects mutating shell commands", () => { + assert.equal(isCacheable("gh api repos/a"), true); + assert.equal(isCacheable("kubectl get pods"), true); + assert.equal(isCacheable("kubectl delete pod x"), false); + assert.equal(isCacheable("kubectl exec pod -- sh"), false); + assert.equal(isCacheable("gh pr create --title x"), false); + assert.equal(isCacheable("gh api -X POST repos/a"), false); + assert.equal(isCacheable("git push origin main"), false); + assert.equal(isCacheable("rm -rf /tmp/x"), false); +}); + +test("isRunnable enforces an allowlisted read-only leader", () => { + assert.equal(isRunnable("gh api repos/a").ok, true); + assert.equal(isRunnable("kubectl get pods -n regression").ok, true); + assert.equal(isRunnable("python3 -c 'print(1)'").ok, false); + assert.equal(isRunnable("sh -c 'echo hi'").ok, false); +}); + +test("isRunnable rejects chaining and redirects, but ACCEPTS pipelines", () => { + assert.equal(isRunnable("gh api a ; rm -rf /").ok, false); + assert.equal(isRunnable("gh api a && kubectl delete pod x").ok, false); + assert.equal(isRunnable("gh api a > /etc/passwd").ok, false); + // `2>&1` is stderr plumbing the wrapper already owns — stripped, not refused. + // Refusing it rejected 134 of 223 real recorded calls and zeroed the hit rate. + assert.equal(isRunnable("gh api a 2>&1").ok, true, "stderr plumbing is normalized away"); + assert.equal(isRunnable("gh api a 2>/dev/null | jq .x").ok, true); + // Pipelines are supported now: refusing them meant the cache applied to + // almost no real traffic, since most fetches are written inline with a filter. + assert.equal(isRunnable("gh api a | jq .x").ok, true); +}); + +test("a FILE redirect is reported as a redirect, not as a mutation", () => { + const r = isRunnable("gh api repos/x > out.json"); + assert.equal(r.ok, false); + assert.match(r.reason, /redirect/i); + assert.doesNotMatch(r.reason, /mutating/i); +}); + +test("stderr plumbing does not change the cache key", () => { + const a = isRunnable("gh api repos/x | jq .a"); + const b = isRunnable("gh api repos/x 2>&1 | jq .b"); + assert.equal(cacheKey(a.fetchText), cacheKey(b.fetchText)); +}); + +test("pipeline plan: only the FETCH is keyed, filters are separate", () => { + const a = isRunnable("gh api repos/x | jq -r .name"); + const b = isRunnable("gh api repos/x | jq -r .branch | tr a-z A-Z"); + assert.equal(a.ok && b.ok, true); + // Same underlying fetch -> same cache key -> one network call serves both. + assert.equal(cacheKey(a.fetchText), cacheKey(b.fetchText)); + assert.deepEqual(a.fetch, ["gh", "api", "repos/x"]); + assert.equal(a.filters.length, 1); + assert.equal(b.filters.length, 2); +}); + +test("only pure text filters may follow the fetch", () => { + assert.equal(isRunnable("gh api repos/x | jq .a").ok, true); + assert.equal(isRunnable("gh api repos/x | grep foo").ok, true); + assert.equal(isRunnable("gh api repos/x | sh").ok, false); + assert.equal(isRunnable("gh api repos/x | bash -c 'x'").ok, false); + assert.equal(isRunnable("gh api repos/x | kubectl delete pod y").ok, false); +}); + +test("splitPipeline ignores a pipe inside quotes", () => { + assert.deepEqual(splitPipeline(`gh pr list --jq '.[] | .number' | head -5`), + ["gh pr list --jq '.[] | .number'", "head -5"]); +}); + +// Regression: the old raw-string guard refused these legitimate read-only +// calls, which is what pushed a coordinator into slower workarounds. +test("isRunnable ALLOWS metacharacters inside quoted arguments", () => { + const jqSemicolon = `gh api repos/o/r/git/trees/main --jq '[.tree[].path|select(test("rcaThree";"i"))]'`; + assert.equal(isRunnable(jqSemicolon).ok, true, "; inside a jq expression is not a shell operator"); + + const urlAmp = "gh api 'search/code?q=foo&per_page=20'"; + assert.equal(isRunnable(urlAmp).ok, true, "& inside a quoted URL is not a shell operator"); + + const jqPipe = `gh pr list -R o/r --json number --jq '.[] | .number'`; + assert.equal(isRunnable(jqPipe).ok, true, "| inside a quoted jq expression is not a shell pipe"); +}); + +test("a quoted metacharacter survives tokenization as ONE literal argument", () => { + const argv = tokenize(`gh api repos/o/r --jq '[.tree[]|select(test("x";"i"))]'`); + assert.equal(argv.length, 5); + assert.equal(argv[4], '[.tree[]|select(test("x";"i"))]'); +}); + +test("tokenize splits like a shell for quoted args, without a shell", () => { + assert.deepEqual(tokenize("gh api repos/a --jq '.items[].path'"), + ["gh", "api", "repos/a", "--jq", ".items[].path"]); + assert.deepEqual(tokenize('kubectl get pods -o "custom:.metadata.name"'), + ["kubectl", "get", "pods", "-o", "custom:.metadata.name"]); + assert.throws(() => tokenize("gh api 'unterminated"), /unterminated quote/); +}); + +test("tokenize keeps injection payloads as ONE literal argument", () => { + // With execFile + these argv, no shell ever sees the metacharacters. + const argv = tokenize(`gh api "repos/a;rm -rf /"`); + assert.deepEqual(argv, ["gh", "api", "repos/a;rm -rf /"]); +}); + +test("MCP: stateful tools are never cacheable", () => { + assert.equal(isCacheableMcp("mcp__grafana__query_loki_logs"), true); + assert.equal(isCacheableMcp("mcp__browserstack__listTestIds"), true); + assert.equal(isCacheableMcp("mcp__browserstack__tfaRcaTurn"), false); + assert.equal(isCacheableMcp("mcp__browserstack__getTfaTurnResult"), false); + assert.equal(isCacheableMcp("mcp__browserstack__triggerRcaReport"), false); +}); + +test("mcpCacheKey is argument-order independent but value sensitive", () => { + const a = mcpCacheKey("t", { b: 2, a: 1 }); + const b = mcpCacheKey("t", { a: 1, b: 2 }); + assert.equal(a, b); + assert.notEqual(a, mcpCacheKey("t", { a: 1, b: 3 })); + assert.notEqual(a, mcpCacheKey("other", { a: 1, b: 2 })); +}); + +test("mcpCacheKey canonicalizes nested objects and arrays", () => { + assert.equal( + mcpCacheKey("t", { q: { z: 1, y: [{ n: 1, m: 2 }] } }), + mcpCacheKey("t", { q: { y: [{ m: 2, n: 1 }], z: 1 } }), + ); +}); + +test("an MCP result round-trips through the shared store", () => { + const k = mcpCacheKey("mcp__grafana__query_loki_logs", { ns: "regression", limit: 50 }); + cachePut(dir, k, { command: "grafana query", writerId: "3889074893", stdout: "0 rows, clean" }, 1000); + assert.equal(cacheGet(dir, k).stdout, "0 rows, clean"); +}); + +test("cache files are owner-only (0600) and the dir owner-only (0700)", () => { + const sub = join(dir, "nested-cache"); + const k = cacheKey("gh api repos/a"); + cachePut(sub, k, { command: "gh api repos/a", stdout: "private repo source" }, 1000); + // The cache sits in a world-readable OS temp dir and holds raw gh/kubectl + // output; redaction is best-effort, so the mode is the real control. + assert.equal(statSync(join(sub, `${k}.json`)).mode & 0o777, 0o600); + assert.equal(statSync(sub).mode & 0o777, 0o700); +}); + +test("no temp file is left behind after an atomic put", () => { + const k = cacheKey("gh api repos/a"); + cachePut(dir, k, { command: "gh api repos/a", stdout: "x" }, 1000); + assert.deepEqual(readdirSync(dir).filter((f) => f.endsWith(".tmp")), []); +}); + +test("CONCURRENCY: same key written twice stays readable and consistent", () => { + const k = cacheKey("gh api repos/a"); + cachePut(dir, k, { command: "gh api repos/a", writerId: "w1", stdout: "same-bytes" }, 1000); + cachePut(dir, k, { command: "gh api repos/a", writerId: "w2", stdout: "same-bytes" }, 2000); + assert.equal(cacheGet(dir, k).stdout, "same-bytes"); +}); diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs new file mode 100644 index 0000000..4992bee --- /dev/null +++ b/workflows/rca-batch.mjs @@ -0,0 +1,166 @@ +export const meta = { + name: "rca-batch", + description: + "Drive autonomous collaborative RCA over all failed tests of a build: cluster representatives run the full loop, siblings one-turn-confirm, ~5 concurrent. Never prompts a user.", + phases: [ + { title: "Representatives", detail: "full multi-turn RCA per cluster" }, + { title: "Siblings", detail: "one-turn confirm against own logs" }, + ], +}; + +// The /rca-build batch orchestration (fully autonomous — the gate closed before +// this runs; nothing here ever asks the user). This is a dynamic-workflow +// script: it runs in the Workflow sandbox (no filesystem, no Date.now/ +// Math.random, agent()/pipeline() as globals). It therefore does NO state I/O +// itself — the orchestrator seeds the CSV, clusters, and builds the validated +// manifest at the gate and passes the work-list via `args`; each dispatched +// `ai-tfa-coordinator` agent (which HAS tool access) claims + flips its own CSV +// row eagerly (WAL); this script orchestrates concurrency and returns the +// structured results for reconciliation. The final glimpse + triggerRcaReport +// step happens back in the orchestrator (SKILL.md Step 6). +// +// args shape: +// { +// csvPath, buildId, +// manifest: { capability: { available, via } }, +// evidenceFilePath, // lib/evidence-file.mjs artifact for this build +// pluginRoot, // absolute path to this plugin, so coordinators can call bin/cached-exec.mjs +// buildEvidence: { baselineRef, isFallback, suspectWindow, reposCovered, workloadsCovered, gaps }, +// // ^ SHRUNK to a cheap summary/pointer only — the full PR list / log +// // sweeps live in the file at evidenceFilePath, read via each +// // coordinator's own Read tool. Repeating the full detail in every +// // dispatch prompt (as before) is exactly the duplication this file +// // removes. +// clusters: [ +// { cluster_id, representative: { testRunId, testName, error_summary }, +// siblings: [ { testRunId, testName, error_summary } ] } +// ] +// } + +const RCA_SCHEMA = { + type: "object", + required: ["testRunId", "status"], + properties: { + testRunId: { type: "string" }, + status: { enum: ["RESOLVED", "PENDING", "failed"] }, + confidence: { enum: ["high", "medium", "low", "unknown"] }, + root_cause: { type: "string" }, + failure_type: { type: "string" }, + view_rca: { type: "string" }, + related_prs: { type: "array", items: { type: "string" } }, + suspect_signals: { type: "array", items: { type: "string" } }, + threadId: { type: "string" }, + turnId: { type: "string" }, + turns_used: { type: "number" }, + asks_fulfilled: { type: "array", items: { type: "string" } }, + asks_skipped: { type: "array", items: { type: "string" } }, + asks_unavailable: { type: "array", items: { type: "string" } }, + cluster_id: { type: "string" }, + }, + additionalProperties: true, +}; + +const ctx = (typeof args === "string" ? JSON.parse(args) : args) ?? {}; +const clusters = ctx.clusters ?? []; +const shared = [ + `CSV state file: ${ctx.csvPath}`, + `Capability manifest: ${JSON.stringify(ctx.manifest ?? {})}`, + `Pre-fetched build-evidence file — READ THIS FIRST (via the Read tool) before making ANY live github/infra/logs gather call: ${ctx.evidenceFilePath}`, + `Build-evidence summary (full detail is in the file above; this is only a pointer — do not re-fetch what the file already covers): ${JSON.stringify(ctx.buildEvidence ?? {})}`, + `If the file's github/logs sections do not name a repo/workload/ask you need, or record a "gap" for it, that is a genuine gap — fall back to a live gather via the capability manifest above exactly as if no file existed. The file is an optimization, never a hard dependency.`, + `The file is read-write: after any live gather that fills a gap or goes deeper than what was there, write it back via contributeGithubEvidence/contributeLogsEvidence (lib/evidence-file.mjs) passing your own testRunId as writerId, before finishing this test — so a sibling dispatched after you, or another cluster sharing the same repo/workload, reads the enriched entry instead of re-fetching it. Each writer owns its own shard file, so concurrent coordinators cannot clobber each other; readers fold base + shards automatically.`, + `Tool cache — route read-only lookups through it so duplicate calls across coordinators become hits. Shell: node ${ctx.pluginRoot ?? "<pluginRoot>"}/bin/cached-exec.mjs <buildId> <yourTestRunId> '<gh|kubectl|curl|git command>' (behaves like the raw command; pipe to jq/grep OUTSIDE the wrapper so different filters share one fetch). MCP data queries: cached-mcp.mjs <buildId> get|put <tool> '<argsJson>' [writerId]. NEVER cache tfaRcaTurn/getTfaTurnResult/triggerRcaReport — they are stateful. Do not re-probe connectors the gate already validated.`, + `Autonomous run — on an evidence gap with no valid connector, report "unavailable" back to TFA (NEVER prompt a user). Best-effort finalize.`, + `PRODUCT_BUG / application-bug mandate: hunt the culprit PR via the github connector (deploy timeline vs last-pass window, changed paths vs failure signature) and feed the PR link(s) to TFA so related_prs populates. No PR after digging to the turn cap → state explicitly "no culprit PR identified after <what was searched>" so the CSV row records the gap.`, + `Soft-PENDING is NOT an answer: tfaRcaTurn abandons its in-call poll at 90s while TFA keeps working. On status PENDING, call getTfaTurnResult(testRunId, turnId) FIRST and keep reading on the softPendingDrain budget (every 5s, <=40 reads / <=10min) until the status is RESOLVED / NEEDS_INFO / BLOCKED, then continue the loop. Reads do NOT count against the turn cap. Never submit a new message onto a turn still in flight. Only a fully spent drain budget ends the test PENDING.`, + `Persist eagerly to the CSV: claim your row before turn 1, flip it on terminal (lib/csv-state.mjs).`, +].join("\n"); + +function resumeLine(row) { + if (!row?.threadId || !row?.turnId) return null; + return [ + `RESUME (do not start a new thread): this test already has an in-flight thread`, + `threadId=${row.threadId} turnId=${row.turnId}.`, + `Call getTfaTurnResult(testRunId, turnId) FIRST to read its current state`, + `(drain any soft-PENDING per the softPendingDrain budget) before submitting`, + `anything further — reuse this threadId for every follow-up on this test.`, + row.last_evidence_digest ? `Prior evidence already gathered (reuse, don't re-fetch): ${row.last_evidence_digest}` : null, + row.root_cause ? `Prior attempt note: ${row.root_cause}` : null, + ].filter(Boolean).join("\n"); +} + +function repPrompt(cluster) { + const r = cluster.representative; + return [ + `You are the ai-tfa-coordinator for cluster ${cluster.cluster_id}.`, + `Run the FULL collaborative RCA loop for the representative test.`, + `testRunId=${r.testRunId} testName=${r.testName ?? ""}`, + `error_digest: ${r.error_summary ?? "(none)"}`, + resumeLine(r), + shared, + `Return the structured RCA_OUTPUT for this test.`, + ].filter(Boolean).join("\n"); +} + +function siblingPrompt(sibling, repResult, cluster) { + return [ + `You are the ai-tfa-coordinator for a SIBLING of cluster ${cluster.cluster_id}.`, + `Pre-seed: the representative resolved as:`, + ` root_cause: ${repResult?.root_cause ?? "(representative did not resolve)"}`, + ` related_prs: ${JSON.stringify(repResult?.related_prs ?? [])}`, + `State this hypothesis on turn 1 and ask TFA to CONFIRM it against THIS test's own logs.`, + `The pre-fetched evidence file's data about your OWN workload is real evidence about YOUR OWN test — reading it is NOT blind inheritance. What must stay independent is the CONFIRMATION judgment: never adopt the representative's verdict just because the file already has the answer in it.`, + `If TFA confirms in one turn → done. If it does NOT (NEEDS_INFO), fall back to the full loop — never blindly inherit.`, + `testRunId=${sibling.testRunId} testName=${sibling.testName ?? ""}`, + `error_digest: ${sibling.error_summary ?? "(none)"}`, + resumeLine(sibling), + shared, + `Return the structured RCA_OUTPUT for this test.`, + ].filter(Boolean).join("\n"); +} + +log(`Batch: ${clusters.length} cluster(s) over build ${ctx.buildId ?? "?"}`); + +// Pipeline: each cluster flows representative → siblings independently (no barrier +// between stages), so a small cluster's siblings confirm while a big cluster's +// representative is still looping. Concurrency is capped by the Workflow runtime +// at min(16, cores-2) — an architectural limit of the tool itself, not something +// this script or config.concurrency (20, see rca.config.json) can raise. That +// config value is an intended soft target/upper bound on THIS path only; the +// runtime queues anything beyond its own cap regardless of what this file says. +const results = await pipeline( + clusters, + (cluster) => + agent(repPrompt(cluster), { + label: `rep:${cluster.representative.testRunId}`, + phase: "Representatives", + agentType: "tfa-rca:ai-tfa-coordinator", + schema: RCA_SCHEMA, + }).then((rca) => ({ cluster, rca })), + ({ cluster, rca }) => + parallel( + (cluster.siblings ?? []).map((sib) => () => + agent(siblingPrompt(sib, rca, cluster), { + label: `sib:${sib.testRunId}`, + phase: "Siblings", + agentType: "tfa-rca:ai-tfa-coordinator", + schema: RCA_SCHEMA, + }), + ), + ).then((sibs) => ({ + cluster_id: cluster.cluster_id, + representative: rca, + siblings: sibs.filter(Boolean), + })), +); + +const flat = results.filter(Boolean); +const all = flat.flatMap((r) => [r.representative, ...(r.siblings ?? [])]).filter(Boolean); +const byStatus = all.reduce((acc, r) => { + acc[r.status] = (acc[r.status] ?? 0) + 1; + return acc; +}, {}); + +log(`Batch complete: ${all.length} test(s) — ${JSON.stringify(byStatus)}`); + +return { clusters: flat.length, tests: all.length, byStatus, results: flat };