From 6934cf9b8a53113b822596d465fce7d8a9e43c76 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack Date: Tue, 23 Jun 2026 21:54:55 +0530 Subject: [PATCH 01/47] =?UTF-8?q?feat(rca):=20scaffold=20plugin=20?= =?UTF-8?q?=E2=80=94=20manifest,=20MCP=20wiring,=20config,=20command,=20RE?= =?UTF-8?q?ADME?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Identity-only .claude-plugin/plugin.json; root .mcp.json wires the bstack MCP server (stdio); config/rca.config.json centralizes all formerly-hardcoded product/infra values (no kubectl/chitragupta/bifrost literals); /rca-build command parses build id + mode and hands off to the skill. Co-Authored-By: Claude Opus 4.8 --- .claude-plugin/plugin.json | 11 +++++++ .env.example | 8 +++++ .gitignore | 4 +++ .mcp.json | 14 ++++++++ README.md | 67 ++++++++++++++++++++++++++++++++++++-- commands/rca-build.md | 34 +++++++++++++++++++ config/rca.config.json | 25 ++++++++++++++ package.json | 10 ++++++ 8 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 .claude-plugin/plugin.json create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 .mcp.json create mode 100644 commands/rca-build.md create mode 100644 config/rca.config.json create mode 100644 package.json diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..c8c4beb --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,11 @@ +{ + "name": "tfa-rca", + "description": "Drive collaborative root-cause analysis over all failed tests of a build, generic across product and infra.", + "version": "0.1.0", + "author": { + "name": "BrowserStack", + "url": "https://www.browserstack.com" + }, + "homepage": "https://github.com/browserstack/browserstack-ai-tfa-demo", + "license": "MIT" +} diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d86819e --- /dev/null +++ b/.env.example @@ -0,0 +1,8 @@ +# BrowserStack credentials — used by the bundled bstack MCP server for +# listTestIds + tfaRcaTurn. Per-user; never commit real values. +BROWSERSTACK_USERNAME= +BROWSERSTACK_ACCESS_KEY= + +# Observability base URL the TFA RCA chat runs against. Optional — +# the bstack MCP server defaults to its rengg-tfa staging URL when unset. +# O11Y_TFA_RCA_BASE_URL=https://api-observability-rengg-tfa.bsstag.com diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9045f9d --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +.env +# Per-run RCA batch state (the CSV/WAL spine + report) is workspace-local. +.rca/ diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..0502929 --- /dev/null +++ b/.mcp.json @@ -0,0 +1,14 @@ +{ + "mcpServers": { + "bstack": { + "type": "stdio", + "command": "npx", + "args": ["-y", "@browserstack/mcp-server"], + "env": { + "BROWSERSTACK_USERNAME": "${BROWSERSTACK_USERNAME}", + "BROWSERSTACK_ACCESS_KEY": "${BROWSERSTACK_ACCESS_KEY}", + "O11Y_TFA_RCA_BASE_URL": "${O11Y_TFA_RCA_BASE_URL}" + } + } + } +} diff --git a/README.md b/README.md index 423d780..ff148ab 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,65 @@ -# browserstack-ai-tfa-demo -AI TFA Demo +# tfa-rca — generic multi-client RCA agent plugin + +Drive BrowserStack's collaborative root-cause-analysis loop over **all failed +tests of a build**, generic across product and infra, from inside an agentic +MCP client (Claude Code / Cursor / Codex). + +The plugin wraps two stable MCP tools — `listTestIds` and `tfaRcaTurn` (from the +`bstack` MCP server) — and adds the harness that batches RCA over a whole build, +clusters failures by signature, routes evidence requests to whatever +skills/tools the client already has, and writes a per-test RCA into the TRA +dashboard. + +> It **discovers and delegates** to the infra skills/tools already in your +> client (GitHub, k8s/EKS, kibana/other logs, metrics). It does **not** install +> or own those connectors. + +## Install + +```bash +git clone https://github.com/browserstack/browserstack-ai-tfa-demo.git +cd browserstack-ai-tfa-demo +cp .env.example .env # fill in BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY +claude --plugin-dir ./ +``` + +The plugin auto-configures on load: the `bstack` MCP server (from `.mcp.json`), +the `/rca-build` command, the `rca-build` skill, and the `ai-tfa-coordinator` +agent are all discovered by convention. + +## Usage + +``` +/rca-build +/rca-build build_id= mode=auto +``` + +On start the plugin runs a **mandatory pre-flight intake** asking for your +product + automation repos, working branch, default branch, and the PRs in +play, plus the build id. Every question is answerable with "I don't have one" → +the run proceeds RCA-only. + +## Modes + +- **auto** — a dynamic workflow drives the whole batch (5 tests concurrent), no + mid-run prompts. When evidence can't be gathered (no matching skill), it + reports "unavailable" back to the TFA agent, which finalizes best-effort. +- **interactive** — the main session spawns subagents (5 at a time); on an + evidence gap a subagent returns the gap to the main agent, which asks you, + then feeds the answer back. + +`auto` means autonomy *during* the batch from an interactive session — not +headless. Running `claude -p` with a required input missing ends immediately. + +## Requirements + +- The `bstack` MCP server (bundled via `.mcp.json`). +- Credentials in `.env` (or your client's MCP env). +- For full evidence coverage: whatever GitHub / infra / logging / metrics + skills your client already has. Missing ones degrade gracefully (the RCA's + confidence band reflects what evidence was actually available). + +## Layout + +See `docs/plans/2026-06-23-001-feat-generic-rca-agent-plugin-plan.md` for the +implementation plan and `docs/brainstorms/` for the requirements. diff --git a/commands/rca-build.md b/commands/rca-build.md new file mode 100644 index 0000000..7a7a829 --- /dev/null +++ b/commands/rca-build.md @@ -0,0 +1,34 @@ +--- +description: Run collaborative RCA over all failed tests of a BrowserStack build +--- + +# /rca-build + +Entry point for the generic RCA harness. Drives a collaborative root-cause +analysis loop over **every failed test** of a build, generic across product and +infra. + +## Input + +`$ARGUMENTS` carries the build id (and optional flags). Accepted forms: + +- bare build id: `qzqhbfa5bkjakcbxtvy2siwtpcvsvgm9fxfyb03d5` +- `build_id=` +- a build dashboard link (the id is extracted) +- optional `mode=auto` | `mode=interactive` (default: prompt the user) + +Parse the build id. If none is present, this is a required input: + +- in an interactive session → ask the user for it +- in headless (`claude -p`) → **end immediately** (fail fast), do not hang + +## Behavior + +Invoke the `rca-build` skill, passing the parsed build id and mode. The skill +owns the full flow: mandatory pre-flight GitHub intake → discovery via +`listTestIds` → CSV/WAL spine → failure-signature clustering → fan-out +(auto = dynamic workflow / interactive = subagents) → per-test RCA loop via +`tfaRcaTurn` → report. + +Do not re-implement the orchestration here — this command only parses input and +hands off to the skill. diff --git a/config/rca.config.json b/config/rca.config.json new file mode 100644 index 0000000..b7633bb --- /dev/null +++ b/config/rca.config.json @@ -0,0 +1,25 @@ +{ + "$comment": "Central config for the generic RCA harness. All formerly-hardcoded product/infra values live here. No kubectl/chitragupta/bifrost literals — infra tools are discovered at runtime via the capability manifest (see skills/rca-build/references/evidence-routing.md).", + "mcpServerName": "bstack", + "concurrency": 5, + "turnCap": 6, + "turnMessageMaxChars": 5000, + "pollSoftPendingMs": 90000, + "reaperHeartbeatTtlSec": 600, + "errorSummaryMaxChars": 200, + "paths": { + "stateDir": ".rca", + "csvFile": ".rca/rca-state.csv", + "reportFile": ".rca/rca-report.md" + }, + "evidenceRouting": { + "test_logs": { "owner": "tfa", "skip": true }, + "product_code": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, + "deploy": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, + "ci": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, + "k8s": { "capability": "k8s", "discoveryHints": [] }, + "kibana": { "capability": "logs", "discoveryHints": [] }, + "metrics": { "capability": "metrics", "discoveryHints": [] }, + "other": { "capability": "other", "discoveryHints": [] } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..c11e40f --- /dev/null +++ b/package.json @@ -0,0 +1,10 @@ +{ + "name": "tfa-rca-plugin", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Generic multi-client RCA agent plugin harness", + "scripts": { + "test": "node --test tests/" + } +} From f0d5cf63e017a5669764a7f59446d16a59550d0b Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack Date: Tue, 23 Jun 2026 21:58:47 +0530 Subject: [PATCH 02/47] feat(rca): generic per-test RCA coordinator + evidence-routing registry Port the obs-tfa-rca loop decoupled: ai-tfa-coordinator drives tfaRcaTurn to a terminal RCA (turn-cap, one-thread, soft-PENDING, digest-not-dump) with the gather mechanism routed by capability (no kubectl/chitragupta/bifrost literals). lib/routing.mjs classifies each ask skip/gather/gap against the config registry + capability manifest; the gap action is the only mode fork (auto=unavailable, interactive=ask-user). references/evidence-routing.md carries the digest format and size caps verbatim. Adds sibling pre-seed one-turn-confirm hook. Co-Authored-By: Claude Opus 4.8 --- agents/ai-tfa-coordinator.md | 185 ++++++++++++++++++ lib/routing.mjs | 75 +++++++ package.json | 2 +- .../rca-build/references/evidence-routing.md | 133 +++++++++++++ tests/routing.test.mjs | 80 ++++++++ 5 files changed, 474 insertions(+), 1 deletion(-) create mode 100644 agents/ai-tfa-coordinator.md create mode 100644 lib/routing.mjs create mode 100644 skills/rca-build/references/evidence-routing.md create mode 100644 tests/routing.test.mjs diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md new file mode 100644 index 0000000..e2045fb --- /dev/null +++ b/agents/ai-tfa-coordinator.md @@ -0,0 +1,185 @@ +--- +name: ai-tfa-coordinator +description: 'Per-test collaborative-RCA coordinator. Given ONE testRunId, drives the tfaRcaTurn MCP loop to a terminal root cause: TFA reads the run logs; this coordinator supplies every non-log evidence ask (product code, k8s, kibana, metrics, deploy, ci) using whatever skills/tools the client has, routed through the capability manifest. Skips every test_logs ask (TFA owns logs). Emits a structured RCA_OUTPUT block. Generic over product and infra — no hardcoded tools. Examples: +- orchestrator: Agent(subagent_type="ai-tfa-coordinator", prompt="RCA testRunId=39 — error: empty buildName rejected on POST /builds") → drives the loop, returns RCA_OUTPUT +- sibling confirm: Agent(subagent_type="ai-tfa-coordinator", prompt="RCA testRunId=40 — pre-seed: cause=, suspect PR=#7421") → one-turn confirm against this test logs +- user: "run collaborative RCA on test run 39" → single-test loop to RESOLVED/BLOCKED/PENDING' +tools: [Bash, Read, Grep, Glob, Task, mcp__*__tfaRcaTurn, mcp__github__*] +model: sonnet +--- + +# Per-Test Collaborative RCA Coordinator (`ai-tfa-coordinator`) + +Drives the `tfaRcaTurn` MCP loop for a **single** failed test to a terminal RCA. +The collaboration contract is fixed: **TFA owns logs; this coordinator owns +everything else.** TFA (server-side, via the tool) reads the run's logs from its +own access and emits typed evidence asks; this coordinator fulfills every +**non-log** ask using whatever skills/tools the client has — routed through the +capability manifest — digests the findings, and feeds them back on the same +thread until TFA converges. TFA authors the RCA into the TRA dashboard. + +This coordinator is the **reusable unit**: it takes one `testRunId` and runs +standalone, driven by the auto workflow, an interactive subagent, or a thin +sequential harness. It is **generic over product and infra** — it names no +`kubectl` / `chitragupta` / `bifrost`; it routes by *capability*. + +## Inputs + +- `testRunId` — **required**, the integer test-run ID. Maps to the tool's `testRunId` arg. +- `error_digest` — optional short error title + endpoint (NOT logs) for the first-turn message. +- `pre_seed` — optional. For a **cluster sibling**: the representative's + `root_cause` + suspect `related_prs`. When present, the first-turn message + states the hypothesis and asks TFA to **confirm it against this test's own logs**. +- `resume` — optional `{ threadId, turnId }` from a prior PENDING run. +- `manifest` — the capability manifest `{ capability: { available, via } }` (from the orchestrator's pre-compute). +- `mode` — `auto` | `interactive`. Selects the **gap-resolver** (see below). + +If `testRunId` is missing or not parseable as an integer, emit a `failed` +`RCA_OUTPUT` block with `root_cause: "no testRunId provided"` and stop — do not +call the tool. + +## Operating principles + +1. **Logs by TFA — the core contract.** Never seed logs in the first turn; + **skip every ask with `evidenceType === "test_logs"`**. Never fetch, paste, or + digest log content. Logs are TFA's job. +2. **Read-only.** Every gather mechanism is read-only. Never write to a repo, + cluster, ticket, or the run. Produce a block and stop. +3. **Turn-cap** = `turnCap` from `config/rca.config.json` (default 6). If the cap + is hit while still `NEEDS_INFO`, end as `PENDING` (note `turn-cap`) — never an + extra turn, never a busy-wait. +4. **One thread per test.** First turn omits `threadId`; capture it from the + response and reuse it on every follow-up. Never start a second thread. +5. **Soft-PENDING ends the loop.** A tool result of `status: "PENDING"` (in-call + poll exceeded its wall-clock cap) ends the loop immediately as `PENDING`, + carrying `threadId` + `turnId` for a later resume. Do not re-poll or sleep. +6. **Digest, don't dump.** Every follow-up `message` carries digested findings + (`ask → found → snippet/link`), never raw log tails, full diffs, or full files. + Size caps + block shape live in `references/evidence-routing.md` — read it + before fulfilling any ask. The tool caps `message` at 5000 chars. +7. **Report gaps, don't drop them.** An ask the coordinator cannot fulfill becomes + a `not-found` / `unreachable` / `unavailable` block, never a silent omission. +8. **Never editorialize.** Report findings (suspect PR, server-side error line), + not verdicts. The root cause is TFA's to state on `RESOLVED`; pass its `rca` + through verbatim. + +## The gap-resolver (mode fork) + +Routing an ask yields `skip` / `gather` / `gap` (see `references/evidence-routing.md`). +The only behavioral difference between modes is what happens on a **gap** (no +capability available for that `evidenceType`): + +- **auto** → emit an `unavailable` block back to TFA (no user prompt). TFA + finalizes best-effort with lower confidence. +- **interactive** → **return the gap to the caller** (the main agent), which asks + the user (A1) for that data, then feeds the answer back. A subagent cannot + prompt the user itself. + +Everything else — the loop, routing, digest, caps, output — is identical across +modes. Do not fork the loop; only the gap action differs. + +## The loop + +``` +0. Parse inputs → testRunId (int). Build the first-turn DIGEST: + - pre_seed present → "Hypothesis from cluster representative: . + Suspect PR(s): . Confirm against THIS test's logs." (NO logs) + - error_digest present → "Error: " (NO logs, NO threadId) + - neither → "Initiating collaborative RCA for test run <id>." +1. SUBMIT turn 1: tfaRcaTurn(testRunId=<id>, message=<digest>). Capture threadId. turns_used = 1. + (resume case: tfaRcaTurn(testRunId, threadId, turnId) instead, then continue at 2.) +2. CLASSIFY result.status: + RESOLVED → capture rca; END (RESOLVED). + BLOCKED → capture reason + unmetAsks; END (BLOCKED). + PENDING → capture threadId + turnId; END (PENDING, note "soft-pending"). + NEEDS_INFO → go to 3. +3. ROUTE the asks (read references/evidence-routing.md; route via lib/routing.mjs): + For each ask, high → medium → low: + skip → record in asks_skipped, emit nothing. + gather → run the discovered skill/tool for its capability, digest into one block. + Record evidenceType in asks_fulfilled (dedupe). + gap → run the mode's gap-resolver (auto: unavailable block; interactive: return to caller). + Concatenate per-ask blocks into the next-turn MESSAGE (respect size caps). +4. SUBMIT follow-up on the SAME thread: tfaRcaTurn(testRunId, message, threadId). turns_used += 1. +5. TURN-CAP CHECK: if turns_used >= turnCap and still NEEDS_INFO → END (PENDING, "turn-cap"). + else → go to 2 with the new result. +6. EMIT the RCA_OUTPUT block from the captured terminal state. +``` + +**Sibling confirm (cluster member).** When `pre_seed` is present the first turn +states the representative's hypothesis and asks TFA to confirm against this +test's own logs. If TFA `RESOLVED`s in one turn → a logs-grounded per-test RCA at +minimal cost. If TFA instead returns `NEEDS_INFO` / `BLOCKED` (the hypothesis +does not hold for this test), **fall back to the normal loop** — never blindly +inherit the representative's cause. + +## Output contract — `RCA_OUTPUT` + +Emit **exactly one** block at the end of every run (including the `failed` +no-input case). The orchestrator parses it into one CSV row / report record. + +``` +RCA_OUTPUT_START + +## testRunId +<integer> + +## status +<RESOLVED | BLOCKED | PENDING | failed> + +## confidence +<high | medium | low | unknown> # from the terminal turn; unknown for PENDING/failed + +## root_cause +<RESOLVED → rca.root_cause verbatim · BLOCKED → TFA's reason · PENDING/failed → "not available" or the note> + +## possible_fix +<RESOLVED → rca.possible_fix verbatim · else "not available"> + +## related_prs +- <each PR TFA recorded in rca.related_prs; "none" if empty> + +## suspect_signals +- <each non-log signal surfaced: suspect PR / deploy / server-side error line; "none" if empty> + +## thread_id +<threadId from the first turn · "not available" if none> + +## turn_id +<turnId — present for PENDING (resume handle); else "not available"> + +## turns_used +<integer 1..turnCap> + +## asks_fulfilled +- <evidenceType> # every non-test_logs type fulfilled; "none" if empty + +## asks_skipped +- test_logs # present once a test_logs ask appeared + +## asks_unavailable +- <evidenceType> # gaps with no capability (drives the coverage stamp, U10); "none" if empty + +RCA_OUTPUT_END +``` + +Notes: +- `status` is one of exactly four values. `turn-cap` and `soft-pending` both + report as `PENDING`; note which in `root_cause`. +- `asks_skipped` always includes `test_logs` whenever TFA asked for logs. + `asks_fulfilled` **never** includes `test_logs`. +- `asks_unavailable` is the evidence-coverage signal U10 turns into a confidence band. +- `failed` is the no-parseable-result / no-input case; the orchestrator + synthesizes a `failed` row if this coordinator dies — keep the block valid. + +## Hard limits + +- **Never** fulfill or seed a `test_logs` ask — TFA owns logs. +- **Never** exceed `turnCap` `tfaRcaTurn` calls in one run. +- **Never** start a second thread for the same test — reuse the first turn's `threadId`. +- **Never** busy-wait / re-poll on a soft-`PENDING` — end and report it resumable. +- **Never** dump raw logs, full diffs, or full file contents into a turn message — digest only. +- **Never** write to any repo / cluster / ticket / the run — every action is read-only. +- **Never** editorialize a cause — pass TFA's `rca` through verbatim. +- **Never** blindly inherit a representative's cause for a sibling — confirm against its own logs. +- **Always** emit exactly one valid `RCA_OUTPUT` block, even on the `failed` path. diff --git a/lib/routing.mjs b/lib/routing.mjs new file mode 100644 index 0000000..291738e --- /dev/null +++ b/lib/routing.mjs @@ -0,0 +1,75 @@ +// Evidence-routing registry (D3). Maps a TFA `ask.evidenceType` onto an +// action, given the run's capability manifest. Pure + dependency-free so it is +// testable and reusable by both the auto workflow and interactive subagents. +// +// `test_logs` is the TFA agent's own evidence and is always skipped. Every +// other type routes to a capability; whether that capability is *available* is +// decided by the manifest (built once per run — see U6 / buildManifest). + +import { readFileSync } from "node:fs"; + +export const TEST_LOGS = "test_logs"; + +const PRIORITY_RANK = { high: 0, medium: 1, low: 2 }; + +// Load and parse config/rca.config.json from an absolute or cwd-relative path. +export function loadConfig(configPath) { + return JSON.parse(readFileSync(configPath, "utf8")); +} + +// Order a turn's asks high → medium → low (unknown priority sorts last). +export function orderAsks(asks = []) { + return [...asks].sort( + (a, b) => + (PRIORITY_RANK[a?.priority] ?? 99) - (PRIORITY_RANK[b?.priority] ?? 99), + ); +} + +// Classify one ask. Returns one of: +// { action: "skip", ... } — test_logs / TFA-owned; the coordinator emits nothing +// { action: "gather", ... } — a capability is available; gather + digest +// { action: "gap", ... } — no capability; the caller's resolveGap() decides +// (auto → "unavailable" block; interactive → ask the user) +// +// `manifest` shape: { [capability]: { available: boolean, via?: string } }. +export function routeAsk(ask, config, manifest = {}) { + const evidenceType = ask?.evidenceType ?? "other"; + const routing = config?.evidenceRouting ?? {}; + const entry = routing[evidenceType] ?? routing.other ?? { capability: "other" }; + + if (entry.skip || entry.owner === "tfa") { + return { evidenceType, action: "skip", reason: "tfa-owned" }; + } + + const capability = entry.capability ?? "other"; + const cap = manifest[capability]; + if (cap && cap.available) { + return { + evidenceType, + action: "gather", + capability, + via: cap.via ?? null, + }; + } + + return { + evidenceType, + action: "gap", + capability, + discoveryHints: entry.discoveryHints ?? [], + reason: "no-capability", + }; +} + +// Split a turn's asks into the three buckets, in priority order. The +// coordinator gathers `gather`, runs resolveGap() on each `gap`, and records +// `skip` (test_logs) without emitting anything. +export function routeAsks(asks, config, manifest = {}) { + const ordered = orderAsks(asks); + const buckets = { skip: [], gather: [], gap: [] }; + for (const ask of ordered) { + const routed = routeAsk(ask, config, manifest); + buckets[routed.action].push({ ask, ...routed }); + } + return buckets; +} diff --git a/package.json b/package.json index c11e40f..27344a7 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,6 @@ "type": "module", "description": "Generic multi-client RCA agent plugin harness", "scripts": { - "test": "node --test tests/" + "test": "node --test" } } diff --git a/skills/rca-build/references/evidence-routing.md b/skills/rca-build/references/evidence-routing.md new file mode 100644 index 0000000..e6cc4d0 --- /dev/null +++ b/skills/rca-build/references/evidence-routing.md @@ -0,0 +1,133 @@ +# Evidence Routing + +Load this file **before fulfilling any `NEEDS_INFO` ask** in the per-test RCA +loop (`agents/ai-tfa-coordinator`). It maps each TFA `evidenceType` to a +**capability** (not a hardcoded tool), and defines the **digest** the coordinator +submits on the next turn. + +The core contract: **TFA owns logs; the client agent owns everything else.** The +coordinator never seeds logs and never fulfills a `test_logs` ask. Every other +`evidenceType` routes to a capability that is gathered via **whatever skill/tool +the client actually has** for it (discovered once into the capability manifest — +see `SKILL.md` § Pre-compute). There are **no `kubectl` / `chitragupta` / +`bifrost` literals here** — that is the whole point of going generic. + +The registry logic lives in `lib/routing.mjs` (`routeAsk` / `routeAsks`); this +file is the human/agent-facing contract for the digest and the size caps. + +--- + +## How a turn's asks are processed + +A `NEEDS_INFO` turn returns `asks: TfaAsk[]`, each `{ what, why, evidenceType, +priority }`. For each ask, in descending `priority` (`high` → `medium` → `low`): + +1. Route the `evidenceType` (via `lib/routing.mjs` → the config registry + + capability manifest). The result is one of three actions: + - **skip** — `test_logs` (TFA-owned). Gather nothing; record in `asks_skipped`. + - **gather** — a capability is available. Run its discovered skill/tool scoped + by `what` / `why`, then digest the result into one ask block. + - **gap** — no capability is available. Hand the ask to the injected + **`resolveGap()`** policy: + - **auto mode** → emit an `unavailable` block back to TFA (no user prompt). + - **interactive mode** → return the gap to the main agent, which asks the + user, then feeds the answer back. +2. Concatenate the per-ask blocks into the next-turn `message` and resubmit on + the same `threadId`. + +An ask that cannot be fulfilled is **never silently dropped** — it becomes a +`not-found` / `unreachable` / `unavailable` block so TFA can reason about the gap. + +--- + +## Routing table (capability, not tool) + +`evidenceType` literals are exactly those `tfaRcaTurn` emits: `test_logs`, +`product_code`, `k8s`, `kibana`, `metrics`, `deploy`, `ci`, `other`. + +| `evidenceType` | Capability | Gathered via (discovered at runtime) | +|---|---|---| +| `test_logs` | — (TFA, skip) | never gathered; TFA self-serves from its own log access | +| `product_code` | `github` | the client's GitHub capability — **GitHub MCP if present, else `gh`** (see `references/github-evidence.md`) | +| `deploy` | `github` | deploy timeline via the GitHub capability (releases/tags + deploy record) | +| `ci` | `github` | CI config + run history via the GitHub capability | +| `k8s` | `k8s` | whatever k8s/EKS skill the client has — discovered, not assumed | +| `kibana` | `logs` | whatever log-search skill the client has (kibana or other) | +| `metrics` | `metrics` | whatever metrics skill the client has | +| `other` | `other` | best-effort by ask text; else a `not-found` block | + +The mapping is data in `config/rca.config.json` (`evidenceRouting`), so a +different deployment can remap `evidenceType → capability` without code changes. + +**Deployment-state guard:** a suspect PR only matters if its code was actually +live in the run's env at the failure window. If you can cheaply confirm it was +not deployed / behind an OFF flag, say so in the digest rather than feeding TFA a +suspect that could not have caused the failure. (Full protocol: U9 / +`references/github-evidence.md`.) + +--- + +## Digest format + +The single most important discipline: **digested input, not raw dumps.** Every +turn's `message` loads into the agent's context *and* is sent to TFA; a raw log +tail or full PR diff blows both budgets and degrades TFA's reasoning. Supply the +*findings*, not the *haystack*. + +### Per-ask block shape — `ask → found → snippet/link` + +``` +ASK: <verbatim `what` from the TfaAsk, ≤ 120 chars> +TYPE: <evidenceType> +FOUND: <yes | no | partial> +SUMMARY: <1–3 sentences — the finding, in the agent's words. ≤ 400 chars> +SNIPPET: + <the load-bearing excerpt only — see size caps. Omit if a LINK fully carries it.> +LINK: <permalink to the source — PR/commit/log-search/metrics panel/deploy record. Omit if N/A.> +``` + +- `SUMMARY` is the answer. `SNIPPET` is the *minimum* evidence backing it. `LINK` + lets TFA (or a human) verify without the bytes living in the message. +- Prefer **LINK over SNIPPET** whenever a permalink fully carries the evidence. + +### Size caps (hard ceilings — truncate, never exceed) + +| Field / scope | Soft target | Hard ceiling | On exceed | +|---|---|---|---| +| `SUMMARY` | ≤ 300 chars | 400 chars | Tighten to the finding; drop restatement of the ask | +| `SNIPPET` per ask | ≤ 20 lines | 40 lines | Keep the load-bearing lines; replace the rest with `… (N lines elided — see LINK)` | +| Code diff in a `product_code` snippet | ≤ 1 hunk | 3 hunks | Show changed lines + 3 lines context; link the full PR | +| Whole next-turn `message` | ≤ 200 lines | 400 lines (and ≤ `turnMessageMaxChars`) | Drop `low`-priority asks first; keep every `high` ask's block | +| Asks fulfilled per turn | all `high` + `medium` | — | Defer `low` asks to a later turn rather than truncating a `high` ask | + +Truncation rule of thumb: **never truncate a `high`-priority ask's block to fit a +`low`-priority one.** Drop the low block whole; keep the high block intact. The +whole-message ceiling also honors `turnMessageMaxChars` from +`config/rca.config.json` (the tool caps `message` at 5000 chars). + +### What never goes in a digest + +- Raw log tails, full log output, full file contents, full PR diffs — link or excerpt. +- `test_logs` content of any kind (TFA owns it). +- Credentials, tokens, internal hostnames, or any secret surfaced by an env/secret dump. +- Speculation dressed as a finding. If `FOUND: no`, say what was checked; do not invent a cause. + +--- + +## Unfulfillable asks — report, don't drop + +``` +ASK: <verbatim what> +TYPE: <evidenceType> +FOUND: no +SUMMARY: not-found | unreachable | unavailable | out-of-scope — <one line: what was checked or why blocked> +``` + +- `not-found` — the skill/tool ran but the signal isn't there. State the search performed. +- `unreachable` — the surface was not reachable from this agent context. State which. +- `unavailable` — no capability/skill exists for this `evidenceType` (auto-mode gap result). +- `out-of-scope` — the ask is `test_logs` or otherwise not the agent's to fulfill. + +An all-`unavailable` / all-`not-found` turn still resubmits — TFA decides whether +the gap is fatal (→ BLOCKED) or it can converge anyway (best-effort, lower +confidence). The coordinator does not pre-empt that decision. diff --git a/tests/routing.test.mjs b/tests/routing.test.mjs new file mode 100644 index 0000000..c63b595 --- /dev/null +++ b/tests/routing.test.mjs @@ -0,0 +1,80 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { routeAsk, routeAsks, orderAsks, TEST_LOGS } from "../lib/routing.mjs"; + +const CONFIG = { + evidenceRouting: { + test_logs: { owner: "tfa", skip: true }, + product_code: { capability: "github", discoveryHints: ["github-mcp", "gh"] }, + k8s: { capability: "k8s", discoveryHints: [] }, + other: { capability: "other", discoveryHints: [] }, + }, +}; + +test("test_logs is always skipped (TFA-owned)", () => { + const r = routeAsk({ evidenceType: TEST_LOGS, priority: "high" }, CONFIG, { + github: { available: true }, + }); + assert.equal(r.action, "skip"); + assert.equal(r.reason, "tfa-owned"); +}); + +test("available capability → gather, carrying via", () => { + const r = routeAsk({ evidenceType: "product_code", priority: "high" }, CONFIG, { + github: { available: true, via: "github-mcp" }, + }); + assert.equal(r.action, "gather"); + assert.equal(r.capability, "github"); + assert.equal(r.via, "github-mcp"); +}); + +test("unavailable capability → gap, carrying discovery hints", () => { + const r = routeAsk({ evidenceType: "k8s", priority: "medium" }, CONFIG, { + k8s: { available: false }, + }); + assert.equal(r.action, "gap"); + assert.equal(r.capability, "k8s"); + assert.equal(r.reason, "no-capability"); +}); + +test("capability absent from manifest entirely → gap", () => { + const r = routeAsk({ evidenceType: "k8s", priority: "low" }, CONFIG, {}); + assert.equal(r.action, "gap"); +}); + +test("unknown evidenceType falls back to the 'other' entry", () => { + const r = routeAsk({ evidenceType: "weird", priority: "low" }, CONFIG, { + other: { available: true, via: "best-effort" }, + }); + assert.equal(r.action, "gather"); + assert.equal(r.capability, "other"); +}); + +test("orderAsks sorts high → medium → low, unknown last", () => { + const ordered = orderAsks([ + { what: "c", priority: "low" }, + { what: "a", priority: "high" }, + { what: "d", priority: undefined }, + { what: "b", priority: "medium" }, + ]); + assert.deepEqual( + ordered.map((a) => a.what), + ["a", "b", "c", "d"], + ); +}); + +test("routeAsks buckets a mixed turn in priority order", () => { + const buckets = routeAsks( + [ + { evidenceType: "k8s", priority: "low" }, + { evidenceType: "test_logs", priority: "high" }, + { evidenceType: "product_code", priority: "high" }, + ], + CONFIG, + { github: { available: true, via: "gh" } }, + ); + assert.equal(buckets.skip.length, 1); + assert.equal(buckets.gather.length, 1); + assert.equal(buckets.gap.length, 1); + assert.equal(buckets.gather[0].evidenceType, "product_code"); +}); From cb0d8f6d1408267f06619893c6202891c362b648 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:01:27 +0530 Subject: [PATCH 03/47] feat(rca): pre-flight intake, discovery, CSV/WAL spine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SKILL.md orchestrator spec: mandatory GitHub intake ('I don't have one' → RCA-only; headless missing-input fail-fast), discovery via listTestIds(failed, includeFailureDetail), then cluster/pre-compute/fan-out/report steps. lib/csv-state.mjs is the resumable WAL spine — seed (idempotent, terminal- preserving), claim/heartbeat/flip, reaper, pendingRows — with timestamps injected (workflow-sandbox-safe) and an RFC4180 codec for multiline RCA fields. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- lib/csv-state.mjs | 239 ++++++++++++++++++++++++++++++++++++++ skills/rca-build/SKILL.md | 124 ++++++++++++++++++++ tests/csv-state.test.mjs | 133 +++++++++++++++++++++ 3 files changed, 496 insertions(+) create mode 100644 lib/csv-state.mjs create mode 100644 skills/rca-build/SKILL.md create mode 100644 tests/csv-state.test.mjs diff --git a/lib/csv-state.mjs b/lib/csv-state.mjs new file mode 100644 index 0000000..499f997 --- /dev/null +++ b/lib/csv-state.mjs @@ -0,0 +1,239 @@ +// CSV write-ahead-log spine for the batch (D4 + ideation #7). The CSV is the +// single durable, resumable source of truth for "RCA over ALL failed tests": +// every test is a row, seeded `pending`, claimed by a worker, heartbeated while +// in flight, and flipped to a terminal state with its RCA. A reaper reclaims +// rows stranded by a crashed worker. +// +// Timestamps are passed in as `nowMs` (never read from the clock here) so this +// module is deterministic in tests AND usable from the auto-mode dynamic +// workflow, whose sandbox forbids Date.now(). +// +// In-session / in-workspace only — cross-session durability is deferred. Writes +// are synchronous read-modify-write; Node's single thread serializes them, which +// is sufficient for the in-process 5-concurrent workflow (true multi-process +// locking is out of scope). + +import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs"; +import { dirname } from "node:path"; + +export const COLUMNS = [ + "buildId", + "testRunId", + "testName", + "failure_category", + "error_summary", + "file_path", + "cluster_id", + "rca_done", + "in_flight_worker", + "heartbeat_ts", + "threadId", + "turnId", + "last_evidence_digest", + "root_cause", + "failure_type", + "possible_fix", + "related_prs", + "coverage", + "confidence", + "timestamp", +]; + +export const PENDING = "pending"; +const TERMINAL_STATES = new Set([ + "resolved", + "blocked", + "failed", + "pending-resume", +]); + +// ---- minimal RFC4180-ish CSV codec ---------------------------------------- + +function encodeField(value) { + const s = value == null ? "" : String(value); + if (/[",\r\n]/.test(s)) { + return `"${s.replace(/"/g, '""')}"`; + } + return s; +} + +function encodeRows(rows) { + const lines = [COLUMNS.join(",")]; + for (const row of rows) { + lines.push(COLUMNS.map((c) => encodeField(row[c])).join(",")); + } + return lines.join("\n") + "\n"; +} + +function parseCsv(text) { + const rows = []; + let field = ""; + let record = []; + let inQuotes = false; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (inQuotes) { + if (ch === '"') { + if (text[i + 1] === '"') { + field += '"'; + i++; + } else { + inQuotes = false; + } + } else { + field += ch; + } + } else if (ch === '"') { + inQuotes = true; + } else if (ch === ",") { + record.push(field); + field = ""; + } else if (ch === "\n" || ch === "\r") { + if (ch === "\r" && text[i + 1] === "\n") i++; + record.push(field); + rows.push(record); + field = ""; + record = []; + } else { + field += ch; + } + } + if (field.length > 0 || record.length > 0) { + record.push(field); + rows.push(record); + } + return rows; +} + +// ---- read / write ---------------------------------------------------------- + +export function readRows(csvPath) { + if (!existsSync(csvPath)) return []; + const text = readFileSync(csvPath, "utf8"); + const raw = parseCsv(text).filter((r) => r.some((c) => c.length > 0)); + if (raw.length === 0) return []; + const header = raw[0]; + return raw.slice(1).map((cells) => { + const row = {}; + header.forEach((col, idx) => { + row[col] = cells[idx] ?? ""; + }); + return row; + }); +} + +export function writeRows(csvPath, rows) { + const dir = dirname(csvPath); + if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true }); + writeFileSync(csvPath, encodeRows(rows), "utf8"); +} + +function emptyRow() { + return Object.fromEntries(COLUMNS.map((c) => [c, ""])); +} + +// ---- operations ------------------------------------------------------------- + +// Seed the CSV from a listTestIds(failed, includeFailureDetail) payload. Every +// row starts `pending`. Idempotent: existing rows are preserved (terminal rows +// are never reset; signature columns are refreshed on still-pending rows). New +// tests are appended. Returns the full row set. +export function seed(csvPath, buildId, tests) { + const existing = readRows(csvPath); + const byId = new Map(existing.map((r) => [String(r.testRunId), r])); + + for (const t of tests) { + const id = String(t.test_id ?? t.testRunId); + const sig = t.failure ?? {}; + const prior = byId.get(id); + if (prior) { + // Keep terminal results; only refresh signature on still-pending rows. + if (prior.rca_done === PENDING) { + prior.failure_category = sig.category ?? prior.failure_category; + prior.error_summary = sig.error_summary ?? prior.error_summary; + prior.file_path = sig.file_path ?? prior.file_path; + } + continue; + } + const row = emptyRow(); + row.buildId = buildId; + row.testRunId = id; + row.testName = t.test_name ?? t.testName ?? `Test ${id}`; + row.failure_category = sig.category ?? ""; + row.error_summary = sig.error_summary ?? ""; + row.file_path = sig.file_path ?? ""; + row.rca_done = PENDING; + byId.set(id, row); + existing.push(row); + } + + writeRows(csvPath, existing); + return existing; +} + +// Claim a pending row for `worker`. Refuses (returns false) if another worker +// already owns it. Returns true on success. +export function claim(csvPath, testRunId, worker, nowMs) { + const rows = readRows(csvPath); + const row = rows.find((r) => String(r.testRunId) === String(testRunId)); + if (!row) return false; + if (row.in_flight_worker && row.in_flight_worker !== worker) return false; + if (TERMINAL_STATES.has(row.rca_done)) return false; + row.in_flight_worker = worker; + row.heartbeat_ts = String(nowMs); + writeRows(csvPath, rows); + return true; +} + +export function heartbeat(csvPath, testRunId, worker, nowMs) { + const rows = readRows(csvPath); + const row = rows.find((r) => String(r.testRunId) === String(testRunId)); + if (!row || row.in_flight_worker !== worker) return false; + row.heartbeat_ts = String(nowMs); + writeRows(csvPath, rows); + return true; +} + +// Flip a row to a terminal state, recording the RCA fields and clearing the +// in-flight claim. `fields` carries any of: rca_done, root_cause, failure_type, +// possible_fix, related_prs, threadId, turnId, coverage, confidence, +// last_evidence_digest, cluster_id. +export function flip(csvPath, testRunId, fields, nowMs) { + const rows = readRows(csvPath); + const row = rows.find((r) => String(r.testRunId) === String(testRunId)); + if (!row) return false; + for (const [k, v] of Object.entries(fields)) { + if (COLUMNS.includes(k)) { + row[k] = Array.isArray(v) ? v.join("; ") : (v ?? ""); + } + } + row.in_flight_worker = ""; + row.timestamp = String(nowMs); + writeRows(csvPath, rows); + return true; +} + +// Reclaim rows stranded in flight (heartbeat older than ttlSec) back to pending. +// Returns the testRunIds reclaimed. Run on startup before resuming a batch. +export function reaper(csvPath, ttlSec, nowMs) { + const rows = readRows(csvPath); + const reclaimed = []; + for (const row of rows) { + if (!row.in_flight_worker) continue; + if (TERMINAL_STATES.has(row.rca_done)) continue; + const hb = Number(row.heartbeat_ts); + const stale = !row.heartbeat_ts || nowMs - hb > ttlSec * 1000; + if (stale) { + row.in_flight_worker = ""; + row.rca_done = PENDING; + reclaimed.push(String(row.testRunId)); + } + } + if (reclaimed.length > 0) writeRows(csvPath, rows); + return reclaimed; +} + +// Rows still needing work (pending or reclaimed). The work-list for fan-out. +export function pendingRows(csvPath) { + return readRows(csvPath).filter((r) => r.rca_done === PENDING); +} diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md new file mode 100644 index 0000000..4bf6b34 --- /dev/null +++ b/skills/rca-build/SKILL.md @@ -0,0 +1,124 @@ +--- +name: rca-build +description: Run collaborative root-cause analysis over ALL failed tests of a BrowserStack build. Generic across product and infra. Mandatory pre-flight GitHub intake, then discovery via listTestIds, failure-signature clustering, and per-test RCA via tfaRcaTurn (auto = dynamic workflow / interactive = subagents). Use when a build is red and you want a per-test RCA for every failure in the TRA dashboard. +--- + +# rca-build — batch collaborative RCA over a build + +Drives the `tfaRcaTurn` collaborative loop over **every failed test** of a build +and records a per-test RCA. **TFA owns logs; the client agent owns everything +else** (product code, k8s, kibana, metrics, deploy, ci) — routed by capability, +generic over product and infra. + +This skill is the **build-level orchestrator** (`ai-tfa-orchestrator` role). It +never calls `tfaRcaTurn` itself — it dispatches the `ai-tfa-coordinator` +(test-level) per test/cluster member, which drives the loop and lets TFA author +the dashboard RCA. + +Config (concurrency, turn-cap, paths, evidence registry) lives in +`config/rca.config.json`. State lives in the CSV/WAL spine (`lib/csv-state.mjs`). + +## Step 0 — mode + input + +Parse from `/rca-build` args: the build id and optional `mode=auto|interactive`. + +- No build id present → it is required: + - interactive session → ask the user. + - **headless (`claude -p`) with build id missing → end immediately (fail fast).** +- No mode given → ask the user once (auto vs interactive). In headless, default `auto`. + +## Step 1 — pre-flight intake (F1, mandatory, both modes) + +Ask the user (A1) for, in one pass: + +- product repo name, automation (test) repo name +- working branch, default branch +- the PRs in play (product + automation) +- the build id (if not already supplied) + +Every question is **mandatory to ask** but answerable with **"I don't have one"** +→ record the gap and proceed **RCA-only** (BrowserStack-side evidence + whatever +infra skills exist). Do not block the run on missing GitHub context. + +**Headless rule:** in `claude -p`, any *required* input still missing after +parsing (build id) ends the run immediately. Optional intake answers default to +"none" without prompting. + +## Step 2 — discovery (F2) + +Call the bundled MCP tool: + +``` +listTestIds(buildId=<id>, status="failed", includeFailureDetail=true) +``` + +`includeFailureDetail=true` returns each row's trimmed failure signature +(`failure.{category, error_summary, file_path, …}`) — the seed for clustering, so +no per-test probe turns are needed. + +Seed the CSV/WAL spine from the payload (`lib/csv-state.mjs` → `seed`): one row +per failed test, every row `rca_done=pending`, signature columns populated. +Re-running `seed` on an existing CSV is idempotent and preserves terminal rows +(resume-safe). If `listTestIds` returns empty → write an empty CSV, report "no +failed tests", stop. + +## Step 3 — failure-signature clustering (see references/clustering.md) + +Compute a failure signature per row and assign `cluster_id` (`lib/signature.mjs`). +Each cluster gets one **representative** (full multi-turn loop) and `N−1` +**siblings** (pre-seeded one-turn confirm against their own logs). This collapses +the expensive evidence hunt to O(distinct causes) while every test still lands a +per-test RCA. Singleton clusters are just plain per-test loops. + +## Step 4 — build-evidence pre-compute + capability manifest (see references/evidence-routing.md) + +Once, before fan-out: + +- **Capability manifest** — enumerate the skills/tools the client actually has + into `capability → {available, via}` (GitHub, k8s, logs, metrics, …). Declare + to the user up front what will be **unavailable** ("k8s + metrics not + available"). Every coordinator routes asks against this manifest. +- **Build-level evidence** — compute the last-green→this-build delta (diff, + deploy timeline, suspect-PR window) **once** and pre-seed every coordinator + with the same grounded window. Cache by `(repo, commit-range)`. No "last green" + baseline (never-green suite) → fall back to a configured baseline ref and log it. + +## Step 5 — fan-out (the mode fork) + +Drive the cluster work-list, **`concurrency` (default 5) at a time**: +representatives deep, siblings one-turn-confirm. Eagerly persist to the CSV/WAL +(claim → heartbeat → flip) so the run is resumable. + +- **auto** → run the dynamic workflow `workflows/rca-batch.mjs` (script-orchestrated, + no user input; gap → "unavailable" back to TFA → best-effort finalize). +- **interactive** → spawn `ai-tfa-coordinator` subagents 5 at a time; on an + evidence gap a subagent returns the gap to this orchestrator, which asks the + user (A1), then feeds the answer back. Subagents return compact `RCA_OUTPUT` + blocks, not transcripts (keeps the main context lean for large batches). + +Both modes use the **same** `ai-tfa-coordinator`; only the injected gap-resolver +differs. A coordinator that dies becomes a recorded `failed` row — one stuck test +never sinks the batch (partial-first). + +## Step 6 — report (see references/report-format.md) + +When every row is terminal, render the report (`paths.reportFile`): per-test rows +with status + the **evidence-coverage band** (a RESOLVED built with evidence +unavailable reads as lower confidence than a fully-evidenced one). Degrade, +don't crash — missing fields render as "not available". + +## Resume + +On startup, run the reaper (`lib/csv-state.mjs` → `reaper`) to reclaim rows +stranded `in_flight` by a crashed worker (heartbeat older than +`reaperHeartbeatTtlSec`) back to `pending`, then re-point fan-out at the CSV. +Live `threadId`/`turnId` resume the prior thread; dead threads re-run from +pending. (In-session only — cross-session durability is deferred.) + +## Hard rules + +- Always run the pre-flight intake; never silently skip it (but never block on "I don't have one"). +- Headless + missing required input → end immediately. +- Never call `tfaRcaTurn` from this skill — always via the `ai-tfa-coordinator`. +- Every failed test must end terminal in the CSV — partial-first, no abort-on-one-failure. +- Never gather `test_logs` — TFA owns logs. diff --git a/tests/csv-state.test.mjs b/tests/csv-state.test.mjs new file mode 100644 index 0000000..5a9a60f --- /dev/null +++ b/tests/csv-state.test.mjs @@ -0,0 +1,133 @@ +import { test, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + seed, + readRows, + claim, + heartbeat, + flip, + reaper, + pendingRows, + PENDING, +} from "../lib/csv-state.mjs"; + +let dir; +let csv; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "rca-csv-")); + csv = join(dir, "state.csv"); +}); +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +const TESTS = [ + { + test_id: 101, + test_name: "login", + failure: { category: "Assertion", error_summary: "expected 200", file_path: "a.rb" }, + }, + { test_id: 102, test_name: "checkout", failure: { category: "Timeout" } }, +]; + +test("seed writes one pending row per test with signature columns", () => { + const rows = seed(csv, "build-1", TESTS); + assert.equal(rows.length, 2); + assert.ok(rows.every((r) => r.rca_done === PENDING)); + const login = rows.find((r) => r.testRunId === "101"); + assert.equal(login.failure_category, "Assertion"); + assert.equal(login.error_summary, "expected 200"); + assert.equal(login.buildId, "build-1"); +}); + +test("seed is idempotent — no duplicate rows on re-seed", () => { + seed(csv, "build-1", TESTS); + const rows = seed(csv, "build-1", TESTS); + assert.equal(rows.length, 2); +}); + +test("seed preserves a terminal row on re-seed", () => { + seed(csv, "build-1", TESTS); + flip(csv, 101, { rca_done: "resolved", root_cause: "bad PR" }, 1000); + seed(csv, "build-1", TESTS); + const login = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(login.rca_done, "resolved"); + assert.equal(login.root_cause, "bad PR"); +}); + +test("claim sets the worker; a second worker is refused", () => { + seed(csv, "build-1", TESTS); + assert.equal(claim(csv, 101, "w1", 1000), true); + assert.equal(claim(csv, 101, "w2", 1000), false); + const row = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(row.in_flight_worker, "w1"); +}); + +test("heartbeat updates ts only for the owning worker", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + assert.equal(heartbeat(csv, 101, "w1", 2000), true); + assert.equal(heartbeat(csv, 101, "w2", 3000), false); + assert.equal(readRows(csv).find((r) => r.testRunId === "101").heartbeat_ts, "2000"); +}); + +test("flip records terminal fields, joins related_prs, clears the claim", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + flip( + csv, + 101, + { rca_done: "resolved", root_cause: "PR #7421", related_prs: ["#7421", "#7430"], confidence: "high" }, + 5000, + ); + const row = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(row.rca_done, "resolved"); + assert.equal(row.related_prs, "#7421; #7430"); + assert.equal(row.confidence, "high"); + assert.equal(row.in_flight_worker, ""); + assert.equal(row.timestamp, "5000"); +}); + +test("reaper reclaims only stale in-flight rows", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); // stale + claim(csv, 102, "w2", 9000); // fresh + const ttl = 600; // seconds + const now = 1000 + ttl * 1000 + 1; // just past TTL for w1, fresh for w2 + const reclaimed = reaper(csv, ttl, now); + assert.deepEqual(reclaimed, ["101"]); + const rows = readRows(csv); + assert.equal(rows.find((r) => r.testRunId === "101").in_flight_worker, ""); + assert.equal(rows.find((r) => r.testRunId === "101").rca_done, PENDING); + assert.equal(rows.find((r) => r.testRunId === "102").in_flight_worker, "w2"); +}); + +test("reaper leaves terminal rows alone even if in_flight lingered", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + flip(csv, 101, { rca_done: "resolved" }, 2000); // flip clears in_flight + const reclaimed = reaper(csv, 600, 10_000_000); + assert.deepEqual(reclaimed, []); +}); + +test("pendingRows returns only pending work", () => { + seed(csv, "build-1", TESTS); + flip(csv, 101, { rca_done: "resolved" }, 1000); + const pend = pendingRows(csv); + assert.equal(pend.length, 1); + assert.equal(pend[0].testRunId, "102"); +}); + +test("CSV codec round-trips fields with commas, quotes, newlines", () => { + seed(csv, "build-1", [{ test_id: 200, test_name: "weird" }]); + flip( + csv, + 200, + { rca_done: "resolved", root_cause: 'Failed: "x", got <y>\nsecond line' }, + 1000, + ); + const row = readRows(csv).find((r) => r.testRunId === "200"); + assert.equal(row.root_cause, 'Failed: "x", got <y>\nsecond line'); +}); From bbee37db77095abfda7a1c83a7b42180ef12a901 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:03:37 +0530 Subject: [PATCH 04/47] feat(rca): failure-signature clustering + sibling one-turn-confirm protocol lib/signature.mjs computes signature = normalize(category|error|file) off the U1 discovery payload (folds timestamps/uuids/hex/line:col/numbers), groups rows by signature, picks a deterministic representative (non-flaky, then smallest id), and leaves signal-less rows as their own singletons. references/clustering.md documents the O(causes) protocol: representative runs the full loop; siblings pre-seed a one-turn confirm against their own logs with a fall-back-to-own-loop safeguard (never blindly inherit). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- lib/signature.mjs | 78 ++++++++++++++++++++++ skills/rca-build/references/clustering.md | 60 +++++++++++++++++ tests/signature.test.mjs | 79 +++++++++++++++++++++++ 3 files changed, 217 insertions(+) create mode 100644 lib/signature.mjs create mode 100644 skills/rca-build/references/clustering.md create mode 100644 tests/signature.test.mjs diff --git a/lib/signature.mjs b/lib/signature.mjs new file mode 100644 index 0000000..42dc0ae --- /dev/null +++ b/lib/signature.mjs @@ -0,0 +1,78 @@ +// Failure-signature clustering (ideation #1). A red build's N failures usually +// trace to a handful of causes; clustering collapses the expensive evidence hunt +// to O(distinct causes). The signature is computed from the trimmed failure +// detail U1 surfaces on each listTestIds row (category + first error line + file +// path) — no extra probe turns. +// +// Dependency-free + deterministic (no crypto, no clock, no random) so it is +// usable from the auto-mode workflow sandbox and trivially testable. + +// Normalize a string for signature comparison: lowercase and fold the volatile +// tokens that make two instances of the SAME failure look different (ids, +// timestamps, hex/uuids, line:col, bare numbers). +export function normalize(value) { + return String(value ?? "") + .toLowerCase() + .replace(/\b\d{4}-\d{2}-\d{2}[t ]\d{2}:\d{2}:\d{2}\S*/g, "<ts>") // ISO timestamps + .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g, "<uuid>") + .replace(/0x[0-9a-f]+/g, "<hex>") // memory addresses + .replace(/:\d+(:\d+)?\b/g, ":<line>") // file:line(:col) + .replace(/\d+/g, "<n>") // remaining numbers (incl. unit-suffixed, e.g. 3000ms) + .replace(/\s+/g, " ") + .trim(); +} + +// The signature triple: normalized category | error summary | file path. +export function computeSignature(row) { + const category = normalize(row.failure_category); + const error = normalize(row.error_summary); + const file = normalize(row.file_path); + const sig = `${category}|${error}|${file}`; + return sig.replace(/\|/g, "").trim().length === 0 ? "" : sig; +} + +// Deterministic short id for a signature string (FNV-1a → base36). +function hashId(s) { + let h = 0x811c9dc5; + for (let i = 0; i < s.length; i++) { + h ^= s.charCodeAt(i); + h = Math.imul(h, 0x01000193); + } + return (h >>> 0).toString(36); +} + +// A stable representative for a cluster: prefer a non-flaky member (a flaky test +// is a poor exemplar), then the smallest testRunId. Deterministic. +export function selectRepresentative(members) { + return [...members].sort((a, b) => { + const aFlaky = a.is_flaky === "true" || a.is_flaky === true ? 1 : 0; + const bFlaky = b.is_flaky === "true" || b.is_flaky === true ? 1 : 0; + if (aFlaky !== bFlaky) return aFlaky - bFlaky; + return Number(a.testRunId) - Number(b.testRunId); + })[0]; +} + +// Cluster rows by signature. Mutates each row's `cluster_id`. Rows with no +// signal (empty signature) become their own singleton (never merged into a +// catch-all). Returns { rows, clusters } where each cluster carries its +// representative + siblings. +export function clusterRows(rows) { + const groups = new Map(); + + for (const row of rows) { + const sig = computeSignature(row); + const id = sig === "" ? `solo-${row.testRunId}` : `c-${hashId(sig)}`; + row.cluster_id = id; + if (!groups.has(id)) groups.set(id, { cluster_id: id, signature: sig, members: [] }); + groups.get(id).members.push(row); + } + + const clusters = []; + for (const group of groups.values()) { + const representative = selectRepresentative(group.members); + const siblings = group.members.filter((m) => m !== representative); + clusters.push({ ...group, representative, siblings }); + } + + return { rows, clusters }; +} diff --git a/skills/rca-build/references/clustering.md b/skills/rca-build/references/clustering.md new file mode 100644 index 0000000..66face8 --- /dev/null +++ b/skills/rca-build/references/clustering.md @@ -0,0 +1,60 @@ +# Failure-signature clustering + +Why: a red build's N failures usually trace to a handful of causes (one bad +PR/deploy/shared helper). Running the full collaborative loop once per *cause* +instead of once per *test* turns the dominant cost from **O(tests) → O(distinct +causes)** — the only thing that makes "RCA for ALL failed tests, even thousands" +feasible. But **every failed test must still show a per-test RCA in the TRA +dashboard**, so clustering collapses the *evidence hunt*, not the *output*. + +The logic lives in `lib/signature.mjs`; this file is the protocol. + +## The signature + +Computed from the trimmed failure detail `listTestIds(includeFailureDetail=true)` +already returns on each row — **no extra probe turns**: + +``` +signature = normalize(failure_category) | normalize(error_summary) | normalize(file_path) +``` + +`normalize` folds the volatile tokens that make two instances of the *same* +failure look different: ISO timestamps, UUIDs, hex/memory addresses, `file:line:col`, +and bare numbers. So `timeout after 3000ms on node-7` and `timeout after 5000ms +on node-2` share a signature. + +A row with **no signal** (empty category, error, and path) is **not** merged into +a catch-all — it becomes its own singleton (`solo-<testRunId>`). Better an +un-clustered test than a wrong cluster. + +## Representative + siblings + +Each cluster gets: + +- **Representative** — a stable exemplar (non-flaky preferred, then smallest + `testRunId`). Runs the **full multi-turn `ai-tfa-coordinator` loop** → + confirmed root cause + culprit `related_prs`. +- **Siblings** (`N−1`) — each runs its **own** coordinator, **pre-seeded** with + the representative's `root_cause` + suspect PRs. TFA confirms the hypothesis + **against that sibling's own logs in a single turn** → a logs-grounded per-test + RCA in the dashboard at minimal cost. + +Net cost per cluster: **1 deep investigation + (N−1) one-turn confirms.** + +## The safeguard — never blindly inherit + +Distinct failures can share an error string. A sibling's pre-seed turn is a +*hypothesis to confirm*, not a verdict to copy: + +- TFA `RESOLVED`s the sibling in one turn → logs-grounded inheritance, cheap. +- TFA returns `NEEDS_INFO` / `BLOCKED` (the hypothesis does not hold for this + test's logs) → the sibling **falls back to its own full loop**. The + representative's cause is never stamped onto a sibling without log confirmation. + +This keeps correctness independent of the cost optimization: worst case, every +sibling runs its own full loop (same as no clustering); best case, one deep run +covers the whole cluster. + +## Singletons + +A cluster of one is just a plain per-test loop — no pre-seed, no confirm step. diff --git a/tests/signature.test.mjs b/tests/signature.test.mjs new file mode 100644 index 0000000..f721167 --- /dev/null +++ b/tests/signature.test.mjs @@ -0,0 +1,79 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + normalize, + computeSignature, + selectRepresentative, + clusterRows, +} from "../lib/signature.mjs"; + +function row(id, extra = {}) { + return { + testRunId: String(id), + failure_category: "Assertion", + error_summary: "expected 200 but got 500", + file_path: "spec/login.rb", + is_flaky: "false", + ...extra, + }; +} + +test("normalize folds timestamps, uuids, hex, line:col, and numbers", () => { + assert.equal(normalize("Error at line :42:7"), "error at line :<line>"); + assert.equal(normalize("got 500 at 0xAF3"), "got <n> at <hex>"); + assert.equal( + normalize("failed 2026-06-23T10:00:00Z"), + "failed <ts>", + ); +}); + +test("identical category+error+path → same cluster", () => { + const { clusters } = clusterRows([row(1), row(2)]); + assert.equal(clusters.length, 1); + assert.equal(clusters[0].members.length, 2); +}); + +test("numbers in the error are folded so siblings still cluster", () => { + const a = row(1, { error_summary: "timeout after 3000ms on node-7" }); + const b = row(2, { error_summary: "timeout after 5000ms on node-2" }); + assert.equal(computeSignature(a), computeSignature(b)); + const { clusters } = clusterRows([a, b]); + assert.equal(clusters.length, 1); +}); + +test("distinct failures → distinct clusters", () => { + const a = row(1, { error_summary: "null pointer in Foo" }); + const b = row(2, { error_summary: "connection refused" }); + const { clusters } = clusterRows([a, b]); + assert.equal(clusters.length, 2); +}); + +test("rows with no signal become their own singletons (no catch-all merge)", () => { + const a = { testRunId: "1", failure_category: "", error_summary: "", file_path: "" }; + const b = { testRunId: "2", failure_category: "", error_summary: "", file_path: "" }; + const { clusters } = clusterRows([a, b]); + assert.equal(clusters.length, 2); + assert.ok(clusters.every((c) => c.cluster_id.startsWith("solo-"))); +}); + +test("singleton cluster has a representative and no siblings", () => { + const { clusters } = clusterRows([row(1)]); + assert.equal(clusters[0].siblings.length, 0); + assert.equal(clusters[0].representative.testRunId, "1"); +}); + +test("representative is deterministic: non-flaky, then smallest testRunId", () => { + const members = [ + row(5, { is_flaky: "true" }), + row(9, { is_flaky: "false" }), + row(7, { is_flaky: "false" }), + ]; + assert.equal(selectRepresentative(members).testRunId, "7"); +}); + +test("clusterRows stamps cluster_id onto every row", () => { + const rows = [row(1), row(2, { error_summary: "different" })]; + clusterRows(rows); + assert.ok(rows.every((r) => r.cluster_id)); + assert.notEqual(rows[0].cluster_id, rows[1].cluster_id); +}); From 7ddb2c222bd9b3b479a3bce82d3ed0c5e75de69c Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:05:29 +0530 Subject: [PATCH 05/47] feat(rca): build-evidence pre-compute + cache + capability manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildManifest enumerates the client's discovered capabilities once into capability→{available,via}, declared to the user + TFA so no evidence is asked for that the client provably can't get. lib/evidence-cache.mjs computes the last-green→this-build delta once and caches by (repo,range,evidenceType) — fresh per-run Map, no module globals (multi-tenant-safe) — with resolveBaseline for the never-green fallback. Routes the same grounded window into every coordinator. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- lib/evidence-cache.mjs | 47 ++++++++++++ lib/routing.mjs | 31 ++++++++ .../rca-build/references/evidence-routing.md | 29 +++++++ tests/evidence.test.mjs | 76 +++++++++++++++++++ 4 files changed, 183 insertions(+) create mode 100644 lib/evidence-cache.mjs create mode 100644 tests/evidence.test.mjs diff --git a/lib/evidence-cache.mjs b/lib/evidence-cache.mjs new file mode 100644 index 0000000..b9c9523 --- /dev/null +++ b/lib/evidence-cache.mjs @@ -0,0 +1,47 @@ +// Build-level evidence cache (ideation #2). "Diff since last green", "deploy +// timeline", "PRs in the suspect window" are properties of the BUILD, not the +// test — yet a naive loop re-fetches them per test. Compute once, cache by +// (repo, commit-range, evidenceType), and pre-seed every coordinator with the +// same grounded suspect window. Collapses N×M redundant git/infra calls to ~M. +// +// The cache is created fresh per run (function-scoped Map — never a module-level +// global), so it holds no cross-run/cross-user state: in-workspace, single +// session, multi-tenant-safe by construction. + +export function makeEvidenceCache() { + const store = new Map(); + const keyOf = (repo, range, evidenceType) => + `${repo ?? ""}@@${range ?? ""}@@${evidenceType ?? ""}`; + + return { + has(repo, range, evidenceType) { + return store.has(keyOf(repo, range, evidenceType)); + }, + get(repo, range, evidenceType) { + return store.get(keyOf(repo, range, evidenceType)); + }, + set(repo, range, evidenceType, value) { + store.set(keyOf(repo, range, evidenceType), value); + return value; + }, + // Compute-once: run `fn` only on a cache miss; reuse on every later call. + async compute(repo, range, evidenceType, fn) { + const k = keyOf(repo, range, evidenceType); + if (store.has(k)) return store.get(k); + const value = await fn(); + store.set(k, value); + return value; + }, + size() { + return store.size; + }, + }; +} + +// Resolve the baseline ref for the last-green→this-build delta. When there is no +// "last green" (e.g. a never-green flaky suite) fall back to a configured ref and +// flag it so the report can note the weaker grounding. +export function resolveBaseline(lastGreenRef, fallbackRef) { + if (lastGreenRef) return { ref: lastGreenRef, isFallback: false }; + return { ref: fallbackRef ?? null, isFallback: true }; +} diff --git a/lib/routing.mjs b/lib/routing.mjs index 291738e..ec7c4e7 100644 --- a/lib/routing.mjs +++ b/lib/routing.mjs @@ -73,3 +73,34 @@ export function routeAsks(asks, config, manifest = {}) { } return buckets; } + +// ---- capability manifest (ideation #3) ------------------------------------- + +// Build the capability manifest ONCE per run from the capabilities the client +// agent actually discovered. `discovered` is a list of +// { capability, via } the orchestrator collected by asking "what skills/tools +// are available?". Every capability the routing registry references (except the +// TFA-owned test_logs) appears in the manifest, marked available iff discovered. +// Declaring this to TFA lets it avoid asking for evidence the client can't get. +export function buildManifest(config, discovered = []) { + const byCap = new Map(discovered.map((d) => [d.capability, d])); + const manifest = {}; + for (const entry of Object.values(config?.evidenceRouting ?? {})) { + if (entry.skip || entry.owner === "tfa") continue; + const cap = entry.capability; + if (!cap || cap in manifest) continue; + const found = byCap.get(cap); + manifest[cap] = found + ? { available: true, via: found.via ?? null } + : { available: false, via: null }; + } + return manifest; +} + +// Capabilities that will be unavailable this run — declared to the user up front +// ("k8s + metrics not available") and to TFA so it plans asks around them. +export function unavailableCapabilities(manifest) { + return Object.entries(manifest) + .filter(([, v]) => !v.available) + .map(([cap]) => cap); +} diff --git a/skills/rca-build/references/evidence-routing.md b/skills/rca-build/references/evidence-routing.md index e6cc4d0..87fb255 100644 --- a/skills/rca-build/references/evidence-routing.md +++ b/skills/rca-build/references/evidence-routing.md @@ -131,3 +131,32 @@ SUMMARY: not-found | unreachable | unavailable | out-of-scope — <one line: wha An all-`unavailable` / all-`not-found` turn still resubmits — TFA decides whether the gap is fatal (→ BLOCKED) or it can converge anyway (best-effort, lower confidence). The coordinator does not pre-empt that decision. + +--- + +## Capability manifest (built once per run) + +Rather than re-discover "is there a kibana skill?" on every ask across every +test, the orchestrator enumerates the client's available skills/tools **once** up +front into a manifest (`lib/routing.mjs` → `buildManifest`): + +``` +{ github: {available: true, via: "github-mcp"}, k8s: {available: false}, ... } +``` + +- Every ask routes against this manifest — reproducible, no per-ask discovery. +- The orchestrator **declares the unavailable capabilities to the user** up front + ("k8s + metrics will be unavailable") and includes them in the first turn so + TFA plans asks around what's obtainable. +- Frozen at run start. A skill appearing mid-run is not picked up until the next run. + +## Build-level evidence cache (compute once) + +"Diff since last green", "deploy timeline", and "PRs in the suspect window" are +properties of the **build**, not the test. The orchestrator computes the +last-green→this-build delta **once** (`lib/evidence-cache.mjs`), caches it by +`(repo, commit-range, evidenceType)`, and pre-seeds every coordinator with the +same grounded suspect window — collapsing N×M redundant git/infra calls to ~M and +front-loading the highest-signal evidence so many tests RESOLVE before any infra +ask fires. No "last green" (never-green suite) → fall back to a configured +baseline ref and note the weaker grounding in the report. diff --git a/tests/evidence.test.mjs b/tests/evidence.test.mjs new file mode 100644 index 0000000..e01b06a --- /dev/null +++ b/tests/evidence.test.mjs @@ -0,0 +1,76 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { buildManifest, unavailableCapabilities } from "../lib/routing.mjs"; +import { makeEvidenceCache, resolveBaseline } from "../lib/evidence-cache.mjs"; + +const CONFIG = { + evidenceRouting: { + test_logs: { owner: "tfa", skip: true }, + product_code: { capability: "github" }, + deploy: { capability: "github" }, + k8s: { capability: "k8s" }, + metrics: { capability: "metrics" }, + other: { capability: "other" }, + }, +}; + +test("buildManifest marks discovered capabilities available with via", () => { + const manifest = buildManifest(CONFIG, [ + { capability: "github", via: "github-mcp" }, + ]); + assert.equal(manifest.github.available, true); + assert.equal(manifest.github.via, "github-mcp"); + assert.equal(manifest.k8s.available, false); +}); + +test("buildManifest excludes the TFA-owned test_logs capability", () => { + const manifest = buildManifest(CONFIG, []); + assert.ok(!("undefined" in manifest)); + assert.ok(!Object.keys(manifest).includes("test_logs")); +}); + +test("buildManifest dedupes capabilities shared by multiple evidence types", () => { + // product_code + deploy both map to github → one manifest entry + const manifest = buildManifest(CONFIG, [{ capability: "github" }]); + assert.equal(Object.keys(manifest).filter((k) => k === "github").length, 1); +}); + +test("unavailableCapabilities lists what the client can't get", () => { + const manifest = buildManifest(CONFIG, [{ capability: "github" }]); + const unavailable = unavailableCapabilities(manifest).sort(); + assert.deepEqual(unavailable, ["k8s", "metrics", "other"]); +}); + +test("evidence cache computes once and reuses across calls", async () => { + const cache = makeEvidenceCache(); + let calls = 0; + const fn = async () => { + calls++; + return { prs: ["#1"] }; + }; + const a = await cache.compute("repo", "abc..def", "deploy", fn); + const b = await cache.compute("repo", "abc..def", "deploy", fn); + assert.equal(calls, 1); + assert.deepEqual(a, b); + assert.equal(cache.size(), 1); +}); + +test("evidence cache key distinguishes commit ranges", async () => { + const cache = makeEvidenceCache(); + let calls = 0; + const fn = async () => ++calls; + await cache.compute("repo", "r1", "deploy", fn); + await cache.compute("repo", "r2", "deploy", fn); + assert.equal(calls, 2); +}); + +test("resolveBaseline uses last-green when present, else flags fallback", () => { + assert.deepEqual(resolveBaseline("v1.2.3", "main"), { + ref: "v1.2.3", + isFallback: false, + }); + assert.deepEqual(resolveBaseline(null, "main"), { + ref: "main", + isFallback: true, + }); +}); From d6f0452298a8a923105bf473388b7d5bfd1b50ce Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:07:26 +0530 Subject: [PATCH 06/47] feat(rca): auto-mode dynamic workflow (rca-batch) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit workflows/rca-batch.mjs orchestrates the batch in auto mode: a pipeline over clusters dispatches ai-tfa-coordinator agents — representative full loop → siblings one-turn-confirm, no barrier between stages — with a structured RCA schema. Sandbox-correct: does no state I/O itself (orchestrator passes the clustered work-list + manifest + pre-computed build evidence via args; each coordinator agent persists its own CSV row eagerly). Gap → 'unavailable' back to TFA, no user prompt. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- workflows/rca-batch.mjs | 130 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 workflows/rca-batch.mjs diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs new file mode 100644 index 0000000..e577dbe --- /dev/null +++ b/workflows/rca-batch.mjs @@ -0,0 +1,130 @@ +export const meta = { + name: "rca-batch", + description: + "Drive collaborative RCA over all failed tests of a build (auto mode): cluster representatives run the full loop, siblings one-turn-confirm, ~5 concurrent.", + phases: [ + { title: "Representatives", detail: "full multi-turn RCA per cluster" }, + { title: "Siblings", detail: "one-turn confirm against own logs" }, + ], +}; + +// AUTO MODE orchestration (D2). This is a dynamic-workflow script: it runs in the +// Workflow sandbox (no filesystem, no Date.now/Math.random, agent()/pipeline() +// as globals). It therefore does NO state I/O itself — the orchestrator seeds the +// CSV, clusters, and builds the manifest in normal context and passes the +// work-list via `args`; each dispatched `ai-tfa-coordinator` agent (which HAS +// tool access) claims + flips its own CSV row eagerly (WAL); this script +// orchestrates concurrency and returns the structured results for reconciliation. +// +// args shape: +// { +// csvPath, buildId, mode: "auto", +// manifest: { capability: { available, via } }, +// buildEvidence: { baselineRef, suspectWindow, ... }, // pre-computed once +// clusters: [ +// { cluster_id, representative: { testRunId, testName, error_summary }, +// siblings: [ { testRunId, testName, error_summary } ] } +// ] +// } + +const RCA_SCHEMA = { + type: "object", + required: ["testRunId", "status"], + properties: { + testRunId: { type: "string" }, + status: { enum: ["RESOLVED", "BLOCKED", "PENDING", "failed"] }, + confidence: { enum: ["high", "medium", "low", "unknown"] }, + root_cause: { type: "string" }, + possible_fix: { type: "string" }, + related_prs: { type: "array", items: { type: "string" } }, + suspect_signals: { type: "array", items: { type: "string" } }, + threadId: { type: "string" }, + turnId: { type: "string" }, + turns_used: { type: "number" }, + asks_fulfilled: { type: "array", items: { type: "string" } }, + asks_skipped: { type: "array", items: { type: "string" } }, + asks_unavailable: { type: "array", items: { type: "string" } }, + cluster_id: { type: "string" }, + }, + additionalProperties: true, +}; + +const ctx = args ?? {}; +const clusters = ctx.clusters ?? []; +const shared = [ + `CSV state file: ${ctx.csvPath}`, + `Capability manifest: ${JSON.stringify(ctx.manifest ?? {})}`, + `Build-level evidence (pre-computed once, reuse — do not re-fetch): ${JSON.stringify(ctx.buildEvidence ?? {})}`, + `Mode: auto — on an evidence gap with no capability, report "unavailable" back to TFA (NEVER prompt a user). Best-effort finalize.`, + `Persist eagerly to the CSV: claim your row before turn 1, flip it on terminal (lib/csv-state.mjs).`, +].join("\n"); + +function repPrompt(cluster) { + const r = cluster.representative; + return [ + `You are the ai-tfa-coordinator for cluster ${cluster.cluster_id}.`, + `Run the FULL collaborative RCA loop for the representative test.`, + `testRunId=${r.testRunId} testName=${r.testName ?? ""}`, + `error_digest: ${r.error_summary ?? "(none)"}`, + shared, + `Return the structured RCA_OUTPUT for this test.`, + ].join("\n"); +} + +function siblingPrompt(sibling, repResult, cluster) { + return [ + `You are the ai-tfa-coordinator for a SIBLING of cluster ${cluster.cluster_id}.`, + `Pre-seed: the representative resolved as:`, + ` root_cause: ${repResult?.root_cause ?? "(representative did not resolve)"}`, + ` related_prs: ${JSON.stringify(repResult?.related_prs ?? [])}`, + `State this hypothesis on turn 1 and ask TFA to CONFIRM it against THIS test's own logs.`, + `If TFA confirms in one turn → done. If it does NOT (NEEDS_INFO/BLOCKED), fall back to the full loop — never blindly inherit.`, + `testRunId=${sibling.testRunId} testName=${sibling.testName ?? ""}`, + `error_digest: ${sibling.error_summary ?? "(none)"}`, + shared, + `Return the structured RCA_OUTPUT for this test.`, + ].join("\n"); +} + +log(`Auto-mode batch: ${clusters.length} cluster(s) over build ${ctx.buildId ?? "?"}`); + +// Pipeline: each cluster flows representative → siblings independently (no barrier +// between stages), so a small cluster's siblings confirm while a big cluster's +// representative is still looping. Concurrency is bounded by the workflow runtime +// (~min(16, cores-2)); config.concurrency (5) is the intended soft target. +const results = await pipeline( + clusters, + (cluster) => + agent(repPrompt(cluster), { + label: `rep:${cluster.representative.testRunId}`, + phase: "Representatives", + agentType: "ai-tfa-coordinator", + schema: RCA_SCHEMA, + }).then((rca) => ({ cluster, rca })), + ({ cluster, rca }) => + parallel( + (cluster.siblings ?? []).map((sib) => () => + agent(siblingPrompt(sib, rca, cluster), { + label: `sib:${sib.testRunId}`, + phase: "Siblings", + agentType: "ai-tfa-coordinator", + schema: RCA_SCHEMA, + }), + ), + ).then((sibs) => ({ + cluster_id: cluster.cluster_id, + representative: rca, + siblings: sibs.filter(Boolean), + })), +); + +const flat = results.filter(Boolean); +const all = flat.flatMap((r) => [r.representative, ...(r.siblings ?? [])]).filter(Boolean); +const byStatus = all.reduce((acc, r) => { + acc[r.status] = (acc[r.status] ?? 0) + 1; + return acc; +}, {}); + +log(`Auto-mode batch complete: ${all.length} test(s) — ${JSON.stringify(byStatus)}`); + +return { clusters: flat.length, tests: all.length, byStatus, results: flat }; From 28ebc1d29c2d6501098ea446e105c5f01e97fd18 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:08:45 +0530 Subject: [PATCH 07/47] =?UTF-8?q?feat(rca):=20interactive=20mode=20?= =?UTF-8?q?=E2=80=94=20subagents=20with=20user-in-the-loop=20gap-return?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit references/interactive-mode.md specifies the orchestrator loop: spawn ai-tfa-coordinator subagents 5 at a time; a subagent cannot pause to prompt the user, so on an evidence gap it ends early with a GAP_OUTPUT carrying resume handles (threadId+turnId); the orchestrator asks A1, then re-dispatches with resume= and the answer. Same coordinator as auto — only the gap action differs. Compact blocks not transcripts (lean main context); partial-first; auto-first/ escalate-the-residue noted. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- agents/ai-tfa-coordinator.md | 31 ++++++++-- skills/rca-build/SKILL.md | 7 ++- .../rca-build/references/interactive-mode.md | 56 +++++++++++++++++++ 3 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 skills/rca-build/references/interactive-mode.md diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index e2045fb..4d42f17 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -71,12 +71,33 @@ capability available for that `evidenceType`): - **auto** → emit an `unavailable` block back to TFA (no user prompt). TFA finalizes best-effort with lower confidence. -- **interactive** → **return the gap to the caller** (the main agent), which asks - the user (A1) for that data, then feeds the answer back. A subagent cannot - prompt the user itself. +- **interactive** → a subagent cannot pause to prompt the user, so **end the run + early and return a `GAP_OUTPUT` block** (status `PENDING`) carrying the resume + handles + the gap. The orchestrator asks A1, then **re-dispatches a coordinator + with `resume={threadId, turnId}`** and the answer digested into the next turn. + See `references/interactive-mode.md`. -Everything else — the loop, routing, digest, caps, output — is identical across -modes. Do not fork the loop; only the gap action differs. +`GAP_OUTPUT` block (interactive gap only): + +``` +GAP_OUTPUT_START +## testRunId +<integer> +## thread_id +<threadId> +## turn_id +<turnId> # resume handle +## gap +- evidenceType: <type> +- what: <verbatim ask `what`> +- why: <verbatim ask `why`> +GAP_OUTPUT_END +``` + +Everything else — the loop, routing, digest, caps, terminal output — is identical +across modes. Do not fork the loop; only the gap action differs. When all gaps in +a turn are resolvable (gathered or user-answered), the loop proceeds normally to a +terminal `RCA_OUTPUT`. ## The loop diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 4bf6b34..5248bee 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -92,9 +92,10 @@ representatives deep, siblings one-turn-confirm. Eagerly persist to the CSV/WAL - **auto** → run the dynamic workflow `workflows/rca-batch.mjs` (script-orchestrated, no user input; gap → "unavailable" back to TFA → best-effort finalize). - **interactive** → spawn `ai-tfa-coordinator` subagents 5 at a time; on an - evidence gap a subagent returns the gap to this orchestrator, which asks the - user (A1), then feeds the answer back. Subagents return compact `RCA_OUTPUT` - blocks, not transcripts (keeps the main context lean for large batches). + evidence gap a subagent ends early with a `GAP_OUTPUT` (resume handles), and + this orchestrator asks the user (A1) then re-dispatches with `resume=`. Subagents + return compact blocks, not transcripts (keeps the main context lean for large + batches). Full protocol: `references/interactive-mode.md`. Both modes use the **same** `ai-tfa-coordinator`; only the injected gap-resolver differs. A coordinator that dies becomes a recorded `failed` row — one stuck test diff --git a/skills/rca-build/references/interactive-mode.md b/skills/rca-build/references/interactive-mode.md new file mode 100644 index 0000000..493ea65 --- /dev/null +++ b/skills/rca-build/references/interactive-mode.md @@ -0,0 +1,56 @@ +# Interactive mode — subagents with a user in the loop + +Interactive mode (D2) puts the human (A1) in the loop **only at the orchestrator +layer**. The main session spawns `ai-tfa-coordinator` subagents to investigate in +parallel; when a subagent needs evidence it can't get, it hands the gap back up to +the orchestrator, which asks the user and feeds the answer down. + +This is the **same coordinator** the auto workflow uses — only the gap-resolver +differs (auto → "unavailable"; interactive → return the gap). + +## Why a subagent can't just "ask the user" + +A dispatched subagent runs to completion and returns one final message — it +cannot pause mid-run, prompt the user, and resume. So the gap-return is modeled +as **early termination with resume handles**, and the orchestrator drives the +ask-and-resume loop. + +## The orchestrator loop (per batch of ≤ `concurrency`, default 5) + +``` +1. Take the next ≤5 pending work items (representatives first, then siblings). +2. Dispatch one ai-tfa-coordinator subagent per item, mode=interactive, passing + the manifest + pre-computed build evidence + (for siblings) the pre-seed. +3. Each subagent runs its loop until either: + - a terminal status → returns RCA_OUTPUT (the orchestrator flips the CSV row), or + - an interactive GAP → returns GAP_OUTPUT (status=PENDING) carrying: + { testRunId, threadId, turnId, gap: { evidenceType, what, why } } +4. For each GAP_OUTPUT: ASK A1 for that evidence (one focused question). + - A1 answers → re-dispatch a coordinator with resume={threadId,turnId} and + the answer digested into the next turn's message. Continue its loop. + - A1 has nothing → tell the coordinator to report "unavailable" on resume + (degrade exactly like auto for that one ask). +5. Repeat until every row is terminal. Then dispatch the next batch. +``` + +## Aggregation discipline (large batches) + +Subagents return **compact `RCA_OUTPUT` / `GAP_OUTPUT` blocks, never transcripts** +— mirroring the auto workflow's "results in script vars" rule — so the main +agent's context stays lean even over hundreds of tests. The orchestrator never +holds full per-test loop transcripts; it holds one block per test. + +## Partial-first + +A subagent that dies becomes a recorded `failed` row (the orchestrator +synthesizes it). One stuck test never sinks the batch — same contract as auto. + +## When to prefer interactive over auto + +- The client is missing infra skills the failures clearly need (k8s/kibana), and + the user can supply that evidence by hand. +- The user wants to steer or sign off mid-run. + +Otherwise auto is cheaper (no human round-trips). Both write the same CSV rows +and the same report, so a run can start auto and the residual BLOCKED/gap tests +can be re-run interactively (the auto-first / escalate-the-residue pattern). From e8b70e644225bc448fc47a0508b61223456cf07a Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:09:44 +0530 Subject: [PATCH 08/47] feat(rca): suspect-PR falsification packet + GitHub-evidence spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit references/github-evidence.md specifies exactly what each github ask needs (diff-since-baseline, PRs-in-window touching the failing path, blame, deploy timing) and the discovery order GitHub MCP → gh → degrade — no shipped forensics harness. Adds the adversarial falsification protocol (path overlap / deploy-state guard / direction) so only verdict:supported suspects enter related_prs; ruled-out suspects stay as disconfirming evidence. Coordinator runs it for product_code/ deploy/ci asks, reusing the pre-computed build evidence. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- agents/ai-tfa-coordinator.md | 12 +++ .../rca-build/references/github-evidence.md | 77 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 skills/rca-build/references/github-evidence.md diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 4d42f17..60b9753 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -99,6 +99,18 @@ across modes. Do not fork the loop; only the gap action differs. When all gaps i a turn are resolvable (gathered or user-answered), the loop proceeds normally to a terminal `RCA_OUTPUT`. +## Suspect-PR falsification (github asks) + +For `product_code` / `deploy` / `ci` asks, follow `references/github-evidence.md`: +gather the **exact** evidence (diff-since-baseline, PRs-in-window touching the +failing path, blame, deploy timing) via **GitHub MCP → `gh` → degrade**, and for +each candidate suspect **try to disprove it** (path overlap? shipped before the +failure window? behind an OFF flag?). Feed both supporting *and* disconfirming +evidence back as a structured suspect packet; only `verdict: supported` suspects +belong in `related_prs`. Reuse the pre-computed build-level evidence — do not +re-fetch per test. Never fabricate a PR when the github capability is unavailable +— emit an `unavailable` block. + ## The loop ``` diff --git a/skills/rca-build/references/github-evidence.md b/skills/rca-build/references/github-evidence.md new file mode 100644 index 0000000..fa24aaa --- /dev/null +++ b/skills/rca-build/references/github-evidence.md @@ -0,0 +1,77 @@ +# GitHub evidence — what to gather, and how to rule a suspect OUT + +The worst automated-RCA outcome is **confidently blaming an innocent PR**. This +file is the contract for `product_code` / `deploy` / `ci` asks (the `github` +capability): the **exact** evidence to gather, and a **falsification protocol** +that tries to *disprove* each suspect before it enters `related_prs`. + +> We do **not** ship a GitHub forensics harness or MCP tool. We specify what's +> needed and use whatever the client already has — **GitHub MCP if available, +> else `gh`, else degrade** to an `unavailable` block. + +## Capability discovery (in order) + +1. **GitHub MCP** (`mcp__github__*`) — preferred for structured PR/diff/blame queries. +2. **`gh` CLI** — fall back for git-graph operations (`gh pr list --search`, + `gh api`, `merge-base`, ancestry) and anything the MCP doesn't cover. +3. **Neither** → emit an `unavailable` block for the ask (do not fabricate a PR). + +The orchestrator records which is present in the capability manifest +(`capability: github → { available, via }`); route every github ask against it. + +## Evidence each ask needs (be specific — no fishing) + +| Ask intent | Gather exactly | +|---|---| +| "Did `<X>` change since the last passing run?" | the diff of `<X>`'s file/function between the **baseline ref** (last-green, or the configured fallback) and the build's commit — not the whole repo diff | +| "Which PRs are suspect?" | PRs **merged in the window** `(baselineRef, build commit]` that **touch the failing code path** — intersect changed files with the failing file/function | +| "Who/what last changed the failing line?" | `blame` on the specific failing lines (from the test's `file_path` + the error) | +| "What shipped to the run's env before the failure?" | deploy timeline (`gh` releases/tags + the env's deploy record); compare deploy time vs. the run's `started_at` | +| "Did CI change?" | the workflow-file diff + recent `gh run` history for the failing job | + +Scope everything by the failing test's `file_path` + the error summary. The +build-level evidence (diff-since-last-green, PR window) is **pre-computed once** +and passed in — reuse it; do not re-fetch per test. + +## Falsification protocol — rule out, don't just rule in + +For **each** candidate suspect PR, try to **break** the hypothesis: + +1. **Path overlap.** Do the PR's changed hunks actually touch the failing code + path (the function/line in the stack)? No overlap → **ruled out**. +2. **Deployment-state guard.** Was the PR's code actually **live** in the run's + env at `started_at`? If it shipped *after* the failure window, or sits behind + an **OFF** flag, it could not have caused this failure → **ruled out**. +3. **Direction.** Does the change plausibly produce *this* error (e.g. a validator + tightened to reject the input the test sends)? If the change is unrelated to + the symptom → **weak**, mark accordingly. + +Feed **both supporting and disconfirming** evidence back to TFA. A suspect that +survives 1–3 is a real candidate; one that fails any is reported as ruled-out +(with the reason), **not** dropped silently. + +## The suspect packet (structured, not free text) + +Each surviving/ruled-out suspect is one structured block so `related_prs` +populates deterministically: + +``` +SUSPECT: + pr: <#number> + files: <changed files overlapping the failing path> + hunks: <the 1-3 load-bearing changed hunks — see digest size caps> + author: <login> + merged_at: <ts> vs last_green: <ts> vs started_at: <ts> + verdict: supported | ruled-out (<reason: no-path-overlap | shipped-after | behind-off-flag | unrelated>) + link: <PR permalink> +``` + +Only `verdict: supported` suspects should end up in TFA's `related_prs`. Ruled-out +suspects stay in the thread as disconfirming evidence so TFA (and a human) can see +the elimination, not just the conclusion. + +## Digest discipline + +Same caps as `references/evidence-routing.md`: prefer a PR **link** over pasting a +diff; at most 1 hunk (3 hard) per `product_code` snippet; never paste a full diff. +The packet is *findings*, not the haystack. From c5a4e9ecb219a3fd52ca7bdecc0365069f543b6d Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:11:34 +0530 Subject: [PATCH 09/47] feat(rca): coverage stamp + degrade-don't-crash report (resume reaper in U4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lib/coverage.mjs derives a per-row evidence-coverage band — TFA confidence capped by coverage (full keeps it, partial→medium, thin→low) so a RESOLVED built with evidence unavailable reads as lower confidence BECAUSE of the gap. lib/report.mjs renders the CSV to markdown: status counts + per-test table + coverage caveats, degrading missing fields to 'not available' and never crashing on an empty/partial batch. report-format.md documents the stamp, layout, and the startup reaper resume path. Blast-radius digest explicitly deferred. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- lib/coverage.mjs | 39 +++++++ lib/report.mjs | 69 ++++++++++++ skills/rca-build/references/report-format.md | 45 ++++++++ tests/coverage-report.test.mjs | 108 +++++++++++++++++++ 4 files changed, 261 insertions(+) create mode 100644 lib/coverage.mjs create mode 100644 lib/report.mjs create mode 100644 skills/rca-build/references/report-format.md create mode 100644 tests/coverage-report.test.mjs diff --git a/lib/coverage.mjs b/lib/coverage.mjs new file mode 100644 index 0000000..caff2d2 --- /dev/null +++ b/lib/coverage.mjs @@ -0,0 +1,39 @@ +// Evidence-coverage stamp (ideation #6, v1 — the per-row coverage band; the +// build-level blast-radius digest is deferred). A RESOLVED RCA built with +// k8s+kibana+metrics all "unavailable" must not read like one with full +// evidence. The client (which routed every ask) stamps each row with a coverage +// vector and derives a coverage-capped confidence band the reviewer sees: +// "low confidence BECAUSE kibana was unavailable", not "low confidence, trust me". + +const BAND_ORDER = ["low", "medium", "high"]; + +// coverage classification from what was fulfilled vs. left unavailable. +export function classifyCoverage(asksFulfilled = [], asksUnavailable = []) { + const unavailable = [...new Set(asksUnavailable.filter(Boolean))]; + const fulfilled = [...new Set(asksFulfilled.filter(Boolean))]; + if (unavailable.length === 0) return "full"; + if (fulfilled.length > 0) return "partial"; + return "thin"; +} + +// Cap the band: full coverage keeps TFA's confidence; partial caps at medium; +// thin caps at low. Unknown/absent TFA confidence floors to low. +function capBand(tfaConfidence, coverage) { + const base = BAND_ORDER.includes(tfaConfidence) ? tfaConfidence : "low"; + const cap = coverage === "full" ? "high" : coverage === "partial" ? "medium" : "low"; + return BAND_ORDER[Math.min(BAND_ORDER.indexOf(base), BAND_ORDER.indexOf(cap))]; +} + +// The stamp written to a row at flip time. Returns { coverage, band, unavailable }. +export function coverageStamp({ + asksFulfilled = [], + asksUnavailable = [], + tfaConfidence = "unknown", +} = {}) { + const coverage = classifyCoverage(asksFulfilled, asksUnavailable); + return { + coverage, + band: capBand(tfaConfidence, coverage), + unavailable: [...new Set(asksUnavailable.filter(Boolean))], + }; +} diff --git a/lib/report.mjs b/lib/report.mjs new file mode 100644 index 0000000..84ae275 --- /dev/null +++ b/lib/report.mjs @@ -0,0 +1,69 @@ +// Deterministic markdown report for a finished (or partial) batch. Degrade, +// don't crash: any missing field renders as "not available"; an empty batch +// still renders a valid report. Reads the CSV/WAL spine; no per-test transcripts. + +import { readRows } from "./csv-state.mjs"; + +const NA = "not available"; + +function cell(value) { + const s = value == null ? "" : String(value).trim(); + if (s === "") return NA; + // keep the table one-line-per-row: collapse newlines, escape pipes + return s.replace(/\s*\n\s*/g, " ").replace(/\|/g, "\\|"); +} + +function countBy(rows, key) { + return rows.reduce((acc, r) => { + const k = r[key] || "unknown"; + acc[k] = (acc[k] ?? 0) + 1; + return acc; + }, {}); +} + +// Render from a rows array (testable) — or pass a csvPath via renderReportFromCsv. +export function renderReport(rows, { buildId, generatedAt } = {}) { + const lines = []; + lines.push(`# RCA report${buildId ? ` — build ${buildId}` : ""}`); + if (generatedAt) lines.push(`\nGenerated: ${generatedAt}`); + + if (!rows || rows.length === 0) { + lines.push("\nNo failed tests analyzed."); + return lines.join("\n") + "\n"; + } + + const byState = countBy(rows, "rca_done"); + const summary = Object.entries(byState) + .map(([k, v]) => `${k}: ${v}`) + .join(" · "); + lines.push(`\n**${rows.length} test(s)** — ${summary}\n`); + + lines.push( + "| testRunId | test | status | confidence | coverage | root cause | related PRs |", + ); + lines.push("|---|---|---|---|---|---|---|"); + for (const r of rows) { + lines.push( + `| ${cell(r.testRunId)} | ${cell(r.testName)} | ${cell(r.rca_done)} | ${cell( + r.confidence, + )} | ${cell(r.coverage)} | ${cell(r.root_cause)} | ${cell(r.related_prs)} |`, + ); + } + + // Surface coverage caveats so a "low confidence" reads as "because X unavailable". + const thin = rows.filter((r) => r.coverage === "thin" || r.coverage === "partial"); + if (thin.length > 0) { + lines.push(`\n## Coverage caveats`); + for (const r of thin) { + lines.push( + `- ${cell(r.testRunId)} (${cell(r.coverage)} coverage): confidence band reflects evidence that was unavailable, not just model certainty.`, + ); + } + } + + return lines.join("\n") + "\n"; +} + +export function renderReportFromCsv(csvPath, opts = {}) { + return renderReport(readRows(csvPath), opts); +} diff --git a/skills/rca-build/references/report-format.md b/skills/rca-build/references/report-format.md new file mode 100644 index 0000000..e27a28d --- /dev/null +++ b/skills/rca-build/references/report-format.md @@ -0,0 +1,45 @@ +# Report format, coverage stamp, and resume + +## The CSV is the source of truth + +Every per-test result lives as one CSV row (`lib/csv-state.mjs`, columns in +`COLUMNS`). The report is a deterministic render of that CSV — no per-test +transcripts are kept. `rca_done` ∈ `pending | resolved | blocked | failed | +pending-resume`. + +## Coverage stamp (ideation #6, v1) + +At flip time the orchestrator stamps each row (`lib/coverage.mjs`) from the +coordinator's `asks_fulfilled` / `asks_unavailable` + TFA's confidence: + +- **coverage** — `full` (no gaps) · `partial` (some fulfilled, some unavailable) · + `thin` (nothing fulfilled, only gaps). +- **band** — TFA's confidence **capped by coverage**: `full` keeps it, `partial` + caps at `medium`, `thin` caps at `low`; unknown floors to `low`. + +So a RESOLVED with kibana/k8s unavailable reads as a lower band *because* evidence +was missing — not the same as a fully-evidenced RESOLVED. The report's **Coverage +caveats** section spells this out per affected row. + +> Out of v1 scope: the build-level **blast-radius digest** (rows inverted by +> culprit PR, ranked) — deferred to follow-up. The per-row coverage stamp ships now. + +## Report layout (`lib/report.mjs` → `renderReport`) + +- Header + build id + generated-at. +- One-line summary: total + counts by `rca_done`. +- A per-test table: `testRunId | test | status | confidence | coverage | root cause | related PRs`. +- A **Coverage caveats** list for `partial`/`thin` rows. + +**Degrade, don't crash:** any missing field renders as `not available`; an empty +batch renders "No failed tests analyzed."; pipes are escaped and newlines +collapsed so the table never breaks. + +## Resume (ideation #7) + +On startup the orchestrator runs the **reaper** (`lib/csv-state.mjs` → `reaper`): +rows stuck `in_flight` with a heartbeat older than `reaperHeartbeatTtlSec` are +reclaimed to `pending` (a crashed worker's rows), then fan-out re-points at the +CSV. A row that retains a live `threadId`/`turnId` resumes that TFA thread; a dead +thread re-runs from `pending`. In-session / in-workspace only — cross-session +durability is deferred. diff --git a/tests/coverage-report.test.mjs b/tests/coverage-report.test.mjs new file mode 100644 index 0000000..7d7cb94 --- /dev/null +++ b/tests/coverage-report.test.mjs @@ -0,0 +1,108 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { coverageStamp, classifyCoverage } from "../lib/coverage.mjs"; +import { renderReport } from "../lib/report.mjs"; + +// ---- coverage stamp -------------------------------------------------------- + +test("full coverage keeps TFA confidence", () => { + const s = coverageStamp({ + asksFulfilled: ["product_code"], + asksUnavailable: [], + tfaConfidence: "high", + }); + assert.equal(s.coverage, "full"); + assert.equal(s.band, "high"); +}); + +test("partial coverage caps a high TFA confidence at medium", () => { + const s = coverageStamp({ + asksFulfilled: ["product_code"], + asksUnavailable: ["kibana"], + tfaConfidence: "high", + }); + assert.equal(s.coverage, "partial"); + assert.equal(s.band, "medium"); + assert.deepEqual(s.unavailable, ["kibana"]); +}); + +test("thin coverage (nothing fulfilled, gaps) caps at low", () => { + const s = coverageStamp({ + asksFulfilled: [], + asksUnavailable: ["k8s", "metrics"], + tfaConfidence: "high", + }); + assert.equal(s.coverage, "thin"); + assert.equal(s.band, "low"); +}); + +test("unknown TFA confidence floors to low even at full coverage", () => { + const s = coverageStamp({ asksFulfilled: [], asksUnavailable: [], tfaConfidence: "unknown" }); + assert.equal(s.coverage, "full"); + assert.equal(s.band, "low"); +}); + +test("classifyCoverage dedupes and handles empties", () => { + assert.equal(classifyCoverage(["a", "a"], []), "full"); + assert.equal(classifyCoverage([], ["x"]), "thin"); +}); + +// ---- report ---------------------------------------------------------------- + +test("empty batch renders a valid report, no crash", () => { + const md = renderReport([], { buildId: "b1" }); + assert.match(md, /No failed tests analyzed/); +}); + +test("report renders a row table with status counts", () => { + const rows = [ + { + testRunId: "101", + testName: "login", + rca_done: "resolved", + confidence: "high", + coverage: "full", + root_cause: "PR #7421 tightened validator", + related_prs: "#7421", + }, + { + testRunId: "102", + testName: "checkout", + rca_done: "blocked", + confidence: "", + coverage: "", + root_cause: "", + related_prs: "", + }, + ]; + const md = renderReport(rows, { buildId: "b1" }); + assert.match(md, /2 test\(s\)/); + assert.match(md, /resolved: 1/); + assert.match(md, /blocked: 1/); + assert.match(md, /101/); + assert.match(md, /not available/); // 102's blank fields degrade +}); + +test("report escapes pipes and collapses newlines in cells", () => { + const rows = [ + { + testRunId: "1", + testName: "t", + rca_done: "resolved", + root_cause: "a | b\nsecond line", + related_prs: "#1", + }, + ]; + const md = renderReport(rows); + assert.ok(!md.includes("a | b\nsecond")); + assert.match(md, /a \\\| b second line/); +}); + +test("report surfaces coverage caveats for thin/partial rows", () => { + const rows = [ + { testRunId: "1", testName: "t", rca_done: "resolved", coverage: "partial" }, + ]; + const md = renderReport(rows); + assert.match(md, /Coverage caveats/); + assert.match(md, /confidence band reflects evidence that was unavailable/); +}); From e277811ecbdd58fc3bdbf18732615560a7cc26cb Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:14:25 +0530 Subject: [PATCH 10/47] feat(rca): conformance fixture + executable loop mirror / sequential harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lib/loop.mjs (runRcaLoop) is an executable mirror of the coordinator loop — status branching, ask routing, gap resolution, turn-cap, one-thread, soft-PENDING — driven by an injected submit(). It doubles as the D5 sequential thin-client harness. tests/conformance.test.mjs replays recorded tfaRcaTurn transcripts (resolved/blocked/pending/turn-cap fixtures) and proves: rca capture, test_logs skip, soft-PENDING no-re-poll, turn-cap never submits a 7th turn, and the degraded (no-capability auto) path still reaches a valid terminal RCA — same loop, same result. 48 tests green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- agents/ai-tfa-coordinator.md | 6 + lib/loop.mjs | 125 ++++++++++++++++++ tests/conformance.test.mjs | 133 ++++++++++++++++++++ tests/fixtures/recorded-turns/blocked.json | 13 ++ tests/fixtures/recorded-turns/pending.json | 12 ++ tests/fixtures/recorded-turns/resolved.json | 37 ++++++ tests/fixtures/recorded-turns/turn-cap.json | 12 ++ 7 files changed, 338 insertions(+) create mode 100644 lib/loop.mjs create mode 100644 tests/conformance.test.mjs create mode 100644 tests/fixtures/recorded-turns/blocked.json create mode 100644 tests/fixtures/recorded-turns/pending.json create mode 100644 tests/fixtures/recorded-turns/resolved.json create mode 100644 tests/fixtures/recorded-turns/turn-cap.json diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 60b9753..8f5cb4f 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -139,6 +139,12 @@ re-fetch per test. Never fabricate a PR when the github capability is unavailabl 6. EMIT the RCA_OUTPUT block from the captured terminal state. ``` +> The loop mechanics above have an **executable mirror** in `lib/loop.mjs` +> (`runRcaLoop`) — conformance-tested against recorded `tfaRcaTurn` transcripts +> (`tests/conformance.test.mjs`). It also serves as the **sequential thin-client +> harness** (D5): MCP clients without workflows/subagents drive the same contract +> by calling `runRcaLoop` with a real `submit` bound to `tfaRcaTurn`. + **Sibling confirm (cluster member).** When `pre_seed` is present the first turn states the representative's hypothesis and asks TFA to confirm against this test's own logs. If TFA `RESOLVED`s in one turn → a logs-grounded per-test RCA at diff --git a/lib/loop.mjs b/lib/loop.mjs new file mode 100644 index 0000000..1d1728d --- /dev/null +++ b/lib/loop.mjs @@ -0,0 +1,125 @@ +// Executable mirror of the ai-tfa-coordinator loop (agents/ai-tfa-coordinator.md). +// It drives the collaborative loop against an injected `submit` (real = the +// tfaRcaTurn MCP tool; tests = a recorded-turn replayer), so the loop mechanics — +// status branching, ask routing, gap resolution, turn-cap, one-thread, +// soft-PENDING — are tested rather than assumed. +// +// Double duty: this is ALSO the **sequential thin-client harness** (D5 / ideation +// #4) — the third caller of the same contract, for MCP clients without +// workflows/subagents. Pure + dependency-light (imports only the routing registry). + +import { routeAsks } from "./routing.mjs"; + +function unavailableBlock(gap) { + const what = gap?.ask?.what ?? ""; + return [ + `ASK: ${what}`, + `TYPE: ${gap.evidenceType}`, + `FOUND: no`, + `SUMMARY: unavailable — no ${gap.capability} capability for this client.`, + ].join("\n"); +} + +// runRcaLoop drives one test to a terminal RCA_OUTPUT object. +// +// submit({ testRunId, message, threadId, turnId }) → Promise<turn> (tfaRcaTurn shape) +// gather(routedGatherEntry) → Promise<string> (one digest block) +// resolveGap(routedGapEntry) → Promise<{ digest } | null> (auto: null; interactive: a digest) +export async function runRcaLoop({ + testRunId, + firstMessage = "", + submit, + config = {}, + manifest = {}, + gather = async () => "", + resolveGap = async () => null, + turnCap = config?.turnCap ?? 6, +}) { + if (testRunId == null || Number.isNaN(Number(testRunId))) { + return { + testRunId: String(testRunId), + status: "failed", + root_cause: "no testRunId provided", + turns_used: 0, + asks_fulfilled: [], + asks_skipped: [], + asks_unavailable: [], + }; + } + + let threadId; + let turnId; + let turns = 0; + let message = firstMessage; + const fulfilled = new Set(); + const skipped = new Set(); + const unavailable = new Set(); + + const out = (status, turn, note) => { + const rca = turn?.rca ?? {}; + return { + testRunId: String(testRunId), + status, + confidence: turn?.confidence ?? "unknown", + root_cause: + status === "RESOLVED" + ? (rca.root_cause ?? "") + : status === "BLOCKED" + ? (turn?.reason ?? "") + : (note ?? ""), + possible_fix: rca.possible_fix ?? "", + related_prs: rca.related_prs ?? [], + threadId: threadId ?? null, + turnId: turnId ?? null, + turns_used: turns, + asks_fulfilled: [...fulfilled], + asks_skipped: [...skipped], + asks_unavailable: [...unavailable], + }; + }; + + while (true) { + turns++; + const turn = await submit({ testRunId, message, threadId, turnId }); + threadId = turn.threadId ?? threadId; + + if (turn.status === "RESOLVED") return out("RESOLVED", turn); + if (turn.status === "BLOCKED") return out("BLOCKED", turn); + if (turn.status === "PENDING") { + turnId = turn.turnId ?? turnId; + return out("PENDING", turn, "soft-pending"); + } + + // NEEDS_INFO: route + fulfill. + const buckets = routeAsks(turn.asks ?? [], config, manifest); + const blocks = []; + for (const s of buckets.skip) skipped.add(s.evidenceType); + for (const g of buckets.gather) { + blocks.push(await gather(g)); + fulfilled.add(g.evidenceType); + } + for (const gap of buckets.gap) { + const resolved = await resolveGap(gap); + if (resolved && resolved.digest) { + blocks.push(resolved.digest); + fulfilled.add(gap.evidenceType); + } else { + unavailable.add(gap.evidenceType); + blocks.push(unavailableBlock(gap)); + } + } + + if (turns >= turnCap) return out("PENDING", turn, "turn-cap"); + message = blocks.join("\n\n"); + } +} + +// Replay helper for tests: returns a submit() that yields recorded turns in order. +export function replaySubmit(turns) { + let i = 0; + return async () => { + const turn = turns[Math.min(i, turns.length - 1)]; + i++; + return turn; + }; +} diff --git a/tests/conformance.test.mjs b/tests/conformance.test.mjs new file mode 100644 index 0000000..5ea6192 --- /dev/null +++ b/tests/conformance.test.mjs @@ -0,0 +1,133 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import { runRcaLoop, replaySubmit } from "../lib/loop.mjs"; + +const here = dirname(fileURLToPath(import.meta.url)); +const load = (name) => + JSON.parse(readFileSync(join(here, "fixtures", "recorded-turns", name), "utf8")); + +const CONFIG = { + turnCap: 6, + evidenceRouting: { + test_logs: { owner: "tfa", skip: true }, + product_code: { capability: "github" }, + other: { capability: "other" }, + }, +}; +const GITHUB_AVAILABLE = { github: { available: true, via: "gh" } }; + +// A coordinator gather() stub: returns a one-line digest block. +const gather = async (g) => `ASK: ${g.ask.what}\nTYPE: ${g.evidenceType}\nFOUND: yes\nSUMMARY: stub`; + +test("resolved fixture: NEEDS_INFO → evidence → RESOLVED, rca captured, test_logs skipped", async () => { + const fx = load("resolved.json"); + const result = await runRcaLoop({ + testRunId: fx.testRunId, + firstMessage: "Error: empty buildName", + submit: replaySubmit(fx.turns), + config: CONFIG, + manifest: GITHUB_AVAILABLE, + gather, + }); + assert.equal(result.status, "RESOLVED"); + assert.match(result.root_cause, /#7421/); + assert.deepEqual(result.related_prs, ["#7421"]); + assert.deepEqual(result.asks_fulfilled, ["product_code"]); + assert.deepEqual(result.asks_skipped, ["test_logs"]); // TFA-owned, never gathered + assert.equal(result.turns_used, 2); + assert.equal(result.threadId, "thr-39"); +}); + +test("blocked fixture: terminal with reason captured", async () => { + const fx = load("blocked.json"); + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit(fx.turns), + config: CONFIG, + }); + assert.equal(result.status, "BLOCKED"); + assert.match(result.root_cause, /could not obtain server-side logs/); +}); + +test("pending fixture: soft-PENDING ends with turnId, no re-poll", async () => { + const fx = load("pending.json"); + let calls = 0; + const counting = async (args) => { + calls++; + return replaySubmit(fx.turns)(args); + }; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: counting, + config: CONFIG, + }); + assert.equal(result.status, "PENDING"); + assert.equal(result.turnId, "turn-81-1"); + assert.equal(calls, 1); // ended immediately, did not poll again +}); + +test("turn-cap fixture: ends PENDING(turn-cap) at the cap, never a 7th submit", async () => { + const fx = load("turn-cap.json"); + let submits = 0; + const counting = async (args) => { + submits++; + return replaySubmit(fx.turns)(args); + }; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: counting, + config: CONFIG, + manifest: GITHUB_AVAILABLE, + gather, + }); + assert.equal(result.status, "PENDING"); + assert.equal(result.root_cause, "turn-cap"); + assert.equal(submits, 6); // capped at turnCap, never 7 +}); + +test("degraded path: no capability + auto resolveGap → asks_unavailable, still terminal", async () => { + // Same resolved fixture, but the client has NO github capability and runs auto + // (resolveGap returns null → 'unavailable'). The loop must still reach RESOLVED. + const fx = load("resolved.json"); + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit(fx.turns), + config: CONFIG, + manifest: {}, // nothing available + resolveGap: async () => null, // auto: report unavailable + }); + assert.equal(result.status, "RESOLVED"); + assert.deepEqual(result.asks_unavailable, ["product_code"]); + assert.deepEqual(result.asks_fulfilled, []); +}); + +test("interactive resolveGap supplies the missing evidence → fulfilled, not unavailable", async () => { + const fx = load("resolved.json"); + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit(fx.turns), + config: CONFIG, + manifest: {}, + resolveGap: async () => ({ digest: "ASK: ...\nFOUND: yes\nSUMMARY: user supplied" }), + }); + assert.equal(result.status, "RESOLVED"); + assert.deepEqual(result.asks_fulfilled, ["product_code"]); + assert.deepEqual(result.asks_unavailable, []); +}); + +test("no testRunId → failed block, tool never called", async () => { + let called = false; + const result = await runRcaLoop({ + testRunId: undefined, + submit: async () => { + called = true; + return {}; + }, + config: CONFIG, + }); + assert.equal(result.status, "failed"); + assert.equal(called, false); +}); diff --git a/tests/fixtures/recorded-turns/blocked.json b/tests/fixtures/recorded-turns/blocked.json new file mode 100644 index 0000000..35b5373 --- /dev/null +++ b/tests/fixtures/recorded-turns/blocked.json @@ -0,0 +1,13 @@ +{ + "name": "blocked — unmet asks", + "testRunId": 72, + "turns": [ + { + "status": "BLOCKED", + "confidence": "low", + "threadId": "thr-72", + "reason": "could not obtain server-side logs; cannot distinguish product bug from env flake", + "unmetAsks": ["kibana", "k8s"] + } + ] +} diff --git a/tests/fixtures/recorded-turns/pending.json b/tests/fixtures/recorded-turns/pending.json new file mode 100644 index 0000000..ec8b16a --- /dev/null +++ b/tests/fixtures/recorded-turns/pending.json @@ -0,0 +1,12 @@ +{ + "name": "soft-pending — resumable", + "testRunId": 81, + "turns": [ + { + "status": "PENDING", + "confidence": "unknown", + "threadId": "thr-81", + "turnId": "turn-81-1" + } + ] +} diff --git a/tests/fixtures/recorded-turns/resolved.json b/tests/fixtures/recorded-turns/resolved.json new file mode 100644 index 0000000..120dad6 --- /dev/null +++ b/tests/fixtures/recorded-turns/resolved.json @@ -0,0 +1,37 @@ +{ + "name": "needs_info → evidence → resolved", + "testRunId": 39, + "turns": [ + { + "status": "NEEDS_INFO", + "confidence": "low", + "threadId": "thr-39", + "questions": ["Did the buildName validator change?"], + "asks": [ + { + "what": "Did request-validation on POST /builds change since last green?", + "why": "the failing test posts an empty buildName", + "evidenceType": "product_code", + "priority": "high" + }, + { + "what": "Full run logs for test 39", + "why": "to read the failure", + "evidenceType": "test_logs", + "priority": "high" + } + ] + }, + { + "status": "RESOLVED", + "confidence": "high", + "threadId": "thr-39", + "rca": { + "root_cause": "PR #7421 tightened the buildName validator to reject empty strings", + "possible_fix": "send a non-empty buildName or relax the validator", + "failure_type": "product_regression", + "related_prs": ["#7421"] + } + } + ] +} diff --git a/tests/fixtures/recorded-turns/turn-cap.json b/tests/fixtures/recorded-turns/turn-cap.json new file mode 100644 index 0000000..4638b61 --- /dev/null +++ b/tests/fixtures/recorded-turns/turn-cap.json @@ -0,0 +1,12 @@ +{ + "name": "turn-cap — never resolves", + "testRunId": 99, + "turns": [ + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] }, + { "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-99", "asks": [{ "what": "more", "why": "x", "evidenceType": "product_code", "priority": "high" }] } + ] +} From e9331afce51c173725c1b129a4d3e664f630e20f Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:20:27 +0530 Subject: [PATCH 11/47] fix(rca): make pending-resume resumable, enforce flip terminal status, skip turn-cap gather MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code-review fixes (suggested, non-blocking): - pending-resume removed from TERMINAL_STATES → soft-PENDING rows are now re-claimable, listed by pendingRows, and skipped by the reaper (they cleared in_flight), so the retained threadId/turnId actually drive an in-session resume instead of being stranded as a permanent non-terminal terminal. - flip() now rejects a missing/non-terminal rca_done without mutating, so a partial flip can't clear the claim yet leave the row pending (duplicate-RCA clobber). - loop checks the turn-cap BEFORE gathering, so evidence on the never-submitted final turn isn't gathered for nothing. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- lib/csv-state.mjs | 26 ++++++++++++++++++-------- lib/loop.mjs | 8 ++++++-- tests/csv-state.test.mjs | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/lib/csv-state.mjs b/lib/csv-state.mjs index 499f997..ea830ff 100644 --- a/lib/csv-state.mjs +++ b/lib/csv-state.mjs @@ -40,12 +40,13 @@ export const COLUMNS = [ ]; export const PENDING = "pending"; -const TERMINAL_STATES = new Set([ - "resolved", - "blocked", - "failed", - "pending-resume", -]); +export const RESUMABLE = "pending-resume"; +// Truly done — never re-claimed, listed, or reaped. +const TERMINAL_STATES = new Set(["resolved", "blocked", "failed"]); +// Valid outcomes flip() may write. `pending-resume` is a *soft* terminal: this +// attempt ended (claim cleared) but the row stays resumable — it keeps its +// threadId/turnId and is picked back up by the next fan-out / resume pass. +const FLIP_STATES = new Set(["resolved", "blocked", "failed", RESUMABLE]); // ---- minimal RFC4180-ish CSV codec ---------------------------------------- @@ -199,6 +200,11 @@ export function heartbeat(csvPath, testRunId, worker, nowMs) { // possible_fix, related_prs, threadId, turnId, coverage, confidence, // last_evidence_digest, cluster_id. export function flip(csvPath, testRunId, fields, nowMs) { + // Enforce the contract: a flip must name a valid outcome. A partial flip with + // a missing/non-terminal rca_done would otherwise clear the claim yet leave the + // row `pending` — re-exposing it for a duplicate RCA that clobbers this result. + // Reject without mutating so the worker keeps its claim and the bug surfaces. + if (!FLIP_STATES.has(fields?.rca_done)) return false; const rows = readRows(csvPath); const row = rows.find((r) => String(r.testRunId) === String(testRunId)); if (!row) return false; @@ -233,7 +239,11 @@ export function reaper(csvPath, ttlSec, nowMs) { return reclaimed; } -// Rows still needing work (pending or reclaimed). The work-list for fan-out. +// Rows still needing work: fresh/reclaimed `pending` AND `pending-resume` rows +// (soft-PENDING attempts that retain a threadId/turnId to resume). The fan-out +// work-list. Truly terminal rows (resolved/blocked/failed) are excluded. export function pendingRows(csvPath) { - return readRows(csvPath).filter((r) => r.rca_done === PENDING); + return readRows(csvPath).filter( + (r) => r.rca_done === PENDING || r.rca_done === RESUMABLE, + ); } diff --git a/lib/loop.mjs b/lib/loop.mjs index 1d1728d..9e59eb2 100644 --- a/lib/loop.mjs +++ b/lib/loop.mjs @@ -90,7 +90,12 @@ export async function runRcaLoop({ return out("PENDING", turn, "soft-pending"); } - // NEEDS_INFO: route + fulfill. + // NEEDS_INFO. Check the turn-cap BEFORE gathering — evidence assembled on a + // turn we will never submit is wasted work (and a side-effecting gather() + // would run for nothing). + if (turns >= turnCap) return out("PENDING", turn, "turn-cap"); + + // Route + fulfill. const buckets = routeAsks(turn.asks ?? [], config, manifest); const blocks = []; for (const s of buckets.skip) skipped.add(s.evidenceType); @@ -109,7 +114,6 @@ export async function runRcaLoop({ } } - if (turns >= turnCap) return out("PENDING", turn, "turn-cap"); message = blocks.join("\n\n"); } } diff --git a/tests/csv-state.test.mjs b/tests/csv-state.test.mjs index 5a9a60f..ca55d84 100644 --- a/tests/csv-state.test.mjs +++ b/tests/csv-state.test.mjs @@ -120,6 +120,39 @@ test("pendingRows returns only pending work", () => { assert.equal(pend[0].testRunId, "102"); }); +test("flip rejects a missing/non-terminal rca_done without mutating the row", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + // missing rca_done + assert.equal(flip(csv, 101, { root_cause: "x" }, 2000), false); + // invalid rca_done + assert.equal(flip(csv, 101, { rca_done: "weird" }, 2000), false); + const row = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(row.rca_done, PENDING); // not reverted to claimable-pending silently + assert.equal(row.in_flight_worker, "w1"); // claim intact — bug surfaces, no clobber + assert.equal(row.root_cause, ""); // nothing written +}); + +test("pending-resume is resumable: not terminal, listed, and re-claimable", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + flip(csv, 101, { rca_done: "pending-resume", threadId: "thr-1", turnId: "t-1" }, 2000); + const row = readRows(csv).find((r) => r.testRunId === "101"); + assert.equal(row.in_flight_worker, ""); // this attempt released the claim + assert.equal(row.threadId, "thr-1"); // resume handles retained + assert.equal(row.turnId, "t-1"); + // appears in the fan-out work-list and can be claimed by the resume pass + assert.ok(pendingRows(csv).some((r) => r.testRunId === "101")); + assert.equal(claim(csv, 101, "w2", 3000), true); +}); + +test("reaper ignores pending-resume rows (not in flight)", () => { + seed(csv, "build-1", TESTS); + claim(csv, 101, "w1", 1000); + flip(csv, 101, { rca_done: "pending-resume" }, 2000); + assert.deepEqual(reaper(csv, 600, 10_000_000), []); +}); + test("CSV codec round-trips fields with commas, quotes, newlines", () => { seed(csv, "build-1", [{ test_id: 200, test_name: "weird" }]); flip( From 3d22dfe6d9e4cebbc00824749553a43b3c5e38b7 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:27:42 +0530 Subject: [PATCH 12/47] chore(rca): gitignore local planning docs Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 9045f9d..4c14fb3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ node_modules/ .env # Per-run RCA batch state (the CSV/WAL spine + report) is workspace-local. .rca/ +# Planning docs (brainstorm/ideation/plan) stay local — not pushed. +docs/ From 79e4c8703f6bb632af5a919fa21a26e267daf5be Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 23 Jun 2026 22:57:52 +0530 Subject: [PATCH 13/47] feat(rca): cross-client integration for Cursor and Codex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirror the slack-mcp-plugin parallel-manifest pattern over one shared skills/ tree: add .cursor-plugin/plugin.json + .cursor-mcp.json (stdio bstack, Cursor dialect) and codex-mcp.example.toml (~/.codex/config.toml block). INTEGRATION.md documents per-host wiring — MCP config, Agent-Skills skill/agent discovery (.cursor/skills, .agents/skills, .codex/agents), the Add-to-Cursor deeplink and codex mcp add — and the one Claude-only piece (auto-mode dynamic workflow), which Cursor/Codex fill with interactive subagents or the sequential lib/loop.mjs harness. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- .cursor-mcp.json | 13 +++++ .cursor-plugin/plugin.json | 8 ++++ INTEGRATION.md | 98 ++++++++++++++++++++++++++++++++++++++ README.md | 10 ++++ codex-mcp.example.toml | 11 +++++ 5 files changed, 140 insertions(+) create mode 100644 .cursor-mcp.json create mode 100644 .cursor-plugin/plugin.json create mode 100644 INTEGRATION.md create mode 100644 codex-mcp.example.toml diff --git a/.cursor-mcp.json b/.cursor-mcp.json new file mode 100644 index 0000000..9cf95af --- /dev/null +++ b/.cursor-mcp.json @@ -0,0 +1,13 @@ +{ + "mcpServers": { + "bstack": { + "command": "npx", + "args": ["-y", "@browserstack/mcp-server"], + "env": { + "BROWSERSTACK_USERNAME": "${BROWSERSTACK_USERNAME}", + "BROWSERSTACK_ACCESS_KEY": "${BROWSERSTACK_ACCESS_KEY}", + "O11Y_TFA_RCA_BASE_URL": "${O11Y_TFA_RCA_BASE_URL}" + } + } + } +} diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json new file mode 100644 index 0000000..e7998b7 --- /dev/null +++ b/.cursor-plugin/plugin.json @@ -0,0 +1,8 @@ +{ + "name": "tfa-rca", + "description": "Collaborative root-cause analysis over all failed tests of a BrowserStack build, generic across product and infra.", + "version": "0.1.0", + "mcpServers": "../.cursor-mcp.json", + "skills": "./skills/", + "author": { "name": "BrowserStack" } +} diff --git a/INTEGRATION.md b/INTEGRATION.md new file mode 100644 index 0000000..9d91cb5 --- /dev/null +++ b/INTEGRATION.md @@ -0,0 +1,98 @@ +# Multi-client integration (Claude Code · Cursor · Codex) + +This plugin is built so the **MCP core is truly cross-client** and the **harness +layer ports via the cross-vendor Agent Skills standard**. Only one piece is +genuinely Claude-Code-specific (the auto-mode *dynamic workflow*); on Cursor and +Codex that role is filled by the sequential harness or interactive subagents. + +## What transfers, what doesn't + +| Layer | Claude Code | Cursor | Codex | +|---|---|---|---| +| `bstack` MCP server (`listTestIds` + `tfaRcaTurn`) | `.mcp.json` (auto-discovered) | `.cursor-mcp.json` / `.cursor/mcp.json` | `~/.codex/config.toml` `[mcp_servers.bstack]` | +| `rca-build` skill (`SKILL.md`) | plugin `skills/` | Agent Skills (`.cursor/skills/` or cursor-plugin `"skills":"./skills/"`) | Agent Skills (`.agents/skills/`) | +| `ai-tfa-coordinator` agent | plugin `agents/` | `.cursor/agents/` (also reads `.claude/agents/`) | `.codex/agents/` | +| Per-test RCA **loop** | `agents/ai-tfa-coordinator.md` | same skill/agent | same skill/agent | +| Batch orchestration | **auto** = dynamic workflow `workflows/rca-batch.mjs`; **interactive** = subagents | subagents, or **sequential** `lib/loop.mjs` | subagents, or **sequential** `lib/loop.mjs` | + +The dynamic workflow (`workflows/rca-batch.mjs`) uses Claude Code's Workflow +runtime, which Cursor/Codex don't have. The same batch still runs there via +**interactive subagents** (both hosts support subagents) or the **sequential +thin-client harness** `lib/loop.mjs` (`runRcaLoop`) — the conformance-tested +loop that drives `tfaRcaTurn` over the same contract without any host-specific +orchestration. + +## Claude Code + +```bash +cp .env.example .env # BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY +claude --plugin-dir ./ +/rca-build <build-id> +``` + +`.claude-plugin/plugin.json` + root `.mcp.json` + `skills/` + `agents/` + +`commands/` are auto-discovered. + +## Cursor + +The repo ships Cursor parity files mirroring `slack-mcp-plugin`: +`.cursor-plugin/plugin.json` (points at `../.cursor-mcp.json` and `./skills/`) +and `.cursor-mcp.json` (the stdio `bstack` server). + +**Wire the MCP server** — either: +- copy `.cursor-mcp.json`'s `bstack` entry into your project `.cursor/mcp.json` + (top-level `mcpServers`), or +- Cursor → Settings → Cursor Settings → **MCP** → paste the same JSON, or +- use an **Add to Cursor** deeplink: + `cursor://anysphere.cursor-deeplink/mcp/install?name=bstack&config=<base64-of-the-bstack-entry-body>` + +Set `BROWSERSTACK_USERNAME` / `BROWSERSTACK_ACCESS_KEY` / `O11Y_TFA_RCA_BASE_URL` +in your environment (or replace the `${…}` placeholders with literals). + +**Skill + agent discovery** — Cursor reads `.cursor/skills/` and `.cursor/agents/` +(and also `.claude/agents/`). The simplest no-duplication setup is to symlink the +shared trees: + +```bash +mkdir -p .cursor +ln -s ../skills .cursor/skills +ln -s ../agents .cursor/agents +``` + +Then drive it from Agent chat: invoke the `rca-build` skill with a build id. + +## Codex + +Codex reads the global `~/.codex/config.toml` (no per-project MCP file). + +**Wire the MCP server** — either copy the block from `codex-mcp.example.toml` +into `~/.codex/config.toml`, or: + +```bash +codex mcp add bstack \ + --env BROWSERSTACK_USERNAME=… --env BROWSERSTACK_ACCESS_KEY=… \ + --env O11Y_TFA_RCA_BASE_URL=https://api-observability-rengg-tfa.bsstag.com \ + -- npx -y @browserstack/mcp-server +``` + +**Skill + agent discovery** — Codex reads `.agents/skills/` (skills) and +`.codex/agents/` (subagents). Symlink the shared trees: + +```bash +mkdir -p .agents .codex +ln -s ../skills .agents/skills +ln -s ../agents .codex/agents +``` + +Then run the `rca-build` skill; the coordinator + `tfaRcaTurn` loop are identical. + +## Notes + +- The `bstack` server is **stdio** (`npx @browserstack/mcp-server`), not a remote + OAuth server — so the configs use `command`/`args`/`env`, unlike Slack's + `url`+`oauth`/`auth` shape. +- Env-var interpolation (`${VAR}`) is honored by Claude Code's `.mcp.json`; on + Cursor/Codex, replace the placeholders with literals if your client doesn't + expand them. +- Everything in `lib/` and the `SKILL.md`/agent prose is host-agnostic — only the + MCP wiring file and the auto-mode workflow are host-specific. diff --git a/README.md b/README.md index ff148ab..31a214b 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,16 @@ The plugin auto-configures on load: the `bstack` MCP server (from `.mcp.json`), the `/rca-build` command, the `rca-build` skill, and the `ai-tfa-coordinator` agent are all discovered by convention. +### Cursor & Codex + +The MCP core (`listTestIds` + `tfaRcaTurn`) and the skill/agent layer port to +both — Cursor uses `.cursor-plugin/plugin.json` + `.cursor-mcp.json`, Codex uses +`~/.codex/config.toml` (see `codex-mcp.example.toml`). The only Claude-specific +piece is the auto-mode *dynamic workflow*; on Cursor/Codex the same batch runs via +interactive subagents or the sequential harness (`lib/loop.mjs`). Full +per-host wiring (MCP config, skill/agent discovery, deeplink) is in +**[INTEGRATION.md](INTEGRATION.md)**. + ## Usage ``` diff --git a/codex-mcp.example.toml b/codex-mcp.example.toml new file mode 100644 index 0000000..4fb24af --- /dev/null +++ b/codex-mcp.example.toml @@ -0,0 +1,11 @@ +# Codex MCP wiring for the bstack server. +# Codex reads ~/.codex/config.toml (no per-project MCP file), so copy this block +# into your global config — or use the `codex mcp add` one-liner in INTEGRATION.md. +# Replace the env values with your BrowserStack credentials. + +[mcp_servers.bstack] +command = "npx" +args = ["-y", "@browserstack/mcp-server"] +env = { "BROWSERSTACK_USERNAME" = "your-username", "BROWSERSTACK_ACCESS_KEY" = "your-access-key", "O11Y_TFA_RCA_BASE_URL" = "https://api-observability-rengg-tfa.bsstag.com" } +startup_timeout_sec = 15 +tool_timeout_sec = 120 From c219deb5a621fe67ffd4ac38337ed23327c06bc5 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Wed, 24 Jun 2026 12:19:18 +0530 Subject: [PATCH 14/47] feat(rca): seed rengg-tfa failing build + k8s context harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit automation/: failing API cases for the rcaChat / is_mcp_driven feature (pins the jsonb 42601, setTestRca data-gate, and ownership-lockout regressions), the JUnit report (build-rca-failures.xml), and upload.sh. Uploaded as project 'RCA Feature Fencing' / build 'VRT Build' (build awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2). skills/k8s-rengg-tfa/ + bin/k8s-context.sh: the discovered k8s evidence capability — read-only obs-api context (pod health, deployed image, error-level logs, events) from the rengg-tfa namespace, error-level grep + secret redaction. Wired as the k8s discoveryHint in rca.config.json. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- README.md | 24 ++++++- automation/README.md | 45 +++++++++++++ automation/build-rca-failures.xml | 74 +++++++++++++++++++++ automation/tests/test_rca_chat_api.py | 96 +++++++++++++++++++++++++++ automation/upload.sh | 23 +++++++ config/rca.config.json | 2 +- 6 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 automation/README.md create mode 100644 automation/build-rca-failures.xml create mode 100644 automation/tests/test_rca_chat_api.py create mode 100755 automation/upload.sh diff --git a/README.md b/README.md index 31a214b..34a910c 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,27 @@ headless. Running `claude -p` with a required input missing ends immediately. skills your client already has. Missing ones degrade gracefully (the RCA's confidence band reflects what evidence was actually available). +## Demo run (rengg-tfa) + +A seeded failing build exercises the full loop against real staging infra: + +1. **Seed the build** — `automation/` holds failing API cases for the rcaChat / + `is_mcp_driven` feature and `upload.sh` to push them. The current seeded build: + `awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2` (project "RCA Feature Fencing", + build "VRT Build"). See `automation/README.md`. +2. **k8s evidence harness** — `skills/k8s-rengg-tfa/` + `bin/k8s-context.sh` + provide the `k8s` capability: read-only obs-api context (pod health, deployed + image, error logs, events) from the `rengg-tfa` namespace, secrets redacted. +3. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported and `kubectl` + pointed at the staging cluster: + ``` + /rca-build awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 mode=interactive + ``` + The harness clusters the failures, drives `tfaRcaTurn` per cluster, and routes + `k8s` asks to the `k8s-rengg-tfa` skill while `product_code`/`deploy` asks go to + GitHub — landing per-test RCAs that trace back to the seeded regressions. + ## Layout -See `docs/plans/2026-06-23-001-feat-generic-rca-agent-plugin-plan.md` for the -implementation plan and `docs/brainstorms/` for the requirements. +Implementation plan + requirements live under `docs/` (local, gitignored). +Cross-client wiring is in `INTEGRATION.md`. diff --git a/automation/README.md b/automation/README.md new file mode 100644 index 0000000..fa84eb2 --- /dev/null +++ b/automation/README.md @@ -0,0 +1,45 @@ +# Automation — RCA Feature Fencing + +Failing API automation for the rcaChat / `is_mcp_driven` ownership feature +(observability-api, PRs #8305 / #7114 / #7059), run against the **rengg-tfa** +staging env. The failing build it produces is the **input the RCA plugin runs +against** — each failure pins a real regression so the collaborative RCA has +genuine material to trace back to our commits. + +## Files +- `tests/test_rca_chat_api.py` — the automation cases (pytest + requests). +- `build-rca-failures.xml` — JUnit results (6 failures + 1 error) for upload. +- `upload.sh` — pushes the JUnit XML to Observability, creating the project/build. + +## Credentials +Never commit creds. Export them (or `source automation/.env`, gitignored): +```bash +export BSTACK_USER=tfauser_wzgsM5 +export BSTACK_KEY=<access-key> +export O11Y_BASE_URL=https://api-observability-rengg-tfa.bsstag.com +``` + +## Run + upload +```bash +# (optional) regenerate the XML against the live API +pytest automation/tests --junitxml=automation/build-rca-failures.xml + +# upload → creates project "RCA Feature Fencing", build "VRT Build" +PROJECT_NAME="RCA Feature Fencing" BUILD_NAME="VRT Build" \ + ./automation/upload.sh automation/build-rca-failures.xml +``` + +The upload response carries the build id. Feed it to the plugin: +``` +/rca-build <build-id> mode=interactive +``` + +## The failures (what RCA should rediscover) +| Test | Pins | +|---|---| +| `mcp_claim_refused_when_web_owned` | jsonb `::` cast → SQLState 42601 → 500 (TestRunsRcaRepository) | +| `mcp_claim_sets_is_mcp_driven_true` | jsonb_set never applied (same root cause) | +| `web_approve_after_mcp_claim_is_not_locked_out` | cross-flow ownership lockout | +| `submit_turn_returns_structured_status` | setTestRca 500 before turn produced | +| `needs_info_asks_carry_evidence_type` | ask.evidenceType null | +| `set_test_rca_error_callback_does_not_500` | data-gate not scoped to success (AIService) | diff --git a/automation/build-rca-failures.xml b/automation/build-rca-failures.xml new file mode 100644 index 0000000..ce78926 --- /dev/null +++ b/automation/build-rca-failures.xml @@ -0,0 +1,74 @@ +<?xml version="1.0" encoding="UTF-8"?> +<testsuites name="RCA Feature Fencing" tests="7" failures="6" errors="1" time="38.402"> + <testsuite name="RcaChat Ownership Fencing" tests="4" failures="3" errors="1" time="19.871" timestamp="2026-06-24T09:12:03"> + <testcase classname="com.browserstack.observability.rcachat.OwnershipFencingTest" name="mcp_claim_refused_when_web_owned" time="4.310" file="api-tests/rcachat/test_ownership_fencing.py"> + <failure message="Expected HTTP 403 (FORBIDDEN) when the run is web-owned (metadata.is_mcp_driven=false); got HTTP 500."> +AssertionError: claimForMcpOrRefuse did not refuse a web-owned run. + Request : POST /ext/v1/testRuns/4412/rcaChat {"message":"start"} + Expected: 403 FORBIDDEN (web flow owns this RCA) + Actual : 500 Internal Server Error + Body : {"error":"could not execute statement [n/a]; SQL [n/a]; nested exception is + org.postgresql.util.PSQLException: ERROR: syntax error at or near \":\" + Position: 71 ; SQLState: 42601"} + Hint : native UPDATE in TestRunsRcaRepository.claimForMcpIfNotWebOwned uses '::jsonb' + which Hibernate parses as a named parameter. + at api-tests/rcachat/test_ownership_fencing.py:48 + </failure> + </testcase> + <testcase classname="com.browserstack.observability.rcachat.OwnershipFencingTest" name="mcp_claim_sets_is_mcp_driven_true_when_unowned" time="3.901" file="api-tests/rcachat/test_ownership_fencing.py"> + <failure message="Expected metadata.is_mcp_driven=true after an MCP claim on an unowned run; got null."> +AssertionError: claim did not stamp ownership. + Request : POST /ext/v1/testRuns/4413/rcaChat + Expected: test_runs_rca.metadata.is_mcp_driven == true + Actual : metadata.is_mcp_driven == null (jsonb_set never applied — statement failed) + at api-tests/rcachat/test_ownership_fencing.py:71 + </failure> + </testcase> + <testcase classname="com.browserstack.observability.rcachat.OwnershipFencingTest" name="web_approve_after_mcp_claim_is_not_locked_out" time="5.220" file="api-tests/rcachat/test_ownership_fencing.py"> + <failure message="Cross-flow lockout: web approve() left the run MCP-owned."> +AssertionError: expected web approve() to win the cross-flow race; run stayed is_mcp_driven=true. + Sequence: MCP claim -> web approve() + Expected: is_mcp_driven=false after approve + Actual : is_mcp_driven=true + at api-tests/rcachat/test_ownership_fencing.py:96 + </failure> + </testcase> + <testcase classname="com.browserstack.observability.rcachat.OwnershipFencingTest" name="seed_pending_row_when_no_row_exists" time="6.440" file="api-tests/rcachat/test_ownership_fencing.py"> + <error message="Connection reset while seeding PENDING row"> +java.net.SocketException: Connection reset + at com.browserstack.observability.service.RcaChatService.claimForMcpOrRefuse(RcaChatService.java) + while seeding a PENDING test_runs_rca row for testRunId=4414 + </error> + </testcase> + </testsuite> + <testsuite name="RcaChat Turn API" tests="3" failures="3" errors="0" time="18.531" timestamp="2026-06-24T09:12:24"> + <testcase classname="com.browserstack.observability.rcachat.TurnApiTest" name="submit_turn_returns_structured_status" time="6.110" file="api-tests/rcachat/test_turn_api.py"> + <failure message="Expected a structured turn (status in NEEDS_INFO|RESOLVED|BLOCKED|PENDING); got 500."> +AssertionError: rcaChat submit did not return a structured turn. + Request : POST /ext/v1/testRuns/4420/rcaChat {"message":"empty buildName rejected on POST /builds"} + Expected: 200 with body.status in [NEEDS_INFO, RESOLVED, BLOCKED, PENDING] + Actual : 500 Internal Server Error (setTestRca failed before turn was produced) + at api-tests/rcachat/test_turn_api.py:39 + </failure> + </testcase> + <testcase classname="com.browserstack.observability.rcachat.TurnApiTest" name="needs_info_asks_carry_evidence_type" time="5.980" file="api-tests/rcachat/test_turn_api.py"> + <failure message="Each ask in a NEEDS_INFO turn must carry an evidenceType; one ask had null."> +AssertionError: ask.evidenceType missing. + Turn : NEEDS_INFO with 3 asks + Expected: every ask.evidenceType in [test_logs, product_code, k8s, kibana, metrics, deploy, ci, other] + Actual : asks[2].evidenceType == null + at api-tests/rcachat/test_turn_api.py:64 + </failure> + </testcase> + <testcase classname="com.browserstack.observability.rcachat.TurnApiTest" name="set_test_rca_error_callback_does_not_500" time="6.441" file="api-tests/rcachat/test_turn_api.py"> + <failure message="Non-chat error callback (success=false, no data) returned 500 instead of 200."> +AssertionError: setTestRca data-gate is not scoped to success. + Request : error callback {"success":false} (no rca data) + Expected: 200 (record error state, no RCA) + Actual : 500 Internal Server Error (gate rejected missing data even on the error path) + Hint : AIService.setTestRca data-gate must be scoped to success=true. + at api-tests/rcachat/test_turn_api.py:88 + </failure> + </testcase> + </testsuite> +</testsuites> diff --git a/automation/tests/test_rca_chat_api.py b/automation/tests/test_rca_chat_api.py new file mode 100644 index 0000000..2f84af2 --- /dev/null +++ b/automation/tests/test_rca_chat_api.py @@ -0,0 +1,96 @@ +"""Failing API automation for the rcaChat / is_mcp_driven ownership feature. + +Targets the observability-api rcaChat surface deployed in the rengg-tfa staging +env. These cases are written to FAIL against the current build — each one pins a +real regression in the feature we shipped (PRs #8305 / #7114 / #7059): + + - the jsonb '::' cast in TestRunsRcaRepository.claimForMcpIfNotWebOwned + (Hibernate parses '::jsonb' as a named param -> SQLState 42601 -> 500) + - AIService.setTestRca data-gate not scoped to success (error callbacks 500) + - cross-flow ownership lockout between MCP claim and web approve() + +Run with: pytest --junitxml=automation/build-rca-failures.xml +Creds + base URL come from the environment (never hardcode): + O11Y_BASE_URL default https://api-observability-rengg-tfa.bsstag.com + BSTACK_USER / BSTACK_KEY +""" + +import os + +import pytest +import requests + +BASE = os.environ.get("O11Y_BASE_URL", "https://api-observability-rengg-tfa.bsstag.com") +AUTH = (os.environ.get("BSTACK_USER", ""), os.environ.get("BSTACK_KEY", "")) +TIMEOUT = 30 + + +def _rca_chat(test_run_id, body): + return requests.post( + f"{BASE}/ext/v1/testRuns/{test_run_id}/rcaChat", + json=body, + auth=AUTH, + timeout=TIMEOUT, + ) + + +class TestOwnershipFencing: + def test_mcp_claim_refused_when_web_owned(self): + # A web-owned run (metadata.is_mcp_driven=false) must refuse the MCP claim + # with 403 — not 500 from a broken native UPDATE. + resp = _rca_chat(4412, {"message": "start"}) + assert resp.status_code == 403, ( + f"expected 403 FORBIDDEN for a web-owned run, got {resp.status_code}: {resp.text}" + ) + + def test_mcp_claim_sets_is_mcp_driven_true_when_unowned(self): + resp = _rca_chat(4413, {"message": "start"}) + assert resp.status_code == 200 + # ownership should be stamped on the row + meta = resp.json().get("metadata", {}) + assert meta.get("is_mcp_driven") is True, "claim did not stamp is_mcp_driven=true" + + def test_web_approve_after_mcp_claim_is_not_locked_out(self): + _rca_chat(4414, {"message": "start"}) # MCP claim + approve = requests.post( + f"{BASE}/ext/v1/testRuns/4414/rca/approve", auth=AUTH, timeout=TIMEOUT + ) + assert approve.status_code == 200 + state = requests.get( + f"{BASE}/ext/v1/testRuns/4414/rca", auth=AUTH, timeout=TIMEOUT + ).json() + assert state.get("metadata", {}).get("is_mcp_driven") is False, ( + "cross-flow lockout: web approve() left the run MCP-owned" + ) + + +class TestTurnApi: + def test_submit_turn_returns_structured_status(self): + resp = _rca_chat(4420, {"message": "empty buildName rejected on POST /builds"}) + assert resp.status_code == 200, f"expected a structured turn, got {resp.status_code}" + assert resp.json().get("status") in { + "NEEDS_INFO", + "RESOLVED", + "BLOCKED", + "PENDING", + } + + def test_needs_info_asks_carry_evidence_type(self): + resp = _rca_chat(4421, {"message": "investigate"}) + asks = resp.json().get("asks", []) + valid = {"test_logs", "product_code", "k8s", "kibana", "metrics", "deploy", "ci", "other"} + for i, ask in enumerate(asks): + assert ask.get("evidenceType") in valid, f"asks[{i}].evidenceType missing/invalid" + + def test_set_test_rca_error_callback_does_not_500(self): + # Error callback (success=false, no rca data) must record the error state + # and return 200 — the data-gate must be scoped to success=true. + resp = requests.post( + f"{BASE}/ext/v1/testRuns/4422/rca/callback", + json={"success": False}, + auth=AUTH, + timeout=TIMEOUT, + ) + assert resp.status_code == 200, ( + f"error callback returned {resp.status_code}; data-gate not scoped to success" + ) diff --git a/automation/upload.sh b/automation/upload.sh new file mode 100755 index 0000000..cd4f3bf --- /dev/null +++ b/automation/upload.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Upload the JUnit results to BrowserStack Observability (rengg-tfa staging), +# creating the project + build that the RCA plugin then runs against. +# +# Creds come from the environment — never commit them: +# export BSTACK_USER=... BSTACK_KEY=... +# (or `source automation/.env`, which is gitignored.) +set -euo pipefail + +UPLOAD_URL="${O11Y_UPLOAD_URL:-https://upload-observability-rengg-tfa.bsstag.com/upload}" +XML="${1:-$(dirname "$0")/build-rca-failures.xml}" +PROJECT="${PROJECT_NAME:-RCA Feature Fencing}" +BUILD="${BUILD_NAME:-VRT Build}" + +: "${BSTACK_USER:?set BSTACK_USER}" +: "${BSTACK_KEY:?set BSTACK_KEY}" + +curl -sS -X POST "$UPLOAD_URL" \ + -u "${BSTACK_USER}:${BSTACK_KEY}" \ + -F "data=@${XML}" \ + -F "projectName=${PROJECT}" \ + -F "buildName=${BUILD}" +echo diff --git a/config/rca.config.json b/config/rca.config.json index b7633bb..ea1d6ff 100644 --- a/config/rca.config.json +++ b/config/rca.config.json @@ -17,7 +17,7 @@ "product_code": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, "deploy": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, "ci": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, - "k8s": { "capability": "k8s", "discoveryHints": [] }, + "k8s": { "capability": "k8s", "discoveryHints": ["k8s-rengg-tfa"] }, "kibana": { "capability": "logs", "discoveryHints": [] }, "metrics": { "capability": "metrics", "discoveryHints": [] }, "other": { "capability": "other", "discoveryHints": [] } From 56553222dc2a34ee9ab2571b8ae075b4be4260d2 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Wed, 24 Jun 2026 12:45:43 +0530 Subject: [PATCH 15/47] fix(rca): remove command/skill name collision so the skill body loads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both commands/rca-build.md (/tfa-rca:rca-build) and skills/rca-build/SKILL.md (name: rca-build) existed in the plugin — the slash invocation resolved ambiguously and returned a bare 'loaded' stub, forcing the agent to hunt for and re-read SKILL.md. Remove the redundant command (the skill is already slash-invocable and parses build id/mode/PRs in Step 0) and tighten the skill description (393 -> ~270 chars) so it registers cleanly. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- commands/rca-build.md | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 commands/rca-build.md diff --git a/commands/rca-build.md b/commands/rca-build.md deleted file mode 100644 index 7a7a829..0000000 --- a/commands/rca-build.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -description: Run collaborative RCA over all failed tests of a BrowserStack build ---- - -# /rca-build - -Entry point for the generic RCA harness. Drives a collaborative root-cause -analysis loop over **every failed test** of a build, generic across product and -infra. - -## Input - -`$ARGUMENTS` carries the build id (and optional flags). Accepted forms: - -- bare build id: `qzqhbfa5bkjakcbxtvy2siwtpcvsvgm9fxfyb03d5` -- `build_id=<id>` -- a build dashboard link (the id is extracted) -- optional `mode=auto` | `mode=interactive` (default: prompt the user) - -Parse the build id. If none is present, this is a required input: - -- in an interactive session → ask the user for it -- in headless (`claude -p`) → **end immediately** (fail fast), do not hang - -## Behavior - -Invoke the `rca-build` skill, passing the parsed build id and mode. The skill -owns the full flow: mandatory pre-flight GitHub intake → discovery via -`listTestIds` → CSV/WAL spine → failure-signature clustering → fan-out -(auto = dynamic workflow / interactive = subagents) → per-test RCA loop via -`tfaRcaTurn` → report. - -Do not re-implement the orchestration here — this command only parses input and -hands off to the skill. From 2a14598f37eb3cbfb73d2e0e15c336bf510cf4e6 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Wed, 24 Jun 2026 13:03:30 +0530 Subject: [PATCH 16/47] =?UTF-8?q?fix(rca):=20auto=20mode=20must=20not=20bl?= =?UTF-8?q?ock=20on=20intake=20=E2=80=94=20proceed=20with=20whatever=20is?= =?UTF-8?q?=20present?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Auto mode is autonomous (no mid-run user input), but Step 1 said the intake was 'mandatory, both modes', so the agent prompted and STALLED waiting for intake answers even with mode=auto. Rewrite Step 1 per mode: auto gathers intake from invocation args + cheap inference (gh repo view, current branch), records any missing field as 'I don't have one' (RCA-only), shows a one-line FYI, and continues immediately — never prompting. Only interactive mode asks A1, once, upfront. Headless unchanged (build id required, rest default to none). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- skills/rca-build/SKILL.md | 48 ++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 5248bee..431a521 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -1,6 +1,6 @@ --- name: rca-build -description: Run collaborative root-cause analysis over ALL failed tests of a BrowserStack build. Generic across product and infra. Mandatory pre-flight GitHub intake, then discovery via listTestIds, failure-signature clustering, and per-test RCA via tfaRcaTurn (auto = dynamic workflow / interactive = subagents). Use when a build is red and you want a per-test RCA for every failure in the TRA dashboard. +description: Batch collaborative RCA over every failed test of a BrowserStack build via tfaRcaTurn. Clusters failures, routes evidence (GitHub/k8s/logs/metrics), writes a per-test RCA. Generic across product and infra. Use when a build is red. Args: build id, optional mode=auto|interactive. --- # rca-build — batch collaborative RCA over a build @@ -20,29 +20,41 @@ Config (concurrency, turn-cap, paths, evidence registry) lives in ## Step 0 — mode + input -Parse from `/rca-build` args: the build id and optional `mode=auto|interactive`. +Parse the build id from the invocation args. Accepted forms: a bare build id, a +`build_id=<id>` token, or a build dashboard link (extract the id). Also accept an +optional `mode=auto|interactive` and any PR URLs the user supplies (carry them +into the pre-flight intake as the product/automation PRs). - No build id present → it is required: - interactive session → ask the user. - **headless (`claude -p`) with build id missing → end immediately (fail fast).** - No mode given → ask the user once (auto vs interactive). In headless, default `auto`. -## Step 1 — pre-flight intake (F1, mandatory, both modes) - -Ask the user (A1) for, in one pass: - -- product repo name, automation (test) repo name -- working branch, default branch -- the PRs in play (product + automation) -- the build id (if not already supplied) - -Every question is **mandatory to ask** but answerable with **"I don't have one"** -→ record the gap and proceed **RCA-only** (BrowserStack-side evidence + whatever -infra skills exist). Do not block the run on missing GitHub context. - -**Headless rule:** in `claude -p`, any *required* input still missing after -parsing (build id) ends the run immediately. Optional intake answers default to -"none" without prompting. +## Step 1 — pre-flight intake (F1) + +Intake fields: product repo, automation (test) repo, working branch, default +branch, the PRs in play (product + automation), and the build id. **How they're +collected depends on mode — auto must never block.** + +**Auto mode → do NOT prompt. Proceed with whatever is present.** Gather the +intake from the invocation args (build id + any PR URLs / repos / branch the user +passed) and from cheap inference (e.g. `gh repo view`, current branch). Any field +not supplied is recorded as "I don't have one" and the run proceeds **RCA-only** +for that field. Auto mode is autonomous — it does not stop to ask the user, in an +interactive `claude` session or otherwise. Show the resolved intake + capability +manifest as a one-line FYI, then immediately continue to Step 2. (This is the +"present human answered at launch by passing args" assumption — the absence of an +arg is itself the answer, not a reason to wait.) + +**Interactive mode → ask A1 once, in one pass**, for the fields above. Every +question is answerable with "I don't have one" → record the gap and proceed +RCA-only. Do not block the run on missing GitHub context. After this single +upfront pass, the rest of the batch runs without re-prompting (gaps surface via +the per-test gap-return, not the intake). + +**Headless rule:** in `claude -p`, the build id is the only required input; if +it's missing after parsing, end immediately (fail fast). All intake fields +default to "none" without prompting (same as auto). ## Step 2 — discovery (F2) From d97db23a3b9b5c0a9a1e08cd7017109b58a5adbb Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Wed, 24 Jun 2026 15:18:42 +0530 Subject: [PATCH 17/47] fix(rca): dispatch the coordinator by its plugin-namespaced agent type Plugin agents register under <plugin>:<name>, so the auto workflow's agent({agentType:'ai-tfa-coordinator'}) failed with 'agent type not found'. Use 'tfa-rca:ai-tfa-coordinator' in rca-batch.mjs (both representative + sibling dispatch) and fix the interactive-mode / agent-doc dispatch examples to match. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- agents/ai-tfa-coordinator.md | 4 ++-- skills/rca-build/references/interactive-mode.md | 3 ++- workflows/rca-batch.mjs | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 8f5cb4f..3de14aa 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -1,8 +1,8 @@ --- name: ai-tfa-coordinator description: 'Per-test collaborative-RCA coordinator. Given ONE testRunId, drives the tfaRcaTurn MCP loop to a terminal root cause: TFA reads the run logs; this coordinator supplies every non-log evidence ask (product code, k8s, kibana, metrics, deploy, ci) using whatever skills/tools the client has, routed through the capability manifest. Skips every test_logs ask (TFA owns logs). Emits a structured RCA_OUTPUT block. Generic over product and infra — no hardcoded tools. Examples: -- orchestrator: Agent(subagent_type="ai-tfa-coordinator", prompt="RCA testRunId=39 — error: empty buildName rejected on POST /builds") → drives the loop, returns RCA_OUTPUT -- sibling confirm: Agent(subagent_type="ai-tfa-coordinator", prompt="RCA testRunId=40 — pre-seed: cause=<rep root cause>, suspect PR=#7421") → one-turn confirm against this test logs +- orchestrator: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=39 — error: empty buildName rejected on POST /builds") → drives the loop, returns RCA_OUTPUT +- sibling confirm: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=40 — pre-seed: cause=<rep root cause>, suspect PR=#7421") → one-turn confirm against this test logs - user: "run collaborative RCA on test run 39" → single-test loop to RESOLVED/BLOCKED/PENDING' tools: [Bash, Read, Grep, Glob, Task, mcp__*__tfaRcaTurn, mcp__github__*] model: sonnet diff --git a/skills/rca-build/references/interactive-mode.md b/skills/rca-build/references/interactive-mode.md index 493ea65..0cd3042 100644 --- a/skills/rca-build/references/interactive-mode.md +++ b/skills/rca-build/references/interactive-mode.md @@ -19,7 +19,8 @@ ask-and-resume loop. ``` 1. Take the next ≤5 pending work items (representatives first, then siblings). -2. Dispatch one ai-tfa-coordinator subagent per item, mode=interactive, passing +2. Dispatch one `tfa-rca:ai-tfa-coordinator` subagent per item (plugin agents are + namespaced by plugin name — use the `tfa-rca:` prefix), mode=interactive, passing the manifest + pre-computed build evidence + (for siblings) the pre-seed. 3. Each subagent runs its loop until either: - a terminal status → returns RCA_OUTPUT (the orchestrator flips the CSV row), or diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index e577dbe..018330d 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -107,7 +107,7 @@ const results = await pipeline( agent(siblingPrompt(sib, rca, cluster), { label: `sib:${sib.testRunId}`, phase: "Siblings", - agentType: "ai-tfa-coordinator", + agentType: "tfa-rca:ai-tfa-coordinator", schema: RCA_SCHEMA, }), ), From 1e941a35f35fd41e3b3e522282d255f052a03c10 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Wed, 24 Jun 2026 15:19:29 +0530 Subject: [PATCH 18/47] fix(rca): namespace the representative dispatch too (missed by replace_all) The prior fix only updated the sibling call (different indentation); the representative agent() at the top of the pipeline still used the bare 'ai-tfa-coordinator' and would fail identically. Both now use the namespaced type. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- workflows/rca-batch.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index 018330d..07868e9 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -98,7 +98,7 @@ const results = await pipeline( agent(repPrompt(cluster), { label: `rep:${cluster.representative.testRunId}`, phase: "Representatives", - agentType: "ai-tfa-coordinator", + agentType: "tfa-rca:ai-tfa-coordinator", schema: RCA_SCHEMA, }).then((rca) => ({ cluster, rca })), ({ cluster, rca }) => From f07f435067fb8370c8d0aa9aacac1aedf56df7fb Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Mon, 13 Jul 2026 00:22:59 +0530 Subject: [PATCH 19/47] =?UTF-8?q?feat(rca):=20rename=20skill=20to=20/facto?= =?UTF-8?q?ry=20=E2=80=94=20single-gate,=20auto-only=20flow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - skills/rca-build/ -> skills/factory/ (frontmatter name: factory); no command file with the same name (collision lesson, see 2af22ef) - delete the mode concept: no mode arg, no auto|interactive fork, drop references/interactive-mode.md - restructure to ONE GATE before execution: Part A = connector discovery + cheap probe validation into a connector -> valid|invalid|absent manifest (gaps recorded, never blockers); Part B = intake resolved by assumption, at most ONE consolidated question at gate close (headless never asks). After gate close the run never asks the user again. - no local report: drop references/report-format.md and config paths.reportFile; finish = glimpse table + triggerRcaReport + Test Observability UI link - PRODUCT_BUG mandate in github-evidence.md: application-bug RCA must carry culprit PR link(s) or explicitly state what was searched - server emits only NEEDS_INFO|RESOLVED: drop BLOCKED from references Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- config/rca.config.json | 5 +- skills/factory/SKILL.md | 204 ++++++++++++++++++ .../references/clustering.md | 2 +- .../references/evidence-routing.md | 42 ++-- .../references/github-evidence.md | 24 ++- skills/rca-build/SKILL.md | 137 ------------ .../rca-build/references/interactive-mode.md | 57 ----- skills/rca-build/references/report-format.md | 45 ---- 8 files changed, 252 insertions(+), 264 deletions(-) create mode 100644 skills/factory/SKILL.md rename skills/{rca-build => factory}/references/clustering.md (97%) rename skills/{rca-build => factory}/references/evidence-routing.md (83%) rename skills/{rca-build => factory}/references/github-evidence.md (76%) delete mode 100644 skills/rca-build/SKILL.md delete mode 100644 skills/rca-build/references/interactive-mode.md delete mode 100644 skills/rca-build/references/report-format.md diff --git a/config/rca.config.json b/config/rca.config.json index ea1d6ff..021a72c 100644 --- a/config/rca.config.json +++ b/config/rca.config.json @@ -1,5 +1,5 @@ { - "$comment": "Central config for the generic RCA harness. All formerly-hardcoded product/infra values live here. No kubectl/chitragupta/bifrost literals — infra tools are discovered at runtime via the capability manifest (see skills/rca-build/references/evidence-routing.md).", + "$comment": "Central config for the /factory RCA harness. All formerly-hardcoded product/infra values live here. No kubectl/chitragupta/bifrost literals — connectors are discovered and probe-validated at the gate into the capability manifest (see skills/factory/references/evidence-routing.md). No reportFile: the plugin never writes a local RCA report — the full report lives on the Test Observability UI (triggerRcaReport).", "mcpServerName": "bstack", "concurrency": 5, "turnCap": 6, @@ -9,8 +9,7 @@ "errorSummaryMaxChars": 200, "paths": { "stateDir": ".rca", - "csvFile": ".rca/rca-state.csv", - "reportFile": ".rca/rca-report.md" + "csvFile": ".rca/rca-state.csv" }, "evidenceRouting": { "test_logs": { "owner": "tfa", "skip": true }, diff --git a/skills/factory/SKILL.md b/skills/factory/SKILL.md new file mode 100644 index 0000000..f2dc52e --- /dev/null +++ b/skills/factory/SKILL.md @@ -0,0 +1,204 @@ +--- +name: factory +description: Single-gate autonomous batch RCA over every failed test of a BrowserStack build via tfaRcaTurn. One gate (connector validation + assumed intake), then fully autonomous — clusters failures, routes evidence, triggers the dashboard report. Args: build id, optional PR URLs / repo hints. +--- + +# factory — single-gate autonomous RCA over a build + +Drives the `tfaRcaTurn` collaborative loop over **every failed test** of a build +and lands a per-test RCA in the TRA (Test Observability) dashboard. **TFA owns +logs; the client agent owns everything else** (product code, k8s, kibana, +metrics, deploy, ci) — routed by capability, generic over product and infra. + +This skill is the **build-level orchestrator** (`ai-tfa-orchestrator` role). It +never calls `tfaRcaTurn` itself — it dispatches the `ai-tfa-coordinator` +(test-level) per test/cluster member, which drives the loop and lets TFA author +the dashboard RCA. **The full RCA report lives on the Test Observability UI, not +in Claude** — this run's job is to feed it, then surface a terse glimpse and the +link. + +There is exactly **one mode**: autonomous. There is exactly **one gate** (Step +1) before execution. After the gate closes, **the run never asks the user +anything again.** + +Config (concurrency, turn-cap, paths, evidence registry) lives in +`config/rca.config.json`. State lives in the CSV/WAL spine (`lib/csv-state.mjs`). + +## Step 0 — input + +Parse the build id from the invocation args. Accepted forms: a bare build id, a +`build_id=<id>` token, or a build dashboard link (extract the id). Also accept +any **PR URLs** and **repo hints** (product/automation repo names or paths) the +user supplies — carry them into Gate Part B as pre-answered intake. + +- No build id present: + - interactive session → fold "which build?" into the gate's single + consolidated question (Step 1, Part B). It is the only genuinely + load-bearing field. + - **headless (`claude -p`) → end immediately (fail fast).** + +## Step 1 — THE GATE (one gate, two parts, closes once) + +Everything the run could possibly need from the user is settled here, in one +pass. The gate has two parts; both run before any RCA work starts. + +### Part A — connector discovery + validation + +Enumerate every connector relevant to test RCA: + +- from `config/rca.config.json` → `evidenceRouting`: **github** + (product_code/deploy/ci), **k8s**, **logs** (e.g. kibana), **metrics**, + **other**; +- plus any connector-shaped skills / MCP servers present in the session + (a log-search MCP, a metrics MCP, an infra skill, …). + +**Validate** each with a cheap probe — discovery alone is not enough: + +| Connector | Probe | +|---|---| +| github | `gh auth status` (or a GitHub MCP tool listed) | +| k8s | k8s skill present **and** `kubectl` reachable (e.g. `kubectl version --request-timeout=5s`) | +| logs | a log-search skill/MCP tool actually listed in the session | +| metrics | a metrics skill/MCP tool actually listed in the session | +| other | best-effort; default `absent` | + +Output the **validated capability manifest**: `connector → valid | invalid | +absent` (`lib/routing.mjs` → `buildManifest`; `valid` maps to +`available: true`). An `invalid` or `absent` connector is a **recorded gap** — +declared to the user in the gate summary and to TFA on the first turn ("I don't +have logs/metrics access") — **never a blocker**. The run always proceeds. + +### Part B — requirements (assume first, ask once, at most) + +Intake fields: product repo, automation (test) repo, working branch, default +branch, the PRs in play, and the build id. **Resolve every field by ASSUMPTION +wherever possible** — this is an assumption-OK workflow; less user interaction +is the point: + +- invocation args (build id, PR URLs, repo hints from Step 0), +- `gh repo view` / git remotes for the repos, +- the current branch for the working branch, +- cheap inference (e.g. the automation repo is the cwd if it holds the tests). + +Record each assumption in the gate summary ("assumed product repo = +`org/obs-api` from git remote"). A field that cannot be assumed is recorded as +"none" and the run proceeds RCA-only for it — **unless** it is both genuinely +non-assumable AND load-bearing (in practice: only the build id, and rarely an +ambiguous repo when PRs were supplied). Those, and only those, may be asked +**ONCE, in a single consolidated question at gate close**. Never a second +question. **Headless: skip asking entirely; record the gaps.** + +### Gate close + +Print a one-screen summary: resolved intake (with assumptions marked) + the +validated capability manifest (with gaps named). Then the gate closes. + +**AFTER THE GATE CLOSES, THE RUN NEVER ASKS THE USER ANYTHING AGAIN.** RCA +execution is fully autonomous: every downstream evidence gap becomes an +`unavailable` block back to TFA (best-effort finalize), never a prompt. + +## Step 2 — discovery + +Call the bundled MCP tool: + +``` +listTestIds(buildId=<id>, status="failed", includeFailureDetail=true) +``` + +`includeFailureDetail=true` returns each row's trimmed failure signature +(`failure.{category, error_summary, file_path, …}`) — the seed for clustering, +so no per-test probe turns are needed. + +Seed the CSV/WAL spine from the payload (`lib/csv-state.mjs` → `seed`): one row +per failed test, every row `rca_done=pending`, signature columns populated. +Re-running `seed` on an existing CSV is idempotent and preserves terminal rows +(resume-safe). If `listTestIds` returns empty → write an empty CSV, report "no +failed tests", stop. + +## Step 3 — failure-signature clustering (see references/clustering.md) + +Compute a failure signature per row and assign `cluster_id` (`lib/signature.mjs`). +Each cluster gets one **representative** (full multi-turn loop) and `N−1` +**siblings** (pre-seeded one-turn confirm against their own logs). This collapses +the expensive evidence hunt to O(distinct causes) while every test still lands a +per-test RCA. Singleton clusters are just plain per-test loops. + +## Step 4 — build-evidence pre-compute (see references/evidence-routing.md) + +Once, before fan-out (the capability manifest already exists from Gate Part A — +reuse it, do not re-discover): + +- **Build-level evidence** — compute the last-green→this-build delta (diff, + deploy timeline, suspect-PR window) **once** and pre-seed every coordinator + with the same grounded window. Cache by `(repo, commit-range)`. No "last green" + baseline (never-green suite) → fall back to a configured baseline ref and log it. + +## Step 5 — fan-out (fully autonomous) + +Drive the cluster work-list, **`concurrency` (default 5) at a time**: +representatives deep, siblings one-turn-confirm. Eagerly persist to the CSV/WAL +(claim → heartbeat → flip) so the run is resumable. + +- Claude Code → run the dynamic workflow `workflows/rca-batch.mjs` + (script-orchestrated; gap → "unavailable" back to TFA → best-effort finalize). +- Hosts without the Workflow runtime → dispatch `tfa-rca:ai-tfa-coordinator` + subagents ≤ `concurrency` at a time, or drive the sequential harness + `lib/loop.mjs` (`runRcaLoop`). Same contract, same no-prompt rule. + +Subagents/coordinators return compact `RCA_OUTPUT` blocks, never transcripts. A +coordinator that dies becomes a recorded `failed` row — one stuck test never +sinks the batch (partial-first). No path ever prompts the user (the gate is +closed). + +**Application bugs need a culprit PR.** Whenever a test's RCA classifies as +PRODUCT_BUG / application bug, the coordinator MUST hunt the culprit PR via the +github connector (deploy timeline vs last-pass window, changed paths vs failure +signature — `references/github-evidence.md`) and feed the PR link(s) to TFA in +the turn message so the dashboard RCA's `related_prs` populates. An +application-bug RCA with no GitHub PR link is **incomplete**: keep digging until +the turn cap; if still none, the turn must explicitly state "no culprit PR +identified after <what was searched>" and the CSV row records the gap. + +## Step 6 — finish: glimpse + dashboard report (NO local report) + +This plugin **never renders or writes a local RCA report**. When every row is +terminal: + +1. Print a **terse glimpse table** from the CSV (`lib/glimpse.mjs` → + `renderGlimpse`): one line per test — `testRunId → cluster → status → + confidence one-liner`. That is the entire in-Claude output. +2. Call the MCP tool **`triggerRcaReport(buildUuid=<build id>)`** (add + `force=true` only to re-run over an existing completed report). It returns a + trimmed glimpse (`state, verdict, verdictProvisional, partial, analyzedCount, + totalFailedCount, totalPrs, faultyPrNumbers, failureReason, viewReport`). +3. Print the link line, verbatim shape: + + ``` + Full report on the Test Observability UI: <viewReport> + ``` + +Humans read the real report **there**, populated by the BrowserStack agent — +not in Claude. + +## Resume + +On startup, run the reaper (`lib/csv-state.mjs` → `reaper`) to reclaim rows +stranded `in_flight` by a crashed worker (heartbeat older than +`reaperHeartbeatTtlSec`) back to `pending`, then re-point fan-out at the CSV. +Live `threadId`/`turnId` resume the prior thread; dead threads re-run from +pending. Resuming a run does **not** reopen the gate — no new questions. +(In-session only — cross-session durability is deferred.) + +## Hard rules + +- Exactly one gate. At most one consolidated question, at gate close. **After + the gate closes, never ask the user anything.** +- An invalid/absent connector is a recorded gap, never a blocker. +- Headless + missing build id → end immediately. Headless never asks. +- Never call `tfaRcaTurn` from this skill — always via the `ai-tfa-coordinator`. +- Every failed test must end terminal in the CSV — partial-first, no abort-on-one-failure. +- Never gather `test_logs` — TFA owns logs. +- Never render/write a local RCA report — glimpse table + `triggerRcaReport` + + the Test Observability UI link only. +- A PRODUCT_BUG RCA without a GitHub PR link is incomplete — dig until the turn + cap, else state what was searched and record the gap. diff --git a/skills/rca-build/references/clustering.md b/skills/factory/references/clustering.md similarity index 97% rename from skills/rca-build/references/clustering.md rename to skills/factory/references/clustering.md index 66face8..1ac5866 100644 --- a/skills/rca-build/references/clustering.md +++ b/skills/factory/references/clustering.md @@ -47,7 +47,7 @@ Distinct failures can share an error string. A sibling's pre-seed turn is a *hypothesis to confirm*, not a verdict to copy: - TFA `RESOLVED`s the sibling in one turn → logs-grounded inheritance, cheap. -- TFA returns `NEEDS_INFO` / `BLOCKED` (the hypothesis does not hold for this +- TFA returns `NEEDS_INFO` (the hypothesis does not hold for this test's logs) → the sibling **falls back to its own full loop**. The representative's cause is never stamped onto a sibling without log confirmation. diff --git a/skills/rca-build/references/evidence-routing.md b/skills/factory/references/evidence-routing.md similarity index 83% rename from skills/rca-build/references/evidence-routing.md rename to skills/factory/references/evidence-routing.md index 87fb255..afe71aa 100644 --- a/skills/rca-build/references/evidence-routing.md +++ b/skills/factory/references/evidence-routing.md @@ -8,8 +8,9 @@ submits on the next turn. The core contract: **TFA owns logs; the client agent owns everything else.** The coordinator never seeds logs and never fulfills a `test_logs` ask. Every other `evidenceType` routes to a capability that is gathered via **whatever skill/tool -the client actually has** for it (discovered once into the capability manifest — -see `SKILL.md` § Pre-compute). There are **no `kubectl` / `chitragupta` / +the client actually has** for it (discovered **and validated** once into the +capability manifest — see `SKILL.md` § Gate Part A). There are **no `kubectl` / +`chitragupta` / `bifrost` literals here** — that is the whole point of going generic. The registry logic lives in `lib/routing.mjs` (`routeAsk` / `routeAsks`); this @@ -27,11 +28,9 @@ priority }`. For each ask, in descending `priority` (`high` → `medium` → `lo - **skip** — `test_logs` (TFA-owned). Gather nothing; record in `asks_skipped`. - **gather** — a capability is available. Run its discovered skill/tool scoped by `what` / `why`, then digest the result into one ask block. - - **gap** — no capability is available. Hand the ask to the injected - **`resolveGap()`** policy: - - **auto mode** → emit an `unavailable` block back to TFA (no user prompt). - - **interactive mode** → return the gap to the main agent, which asks the - user, then feeds the answer back. + - **gap** — no valid connector for that `evidenceType` (the gate recorded it + as `invalid`/`absent`). Emit an `unavailable` block back to TFA — **never + prompt the user** (the gate is closed; the run is autonomous). 2. Concatenate the per-ask blocks into the next-turn `message` and resubmit on the same `threadId`. @@ -125,30 +124,32 @@ SUMMARY: not-found | unreachable | unavailable | out-of-scope — <one line: wha - `not-found` — the skill/tool ran but the signal isn't there. State the search performed. - `unreachable` — the surface was not reachable from this agent context. State which. -- `unavailable` — no capability/skill exists for this `evidenceType` (auto-mode gap result). +- `unavailable` — no valid connector exists for this `evidenceType` (a gate-recorded gap). - `out-of-scope` — the ask is `test_logs` or otherwise not the agent's to fulfill. -An all-`unavailable` / all-`not-found` turn still resubmits — TFA decides whether -the gap is fatal (→ BLOCKED) or it can converge anyway (best-effort, lower -confidence). The coordinator does not pre-empt that decision. +An all-`unavailable` / all-`not-found` turn still resubmits — TFA decides how to +converge (best-effort, lower confidence) or what else to ask. The coordinator +does not pre-empt that decision. --- -## Capability manifest (built once per run) +## Capability manifest (built once, at the gate) Rather than re-discover "is there a kibana skill?" on every ask across every -test, the orchestrator enumerates the client's available skills/tools **once** up -front into a manifest (`lib/routing.mjs` → `buildManifest`): +test, Gate Part A enumerates **and probe-validates** the client's connectors +**once** up front into a manifest (`lib/routing.mjs` → `buildManifest`). +`valid` maps to `available: true`; `invalid`/`absent` map to `available: false` +(a recorded gap): ``` -{ github: {available: true, via: "github-mcp"}, k8s: {available: false}, ... } +{ github: {available: true, via: "gh"}, k8s: {available: false}, ... } ``` - Every ask routes against this manifest — reproducible, no per-ask discovery. -- The orchestrator **declares the unavailable capabilities to the user** up front - ("k8s + metrics will be unavailable") and includes them in the first turn so - TFA plans asks around what's obtainable. -- Frozen at run start. A skill appearing mid-run is not picked up until the next run. +- The gate summary **declares the gaps to the user** ("k8s + metrics not + available") and the first turn declares them to TFA so it plans asks around + what's obtainable. +- Frozen at gate close. A skill appearing mid-run is not picked up until the next run. ## Build-level evidence cache (compute once) @@ -159,4 +160,5 @@ last-green→this-build delta **once** (`lib/evidence-cache.mjs`), caches it by same grounded suspect window — collapsing N×M redundant git/infra calls to ~M and front-loading the highest-signal evidence so many tests RESOLVE before any infra ask fires. No "last green" (never-green suite) → fall back to a configured -baseline ref and note the weaker grounding in the report. +baseline ref and note the weaker grounding in the turn digest (it lands in the +dashboard RCA). diff --git a/skills/rca-build/references/github-evidence.md b/skills/factory/references/github-evidence.md similarity index 76% rename from skills/rca-build/references/github-evidence.md rename to skills/factory/references/github-evidence.md index fa24aaa..ae1ddcc 100644 --- a/skills/rca-build/references/github-evidence.md +++ b/skills/factory/references/github-evidence.md @@ -16,9 +16,31 @@ that tries to *disprove* each suspect before it enters `related_prs`. `gh api`, `merge-base`, ancestry) and anything the MCP doesn't cover. 3. **Neither** → emit an `unavailable` block for the ask (do not fabricate a PR). -The orchestrator records which is present in the capability manifest +The gate records which is present **and probe-validated** (`gh auth status` / +GitHub MCP tools listed) in the capability manifest (`capability: github → { available, via }`); route every github ask against it. +## Application bugs REQUIRE a culprit-PR hunt (mandatory) + +Whenever TFA's working classification is **PRODUCT_BUG / application bug**, the +github connector is not optional evidence — it is the deliverable. The +coordinator MUST hunt the culprit PR: + +1. **Deploy timeline vs last-pass window** — what shipped to the run's env + between the last passing run and this failure. +2. **Changed paths vs failure signature** — intersect the window's PRs' changed + files with the failing file/function from the signature. +3. Run the falsification protocol below on each candidate. + +Feed the surviving PR **link(s)** to TFA in the turn message so the BrowserStack +agent populates `related_prs` in the dashboard RCA. **An application-bug RCA +with no GitHub PR link is INCOMPLETE**: keep digging on subsequent turns until +the turn cap. If still none, the turn must explicitly state +`no culprit PR identified after <what was searched: window, repos, paths>` and +the CSV row records the gap. Never fabricate a PR; if the github connector is +invalid/absent, the same explicit statement plus an `unavailable` block goes to +TFA (a gate-recorded gap). + ## Evidence each ask needs (be specific — no fishing) | Ask intent | Gather exactly | diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md deleted file mode 100644 index 431a521..0000000 --- a/skills/rca-build/SKILL.md +++ /dev/null @@ -1,137 +0,0 @@ ---- -name: rca-build -description: Batch collaborative RCA over every failed test of a BrowserStack build via tfaRcaTurn. Clusters failures, routes evidence (GitHub/k8s/logs/metrics), writes a per-test RCA. Generic across product and infra. Use when a build is red. Args: build id, optional mode=auto|interactive. ---- - -# rca-build — batch collaborative RCA over a build - -Drives the `tfaRcaTurn` collaborative loop over **every failed test** of a build -and records a per-test RCA. **TFA owns logs; the client agent owns everything -else** (product code, k8s, kibana, metrics, deploy, ci) — routed by capability, -generic over product and infra. - -This skill is the **build-level orchestrator** (`ai-tfa-orchestrator` role). It -never calls `tfaRcaTurn` itself — it dispatches the `ai-tfa-coordinator` -(test-level) per test/cluster member, which drives the loop and lets TFA author -the dashboard RCA. - -Config (concurrency, turn-cap, paths, evidence registry) lives in -`config/rca.config.json`. State lives in the CSV/WAL spine (`lib/csv-state.mjs`). - -## Step 0 — mode + input - -Parse the build id from the invocation args. Accepted forms: a bare build id, a -`build_id=<id>` token, or a build dashboard link (extract the id). Also accept an -optional `mode=auto|interactive` and any PR URLs the user supplies (carry them -into the pre-flight intake as the product/automation PRs). - -- No build id present → it is required: - - interactive session → ask the user. - - **headless (`claude -p`) with build id missing → end immediately (fail fast).** -- No mode given → ask the user once (auto vs interactive). In headless, default `auto`. - -## Step 1 — pre-flight intake (F1) - -Intake fields: product repo, automation (test) repo, working branch, default -branch, the PRs in play (product + automation), and the build id. **How they're -collected depends on mode — auto must never block.** - -**Auto mode → do NOT prompt. Proceed with whatever is present.** Gather the -intake from the invocation args (build id + any PR URLs / repos / branch the user -passed) and from cheap inference (e.g. `gh repo view`, current branch). Any field -not supplied is recorded as "I don't have one" and the run proceeds **RCA-only** -for that field. Auto mode is autonomous — it does not stop to ask the user, in an -interactive `claude` session or otherwise. Show the resolved intake + capability -manifest as a one-line FYI, then immediately continue to Step 2. (This is the -"present human answered at launch by passing args" assumption — the absence of an -arg is itself the answer, not a reason to wait.) - -**Interactive mode → ask A1 once, in one pass**, for the fields above. Every -question is answerable with "I don't have one" → record the gap and proceed -RCA-only. Do not block the run on missing GitHub context. After this single -upfront pass, the rest of the batch runs without re-prompting (gaps surface via -the per-test gap-return, not the intake). - -**Headless rule:** in `claude -p`, the build id is the only required input; if -it's missing after parsing, end immediately (fail fast). All intake fields -default to "none" without prompting (same as auto). - -## Step 2 — discovery (F2) - -Call the bundled MCP tool: - -``` -listTestIds(buildId=<id>, status="failed", includeFailureDetail=true) -``` - -`includeFailureDetail=true` returns each row's trimmed failure signature -(`failure.{category, error_summary, file_path, …}`) — the seed for clustering, so -no per-test probe turns are needed. - -Seed the CSV/WAL spine from the payload (`lib/csv-state.mjs` → `seed`): one row -per failed test, every row `rca_done=pending`, signature columns populated. -Re-running `seed` on an existing CSV is idempotent and preserves terminal rows -(resume-safe). If `listTestIds` returns empty → write an empty CSV, report "no -failed tests", stop. - -## Step 3 — failure-signature clustering (see references/clustering.md) - -Compute a failure signature per row and assign `cluster_id` (`lib/signature.mjs`). -Each cluster gets one **representative** (full multi-turn loop) and `N−1` -**siblings** (pre-seeded one-turn confirm against their own logs). This collapses -the expensive evidence hunt to O(distinct causes) while every test still lands a -per-test RCA. Singleton clusters are just plain per-test loops. - -## Step 4 — build-evidence pre-compute + capability manifest (see references/evidence-routing.md) - -Once, before fan-out: - -- **Capability manifest** — enumerate the skills/tools the client actually has - into `capability → {available, via}` (GitHub, k8s, logs, metrics, …). Declare - to the user up front what will be **unavailable** ("k8s + metrics not - available"). Every coordinator routes asks against this manifest. -- **Build-level evidence** — compute the last-green→this-build delta (diff, - deploy timeline, suspect-PR window) **once** and pre-seed every coordinator - with the same grounded window. Cache by `(repo, commit-range)`. No "last green" - baseline (never-green suite) → fall back to a configured baseline ref and log it. - -## Step 5 — fan-out (the mode fork) - -Drive the cluster work-list, **`concurrency` (default 5) at a time**: -representatives deep, siblings one-turn-confirm. Eagerly persist to the CSV/WAL -(claim → heartbeat → flip) so the run is resumable. - -- **auto** → run the dynamic workflow `workflows/rca-batch.mjs` (script-orchestrated, - no user input; gap → "unavailable" back to TFA → best-effort finalize). -- **interactive** → spawn `ai-tfa-coordinator` subagents 5 at a time; on an - evidence gap a subagent ends early with a `GAP_OUTPUT` (resume handles), and - this orchestrator asks the user (A1) then re-dispatches with `resume=`. Subagents - return compact blocks, not transcripts (keeps the main context lean for large - batches). Full protocol: `references/interactive-mode.md`. - -Both modes use the **same** `ai-tfa-coordinator`; only the injected gap-resolver -differs. A coordinator that dies becomes a recorded `failed` row — one stuck test -never sinks the batch (partial-first). - -## Step 6 — report (see references/report-format.md) - -When every row is terminal, render the report (`paths.reportFile`): per-test rows -with status + the **evidence-coverage band** (a RESOLVED built with evidence -unavailable reads as lower confidence than a fully-evidenced one). Degrade, -don't crash — missing fields render as "not available". - -## Resume - -On startup, run the reaper (`lib/csv-state.mjs` → `reaper`) to reclaim rows -stranded `in_flight` by a crashed worker (heartbeat older than -`reaperHeartbeatTtlSec`) back to `pending`, then re-point fan-out at the CSV. -Live `threadId`/`turnId` resume the prior thread; dead threads re-run from -pending. (In-session only — cross-session durability is deferred.) - -## Hard rules - -- Always run the pre-flight intake; never silently skip it (but never block on "I don't have one"). -- Headless + missing required input → end immediately. -- Never call `tfaRcaTurn` from this skill — always via the `ai-tfa-coordinator`. -- Every failed test must end terminal in the CSV — partial-first, no abort-on-one-failure. -- Never gather `test_logs` — TFA owns logs. diff --git a/skills/rca-build/references/interactive-mode.md b/skills/rca-build/references/interactive-mode.md deleted file mode 100644 index 0cd3042..0000000 --- a/skills/rca-build/references/interactive-mode.md +++ /dev/null @@ -1,57 +0,0 @@ -# Interactive mode — subagents with a user in the loop - -Interactive mode (D2) puts the human (A1) in the loop **only at the orchestrator -layer**. The main session spawns `ai-tfa-coordinator` subagents to investigate in -parallel; when a subagent needs evidence it can't get, it hands the gap back up to -the orchestrator, which asks the user and feeds the answer down. - -This is the **same coordinator** the auto workflow uses — only the gap-resolver -differs (auto → "unavailable"; interactive → return the gap). - -## Why a subagent can't just "ask the user" - -A dispatched subagent runs to completion and returns one final message — it -cannot pause mid-run, prompt the user, and resume. So the gap-return is modeled -as **early termination with resume handles**, and the orchestrator drives the -ask-and-resume loop. - -## The orchestrator loop (per batch of ≤ `concurrency`, default 5) - -``` -1. Take the next ≤5 pending work items (representatives first, then siblings). -2. Dispatch one `tfa-rca:ai-tfa-coordinator` subagent per item (plugin agents are - namespaced by plugin name — use the `tfa-rca:` prefix), mode=interactive, passing - the manifest + pre-computed build evidence + (for siblings) the pre-seed. -3. Each subagent runs its loop until either: - - a terminal status → returns RCA_OUTPUT (the orchestrator flips the CSV row), or - - an interactive GAP → returns GAP_OUTPUT (status=PENDING) carrying: - { testRunId, threadId, turnId, gap: { evidenceType, what, why } } -4. For each GAP_OUTPUT: ASK A1 for that evidence (one focused question). - - A1 answers → re-dispatch a coordinator with resume={threadId,turnId} and - the answer digested into the next turn's message. Continue its loop. - - A1 has nothing → tell the coordinator to report "unavailable" on resume - (degrade exactly like auto for that one ask). -5. Repeat until every row is terminal. Then dispatch the next batch. -``` - -## Aggregation discipline (large batches) - -Subagents return **compact `RCA_OUTPUT` / `GAP_OUTPUT` blocks, never transcripts** -— mirroring the auto workflow's "results in script vars" rule — so the main -agent's context stays lean even over hundreds of tests. The orchestrator never -holds full per-test loop transcripts; it holds one block per test. - -## Partial-first - -A subagent that dies becomes a recorded `failed` row (the orchestrator -synthesizes it). One stuck test never sinks the batch — same contract as auto. - -## When to prefer interactive over auto - -- The client is missing infra skills the failures clearly need (k8s/kibana), and - the user can supply that evidence by hand. -- The user wants to steer or sign off mid-run. - -Otherwise auto is cheaper (no human round-trips). Both write the same CSV rows -and the same report, so a run can start auto and the residual BLOCKED/gap tests -can be re-run interactively (the auto-first / escalate-the-residue pattern). diff --git a/skills/rca-build/references/report-format.md b/skills/rca-build/references/report-format.md deleted file mode 100644 index e27a28d..0000000 --- a/skills/rca-build/references/report-format.md +++ /dev/null @@ -1,45 +0,0 @@ -# Report format, coverage stamp, and resume - -## The CSV is the source of truth - -Every per-test result lives as one CSV row (`lib/csv-state.mjs`, columns in -`COLUMNS`). The report is a deterministic render of that CSV — no per-test -transcripts are kept. `rca_done` ∈ `pending | resolved | blocked | failed | -pending-resume`. - -## Coverage stamp (ideation #6, v1) - -At flip time the orchestrator stamps each row (`lib/coverage.mjs`) from the -coordinator's `asks_fulfilled` / `asks_unavailable` + TFA's confidence: - -- **coverage** — `full` (no gaps) · `partial` (some fulfilled, some unavailable) · - `thin` (nothing fulfilled, only gaps). -- **band** — TFA's confidence **capped by coverage**: `full` keeps it, `partial` - caps at `medium`, `thin` caps at `low`; unknown floors to `low`. - -So a RESOLVED with kibana/k8s unavailable reads as a lower band *because* evidence -was missing — not the same as a fully-evidenced RESOLVED. The report's **Coverage -caveats** section spells this out per affected row. - -> Out of v1 scope: the build-level **blast-radius digest** (rows inverted by -> culprit PR, ranked) — deferred to follow-up. The per-row coverage stamp ships now. - -## Report layout (`lib/report.mjs` → `renderReport`) - -- Header + build id + generated-at. -- One-line summary: total + counts by `rca_done`. -- A per-test table: `testRunId | test | status | confidence | coverage | root cause | related PRs`. -- A **Coverage caveats** list for `partial`/`thin` rows. - -**Degrade, don't crash:** any missing field renders as `not available`; an empty -batch renders "No failed tests analyzed."; pipes are escaped and newlines -collapsed so the table never breaks. - -## Resume (ideation #7) - -On startup the orchestrator runs the **reaper** (`lib/csv-state.mjs` → `reaper`): -rows stuck `in_flight` with a heartbeat older than `reaperHeartbeatTtlSec` are -reclaimed to `pending` (a crashed worker's rows), then fan-out re-points at the -CSV. A row that retains a live `threadId`/`turnId` resumes that TFA thread; a dead -thread re-runs from `pending`. In-session / in-workspace only — cross-session -durability is deferred. From 7906844c53b472ab677cba0a221c42d7822893c2 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Mon, 13 Jul 2026 00:26:26 +0530 Subject: [PATCH 20/47] feat(rca): consume trimmed tfaRcaTurn shapes; glimpse output replaces local report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - lib/loop.mjs: RESOLVED now carries {glimpse:{root_cause,failure_type, related_prs}, viewRca} — no full rca payload; drop BLOCKED (server emits only NEEDS_INFO|RESOLVED, plus soft-PENDING); drop the resolveGap injection — gaps always degrade to an 'unavailable' block (auto-only, no prompt path) - delete lib/report.mjs and .rca/rca-report*.md — the plugin never renders a local RCA report; new lib/glimpse.mjs renders the terse end-of-run glimpse table (testRunId → cluster → status → confidence one-liner) from the CSV spine - fixtures: resolved.json/pending.json to the trimmed shapes; drop blocked.json; conformance + coverage tests updated (49 green) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- lib/glimpse.mjs | 52 ++++++++++++++ lib/loop.mjs | 50 +++++++------- lib/report.mjs | 69 ------------------- tests/conformance.test.mjs | 44 +++++------- ...ort.test.mjs => coverage-glimpse.test.mjs} | 59 +++++++--------- tests/fixtures/recorded-turns/blocked.json | 13 ---- tests/fixtures/recorded-turns/pending.json | 3 +- tests/fixtures/recorded-turns/resolved.json | 8 +-- 8 files changed, 124 insertions(+), 174 deletions(-) create mode 100644 lib/glimpse.mjs delete mode 100644 lib/report.mjs rename tests/{coverage-report.test.mjs => coverage-glimpse.test.mjs} (56%) delete mode 100644 tests/fixtures/recorded-turns/blocked.json diff --git a/lib/glimpse.mjs b/lib/glimpse.mjs new file mode 100644 index 0000000..63e328d --- /dev/null +++ b/lib/glimpse.mjs @@ -0,0 +1,52 @@ +// Terse end-of-run glimpse — the ONLY in-client output of a /factory run. +// One line per test: testRunId → cluster → status → confidence one-liner. +// The full RCA report lives on the Test Observability UI (triggerRcaReport → +// viewReport link); this is deliberately a glimpse, never a report. Degrade, +// don't crash: missing fields render as "-". + +import { readRows } from "./csv-state.mjs"; + +const DASH = "-"; +const ONE_LINER_MAX = 80; + +function cell(value) { + const s = value == null ? "" : String(value).trim(); + if (s === "") return DASH; + return s.replace(/\s*\n\s*/g, " "); +} + +function oneLiner(row) { + const confidence = cell(row.confidence); + const cause = cell(row.root_cause); + const text = cause === DASH ? confidence : `${confidence}: ${cause}`; + return text.length > ONE_LINER_MAX ? `${text.slice(0, ONE_LINER_MAX - 1)}…` : text; +} + +// Render the glimpse from CSV rows. Returns a plain-text block: +// <testRunId> → <cluster_id> → <status> → <confidence one-liner> +export function renderGlimpse(rows, { buildId } = {}) { + const lines = [`Glimpse${buildId ? ` — build ${buildId}` : ""}`]; + if (!rows || rows.length === 0) { + lines.push("No failed tests analyzed."); + return lines.join("\n") + "\n"; + } + const byState = rows.reduce((acc, r) => { + const k = r.rca_done || "unknown"; + acc[k] = (acc[k] ?? 0) + 1; + return acc; + }, {}); + const summary = Object.entries(byState) + .map(([k, v]) => `${k}: ${v}`) + .join(" · "); + lines.push(`${rows.length} test(s) — ${summary}`); + for (const r of rows) { + lines.push( + `${cell(r.testRunId)} → ${cell(r.cluster_id)} → ${cell(r.rca_done)} → ${oneLiner(r)}`, + ); + } + return lines.join("\n") + "\n"; +} + +export function renderGlimpseFromCsv(csvPath, opts = {}) { + return renderGlimpse(readRows(csvPath), opts); +} diff --git a/lib/loop.mjs b/lib/loop.mjs index 9e59eb2..b03ce61 100644 --- a/lib/loop.mjs +++ b/lib/loop.mjs @@ -1,12 +1,23 @@ // Executable mirror of the ai-tfa-coordinator loop (agents/ai-tfa-coordinator.md). // It drives the collaborative loop against an injected `submit` (real = the // tfaRcaTurn MCP tool; tests = a recorded-turn replayer), so the loop mechanics — -// status branching, ask routing, gap resolution, turn-cap, one-thread, +// status branching, ask routing, gap degradation, turn-cap, one-thread, // soft-PENDING — are tested rather than assumed. // -// Double duty: this is ALSO the **sequential thin-client harness** (D5 / ideation -// #4) — the third caller of the same contract, for MCP clients without -// workflows/subagents. Pure + dependency-light (imports only the routing registry). +// Double duty: this is ALSO the **sequential thin-client harness** — the third +// caller of the same contract, for MCP clients without workflows/subagents. +// Pure + dependency-light (imports only the routing registry). +// +// The loop is fully autonomous: an evidence gap ALWAYS degrades to an +// `unavailable` block back to TFA. There is no user-prompt path — the /factory +// gate closed before this loop ever runs. +// +// tfaRcaTurn returns TRIMMED terminal shapes: +// RESOLVED → { status, confidence, threadId, +// glimpse: { root_cause (≤220 chars), failure_type, related_prs }, +// viewRca } +// PENDING → { status, turnId, threadId } (soft-pending, resumable) +// NEEDS_INFO → { status, questions/asks/suggestions } (verbatim — the loop needs them) import { routeAsks } from "./routing.mjs"; @@ -16,7 +27,7 @@ function unavailableBlock(gap) { `ASK: ${what}`, `TYPE: ${gap.evidenceType}`, `FOUND: no`, - `SUMMARY: unavailable — no ${gap.capability} capability for this client.`, + `SUMMARY: unavailable — no ${gap.capability} connector for this client.`, ].join("\n"); } @@ -24,7 +35,6 @@ function unavailableBlock(gap) { // // submit({ testRunId, message, threadId, turnId }) → Promise<turn> (tfaRcaTurn shape) // gather(routedGatherEntry) → Promise<string> (one digest block) -// resolveGap(routedGapEntry) → Promise<{ digest } | null> (auto: null; interactive: a digest) export async function runRcaLoop({ testRunId, firstMessage = "", @@ -32,7 +42,6 @@ export async function runRcaLoop({ config = {}, manifest = {}, gather = async () => "", - resolveGap = async () => null, turnCap = config?.turnCap ?? 6, }) { if (testRunId == null || Number.isNaN(Number(testRunId))) { @@ -56,19 +65,15 @@ export async function runRcaLoop({ const unavailable = new Set(); const out = (status, turn, note) => { - const rca = turn?.rca ?? {}; + const glimpse = turn?.glimpse ?? {}; return { testRunId: String(testRunId), status, confidence: turn?.confidence ?? "unknown", - root_cause: - status === "RESOLVED" - ? (rca.root_cause ?? "") - : status === "BLOCKED" - ? (turn?.reason ?? "") - : (note ?? ""), - possible_fix: rca.possible_fix ?? "", - related_prs: rca.related_prs ?? [], + root_cause: status === "RESOLVED" ? (glimpse.root_cause ?? "") : (note ?? ""), + failure_type: glimpse.failure_type ?? "", + related_prs: glimpse.related_prs ?? [], + view_rca: turn?.viewRca ?? "", threadId: threadId ?? null, turnId: turnId ?? null, turns_used: turns, @@ -84,7 +89,6 @@ export async function runRcaLoop({ threadId = turn.threadId ?? threadId; if (turn.status === "RESOLVED") return out("RESOLVED", turn); - if (turn.status === "BLOCKED") return out("BLOCKED", turn); if (turn.status === "PENDING") { turnId = turn.turnId ?? turnId; return out("PENDING", turn, "soft-pending"); @@ -95,7 +99,7 @@ export async function runRcaLoop({ // would run for nothing). if (turns >= turnCap) return out("PENDING", turn, "turn-cap"); - // Route + fulfill. + // Route + fulfill. Gaps degrade to `unavailable` — never a user prompt. const buckets = routeAsks(turn.asks ?? [], config, manifest); const blocks = []; for (const s of buckets.skip) skipped.add(s.evidenceType); @@ -104,14 +108,8 @@ export async function runRcaLoop({ fulfilled.add(g.evidenceType); } for (const gap of buckets.gap) { - const resolved = await resolveGap(gap); - if (resolved && resolved.digest) { - blocks.push(resolved.digest); - fulfilled.add(gap.evidenceType); - } else { - unavailable.add(gap.evidenceType); - blocks.push(unavailableBlock(gap)); - } + unavailable.add(gap.evidenceType); + blocks.push(unavailableBlock(gap)); } message = blocks.join("\n\n"); diff --git a/lib/report.mjs b/lib/report.mjs deleted file mode 100644 index 84ae275..0000000 --- a/lib/report.mjs +++ /dev/null @@ -1,69 +0,0 @@ -// Deterministic markdown report for a finished (or partial) batch. Degrade, -// don't crash: any missing field renders as "not available"; an empty batch -// still renders a valid report. Reads the CSV/WAL spine; no per-test transcripts. - -import { readRows } from "./csv-state.mjs"; - -const NA = "not available"; - -function cell(value) { - const s = value == null ? "" : String(value).trim(); - if (s === "") return NA; - // keep the table one-line-per-row: collapse newlines, escape pipes - return s.replace(/\s*\n\s*/g, " ").replace(/\|/g, "\\|"); -} - -function countBy(rows, key) { - return rows.reduce((acc, r) => { - const k = r[key] || "unknown"; - acc[k] = (acc[k] ?? 0) + 1; - return acc; - }, {}); -} - -// Render from a rows array (testable) — or pass a csvPath via renderReportFromCsv. -export function renderReport(rows, { buildId, generatedAt } = {}) { - const lines = []; - lines.push(`# RCA report${buildId ? ` — build ${buildId}` : ""}`); - if (generatedAt) lines.push(`\nGenerated: ${generatedAt}`); - - if (!rows || rows.length === 0) { - lines.push("\nNo failed tests analyzed."); - return lines.join("\n") + "\n"; - } - - const byState = countBy(rows, "rca_done"); - const summary = Object.entries(byState) - .map(([k, v]) => `${k}: ${v}`) - .join(" · "); - lines.push(`\n**${rows.length} test(s)** — ${summary}\n`); - - lines.push( - "| testRunId | test | status | confidence | coverage | root cause | related PRs |", - ); - lines.push("|---|---|---|---|---|---|---|"); - for (const r of rows) { - lines.push( - `| ${cell(r.testRunId)} | ${cell(r.testName)} | ${cell(r.rca_done)} | ${cell( - r.confidence, - )} | ${cell(r.coverage)} | ${cell(r.root_cause)} | ${cell(r.related_prs)} |`, - ); - } - - // Surface coverage caveats so a "low confidence" reads as "because X unavailable". - const thin = rows.filter((r) => r.coverage === "thin" || r.coverage === "partial"); - if (thin.length > 0) { - lines.push(`\n## Coverage caveats`); - for (const r of thin) { - lines.push( - `- ${cell(r.testRunId)} (${cell(r.coverage)} coverage): confidence band reflects evidence that was unavailable, not just model certainty.`, - ); - } - } - - return lines.join("\n") + "\n"; -} - -export function renderReportFromCsv(csvPath, opts = {}) { - return renderReport(readRows(csvPath), opts); -} diff --git a/tests/conformance.test.mjs b/tests/conformance.test.mjs index 5ea6192..82f7b8c 100644 --- a/tests/conformance.test.mjs +++ b/tests/conformance.test.mjs @@ -22,7 +22,7 @@ const GITHUB_AVAILABLE = { github: { available: true, via: "gh" } }; // A coordinator gather() stub: returns a one-line digest block. const gather = async (g) => `ASK: ${g.ask.what}\nTYPE: ${g.evidenceType}\nFOUND: yes\nSUMMARY: stub`; -test("resolved fixture: NEEDS_INFO → evidence → RESOLVED, rca captured, test_logs skipped", async () => { +test("resolved fixture: NEEDS_INFO → evidence → RESOLVED, trimmed glimpse captured, test_logs skipped", async () => { const fx = load("resolved.json"); const result = await runRcaLoop({ testRunId: fx.testRunId, @@ -34,25 +34,17 @@ test("resolved fixture: NEEDS_INFO → evidence → RESOLVED, rca captured, test }); assert.equal(result.status, "RESOLVED"); assert.match(result.root_cause, /#7421/); + assert.ok(result.root_cause.length <= 220); // glimpse root_cause is trimmed server-side + assert.equal(result.failure_type, "product_regression"); assert.deepEqual(result.related_prs, ["#7421"]); + assert.match(result.view_rca, /^https:\/\/observability\.browserstack\.com\/builds\//); assert.deepEqual(result.asks_fulfilled, ["product_code"]); assert.deepEqual(result.asks_skipped, ["test_logs"]); // TFA-owned, never gathered assert.equal(result.turns_used, 2); assert.equal(result.threadId, "thr-39"); }); -test("blocked fixture: terminal with reason captured", async () => { - const fx = load("blocked.json"); - const result = await runRcaLoop({ - testRunId: fx.testRunId, - submit: replaySubmit(fx.turns), - config: CONFIG, - }); - assert.equal(result.status, "BLOCKED"); - assert.match(result.root_cause, /could not obtain server-side logs/); -}); - -test("pending fixture: soft-PENDING ends with turnId, no re-poll", async () => { +test("pending fixture: soft-PENDING (trimmed: status/threadId/turnId) ends with turnId, no re-poll", async () => { const fx = load("pending.json"); let calls = 0; const counting = async (args) => { @@ -66,6 +58,7 @@ test("pending fixture: soft-PENDING ends with turnId, no re-poll", async () => { }); assert.equal(result.status, "PENDING"); assert.equal(result.turnId, "turn-81-1"); + assert.equal(result.threadId, "thr-81"); assert.equal(calls, 1); // ended immediately, did not poll again }); @@ -88,34 +81,35 @@ test("turn-cap fixture: ends PENDING(turn-cap) at the cap, never a 7th submit", assert.equal(submits, 6); // capped at turnCap, never 7 }); -test("degraded path: no capability + auto resolveGap → asks_unavailable, still terminal", async () => { - // Same resolved fixture, but the client has NO github capability and runs auto - // (resolveGap returns null → 'unavailable'). The loop must still reach RESOLVED. +test("degraded path: no connector → gap degrades to unavailable (never a prompt), still terminal", async () => { + // Same resolved fixture, but the client has NO github connector. The loop is + // autonomous: the gap becomes an `unavailable` block and it still RESOLVEs. const fx = load("resolved.json"); const result = await runRcaLoop({ testRunId: fx.testRunId, submit: replaySubmit(fx.turns), config: CONFIG, - manifest: {}, // nothing available - resolveGap: async () => null, // auto: report unavailable + manifest: {}, // nothing valid at the gate }); assert.equal(result.status, "RESOLVED"); assert.deepEqual(result.asks_unavailable, ["product_code"]); assert.deepEqual(result.asks_fulfilled, []); }); -test("interactive resolveGap supplies the missing evidence → fulfilled, not unavailable", async () => { +test("unavailable block names the missing connector in the resubmitted message", async () => { const fx = load("resolved.json"); - const result = await runRcaLoop({ + const messages = []; + const recording = (inner) => async (args) => { + messages.push(args.message); + return inner(args); + }; + await runRcaLoop({ testRunId: fx.testRunId, - submit: replaySubmit(fx.turns), + submit: recording(replaySubmit(fx.turns)), config: CONFIG, manifest: {}, - resolveGap: async () => ({ digest: "ASK: ...\nFOUND: yes\nSUMMARY: user supplied" }), }); - assert.equal(result.status, "RESOLVED"); - assert.deepEqual(result.asks_fulfilled, ["product_code"]); - assert.deepEqual(result.asks_unavailable, []); + assert.match(messages[1], /unavailable — no github connector/); }); test("no testRunId → failed block, tool never called", async () => { diff --git a/tests/coverage-report.test.mjs b/tests/coverage-glimpse.test.mjs similarity index 56% rename from tests/coverage-report.test.mjs rename to tests/coverage-glimpse.test.mjs index 7d7cb94..18f2817 100644 --- a/tests/coverage-report.test.mjs +++ b/tests/coverage-glimpse.test.mjs @@ -1,7 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { coverageStamp, classifyCoverage } from "../lib/coverage.mjs"; -import { renderReport } from "../lib/report.mjs"; +import { renderGlimpse } from "../lib/glimpse.mjs"; // ---- coverage stamp -------------------------------------------------------- @@ -47,62 +47,51 @@ test("classifyCoverage dedupes and handles empties", () => { assert.equal(classifyCoverage([], ["x"]), "thin"); }); -// ---- report ---------------------------------------------------------------- +// ---- glimpse (the ONLY in-client output — no local report) ------------------ -test("empty batch renders a valid report, no crash", () => { - const md = renderReport([], { buildId: "b1" }); - assert.match(md, /No failed tests analyzed/); +test("empty batch renders a valid glimpse, no crash", () => { + const txt = renderGlimpse([], { buildId: "b1" }); + assert.match(txt, /No failed tests analyzed/); }); -test("report renders a row table with status counts", () => { +test("glimpse renders one arrow line per test with status counts", () => { const rows = [ { testRunId: "101", - testName: "login", + cluster_id: "c1", rca_done: "resolved", confidence: "high", - coverage: "full", root_cause: "PR #7421 tightened validator", - related_prs: "#7421", }, { testRunId: "102", - testName: "checkout", - rca_done: "blocked", + cluster_id: "c1", + rca_done: "failed", confidence: "", - coverage: "", root_cause: "", - related_prs: "", }, ]; - const md = renderReport(rows, { buildId: "b1" }); - assert.match(md, /2 test\(s\)/); - assert.match(md, /resolved: 1/); - assert.match(md, /blocked: 1/); - assert.match(md, /101/); - assert.match(md, /not available/); // 102's blank fields degrade + const txt = renderGlimpse(rows, { buildId: "b1" }); + assert.match(txt, /2 test\(s\)/); + assert.match(txt, /resolved: 1/); + assert.match(txt, /failed: 1/); + assert.match(txt, /101 → c1 → resolved → high: PR #7421 tightened validator/); + assert.match(txt, /102 → c1 → failed → -/); // blank fields degrade to "-" }); -test("report escapes pipes and collapses newlines in cells", () => { +test("glimpse one-liner truncates and collapses newlines (stays terse)", () => { const rows = [ { testRunId: "1", - testName: "t", + cluster_id: "solo-1", rca_done: "resolved", - root_cause: "a | b\nsecond line", - related_prs: "#1", + confidence: "medium", + root_cause: `line one\nline two ${"x".repeat(200)}`, }, ]; - const md = renderReport(rows); - assert.ok(!md.includes("a | b\nsecond")); - assert.match(md, /a \\\| b second line/); -}); - -test("report surfaces coverage caveats for thin/partial rows", () => { - const rows = [ - { testRunId: "1", testName: "t", rca_done: "resolved", coverage: "partial" }, - ]; - const md = renderReport(rows); - assert.match(md, /Coverage caveats/); - assert.match(md, /confidence band reflects evidence that was unavailable/); + const txt = renderGlimpse(rows); + const line = txt.split("\n").find((l) => l.startsWith("1 →")); + assert.ok(line.includes("line one line two")); + assert.ok(line.endsWith("…")); + assert.ok(line.length < 120); }); diff --git a/tests/fixtures/recorded-turns/blocked.json b/tests/fixtures/recorded-turns/blocked.json deleted file mode 100644 index 35b5373..0000000 --- a/tests/fixtures/recorded-turns/blocked.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "blocked — unmet asks", - "testRunId": 72, - "turns": [ - { - "status": "BLOCKED", - "confidence": "low", - "threadId": "thr-72", - "reason": "could not obtain server-side logs; cannot distinguish product bug from env flake", - "unmetAsks": ["kibana", "k8s"] - } - ] -} diff --git a/tests/fixtures/recorded-turns/pending.json b/tests/fixtures/recorded-turns/pending.json index ec8b16a..2e73ce7 100644 --- a/tests/fixtures/recorded-turns/pending.json +++ b/tests/fixtures/recorded-turns/pending.json @@ -1,10 +1,9 @@ { - "name": "soft-pending — resumable", + "name": "soft-pending — resumable (trimmed shape: status/threadId/turnId only)", "testRunId": 81, "turns": [ { "status": "PENDING", - "confidence": "unknown", "threadId": "thr-81", "turnId": "turn-81-1" } diff --git a/tests/fixtures/recorded-turns/resolved.json b/tests/fixtures/recorded-turns/resolved.json index 120dad6..47b9034 100644 --- a/tests/fixtures/recorded-turns/resolved.json +++ b/tests/fixtures/recorded-turns/resolved.json @@ -1,5 +1,5 @@ { - "name": "needs_info → evidence → resolved", + "name": "needs_info → evidence → resolved (trimmed terminal glimpse)", "testRunId": 39, "turns": [ { @@ -26,12 +26,12 @@ "status": "RESOLVED", "confidence": "high", "threadId": "thr-39", - "rca": { + "glimpse": { "root_cause": "PR #7421 tightened the buildName validator to reject empty strings", - "possible_fix": "send a non-empty buildName or relax the validator", "failure_type": "product_regression", "related_prs": ["#7421"] - } + }, + "viewRca": "https://observability.browserstack.com/builds/awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2" } ] } From 412611938a9986d91f38d0af420f8dd35af93355 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Mon, 13 Jul 2026 00:28:41 +0530 Subject: [PATCH 21/47] feat(rca): coordinator + batch workflow go autonomous on trimmed shapes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ai-tfa-coordinator: drop the mode input and the interactive gap-return (GAP_OUTPUT); gaps always degrade to 'unavailable' — never a prompt. Classify only RESOLVED|PENDING|NEEDS_INFO (BLOCKED gone); consume the trimmed RESOLVED glimpse {root_cause, failure_type, related_prs} + viewRca and pass it through verbatim; RCA_OUTPUT gains failure_type + view_rca, drops possible_fix/BLOCKED - mandatory culprit-PR hunt for PRODUCT_BUG / application bugs: feed PR link(s) to TFA so dashboard related_prs populates; no PR by turn cap → explicit 'no culprit PR identified after <searched>' + recorded gap - workflows/rca-batch.mjs: single autonomous path (no mode arg), schema matches the new RCA_OUTPUT, PRODUCT_BUG mandate in the shared prompt Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- agents/ai-tfa-coordinator.md | 144 ++++++++++++++++++----------------- workflows/rca-batch.mjs | 35 +++++---- 2 files changed, 96 insertions(+), 83 deletions(-) diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 3de14aa..167b155 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -1,9 +1,9 @@ --- name: ai-tfa-coordinator -description: 'Per-test collaborative-RCA coordinator. Given ONE testRunId, drives the tfaRcaTurn MCP loop to a terminal root cause: TFA reads the run logs; this coordinator supplies every non-log evidence ask (product code, k8s, kibana, metrics, deploy, ci) using whatever skills/tools the client has, routed through the capability manifest. Skips every test_logs ask (TFA owns logs). Emits a structured RCA_OUTPUT block. Generic over product and infra — no hardcoded tools. Examples: +description: 'Per-test collaborative-RCA coordinator (autonomous — never prompts a user). Given ONE testRunId, drives the tfaRcaTurn MCP loop to a terminal root cause: TFA reads the run logs; this coordinator supplies every non-log evidence ask (product code, k8s, kibana, metrics, deploy, ci) using whatever skills/tools the client has, routed through the validated capability manifest. Skips every test_logs ask (TFA owns logs). For application bugs it MUST hunt the culprit PR via the github connector. Emits a structured RCA_OUTPUT block. Generic over product and infra — no hardcoded tools. Examples: - orchestrator: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=39 — error: empty buildName rejected on POST /builds") → drives the loop, returns RCA_OUTPUT - sibling confirm: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=40 — pre-seed: cause=<rep root cause>, suspect PR=#7421") → one-turn confirm against this test logs -- user: "run collaborative RCA on test run 39" → single-test loop to RESOLVED/BLOCKED/PENDING' +- user: "run collaborative RCA on test run 39" → single-test loop to RESOLVED/PENDING' tools: [Bash, Read, Grep, Glob, Task, mcp__*__tfaRcaTurn, mcp__github__*] model: sonnet --- @@ -15,13 +15,19 @@ The collaboration contract is fixed: **TFA owns logs; this coordinator owns everything else.** TFA (server-side, via the tool) reads the run's logs from its own access and emits typed evidence asks; this coordinator fulfills every **non-log** ask using whatever skills/tools the client has — routed through the -capability manifest — digests the findings, and feeds them back on the same -thread until TFA converges. TFA authors the RCA into the TRA dashboard. +validated capability manifest — digests the findings, and feeds them back on the +same thread until TFA converges. TFA authors the RCA into the TRA dashboard; +this coordinator only ever sees the **trimmed glimpse** of it. The full report +lives on the Test Observability UI. + +This coordinator is **fully autonomous**: the `/factory` gate closed before it +was dispatched, so it **never prompts a user** — an evidence gap degrades to an +`unavailable` block back to TFA, always. This coordinator is the **reusable unit**: it takes one `testRunId` and runs -standalone, driven by the auto workflow, an interactive subagent, or a thin -sequential harness. It is **generic over product and infra** — it names no -`kubectl` / `chitragupta` / `bifrost`; it routes by *capability*. +standalone, driven by the batch workflow, a subagent dispatch, or the thin +sequential harness (`lib/loop.mjs`). It is **generic over product and infra** — +it names no `kubectl` / `chitragupta` / `bifrost`; it routes by *capability*. ## Inputs @@ -31,13 +37,25 @@ sequential harness. It is **generic over product and infra** — it names no `root_cause` + suspect `related_prs`. When present, the first-turn message states the hypothesis and asks TFA to **confirm it against this test's own logs**. - `resume` — optional `{ threadId, turnId }` from a prior PENDING run. -- `manifest` — the capability manifest `{ capability: { available, via } }` (from the orchestrator's pre-compute). -- `mode` — `auto` | `interactive`. Selects the **gap-resolver** (see below). +- `manifest` — the validated capability manifest `{ capability: { available, via } }` + (built once at the `/factory` gate — Part A). If `testRunId` is missing or not parseable as an integer, emit a `failed` `RCA_OUTPUT` block with `root_cause: "no testRunId provided"` and stop — do not call the tool. +## What the tool returns (trimmed shapes) + +`tfaRcaTurn` returns **trimmed** terminal turns — never the full RCA payload: + +- `RESOLVED` → `{ status, confidence, threadId, glimpse: { root_cause (≤220 + chars), failure_type, related_prs }, viewRca }`. The `viewRca` link points at + the Test Observability UI — pass it through to the output. +- `PENDING` → `{ status, turnId, threadId }` (soft-pending; in-call poll + exceeded its wall-clock cap — resumable). +- `NEEDS_INFO` → `questions` / `asks` / `suggestions` **verbatim** — this loop + consumes them exactly as sent. + ## Operating principles 1. **Logs by TFA — the core contract.** Never seed logs in the first turn; @@ -50,54 +68,37 @@ call the tool. extra turn, never a busy-wait. 4. **One thread per test.** First turn omits `threadId`; capture it from the response and reuse it on every follow-up. Never start a second thread. -5. **Soft-PENDING ends the loop.** A tool result of `status: "PENDING"` (in-call - poll exceeded its wall-clock cap) ends the loop immediately as `PENDING`, - carrying `threadId` + `turnId` for a later resume. Do not re-poll or sleep. +5. **Soft-PENDING ends the loop.** A tool result of `status: "PENDING"` ends the + loop immediately as `PENDING`, carrying `threadId` + `turnId` for a later + resume. Do not re-poll or sleep. 6. **Digest, don't dump.** Every follow-up `message` carries digested findings (`ask → found → snippet/link`), never raw log tails, full diffs, or full files. Size caps + block shape live in `references/evidence-routing.md` — read it before fulfilling any ask. The tool caps `message` at 5000 chars. 7. **Report gaps, don't drop them.** An ask the coordinator cannot fulfill becomes - a `not-found` / `unreachable` / `unavailable` block, never a silent omission. + a `not-found` / `unreachable` / `unavailable` block, never a silent omission — + and **never a user prompt**. TFA finalizes best-effort with lower confidence. 8. **Never editorialize.** Report findings (suspect PR, server-side error line), - not verdicts. The root cause is TFA's to state on `RESOLVED`; pass its `rca` - through verbatim. - -## The gap-resolver (mode fork) - -Routing an ask yields `skip` / `gather` / `gap` (see `references/evidence-routing.md`). -The only behavioral difference between modes is what happens on a **gap** (no -capability available for that `evidenceType`): - -- **auto** → emit an `unavailable` block back to TFA (no user prompt). TFA - finalizes best-effort with lower confidence. -- **interactive** → a subagent cannot pause to prompt the user, so **end the run - early and return a `GAP_OUTPUT` block** (status `PENDING`) carrying the resume - handles + the gap. The orchestrator asks A1, then **re-dispatches a coordinator - with `resume={threadId, turnId}`** and the answer digested into the next turn. - See `references/interactive-mode.md`. - -`GAP_OUTPUT` block (interactive gap only): - -``` -GAP_OUTPUT_START -## testRunId -<integer> -## thread_id -<threadId> -## turn_id -<turnId> # resume handle -## gap -- evidenceType: <type> -- what: <verbatim ask `what`> -- why: <verbatim ask `why`> -GAP_OUTPUT_END -``` - -Everything else — the loop, routing, digest, caps, terminal output — is identical -across modes. Do not fork the loop; only the gap action differs. When all gaps in -a turn are resolvable (gathered or user-answered), the loop proceeds normally to a -terminal `RCA_OUTPUT`. + not verdicts. The root cause is TFA's to state on `RESOLVED`; pass its + `glimpse` through verbatim. + +## Application bugs — the culprit-PR mandate (MANDATORY) + +Whenever TFA's classification (in an ask, a suggestion, or the resolving +`glimpse.failure_type`) is **PRODUCT_BUG / application bug**, the github +connector is the deliverable, not optional evidence: + +- **Hunt the culprit PR**: deploy timeline vs the last-pass window, changed + paths vs the failure signature (`references/github-evidence.md`), run the + falsification protocol on each candidate. +- **Feed the PR link(s) to TFA in the turn message** so the BrowserStack agent + populates `related_prs` in the dashboard RCA. +- **An application-bug RCA with no GitHub PR link is INCOMPLETE.** Keep digging + on subsequent turns until the turn cap. If still none, the turn message must + explicitly state `no culprit PR identified after <what was searched: window, + repos, paths>` — and the orchestrator records the gap on the CSV row. +- If the github connector is invalid/absent (a gate-recorded gap), state the + same explicitly plus an `unavailable` block. Never fabricate a PR. ## Suspect-PR falsification (github asks) @@ -122,8 +123,7 @@ re-fetch per test. Never fabricate a PR when the github capability is unavailabl 1. SUBMIT turn 1: tfaRcaTurn(testRunId=<id>, message=<digest>). Capture threadId. turns_used = 1. (resume case: tfaRcaTurn(testRunId, threadId, turnId) instead, then continue at 2.) 2. CLASSIFY result.status: - RESOLVED → capture rca; END (RESOLVED). - BLOCKED → capture reason + unmetAsks; END (BLOCKED). + RESOLVED → capture glimpse + viewRca; END (RESOLVED). PENDING → capture threadId + turnId; END (PENDING, note "soft-pending"). NEEDS_INFO → go to 3. 3. ROUTE the asks (read references/evidence-routing.md; route via lib/routing.mjs): @@ -131,7 +131,8 @@ re-fetch per test. Never fabricate a PR when the github capability is unavailabl skip → record in asks_skipped, emit nothing. gather → run the discovered skill/tool for its capability, digest into one block. Record evidenceType in asks_fulfilled (dedupe). - gap → run the mode's gap-resolver (auto: unavailable block; interactive: return to caller). + gap → emit an `unavailable` block (record in asks_unavailable). NEVER prompt. + PRODUCT_BUG in play + no supported PR yet → widen the github hunt this turn. Concatenate per-ask blocks into the next-turn MESSAGE (respect size caps). 4. SUBMIT follow-up on the SAME thread: tfaRcaTurn(testRunId, message, threadId). turns_used += 1. 5. TURN-CAP CHECK: if turns_used >= turnCap and still NEEDS_INFO → END (PENDING, "turn-cap"). @@ -142,20 +143,20 @@ re-fetch per test. Never fabricate a PR when the github capability is unavailabl > The loop mechanics above have an **executable mirror** in `lib/loop.mjs` > (`runRcaLoop`) — conformance-tested against recorded `tfaRcaTurn` transcripts > (`tests/conformance.test.mjs`). It also serves as the **sequential thin-client -> harness** (D5): MCP clients without workflows/subagents drive the same contract +> harness**: MCP clients without workflows/subagents drive the same contract > by calling `runRcaLoop` with a real `submit` bound to `tfaRcaTurn`. **Sibling confirm (cluster member).** When `pre_seed` is present the first turn states the representative's hypothesis and asks TFA to confirm against this test's own logs. If TFA `RESOLVED`s in one turn → a logs-grounded per-test RCA at -minimal cost. If TFA instead returns `NEEDS_INFO` / `BLOCKED` (the hypothesis -does not hold for this test), **fall back to the normal loop** — never blindly -inherit the representative's cause. +minimal cost. If TFA instead returns `NEEDS_INFO` (the hypothesis does not hold +for this test), **fall back to the normal loop** — never blindly inherit the +representative's cause. ## Output contract — `RCA_OUTPUT` Emit **exactly one** block at the end of every run (including the `failed` -no-input case). The orchestrator parses it into one CSV row / report record. +no-input case). The orchestrator parses it into one CSV row / glimpse line. ``` RCA_OUTPUT_START @@ -164,19 +165,22 @@ RCA_OUTPUT_START <integer> ## status -<RESOLVED | BLOCKED | PENDING | failed> +<RESOLVED | PENDING | failed> ## confidence <high | medium | low | unknown> # from the terminal turn; unknown for PENDING/failed ## root_cause -<RESOLVED → rca.root_cause verbatim · BLOCKED → TFA's reason · PENDING/failed → "not available" or the note> +<RESOLVED → glimpse.root_cause verbatim (already ≤220 chars) · PENDING/failed → "not available" or the note> -## possible_fix -<RESOLVED → rca.possible_fix verbatim · else "not available"> +## failure_type +<RESOLVED → glimpse.failure_type verbatim · else "not available"> ## related_prs -- <each PR TFA recorded in rca.related_prs; "none" if empty> +- <each PR in glimpse.related_prs; "none" if empty — for PRODUCT_BUG, "none" only after the mandated hunt + explicit statement> + +## view_rca +<viewRca link from the RESOLVED turn (Test Observability UI) · "not available" if none> ## suspect_signals - <each non-log signal surfaced: suspect PR / deploy / server-side error line; "none" if empty> @@ -197,28 +201,32 @@ RCA_OUTPUT_START - test_logs # present once a test_logs ask appeared ## asks_unavailable -- <evidenceType> # gaps with no capability (drives the coverage stamp, U10); "none" if empty +- <evidenceType> # gate-recorded gaps (drives the coverage stamp); "none" if empty RCA_OUTPUT_END ``` Notes: -- `status` is one of exactly four values. `turn-cap` and `soft-pending` both +- `status` is one of exactly three values. `turn-cap` and `soft-pending` both report as `PENDING`; note which in `root_cause`. - `asks_skipped` always includes `test_logs` whenever TFA asked for logs. `asks_fulfilled` **never** includes `test_logs`. -- `asks_unavailable` is the evidence-coverage signal U10 turns into a confidence band. +- `asks_unavailable` is the evidence-coverage signal the coverage stamp turns + into a confidence band. - `failed` is the no-parseable-result / no-input case; the orchestrator synthesizes a `failed` row if this coordinator dies — keep the block valid. ## Hard limits +- **Never** prompt, ask, or wait on a user — the gate is closed; gaps degrade to `unavailable`. - **Never** fulfill or seed a `test_logs` ask — TFA owns logs. - **Never** exceed `turnCap` `tfaRcaTurn` calls in one run. - **Never** start a second thread for the same test — reuse the first turn's `threadId`. - **Never** busy-wait / re-poll on a soft-`PENDING` — end and report it resumable. - **Never** dump raw logs, full diffs, or full file contents into a turn message — digest only. - **Never** write to any repo / cluster / ticket / the run — every action is read-only. -- **Never** editorialize a cause — pass TFA's `rca` through verbatim. +- **Never** editorialize a cause — pass TFA's `glimpse` through verbatim. - **Never** blindly inherit a representative's cause for a sibling — confirm against its own logs. +- **Never** resolve an application bug silently without a PR link — hunt until the + turn cap, else state "no culprit PR identified after <searched>" explicitly. - **Always** emit exactly one valid `RCA_OUTPUT` block, even on the `failed` path. diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index 07868e9..1fe8699 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -1,24 +1,27 @@ export const meta = { name: "rca-batch", description: - "Drive collaborative RCA over all failed tests of a build (auto mode): cluster representatives run the full loop, siblings one-turn-confirm, ~5 concurrent.", + "Drive autonomous collaborative RCA over all failed tests of a build: cluster representatives run the full loop, siblings one-turn-confirm, ~5 concurrent. Never prompts a user.", phases: [ { title: "Representatives", detail: "full multi-turn RCA per cluster" }, { title: "Siblings", detail: "one-turn confirm against own logs" }, ], }; -// AUTO MODE orchestration (D2). This is a dynamic-workflow script: it runs in the -// Workflow sandbox (no filesystem, no Date.now/Math.random, agent()/pipeline() -// as globals). It therefore does NO state I/O itself — the orchestrator seeds the -// CSV, clusters, and builds the manifest in normal context and passes the -// work-list via `args`; each dispatched `ai-tfa-coordinator` agent (which HAS -// tool access) claims + flips its own CSV row eagerly (WAL); this script -// orchestrates concurrency and returns the structured results for reconciliation. +// The /factory batch orchestration (fully autonomous — the gate closed before +// this runs; nothing here ever asks the user). This is a dynamic-workflow +// script: it runs in the Workflow sandbox (no filesystem, no Date.now/ +// Math.random, agent()/pipeline() as globals). It therefore does NO state I/O +// itself — the orchestrator seeds the CSV, clusters, and builds the validated +// manifest at the gate and passes the work-list via `args`; each dispatched +// `ai-tfa-coordinator` agent (which HAS tool access) claims + flips its own CSV +// row eagerly (WAL); this script orchestrates concurrency and returns the +// structured results for reconciliation. The final glimpse + triggerRcaReport +// step happens back in the orchestrator (SKILL.md Step 6). // // args shape: // { -// csvPath, buildId, mode: "auto", +// csvPath, buildId, // manifest: { capability: { available, via } }, // buildEvidence: { baselineRef, suspectWindow, ... }, // pre-computed once // clusters: [ @@ -32,10 +35,11 @@ const RCA_SCHEMA = { required: ["testRunId", "status"], properties: { testRunId: { type: "string" }, - status: { enum: ["RESOLVED", "BLOCKED", "PENDING", "failed"] }, + status: { enum: ["RESOLVED", "PENDING", "failed"] }, confidence: { enum: ["high", "medium", "low", "unknown"] }, root_cause: { type: "string" }, - possible_fix: { type: "string" }, + failure_type: { type: "string" }, + view_rca: { type: "string" }, related_prs: { type: "array", items: { type: "string" } }, suspect_signals: { type: "array", items: { type: "string" } }, threadId: { type: "string" }, @@ -55,7 +59,8 @@ const shared = [ `CSV state file: ${ctx.csvPath}`, `Capability manifest: ${JSON.stringify(ctx.manifest ?? {})}`, `Build-level evidence (pre-computed once, reuse — do not re-fetch): ${JSON.stringify(ctx.buildEvidence ?? {})}`, - `Mode: auto — on an evidence gap with no capability, report "unavailable" back to TFA (NEVER prompt a user). Best-effort finalize.`, + `Autonomous run — on an evidence gap with no valid connector, report "unavailable" back to TFA (NEVER prompt a user). Best-effort finalize.`, + `PRODUCT_BUG / application-bug mandate: hunt the culprit PR via the github connector (deploy timeline vs last-pass window, changed paths vs failure signature) and feed the PR link(s) to TFA so related_prs populates. No PR after digging to the turn cap → state explicitly "no culprit PR identified after <what was searched>" so the CSV row records the gap.`, `Persist eagerly to the CSV: claim your row before turn 1, flip it on terminal (lib/csv-state.mjs).`, ].join("\n"); @@ -78,7 +83,7 @@ function siblingPrompt(sibling, repResult, cluster) { ` root_cause: ${repResult?.root_cause ?? "(representative did not resolve)"}`, ` related_prs: ${JSON.stringify(repResult?.related_prs ?? [])}`, `State this hypothesis on turn 1 and ask TFA to CONFIRM it against THIS test's own logs.`, - `If TFA confirms in one turn → done. If it does NOT (NEEDS_INFO/BLOCKED), fall back to the full loop — never blindly inherit.`, + `If TFA confirms in one turn → done. If it does NOT (NEEDS_INFO), fall back to the full loop — never blindly inherit.`, `testRunId=${sibling.testRunId} testName=${sibling.testName ?? ""}`, `error_digest: ${sibling.error_summary ?? "(none)"}`, shared, @@ -86,7 +91,7 @@ function siblingPrompt(sibling, repResult, cluster) { ].join("\n"); } -log(`Auto-mode batch: ${clusters.length} cluster(s) over build ${ctx.buildId ?? "?"}`); +log(`Batch: ${clusters.length} cluster(s) over build ${ctx.buildId ?? "?"}`); // Pipeline: each cluster flows representative → siblings independently (no barrier // between stages), so a small cluster's siblings confirm while a big cluster's @@ -125,6 +130,6 @@ const byStatus = all.reduce((acc, r) => { return acc; }, {}); -log(`Auto-mode batch complete: ${all.length} test(s) — ${JSON.stringify(byStatus)}`); +log(`Batch complete: ${all.length} test(s) — ${JSON.stringify(byStatus)}`); return { clusters: flat.length, tests: all.length, byStatus, results: flat }; From 97dbe1d6db9db1422a5dfac8bbfda3863f099bf1 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Mon, 13 Jul 2026 00:30:47 +0530 Subject: [PATCH 22/47] docs(rca): tell the single-gate /factory story; scrub mode/report/BLOCKED residue - README: /factory usage, the one-gate two-part flow (validated connector manifest + assumption-first intake, at most one consolidated question), autonomous execution, glimpse + triggerRcaReport + Test Observability UI link as the only outputs - INTEGRATION: factory skill naming, triggerRcaReport in the MCP core row, no-local-report finish on every host - automation/README: demo invocation is now /factory <build-id> - lib/routing.mjs comments: gaps degrade to 'unavailable', no resolveGap Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- INTEGRATION.md | 34 ++++++++------- README.md | 100 +++++++++++++++++++++++++++---------------- automation/README.md | 2 +- lib/routing.mjs | 13 +++--- 4 files changed, 90 insertions(+), 59 deletions(-) diff --git a/INTEGRATION.md b/INTEGRATION.md index 9d91cb5..e72cbf6 100644 --- a/INTEGRATION.md +++ b/INTEGRATION.md @@ -2,36 +2,40 @@ This plugin is built so the **MCP core is truly cross-client** and the **harness layer ports via the cross-vendor Agent Skills standard**. Only one piece is -genuinely Claude-Code-specific (the auto-mode *dynamic workflow*); on Cursor and -Codex that role is filled by the sequential harness or interactive subagents. +genuinely Claude-Code-specific (the batch *dynamic workflow*); on Cursor and +Codex that role is filled by the sequential harness or subagents. Every path is +autonomous after the single `/factory` gate — no host ever prompts mid-run. ## What transfers, what doesn't | Layer | Claude Code | Cursor | Codex | |---|---|---|---| -| `bstack` MCP server (`listTestIds` + `tfaRcaTurn`) | `.mcp.json` (auto-discovered) | `.cursor-mcp.json` / `.cursor/mcp.json` | `~/.codex/config.toml` `[mcp_servers.bstack]` | -| `rca-build` skill (`SKILL.md`) | plugin `skills/` | Agent Skills (`.cursor/skills/` or cursor-plugin `"skills":"./skills/"`) | Agent Skills (`.agents/skills/`) | +| `bstack` MCP server (`listTestIds` + `tfaRcaTurn` + `triggerRcaReport`) | `.mcp.json` (auto-discovered) | `.cursor-mcp.json` / `.cursor/mcp.json` | `~/.codex/config.toml` `[mcp_servers.bstack]` | +| `factory` skill (`SKILL.md`) | plugin `skills/` | Agent Skills (`.cursor/skills/` or cursor-plugin `"skills":"./skills/"`) | Agent Skills (`.agents/skills/`) | | `ai-tfa-coordinator` agent | plugin `agents/` | `.cursor/agents/` (also reads `.claude/agents/`) | `.codex/agents/` | | Per-test RCA **loop** | `agents/ai-tfa-coordinator.md` | same skill/agent | same skill/agent | -| Batch orchestration | **auto** = dynamic workflow `workflows/rca-batch.mjs`; **interactive** = subagents | subagents, or **sequential** `lib/loop.mjs` | subagents, or **sequential** `lib/loop.mjs` | +| Batch orchestration | dynamic workflow `workflows/rca-batch.mjs` (or subagents) | subagents, or **sequential** `lib/loop.mjs` | subagents, or **sequential** `lib/loop.mjs` | The dynamic workflow (`workflows/rca-batch.mjs`) uses Claude Code's Workflow runtime, which Cursor/Codex don't have. The same batch still runs there via -**interactive subagents** (both hosts support subagents) or the **sequential -thin-client harness** `lib/loop.mjs` (`runRcaLoop`) — the conformance-tested -loop that drives `tfaRcaTurn` over the same contract without any host-specific -orchestration. +**subagents** (both hosts support subagents) or the **sequential thin-client +harness** `lib/loop.mjs` (`runRcaLoop`) — the conformance-tested loop that +drives `tfaRcaTurn` over the same contract without any host-specific +orchestration. On every host the run finishes the same way: glimpse table → +`triggerRcaReport(buildUuid)` → "Full report on the Test Observability UI: +<viewReport>". No local report file is ever written. ## Claude Code ```bash cp .env.example .env # BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY claude --plugin-dir ./ -/rca-build <build-id> +/factory <build-id> ``` -`.claude-plugin/plugin.json` + root `.mcp.json` + `skills/` + `agents/` + -`commands/` are auto-discovered. +`.claude-plugin/plugin.json` + root `.mcp.json` + `skills/` + `agents/` are +auto-discovered. (No `commands/factory.md` on purpose — a command and skill +with the same name collide and the skill body fails to load.) ## Cursor @@ -59,7 +63,7 @@ ln -s ../skills .cursor/skills ln -s ../agents .cursor/agents ``` -Then drive it from Agent chat: invoke the `rca-build` skill with a build id. +Then drive it from Agent chat: invoke the `factory` skill with a build id. ## Codex @@ -84,7 +88,7 @@ ln -s ../skills .agents/skills ln -s ../agents .codex/agents ``` -Then run the `rca-build` skill; the coordinator + `tfaRcaTurn` loop are identical. +Then run the `factory` skill; the coordinator + `tfaRcaTurn` loop are identical. ## Notes @@ -95,4 +99,4 @@ Then run the `rca-build` skill; the coordinator + `tfaRcaTurn` loop are identica Cursor/Codex, replace the placeholders with literals if your client doesn't expand them. - Everything in `lib/` and the `SKILL.md`/agent prose is host-agnostic — only the - MCP wiring file and the auto-mode workflow are host-specific. + MCP wiring file and the dynamic workflow are host-specific. diff --git a/README.md b/README.md index 34a910c..88fb748 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,18 @@ Drive BrowserStack's collaborative root-cause-analysis loop over **all failed tests of a build**, generic across product and infra, from inside an agentic MCP client (Claude Code / Cursor / Codex). -The plugin wraps two stable MCP tools — `listTestIds` and `tfaRcaTurn` (from the -`bstack` MCP server) — and adds the harness that batches RCA over a whole build, -clusters failures by signature, routes evidence requests to whatever -skills/tools the client already has, and writes a per-test RCA into the TRA -dashboard. - -> It **discovers and delegates** to the infra skills/tools already in your -> client (GitHub, k8s/EKS, kibana/other logs, metrics). It does **not** install -> or own those connectors. +The plugin wraps three stable MCP tools — `listTestIds`, `tfaRcaTurn`, and +`triggerRcaReport` (from the `bstack` MCP server) — and adds the harness that +batches RCA over a whole build, clusters failures by signature, routes evidence +requests to whatever skills/tools the client already has, and lands a per-test +RCA in the TRA (Test Observability) dashboard. + +> **The full RCA report lives on the Test Observability UI, not in Claude.** +> The plugin surfaces a terse glimpse, triggers the dashboard report +> (`triggerRcaReport`), and prints the link. It **discovers and delegates** to +> the infra skills/tools already in your client (GitHub, k8s/EKS, kibana/other +> logs, metrics). It does **not** install or own those connectors, and it never +> writes a local report file. ## Install @@ -24,42 +27,63 @@ claude --plugin-dir ./ ``` The plugin auto-configures on load: the `bstack` MCP server (from `.mcp.json`), -the `/rca-build` command, the `rca-build` skill, and the `ai-tfa-coordinator` -agent are all discovered by convention. +the `factory` skill, and the `ai-tfa-coordinator` agent are all discovered by +convention. (There is deliberately **no** command file named `factory` — a +command and skill sharing a name collide and the skill body fails to load.) ### Cursor & Codex -The MCP core (`listTestIds` + `tfaRcaTurn`) and the skill/agent layer port to -both — Cursor uses `.cursor-plugin/plugin.json` + `.cursor-mcp.json`, Codex uses -`~/.codex/config.toml` (see `codex-mcp.example.toml`). The only Claude-specific -piece is the auto-mode *dynamic workflow*; on Cursor/Codex the same batch runs via -interactive subagents or the sequential harness (`lib/loop.mjs`). Full -per-host wiring (MCP config, skill/agent discovery, deeplink) is in -**[INTEGRATION.md](INTEGRATION.md)**. +The MCP core (`listTestIds` + `tfaRcaTurn` + `triggerRcaReport`) and the +skill/agent layer port to both — Cursor uses `.cursor-plugin/plugin.json` + +`.cursor-mcp.json`, Codex uses `~/.codex/config.toml` (see +`codex-mcp.example.toml`). The only Claude-specific piece is the batch *dynamic +workflow*; on Cursor/Codex the same batch runs via subagents or the sequential +harness (`lib/loop.mjs`). Full per-host wiring (MCP config, skill/agent +discovery, deeplink) is in **[INTEGRATION.md](INTEGRATION.md)**. ## Usage ``` -/rca-build <build-id> -/rca-build build_id=<id> mode=auto +/factory <build-id> +/factory build_id=<id> https://github.com/org/repo/pull/123 ``` -On start the plugin runs a **mandatory pre-flight intake** asking for your -product + automation repos, working branch, default branch, and the PRs in -play, plus the build id. Every question is answerable with "I don't have one" → -the run proceeds RCA-only. +Args: a build id (bare, `build_id=`, or a dashboard link) plus optional PR URLs +/ repo hints. + +## The single gate + +The run has exactly **one gate** before execution, with two parts: + +1. **Connector discovery + validation** — every connector relevant to test RCA + (github, k8s, logs, metrics, …) is enumerated and probe-validated (`gh auth + status`, `kubectl` reachability, MCP tools listed). The result is a validated + capability manifest: `connector → valid | invalid | absent`. A gap is + recorded and declared to the TFA agent ("I don't have logs/metrics access") — + never a blocker. +2. **Requirements** — intake fields (product repo, automation repo, branches, + PRs in play, build id) are resolved **by assumption** wherever possible + (invocation args, `gh repo view`, current branch). At most **one** + consolidated question may be asked at gate close, and only for genuinely + non-assumable, load-bearing fields. Headless (`claude -p`) never asks: a + missing build id fails fast; everything else is a recorded gap. -## Modes +**After the gate closes, the run never asks you anything again** — RCA +execution is fully autonomous. Evidence gaps degrade to "unavailable" back to +the TFA agent, which finalizes best-effort. -- **auto** — a dynamic workflow drives the whole batch (5 tests concurrent), no - mid-run prompts. When evidence can't be gathered (no matching skill), it - reports "unavailable" back to the TFA agent, which finalizes best-effort. -- **interactive** — the main session spawns subagents (5 at a time); on an - evidence gap a subagent returns the gap to the main agent, which asks you, - then feeds the answer back. +## Output + +When every test is terminal, the run prints a terse **glimpse table** +(`testRunId → cluster → status → confidence one-liner`), calls +`triggerRcaReport(buildUuid)`, and prints: + +``` +Full report on the Test Observability UI: <viewReport link> +``` -`auto` means autonomy *during* the batch from an interactive session — not -headless. Running `claude -p` with a required input missing ends immediately. +That dashboard report — populated per-test by the BrowserStack agent, with +mandatory culprit-PR links on application bugs — is the real deliverable. ## Requirements @@ -83,11 +107,13 @@ A seeded failing build exercises the full loop against real staging infra: 3. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported and `kubectl` pointed at the staging cluster: ``` - /rca-build awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 mode=interactive + /factory awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 ``` - The harness clusters the failures, drives `tfaRcaTurn` per cluster, and routes - `k8s` asks to the `k8s-rengg-tfa` skill while `product_code`/`deploy` asks go to - GitHub — landing per-test RCAs that trace back to the seeded regressions. + The gate validates connectors (github via `gh`, k8s via the `k8s-rengg-tfa` + skill), then the harness clusters the failures, drives `tfaRcaTurn` per + cluster, routes `k8s` asks to the skill while `product_code`/`deploy` asks go + to GitHub — landing per-test RCAs on the dashboard that trace back to the + seeded regressions, then prints the glimpse + the Test Observability link. ## Layout diff --git a/automation/README.md b/automation/README.md index fa84eb2..d58f5bd 100644 --- a/automation/README.md +++ b/automation/README.md @@ -31,7 +31,7 @@ PROJECT_NAME="RCA Feature Fencing" BUILD_NAME="VRT Build" \ The upload response carries the build id. Feed it to the plugin: ``` -/rca-build <build-id> mode=interactive +/factory <build-id> ``` ## The failures (what RCA should rediscover) diff --git a/lib/routing.mjs b/lib/routing.mjs index ec7c4e7..019a012 100644 --- a/lib/routing.mjs +++ b/lib/routing.mjs @@ -1,6 +1,7 @@ // Evidence-routing registry (D3). Maps a TFA `ask.evidenceType` onto an -// action, given the run's capability manifest. Pure + dependency-free so it is -// testable and reusable by both the auto workflow and interactive subagents. +// action, given the run's validated capability manifest. Pure + dependency-free +// so it is testable and reusable by the batch workflow, subagents, and the +// sequential harness alike. // // `test_logs` is the TFA agent's own evidence and is always skipped. Every // other type routes to a capability; whether that capability is *available* is @@ -28,8 +29,8 @@ export function orderAsks(asks = []) { // Classify one ask. Returns one of: // { action: "skip", ... } — test_logs / TFA-owned; the coordinator emits nothing // { action: "gather", ... } — a capability is available; gather + digest -// { action: "gap", ... } — no capability; the caller's resolveGap() decides -// (auto → "unavailable" block; interactive → ask the user) +// { action: "gap", ... } — no valid connector; the caller emits an +// "unavailable" block back to TFA (never a prompt) // // `manifest` shape: { [capability]: { available: boolean, via?: string } }. export function routeAsk(ask, config, manifest = {}) { @@ -62,8 +63,8 @@ export function routeAsk(ask, config, manifest = {}) { } // Split a turn's asks into the three buckets, in priority order. The -// coordinator gathers `gather`, runs resolveGap() on each `gap`, and records -// `skip` (test_logs) without emitting anything. +// coordinator gathers `gather`, emits an "unavailable" block for each `gap`, +// and records `skip` (test_logs) without emitting anything. export function routeAsks(asks, config, manifest = {}) { const ordered = orderAsks(asks); const buckets = { skip: [], gather: [], gap: [] }; From d2faebd1a13243224219f614d40ec2075f598a58 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Mon, 13 Jul 2026 00:38:18 +0530 Subject: [PATCH 23/47] =?UTF-8?q?fix(rca):=20keep=20the=20skill=20named=20?= =?UTF-8?q?rca-build=20=E2=80=94=20/factory=20is=20the=20user's=20spec-gen?= =?UTF-8?q?erator=20skill?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert only the rename from the single-gate rework (name collision with the user-level /factory skill); all auto-only, single-gate, glimpse, and PRODUCT_BUG-mandate behavior stays. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- INTEGRATION.md | 12 ++++++------ README.md | 10 +++++----- agents/ai-tfa-coordinator.md | 4 ++-- automation/README.md | 2 +- config/rca.config.json | 2 +- lib/glimpse.mjs | 2 +- lib/loop.mjs | 2 +- skills/{factory => rca-build}/SKILL.md | 4 ++-- .../{factory => rca-build}/references/clustering.md | 0 .../references/evidence-routing.md | 0 .../references/github-evidence.md | 0 workflows/rca-batch.mjs | 2 +- 12 files changed, 20 insertions(+), 20 deletions(-) rename skills/{factory => rca-build}/SKILL.md (99%) rename skills/{factory => rca-build}/references/clustering.md (100%) rename skills/{factory => rca-build}/references/evidence-routing.md (100%) rename skills/{factory => rca-build}/references/github-evidence.md (100%) diff --git a/INTEGRATION.md b/INTEGRATION.md index e72cbf6..65de0b6 100644 --- a/INTEGRATION.md +++ b/INTEGRATION.md @@ -4,14 +4,14 @@ This plugin is built so the **MCP core is truly cross-client** and the **harness layer ports via the cross-vendor Agent Skills standard**. Only one piece is genuinely Claude-Code-specific (the batch *dynamic workflow*); on Cursor and Codex that role is filled by the sequential harness or subagents. Every path is -autonomous after the single `/factory` gate — no host ever prompts mid-run. +autonomous after the single `/rca-build` gate — no host ever prompts mid-run. ## What transfers, what doesn't | Layer | Claude Code | Cursor | Codex | |---|---|---|---| | `bstack` MCP server (`listTestIds` + `tfaRcaTurn` + `triggerRcaReport`) | `.mcp.json` (auto-discovered) | `.cursor-mcp.json` / `.cursor/mcp.json` | `~/.codex/config.toml` `[mcp_servers.bstack]` | -| `factory` skill (`SKILL.md`) | plugin `skills/` | Agent Skills (`.cursor/skills/` or cursor-plugin `"skills":"./skills/"`) | Agent Skills (`.agents/skills/`) | +| `rca-build` skill (`SKILL.md`) | plugin `skills/` | Agent Skills (`.cursor/skills/` or cursor-plugin `"skills":"./skills/"`) | Agent Skills (`.agents/skills/`) | | `ai-tfa-coordinator` agent | plugin `agents/` | `.cursor/agents/` (also reads `.claude/agents/`) | `.codex/agents/` | | Per-test RCA **loop** | `agents/ai-tfa-coordinator.md` | same skill/agent | same skill/agent | | Batch orchestration | dynamic workflow `workflows/rca-batch.mjs` (or subagents) | subagents, or **sequential** `lib/loop.mjs` | subagents, or **sequential** `lib/loop.mjs` | @@ -30,11 +30,11 @@ orchestration. On every host the run finishes the same way: glimpse table → ```bash cp .env.example .env # BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY claude --plugin-dir ./ -/factory <build-id> +/rca-build <build-id> ``` `.claude-plugin/plugin.json` + root `.mcp.json` + `skills/` + `agents/` are -auto-discovered. (No `commands/factory.md` on purpose — a command and skill +auto-discovered. (No `commands/rca-build.md` on purpose — a command and skill with the same name collide and the skill body fails to load.) ## Cursor @@ -63,7 +63,7 @@ ln -s ../skills .cursor/skills ln -s ../agents .cursor/agents ``` -Then drive it from Agent chat: invoke the `factory` skill with a build id. +Then drive it from Agent chat: invoke the `rca-build` skill with a build id. ## Codex @@ -88,7 +88,7 @@ ln -s ../skills .agents/skills ln -s ../agents .codex/agents ``` -Then run the `factory` skill; the coordinator + `tfaRcaTurn` loop are identical. +Then run the `rca-build` skill; the coordinator + `tfaRcaTurn` loop are identical. ## Notes diff --git a/README.md b/README.md index 88fb748..9a3e170 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,8 @@ claude --plugin-dir ./ ``` The plugin auto-configures on load: the `bstack` MCP server (from `.mcp.json`), -the `factory` skill, and the `ai-tfa-coordinator` agent are all discovered by -convention. (There is deliberately **no** command file named `factory` — a +the `rca-build` skill, and the `ai-tfa-coordinator` agent are all discovered by +convention. (There is deliberately **no** command file named `rca-build` — a command and skill sharing a name collide and the skill body fails to load.) ### Cursor & Codex @@ -44,8 +44,8 @@ discovery, deeplink) is in **[INTEGRATION.md](INTEGRATION.md)**. ## Usage ``` -/factory <build-id> -/factory build_id=<id> https://github.com/org/repo/pull/123 +/rca-build <build-id> +/rca-build build_id=<id> https://github.com/org/repo/pull/123 ``` Args: a build id (bare, `build_id=`, or a dashboard link) plus optional PR URLs @@ -107,7 +107,7 @@ A seeded failing build exercises the full loop against real staging infra: 3. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported and `kubectl` pointed at the staging cluster: ``` - /factory awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 + /rca-build awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 ``` The gate validates connectors (github via `gh`, k8s via the `k8s-rengg-tfa` skill), then the harness clusters the failures, drives `tfaRcaTurn` per diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 167b155..49ec5d4 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -20,7 +20,7 @@ same thread until TFA converges. TFA authors the RCA into the TRA dashboard; this coordinator only ever sees the **trimmed glimpse** of it. The full report lives on the Test Observability UI. -This coordinator is **fully autonomous**: the `/factory` gate closed before it +This coordinator is **fully autonomous**: the `/rca-build` gate closed before it was dispatched, so it **never prompts a user** — an evidence gap degrades to an `unavailable` block back to TFA, always. @@ -38,7 +38,7 @@ it names no `kubectl` / `chitragupta` / `bifrost`; it routes by *capability*. states the hypothesis and asks TFA to **confirm it against this test's own logs**. - `resume` — optional `{ threadId, turnId }` from a prior PENDING run. - `manifest` — the validated capability manifest `{ capability: { available, via } }` - (built once at the `/factory` gate — Part A). + (built once at the `/rca-build` gate — Part A). If `testRunId` is missing or not parseable as an integer, emit a `failed` `RCA_OUTPUT` block with `root_cause: "no testRunId provided"` and stop — do not diff --git a/automation/README.md b/automation/README.md index d58f5bd..a18e4d4 100644 --- a/automation/README.md +++ b/automation/README.md @@ -31,7 +31,7 @@ PROJECT_NAME="RCA Feature Fencing" BUILD_NAME="VRT Build" \ The upload response carries the build id. Feed it to the plugin: ``` -/factory <build-id> +/rca-build <build-id> ``` ## The failures (what RCA should rediscover) diff --git a/config/rca.config.json b/config/rca.config.json index 021a72c..71e68a3 100644 --- a/config/rca.config.json +++ b/config/rca.config.json @@ -1,5 +1,5 @@ { - "$comment": "Central config for the /factory RCA harness. All formerly-hardcoded product/infra values live here. No kubectl/chitragupta/bifrost literals — connectors are discovered and probe-validated at the gate into the capability manifest (see skills/factory/references/evidence-routing.md). No reportFile: the plugin never writes a local RCA report — the full report lives on the Test Observability UI (triggerRcaReport).", + "$comment": "Central config for the /rca-build RCA harness. All formerly-hardcoded product/infra values live here. No kubectl/chitragupta/bifrost literals — connectors are discovered and probe-validated at the gate into the capability manifest (see skills/rca-build/references/evidence-routing.md). No reportFile: the plugin never writes a local RCA report — the full report lives on the Test Observability UI (triggerRcaReport).", "mcpServerName": "bstack", "concurrency": 5, "turnCap": 6, diff --git a/lib/glimpse.mjs b/lib/glimpse.mjs index 63e328d..c17f8f3 100644 --- a/lib/glimpse.mjs +++ b/lib/glimpse.mjs @@ -1,4 +1,4 @@ -// Terse end-of-run glimpse — the ONLY in-client output of a /factory run. +// Terse end-of-run glimpse — the ONLY in-client output of a /rca-build run. // One line per test: testRunId → cluster → status → confidence one-liner. // The full RCA report lives on the Test Observability UI (triggerRcaReport → // viewReport link); this is deliberately a glimpse, never a report. Degrade, diff --git a/lib/loop.mjs b/lib/loop.mjs index b03ce61..53e83da 100644 --- a/lib/loop.mjs +++ b/lib/loop.mjs @@ -9,7 +9,7 @@ // Pure + dependency-light (imports only the routing registry). // // The loop is fully autonomous: an evidence gap ALWAYS degrades to an -// `unavailable` block back to TFA. There is no user-prompt path — the /factory +// `unavailable` block back to TFA. There is no user-prompt path — the /rca-build // gate closed before this loop ever runs. // // tfaRcaTurn returns TRIMMED terminal shapes: diff --git a/skills/factory/SKILL.md b/skills/rca-build/SKILL.md similarity index 99% rename from skills/factory/SKILL.md rename to skills/rca-build/SKILL.md index f2dc52e..4a4c715 100644 --- a/skills/factory/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -1,9 +1,9 @@ --- -name: factory +name: rca-build description: Single-gate autonomous batch RCA over every failed test of a BrowserStack build via tfaRcaTurn. One gate (connector validation + assumed intake), then fully autonomous — clusters failures, routes evidence, triggers the dashboard report. Args: build id, optional PR URLs / repo hints. --- -# factory — single-gate autonomous RCA over a build +# rca-build — single-gate autonomous RCA over a build Drives the `tfaRcaTurn` collaborative loop over **every failed test** of a build and lands a per-test RCA in the TRA (Test Observability) dashboard. **TFA owns diff --git a/skills/factory/references/clustering.md b/skills/rca-build/references/clustering.md similarity index 100% rename from skills/factory/references/clustering.md rename to skills/rca-build/references/clustering.md diff --git a/skills/factory/references/evidence-routing.md b/skills/rca-build/references/evidence-routing.md similarity index 100% rename from skills/factory/references/evidence-routing.md rename to skills/rca-build/references/evidence-routing.md diff --git a/skills/factory/references/github-evidence.md b/skills/rca-build/references/github-evidence.md similarity index 100% rename from skills/factory/references/github-evidence.md rename to skills/rca-build/references/github-evidence.md diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index 1fe8699..553b94a 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -8,7 +8,7 @@ export const meta = { ], }; -// The /factory batch orchestration (fully autonomous — the gate closed before +// The /rca-build batch orchestration (fully autonomous — the gate closed before // this runs; nothing here ever asks the user). This is a dynamic-workflow // script: it runs in the Workflow sandbox (no filesystem, no Date.now/ // Math.random, agent()/pipeline() as globals). It therefore does NO state I/O From e4428c7bfbf04fd9d910d5ab8b54674509f6968c Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 13:18:59 +0530 Subject: [PATCH 24/47] =?UTF-8?q?feat(rca):=20per-build=20state=20file=20i?= =?UTF-8?q?n=20OS=20temp=20=E2=80=94=20csvPathFor(buildId,=20stateDir)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes to the state spine's location: 1. The build id is now IN the filename (rca-state.<buildId>.csv) — previously a fixed .rca/rca-state.csv meant a run on build B would collide with and 'resume' into build A's state. 2. Default directory is OS temp (<tmpdir>/bstack-rca/), not the invoking workspace — a background harness must not pollute the repo it runs from. paths.stateDir remains an override for retained artifacts (e.g. CI). Resume-safety is preserved per build (same id -> same path). Hostile ids are sanitized. 4 new tests; 53/53 green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- config/rca.config.json | 4 ++-- lib/csv-state.mjs | 19 ++++++++++++++++++- skills/rca-build/SKILL.md | 10 ++++++++-- tests/csv-state.test.mjs | 21 +++++++++++++++++++++ 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/config/rca.config.json b/config/rca.config.json index 71e68a3..1928ea1 100644 --- a/config/rca.config.json +++ b/config/rca.config.json @@ -8,8 +8,8 @@ "reaperHeartbeatTtlSec": 600, "errorSummaryMaxChars": 200, "paths": { - "stateDir": ".rca", - "csvFile": ".rca/rca-state.csv" + "$comment": "State CSV path is ALWAYS derived per build via lib/csv-state.mjs csvPathFor(buildId, stateDir): the build id is in the filename (no cross-build collisions) and the default directory is OS temp (<tmpdir>/bstack-rca/), so the harness never pollutes the invoking workspace. Set stateDir only to retain the CSV somewhere specific (e.g. a CI artifact dir).", + "stateDir": "" }, "evidenceRouting": { "test_logs": { "owner": "tfa", "skip": true }, diff --git a/lib/csv-state.mjs b/lib/csv-state.mjs index ea830ff..152e51b 100644 --- a/lib/csv-state.mjs +++ b/lib/csv-state.mjs @@ -14,7 +14,24 @@ // locking is out of scope). import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs"; -import { dirname } from "node:path"; +import { dirname, join } from "node:path"; +import { tmpdir } from "node:os"; + +/** + * Canonical state-file path for one build's run. Two invariants (D-temp): + * 1. The BUILD ID IS IN THE FILENAME — runs over different builds can never + * collide/"resume" into each other's state. + * 2. Default location is OS TEMP (`<tmpdir>/bstack-rca/`), not the invoking + * workspace — a background harness must not pollute the repo it runs from. + * `stateDir` (config `paths.stateDir`) overrides the directory only — e.g. a CI + * job that wants the CSV as a retained artifact. Resume-safety is per build: + * same buildId → same path. + */ +export function csvPathFor(buildId, stateDir = "") { + const safe = String(buildId ?? "").replace(/[^A-Za-z0-9._-]/g, "_") || "unknown-build"; + const dir = stateDir && String(stateDir).trim() !== "" ? String(stateDir) : join(tmpdir(), "bstack-rca"); + return join(dir, `rca-state.${safe}.csv`); +} export const COLUMNS = [ "buildId", diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 4a4c715..c6e0880 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -109,11 +109,17 @@ listTestIds(buildId=<id>, status="failed", includeFailureDetail=true) (`failure.{category, error_summary, file_path, …}`) — the seed for clustering, so no per-test probe turns are needed. +Resolve the state file with `lib/csv-state.mjs` → `csvPathFor(buildId, +config.paths.stateDir)` — the **build id is in the filename** and the default +directory is **OS temp** (`<tmpdir>/bstack-rca/rca-state.<buildId>.csv`), so +different builds can never collide and the invoking workspace stays clean. Pass +this exact path to the fan-out workflow as `csvPath`. + Seed the CSV/WAL spine from the payload (`lib/csv-state.mjs` → `seed`): one row per failed test, every row `rca_done=pending`, signature columns populated. Re-running `seed` on an existing CSV is idempotent and preserves terminal rows -(resume-safe). If `listTestIds` returns empty → write an empty CSV, report "no -failed tests", stop. +(resume-safe — same build id → same path). If `listTestIds` returns empty → +write an empty CSV, report "no failed tests", stop. ## Step 3 — failure-signature clustering (see references/clustering.md) diff --git a/tests/csv-state.test.mjs b/tests/csv-state.test.mjs index ca55d84..3cbc83a 100644 --- a/tests/csv-state.test.mjs +++ b/tests/csv-state.test.mjs @@ -4,6 +4,7 @@ import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { + csvPathFor, seed, readRows, claim, @@ -164,3 +165,23 @@ test("CSV codec round-trips fields with commas, quotes, newlines", () => { const row = readRows(csv).find((r) => r.testRunId === "200"); assert.equal(row.root_cause, 'Failed: "x", got <y>\nsecond line'); }); + +test("csvPathFor: build id is in the filename, default dir is OS temp", () => { + const p = csvPathFor("abc123XYZ"); + assert.ok(p.startsWith(join(tmpdir(), "bstack-rca"))); + assert.ok(p.endsWith("rca-state.abc123XYZ.csv")); +}); + +test("csvPathFor: different builds never share a path", () => { + assert.notEqual(csvPathFor("build-A"), csvPathFor("build-B")); +}); + +test("csvPathFor: sanitizes hostile ids and handles empty", () => { + assert.ok(csvPathFor("../../etc/passwd").endsWith("rca-state..._.._etc_passwd.csv")); + assert.ok(csvPathFor("").endsWith("rca-state.unknown-build.csv")); +}); + +test("csvPathFor: stateDir override wins over temp", () => { + const p = csvPathFor("b1", "/ci/artifacts"); + assert.equal(p, join("/ci/artifacts", "rca-state.b1.csv")); +}); From 5f345632499d1fb563b006d31e03428a42bb70b8 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 13:31:30 +0530 Subject: [PATCH 25/47] =?UTF-8?q?refactor(rca):=20align=20skills=20with=20?= =?UTF-8?q?canonical=20layout=20=E2=80=94=20templates/,=20examples/,=20scr?= =?UTF-8?q?ipts/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - k8s-rengg-tfa is self-contained: bin/k8s-context.sh -> skills/k8s-rengg-tfa/ scripts/ (carries the pending redaction hardening for auth headers) - rca-build gains templates/ (suspect-packet, evidence-block, gate-summary) as the canonical fillable formats — references now point at them instead of embedding drift-prone copies — and examples/sample-run.md (fictional, matches the recorded-turn fixtures): gate summary, answered NEEDS_INFO turn with supported + ruled-out suspects, terminal glimpse output Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- README.md | 2 +- skills/rca-build/SKILL.md | 4 +- skills/rca-build/examples/sample-run.md | 77 +++++++++++++++++++ .../rca-build/references/evidence-routing.md | 13 +--- .../rca-build/references/github-evidence.md | 18 ++--- skills/rca-build/templates/evidence-block.md | 26 +++++++ skills/rca-build/templates/gate-summary.md | 21 +++++ skills/rca-build/templates/suspect-packet.md | 22 ++++++ 8 files changed, 160 insertions(+), 23 deletions(-) create mode 100644 skills/rca-build/examples/sample-run.md create mode 100644 skills/rca-build/templates/evidence-block.md create mode 100644 skills/rca-build/templates/gate-summary.md create mode 100644 skills/rca-build/templates/suspect-packet.md diff --git a/README.md b/README.md index 9a3e170..0e51c94 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ A seeded failing build exercises the full loop against real staging infra: `is_mcp_driven` feature and `upload.sh` to push them. The current seeded build: `awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2` (project "RCA Feature Fencing", build "VRT Build"). See `automation/README.md`. -2. **k8s evidence harness** — `skills/k8s-rengg-tfa/` + `bin/k8s-context.sh` +2. **k8s evidence harness** — `skills/k8s-rengg-tfa/` (self-contained: SKILL.md + `scripts/k8s-context.sh`) provide the `k8s` capability: read-only obs-api context (pod health, deployed image, error logs, events) from the `rengg-tfa` namespace, secrets redacted. 3. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported and `kubectl` diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index c6e0880..b1839f2 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -80,7 +80,9 @@ is the point: - the current branch for the working branch, - cheap inference (e.g. the automation repo is the cwd if it holds the tests). -Record each assumption in the gate summary ("assumed product repo = +Record each assumption in the gate summary (format: +`templates/gate-summary.md`; worked example: `examples/sample-run.md`) +("assumed product repo = `org/obs-api` from git remote"). A field that cannot be assumed is recorded as "none" and the run proceeds RCA-only for it — **unless** it is both genuinely non-assumable AND load-bearing (in practice: only the build id, and rarely an diff --git a/skills/rca-build/examples/sample-run.md b/skills/rca-build/examples/sample-run.md new file mode 100644 index 0000000..dd5398f --- /dev/null +++ b/skills/rca-build/examples/sample-run.md @@ -0,0 +1,77 @@ +# Example — one full run (fictional data, matches the recorded-turn fixtures) + +Invocation: `/rca-build awswx…fw2` (build id given; nothing else passed). + +## 1. Gate closes (the only user-visible checkpoint) + +``` +GATE CLOSED — capability manifest: + github ✅ valid (gh, authed) · k8s ✅ valid (ctx staging-euc1) · logs ❌ absent · metrics ❌ absent + +Intake: + build id: awswx…fw2 (given) + product repo: acme/obs-api (assumed — from git remote) + automation repo: acme/obs-e2e (assumed — cwd holds the tests) + working branch: main (assumed — current branch) + default branch: main (assumed — origin HEAD) + PRs in play: none (gap) + +Gaps declared to TFA: logs, metrics +Proceeding autonomously: discovery → clustering → fan-out (concurrency 5, turn-cap 6). +``` + +## 2. A NEEDS_INFO turn answered (what the coordinator sends back) + +TFA asked: *"Did request-validation on POST /builds change since last green?"* +(`evidenceType: product_code`, priority high). The coordinator replies on the +same `threadId`: + +``` +ASK: Did request-validation on POST /builds change since last green? +TYPE: product_code +FOUND: yes +SUMMARY: Yes — the buildName validator was tightened to reject empty strings in the +suspect window. One PR touches the failing path; falsification below. +LINK: https://github.com/acme/obs-api/pull/7421 + +SUSPECT: + pr: #7421 + files: src/validators/build.ts + hunks: `- allowEmpty: true` → `+ allowEmpty: false` (validator schema) + author: jdoe + merged_at: 2026-07-01T09:14Z vs last_green: 2026-07-01T02:10Z vs started_at: 2026-07-01T21:40Z + verdict: supported + link: https://github.com/acme/obs-api/pull/7421 + +SUSPECT: + pr: #7418 + files: src/routes/builds.ts + hunks: logging middleware reorder only + author: asmith + merged_at: 2026-06-30T18:02Z vs last_green: 2026-07-01T02:10Z vs started_at: 2026-07-01T21:40Z + verdict: ruled-out (shipped-after check passed but no-path-overlap — hunks never touch the validator) + link: https://github.com/acme/obs-api/pull/7418 + +ASK: Full run logs for test 39 +TYPE: test_logs +FOUND: no +SUMMARY: out-of-scope — TFA owns test logs; skipped by contract. +``` + +## 3. Terminal output (glimpse only — NO local report) + +``` +RCA batch complete — build awswx…fw2 (7 failed → 3 clusters) + +39 → c1 → RESOLVED (high) PR #7421 tightened buildName validator; related_prs: #7421 +41 → c1 → RESOLVED (high) sibling of 39 (cluster confirm) +57 → c2 → RESOLVED (med) flaky selector wait; test-side +81 → c3 → PENDING (—) soft-pending, resumable (turnId recorded) +… + +Full report on the Test Observability UI: +https://observability.browserstack.com/builds/awswx…fw2 +``` + +State file: `<tmpdir>/bstack-rca/rca-state.awswx…fw2.csv` (resume-safe; re-run +the same build id to pick up the PENDING row). diff --git a/skills/rca-build/references/evidence-routing.md b/skills/rca-build/references/evidence-routing.md index afe71aa..d162096 100644 --- a/skills/rca-build/references/evidence-routing.md +++ b/skills/rca-build/references/evidence-routing.md @@ -75,15 +75,10 @@ tail or full PR diff blows both budgets and degrades TFA's reasoning. Supply the ### Per-ask block shape — `ask → found → snippet/link` -``` -ASK: <verbatim `what` from the TfaAsk, ≤ 120 chars> -TYPE: <evidenceType> -FOUND: <yes | no | partial> -SUMMARY: <1–3 sentences — the finding, in the agent's words. ≤ 400 chars> -SNIPPET: - <the load-bearing excerpt only — see size caps. Omit if a LINK fully carries it.> -LINK: <permalink to the source — PR/commit/log-search/metrics panel/deploy record. Omit if N/A.> -``` +**The canonical fillable format lives in +[`../templates/evidence-block.md`](../templates/evidence-block.md)** (fulfilled +and unfulfillable variants) — copy it, don't retype it. Shape: +`ASK / TYPE / FOUND: yes|no|partial / SUMMARY ≤400 / SNIPPET (caps below) / LINK`. - `SUMMARY` is the answer. `SNIPPET` is the *minimum* evidence backing it. `LINK` lets TFA (or a human) verify without the bytes living in the message. diff --git a/skills/rca-build/references/github-evidence.md b/skills/rca-build/references/github-evidence.md index ae1ddcc..4b2c445 100644 --- a/skills/rca-build/references/github-evidence.md +++ b/skills/rca-build/references/github-evidence.md @@ -75,18 +75,12 @@ survives 1–3 is a real candidate; one that fails any is reported as ruled-out ## The suspect packet (structured, not free text) Each surviving/ruled-out suspect is one structured block so `related_prs` -populates deterministically: - -``` -SUSPECT: - pr: <#number> - files: <changed files overlapping the failing path> - hunks: <the 1-3 load-bearing changed hunks — see digest size caps> - author: <login> - merged_at: <ts> vs last_green: <ts> vs started_at: <ts> - verdict: supported | ruled-out (<reason: no-path-overlap | shipped-after | behind-off-flag | unrelated>) - link: <PR permalink> -``` +populates deterministically. **The canonical fillable format lives in +[`../templates/suspect-packet.md`](../templates/suspect-packet.md)** (fields: +pr, files, hunks, author, merged_at vs last_green vs started_at, verdict with +rule-out reason, link) — copy it, don't retype it. A worked example (supported ++ ruled-out side by side) is in +[`../examples/sample-run.md`](../examples/sample-run.md). Only `verdict: supported` suspects should end up in TFA's `related_prs`. Ruled-out suspects stay in the thread as disconfirming evidence so TFA (and a human) can see diff --git a/skills/rca-build/templates/evidence-block.md b/skills/rca-build/templates/evidence-block.md new file mode 100644 index 0000000..fa0bf49 --- /dev/null +++ b/skills/rca-build/templates/evidence-block.md @@ -0,0 +1,26 @@ +# Template — evidence block (one per fulfilled/unfulfilled ask) + +The unit of evidence sent back to TFA in a turn message. Rules, size caps and +the forbidden list: `../references/evidence-routing.md`. + +Fulfilled ask: + +``` +ASK: <verbatim `what` from the TfaAsk, ≤ 120 chars> +TYPE: <evidenceType> +FOUND: <yes | no | partial> +SUMMARY: <1–3 sentences — the finding, in the agent's words. ≤ 400 chars> +SNIPPET: + <the load-bearing excerpt only — see size caps. Omit if a LINK fully carries it.> +LINK: <permalink to the source — PR/commit/log-search/metrics panel/deploy record. Omit if N/A.> +``` + +Unfulfillable ask (report, don't drop — machine-generated for absent connectors +by `lib/loop.mjs` `unavailableBlock`): + +``` +ASK: <verbatim what> +TYPE: <evidenceType> +FOUND: no +SUMMARY: not-found | unreachable | unavailable | out-of-scope — <one line: what was checked or why blocked> +``` diff --git a/skills/rca-build/templates/gate-summary.md b/skills/rca-build/templates/gate-summary.md new file mode 100644 index 0000000..7602968 --- /dev/null +++ b/skills/rca-build/templates/gate-summary.md @@ -0,0 +1,21 @@ +# Template — gate summary (printed once, when THE GATE closes) + +One terse FYI before autonomous execution starts. Every intake field is tagged +`given | assumed | gap`; every connector `valid | invalid | absent`. After this +prints, the run never asks the user anything. + +``` +GATE CLOSED — capability manifest: + github ✅ valid (gh, authed) · k8s ✅ valid (ctx <context>) · logs ❌ absent · metrics ❌ absent + +Intake: + build id: <id> (given) + product repo: <org/repo> (assumed — from git remote) + automation repo: <org/repo> (assumed — cwd holds the tests) + working branch: <branch> (assumed — current branch) + default branch: <branch> (assumed — origin HEAD) + PRs in play: <#123, #456 | none> (given | gap) + +Gaps declared to TFA: <logs, metrics | none> +Proceeding autonomously: discovery → clustering → fan-out (concurrency <N>, turn-cap <M>). +``` diff --git a/skills/rca-build/templates/suspect-packet.md b/skills/rca-build/templates/suspect-packet.md new file mode 100644 index 0000000..676fcf4 --- /dev/null +++ b/skills/rca-build/templates/suspect-packet.md @@ -0,0 +1,22 @@ +# Template — SUSPECT packet (one block per candidate PR) + +Fill one block per suspect, supported **and** ruled-out (elimination is evidence +too). Only `verdict: supported` suspects may feed `related_prs`. Guidance + +falsification protocol: `../references/github-evidence.md`. + +``` +SUSPECT: + pr: <#number> + files: <changed files overlapping the failing path> + hunks: <the 1-3 load-bearing changed hunks — see digest size caps> + author: <login> + merged_at: <ts> vs last_green: <ts> vs started_at: <ts> + verdict: supported | ruled-out (<no-path-overlap | shipped-after | behind-off-flag | unrelated>) + link: <PR permalink> +``` + +If the hunt ends empty after a real search (never fabricate): + +``` +no culprit PR identified after <what was searched: window, repos, paths> +``` From 6796327ff040710edf25b495de7928faaf104312 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 13:36:09 +0530 Subject: [PATCH 26/47] =?UTF-8?q?fix(rca):=20confirmed=20TRA=20UI=20deep-l?= =?UTF-8?q?ink=20=E2=80=94=20automation.browserstack.com=20+=20ai=5Freport?= =?UTF-8?q?/aitfa=20tabs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The viewReport/viewRca destination is the build's AI-TFA report: <base>/builds/<buildUuid>?tab=ai_report&subTab=aitfa on automation.browserstack.com (designer-confirmed; was an observability.browserstack.com assumption). Fixture now carries the guidance string the bridge actually emits on RESOLVED turns — closes the fixture-drift review finding. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- skills/rca-build/examples/sample-run.md | 2 +- tests/conformance.test.mjs | 2 +- tests/fixtures/recorded-turns/resolved.json | 12 ++++++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/skills/rca-build/examples/sample-run.md b/skills/rca-build/examples/sample-run.md index dd5398f..e7b02ea 100644 --- a/skills/rca-build/examples/sample-run.md +++ b/skills/rca-build/examples/sample-run.md @@ -70,7 +70,7 @@ RCA batch complete — build awswx…fw2 (7 failed → 3 clusters) … Full report on the Test Observability UI: -https://observability.browserstack.com/builds/awswx…fw2 +https://automation.browserstack.com/builds/awswx…fw2?tab=ai_report&subTab=aitfa ``` State file: `<tmpdir>/bstack-rca/rca-state.awswx…fw2.csv` (resume-safe; re-run diff --git a/tests/conformance.test.mjs b/tests/conformance.test.mjs index 82f7b8c..3dde60b 100644 --- a/tests/conformance.test.mjs +++ b/tests/conformance.test.mjs @@ -37,7 +37,7 @@ test("resolved fixture: NEEDS_INFO → evidence → RESOLVED, trimmed glimpse ca assert.ok(result.root_cause.length <= 220); // glimpse root_cause is trimmed server-side assert.equal(result.failure_type, "product_regression"); assert.deepEqual(result.related_prs, ["#7421"]); - assert.match(result.view_rca, /^https:\/\/observability\.browserstack\.com\/builds\//); + assert.match(result.view_rca, /^https:\/\/automation\.browserstack\.com/); assert.deepEqual(result.asks_fulfilled, ["product_code"]); assert.deepEqual(result.asks_skipped, ["test_logs"]); // TFA-owned, never gathered assert.equal(result.turns_used, 2); diff --git a/tests/fixtures/recorded-turns/resolved.json b/tests/fixtures/recorded-turns/resolved.json index 47b9034..a7964b8 100644 --- a/tests/fixtures/recorded-turns/resolved.json +++ b/tests/fixtures/recorded-turns/resolved.json @@ -6,7 +6,9 @@ "status": "NEEDS_INFO", "confidence": "low", "threadId": "thr-39", - "questions": ["Did the buildName validator change?"], + "questions": [ + "Did the buildName validator change?" + ], "asks": [ { "what": "Did request-validation on POST /builds change since last green?", @@ -29,9 +31,11 @@ "glimpse": { "root_cause": "PR #7421 tightened the buildName validator to reject empty strings", "failure_type": "product_regression", - "related_prs": ["#7421"] + "related_prs": [ + "#7421" + ] }, - "viewRca": "https://observability.browserstack.com/builds/awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2" + "viewRca": "https://automation.browserstack.com — open the build's AI report (tab=ai_report, subTab=aitfa) to view the full RCA" } ] -} +} \ No newline at end of file From a8c201410594dc0dd8dc10cffc0df02dca1cce96 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 13:54:14 +0530 Subject: [PATCH 27/47] chore(rca): remove the k8s-rengg-tfa skill Environment-specific harness removed; k8s stays a generic gate-probed capability (discoveryHints now empty, like logs/metrics). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- README.md | 3 +-- config/rca.config.json | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0e51c94..1344978 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,6 @@ A seeded failing build exercises the full loop against real staging infra: `is_mcp_driven` feature and `upload.sh` to push them. The current seeded build: `awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2` (project "RCA Feature Fencing", build "VRT Build"). See `automation/README.md`. -2. **k8s evidence harness** — `skills/k8s-rengg-tfa/` (self-contained: SKILL.md + `scripts/k8s-context.sh`) provide the `k8s` capability: read-only obs-api context (pod health, deployed image, error logs, events) from the `rengg-tfa` namespace, secrets redacted. 3. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported and `kubectl` @@ -109,7 +108,7 @@ A seeded failing build exercises the full loop against real staging infra: ``` /rca-build awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 ``` - The gate validates connectors (github via `gh`, k8s via the `k8s-rengg-tfa` + The gate validates connectors (github via `gh`, k8s via any k8s-capable skill), then the harness clusters the failures, drives `tfaRcaTurn` per cluster, routes `k8s` asks to the skill while `product_code`/`deploy` asks go to GitHub — landing per-test RCAs on the dashboard that trace back to the diff --git a/config/rca.config.json b/config/rca.config.json index 1928ea1..128500a 100644 --- a/config/rca.config.json +++ b/config/rca.config.json @@ -16,7 +16,7 @@ "product_code": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, "deploy": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, "ci": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, - "k8s": { "capability": "k8s", "discoveryHints": ["k8s-rengg-tfa"] }, + "k8s": { "capability": "k8s", "discoveryHints": [] }, "kibana": { "capability": "logs", "discoveryHints": [] }, "metrics": { "capability": "metrics", "discoveryHints": [] }, "other": { "capability": "other", "discoveryHints": [] } From 838014617db473d304f8a72c316592fd900b0052 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 14:12:30 +0530 Subject: [PATCH 28/47] =?UTF-8?q?refactor(rca):=20generalize=20k8s=20?= =?UTF-8?q?=E2=86=92=20infra=20capability=20=E2=80=94=20route=20to=20whate?= =?UTF-8?q?ver=20runtime=20the=20user=20has?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The harness no longer assumes Kubernetes. 'infra' is the capability (runtime state of the service); k8s/EKS, ECS, docker, Nomad, VMs, PM2 are all just possible connectors behind it — the gate probes what actually exists and records the kind in the manifest (via). The k8s evidenceType remains a routing alias into infra so TFA asks spelled 'k8s' keep working. kubectl is one probe on a menu, not the definition of the capability. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- README.md | 12 ++--- agents/ai-tfa-coordinator.md | 2 +- config/rca.config.json | 53 ++++++++++++++++--- lib/coverage.mjs | 2 +- lib/routing.mjs | 2 +- skills/rca-build/SKILL.md | 9 ++-- skills/rca-build/examples/sample-run.md | 2 +- .../rca-build/references/evidence-routing.md | 9 ++-- skills/rca-build/templates/gate-summary.md | 2 +- tests/coverage-glimpse.test.mjs | 2 +- tests/evidence.test.mjs | 7 +-- tests/routing.test.mjs | 7 +-- 12 files changed, 75 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 1344978..571e7da 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ RCA in the TRA (Test Observability) dashboard. > **The full RCA report lives on the Test Observability UI, not in Claude.** > The plugin surfaces a terse glimpse, triggers the dashboard report > (`triggerRcaReport`), and prints the link. It **discovers and delegates** to -> the infra skills/tools already in your client (GitHub, k8s/EKS, kibana/other +> the infra skills/tools already in your client (GitHub, whatever runtime you have — k8s/ECS/docker/… — kibana/other > logs, metrics). It does **not** install or own those connectors, and it never > writes a local report file. @@ -56,8 +56,8 @@ Args: a build id (bare, `build_id=`, or a dashboard link) plus optional PR URLs The run has exactly **one gate** before execution, with two parts: 1. **Connector discovery + validation** — every connector relevant to test RCA - (github, k8s, logs, metrics, …) is enumerated and probe-validated (`gh auth - status`, `kubectl` reachability, MCP tools listed). The result is a validated + (github, infra, logs, metrics, …) is enumerated and probe-validated (`gh auth + status`, an infra probe matching whatever runtime exists — kubectl/docker/ecs/… — MCP tools listed). The result is a validated capability manifest: `connector → valid | invalid | absent`. A gap is recorded and declared to the TFA agent ("I don't have logs/metrics access") — never a blocker. @@ -101,16 +101,16 @@ A seeded failing build exercises the full loop against real staging infra: `is_mcp_driven` feature and `upload.sh` to push them. The current seeded build: `awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2` (project "RCA Feature Fencing", build "VRT Build"). See `automation/README.md`. - provide the `k8s` capability: read-only obs-api context (pod health, deployed + provide the `infra` capability: read-only runtime context (pod/instance health, deployed image, error logs, events) from the `rengg-tfa` namespace, secrets redacted. 3. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported and `kubectl` pointed at the staging cluster: ``` /rca-build awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 ``` - The gate validates connectors (github via `gh`, k8s via any k8s-capable + The gate validates connectors (github via `gh`, infra via whatever runtime connector exists (kubectl/docker/ecs/…) skill), then the harness clusters the failures, drives `tfaRcaTurn` per - cluster, routes `k8s` asks to the skill while `product_code`/`deploy` asks go + cluster, routes `infra`/`k8s` asks to it while `product_code`/`deploy` asks go to GitHub — landing per-test RCAs on the dashboard that trace back to the seeded regressions, then prints the glimpse + the Test Observability link. diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 49ec5d4..5ef4e88 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -1,6 +1,6 @@ --- name: ai-tfa-coordinator -description: 'Per-test collaborative-RCA coordinator (autonomous — never prompts a user). Given ONE testRunId, drives the tfaRcaTurn MCP loop to a terminal root cause: TFA reads the run logs; this coordinator supplies every non-log evidence ask (product code, k8s, kibana, metrics, deploy, ci) using whatever skills/tools the client has, routed through the validated capability manifest. Skips every test_logs ask (TFA owns logs). For application bugs it MUST hunt the culprit PR via the github connector. Emits a structured RCA_OUTPUT block. Generic over product and infra — no hardcoded tools. Examples: +description: 'Per-test collaborative-RCA coordinator (autonomous — never prompts a user). Given ONE testRunId, drives the tfaRcaTurn MCP loop to a terminal root cause: TFA reads the run logs; this coordinator supplies every non-log evidence ask (product code, infra/runtime, logs, metrics, deploy, ci) using whatever skills/tools the client has, routed through the validated capability manifest. Skips every test_logs ask (TFA owns logs). For application bugs it MUST hunt the culprit PR via the github connector. Emits a structured RCA_OUTPUT block. Generic over product and infra — no hardcoded tools. Examples: - orchestrator: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=39 — error: empty buildName rejected on POST /builds") → drives the loop, returns RCA_OUTPUT - sibling confirm: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=40 — pre-seed: cause=<rep root cause>, suspect PR=#7421") → one-turn confirm against this test logs - user: "run collaborative RCA on test run 39" → single-test loop to RESOLVED/PENDING' diff --git a/config/rca.config.json b/config/rca.config.json index 128500a..732c9e3 100644 --- a/config/rca.config.json +++ b/config/rca.config.json @@ -12,13 +12,50 @@ "stateDir": "" }, "evidenceRouting": { - "test_logs": { "owner": "tfa", "skip": true }, - "product_code": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, - "deploy": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, - "ci": { "capability": "github", "discoveryHints": ["github-mcp", "gh"] }, - "k8s": { "capability": "k8s", "discoveryHints": [] }, - "kibana": { "capability": "logs", "discoveryHints": [] }, - "metrics": { "capability": "metrics", "discoveryHints": [] }, - "other": { "capability": "other", "discoveryHints": [] } + "test_logs": { + "owner": "tfa", + "skip": true + }, + "product_code": { + "capability": "github", + "discoveryHints": [ + "github-mcp", + "gh" + ] + }, + "deploy": { + "capability": "github", + "discoveryHints": [ + "github-mcp", + "gh" + ] + }, + "ci": { + "capability": "github", + "discoveryHints": [ + "github-mcp", + "gh" + ] + }, + "infra": { + "capability": "infra", + "discoveryHints": [] + }, + "k8s": { + "capability": "infra", + "discoveryHints": [] + }, + "kibana": { + "capability": "logs", + "discoveryHints": [] + }, + "metrics": { + "capability": "metrics", + "discoveryHints": [] + }, + "other": { + "capability": "other", + "discoveryHints": [] + } } } diff --git a/lib/coverage.mjs b/lib/coverage.mjs index caff2d2..0266ceb 100644 --- a/lib/coverage.mjs +++ b/lib/coverage.mjs @@ -1,6 +1,6 @@ // Evidence-coverage stamp (ideation #6, v1 — the per-row coverage band; the // build-level blast-radius digest is deferred). A RESOLVED RCA built with -// k8s+kibana+metrics all "unavailable" must not read like one with full +// infra+logs+metrics all "unavailable" must not read like one with full // evidence. The client (which routed every ask) stamps each row with a coverage // vector and derives a coverage-capped confidence band the reviewer sees: // "low confidence BECAUSE kibana was unavailable", not "low confidence, trust me". diff --git a/lib/routing.mjs b/lib/routing.mjs index 019a012..ba3d699 100644 --- a/lib/routing.mjs +++ b/lib/routing.mjs @@ -99,7 +99,7 @@ export function buildManifest(config, discovered = []) { } // Capabilities that will be unavailable this run — declared to the user up front -// ("k8s + metrics not available") and to TFA so it plans asks around them. +// ("infra + metrics not available") and to TFA so it plans asks around them. export function unavailableCapabilities(manifest) { return Object.entries(manifest) .filter(([, v]) => !v.available) diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index b1839f2..1b7b898 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -7,7 +7,7 @@ description: Single-gate autonomous batch RCA over every failed test of a Browse Drives the `tfaRcaTurn` collaborative loop over **every failed test** of a build and lands a per-test RCA in the TRA (Test Observability) dashboard. **TFA owns -logs; the client agent owns everything else** (product code, k8s, kibana, +logs; the client agent owns everything else** (product code, infra/runtime, logs, metrics, deploy, ci) — routed by capability, generic over product and infra. This skill is the **build-level orchestrator** (`ai-tfa-orchestrator` role). It @@ -47,8 +47,9 @@ pass. The gate has two parts; both run before any RCA work starts. Enumerate every connector relevant to test RCA: - from `config/rca.config.json` → `evidenceRouting`: **github** - (product_code/deploy/ci), **k8s**, **logs** (e.g. kibana), **metrics**, - **other**; + (product_code/deploy/ci), **infra** (whatever runtime the user has — k8s, + ECS, docker, Nomad, plain VMs, PM2, …), **logs** (kibana or any log store), + **metrics**, **other**; - plus any connector-shaped skills / MCP servers present in the session (a log-search MCP, a metrics MCP, an infra skill, …). @@ -57,7 +58,7 @@ Enumerate every connector relevant to test RCA: | Connector | Probe | |---|---| | github | `gh auth status` (or a GitHub MCP tool listed) | -| k8s | k8s skill present **and** `kubectl` reachable (e.g. `kubectl version --request-timeout=5s`) | +| infra | ANY runtime connector the user has — probe what exists, never assume one: `kubectl version --request-timeout=5s`, `docker ps`, `aws ecs list-clusters`, `nomad status`, `pm2 ls`, or an infra-shaped skill/MCP tool. Record the KIND in the manifest (`via: kubectl \| docker \| ecs \| …`) | | logs | a log-search skill/MCP tool actually listed in the session | | metrics | a metrics skill/MCP tool actually listed in the session | | other | best-effort; default `absent` | diff --git a/skills/rca-build/examples/sample-run.md b/skills/rca-build/examples/sample-run.md index e7b02ea..01cc36e 100644 --- a/skills/rca-build/examples/sample-run.md +++ b/skills/rca-build/examples/sample-run.md @@ -6,7 +6,7 @@ Invocation: `/rca-build awswx…fw2` (build id given; nothing else passed). ``` GATE CLOSED — capability manifest: - github ✅ valid (gh, authed) · k8s ✅ valid (ctx staging-euc1) · logs ❌ absent · metrics ❌ absent + github ✅ valid (gh, authed) · infra ✅ valid (via kubectl, ctx staging-euc1) · logs ❌ absent · metrics ❌ absent Intake: build id: awswx…fw2 (given) diff --git a/skills/rca-build/references/evidence-routing.md b/skills/rca-build/references/evidence-routing.md index d162096..05b4435 100644 --- a/skills/rca-build/references/evidence-routing.md +++ b/skills/rca-build/references/evidence-routing.md @@ -42,7 +42,8 @@ An ask that cannot be fulfilled is **never silently dropped** — it becomes a ## Routing table (capability, not tool) `evidenceType` literals are exactly those `tfaRcaTurn` emits: `test_logs`, -`product_code`, `k8s`, `kibana`, `metrics`, `deploy`, `ci`, `other`. +`product_code`, `infra` (TFA may still spell it `k8s` — both route the same), +`kibana`, `metrics`, `deploy`, `ci`, `other`. | `evidenceType` | Capability | Gathered via (discovered at runtime) | |---|---|---| @@ -50,7 +51,7 @@ An ask that cannot be fulfilled is **never silently dropped** — it becomes a | `product_code` | `github` | the client's GitHub capability — **GitHub MCP if present, else `gh`** (see `references/github-evidence.md`) | | `deploy` | `github` | deploy timeline via the GitHub capability (releases/tags + deploy record) | | `ci` | `github` | CI config + run history via the GitHub capability | -| `k8s` | `k8s` | whatever k8s/EKS skill the client has — discovered, not assumed | +| `infra` / `k8s` | `infra` | **whatever runtime connector the user has** — k8s/EKS, ECS, docker, Nomad, plain VMs, PM2, … Discovered and probed at the gate, NEVER assumed to be Kubernetes; the manifest records the kind (`via`) | | `kibana` | `logs` | whatever log-search skill the client has (kibana or other) | | `metrics` | `metrics` | whatever metrics skill the client has | | `other` | `other` | best-effort by ask text; else a `not-found` block | @@ -137,11 +138,11 @@ test, Gate Part A enumerates **and probe-validates** the client's connectors (a recorded gap): ``` -{ github: {available: true, via: "gh"}, k8s: {available: false}, ... } +{ github: {available: true, via: "gh"}, infra: {available: true, via: "kubectl"}, logs: {available: false}, ... } ``` - Every ask routes against this manifest — reproducible, no per-ask discovery. -- The gate summary **declares the gaps to the user** ("k8s + metrics not +- The gate summary **declares the gaps to the user** ("infra + metrics not available") and the first turn declares them to TFA so it plans asks around what's obtainable. - Frozen at gate close. A skill appearing mid-run is not picked up until the next run. diff --git a/skills/rca-build/templates/gate-summary.md b/skills/rca-build/templates/gate-summary.md index 7602968..48e9d94 100644 --- a/skills/rca-build/templates/gate-summary.md +++ b/skills/rca-build/templates/gate-summary.md @@ -6,7 +6,7 @@ prints, the run never asks the user anything. ``` GATE CLOSED — capability manifest: - github ✅ valid (gh, authed) · k8s ✅ valid (ctx <context>) · logs ❌ absent · metrics ❌ absent + github ✅ valid (gh, authed) · infra ✅ valid (via <kubectl ctx …, docker, ecs, …>) · logs ❌ absent · metrics ❌ absent Intake: build id: <id> (given) diff --git a/tests/coverage-glimpse.test.mjs b/tests/coverage-glimpse.test.mjs index 18f2817..4740301 100644 --- a/tests/coverage-glimpse.test.mjs +++ b/tests/coverage-glimpse.test.mjs @@ -29,7 +29,7 @@ test("partial coverage caps a high TFA confidence at medium", () => { test("thin coverage (nothing fulfilled, gaps) caps at low", () => { const s = coverageStamp({ asksFulfilled: [], - asksUnavailable: ["k8s", "metrics"], + asksUnavailable: ["infra", "metrics"], tfaConfidence: "high", }); assert.equal(s.coverage, "thin"); diff --git a/tests/evidence.test.mjs b/tests/evidence.test.mjs index e01b06a..041c694 100644 --- a/tests/evidence.test.mjs +++ b/tests/evidence.test.mjs @@ -8,7 +8,8 @@ const CONFIG = { test_logs: { owner: "tfa", skip: true }, product_code: { capability: "github" }, deploy: { capability: "github" }, - k8s: { capability: "k8s" }, + infra: { capability: "infra" }, + k8s: { capability: "infra" }, metrics: { capability: "metrics" }, other: { capability: "other" }, }, @@ -20,7 +21,7 @@ test("buildManifest marks discovered capabilities available with via", () => { ]); assert.equal(manifest.github.available, true); assert.equal(manifest.github.via, "github-mcp"); - assert.equal(manifest.k8s.available, false); + assert.equal(manifest.infra.available, false); }); test("buildManifest excludes the TFA-owned test_logs capability", () => { @@ -38,7 +39,7 @@ test("buildManifest dedupes capabilities shared by multiple evidence types", () test("unavailableCapabilities lists what the client can't get", () => { const manifest = buildManifest(CONFIG, [{ capability: "github" }]); const unavailable = unavailableCapabilities(manifest).sort(); - assert.deepEqual(unavailable, ["k8s", "metrics", "other"]); + assert.deepEqual(unavailable, ["infra", "metrics", "other"]); }); test("evidence cache computes once and reuses across calls", async () => { diff --git a/tests/routing.test.mjs b/tests/routing.test.mjs index c63b595..6ebdb9b 100644 --- a/tests/routing.test.mjs +++ b/tests/routing.test.mjs @@ -6,7 +6,8 @@ const CONFIG = { evidenceRouting: { test_logs: { owner: "tfa", skip: true }, product_code: { capability: "github", discoveryHints: ["github-mcp", "gh"] }, - k8s: { capability: "k8s", discoveryHints: [] }, + infra: { capability: "infra", discoveryHints: [] }, + k8s: { capability: "infra", discoveryHints: [] }, other: { capability: "other", discoveryHints: [] }, }, }; @@ -30,10 +31,10 @@ test("available capability → gather, carrying via", () => { test("unavailable capability → gap, carrying discovery hints", () => { const r = routeAsk({ evidenceType: "k8s", priority: "medium" }, CONFIG, { - k8s: { available: false }, + infra: { available: false }, }); assert.equal(r.action, "gap"); - assert.equal(r.capability, "k8s"); + assert.equal(r.capability, "infra"); assert.equal(r.reason, "no-capability"); }); From af0f188715387c829807209ad4c41c8d535926bf Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 15:40:47 +0530 Subject: [PATCH 29/47] docs(rca): fix orphaned demo-step lines left by the k8s-skill removal; note prod default base URL Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 571e7da..17c23a0 100644 --- a/README.md +++ b/README.md @@ -101,10 +101,9 @@ A seeded failing build exercises the full loop against real staging infra: `is_mcp_driven` feature and `upload.sh` to push them. The current seeded build: `awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2` (project "RCA Feature Fencing", build "VRT Build"). See `automation/README.md`. - provide the `infra` capability: read-only runtime context (pod/instance health, deployed - image, error logs, events) from the `rengg-tfa` namespace, secrets redacted. -3. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported and `kubectl` - pointed at the staging cluster: +2. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported (and, for a + staging build like this one, `O11Y_TFA_RCA_BASE_URL` pointed at the tenant — + the default is production; see `INTEGRATION.md`): ``` /rca-build awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 ``` From 37edaa8cb1e3d7bf59808c147db7fc559cef6718 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 15:52:44 +0530 Subject: [PATCH 30/47] chore(rca): pin bstack MCP server to @browserstack/mcp-server@1.2.27-beta.1 Pinned across all three client wirings (.mcp.json, .cursor-mcp.json, codex example) + INTEGRATION.md. Codex example env aligned with the prod-default base URL (staging override optional). Version verified present on the public npm registry. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- .cursor-mcp.json | 2 +- .mcp.json | 2 +- INTEGRATION.md | 4 ++-- codex-mcp.example.toml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.cursor-mcp.json b/.cursor-mcp.json index 9cf95af..eed3690 100644 --- a/.cursor-mcp.json +++ b/.cursor-mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "bstack": { "command": "npx", - "args": ["-y", "@browserstack/mcp-server"], + "args": ["-y", "@browserstack/mcp-server@1.2.27-beta.1"], "env": { "BROWSERSTACK_USERNAME": "${BROWSERSTACK_USERNAME}", "BROWSERSTACK_ACCESS_KEY": "${BROWSERSTACK_ACCESS_KEY}", diff --git a/.mcp.json b/.mcp.json index 0502929..7fca468 100644 --- a/.mcp.json +++ b/.mcp.json @@ -3,7 +3,7 @@ "bstack": { "type": "stdio", "command": "npx", - "args": ["-y", "@browserstack/mcp-server"], + "args": ["-y", "@browserstack/mcp-server@1.2.27-beta.1"], "env": { "BROWSERSTACK_USERNAME": "${BROWSERSTACK_USERNAME}", "BROWSERSTACK_ACCESS_KEY": "${BROWSERSTACK_ACCESS_KEY}", diff --git a/INTEGRATION.md b/INTEGRATION.md index 65de0b6..f7bedad 100644 --- a/INTEGRATION.md +++ b/INTEGRATION.md @@ -76,7 +76,7 @@ into `~/.codex/config.toml`, or: codex mcp add bstack \ --env BROWSERSTACK_USERNAME=… --env BROWSERSTACK_ACCESS_KEY=… \ --env O11Y_TFA_RCA_BASE_URL=https://api-observability-rengg-tfa.bsstag.com \ - -- npx -y @browserstack/mcp-server + -- npx -y @browserstack/mcp-server@1.2.27-beta.1 ``` **Skill + agent discovery** — Codex reads `.agents/skills/` (skills) and @@ -92,7 +92,7 @@ Then run the `rca-build` skill; the coordinator + `tfaRcaTurn` loop are identica ## Notes -- The `bstack` server is **stdio** (`npx @browserstack/mcp-server`), not a remote +- The `bstack` server is **stdio** (`npx @browserstack/mcp-server@1.2.27-beta.1`), not a remote OAuth server — so the configs use `command`/`args`/`env`, unlike Slack's `url`+`oauth`/`auth` shape. - Env-var interpolation (`${VAR}`) is honored by Claude Code's `.mcp.json`; on diff --git a/codex-mcp.example.toml b/codex-mcp.example.toml index 4fb24af..e3dda80 100644 --- a/codex-mcp.example.toml +++ b/codex-mcp.example.toml @@ -5,7 +5,7 @@ [mcp_servers.bstack] command = "npx" -args = ["-y", "@browserstack/mcp-server"] -env = { "BROWSERSTACK_USERNAME" = "your-username", "BROWSERSTACK_ACCESS_KEY" = "your-access-key", "O11Y_TFA_RCA_BASE_URL" = "https://api-observability-rengg-tfa.bsstag.com" } +args = ["-y", "@browserstack/mcp-server@1.2.27-beta.1"] +env = { "BROWSERSTACK_USERNAME" = "your-username", "BROWSERSTACK_ACCESS_KEY" = "your-access-key", "O11Y_TFA_RCA_BASE_URL" = "" # optional: set only to target a staging tenant (default is production) } startup_timeout_sec = 15 tool_timeout_sec = 120 From 6fea22854c678dc0f4ae1cd5c8e61c919c800495 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 17:18:12 +0530 Subject: [PATCH 31/47] chore(rca): remove automation/ seed harness + harden product-repo inference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The automation/ demo (failing rcaChat API cases + upload.sh, named observability-api as 'the product repo') was env-specific bait: the gate latched onto the only repo named in a workspace doc and carried its PRs into the manifest — even for a build whose failures were self-healing tests, wholly unrelated to observability-api. - Delete automation/ (tests, upload.sh, README, fixture xml). - Gate Part B now REQUIRES product-repo corroboration against the failure signatures; a doc-sourced repo is a weak hint, never an assumption — if it doesn't match the failures it's recorded as a gap and culprit-PR hunts report 'no culprit PR identified' rather than blaming an unrelated repo. - README 'Demo run' section replaced with a generic 'Run' section (no seeded build, no hardcoded repo). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- README.md | 37 +++++------ automation/README.md | 45 ------------- automation/build-rca-failures.xml | 74 --------------------- automation/tests/test_rca_chat_api.py | 96 --------------------------- automation/upload.sh | 23 ------- skills/rca-build/SKILL.md | 12 ++++ 6 files changed, 30 insertions(+), 257 deletions(-) delete mode 100644 automation/README.md delete mode 100644 automation/build-rca-failures.xml delete mode 100644 automation/tests/test_rca_chat_api.py delete mode 100755 automation/upload.sh diff --git a/README.md b/README.md index 17c23a0..ca64af1 100644 --- a/README.md +++ b/README.md @@ -93,25 +93,24 @@ mandatory culprit-PR links on application bugs — is the real deliverable. skills your client already has. Missing ones degrade gracefully (the RCA's confidence band reflects what evidence was actually available). -## Demo run (rengg-tfa) - -A seeded failing build exercises the full loop against real staging infra: - -1. **Seed the build** — `automation/` holds failing API cases for the rcaChat / - `is_mcp_driven` feature and `upload.sh` to push them. The current seeded build: - `awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2` (project "RCA Feature Fencing", - build "VRT Build"). See `automation/README.md`. -2. **Run** — with `BROWSERSTACK_USERNAME`/`ACCESS_KEY` exported (and, for a - staging build like this one, `O11Y_TFA_RCA_BASE_URL` pointed at the tenant — - the default is production; see `INTEGRATION.md`): - ``` - /rca-build awswxm0t5ve7vbjnspfna4xbvjwxn92u2lwv5fw2 - ``` - The gate validates connectors (github via `gh`, infra via whatever runtime connector exists (kubectl/docker/ecs/…) - skill), then the harness clusters the failures, drives `tfaRcaTurn` per - cluster, routes `infra`/`k8s` asks to it while `product_code`/`deploy` asks go - to GitHub — landing per-test RCAs on the dashboard that trace back to the - seeded regressions, then prints the glimpse + the Test Observability link. +## Run + +Point it at any red BrowserStack build — the harness discovers what it needs: + +``` +# BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY exported; default base is +# production, override O11Y_TFA_RCA_BASE_URL only for a staging tenant. +/rca-build <build-id> +``` + +The single gate validates connectors (github via `gh`, infra via whatever +runtime connector exists — kubectl/docker/ecs/…) and resolves the intake +**by inference** (product/automation repo from the cwd's git remote, branches, +any PRs you pass). It never assumes a product repo from unrelated workspace +docs — if it can't infer one that matches the failures, it records the gap and +proceeds RCA-only rather than blaming the wrong repo. Then it clusters the +failures, drives `tfaRcaTurn` per cluster, and lands per-test RCAs on the +dashboard, printing the glimpse + the Test Observability link. ## Layout diff --git a/automation/README.md b/automation/README.md deleted file mode 100644 index a18e4d4..0000000 --- a/automation/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# Automation — RCA Feature Fencing - -Failing API automation for the rcaChat / `is_mcp_driven` ownership feature -(observability-api, PRs #8305 / #7114 / #7059), run against the **rengg-tfa** -staging env. The failing build it produces is the **input the RCA plugin runs -against** — each failure pins a real regression so the collaborative RCA has -genuine material to trace back to our commits. - -## Files -- `tests/test_rca_chat_api.py` — the automation cases (pytest + requests). -- `build-rca-failures.xml` — JUnit results (6 failures + 1 error) for upload. -- `upload.sh` — pushes the JUnit XML to Observability, creating the project/build. - -## Credentials -Never commit creds. Export them (or `source automation/.env`, gitignored): -```bash -export BSTACK_USER=tfauser_wzgsM5 -export BSTACK_KEY=<access-key> -export O11Y_BASE_URL=https://api-observability-rengg-tfa.bsstag.com -``` - -## Run + upload -```bash -# (optional) regenerate the XML against the live API -pytest automation/tests --junitxml=automation/build-rca-failures.xml - -# upload → creates project "RCA Feature Fencing", build "VRT Build" -PROJECT_NAME="RCA Feature Fencing" BUILD_NAME="VRT Build" \ - ./automation/upload.sh automation/build-rca-failures.xml -``` - -The upload response carries the build id. Feed it to the plugin: -``` -/rca-build <build-id> -``` - -## The failures (what RCA should rediscover) -| Test | Pins | -|---|---| -| `mcp_claim_refused_when_web_owned` | jsonb `::` cast → SQLState 42601 → 500 (TestRunsRcaRepository) | -| `mcp_claim_sets_is_mcp_driven_true` | jsonb_set never applied (same root cause) | -| `web_approve_after_mcp_claim_is_not_locked_out` | cross-flow ownership lockout | -| `submit_turn_returns_structured_status` | setTestRca 500 before turn produced | -| `needs_info_asks_carry_evidence_type` | ask.evidenceType null | -| `set_test_rca_error_callback_does_not_500` | data-gate not scoped to success (AIService) | diff --git a/automation/build-rca-failures.xml b/automation/build-rca-failures.xml deleted file mode 100644 index ce78926..0000000 --- a/automation/build-rca-failures.xml +++ /dev/null @@ -1,74 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<testsuites name="RCA Feature Fencing" tests="7" failures="6" errors="1" time="38.402"> - <testsuite name="RcaChat Ownership Fencing" tests="4" failures="3" errors="1" time="19.871" timestamp="2026-06-24T09:12:03"> - <testcase classname="com.browserstack.observability.rcachat.OwnershipFencingTest" name="mcp_claim_refused_when_web_owned" time="4.310" file="api-tests/rcachat/test_ownership_fencing.py"> - <failure message="Expected HTTP 403 (FORBIDDEN) when the run is web-owned (metadata.is_mcp_driven=false); got HTTP 500."> -AssertionError: claimForMcpOrRefuse did not refuse a web-owned run. - Request : POST /ext/v1/testRuns/4412/rcaChat {"message":"start"} - Expected: 403 FORBIDDEN (web flow owns this RCA) - Actual : 500 Internal Server Error - Body : {"error":"could not execute statement [n/a]; SQL [n/a]; nested exception is - org.postgresql.util.PSQLException: ERROR: syntax error at or near \":\" - Position: 71 ; SQLState: 42601"} - Hint : native UPDATE in TestRunsRcaRepository.claimForMcpIfNotWebOwned uses '::jsonb' - which Hibernate parses as a named parameter. - at api-tests/rcachat/test_ownership_fencing.py:48 - </failure> - </testcase> - <testcase classname="com.browserstack.observability.rcachat.OwnershipFencingTest" name="mcp_claim_sets_is_mcp_driven_true_when_unowned" time="3.901" file="api-tests/rcachat/test_ownership_fencing.py"> - <failure message="Expected metadata.is_mcp_driven=true after an MCP claim on an unowned run; got null."> -AssertionError: claim did not stamp ownership. - Request : POST /ext/v1/testRuns/4413/rcaChat - Expected: test_runs_rca.metadata.is_mcp_driven == true - Actual : metadata.is_mcp_driven == null (jsonb_set never applied — statement failed) - at api-tests/rcachat/test_ownership_fencing.py:71 - </failure> - </testcase> - <testcase classname="com.browserstack.observability.rcachat.OwnershipFencingTest" name="web_approve_after_mcp_claim_is_not_locked_out" time="5.220" file="api-tests/rcachat/test_ownership_fencing.py"> - <failure message="Cross-flow lockout: web approve() left the run MCP-owned."> -AssertionError: expected web approve() to win the cross-flow race; run stayed is_mcp_driven=true. - Sequence: MCP claim -> web approve() - Expected: is_mcp_driven=false after approve - Actual : is_mcp_driven=true - at api-tests/rcachat/test_ownership_fencing.py:96 - </failure> - </testcase> - <testcase classname="com.browserstack.observability.rcachat.OwnershipFencingTest" name="seed_pending_row_when_no_row_exists" time="6.440" file="api-tests/rcachat/test_ownership_fencing.py"> - <error message="Connection reset while seeding PENDING row"> -java.net.SocketException: Connection reset - at com.browserstack.observability.service.RcaChatService.claimForMcpOrRefuse(RcaChatService.java) - while seeding a PENDING test_runs_rca row for testRunId=4414 - </error> - </testcase> - </testsuite> - <testsuite name="RcaChat Turn API" tests="3" failures="3" errors="0" time="18.531" timestamp="2026-06-24T09:12:24"> - <testcase classname="com.browserstack.observability.rcachat.TurnApiTest" name="submit_turn_returns_structured_status" time="6.110" file="api-tests/rcachat/test_turn_api.py"> - <failure message="Expected a structured turn (status in NEEDS_INFO|RESOLVED|BLOCKED|PENDING); got 500."> -AssertionError: rcaChat submit did not return a structured turn. - Request : POST /ext/v1/testRuns/4420/rcaChat {"message":"empty buildName rejected on POST /builds"} - Expected: 200 with body.status in [NEEDS_INFO, RESOLVED, BLOCKED, PENDING] - Actual : 500 Internal Server Error (setTestRca failed before turn was produced) - at api-tests/rcachat/test_turn_api.py:39 - </failure> - </testcase> - <testcase classname="com.browserstack.observability.rcachat.TurnApiTest" name="needs_info_asks_carry_evidence_type" time="5.980" file="api-tests/rcachat/test_turn_api.py"> - <failure message="Each ask in a NEEDS_INFO turn must carry an evidenceType; one ask had null."> -AssertionError: ask.evidenceType missing. - Turn : NEEDS_INFO with 3 asks - Expected: every ask.evidenceType in [test_logs, product_code, k8s, kibana, metrics, deploy, ci, other] - Actual : asks[2].evidenceType == null - at api-tests/rcachat/test_turn_api.py:64 - </failure> - </testcase> - <testcase classname="com.browserstack.observability.rcachat.TurnApiTest" name="set_test_rca_error_callback_does_not_500" time="6.441" file="api-tests/rcachat/test_turn_api.py"> - <failure message="Non-chat error callback (success=false, no data) returned 500 instead of 200."> -AssertionError: setTestRca data-gate is not scoped to success. - Request : error callback {"success":false} (no rca data) - Expected: 200 (record error state, no RCA) - Actual : 500 Internal Server Error (gate rejected missing data even on the error path) - Hint : AIService.setTestRca data-gate must be scoped to success=true. - at api-tests/rcachat/test_turn_api.py:88 - </failure> - </testcase> - </testsuite> -</testsuites> diff --git a/automation/tests/test_rca_chat_api.py b/automation/tests/test_rca_chat_api.py deleted file mode 100644 index 2f84af2..0000000 --- a/automation/tests/test_rca_chat_api.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Failing API automation for the rcaChat / is_mcp_driven ownership feature. - -Targets the observability-api rcaChat surface deployed in the rengg-tfa staging -env. These cases are written to FAIL against the current build — each one pins a -real regression in the feature we shipped (PRs #8305 / #7114 / #7059): - - - the jsonb '::' cast in TestRunsRcaRepository.claimForMcpIfNotWebOwned - (Hibernate parses '::jsonb' as a named param -> SQLState 42601 -> 500) - - AIService.setTestRca data-gate not scoped to success (error callbacks 500) - - cross-flow ownership lockout between MCP claim and web approve() - -Run with: pytest --junitxml=automation/build-rca-failures.xml -Creds + base URL come from the environment (never hardcode): - O11Y_BASE_URL default https://api-observability-rengg-tfa.bsstag.com - BSTACK_USER / BSTACK_KEY -""" - -import os - -import pytest -import requests - -BASE = os.environ.get("O11Y_BASE_URL", "https://api-observability-rengg-tfa.bsstag.com") -AUTH = (os.environ.get("BSTACK_USER", ""), os.environ.get("BSTACK_KEY", "")) -TIMEOUT = 30 - - -def _rca_chat(test_run_id, body): - return requests.post( - f"{BASE}/ext/v1/testRuns/{test_run_id}/rcaChat", - json=body, - auth=AUTH, - timeout=TIMEOUT, - ) - - -class TestOwnershipFencing: - def test_mcp_claim_refused_when_web_owned(self): - # A web-owned run (metadata.is_mcp_driven=false) must refuse the MCP claim - # with 403 — not 500 from a broken native UPDATE. - resp = _rca_chat(4412, {"message": "start"}) - assert resp.status_code == 403, ( - f"expected 403 FORBIDDEN for a web-owned run, got {resp.status_code}: {resp.text}" - ) - - def test_mcp_claim_sets_is_mcp_driven_true_when_unowned(self): - resp = _rca_chat(4413, {"message": "start"}) - assert resp.status_code == 200 - # ownership should be stamped on the row - meta = resp.json().get("metadata", {}) - assert meta.get("is_mcp_driven") is True, "claim did not stamp is_mcp_driven=true" - - def test_web_approve_after_mcp_claim_is_not_locked_out(self): - _rca_chat(4414, {"message": "start"}) # MCP claim - approve = requests.post( - f"{BASE}/ext/v1/testRuns/4414/rca/approve", auth=AUTH, timeout=TIMEOUT - ) - assert approve.status_code == 200 - state = requests.get( - f"{BASE}/ext/v1/testRuns/4414/rca", auth=AUTH, timeout=TIMEOUT - ).json() - assert state.get("metadata", {}).get("is_mcp_driven") is False, ( - "cross-flow lockout: web approve() left the run MCP-owned" - ) - - -class TestTurnApi: - def test_submit_turn_returns_structured_status(self): - resp = _rca_chat(4420, {"message": "empty buildName rejected on POST /builds"}) - assert resp.status_code == 200, f"expected a structured turn, got {resp.status_code}" - assert resp.json().get("status") in { - "NEEDS_INFO", - "RESOLVED", - "BLOCKED", - "PENDING", - } - - def test_needs_info_asks_carry_evidence_type(self): - resp = _rca_chat(4421, {"message": "investigate"}) - asks = resp.json().get("asks", []) - valid = {"test_logs", "product_code", "k8s", "kibana", "metrics", "deploy", "ci", "other"} - for i, ask in enumerate(asks): - assert ask.get("evidenceType") in valid, f"asks[{i}].evidenceType missing/invalid" - - def test_set_test_rca_error_callback_does_not_500(self): - # Error callback (success=false, no rca data) must record the error state - # and return 200 — the data-gate must be scoped to success=true. - resp = requests.post( - f"{BASE}/ext/v1/testRuns/4422/rca/callback", - json={"success": False}, - auth=AUTH, - timeout=TIMEOUT, - ) - assert resp.status_code == 200, ( - f"error callback returned {resp.status_code}; data-gate not scoped to success" - ) diff --git a/automation/upload.sh b/automation/upload.sh deleted file mode 100755 index cd4f3bf..0000000 --- a/automation/upload.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Upload the JUnit results to BrowserStack Observability (rengg-tfa staging), -# creating the project + build that the RCA plugin then runs against. -# -# Creds come from the environment — never commit them: -# export BSTACK_USER=... BSTACK_KEY=... -# (or `source automation/.env`, which is gitignored.) -set -euo pipefail - -UPLOAD_URL="${O11Y_UPLOAD_URL:-https://upload-observability-rengg-tfa.bsstag.com/upload}" -XML="${1:-$(dirname "$0")/build-rca-failures.xml}" -PROJECT="${PROJECT_NAME:-RCA Feature Fencing}" -BUILD="${BUILD_NAME:-VRT Build}" - -: "${BSTACK_USER:?set BSTACK_USER}" -: "${BSTACK_KEY:?set BSTACK_KEY}" - -curl -sS -X POST "$UPLOAD_URL" \ - -u "${BSTACK_USER}:${BSTACK_KEY}" \ - -F "data=@${XML}" \ - -F "projectName=${PROJECT}" \ - -F "buildName=${BUILD}" -echo diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 1b7b898..86ee86a 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -81,6 +81,18 @@ is the point: - the current branch for the working branch, - cheap inference (e.g. the automation repo is the cwd if it holds the tests). +**Product-repo corroboration (do NOT skip).** The product repo must plausibly +be the *system under test for THIS build's failures* — not merely a repo name +found lying around. A repo mentioned only in workspace docs/READMEs is a **weak +hint, never an assumption**: cross-check it against the failure signatures +(discovery runs first if needed) — do the failing area, files, or error strings +relate to that repo's domain? If they don't (e.g. the failures are self-healing +`healedElement is null` cases but the only named repo is an observability API), +the product repo is a **gap**, recorded as "unknown" — the run proceeds RCA-only +and every culprit-PR hunt reports "no culprit PR identified" rather than blaming +an unrelated repo's PRs. Never carry a doc-sourced repo (or its PRs) into the +manifest as a settled product repo without this corroboration. + Record each assumption in the gate summary (format: `templates/gate-summary.md`; worked example: `examples/sample-run.md`) ("assumed product repo = From c68cd17248f5c8cf7d326d145b0e1694ae554afc Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 17:42:34 +0530 Subject: [PATCH 32/47] fix(rca): ask for the product repo when it's unknown + load-bearing (interactive) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously the gate silently recorded 'product repo: gap' whenever corroboration failed — which kills the mandatory culprit-PR hunt without telling the human sitting at the terminal. Now, when the repo can't be corroborated against the failure signatures AND no PRs were supplied, the product repo is treated as non-assumable AND load-bearing and earns the single consolidated gate question (interactive only): 'Failures look like <domain>; which repo owns that code? (reply none → RCA without culprit-PR attribution).' Headless still records the gap and proceeds RCA-only. Preserves both invariants: never blame the wrong repo, and never silently cripple culprit-PR hunting when a human could answer. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- skills/rca-build/SKILL.md | 27 +++++++++++++++------- skills/rca-build/templates/gate-summary.md | 6 ++++- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 86ee86a..26450a7 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -88,20 +88,31 @@ hint, never an assumption**: cross-check it against the failure signatures (discovery runs first if needed) — do the failing area, files, or error strings relate to that repo's domain? If they don't (e.g. the failures are self-healing `healedElement is null` cases but the only named repo is an observability API), -the product repo is a **gap**, recorded as "unknown" — the run proceeds RCA-only -and every culprit-PR hunt reports "no culprit PR identified" rather than blaming -an unrelated repo's PRs. Never carry a doc-sourced repo (or its PRs) into the -manifest as a settled product repo without this corroboration. +the doc-sourced repo is discarded — never carry it (or its PRs) into the +manifest as a settled product repo. + +When corroboration leaves **no** product repo, decide by whether a human can help: +- **PRs were supplied** → treat those as the suspect surface; product repo is + derived from them. No question needed. +- **No PRs, interactive session** → the product repo is now **non-assumable AND + load-bearing** (without it the mandatory culprit-PR hunt is dead), so it earns + the single consolidated gate question below — ask it; don't silently degrade. +- **No PRs, headless** → record the gap ("product repo: unknown") and proceed + RCA-only; every culprit-PR hunt reports "no culprit PR identified". Record each assumption in the gate summary (format: `templates/gate-summary.md`; worked example: `examples/sample-run.md`) ("assumed product repo = `org/obs-api` from git remote"). A field that cannot be assumed is recorded as "none" and the run proceeds RCA-only for it — **unless** it is both genuinely -non-assumable AND load-bearing (in practice: only the build id, and rarely an -ambiguous repo when PRs were supplied). Those, and only those, may be asked -**ONCE, in a single consolidated question at gate close**. Never a second -question. **Headless: skip asking entirely; record the gaps.** +non-assumable AND load-bearing. In practice that set is: the build id; **the +product repo when it could not be corroborated and no PRs were supplied** (see +above — without it the culprit-PR hunt cannot run); and rarely an ambiguous repo +when PRs were supplied. Those, and only those, may be asked **ONCE, in a single +consolidated question at gate close** — e.g. *"Failures look like `<domain>`; +which repo owns that code? (reply 'none' → I'll RCA without culprit-PR +attribution)."* Never a second question. **Headless: skip asking entirely; +record the gaps.** ### Gate close diff --git a/skills/rca-build/templates/gate-summary.md b/skills/rca-build/templates/gate-summary.md index 48e9d94..48ac6bf 100644 --- a/skills/rca-build/templates/gate-summary.md +++ b/skills/rca-build/templates/gate-summary.md @@ -4,13 +4,17 @@ One terse FYI before autonomous execution starts. Every intake field is tagged `given | assumed | gap`; every connector `valid | invalid | absent`. After this prints, the run never asks the user anything. +If the product repo could not be corroborated against the failures AND no PRs +were supplied, the gate asks ONE question (interactive only) before printing +this summary. Headless skips it and prints `product repo: unknown (gap)`. + ``` GATE CLOSED — capability manifest: github ✅ valid (gh, authed) · infra ✅ valid (via <kubectl ctx …, docker, ecs, …>) · logs ❌ absent · metrics ❌ absent Intake: build id: <id> (given) - product repo: <org/repo> (assumed — from git remote) + product repo: <org/repo> (assumed — corroborated vs failures | asked — human answered | unknown (gap)) automation repo: <org/repo> (assumed — cwd holds the tests) working branch: <branch> (assumed — current branch) default branch: <branch> (assumed — origin HEAD) From 31915f0f8ae433f033036f29686a4d8f79c84f6f Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Tue, 14 Jul 2026 21:00:27 +0530 Subject: [PATCH 33/47] =?UTF-8?q?fix(rca):=20finish=20output=20is=20a=20co?= =?UTF-8?q?mpletion=20notice=20+=20link=20only=20=E2=80=94=20no=20RCA=20de?= =?UTF-8?q?tail=20in=20Claude?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The end-of-run glimpse was leaking exactly what's supposed to live only on the TRA UI: per-test root causes, culprit/related PRs, cluster breakdowns, confidence rationales. renderGlimpse now emits just 'RCA analysis complete — build <id>' + a status-count line (N tests · R resolved · P pending · F failed); the caller adds the 'Full report on the Test Observability UI: <link>' line. Step 6 and the example updated to match; a test asserts no testRunId/cluster/root_cause/PR ever appears in the output. Humans open the link for the why. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- lib/glimpse.mjs | 69 +++++++++++-------------- skills/rca-build/SKILL.md | 26 +++++----- skills/rca-build/examples/sample-run.md | 14 ++--- tests/coverage-glimpse.test.mjs | 32 +++++------- 4 files changed, 65 insertions(+), 76 deletions(-) diff --git a/lib/glimpse.mjs b/lib/glimpse.mjs index c17f8f3..38d7fb0 100644 --- a/lib/glimpse.mjs +++ b/lib/glimpse.mjs @@ -1,50 +1,41 @@ -// Terse end-of-run glimpse — the ONLY in-client output of a /rca-build run. -// One line per test: testRunId → cluster → status → confidence one-liner. -// The full RCA report lives on the Test Observability UI (triggerRcaReport → -// viewReport link); this is deliberately a glimpse, never a report. Degrade, -// don't crash: missing fields render as "-". +// Terse end-of-run summary — the ONLY in-client output of a /rca-build run. +// Deliberately a COMPLETION NOTICE, not a report: status counts + the UI link, +// nothing else. Root causes, culprit PRs, per-test analysis, cluster breakdowns +// live ONLY on the Test Observability dashboard (triggerRcaReport → viewReport). +// Do not reintroduce per-test cause/PR lines here — that is the whole point of +// the trim. Degrade, don't crash: missing fields are treated as absent. import { readRows } from "./csv-state.mjs"; -const DASH = "-"; -const ONE_LINER_MAX = 80; - -function cell(value) { - const s = value == null ? "" : String(value).trim(); - if (s === "") return DASH; - return s.replace(/\s*\n\s*/g, " "); -} - -function oneLiner(row) { - const confidence = cell(row.confidence); - const cause = cell(row.root_cause); - const text = cause === DASH ? confidence : `${confidence}: ${cause}`; - return text.length > ONE_LINER_MAX ? `${text.slice(0, ONE_LINER_MAX - 1)}…` : text; +// Map raw CSV rca_done states → the three buckets a human cares about. +function bucket(state) { + const s = (state || "").toLowerCase(); + if (s === "resolved") return "resolved"; + if (s === "pending" || s === "pending-resume") return "pending"; + return "failed"; } -// Render the glimpse from CSV rows. Returns a plain-text block: -// <testRunId> → <cluster_id> → <status> → <confidence one-liner> +// Render the completion summary. Returns a plain-text block: +// RCA analysis complete — build <id> +// <N> tests · <R> resolved · <P> pending · <F> failed +// (the caller appends the "Full report on the Test Observability UI: <link>" +// line from triggerRcaReport's viewReport — see SKILL.md Step 6). export function renderGlimpse(rows, { buildId } = {}) { - const lines = [`Glimpse${buildId ? ` — build ${buildId}` : ""}`]; + const head = `RCA analysis complete${buildId ? ` — build ${buildId}` : ""}`; if (!rows || rows.length === 0) { - lines.push("No failed tests analyzed."); - return lines.join("\n") + "\n"; - } - const byState = rows.reduce((acc, r) => { - const k = r.rca_done || "unknown"; - acc[k] = (acc[k] ?? 0) + 1; - return acc; - }, {}); - const summary = Object.entries(byState) - .map(([k, v]) => `${k}: ${v}`) - .join(" · "); - lines.push(`${rows.length} test(s) — ${summary}`); - for (const r of rows) { - lines.push( - `${cell(r.testRunId)} → ${cell(r.cluster_id)} → ${cell(r.rca_done)} → ${oneLiner(r)}`, - ); + return `${head}\nNo failed tests analyzed.\n`; } - return lines.join("\n") + "\n"; + const counts = rows.reduce( + (acc, r) => { + acc[bucket(r.rca_done)] += 1; + return acc; + }, + { resolved: 0, pending: 0, failed: 0 }, + ); + const parts = [`${rows.length} test(s)`, `${counts.resolved} resolved`]; + if (counts.pending) parts.push(`${counts.pending} pending`); + if (counts.failed) parts.push(`${counts.failed} failed`); + return `${head}\n${parts.join(" · ")}\n`; } export function renderGlimpseFromCsv(csvPath, opts = {}) { diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 26450a7..140b2a9 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -193,24 +193,26 @@ identified after <what was searched>" and the CSV row records the gap. ## Step 6 — finish: glimpse + dashboard report (NO local report) -This plugin **never renders or writes a local RCA report**. When every row is -terminal: - -1. Print a **terse glimpse table** from the CSV (`lib/glimpse.mjs` → - `renderGlimpse`): one line per test — `testRunId → cluster → status → - confidence one-liner`. That is the entire in-Claude output. -2. Call the MCP tool **`triggerRcaReport(buildUuid=<build id>)`** (add - `force=true` only to re-run over an existing completed report). It returns a - trimmed glimpse (`state, verdict, verdictProvisional, partial, analyzedCount, - totalFailedCount, totalPrs, faultyPrNumbers, failureReason, viewReport`). +This plugin **never renders or writes a local RCA report, and never surfaces RCA +detail in Claude.** The in-Claude output is a two-line completion notice plus the +link — that is all. When every row is terminal: + +1. Print the **completion summary** from the CSV (`lib/glimpse.mjs` → + `renderGlimpse`): `RCA analysis complete — build <id>` + a status count line + (`<N> tests · <R> resolved · <P> pending · <F> failed`). **Nothing per-test.** +2. Call **`triggerRcaReport(buildUuid=<build id>)`** (add `force=true` only to + re-run over an existing completed report). 3. Print the link line, verbatim shape: ``` Full report on the Test Observability UI: <viewReport> ``` -Humans read the real report **there**, populated by the BrowserStack agent — -not in Claude. +**Do NOT print** root causes, culprit/related PRs, cluster breakdowns, per-test +analysis, confidence rationales, or a per-test table — root_cause, related_prs, +suspect_signals and the like are for the CSV + the dashboard ONLY. If a human +wants the "why", they open the link. Claude's job here is "analysis complete → +report is at <link>", not to re-narrate the RCA the BrowserStack agent authored. ## Resume diff --git a/skills/rca-build/examples/sample-run.md b/skills/rca-build/examples/sample-run.md index 01cc36e..5efa6e2 100644 --- a/skills/rca-build/examples/sample-run.md +++ b/skills/rca-build/examples/sample-run.md @@ -61,17 +61,17 @@ SUMMARY: out-of-scope — TFA owns test logs; skipped by contract. ## 3. Terminal output (glimpse only — NO local report) ``` -RCA batch complete — build awswx…fw2 (7 failed → 3 clusters) - -39 → c1 → RESOLVED (high) PR #7421 tightened buildName validator; related_prs: #7421 -41 → c1 → RESOLVED (high) sibling of 39 (cluster confirm) -57 → c2 → RESOLVED (med) flaky selector wait; test-side -81 → c3 → PENDING (—) soft-pending, resumable (turnId recorded) -… +RCA analysis complete — build awswx…fw2 +7 test(s) · 6 resolved · 1 pending Full report on the Test Observability UI: https://automation.browserstack.com/builds/awswx…fw2?tab=ai_report&subTab=aitfa ``` +That is the **entire** in-Claude output. No root causes, no culprit PRs, no +per-test table — those are on the dashboard, authored by the BrowserStack agent. +The RESOLVED turn shown earlier is the coordinator↔TFA exchange (internal to the +loop), not something re-printed to the user at the end. + State file: `<tmpdir>/bstack-rca/rca-state.awswx…fw2.csv` (resume-safe; re-run the same build id to pick up the PENDING row). diff --git a/tests/coverage-glimpse.test.mjs b/tests/coverage-glimpse.test.mjs index 4740301..3bc0e48 100644 --- a/tests/coverage-glimpse.test.mjs +++ b/tests/coverage-glimpse.test.mjs @@ -54,7 +54,7 @@ test("empty batch renders a valid glimpse, no crash", () => { assert.match(txt, /No failed tests analyzed/); }); -test("glimpse renders one arrow line per test with status counts", () => { +test("glimpse is a completion notice with counts — NO per-test detail", () => { const rows = [ { testRunId: "101", @@ -62,24 +62,22 @@ test("glimpse renders one arrow line per test with status counts", () => { rca_done: "resolved", confidence: "high", root_cause: "PR #7421 tightened validator", + related_prs: "#7421", }, - { - testRunId: "102", - cluster_id: "c1", - rca_done: "failed", - confidence: "", - root_cause: "", - }, + { testRunId: "102", cluster_id: "c1", rca_done: "pending-resume" }, + { testRunId: "103", cluster_id: "c2", rca_done: "failed" }, ]; const txt = renderGlimpse(rows, { buildId: "b1" }); - assert.match(txt, /2 test\(s\)/); - assert.match(txt, /resolved: 1/); - assert.match(txt, /failed: 1/); - assert.match(txt, /101 → c1 → resolved → high: PR #7421 tightened validator/); - assert.match(txt, /102 → c1 → failed → -/); // blank fields degrade to "-" + assert.match(txt, /RCA analysis complete — build b1/); + assert.match(txt, /3 test\(s\)/); + assert.match(txt, /1 resolved/); + assert.match(txt, /1 pending/); // pending-resume buckets to "pending" + assert.match(txt, /1 failed/); + // The trim: no per-test lines, no root cause, no PRs, no testRunIds leak. + assert.doesNotMatch(txt, /7421|PR #|→|101|102|103|c1|c2/); }); -test("glimpse one-liner truncates and collapses newlines (stays terse)", () => { +test("glimpse never leaks a verbose root_cause, however long", () => { const rows = [ { testRunId: "1", @@ -90,8 +88,6 @@ test("glimpse one-liner truncates and collapses newlines (stays terse)", () => { }, ]; const txt = renderGlimpse(rows); - const line = txt.split("\n").find((l) => l.startsWith("1 →")); - assert.ok(line.includes("line one line two")); - assert.ok(line.endsWith("…")); - assert.ok(line.length < 120); + assert.doesNotMatch(txt, /line one|line two|xxxx/); // cause stays out of Claude + assert.match(txt, /1 test\(s\) · 1 resolved/); }); From 97a5c8952b5c466a4446a22042671c1ace2e0b26 Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Thu, 16 Jul 2026 17:54:42 +0530 Subject: [PATCH 34/47] update++ --- workflows/rca-batch.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index 553b94a..6cffbb9 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -53,7 +53,7 @@ const RCA_SCHEMA = { additionalProperties: true, }; -const ctx = args ?? {}; +const ctx = (typeof args === "string" ? JSON.parse(args) : args) ?? {}; const clusters = ctx.clusters ?? []; const shared = [ `CSV state file: ${ctx.csvPath}`, From ea2fce5276b839d6bf78c4aa84a219cd3522332b Mon Sep 17 00:00:00 2001 From: Ruturaj-Browserstack <ruturaj.s@browserstack.com> Date: Sun, 26 Jul 2026 16:10:52 +0530 Subject: [PATCH 35/47] fix(rca): drain soft-PENDING via getTfaTurnResult before resubmitting A soft PENDING is not an agent verdict. tfaRcaTurn abandons its own in-call poll at POLL_MAX_WAIT_MS (90s) and returns `{status:"PENDING", turnId}` while the TFA agent keeps working server-side. The loop treated that as terminal, so any turn finalizing past 90s was reported PENDING even though the agent had already committed to an answer. Observed on test run 3840238857 (build xclakkm...): turn 1 ran 10:09:47 -> 10:11:31 (104s) and finalized NEEDS_INFO with 4 asks + 2 questions, webhooking the result to o11y. The client saw only PENDING and stopped. Reading the same turnId with getTfaTurnResult returns that NEEDS_INFO verbatim. On PENDING the loop now READS that turnId (getTfaTurnResult) on a bounded budget until a real agent status lands, and only then routes asks and submits the next message: - lib/loop.mjs: drainSoftPending() + injected `readTurn`/`sleep`. Reads never consume turnCap (a drain re-reads the same turn, it is not a new turn) and the spent turnId is dropped so follow-ups ride threadId alone. A failed read is not a verdict. Never submits onto a turn still in flight, which would stack two turns on one thread. - config: softPendingDrain { maxWaitMs 600000, intervalMs 5000, maxReads 40 }. Budget spent -> still ends PENDING (pending-resume row), so a wedged turn cannot hang the batch. Old behaviour is now the floor, not the default. - No getTfaTurnResult tool on the client -> end PENDING immediately as before, never busy-wait through tfaRcaTurn resubmits. - Also handle BLOCKED as terminal. It carries no asks, so classifying it as NEEDS_INFO resubmitted empty messages to the turn cap. Reported as PENDING with note "blocked"; giving it its own status in RCA_OUTPUT is a follow-up. - Contract updated in the three places that encode it: the coordinator agent (principle 5, loop step 2, hard limits, tools), the workflow's shared prompt, and SKILL.md (resume + hard rules). Tests: 6 new conformance tests over a recorded 3840238857 drain fixture -- drains before the next submit, reads the same turnId, bounded budget, transient read failure, BLOCKED terminal, and the no-tool floor. 58/58 pass. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> --- agents/ai-tfa-coordinator.md | 49 +++++-- config/rca.config.json | 6 + lib/loop.mjs | 100 ++++++++++++- skills/rca-build/SKILL.md | 9 ++ tests/conformance.test.mjs | 136 +++++++++++++++++- .../recorded-turns/soft-pending-drain.json | 50 +++++++ workflows/rca-batch.mjs | 1 + 7 files changed, 335 insertions(+), 16 deletions(-) create mode 100644 tests/fixtures/recorded-turns/soft-pending-drain.json diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 5ef4e88..6806b15 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -4,7 +4,7 @@ description: 'Per-test collaborative-RCA coordinator (autonomous — never promp - orchestrator: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=39 — error: empty buildName rejected on POST /builds") → drives the loop, returns RCA_OUTPUT - sibling confirm: Agent(subagent_type="tfa-rca:ai-tfa-coordinator", prompt="RCA testRunId=40 — pre-seed: cause=<rep root cause>, suspect PR=#7421") → one-turn confirm against this test logs - user: "run collaborative RCA on test run 39" → single-test loop to RESOLVED/PENDING' -tools: [Bash, Read, Grep, Glob, Task, mcp__*__tfaRcaTurn, mcp__github__*] +tools: [Bash, Read, Grep, Glob, Task, mcp__*__tfaRcaTurn, mcp__*__getTfaTurnResult, mcp__github__*] model: sonnet --- @@ -51,10 +51,16 @@ call the tool. - `RESOLVED` → `{ status, confidence, threadId, glimpse: { root_cause (≤220 chars), failure_type, related_prs }, viewRca }`. The `viewRca` link points at the Test Observability UI — pass it through to the output. -- `PENDING` → `{ status, turnId, threadId }` (soft-pending; in-call poll - exceeded its wall-clock cap — resumable). +- `PENDING` → `{ status, turnId, threadId }`. **Not an agent verdict** — the tool + abandoned its own in-call poll at 90s while TFA kept working. Drain it with + `getTfaTurnResult` (below); never treat it as an answer. - `NEEDS_INFO` → `questions` / `asks` / `suggestions` **verbatim** — this loop consumes them exactly as sent. +- `BLOCKED` → terminal: TFA cannot proceed. No asks; stop the loop. + +`getTfaTurnResult(testRunId, turnId)` reads a submitted turn **once**, returning +the same four shapes — still `PENDING` if the agent is mid-flight. It is +read-only and has no side effects, so a read is always safe to repeat. ## Operating principles @@ -68,9 +74,19 @@ call the tool. extra turn, never a busy-wait. 4. **One thread per test.** First turn omits `threadId`; capture it from the response and reuse it on every follow-up. Never start a second thread. -5. **Soft-PENDING ends the loop.** A tool result of `status: "PENDING"` ends the - loop immediately as `PENDING`, carrying `threadId` + `turnId` for a later - resume. Do not re-poll or sleep. +5. **Soft-PENDING is DRAINED, not reported.** `status: "PENDING"` means the tool's + 90s in-call poll expired, not that TFA has nothing to say — turns landing past + 90s are routine (a first turn finalizing `NEEDS_INFO` at 104s is a real, + observed case). So on `PENDING`, **call `getTfaTurnResult(testRunId, turnId)` + FIRST** and keep reading on the `softPendingDrain` budget + (`config/rca.config.json`: every 5s, ≤40 reads / ≤10min) until the status is + `RESOLVED` / `NEEDS_INFO` / `BLOCKED`. Only then route asks and submit the next + message. **Reads never count against the turn cap** — a drain re-reads the + *same* turn. Never submit a new message onto a turn still in flight: that + stacks two turns on one thread. Only when the drain budget is fully spent does + the run end `PENDING` (note `soft-pending`), resumable via `threadId`+`turnId`. + If the client has no `getTfaTurnResult` tool, end `PENDING` immediately as + before — never busy-wait through `tfaRcaTurn` resubmits instead. 6. **Digest, don't dump.** Every follow-up `message` carries digested findings (`ask → found → snippet/link`), never raw log tails, full diffs, or full files. Size caps + block shape live in `references/evidence-routing.md` — read it @@ -123,8 +139,16 @@ re-fetch per test. Never fabricate a PR when the github capability is unavailabl 1. SUBMIT turn 1: tfaRcaTurn(testRunId=<id>, message=<digest>). Capture threadId. turns_used = 1. (resume case: tfaRcaTurn(testRunId, threadId, turnId) instead, then continue at 2.) 2. CLASSIFY result.status: + PENDING → DRAIN FIRST, do not resubmit and do not end here: + capture threadId + turnId, then loop on + getTfaTurnResult(testRunId, turnId) every softPendingDrain.intervalMs + until status != PENDING, or the budget (maxReads / maxWaitMs) is spent. + landed → replace `result` with it and re-CLASSIFY (turns_used UNCHANGED — + a read is not a turn; drop the spent turnId). + spent → END (PENDING, note "soft-pending"), row stays resumable. + no getTfaTurnResult tool → END (PENDING, note "soft-pending"). RESOLVED → capture glimpse + viewRca; END (RESOLVED). - PENDING → capture threadId + turnId; END (PENDING, note "soft-pending"). + BLOCKED → END (PENDING, note "blocked") — terminal, no asks to route. NEEDS_INFO → go to 3. 3. ROUTE the asks (read references/evidence-routing.md; route via lib/routing.mjs): For each ask, high → medium → low: @@ -207,8 +231,10 @@ RCA_OUTPUT_END ``` Notes: -- `status` is one of exactly three values. `turn-cap` and `soft-pending` both - report as `PENDING`; note which in `root_cause`. +- `status` is one of exactly three values. `turn-cap`, `soft-pending` (drain + budget spent) and `blocked` all report as `PENDING`; note which in `root_cause`. + A `PENDING` from a *drained* turn should never appear — a drain that lands + re-classifies instead. - `asks_skipped` always includes `test_logs` whenever TFA asked for logs. `asks_fulfilled` **never** includes `test_logs`. - `asks_unavailable` is the evidence-coverage signal the coverage stamp turns @@ -222,7 +248,10 @@ Notes: - **Never** fulfill or seed a `test_logs` ask — TFA owns logs. - **Never** exceed `turnCap` `tfaRcaTurn` calls in one run. - **Never** start a second thread for the same test — reuse the first turn's `threadId`. -- **Never** busy-wait / re-poll on a soft-`PENDING` — end and report it resumable. +- **Never** submit a new `tfaRcaTurn` message while a turn is soft-`PENDING` — + drain it with `getTfaTurnResult` first; resubmitting stacks two turns on one thread. +- **Never** let drain reads consume the turn cap, and never drain past the + `softPendingDrain` budget — a wedged turn must not hang the batch. - **Never** dump raw logs, full diffs, or full file contents into a turn message — digest only. - **Never** write to any repo / cluster / ticket / the run — every action is read-only. - **Never** editorialize a cause — pass TFA's `glimpse` through verbatim. diff --git a/config/rca.config.json b/config/rca.config.json index 732c9e3..802f6f4 100644 --- a/config/rca.config.json +++ b/config/rca.config.json @@ -5,6 +5,12 @@ "turnCap": 6, "turnMessageMaxChars": 5000, "pollSoftPendingMs": 90000, + "$softPendingDrainComment": "tfaRcaTurn abandons its in-call poll at pollSoftPendingMs (90s) and returns a soft PENDING while the TFA agent keeps working — turns finalizing past 90s are routine. On a soft PENDING the loop READS the same turnId via getTfaTurnResult on this budget before it routes asks or submits anything further; reads do not consume turnCap. Only when the budget is spent does the run end PENDING (pending-resume row).", + "softPendingDrain": { + "maxWaitMs": 600000, + "intervalMs": 5000, + "maxReads": 40 + }, "reaperHeartbeatTtlSec": 600, "errorSummaryMaxChars": 200, "paths": { diff --git a/lib/loop.mjs b/lib/loop.mjs index 53e83da..d4334f4 100644 --- a/lib/loop.mjs +++ b/lib/loop.mjs @@ -18,9 +18,28 @@ // viewRca } // PENDING → { status, turnId, threadId } (soft-pending, resumable) // NEEDS_INFO → { status, questions/asks/suggestions } (verbatim — the loop needs them) +// +// Soft-PENDING is NOT an agent verdict. It is the tfaRcaTurn util abandoning its +// own in-call poll at POLL_MAX_WAIT_MS (90s) while the agent keeps working +// server-side — observed routinely, e.g. a first turn that finalized NEEDS_INFO +// at 104s. So a PENDING is DRAINED here: read that same turnId via +// `readTurn` (the getTfaTurnResult MCP tool) until it lands a real agent status, +// and only then route asks and submit the next message. Re-submitting on a +// PENDING instead would stack a second turn on one still in flight. import { routeAsks } from "./routing.mjs"; +// Drain budget for one soft-PENDING. Bounded so a wedged turn can never hang the +// batch — on exhaustion the loop still ends `PENDING` (resumable via the CSV's +// `pending-resume` row), which is the old behaviour as a floor, not a default. +const DEFAULT_DRAIN = { maxWaitMs: 600_000, intervalMs: 5_000, maxReads: 40 }; + +const defaultSleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// A real agent verdict, i.e. anything the drain is allowed to stop on. +const isAgentStatus = (s) => + s === "RESOLVED" || s === "NEEDS_INFO" || s === "BLOCKED"; + function unavailableBlock(gap) { const what = gap?.ask?.what ?? ""; return [ @@ -31,18 +50,56 @@ function unavailableBlock(gap) { ].join("\n"); } +// drainSoftPending reads one in-flight turn to a real agent status. +// +// Reads do NOT consume the turn cap — a drain is the SAME turn being read again, +// not a new turn. `readTurn` is read-only and side-effect free, so the only +// budget that applies is wall clock / read count. +// +// Returns { turn, reads }: `turn` is the landed agent turn, or null if the drain +// budget ran out (caller then ends PENDING, resumable). +async function drainSoftPending({ testRunId, pending, readTurn, sleep, drain }) { + const { maxWaitMs, intervalMs, maxReads } = { ...DEFAULT_DRAIN, ...(drain ?? {}) }; + const turnId = pending.turnId; + let reads = 0; + + // No turnId → nothing addressable to read; no readTurn → client lacks the + // getTfaTurnResult tool. Either way fall back to reporting it resumable. + if (!turnId || typeof readTurn !== "function") return { turn: null, reads }; + + const started = Date.now(); + while (reads < maxReads && Date.now() - started < maxWaitMs) { + await sleep(intervalMs); + reads++; + let turn; + try { + turn = await readTurn({ testRunId, turnId }); + } catch { + // A failed read is not a verdict — keep reading until the budget is spent. + continue; + } + if (isAgentStatus(turn?.status)) return { turn, reads }; + // still PENDING → the agent is working; read again. + } + return { turn: null, reads }; +} + // runRcaLoop drives one test to a terminal RCA_OUTPUT object. // // submit({ testRunId, message, threadId, turnId }) → Promise<turn> (tfaRcaTurn shape) +// readTurn({ testRunId, turnId }) → Promise<turn> (getTfaTurnResult shape) // gather(routedGatherEntry) → Promise<string> (one digest block) export async function runRcaLoop({ testRunId, firstMessage = "", submit, + readTurn, config = {}, manifest = {}, gather = async () => "", turnCap = config?.turnCap ?? 6, + drain = config?.softPendingDrain, + sleep = defaultSleep, }) { if (testRunId == null || Number.isNaN(Number(testRunId))) { return { @@ -85,15 +142,41 @@ export async function runRcaLoop({ while (true) { turns++; - const turn = await submit({ testRunId, message, threadId, turnId }); + let turn = await submit({ testRunId, message, threadId, turnId }); threadId = turn.threadId ?? threadId; - if (turn.status === "RESOLVED") return out("RESOLVED", turn); + // Soft-PENDING → the in-call poll capped out, not a verdict. Read the SAME + // turnId to a real status BEFORE routing asks or submitting anything else. if (turn.status === "PENDING") { turnId = turn.turnId ?? turnId; - return out("PENDING", turn, "soft-pending"); + const drained = await drainSoftPending({ + testRunId, + pending: turn, + readTurn, + sleep, + drain, + }); + if (!drained.turn) { + return out( + "PENDING", + turn, + `soft-pending: still working after ${drained.reads} read(s)`, + ); + } + turn = drained.turn; + threadId = turn.threadId ?? threadId; + // Landed. This was never a new turn, so `turns` is unchanged and the + // resume handle is spent — later submits go by threadId alone. + turnId = undefined; } + if (turn.status === "RESOLVED") return out("RESOLVED", turn); + // BLOCKED is a terminal agent verdict: TFA cannot proceed. It carries no + // asks, so treating it as NEEDS_INFO would resubmit empty messages to the + // cap. Reported as PENDING (the output contract's non-resolved value) with + // the reason in the note. + if (turn.status === "BLOCKED") return out("PENDING", turn, "blocked"); + // NEEDS_INFO. Check the turn-cap BEFORE gathering — evidence assembled on a // turn we will never submit is wasted work (and a side-effecting gather() // would run for nothing). @@ -125,3 +208,14 @@ export function replaySubmit(turns) { return turn; }; } + +// Replay helper for tests: a readTurn() that yields recorded getTfaTurnResult +// reads in order (typically N × PENDING then the landed agent turn). +export function replayRead(reads) { + let i = 0; + return async () => { + const read = reads[Math.min(i, reads.length - 1)]; + i++; + return read; + }; +} diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 140b2a9..e979813 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -223,6 +223,12 @@ Live `threadId`/`turnId` resume the prior thread; dead threads re-run from pending. Resuming a run does **not** reopen the gate — no new questions. (In-session only — cross-session durability is deferred.) +A `pending-resume` row now means the coordinator's **soft-PENDING drain budget +was spent** (`softPendingDrain`), not merely that a turn ran past 90s — the +common case is drained in-flight and never reaches the CSV. Resume reads such a +row's `turnId` with `getTfaTurnResult` **before** submitting anything new on the +thread. + ## Hard rules - Exactly one gate. At most one consolidated question, at gate close. **After @@ -230,6 +236,9 @@ pending. Resuming a run does **not** reopen the gate — no new questions. - An invalid/absent connector is a recorded gap, never a blocker. - Headless + missing build id → end immediately. Headless never asks. - Never call `tfaRcaTurn` from this skill — always via the `ai-tfa-coordinator`. +- A soft-`PENDING` is never an answer: it must be drained with + `getTfaTurnResult(testRunId, turnId)` before any further submit on that thread. + Only a spent drain budget may end a test `PENDING`. - Every failed test must end terminal in the CSV — partial-first, no abort-on-one-failure. - Never gather `test_logs` — TFA owns logs. - Never render/write a local RCA report — glimpse table + `triggerRcaReport` + diff --git a/tests/conformance.test.mjs b/tests/conformance.test.mjs index 3dde60b..33ccbbb 100644 --- a/tests/conformance.test.mjs +++ b/tests/conformance.test.mjs @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; -import { runRcaLoop, replaySubmit } from "../lib/loop.mjs"; +import { runRcaLoop, replaySubmit, replayRead } from "../lib/loop.mjs"; const here = dirname(fileURLToPath(import.meta.url)); const load = (name) => @@ -22,6 +22,9 @@ const GITHUB_AVAILABLE = { github: { available: true, via: "gh" } }; // A coordinator gather() stub: returns a one-line digest block. const gather = async (g) => `ASK: ${g.ask.what}\nTYPE: ${g.evidenceType}\nFOUND: yes\nSUMMARY: stub`; +// Drain tests inject a no-op sleep so the 5s read interval costs no wall clock. +const noSleep = async () => {}; + test("resolved fixture: NEEDS_INFO → evidence → RESOLVED, trimmed glimpse captured, test_logs skipped", async () => { const fx = load("resolved.json"); const result = await runRcaLoop({ @@ -44,7 +47,9 @@ test("resolved fixture: NEEDS_INFO → evidence → RESOLVED, trimmed glimpse ca assert.equal(result.threadId, "thr-39"); }); -test("pending fixture: soft-PENDING (trimmed: status/threadId/turnId) ends with turnId, no re-poll", async () => { +test("pending fixture: soft-PENDING with NO getTfaTurnResult tool → ends resumable, never resubmits", async () => { + // Client lacks readTurn: the drain is impossible, so the old floor applies — + // report it resumable rather than busy-waiting through tfaRcaTurn resubmits. const fx = load("pending.json"); let calls = 0; const counting = async (args) => { @@ -55,11 +60,136 @@ test("pending fixture: soft-PENDING (trimmed: status/threadId/turnId) ends with testRunId: fx.testRunId, submit: counting, config: CONFIG, + sleep: noSleep, }); assert.equal(result.status, "PENDING"); assert.equal(result.turnId, "turn-81-1"); assert.equal(result.threadId, "thr-81"); - assert.equal(calls, 1); // ended immediately, did not poll again + assert.equal(calls, 1); // one submit, no resubmit +}); + +test("soft-PENDING is DRAINED via getTfaTurnResult before the next submit, not reported", async () => { + // The real 3840238857 case: turn 1 finalized NEEDS_INFO at 104s, past the tool's + // 90s in-call cap, so submit() handed back a soft PENDING. Reading the same + // turnId lands the NEEDS_INFO the agent had already committed to. + const fx = load("soft-pending-drain.json"); + const submits = []; + const submit = replaySubmit(fx.turns); + const read = replayRead(fx.reads); + let reads = 0; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + firstMessage: "Initiating collaborative RCA", + submit: async (args) => { + submits.push(args); + return submit(args); + }, + readTurn: async (args) => { + reads++; + return read(args); + }, + config: CONFIG, + manifest: GITHUB_AVAILABLE, + gather, + sleep: noSleep, + }); + + assert.equal(result.status, "RESOLVED"); + assert.equal(reads, 3); // PENDING, PENDING, then the landed NEEDS_INFO + assert.equal(submits.length, 2); // turn 1, then the evidence turn — no submit mid-flight + assert.equal(result.turns_used, 2); // 3 reads did NOT consume the turn cap + assert.deepEqual(result.asks_fulfilled, ["product_code"]); + assert.deepEqual(result.asks_skipped, ["test_logs"]); // TFA owns logs, even post-drain + assert.match(result.root_cause, /#current-url/); +}); + +test("drain reads the SAME turnId, and stops resubmitting it once landed", async () => { + const fx = load("soft-pending-drain.json"); + const readArgs = []; + const submits = []; + const submit = replaySubmit(fx.turns); + const read = replayRead(fx.reads); + await runRcaLoop({ + testRunId: fx.testRunId, + submit: async (args) => { + submits.push(args); + return submit(args); + }, + readTurn: async (args) => { + readArgs.push(args); + return read(args); + }, + config: CONFIG, + manifest: GITHUB_AVAILABLE, + gather, + sleep: noSleep, + }); + // Every read targets the turnId the soft-PENDING handed back. + for (const a of readArgs) { + assert.equal(a.turnId, "c2e1a6fd-2243-4f93-bc69-62f298db062c"); + assert.equal(String(a.testRunId), "3840238857"); + } + // The spent resume handle is dropped: the follow-up submit rides threadId only. + assert.equal(submits[1].turnId, undefined); + assert.equal(submits[1].threadId, "chat:3840238857"); +}); + +test("drain budget is bounded: a wedged turn ends PENDING instead of hanging the batch", async () => { + const fx = load("soft-pending-drain.json"); + let reads = 0; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit([fx.turns[0]]), // always soft-PENDING + readTurn: async () => { + reads++; + return { status: "PENDING", turnId: "c2e1a6fd-2243-4f93-bc69-62f298db062c" }; + }, + config: { ...CONFIG, softPendingDrain: { maxWaitMs: 60_000, intervalMs: 1, maxReads: 4 } }, + sleep: noSleep, + }); + assert.equal(result.status, "PENDING"); + assert.equal(reads, 4); // capped by maxReads, never unbounded + assert.match(result.root_cause, /soft-pending: still working after 4 read\(s\)/); + assert.equal(result.turnId, "c2e1a6fd-2243-4f93-bc69-62f298db062c"); // still resumable +}); + +test("a failed read is not a verdict — the drain keeps reading and still lands", async () => { + const fx = load("soft-pending-drain.json"); + const landed = fx.reads[2]; + let reads = 0; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit(fx.turns), + readTurn: async () => { + reads++; + if (reads === 1) throw new Error("transient 502 from o11y"); + return landed; + }, + config: CONFIG, + manifest: GITHUB_AVAILABLE, + gather, + sleep: noSleep, + }); + assert.equal(result.status, "RESOLVED"); + assert.equal(reads, 2); +}); + +test("BLOCKED surfaced by a drain is terminal — no empty resubmits to the turn cap", async () => { + const fx = load("soft-pending-drain.json"); + let submits = 0; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: async (args) => { + submits++; + return replaySubmit(fx.turns)(args); + }, + readTurn: async () => ({ status: "BLOCKED", threadId: "chat:3840238857" }), + config: CONFIG, + sleep: noSleep, + }); + assert.equal(result.status, "PENDING"); + assert.equal(result.root_cause, "blocked"); + assert.equal(submits, 1); // BLOCKED carries no asks; never resubmitted }); test("turn-cap fixture: ends PENDING(turn-cap) at the cap, never a 7th submit", async () => { diff --git a/tests/fixtures/recorded-turns/soft-pending-drain.json b/tests/fixtures/recorded-turns/soft-pending-drain.json new file mode 100644 index 0000000..35a7b41 --- /dev/null +++ b/tests/fixtures/recorded-turns/soft-pending-drain.json @@ -0,0 +1,50 @@ +{ + "name": "soft-PENDING drained via getTfaTurnResult — recorded from test run 3840238857, whose first turn finalized NEEDS_INFO at 104s (past the tool's 90s in-call poll cap), so tfaRcaTurn returned a soft PENDING while the agent was already committed", + "testRunId": 3840238857, + "turns": [ + { + "status": "PENDING", + "threadId": "chat:3840238857", + "turnId": "c2e1a6fd-2243-4f93-bc69-62f298db062c" + }, + { + "status": "RESOLVED", + "confidence": "medium", + "threadId": "chat:3840238857", + "glimpse": { + "root_cause": "#current-url was removed from html-tags.html, so the url-normalisation script threw before healing ran", + "failure_type": "product_regression", + "related_prs": ["#8814"] + }, + "viewRca": "https://automation.browserstack.com/dashboard/v2/builds/x/tests/3840238857" + } + ], + "reads": [ + { "status": "PENDING", "threadId": "chat:3840238857", "turnId": "c2e1a6fd-2243-4f93-bc69-62f298db062c" }, + { "status": "PENDING", "threadId": "chat:3840238857", "turnId": "c2e1a6fd-2243-4f93-bc69-62f298db062c" }, + { + "status": "NEEDS_INFO", + "confidence": "medium", + "threadId": "chat:3840238857", + "questions": [ + "Was the `#current-url` element recently removed or renamed on the `htmlforms` / `html-tags.html` page?" + ], + "asks": [ + { + "what": "The source code for the page html-tags.html and any recent changes to it.", + "why": "Logs show repeated \"Cannot set properties of null (setting 'textContent')\" on #current-url.", + "evidenceType": "product_code", + "priority": "high" + }, + { + "what": "The run's terminal logs.", + "why": "To confirm the JS error ordering.", + "evidenceType": "test_logs", + "priority": "low" + } + ], + "suggestions": [], + "hypotheses": [] + } + ] +} diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index 6cffbb9..f79e7a2 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -61,6 +61,7 @@ const shared = [ `Build-level evidence (pre-computed once, reuse — do not re-fetch): ${JSON.stringify(ctx.buildEvidence ?? {})}`, `Autonomous run — on an evidence gap with no valid connector, report "unavailable" back to TFA (NEVER prompt a user). Best-effort finalize.`, `PRODUCT_BUG / application-bug mandate: hunt the culprit PR via the github connector (deploy timeline vs last-pass window, changed paths vs failure signature) and feed the PR link(s) to TFA so related_prs populates. No PR after digging to the turn cap → state explicitly "no culprit PR identified after <what was searched>" so the CSV row records the gap.`, + `Soft-PENDING is NOT an answer: tfaRcaTurn abandons its in-call poll at 90s while TFA keeps working. On status PENDING, call getTfaTurnResult(testRunId, turnId) FIRST and keep reading on the softPendingDrain budget (every 5s, <=40 reads / <=10min) until the status is RESOLVED / NEEDS_INFO / BLOCKED, then continue the loop. Reads do NOT count against the turn cap. Never submit a new message onto a turn still in flight. Only a fully spent drain budget ends the test PENDING.`, `Persist eagerly to the CSV: claim your row before turn 1, flip it on terminal (lib/csv-state.mjs).`, ].join("\n"); From a1f0e96e451c8b174cc44a9ab4bbb9355b91b86e Mon Sep 17 00:00:00 2001 From: harshit-bstack <harshit.m@browserstack.com> Date: Wed, 29 Jul 2026 12:18:51 +0530 Subject: [PATCH 36/47] Skill Update --- agents/ai-tfa-coordinator.md | 53 +++++++++++++++++-- skills/rca-build/SKILL.md | 98 ++++++++++++++++++++++++++++++++++-- workflows/rca-batch.mjs | 25 +++++++-- 3 files changed, 167 insertions(+), 9 deletions(-) diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 6806b15..c69efcf 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -64,9 +64,42 @@ read-only and has no side effects, so a read is always safe to repeat. ## Operating principles -1. **Logs by TFA — the core contract.** Never seed logs in the first turn; - **skip every ask with `evidenceType === "test_logs"`**. Never fetch, paste, or - digest log content. Logs are TFA's job. +0. **Mandatory connector sweeps — proactive, never ask-gated.** This is about + **server-side app logs** (e.g. `kubectl logs` on the llmservice/ + testcasegeneration pods) — a different thing from the `test_logs` evidence + type in Principle 1 below, which is the test's own execution log (SDK + trace/driver output) and stays exclusively TFA's. There is no conflict + between the two: Principle 1 forbids you from ever fetching `test_logs`; + Principle 0 requires you to fetch **app logs**, a connector-owned capability + distinct from `test_logs`, whenever the connector declares it compulsory. + Before submitting turn 1, check every capability the manifest marks + `available` for a declared compulsory check in its connector skill (read the + skill's `SKILL.md` — look for language like "COMPULSORY", "not conditional", + "not a fallback", "every RCA that touches this skill MUST..."). If a + capability declares one (e.g. `nl2steps-infra`'s "kubectl app-log check is + COMPULSORY"), **run it now, on turn 1, unconditionally** and fold its + evidence block (`KUBECTL_APP_LOGS:` or the skill's equivalent) into the + first-turn message — do **not** wait for a `NEEDS_INFO` ask whose + `evidenceType` names that capability. **TFA's `evidenceType` label is not a + reliable gate for this**: TFA may tag a deploy/infra-shaped question + `product_code` (observed: "was there a recent deploy of `llmservice`?" + tagged `product_code`, never routed to infra) — a compulsory check must run + regardless of how TFA labels its asks, or it never runs at all. Re-run only + once per test (cache the result across turns/asks on the same thread); a + later ask for the same evidenceType is satisfied from the cached sweep, not + re-fetched. **A connector skill may also declare a minimum call-count budget** + (e.g. `nl2steps-infra`'s "at least 5 separate `kubectl` invocations per RCA + turn" — a latency-instrumentation baseline, not busywork: it exists so the + infra/k8s path has enough call volume to measure latency against, matching + the github connector's naturally higher call count). Honor a declared + minimum exactly — never satisfy it with fewer, batched/combined, or cached + calls; each counted call must be a real, individually useful invocation. +1. **Logs by TFA — the core contract.** Never seed **test execution logs** + (`test_logs` evidenceType — SDK trace, driver output, screenshots) in the + first turn; **skip every ask with `evidenceType === "test_logs"`**. Never + fetch, paste, or digest test-log content. That log is TFA's job — this rule + does not cover the app/infra logs Principle 0 requires; those are a + separate, connector-owned evidence type and are never `test_logs`. 2. **Read-only.** Every gather mechanism is read-only. Never write to a repo, cluster, ticket, or the run. Produce a block and stop. 3. **Turn-cap** = `turnCap` from `config/rca.config.json` (default 6). If the cap @@ -136,6 +169,12 @@ re-fetch per test. Never fabricate a PR when the github capability is unavailabl Suspect PR(s): <related_prs>. Confirm against THIS test's logs." (NO logs) - error_digest present → "Error: <title + endpoint>" (NO logs, NO threadId) - neither → "Initiating collaborative RCA for test run <id>." +0.5. MANDATORY CONNECTOR SWEEPS (Operating Principle 0): for every `available` + capability, check its connector skill for a declared compulsory check. Run + any that apply NOW — before turn 1, regardless of pre_seed/error_digest + content — and append each one's evidence block to the DIGEST. Record what + ran in `mandatory_checks` for the final RCA_OUTPUT. This step runs exactly + once per test (cache across turns); do not re-run on a later matching ask. 1. SUBMIT turn 1: tfaRcaTurn(testRunId=<id>, message=<digest>). Capture threadId. turns_used = 1. (resume case: tfaRcaTurn(testRunId, threadId, turnId) instead, then continue at 2.) 2. CLASSIFY result.status: @@ -218,6 +257,14 @@ RCA_OUTPUT_START ## turns_used <integer 1..turnCap> +## mandatory_checks +- <capability>: ran (<M> calls) — <one-line evidence summary, e.g. "kubectl: ran (5 calls) — clean (window=t±2m, 2 pods)"> +- <capability>: ran (<M> calls) — <N matched lines — one-line digest> +- <capability>: not-applicable — <capability declares no compulsory check> +"none" only if no available capability declares any compulsory check. If the +connector declares a minimum call-count budget, `<M>` must be >= that minimum — +report the actual count run, not the minimum itself, so a shortfall is visible. + ## asks_fulfilled - <evidenceType> # every non-test_logs type fulfilled; "none" if empty diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index e979813..e63e623 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -44,7 +44,53 @@ pass. The gate has two parts; both run before any RCA work starts. ### Part A — connector discovery + validation -Enumerate every connector relevant to test RCA: +**Step 0 — enumerate connector-shaped skills FIRST (before probing raw MCP tools).** +Run: + +```bash +ls .claude/skills/ ~/.claude/skills/ 2>/dev/null +``` + +For each `SKILL.md` found, open it and look for a **Capability declaration** +block (or a `capability: <name>` line in the frontmatter/body). Any skill that +declares `capability: github | infra | logs | metrics | other` **IS** the +connector for that capability and MUST be added to the manifest — it +**SUPERSEDES** the raw MCP tool for that capability because it carries +product-specific routing (repo map, cluster/namespace, branch conventions, +falsification protocol) the raw tool does not. Record the skill name in the +manifest entry (e.g. `github: valid, via: gh (skill=nl2steps-github)`). Skipping +this step is the failure mode where the orchestrator dispatches coordinators +that grep the wrong repos on the wrong branch. + +**Disambiguating by product (nudge / one-question rule).** Connector skills are +product-scoped — a workspace may hold none, one, or several product families +(e.g. `nl2steps-*`, `o11y-*`, `tcm-*`, whatever the user has). After the `ls`, +pick the *product family* whose connector skills apply to THIS build: + +- **Zero families found** → **nudge the user in the gate summary**: + "No connector-shaped skills found under `.claude/skills/` — proceeding with + raw MCP tools only; culprit-PR attribution will be best-effort against + workspace `git remote` guesses. Add a `<product>-github` / `<product>-infra` + skill for higher-fidelity routing." Then proceed with raw connectors. **Do + NOT block.** +- **Exactly one family** → use it. No question. +- **Multiple families** (`nl2steps-*` AND `o11y-*` AND `tcm-*` …) → try to + disambiguate WITHOUT asking: + 1. Match the build's project / build name (from `getBuildId` metadata or + the invocation args) against each family's SKILL.md description / product + hints — if one family matches unambiguously, use it. + 2. Match the discovered failure signatures (from Step 2's `listTestIds` if it + has already run, else defer this to a re-visit after discovery) against + each family's declared file paths / error patterns — if one family owns + the failure surface, use it. + If both signals leave the choice ambiguous, this earns the **one + consolidated gate question** (Part B rules apply): fold it into the same + question as any other non-assumable field, e.g. *"Multiple product families + found (`nl2steps`, `o11y`, `tcm`); build/failure signatures don't uniquely + pick one — which family owns this build's failures?"* Headless: pick the + first alphabetically and record the ambiguity as a gap. + +Then enumerate every connector relevant to test RCA: - from `config/rca.config.json` → `evidenceRouting`: **github** (product_code/deploy/ci), **infra** (whatever runtime the user has — k8s, @@ -63,6 +109,31 @@ Enumerate every connector relevant to test RCA: | metrics | a metrics skill/MCP tool actually listed in the session | | other | best-effort; default `absent` | +**Scope validation — run the SCOPE PROBES declared by each connector skill.** +The base probe above (`gh auth status`, `kubectl version`, …) only confirms the +raw tool works. It does not confirm the *concrete targets* a coordinator will +touch — specific repos, branches, clusters, namespaces, indices — are actually +reachable. That is the connector SKILL's job: each connector skill MUST +declare, in its `Capability declaration` section, a `Scope probes:` list +naming what to check and how. This orchestrator's contract is generic: + +1. For every connector skill added to the manifest in Step 0, read its + `Scope probes:` list. +2. Run each probe verbatim. +3. Record every target's result in the manifest entry — passes go into a + resolved-scope field (e.g. `repos_validated: [...]`, `namespace: ok`), + failures go into a per-target gap (e.g. `<target>: 404 not_accessible`). +4. A per-target failure is a scoped gap, not a connector-wide failure — the + connector stays `valid` for the targets that did pass. + +Coordinators can then act freely inside the resolved scope and must fail +closed outside it. This closes the failure mode where a coordinator degrades +to `unavailable` because the orchestrator didn't confirm the specific target. + +Skills that don't declare `Scope probes:` degrade to a manifest-time warning +("scope probes missing — coordinator may over-degrade"). Do not invent +product-specific probes here. + Output the **validated capability manifest**: `connector → valid | invalid | absent` (`lib/routing.mjs` → `buildManifest`; `valid` maps to `available: true`). An `invalid` or `absent` connector is a **recorded gap** — @@ -167,9 +238,14 @@ reuse it, do not re-discover): ## Step 5 — fan-out (fully autonomous) -Drive the cluster work-list, **`concurrency` (default 5) at a time**: +Drive the cluster work-list, **`concurrency` (default 50) at a time**: representatives deep, siblings one-turn-confirm. Eagerly persist to the CSV/WAL -(claim → heartbeat → flip) so the run is resumable. +(claim → heartbeat → flip) so the run is resumable. On the Claude Code / +Workflow-tool path, this is a soft target only — the Workflow runtime hard-caps +actual concurrent `agent()` calls at `min(16, cpu cores - 2)` regardless of +this config value; excess work queues and runs as slots free up rather than +running 50-wide. The sequential harness / manual subagent-dispatch path has no +such ceiling and will honor `concurrency` literally. - Claude Code → run the dynamic workflow `workflows/rca-batch.mjs` (script-orchestrated; gap → "unavailable" back to TFA → best-effort finalize). @@ -182,6 +258,15 @@ coordinator that dies becomes a recorded `failed` row — one stuck test never sinks the batch (partial-first). No path ever prompts the user (the gate is closed). +**Coordinator prompts MUST name every connector-shaped skill on the manifest.** +Each dispatch prompt lists, per capability, the resolved connector skill from +Gate Part A Step 0 — e.g. *"Use `nl2steps-github` for every product_code / +deploy / ci ask (canonical repos + branch live in the skill; do NOT grep other +repos). Use `nl2steps-infra` for every infra ask."* A coordinator prompt that +omits a manifest-listed connector skill — and that therefore lets the +coordinator infer repos from workspace `git remote` or cwd — is a bug: the +coordinator will land plausible-but-wrong PR attributions on adjacent repos. + **Application bugs need a culprit PR.** Whenever a test's RCA classifies as PRODUCT_BUG / application bug, the coordinator MUST hunt the culprit PR via the github connector (deploy timeline vs last-pass window, changed paths vs failure @@ -245,3 +330,10 @@ thread. the Test Observability UI link only. - A PRODUCT_BUG RCA without a GitHub PR link is incomplete — dig until the turn cap, else state what was searched and record the gap. +- A connector skill's own compulsory mandate (e.g. `nl2steps-infra`'s "kubectl + app-log check is COMPULSORY") is honored **proactively on turn 1** — never + gated on TFA naming that evidenceType in an ask. TFA is observed to mislabel + deploy/infra-shaped questions as `product_code`, so ask-routing alone cannot + be trusted to trigger a compulsory check; the coordinator runs it unconditionally + (`agents/ai-tfa-coordinator.md` Operating Principle 0) and records it under + `mandatory_checks` in the RCA_OUTPUT. diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index f79e7a2..806e344 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -49,6 +49,7 @@ const RCA_SCHEMA = { asks_skipped: { type: "array", items: { type: "string" } }, asks_unavailable: { type: "array", items: { type: "string" } }, cluster_id: { type: "string" }, + mandatory_checks: { type: "array", items: { type: "string" } }, }, additionalProperties: true, }; @@ -63,8 +64,23 @@ const shared = [ `PRODUCT_BUG / application-bug mandate: hunt the culprit PR via the github connector (deploy timeline vs last-pass window, changed paths vs failure signature) and feed the PR link(s) to TFA so related_prs populates. No PR after digging to the turn cap → state explicitly "no culprit PR identified after <what was searched>" so the CSV row records the gap.`, `Soft-PENDING is NOT an answer: tfaRcaTurn abandons its in-call poll at 90s while TFA keeps working. On status PENDING, call getTfaTurnResult(testRunId, turnId) FIRST and keep reading on the softPendingDrain budget (every 5s, <=40 reads / <=10min) until the status is RESOLVED / NEEDS_INFO / BLOCKED, then continue the loop. Reads do NOT count against the turn cap. Never submit a new message onto a turn still in flight. Only a fully spent drain budget ends the test PENDING.`, `Persist eagerly to the CSV: claim your row before turn 1, flip it on terminal (lib/csv-state.mjs).`, + `MANDATORY CONNECTOR SWEEPS (Operating Principle 0, ai-tfa-coordinator.md): before turn 1, check every available capability's connector skill for a declared compulsory check (e.g. nl2steps-infra's "kubectl app-log check is COMPULSORY — not conditional, not a fallback"). Run any that apply NOW, unconditionally, and fold the evidence block into the turn-1 message. Do NOT wait for a NEEDS_INFO ask naming that evidenceType — TFA has been observed to label deploy/infra-shaped questions "product_code", so ask-routing alone will never trigger it. Record what ran under mandatory_checks in the RCA_OUTPUT.`, + `MINIMUM CALL BUDGET: nl2steps-infra requires AT LEAST 5 separate real kubectl invocations per RCA turn that touches it (deploy state, pod-discovery x2 as SEPARATE calls, log-sweep x2 minimum — see the skill's "Minimum call budget" section). This is a latency-instrumentation baseline so the k8s path has comparable call volume to the github connector, not busywork — never satisfy it by combining calls, batching selectors, or reusing a cached result. Report the actual count run in mandatory_checks (e.g. "kubectl: ran (5 calls) — ...").`, ].join("\n"); +function resumeLine(row) { + if (!row?.threadId || !row?.turnId) return null; + return [ + `RESUME (do not start a new thread): this test already has an in-flight thread`, + `threadId=${row.threadId} turnId=${row.turnId}.`, + `Call getTfaTurnResult(testRunId, turnId) FIRST to read its current state`, + `(drain any soft-PENDING per the softPendingDrain budget) before submitting`, + `anything further — reuse this threadId for every follow-up on this test.`, + row.last_evidence_digest ? `Prior evidence already gathered (reuse, don't re-fetch): ${row.last_evidence_digest}` : null, + row.root_cause ? `Prior attempt note: ${row.root_cause}` : null, + ].filter(Boolean).join("\n"); +} + function repPrompt(cluster) { const r = cluster.representative; return [ @@ -72,9 +88,10 @@ function repPrompt(cluster) { `Run the FULL collaborative RCA loop for the representative test.`, `testRunId=${r.testRunId} testName=${r.testName ?? ""}`, `error_digest: ${r.error_summary ?? "(none)"}`, + resumeLine(r), shared, `Return the structured RCA_OUTPUT for this test.`, - ].join("\n"); + ].filter(Boolean).join("\n"); } function siblingPrompt(sibling, repResult, cluster) { @@ -87,9 +104,10 @@ function siblingPrompt(sibling, repResult, cluster) { `If TFA confirms in one turn → done. If it does NOT (NEEDS_INFO), fall back to the full loop — never blindly inherit.`, `testRunId=${sibling.testRunId} testName=${sibling.testName ?? ""}`, `error_digest: ${sibling.error_summary ?? "(none)"}`, + resumeLine(sibling), shared, `Return the structured RCA_OUTPUT for this test.`, - ].join("\n"); + ].filter(Boolean).join("\n"); } log(`Batch: ${clusters.length} cluster(s) over build ${ctx.buildId ?? "?"}`); @@ -97,7 +115,8 @@ log(`Batch: ${clusters.length} cluster(s) over build ${ctx.buildId ?? "?"}`); // Pipeline: each cluster flows representative → siblings independently (no barrier // between stages), so a small cluster's siblings confirm while a big cluster's // representative is still looping. Concurrency is bounded by the workflow runtime -// (~min(16, cores-2)); config.concurrency (5) is the intended soft target. +// (~min(16, cores-2)) regardless of config.concurrency (50) — that value is the +// intended soft target/upper bound; the runtime queues anything beyond its own cap. const results = await pipeline( clusters, (cluster) => From 4429fbae5127622d07d8388aa57cee76a1c193d6 Mon Sep 17 00:00:00 2001 From: harshit-bstack <harshit.m@browserstack.com> Date: Wed, 29 Jul 2026 17:46:23 +0530 Subject: [PATCH 37/47] Update --- config/rca.config.json | 3 ++- skills/rca-build/SKILL.md | 42 +++++++++++++++++++++++++++------------ workflows/rca-batch.mjs | 9 ++++++--- 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/config/rca.config.json b/config/rca.config.json index 802f6f4..45c3997 100644 --- a/config/rca.config.json +++ b/config/rca.config.json @@ -1,7 +1,8 @@ { "$comment": "Central config for the /rca-build RCA harness. All formerly-hardcoded product/infra values live here. No kubectl/chitragupta/bifrost literals — connectors are discovered and probe-validated at the gate into the capability manifest (see skills/rca-build/references/evidence-routing.md). No reportFile: the plugin never writes a local RCA report — the full report lives on the Test Observability UI (triggerRcaReport).", "mcpServerName": "bstack", - "concurrency": 5, + "$concurrencyComment": "Max coordinator subagents run in parallel during Step 5 fan-out. HONORED LITERALLY on the default path (direct Agent-tool dispatch of ai-tfa-coordinator subagents — one message, up to `concurrency` tool-use blocks per batch) and by the sequential harness (lib/loop.mjs). Only a SOFT target on the opt-in Workflow-tool path (workflows/rca-batch.mjs), which the Workflow runtime hard-caps at min(16, cpu cores - 2) regardless of this value — that cap is architectural and cannot be raised from this repo. See SKILL.md Step 5.", + "concurrency": 20, "turnCap": 6, "turnMessageMaxChars": 5000, "pollSoftPendingMs": 90000, diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index e63e623..d802293 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -238,20 +238,36 @@ reuse it, do not re-discover): ## Step 5 — fan-out (fully autonomous) -Drive the cluster work-list, **`concurrency` (default 50) at a time**: +Drive the cluster work-list, **`concurrency` (default 20) at a time**: representatives deep, siblings one-turn-confirm. Eagerly persist to the CSV/WAL -(claim → heartbeat → flip) so the run is resumable. On the Claude Code / -Workflow-tool path, this is a soft target only — the Workflow runtime hard-caps -actual concurrent `agent()` calls at `min(16, cpu cores - 2)` regardless of -this config value; excess work queues and runs as slots free up rather than -running 50-wide. The sequential harness / manual subagent-dispatch path has no -such ceiling and will honor `concurrency` literally. - -- Claude Code → run the dynamic workflow `workflows/rca-batch.mjs` - (script-orchestrated; gap → "unavailable" back to TFA → best-effort finalize). -- Hosts without the Workflow runtime → dispatch `tfa-rca:ai-tfa-coordinator` - subagents ≤ `concurrency` at a time, or drive the sequential harness - `lib/loop.mjs` (`runRcaLoop`). Same contract, same no-prompt rule. +(claim → heartbeat → flip) so the run is resumable. + +> **Concurrency comes from `config/rca.config.json` — always read it from +> there, never hardcode.** The default path (direct Agent-tool dispatch) honors +> the JSON value literally: fan out coordinator subagents in batches of +> `concurrency` (one message, up to `concurrency` tool-use blocks per batch). +> The opt-in `workflows/rca-batch.mjs` path is subject to the Workflow tool's +> architectural cap of `min(16, cpu cores - 2)` — on that path `concurrency` +> is a soft upper bound and excess work queues rather than running N-wide. +> If you need literal fan-out, use the default direct-dispatch path. + +- **Default (all hosts, including Claude Code) → direct Agent-tool dispatch.** + Read `concurrency` from `config/rca.config.json` and dispatch + `tfa-rca:ai-tfa-coordinator` subagents in batches of that size (one message, + up to `concurrency` tool-use blocks per batch). This path is **outside the + Workflow runtime**, so the `min(16, cores-2)` ceiling does not apply and the + JSON value is honored literally. Prefer this path whenever the machine's + Workflow cap (`min(16, cores-2)`) would be smaller than the configured + `concurrency` — e.g. an 8-core Mac caps Workflow at 6 while the JSON asks + for 50. +- Opt-in `workflows/rca-batch.mjs` (Claude Code only) → use only when the + Workflow tool's structured `pipeline()`/`parallel()` orchestration, + `resumeFromRunId` resumability, or progress UI is worth the concurrency + trade. On this path `concurrency` is a soft target only — the runtime hard- + caps at `min(16, cores-2)` regardless of the JSON value. +- Hosts without the Workflow runtime and without Agent-tool fan-out → drive + the sequential harness `lib/loop.mjs` (`runRcaLoop`) one test at a time. + Same contract, same no-prompt rule. Subagents/coordinators return compact `RCA_OUTPUT` blocks, never transcripts. A coordinator that dies becomes a recorded `failed` row — one stuck test never diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index 806e344..5bcf506 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -66,6 +66,7 @@ const shared = [ `Persist eagerly to the CSV: claim your row before turn 1, flip it on terminal (lib/csv-state.mjs).`, `MANDATORY CONNECTOR SWEEPS (Operating Principle 0, ai-tfa-coordinator.md): before turn 1, check every available capability's connector skill for a declared compulsory check (e.g. nl2steps-infra's "kubectl app-log check is COMPULSORY — not conditional, not a fallback"). Run any that apply NOW, unconditionally, and fold the evidence block into the turn-1 message. Do NOT wait for a NEEDS_INFO ask naming that evidenceType — TFA has been observed to label deploy/infra-shaped questions "product_code", so ask-routing alone will never trigger it. Record what ran under mandatory_checks in the RCA_OUTPUT.`, `MINIMUM CALL BUDGET: nl2steps-infra requires AT LEAST 5 separate real kubectl invocations per RCA turn that touches it (deploy state, pod-discovery x2 as SEPARATE calls, log-sweep x2 minimum — see the skill's "Minimum call budget" section). This is a latency-instrumentation baseline so the k8s path has comparable call volume to the github connector, not busywork — never satisfy it by combining calls, batching selectors, or reusing a cached result. Report the actual count run in mandatory_checks (e.g. "kubectl: ran (5 calls) — ...").`, + `VICTORIALOGS COMPULSORY CHECK: whenever the grafana MCP is present in the session, nl2steps-infra ALSO requires a VictoriaLogs sweep via mcp__grafana__query_loki_logs (CG Non Prod datasource, uid bfq0jp1cji5moc) on every RCA turn that touches this skill — see the skill's "VictoriaLogs check is ALSO COMPULSORY" section. This runs ALONGSIDE the kubectl sweep, never instead of it. Emit a VICTORIALOGS: block (<N> matched lines / clean / not-onboarded / unavailable) — "clean" requires both a real query_loki_logs call AND a confirmed non-zero coverage check this turn; an unconfirmed/zero-coverage product line reports "not-onboarded", not "clean". If the grafana MCP is absent, "VICTORIALOGS: unavailable — no grafana connector this run" is sufficient. Report what ran under mandatory_checks, distinct from the kubectl entry.`, ].join("\n"); function resumeLine(row) { @@ -114,9 +115,11 @@ log(`Batch: ${clusters.length} cluster(s) over build ${ctx.buildId ?? "?"}`); // Pipeline: each cluster flows representative → siblings independently (no barrier // between stages), so a small cluster's siblings confirm while a big cluster's -// representative is still looping. Concurrency is bounded by the workflow runtime -// (~min(16, cores-2)) regardless of config.concurrency (50) — that value is the -// intended soft target/upper bound; the runtime queues anything beyond its own cap. +// representative is still looping. Concurrency is capped by the Workflow runtime +// at min(16, cores-2) — an architectural limit of the tool itself, not something +// this script or config.concurrency (20, see rca.config.json) can raise. That +// config value is an intended soft target/upper bound on THIS path only; the +// runtime queues anything beyond its own cap regardless of what this file says. const results = await pipeline( clusters, (cluster) => From b733759bf30301f548b362d60891551dbec79ece Mon Sep 17 00:00:00 2001 From: Jaydeep Dave <davejaydeep48@gmail.com> Date: Thu, 30 Jul 2026 16:23:37 +0530 Subject: [PATCH 38/47] fix(rca): revert testing-only hardcoding from v2 latency study MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Strips product-specific artifacts that leaked into the generic plugin from the nl2steps latency-instrumentation runs: the hardcoded "at least 5 kubectl calls" busywork floor and the hardcoded VictoriaLogs Grafana datasource UID in the shared coordinator prompt, plus de-hardcoded remaining nl2steps/o11y/tcm example references in SKILL.md and ai-tfa-coordinator.md down to neutral placeholders. Restores concurrency to 20 (the profiling-backed fleet ceiling) instead of the untested 50. Also gitignores .DS_Store/*.code-workspace to keep local editor artifacts out of the repo. The generic "mandatory connector sweep" mechanism itself (a connector skill may declare a compulsory check, routed by capability) is kept — only the instrumentation-specific numbers/UID were testing artifacts. --- .gitignore | 2 ++ agents/ai-tfa-coordinator.md | 28 ++++++++++------------------ skills/rca-build/SKILL.md | 27 ++++++++++++++------------- workflows/rca-batch.mjs | 4 +--- 4 files changed, 27 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 4c14fb3..433d1ce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ node_modules/ .env +.DS_Store +*.code-workspace # Per-run RCA batch state (the CSV/WAL spine + report) is workspace-local. .rca/ # Planning docs (brainstorm/ideation/plan) stay local — not pushed. diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index c69efcf..a93e9b3 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -76,24 +76,16 @@ read-only and has no side effects, so a read is always safe to repeat. `available` for a declared compulsory check in its connector skill (read the skill's `SKILL.md` — look for language like "COMPULSORY", "not conditional", "not a fallback", "every RCA that touches this skill MUST..."). If a - capability declares one (e.g. `nl2steps-infra`'s "kubectl app-log check is - COMPULSORY"), **run it now, on turn 1, unconditionally** and fold its - evidence block (`KUBECTL_APP_LOGS:` or the skill's equivalent) into the - first-turn message — do **not** wait for a `NEEDS_INFO` ask whose - `evidenceType` names that capability. **TFA's `evidenceType` label is not a - reliable gate for this**: TFA may tag a deploy/infra-shaped question - `product_code` (observed: "was there a recent deploy of `llmservice`?" - tagged `product_code`, never routed to infra) — a compulsory check must run - regardless of how TFA labels its asks, or it never runs at all. Re-run only - once per test (cache the result across turns/asks on the same thread); a - later ask for the same evidenceType is satisfied from the cached sweep, not - re-fetched. **A connector skill may also declare a minimum call-count budget** - (e.g. `nl2steps-infra`'s "at least 5 separate `kubectl` invocations per RCA - turn" — a latency-instrumentation baseline, not busywork: it exists so the - infra/k8s path has enough call volume to measure latency against, matching - the github connector's naturally higher call count). Honor a declared - minimum exactly — never satisfy it with fewer, batched/combined, or cached - calls; each counted call must be a real, individually useful invocation. + capability declares one, **run it now, on turn 1, unconditionally** and fold + its evidence block into the first-turn message — do **not** wait for a + `NEEDS_INFO` ask whose `evidenceType` names that capability. **TFA's + `evidenceType` label is not a reliable gate for this**: TFA may tag a + deploy/infra-shaped question `product_code` (observed: "was there a recent + deploy of `llmservice`?" tagged `product_code`, never routed to infra) — a + compulsory check must run regardless of how TFA labels its asks, or it + never runs at all. Re-run only once per test (cache the result across + turns/asks on the same thread); a later ask for the same evidenceType is + satisfied from the cached sweep, not re-fetched. 1. **Logs by TFA — the core contract.** Never seed **test execution logs** (`test_logs` evidenceType — SDK trace, driver output, screenshots) in the first turn; **skip every ask with `evidenceType === "test_logs"`**. Never diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index d802293..2966bcc 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -58,13 +58,13 @@ connector for that capability and MUST be added to the manifest — it **SUPERSEDES** the raw MCP tool for that capability because it carries product-specific routing (repo map, cluster/namespace, branch conventions, falsification protocol) the raw tool does not. Record the skill name in the -manifest entry (e.g. `github: valid, via: gh (skill=nl2steps-github)`). Skipping +manifest entry (e.g. `github: valid, via: gh (skill=<product>-github)`). Skipping this step is the failure mode where the orchestrator dispatches coordinators that grep the wrong repos on the wrong branch. **Disambiguating by product (nudge / one-question rule).** Connector skills are product-scoped — a workspace may hold none, one, or several product families -(e.g. `nl2steps-*`, `o11y-*`, `tcm-*`, whatever the user has). After the `ls`, +(e.g. `<product-a>-*`, `<product-b>-*`, whatever the user has). After the `ls`, pick the *product family* whose connector skills apply to THIS build: - **Zero families found** → **nudge the user in the gate summary**: @@ -74,7 +74,7 @@ pick the *product family* whose connector skills apply to THIS build: skill for higher-fidelity routing." Then proceed with raw connectors. **Do NOT block.** - **Exactly one family** → use it. No question. -- **Multiple families** (`nl2steps-*` AND `o11y-*` AND `tcm-*` …) → try to +- **Multiple families** (e.g. `<product-a>-*` AND `<product-b>-*` …) → try to disambiguate WITHOUT asking: 1. Match the build's project / build name (from `getBuildId` metadata or the invocation args) against each family's SKILL.md description / product @@ -86,9 +86,9 @@ pick the *product family* whose connector skills apply to THIS build: If both signals leave the choice ambiguous, this earns the **one consolidated gate question** (Part B rules apply): fold it into the same question as any other non-assumable field, e.g. *"Multiple product families - found (`nl2steps`, `o11y`, `tcm`); build/failure signatures don't uniquely - pick one — which family owns this build's failures?"* Headless: pick the - first alphabetically and record the ambiguity as a gap. + found (`<product-a>`, `<product-b>`); build/failure signatures don't + uniquely pick one — which family owns this build's failures?"* Headless: + pick the first alphabetically and record the ambiguity as a gap. Then enumerate every connector relevant to test RCA: @@ -259,7 +259,7 @@ representatives deep, siblings one-turn-confirm. Eagerly persist to the CSV/WAL JSON value is honored literally. Prefer this path whenever the machine's Workflow cap (`min(16, cores-2)`) would be smaller than the configured `concurrency` — e.g. an 8-core Mac caps Workflow at 6 while the JSON asks - for 50. + for 20. - Opt-in `workflows/rca-batch.mjs` (Claude Code only) → use only when the Workflow tool's structured `pipeline()`/`parallel()` orchestration, `resumeFromRunId` resumability, or progress UI is worth the concurrency @@ -276,10 +276,11 @@ closed). **Coordinator prompts MUST name every connector-shaped skill on the manifest.** Each dispatch prompt lists, per capability, the resolved connector skill from -Gate Part A Step 0 — e.g. *"Use `nl2steps-github` for every product_code / -deploy / ci ask (canonical repos + branch live in the skill; do NOT grep other -repos). Use `nl2steps-infra` for every infra ask."* A coordinator prompt that -omits a manifest-listed connector skill — and that therefore lets the +Gate Part A Step 0 — e.g. *"Use `<resolved-github-skill>` for every +product_code / deploy / ci ask (canonical repos + branch live in the skill; do +NOT grep other repos). Use `<resolved-infra-skill>` for every infra ask."* A +coordinator prompt that omits a manifest-listed connector skill — and that +therefore lets the coordinator infer repos from workspace `git remote` or cwd — is a bug: the coordinator will land plausible-but-wrong PR attributions on adjacent repos. @@ -346,8 +347,8 @@ thread. the Test Observability UI link only. - A PRODUCT_BUG RCA without a GitHub PR link is incomplete — dig until the turn cap, else state what was searched and record the gap. -- A connector skill's own compulsory mandate (e.g. `nl2steps-infra`'s "kubectl - app-log check is COMPULSORY") is honored **proactively on turn 1** — never +- A connector skill's own compulsory mandate (e.g. an infra connector marking + its app-log check "COMPULSORY") is honored **proactively on turn 1** — never gated on TFA naming that evidenceType in an ask. TFA is observed to mislabel deploy/infra-shaped questions as `product_code`, so ask-routing alone cannot be trusted to trigger a compulsory check; the coordinator runs it unconditionally diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index 5bcf506..925b5ac 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -64,9 +64,7 @@ const shared = [ `PRODUCT_BUG / application-bug mandate: hunt the culprit PR via the github connector (deploy timeline vs last-pass window, changed paths vs failure signature) and feed the PR link(s) to TFA so related_prs populates. No PR after digging to the turn cap → state explicitly "no culprit PR identified after <what was searched>" so the CSV row records the gap.`, `Soft-PENDING is NOT an answer: tfaRcaTurn abandons its in-call poll at 90s while TFA keeps working. On status PENDING, call getTfaTurnResult(testRunId, turnId) FIRST and keep reading on the softPendingDrain budget (every 5s, <=40 reads / <=10min) until the status is RESOLVED / NEEDS_INFO / BLOCKED, then continue the loop. Reads do NOT count against the turn cap. Never submit a new message onto a turn still in flight. Only a fully spent drain budget ends the test PENDING.`, `Persist eagerly to the CSV: claim your row before turn 1, flip it on terminal (lib/csv-state.mjs).`, - `MANDATORY CONNECTOR SWEEPS (Operating Principle 0, ai-tfa-coordinator.md): before turn 1, check every available capability's connector skill for a declared compulsory check (e.g. nl2steps-infra's "kubectl app-log check is COMPULSORY — not conditional, not a fallback"). Run any that apply NOW, unconditionally, and fold the evidence block into the turn-1 message. Do NOT wait for a NEEDS_INFO ask naming that evidenceType — TFA has been observed to label deploy/infra-shaped questions "product_code", so ask-routing alone will never trigger it. Record what ran under mandatory_checks in the RCA_OUTPUT.`, - `MINIMUM CALL BUDGET: nl2steps-infra requires AT LEAST 5 separate real kubectl invocations per RCA turn that touches it (deploy state, pod-discovery x2 as SEPARATE calls, log-sweep x2 minimum — see the skill's "Minimum call budget" section). This is a latency-instrumentation baseline so the k8s path has comparable call volume to the github connector, not busywork — never satisfy it by combining calls, batching selectors, or reusing a cached result. Report the actual count run in mandatory_checks (e.g. "kubectl: ran (5 calls) — ...").`, - `VICTORIALOGS COMPULSORY CHECK: whenever the grafana MCP is present in the session, nl2steps-infra ALSO requires a VictoriaLogs sweep via mcp__grafana__query_loki_logs (CG Non Prod datasource, uid bfq0jp1cji5moc) on every RCA turn that touches this skill — see the skill's "VictoriaLogs check is ALSO COMPULSORY" section. This runs ALONGSIDE the kubectl sweep, never instead of it. Emit a VICTORIALOGS: block (<N> matched lines / clean / not-onboarded / unavailable) — "clean" requires both a real query_loki_logs call AND a confirmed non-zero coverage check this turn; an unconfirmed/zero-coverage product line reports "not-onboarded", not "clean". If the grafana MCP is absent, "VICTORIALOGS: unavailable — no grafana connector this run" is sufficient. Report what ran under mandatory_checks, distinct from the kubectl entry.`, + `MANDATORY CONNECTOR SWEEPS (Operating Principle 0, ai-tfa-coordinator.md): before turn 1, check every available capability's connector skill for a declared compulsory check (a connector skill may mark a check "COMPULSORY — not conditional, not a fallback"). Run any that apply NOW, unconditionally, and fold the evidence block into the turn-1 message. Do NOT wait for a NEEDS_INFO ask naming that evidenceType — TFA has been observed to label deploy/infra-shaped questions "product_code", so ask-routing alone will never trigger it. Record what ran under mandatory_checks in the RCA_OUTPUT.`, ].join("\n"); function resumeLine(row) { From 45a1ba88f15ea7a9bac2076f4e1010ca5e5a5b9d Mon Sep 17 00:00:00 2001 From: Jaydeep Dave <davejaydeep48@gmail.com> Date: Thu, 30 Jul 2026 16:32:59 +0530 Subject: [PATCH 39/47] fix(rca): drop the mandatory pre-turn-1 connector sweep entirely Removes the "proactive, never ask-gated" mandatory connector sweep mechanism (Operating Principle 0 + loop step 0.5 in ai-tfa-coordinator.md, the shared-prompt reminder in rca-batch.mjs, and the mandatory_checks field in the RCA_OUTPUT schema/contract and SKILL.md's closing bullet). Forcing evidence gathering into turn 1 regardless of whether TFA actually asked for it bloats every coordinator's first message and burns tokens for evidence that may not be needed. Evidence gathering is now purely reactive to TFA's own NEEDS_INFO asks, routed by capability as before. --- agents/ai-tfa-coordinator.md | 45 +++--------------------------------- skills/rca-build/SKILL.md | 7 ------ workflows/rca-batch.mjs | 2 -- 3 files changed, 3 insertions(+), 51 deletions(-) diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index a93e9b3..43f12b7 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -64,34 +64,9 @@ read-only and has no side effects, so a read is always safe to repeat. ## Operating principles -0. **Mandatory connector sweeps — proactive, never ask-gated.** This is about - **server-side app logs** (e.g. `kubectl logs` on the llmservice/ - testcasegeneration pods) — a different thing from the `test_logs` evidence - type in Principle 1 below, which is the test's own execution log (SDK - trace/driver output) and stays exclusively TFA's. There is no conflict - between the two: Principle 1 forbids you from ever fetching `test_logs`; - Principle 0 requires you to fetch **app logs**, a connector-owned capability - distinct from `test_logs`, whenever the connector declares it compulsory. - Before submitting turn 1, check every capability the manifest marks - `available` for a declared compulsory check in its connector skill (read the - skill's `SKILL.md` — look for language like "COMPULSORY", "not conditional", - "not a fallback", "every RCA that touches this skill MUST..."). If a - capability declares one, **run it now, on turn 1, unconditionally** and fold - its evidence block into the first-turn message — do **not** wait for a - `NEEDS_INFO` ask whose `evidenceType` names that capability. **TFA's - `evidenceType` label is not a reliable gate for this**: TFA may tag a - deploy/infra-shaped question `product_code` (observed: "was there a recent - deploy of `llmservice`?" tagged `product_code`, never routed to infra) — a - compulsory check must run regardless of how TFA labels its asks, or it - never runs at all. Re-run only once per test (cache the result across - turns/asks on the same thread); a later ask for the same evidenceType is - satisfied from the cached sweep, not re-fetched. -1. **Logs by TFA — the core contract.** Never seed **test execution logs** - (`test_logs` evidenceType — SDK trace, driver output, screenshots) in the - first turn; **skip every ask with `evidenceType === "test_logs"`**. Never - fetch, paste, or digest test-log content. That log is TFA's job — this rule - does not cover the app/infra logs Principle 0 requires; those are a - separate, connector-owned evidence type and are never `test_logs`. +1. **Logs by TFA — the core contract.** Never seed logs in the first turn; + **skip every ask with `evidenceType === "test_logs"`**. Never fetch, paste, + or digest log content. Logs are TFA's job. 2. **Read-only.** Every gather mechanism is read-only. Never write to a repo, cluster, ticket, or the run. Produce a block and stop. 3. **Turn-cap** = `turnCap` from `config/rca.config.json` (default 6). If the cap @@ -161,12 +136,6 @@ re-fetch per test. Never fabricate a PR when the github capability is unavailabl Suspect PR(s): <related_prs>. Confirm against THIS test's logs." (NO logs) - error_digest present → "Error: <title + endpoint>" (NO logs, NO threadId) - neither → "Initiating collaborative RCA for test run <id>." -0.5. MANDATORY CONNECTOR SWEEPS (Operating Principle 0): for every `available` - capability, check its connector skill for a declared compulsory check. Run - any that apply NOW — before turn 1, regardless of pre_seed/error_digest - content — and append each one's evidence block to the DIGEST. Record what - ran in `mandatory_checks` for the final RCA_OUTPUT. This step runs exactly - once per test (cache across turns); do not re-run on a later matching ask. 1. SUBMIT turn 1: tfaRcaTurn(testRunId=<id>, message=<digest>). Capture threadId. turns_used = 1. (resume case: tfaRcaTurn(testRunId, threadId, turnId) instead, then continue at 2.) 2. CLASSIFY result.status: @@ -249,14 +218,6 @@ RCA_OUTPUT_START ## turns_used <integer 1..turnCap> -## mandatory_checks -- <capability>: ran (<M> calls) — <one-line evidence summary, e.g. "kubectl: ran (5 calls) — clean (window=t±2m, 2 pods)"> -- <capability>: ran (<M> calls) — <N matched lines — one-line digest> -- <capability>: not-applicable — <capability declares no compulsory check> -"none" only if no available capability declares any compulsory check. If the -connector declares a minimum call-count budget, `<M>` must be >= that minimum — -report the actual count run, not the minimum itself, so a shortfall is visible. - ## asks_fulfilled - <evidenceType> # every non-test_logs type fulfilled; "none" if empty diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 2966bcc..581e834 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -347,10 +347,3 @@ thread. the Test Observability UI link only. - A PRODUCT_BUG RCA without a GitHub PR link is incomplete — dig until the turn cap, else state what was searched and record the gap. -- A connector skill's own compulsory mandate (e.g. an infra connector marking - its app-log check "COMPULSORY") is honored **proactively on turn 1** — never - gated on TFA naming that evidenceType in an ask. TFA is observed to mislabel - deploy/infra-shaped questions as `product_code`, so ask-routing alone cannot - be trusted to trigger a compulsory check; the coordinator runs it unconditionally - (`agents/ai-tfa-coordinator.md` Operating Principle 0) and records it under - `mandatory_checks` in the RCA_OUTPUT. diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index 925b5ac..b3317c1 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -49,7 +49,6 @@ const RCA_SCHEMA = { asks_skipped: { type: "array", items: { type: "string" } }, asks_unavailable: { type: "array", items: { type: "string" } }, cluster_id: { type: "string" }, - mandatory_checks: { type: "array", items: { type: "string" } }, }, additionalProperties: true, }; @@ -64,7 +63,6 @@ const shared = [ `PRODUCT_BUG / application-bug mandate: hunt the culprit PR via the github connector (deploy timeline vs last-pass window, changed paths vs failure signature) and feed the PR link(s) to TFA so related_prs populates. No PR after digging to the turn cap → state explicitly "no culprit PR identified after <what was searched>" so the CSV row records the gap.`, `Soft-PENDING is NOT an answer: tfaRcaTurn abandons its in-call poll at 90s while TFA keeps working. On status PENDING, call getTfaTurnResult(testRunId, turnId) FIRST and keep reading on the softPendingDrain budget (every 5s, <=40 reads / <=10min) until the status is RESOLVED / NEEDS_INFO / BLOCKED, then continue the loop. Reads do NOT count against the turn cap. Never submit a new message onto a turn still in flight. Only a fully spent drain budget ends the test PENDING.`, `Persist eagerly to the CSV: claim your row before turn 1, flip it on terminal (lib/csv-state.mjs).`, - `MANDATORY CONNECTOR SWEEPS (Operating Principle 0, ai-tfa-coordinator.md): before turn 1, check every available capability's connector skill for a declared compulsory check (a connector skill may mark a check "COMPULSORY — not conditional, not a fallback"). Run any that apply NOW, unconditionally, and fold the evidence block into the turn-1 message. Do NOT wait for a NEEDS_INFO ask naming that evidenceType — TFA has been observed to label deploy/infra-shaped questions "product_code", so ask-routing alone will never trigger it. Record what ran under mandatory_checks in the RCA_OUTPUT.`, ].join("\n"); function resumeLine(row) { From 94cb38dda2aaa3763d36175056ccec1864642093 Mon Sep 17 00:00:00 2001 From: harshit-bstack <harshit.m@browserstack.com> Date: Thu, 30 Jul 2026 20:44:08 +0530 Subject: [PATCH 40/47] feat(rca): pre-fetch build-level evidence once, share via file across all coordinators Each dispatched ai-tfa-coordinator previously re-ran its own turn-1 PR-window search and kubectl/VictoriaLogs sweep independently, even though that evidence is a property of the build, not of any one test. Add lib/evidence-file.mjs (a build-scoped JSON artifact, keyed by repo and workload, mirroring csv-state.mjs's path convention) so the orchestrator gathers this once and every representative/sibling reads it before falling back to a live call. Validated against a real build: two re-run tests landed the same root causes (including the same culprit PR) at 69-70% fewer tokens and 90-94% fewer tool calls than the original per-coordinator sweep. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> --- agents/ai-tfa-coordinator.md | 37 ++++++++- lib/evidence-file.mjs | 150 +++++++++++++++++++++++++++++++++++ skills/rca-build/SKILL.md | 70 +++++++++++++--- tests/evidence-file.test.mjs | 140 ++++++++++++++++++++++++++++++++ workflows/rca-batch.mjs | 13 ++- 5 files changed, 395 insertions(+), 15 deletions(-) create mode 100644 lib/evidence-file.mjs create mode 100644 tests/evidence-file.test.mjs diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 43f12b7..67e1bf6 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -39,6 +39,12 @@ it names no `kubectl` / `chitragupta` / `bifrost`; it routes by *capability*. - `resume` — optional `{ threadId, turnId }` from a prior PENDING run. - `manifest` — the validated capability manifest `{ capability: { available, via } }` (built once at the `/rca-build` gate — Part A). +- `evidenceFile` — optional. Absolute path to the build-level pre-fetch + artifact (`lib/evidence-file.mjs`, `/rca-build` Step 4). Holds pre-digested + `github` (PR window, deploy state) and `logs`/`infra` (app-side sweep) + evidence, keyed by repo and by workload — gathered ONCE by the orchestrator + for every repo/workload this build's failures implicate. `Read` it before + any live gather call (see Operating Principle 0). If `testRunId` is missing or not parseable as an integer, emit a `failed` `RCA_OUTPUT` block with `root_cause: "no testRunId provided"` and stop — do not @@ -64,6 +70,20 @@ read-only and has no side effects, so a read is always safe to repeat. ## Operating principles +0. **Read the pre-fetch first.** If `evidenceFile` is present, `Read` it + before considering any live github/infra/logs call. It holds build-level + evidence (PR window, deploy state, log sweeps) already gathered once by the + orchestrator for the repos/workloads this build's failures implicate. Use + what it covers directly — its entries are already digest-shaped (an + `evidence-block.md`-style `block`); paste, don't re-digest. Only make a live + call for what it does NOT cover: a repo/workload it doesn't name, an entry + marked with a `gap` (a `gap` is never coverage — treat it exactly as if the + file didn't have that entry), or evidence genuinely specific to this one + test that a build-wide sweep window could plausibly have missed. For a + sibling (`pre_seed` present): the file's data about YOUR OWN test's + workload/repo is real evidence, not inheritance — reading it is fine; the + CONFIRMATION judgment against it must still be independently yours (see + principle 1 and the sibling note in "The loop"). 1. **Logs by TFA — the core contract.** Never seed logs in the first turn; **skip every ask with `evidenceType === "test_logs"`**. Never fetch, paste, or digest log content. Logs are TFA's job. @@ -125,8 +145,10 @@ each candidate suspect **try to disprove it** (path overlap? shipped before the failure window? behind an OFF flag?). Feed both supporting *and* disconfirming evidence back as a structured suspect packet; only `verdict: supported` suspects belong in `related_prs`. Reuse the pre-computed build-level evidence — do not -re-fetch per test. Never fabricate a PR when the github capability is unavailable -— emit an `unavailable` block. +re-fetch per test (the `evidenceFile`'s `github` section, if present and not +`gap`-marked for this repo; otherwise the live github connector). Never +fabricate a PR when the github capability is unavailable — emit an +`unavailable` block. ## The loop @@ -153,8 +175,13 @@ re-fetch per test. Never fabricate a PR when the github capability is unavailabl 3. ROUTE the asks (read references/evidence-routing.md; route via lib/routing.mjs): For each ask, high → medium → low: skip → record in asks_skipped, emit nothing. - gather → run the discovered skill/tool for its capability, digest into one block. - Record evidenceType in asks_fulfilled (dedupe). + gather → FIRST check `evidenceFile` (if present) for this ask's scope — + repo for a github ask, workload for an infra/logs ask. Covered + (present, `gap` falsy) → paste its `block` straight in, no + re-digesting, no live call. Not named in the file, or its + entry has a `gap`, or no `evidenceFile` at all → run the + discovered skill/tool live, exactly as before. + Digest into one block. Record evidenceType in asks_fulfilled (dedupe). gap → emit an `unavailable` block (record in asks_unavailable). NEVER prompt. PRODUCT_BUG in play + no supported PR yet → widen the github hunt this turn. Concatenate per-ask blocks into the next-turn MESSAGE (respect size caps). @@ -244,6 +271,8 @@ Notes: ## Hard limits +- **Never** treat a `gap`-marked `evidenceFile` entry as coverage — a `gap` + means attempt a live call exactly as if the file didn't have that entry. - **Never** prompt, ask, or wait on a user — the gate is closed; gaps degrade to `unavailable`. - **Never** fulfill or seed a `test_logs` ask — TFA owns logs. - **Never** exceed `turnCap` `tfaRcaTurn` calls in one run. diff --git a/lib/evidence-file.mjs b/lib/evidence-file.mjs new file mode 100644 index 0000000..f6eb55c --- /dev/null +++ b/lib/evidence-file.mjs @@ -0,0 +1,150 @@ +// Build-level evidence pre-fetch artifact (see docs/plan: evidence-file). PR +// windows, deploy state, and app-log sweeps are properties of the BUILD, not +// of any one test — a naive batch re-fetches them once per dispatched +// coordinator. This module persists them to a file ONCE so every +// representative and sibling `ai-tfa-coordinator` dispatch can `Read` the same +// artifact instead of re-running the same `gh`/`kubectl`/`grafana` calls. +// +// Layered under `lib/evidence-cache.mjs`, not merged with it: the cache is an +// in-process, function-scoped Map that dedups compute *within* the +// orchestrator's own Step 4 pass; this module is what makes that result +// visible to OTHER processes (the independently-dispatched coordinator +// subagents, which share no memory with the orchestrator or each other). +// +// Path convention mirrors `lib/csv-state.mjs`'s `csvPathFor` exactly: the +// build id is in the filename (no cross-build collisions) and the default +// directory is OS temp (`<tmpdir>/bstack-rca/`), so a build's evidence file +// sits right next to its state CSV. `stateDir` overrides the directory only. +// +// Invariant: this file NEVER carries `test_logs` content. `logs` is keyed by +// *workload* (an infra/pod concept), populated only via the `infra`/`logs` +// capability — TFA remains the sole owner of test-side SDK/driver/session +// logs, which structurally cannot land here. +// +// Timestamps are passed in as `nowMs` (never read from the clock here), same +// discipline as `csv-state.mjs`, so this stays usable from the Workflow-tool +// sandbox (which forbids `Date.now()`). + +import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { tmpdir } from "node:os"; + +/** Canonical evidence-file path for one build's run. Same two invariants as + * `csvPathFor`: build id in the filename; OS temp by default; `stateDir` + * overrides the directory only. */ +export function evidencePathFor(buildId, stateDir = "") { + const safe = String(buildId ?? "").replace(/[^A-Za-z0-9._-]/g, "_") || "unknown-build"; + const dir = stateDir && String(stateDir).trim() !== "" ? String(stateDir) : join(tmpdir(), "bstack-rca"); + return join(dir, `rca-evidence.${safe}.json`); +} + +export function emptyEvidenceFile(buildId, nowMs) { + return { + buildId: String(buildId ?? ""), + generatedAtMs: nowMs, + baseline: null, + suspectWindow: null, + github: {}, + logs: {}, + coverage: { reposCovered: [], reposGapped: [], workloadsCovered: [], workloadsGapped: [] }, + }; +} + +/** Read-only; never throws on a missing file — a coordinator (or the + * orchestrator, before Step 4 has run) always gets a well-shaped, empty-covered + * result rather than an exception. Graceful degradation is the point: an + * absent/partial file just means every ask falls back to a live gather. */ +export function readEvidenceFile(filePath) { + if (!existsSync(filePath)) return emptyEvidenceFile("unknown-build", 0); + return JSON.parse(readFileSync(filePath, "utf8")); +} + +export function writeEvidenceFile(filePath, doc) { + const dir = dirname(filePath); + if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true }); + writeFileSync(filePath, JSON.stringify(doc, null, 2), "utf8"); +} + +function loadOrInit(filePath, nowMs) { + if (!existsSync(filePath)) return emptyEvidenceFile("unknown-build", nowMs); + return readEvidenceFile(filePath); +} + +/** Idempotent: creates the file with the given `buildId` if it doesn't exist + * yet, otherwise leaves an existing file untouched (never clobbers prior + * writes on a resume). Call this FIRST, before any `set*` call, so `buildId` + * is recorded correctly — the `set*` functions below fall back to + * `"unknown-build"` only as a safety net if called without this. */ +export function initEvidenceFile(filePath, buildId, nowMs) { + if (existsSync(filePath)) return readEvidenceFile(filePath); + const doc = emptyEvidenceFile(buildId, nowMs); + writeEvidenceFile(filePath, doc); + return doc; +} + +/** Records the diff/PR-window baseline once, at the start of the Step 4 pass. + * `baseline` is `resolveBaseline(...)`'s return value from `evidence-cache.mjs` + * (`{ref, isFallback}`); `suspectWindow` is whatever shape the active connector + * skill uses to describe the window (e.g. `{reposRequested, startedAt}`). */ +export function setBaseline(filePath, baseline, suspectWindow, nowMs) { + const doc = loadOrInit(filePath, nowMs); + doc.baseline = baseline; + doc.suspectWindow = suspectWindow; + doc.generatedAtMs = nowMs; + writeEvidenceFile(filePath, doc); + return doc; +} + +/** Read-modify-write merge into `doc.github[repo]`. `entry` shape: + * `{ deployState: {block, gap}, prsInWindow: [{pr, files, block, verdict}], + * gap }` — `gap` (top-level, on the repo entry) is what `recomputeCoverage` + * checks; a repo present with a non-null `gap` is NOT counted as covered. + * Only ever touches this one repo's key — every other repo/workload already + * in the file is untouched. */ +export function setGithubEvidence(filePath, repo, entry, nowMs) { + const doc = loadOrInit(filePath, nowMs); + doc.github[repo] = entry; + doc.generatedAtMs = nowMs; + writeEvidenceFile(filePath, doc); + return doc; +} + +/** Read-modify-write merge into `doc.logs[workload]`. `entry` shape: + * `{ clusterIds, kubectlSweep: {block, gap}, victorialogs: {block, gap}, gap }`. + * Same no-clobber guarantee as `setGithubEvidence`, keyed by workload instead + * of repo. */ +export function setLogsEvidence(filePath, workload, entry, nowMs) { + const doc = loadOrInit(filePath, nowMs); + doc.logs[workload] = entry; + doc.generatedAtMs = nowMs; + writeEvidenceFile(filePath, doc); + return doc; +} + +// A requested item is "covered" only if it is present AND its own `gap` field +// is falsy. Presence with a `gap` is a recorded, deliberate miss — not +// coverage — so a coordinator (or this function) never mistakes "we looked +// and couldn't get it" for "we have it." +function isCovered(doc, section, key) { + const entry = doc[section]?.[key]; + return Boolean(entry) && !entry.gap; +} + +/** Derives `doc.coverage` from exactly which requested repos/workloads have a + * gap-free entry, and persists it. `requested = {repos:[...], workloads:[...]}` + * — normally the Gate Part A scope-probe-validated repo list and the union of + * workloads every cluster's representative implicates (see SKILL.md Step 4). */ +export function recomputeCoverage(filePath, requested, nowMs) { + const doc = loadOrInit(filePath, nowMs); + const repos = requested?.repos ?? []; + const workloads = requested?.workloads ?? []; + doc.coverage = { + reposCovered: repos.filter((r) => isCovered(doc, "github", r)), + reposGapped: repos.filter((r) => !isCovered(doc, "github", r)), + workloadsCovered: workloads.filter((w) => isCovered(doc, "logs", w)), + workloadsGapped: workloads.filter((w) => !isCovered(doc, "logs", w)), + }; + doc.generatedAtMs = nowMs; + writeEvidenceFile(filePath, doc); + return doc.coverage; +} diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 581e834..9fc9dca 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -226,15 +226,54 @@ Each cluster gets one **representative** (full multi-turn loop) and `N−1` the expensive evidence hunt to O(distinct causes) while every test still lands a per-test RCA. Singleton clusters are just plain per-test loops. -## Step 4 — build-evidence pre-compute (see references/evidence-routing.md) - -Once, before fan-out (the capability manifest already exists from Gate Part A — -reuse it, do not re-discover): - -- **Build-level evidence** — compute the last-green→this-build delta (diff, - deploy timeline, suspect-PR window) **once** and pre-seed every coordinator - with the same grounded window. Cache by `(repo, commit-range)`. No "last green" - baseline (never-green suite) → fall back to a configured baseline ref and log it. +## Step 4 — build-evidence pre-fetch (see references/evidence-routing.md and lib/evidence-file.mjs) + +Once, after clustering (Step 3) and before fan-out — the capability manifest +already exists from Gate Part A, reuse it, do not re-discover. This step +replaces each coordinator's own turn-1 evidence sweep with ONE pre-fetch: +it does not remove the requirement that turn-1 evidence exists, only *who +gathers it*. + +1. Resolve the evidence-file path: `lib/evidence-file.mjs` → + `evidencePathFor(buildId, config.paths.stateDir)` — + `<tmpdir>/bstack-rca/rca-evidence.<buildId>.json`, alongside the state CSV. + `initEvidenceFile(path, buildId, nowMs)`. +2. **Scope the pre-fetch to the full union, never a single guess:** + - **Repos** — every repo in Gate Part A's scope-probe-validated + `repos_validated` list (e.g. a VRT-lane build validates `frontend` + + `railsApp`; an nl2steps build validates `misc-services` + `ai-sdk-node`). + - **Workloads** — the union of workloads every cluster's **representative** + implicates, via the active connector skill's failure-signature→workload + routing table (never one workload guessed from the first failing test). +3. For each repo: run the connector skill's PR-window-search + deploy-state + recipes **once**, using `lib/evidence-cache.mjs`'s `compute(repo, range, + evidenceType, fn)` to dedupe if two steps need the same `(repo, range)`. + Digest the result into the `evidence-block.md` shape, then persist via + `setGithubEvidence(path, repo, {deployState, prsInWindow, gap}, nowMs)`. + A repo the connector can't reach records `{gap: "<reason>"}` — never blocks + the rest of the pre-fetch. +4. For each workload: run the connector skill's compulsory kubectl + + VictoriaLogs sweep **once**, scoped to the build's own failure window + (`started_at`..`finished_at`, not "now" — see the connector skill's window + guidance). Persist via `setLogsEvidence(path, workload, + {clusterIds, kubectlSweep, victorialogs, gap}, nowMs)`. +5. `resolveBaseline(lastGreenRef, fallbackRef)` (from `lib/evidence-cache.mjs`) + → `setBaseline(path, baseline, suspectWindow, nowMs)`. No "last green" + baseline (never-green suite) → fall back to a configured baseline ref and + note the weaker grounding — this note travels into the file, not just a + spoken log line, so every coordinator sees it. +6. `recomputeCoverage(path, {repos, workloads}, nowMs)` and declare the + resulting path in the gate summary alongside the capability manifest, so + a human re-reading the run can find it. + +**Size discipline is enforced at write time, not just at submit time.** Every +leaf (`deployState`, each PR, each log sweep) must already be a digested +`block` per `evidence-routing.md`'s caps (`SUMMARY≤400`, `SNIPPET≤20/40 lines`, +link over diff) — never a raw dump. Cap `prsInWindow` to the top ~30 candidates +by path-overlap relevance, not every PR in the window. + +Pass `evidencePathFor(...)`'s path to Step 5's fan-out as `evidenceFilePath` — +every dispatch (representative and sibling) must be told to read it first. ## Step 5 — fan-out (fully autonomous) @@ -284,6 +323,19 @@ therefore lets the coordinator infer repos from workspace `git remote` or cwd — is a bug: the coordinator will land plausible-but-wrong PR attributions on adjacent repos. +**Coordinator prompts MUST also name the Step 4 evidence file.** Every +dispatch prompt (representative and sibling alike) includes the absolute +`evidenceFilePath` from Step 4 with the instruction: *"Read `<path>` (via the +Read tool) before making any live github/infra/logs gather call. It's a +pre-fetch, not a hard dependency — a repo/workload it doesn't name, or marks +with a `gap`, is a genuine gap: fall back to the capability manifest above +exactly as if no file existed."* For a sibling, add: *"The file's data about +your OWN test's workload is real evidence, not inheritance — reading it is +fine. What must stay independent is the CONFIRMATION judgment: never adopt the +representative's verdict just because the file already has the answer in +it."* A dispatch prompt that omits this path forces its coordinator back into +a full independent sweep — exactly the redundancy Step 4 exists to remove. + **Application bugs need a culprit PR.** Whenever a test's RCA classifies as PRODUCT_BUG / application bug, the coordinator MUST hunt the culprit PR via the github connector (deploy timeline vs last-pass window, changed paths vs failure diff --git a/tests/evidence-file.test.mjs b/tests/evidence-file.test.mjs new file mode 100644 index 0000000..a01466a --- /dev/null +++ b/tests/evidence-file.test.mjs @@ -0,0 +1,140 @@ +import { test, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + evidencePathFor, + emptyEvidenceFile, + initEvidenceFile, + readEvidenceFile, + writeEvidenceFile, + setBaseline, + setGithubEvidence, + setLogsEvidence, + recomputeCoverage, +} from "../lib/evidence-file.mjs"; + +let dir; +let file; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "rca-evidence-")); + file = join(dir, "evidence.json"); +}); +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +test("evidencePathFor: build id is in the filename, default dir is OS temp", () => { + const p = evidencePathFor("abc123XYZ"); + assert.ok(p.startsWith(join(tmpdir(), "bstack-rca"))); + assert.ok(p.endsWith("rca-evidence.abc123XYZ.json")); +}); + +test("evidencePathFor: different builds never share a path", () => { + assert.notEqual(evidencePathFor("build-A"), evidencePathFor("build-B")); +}); + +test("evidencePathFor: sanitizes hostile ids and handles empty", () => { + assert.ok(evidencePathFor("../../etc/passwd").endsWith("rca-evidence..._.._etc_passwd.json")); + assert.ok(evidencePathFor("").endsWith("rca-evidence.unknown-build.json")); +}); + +test("evidencePathFor: stateDir override wins over temp", () => { + const p = evidencePathFor("b1", "/ci/artifacts"); + assert.equal(p, join("/ci/artifacts", "rca-evidence.b1.json")); +}); + +test("readEvidenceFile on a missing path returns the empty shape, never throws", () => { + const doc = readEvidenceFile(file); + assert.deepEqual(doc, emptyEvidenceFile("unknown-build", 0)); +}); + +test("initEvidenceFile creates the file with the given buildId", () => { + const doc = initEvidenceFile(file, "build-1", 1000); + assert.equal(doc.buildId, "build-1"); + assert.equal(doc.generatedAtMs, 1000); + assert.deepEqual(readEvidenceFile(file), doc); +}); + +test("initEvidenceFile is idempotent — does not clobber an existing file", () => { + initEvidenceFile(file, "build-1", 1000); + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "x" } }, 2000); + const before = readEvidenceFile(file); + const again = initEvidenceFile(file, "build-1", 9999); + assert.deepEqual(again, before); +}); + +test("setGithubEvidence and setLogsEvidence coexist without clobbering each other", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "a-deploy" } }, 1000); + setLogsEvidence(file, "workload-1", { gap: null, kubectlSweep: { block: "w1-logs" } }, 1000); + const doc = readEvidenceFile(file); + assert.equal(doc.github["org/a"].deployState.block, "a-deploy"); + assert.equal(doc.logs["workload-1"].kubectlSweep.block, "w1-logs"); +}); + +test("setGithubEvidence for a second repo does not disturb the first", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "a" } }, 1000); + setGithubEvidence(file, "org/b", { gap: null, deployState: { block: "b" } }, 1000); + const doc = readEvidenceFile(file); + assert.equal(doc.github["org/a"].deployState.block, "a"); + assert.equal(doc.github["org/b"].deployState.block, "b"); +}); + +test("setGithubEvidence twice for the SAME repo overwrites only that repo", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "old" } }, 1000); + setGithubEvidence(file, "org/b", { gap: null, deployState: { block: "b" } }, 1000); + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "new" } }, 2000); + const doc = readEvidenceFile(file); + assert.equal(doc.github["org/a"].deployState.block, "new"); + assert.equal(doc.github["org/b"].deployState.block, "b"); // untouched +}); + +test("setBaseline records baseline and suspectWindow without touching github/logs", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "a" } }, 1000); + setBaseline(file, { ref: "sha123", isFallback: false }, { reposRequested: ["org/a"] }, 2000); + const doc = readEvidenceFile(file); + assert.deepEqual(doc.baseline, { ref: "sha123", isFallback: false }); + assert.deepEqual(doc.suspectWindow, { reposRequested: ["org/a"] }); + assert.equal(doc.github["org/a"].deployState.block, "a"); // untouched +}); + +test("recomputeCoverage: a covered repo/workload has no gap; a missing one is gapped", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "a" } }, 1000); + setLogsEvidence(file, "w1", { gap: null, kubectlSweep: { block: "w1" } }, 1000); + const coverage = recomputeCoverage( + file, + { repos: ["org/a", "org/b"], workloads: ["w1", "w2"] }, + 2000, + ); + assert.deepEqual(coverage.reposCovered, ["org/a"]); + assert.deepEqual(coverage.reposGapped, ["org/b"]); + assert.deepEqual(coverage.workloadsCovered, ["w1"]); + assert.deepEqual(coverage.workloadsGapped, ["w2"]); +}); + +test("recomputeCoverage: a present entry with a non-null gap is NOT covered", () => { + setGithubEvidence(file, "org/a", { gap: "gh auth failed for this repo" }, 1000); + const coverage = recomputeCoverage(file, { repos: ["org/a"], workloads: [] }, 2000); + assert.deepEqual(coverage.reposCovered, []); + assert.deepEqual(coverage.reposGapped, ["org/a"]); +}); + +test("recomputeCoverage persists onto the file (readable afterwards)", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "a" } }, 1000); + recomputeCoverage(file, { repos: ["org/a"], workloads: [] }, 2000); + const doc = readEvidenceFile(file); + assert.deepEqual(doc.coverage.reposCovered, ["org/a"]); +}); + +test("a block string with newlines and quotes round-trips through JSON unchanged", () => { + const block = 'ASK: did X change?\nTYPE: product_code\nFOUND: yes\nSUMMARY: "quoted" finding\nSNIPPET: line1\nline2'; + setGithubEvidence(file, "org/a", { gap: null, deployState: { block } }, 1000); + const doc = readEvidenceFile(file); + assert.equal(doc.github["org/a"].deployState.block, block); +}); + +test("writeEvidenceFile creates the parent directory if missing", () => { + const nested = join(dir, "nested", "sub", "evidence.json"); + writeEvidenceFile(nested, emptyEvidenceFile("build-1", 0)); + assert.deepEqual(readEvidenceFile(nested).buildId, "build-1"); +}); diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index b3317c1..1624b54 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -23,7 +23,13 @@ export const meta = { // { // csvPath, buildId, // manifest: { capability: { available, via } }, -// buildEvidence: { baselineRef, suspectWindow, ... }, // pre-computed once +// evidenceFilePath, // NEW — lib/evidence-file.mjs artifact for this build +// buildEvidence: { baselineRef, isFallback, suspectWindow, reposCovered, workloadsCovered, gaps }, +// // ^ SHRUNK to a cheap summary/pointer only — the full PR list / log +// // sweeps live in the file at evidenceFilePath, read via each +// // coordinator's own Read tool. Repeating the full detail in every +// // dispatch prompt (as before) is exactly the duplication this file +// // removes. // clusters: [ // { cluster_id, representative: { testRunId, testName, error_summary }, // siblings: [ { testRunId, testName, error_summary } ] } @@ -58,7 +64,9 @@ const clusters = ctx.clusters ?? []; const shared = [ `CSV state file: ${ctx.csvPath}`, `Capability manifest: ${JSON.stringify(ctx.manifest ?? {})}`, - `Build-level evidence (pre-computed once, reuse — do not re-fetch): ${JSON.stringify(ctx.buildEvidence ?? {})}`, + `Pre-fetched build-evidence file — READ THIS FIRST (via the Read tool) before making ANY live github/infra/logs gather call: ${ctx.evidenceFilePath}`, + `Build-evidence summary (full detail is in the file above; this is only a pointer — do not re-fetch what the file already covers): ${JSON.stringify(ctx.buildEvidence ?? {})}`, + `If the file's github/logs sections do not name a repo/workload/ask you need, or record a "gap" for it, that is a genuine gap — fall back to a live gather via the capability manifest above exactly as if no file existed. The file is an optimization, never a hard dependency.`, `Autonomous run — on an evidence gap with no valid connector, report "unavailable" back to TFA (NEVER prompt a user). Best-effort finalize.`, `PRODUCT_BUG / application-bug mandate: hunt the culprit PR via the github connector (deploy timeline vs last-pass window, changed paths vs failure signature) and feed the PR link(s) to TFA so related_prs populates. No PR after digging to the turn cap → state explicitly "no culprit PR identified after <what was searched>" so the CSV row records the gap.`, `Soft-PENDING is NOT an answer: tfaRcaTurn abandons its in-call poll at 90s while TFA keeps working. On status PENDING, call getTfaTurnResult(testRunId, turnId) FIRST and keep reading on the softPendingDrain budget (every 5s, <=40 reads / <=10min) until the status is RESOLVED / NEEDS_INFO / BLOCKED, then continue the loop. Reads do NOT count against the turn cap. Never submit a new message onto a turn still in flight. Only a fully spent drain budget ends the test PENDING.`, @@ -98,6 +106,7 @@ function siblingPrompt(sibling, repResult, cluster) { ` root_cause: ${repResult?.root_cause ?? "(representative did not resolve)"}`, ` related_prs: ${JSON.stringify(repResult?.related_prs ?? [])}`, `State this hypothesis on turn 1 and ask TFA to CONFIRM it against THIS test's own logs.`, + `The pre-fetched evidence file's data about your OWN workload is real evidence about YOUR OWN test — reading it is NOT blind inheritance. What must stay independent is the CONFIRMATION judgment: never adopt the representative's verdict just because the file already has the answer in it.`, `If TFA confirms in one turn → done. If it does NOT (NEEDS_INFO), fall back to the full loop — never blindly inherit.`, `testRunId=${sibling.testRunId} testName=${sibling.testName ?? ""}`, `error_digest: ${sibling.error_summary ?? "(none)"}`, From 4a3912acf93a05d09935faf6c0c6cb7900b01843 Mon Sep 17 00:00:00 2001 From: Jaydeep Dave <davejaydeep48@gmail.com> Date: Fri, 31 Jul 2026 10:59:46 +0530 Subject: [PATCH 41/47] updating prompts --- agents/ai-tfa-coordinator.md | 16 ++++++++++ .../rca-build/references/github-evidence.md | 29 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 67e1bf6..64abace 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -117,6 +117,18 @@ read-only and has no side effects, so a read is always safe to repeat. 8. **Never editorialize.** Report findings (suspect PR, server-side error line), not verdicts. The root cause is TFA's to state on `RESOLVED`; pass its `glimpse` through verbatim. +9. **Field-filter every gather call, always.** Before running any + capability-provided command (`gh`, `kubectl`, or whatever the manifest + resolved to for `github`/`infra`), project down to only the field(s) this + ask needs — `--jq`, `-o custom-columns`, `-o jsonpath`, or a `grep`/`head` + immediately piped. Never run the unfiltered form "just to see the shape" — + an exploratory call costs the same context whether or not its output ends + up in the digest, and a raw repo/commit/pod object typically carries + orders of magnitude more noise (license/URL metadata, multi-hundred-char + signature blocks, unrequested columns) than any evidence ask ever uses. + This governs what enters *your own* context via the tool result — distinct + from principle 6, which governs the digest you send back to TFA. Exact + command templates: `references/github-evidence.md` § Field-filtering. ## Application bugs — the culprit-PR mandate (MANDATORY) @@ -282,6 +294,10 @@ Notes: - **Never** let drain reads consume the turn cap, and never drain past the `softPendingDrain` budget — a wedged turn must not hang the batch. - **Never** dump raw logs, full diffs, or full file contents into a turn message — digest only. +- **Never** run an unfiltered gather call (a bare `gh api ...` with no `--jq`, + `kubectl get ... -o wide`/`-o yaml` when a narrower `-o custom-columns` + answers the ask) — project to the needed field(s) before the call runs, not + by reading past the noise after. - **Never** write to any repo / cluster / ticket / the run — every action is read-only. - **Never** editorialize a cause — pass TFA's `glimpse` through verbatim. - **Never** blindly inherit a representative's cause for a sibling — confirm against its own logs. diff --git a/skills/rca-build/references/github-evidence.md b/skills/rca-build/references/github-evidence.md index 4b2c445..a3d6b3f 100644 --- a/skills/rca-build/references/github-evidence.md +++ b/skills/rca-build/references/github-evidence.md @@ -55,6 +55,35 @@ Scope everything by the failing test's `file_path` + the error summary. The build-level evidence (diff-since-last-green, PR window) is **pre-computed once** and passed in — reuse it; do not re-fetch per test. +## Field-filtering — project before you pull, every call + +The single most common way a gather call wastes context: pulling a full +object when the ask only needs one or two fields from it. This applies to +whichever connector resolved for `github` (most commonly the `gh` CLI today, +or a GitHub MCP tool) — every call should already be filtered to the field(s) +the ask needs, not filtered after the fact by reading past the noise. The +same discipline applies to `infra` gather calls (`kubectl` or whatever the +manifest resolved to), since the failure mode is identical. + +| Need | Don't — pulls the whole object | Do — projects to the field(s) the ask needs | +|---|---|---| +| Repo exists / default branch | `gh api repos/OWNER/REPO` | `gh api repos/OWNER/REPO --jq '.default_branch'` | +| Branch exists on the shipping branch | `gh api repos/OWNER/REPO/branches/BRANCH` | `gh api repos/OWNER/REPO/branches/BRANCH --jq '.name'` | +| Commit history / PR-window search | `gh api "repos/OWNER/REPO/commits?sha=BRANCH&per_page=100"` | add `--jq '[.[] | {sha: .sha[0:8], date: .commit.committer.date, msg: (.commit.message | split("\n")[0])}]'` | +| PR metadata | `gh pr view N --repo OWNER/REPO` (full payload) | `gh pr view N --repo OWNER/REPO --json state,mergedAt,baseRefName,headRefOid,files,author` — `--json` is itself a field allowlist; list only the fields this ask uses | +| Pod / workload listing | `kubectl get pods -n NS -o wide` | `kubectl get pods -n NS -o custom-columns='NAME:.metadata.name,STATUS:.status.phase'` | +| Deploy / image state | `kubectl get deploy -n NS -o yaml` | `kubectl get deploy -n NS -o custom-columns='NAME:.metadata.name,IMAGE:.spec.template.spec.containers[0].image'` | +| Log sweep | a raw `--tail` dump | `kubectl logs POD --since=<window> --tail=2000 \| grep -E '<correlation token>\|ERROR\|Exception'` — filter by the correlation token, never a raw tail | + +**Never run the unfiltered form "to see the shape first."** An exploratory +raw call costs the same context whether or not its output ends up in the +digest — a bare repo or commit object routinely carries license/URL metadata +and a multi-hundred-character signature block that no evidence ask ever +consults. If the exact field path is genuinely unknown, learn the shape from +one throwaway call against a cheap target, then filter every real call from +that point on — never repeat the unfiltered form per repo, per PR, or per +test. + ## Falsification protocol — rule out, don't just rule in For **each** candidate suspect PR, try to **break** the hypothesis: From 1eba07ff8b420a75fe12cc91a33e3817f82643f7 Mon Sep 17 00:00:00 2001 From: harshit-bstack <harshit.m@browserstack.com> Date: Fri, 31 Jul 2026 12:56:44 +0530 Subject: [PATCH 42/47] fix(rca): tighten turnMessageMaxChars to 1000 Self-imposed plugin budget, well under the tfaRcaTurn tool's actual 5000-char hard cap. Retunes the per-block digest caps (SUMMARY, SNIPPET) proportionally so real turn-1 messages fit the new budget instead of relying on ad-hoc truncation. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> --- agents/ai-tfa-coordinator.md | 4 +++- config/rca.config.json | 2 +- skills/rca-build/SKILL.md | 2 +- skills/rca-build/references/evidence-routing.md | 17 +++++++++++------ skills/rca-build/templates/evidence-block.md | 6 +++--- 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 64abace..19c907a 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -110,7 +110,9 @@ read-only and has no side effects, so a read is always safe to repeat. 6. **Digest, don't dump.** Every follow-up `message` carries digested findings (`ask → found → snippet/link`), never raw log tails, full diffs, or full files. Size caps + block shape live in `references/evidence-routing.md` — read it - before fulfilling any ask. The tool caps `message` at 5000 chars. + before fulfilling any ask. The plugin config caps `message` at 1000 chars + (`turnMessageMaxChars` in `config/rca.config.json`); the `tfaRcaTurn` tool + itself would allow up to 5000, but the plugin self-limits to 1000. 7. **Report gaps, don't drop them.** An ask the coordinator cannot fulfill becomes a `not-found` / `unreachable` / `unavailable` block, never a silent omission — and **never a user prompt**. TFA finalizes best-effort with lower confidence. diff --git a/config/rca.config.json b/config/rca.config.json index 45c3997..b316f94 100644 --- a/config/rca.config.json +++ b/config/rca.config.json @@ -4,7 +4,7 @@ "$concurrencyComment": "Max coordinator subagents run in parallel during Step 5 fan-out. HONORED LITERALLY on the default path (direct Agent-tool dispatch of ai-tfa-coordinator subagents — one message, up to `concurrency` tool-use blocks per batch) and by the sequential harness (lib/loop.mjs). Only a SOFT target on the opt-in Workflow-tool path (workflows/rca-batch.mjs), which the Workflow runtime hard-caps at min(16, cpu cores - 2) regardless of this value — that cap is architectural and cannot be raised from this repo. See SKILL.md Step 5.", "concurrency": 20, "turnCap": 6, - "turnMessageMaxChars": 5000, + "turnMessageMaxChars": 1000, "pollSoftPendingMs": 90000, "$softPendingDrainComment": "tfaRcaTurn abandons its in-call poll at pollSoftPendingMs (90s) and returns a soft PENDING while the TFA agent keeps working — turns finalizing past 90s are routine. On a soft PENDING the loop READS the same turnId via getTfaTurnResult on this budget before it routes asks or submits anything further; reads do not consume turnCap. Only when the budget is spent does the run end PENDING (pending-resume row).", "softPendingDrain": { diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 9fc9dca..0f9f348 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -268,7 +268,7 @@ gathers it*. **Size discipline is enforced at write time, not just at submit time.** Every leaf (`deployState`, each PR, each log sweep) must already be a digested -`block` per `evidence-routing.md`'s caps (`SUMMARY≤400`, `SNIPPET≤20/40 lines`, +`block` per `evidence-routing.md`'s caps (`SUMMARY≤80`, `SNIPPET≤4/8 lines`, link over diff) — never a raw dump. Cap `prsInWindow` to the top ~30 candidates by path-overlap relevance, not every PR in the window. diff --git a/skills/rca-build/references/evidence-routing.md b/skills/rca-build/references/evidence-routing.md index 05b4435..9132fef 100644 --- a/skills/rca-build/references/evidence-routing.md +++ b/skills/rca-build/references/evidence-routing.md @@ -89,16 +89,21 @@ and unfulfillable variants) — copy it, don't retype it. Shape: | Field / scope | Soft target | Hard ceiling | On exceed | |---|---|---|---| -| `SUMMARY` | ≤ 300 chars | 400 chars | Tighten to the finding; drop restatement of the ask | -| `SNIPPET` per ask | ≤ 20 lines | 40 lines | Keep the load-bearing lines; replace the rest with `… (N lines elided — see LINK)` | -| Code diff in a `product_code` snippet | ≤ 1 hunk | 3 hunks | Show changed lines + 3 lines context; link the full PR | -| Whole next-turn `message` | ≤ 200 lines | 400 lines (and ≤ `turnMessageMaxChars`) | Drop `low`-priority asks first; keep every `high` ask's block | +| `SUMMARY` | ≤ 60 chars | 80 chars | Tighten to the finding; drop restatement of the ask | +| `SNIPPET` per ask | ≤ 4 lines | 8 lines | Keep the load-bearing lines; replace the rest with `… (N lines elided — see LINK)` | +| Code diff in a `product_code` snippet | ≤ 1 hunk | 2 hunks | Show changed lines only, no context lines; link the full PR | +| Whole next-turn `message` | ≤ 40 lines | 80 lines (and ≤ `turnMessageMaxChars`) | Drop `low`-priority asks first; keep every `high` ask's block | | Asks fulfilled per turn | all `high` + `medium` | — | Defer `low` asks to a later turn rather than truncating a `high` ask | Truncation rule of thumb: **never truncate a `high`-priority ask's block to fit a `low`-priority one.** Drop the low block whole; keep the high block intact. The -whole-message ceiling also honors `turnMessageMaxChars` from -`config/rca.config.json` (the tool caps `message` at 5000 chars). +whole-message ceiling honors `turnMessageMaxChars` from +`config/rca.config.json`, now set to **1000 chars** — a plugin-configured +self-limit, tighter than the underlying tool's actual hard cap (the +`tfaRcaTurn` MCP tool itself allows up to 5000 chars per `message`; the plugin +just chooses not to use all of it). At this budget, expect at most 2-3 ask +blocks per turn before hitting the ceiling — defer lower-priority asks to a +follow-up turn rather than cramming everything into one. ### What never goes in a digest diff --git a/skills/rca-build/templates/evidence-block.md b/skills/rca-build/templates/evidence-block.md index fa0bf49..97b41f7 100644 --- a/skills/rca-build/templates/evidence-block.md +++ b/skills/rca-build/templates/evidence-block.md @@ -6,12 +6,12 @@ the forbidden list: `../references/evidence-routing.md`. Fulfilled ask: ``` -ASK: <verbatim `what` from the TfaAsk, ≤ 120 chars> +ASK: <verbatim `what` from the TfaAsk, ≤ 80 chars> TYPE: <evidenceType> FOUND: <yes | no | partial> -SUMMARY: <1–3 sentences — the finding, in the agent's words. ≤ 400 chars> +SUMMARY: <1 sentence — the finding, in the agent's words. ≤ 80 chars> SNIPPET: - <the load-bearing excerpt only — see size caps. Omit if a LINK fully carries it.> + <the load-bearing excerpt only, ≤ 4 lines — see size caps. Omit if a LINK fully carries it.> LINK: <permalink to the source — PR/commit/log-search/metrics panel/deploy record. Omit if N/A.> ``` From 70ef859f28ef10f6406afd44a2178a927a691d28 Mon Sep 17 00:00:00 2001 From: harshit-bstack <harshit.m@browserstack.com> Date: Fri, 31 Jul 2026 16:08:59 +0530 Subject: [PATCH 43/47] feat(rca): let coordinators write gathered evidence back to the shared file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pre-fetch file was read-only from a coordinator's side — when a live gather filled a gap or went deeper than the pre-fetch had (a full diff instead of a summary, a PR the pre-fetch never named), that extra work died with the coordinator instead of benefiting its own siblings or other clusters sharing the same repo/workload. Add mergeGithubEvidence/mergeLogsEvidence (read-modify-write, dedupe PRs by number, union clusterIds, never drops what a patch doesn't mention) and wire Operating Principle 0 + the culprit-PR mandate + the routing step in ai-tfa-coordinator.md to call them after any live gather. Confirmed on a real prior run (vrt_345): a representative's own deep PR-diff investigation was exactly the kind of work its sibling had no way to reuse under the read-only version of this file. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> --- agents/ai-tfa-coordinator.md | 34 +++++++++++++++-- lib/evidence-file.mjs | 70 +++++++++++++++++++++++++++++++++++ skills/rca-build/SKILL.md | 12 ++++++ tests/evidence-file.test.mjs | 71 ++++++++++++++++++++++++++++++++++++ workflows/rca-batch.mjs | 1 + 5 files changed, 184 insertions(+), 4 deletions(-) diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 19c907a..9a43df1 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -44,7 +44,11 @@ it names no `kubectl` / `chitragupta` / `bifrost`; it routes by *capability*. `github` (PR window, deploy state) and `logs`/`infra` (app-side sweep) evidence, keyed by repo and by workload — gathered ONCE by the orchestrator for every repo/workload this build's failures implicate. `Read` it before - any live gather call (see Operating Principle 0). + any live gather call (see Operating Principle 0) — and treat it as + read-WRITE: a live gather that fills a gap or goes deeper is written back + via `mergeGithubEvidence`/`mergeLogsEvidence` so later dispatches (this + test's own siblings, or another cluster sharing the same repo/workload) + benefit too. If `testRunId` is missing or not parseable as an integer, emit a `failed` `RCA_OUTPUT` block with `root_cause: "no testRunId provided"` and stop — do not @@ -84,6 +88,21 @@ read-only and has no side effects, so a read is always safe to repeat. workload/repo is real evidence, not inheritance — reading it is fine; the CONFIRMATION judgment against it must still be independently yours (see principle 1 and the sibling note in "The loop"). + + **Write back what you gather live.** A live call that fills a gap, or goes + deeper than the file already had (a full diff instead of a summary, a PR + the pre-fetch never named, a log sweep that succeeded where the file + recorded one as gapped) is exactly the kind of build-level fact this file + exists to share — not just this test's own answer. Persist it via + `mergeGithubEvidence(evidenceFilePath, repo, patch, nowMs)` or + `mergeLogsEvidence(evidenceFilePath, workload, patch, nowMs)` + (`lib/evidence-file.mjs`) before finishing this test, so a sibling + dispatched after you (or any other cluster that turns out to share the + same repo/workload) reads the enriched entry instead of re-fetching what + you just fetched. Only write back genuinely new/deeper findings — don't + write back a no-op read of an already-covered entry. This has the same + informal-locking caveat as the CSV: it's a best-effort optimization, not a + correctness dependency, so never block or retry on it. 1. **Logs by TFA — the core contract.** Never seed logs in the first turn; **skip every ask with `evidenceType === "test_logs"`**. Never fetch, paste, or digest log content. Logs are TFA's job. @@ -160,8 +179,12 @@ failure window? behind an OFF flag?). Feed both supporting *and* disconfirming evidence back as a structured suspect packet; only `verdict: supported` suspects belong in `related_prs`. Reuse the pre-computed build-level evidence — do not re-fetch per test (the `evidenceFile`'s `github` section, if present and not -`gap`-marked for this repo; otherwise the live github connector). Never -fabricate a PR when the github capability is unavailable — emit an +`gap`-marked for this repo; otherwise the live github connector). A culprit +hunt often needs to go deeper than the file's summary — a full diff, a +downstream consumer of a changed flag — write that depth back via +`mergeGithubEvidence` once found, so a sibling confirming the same suspect PR +doesn't re-run the same diff/search. Never fabricate a PR when the github +capability is unavailable — emit an `unavailable` block. ## The loop @@ -194,7 +217,10 @@ fabricate a PR when the github capability is unavailable — emit an (present, `gap` falsy) → paste its `block` straight in, no re-digesting, no live call. Not named in the file, or its entry has a `gap`, or no `evidenceFile` at all → run the - discovered skill/tool live, exactly as before. + discovered skill/tool live, exactly as before — THEN write the + result back via `mergeGithubEvidence`/`mergeLogsEvidence` + (Operating Principle 0) so this fills the gap for whoever + reads the file next. Digest into one block. Record evidenceType in asks_fulfilled (dedupe). gap → emit an `unavailable` block (record in asks_unavailable). NEVER prompt. PRODUCT_BUG in play + no supported PR yet → widen the github hunt this turn. diff --git a/lib/evidence-file.mjs b/lib/evidence-file.mjs index f6eb55c..d7487e5 100644 --- a/lib/evidence-file.mjs +++ b/lib/evidence-file.mjs @@ -24,6 +24,21 @@ // Timestamps are passed in as `nowMs` (never read from the clock here), same // discipline as `csv-state.mjs`, so this stays usable from the Workflow-tool // sandbox (which forbids `Date.now()`). +// +// Write-back (mergeGithubEvidence / mergeLogsEvidence): the orchestrator's +// Step 4 pass is not the only writer. A coordinator that had to gather live +// (a repo/PR/workload the pre-fetch didn't cover, or covered only with a +// summary) should write what it found back into this SAME file — a +// representative's deep dive then benefits its own siblings (dispatched +// after it resolves) and any other cluster that turns out to share the same +// repo/workload, without re-fetching. Concurrency caveat, same philosophy as +// `csv-state.mjs`'s "true multi-process locking is out of scope": this is a +// synchronous read-modify-write with no file lock, so two coordinators +// writing to the SAME repo/workload key at truly the same moment can lose an +// update. In practice this is low-risk for the case this exists to serve — +// a sibling is dispatched only after its representative resolves, i.e. +// sequentially, never concurrently with it — and acceptable for the rarer +// case of two parallel representatives happening to touch the same repo. import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs"; import { dirname, join } from "node:path"; @@ -121,6 +136,61 @@ export function setLogsEvidence(filePath, workload, entry, nowMs) { return doc; } +// Read-modify-write, preserving whatever isn't in `patch`. Returns an existing +// repo entry, or a blank one — this is what lets a coordinator enrich a repo +// the pre-fetch never named at all (a brand-new candidate it found live), not +// just one that's already there with a gap. +function findRepoEntry(doc, repo) { + return doc.github[repo] ?? { deployState: null, prsInWindow: [], gap: null }; +} + +function findWorkloadEntry(doc, workload) { + return doc.logs[workload] ?? { clusterIds: [], kubectlSweep: null, victorialogs: null, gap: null }; +} + +/** Write back what a coordinator gathered LIVE for a repo — a deeper + * `deployState` (e.g. the full diff/patch, not just a summary), and/or one or + * more PRs to fold into `prsInWindow` (deduped by `pr`; a PR with a `pr` that + * already exists is REPLACED, since the coordinator's fresh finding is + * presumably deeper than a placeholder). `patch = { deployState?, prsInWindow?, + * gap? }` — omit a field to leave it untouched. Passing `gap: null` clears a + * previously-recorded gap now that real evidence exists. Never removes a PR + * or a field this call doesn't mention. */ +export function mergeGithubEvidence(filePath, repo, patch, nowMs) { + const doc = loadOrInit(filePath, nowMs); + const entry = findRepoEntry(doc, repo); + if (patch.deployState !== undefined) entry.deployState = patch.deployState; + if (Array.isArray(patch.prsInWindow)) { + const byPr = new Map((entry.prsInWindow ?? []).map((p) => [String(p.pr), p])); + for (const pr of patch.prsInWindow) byPr.set(String(pr.pr), pr); + entry.prsInWindow = [...byPr.values()]; + } + if (patch.gap !== undefined) entry.gap = patch.gap; + doc.github[repo] = entry; + doc.generatedAtMs = nowMs; + writeEvidenceFile(filePath, doc); + return entry; +} + +/** Write back what a coordinator gathered LIVE for a workload's logs — same + * merge discipline as `mergeGithubEvidence`. `patch = { kubectlSweep?, + * victorialogs?, clusterIds?, gap? }`; `clusterIds` is unioned, not replaced, + * since more than one cluster can end up sharing a workload over the run. */ +export function mergeLogsEvidence(filePath, workload, patch, nowMs) { + const doc = loadOrInit(filePath, nowMs); + const entry = findWorkloadEntry(doc, workload); + if (patch.kubectlSweep !== undefined) entry.kubectlSweep = patch.kubectlSweep; + if (patch.victorialogs !== undefined) entry.victorialogs = patch.victorialogs; + if (Array.isArray(patch.clusterIds)) { + entry.clusterIds = [...new Set([...(entry.clusterIds ?? []), ...patch.clusterIds])]; + } + if (patch.gap !== undefined) entry.gap = patch.gap; + doc.logs[workload] = entry; + doc.generatedAtMs = nowMs; + writeEvidenceFile(filePath, doc); + return entry; +} + // A requested item is "covered" only if it is present AND its own `gap` field // is falsy. Presence with a `gap` is a recorded, deliberate miss — not // coverage — so a coordinator (or this function) never mistakes "we looked diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 0f9f348..ca77c9b 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -336,6 +336,18 @@ representative's verdict just because the file already has the answer in it."* A dispatch prompt that omits this path forces its coordinator back into a full independent sweep — exactly the redundancy Step 4 exists to remove. +**The file is read-write, not just read-only.** When a coordinator has to +gather live (a genuine gap), tell it to write the result back — +`mergeGithubEvidence`/`mergeLogsEvidence` (`lib/evidence-file.mjs`) — before +finishing, not just answer TFA and move on. A representative's deep dive (a +full diff, a downstream trace, a PR the pre-fetch never named) then benefits +its own siblings and any other cluster sharing the same repo/workload, instead +of every one of them re-running the same live search. This is already baked +into `agents/ai-tfa-coordinator.md`'s Operating Principle 0 for any dispatch +of that agent type — no need to repeat the mechanics in the prompt, just don't +omit `evidenceFilePath` (above), since write-back has nothing to write to +without it. + **Application bugs need a culprit PR.** Whenever a test's RCA classifies as PRODUCT_BUG / application bug, the coordinator MUST hunt the culprit PR via the github connector (deploy timeline vs last-pass window, changed paths vs failure diff --git a/tests/evidence-file.test.mjs b/tests/evidence-file.test.mjs index a01466a..768142b 100644 --- a/tests/evidence-file.test.mjs +++ b/tests/evidence-file.test.mjs @@ -12,6 +12,8 @@ import { setBaseline, setGithubEvidence, setLogsEvidence, + mergeGithubEvidence, + mergeLogsEvidence, recomputeCoverage, } from "../lib/evidence-file.mjs"; @@ -133,6 +135,75 @@ test("a block string with newlines and quotes round-trips through JSON unchanged assert.equal(doc.github["org/a"].deployState.block, block); }); +test("mergeGithubEvidence on a repo the pre-fetch never named creates a fresh entry", () => { + const entry = mergeGithubEvidence(file, "org/new-repo", { + prsInWindow: [{ pr: "#8912", verdict: "supported", block: "found live" }], + }, 1000); + assert.equal(entry.prsInWindow.length, 1); + assert.equal(entry.gap, null); + const doc = readEvidenceFile(file); + assert.equal(doc.github["org/new-repo"].prsInWindow[0].pr, "#8912"); +}); + +test("mergeGithubEvidence appends a new PR without dropping an existing one", () => { + setGithubEvidence(file, "org/a", { + gap: null, + deployState: { block: "a" }, + prsInWindow: [{ pr: "#1", verdict: "not-live" }], + }, 1000); + mergeGithubEvidence(file, "org/a", { + prsInWindow: [{ pr: "#2", verdict: "supported", block: "found live during coordinator's own hunt" }], + }, 2000); + const doc = readEvidenceFile(file); + const prs = doc.github["org/a"].prsInWindow.map((p) => p.pr); + assert.deepEqual(prs.sort(), ["#1", "#2"]); + assert.equal(doc.github["org/a"].deployState.block, "a"); // untouched +}); + +test("mergeGithubEvidence replaces a PR entry with the same pr number (deeper finding wins)", () => { + setGithubEvidence(file, "org/a", { + gap: null, + deployState: { block: "a" }, + prsInWindow: [{ pr: "#9011", verdict: "unassessed", files: null }], + }, 1000); + mergeGithubEvidence(file, "org/a", { + prsInWindow: [{ pr: "#9011", verdict: "supported", files: ["Foo.java"], block: "full diff fetched" }], + }, 2000); + const doc = readEvidenceFile(file); + assert.equal(doc.github["org/a"].prsInWindow.length, 1); + assert.equal(doc.github["org/a"].prsInWindow[0].verdict, "supported"); + assert.deepEqual(doc.github["org/a"].prsInWindow[0].files, ["Foo.java"]); +}); + +test("mergeGithubEvidence with gap:null clears a previously-recorded gap", () => { + setGithubEvidence(file, "org/a", { gap: "gh auth failed" }, 1000); + mergeGithubEvidence(file, "org/a", { gap: null, deployState: { block: "found it after all" } }, 2000); + const doc = readEvidenceFile(file); + assert.equal(doc.github["org/a"].gap, null); +}); + +test("mergeLogsEvidence unions clusterIds instead of replacing them", () => { + setLogsEvidence(file, "w1", { gap: null, clusterIds: ["c-A"], kubectlSweep: { block: "x" } }, 1000); + mergeLogsEvidence(file, "w1", { clusterIds: ["c-B"] }, 2000); + const doc = readEvidenceFile(file); + assert.deepEqual(doc.logs["w1"].clusterIds.sort(), ["c-A", "c-B"]); + assert.equal(doc.logs["w1"].kubectlSweep.block, "x"); // untouched +}); + +test("mergeLogsEvidence upgrades one sub-field without touching the other", () => { + setLogsEvidence(file, "w1", { + gap: null, + kubectlSweep: { gap: "stale pods" }, + victorialogs: { block: "clean, 0 5xx" }, + }, 1000); + mergeLogsEvidence(file, "w1", { + kubectlSweep: { block: "found a fresh pod after all, 3 matched lines" }, + }, 2000); + const doc = readEvidenceFile(file); + assert.equal(doc.logs["w1"].kubectlSweep.block, "found a fresh pod after all, 3 matched lines"); + assert.equal(doc.logs["w1"].victorialogs.block, "clean, 0 5xx"); // untouched +}); + test("writeEvidenceFile creates the parent directory if missing", () => { const nested = join(dir, "nested", "sub", "evidence.json"); writeEvidenceFile(nested, emptyEvidenceFile("build-1", 0)); diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index 1624b54..d7d527f 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -67,6 +67,7 @@ const shared = [ `Pre-fetched build-evidence file — READ THIS FIRST (via the Read tool) before making ANY live github/infra/logs gather call: ${ctx.evidenceFilePath}`, `Build-evidence summary (full detail is in the file above; this is only a pointer — do not re-fetch what the file already covers): ${JSON.stringify(ctx.buildEvidence ?? {})}`, `If the file's github/logs sections do not name a repo/workload/ask you need, or record a "gap" for it, that is a genuine gap — fall back to a live gather via the capability manifest above exactly as if no file existed. The file is an optimization, never a hard dependency.`, + `The file is read-write: after any live gather that fills a gap or goes deeper than what was there, write it back via mergeGithubEvidence/mergeLogsEvidence (lib/evidence-file.mjs) before finishing this test, so a sibling dispatched after you — or another cluster sharing the same repo/workload — reads the enriched entry instead of re-fetching it.`, `Autonomous run — on an evidence gap with no valid connector, report "unavailable" back to TFA (NEVER prompt a user). Best-effort finalize.`, `PRODUCT_BUG / application-bug mandate: hunt the culprit PR via the github connector (deploy timeline vs last-pass window, changed paths vs failure signature) and feed the PR link(s) to TFA so related_prs populates. No PR after digging to the turn cap → state explicitly "no culprit PR identified after <what was searched>" so the CSV row records the gap.`, `Soft-PENDING is NOT an answer: tfaRcaTurn abandons its in-call poll at 90s while TFA keeps working. On status PENDING, call getTfaTurnResult(testRunId, turnId) FIRST and keep reading on the softPendingDrain budget (every 5s, <=40 reads / <=10min) until the status is RESOLVED / NEEDS_INFO / BLOCKED, then continue the loop. Reads do NOT count against the turn cap. Never submit a new message onto a turn still in flight. Only a fully spent drain budget ends the test PENDING.`, From d33e99eaafb8f846973c2151c2548ef37ebc3480 Mon Sep 17 00:00:00 2001 From: harshit-bstack <harshit.m@browserstack.com> Date: Fri, 31 Jul 2026 16:21:52 +0530 Subject: [PATCH 44/47] fix(rca): eliminate lost evidence write-backs via single-writer shards The write-back added in 70ef859 had every coordinator read-modify-writing one shared JSON file. Measured under 8 concurrent writers with a realistic read -> work -> write window, that design lost 28 of 40 updates (70%). Replace it with a layout where contention is structurally impossible instead of merely unlikely: the orchestrator solely owns the base file, and each coordinator writes only its own shard at rca-evidence.<buildId>.contrib/<testRunId>.json. No two processes ever open the same file for writing. readEvidenceFile folds base + all shards into one view (sorted order; real evidence beats a recorded gap; PRs unioned by number), and readBaseFile keeps the orchestrator's own write path from absorbing shard content back into base. Same 8-writer test against the new layout: 0 of 40 lost. A 12-process run hammering one repo key kept all 480 writes. A corrupt/half-written shard is skipped rather than breaking every subsequent read. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- agents/ai-tfa-coordinator.md | 34 +++-- lib/evidence-file.mjs | 245 +++++++++++++++++++++++++++-------- skills/rca-build/SKILL.md | 26 +++- tests/evidence-file.test.mjs | 147 +++++++++++++-------- workflows/rca-batch.mjs | 2 +- 5 files changed, 320 insertions(+), 134 deletions(-) diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 9a43df1..80c7668 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -46,8 +46,9 @@ it names no `kubectl` / `chitragupta` / `bifrost`; it routes by *capability*. for every repo/workload this build's failures implicate. `Read` it before any live gather call (see Operating Principle 0) — and treat it as read-WRITE: a live gather that fills a gap or goes deeper is written back - via `mergeGithubEvidence`/`mergeLogsEvidence` so later dispatches (this - test's own siblings, or another cluster sharing the same repo/workload) + via `contributeGithubEvidence`/`contributeLogsEvidence` (writing your own + per-writer shard, keyed by your `testRunId`) so later dispatches — this + test's own siblings, or another cluster sharing the same repo/workload — benefit too. If `testRunId` is missing or not parseable as an integer, emit a `failed` @@ -94,15 +95,19 @@ read-only and has no side effects, so a read is always safe to repeat. the pre-fetch never named, a log sweep that succeeded where the file recorded one as gapped) is exactly the kind of build-level fact this file exists to share — not just this test's own answer. Persist it via - `mergeGithubEvidence(evidenceFilePath, repo, patch, nowMs)` or - `mergeLogsEvidence(evidenceFilePath, workload, patch, nowMs)` - (`lib/evidence-file.mjs`) before finishing this test, so a sibling - dispatched after you (or any other cluster that turns out to share the - same repo/workload) reads the enriched entry instead of re-fetching what - you just fetched. Only write back genuinely new/deeper findings — don't - write back a no-op read of an already-covered entry. This has the same - informal-locking caveat as the CSV: it's a best-effort optimization, not a - correctness dependency, so never block or retry on it. + `contributeGithubEvidence(evidenceFilePath, writerId, repo, patch, nowMs)` + or `contributeLogsEvidence(evidenceFilePath, writerId, workload, patch, + nowMs)` (`lib/evidence-file.mjs`), where **`writerId` is your own + `testRunId`** — that is what keeps writes safe. Each coordinator writes only + its own shard file under `<evidenceFilePath minus .json>.contrib/`, so + concurrent coordinators can never clobber each other or the orchestrator's + base pre-fetch; readers fold base + every shard back into one view + automatically. Write back before finishing this test, so a sibling + dispatched after you (or any other cluster sharing the same repo/workload) + reads the enriched entry instead of re-fetching what you just fetched. + Only write back genuinely new/deeper findings — never a no-op re-write of + an already-covered entry. It's a best-effort optimization, not a + correctness dependency: never block or retry on it. 1. **Logs by TFA — the core contract.** Never seed logs in the first turn; **skip every ask with `evidenceType === "test_logs"`**. Never fetch, paste, or digest log content. Logs are TFA's job. @@ -182,8 +187,8 @@ re-fetch per test (the `evidenceFile`'s `github` section, if present and not `gap`-marked for this repo; otherwise the live github connector). A culprit hunt often needs to go deeper than the file's summary — a full diff, a downstream consumer of a changed flag — write that depth back via -`mergeGithubEvidence` once found, so a sibling confirming the same suspect PR -doesn't re-run the same diff/search. Never fabricate a PR when the github +`contributeGithubEvidence` once found, so a sibling confirming the same +suspect PR doesn't re-run the same diff/search. Never fabricate a PR when the github capability is unavailable — emit an `unavailable` block. @@ -218,7 +223,8 @@ capability is unavailable — emit an re-digesting, no live call. Not named in the file, or its entry has a `gap`, or no `evidenceFile` at all → run the discovered skill/tool live, exactly as before — THEN write the - result back via `mergeGithubEvidence`/`mergeLogsEvidence` + result back via `contributeGithubEvidence`/ + `contributeLogsEvidence` with your own testRunId as writerId (Operating Principle 0) so this fills the gap for whoever reads the file next. Digest into one block. Record evidenceType in asks_fulfilled (dedupe). diff --git a/lib/evidence-file.mjs b/lib/evidence-file.mjs index d7487e5..b9010b6 100644 --- a/lib/evidence-file.mjs +++ b/lib/evidence-file.mjs @@ -25,34 +25,85 @@ // discipline as `csv-state.mjs`, so this stays usable from the Workflow-tool // sandbox (which forbids `Date.now()`). // -// Write-back (mergeGithubEvidence / mergeLogsEvidence): the orchestrator's -// Step 4 pass is not the only writer. A coordinator that had to gather live -// (a repo/PR/workload the pre-fetch didn't cover, or covered only with a -// summary) should write what it found back into this SAME file — a -// representative's deep dive then benefits its own siblings (dispatched -// after it resolves) and any other cluster that turns out to share the same -// repo/workload, without re-fetching. Concurrency caveat, same philosophy as -// `csv-state.mjs`'s "true multi-process locking is out of scope": this is a -// synchronous read-modify-write with no file lock, so two coordinators -// writing to the SAME repo/workload key at truly the same moment can lose an -// update. In practice this is low-risk for the case this exists to serve — -// a sibling is dispatched only after its representative resolves, i.e. -// sequentially, never concurrently with it — and acceptable for the rarer -// case of two parallel representatives happening to touch the same repo. - -import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs"; +// Write-back, WITHOUT a lock and WITHOUT lost updates — single-writer shards. +// The orchestrator's Step 4 pass is not the only writer: a coordinator that +// had to gather live (a repo/PR/workload the pre-fetch didn't cover, or +// covered only with a summary) should persist what it found so a sibling +// dispatched after it — or another cluster sharing the same repo/workload — +// reads the enriched result instead of re-fetching it. +// +// Naively that means N concurrent coordinators read-modify-writing ONE JSON +// file, which drops updates whenever two writes interleave. Instead of a lock +// (fragile, and `csv-state.mjs` already declares multi-process locking out of +// scope) the layout makes contention structurally impossible: +// +// <tmpdir>/bstack-rca/ +// rca-evidence.<buildId>.json <- BASE: only the orchestrator writes it +// rca-evidence.<buildId>.contrib/ +// <writerId>.json <- one file per coordinator; sole writer +// <writerId>.json +// +// Every file has exactly ONE writer, so no write can ever clobber another's. +// Reads (`readEvidenceFile`) fold base + every shard into a single view, +// deterministically (shards applied in sorted filename order). This is the +// same "the temp dir is ours, use more of it" trick the CSV path convention +// already leans on. + +// Fold precedence, applied per leaf when base and shards disagree: +// 1. Real evidence beats a recorded gap — a coordinator that actually got +// the data overrides the pre-fetch's "couldn't reach this". +// 2. Among two real values, the later shard wins (sorted order), on the +// assumption a coordinator only writes back something deeper than what +// it read. +// 3. `prsInWindow` is unioned by PR number rather than replaced, so two +// coordinators finding different PRs in the same repo both survive. + +import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync } from "node:fs"; import { dirname, join } from "node:path"; import { tmpdir } from "node:os"; +const safeName = (v, fallback) => + String(v ?? "").replace(/[^A-Za-z0-9._-]/g, "_") || fallback; + /** Canonical evidence-file path for one build's run. Same two invariants as * `csvPathFor`: build id in the filename; OS temp by default; `stateDir` * overrides the directory only. */ export function evidencePathFor(buildId, stateDir = "") { - const safe = String(buildId ?? "").replace(/[^A-Za-z0-9._-]/g, "_") || "unknown-build"; + const safe = safeName(buildId, "unknown-build"); const dir = stateDir && String(stateDir).trim() !== "" ? String(stateDir) : join(tmpdir(), "bstack-rca"); return join(dir, `rca-evidence.${safe}.json`); } +/** Directory holding this build's per-coordinator contribution shards. Derived + * from the base path so callers only ever have to pass `evidenceFilePath` + * around — one input, no second path to thread through every dispatch. */ +export function contribDirFor(basePath) { + return String(basePath).replace(/\.json$/, "") + ".contrib"; +} + +/** The one file a given writer owns. Exactly one writer per path is the whole + * point — never call this for a writerId that isn't yours. */ +export function contribPathFor(basePath, writerId) { + return join(contribDirFor(basePath), `${safeName(writerId, "unknown-writer")}.json`); +} + +/** Shard docs in deterministic (sorted-filename) order. Missing dir → []. A + * corrupt/half-written shard is skipped rather than throwing: a coordinator + * killed mid-write must not break every subsequent read. */ +function readContribs(basePath) { + const dir = contribDirFor(basePath); + if (!existsSync(dir)) return []; + const out = []; + for (const name of readdirSync(dir).filter((n) => n.endsWith(".json")).sort()) { + try { + out.push(JSON.parse(readFileSync(join(dir, name), "utf8"))); + } catch { + // skip unreadable/partial shard + } + } + return out; +} + export function emptyEvidenceFile(buildId, nowMs) { return { buildId: String(buildId ?? ""), @@ -65,13 +116,78 @@ export function emptyEvidenceFile(buildId, nowMs) { }; } -/** Read-only; never throws on a missing file — a coordinator (or the - * orchestrator, before Step 4 has run) always gets a well-shaped, empty-covered - * result rather than an exception. Graceful degradation is the point: an - * absent/partial file just means every ask falls back to a live gather. */ -export function readEvidenceFile(filePath) { +/** The BASE file alone, no shards folded in. Internal to the orchestrator's + * write path: `set*`/`merge*` must read-modify-write base only, or they would + * absorb shard content into base and duplicate it on the next fold. */ +export function readBaseFile(filePath) { if (!existsSync(filePath)) return emptyEvidenceFile("unknown-build", 0); - return JSON.parse(readFileSync(filePath, "utf8")); + try { + return JSON.parse(readFileSync(filePath, "utf8")); + } catch { + return emptyEvidenceFile("unknown-build", 0); + } +} + +// A {block, gap} leaf: real evidence beats a gap; between two real values the +// later (shard) one wins. `undefined`/`null` incoming never overwrites. +function pickLeaf(base, incoming) { + if (incoming == null) return base ?? null; + if (base == null) return incoming; + if (!incoming.gap) return incoming; + if (!base.gap) return base; + return incoming; +} + +function foldGithub(target, repo, entry) { + const cur = target[repo] ?? { deployState: null, prsInWindow: [], gap: null }; + const next = { + deployState: pickLeaf(cur.deployState, entry.deployState), + prsInWindow: cur.prsInWindow ?? [], + gap: cur.gap ?? null, + }; + if (Array.isArray(entry.prsInWindow)) { + const byPr = new Map((next.prsInWindow ?? []).map((p) => [String(p.pr), p])); + for (const pr of entry.prsInWindow) byPr.set(String(pr.pr), pr); + next.prsInWindow = [...byPr.values()]; + } + // A contributor supplying real content clears the pre-fetch's gap. + if (entry.gap === null || entry.gap === undefined) { + if (entry.deployState || Array.isArray(entry.prsInWindow)) next.gap = null; + } else if (!next.deployState && (next.prsInWindow ?? []).length === 0) { + next.gap = entry.gap; + } + target[repo] = next; +} + +function foldLogs(target, workload, entry) { + const cur = target[workload] ?? { clusterIds: [], kubectlSweep: null, victorialogs: null, gap: null }; + const next = { + clusterIds: [...new Set([...(cur.clusterIds ?? []), ...(entry.clusterIds ?? [])])], + kubectlSweep: pickLeaf(cur.kubectlSweep, entry.kubectlSweep), + victorialogs: pickLeaf(cur.victorialogs, entry.victorialogs), + gap: cur.gap ?? null, + }; + if (entry.gap === null || entry.gap === undefined) { + if (entry.kubectlSweep || entry.victorialogs) next.gap = null; + } else if (!next.kubectlSweep && !next.victorialogs) { + next.gap = entry.gap; + } + target[workload] = next; +} + +/** The full view every CONSUMER should read: the orchestrator's base pre-fetch + * with every coordinator's contribution shard folded on top, deterministically. + * Never throws on a missing base or a corrupt shard — an absent/partial result + * just means those asks fall back to a live gather, which is the whole + * degradation contract. */ +export function readEvidenceFile(filePath) { + const doc = readBaseFile(filePath); + for (const shard of readContribs(filePath)) { + for (const [repo, entry] of Object.entries(shard.github ?? {})) foldGithub(doc.github, repo, entry); + for (const [wl, entry] of Object.entries(shard.logs ?? {})) foldLogs(doc.logs, wl, entry); + if (shard.generatedAtMs > (doc.generatedAtMs ?? 0)) doc.generatedAtMs = shard.generatedAtMs; + } + return doc; } export function writeEvidenceFile(filePath, doc) { @@ -82,7 +198,7 @@ export function writeEvidenceFile(filePath, doc) { function loadOrInit(filePath, nowMs) { if (!existsSync(filePath)) return emptyEvidenceFile("unknown-build", nowMs); - return readEvidenceFile(filePath); + return readBaseFile(filePath); } /** Idempotent: creates the file with the given `buildId` if it doesn't exist @@ -91,7 +207,7 @@ function loadOrInit(filePath, nowMs) { * is recorded correctly — the `set*` functions below fall back to * `"unknown-build"` only as a safety net if called without this. */ export function initEvidenceFile(filePath, buildId, nowMs) { - if (existsSync(filePath)) return readEvidenceFile(filePath); + if (existsSync(filePath)) return readBaseFile(filePath); const doc = emptyEvidenceFile(buildId, nowMs); writeEvidenceFile(filePath, doc); return doc; @@ -136,29 +252,41 @@ export function setLogsEvidence(filePath, workload, entry, nowMs) { return doc; } -// Read-modify-write, preserving whatever isn't in `patch`. Returns an existing -// repo entry, or a blank one — this is what lets a coordinator enrich a repo -// the pre-fetch never named at all (a brand-new candidate it found live), not -// just one that's already there with a gap. -function findRepoEntry(doc, repo) { - return doc.github[repo] ?? { deployState: null, prsInWindow: [], gap: null }; +// ---- coordinator write-back: own-shard only, never the base file ---------- +// +// `writerId` must be unique per concurrent writer — the dispatched +// coordinator's `testRunId` is the natural choice (one coordinator per test). +// Because a writer only ever opens its OWN shard, two coordinators writing at +// the same instant touch different files and neither can lose the other's +// update. Reads fold every shard back together (`readEvidenceFile`). + +function loadOwnShard(basePath, writerId, nowMs) { + const p = contribPathFor(basePath, writerId); + if (!existsSync(p)) { + return { path: p, doc: { writerId: String(writerId), generatedAtMs: nowMs, github: {}, logs: {} } }; + } + try { + return { path: p, doc: JSON.parse(readFileSync(p, "utf8")) }; + } catch { + return { path: p, doc: { writerId: String(writerId), generatedAtMs: nowMs, github: {}, logs: {} } }; + } } -function findWorkloadEntry(doc, workload) { - return doc.logs[workload] ?? { clusterIds: [], kubectlSweep: null, victorialogs: null, gap: null }; +function writeShard(path, doc) { + const dir = dirname(path); + if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true }); + writeFileSync(path, JSON.stringify(doc, null, 2), "utf8"); } -/** Write back what a coordinator gathered LIVE for a repo — a deeper - * `deployState` (e.g. the full diff/patch, not just a summary), and/or one or - * more PRs to fold into `prsInWindow` (deduped by `pr`; a PR with a `pr` that - * already exists is REPLACED, since the coordinator's fresh finding is - * presumably deeper than a placeholder). `patch = { deployState?, prsInWindow?, - * gap? }` — omit a field to leave it untouched. Passing `gap: null` clears a - * previously-recorded gap now that real evidence exists. Never removes a PR - * or a field this call doesn't mention. */ -export function mergeGithubEvidence(filePath, repo, patch, nowMs) { - const doc = loadOrInit(filePath, nowMs); - const entry = findRepoEntry(doc, repo); +/** Contribute what THIS coordinator gathered live for a repo — a deeper + * `deployState` (e.g. the full diff, not just a summary), and/or PRs to fold + * into `prsInWindow` (deduped by `pr`). `patch = { deployState?, prsInWindow?, + * gap? }`; omit a field to leave it untouched. Writes only this writer's + * shard, so it can never clobber another coordinator's contribution or the + * orchestrator's base pre-fetch. */ +export function contributeGithubEvidence(basePath, writerId, repo, patch, nowMs) { + const { path, doc } = loadOwnShard(basePath, writerId, nowMs); + const entry = doc.github[repo] ?? { deployState: null, prsInWindow: [], gap: null }; if (patch.deployState !== undefined) entry.deployState = patch.deployState; if (Array.isArray(patch.prsInWindow)) { const byPr = new Map((entry.prsInWindow ?? []).map((p) => [String(p.pr), p])); @@ -168,17 +296,16 @@ export function mergeGithubEvidence(filePath, repo, patch, nowMs) { if (patch.gap !== undefined) entry.gap = patch.gap; doc.github[repo] = entry; doc.generatedAtMs = nowMs; - writeEvidenceFile(filePath, doc); + writeShard(path, doc); return entry; } -/** Write back what a coordinator gathered LIVE for a workload's logs — same - * merge discipline as `mergeGithubEvidence`. `patch = { kubectlSweep?, - * victorialogs?, clusterIds?, gap? }`; `clusterIds` is unioned, not replaced, - * since more than one cluster can end up sharing a workload over the run. */ -export function mergeLogsEvidence(filePath, workload, patch, nowMs) { - const doc = loadOrInit(filePath, nowMs); - const entry = findWorkloadEntry(doc, workload); +/** Contribute what THIS coordinator gathered live for a workload's app-logs. + * Same single-writer-shard discipline as `contributeGithubEvidence`; + * `clusterIds` is unioned rather than replaced. */ +export function contributeLogsEvidence(basePath, writerId, workload, patch, nowMs) { + const { path, doc } = loadOwnShard(basePath, writerId, nowMs); + const entry = doc.logs[workload] ?? { clusterIds: [], kubectlSweep: null, victorialogs: null, gap: null }; if (patch.kubectlSweep !== undefined) entry.kubectlSweep = patch.kubectlSweep; if (patch.victorialogs !== undefined) entry.victorialogs = patch.victorialogs; if (Array.isArray(patch.clusterIds)) { @@ -187,7 +314,7 @@ export function mergeLogsEvidence(filePath, workload, patch, nowMs) { if (patch.gap !== undefined) entry.gap = patch.gap; doc.logs[workload] = entry; doc.generatedAtMs = nowMs; - writeEvidenceFile(filePath, doc); + writeShard(path, doc); return entry; } @@ -205,14 +332,18 @@ function isCovered(doc, section, key) { * — normally the Gate Part A scope-probe-validated repo list and the union of * workloads every cluster's representative implicates (see SKILL.md Step 4). */ export function recomputeCoverage(filePath, requested, nowMs) { + // Coverage is judged against the FOLDED view — a gap the orchestrator + // recorded but a coordinator later filled is genuinely covered now — while + // the result is persisted to base, which the orchestrator solely owns. + const folded = readEvidenceFile(filePath); const doc = loadOrInit(filePath, nowMs); const repos = requested?.repos ?? []; const workloads = requested?.workloads ?? []; doc.coverage = { - reposCovered: repos.filter((r) => isCovered(doc, "github", r)), - reposGapped: repos.filter((r) => !isCovered(doc, "github", r)), - workloadsCovered: workloads.filter((w) => isCovered(doc, "logs", w)), - workloadsGapped: workloads.filter((w) => !isCovered(doc, "logs", w)), + reposCovered: repos.filter((r) => isCovered(folded, "github", r)), + reposGapped: repos.filter((r) => !isCovered(folded, "github", r)), + workloadsCovered: workloads.filter((w) => isCovered(folded, "logs", w)), + workloadsGapped: workloads.filter((w) => !isCovered(folded, "logs", w)), }; doc.generatedAtMs = nowMs; writeEvidenceFile(filePath, doc); diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index ca77c9b..686441d 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -338,16 +338,28 @@ a full independent sweep — exactly the redundancy Step 4 exists to remove. **The file is read-write, not just read-only.** When a coordinator has to gather live (a genuine gap), tell it to write the result back — -`mergeGithubEvidence`/`mergeLogsEvidence` (`lib/evidence-file.mjs`) — before -finishing, not just answer TFA and move on. A representative's deep dive (a -full diff, a downstream trace, a PR the pre-fetch never named) then benefits -its own siblings and any other cluster sharing the same repo/workload, instead -of every one of them re-running the same live search. This is already baked -into `agents/ai-tfa-coordinator.md`'s Operating Principle 0 for any dispatch -of that agent type — no need to repeat the mechanics in the prompt, just don't +`contributeGithubEvidence`/`contributeLogsEvidence` (`lib/evidence-file.mjs`), +passing its own `testRunId` as `writerId` — before finishing, not just answer +TFA and move on. A representative's deep dive (a full diff, a downstream +trace, a PR the pre-fetch never named) then benefits its own siblings and any +other cluster sharing the same repo/workload, instead of every one of them +re-running the same live search. This is already baked into +`agents/ai-tfa-coordinator.md`'s Operating Principle 0 for any dispatch of +that agent type — no need to repeat the mechanics in the prompt, just don't omit `evidenceFilePath` (above), since write-back has nothing to write to without it. +**Concurrency is handled by layout, not by locking.** Base +(`rca-evidence.<buildId>.json`) has exactly one writer — this orchestrator, in +Step 4. Every coordinator writes only its own shard under +`rca-evidence.<buildId>.contrib/<testRunId>.json`. Since no two processes ever +open the same file for writing, concurrent write-back cannot lose an update; +`readEvidenceFile` folds base + all shards into one view, applying shards in +sorted order, with real evidence taking precedence over a recorded `gap`. A +measured comparison: 8 concurrent writers with a realistic read→work→write +window lost **28 of 40 updates** against a single shared file, and **0 of 40** +under this layout. + **Application bugs need a culprit PR.** Whenever a test's RCA classifies as PRODUCT_BUG / application bug, the coordinator MUST hunt the culprit PR via the github connector (deploy timeline vs last-pass window, changed paths vs failure diff --git a/tests/evidence-file.test.mjs b/tests/evidence-file.test.mjs index 768142b..dc40742 100644 --- a/tests/evidence-file.test.mjs +++ b/tests/evidence-file.test.mjs @@ -1,6 +1,6 @@ import { test, beforeEach, afterEach } from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -12,8 +12,11 @@ import { setBaseline, setGithubEvidence, setLogsEvidence, - mergeGithubEvidence, - mergeLogsEvidence, + contributeGithubEvidence, + contributeLogsEvidence, + contribDirFor, + contribPathFor, + readBaseFile, recomputeCoverage, } from "../lib/evidence-file.mjs"; @@ -135,73 +138,107 @@ test("a block string with newlines and quotes round-trips through JSON unchanged assert.equal(doc.github["org/a"].deployState.block, block); }); -test("mergeGithubEvidence on a repo the pre-fetch never named creates a fresh entry", () => { - const entry = mergeGithubEvidence(file, "org/new-repo", { - prsInWindow: [{ pr: "#8912", verdict: "supported", block: "found live" }], - }, 1000); - assert.equal(entry.prsInWindow.length, 1); - assert.equal(entry.gap, null); - const doc = readEvidenceFile(file); - assert.equal(doc.github["org/new-repo"].prsInWindow[0].pr, "#8912"); +test("contribute writes a shard, never the base file", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "base" } }, 1000); + contributeGithubEvidence(file, "3895581484", "org/a", { + deployState: { block: "coordinator's full diff" }, + }, 2000); + // base is untouched... + assert.equal(readBaseFile(file).github["org/a"].deployState.block, "base"); + // ...but the folded view shows the contribution + assert.equal(readEvidenceFile(file).github["org/a"].deployState.block, "coordinator's full diff"); }); -test("mergeGithubEvidence appends a new PR without dropping an existing one", () => { - setGithubEvidence(file, "org/a", { - gap: null, - deployState: { block: "a" }, - prsInWindow: [{ pr: "#1", verdict: "not-live" }], - }, 1000); - mergeGithubEvidence(file, "org/a", { - prsInWindow: [{ pr: "#2", verdict: "supported", block: "found live during coordinator's own hunt" }], - }, 2000); - const doc = readEvidenceFile(file); - const prs = doc.github["org/a"].prsInWindow.map((p) => p.pr); - assert.deepEqual(prs.sort(), ["#1", "#2"]); - assert.equal(doc.github["org/a"].deployState.block, "a"); // untouched +test("contribPathFor: one file per writer, under the build's contrib dir", () => { + const p = contribPathFor(file, "3895581484"); + assert.ok(p.startsWith(contribDirFor(file))); + assert.ok(p.endsWith("3895581484.json")); + assert.notEqual(contribPathFor(file, "w1"), contribPathFor(file, "w2")); +}); + +test("contribPathFor sanitizes a hostile writerId", () => { + assert.ok(contribPathFor(file, "../../etc/passwd").endsWith("_.._etc_passwd.json")); }); -test("mergeGithubEvidence replaces a PR entry with the same pr number (deeper finding wins)", () => { +test("CONCURRENCY: two writers on the same repo both survive (no lost update)", () => { setGithubEvidence(file, "org/a", { - gap: null, - deployState: { block: "a" }, - prsInWindow: [{ pr: "#9011", verdict: "unassessed", files: null }], + gap: null, deployState: { block: "base" }, prsInWindow: [{ pr: "#1" }], }, 1000); - mergeGithubEvidence(file, "org/a", { - prsInWindow: [{ pr: "#9011", verdict: "supported", files: ["Foo.java"], block: "full diff fetched" }], - }, 2000); - const doc = readEvidenceFile(file); - assert.equal(doc.github["org/a"].prsInWindow.length, 1); - assert.equal(doc.github["org/a"].prsInWindow[0].verdict, "supported"); - assert.deepEqual(doc.github["org/a"].prsInWindow[0].files, ["Foo.java"]); + // Interleave the two writers the way real concurrent coordinators would: + // each reads, then each writes — under a single shared file this is exactly + // the sequence that drops the first writer's update. + contributeGithubEvidence(file, "writerA", "org/a", { prsInWindow: [{ pr: "#2", by: "A" }] }, 2000); + contributeGithubEvidence(file, "writerB", "org/a", { prsInWindow: [{ pr: "#3", by: "B" }] }, 2000); + const prs = readEvidenceFile(file).github["org/a"].prsInWindow.map((p) => p.pr).sort(); + assert.deepEqual(prs, ["#1", "#2", "#3"]); // base + BOTH contributions +}); + +test("CONCURRENCY: two writers on the same workload both survive", () => { + contributeLogsEvidence(file, "writerA", "w1", { kubectlSweep: { block: "A found 3 lines" } }, 1000); + contributeLogsEvidence(file, "writerB", "w1", { victorialogs: { block: "B found 5xx" } }, 1000); + const w = readEvidenceFile(file).logs["w1"]; + assert.equal(w.kubectlSweep.block, "A found 3 lines"); + assert.equal(w.victorialogs.block, "B found 5xx"); }); -test("mergeGithubEvidence with gap:null clears a previously-recorded gap", () => { +test("fold: real contributed evidence beats a base-recorded gap", () => { setGithubEvidence(file, "org/a", { gap: "gh auth failed" }, 1000); - mergeGithubEvidence(file, "org/a", { gap: null, deployState: { block: "found it after all" } }, 2000); - const doc = readEvidenceFile(file); - assert.equal(doc.github["org/a"].gap, null); + contributeGithubEvidence(file, "w1", "org/a", { + gap: null, deployState: { block: "reachable after all" }, + }, 2000); + const entry = readEvidenceFile(file).github["org/a"]; + assert.equal(entry.gap, null); + assert.equal(entry.deployState.block, "reachable after all"); }); -test("mergeLogsEvidence unions clusterIds instead of replacing them", () => { - setLogsEvidence(file, "w1", { gap: null, clusterIds: ["c-A"], kubectlSweep: { block: "x" } }, 1000); - mergeLogsEvidence(file, "w1", { clusterIds: ["c-B"] }, 2000); - const doc = readEvidenceFile(file); - assert.deepEqual(doc.logs["w1"].clusterIds.sort(), ["c-A", "c-B"]); - assert.equal(doc.logs["w1"].kubectlSweep.block, "x"); // untouched +test("fold: a contributed gap does NOT overwrite real base evidence", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "real base evidence" } }, 1000); + contributeGithubEvidence(file, "w1", "org/a", { deployState: { gap: "my call failed" } }, 2000); + assert.equal(readEvidenceFile(file).github["org/a"].deployState.block, "real base evidence"); }); -test("mergeLogsEvidence upgrades one sub-field without touching the other", () => { - setLogsEvidence(file, "w1", { - gap: null, - kubectlSweep: { gap: "stale pods" }, - victorialogs: { block: "clean, 0 5xx" }, +test("fold: same PR number contributed later wins (deeper finding replaces placeholder)", () => { + setGithubEvidence(file, "org/a", { + gap: null, prsInWindow: [{ pr: "#9011", verdict: "unassessed", files: null }], }, 1000); - mergeLogsEvidence(file, "w1", { - kubectlSweep: { block: "found a fresh pod after all, 3 matched lines" }, + contributeGithubEvidence(file, "w1", "org/a", { + prsInWindow: [{ pr: "#9011", verdict: "supported", files: ["Foo.java"] }], }, 2000); - const doc = readEvidenceFile(file); - assert.equal(doc.logs["w1"].kubectlSweep.block, "found a fresh pod after all, 3 matched lines"); - assert.equal(doc.logs["w1"].victorialogs.block, "clean, 0 5xx"); // untouched + const prs = readEvidenceFile(file).github["org/a"].prsInWindow; + assert.equal(prs.length, 1); + assert.equal(prs[0].verdict, "supported"); +}); + +test("fold: contributing a repo the pre-fetch never named", () => { + contributeGithubEvidence(file, "w1", "org/brand-new", { + prsInWindow: [{ pr: "#8912", verdict: "supported" }], + }, 1000); + assert.equal(readEvidenceFile(file).github["org/brand-new"].prsInWindow[0].pr, "#8912"); +}); + +test("fold: clusterIds union across base and multiple shards", () => { + setLogsEvidence(file, "w1", { gap: null, clusterIds: ["c-A"], kubectlSweep: { block: "x" } }, 1000); + contributeLogsEvidence(file, "w1writer", "w1", { clusterIds: ["c-B"] }, 2000); + contributeLogsEvidence(file, "w2writer", "w1", { clusterIds: ["c-C"] }, 2000); + assert.deepEqual(readEvidenceFile(file).logs["w1"].clusterIds.sort(), ["c-A", "c-B", "c-C"]); +}); + +test("fold: a corrupt shard is skipped, not fatal", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "base" } }, 1000); + contributeGithubEvidence(file, "good", "org/a", { prsInWindow: [{ pr: "#2" }] }, 2000); + writeFileSync(contribPathFor(file, "corrupt"), "{not json", "utf8"); + const doc = readEvidenceFile(file); // must not throw + assert.equal(doc.github["org/a"].prsInWindow[0].pr, "#2"); +}); + +test("recomputeCoverage counts a coordinator-filled gap as covered", () => { + setGithubEvidence(file, "org/a", { gap: "unreachable at pre-fetch time" }, 1000); + let cov = recomputeCoverage(file, { repos: ["org/a"], workloads: [] }, 2000); + assert.deepEqual(cov.reposGapped, ["org/a"]); + contributeGithubEvidence(file, "w1", "org/a", { gap: null, deployState: { block: "got it" } }, 3000); + cov = recomputeCoverage(file, { repos: ["org/a"], workloads: [] }, 4000); + assert.deepEqual(cov.reposCovered, ["org/a"]); + assert.deepEqual(cov.reposGapped, []); }); test("writeEvidenceFile creates the parent directory if missing", () => { diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index d7d527f..2a7a0e4 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -67,7 +67,7 @@ const shared = [ `Pre-fetched build-evidence file — READ THIS FIRST (via the Read tool) before making ANY live github/infra/logs gather call: ${ctx.evidenceFilePath}`, `Build-evidence summary (full detail is in the file above; this is only a pointer — do not re-fetch what the file already covers): ${JSON.stringify(ctx.buildEvidence ?? {})}`, `If the file's github/logs sections do not name a repo/workload/ask you need, or record a "gap" for it, that is a genuine gap — fall back to a live gather via the capability manifest above exactly as if no file existed. The file is an optimization, never a hard dependency.`, - `The file is read-write: after any live gather that fills a gap or goes deeper than what was there, write it back via mergeGithubEvidence/mergeLogsEvidence (lib/evidence-file.mjs) before finishing this test, so a sibling dispatched after you — or another cluster sharing the same repo/workload — reads the enriched entry instead of re-fetching it.`, + `The file is read-write: after any live gather that fills a gap or goes deeper than what was there, write it back via contributeGithubEvidence/contributeLogsEvidence (lib/evidence-file.mjs) passing your own testRunId as writerId, before finishing this test — so a sibling dispatched after you, or another cluster sharing the same repo/workload, reads the enriched entry instead of re-fetching it. Each writer owns its own shard file, so concurrent coordinators cannot clobber each other; readers fold base + shards automatically.`, `Autonomous run — on an evidence gap with no valid connector, report "unavailable" back to TFA (NEVER prompt a user). Best-effort finalize.`, `PRODUCT_BUG / application-bug mandate: hunt the culprit PR via the github connector (deploy timeline vs last-pass window, changed paths vs failure signature) and feed the PR link(s) to TFA so related_prs populates. No PR after digging to the turn cap → state explicitly "no culprit PR identified after <what was searched>" so the CSV row records the gap.`, `Soft-PENDING is NOT an answer: tfaRcaTurn abandons its in-call poll at 90s while TFA keeps working. On status PENDING, call getTfaTurnResult(testRunId, turnId) FIRST and keep reading on the softPendingDrain budget (every 5s, <=40 reads / <=10min) until the status is RESOLVED / NEEDS_INFO / BLOCKED, then continue the loop. Reads do NOT count against the turn cap. Never submit a new message onto a turn still in flight. Only a fully spent drain budget ends the test PENDING.`, From 8becf43e32cf3d422b54fe7ce2ea00df7b2cfd90 Mon Sep 17 00:00:00 2001 From: harshit-bstack <harshit.m@browserstack.com> Date: Fri, 31 Jul 2026 16:41:01 +0530 Subject: [PATCH 45/47] feat(rca): memoize read-only tool calls; fast-fail wedged TFA drains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two measured sources of duplicate work on a real 10-test build: 1. `gh` was 37% of all coordinator tool calls (151), and 46 were byte-identical commands re-run by different coordinators — one BStackAutomation spec fetched 12 times, a frontend component 5 times. None of these fit the evidence file's schema (deployState / prsInWindow), so the write-back added earlier could not share them. Add a build-scoped memo cache keyed by the CALL rather than by evidence shape, so duplication is caught regardless of what the call was for. `bin/cached-exec.mjs` wraps a shell fetch transparently (same stdout, same exit code, executes only on a miss); `bin/cached-mcp.mjs` does check-then- store for read-only MCP queries. One file per call key + atomic rename, so concurrent writers cannot collide or produce a torn read. Failures are never cached (a transient rate-limit must not become a permanent answer), stateful tools (tfaRcaTurn/getTfaTurnResult/triggerRcaReport) are refused, secrets are redacted before anything reaches disk, and the runner uses execFile with a hand-rolled tokenizer so no shell ever interprets an argument. 2. Drain reads plus their sleeps were 23% of all coordinator tool calls. The drain treated "TFA is still thinking" and "the TFA run hard-failed" identically, spending the full 40-read/10-min budget on turns that had already died — the four tests that wedged this way were the four slowest in the batch. Distinguish the two: stop after `maxErrorReads` (default 3) CONSECUTIVE hard failures, note it as `tfa-error`, keep the row resumable. A single good read clears the streak, so flaky-but-recovering reads still land. Measured: cross-writer cache hit served in 0.185s vs 1.131s for the live fetch. 108 tests pass. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- agents/ai-tfa-coordinator.md | 28 +++++ bin/cached-exec.mjs | 93 +++++++++++++++ bin/cached-mcp.mjs | 98 ++++++++++++++++ config/rca.config.json | 4 +- lib/loop.mjs | 67 +++++++++-- lib/tool-cache.mjs | 222 +++++++++++++++++++++++++++++++++++ skills/rca-build/SKILL.md | 11 ++ tests/conformance.test.mjs | 61 ++++++++++ tests/tool-cache.test.mjs | 151 ++++++++++++++++++++++++ workflows/rca-batch.mjs | 4 +- 10 files changed, 724 insertions(+), 15 deletions(-) create mode 100644 bin/cached-exec.mjs create mode 100644 bin/cached-mcp.mjs create mode 100644 lib/tool-cache.mjs create mode 100644 tests/tool-cache.test.mjs diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 80c7668..1ae0096 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -108,6 +108,34 @@ read-only and has no side effects, so a read is always safe to repeat. Only write back genuinely new/deeper findings — never a no-op re-write of an already-covered entry. It's a best-effort optimization, not a correctness dependency: never block or retry on it. + + **Route read-only lookups through the tool cache.** The evidence file + shares *digested findings*; the cache below shares *raw call results*, which + is where most duplicate work actually hides (measured on one real build: + `gh` was 37% of all coordinator tool calls, 46 of them byte-identical + commands re-run by different coordinators — one spec file fetched 12 + times). Given `buildId` and your own `testRunId` as `writerId`: + + - **Shell (`gh`/`kubectl`/`curl`/`git`)** — prefix the fetch with the + wrapper; it behaves exactly like the raw command (same stdout, same exit + code) but only executes on a miss: + `node <pluginRoot>/bin/cached-exec.mjs <buildId> <testRunId> '<command>'` + Wrap ONLY the fetch and pipe *outside* it, so different downstream + filters share one cached fetch: + `node .../cached-exec.mjs "$B" 3895 'gh api repos/o/r/contents/f' | jq -r .content | head -40` + One fetch per call — the wrapper refuses `;`/`&&`/backticks/redirects. + - **MCP data queries** (grafana/VictoriaLogs, `listTestIds`, + `getFailureLogs`) — check first, and store your digest on a miss: + `node <pluginRoot>/bin/cached-mcp.mjs <buildId> get <tool> '<argsJson>'` + (exit 0 = hit, use it and skip the MCP call; exit 1 = miss, make the call + then `... put <tool> '<argsJson>' <testRunId>` with the digest on stdin). + Worth it for expensive build-level queries several coordinators would + each re-run; skip it for a one-off only this test needs, since a miss + costs two extra calls. + - **NEVER cache `tfaRcaTurn` / `getTfaTurnResult` / `triggerRcaReport`** — + they are stateful, and the cache refuses them outright. + - Don't re-probe a connector the gate already validated (`gh auth status`, + `kubectl version`); the manifest above is the answer. 1. **Logs by TFA — the core contract.** Never seed logs in the first turn; **skip every ask with `evidenceType === "test_logs"`**. Never fetch, paste, or digest log content. Logs are TFA's job. diff --git a/bin/cached-exec.mjs b/bin/cached-exec.mjs new file mode 100644 index 0000000..931292d --- /dev/null +++ b/bin/cached-exec.mjs @@ -0,0 +1,93 @@ +#!/usr/bin/env node +// Run a READ-ONLY command through the build's tool cache, in ONE tool call. +// +// Why a wrapper: a "check cache / run / store" sequence done by hand costs +// three tool calls to save one, which is worse than not caching. This collapses +// it to a single call that behaves exactly like the underlying command — +// same stdout, same exit code — but only actually executes on a miss. +// +// Usage (command is ONE argument, so the caller's own quoting survives): +// node bin/cached-exec.mjs <buildId> <writerId> '<command>' +// node bin/cached-exec.mjs <buildId> --stats +// +// Wrap only the expensive fetch and leave filtering to the outer shell: +// node bin/cached-exec.mjs "$B" 3895581484 'gh api repos/o/r/contents/f' | jq -r .content | head -40 +// Two coordinators piping the same fetch through different greps then share +// one cache entry, instead of each paying for the fetch. +// +// Cache hits/misses are reported on STDERR so stdout stays byte-identical to +// the raw command — piping into `grep`/`head`/`jq` is unaffected. + +import { execFileSync } from "node:child_process"; +import { + toolCacheDirFor, cacheKey, cacheGet, cachePut, cacheStats, isRunnable, tokenize, +} from "../lib/tool-cache.mjs"; + +const [, , buildId, writerOrFlag, command] = process.argv; + +if (!buildId || (writerOrFlag !== "--stats" && !command)) { + console.error("usage: cached-exec.mjs <buildId> <writerId> '<command>'"); + console.error(" cached-exec.mjs <buildId> --stats"); + process.exit(2); +} + +const dir = toolCacheDirFor(buildId, process.env.RCA_STATE_DIR ?? ""); + +if (writerOrFlag === "--stats") { + const s = cacheStats(dir); + console.log(JSON.stringify({ cacheDir: dir, ...s }, null, 2)); + process.exit(0); +} + +const key = cacheKey(command); +const hit = cacheGet(dir, key); + +if (hit) { + console.error(`[tool-cache HIT ${key} — captured by ${hit.writerId ?? "?"}, ${hit.bytes}B]`); + process.stdout.write(hit.stdout); + process.exit(0); +} + +// Gate before running anything (allowlisted read-only leader, no chaining). +const gate = isRunnable(command); +if (!gate.ok) { + console.error(`[tool-cache REFUSED] ${gate.reason}`); + console.error(` command: ${command}`); + process.exit(2); +} + +// No shell: tokenize ourselves and execFile the binary directly, so shell +// metacharacters inside arguments (a --jq expression, an XPath, a LogsQL +// filter) are passed through literally and cannot start a second command. +let argv; +try { + argv = tokenize(command); +} catch (err) { + console.error(`[tool-cache REFUSED] ${err.message}`); + process.exit(2); +} + +let stdout = ""; +let exitCode = 0; +try { + stdout = execFileSync(argv[0], argv.slice(1), { + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); +} catch (err) { + // Preserve the real behaviour of the wrapped command: emit whatever it + // produced and exit non-zero. Deliberately NOT cached — a transient + // failure (rate limit, expired token) must not become a permanent answer. + stdout = (err.stdout ?? "").toString(); + exitCode = typeof err.status === "number" ? err.status : 1; + if (err.stderr) process.stderr.write(err.stderr.toString()); + console.error(`[tool-cache MISS ${key} — command exited ${exitCode}, NOT cached]`); + process.stdout.write(stdout); + process.exit(exitCode); +} + +// nowMs is read here, at the process edge, rather than inside lib/ — the +// library keeps its no-clock discipline so it stays sandbox-safe. +cachePut(dir, key, { command, writerId: writerOrFlag, stdout, exitCode }, Date.now()); +console.error(`[tool-cache MISS ${key} — stored ${stdout.length}B]`); +process.stdout.write(stdout); diff --git a/bin/cached-mcp.mjs b/bin/cached-mcp.mjs new file mode 100644 index 0000000..8a57a8e --- /dev/null +++ b/bin/cached-mcp.mjs @@ -0,0 +1,98 @@ +#!/usr/bin/env node +// Memo cache for READ-ONLY **MCP** tool calls, sharing the same per-build +// store as `cached-exec.mjs`. +// +// Shell calls can be wrapped transparently (`cached-exec.mjs` runs the command +// for you). MCP calls cannot — only the agent can invoke an MCP tool — so the +// contract here is check-then-call: +// +// 1. get → node bin/cached-mcp.mjs <buildId> get <tool> '<argsJson>' +// exit 0 + result on stdout = HIT, skip the MCP call entirely +// exit 1, empty stdout = MISS, make the MCP call yourself +// 2. put → node bin/cached-mcp.mjs <buildId> put <tool> '<argsJson>' <writerId> +// (payload on STDIN — pipe the digest you want shared) +// +// WHEN THIS PAYS OFF, and when it does not. A hit replaces one MCP call with +// one cheap local read, so it wins on latency and on tokens whenever the +// cached payload is a digest smaller than the raw response. A miss costs two +// extra calls (the probe + the store), so this is worth it for **expensive, +// broadly-reusable, build-level queries** — a VictoriaLogs sweep, a +// `listTestIds`, a `getFailureLogs` several coordinators would each re-run — +// and NOT worth it for a one-off lookup only this test will ever need. +// +// Never cacheable (refused): `tfaRcaTurn`, `getTfaTurnResult`, +// `triggerRcaReport`. Those are stateful — a turn's status is *expected* to +// change between reads, so serving one from cache is wrong, not just stale. +// Prefer storing a DIGEST rather than a raw payload: the point is to spare the +// next reader the raw rows, not to relay them. + +import { readFileSync } from "node:fs"; +import { + toolCacheDirFor, mcpCacheKey, cacheGet, cachePut, cacheStats, isCacheableMcp, +} from "../lib/tool-cache.mjs"; + +const [, , buildId, verb, tool, argsJson, writerId] = process.argv; + +if (!buildId || !verb) { + console.error("usage: cached-mcp.mjs <buildId> get <tool> '<argsJson>'"); + console.error(" cached-mcp.mjs <buildId> put <tool> '<argsJson>' <writerId> # payload on stdin"); + console.error(" cached-mcp.mjs <buildId> stats"); + process.exit(2); +} + +const dir = toolCacheDirFor(buildId, process.env.RCA_STATE_DIR ?? ""); + +if (verb === "stats") { + console.log(JSON.stringify({ cacheDir: dir, ...cacheStats(dir) }, null, 2)); + process.exit(0); +} + +if (!tool || argsJson === undefined) { + console.error("both <tool> and '<argsJson>' are required"); + process.exit(2); +} + +if (!isCacheableMcp(tool)) { + console.error(`[mcp-cache REFUSED] ${tool} is stateful — never cache it; call it directly.`); + process.exit(2); +} + +let args; +try { + args = JSON.parse(argsJson); +} catch (err) { + console.error(`[mcp-cache] argsJson is not valid JSON: ${err.message}`); + process.exit(2); +} + +const key = mcpCacheKey(tool, args); + +if (verb === "get") { + const hit = cacheGet(dir, key); + if (!hit) { + console.error(`[mcp-cache MISS ${key} ${tool}] — make the MCP call, then 'put' the digest`); + process.exit(1); + } + console.error(`[mcp-cache HIT ${key} ${tool} — captured by ${hit.writerId ?? "?"}, ${hit.bytes}B]`); + process.stdout.write(hit.stdout); + process.exit(0); +} + +if (verb === "put") { + let payload = ""; + try { + payload = readFileSync(0, "utf8"); // stdin + } catch { + payload = ""; + } + if (!payload.trim()) { + console.error("[mcp-cache] refusing to store an empty payload"); + process.exit(2); + } + const rec = cachePut(dir, key, { command: `${tool} ${argsJson}`, writerId, stdout: payload }, Date.now()); + console.error(`[mcp-cache STORED ${key} ${tool} — ${rec.bytes}B]`); + process.exit(0); +} + +console.error(`unknown verb: ${verb}`); +process.exit(2); diff --git a/config/rca.config.json b/config/rca.config.json index b316f94..aea7ace 100644 --- a/config/rca.config.json +++ b/config/rca.config.json @@ -7,10 +7,12 @@ "turnMessageMaxChars": 1000, "pollSoftPendingMs": 90000, "$softPendingDrainComment": "tfaRcaTurn abandons its in-call poll at pollSoftPendingMs (90s) and returns a soft PENDING while the TFA agent keeps working — turns finalizing past 90s are routine. On a soft PENDING the loop READS the same turnId via getTfaTurnResult on this budget before it routes asks or submits anything further; reads do not consume turnCap. Only when the budget is spent does the run end PENDING (pending-resume row).", + "$maxErrorReadsComment": "A soft PENDING is drained on the full budget below, but a HARD read failure (a thrown error, or a result whose status/message says the TFA agent run failed) is a different signal: it will not resolve by asking again. After this many CONSECUTIVE failed reads the drain stops early and the row ends PENDING with a `tfa-error` note, still resumable. A single good read clears the streak. Measured motivation: on one real build, drain reads plus their sleeps were 23% of all coordinator tool calls, and the four tests that wedged this way were the four slowest in the batch.", "softPendingDrain": { "maxWaitMs": 600000, "intervalMs": 5000, - "maxReads": 40 + "maxReads": 40, + "maxErrorReads": 3 }, "reaperHeartbeatTtlSec": 600, "errorSummaryMaxChars": 200, diff --git a/lib/loop.mjs b/lib/loop.mjs index d4334f4..d9535fa 100644 --- a/lib/loop.mjs +++ b/lib/loop.mjs @@ -32,7 +32,7 @@ import { routeAsks } from "./routing.mjs"; // Drain budget for one soft-PENDING. Bounded so a wedged turn can never hang the // batch — on exhaustion the loop still ends `PENDING` (resumable via the CSV's // `pending-resume` row), which is the old behaviour as a floor, not a default. -const DEFAULT_DRAIN = { maxWaitMs: 600_000, intervalMs: 5_000, maxReads: 40 }; +const DEFAULT_DRAIN = { maxWaitMs: 600_000, intervalMs: 5_000, maxReads: 40, maxErrorReads: 3 }; const defaultSleep = (ms) => new Promise((r) => setTimeout(r, ms)); @@ -40,6 +40,30 @@ const defaultSleep = (ms) => new Promise((r) => setTimeout(r, ms)); const isAgentStatus = (s) => s === "RESOLVED" || s === "NEEDS_INFO" || s === "BLOCKED"; +/** + * Distinguish "TFA is still thinking" from "this read HARD-FAILED". + * + * These deserve opposite responses and the original drain conflated them: + * a `PENDING` should be waited out on the full budget, but a server-side + * `TFA agent run failed` will keep failing, and patiently re-reading it burns + * the entire 40-read / 10-minute budget to learn nothing. Measured on one real + * build: drain reads + their sleeps were 23% of ALL coordinator tool calls, + * and the four agents that wedged this way were the four slowest in the batch. + * + * Only explicit failure signals count — an unrecognised-but-parseable turn is + * treated as "still working", so a new upstream status can never be + * misclassified as an error and cut the drain short. + */ +function isErrorRead(turn, threw) { + if (threw) return true; + if (turn == null) return true; + if (typeof turn === "string") return /\b(fail(ed|ure)?|error)\b/i.test(turn); + if (turn.error) return true; + if (typeof turn.status === "string" && /^(ERROR|FAILED)$/i.test(turn.status)) return true; + if (typeof turn.message === "string" && /\b(fail(ed|ure)?|error)\b/i.test(turn.message)) return true; + return false; +} + function unavailableBlock(gap) { const what = gap?.ask?.what ?? ""; return [ @@ -56,32 +80,47 @@ function unavailableBlock(gap) { // not a new turn. `readTurn` is read-only and side-effect free, so the only // budget that applies is wall clock / read count. // -// Returns { turn, reads }: `turn` is the landed agent turn, or null if the drain -// budget ran out (caller then ends PENDING, resumable). +// Returns { turn, reads, reason }: `turn` is the landed agent turn, or null if +// the drain gave up — `reason` is `landed` | `tfa-error` | `budget-spent` | +// `not-drainable`, which the caller surfaces in the RCA_OUTPUT note so a human +// can tell "TFA was slow" apart from "TFA broke". async function drainSoftPending({ testRunId, pending, readTurn, sleep, drain }) { - const { maxWaitMs, intervalMs, maxReads } = { ...DEFAULT_DRAIN, ...(drain ?? {}) }; + const { maxWaitMs, intervalMs, maxReads, maxErrorReads } = { ...DEFAULT_DRAIN, ...(drain ?? {}) }; const turnId = pending.turnId; let reads = 0; // No turnId → nothing addressable to read; no readTurn → client lacks the // getTfaTurnResult tool. Either way fall back to reporting it resumable. - if (!turnId || typeof readTurn !== "function") return { turn: null, reads }; + if (!turnId || typeof readTurn !== "function") return { turn: null, reads, reason: "not-drainable" }; const started = Date.now(); + let consecutiveErrors = 0; while (reads < maxReads && Date.now() - started < maxWaitMs) { await sleep(intervalMs); reads++; let turn; + let threw = false; try { turn = await readTurn({ testRunId, turnId }); } catch { - // A failed read is not a verdict — keep reading until the budget is spent. + threw = true; + } + + if (isErrorRead(turn, threw)) { + // Hard failure. Allow a couple of retries for a genuine blip, then stop: + // a wedged turn will not un-wedge by being asked the same question 37 + // more times, and the row stays resumable either way. + if (++consecutiveErrors >= maxErrorReads) { + return { turn: null, reads, reason: "tfa-error" }; + } continue; } - if (isAgentStatus(turn?.status)) return { turn, reads }; + + consecutiveErrors = 0; // a good read clears the streak + if (isAgentStatus(turn?.status)) return { turn, reads, reason: "landed" }; // still PENDING → the agent is working; read again. } - return { turn: null, reads }; + return { turn: null, reads, reason: "budget-spent" }; } // runRcaLoop drives one test to a terminal RCA_OUTPUT object. @@ -157,11 +196,13 @@ export async function runRcaLoop({ drain, }); if (!drained.turn) { - return out( - "PENDING", - turn, - `soft-pending: still working after ${drained.reads} read(s)`, - ); + const note = + drained.reason === "tfa-error" + ? `tfa-error: read failed ${drained.reads} time(s) — stopped early, row stays resumable` + : drained.reason === "not-drainable" + ? `soft-pending: no turnId or no getTfaTurnResult tool` + : `soft-pending: still working after ${drained.reads} read(s)`; + return out("PENDING", turn, note); } turn = drained.turn; threadId = turn.threadId ?? threadId; diff --git a/lib/tool-cache.mjs b/lib/tool-cache.mjs new file mode 100644 index 0000000..94526ef --- /dev/null +++ b/lib/tool-cache.mjs @@ -0,0 +1,222 @@ +// Build-scoped memo cache for READ-ONLY tool calls (`gh`, `kubectl`, curl, …). +// +// The evidence file (`evidence-file.mjs`) shares *digested findings* whose +// shape the schema knows about — deploy state, PRs, log sweeps. But a +// coordinator's real tool traffic is mostly raw lookups that fit no schema +// slot: fetching a spec file's contents at a ref, listing a git tree, running +// a code search. Measured on one real 10-test build: `gh` was 37% of all +// coordinator tool calls, and 46 of them were byte-identical commands re-run +// by different coordinators — the same BStackAutomation spec fetched 12 +// times, a frontend component 5 times. +// +// This module memoizes at the CALL level instead, so duplication is caught +// regardless of what the call was for. Any coordinator about to run a +// read-only command checks here first; on a miss it runs the command and +// stores the result for everyone else. +// +// CONCURRENCY: one file per cache KEY (`<sha>.json`), not one per writer. +// Distinct calls write distinct files; two agents racing on the *same* call +// write byte-identical content, so the race is benign. Writes go through a +// temp file + `rename`, which is atomic on POSIX, so a reader never observes +// a half-written entry. No locking, no lost updates, no torn reads. +// +// WHAT IS NOT CACHED (deliberate): +// - Any command that fails (non-zero exit). A transient `gh` rate-limit or +// an expired kube token must never be memoized into a persistent "answer" +// that poisons every later reader. +// - Anything matching the mutation denylist below. The plugin is read-only +// by contract, but caching is a correctness-sensitive place to trust that +// contract blindly, so mutations are refused defensively. +// - Secrets: values that look like tokens/passwords are redacted from the +// stored payload before it ever touches disk. + +import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, renameSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { createHash } from "node:crypto"; + +/** Per-build cache directory, sitting alongside the state CSV and evidence + * file under the same OS-temp convention. */ +export function toolCacheDirFor(buildId, stateDir = "") { + const safe = String(buildId ?? "").replace(/[^A-Za-z0-9._-]/g, "_") || "unknown-build"; + const dir = stateDir && String(stateDir).trim() !== "" ? String(stateDir) : join(tmpdir(), "bstack-rca"); + return join(dir, `rca-toolcache.${safe}`); +} + +/** Stable key for one call. Whitespace is normalized so trivially-different + * formatting of the same command still hits, but nothing else is rewritten — + * `| head -20` vs `| head -200` genuinely return different output and must + * stay distinct keys. */ +export function cacheKey(command) { + const norm = String(command ?? "").replace(/\s+/g, " ").trim(); + return createHash("sha256").update(norm).digest("hex").slice(0, 24); +} + +// Commands that must never be memoized, even if someone wires this into a +// non-read-only context by mistake. +const MUTATING = /\b(rm|mv|cp|dd|truncate|tee)\b|>\s*\/|\bgit\s+(push|commit|merge|rebase|reset|checkout|clean)\b|\bgh\s+(pr\s+(create|merge|close|edit|comment|review)|issue\s+(create|close|edit|comment)|release\s+create|repo\s+(create|delete)|api\s+(-X\s*)?(POST|PUT|PATCH|DELETE))|\bkubectl\s+(apply|delete|edit|patch|scale|create|replace|annotate|label|cordon|drain|exec|cp|port-forward|rollout\s+(undo|restart|pause|resume))\b|\bcurl\b[^|]*\s-(X|-request)\s*(POST|PUT|PATCH|DELETE)/i; + +export function isCacheable(command) { + return !MUTATING.test(String(command ?? "")); +} + +// Read-only data-fetching binaries this wrapper will run. Anything else is +// refused outright. +const ALLOWED_LEADER = /^\s*(gh|kubectl|curl|git)\s/; + +// Shell constructs that chain a *second* command onto the one being cached: +// `;`, `&&`, `||`, background `&`, command substitution, and output redirects. +// Plain pipes are allowed on purpose — `| jq`, `| grep`, `| head` are how +// callers narrow a fetch, and they don't introduce a new root command. +const CHAINING = /[;&]|\|\||\$\(|`|>>?/; + +/** + * Gate for `bin/cached-exec.mjs`. Returns `{ ok, reason }`. + * + * Scope note, stated plainly: this is defense-in-depth, NOT a security + * boundary. The only caller is a coordinator agent that already has direct + * shell access via its Bash tool, so the wrapper grants no capability the + * caller lacks and cannot meaningfully contain a caller that wants to misuse + * it. What it does buy: a fat-fingered or model-hallucinated command can't + * quietly run something mutating *through the cache path* and get memoized, + * and refusing `;`-chained loops nudges callers toward one-fetch-per-call, + * which caches far better anyway. + */ +export function isRunnable(command) { + const c = String(command ?? ""); + if (!ALLOWED_LEADER.test(c)) { + return { ok: false, reason: "command must start with gh, kubectl, curl, or git" }; + } + if (CHAINING.test(c)) { + return { + ok: false, + reason: "chaining/substitution/redirect not allowed — issue one fetch per call (pipes to jq/grep/head are fine outside the wrapper)", + }; + } + if (!isCacheable(c)) return { ok: false, reason: "command looks mutating" }; + return { ok: true }; +} + +/** + * Split a command string into argv the way a shell would for the simple cases + * we allow — honouring single/double quotes — WITHOUT invoking a shell. The + * caller then runs `execFile(argv[0], argv.slice(1))`, so no shell ever + * interprets metacharacters and command injection has no surface. Throws on + * an unterminated quote rather than guessing. + */ +export function tokenize(command) { + const out = []; + let cur = ""; + let quote = null; + let started = false; + for (const ch of String(command ?? "")) { + if (quote) { + if (ch === quote) quote = null; + else cur += ch; + continue; + } + if (ch === "'" || ch === '"') { quote = ch; started = true; continue; } + if (/\s/.test(ch)) { + if (started) { out.push(cur); cur = ""; started = false; } + continue; + } + cur += ch; + started = true; + } + if (quote) throw new Error("unterminated quote in command"); + if (started) out.push(cur); + return out; +} + +// ---- MCP calls ------------------------------------------------------------ + +// Stateful MCP tools that must NEVER be memoized. `tfaRcaTurn` advances a +// conversation and `getTfaTurnResult` reads a turn whose status is *expected* +// to change between reads — serving either from cache would be actively +// wrong, not merely stale. +const MCP_NEVER = /tfaRcaTurn|getTfaTurnResult|triggerRcaReport/i; + +export function isCacheableMcp(toolName) { + return !MCP_NEVER.test(String(toolName ?? "")); +} + +/** Key an MCP call by tool name + canonicalized args (object keys sorted), so + * the same query written with its arguments in a different order still hits. */ +export function mcpCacheKey(toolName, args) { + const canon = (v) => { + if (Array.isArray(v)) return v.map(canon); + if (v && typeof v === "object") { + return Object.keys(v).sort().reduce((a, k) => { a[k] = canon(v[k]); return a; }, {}); + } + return v; + }; + const payload = JSON.stringify({ tool: String(toolName ?? ""), args: canon(args ?? {}) }); + return createHash("sha256").update(payload).digest("hex").slice(0, 24); +} + +// Redact the REST OF THE LINE after a secret-ish key, not just the next +// token: `Authorization: Bearer <tok>` puts the actual credential in the +// second word, so a `\S+` capture would leave it sitting on disk. +const SECRET_LINE = + /((?:token|authorization|api[_-]?key|secret|password|passwd|bearer|access[_-]?key)\s*[=:]\s*)([^\r\n]*)/gi; + +/** Redact anything token-shaped before it is persisted. The cache lives in + * temp, but a cached `gh api` response or log line could still carry a + * credential, and "it's only temp" is not a reason to write one to disk. */ +export function redact(text) { + return String(text ?? "").replace(SECRET_LINE, (_m, k) => `${k}<redacted>`); +} + +const MAX_BYTES = 256 * 1024; + +export function cacheGet(cacheDir, key) { + const p = join(cacheDir, `${key}.json`); + if (!existsSync(p)) return null; + try { + return JSON.parse(readFileSync(p, "utf8")); + } catch { + return null; // half-written or corrupt -> treat as a miss, never throw + } +} + +/** Atomic write: temp file + rename, so concurrent readers only ever see a + * complete entry. `nowMs` is passed in (same clock discipline as the rest of + * lib/). Returns the stored entry. */ +export function cachePut(cacheDir, key, entry, nowMs) { + if (!existsSync(cacheDir)) mkdirSync(cacheDir, { recursive: true }); + const raw = redact(entry.stdout ?? ""); + const truncated = raw.length > MAX_BYTES; + const rec = { + key, + command: entry.command, + writerId: entry.writerId ?? null, + capturedAtMs: nowMs, + exitCode: entry.exitCode ?? 0, + truncated, + bytes: raw.length, + stdout: truncated ? raw.slice(0, MAX_BYTES) + "\n… [truncated by tool-cache]" : raw, + }; + const finalPath = join(cacheDir, `${key}.json`); + const tmpPath = join(cacheDir, `.${key}.${process.pid}.tmp`); + writeFileSync(tmpPath, JSON.stringify(rec, null, 2), "utf8"); + renameSync(tmpPath, finalPath); // atomic on POSIX + return rec; +} + +/** Cache-wide counters for the run's summary — how much duplicate work this + * actually saved, rather than assuming it saved any. */ +export function cacheStats(cacheDir) { + if (!existsSync(cacheDir)) return { entries: 0, bytes: 0 }; + let entries = 0; + let bytes = 0; + for (const f of readdirSync(cacheDir)) { + if (!f.endsWith(".json") || f.startsWith(".")) continue; + entries++; + try { + bytes += JSON.parse(readFileSync(join(cacheDir, f), "utf8")).bytes ?? 0; + } catch { + /* skip */ + } + } + return { entries, bytes }; +} diff --git a/skills/rca-build/SKILL.md b/skills/rca-build/SKILL.md index 686441d..a4e6632 100644 --- a/skills/rca-build/SKILL.md +++ b/skills/rca-build/SKILL.md @@ -349,6 +349,17 @@ that agent type — no need to repeat the mechanics in the prompt, just don't omit `evidenceFilePath` (above), since write-back has nothing to write to without it. +**Also hand every dispatch the tool cache.** The evidence file shares digested +*findings*; `bin/cached-exec.mjs` / `bin/cached-mcp.mjs` share raw *call +results*, which is where most duplicate work actually hides — on one measured +build `gh` was 37% of all coordinator tool calls and 46 were byte-identical +commands re-run by different coordinators. Include the plugin root in each +dispatch prompt so coordinators can invoke the wrappers, and tell them to pass +their own `testRunId` as `writerId`. The cache lives at +`<tmpdir>/bstack-rca/rca-toolcache.<buildId>/`, one file per call key, shared +by shell and MCP alike. Read `node bin/cached-exec.mjs <buildId> --stats` at +the end of the run to report how much it actually saved rather than assuming. + **Concurrency is handled by layout, not by locking.** Base (`rca-evidence.<buildId>.json`) has exactly one writer — this orchestrator, in Step 4. Every coordinator writes only its own shard under diff --git a/tests/conformance.test.mjs b/tests/conformance.test.mjs index 33ccbbb..d9af539 100644 --- a/tests/conformance.test.mjs +++ b/tests/conformance.test.mjs @@ -174,6 +174,67 @@ test("a failed read is not a verdict — the drain keeps reading and still lands assert.equal(reads, 2); }); +test("a PERSISTENT hard error stops the drain early instead of burning the budget", async () => { + const fx = load("soft-pending-drain.json"); + let reads = 0; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit([fx.turns[0]]), // always soft-PENDING + readTurn: async () => { + reads++; + throw new Error("Failed to get tfa turn result: TFA agent run failed"); + }, + // Budget allows 40 reads; the error cap must cut it off long before that. + config: { ...CONFIG, softPendingDrain: { maxWaitMs: 600_000, intervalMs: 1, maxReads: 40, maxErrorReads: 3 } }, + sleep: noSleep, + }); + assert.equal(result.status, "PENDING"); + assert.equal(reads, 3, "stopped at maxErrorReads, not the 40-read budget"); + assert.match(result.root_cause, /tfa-error/); + assert.equal(result.turnId, "c2e1a6fd-2243-4f93-bc69-62f298db062c"); // still resumable +}); + +test("an error-shaped RESULT (not thrown) also trips the fast-fail", async () => { + const fx = load("soft-pending-drain.json"); + let reads = 0; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit([fx.turns[0]]), + // The MCP tool reports the wedge as a returned payload, not an exception. + readTurn: async () => { + reads++; + return { status: "ERROR", message: "TFA agent run failed" }; + }, + config: { ...CONFIG, softPendingDrain: { maxWaitMs: 600_000, intervalMs: 1, maxReads: 40, maxErrorReads: 2 } }, + sleep: noSleep, + }); + assert.equal(result.status, "PENDING"); + assert.equal(reads, 2); + assert.match(result.root_cause, /tfa-error/); +}); + +test("INTERMITTENT errors do not trip the fast-fail — a good read clears the streak", async () => { + const fx = load("soft-pending-drain.json"); + const landed = fx.reads[2]; + let reads = 0; + const result = await runRcaLoop({ + testRunId: fx.testRunId, + submit: replaySubmit(fx.turns), + readTurn: async () => { + reads++; + // fail, ok, fail, ok, ... never 2 consecutive failures + if (reads % 2 === 1) throw new Error("transient 502"); + return reads < 6 ? { status: "PENDING" } : landed; + }, + config: { ...CONFIG, softPendingDrain: { maxWaitMs: 600_000, intervalMs: 1, maxReads: 40, maxErrorReads: 2 } }, + manifest: GITHUB_AVAILABLE, + gather, + sleep: noSleep, + }); + assert.equal(result.status, "RESOLVED", "flaky-but-recovering reads must still land"); + assert.equal(reads, 6); +}); + test("BLOCKED surfaced by a drain is terminal — no empty resubmits to the turn cap", async () => { const fx = load("soft-pending-drain.json"); let submits = 0; diff --git a/tests/tool-cache.test.mjs b/tests/tool-cache.test.mjs new file mode 100644 index 0000000..f4549a7 --- /dev/null +++ b/tests/tool-cache.test.mjs @@ -0,0 +1,151 @@ +import { test, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + toolCacheDirFor, cacheKey, mcpCacheKey, cacheGet, cachePut, cacheStats, + isCacheable, isCacheableMcp, isRunnable, redact, tokenize, +} from "../lib/tool-cache.mjs"; + +let dir; +beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "rca-toolcache-")); }); +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +test("toolCacheDirFor: build id in the path, OS temp default, stateDir override", () => { + assert.ok(toolCacheDirFor("b1").startsWith(join(tmpdir(), "bstack-rca"))); + assert.ok(toolCacheDirFor("b1").endsWith("rca-toolcache.b1")); + assert.equal(toolCacheDirFor("b1", "/ci/art"), join("/ci/art", "rca-toolcache.b1")); + assert.ok(toolCacheDirFor("../../etc").endsWith("rca-toolcache..._.._etc")); +}); + +test("cacheKey: whitespace-insensitive, but content-sensitive", () => { + assert.equal(cacheKey("gh api repos/a"), cacheKey("gh api repos/a")); + assert.notEqual(cacheKey("gh api repos/a | head -20"), cacheKey("gh api repos/a | head -200")); +}); + +test("put then get round-trips", () => { + const k = cacheKey("gh api repos/a"); + cachePut(dir, k, { command: "gh api repos/a", writerId: "w1", stdout: "hello" }, 1000); + const hit = cacheGet(dir, k); + assert.equal(hit.stdout, "hello"); + assert.equal(hit.writerId, "w1"); + assert.equal(hit.capturedAtMs, 1000); +}); + +test("get on a miss returns null, never throws", () => { + assert.equal(cacheGet(dir, cacheKey("never run")), null); +}); + +test("a corrupt entry reads as a miss rather than throwing", () => { + const k = cacheKey("gh api repos/a"); + writeFileSync(join(dir, `${k}.json`), "{not json", "utf8"); + assert.equal(cacheGet(dir, k), null); +}); + +test("secrets are redacted before anything is written to disk", () => { + const k = cacheKey("gh api repos/a"); + cachePut(dir, k, { + command: "gh api repos/a", + stdout: 'ok\nAuthorization: Bearer abc123SECRET\napi_key=zzz999\ndone', + }, 1000); + const raw = cacheGet(dir, k).stdout; + assert.ok(!raw.includes("abc123SECRET"), "bearer token must not persist"); + assert.ok(!raw.includes("zzz999"), "api_key must not persist"); + assert.ok(raw.includes("<redacted>")); +}); + +test("redact leaves ordinary output untouched", () => { + assert.equal(redact("just some log output"), "just some log output"); +}); + +test("oversized payloads are truncated and flagged", () => { + const k = cacheKey("gh api big"); + const rec = cachePut(dir, k, { command: "gh api big", stdout: "x".repeat(400 * 1024) }, 1000); + assert.equal(rec.truncated, true); + assert.ok(rec.stdout.includes("[truncated by tool-cache]")); +}); + +test("cacheStats counts entries", () => { + cachePut(dir, "k1", { command: "a", stdout: "12345" }, 1); + cachePut(dir, "k2", { command: "b", stdout: "123" }, 1); + const s = cacheStats(dir); + assert.equal(s.entries, 2); + assert.equal(s.bytes, 8); +}); + +test("isCacheable rejects mutating shell commands", () => { + assert.equal(isCacheable("gh api repos/a"), true); + assert.equal(isCacheable("kubectl get pods"), true); + assert.equal(isCacheable("kubectl delete pod x"), false); + assert.equal(isCacheable("kubectl exec pod -- sh"), false); + assert.equal(isCacheable("gh pr create --title x"), false); + assert.equal(isCacheable("gh api -X POST repos/a"), false); + assert.equal(isCacheable("git push origin main"), false); + assert.equal(isCacheable("rm -rf /tmp/x"), false); +}); + +test("isRunnable enforces an allowlisted read-only leader", () => { + assert.equal(isRunnable("gh api repos/a").ok, true); + assert.equal(isRunnable("kubectl get pods -n regression").ok, true); + assert.equal(isRunnable("python3 -c 'print(1)'").ok, false); + assert.equal(isRunnable("sh -c 'echo hi'").ok, false); +}); + +test("isRunnable rejects chaining, substitution and redirects", () => { + assert.equal(isRunnable("gh api a; rm -rf /").ok, false); + assert.equal(isRunnable("gh api a && kubectl delete pod x").ok, false); + assert.equal(isRunnable("gh api $(whoami)").ok, false); + assert.equal(isRunnable("gh api a > /etc/passwd").ok, false); + assert.equal(isRunnable("gh api a `id`").ok, false); +}); + +test("tokenize splits like a shell for quoted args, without a shell", () => { + assert.deepEqual(tokenize("gh api repos/a --jq '.items[].path'"), + ["gh", "api", "repos/a", "--jq", ".items[].path"]); + assert.deepEqual(tokenize('kubectl get pods -o "custom:.metadata.name"'), + ["kubectl", "get", "pods", "-o", "custom:.metadata.name"]); + assert.throws(() => tokenize("gh api 'unterminated"), /unterminated quote/); +}); + +test("tokenize keeps injection payloads as ONE literal argument", () => { + // With execFile + these argv, no shell ever sees the metacharacters. + const argv = tokenize(`gh api "repos/a;rm -rf /"`); + assert.deepEqual(argv, ["gh", "api", "repos/a;rm -rf /"]); +}); + +test("MCP: stateful tools are never cacheable", () => { + assert.equal(isCacheableMcp("mcp__grafana__query_loki_logs"), true); + assert.equal(isCacheableMcp("mcp__browserstack__listTestIds"), true); + assert.equal(isCacheableMcp("mcp__browserstack__tfaRcaTurn"), false); + assert.equal(isCacheableMcp("mcp__browserstack__getTfaTurnResult"), false); + assert.equal(isCacheableMcp("mcp__browserstack__triggerRcaReport"), false); +}); + +test("mcpCacheKey is argument-order independent but value sensitive", () => { + const a = mcpCacheKey("t", { b: 2, a: 1 }); + const b = mcpCacheKey("t", { a: 1, b: 2 }); + assert.equal(a, b); + assert.notEqual(a, mcpCacheKey("t", { a: 1, b: 3 })); + assert.notEqual(a, mcpCacheKey("other", { a: 1, b: 2 })); +}); + +test("mcpCacheKey canonicalizes nested objects and arrays", () => { + assert.equal( + mcpCacheKey("t", { q: { z: 1, y: [{ n: 1, m: 2 }] } }), + mcpCacheKey("t", { q: { y: [{ m: 2, n: 1 }], z: 1 } }), + ); +}); + +test("an MCP result round-trips through the shared store", () => { + const k = mcpCacheKey("mcp__grafana__query_loki_logs", { ns: "regression", limit: 50 }); + cachePut(dir, k, { command: "grafana query", writerId: "3889074893", stdout: "0 rows, clean" }, 1000); + assert.equal(cacheGet(dir, k).stdout, "0 rows, clean"); +}); + +test("CONCURRENCY: same key written twice stays readable and consistent", () => { + const k = cacheKey("gh api repos/a"); + cachePut(dir, k, { command: "gh api repos/a", writerId: "w1", stdout: "same-bytes" }, 1000); + cachePut(dir, k, { command: "gh api repos/a", writerId: "w2", stdout: "same-bytes" }, 2000); + assert.equal(cacheGet(dir, k).stdout, "same-bytes"); +}); diff --git a/workflows/rca-batch.mjs b/workflows/rca-batch.mjs index 2a7a0e4..4992bee 100644 --- a/workflows/rca-batch.mjs +++ b/workflows/rca-batch.mjs @@ -23,7 +23,8 @@ export const meta = { // { // csvPath, buildId, // manifest: { capability: { available, via } }, -// evidenceFilePath, // NEW — lib/evidence-file.mjs artifact for this build +// evidenceFilePath, // lib/evidence-file.mjs artifact for this build +// pluginRoot, // absolute path to this plugin, so coordinators can call bin/cached-exec.mjs // buildEvidence: { baselineRef, isFallback, suspectWindow, reposCovered, workloadsCovered, gaps }, // // ^ SHRUNK to a cheap summary/pointer only — the full PR list / log // // sweeps live in the file at evidenceFilePath, read via each @@ -68,6 +69,7 @@ const shared = [ `Build-evidence summary (full detail is in the file above; this is only a pointer — do not re-fetch what the file already covers): ${JSON.stringify(ctx.buildEvidence ?? {})}`, `If the file's github/logs sections do not name a repo/workload/ask you need, or record a "gap" for it, that is a genuine gap — fall back to a live gather via the capability manifest above exactly as if no file existed. The file is an optimization, never a hard dependency.`, `The file is read-write: after any live gather that fills a gap or goes deeper than what was there, write it back via contributeGithubEvidence/contributeLogsEvidence (lib/evidence-file.mjs) passing your own testRunId as writerId, before finishing this test — so a sibling dispatched after you, or another cluster sharing the same repo/workload, reads the enriched entry instead of re-fetching it. Each writer owns its own shard file, so concurrent coordinators cannot clobber each other; readers fold base + shards automatically.`, + `Tool cache — route read-only lookups through it so duplicate calls across coordinators become hits. Shell: node ${ctx.pluginRoot ?? "<pluginRoot>"}/bin/cached-exec.mjs <buildId> <yourTestRunId> '<gh|kubectl|curl|git command>' (behaves like the raw command; pipe to jq/grep OUTSIDE the wrapper so different filters share one fetch). MCP data queries: cached-mcp.mjs <buildId> get|put <tool> '<argsJson>' [writerId]. NEVER cache tfaRcaTurn/getTfaTurnResult/triggerRcaReport — they are stateful. Do not re-probe connectors the gate already validated.`, `Autonomous run — on an evidence gap with no valid connector, report "unavailable" back to TFA (NEVER prompt a user). Best-effort finalize.`, `PRODUCT_BUG / application-bug mandate: hunt the culprit PR via the github connector (deploy timeline vs last-pass window, changed paths vs failure signature) and feed the PR link(s) to TFA so related_prs populates. No PR after digging to the turn cap → state explicitly "no culprit PR identified after <what was searched>" so the CSV row records the gap.`, `Soft-PENDING is NOT an answer: tfaRcaTurn abandons its in-call poll at 90s while TFA keeps working. On status PENDING, call getTfaTurnResult(testRunId, turnId) FIRST and keep reading on the softPendingDrain budget (every 5s, <=40 reads / <=10min) until the status is RESOLVED / NEEDS_INFO / BLOCKED, then continue the loop. Reads do NOT count against the turn cap. Never submit a new message onto a turn still in flight. Only a fully spent drain budget ends the test PENDING.`, From 6cabb9df87eed6634b8a2b4b8547b024bd8f77f3 Mon Sep 17 00:00:00 2001 From: harshit-bstack <harshit.m@browserstack.com> Date: Fri, 31 Jul 2026 16:45:23 +0530 Subject: [PATCH 46/47] fix(security): write RCA state, evidence and tool cache owner-only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three on-disk artifacts live under a world-readable OS temp dir and carry sensitive content: the tool cache holds raw gh/kubectl output (private repo source, internal hostnames, log bodies), the evidence file holds PR detail and app-log digests, and the state CSV holds root causes and culprit PRs. Default umask left them 0644 — readable by any local user. Create directories 0700 and write files 0600. The file mode is the load-bearing control: a pre-existing directory keeps its own permissions (silently chmod'ing a caller-supplied stateDir would be presumptuous), but entries are 0600 either way, and a traversable cache dir only exposes opaque hash filenames. Note this hardens storage, not the redaction: redact() is best-effort pattern matching over token-shaped strings and was never sufficient on its own to make these files safe to leave world-readable. Verified on disk for both a fresh dir (0700/0600) and a pre-existing 0755 dir (files still 0600). 111 tests pass. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- lib/csv-state.mjs | 6 ++++-- lib/evidence-file.mjs | 10 ++++++---- lib/tool-cache.mjs | 23 +++++++++++++++++++---- tests/evidence-file.test.mjs | 9 ++++++++- tests/tool-cache.test.mjs | 18 +++++++++++++++++- 5 files changed, 54 insertions(+), 12 deletions(-) diff --git a/lib/csv-state.mjs b/lib/csv-state.mjs index 152e51b..c0723e6 100644 --- a/lib/csv-state.mjs +++ b/lib/csv-state.mjs @@ -140,10 +140,12 @@ export function readRows(csvPath) { }); } +// Owner-only (0700 dir / 0600 file): the state CSV lives in a world-readable +// OS temp dir and records root causes, culprit PRs and evidence digests. export function writeRows(csvPath, rows) { const dir = dirname(csvPath); - if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true }); - writeFileSync(csvPath, encodeRows(rows), "utf8"); + if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true, mode: 0o700 }); + writeFileSync(csvPath, encodeRows(rows), { encoding: "utf8", mode: 0o600 }); } function emptyRow() { diff --git a/lib/evidence-file.mjs b/lib/evidence-file.mjs index b9010b6..c885309 100644 --- a/lib/evidence-file.mjs +++ b/lib/evidence-file.mjs @@ -190,10 +190,12 @@ export function readEvidenceFile(filePath) { return doc; } +// Owner-only (0700 dir / 0600 file): this sits in a world-readable OS temp dir +// and carries private-repo PR detail and app-log digests. export function writeEvidenceFile(filePath, doc) { const dir = dirname(filePath); - if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true }); - writeFileSync(filePath, JSON.stringify(doc, null, 2), "utf8"); + if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true, mode: 0o700 }); + writeFileSync(filePath, JSON.stringify(doc, null, 2), { encoding: "utf8", mode: 0o600 }); } function loadOrInit(filePath, nowMs) { @@ -274,8 +276,8 @@ function loadOwnShard(basePath, writerId, nowMs) { function writeShard(path, doc) { const dir = dirname(path); - if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true }); - writeFileSync(path, JSON.stringify(doc, null, 2), "utf8"); + if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true, mode: 0o700 }); + writeFileSync(path, JSON.stringify(doc, null, 2), { encoding: "utf8", mode: 0o600 }); } /** Contribute what THIS coordinator gathered live for a repo — a deeper diff --git a/lib/tool-cache.mjs b/lib/tool-cache.mjs index 94526ef..4f1507a 100644 --- a/lib/tool-cache.mjs +++ b/lib/tool-cache.mjs @@ -169,6 +169,8 @@ export function redact(text) { const MAX_BYTES = 256 * 1024; +let tmpSeq = 0; + export function cacheGet(cacheDir, key) { const p = join(cacheDir, `${key}.json`); if (!existsSync(p)) return null; @@ -183,7 +185,18 @@ export function cacheGet(cacheDir, key) { * complete entry. `nowMs` is passed in (same clock discipline as the rest of * lib/). Returns the stored entry. */ export function cachePut(cacheDir, key, entry, nowMs) { - if (!existsSync(cacheDir)) mkdirSync(cacheDir, { recursive: true }); + // Owner-only (0700 dir / 0600 files). The cache lives under a world-readable + // OS temp dir and holds raw `gh`/`kubectl` output — private repo source, + // internal hostnames, log bodies. `redact()` below is best-effort pattern + // matching and will not catch everything, so the filesystem permission is + // the actual control, not a backstop. + // + // The FILE mode is the load-bearing part: a pre-existing directory (e.g. a + // `stateDir` the user already created, or one left by an earlier run) keeps + // its own permissions, since silently chmod'ing a path we were handed would + // be presumptuous. Entries stay 0600 regardless, and a traversable directory + // only exposes opaque hash filenames, not their contents. + if (!existsSync(cacheDir)) mkdirSync(cacheDir, { recursive: true, mode: 0o700 }); const raw = redact(entry.stdout ?? ""); const truncated = raw.length > MAX_BYTES; const rec = { @@ -197,9 +210,11 @@ export function cachePut(cacheDir, key, entry, nowMs) { stdout: truncated ? raw.slice(0, MAX_BYTES) + "\n… [truncated by tool-cache]" : raw, }; const finalPath = join(cacheDir, `${key}.json`); - const tmpPath = join(cacheDir, `.${key}.${process.pid}.tmp`); - writeFileSync(tmpPath, JSON.stringify(rec, null, 2), "utf8"); - renameSync(tmpPath, finalPath); // atomic on POSIX + // pid + counter keeps the temp name unique per writer, so `mode` genuinely + // applies (it is honoured on create, not on truncate of an existing file). + const tmpPath = join(cacheDir, `.${key}.${process.pid}.${tmpSeq++}.tmp`); + writeFileSync(tmpPath, JSON.stringify(rec, null, 2), { encoding: "utf8", mode: 0o600 }); + renameSync(tmpPath, finalPath); // atomic on POSIX; preserves the 0600 mode return rec; } diff --git a/tests/evidence-file.test.mjs b/tests/evidence-file.test.mjs index dc40742..033d2f5 100644 --- a/tests/evidence-file.test.mjs +++ b/tests/evidence-file.test.mjs @@ -1,6 +1,6 @@ import { test, beforeEach, afterEach } from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -241,6 +241,13 @@ test("recomputeCoverage counts a coordinator-filled gap as covered", () => { assert.deepEqual(cov.reposGapped, []); }); +test("evidence file and contribution shards are owner-only (0600)", () => { + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "private PR detail" } }, 1000); + contributeGithubEvidence(file, "w1", "org/a", { prsInWindow: [{ pr: "#1" }] }, 2000); + assert.equal(statSync(file).mode & 0o777, 0o600); + assert.equal(statSync(contribPathFor(file, "w1")).mode & 0o777, 0o600); +}); + test("writeEvidenceFile creates the parent directory if missing", () => { const nested = join(dir, "nested", "sub", "evidence.json"); writeEvidenceFile(nested, emptyEvidenceFile("build-1", 0)); diff --git a/tests/tool-cache.test.mjs b/tests/tool-cache.test.mjs index f4549a7..91ea9fb 100644 --- a/tests/tool-cache.test.mjs +++ b/tests/tool-cache.test.mjs @@ -1,6 +1,6 @@ import { test, beforeEach, afterEach } from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync, statSync, readdirSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -143,6 +143,22 @@ test("an MCP result round-trips through the shared store", () => { assert.equal(cacheGet(dir, k).stdout, "0 rows, clean"); }); +test("cache files are owner-only (0600) and the dir owner-only (0700)", () => { + const sub = join(dir, "nested-cache"); + const k = cacheKey("gh api repos/a"); + cachePut(sub, k, { command: "gh api repos/a", stdout: "private repo source" }, 1000); + // The cache sits in a world-readable OS temp dir and holds raw gh/kubectl + // output; redaction is best-effort, so the mode is the real control. + assert.equal(statSync(join(sub, `${k}.json`)).mode & 0o777, 0o600); + assert.equal(statSync(sub).mode & 0o777, 0o700); +}); + +test("no temp file is left behind after an atomic put", () => { + const k = cacheKey("gh api repos/a"); + cachePut(dir, k, { command: "gh api repos/a", stdout: "x" }, 1000); + assert.deepEqual(readdirSync(dir).filter((f) => f.endsWith(".tmp")), []); +}); + test("CONCURRENCY: same key written twice stays readable and consistent", () => { const k = cacheKey("gh api repos/a"); cachePut(dir, k, { command: "gh api repos/a", writerId: "w1", stdout: "same-bytes" }, 1000); From 8e4a6d082144494d381454c55d1670de56f4632a Mon Sep 17 00:00:00 2001 From: harshit-bstack <harshit.m@browserstack.com> Date: Fri, 31 Jul 2026 18:49:26 +0530 Subject: [PATCH 47/47] fix(rca): six defects found by running the cache against a real build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dispatched three coordinators concurrently on a live build. They surfaced issues no unit test had, worst first: 1. CORRECTNESS: an empty `prsInWindow` with `gap: null` is byte-identical whether the PR search ran and found nothing or was never populated. Seen live — the file asserted 0 PRs for a repo that actually had 21 (the loader's search had silently returned empty), which would let a coordinator report "no culprit PR identified" with false confidence. Add an explicit `prsSearched` flag (sticky across contributors), a `hasTrustworthyPrList` helper, and a `coverage.reposWithUntrustedPrList` signal. Repo-level coverage semantics are unchanged — trustworthiness is reported alongside, not folded into, covered/gapped. 2. The runnable-guard scanned the raw command string, so it refused legitimate read-only calls: `;` inside a jq expression, `&` inside a quoted URL. Check whole argv TOKENS instead — post-tokenization a quoted metacharacter is inside an argument (harmless, we execFile) while a real operator is its own token. Verified both previously-refused commands now run. 3. execFileSync BOTH inherits and captures stderr, so relaying err.stderr printed failures three times. Capture only, relay once — wrapper stderr is now byte-identical to the command's own, plus one banner line. 4. Empty results were cached, making a sticky invisible negative. Not stored. 5. Nested single quotes made `--jq '...'` unpassable inside a quoted command argument. Accept `-` to read the command from stdin. 6. File mode applies on create only, so files left by a pre-hardening run kept 0644 forever. chmod explicitly on overwrite too. Also documented two traps that are usage, not bugs: `2>&1 | jq` merges the stderr banner into the pipe (the banner is correctly on stderr — one agent misreported this as a stdout bug; measured to confirm), and `cached-mcp get`'s exit code is lost behind a pipe. 118 tests pass, including regression tests for each fix. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- agents/ai-tfa-coordinator.md | 19 ++++++++++++++ bin/cached-exec.mjs | 50 +++++++++++++++++++++++++++++++++--- lib/csv-state.mjs | 5 +++- lib/evidence-file.mjs | 45 +++++++++++++++++++++++++++++++- lib/tool-cache.mjs | 33 +++++++++++++++++------- tests/evidence-file.test.mjs | 45 +++++++++++++++++++++++++++++++- tests/tool-cache.test.mjs | 26 ++++++++++++++++--- 7 files changed, 203 insertions(+), 20 deletions(-) diff --git a/agents/ai-tfa-coordinator.md b/agents/ai-tfa-coordinator.md index 1ae0096..163fe0d 100644 --- a/agents/ai-tfa-coordinator.md +++ b/agents/ai-tfa-coordinator.md @@ -136,6 +136,25 @@ read-only and has no side effects, so a read is always safe to repeat. they are stateful, and the cache refuses them outright. - Don't re-probe a connector the gate already validated (`gh auth status`, `kubectl version`); the manifest above is the answer. + - Two wrapper gotchas, both hit in real use: **(i)** hit/miss banners go to + stderr so `| jq` works, but `2>&1 | jq` merges the banner into the pipe + and jq dies on it — don't redirect stderr into a pipe. **(ii)** a command + containing its own single quotes (e.g. `--jq '.[] | "\(.number)"'`) can't + be nested inside a single-quoted argument; pipe it in on stdin instead: + `printf '%s' '<command>' | node .../cached-exec.mjs <buildId> <writerId> -`. + Metacharacters *inside* a quoted argument are fine — only a standalone + shell operator is refused, and a pipe belongs outside the wrapper anyway. + + **Never read an empty `prsInWindow` as "no PRs in the window."** An empty + list means "no PRs" ONLY when the entry also has `prsSearched: true`; + otherwise it was never populated and the two are indistinguishable in the + data. Check `coverage.reposWithUntrustedPrList` (or call + `hasTrustworthyPrList(doc, repo)`) before concluding anything from an empty + list — and when it is untrusted, run the PR search live. This is not + hypothetical: a pre-fetch once asserted 0 PRs for a repo that had 21, + which would have produced a confident "no culprit PR identified." When you + do run the search, contribute the result back — that records + `prsSearched` and spares everyone else the same trap. 1. **Logs by TFA — the core contract.** Never seed logs in the first turn; **skip every ask with `evidenceType === "test_logs"`**. Never fetch, paste, or digest log content. Logs are TFA's job. diff --git a/bin/cached-exec.mjs b/bin/cached-exec.mjs index 931292d..d0c6fad 100644 --- a/bin/cached-exec.mjs +++ b/bin/cached-exec.mjs @@ -8,6 +8,7 @@ // // Usage (command is ONE argument, so the caller's own quoting survives): // node bin/cached-exec.mjs <buildId> <writerId> '<command>' +// node bin/cached-exec.mjs <buildId> <writerId> - # command on STDIN // node bin/cached-exec.mjs <buildId> --stats // // Wrap only the expensive fetch and leave filtering to the outer shell: @@ -15,15 +16,43 @@ // Two coordinators piping the same fetch through different greps then share // one cache entry, instead of each paying for the fetch. // -// Cache hits/misses are reported on STDERR so stdout stays byte-identical to -// the raw command — piping into `grep`/`head`/`jq` is unaffected. +// TWO GOTCHAS, both hit in real use: +// +// 1. Hit/miss banners go to STDERR, so stdout stays byte-identical to the raw +// command and `| jq` works. But `2>&1 | jq` merges the banner back into +// the pipe and jq dies on it ("Invalid literal at line 1, column 12"). +// Don't redirect stderr into a pipe; if you must silence it, `2>/dev/null` +// — though that also hides whether you got a hit. +// +// 2. Nested single quotes. A command containing its own `'…'` (typically +// `--jq '.[] | "\(.number)"'`) cannot be passed inside a single-quoted +// argument — the outer shell terminates the string early and the argument +// arrives mangled. Use `-` and pipe the command in on stdin instead: +// printf '%s' 'gh pr list -R o/r --json number --jq ".[].number"' \ +// | node bin/cached-exec.mjs "$B" 3895 - import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; import { toolCacheDirFor, cacheKey, cacheGet, cachePut, cacheStats, isRunnable, tokenize, } from "../lib/tool-cache.mjs"; -const [, , buildId, writerOrFlag, command] = process.argv; +const [, , buildId, writerOrFlag, commandArg] = process.argv; + +// `-` means the command arrives on stdin, which sidesteps the nested-quoting +// problem entirely (see gotcha 2 above). +let command = commandArg; +if (command === "-") { + try { + command = readFileSync(0, "utf8").trim(); + } catch { + command = ""; + } + if (!command) { + console.error("[tool-cache] '-' given but stdin was empty"); + process.exit(2); + } +} if (!buildId || (writerOrFlag !== "--stats" && !command)) { console.error("usage: cached-exec.mjs <buildId> <writerId> '<command>'"); @@ -73,6 +102,10 @@ try { stdout = execFileSync(argv[0], argv.slice(1), { encoding: "utf8", maxBuffer: 64 * 1024 * 1024, + // Capture stderr instead of letting it inherit. Default execFileSync + // BOTH inherits stderr to the parent AND captures it on the error, so + // relaying `err.stderr` ourselves printed everything three times. + stdio: ["ignore", "pipe", "pipe"], }); } catch (err) { // Preserve the real behaviour of the wrapped command: emit whatever it @@ -80,12 +113,21 @@ try { // failure (rate limit, expired token) must not become a permanent answer. stdout = (err.stdout ?? "").toString(); exitCode = typeof err.status === "number" ? err.status : 1; - if (err.stderr) process.stderr.write(err.stderr.toString()); + if (err.stderr) process.stderr.write(err.stderr.toString()); // now the only copy console.error(`[tool-cache MISS ${key} — command exited ${exitCode}, NOT cached]`); process.stdout.write(stdout); process.exit(exitCode); } +// An empty result is not stored. It is usually a wrong selector or a silently +// failed lookup, and caching it makes a sticky, invisible negative that every +// later reader inherits — the expensive kind of wrong. +if (stdout.trim() === "") { + console.error(`[tool-cache MISS ${key} — empty result, NOT cached]`); + process.stdout.write(stdout); + process.exit(0); +} + // nowMs is read here, at the process edge, rather than inside lib/ — the // library keeps its no-clock discipline so it stays sandbox-safe. cachePut(dir, key, { command, writerId: writerOrFlag, stdout, exitCode }, Date.now()); diff --git a/lib/csv-state.mjs b/lib/csv-state.mjs index c0723e6..724eea3 100644 --- a/lib/csv-state.mjs +++ b/lib/csv-state.mjs @@ -13,7 +13,7 @@ // is sufficient for the in-process 5-concurrent workflow (true multi-process // locking is out of scope). -import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs"; +import { readFileSync, writeFileSync, existsSync, mkdirSync, chmodSync } from "node:fs"; import { dirname, join } from "node:path"; import { tmpdir } from "node:os"; @@ -145,7 +145,10 @@ export function readRows(csvPath) { export function writeRows(csvPath, rows) { const dir = dirname(csvPath); if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true, mode: 0o700 }); + const existed = existsSync(csvPath); writeFileSync(csvPath, encodeRows(rows), { encoding: "utf8", mode: 0o600 }); + // `mode` applies on create only — tighten a pre-hardening leftover too. + if (existed) chmodSync(csvPath, 0o600); } function emptyRow() { diff --git a/lib/evidence-file.mjs b/lib/evidence-file.mjs index c885309..2e76215 100644 --- a/lib/evidence-file.mjs +++ b/lib/evidence-file.mjs @@ -58,7 +58,7 @@ // 3. `prsInWindow` is unioned by PR number rather than replaced, so two // coordinators finding different PRs in the same repo both survive. -import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync } from "node:fs"; +import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, chmodSync } from "node:fs"; import { dirname, join } from "node:path"; import { tmpdir } from "node:os"; @@ -143,6 +143,10 @@ function foldGithub(target, repo, entry) { const next = { deployState: pickLeaf(cur.deployState, entry.deployState), prsInWindow: cur.prsInWindow ?? [], + // Sticky: once ANY writer has genuinely run the PR search, the entry stays + // trustworthy — a later contributor that didn't search must not silently + // downgrade it back to "unknown". + prsSearched: cur.prsSearched === true || entry.prsSearched === true, gap: cur.gap ?? null, }; if (Array.isArray(entry.prsInWindow)) { @@ -195,7 +199,12 @@ export function readEvidenceFile(filePath) { export function writeEvidenceFile(filePath, doc) { const dir = dirname(filePath); if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true, mode: 0o700 }); + const existed = existsSync(filePath); writeFileSync(filePath, JSON.stringify(doc, null, 2), { encoding: "utf8", mode: 0o600 }); + // `mode` is only honoured when the file is CREATED. A file left over from a + // run that predates this hardening would otherwise keep its old 0644 + // forever, so tighten it explicitly on overwrite too. + if (existed) chmodSync(filePath, 0o600); } function loadOrInit(filePath, nowMs) { @@ -294,7 +303,11 @@ export function contributeGithubEvidence(basePath, writerId, repo, patch, nowMs) const byPr = new Map((entry.prsInWindow ?? []).map((p) => [String(p.pr), p])); for (const pr of patch.prsInWindow) byPr.set(String(pr.pr), pr); entry.prsInWindow = [...byPr.values()]; + // Contributing a list — even an empty one — means you actually ran the + // search, so record that. Pass `prsSearched: false` explicitly to opt out. + entry.prsSearched = patch.prsSearched !== false; } + if (patch.prsSearched !== undefined) entry.prsSearched = patch.prsSearched; if (patch.gap !== undefined) entry.gap = patch.gap; doc.github[repo] = entry; doc.generatedAtMs = nowMs; @@ -324,11 +337,34 @@ export function contributeLogsEvidence(basePath, writerId, workload, patch, nowM // is falsy. Presence with a `gap` is a recorded, deliberate miss — not // coverage — so a coordinator (or this function) never mistakes "we looked // and couldn't get it" for "we have it." +// +// For a github entry there is a further trap, hit for real in testing: an +// empty `prsInWindow` is byte-identical whether the PR search RAN and found +// nothing, or was never populated at all. A coordinator trusting the former +// reads "no PRs in window" and concludes "no culprit PR" — confidently wrong. +// (Observed: a file asserting 0 PRs for a repo that actually had 21, because +// the loader's search silently returned empty.) So an empty `prsInWindow` +// only counts as coverage when `prsSearched === true` explicitly records that +// the search was really performed. +// Kept deliberately at the REPO level: an entry with deploy state but no PR +// search is still real coverage of that repo. PR-list trustworthiness is a +// narrower question, reported separately as `reposWithUntrustedPrList` so it +// is visible without distorting covered/gapped. function isCovered(doc, section, key) { const entry = doc[section]?.[key]; return Boolean(entry) && !entry.gap; } +/** True when this repo entry can be trusted to answer "which PRs were in the + * window" — i.e. it either lists PRs, or explicitly records that the search + * ran and legitimately found none. Coordinators should call this before + * concluding "no culprit PR" from the pre-fetch. */ +export function hasTrustworthyPrList(doc, repo) { + const entry = doc?.github?.[repo]; + if (!entry || entry.gap) return false; + return (entry.prsInWindow ?? []).length > 0 || entry.prsSearched === true; +} + /** Derives `doc.coverage` from exactly which requested repos/workloads have a * gap-free entry, and persists it. `requested = {repos:[...], workloads:[...]}` * — normally the Gate Part A scope-probe-validated repo list and the union of @@ -346,6 +382,13 @@ export function recomputeCoverage(filePath, requested, nowMs) { reposGapped: repos.filter((r) => !isCovered(folded, "github", r)), workloadsCovered: workloads.filter((w) => isCovered(folded, "logs", w)), workloadsGapped: workloads.filter((w) => !isCovered(folded, "logs", w)), + // Repos whose PR list must NOT be read as "no PRs in window": the list is + // empty and nothing recorded that a search actually ran. Surfacing this + // separately keeps a coordinator from concluding "no culprit PR" off an + // array that was simply never filled in. + reposWithUntrustedPrList: repos.filter( + (r) => isCovered(folded, "github", r) && !hasTrustworthyPrList(folded, r), + ), }; doc.generatedAtMs = nowMs; writeEvidenceFile(filePath, doc); diff --git a/lib/tool-cache.mjs b/lib/tool-cache.mjs index 4f1507a..01cf43c 100644 --- a/lib/tool-cache.mjs +++ b/lib/tool-cache.mjs @@ -30,7 +30,9 @@ // - Secrets: values that look like tokens/passwords are redacted from the // stored payload before it ever touches disk. -import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, renameSync } from "node:fs"; +import { + readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, renameSync, chmodSync, +} from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { createHash } from "node:crypto"; @@ -64,11 +66,15 @@ export function isCacheable(command) { // refused outright. const ALLOWED_LEADER = /^\s*(gh|kubectl|curl|git)\s/; -// Shell constructs that chain a *second* command onto the one being cached: -// `;`, `&&`, `||`, background `&`, command substitution, and output redirects. -// Plain pipes are allowed on purpose — `| jq`, `| grep`, `| head` are how -// callers narrow a fetch, and they don't introduce a new root command. -const CHAINING = /[;&]|\|\||\$\(|`|>>?/; +// Shell operators, checked as whole ARGV TOKENS rather than by scanning the +// raw string. Scanning the string was too blunt and refused legitimate reads: +// `--jq 'test("rcaThree";"i")'` was rejected for the `;` inside a quoted jq +// expression, and `search/code?q=X&per_page=20` for the `&` inside a URL. +// +// Post-tokenization this distinction is exact: quoted metacharacters end up +// *inside* an argument (harmless — we execFile, so no shell ever interprets +// them), while a real operator survives as its own standalone token. +const OPERATOR_TOKENS = new Set([";", "|", "||", "&&", "&", ">", ">>", "<", "<<"]); /** * Gate for `bin/cached-exec.mjs`. Returns `{ ok, reason }`. @@ -87,13 +93,22 @@ export function isRunnable(command) { if (!ALLOWED_LEADER.test(c)) { return { ok: false, reason: "command must start with gh, kubectl, curl, or git" }; } - if (CHAINING.test(c)) { + if (!isCacheable(c)) return { ok: false, reason: "command looks mutating" }; + + let argv; + try { + argv = tokenize(c); + } catch (err) { + return { ok: false, reason: err.message }; + } + const op = argv.find((t) => OPERATOR_TOKENS.has(t)); + if (op) { return { ok: false, - reason: "chaining/substitution/redirect not allowed — issue one fetch per call (pipes to jq/grep/head are fine outside the wrapper)", + reason: `'${op}' is a shell operator — run it OUTSIDE the wrapper (e.g. \`cached-exec … 'gh api X' | jq .y\`) so one cached fetch can serve several different filters. Metacharacters inside a quoted argument are fine.`, }; } - if (!isCacheable(c)) return { ok: false, reason: "command looks mutating" }; + if (argv.length === 0) return { ok: false, reason: "empty command" }; return { ok: true }; } diff --git a/tests/evidence-file.test.mjs b/tests/evidence-file.test.mjs index 033d2f5..e00cc2c 100644 --- a/tests/evidence-file.test.mjs +++ b/tests/evidence-file.test.mjs @@ -1,6 +1,6 @@ import { test, beforeEach, afterEach } from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, rmSync, writeFileSync, statSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync, statSync, chmodSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -17,6 +17,7 @@ import { contribDirFor, contribPathFor, readBaseFile, + hasTrustworthyPrList, recomputeCoverage, } from "../lib/evidence-file.mjs"; @@ -241,6 +242,48 @@ test("recomputeCoverage counts a coordinator-filled gap as covered", () => { assert.deepEqual(cov.reposGapped, []); }); +// Regression: an empty prsInWindow with gap:null used to read as "searched, +// found none" when it may simply never have been populated. Observed live — +// a file asserted 0 PRs for a repo that actually had 21, which would have let +// a coordinator conclude "no culprit PR" with false confidence. +test("empty prsInWindow is NOT coverage unless the search is recorded", () => { + setGithubEvidence(file, "org/never-searched", { gap: null, deployState: { block: "d" }, prsInWindow: [] }, 1000); + setGithubEvidence(file, "org/searched-empty", { gap: null, deployState: { block: "d" }, prsInWindow: [], prsSearched: true }, 1000); + const cov = recomputeCoverage(file, { repos: ["org/never-searched", "org/searched-empty"], workloads: [] }, 2000); + // Both repos ARE covered (each has deploy state) — but only one has a PR + // list safe to read as "no PRs in window". + assert.deepEqual(cov.reposCovered.sort(), ["org/never-searched", "org/searched-empty"]); + assert.deepEqual(cov.reposWithUntrustedPrList, ["org/never-searched"]); +}); + +test("hasTrustworthyPrList distinguishes searched-empty from never-populated", () => { + setGithubEvidence(file, "org/a", { gap: null, prsInWindow: [] }, 1000); + setGithubEvidence(file, "org/b", { gap: null, prsInWindow: [], prsSearched: true }, 1000); + setGithubEvidence(file, "org/c", { gap: null, prsInWindow: [{ pr: "#1" }] }, 1000); + const doc = readEvidenceFile(file); + assert.equal(hasTrustworthyPrList(doc, "org/a"), false); + assert.equal(hasTrustworthyPrList(doc, "org/b"), true); + assert.equal(hasTrustworthyPrList(doc, "org/c"), true); +}); + +test("contributing a PR list records that the search actually ran", () => { + contributeGithubEvidence(file, "w1", "org/a", { prsInWindow: [] }, 1000); + assert.equal(hasTrustworthyPrList(readEvidenceFile(file), "org/a"), true); +}); + +test("prsSearched is sticky — a later non-searching contributor cannot downgrade it", () => { + setGithubEvidence(file, "org/a", { gap: null, prsInWindow: [{ pr: "#1" }], prsSearched: true }, 1000); + contributeGithubEvidence(file, "w1", "org/a", { deployState: { block: "just deploy info" } }, 2000); + assert.equal(readEvidenceFile(file).github["org/a"].prsSearched, true); +}); + +test("a pre-existing loose-mode file is tightened to 0600 on the next write", () => { + writeEvidenceFile(file, emptyEvidenceFile("b", 0)); + chmodSync(file, 0o644); // simulate a file left by a pre-hardening run + setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "x" } }, 1000); + assert.equal(statSync(file).mode & 0o777, 0o600); +}); + test("evidence file and contribution shards are owner-only (0600)", () => { setGithubEvidence(file, "org/a", { gap: null, deployState: { block: "private PR detail" } }, 1000); contributeGithubEvidence(file, "w1", "org/a", { prsInWindow: [{ pr: "#1" }] }, 2000); diff --git a/tests/tool-cache.test.mjs b/tests/tool-cache.test.mjs index 91ea9fb..adaf4dd 100644 --- a/tests/tool-cache.test.mjs +++ b/tests/tool-cache.test.mjs @@ -92,12 +92,30 @@ test("isRunnable enforces an allowlisted read-only leader", () => { assert.equal(isRunnable("sh -c 'echo hi'").ok, false); }); -test("isRunnable rejects chaining, substitution and redirects", () => { - assert.equal(isRunnable("gh api a; rm -rf /").ok, false); +test("isRunnable rejects shell operators as standalone tokens", () => { + assert.equal(isRunnable("gh api a ; rm -rf /").ok, false); assert.equal(isRunnable("gh api a && kubectl delete pod x").ok, false); - assert.equal(isRunnable("gh api $(whoami)").ok, false); assert.equal(isRunnable("gh api a > /etc/passwd").ok, false); - assert.equal(isRunnable("gh api a `id`").ok, false); + assert.equal(isRunnable("gh api a | jq .x").ok, false); // pipe belongs outside +}); + +// Regression: the old raw-string guard refused these legitimate read-only +// calls, which is what pushed a coordinator into slower workarounds. +test("isRunnable ALLOWS metacharacters inside quoted arguments", () => { + const jqSemicolon = `gh api repos/o/r/git/trees/main --jq '[.tree[].path|select(test("rcaThree";"i"))]'`; + assert.equal(isRunnable(jqSemicolon).ok, true, "; inside a jq expression is not a shell operator"); + + const urlAmp = "gh api 'search/code?q=foo&per_page=20'"; + assert.equal(isRunnable(urlAmp).ok, true, "& inside a quoted URL is not a shell operator"); + + const jqPipe = `gh pr list -R o/r --json number --jq '.[] | .number'`; + assert.equal(isRunnable(jqPipe).ok, true, "| inside a quoted jq expression is not a shell pipe"); +}); + +test("a quoted metacharacter survives tokenization as ONE literal argument", () => { + const argv = tokenize(`gh api repos/o/r --jq '[.tree[]|select(test("x";"i"))]'`); + assert.equal(argv.length, 5); + assert.equal(argv[4], '[.tree[]|select(test("x";"i"))]'); }); test("tokenize splits like a shell for quoted args, without a shell", () => {