From 3cb995f61bee6afac25e19dfcf2008708f6aa720 Mon Sep 17 00:00:00 2001 From: bigboateng Date: Sat, 1 Aug 2026 16:59:41 +0100 Subject: [PATCH] feat(yield): add evaluation evidence surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disclosure-Reviewed: reviewed — public-safe only, private facet kept out of this commit --- .github/workflows/yield-lab.yml | 10 +++ .../2026-08-01-evaluation-surface.md | 9 ++ labs/22-yield/public-readme/README.md | 2 + labs/22-yield/publish.config.json | 3 + labs/22-yield/yield/docs/examples.md | 12 +++ labs/22-yield/yield/evals/.gitignore | 5 ++ labs/22-yield/yield/evals/README.md | 39 +++++++++ labs/22-yield/yield/evals/cases/README.md | 15 ++++ .../evals/cases/actions-auditor/SKILL.md | 17 ++++ .../evals/cases/actions-auditor/workflow.ts | 13 +++ .../evals/cases/doc-coauthoring/SKILL.md | 17 ++++ .../evals/cases/doc-coauthoring/workflow.ts | 15 ++++ .../yield/evals/cases/gstack-review/SKILL.md | 18 ++++ .../evals/cases/gstack-review/workflow.ts | 12 +++ labs/22-yield/yield/evals/cases/index.json | 84 +++++++++++++++++++ .../yield/evals/cases/mcp-builder/SKILL.md | 17 ++++ .../yield/evals/cases/mcp-builder/workflow.ts | 19 +++++ .../evals/cases/systematic-debugging/SKILL.md | 18 ++++ .../cases/systematic-debugging/workflow.ts | 16 ++++ .../yield/evals/cases/vercel-deploy/SKILL.md | 14 ++++ .../evals/cases/vercel-deploy/workflow.ts | 18 ++++ labs/22-yield/yield/evals/package-lock.json | 21 +++++ labs/22-yield/yield/evals/package.json | 13 +++ labs/22-yield/yield/evals/results/README.md | 17 ++++ labs/22-yield/yield/evals/results/latest.json | 41 +++++++++ .../yield/evals/scripts/measure-source.mjs | 59 +++++++++++++ .../22-yield/yield/evals/scripts/validate.mjs | 82 ++++++++++++++++++ 27 files changed, 606 insertions(+) create mode 100644 labs/22-yield/distribution/release-notes/2026-08-01-evaluation-surface.md create mode 100644 labs/22-yield/yield/evals/.gitignore create mode 100644 labs/22-yield/yield/evals/README.md create mode 100644 labs/22-yield/yield/evals/cases/README.md create mode 100644 labs/22-yield/yield/evals/cases/actions-auditor/SKILL.md create mode 100644 labs/22-yield/yield/evals/cases/actions-auditor/workflow.ts create mode 100644 labs/22-yield/yield/evals/cases/doc-coauthoring/SKILL.md create mode 100644 labs/22-yield/yield/evals/cases/doc-coauthoring/workflow.ts create mode 100644 labs/22-yield/yield/evals/cases/gstack-review/SKILL.md create mode 100644 labs/22-yield/yield/evals/cases/gstack-review/workflow.ts create mode 100644 labs/22-yield/yield/evals/cases/index.json create mode 100644 labs/22-yield/yield/evals/cases/mcp-builder/SKILL.md create mode 100644 labs/22-yield/yield/evals/cases/mcp-builder/workflow.ts create mode 100644 labs/22-yield/yield/evals/cases/systematic-debugging/SKILL.md create mode 100644 labs/22-yield/yield/evals/cases/systematic-debugging/workflow.ts create mode 100644 labs/22-yield/yield/evals/cases/vercel-deploy/SKILL.md create mode 100644 labs/22-yield/yield/evals/cases/vercel-deploy/workflow.ts create mode 100644 labs/22-yield/yield/evals/package-lock.json create mode 100644 labs/22-yield/yield/evals/package.json create mode 100644 labs/22-yield/yield/evals/results/README.md create mode 100644 labs/22-yield/yield/evals/results/latest.json create mode 100644 labs/22-yield/yield/evals/scripts/measure-source.mjs create mode 100644 labs/22-yield/yield/evals/scripts/validate.mjs diff --git a/.github/workflows/yield-lab.yml b/.github/workflows/yield-lab.yml index 27d10b87..457f0d8f 100644 --- a/.github/workflows/yield-lab.yml +++ b/.github/workflows/yield-lab.yml @@ -22,6 +22,11 @@ jobs: - uses: actions/setup-node@v4 with: node-version: "24" + - name: Validate evaluation cases and published summaries + working-directory: labs/22-yield/yield/evals + run: | + npm ci + npm run validate - name: Build and smoke-test the packed TypeScript SDK working-directory: labs/22-yield/yield/sdk/typescript run: | @@ -80,6 +85,11 @@ jobs: - uses: actions/setup-node@v4 with: node-version: "24" + - name: Validate projected evaluation cases and summaries + working-directory: ${{ runner.temp }}/yield-projected/evals + run: | + npm ci + npm run validate - name: Build and smoke-test the projected TypeScript package working-directory: ${{ runner.temp }}/yield-projected/sdk/typescript run: | diff --git a/labs/22-yield/distribution/release-notes/2026-08-01-evaluation-surface.md b/labs/22-yield/distribution/release-notes/2026-08-01-evaluation-surface.md new file mode 100644 index 00000000..ccc25656 --- /dev/null +++ b/labs/22-yield/distribution/release-notes/2026-08-01-evaluation-surface.md @@ -0,0 +1,9 @@ +# Evaluation cases and evidence boundary + +- Add six provenance-bound skill-conversion cases with their thin model-facing + skills and Yield workflow programs. +- Add a reproducible source-size harness and fail-closed summary validator. +- Keep raw transcripts, temporary repositories, and large run artifacts out of + Git; published behavior summaries must bind an immutable artifact digest. +- Validate the same evaluation surface in both the source tree and projected + public repository. diff --git a/labs/22-yield/public-readme/README.md b/labs/22-yield/public-readme/README.md index cab8eaf0..03565e08 100644 --- a/labs/22-yield/public-readme/README.md +++ b/labs/22-yield/public-readme/README.md @@ -81,6 +81,8 @@ use the documentation by job: - [tutorials](docs/tutorials/README.md) — review, approval, environment repair, bounded debugging, and migration; - [examples](docs/examples.md) — working programs in all four languages; +- [evaluations](evals/README.md) — pinned conversion cases, reproducible + measurements, and the raw-evidence publication boundary; - [convert an existing skill](docs/convert-existing-skill.md) — move control flow into code without claiming that fixture execution proves every reading of the original prose; diff --git a/labs/22-yield/publish.config.json b/labs/22-yield/publish.config.json index d338d5fb..46a5be4e 100644 --- a/labs/22-yield/publish.config.json +++ b/labs/22-yield/publish.config.json @@ -31,6 +31,9 @@ "dist", "build", "runs", + "raw", + "artifacts", + ".worktrees", "target" ] }, diff --git a/labs/22-yield/yield/docs/examples.md b/labs/22-yield/yield/docs/examples.md index cd14bdf2..e5638ad0 100644 --- a/labs/22-yield/yield/docs/examples.md +++ b/labs/22-yield/yield/docs/examples.md @@ -60,3 +60,15 @@ YSKILL=/tmp/yskill /tmp/yskill test examples/convert-skill When adapting an example, change the repository-specific commands and model instructions. Keep stable operation IDs for existing steps so saved runs can replay them. + +## Conversion cases used by evaluations + +The [evaluation cases](../evals/cases/) are different from the example library. +Each one pins a public third-party skill to an exact commit and digest, then +shows the smaller model-facing `SKILL.md` beside the TypeScript workflow that +owns its order, commands, gates, and completion. + +The source-size harness and early summaries live in [`evals/`](../evals/). +Large transcripts and temporary repositories are external artifacts, never +committed source. A behavioral result is publishable only when its summary +binds the exact artifact URI and SHA-256. diff --git a/labs/22-yield/yield/evals/.gitignore b/labs/22-yield/yield/evals/.gitignore new file mode 100644 index 00000000..63dfb88b --- /dev/null +++ b/labs/22-yield/yield/evals/.gitignore @@ -0,0 +1,5 @@ +runs/ +raw/ +artifacts/ +.worktrees/ +node_modules/ diff --git a/labs/22-yield/yield/evals/README.md b/labs/22-yield/yield/evals/README.md new file mode 100644 index 00000000..a8d144df --- /dev/null +++ b/labs/22-yield/yield/evals/README.md @@ -0,0 +1,39 @@ +# Yield evaluations + +This directory contains the public, reviewable part of Yield's evaluation +system: case definitions, pinned source identities, conversion programs, +measurement code, validation rules, and small result summaries. + +Raw agent transcripts, temporary repositories, command logs, and model +responses do not belong in Git. A full campaign uploads those files as one +immutable artifact bundle and records its URI and SHA-256 in the published +summary. Until that bundle exists, the summary must say `unpublished`. + +## Layout + +- `cases/` — pinned public source identities plus the thin skill and Yield + program used for each conversion. +- `scripts/measure-source.mjs` — reproduces the source-size comparison from + pinned upstream files. +- `scripts/validate.mjs` — fail-closed validation for cases and summaries. +- `results/latest.json` — small website-safe summary. It is not raw evidence. +- `runs/` — local or CI output; ignored by Git and projected releases. + +## Run + +```bash +npm install +npm run validate +npm run measure +``` + +`npm run measure` writes a fresh summary under `runs/`. Publishing that result +requires a separate promotion step that binds the raw artifact digest, the +exact Yield commit, model identity, harness version, and case-set digest. + +## Claim boundary + +Source-size measurements show how much model-facing text and workflow source +the prototypes contain. They do not prove behavioral equivalence. Behavioral +claims require executable fixtures, held-out oracles, repeated model runs, and +the immutable raw artifact named by the result summary. diff --git a/labs/22-yield/yield/evals/cases/README.md b/labs/22-yield/yield/evals/cases/README.md new file mode 100644 index 00000000..0f606b1b --- /dev/null +++ b/labs/22-yield/yield/evals/cases/README.md @@ -0,0 +1,15 @@ +# Conversion cases + +These are measured rewrites, not automatic equivalence claims. Each case keeps +the model-facing judgment in a short `SKILL.md` and moves repeatable control +flow into a TypeScript Yield program. The pinned original remains in its source +repository and is identified by commit plus SHA-256 in `index.json`. + +| Case | Thin skill | Yield program | Pinned original | +|---|---|---|---| +| GStack review | [SKILL.md](gstack-review/SKILL.md) | [workflow.ts](gstack-review/workflow.ts) | [source](https://github.com/garrytan/gstack/blob/a3259400a366593e0c909dd9ac3e59752efd2488/review/SKILL.md) | +| Anthropic doc co-authoring | [SKILL.md](doc-coauthoring/SKILL.md) | [workflow.ts](doc-coauthoring/workflow.ts) | [source](https://github.com/anthropics/skills/blob/b29e7cf65e5cb78a5ac33d582270551bc74a14eb/skills/doc-coauthoring/SKILL.md) | +| Superpowers systematic debugging | [SKILL.md](systematic-debugging/SKILL.md) | [workflow.ts](systematic-debugging/workflow.ts) | [source](https://github.com/obra/superpowers/blob/44c9b2d6e889982ac18c27d05a19fefe335194e1/skills/systematic-debugging/SKILL.md) | +| Vercel deploy | [SKILL.md](vercel-deploy/SKILL.md) | [workflow.ts](vercel-deploy/workflow.ts) | [source](https://github.com/vercel-labs/agent-skills/blob/7c180d9044c9ae2b442b567aad4e42a28dd5ed62/skills/deploy-to-vercel/SKILL.md) | +| Microsoft MCP builder | [SKILL.md](mcp-builder/SKILL.md) | [workflow.ts](mcp-builder/workflow.ts) | [source](https://github.com/microsoft/skills/blob/4a2873faffc1b101a33a0b59c24713d4ed78142f/.github/skills/mcp-builder/SKILL.md) | +| Trail of Bits actions auditor | [SKILL.md](actions-auditor/SKILL.md) | [workflow.ts](actions-auditor/workflow.ts) | [source](https://github.com/trailofbits/skills/blob/1256982d4d925a0acfe11e26c2253c32052c6247/plugins/agentic-actions-auditor/skills/agentic-actions-auditor/SKILL.md) | diff --git a/labs/22-yield/yield/evals/cases/actions-auditor/SKILL.md b/labs/22-yield/yield/evals/cases/actions-auditor/SKILL.md new file mode 100644 index 00000000..ffd83cee --- /dev/null +++ b/labs/22-yield/yield/evals/cases/actions-auditor/SKILL.md @@ -0,0 +1,17 @@ +--- +name: agentic-actions-auditor +description: Audit AI-enabled CI workflows for concrete security risks. +--- + +# Agentic actions auditor + +Inspect the supplied workflow files and repository context. Trace untrusted input +to agent prompts, tools, credentials, write permissions, network access, and +mutable dependencies. Report only findings with a concrete attack path. + +Each finding must include severity, workflow and line, source, capability reached, +impact, evidence, and remediation. Distinguish exploitable paths from hardening +advice. State coverage gaps explicitly. + +Yield owns file discovery, scope, evidence capture, required fields, report +generation, and completion. diff --git a/labs/22-yield/yield/evals/cases/actions-auditor/workflow.ts b/labs/22-yield/yield/evals/cases/actions-auditor/workflow.ts new file mode 100644 index 00000000..56aadc91 --- /dev/null +++ b/labs/22-yield/yield/evals/cases/actions-auditor/workflow.ts @@ -0,0 +1,13 @@ +export default defineSkill((ctx) => { + const files = ctx.runCommand("discover", "find .github/workflows -type f -name '*.yml' -o -name '*.yaml'", 30) + ctx.require(files.exit_code === 0 && files.stdout.length > 0, "workflow files found", files) + + const context = ctx.runCommand("context", "git ls-files && git status --short", 30) + const audit = ctx.agentTask("audit", prompts.audit, { files: files.stdout, context }) + ctx.require(audit.coverage.reviewed === audit.coverage.discovered, "all workflows reviewed", audit.coverage) + ctx.require(audit.findings.every((finding) => finding.evidence), "every finding has evidence", audit) + + const report = ctx.runCommand("report", "node scripts/render-audit.mjs", 60, { stdin: audit }) + ctx.require(report.exit_code === 0, "report generated", report) + return ctx.complete({ findings: audit.findings, report: report.stdout }) +}) diff --git a/labs/22-yield/yield/evals/cases/doc-coauthoring/SKILL.md b/labs/22-yield/yield/evals/cases/doc-coauthoring/SKILL.md new file mode 100644 index 00000000..7e7d62b5 --- /dev/null +++ b/labs/22-yield/yield/evals/cases/doc-coauthoring/SKILL.md @@ -0,0 +1,17 @@ +--- +name: doc-coauthoring +description: Help a user turn context into a clear document for a named reader. +--- + +# Doc co-authoring + +At each model step, ask only the questions needed for the current stage. Keep +the author's voice. Make claims concrete, expose missing evidence, and organize +the document around what its intended reader must understand or decide. + +For reader testing, act as a fresh reader with no hidden context. List unclear +terms, unanswered questions, and assumptions the document makes. Suggest the +smallest edits that resolve them. + +Yield owns stage order, saved answers, iteration limits, user choices, and the +definition of done. diff --git a/labs/22-yield/yield/evals/cases/doc-coauthoring/workflow.ts b/labs/22-yield/yield/evals/cases/doc-coauthoring/workflow.ts new file mode 100644 index 00000000..7ba13a57 --- /dev/null +++ b/labs/22-yield/yield/evals/cases/doc-coauthoring/workflow.ts @@ -0,0 +1,15 @@ +export default defineSkill((ctx) => { + const audience = ctx.askUser("audience", "Who will read this document?") + const outcome = ctx.askUser("outcome", "What should the reader know or decide?") + const context = ctx.askUser("context", "Paste the source context.") + + const outline = ctx.agentTask("outline", prompts.outline, { audience, outcome, context }) + const chosen = ctx.askUser("outline-approval", "Use this outline?", ["use", "revise"]) + ctx.require(chosen === "use", "outline approved", outline) + + const draft = ctx.agentTask("draft", prompts.draft, { outline, context }) + const test = ctx.agentTask("reader-test", prompts.readerTest, { audience, draft }) + const final = ctx.agentTask("revise", prompts.revise, { draft, test }) + ctx.require(test.blocking_questions.length === 0, "reader has no blocking questions", test) + return ctx.complete(final) +}) diff --git a/labs/22-yield/yield/evals/cases/gstack-review/SKILL.md b/labs/22-yield/yield/evals/cases/gstack-review/SKILL.md new file mode 100644 index 00000000..0c64cbbd --- /dev/null +++ b/labs/22-yield/yield/evals/cases/gstack-review/SKILL.md @@ -0,0 +1,18 @@ +--- +name: review +description: Review the current branch for important defects before shipping. +--- + +# Review + +Inspect the supplied diff and evidence. Look for failures tests may miss: +trust-boundary mistakes, unsafe side effects, incomplete error handling, +concurrency problems, and behavior that contradicts the surrounding code. + +Return structured findings with `severity`, `confidence`, `category`, `file`, +`line`, `problem`, and `fix`. Use only `critical` or `informational`. Count only +defects caused by this branch. Prefer a small number of specific findings over +general advice. + +Yield owns repository checks, ordering, policy validation, saved state, and +completion. Do not reproduce those rules in prose. diff --git a/labs/22-yield/yield/evals/cases/gstack-review/workflow.ts b/labs/22-yield/yield/evals/cases/gstack-review/workflow.ts new file mode 100644 index 00000000..14b7a617 --- /dev/null +++ b/labs/22-yield/yield/evals/cases/gstack-review/workflow.ts @@ -0,0 +1,12 @@ +export default defineSkill((ctx) => { + const diff = ctx.runCommand("diff", "git diff --merge-base origin/main HEAD", 60) + ctx.require(diff.exit_code === 0 && diff.stdout.length > 0, "branch has a readable diff", diff) + + const checks = ctx.runCommand("checks", "npm test && npm run typecheck", 600) + ctx.require(checks.exit_code === 0, "tests and types pass", checks) + + const review = ctx.agentTask("review", prompts.review, { diff: diff.stdout, checks }) + const critical = review.findings.some((finding) => finding.severity === "critical") + ctx.require(!critical, "no unresolved critical findings", review) + return ctx.complete(review) +}) diff --git a/labs/22-yield/yield/evals/cases/index.json b/labs/22-yield/yield/evals/cases/index.json new file mode 100644 index 00000000..c92575ab --- /dev/null +++ b/labs/22-yield/yield/evals/cases/index.json @@ -0,0 +1,84 @@ +{ + "schema_version": 1, + "methodology_version": "0.1", + "cases": [ + { + "id": "gstack-review", + "label": "GStack review", + "source": { + "repo": "garrytan/gstack", + "commit": "a3259400a366593e0c909dd9ac3e59752efd2488", + "path": "review/SKILL.md", + "license": "MIT", + "sha256": "92ee16af71d5e0088326869b0a211c50f94b9261eeae75656bc21f9bcfae2031" + }, + "thin_skill": "gstack-review/SKILL.md", + "workflow": "gstack-review/workflow.ts" + }, + { + "id": "doc-coauthoring", + "label": "Anthropic doc co-authoring", + "source": { + "repo": "anthropics/skills", + "commit": "b29e7cf65e5cb78a5ac33d582270551bc74a14eb", + "path": "skills/doc-coauthoring/SKILL.md", + "license": "See source repository", + "sha256": "2e47d78846faeea4a56e9809c52700087a15a2155a3f293a3efbaded81398ef4" + }, + "thin_skill": "doc-coauthoring/SKILL.md", + "workflow": "doc-coauthoring/workflow.ts" + }, + { + "id": "systematic-debugging", + "label": "Superpowers systematic debugging", + "source": { + "repo": "obra/superpowers", + "commit": "44c9b2d6e889982ac18c27d05a19fefe335194e1", + "path": "skills/systematic-debugging/SKILL.md", + "license": "MIT", + "sha256": "808fc5717aa88ad65efff312b11c186294d3e6ee301afb584e2f86599b137787" + }, + "thin_skill": "systematic-debugging/SKILL.md", + "workflow": "systematic-debugging/workflow.ts" + }, + { + "id": "vercel-deploy", + "label": "Vercel deploy", + "source": { + "repo": "vercel-labs/agent-skills", + "commit": "7c180d9044c9ae2b442b567aad4e42a28dd5ed62", + "path": "skills/deploy-to-vercel/SKILL.md", + "license": "See source repository", + "sha256": "cfcc3dd479ab2e0ae721ddf39b8af84d977321487672f1487c8d6855f576927b" + }, + "thin_skill": "vercel-deploy/SKILL.md", + "workflow": "vercel-deploy/workflow.ts" + }, + { + "id": "mcp-builder", + "label": "Microsoft MCP builder", + "source": { + "repo": "microsoft/skills", + "commit": "4a2873faffc1b101a33a0b59c24713d4ed78142f", + "path": ".github/skills/mcp-builder/SKILL.md", + "license": "MIT", + "sha256": "621e771c22224140752ddf933923467b2d1148580194ad59d3f1f21f9f27bdc9" + }, + "thin_skill": "mcp-builder/SKILL.md", + "workflow": "mcp-builder/workflow.ts" + }, + { + "id": "actions-auditor", + "label": "Trail of Bits actions auditor", + "source": { + "repo": "trailofbits/skills", + "commit": "1256982d4d925a0acfe11e26c2253c32052c6247", + "path": "plugins/agentic-actions-auditor/skills/agentic-actions-auditor/SKILL.md", + "license": "CC-BY-SA-4.0", + "sha256": "80e36ab06e3ee667ac45bab036ed395fc425c4d54fa72c4b4981a5b982a389aa" + }, + "thin_skill": "actions-auditor/SKILL.md", + "workflow": "actions-auditor/workflow.ts" + } + ] +} diff --git a/labs/22-yield/yield/evals/cases/mcp-builder/SKILL.md b/labs/22-yield/yield/evals/cases/mcp-builder/SKILL.md new file mode 100644 index 00000000..a9b860ee --- /dev/null +++ b/labs/22-yield/yield/evals/cases/mcp-builder/SKILL.md @@ -0,0 +1,17 @@ +--- +name: mcp-builder +description: Design and review a small MCP server around a defined use case. +--- + +# MCP builder + +Turn the supplied use case into a minimal tool surface. For each tool, define a +specific name, typed inputs, bounded output, error behavior, and one realistic +example. Prefer fewer composable tools. Keep secrets out of arguments and make +destructive effects explicit. + +During review, check schema clarity, transport errors, authentication boundaries, +pagination, idempotency, and whether evaluations cover success and failure paths. + +Yield owns research approval, scaffold and test commands, evaluation thresholds, +saved artifacts, and completion. diff --git a/labs/22-yield/yield/evals/cases/mcp-builder/workflow.ts b/labs/22-yield/yield/evals/cases/mcp-builder/workflow.ts new file mode 100644 index 00000000..ac5cc775 --- /dev/null +++ b/labs/22-yield/yield/evals/cases/mcp-builder/workflow.ts @@ -0,0 +1,19 @@ +export default defineSkill((ctx) => { + const useCase = ctx.askUser("use-case", "What must this MCP server enable?") + const constraints = ctx.askUser("constraints", "Which APIs, auth, and runtime apply?") + const design = ctx.agentTask("design", prompts.design, { useCase, constraints }) + ctx.require(design.tools.length <= 8, "tool surface stays small", design) + + const approval = ctx.askUser("design-approval", "Build this tool surface?", ["build", "revise"]) + if (approval !== "build") ctx.blocked("design needs revision") + const scaffold = ctx.runCommand("scaffold", "npm run scaffold:mcp", 120) + ctx.require(scaffold.exit_code === 0, "server scaffolds", scaffold) + const tests = ctx.runCommand("tests", "npm test", 600) + ctx.require(tests.exit_code === 0, "tests pass", tests) + + const review = ctx.agentTask("review", prompts.review, { design, tests }) + ctx.require(review.blockers.length === 0, "review has no blockers", review) + const evals = ctx.runCommand("evals", "npm run evals", 900) + ctx.require(evals.exit_code === 0, "evaluations pass", evals) + return ctx.complete({ design, review, evals }) +}) diff --git a/labs/22-yield/yield/evals/cases/systematic-debugging/SKILL.md b/labs/22-yield/yield/evals/cases/systematic-debugging/SKILL.md new file mode 100644 index 00000000..610fb8f0 --- /dev/null +++ b/labs/22-yield/yield/evals/cases/systematic-debugging/SKILL.md @@ -0,0 +1,18 @@ +--- +name: systematic-debugging +description: Diagnose a reproducible failure from evidence before proposing a fix. +--- + +# Systematic debugging + +Use the supplied failure output and code context to identify one falsifiable +root-cause hypothesis. Separate observations from inference. Name the mechanism, +the evidence that supports it, and the smallest experiment that could disprove +it. Do not recommend a fix until the hypothesis survives that experiment. + +When reviewing a candidate fix, check that it addresses the mechanism rather +than hiding the symptom and that the original failure now passes without a new +regression. + +Yield owns phase order, experiment bounds, commands, saved evidence, and exit +conditions. diff --git a/labs/22-yield/yield/evals/cases/systematic-debugging/workflow.ts b/labs/22-yield/yield/evals/cases/systematic-debugging/workflow.ts new file mode 100644 index 00000000..815e5b32 --- /dev/null +++ b/labs/22-yield/yield/evals/cases/systematic-debugging/workflow.ts @@ -0,0 +1,16 @@ +export default defineSkill((ctx) => { + const failure = ctx.runCommand("reproduce", "npm test -- --runInBand", 300) + ctx.require(failure.exit_code !== 0, "failure reproduces", failure) + + const hypothesis = ctx.agentTask("hypothesis", prompts.hypothesis, { failure }) + ctx.require(Boolean(hypothesis.experiment), "hypothesis is falsifiable", hypothesis) + const experiment = ctx.runCommand("experiment", hypothesis.experiment, 300) + const verdict = ctx.agentTask("evaluate", prompts.evaluate, { hypothesis, experiment }) + ctx.require(verdict.supported, "root cause supported by evidence", verdict) + + const fix = ctx.askUser("fix", "Apply the proposed fix?", ["apply", "stop"]) + if (fix !== "apply") ctx.blocked("fix not approved") + const verification = ctx.runCommand("verify", "npm test", 600) + ctx.require(verification.exit_code === 0, "full test suite passes", verification) + return ctx.complete({ hypothesis, verification }) +}) diff --git a/labs/22-yield/yield/evals/cases/vercel-deploy/SKILL.md b/labs/22-yield/yield/evals/cases/vercel-deploy/SKILL.md new file mode 100644 index 00000000..159eebd3 --- /dev/null +++ b/labs/22-yield/yield/evals/cases/vercel-deploy/SKILL.md @@ -0,0 +1,14 @@ +--- +name: deploy-to-vercel +description: Interpret deployment evidence and explain a failed Vercel release. +--- + +# Deploy to Vercel + +Given detected project state and command output, explain the selected deployment +path in plain language. If deployment fails, identify the failing boundary and +suggest one next action grounded in the output. Never claim a deployment is +live without a successful command and HTTP verification. + +Yield owns state detection, method selection, team choice, command execution, +timeouts, verification, and the final success gate. diff --git a/labs/22-yield/yield/evals/cases/vercel-deploy/workflow.ts b/labs/22-yield/yield/evals/cases/vercel-deploy/workflow.ts new file mode 100644 index 00000000..ef9425c5 --- /dev/null +++ b/labs/22-yield/yield/evals/cases/vercel-deploy/workflow.ts @@ -0,0 +1,18 @@ +export default defineSkill((ctx) => { + const git = ctx.runCommand("git-state", "git remote get-url origin", 20) + const linked = ctx.runCommand("vercel-state", "test -f .vercel/project.json", 20) + const auth = ctx.runCommand("auth", "vercel whoami", 30) + + const state = { git: git.exit_code === 0, linked: linked.exit_code === 0, auth: auth.exit_code === 0 } + const plan = ctx.agentTask("explain-plan", prompts.plan, state) + const approval = ctx.askUser("deploy", "Deploy using this plan?", ["deploy", "cancel"]) + if (approval !== "deploy") ctx.refused("deployment cancelled") + if (!state.auth) ctx.blocked("Vercel authentication required") + if (!state.linked) ctx.runCommand("link", "vercel link", 120) + + const deploy = ctx.runCommand("deploy", "vercel deploy --prod --yes", 900) + ctx.require(deploy.exit_code === 0, "deploy command succeeds", deploy) + const verify = ctx.runCommand("verify", `curl -fsS ${deploy.url}`, 120) + ctx.require(verify.exit_code === 0, "deployment responds over HTTPS", verify) + return ctx.complete({ url: deploy.url, plan }) +}) diff --git a/labs/22-yield/yield/evals/package-lock.json b/labs/22-yield/yield/evals/package-lock.json new file mode 100644 index 00000000..6b3084d7 --- /dev/null +++ b/labs/22-yield/yield/evals/package-lock.json @@ -0,0 +1,21 @@ +{ + "name": "@operatorstack/yield-evals", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@operatorstack/yield-evals", + "version": "0.1.0", + "dependencies": { + "gpt-tokenizer": "3.4.0" + } + }, + "node_modules/gpt-tokenizer": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/gpt-tokenizer/-/gpt-tokenizer-3.4.0.tgz", + "integrity": "sha512-wxFLnhIXTDjYebd9A9pGl3e31ZpSypbpIJSOswbgop5jLte/AsZVDvjlbEuVFlsqZixVKqbcoNmRlFDf6pz/UQ==", + "license": "MIT" + } + } +} diff --git a/labs/22-yield/yield/evals/package.json b/labs/22-yield/yield/evals/package.json new file mode 100644 index 00000000..7d3ba68e --- /dev/null +++ b/labs/22-yield/yield/evals/package.json @@ -0,0 +1,13 @@ +{ + "name": "@operatorstack/yield-evals", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "validate": "node scripts/validate.mjs", + "measure": "node scripts/measure-source.mjs" + }, + "dependencies": { + "gpt-tokenizer": "3.4.0" + } +} diff --git a/labs/22-yield/yield/evals/results/README.md b/labs/22-yield/yield/evals/results/README.md new file mode 100644 index 00000000..cbe99db5 --- /dev/null +++ b/labs/22-yield/yield/evals/results/README.md @@ -0,0 +1,17 @@ +# Published summaries + +`latest.json` is intentionally small enough for documentation and websites. +It may point to a large raw artifact, but it never embeds transcripts or +temporary repositories. + +A promoted behavior result must contain: + +- the exact Yield commit and eval case-set digest; +- model and harness versions; +- fixture, repeat, and arm counts; +- aggregate metrics with uncertainty intervals; +- an immutable artifact URI and SHA-256; +- explicit exclusions and untested boundaries. + +The current result is marked early and its raw artifact is unpublished. It is +useful for building the reporting surface, not as a final launch claim. diff --git a/labs/22-yield/yield/evals/results/latest.json b/labs/22-yield/yield/evals/results/latest.json new file mode 100644 index 00000000..f7fac150 --- /dev/null +++ b/labs/22-yield/yield/evals/results/latest.json @@ -0,0 +1,41 @@ +{ + "schema_version": 1, + "status": "early", + "source_size": { + "methodology_version": "0.1", + "generated_at": "2026-08-01T15:57:22.533Z", + "tokenizer": "gpt-tokenizer 3.4.0 / cl100k_base", + "summary": { + "original_tokens": 43069, + "prompt_tokens": 791, + "workflow_tokens": 1398, + "maintained_tokens": 2189, + "maintained_reduction_pct": 94.9 + }, + "rows": [ + {"id":"gstack-review","label":"GStack review","original_tokens":27162,"prompt_tokens":146,"workflow_tokens":160,"maintained_tokens":306,"maintained_change_pct":-98.9}, + {"id":"doc-coauthoring","label":"Anthropic doc co-authoring","original_tokens":3289,"prompt_tokens":137,"workflow_tokens":209,"maintained_tokens":346,"maintained_change_pct":-89.5}, + {"id":"systematic-debugging","label":"Superpowers systematic debugging","original_tokens":2226,"prompt_tokens":137,"workflow_tokens":226,"maintained_tokens":363,"maintained_change_pct":-83.7}, + {"id":"vercel-deploy","label":"Vercel deploy","original_tokens":2898,"prompt_tokens":108,"workflow_tokens":290,"maintained_tokens":398,"maintained_change_pct":-86.3}, + {"id":"mcp-builder","label":"Microsoft MCP builder","original_tokens":2648,"prompt_tokens":131,"workflow_tokens":291,"maintained_tokens":422,"maintained_change_pct":-84.1}, + {"id":"actions-auditor","label":"Trail of Bits actions auditor","original_tokens":4846,"prompt_tokens":132,"workflow_tokens":222,"maintained_tokens":354,"maintained_change_pct":-92.7} + ] + }, + "behavior": { + "methodology_version": "0.3", + "generated_at": "2026-08-01T13:59:25.489Z", + "model": "gpt-5.6-terra", + "fixture_count": 4, + "repeats": 3, + "paid_model_runs": 36, + "arms": { + "original": {"oracle_passes":11,"samples":12,"detections":5,"positives":6,"clean_passes":6,"clean_controls":6,"median_input_tokens":235477.5}, + "compressed": {"oracle_passes":12,"samples":12,"detections":6,"positives":6,"clean_passes":6,"clean_controls":6,"median_input_tokens":71905.5}, + "yield": {"oracle_passes":12,"samples":12,"detections":6,"positives":6,"clean_passes":6,"clean_controls":6,"median_input_tokens":14027.5,"deterministic_replays":12} + }, + "artifact": { + "status": "unpublished", + "reason": "The precursor run completed locally; its raw bundle has not been promoted to immutable public storage." + } + } +} diff --git a/labs/22-yield/yield/evals/scripts/measure-source.mjs b/labs/22-yield/yield/evals/scripts/measure-source.mjs new file mode 100644 index 00000000..273c0cfc --- /dev/null +++ b/labs/22-yield/yield/evals/scripts/measure-source.mjs @@ -0,0 +1,59 @@ +import { createHash } from "node:crypto" +import { mkdir, readFile, writeFile } from "node:fs/promises" +import { dirname, join, resolve } from "node:path" +import { fileURLToPath } from "node:url" +import { countTokens } from "gpt-tokenizer/encoding/cl100k_base" + +const root = resolve(dirname(fileURLToPath(import.meta.url)), "..") +const index = JSON.parse(await readFile(join(root, "cases/index.json"), "utf8")) +const measure = (text) => ({ + bytes: Buffer.byteLength(text), + lines: text === "" ? 0 : text.split(/\r?\n/).length, + tokens: countTokens(text), +}) +const sha256 = (text) => createHash("sha256").update(text).digest("hex") +const round = (value) => Math.round(value * 10) / 10 +const rows = [] + +for (const item of index.cases) { + const rawUrl = `https://raw.githubusercontent.com/${item.source.repo}/${item.source.commit}/${item.source.path}` + const response = await fetch(rawUrl, { headers: { "user-agent": "yield-evals/0.1" } }) + if (!response.ok) throw new Error(`${item.id}: source fetch failed (${response.status})`) + const originalText = await response.text() + if (sha256(originalText) !== item.source.sha256) throw new Error(`${item.id}: pinned source digest changed`) + + const skillText = await readFile(join(root, "cases", item.thin_skill), "utf8") + const workflowText = await readFile(join(root, "cases", item.workflow), "utf8") + const original = measure(originalText) + const prompt = measure(skillText) + const workflow = measure(workflowText) + rows.push({ + id: item.id, + label: item.label, + source_url: `https://github.com/${item.source.repo}/blob/${item.source.commit}/${item.source.path}`, + original_tokens: original.tokens, + prompt_tokens: prompt.tokens, + workflow_tokens: workflow.tokens, + maintained_tokens: prompt.tokens + workflow.tokens, + maintained_change_pct: round(((prompt.tokens + workflow.tokens) / original.tokens - 1) * 100), + }) +} + +const sum = (field) => rows.reduce((total, row) => total + row[field], 0) +const output = { + schema_version: 1, + methodology_version: index.methodology_version, + generated_at: new Date().toISOString(), + tokenizer: "gpt-tokenizer 3.4.0 / cl100k_base", + summary: { + original_tokens: sum("original_tokens"), + prompt_tokens: sum("prompt_tokens"), + workflow_tokens: sum("workflow_tokens"), + maintained_tokens: sum("maintained_tokens"), + }, + rows, +} +const stamp = output.generated_at.replaceAll(":", "-") +await mkdir(join(root, "runs"), { recursive: true }) +await writeFile(join(root, "runs", `source-size-${stamp}.json`), JSON.stringify(output, null, 2) + "\n") +console.log(JSON.stringify(output.summary, null, 2)) diff --git a/labs/22-yield/yield/evals/scripts/validate.mjs b/labs/22-yield/yield/evals/scripts/validate.mjs new file mode 100644 index 00000000..651f0075 --- /dev/null +++ b/labs/22-yield/yield/evals/scripts/validate.mjs @@ -0,0 +1,82 @@ +import { readdir, readFile, stat } from "node:fs/promises" +import { dirname, join, resolve } from "node:path" +import { fileURLToPath } from "node:url" +import { countTokens } from "gpt-tokenizer/encoding/cl100k_base" + +const root = resolve(dirname(fileURLToPath(import.meta.url)), "..") +const readJson = async (path) => JSON.parse(await readFile(join(root, path), "utf8")) +const fail = (message) => { throw new Error(message) } +const isSha256 = (value) => /^[0-9a-f]{64}$/.test(value) +const isCommit = (value) => /^[0-9a-f]{40}$/.test(value) +const round = (value) => Math.round(value * 10) / 10 + +const index = await readJson("cases/index.json") +const result = await readJson("results/latest.json") +if (index.schema_version !== 1) fail("unsupported case schema") +if (result.schema_version !== 1) fail("unsupported result schema") + +const ids = new Set() +const localMeasurements = new Map() +for (const item of index.cases) { + if (!item.id || ids.has(item.id)) fail(`duplicate or empty case id: ${item.id}`) + ids.add(item.id) + if (!isCommit(item.source.commit)) fail(`${item.id}: source commit must be a full SHA`) + if (!isSha256(item.source.sha256)) fail(`${item.id}: source sha256 is invalid`) + const skill = await readFile(join(root, "cases", item.thin_skill), "utf8") + const workflow = await readFile(join(root, "cases", item.workflow), "utf8") + if (!skill.trim() || !workflow.trim()) fail(`${item.id}: conversion source is empty`) + localMeasurements.set(item.id, { + prompt_tokens: countTokens(skill), + workflow_tokens: countTokens(workflow), + }) +} + +const rows = result.source_size.rows +if (rows.length !== ids.size) fail("result row count does not match cases") +for (const row of rows) { + if (!ids.has(row.id)) fail(`result references unknown case: ${row.id}`) + const local = localMeasurements.get(row.id) + if (row.prompt_tokens !== local.prompt_tokens || row.workflow_tokens !== local.workflow_tokens) { + fail(`${row.id}: published source-size row does not match the committed conversion`) + } + if (row.maintained_tokens !== row.prompt_tokens + row.workflow_tokens) { + fail(`${row.id}: maintained token total is inconsistent`) + } + const change = round((row.maintained_tokens / row.original_tokens - 1) * 100) + if (change !== row.maintained_change_pct) fail(`${row.id}: maintained change is inconsistent`) +} + +const sum = (field) => rows.reduce((total, row) => total + row[field], 0) +const totals = result.source_size.summary +for (const [field, rowField] of [ + ["original_tokens", "original_tokens"], + ["prompt_tokens", "prompt_tokens"], + ["workflow_tokens", "workflow_tokens"], + ["maintained_tokens", "maintained_tokens"], +]) { + if (totals[field] !== sum(rowField)) fail(`summary ${field} is inconsistent`) +} +const maintainedReduction = round((1 - totals.maintained_tokens / totals.original_tokens) * 100) +if (maintainedReduction !== totals.maintained_reduction_pct) fail("summary maintained reduction is inconsistent") + +const artifact = result.behavior.artifact +if (!["published", "unpublished"].includes(artifact.status)) fail("invalid artifact status") +if (artifact.status === "published" && (!artifact.uri || !isSha256(artifact.sha256))) { + fail("published behavior result requires an artifact URI and SHA-256") +} + +const forbidden = new Set(["runs", "raw", "artifacts", ".worktrees"]) +const walk = async (directory) => { + for (const entry of await readdir(directory, { withFileTypes: true })) { + if (entry.name === "node_modules") continue + const path = join(directory, entry.name) + if (entry.isDirectory()) { + if (directory !== root && forbidden.has(entry.name)) fail(`raw artifact directory is committed: ${path}`) + await walk(path) + } else if ((await stat(path)).size > 262144) { + fail(`evaluation source file exceeds 256 KiB: ${path}`) + } + } +} +await walk(root) +console.log(`validated ${ids.size} conversion cases and ${rows.length} result rows`)