diff --git a/README.md b/README.md index cab8eaf..03565e0 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,8 @@ use the documentation by job: - [tutorials](docs/tutorials/README.md) — review, approval, environment repair, bounded debugging, and migration; - [examples](docs/examples.md) — working programs in all four languages; +- [evaluations](evals/README.md) — pinned conversion cases, reproducible + measurements, and the raw-evidence publication boundary; - [convert an existing skill](docs/convert-existing-skill.md) — move control flow into code without claiming that fixture execution proves every reading of the original prose; diff --git a/UPSTREAM.json b/UPSTREAM.json index 7d5c35c..457e0f2 100644 --- a/UPSTREAM.json +++ b/UPSTREAM.json @@ -2,11 +2,11 @@ "files": { ".gitignore": "803c5f79d6da7f2c5a1dc0ce27c53b8e5c059309b165782831c4c472af058a4c", "LICENSE": "fff261ce507eabd57666c283a621f33e183a3aedebda04c4ecbc6309a62f5edf", - "README.md": "eba1fb9247b6512650fdd9ede5627fd0654a60f86eca66ccd4ad93db4b085340", + "README.md": "d3a13ebcd016ab1ba6a4fac64ca4a555ff738e9ce4c5c9744ee4e046e9dc08ef", "cmd/yskill/main.go": "a88bdd133118aa7b86cdf021e67052bd8e64e63b3792c091f7612699af89648b", "docs/README.md": "cd8a4105f4f04143172661b39789e3c533d031ad46437aa2c078625a29a0a9db", "docs/convert-existing-skill.md": "0dd538e908a1f2d3a1958e4b2cc1efede2a8e7a73341e32c1a75fec69a9acb8a", - "docs/examples.md": "e2614bd275c1ce58fa08050c063887ee72dbe19142259dd67f863f74d98e4227", + "docs/examples.md": "8c81f990b8d3a06b42cedcf34813a11dbe0606201782f72903a8c14d91d1ae23", "docs/locus-conformance.md": "a71fd5a72678dcad344afc71cd7a788090ef008a8e5e82d925da5a3db41af6a0", "docs/locus-converter.md": "b8d1bdb63cb0283ace524f572fb13f0d1cb535b130c5ea3e112d2efae55a8681", "docs/locus-yield.md": "c29d0c5801f32591828fc348385fb25c1850b39afc22770098eae0614fc3a417", @@ -39,6 +39,28 @@ "docs/tutorials/code-review.md": "275d9be9eeb8b10c840a58cb861258803af7629a54c4379d0d0336a9e8678898", "docs/tutorials/data-migration.md": "6699fc47cda84c4f46a3703ca4a69bcced7cc4a9df339a308e807ea3a99578bd", "docs/tutorials/environment-repair.md": "289c9b261e3768c2c58d60d9ef7a09e85f89038438a08febb4b9c12ebada88d1", + "evals/.gitignore": "0d5020173666118bafe31c857b96aa325809f41d159ca51324ceaf239e043347", + "evals/README.md": "c6fb08381877d3cddce22afe59746902ba73deebd78e52a414f85bcbbf441b22", + "evals/cases/README.md": "6f478571ce1850942c3797f31eab13059b97dbdfb67bbbaf62b95cbb50eaa115", + "evals/cases/actions-auditor/SKILL.md": "bdc81816f25144bcc44badb14ff3b46a234365466a8eeb6b3a6f6128ed57fd9b", + "evals/cases/actions-auditor/workflow.ts": "622f6dd037548a5ae0eb097a77392dbcc92eaa294691a4c4e30c4abfaa9544ec", + "evals/cases/doc-coauthoring/SKILL.md": "8ab3e342ef202f22fe2763d1d5e0d5d6918ccdf5a30426abfc02f71b8a0b00d8", + "evals/cases/doc-coauthoring/workflow.ts": "df78a605f5c0d7e52ba63ef2b91757dda1f4a5229fbe66fcb7c9adcb9aac9efb", + "evals/cases/gstack-review/SKILL.md": "9b4914d4e569cf70fdc794d88c1c1714d4e88b4646ab9762d29da2cc2b892fba", + "evals/cases/gstack-review/workflow.ts": "0743492bc14aeacd941cfbe90ef909ffe47f474941fd1cc28c33a38147ae6910", + "evals/cases/index.json": "b363426b292939f977360daaaad3ea4875a447f59657fed74275f6cc4084de43", + "evals/cases/mcp-builder/SKILL.md": "0a5c45ae7ad2e902f2a5c2c782e78db995b8e42869f36fad42839993e346dd5e", + "evals/cases/mcp-builder/workflow.ts": "fdf1ed76dc0cbe0eafd95ce1c8da7f8e65419e1f655c2df9ce88eaf285eee8a5", + "evals/cases/systematic-debugging/SKILL.md": "2d130b11d680d43f02ef126a77b462c585101e4df1632b89873597acdd7b189b", + "evals/cases/systematic-debugging/workflow.ts": "86aaacc7d1fab4e320e5ce0fab7bd9972c137d6b17e8ebddc6d118299d2a1a80", + "evals/cases/vercel-deploy/SKILL.md": "4ac80ad71f42c327f63685530b6b98ce3e287459527d2c0bafb5e1c716c313da", + "evals/cases/vercel-deploy/workflow.ts": "78667cea9b4b4d3dfc54ef6944ca9aa75d837b0170a4d3811b8f55172f63d6f2", + "evals/package-lock.json": "22c099aa5f2d9959084d6703e4a94ff7fa34216cac21b9d81137e2dd155c09d5", + "evals/package.json": "3361b8d579664bf9d6642eaeca6a906a1bca43fd0b1b543b00b2d5e0aa021b17", + "evals/results/README.md": "8eda7b901660466e60e5e32609cb2a3dcca7d9d512a93f2f379090b16da15525", + "evals/results/latest.json": "339a15841921fa270c668aa5c55cd33a271493735b4b85beaeaba1b25807216a", + "evals/scripts/measure-source.mjs": "064a62073422c067e12b0f2b77cc65a38f7d3440efb1a91020595a941470db7a", + "evals/scripts/validate.mjs": "eff5f24a95ebb40f86d561930b8ec7f47ec1437c0d358feb040f81b4fcc7d160", "examples/convert-skill/SKILL.md": "e6376f34365d4ac030d316db55e91f0c606a501668099f4ecf1e27d43ec806a2", "examples/convert-skill/fixtures/responses.json": "5b5f27b0ae5962360ac7b0e779992f430c655f352a38da37debb3505c32cd60a", "examples/convert-skill/main.go": "eb987ed1ce9139e6d0d40a2fae20c532c2fba3b27357e31555db2f0f52db4e39", @@ -255,6 +277,7 @@ "ir/yield.v1/request-envelope.schema.json": "2d5f34b04638450f1bd87ec1305ba7ecf0de2ec0ef6948365e6f46c2f2ee1c35", "ir/yield.v1/response-envelope.schema.json": "698fc20510bf1362cac17f332b8ec4b4dd336d2bee4294b1949ed3257b535639", "release-notes/2026-08-01-docs-and-typescript-package.md": "93382375cb47187092aff9447b280853151c738b140879828c74576a109195b8", + "release-notes/2026-08-01-evaluation-surface.md": "64fb5e2fdad3ccd41967e028cf4675c45939174000281c443f4b28d96dc07549", "release-notes/2026-08-01-example-library.md": "14d6ca40529a6aeeb72872295e57bb0d7dcc7824df31e029e497602060ce4c97", "release-notes/2026-08-01-initial-projection.md": "d38f0832b5552fb97237b30d19bb63369442ddde69e678b752ac07eedeb7ba3d", "release-notes/2026-08-01-multi-language-and-converter.md": "d0cf62d191e6a58e827f3b35d442f88b83be4dde98aa25ad54e4ae3ff787a453", @@ -272,7 +295,7 @@ "generator": "operatorstack/yield:project", "schema_version": 1, "source": { - "commit": "15a327cc48f28b7d7292b9412d44882c426152e4", + "commit": "ba7ac8e086564039b94ac3dd4cf5b82a247ba82f", "path": "labs/22-yield", "repository": "operatorstack/intelligence-flow" } diff --git a/docs/examples.md b/docs/examples.md index cd14bdf..e5638ad 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -60,3 +60,15 @@ YSKILL=/tmp/yskill /tmp/yskill test examples/convert-skill When adapting an example, change the repository-specific commands and model instructions. Keep stable operation IDs for existing steps so saved runs can replay them. + +## Conversion cases used by evaluations + +The [evaluation cases](../evals/cases/) are different from the example library. +Each one pins a public third-party skill to an exact commit and digest, then +shows the smaller model-facing `SKILL.md` beside the TypeScript workflow that +owns its order, commands, gates, and completion. + +The source-size harness and early summaries live in [`evals/`](../evals/). +Large transcripts and temporary repositories are external artifacts, never +committed source. A behavioral result is publishable only when its summary +binds the exact artifact URI and SHA-256. diff --git a/evals/.gitignore b/evals/.gitignore new file mode 100644 index 0000000..63dfb88 --- /dev/null +++ b/evals/.gitignore @@ -0,0 +1,5 @@ +runs/ +raw/ +artifacts/ +.worktrees/ +node_modules/ diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 0000000..a8d144d --- /dev/null +++ b/evals/README.md @@ -0,0 +1,39 @@ +# Yield evaluations + +This directory contains the public, reviewable part of Yield's evaluation +system: case definitions, pinned source identities, conversion programs, +measurement code, validation rules, and small result summaries. + +Raw agent transcripts, temporary repositories, command logs, and model +responses do not belong in Git. A full campaign uploads those files as one +immutable artifact bundle and records its URI and SHA-256 in the published +summary. Until that bundle exists, the summary must say `unpublished`. + +## Layout + +- `cases/` — pinned public source identities plus the thin skill and Yield + program used for each conversion. +- `scripts/measure-source.mjs` — reproduces the source-size comparison from + pinned upstream files. +- `scripts/validate.mjs` — fail-closed validation for cases and summaries. +- `results/latest.json` — small website-safe summary. It is not raw evidence. +- `runs/` — local or CI output; ignored by Git and projected releases. + +## Run + +```bash +npm install +npm run validate +npm run measure +``` + +`npm run measure` writes a fresh summary under `runs/`. Publishing that result +requires a separate promotion step that binds the raw artifact digest, the +exact Yield commit, model identity, harness version, and case-set digest. + +## Claim boundary + +Source-size measurements show how much model-facing text and workflow source +the prototypes contain. They do not prove behavioral equivalence. Behavioral +claims require executable fixtures, held-out oracles, repeated model runs, and +the immutable raw artifact named by the result summary. diff --git a/evals/cases/README.md b/evals/cases/README.md new file mode 100644 index 0000000..0f606b1 --- /dev/null +++ b/evals/cases/README.md @@ -0,0 +1,15 @@ +# Conversion cases + +These are measured rewrites, not automatic equivalence claims. Each case keeps +the model-facing judgment in a short `SKILL.md` and moves repeatable control +flow into a TypeScript Yield program. The pinned original remains in its source +repository and is identified by commit plus SHA-256 in `index.json`. + +| Case | Thin skill | Yield program | Pinned original | +|---|---|---|---| +| GStack review | [SKILL.md](gstack-review/SKILL.md) | [workflow.ts](gstack-review/workflow.ts) | [source](https://github.com/garrytan/gstack/blob/a3259400a366593e0c909dd9ac3e59752efd2488/review/SKILL.md) | +| Anthropic doc co-authoring | [SKILL.md](doc-coauthoring/SKILL.md) | [workflow.ts](doc-coauthoring/workflow.ts) | [source](https://github.com/anthropics/skills/blob/b29e7cf65e5cb78a5ac33d582270551bc74a14eb/skills/doc-coauthoring/SKILL.md) | +| Superpowers systematic debugging | [SKILL.md](systematic-debugging/SKILL.md) | [workflow.ts](systematic-debugging/workflow.ts) | [source](https://github.com/obra/superpowers/blob/44c9b2d6e889982ac18c27d05a19fefe335194e1/skills/systematic-debugging/SKILL.md) | +| Vercel deploy | [SKILL.md](vercel-deploy/SKILL.md) | [workflow.ts](vercel-deploy/workflow.ts) | [source](https://github.com/vercel-labs/agent-skills/blob/7c180d9044c9ae2b442b567aad4e42a28dd5ed62/skills/deploy-to-vercel/SKILL.md) | +| Microsoft MCP builder | [SKILL.md](mcp-builder/SKILL.md) | [workflow.ts](mcp-builder/workflow.ts) | [source](https://github.com/microsoft/skills/blob/4a2873faffc1b101a33a0b59c24713d4ed78142f/.github/skills/mcp-builder/SKILL.md) | +| Trail of Bits actions auditor | [SKILL.md](actions-auditor/SKILL.md) | [workflow.ts](actions-auditor/workflow.ts) | [source](https://github.com/trailofbits/skills/blob/1256982d4d925a0acfe11e26c2253c32052c6247/plugins/agentic-actions-auditor/skills/agentic-actions-auditor/SKILL.md) | diff --git a/evals/cases/actions-auditor/SKILL.md b/evals/cases/actions-auditor/SKILL.md new file mode 100644 index 0000000..ffd83ce --- /dev/null +++ b/evals/cases/actions-auditor/SKILL.md @@ -0,0 +1,17 @@ +--- +name: agentic-actions-auditor +description: Audit AI-enabled CI workflows for concrete security risks. +--- + +# Agentic actions auditor + +Inspect the supplied workflow files and repository context. Trace untrusted input +to agent prompts, tools, credentials, write permissions, network access, and +mutable dependencies. Report only findings with a concrete attack path. + +Each finding must include severity, workflow and line, source, capability reached, +impact, evidence, and remediation. Distinguish exploitable paths from hardening +advice. State coverage gaps explicitly. + +Yield owns file discovery, scope, evidence capture, required fields, report +generation, and completion. diff --git a/evals/cases/actions-auditor/workflow.ts b/evals/cases/actions-auditor/workflow.ts new file mode 100644 index 0000000..56aadc9 --- /dev/null +++ b/evals/cases/actions-auditor/workflow.ts @@ -0,0 +1,13 @@ +export default defineSkill((ctx) => { + const files = ctx.runCommand("discover", "find .github/workflows -type f -name '*.yml' -o -name '*.yaml'", 30) + ctx.require(files.exit_code === 0 && files.stdout.length > 0, "workflow files found", files) + + const context = ctx.runCommand("context", "git ls-files && git status --short", 30) + const audit = ctx.agentTask("audit", prompts.audit, { files: files.stdout, context }) + ctx.require(audit.coverage.reviewed === audit.coverage.discovered, "all workflows reviewed", audit.coverage) + ctx.require(audit.findings.every((finding) => finding.evidence), "every finding has evidence", audit) + + const report = ctx.runCommand("report", "node scripts/render-audit.mjs", 60, { stdin: audit }) + ctx.require(report.exit_code === 0, "report generated", report) + return ctx.complete({ findings: audit.findings, report: report.stdout }) +}) diff --git a/evals/cases/doc-coauthoring/SKILL.md b/evals/cases/doc-coauthoring/SKILL.md new file mode 100644 index 0000000..7e7d62b --- /dev/null +++ b/evals/cases/doc-coauthoring/SKILL.md @@ -0,0 +1,17 @@ +--- +name: doc-coauthoring +description: Help a user turn context into a clear document for a named reader. +--- + +# Doc co-authoring + +At each model step, ask only the questions needed for the current stage. Keep +the author's voice. Make claims concrete, expose missing evidence, and organize +the document around what its intended reader must understand or decide. + +For reader testing, act as a fresh reader with no hidden context. List unclear +terms, unanswered questions, and assumptions the document makes. Suggest the +smallest edits that resolve them. + +Yield owns stage order, saved answers, iteration limits, user choices, and the +definition of done. diff --git a/evals/cases/doc-coauthoring/workflow.ts b/evals/cases/doc-coauthoring/workflow.ts new file mode 100644 index 0000000..7ba13a5 --- /dev/null +++ b/evals/cases/doc-coauthoring/workflow.ts @@ -0,0 +1,15 @@ +export default defineSkill((ctx) => { + const audience = ctx.askUser("audience", "Who will read this document?") + const outcome = ctx.askUser("outcome", "What should the reader know or decide?") + const context = ctx.askUser("context", "Paste the source context.") + + const outline = ctx.agentTask("outline", prompts.outline, { audience, outcome, context }) + const chosen = ctx.askUser("outline-approval", "Use this outline?", ["use", "revise"]) + ctx.require(chosen === "use", "outline approved", outline) + + const draft = ctx.agentTask("draft", prompts.draft, { outline, context }) + const test = ctx.agentTask("reader-test", prompts.readerTest, { audience, draft }) + const final = ctx.agentTask("revise", prompts.revise, { draft, test }) + ctx.require(test.blocking_questions.length === 0, "reader has no blocking questions", test) + return ctx.complete(final) +}) diff --git a/evals/cases/gstack-review/SKILL.md b/evals/cases/gstack-review/SKILL.md new file mode 100644 index 0000000..0c64cbb --- /dev/null +++ b/evals/cases/gstack-review/SKILL.md @@ -0,0 +1,18 @@ +--- +name: review +description: Review the current branch for important defects before shipping. +--- + +# Review + +Inspect the supplied diff and evidence. Look for failures tests may miss: +trust-boundary mistakes, unsafe side effects, incomplete error handling, +concurrency problems, and behavior that contradicts the surrounding code. + +Return structured findings with `severity`, `confidence`, `category`, `file`, +`line`, `problem`, and `fix`. Use only `critical` or `informational`. Count only +defects caused by this branch. Prefer a small number of specific findings over +general advice. + +Yield owns repository checks, ordering, policy validation, saved state, and +completion. Do not reproduce those rules in prose. diff --git a/evals/cases/gstack-review/workflow.ts b/evals/cases/gstack-review/workflow.ts new file mode 100644 index 0000000..14b7a61 --- /dev/null +++ b/evals/cases/gstack-review/workflow.ts @@ -0,0 +1,12 @@ +export default defineSkill((ctx) => { + const diff = ctx.runCommand("diff", "git diff --merge-base origin/main HEAD", 60) + ctx.require(diff.exit_code === 0 && diff.stdout.length > 0, "branch has a readable diff", diff) + + const checks = ctx.runCommand("checks", "npm test && npm run typecheck", 600) + ctx.require(checks.exit_code === 0, "tests and types pass", checks) + + const review = ctx.agentTask("review", prompts.review, { diff: diff.stdout, checks }) + const critical = review.findings.some((finding) => finding.severity === "critical") + ctx.require(!critical, "no unresolved critical findings", review) + return ctx.complete(review) +}) diff --git a/evals/cases/index.json b/evals/cases/index.json new file mode 100644 index 0000000..c92575a --- /dev/null +++ b/evals/cases/index.json @@ -0,0 +1,84 @@ +{ + "schema_version": 1, + "methodology_version": "0.1", + "cases": [ + { + "id": "gstack-review", + "label": "GStack review", + "source": { + "repo": "garrytan/gstack", + "commit": "a3259400a366593e0c909dd9ac3e59752efd2488", + "path": "review/SKILL.md", + "license": "MIT", + "sha256": "92ee16af71d5e0088326869b0a211c50f94b9261eeae75656bc21f9bcfae2031" + }, + "thin_skill": "gstack-review/SKILL.md", + "workflow": "gstack-review/workflow.ts" + }, + { + "id": "doc-coauthoring", + "label": "Anthropic doc co-authoring", + "source": { + "repo": "anthropics/skills", + "commit": "b29e7cf65e5cb78a5ac33d582270551bc74a14eb", + "path": "skills/doc-coauthoring/SKILL.md", + "license": "See source repository", + "sha256": "2e47d78846faeea4a56e9809c52700087a15a2155a3f293a3efbaded81398ef4" + }, + "thin_skill": "doc-coauthoring/SKILL.md", + "workflow": "doc-coauthoring/workflow.ts" + }, + { + "id": "systematic-debugging", + "label": "Superpowers systematic debugging", + "source": { + "repo": "obra/superpowers", + "commit": "44c9b2d6e889982ac18c27d05a19fefe335194e1", + "path": "skills/systematic-debugging/SKILL.md", + "license": "MIT", + "sha256": "808fc5717aa88ad65efff312b11c186294d3e6ee301afb584e2f86599b137787" + }, + "thin_skill": "systematic-debugging/SKILL.md", + "workflow": "systematic-debugging/workflow.ts" + }, + { + "id": "vercel-deploy", + "label": "Vercel deploy", + "source": { + "repo": "vercel-labs/agent-skills", + "commit": "7c180d9044c9ae2b442b567aad4e42a28dd5ed62", + "path": "skills/deploy-to-vercel/SKILL.md", + "license": "See source repository", + "sha256": "cfcc3dd479ab2e0ae721ddf39b8af84d977321487672f1487c8d6855f576927b" + }, + "thin_skill": "vercel-deploy/SKILL.md", + "workflow": "vercel-deploy/workflow.ts" + }, + { + "id": "mcp-builder", + "label": "Microsoft MCP builder", + "source": { + "repo": "microsoft/skills", + "commit": "4a2873faffc1b101a33a0b59c24713d4ed78142f", + "path": ".github/skills/mcp-builder/SKILL.md", + "license": "MIT", + "sha256": "621e771c22224140752ddf933923467b2d1148580194ad59d3f1f21f9f27bdc9" + }, + "thin_skill": "mcp-builder/SKILL.md", + "workflow": "mcp-builder/workflow.ts" + }, + { + "id": "actions-auditor", + "label": "Trail of Bits actions auditor", + "source": { + "repo": "trailofbits/skills", + "commit": "1256982d4d925a0acfe11e26c2253c32052c6247", + "path": "plugins/agentic-actions-auditor/skills/agentic-actions-auditor/SKILL.md", + "license": "CC-BY-SA-4.0", + "sha256": "80e36ab06e3ee667ac45bab036ed395fc425c4d54fa72c4b4981a5b982a389aa" + }, + "thin_skill": "actions-auditor/SKILL.md", + "workflow": "actions-auditor/workflow.ts" + } + ] +} diff --git a/evals/cases/mcp-builder/SKILL.md b/evals/cases/mcp-builder/SKILL.md new file mode 100644 index 0000000..a9b860e --- /dev/null +++ b/evals/cases/mcp-builder/SKILL.md @@ -0,0 +1,17 @@ +--- +name: mcp-builder +description: Design and review a small MCP server around a defined use case. +--- + +# MCP builder + +Turn the supplied use case into a minimal tool surface. For each tool, define a +specific name, typed inputs, bounded output, error behavior, and one realistic +example. Prefer fewer composable tools. Keep secrets out of arguments and make +destructive effects explicit. + +During review, check schema clarity, transport errors, authentication boundaries, +pagination, idempotency, and whether evaluations cover success and failure paths. + +Yield owns research approval, scaffold and test commands, evaluation thresholds, +saved artifacts, and completion. diff --git a/evals/cases/mcp-builder/workflow.ts b/evals/cases/mcp-builder/workflow.ts new file mode 100644 index 0000000..ac5cc77 --- /dev/null +++ b/evals/cases/mcp-builder/workflow.ts @@ -0,0 +1,19 @@ +export default defineSkill((ctx) => { + const useCase = ctx.askUser("use-case", "What must this MCP server enable?") + const constraints = ctx.askUser("constraints", "Which APIs, auth, and runtime apply?") + const design = ctx.agentTask("design", prompts.design, { useCase, constraints }) + ctx.require(design.tools.length <= 8, "tool surface stays small", design) + + const approval = ctx.askUser("design-approval", "Build this tool surface?", ["build", "revise"]) + if (approval !== "build") ctx.blocked("design needs revision") + const scaffold = ctx.runCommand("scaffold", "npm run scaffold:mcp", 120) + ctx.require(scaffold.exit_code === 0, "server scaffolds", scaffold) + const tests = ctx.runCommand("tests", "npm test", 600) + ctx.require(tests.exit_code === 0, "tests pass", tests) + + const review = ctx.agentTask("review", prompts.review, { design, tests }) + ctx.require(review.blockers.length === 0, "review has no blockers", review) + const evals = ctx.runCommand("evals", "npm run evals", 900) + ctx.require(evals.exit_code === 0, "evaluations pass", evals) + return ctx.complete({ design, review, evals }) +}) diff --git a/evals/cases/systematic-debugging/SKILL.md b/evals/cases/systematic-debugging/SKILL.md new file mode 100644 index 0000000..610fb8f --- /dev/null +++ b/evals/cases/systematic-debugging/SKILL.md @@ -0,0 +1,18 @@ +--- +name: systematic-debugging +description: Diagnose a reproducible failure from evidence before proposing a fix. +--- + +# Systematic debugging + +Use the supplied failure output and code context to identify one falsifiable +root-cause hypothesis. Separate observations from inference. Name the mechanism, +the evidence that supports it, and the smallest experiment that could disprove +it. Do not recommend a fix until the hypothesis survives that experiment. + +When reviewing a candidate fix, check that it addresses the mechanism rather +than hiding the symptom and that the original failure now passes without a new +regression. + +Yield owns phase order, experiment bounds, commands, saved evidence, and exit +conditions. diff --git a/evals/cases/systematic-debugging/workflow.ts b/evals/cases/systematic-debugging/workflow.ts new file mode 100644 index 0000000..815e5b3 --- /dev/null +++ b/evals/cases/systematic-debugging/workflow.ts @@ -0,0 +1,16 @@ +export default defineSkill((ctx) => { + const failure = ctx.runCommand("reproduce", "npm test -- --runInBand", 300) + ctx.require(failure.exit_code !== 0, "failure reproduces", failure) + + const hypothesis = ctx.agentTask("hypothesis", prompts.hypothesis, { failure }) + ctx.require(Boolean(hypothesis.experiment), "hypothesis is falsifiable", hypothesis) + const experiment = ctx.runCommand("experiment", hypothesis.experiment, 300) + const verdict = ctx.agentTask("evaluate", prompts.evaluate, { hypothesis, experiment }) + ctx.require(verdict.supported, "root cause supported by evidence", verdict) + + const fix = ctx.askUser("fix", "Apply the proposed fix?", ["apply", "stop"]) + if (fix !== "apply") ctx.blocked("fix not approved") + const verification = ctx.runCommand("verify", "npm test", 600) + ctx.require(verification.exit_code === 0, "full test suite passes", verification) + return ctx.complete({ hypothesis, verification }) +}) diff --git a/evals/cases/vercel-deploy/SKILL.md b/evals/cases/vercel-deploy/SKILL.md new file mode 100644 index 0000000..159eebd --- /dev/null +++ b/evals/cases/vercel-deploy/SKILL.md @@ -0,0 +1,14 @@ +--- +name: deploy-to-vercel +description: Interpret deployment evidence and explain a failed Vercel release. +--- + +# Deploy to Vercel + +Given detected project state and command output, explain the selected deployment +path in plain language. If deployment fails, identify the failing boundary and +suggest one next action grounded in the output. Never claim a deployment is +live without a successful command and HTTP verification. + +Yield owns state detection, method selection, team choice, command execution, +timeouts, verification, and the final success gate. diff --git a/evals/cases/vercel-deploy/workflow.ts b/evals/cases/vercel-deploy/workflow.ts new file mode 100644 index 0000000..ef9425c --- /dev/null +++ b/evals/cases/vercel-deploy/workflow.ts @@ -0,0 +1,18 @@ +export default defineSkill((ctx) => { + const git = ctx.runCommand("git-state", "git remote get-url origin", 20) + const linked = ctx.runCommand("vercel-state", "test -f .vercel/project.json", 20) + const auth = ctx.runCommand("auth", "vercel whoami", 30) + + const state = { git: git.exit_code === 0, linked: linked.exit_code === 0, auth: auth.exit_code === 0 } + const plan = ctx.agentTask("explain-plan", prompts.plan, state) + const approval = ctx.askUser("deploy", "Deploy using this plan?", ["deploy", "cancel"]) + if (approval !== "deploy") ctx.refused("deployment cancelled") + if (!state.auth) ctx.blocked("Vercel authentication required") + if (!state.linked) ctx.runCommand("link", "vercel link", 120) + + const deploy = ctx.runCommand("deploy", "vercel deploy --prod --yes", 900) + ctx.require(deploy.exit_code === 0, "deploy command succeeds", deploy) + const verify = ctx.runCommand("verify", `curl -fsS ${deploy.url}`, 120) + ctx.require(verify.exit_code === 0, "deployment responds over HTTPS", verify) + return ctx.complete({ url: deploy.url, plan }) +}) diff --git a/evals/package-lock.json b/evals/package-lock.json new file mode 100644 index 0000000..6b3084d --- /dev/null +++ b/evals/package-lock.json @@ -0,0 +1,21 @@ +{ + "name": "@operatorstack/yield-evals", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@operatorstack/yield-evals", + "version": "0.1.0", + "dependencies": { + "gpt-tokenizer": "3.4.0" + } + }, + "node_modules/gpt-tokenizer": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/gpt-tokenizer/-/gpt-tokenizer-3.4.0.tgz", + "integrity": "sha512-wxFLnhIXTDjYebd9A9pGl3e31ZpSypbpIJSOswbgop5jLte/AsZVDvjlbEuVFlsqZixVKqbcoNmRlFDf6pz/UQ==", + "license": "MIT" + } + } +} diff --git a/evals/package.json b/evals/package.json new file mode 100644 index 0000000..7d3ba68 --- /dev/null +++ b/evals/package.json @@ -0,0 +1,13 @@ +{ + "name": "@operatorstack/yield-evals", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "validate": "node scripts/validate.mjs", + "measure": "node scripts/measure-source.mjs" + }, + "dependencies": { + "gpt-tokenizer": "3.4.0" + } +} diff --git a/evals/results/README.md b/evals/results/README.md new file mode 100644 index 0000000..cbe99db --- /dev/null +++ b/evals/results/README.md @@ -0,0 +1,17 @@ +# Published summaries + +`latest.json` is intentionally small enough for documentation and websites. +It may point to a large raw artifact, but it never embeds transcripts or +temporary repositories. + +A promoted behavior result must contain: + +- the exact Yield commit and eval case-set digest; +- model and harness versions; +- fixture, repeat, and arm counts; +- aggregate metrics with uncertainty intervals; +- an immutable artifact URI and SHA-256; +- explicit exclusions and untested boundaries. + +The current result is marked early and its raw artifact is unpublished. It is +useful for building the reporting surface, not as a final launch claim. diff --git a/evals/results/latest.json b/evals/results/latest.json new file mode 100644 index 0000000..f7fac15 --- /dev/null +++ b/evals/results/latest.json @@ -0,0 +1,41 @@ +{ + "schema_version": 1, + "status": "early", + "source_size": { + "methodology_version": "0.1", + "generated_at": "2026-08-01T15:57:22.533Z", + "tokenizer": "gpt-tokenizer 3.4.0 / cl100k_base", + "summary": { + "original_tokens": 43069, + "prompt_tokens": 791, + "workflow_tokens": 1398, + "maintained_tokens": 2189, + "maintained_reduction_pct": 94.9 + }, + "rows": [ + {"id":"gstack-review","label":"GStack review","original_tokens":27162,"prompt_tokens":146,"workflow_tokens":160,"maintained_tokens":306,"maintained_change_pct":-98.9}, + {"id":"doc-coauthoring","label":"Anthropic doc co-authoring","original_tokens":3289,"prompt_tokens":137,"workflow_tokens":209,"maintained_tokens":346,"maintained_change_pct":-89.5}, + {"id":"systematic-debugging","label":"Superpowers systematic debugging","original_tokens":2226,"prompt_tokens":137,"workflow_tokens":226,"maintained_tokens":363,"maintained_change_pct":-83.7}, + {"id":"vercel-deploy","label":"Vercel deploy","original_tokens":2898,"prompt_tokens":108,"workflow_tokens":290,"maintained_tokens":398,"maintained_change_pct":-86.3}, + {"id":"mcp-builder","label":"Microsoft MCP builder","original_tokens":2648,"prompt_tokens":131,"workflow_tokens":291,"maintained_tokens":422,"maintained_change_pct":-84.1}, + {"id":"actions-auditor","label":"Trail of Bits actions auditor","original_tokens":4846,"prompt_tokens":132,"workflow_tokens":222,"maintained_tokens":354,"maintained_change_pct":-92.7} + ] + }, + "behavior": { + "methodology_version": "0.3", + "generated_at": "2026-08-01T13:59:25.489Z", + "model": "gpt-5.6-terra", + "fixture_count": 4, + "repeats": 3, + "paid_model_runs": 36, + "arms": { + "original": {"oracle_passes":11,"samples":12,"detections":5,"positives":6,"clean_passes":6,"clean_controls":6,"median_input_tokens":235477.5}, + "compressed": {"oracle_passes":12,"samples":12,"detections":6,"positives":6,"clean_passes":6,"clean_controls":6,"median_input_tokens":71905.5}, + "yield": {"oracle_passes":12,"samples":12,"detections":6,"positives":6,"clean_passes":6,"clean_controls":6,"median_input_tokens":14027.5,"deterministic_replays":12} + }, + "artifact": { + "status": "unpublished", + "reason": "The precursor run completed locally; its raw bundle has not been promoted to immutable public storage." + } + } +} diff --git a/evals/scripts/measure-source.mjs b/evals/scripts/measure-source.mjs new file mode 100644 index 0000000..273c0cf --- /dev/null +++ b/evals/scripts/measure-source.mjs @@ -0,0 +1,59 @@ +import { createHash } from "node:crypto" +import { mkdir, readFile, writeFile } from "node:fs/promises" +import { dirname, join, resolve } from "node:path" +import { fileURLToPath } from "node:url" +import { countTokens } from "gpt-tokenizer/encoding/cl100k_base" + +const root = resolve(dirname(fileURLToPath(import.meta.url)), "..") +const index = JSON.parse(await readFile(join(root, "cases/index.json"), "utf8")) +const measure = (text) => ({ + bytes: Buffer.byteLength(text), + lines: text === "" ? 0 : text.split(/\r?\n/).length, + tokens: countTokens(text), +}) +const sha256 = (text) => createHash("sha256").update(text).digest("hex") +const round = (value) => Math.round(value * 10) / 10 +const rows = [] + +for (const item of index.cases) { + const rawUrl = `https://raw.githubusercontent.com/${item.source.repo}/${item.source.commit}/${item.source.path}` + const response = await fetch(rawUrl, { headers: { "user-agent": "yield-evals/0.1" } }) + if (!response.ok) throw new Error(`${item.id}: source fetch failed (${response.status})`) + const originalText = await response.text() + if (sha256(originalText) !== item.source.sha256) throw new Error(`${item.id}: pinned source digest changed`) + + const skillText = await readFile(join(root, "cases", item.thin_skill), "utf8") + const workflowText = await readFile(join(root, "cases", item.workflow), "utf8") + const original = measure(originalText) + const prompt = measure(skillText) + const workflow = measure(workflowText) + rows.push({ + id: item.id, + label: item.label, + source_url: `https://github.com/${item.source.repo}/blob/${item.source.commit}/${item.source.path}`, + original_tokens: original.tokens, + prompt_tokens: prompt.tokens, + workflow_tokens: workflow.tokens, + maintained_tokens: prompt.tokens + workflow.tokens, + maintained_change_pct: round(((prompt.tokens + workflow.tokens) / original.tokens - 1) * 100), + }) +} + +const sum = (field) => rows.reduce((total, row) => total + row[field], 0) +const output = { + schema_version: 1, + methodology_version: index.methodology_version, + generated_at: new Date().toISOString(), + tokenizer: "gpt-tokenizer 3.4.0 / cl100k_base", + summary: { + original_tokens: sum("original_tokens"), + prompt_tokens: sum("prompt_tokens"), + workflow_tokens: sum("workflow_tokens"), + maintained_tokens: sum("maintained_tokens"), + }, + rows, +} +const stamp = output.generated_at.replaceAll(":", "-") +await mkdir(join(root, "runs"), { recursive: true }) +await writeFile(join(root, "runs", `source-size-${stamp}.json`), JSON.stringify(output, null, 2) + "\n") +console.log(JSON.stringify(output.summary, null, 2)) diff --git a/evals/scripts/validate.mjs b/evals/scripts/validate.mjs new file mode 100644 index 0000000..651f007 --- /dev/null +++ b/evals/scripts/validate.mjs @@ -0,0 +1,82 @@ +import { readdir, readFile, stat } from "node:fs/promises" +import { dirname, join, resolve } from "node:path" +import { fileURLToPath } from "node:url" +import { countTokens } from "gpt-tokenizer/encoding/cl100k_base" + +const root = resolve(dirname(fileURLToPath(import.meta.url)), "..") +const readJson = async (path) => JSON.parse(await readFile(join(root, path), "utf8")) +const fail = (message) => { throw new Error(message) } +const isSha256 = (value) => /^[0-9a-f]{64}$/.test(value) +const isCommit = (value) => /^[0-9a-f]{40}$/.test(value) +const round = (value) => Math.round(value * 10) / 10 + +const index = await readJson("cases/index.json") +const result = await readJson("results/latest.json") +if (index.schema_version !== 1) fail("unsupported case schema") +if (result.schema_version !== 1) fail("unsupported result schema") + +const ids = new Set() +const localMeasurements = new Map() +for (const item of index.cases) { + if (!item.id || ids.has(item.id)) fail(`duplicate or empty case id: ${item.id}`) + ids.add(item.id) + if (!isCommit(item.source.commit)) fail(`${item.id}: source commit must be a full SHA`) + if (!isSha256(item.source.sha256)) fail(`${item.id}: source sha256 is invalid`) + const skill = await readFile(join(root, "cases", item.thin_skill), "utf8") + const workflow = await readFile(join(root, "cases", item.workflow), "utf8") + if (!skill.trim() || !workflow.trim()) fail(`${item.id}: conversion source is empty`) + localMeasurements.set(item.id, { + prompt_tokens: countTokens(skill), + workflow_tokens: countTokens(workflow), + }) +} + +const rows = result.source_size.rows +if (rows.length !== ids.size) fail("result row count does not match cases") +for (const row of rows) { + if (!ids.has(row.id)) fail(`result references unknown case: ${row.id}`) + const local = localMeasurements.get(row.id) + if (row.prompt_tokens !== local.prompt_tokens || row.workflow_tokens !== local.workflow_tokens) { + fail(`${row.id}: published source-size row does not match the committed conversion`) + } + if (row.maintained_tokens !== row.prompt_tokens + row.workflow_tokens) { + fail(`${row.id}: maintained token total is inconsistent`) + } + const change = round((row.maintained_tokens / row.original_tokens - 1) * 100) + if (change !== row.maintained_change_pct) fail(`${row.id}: maintained change is inconsistent`) +} + +const sum = (field) => rows.reduce((total, row) => total + row[field], 0) +const totals = result.source_size.summary +for (const [field, rowField] of [ + ["original_tokens", "original_tokens"], + ["prompt_tokens", "prompt_tokens"], + ["workflow_tokens", "workflow_tokens"], + ["maintained_tokens", "maintained_tokens"], +]) { + if (totals[field] !== sum(rowField)) fail(`summary ${field} is inconsistent`) +} +const maintainedReduction = round((1 - totals.maintained_tokens / totals.original_tokens) * 100) +if (maintainedReduction !== totals.maintained_reduction_pct) fail("summary maintained reduction is inconsistent") + +const artifact = result.behavior.artifact +if (!["published", "unpublished"].includes(artifact.status)) fail("invalid artifact status") +if (artifact.status === "published" && (!artifact.uri || !isSha256(artifact.sha256))) { + fail("published behavior result requires an artifact URI and SHA-256") +} + +const forbidden = new Set(["runs", "raw", "artifacts", ".worktrees"]) +const walk = async (directory) => { + for (const entry of await readdir(directory, { withFileTypes: true })) { + if (entry.name === "node_modules") continue + const path = join(directory, entry.name) + if (entry.isDirectory()) { + if (directory !== root && forbidden.has(entry.name)) fail(`raw artifact directory is committed: ${path}`) + await walk(path) + } else if ((await stat(path)).size > 262144) { + fail(`evaluation source file exceeds 256 KiB: ${path}`) + } + } +} +await walk(root) +console.log(`validated ${ids.size} conversion cases and ${rows.length} result rows`) diff --git a/release-notes/2026-08-01-evaluation-surface.md b/release-notes/2026-08-01-evaluation-surface.md new file mode 100644 index 0000000..ccc2565 --- /dev/null +++ b/release-notes/2026-08-01-evaluation-surface.md @@ -0,0 +1,9 @@ +# Evaluation cases and evidence boundary + +- Add six provenance-bound skill-conversion cases with their thin model-facing + skills and Yield workflow programs. +- Add a reproducible source-size harness and fail-closed summary validator. +- Keep raw transcripts, temporary repositories, and large run artifacts out of + Git; published behavior summaries must bind an immutable artifact digest. +- Validate the same evaluation surface in both the source tree and projected + public repository.