From 1cf8e8281285355f48c8ffb0e7173905b4d81f1a Mon Sep 17 00:00:00 2001 From: bigboateng Date: Sat, 1 Aug 2026 18:49:44 +0100 Subject: [PATCH 1/2] fix(yield): preserve append-only release history MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disclosure-Reviewed: reviewed — public-safe only, private facet kept out of this commit --- .../release-notes/2026-08-01-evaluation-surface.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 labs/22-yield/distribution/release-notes/2026-08-01-evaluation-surface.md diff --git a/labs/22-yield/distribution/release-notes/2026-08-01-evaluation-surface.md b/labs/22-yield/distribution/release-notes/2026-08-01-evaluation-surface.md new file mode 100644 index 00000000..ccc25656 --- /dev/null +++ b/labs/22-yield/distribution/release-notes/2026-08-01-evaluation-surface.md @@ -0,0 +1,9 @@ +# Evaluation cases and evidence boundary + +- Add six provenance-bound skill-conversion cases with their thin model-facing + skills and Yield workflow programs. +- Add a reproducible source-size harness and fail-closed summary validator. +- Keep raw transcripts, temporary repositories, and large run artifacts out of + Git; published behavior summaries must bind an immutable artifact digest. +- Validate the same evaluation surface in both the source tree and projected + public repository. From 4c338abc0d14f017eb3501885adca7594bd5b6f4 Mon Sep 17 00:00:00 2001 From: bigboateng Date: Sat, 1 Aug 2026 18:54:12 +0100 Subject: [PATCH 2/2] docs(yield): simplify evaluation language MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disclosure-Reviewed: reviewed — public-safe only, private facet kept out of this commit --- labs/22-yield/yield/docs/examples.md | 10 ++++---- labs/22-yield/yield/evals/README.md | 23 ++++++++++--------- labs/22-yield/yield/evals/results/README.md | 6 ++--- labs/22-yield/yield/evals/results/latest.json | 14 +++++------ labs/22-yield/yield/evals/scripts/run.mjs | 14 +++++------ .../22-yield/yield/evals/scripts/validate.mjs | 8 +++---- 6 files changed, 38 insertions(+), 37 deletions(-) diff --git a/labs/22-yield/yield/docs/examples.md b/labs/22-yield/yield/docs/examples.md index 172d4160..31ce6c3a 100644 --- a/labs/22-yield/yield/docs/examples.md +++ b/labs/22-yield/yield/docs/examples.md @@ -64,9 +64,9 @@ replay them. ## Evaluated examples The [first-party evaluation suite](../evals/) runs every example-library -pattern through TypeScript, Python, Go, and Rust fixtures. It also checks the -supervisor's resume, replay, divergence, blocking, and source-change behavior. +pattern through TypeScript, Python, Go, and Rust. It also checks resume, replay, +changed behavior, blocking, and changed source. -These results establish that the tested Yield revision executes the declared -control flow. They do not compare Yield with prose or evaluate whether an -agent's judgment is correct. +These results show that the tested Yield version runs the workflow steps in +code. They do not compare Yield with prose or test whether an agent's judgment +is correct. diff --git a/labs/22-yield/yield/evals/README.md b/labs/22-yield/yield/evals/README.md index d4f42463..a7a813a1 100644 --- a/labs/22-yield/yield/evals/README.md +++ b/labs/22-yield/yield/evals/README.md @@ -5,18 +5,19 @@ tool, company, prompt, or skill. The suite answers two questions: -1. Can each checked-in example workflow reach its declared terminal result +1. Can each checked-in example workflow reach its expected final result through every supported SDK? -2. Does the supervisor preserve its core runtime invariants when a run resumes, - replays, blocks, or encounters changed code? +2. Does the runtime behave correctly when a run resumes, replays, blocks, or + encounters changed code? ## Current coverage - 10 workflow patterns written by this project. - 4 SDKs: TypeScript, Python, Go, and Rust. -- 40 end-to-end fixture runs. -- 5 runtime invariants: resume and complete, deterministic replay, loud replay - divergence, failed-requirement blocking, and source-change refusal/migration. +- 40 end-to-end workflow tests. +- 5 runtime checks: resume and complete, repeat the same saved step, stop when + behavior changes, block when a rule fails, and require approval for changed + source. Run the exact suite and refresh the checked-in result: @@ -35,20 +36,20 @@ npm test A passing result proves that the tested Yield revision: -- executes each owned workflow fixture to `completed`; +- executes each owned workflow test to `completed`; - runs command steps rather than asking the model to invent their outputs; - presents requests in the program-defined order; - resumes from recorded responses; -- reproduces a saved frontier during replay; +- returns to the same saved step during replay; - stops on changed behavior or failed requirements. ## What it does not prove This suite does not prove that Yield is better than prose, that an agent's judgment is correct, or that illustrative commands are production-safe. The -fixtures supply deterministic agent and human responses so the suite can test -the coding and supervision layer in isolation. +Fixed test data supplies agent and human responses so the suite can test only +the code-controlled workflow layer. -`results/latest.json` is a compact, website-safe result. Its source digest is +`results/latest.json` is a compact, website-safe result. Its source hash is computed from the CLI, engine, protocol, SDKs, example workflows, fixtures, and evaluation harness. CI reruns the suite instead of trusting that file alone. diff --git a/labs/22-yield/yield/evals/results/README.md b/labs/22-yield/yield/evals/results/README.md index e445de56..e2389c8c 100644 --- a/labs/22-yield/yield/evals/results/README.md +++ b/labs/22-yield/yield/evals/results/README.md @@ -1,8 +1,8 @@ # Published result -`latest.json` records the latest first-party conformance run. It contains no +`latest.json` records the latest first-party test run. It contains no model transcripts and no third-party comparison data. -The result is publishable only when `npm test` reruns all workflow fixtures and -runtime invariants successfully and reproduces the same source digest, counts, +The result is publishable only when `npm test` reruns all workflow tests and +runtime checks successfully and reproduces the same source hash, counts, and case identities. diff --git a/labs/22-yield/yield/evals/results/latest.json b/labs/22-yield/yield/evals/results/latest.json index 02cf344c..b521e6cd 100644 --- a/labs/22-yield/yield/evals/results/latest.json +++ b/labs/22-yield/yield/evals/results/latest.json @@ -1,8 +1,8 @@ { "schema_version": 2, "methodology_version": "1.0", - "generated_at": "2026-08-01T17:40:24.932Z", - "source_digest": "51a8f4ecc511324915e1cab9f8bbecbba4d9d1c21ab6d285e7a381ba733be6dd", + "generated_at": "2026-08-01T17:53:20.762Z", + "source_digest": "8426ef59771921109fa7aa2d242e089d13f391aea60f2aebf509da93b7b534ae", "status": "passed", "workflow_conformance": { "passed": 40, @@ -270,7 +270,7 @@ { "id": "deterministic-replay", "test": "TestReplayIsDeterministic", - "assertion": "the saved log reproduces the same frontier", + "assertion": "the saved log returns to the same next step", "status": "passed" }, { @@ -282,24 +282,24 @@ { "id": "requirement-block", "test": "TestFailedRequirementBlocksRun", - "assertion": "a failed requirement produces a blocked terminal result", + "assertion": "a failed rule ends the run as blocked", "status": "passed" }, { "id": "source-change", "test": "TestDigestMismatchRefusedThenMigrates", - "assertion": "changed source is refused until migration is explicit", + "assertion": "changed source is refused until the user accepts the change", "status": "passed" } ] }, "claim_boundary": { - "summary": "Yield executes the tested workflows and preserves the tested supervisor invariants.", + "summary": "Yield executes the tested workflows and passes the tested runtime checks.", "compares_tools": false, "tests_agent_judgment": false, "exclusions": [ "No claim that Yield is better than prompts or another tool.", - "Fixtures replace agent and human judgment with deterministic responses.", + "Fixed test data replaces agent and human judgment.", "Illustrative commands do not establish production safety for a real project." ] } diff --git a/labs/22-yield/yield/evals/scripts/run.mjs b/labs/22-yield/yield/evals/scripts/run.mjs index bcf5c0b0..9ad0f74e 100644 --- a/labs/22-yield/yield/evals/scripts/run.mjs +++ b/labs/22-yield/yield/evals/scripts/run.mjs @@ -11,10 +11,10 @@ const libraryRoot = join(yieldRoot, "examples/library") const languages = ["typescript", "python", "go", "rust"] const runtimeCases = [ ["resume-complete", "TestEndToEndRunResumeComplete", "a recorded response advances the run to completion"], - ["deterministic-replay", "TestReplayIsDeterministic", "the saved log reproduces the same frontier"], + ["deterministic-replay", "TestReplayIsDeterministic", "the saved log returns to the same next step"], ["replay-divergence", "TestReplayDivergenceFailsLoudly", "changed behavior stops replay instead of reusing the wrong result"], - ["requirement-block", "TestFailedRequirementBlocksRun", "a failed requirement produces a blocked terminal result"], - ["source-change", "TestDigestMismatchRefusedThenMigrates", "changed source is refused until migration is explicit"], + ["requirement-block", "TestFailedRequirementBlocksRun", "a failed rule ends the run as blocked"], + ["source-change", "TestDigestMismatchRefusedThenMigrates", "changed source is refused until the user accepts the change"], ] const excludedDirectories = new Set([ ".git", ".yield", "node_modules", "runs", "raw", "artifacts", @@ -105,12 +105,12 @@ async function evaluate() { cases: invariants, }, claim_boundary: { - summary: "Yield executes the tested workflows and preserves the tested supervisor invariants.", + summary: "Yield executes the tested workflows and passes the tested runtime checks.", compares_tools: false, tests_agent_judgment: false, exclusions: [ "No claim that Yield is better than prompts or another tool.", - "Fixtures replace agent and human judgment with deterministic responses.", + "Fixed test data replaces agent and human judgment.", "Illustrative commands do not establish production safety for a real project.", ], }, @@ -133,8 +133,8 @@ if (process.argv.includes("--write")) { for (const field of ["workflow_conformance", "runtime_invariants", "claim_boundary"]) { if (JSON.stringify(published[field]) !== JSON.stringify(result[field])) throw new Error(`published ${field} is stale`) } - console.log(`passed ${result.workflow_conformance.passed}/${result.workflow_conformance.total} workflow fixtures`) - console.log(`passed ${result.runtime_invariants.passed}/${result.runtime_invariants.total} runtime invariants`) + console.log(`passed ${result.workflow_conformance.passed}/${result.workflow_conformance.total} workflow tests`) + console.log(`passed ${result.runtime_invariants.passed}/${result.runtime_invariants.total} runtime checks`) } else { console.log(JSON.stringify(result, null, 2)) } diff --git a/labs/22-yield/yield/evals/scripts/validate.mjs b/labs/22-yield/yield/evals/scripts/validate.mjs index 277f90b7..2ff0a758 100644 --- a/labs/22-yield/yield/evals/scripts/validate.mjs +++ b/labs/22-yield/yield/evals/scripts/validate.mjs @@ -8,7 +8,7 @@ const fail = (message) => { throw new Error(message) } if (result.schema_version !== 2) fail("unsupported result schema") if (result.methodology_version !== "1.0") fail("unsupported methodology") -if (!/^[0-9a-f]{64}$/.test(result.source_digest)) fail("source digest is invalid") +if (!/^[0-9a-f]{64}$/.test(result.source_digest)) fail("source hash is invalid") if (result.status !== "passed") fail("published result is not passing") const workflows = result.workflow_conformance @@ -16,17 +16,17 @@ if (workflows.patterns !== 10 || workflows.languages.length !== 4) { fail("workflow matrix is incomplete") } if (workflows.total !== 40 || workflows.passed !== workflows.total) { - fail("not every workflow fixture passed") + fail("not every workflow test passed") } if (workflows.cases.length !== workflows.total) fail("workflow case list is incomplete") const runtime = result.runtime_invariants if (runtime.total !== 5 || runtime.passed !== runtime.total) { - fail("not every runtime invariant passed") + fail("not every runtime check passed") } if (runtime.cases.length !== runtime.total) fail("runtime case list is incomplete") if (result.claim_boundary.compares_tools !== false) fail("result must not compare tools") if (result.claim_boundary.tests_agent_judgment !== false) fail("result must isolate agent judgment") -console.log(`validated ${workflows.passed} workflow fixtures and ${runtime.passed} runtime invariants`) +console.log(`validated ${workflows.passed} workflow tests and ${runtime.passed} runtime checks`)