diff --git a/labs/22-yield/yield/cmd/yskill/main.go b/labs/22-yield/yield/cmd/yskill/main.go index aac22d91..99ecf8a2 100644 --- a/labs/22-yield/yield/cmd/yskill/main.go +++ b/labs/22-yield/yield/cmd/yskill/main.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/operatorstack/yield/internal/engine" "github.com/operatorstack/yield/internal/protocol" @@ -60,7 +61,7 @@ func main() { func cmdRun(args []string) error { fs := flag.NewFlagSet("run", flag.ExitOnError) input := fs.String("input", "", "path to a JSON input file") - if err := fs.Parse(args); err != nil { + if err := parseOnePositional(fs, args); err != nil { return err } if fs.NArg() != 1 { @@ -90,7 +91,7 @@ func cmdResume(args []string) error { response := fs.String("response", "", "path to the response envelope JSON") skillDir := fs.String("skill", ".", "skill directory the run belongs to") migrate := fs.Bool("accept-new-digest", false, "explicitly rebind the run to the current skill source digest") - if err := fs.Parse(args); err != nil { + if err := parseOnePositional(fs, args); err != nil { return err } if fs.NArg() != 1 || *response == "" { @@ -114,7 +115,7 @@ func cmdResume(args []string) error { func cmdInspect(args []string) error { fs := flag.NewFlagSet("inspect", flag.ExitOnError) skillDir := fs.String("skill", ".", "skill directory the run belongs to") - if err := fs.Parse(args); err != nil { + if err := parseOnePositional(fs, args); err != nil { return err } e, err := engine.New(*skillDir) @@ -144,7 +145,7 @@ func cmdInspect(args []string) error { func cmdReplay(args []string) error { fs := flag.NewFlagSet("replay", flag.ExitOnError) skillDir := fs.String("skill", ".", "skill directory the run belongs to") - if err := fs.Parse(args); err != nil { + if err := parseOnePositional(fs, args); err != nil { return err } if fs.NArg() != 1 { @@ -167,7 +168,7 @@ func cmdReplay(args []string) error { // real; everything else is answered from the script. func cmdTest(args []string) error { fs := flag.NewFlagSet("test", flag.ExitOnError) - if err := fs.Parse(args); err != nil { + if err := parseOnePositional(fs, args); err != nil { return err } if fs.NArg() != 1 { @@ -213,6 +214,17 @@ func cmdTest(args []string) error { return nil } +// parseOnePositional accepts the documented command shape where the target +// comes first and flags follow it. The standard flag package stops parsing at +// the first positional argument, so move that one target behind the flags. +// Flag-first calls keep working unchanged. +func parseOnePositional(fs *flag.FlagSet, args []string) error { + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + args = append(append([]string{}, args[1:]...), args[0]) + } + return fs.Parse(args) +} + func printProgress(p *engine.Progress) error { if p.Terminal != nil { fmt.Printf("run %s: %s\n", p.RunID, p.Terminal.Status) diff --git a/labs/22-yield/yield/cmd/yskill/main_test.go b/labs/22-yield/yield/cmd/yskill/main_test.go new file mode 100644 index 00000000..ace1a0e0 --- /dev/null +++ b/labs/22-yield/yield/cmd/yskill/main_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "flag" + "testing" +) + +func TestParseOnePositionalAllowsDocumentedFlagOrder(t *testing.T) { + fs := flag.NewFlagSet("resume", flag.ContinueOnError) + response := fs.String("response", "", "response file") + skill := fs.String("skill", ".", "skill directory") + if err := parseOnePositional(fs, []string{"run_123", "--response", "response.json", "--skill", "skills/release"}); err != nil { + t.Fatal(err) + } + if fs.NArg() != 1 || fs.Arg(0) != "run_123" { + t.Fatalf("positional args = %q, want run_123", fs.Args()) + } + if *response != "response.json" || *skill != "skills/release" { + t.Fatalf("flags = response %q skill %q", *response, *skill) + } +} + +func TestParseOnePositionalKeepsFlagFirstOrder(t *testing.T) { + fs := flag.NewFlagSet("resume", flag.ContinueOnError) + response := fs.String("response", "", "response file") + if err := parseOnePositional(fs, []string{"--response", "response.json", "run_123"}); err != nil { + t.Fatal(err) + } + if fs.NArg() != 1 || fs.Arg(0) != "run_123" || *response != "response.json" { + t.Fatalf("args = %q response = %q", fs.Args(), *response) + } +} diff --git a/labs/22-yield/yield/evals/README.md b/labs/22-yield/yield/evals/README.md index a7a813a1..35cc3e5e 100644 --- a/labs/22-yield/yield/evals/README.md +++ b/labs/22-yield/yield/evals/README.md @@ -1,9 +1,9 @@ # Yield evaluations These evaluations test Yield itself. They do not compare Yield with another -tool, company, prompt, or skill. +tool or company. -The suite answers two questions: +The deterministic suite answers two questions: 1. Can each checked-in example workflow reach its expected final result through every supported SDK? @@ -47,9 +47,21 @@ A passing result proves that the tested Yield revision: This suite does not prove that Yield is better than prose, that an agent's judgment is correct, or that illustrative commands are production-safe. The -Fixed test data supplies agent and human responses so the suite can test only +fixed test data supplies agent and human responses so the suite can test only the code-controlled workflow layer. `results/latest.json` is a compact, website-safe result. Its source hash is computed from the CLI, engine, protocol, SDKs, example workflows, fixtures, and evaluation harness. CI reruns the suite instead of trusting that file alone. + +## Coding-agent workflow check + +The separate `agent/` suite runs the same owned workflow through a real coding +agent in two forms: a long skill, and a thin skill backed by Yield code. It +checks matching step order, gates, responses, and final status. It does not +score the agent's domain judgment or claim that one form is better. + +```bash +npm run eval:agent +npm run test:agent +``` diff --git a/labs/22-yield/yield/evals/agent/README.md b/labs/22-yield/yield/evals/agent/README.md new file mode 100644 index 00000000..9de5fc73 --- /dev/null +++ b/labs/22-yield/yield/evals/agent/README.md @@ -0,0 +1,46 @@ +# Coding-agent workflow equivalence + +This suite checks one Yield product claim with a real coding agent: + +> Moving workflow control from a long skill into Yield code preserves the +> tested step order and final result. + +It does not score general coding ability. The release data, command outcomes, +and user answers are deliberately simple. The same Codex model runs two owned +representations: + +1. a long `SKILL.md` containing every step and branch; +2. a thin `SKILL.md` that follows the same workflow in Yield code. + +Six cases cover the important branches: failed tests, failed review rule, user +refusal, failed publish, failed verification, and successful completion. + +## Evidence + +The long-skill arm records actual command calls and structured step results. +The Yield arm is scored from its append-only run log. The scorer verifies: + +- the same ordered step IDs; +- the same final status; +- real command evidence for every command step; +- accepted `agent_task` and `ask_user` responses in the Yield log; +- matching requirement results and terminal events; +- no rejected response envelopes. + +Raw Codex transcripts and run logs stay under ignored `evals/runs/`. The compact +result in `results/latest-agent.json` contains counts, normalized traces, model +identity, CLI version, token usage, timings, and a source hash covering the +harness, fixtures, CLI, engine, protocol, and TypeScript SDK. + +## Run + +Use the current Codex login: + +```bash +cd evals +npm run eval:agent +npm run test:agent +``` + +CI can instead provide `CODEX_API_KEY`. Set `EVAL_AGENT_MODEL` and +`EVAL_AGENT_REASONING` to make a different model configuration explicit. diff --git a/labs/22-yield/yield/evals/agent/cases.json b/labs/22-yield/yield/evals/agent/cases.json new file mode 100644 index 00000000..86f91747 --- /dev/null +++ b/labs/22-yield/yield/evals/agent/cases.json @@ -0,0 +1,68 @@ +[ + { + "id": "preflight-blocks", + "commands": { + "test-package": { "exit_code": 1, "stdout": "", "stderr": "tests failed\n" }, + "publish-package": { "exit_code": 0, "stdout": "published\n", "stderr": "" }, + "verify-package": { "exit_code": 0, "stdout": "verified\n", "stderr": "" } + }, + "review": { "status": "pass", "critical": 0, "summary": "Release metadata is complete." }, + "answers": { "approve-publish": "continue" }, + "expected": { "steps": ["test-package"], "terminal": "blocked" } + }, + { + "id": "review-blocks", + "commands": { + "test-package": { "exit_code": 0, "stdout": "tests passed\n", "stderr": "" }, + "publish-package": { "exit_code": 0, "stdout": "published\n", "stderr": "" }, + "verify-package": { "exit_code": 0, "stdout": "verified\n", "stderr": "" } + }, + "review": { "status": "needs_work", "critical": 1, "summary": "The release notes omit a breaking API change." }, + "answers": { "approve-publish": "continue" }, + "expected": { "steps": ["test-package", "review-release"], "terminal": "blocked" } + }, + { + "id": "approval-refuses", + "commands": { + "test-package": { "exit_code": 0, "stdout": "tests passed\n", "stderr": "" }, + "publish-package": { "exit_code": 0, "stdout": "published\n", "stderr": "" }, + "verify-package": { "exit_code": 0, "stdout": "verified\n", "stderr": "" } + }, + "review": { "status": "pass", "critical": 0, "summary": "Release metadata is complete." }, + "answers": { "approve-publish": "stop" }, + "expected": { "steps": ["test-package", "review-release", "approve-publish"], "terminal": "refused" } + }, + { + "id": "publish-blocks", + "commands": { + "test-package": { "exit_code": 0, "stdout": "tests passed\n", "stderr": "" }, + "publish-package": { "exit_code": 1, "stdout": "", "stderr": "registry unavailable\n" }, + "verify-package": { "exit_code": 0, "stdout": "verified\n", "stderr": "" } + }, + "review": { "status": "pass", "critical": 0, "summary": "Release metadata is complete." }, + "answers": { "approve-publish": "continue" }, + "expected": { "steps": ["test-package", "review-release", "approve-publish", "publish-package"], "terminal": "blocked" } + }, + { + "id": "verification-blocks", + "commands": { + "test-package": { "exit_code": 0, "stdout": "tests passed\n", "stderr": "" }, + "publish-package": { "exit_code": 0, "stdout": "published\n", "stderr": "" }, + "verify-package": { "exit_code": 1, "stdout": "", "stderr": "package not visible\n" } + }, + "review": { "status": "pass", "critical": 0, "summary": "Release metadata is complete." }, + "answers": { "approve-publish": "continue" }, + "expected": { "steps": ["test-package", "review-release", "approve-publish", "publish-package", "verify-package"], "terminal": "blocked" } + }, + { + "id": "success", + "commands": { + "test-package": { "exit_code": 0, "stdout": "tests passed\n", "stderr": "" }, + "publish-package": { "exit_code": 0, "stdout": "published\n", "stderr": "" }, + "verify-package": { "exit_code": 0, "stdout": "verified\n", "stderr": "" } + }, + "review": { "status": "pass", "critical": 0, "summary": "Release metadata is complete." }, + "answers": { "approve-publish": "continue" }, + "expected": { "steps": ["test-package", "review-release", "approve-publish", "publish-package", "verify-package"], "terminal": "completed" } + } +] diff --git a/labs/22-yield/yield/evals/agent/fixtures/long/SKILL.md b/labs/22-yield/yield/evals/agent/fixtures/long/SKILL.md new file mode 100644 index 00000000..7307b8d4 --- /dev/null +++ b/labs/22-yield/yield/evals/agent/fixtures/long/SKILL.md @@ -0,0 +1,34 @@ +--- +name: release-package-long +description: Test, review, approve, publish, and verify a package release. +--- + +Run this workflow in order. Do not skip a step. Do not continue after a failed +rule. The `node bin/record.mjs` calls are eval instrumentation and must run. + +1. Run `node bin/step.mjs test-package`. +2. Record whether the command passed: + `node bin/record.mjs requirement package-tests '{"passed":true}'` or use + `false`. If it failed, run + `node bin/record.mjs terminal blocked '{"reason":"the package tests pass"}'` + and stop. +3. Read `.eval/release.json`. Use its exact JSON object as the structured result + of the `review-release` task. Record it with + `node bin/record.mjs agent_task review-release ''`. +4. The review passes only when `status` is `pass` and `critical` is `0`. Record + that rule with `node bin/record.mjs requirement review-ready + '{"passed":true}'` or use `false`. If it failed, record terminal `blocked` + with reason `the package is ready to publish` and stop. +5. Get the user's answer by running + `node bin/user-answer.mjs approve-publish`. The helper records this user + step. If it is not `continue`, record terminal `refused` with reason `the + user declined to continue` and stop. +6. Run `node bin/step.mjs publish-package`. Record requirement `publish-passed`. + If it failed, record terminal `blocked` with reason + `the package publish command succeeds` and stop. +7. Run `node bin/step.mjs verify-package`. Record requirement `verify-passed`. + If it failed, record terminal `blocked` with reason + `the published package resolves from the registry` and stop. +8. Record terminal `completed` with the review summary. + +Every recorder result must be valid JSON. End with a short status report. diff --git a/labs/22-yield/yield/evals/agent/fixtures/shared/.gitignore b/labs/22-yield/yield/evals/agent/fixtures/shared/.gitignore new file mode 100644 index 00000000..042c734d --- /dev/null +++ b/labs/22-yield/yield/evals/agent/fixtures/shared/.gitignore @@ -0,0 +1,3 @@ +.eval/observed.jsonl +.eval/codex-last.json +skills/release/.yield/ diff --git a/labs/22-yield/yield/evals/agent/fixtures/shared/bin/record.mjs b/labs/22-yield/yield/evals/agent/fixtures/shared/bin/record.mjs new file mode 100644 index 00000000..5890064c --- /dev/null +++ b/labs/22-yield/yield/evals/agent/fixtures/shared/bin/record.mjs @@ -0,0 +1,10 @@ +#!/usr/bin/env node +import { appendFile } from "node:fs/promises" +import { dirname, join, resolve } from "node:path" +import { fileURLToPath } from "node:url" + +const root = resolve(dirname(fileURLToPath(import.meta.url)), "..") +const [kind, id, raw = "null"] = process.argv.slice(2) +if (!["agent_task", "requirement", "terminal"].includes(kind)) throw new Error(`unsupported event kind: ${kind}`) +const event = { kind, id, result: JSON.parse(raw) } +await appendFile(join(root, ".eval/observed.jsonl"), JSON.stringify(event) + "\n") diff --git a/labs/22-yield/yield/evals/agent/fixtures/shared/bin/step.mjs b/labs/22-yield/yield/evals/agent/fixtures/shared/bin/step.mjs new file mode 100644 index 00000000..bfd80189 --- /dev/null +++ b/labs/22-yield/yield/evals/agent/fixtures/shared/bin/step.mjs @@ -0,0 +1,14 @@ +#!/usr/bin/env node +import { appendFile, readFile } from "node:fs/promises" +import { dirname, join, resolve } from "node:path" +import { fileURLToPath } from "node:url" + +const root = resolve(dirname(fileURLToPath(import.meta.url)), "..") +const id = process.argv[2] +const testCase = JSON.parse(await readFile(join(root, ".eval/case.json"), "utf8")) +const command = testCase.commands[id] +if (!command) throw new Error(`unknown command step: ${id}`) +await appendFile(join(root, ".eval/observed.jsonl"), JSON.stringify({ kind: "run_command", id, exit_code: command.exit_code }) + "\n") +process.stdout.write(command.stdout) +process.stderr.write(command.stderr) +process.exitCode = command.exit_code diff --git a/labs/22-yield/yield/evals/agent/fixtures/shared/bin/user-answer.mjs b/labs/22-yield/yield/evals/agent/fixtures/shared/bin/user-answer.mjs new file mode 100644 index 00000000..3cc654ac --- /dev/null +++ b/labs/22-yield/yield/evals/agent/fixtures/shared/bin/user-answer.mjs @@ -0,0 +1,12 @@ +#!/usr/bin/env node +import { appendFile, readFile } from "node:fs/promises" +import { dirname, join, resolve } from "node:path" +import { fileURLToPath } from "node:url" + +const root = resolve(dirname(fileURLToPath(import.meta.url)), "..") +const id = process.argv[2] +const testCase = JSON.parse(await readFile(join(root, ".eval/case.json"), "utf8")) +const value = testCase.answers[id] +if (typeof value !== "string") throw new Error(`no user answer for: ${id}`) +await appendFile(join(root, ".eval/observed.jsonl"), JSON.stringify({ kind: "ask_user", id, result: { value } }) + "\n") +process.stdout.write(value + "\n") diff --git a/labs/22-yield/yield/evals/agent/fixtures/yield/SKILL.md b/labs/22-yield/yield/evals/agent/fixtures/yield/SKILL.md new file mode 100644 index 00000000..b11ccf08 --- /dev/null +++ b/labs/22-yield/yield/evals/agent/fixtures/yield/SKILL.md @@ -0,0 +1,24 @@ +--- +name: release-package-yield +description: Run the code-controlled package release workflow. +--- + +Run `bin/yskill run skills/release` and follow every returned operation. + +- For `agent_task`, perform its instruction and return schema-valid JSON. +- For `ask_user`, run `node bin/user-answer.mjs ` and use the + returned value as the user's answer. +- For either kind, write a response file using the values in the current + request envelope: + + ```json + {"run_id":"","sequence":1,"request_id":"","status":"completed","result":{}} + ``` + + Replace `sequence` and `result` with the current request's sequence and the + schema-valid result. Then run `bin/yskill resume --response + --skill skills/release`. + +Do not run workflow commands yourself. Yield runs them. Do not skip a request, +invent an answer, edit the workflow, or use test expectations. End with the +terminal status reported by Yield. diff --git a/labs/22-yield/yield/evals/agent/fixtures/yield/skills/release/main.ts b/labs/22-yield/yield/evals/agent/fixtures/yield/skills/release/main.ts new file mode 100644 index 00000000..f36d98f5 --- /dev/null +++ b/labs/22-yield/yield/evals/agent/fixtures/yield/skills/release/main.ts @@ -0,0 +1,39 @@ +import { defineSkill } from "./yield-sdk.ts" + +type Review = { status: "pass" | "needs_work"; critical: number; summary: string } +const reviewSchema = { + type: "object", + required: ["status", "critical", "summary"], + properties: { + status: { enum: ["pass", "needs_work"] }, + critical: { type: "integer", minimum: 0 }, + summary: { type: "string", minLength: 1 }, + }, +} + +defineSkill((ctx) => { + const tests = ctx.runCommand("test-package", "node ../../bin/step.mjs test-package", 60) + ctx.require(tests.exit_code === 0, "the package tests pass", tests) + + const review = ctx.agentTask( + "review-release", + "Read .eval/release.json from the project root. Return that exact JSON object as the release review result.", + { stdout: tests.stdout, stderr: tests.stderr }, + reviewSchema, + ) + ctx.require(review.status === "pass" && review.critical === 0, "the package is ready to publish", review) + + const approval = ctx.askUser("approve-publish", "Publish this package release?", [ + { value: "continue", label: "Continue" }, + { value: "stop", label: "Stop" }, + ]) + if (approval !== "continue") ctx.refused("the user declined to continue") + + const publish = ctx.runCommand("publish-package", "node ../../bin/step.mjs publish-package", 60) + ctx.require(publish.exit_code === 0, "the package publish command succeeds", publish) + + const verify = ctx.runCommand("verify-package", "node ../../bin/step.mjs verify-package", 60) + ctx.require(verify.exit_code === 0, "the published package resolves from the registry", verify) + + return { workflow: "release-package", summary: review.summary } +}) diff --git a/labs/22-yield/yield/evals/agent/fixtures/yield/skills/release/skill.json b/labs/22-yield/yield/evals/agent/fixtures/yield/skills/release/skill.json new file mode 100644 index 00000000..8b8a91c9 --- /dev/null +++ b/labs/22-yield/yield/evals/agent/fixtures/yield/skills/release/skill.json @@ -0,0 +1 @@ +{"run":["node","main.ts"]} diff --git a/labs/22-yield/yield/evals/agent/scripts/run.mjs b/labs/22-yield/yield/evals/agent/scripts/run.mjs new file mode 100644 index 00000000..43baa1fd --- /dev/null +++ b/labs/22-yield/yield/evals/agent/scripts/run.mjs @@ -0,0 +1,292 @@ +import { createHash } from "node:crypto" +import { execFileSync, spawnSync } from "node:child_process" +import { chmod, cp, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises" +import { homedir, tmpdir } from "node:os" +import { dirname, join, relative, resolve } from "node:path" +import { fileURLToPath } from "node:url" + +const agentRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..") +const evalRoot = resolve(agentRoot, "..") +const yieldRoot = resolve(evalRoot, "..") +const fixturesRoot = join(agentRoot, "fixtures") +const runsRoot = join(evalRoot, "runs/agent") +const selectedCase = valueAfter("--case") +const selectedArm = valueAfter("--arm") +const repeat = Number(valueAfter("--repeat") ?? "1") +const model = process.env.EVAL_AGENT_MODEL ?? "gpt-5.6-terra" +const reasoning = process.env.EVAL_AGENT_REASONING ?? "medium" +const arms = selectedArm ? [selectedArm] : ["long", "yield"] + +if (!Number.isInteger(repeat) || repeat < 1) throw new Error("--repeat must be a positive integer") +if (arms.some((arm) => !["long", "yield"].includes(arm))) throw new Error("--arm must be long or yield") + +function valueAfter(flag) { + const index = process.argv.indexOf(flag) + return index === -1 ? undefined : process.argv[index + 1] +} + +function command(command, args, cwd = yieldRoot) { + return execFileSync(command, args, { cwd, encoding: "utf8", env: process.env }).trim() +} + +async function sourceHash() { + const roots = [ + [evalRoot, "agent/cases.json"], + [evalRoot, "agent/fixtures"], + [evalRoot, "agent/scripts"], + [yieldRoot, "cmd/yskill"], + [yieldRoot, "internal"], + [yieldRoot, "sdk/typescript/src/index.ts"], + ] + const files = [] + for (const [base, root] of roots) files.push(...await filesUnder(join(base, root))) + files.sort() + const hash = createHash("sha256") + for (const path of files) { + hash.update(relative(yieldRoot, path)) + hash.update("\0") + hash.update(await readFile(path)) + hash.update("\0") + } + return hash.digest("hex") +} + +async function filesUnder(path) { + if ((await stat(path)).isFile()) return [path] + const entries = await readdir(path, { withFileTypes: true }) + const files = [] + for (const entry of entries) { + const child = join(path, entry.name) + if (entry.isDirectory()) files.push(...await filesUnder(child)) + else files.push(child) + } + return files +} + +async function prepareAuthHome(parent) { + const target = join(parent, "codex-home") + await mkdir(target) + if (!process.env.CODEX_API_KEY) { + const source = join(process.env.CODEX_HOME ?? join(homedir(), ".codex"), "auth.json") + await cp(source, join(target, "auth.json")) + await chmod(join(target, "auth.json"), 0o600) + } + return target +} + +async function prepareRepo(parent, arm, testCase, yskill, repeatIndex) { + const repo = join(parent, `${testCase.id}-${arm}-${repeatIndex}`) + await mkdir(repo) + await cp(join(fixturesRoot, "shared"), repo, { recursive: true }) + await cp(join(fixturesRoot, arm), repo, { recursive: true }) + await mkdir(join(repo, ".eval"), { recursive: true }) + await writeFile(join(repo, ".eval/case.json"), JSON.stringify({ commands: testCase.commands, answers: testCase.answers }, null, 2) + "\n") + await writeFile(join(repo, ".eval/release.json"), JSON.stringify(testCase.review, null, 2) + "\n") + await writeFile(join(repo, ".eval/output-schema.json"), JSON.stringify({ + type: "object", + required: ["status", "summary"], + properties: { + status: { enum: ["completed", "blocked", "refused", "failed"] }, + summary: { type: "string" }, + }, + additionalProperties: false, + }, null, 2) + "\n") + if (arm === "yield") { + await cp(join(yieldRoot, "sdk/typescript/src/index.ts"), join(repo, "skills/release/yield-sdk.ts")) + await cp(yskill, join(repo, "bin/yskill")) + await chmod(join(repo, "bin/yskill"), 0o755) + } + command("git", ["init", "-b", "main"], repo) + command("git", ["config", "user.email", "eval@operatorstack.systems"], repo) + command("git", ["config", "user.name", "Yield Eval"], repo) + command("git", ["add", "."], repo) + command("git", ["commit", "-m", "evaluation fixture"], repo) + return repo +} + +function parseCodexUsage(stdout) { + let usage = { input_tokens: 0, cached_input_tokens: 0, output_tokens: 0, reasoning_output_tokens: 0 } + for (const line of stdout.split("\n")) { + if (!line.startsWith("{")) continue + try { + const event = JSON.parse(line) + if (event.type === "turn.completed" && event.usage) usage = event.usage + } catch {} + } + return usage +} + +async function runCodex(repo, authHome, evidenceDir) { + const prompt = [ + "Read SKILL.md and execute that workflow exactly.", + "This evaluates workflow control, not your domain knowledge.", + "Do not edit SKILL.md, workflow code, bin scripts, or files under .eval.", + "Do not read .eval/case.json. Read .eval/release.json only when the workflow asks.", + "Do not infer hidden expectations. Report the terminal status you actually reach.", + ].join(" ") + const started = Date.now() + const execution = spawnSync("codex", [ + "exec", "--ephemeral", "--ignore-user-config", "--ignore-rules", + "--disable", "plugins", "--disable", "remote_plugin", "--disable", "apps", + "--disable", "memories", "--disable", "goals", "--disable", "multi_agent", + "--disable", "browser_use", "--disable", "computer_use", "--disable", "image_generation", + "--disable", "skill_search", "--disable", "workspace_dependencies", + "--json", "--sandbox", "workspace-write", "-C", repo, + "--model", model, "-c", `model_reasoning_effort=\"${reasoning}\"`, + "--output-schema", join(repo, ".eval/output-schema.json"), + "--output-last-message", join(repo, ".eval/codex-last.json"), + prompt, + ], { + cwd: repo, + encoding: "utf8", + timeout: 10 * 60 * 1000, + maxBuffer: 64 * 1024 * 1024, + env: { ...process.env, CODEX_HOME: authHome }, + }) + await mkdir(evidenceDir, { recursive: true }) + await writeFile(join(evidenceDir, "transcript.jsonl"), execution.stdout ?? "") + await writeFile(join(evidenceDir, "stderr.log"), execution.stderr ?? "") + if (execution.status !== 0) throw new Error(`Codex exited ${execution.status}; see ${evidenceDir}`) + return { duration_ms: Date.now() - started, usage: parseCodexUsage(execution.stdout ?? "") } +} + +async function readJSONL(path) { + const text = await readFile(path, "utf8") + return text.split("\n").filter(Boolean).map((line) => JSON.parse(line)) +} + +async function scoreLong(repo) { + const events = await readJSONL(join(repo, ".eval/observed.jsonl")) + const steps = events.filter((event) => ["run_command", "agent_task", "ask_user"].includes(event.kind)).map((event) => event.id) + const terminals = events.filter((event) => event.kind === "terminal") + if (terminals.length !== 1) throw new Error(`long arm wrote ${terminals.length} terminal events`) + return { + steps, + terminal: terminals[0].id, + requirements: events.filter((event) => event.kind === "requirement").map((event) => Boolean(event.result?.passed)), + command_evidence: events.filter((event) => event.kind === "run_command"), + accepted_agent_tasks: events.filter((event) => event.kind === "agent_task").length, + accepted_user_answers: events.filter((event) => event.kind === "ask_user").length, + } +} + +async function scoreYield(repo) { + const runsDir = join(repo, "skills/release/.yield/runs") + const logs = (await readdir(runsDir)).filter((name) => name.endsWith(".jsonl")) + if (logs.length !== 1) throw new Error(`Yield arm produced ${logs.length} run logs`) + const events = await readJSONL(join(runsDir, logs[0])) + const requests = events.filter((event) => event.type === "operation.requested").map((event) => event.data.request) + const completed = new Set(events.filter((event) => event.type === "operation.completed").map((event) => event.data.request_id)) + const terminalEvent = events.findLast((event) => ["run.completed", "run.blocked", "run.refused"].includes(event.type)) + if (!terminalEvent) throw new Error("Yield arm did not reach a terminal event") + const terminal = terminalEvent.type.slice(4) + const rejected = events.filter((event) => event.type === "response.rejected") + const unanswered = requests.filter((request) => !completed.has(request.id)) + if (unanswered.length) throw new Error(`Yield log has unanswered requests: ${unanswered.map((request) => request.id).join(", ")}`) + const observed = await readJSONL(join(repo, ".eval/observed.jsonl")) + return { + steps: requests.map((request) => request.id), + terminal, + requirements: events.filter((event) => event.type.startsWith("requirement.")).map((event) => event.type === "requirement.passed"), + command_evidence: observed.filter((event) => event.kind === "run_command"), + accepted_agent_tasks: requests.filter((request) => request.kind === "agent_task" && completed.has(request.id)).length, + accepted_user_answers: requests.filter((request) => request.kind === "ask_user" && completed.has(request.id)).length, + response_rejections: rejected.map((event) => event.data), + event_counts: Object.fromEntries([...new Set(events.map((event) => event.type))].sort().map((type) => [type, events.filter((event) => event.type === type).length])), + } +} + +function same(a, b) { return JSON.stringify(a) === JSON.stringify(b) } + +function scoreAgainstExpected(observed, expected) { + const failures = [] + if (!same(observed.steps, expected.steps)) failures.push(`steps ${JSON.stringify(observed.steps)} != ${JSON.stringify(expected.steps)}`) + if (observed.terminal !== expected.terminal) failures.push(`terminal ${observed.terminal} != ${expected.terminal}`) + const commandSteps = observed.steps.filter((step) => ["test-package", "publish-package", "verify-package"].includes(step)) + if (!same(observed.command_evidence.map((event) => event.id), commandSteps)) failures.push("actual command evidence does not match the workflow trace") + if (observed.response_rejections?.length) failures.push(`Yield rejected ${observed.response_rejections.length} response(s)`) + return failures +} + +async function main() { + const allCases = JSON.parse(await readFile(join(agentRoot, "cases.json"), "utf8")) + const cases = selectedCase ? allCases.filter((testCase) => testCase.id === selectedCase) : allCases + if (!cases.length) throw new Error(`unknown case: ${selectedCase}`) + const session = await mkdtemp(join(tmpdir(), "yield-agent-eval-")) + const runStamp = new Date().toISOString().replaceAll(":", "-") + const evidenceRoot = join(runsRoot, runStamp) + const yskill = join(session, "yskill") + command("go", ["build", "-o", yskill, "./cmd/yskill"]) + const authHome = await prepareAuthHome(session) + const runs = [] + try { + for (let index = 0; index < repeat; index++) { + for (const testCase of cases) { + for (const arm of arms) { + process.stdout.write(`running ${testCase.id} / ${arm} / ${index + 1}\n`) + const repo = await prepareRepo(session, arm, testCase, yskill, index + 1) + const evidenceDir = join(evidenceRoot, testCase.id, arm, String(index + 1)) + const execution = await runCodex(repo, authHome, evidenceDir) + const observed = arm === "long" ? await scoreLong(repo) : await scoreYield(repo) + const failures = scoreAgainstExpected(observed, testCase.expected) + await cp(join(repo, ".eval/observed.jsonl"), join(evidenceDir, "observed.jsonl")) + if (arm === "yield") await cp(join(repo, "skills/release/.yield/runs"), join(evidenceDir, "yield-runs"), { recursive: true }) + runs.push({ case: testCase.id, arm, repeat: index + 1, passed: failures.length === 0, failures, ...execution, ...observed }) + process.stdout.write(`${failures.length ? "failed" : "passed"} ${testCase.id} / ${arm}\n`) + } + } + } + } finally { + await rm(session, { recursive: true, force: true }) + } + + const comparisons = [] + for (let index = 1; index <= repeat; index++) { + for (const testCase of cases) { + const long = runs.find((run) => run.case === testCase.id && run.arm === "long" && run.repeat === index) + const yieldRun = runs.find((run) => run.case === testCase.id && run.arm === "yield" && run.repeat === index) + if (!long || !yieldRun) continue + const equivalent = long.passed && yieldRun.passed && same(long.steps, yieldRun.steps) && + long.terminal === yieldRun.terminal && same(long.requirements, yieldRun.requirements) && + long.accepted_agent_tasks === yieldRun.accepted_agent_tasks && + long.accepted_user_answers === yieldRun.accepted_user_answers + comparisons.push({ + case: testCase.id, + repeat: index, + equivalent, + long_steps: long.steps, + yield_steps: yieldRun.steps, + long_terminal: long.terminal, + yield_terminal: yieldRun.terminal, + long_requirements: long.requirements, + yield_requirements: yieldRun.requirements, + accepted_agent_tasks: { long: long.accepted_agent_tasks, yield: yieldRun.accepted_agent_tasks }, + accepted_user_answers: { long: long.accepted_user_answers, yield: yieldRun.accepted_user_answers }, + }) + } + } + const result = { + schema_version: 1, + methodology_version: "agent-equivalence-v1", + generated_at: new Date().toISOString(), + source_hash: await sourceHash(), + status: runs.every((run) => run.passed) && comparisons.every((comparison) => comparison.equivalent) ? "passed" : "failed", + agent: { product: "Codex CLI", cli_version: command("codex", ["--version"]), model, reasoning }, + coverage: { cases: cases.length, arms, repeats: repeat, agent_runs: runs.length }, + runs, + equivalence: { passed: comparisons.filter((comparison) => comparison.equivalent).length, total: comparisons.length, comparisons }, + claim_boundary: { + summary: "The same Codex model followed the owned long skill and the thin-skill-plus-Yield workflow with the same tested step order and final status.", + exclusions: [ + "This does not measure general coding-agent quality.", + "This does not claim Yield is better than a long skill.", + "This covers the declared cases, not every possible workflow.", + ], + }, + } + if (process.argv.includes("--write")) await writeFile(join(evalRoot, "results/latest-agent.json"), JSON.stringify(result, null, 2) + "\n") + process.stdout.write(JSON.stringify({ status: result.status, coverage: result.coverage, equivalence: result.equivalence, evidence: relative(evalRoot, evidenceRoot) }, null, 2) + "\n") + if (result.status !== "passed") process.exitCode = 1 +} + +await main() diff --git a/labs/22-yield/yield/evals/agent/scripts/validate.mjs b/labs/22-yield/yield/evals/agent/scripts/validate.mjs new file mode 100644 index 00000000..537231c7 --- /dev/null +++ b/labs/22-yield/yield/evals/agent/scripts/validate.mjs @@ -0,0 +1,49 @@ +import { createHash } from "node:crypto" +import { readFile, readdir, stat } from "node:fs/promises" +import { dirname, join, relative, resolve } from "node:path" +import { fileURLToPath } from "node:url" + +const agentRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..") +const evalRoot = resolve(agentRoot, "..") +const yieldRoot = resolve(evalRoot, "..") +const result = JSON.parse(await readFile(join(evalRoot, "results/latest-agent.json"), "utf8")) +const fail = (message) => { throw new Error(message) } + +async function filesUnder(path) { + if ((await stat(path)).isFile()) return [path] + const files = [] + for (const entry of await readdir(path, { withFileTypes: true })) { + const child = join(path, entry.name) + if (entry.isDirectory()) files.push(...await filesUnder(child)) + else files.push(child) + } + return files +} + +async function sourceHash() { + const roots = [ + [evalRoot, "agent/cases.json"], + [evalRoot, "agent/fixtures"], + [evalRoot, "agent/scripts"], + [yieldRoot, "cmd/yskill"], + [yieldRoot, "internal"], + [yieldRoot, "sdk/typescript/src/index.ts"], + ] + const files = [] + for (const [base, root] of roots) files.push(...await filesUnder(join(base, root))) + files.sort() + const hash = createHash("sha256") + for (const path of files) { + hash.update(relative(yieldRoot, path)); hash.update("\0"); hash.update(await readFile(path)); hash.update("\0") + } + return hash.digest("hex") +} + +if (result.schema_version !== 1) fail("unsupported agent result schema") +if (result.methodology_version !== "agent-equivalence-v1") fail("unsupported agent test method") +if (result.source_hash !== await sourceHash()) fail("published agent result has a stale source hash") +if (result.status !== "passed") fail("published agent result is not passing") +if (result.coverage.cases !== 6 || result.coverage.arms.length !== 2) fail("agent branch coverage is incomplete") +if (result.equivalence.total < 6 || result.equivalence.passed !== result.equivalence.total) fail("not every paired run is equivalent") +if (result.runs.some((run) => !run.passed)) fail("an agent run failed its expected trace") +console.log(`validated ${result.equivalence.passed}/${result.equivalence.total} paired agent runs`) diff --git a/labs/22-yield/yield/evals/package.json b/labs/22-yield/yield/evals/package.json index ad898e49..4f74746c 100644 --- a/labs/22-yield/yield/evals/package.json +++ b/labs/22-yield/yield/evals/package.json @@ -5,7 +5,10 @@ "type": "module", "scripts": { "eval": "node scripts/run.mjs --write", + "eval:agent": "node agent/scripts/run.mjs --write", + "eval:agent:smoke": "node agent/scripts/run.mjs --case success --repeat 1", "test": "node scripts/validate.mjs && node scripts/run.mjs --check", + "test:agent": "node agent/scripts/validate.mjs", "validate": "node scripts/validate.mjs" } } diff --git a/labs/22-yield/yield/evals/results/latest-agent.json b/labs/22-yield/yield/evals/results/latest-agent.json new file mode 100644 index 00000000..d0ae40d1 --- /dev/null +++ b/labs/22-yield/yield/evals/results/latest-agent.json @@ -0,0 +1,769 @@ +{ + "schema_version": 1, + "methodology_version": "agent-equivalence-v1", + "generated_at": "2026-08-01T18:41:10.110Z", + "source_hash": "0b51a9b75ae3e788cdc0a80c633c003913ac7be2f50832b809a2a5082cbd89b0", + "status": "passed", + "agent": { + "product": "Codex CLI", + "cli_version": "codex-cli 0.145.0", + "model": "gpt-5.6-terra", + "reasoning": "medium" + }, + "coverage": { + "cases": 6, + "arms": [ + "long", + "yield" + ], + "repeats": 1, + "agent_runs": 12 + }, + "runs": [ + { + "case": "preflight-blocks", + "arm": "long", + "repeat": 1, + "passed": true, + "failures": [], + "duration_ms": 8797, + "usage": { + "input_tokens": 61796, + "cached_input_tokens": 48783, + "cache_write_input_tokens": 12998, + "output_tokens": 748, + "reasoning_output_tokens": 66 + }, + "steps": [ + "test-package" + ], + "terminal": "blocked", + "requirements": [ + false + ], + "command_evidence": [ + { + "kind": "run_command", + "id": "test-package", + "exit_code": 1 + } + ], + "accepted_agent_tasks": 0, + "accepted_user_answers": 0 + }, + { + "case": "preflight-blocks", + "arm": "yield", + "repeat": 1, + "passed": true, + "failures": [], + "duration_ms": 10958, + "usage": { + "input_tokens": 61391, + "cached_input_tokens": 48552, + "cache_write_input_tokens": 12824, + "output_tokens": 742, + "reasoning_output_tokens": 98 + }, + "steps": [ + "test-package" + ], + "terminal": "blocked", + "requirements": [ + false + ], + "command_evidence": [ + { + "kind": "run_command", + "id": "test-package", + "exit_code": 1 + } + ], + "accepted_agent_tasks": 0, + "accepted_user_answers": 0, + "response_rejections": [], + "event_counts": { + "operation.completed": 1, + "operation.requested": 1, + "requirement.failed": 1, + "run.blocked": 1, + "run.started": 1 + } + }, + { + "case": "review-blocks", + "arm": "long", + "repeat": 1, + "passed": true, + "failures": [], + "duration_ms": 17218, + "usage": { + "input_tokens": 142302, + "cached_input_tokens": 128466, + "cache_write_input_tokens": 13803, + "output_tokens": 1400, + "reasoning_output_tokens": 217 + }, + "steps": [ + "test-package", + "review-release" + ], + "terminal": "blocked", + "requirements": [ + true, + false + ], + "command_evidence": [ + { + "kind": "run_command", + "id": "test-package", + "exit_code": 0 + } + ], + "accepted_agent_tasks": 1, + "accepted_user_answers": 0 + }, + { + "case": "review-blocks", + "arm": "yield", + "repeat": 1, + "passed": true, + "failures": [], + "duration_ms": 11800, + "usage": { + "input_tokens": 88064, + "cached_input_tokens": 74740, + "cache_write_input_tokens": 13303, + "output_tokens": 929, + "reasoning_output_tokens": 158 + }, + "steps": [ + "test-package", + "review-release" + ], + "terminal": "blocked", + "requirements": [ + true, + false + ], + "command_evidence": [ + { + "kind": "run_command", + "id": "test-package", + "exit_code": 0 + } + ], + "accepted_agent_tasks": 1, + "accepted_user_answers": 0, + "response_rejections": [], + "event_counts": { + "operation.completed": 2, + "operation.requested": 2, + "requirement.failed": 1, + "requirement.passed": 1, + "run.blocked": 1, + "run.started": 1 + } + }, + { + "case": "approval-refuses", + "arm": "long", + "repeat": 1, + "passed": true, + "failures": [], + "duration_ms": 15068, + "usage": { + "input_tokens": 128054, + "cached_input_tokens": 114318, + "cache_write_input_tokens": 13706, + "output_tokens": 1194, + "reasoning_output_tokens": 83 + }, + "steps": [ + "test-package", + "review-release", + "approve-publish" + ], + "terminal": "refused", + "requirements": [ + true, + true + ], + "command_evidence": [ + { + "kind": "run_command", + "id": "test-package", + "exit_code": 0 + } + ], + "accepted_agent_tasks": 1, + "accepted_user_answers": 1 + }, + { + "case": "approval-refuses", + "arm": "yield", + "repeat": 1, + "passed": true, + "failures": [], + "duration_ms": 15466, + "usage": { + "input_tokens": 129932, + "cached_input_tokens": 115871, + "cache_write_input_tokens": 14031, + "output_tokens": 1339, + "reasoning_output_tokens": 151 + }, + "steps": [ + "test-package", + "review-release", + "approve-publish" + ], + "terminal": "refused", + "requirements": [ + true, + true + ], + "command_evidence": [ + { + "kind": "run_command", + "id": "test-package", + "exit_code": 0 + } + ], + "accepted_agent_tasks": 1, + "accepted_user_answers": 1, + "response_rejections": [], + "event_counts": { + "operation.completed": 3, + "operation.requested": 3, + "requirement.passed": 2, + "run.refused": 1, + "run.started": 1 + } + }, + { + "case": "publish-blocks", + "arm": "long", + "repeat": 1, + "passed": true, + "failures": [], + "duration_ms": 14641, + "usage": { + "input_tokens": 141305, + "cached_input_tokens": 127522, + "cache_write_input_tokens": 13750, + "output_tokens": 1347, + "reasoning_output_tokens": 96 + }, + "steps": [ + "test-package", + "review-release", + "approve-publish", + "publish-package" + ], + "terminal": "blocked", + "requirements": [ + true, + true, + false + ], + "command_evidence": [ + { + "kind": "run_command", + "id": "test-package", + "exit_code": 0 + }, + { + "kind": "run_command", + "id": "publish-package", + "exit_code": 1 + } + ], + "accepted_agent_tasks": 1, + "accepted_user_answers": 1 + }, + { + "case": "publish-blocks", + "arm": "yield", + "repeat": 1, + "passed": true, + "failures": [], + "duration_ms": 16878, + "usage": { + "input_tokens": 131091, + "cached_input_tokens": 116928, + "cache_write_input_tokens": 14133, + "output_tokens": 1292, + "reasoning_output_tokens": 162 + }, + "steps": [ + "test-package", + "review-release", + "approve-publish", + "publish-package" + ], + "terminal": "blocked", + "requirements": [ + true, + true, + false + ], + "command_evidence": [ + { + "kind": "run_command", + "id": "test-package", + "exit_code": 0 + }, + { + "kind": "run_command", + "id": "publish-package", + "exit_code": 1 + } + ], + "accepted_agent_tasks": 1, + "accepted_user_answers": 1, + "response_rejections": [], + "event_counts": { + "operation.completed": 4, + "operation.requested": 4, + "requirement.failed": 1, + "requirement.passed": 2, + "run.blocked": 1, + "run.started": 1 + } + }, + { + "case": "verification-blocks", + "arm": "long", + "repeat": 1, + "passed": true, + "failures": [], + "duration_ms": 23277, + "usage": { + "input_tokens": 184440, + "cached_input_tokens": 169988, + "cache_write_input_tokens": 14410, + "output_tokens": 1660, + "reasoning_output_tokens": 114 + }, + "steps": [ + "test-package", + "review-release", + "approve-publish", + "publish-package", + "verify-package" + ], + "terminal": "blocked", + "requirements": [ + true, + true, + true, + false + ], + "command_evidence": [ + { + "kind": "run_command", + "id": "test-package", + "exit_code": 0 + }, + { + "kind": "run_command", + "id": "publish-package", + "exit_code": 0 + }, + { + "kind": "run_command", + "id": "verify-package", + "exit_code": 1 + } + ], + "accepted_agent_tasks": 1, + "accepted_user_answers": 1 + }, + { + "case": "verification-blocks", + "arm": "yield", + "repeat": 1, + "passed": true, + "failures": [], + "duration_ms": 15430, + "usage": { + "input_tokens": 129225, + "cached_input_tokens": 115301, + "cache_write_input_tokens": 13894, + "output_tokens": 1270, + "reasoning_output_tokens": 167 + }, + "steps": [ + "test-package", + "review-release", + "approve-publish", + "publish-package", + "verify-package" + ], + "terminal": "blocked", + "requirements": [ + true, + true, + true, + false + ], + "command_evidence": [ + { + "kind": "run_command", + "id": "test-package", + "exit_code": 0 + }, + { + "kind": "run_command", + "id": "publish-package", + "exit_code": 0 + }, + { + "kind": "run_command", + "id": "verify-package", + "exit_code": 1 + } + ], + "accepted_agent_tasks": 1, + "accepted_user_answers": 1, + "response_rejections": [], + "event_counts": { + "operation.completed": 5, + "operation.requested": 5, + "requirement.failed": 1, + "requirement.passed": 3, + "run.blocked": 1, + "run.started": 1 + } + }, + { + "case": "success", + "arm": "long", + "repeat": 1, + "passed": true, + "failures": [], + "duration_ms": 21019, + "usage": { + "input_tokens": 156788, + "cached_input_tokens": 142604, + "cache_write_input_tokens": 14148, + "output_tokens": 1557, + "reasoning_output_tokens": 215 + }, + "steps": [ + "test-package", + "review-release", + "approve-publish", + "publish-package", + "verify-package" + ], + "terminal": "completed", + "requirements": [ + true, + true, + true, + true + ], + "command_evidence": [ + { + "kind": "run_command", + "id": "test-package", + "exit_code": 0 + }, + { + "kind": "run_command", + "id": "publish-package", + "exit_code": 0 + }, + { + "kind": "run_command", + "id": "verify-package", + "exit_code": 0 + } + ], + "accepted_agent_tasks": 1, + "accepted_user_answers": 1 + }, + { + "case": "success", + "arm": "yield", + "repeat": 1, + "passed": true, + "failures": [], + "duration_ms": 17399, + "usage": { + "input_tokens": 129615, + "cached_input_tokens": 115624, + "cache_write_input_tokens": 13961, + "output_tokens": 1284, + "reasoning_output_tokens": 162 + }, + "steps": [ + "test-package", + "review-release", + "approve-publish", + "publish-package", + "verify-package" + ], + "terminal": "completed", + "requirements": [ + true, + true, + true, + true + ], + "command_evidence": [ + { + "kind": "run_command", + "id": "test-package", + "exit_code": 0 + }, + { + "kind": "run_command", + "id": "publish-package", + "exit_code": 0 + }, + { + "kind": "run_command", + "id": "verify-package", + "exit_code": 0 + } + ], + "accepted_agent_tasks": 1, + "accepted_user_answers": 1, + "response_rejections": [], + "event_counts": { + "operation.completed": 5, + "operation.requested": 5, + "requirement.passed": 4, + "run.completed": 1, + "run.started": 1 + } + } + ], + "equivalence": { + "passed": 6, + "total": 6, + "comparisons": [ + { + "case": "preflight-blocks", + "repeat": 1, + "equivalent": true, + "long_steps": [ + "test-package" + ], + "yield_steps": [ + "test-package" + ], + "long_terminal": "blocked", + "yield_terminal": "blocked", + "long_requirements": [ + false + ], + "yield_requirements": [ + false + ], + "accepted_agent_tasks": { + "long": 0, + "yield": 0 + }, + "accepted_user_answers": { + "long": 0, + "yield": 0 + } + }, + { + "case": "review-blocks", + "repeat": 1, + "equivalent": true, + "long_steps": [ + "test-package", + "review-release" + ], + "yield_steps": [ + "test-package", + "review-release" + ], + "long_terminal": "blocked", + "yield_terminal": "blocked", + "long_requirements": [ + true, + false + ], + "yield_requirements": [ + true, + false + ], + "accepted_agent_tasks": { + "long": 1, + "yield": 1 + }, + "accepted_user_answers": { + "long": 0, + "yield": 0 + } + }, + { + "case": "approval-refuses", + "repeat": 1, + "equivalent": true, + "long_steps": [ + "test-package", + "review-release", + "approve-publish" + ], + "yield_steps": [ + "test-package", + "review-release", + "approve-publish" + ], + "long_terminal": "refused", + "yield_terminal": "refused", + "long_requirements": [ + true, + true + ], + "yield_requirements": [ + true, + true + ], + "accepted_agent_tasks": { + "long": 1, + "yield": 1 + }, + "accepted_user_answers": { + "long": 1, + "yield": 1 + } + }, + { + "case": "publish-blocks", + "repeat": 1, + "equivalent": true, + "long_steps": [ + "test-package", + "review-release", + "approve-publish", + "publish-package" + ], + "yield_steps": [ + "test-package", + "review-release", + "approve-publish", + "publish-package" + ], + "long_terminal": "blocked", + "yield_terminal": "blocked", + "long_requirements": [ + true, + true, + false + ], + "yield_requirements": [ + true, + true, + false + ], + "accepted_agent_tasks": { + "long": 1, + "yield": 1 + }, + "accepted_user_answers": { + "long": 1, + "yield": 1 + } + }, + { + "case": "verification-blocks", + "repeat": 1, + "equivalent": true, + "long_steps": [ + "test-package", + "review-release", + "approve-publish", + "publish-package", + "verify-package" + ], + "yield_steps": [ + "test-package", + "review-release", + "approve-publish", + "publish-package", + "verify-package" + ], + "long_terminal": "blocked", + "yield_terminal": "blocked", + "long_requirements": [ + true, + true, + true, + false + ], + "yield_requirements": [ + true, + true, + true, + false + ], + "accepted_agent_tasks": { + "long": 1, + "yield": 1 + }, + "accepted_user_answers": { + "long": 1, + "yield": 1 + } + }, + { + "case": "success", + "repeat": 1, + "equivalent": true, + "long_steps": [ + "test-package", + "review-release", + "approve-publish", + "publish-package", + "verify-package" + ], + "yield_steps": [ + "test-package", + "review-release", + "approve-publish", + "publish-package", + "verify-package" + ], + "long_terminal": "completed", + "yield_terminal": "completed", + "long_requirements": [ + true, + true, + true, + true + ], + "yield_requirements": [ + true, + true, + true, + true + ], + "accepted_agent_tasks": { + "long": 1, + "yield": 1 + }, + "accepted_user_answers": { + "long": 1, + "yield": 1 + } + } + ] + }, + "claim_boundary": { + "summary": "The same Codex model followed the owned long skill and the thin-skill-plus-Yield workflow with the same tested step order and final status.", + "exclusions": [ + "This does not measure general coding-agent quality.", + "This does not claim Yield is better than a long skill.", + "This covers the declared cases, not every possible workflow." + ] + } +} diff --git a/labs/22-yield/yield/evals/results/latest.json b/labs/22-yield/yield/evals/results/latest.json index b521e6cd..2c097da3 100644 --- a/labs/22-yield/yield/evals/results/latest.json +++ b/labs/22-yield/yield/evals/results/latest.json @@ -1,8 +1,8 @@ { "schema_version": 2, "methodology_version": "1.0", - "generated_at": "2026-08-01T17:53:20.762Z", - "source_digest": "8426ef59771921109fa7aa2d242e089d13f391aea60f2aebf509da93b7b534ae", + "generated_at": "2026-08-01T18:42:40.588Z", + "source_digest": "864ef22f4ea4d990b0576a69ba1208f58964c4aec4791373569053f2f7626edb", "status": "passed", "workflow_conformance": { "passed": 40,