From 72019f4f5bf19a690facfeaa00d3ef5a394e7f0d Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 05:32:58 +0000 Subject: [PATCH 1/5] test: pin remaining Grok terminal evidence defects (#78) Reproduce cancellation classification, lost provider metadata, and missing full stream artifacts on current main without changing Grok permissions. Keep this red checkpoint local until the implementation passes. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .../scripts/runner/parse-output.test.ts | 41 ++++++++++++ .../poteto-mode/scripts/runner/run.test.ts | 62 +++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/plugins/pstack/skills/poteto-mode/scripts/runner/parse-output.test.ts b/plugins/pstack/skills/poteto-mode/scripts/runner/parse-output.test.ts index b4ebc041..dc0e3c92 100644 --- a/plugins/pstack/skills/poteto-mode/scripts/runner/parse-output.test.ts +++ b/plugins/pstack/skills/poteto-mode/scripts/runner/parse-output.test.ts @@ -120,6 +120,47 @@ describe("parseProviderOutput", () => { expect(reportedModelMatches("grok", "fable", "claude-fable-9-9")).toBe(false); }); + for (const [stopReason, expectedStatus] of [ + ["cancelled", "cancelled"], ["canceled", "cancelled"], ["end_turn", "child-failed"], + ]) { + it(`issue78 retains terminal metadata for ${stopReason}`, () => { + let failure: unknown; + try { + parseProviderOutput("grok", JSON.stringify({ + type: "result", subtype: "error_during_execution", is_error: true, + stop_reason: stopReason, errors: ["Exact provider reason"], + session_id: "terminal-session", modelUsage: { "grok-4.6-build": {} }, + usage: { input_tokens: 30, output_tokens: 4 }, total_cost_usd: 0.02, + }), "", "grok-4.6"); + } catch (error) { + failure = error; + } + expect(failure).toMatchObject({ + message: "Exact provider reason", status: expectedStatus, + metadata: { + reportedModel: "grok-4.6-build", sessionId: "terminal-session", + usage: { inputTokens: 30, outputTokens: 4 }, costUsd: 0.02, + }, + }); + }); + } + + it("issue78 rejects incomplete terminal status", () => { + for (const terminal of [ + { type: "result", result: "text" }, + { type: "result", subtype: "success", result: "text" }, + { type: "result", subtype: "", is_error: true }, + { type: "result", subtype: "api_error", is_error: "true" }, + ]) { + expect(() => parseProviderOutput("grok", JSON.stringify(terminal), "", "grok-4.6")) + .toThrow("valid terminal status"); + } + expect(() => parseProviderOutput("grok", "not-json", "", "grok-4.6")) + .toThrow("non-JSON event"); + expect(() => parseProviderOutput("grok", '{"type":"assistant"}', "", "grok-4.6")) + .toThrow("terminal event"); + }); + it("rejects malformed or textless responses", () => { expect(() => parseProviderOutput("claude", "not-json", "", "fable") diff --git a/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts b/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts index c114d102..ad4302af 100644 --- a/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts +++ b/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts @@ -276,6 +276,68 @@ afterEach(() => { rmSync(scratch, { recursive: true, force: true }); }); +describe("issue78 terminal results", () => { + for (const scenario of ["cancel-zero", "cancel-nonzero", "api-error", "success-streams"]) { + it(`issue78 ${scenario}`, async () => { + const input = options("grok"); + const reason = scenario === "api-error" + ? "API unavailable" + : "User cancelled the execution for tool run_terminal_command"; + const terminal = { + type: "result", + subtype: scenario === "api-error" ? "api_error" + : scenario === "success-streams" ? "success" : "error_during_execution", + is_error: scenario !== "success-streams", + stop_reason: scenario.startsWith("cancel") ? "cancelled" : "end_turn", + errors: [reason], + result: "GROK_OK", + session_id: "terminal-session", + usage: { input_tokens: 30, output_tokens: 4 }, + total_cost_usd: 0.02, + modelUsage: { "grok-4.6-build": {} }, + }; + const stdout = JSON.stringify(terminal) + "\n"; + const stderr = "startup warning\n".repeat(400) + "\n"; + writeFileSync(join(bin, "grok"), `#!/usr/bin/env bun +if (process.argv[2] === "models") { + console.log("You are logged in with grok.com.\\nAvailable models:\\n * grok-4.6 (default)"); + process.exit(0); +} +process.stdout.write(${JSON.stringify(stdout)}); +process.stderr.write(${JSON.stringify(stderr)}); +process.exit(${scenario === "cancel-nonzero" ? 1 : 0}); +`); + const result = await runLane(input); + const saved = JSON.parse(readFileSync(input.receiptPath, "utf8")); + expect(saved.status).toBe(scenario === "success-streams" ? "complete" + : scenario === "api-error" ? "child-failed" : "cancelled"); + expect(result.exitCode).toBe(scenario === "success-streams" ? 0 + : scenario === "api-error" ? 70 : 130); + expect(saved.exitCode).toBe(scenario === "cancel-nonzero" ? 1 : 0); + expect(saved.reportedModel).toBe("grok-4.6-build"); + expect(saved.modelVerified).toBe(true); + expect(saved.modelEvidence).toBe("provider-report"); + expect(saved.sessionId).toBe("terminal-session"); + expect(saved.usage).toEqual({ inputTokens: 30, outputTokens: 4 }); + expect(saved.costUsd).toBe(0.02); + if (scenario !== "success-streams") { + expect(saved.error.message).toBe(reason); + expect(saved.error.evidence.startsWith(reason)).toBe(true); + expect(saved.error.evidence.length).toBeLessThanOrEqual(4_000); + expect(existsSync(input.outputPath)).toBe(false); + } + expect(saved.stdoutPath).toBe(`${input.receiptPath}.stdout`); + expect(saved.stderrPath).toBe(`${input.receiptPath}.stderr`); + expect(readFileSync(saved.stdoutPath)).toEqual(Buffer.from(stdout)); + expect(readFileSync(saved.stderrPath)).toEqual(Buffer.from(stderr)); + expect(statSync(saved.stderrPath).size).toBe(6_401); + for (const path of [saved.stdoutPath, saved.stderrPath, input.receiptPath]) { + expect(statSync(path).mode & 0o777).toBe(0o600); + } + }); + } +}); + describe("runLane", () => { for (const provider of ["claude", "codex", "grok"] as const) { it(`executes and receipts the ${provider} external lane`, async () => { From bdd51f6e1ab5f1216c849af7b31e48effafd04aa Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 05:37:44 +0000 Subject: [PATCH 2/5] fix: retain Grok terminal failure classification and metadata (#78) Parse terminal provider results after ordinary nonzero exits while preserving launcher cancellation and timeout precedence. Keep exact provider reasons and model proof rather than treating valid failures as malformed. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .../scripts/runner/parse-output.ts | 37 ++++++++++++--- .../poteto-mode/scripts/runner/run.test.ts | 10 ++--- .../skills/poteto-mode/scripts/runner/run.ts | 45 ++++++++++++------- 3 files changed, 63 insertions(+), 29 deletions(-) diff --git a/plugins/pstack/skills/poteto-mode/scripts/runner/parse-output.ts b/plugins/pstack/skills/poteto-mode/scripts/runner/parse-output.ts index 81ed53d4..114c0683 100644 --- a/plugins/pstack/skills/poteto-mode/scripts/runner/parse-output.ts +++ b/plugins/pstack/skills/poteto-mode/scripts/runner/parse-output.ts @@ -8,6 +8,17 @@ import { isRollingClaudeAlias, } from "./model-aliases.ts"; +export class ProviderResultError extends Error { + constructor( + message: string, + readonly status: "cancelled" | "child-failed", + readonly metadata: Omit + ) { + super(message); + this.name = "ProviderResultError"; + } +} + type JsonObject = Record; function object(value: unknown): JsonObject | null { @@ -99,19 +110,31 @@ function parseGrok(stdout: string, requestedModel: string): ParsedOutput { } if (result === null) throw new Error("grok result did not contain a terminal event"); - if (result.is_error === true || result.subtype !== "success") { - throw new Error("grok reported an error result"); + if (nullableString(result.subtype) === null || typeof result.is_error !== "boolean") { + throw new Error("grok result did not contain a valid terminal status"); } - const text = nullableString(result.result); - if (text === null) throw new Error("grok result did not contain final text"); - - return { - text, + const metadata = { reportedModel: modelFromUsage(result.modelUsage, "grok", requestedModel), sessionId: nullableString(result.session_id), usage: normalizedUsage(result.usage), costUsd: finiteNumber(result.total_cost_usd) ?? null, }; + if (result.is_error || result.subtype !== "success") { + const errors = Array.isArray(result.errors) + ? result.errors.filter((entry): entry is string => typeof entry === "string" && entry.length > 0) + : []; + const message = errors.join("\n") || nullableString(result.error) + || nullableString(object(result.error)?.message) || nullableString(result.result) + || `grok reported ${result.subtype}`; + const cancelled = result.stop_reason === "cancelled" || result.stop_reason === "canceled" + || result.subtype === "cancelled" || result.subtype === "canceled" + || /^(?:User cancel(?:led|ed) the execution|PermissionCancelled)/i.test(message); + throw new ProviderResultError(message, cancelled ? "cancelled" : "child-failed", metadata); + } + const text = nullableString(result.result); + if (text === null) throw new Error("grok result did not contain final text"); + + return { text, ...metadata }; } function parseCodex(stdout: string): ParsedOutput { diff --git a/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts b/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts index ad4302af..0bef5730 100644 --- a/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts +++ b/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts @@ -374,19 +374,19 @@ describe("runLane", () => { }); }); - it("keeps Grok's terminal error result in malformed-output evidence", async () => { + it("keeps Grok's terminal cancellation reason in bounded evidence", async () => { process.env.FAKE_GROK_ERROR_RESULT = "1"; const input = options("grok", "grok-error-result"); const result = await runLane(input); - expect(result.exitCode).toBe(65); + expect(result.exitCode).toBe(130); expect(existsSync(input.outputPath)).toBe(false); const recorded = receipt(input.receiptPath); expect(recorded).toMatchObject({ - status: "malformed-output", + status: "cancelled", exitCode: 0, - error: { message: "grok reported an error result" }, + error: { message: "cancelled" }, }); - expect(recorded.error?.evidence).toContain('"stop_reason":"cancelled","errors":["cancelled"]'); + expect(recorded.error?.evidence).toStartWith("cancelled"); expect(recorded.error?.evidence.length).toBeLessThanOrEqual(4_000); }); diff --git a/plugins/pstack/skills/poteto-mode/scripts/runner/run.ts b/plugins/pstack/skills/poteto-mode/scripts/runner/run.ts index aa887f00..ce2d1cce 100644 --- a/plugins/pstack/skills/poteto-mode/scripts/runner/run.ts +++ b/plugins/pstack/skills/poteto-mode/scripts/runner/run.ts @@ -11,8 +11,9 @@ import { import { dirname, resolve } from "node:path"; import { invocationCommand, preflightCommand, type CommandSpec } from "./commands.ts"; import { versionedClaudeAlias } from "./model-aliases.ts"; -import { parseProviderOutput, reportedModelMatches } from "./parse-output.ts"; +import { parseProviderOutput, ProviderResultError, reportedModelMatches } from "./parse-output.ts"; import type { + ParsedOutput, Provider, ReceiptStatus, RunnerOptions, @@ -770,23 +771,37 @@ async function executeLane( signal: result.signal, } as const; - if (result.cancelledBy !== null || result.timedOut || result.exitCode !== 0) { + let parsed: ParsedOutput | null = null; + let parseError: unknown = null; + if (result.cancelledBy === null && !result.timedOut + && (result.exitCode === 0 || (options.provider === "grok" && result.stdout.trim().length > 0))) { + try { + parsed = parseProviderOutput(options.provider, result.stdout, result.stderr, options.model); + } catch (error) { + parseError = error; + } + } + const providerFailure = parseError instanceof ProviderResultError ? parseError : null; + const metadata = providerFailure?.metadata ?? parsed; + if (result.cancelledBy !== null || result.timedOut || providerFailure !== null + || (result.exitCode !== 0 && parseError === null)) { const rawFailureEvidence = `${result.stderr}\n${result.stdout}`; - const failureEvidence = evidence(rawFailureEvidence); + const failureEvidence = evidence(providerFailure === null + ? rawFailureEvidence : `${providerFailure.message}\n${rawFailureEvidence}`); const status: ReceiptStatus = result.cancelledBy !== null ? "cancelled" : result.timedOut ? "timed-out" - : unavailableStatus(rawFailureEvidence); + : providerFailure?.status ?? unavailableStatus(rawFailureEvidence); receipt = completeReceipt(options, { ...base, status, - reportedModel: null, - modelVerified: false, - modelEvidence: null, - sessionId: null, - usage: null, - costUsd: null, + ...(metadata === null + ? { reportedModel: null, modelVerified: false, modelEvidence: null } + : modelProof(options.provider, options.model, metadata.reportedModel)), + sessionId: metadata?.sessionId ?? null, + usage: metadata?.usage ?? null, + costUsd: metadata?.costUsd ?? null, error: { message: result.cancelledBy !== null ? result.signal === result.cancelledBy @@ -794,7 +809,7 @@ async function executeLane( : `launcher received ${result.cancelledBy} after child exited` : result.timedOut ? `launcher exceeded the explicit ${options.timeoutMs}ms deadline` - : `child exited with status ${result.exitCode}`, + : providerFailure?.message ?? `child exited with status ${result.exitCode}`, evidence: failureEvidence, }, }); @@ -804,12 +819,8 @@ async function executeLane( } try { - const parsed = parseProviderOutput( - options.provider, - result.stdout, - result.stderr, - options.model - ); + if (parseError !== null) throw parseError; + if (parsed === null) throw new Error("provider result was not parsed"); const proof = modelProof( options.provider, options.model, From 19d6d5b15ecd394de68fa3e95f9f70c9c9a98824 Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 06:04:40 +0000 Subject: [PATCH 3/5] fix: preserve complete private runner streams on failures (#78) Reserve collision-safe stdout and stderr artifacts, retain raw bytes and stop the child promptly if artifact writes fail. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .../poteto-mode/scripts/runner/run.test.ts | 171 ++++++++++++++++-- .../skills/poteto-mode/scripts/runner/run.ts | 70 +++++-- .../poteto-mode/scripts/runner/types.ts | 2 + 3 files changed, 215 insertions(+), 28 deletions(-) diff --git a/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts b/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts index 0bef5730..ca96d442 100644 --- a/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts +++ b/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts @@ -1,4 +1,5 @@ -import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { afterEach, beforeEach, describe, expect, it, spyOn } from "bun:test"; +import * as fs from "node:fs"; import { chmodSync, cpSync, @@ -20,6 +21,11 @@ let scratch = ""; let bin = ""; let previousPath: string | undefined; +function streamPath(path: string | null): string { + if (path === null) throw new Error("expected a reserved stream artifact path"); + return path; +} + const fake = `#!/usr/bin/env bun import { appendFileSync, existsSync, unlinkSync, writeFileSync } from "node:fs"; const args = process.argv.slice(2); @@ -277,15 +283,15 @@ afterEach(() => { }); describe("issue78 terminal results", () => { - for (const scenario of ["cancel-zero", "cancel-nonzero", "api-error", "success-streams"]) { + for (const scenario of ["cancel-zero", "cancel-nonzero", "api-error", "api-error-nonzero", "success-streams"]) { it(`issue78 ${scenario}`, async () => { const input = options("grok"); - const reason = scenario === "api-error" + const reason = scenario.startsWith("api-error") ? "API unavailable" : "User cancelled the execution for tool run_terminal_command"; const terminal = { type: "result", - subtype: scenario === "api-error" ? "api_error" + subtype: scenario.startsWith("api-error") ? "api_error" : scenario === "success-streams" ? "success" : "error_during_execution", is_error: scenario !== "success-streams", stop_reason: scenario.startsWith("cancel") ? "cancelled" : "end_turn", @@ -305,15 +311,15 @@ if (process.argv[2] === "models") { } process.stdout.write(${JSON.stringify(stdout)}); process.stderr.write(${JSON.stringify(stderr)}); -process.exit(${scenario === "cancel-nonzero" ? 1 : 0}); +process.exit(${scenario.endsWith("nonzero") ? 1 : 0}); `); const result = await runLane(input); const saved = JSON.parse(readFileSync(input.receiptPath, "utf8")); expect(saved.status).toBe(scenario === "success-streams" ? "complete" - : scenario === "api-error" ? "child-failed" : "cancelled"); + : scenario.startsWith("api-error") ? "child-failed" : "cancelled"); expect(result.exitCode).toBe(scenario === "success-streams" ? 0 - : scenario === "api-error" ? 70 : 130); - expect(saved.exitCode).toBe(scenario === "cancel-nonzero" ? 1 : 0); + : scenario.startsWith("api-error") ? 70 : 130); + expect(saved.exitCode).toBe(scenario.endsWith("nonzero") ? 1 : 0); expect(saved.reportedModel).toBe("grok-4.6-build"); expect(saved.modelVerified).toBe(true); expect(saved.modelEvidence).toBe("provider-report"); @@ -328,14 +334,149 @@ process.exit(${scenario === "cancel-nonzero" ? 1 : 0}); } expect(saved.stdoutPath).toBe(`${input.receiptPath}.stdout`); expect(saved.stderrPath).toBe(`${input.receiptPath}.stderr`); - expect(readFileSync(saved.stdoutPath)).toEqual(Buffer.from(stdout)); - expect(readFileSync(saved.stderrPath)).toEqual(Buffer.from(stderr)); - expect(statSync(saved.stderrPath).size).toBe(6_401); + expect(readFileSync(streamPath(saved.stdoutPath))).toEqual(Buffer.from(stdout)); + expect(readFileSync(streamPath(saved.stderrPath))).toEqual(Buffer.from(stderr)); + expect(statSync(streamPath(saved.stderrPath)).size).toBe(6_401); + expect(saved.schemaVersion).toBe(1); for (const path of [saved.stdoutPath, saved.stderrPath, input.receiptPath]) { - expect(statSync(path).mode & 0o777).toBe(0o600); + expect(statSync(streamPath(path)).mode & 0o777).toBe(0o600); + } + }); + } + + for (const stream of ["stdout", "stderr"]) { + it(`rolls back reservations without overwriting an existing ${stream} sidecar`, async () => { + const input = options("grok"); + const conflict = `${input.receiptPath}.${stream}`; + writeFileSync(conflict, "operator evidence"); + await expect(runLane(input)).rejects.toThrow(); + expect(readFileSync(conflict, "utf8")).toBe("operator evidence"); + for (const path of [input.outputPath, input.receiptPath, + `${input.receiptPath}.${stream === "stdout" ? "stderr" : "stdout"}`]) { + expect(existsSync(path)).toBe(false); + } + }); + + it(`rejects a ${stream} sidecar colliding with the prompt or output`, async () => { + const input = options("grok"); + const conflict = `${input.receiptPath}.${stream}`; + writeFileSync(conflict, "operator prompt"); + await expect(runLane({ ...input, promptPath: conflict })).rejects.toThrow("must be distinct"); + expect(readFileSync(conflict, "utf8")).toBe("operator prompt"); + rmSync(conflict); + await expect(runLane({ ...input, outputPath: conflict })).rejects.toThrow("must be distinct"); + for (const path of [input.outputPath, input.receiptPath, + `${input.receiptPath}.stdout`, `${input.receiptPath}.stderr`]) { + expect(existsSync(path)).toBe(false); } }); } + + it("retains complete streams when writes are partial", async () => { + const input = options("grok"); + const stdout = JSON.stringify({ type: "result", subtype: "success", is_error: false, + result: "GROK_OK", modelUsage: { "grok-4.6-build": {} } }) + "\n"; + const stderr = "complete stderr despite partial writes\n"; + writeFileSync(join(bin, "grok"), fake.replace("const modelIndex =", + `process.stdout.write(${JSON.stringify(stdout)}); process.stderr.write(${JSON.stringify(stderr)}); process.exit(0);\nconst modelIndex =`)); + const originalWrite = fs.writeSync; + let partialWrites = 0; + const write = spyOn(fs, "writeSync").mockImplementation((descriptor, data, offset, length) => { + if (typeof data === "string") return originalWrite(descriptor, data, offset); + partialWrites++; + return originalWrite(descriptor, data, offset, + Math.min(typeof length === "number" ? length : data.byteLength, 7)); + }); + try { + const result = await runLane(input); + expect(result.exitCode).toBe(0); + expect(partialWrites).toBeGreaterThan(1); + expect(readFileSync(streamPath(result.receipt.stdoutPath))).toEqual(Buffer.from(stdout)); + expect(readFileSync(streamPath(result.receipt.stderrPath))).toEqual(Buffer.from(stderr)); + } finally { + write.mockRestore(); + } + }); + + it("terminalizes a sidecar write failure without waiting for the child", async () => { + const input = options("grok"); + writeFileSync(join(bin, "grok"), fake.replace("const modelIndex =", + 'process.stdout.write("stream data"); await Bun.sleep(10_000);\nconst modelIndex =')); + const originalWrite = fs.writeSync; + const write = spyOn(fs, "writeSync").mockImplementation((descriptor, data, offset, length) => { + if (typeof data === "string") return originalWrite(descriptor, data, offset); + throw new Error("sidecar storage unavailable"); + }); + try { + const result = await runLane(input); + expect(result.exitCode).toBe(70); + expect(result.receipt.status).toBe("child-failed"); + expect(result.receipt.error?.evidence).toContain("sidecar storage unavailable"); + expect(existsSync(input.outputPath)).toBe(false); + expect(existsSync(streamPath(result.receipt.stdoutPath))).toBe(true); + expect(existsSync(streamPath(result.receipt.stderrPath))).toBe(true); + } finally { + write.mockRestore(); + } + }); + + it("retains undecodable stderr bytes rather than re-encoding captured text", async () => { + const input = options("grok"); + const bytes = Buffer.from([0, 255, 195, 169, 13, 10]); + writeFileSync(join(bin, "grok"), fake.replace("const modelIndex =", + `process.stderr.write(Buffer.from(${JSON.stringify([...bytes])}));\nconst modelIndex =`)); + const result = await runLane(input); + expect(result.exitCode).toBe(0); + expect(readFileSync(streamPath(result.receipt.stderrPath))).toEqual(bytes); + }); + + for (const exitCode of [0, 1]) { + it(`keeps malformed terminal data distinct after child exit ${exitCode}`, async () => { + const input = options("grok"); + const stdout = '{"type":"result","is_error":true}\n'; + writeFileSync(join(bin, "grok"), fake.replace( + 'console.log(JSON.stringify({type:"result",subtype:"error_during_execution",is_error:true,stop_reason:"cancelled",errors:["cancelled"]}));', + `process.stdout.write(${JSON.stringify(stdout)}); process.exit(${exitCode});`)); + process.env.FAKE_GROK_ERROR_RESULT = "1"; + const result = await runLane(input); + expect(result.exitCode).toBe(65); + expect(result.receipt).toMatchObject({ + status: "malformed-output", exitCode, reportedModel: null, modelVerified: false, + sessionId: null, error: { message: "grok result did not contain a valid terminal status" }, + }); + expect(result.receipt.error?.evidence).toEndWith(stdout.trim()); + expect(readFileSync(streamPath(result.receipt.stdoutPath), "utf8")).toEndWith(stdout); + expect(existsSync(input.outputPath)).toBe(false); + }); + } + + it("does not invent metadata for a valid provider failure", async () => { + const input = options("grok"); + writeFileSync(join(bin, "grok"), fake.replace( + 'console.log(JSON.stringify({type:"result",subtype:"error_during_execution",is_error:true,stop_reason:"cancelled",errors:["cancelled"]}));', + 'console.log(JSON.stringify({type:"result",subtype:"api_error",is_error:true,errors:["API unavailable"]})); process.exit(1);')); + process.env.FAKE_GROK_ERROR_RESULT = "1"; + const result = await runLane(input); + expect(result.exitCode).toBe(70); + expect(result.receipt).toMatchObject({ + status: "child-failed", exitCode: 1, reportedModel: null, modelVerified: false, + modelEvidence: null, sessionId: null, usage: null, costUsd: null, + error: { message: "API unavailable" }, + }); + expect(readFileSync(streamPath(result.receipt.stdoutPath), "utf8")).toContain("api_error"); + expect(existsSync(input.outputPath)).toBe(false); + }); + + it("retains strict model verification for successful Grok results", async () => { + const input = options("grok"); + writeFileSync(join(bin, "grok"), fake.replace('[model + "-build"]', '["grok-unexpected"]')); + const result = await runLane(input); + expect(result.exitCode).toBe(65); + expect(result.receipt).toMatchObject({ status: "malformed-output", modelVerified: false }); + expect(result.receipt.error?.message).toBe("requested model grok-4.6 was not reported by grok"); + expect(readFileSync(streamPath(result.receipt.stdoutPath), "utf8")).toContain("grok-unexpected"); + expect(existsSync(input.outputPath)).toBe(false); + }); }); describe("runLane", () => { @@ -355,6 +496,12 @@ describe("runLane", () => { modelEvidence: provider === "codex" ? "pinned-argv" : "provider-report", preflight: { status: "passed" }, }); + const saved = receipt(input.receiptPath); + expect(readFileSync(streamPath(saved.stdoutPath), "utf8")).toContain(provider.toUpperCase()); + expect(readFileSync(streamPath(saved.stderrPath))).toEqual(Buffer.alloc(0)); + for (const path of [saved.stdoutPath, saved.stderrPath]) { + expect(statSync(streamPath(path)).mode & 0o777).toBe(0o600); + } if (provider === "claude") { expect(receipt(input.receiptPath).reportedModel).toBe("claude-fable-9-9"); } diff --git a/plugins/pstack/skills/poteto-mode/scripts/runner/run.ts b/plugins/pstack/skills/poteto-mode/scripts/runner/run.ts index ce2d1cce..c12a48f7 100644 --- a/plugins/pstack/skills/poteto-mode/scripts/runner/run.ts +++ b/plugins/pstack/skills/poteto-mode/scripts/runner/run.ts @@ -7,6 +7,7 @@ import { statSync, unlinkSync, writeFileSync, + writeSync, } from "node:fs"; import { dirname, resolve } from "node:path"; import { invocationCommand, preflightCommand, type CommandSpec } from "./commands.ts"; @@ -68,15 +69,36 @@ function reserve(path: string): void { closeSync(descriptor); } -function reserveOutputs(options: RunnerOptions): void { - if (options.outputPath === options.receiptPath) { - throw new UsageError("output and receipt paths must differ"); +interface ModelStreams { + readonly stdout: number; + readonly stderr: number; +} + +function modelStreamPaths(receiptPath: string): { stdoutPath: string; stderrPath: string } { + return { stdoutPath: `${receiptPath}.stdout`, stderrPath: `${receiptPath}.stderr` }; +} + +function reserveOutputs(options: RunnerOptions): ModelStreams { + const { stdoutPath, stderrPath } = modelStreamPaths(options.receiptPath); + const paths = [options.outputPath, options.receiptPath, stdoutPath, stderrPath]; + const resolved = paths.map((path) => resolve(path)); + if (new Set(resolved).size !== paths.length || resolved.includes(resolve(options.promptPath))) { + throw new UsageError("prompt, output, receipt, and stream paths must be distinct"); } - reserve(options.outputPath); + const created: string[] = []; + let stdout: number | null = null; try { - reserve(options.receiptPath); + for (const path of paths.slice(0, 2)) { + reserve(path); + created.push(path); + } + stdout = openSync(stdoutPath, "wx", 0o600); + created.push(stdoutPath); + const stderr = openSync(stderrPath, "wx", 0o600); + return { stdout, stderr }; } catch (error) { - removeIfExists(options.outputPath); + if (stdout !== null) closeSync(stdout); + for (const path of created) removeIfExists(path); throw error; } } @@ -179,7 +201,7 @@ interface StreamCapture { cancel(): Promise; } -function captureStream(stream: ReadableStream): StreamCapture { +function captureStream(stream: ReadableStream, descriptor?: number): StreamCapture { const reader = stream.getReader(); const decoder = new TextDecoder(); let text = ""; @@ -190,6 +212,12 @@ function captureStream(stream: ReadableStream): StreamCapture { while (true) { const next = await reader.read(); if (next.done) break; + if (descriptor !== undefined) { + let offset = 0; + while (offset < next.value.byteLength) { + offset += writeSync(descriptor, next.value, offset, next.value.byteLength - offset); + } + } text += decoder.decode(next.value, { stream: true }); } text += decoder.decode(); @@ -228,7 +256,8 @@ async function runProcess( env: NodeJS.ProcessEnv, prompt: string, deadlineAt: number | null, - cancellation: RunCancellation + cancellation: RunCancellation, + streamFiles?: ModelStreams ): Promise { const child = Bun.spawn([executable, ...spec.args], { cwd, @@ -238,8 +267,8 @@ async function runProcess( stderr: "pipe", }); let deadlineTimer: ReturnType | null = null; - const stdoutCapture = captureStream(child.stdout); - const stderrCapture = captureStream(child.stderr); + const stdoutCapture = captureStream(child.stdout, streamFiles?.stdout); + const stderrCapture = captureStream(child.stderr, streamFiles?.stderr); const streams = Promise.all([stdoutCapture.result, stderrCapture.result]); const exited = child.exited.then((exitCode): ProcessEvent => ({ kind: "exited", @@ -270,7 +299,7 @@ async function runProcess( stdin.end(); } - const completions = [exited, cancelled]; + const completions = [exited, cancelled, streams.then(() => exited)]; if (deadline !== null) completions.push(deadline); const first = await Promise.race(completions); @@ -472,7 +501,7 @@ function modelProof( function completeReceipt( options: RunnerOptions, - partial: Omit + partial: Omit ): RunnerReceipt { return { schemaVersion: 1, @@ -484,6 +513,7 @@ function completeReceipt( cwd: options.cwd, promptPath: options.promptPath, outputPath: options.outputPath, + ...modelStreamPaths(options.receiptPath), ...partial, }; } @@ -536,7 +566,8 @@ async function executeLane( deadlineAt: number | null, invocation: CommandSpec, preflight: CommandSpec, - progress: LaneProgress + progress: LaneProgress, + streamFiles: ModelStreams ): Promise { const startedAt = new Date(started).toISOString(); const prompt = readFileSync(options.promptPath, "utf8"); @@ -757,7 +788,8 @@ async function executeLane( env, prompt, deadlineAt, - cancellation + cancellation, + streamFiles ); const completed = Date.now(); const base = { @@ -882,8 +914,9 @@ export async function runLane( argv: [invocation.command, ...invocation.args], }; const cancellation = installRunCancellation(); + let streamFiles: ModelStreams | null = null; try { - reserveOutputs(options); + streamFiles = reserveOutputs(options); try { return await executeLane( options, @@ -892,7 +925,8 @@ export async function runLane( deadlineAt, invocation, preflight, - progress + progress, + streamFiles ); } catch (error) { const completed = Date.now(); @@ -937,6 +971,10 @@ export async function runLane( } } finally { cancellation.dispose(); + if (streamFiles !== null) { + closeSync(streamFiles.stdout); + closeSync(streamFiles.stderr); + } } } diff --git a/plugins/pstack/skills/poteto-mode/scripts/runner/types.ts b/plugins/pstack/skills/poteto-mode/scripts/runner/types.ts index 11c6dfb9..69aa7871 100644 --- a/plugins/pstack/skills/poteto-mode/scripts/runner/types.ts +++ b/plugins/pstack/skills/poteto-mode/scripts/runner/types.ts @@ -59,6 +59,8 @@ export interface RunnerReceipt { readonly cwd: string; readonly promptPath: string; readonly outputPath: string; + readonly stdoutPath: string | null; + readonly stderrPath: string | null; readonly startedAt: string; readonly completedAt: string; readonly elapsedMs: number; From 21e134ffd993ab4f1952382dd1c2717e51b3c3cf Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 06:06:31 +0000 Subject: [PATCH 4/5] docs: explain retained provider failure evidence (#78) Document private sidecars and distinguish provider cancellation from launcher signals without changing Grok policy. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .../pstack/skills/poteto-mode/references/provider-dispatch.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/pstack/skills/poteto-mode/references/provider-dispatch.md b/plugins/pstack/skills/poteto-mode/references/provider-dispatch.md index 005c5620..854fd7ce 100644 --- a/plugins/pstack/skills/poteto-mode/references/provider-dispatch.md +++ b/plugins/pstack/skills/poteto-mode/references/provider-dispatch.md @@ -82,7 +82,7 @@ The runner and its preflight have no implicit timeout. Do not invent a duration Read-only mode maps to Claude plan mode with project-only settings and an explicit tool list, Codex's read-only sandbox, and Grok auto mode plus its `read-only` sandbox and read-oriented tool list. Headless Grok cancels the whole turn on a permission prompt, which its `plan` mode raises for any shell command outside its built-in read-only list; auto mode reports a blocked call to the model instead. Grok's built-in read-only profile deliberately keeps its own state and system temporary directories writable, so point a read-only Grok lane at the actual checkout rather than a worktree under `/tmp`, `/var/tmp`, or the host's temporary directory. `isolated-write` maps to Claude `acceptEdits` with project-only settings, Codex `workspace-write`, and Grok auto mode plus its `workspace` sandbox and write-capable tool list. Grok uses auto mode in both access modes because `acceptEdits` raises the same turn-cancelling prompt for a shell command outside Grok's built-in list. Give every writer only a dedicated worktree or output directory. Never route a writer into the primary checkout. -Every concurrent external lane needs distinct prompt, output, and receipt paths. The launcher reserves output and receipt paths exclusively and refuses to overwrite them. +Every concurrent external lane needs distinct prompt, output, and receipt paths. The launcher exclusively reserves the output, receipt, and `.stdout` / `.stderr` sidecars with private (`0600`) permissions and refuses to overwrite existing files. All these paths must be distinct from each other and the prompt; a failed reservation rolls back only files created by that attempt. Schema-version-1 receipts add nullable `stdoutPath` and `stderrPath` fields identifying reserved artifacts. Sidecars retain the model process's raw stdout/stderr bytes, including on dropouts; they do not include authentication-preflight output. Cancellation and timeout retain every byte captured before the drain stops. Keep these private local files with the receipt; quote only operator-selected excerpts in public evidence. ## Completion and dropouts @@ -95,6 +95,6 @@ Success requires all of these: The receipt also carries elapsed time, token usage when the CLI exposes it, and cost when available. Keep it with the arena or review artifacts so parent-harness comparisons are evidence-based. -Any missing CLI, failed login, unavailable model, explicit timeout, cancellation, catchable post-reservation launcher failure, non-zero child exit, malformed result, or model mismatch is a receipt-bearing dropout. Record it and apply the calling skill's existing dropout policy. A `cancelled` receipt proves that the runner received the signal; its `signal` field is non-null only when the runner sent that signal to a still-active direct CLI child, and remains null when cancellation only stopped a post-exit pipe drain. The provider CLI owns any processes it starts beneath that direct child; the receipt does not claim a process-tree kill. Do not delete or overwrite the receipt. Never substitute the parent model, retry another provider, or reinterpret an external descriptor as a native model slug. +Any missing CLI, failed login, unavailable model, explicit timeout, cancellation, catchable post-reservation launcher failure, non-zero child exit, malformed result, or model mismatch is a receipt-bearing dropout. Record it and apply the calling skill's existing dropout policy. A `cancelled` receipt can represent either a launcher signal or a well-formed Grok terminal cancellation. Provider cancellations return wrapper exit 130; other valid Grok terminal failures return `child-failed`/70, while invalid or incomplete terminal data remains `malformed-output`/65. These provider failures preserve the exact reason in `error.message`, put it first in bounded evidence, and retain reported model/session/usage/cost and the actual child exit code, even after an ordinary nonzero child exit. Missing provider metadata stays null. Launcher cancellation and timeout take precedence. The `signal` field is non-null only when the runner sent that signal to a still-active direct CLI child, and remains null for provider-only cancellation or when launcher cancellation only stopped a post-exit pipe drain. The provider CLI owns any processes it starts beneath that direct child; the receipt does not claim a process-tree kill. Do not delete or overwrite the receipt. Never substitute the parent model, retry another provider, or reinterpret an external descriptor as a native model slug. Start native and external lanes in the same fan-out phase, then wait for all of them before judging. A judge must not read candidate paths while their owners are still writing. From ef6c5c439fe3cea47fc880d68803af5f9c0609cc Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 09:02:46 +0000 Subject: [PATCH 5/5] fix: preserve nonzero Grok dropout classification (#78) Use typed terminal failures only; otherwise retain the child exit failure path rather than treating startup output as malformed. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .../poteto-mode/scripts/runner/run.test.ts | 29 ++++++++++++++++--- .../skills/poteto-mode/scripts/runner/run.ts | 4 +-- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts b/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts index ca96d442..7a4b0008 100644 --- a/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts +++ b/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts @@ -439,17 +439,38 @@ process.exit(${scenario.endsWith("nonzero") ? 1 : 0}); `process.stdout.write(${JSON.stringify(stdout)}); process.exit(${exitCode});`)); process.env.FAKE_GROK_ERROR_RESULT = "1"; const result = await runLane(input); - expect(result.exitCode).toBe(65); + expect(result.exitCode).toBe(exitCode === 0 ? 65 : 70); expect(result.receipt).toMatchObject({ - status: "malformed-output", exitCode, reportedModel: null, modelVerified: false, - sessionId: null, error: { message: "grok result did not contain a valid terminal status" }, + status: exitCode === 0 ? "malformed-output" : "child-failed", + exitCode, reportedModel: null, modelVerified: false, + sessionId: null, error: { message: exitCode === 0 + ? "grok result did not contain a valid terminal status" : "child exited with status 1" }, }); - expect(result.receipt.error?.evidence).toEndWith(stdout.trim()); expect(readFileSync(streamPath(result.receipt.stdoutPath), "utf8")).toEndWith(stdout); expect(existsSync(input.outputPath)).toBe(false); }); } + for (const stdout of ['{"type":"system","subtype":"init"}\n', "startup failed\n"]) { + for (const exitCode of [0, 1]) { + it(`handles missing Grok terminal output ${JSON.stringify(stdout)} after exit ${exitCode}`, async () => { + const input = options("grok"); + const stderr = "sandbox startup refused\n"; + writeFileSync(join(bin, "grok"), fake.replace("const modelIndex =", + `process.stdout.write(${JSON.stringify(stdout)}); process.stderr.write(${JSON.stringify(stderr)}); process.exit(${exitCode});\nconst modelIndex =`)); + const result = await runLane(input); + expect(result.exitCode).toBe(exitCode === 0 ? 65 : 70); + expect(result.receipt).toMatchObject({ + status: exitCode === 0 ? "malformed-output" : "child-failed", + exitCode, reportedModel: null, sessionId: null, + }); + expect(readFileSync(streamPath(result.receipt.stdoutPath), "utf8")).toBe(stdout); + expect(readFileSync(streamPath(result.receipt.stderrPath), "utf8")).toBe(stderr); + expect(existsSync(input.outputPath)).toBe(false); + }); + } + } + it("does not invent metadata for a valid provider failure", async () => { const input = options("grok"); writeFileSync(join(bin, "grok"), fake.replace( diff --git a/plugins/pstack/skills/poteto-mode/scripts/runner/run.ts b/plugins/pstack/skills/poteto-mode/scripts/runner/run.ts index c12a48f7..ce4ad937 100644 --- a/plugins/pstack/skills/poteto-mode/scripts/runner/run.ts +++ b/plugins/pstack/skills/poteto-mode/scripts/runner/run.ts @@ -814,9 +814,9 @@ async function executeLane( } } const providerFailure = parseError instanceof ProviderResultError ? parseError : null; - const metadata = providerFailure?.metadata ?? parsed; + const metadata = providerFailure?.metadata ?? (result.exitCode === 0 ? parsed : null); if (result.cancelledBy !== null || result.timedOut || providerFailure !== null - || (result.exitCode !== 0 && parseError === null)) { + || result.exitCode !== 0) { const rawFailureEvidence = `${result.stderr}\n${result.stdout}`; const failureEvidence = evidence(providerFailure === null ? rawFailureEvidence : `${providerFailure.message}\n${rawFailureEvidence}`);