From 28f55a9604616e04d86bf713e049f83f7a8af9af Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:33:18 +0000 Subject: [PATCH 01/20] feat(#90): define fail-closed project verification coverage Keep the live-verification model and independent tooling outside the shipped plugin; require separate evidence for each selected skill. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .claude/skills/verify-open-pstack/SKILL.md | 59 ++++++++++ .claude/skills/verify-open-pstack/bun.lock | 22 ++++ .../verify-open-pstack/features/README.md | 23 ++++ .../verify-open-pstack/features/registry.json | 49 ++++++++ .../verify-open-pstack/features/runner.md | 17 +++ .../verify-open-pstack/features/setup.md | 17 +++ .../features/shipped-tools.md | 17 +++ .../features/skill-invocation.md | 19 ++++ .../skills/verify-open-pstack/package.json | 7 ++ .../verify-open-pstack/scripts/core.test.ts | 107 ++++++++++++++++++ .../skills/verify-open-pstack/scripts/core.ts | 76 +++++++++++++ .../verify-open-pstack/scripts/types.ts | 46 ++++++++ .../skills/verify-open-pstack/tsconfig.json | 8 ++ 13 files changed, 467 insertions(+) create mode 100644 .claude/skills/verify-open-pstack/SKILL.md create mode 100644 .claude/skills/verify-open-pstack/bun.lock create mode 100644 .claude/skills/verify-open-pstack/features/README.md create mode 100644 .claude/skills/verify-open-pstack/features/registry.json create mode 100644 .claude/skills/verify-open-pstack/features/runner.md create mode 100644 .claude/skills/verify-open-pstack/features/setup.md create mode 100644 .claude/skills/verify-open-pstack/features/shipped-tools.md create mode 100644 .claude/skills/verify-open-pstack/features/skill-invocation.md create mode 100644 .claude/skills/verify-open-pstack/package.json create mode 100644 .claude/skills/verify-open-pstack/scripts/core.test.ts create mode 100644 .claude/skills/verify-open-pstack/scripts/core.ts create mode 100644 .claude/skills/verify-open-pstack/scripts/types.ts create mode 100644 .claude/skills/verify-open-pstack/tsconfig.json diff --git a/.claude/skills/verify-open-pstack/SKILL.md b/.claude/skills/verify-open-pstack/SKILL.md new file mode 100644 index 00000000..51252c85 --- /dev/null +++ b/.claude/skills/verify-open-pstack/SKILL.md @@ -0,0 +1,59 @@ +--- +name: verify-open-pstack +description: Verify an Open Pstack PR's exact head in isolated Claude Code and Codex sessions, retain operator-reviewed real-surface evidence, and publish the live gate. +--- + +# Verify Open Pstack + +## Launch + +This is a repository-local, non-shipped skill shared with Codex through `.agents/skills/verify-open-pstack`. Run from a trusted checkout on the operator's Mac. Never run a candidate's arbitrary scripts with GitHub credentials. Review this verifier before using it; it creates GitHub comments/statuses and can mark a draft ready. It never queues or merges. + +```sh +(cd .claude/skills/verify-open-pstack && bun install --frozen-lockfile) +PR=123 # supplied delivery PR number +EVIDENCE="$(mktemp -d "${TMPDIR:-/tmp}/open-pstack-evidence.XXXXXX")" +.claude/skills/verify-open-pstack/scripts/verify.sh doctor --output "$EVIDENCE" +.claude/skills/verify-open-pstack/scripts/verify.sh run --pr "$PR" --self-test --output "$EVIDENCE" +``` + +Omit `--self-test` for ordinary plugin verification; it is mandatory for this skill's own delivery. Use a new evidence directory for every run, even if the head is unchanged. A previous run cannot resume or transfer success. The helper pins the open, same-repository PR's 40-character head SHA, fetches that exact object, classifies old/new changed paths, and checks the head at every boundary and around publication. Unknown paths abort. + +## Doctor + +Requires Darwin, Bun, gh authentication, both harness CLIs, and operator access to every selected provider. `doctor` writes `doctor.json` outside the checkout. It checks the supported isolation flags but does not claim that installation or live behavior works. + +The approved October 2 Mac probe (issue #90, comment 5961645184) reported Darwin, Claude Code 2.1.283, codex-cli 0.160.0, Bun 1.4.0, gh 2.97.0. Selected interfaces: Claude `--plugin-dir`, `--settings`, `--setting-sources`, `CLAUDE_CONFIG_DIR`; Codex local `plugin marketplace add`/`plugin add`, `CODEX_HOME`. Each harness also receives a run-owned `HOME`. Installation/provenance checks validate those choices during the actual run. Missing interfaces or authentication fail closed: never use a daily install. + +## Drive + +Read `features/README.md` and each selected feature document. Runtime markdown and consumed references are runtime. Shared paths select all consumers conservatively. `no runtime change` launches no harness unless the separate `--self-test` was requested. + +The runner creates a detached exact-head candidate, separate homes and fixture workspaces, and executable launchers. Claude loads the candidate with its explicit plugin directory; Codex installs the candidate's local marketplace into its isolated home. The installed trees must match the candidate's content digest; a shared version string is insufficient. Fresh fixture sessions discover the canonical project skill and Codex link copied from this head. + +The operator authenticates only these isolated homes when needed, using the printed launcher with `auth login` (Claude) or `login` (Codex); do not copy daily auth/configuration. GitHub tokens and daily provider secrets are excluded from harness subprocess environments. A missing configured provider remains a feature failure. Never add a timeout or weaker-model fallback. + +For each selected feature in **each** harness, the runner opens a fresh native session with instructions from the feature map. Use a safe disposable fixture, exercise all changed sub-features, and observe actual tool calls and effects. Save and redact the native transcript and result artifacts outside the checkout. The runner asks for the surface, action, observed result, transcript path, and artifact paths, then requires explicit operator approval. Empty or missing evidence fails. A model's assertion that it passed does not count. Direct CLI proof is supplemental and must be labeled separately. + +For `--self-test`, invoke this discovered project skill in both sessions and run `doctor` into a child evidence directory; save invocation traces and that output. Do not recursively run verification or publish from those sessions. This self-test cannot replace plugin-runtime exercises. + +## Evidence + +Retain `doctor.json`, `receipt.json`, installed provenance, redacted transcripts, artifacts, and cleanup outcome in `$EVIDENCE`. Artifact content is hashed before publication. The helper rechecks installation digests and artifact hashes after exercises. The PR comment includes `Live evidence:`, head SHA, installed version/location/tree, surface/action/observed result, artifact hashes, and failure or `no runtime change`. + +Only complete evidence publishes `live-gate=success` on the pinned SHA, targeting the evidence-comment URL; only then may this run mark a draft ready. Failure records its reason and, where possible, publishes failure against that same SHA. A changed head aborts; detected post-write races withdraw old-head success and restore draft if this run made it ready. GitHub mutations are not atomic; failed compensation is recorded prominently. Never attach the old run to a new head. + +The builder opens this delivery as a draft and does **not** run publication. The Mac operator supplies live evidence. Workflow changes require explicit operator queue submission. If Mergify creates another head, rerun on it; never hand-post a replacement status or reuse an old receipt. Queue policy/recovery is outside this skill. + +## Cleanup + +The runner leaves all run-owned state and evidence intact; it does not delete daily state or kill unrelated processes. Quit spawned native sessions normally. After reviewing and archiving evidence, the operator may remove only this run's candidate/homes/workspaces. Record that action in the retained receipt's cleanup field; retain redacted transcripts and artifact hashes. No implicit cleanup or runtime timeout. + +## Helpers + +- `scripts/verify.sh doctor --output `: capability report, no installation/publication. +- `scripts/verify.sh run --pr [--self-test] --output `: supervised exact-head live verification and publication. Requires an interactive terminal for mapped exercises. +- `bun run test` and `bun run typecheck` in this skill directory: orchestration tests and strict types; not live proof. +- `features/registry.json`: maintained path ownership; unknown runtime paths block rather than guessing. + +Read receipts as data, never shell input. Do not put secrets in prompts/artifacts. All external commands use argument arrays; PR metadata is never evaluated as code. diff --git a/.claude/skills/verify-open-pstack/bun.lock b/.claude/skills/verify-open-pstack/bun.lock new file mode 100644 index 00000000..d1145521 --- /dev/null +++ b/.claude/skills/verify-open-pstack/bun.lock @@ -0,0 +1,22 @@ +{ + "lockfileVersion": 2, + "configVersion": 1, + "workspaces": { + "": { + "name": "@open-pstack/project-verification", + "devDependencies": { + "bun-types": "1.4.0", + "typescript": "5.9.3", + }, + }, + }, + "packages": { + "@types/node": ["@types/node@26.6.4", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-ldVPDCzj7fsaGZrLB0NuHuTvJcsNasysBAqMolr/cgxrLd1xbqxIr3XJiPnHHJUCxj5sNF1vnRj9aWnrVh5Jcg=="], + + "bun-types": ["bun-types@1.4.0", "", { "dependencies": { "@types/node": "*" } }, "sha512-iIKw23BspnQQYd3prITOBxeUsxBHnwzX6YJfGMuNOZzeNcMmVqzIIVGRm1l69ogaPQmb4wB6BN8mA5bE9YuC5Q=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@8.9.0", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="], + } +} diff --git a/.claude/skills/verify-open-pstack/features/README.md b/.claude/skills/verify-open-pstack/features/README.md new file mode 100644 index 00000000..4ce77077 --- /dev/null +++ b/.claude/skills/verify-open-pstack/features/README.md @@ -0,0 +1,23 @@ +# Maintained feature map + +`registry.json` is the machine-readable ownership map. `core.ts` supports exact paths and terminal `/**` only. Unknown paths fail closed; rename classification includes both paths. Runtime markdown, references, playbooks, hooks, manifests, and scripts are runtime, even when the extension is `.md`. + +| Ownership | Required real surfaces | +| --- | --- | +| `skills` | One `skill-invocation:` exercise per selected skill, in both harnesses | +| `shared` | All listed skills, setup, runner, shipped tools in both harnesses | +| `setup` | Installed setup/model configuration | +| `runner` | Installed parent invoking real configured provider lanes | +| `tools` | Installed parent invoking the changed shipped CLI | +| `nonRuntime` | Explicit repository documentation/CI/assets/tests; no plugin harness launch | + +Skill-owned `SKILL.md`, `references/**`, and `playbooks/**` select their skill. Additional executable paths must be registered explicitly. Shared consumed mappings select every dependent skill conservatively. The coverage test inventories all tracked plugin files; new unmapped files block CI and runs. Deleted skill surfaces cannot silently disappear from evidence: choose a surviving documented entry point demonstrating the intended removal, or report blocked. + +The four documents below define sub-features, user entry points, driving instructions, and gotchas. Review each changed sub-feature in the diff; one aggregate observation cannot replace distinct selected skills. Record all changed sub-feature actions in the feature's transcript and artifact. Do not run destructive application work in the maintainer checkout. + +- `skill-invocation.md`: installed native skills, principle leaves, and project-skill self-test. +- `setup.md`: setup/routing/model configuration. +- `runner.md`: parent-to-child provider dispatch and receipts. +- `shipped-tools.md`: orchestration, watch, plan checks, audit, and evidence logging. + +An ordinary repository-only change is `no runtime change`. `--self-test` adds a separate `project-skill` exercise in each harness without changing that classification. It is mandatory for this skill's delivery PR. Plugin-runtime proof and local CLI proof remain separate. diff --git a/.claude/skills/verify-open-pstack/features/registry.json b/.claude/skills/verify-open-pstack/features/registry.json new file mode 100644 index 00000000..25123e3d --- /dev/null +++ b/.claude/skills/verify-open-pstack/features/registry.json @@ -0,0 +1,49 @@ +{ + "skills": [ + "architect", "arena", "automate-me", "babysit", "blast-radius", "bro", + "create-verification-skill", "deslop", "figure-it-out", "fix-ci", "fix-merge-conflicts", + "get-pr-comments", "how", "interrogate", "maintain-verification-skill", + "make-pr-easy-to-review", "no-comments", "poteto-mode", + "principle-attack-the-premise", "principle-boundary-discipline", "principle-build-the-lever", + "principle-encode-lessons-in-structure", "principle-exhaust-the-design-space", + "principle-experience-first", "principle-fix-root-causes", "principle-foundational-thinking", + "principle-guard-the-context-window", "principle-laziness-protocol", + "principle-make-operations-idempotent", "principle-migrate-callers-then-delete-legacy-apis", + "principle-minimize-reader-load", "principle-model-the-domain", "principle-never-block-on-the-human", + "principle-outcome-oriented-execution", "principle-prove-it-works", + "principle-redesign-from-first-principles", "principle-separate-before-serializing-shared-state", + "principle-sequence-verifiable-units", "principle-subtract-before-you-add", + "principle-test-behavior-not-implementation", "principle-type-system-discipline", + "recall", "reflect", "setup-pstack", "show-me-your-work", "swarm", "tdd", "teach", + "technical-writing", "thermo-nuclear-code-quality-review", "typescript-best-practices", + "unslop", "what-did-i-get-done", "why" + ], + "shared": [ + "plugins/pstack/.claude-plugin/plugin.json", "plugins/pstack/.codex-plugin/plugin.json", + ".claude-plugin/marketplace.json", ".agents/plugins/marketplace.json", + "plugins/pstack/agents/**", "plugins/pstack/hooks/**", + "plugins/pstack/skills/poteto-mode/references/**" + ], + "setup": ["plugins/pstack/skills/setup-pstack/SKILL.md"], + "runner": [ + "plugins/pstack/skills/poteto-mode/scripts/runner/**", + "plugins/pstack/skills/poteto-mode/scripts/bootstrap.ts" + ], + "tools": [ + "plugins/pstack/skills/poteto-mode/scripts/orch/**", + "plugins/pstack/skills/poteto-mode/scripts/watch-pr/**", + "plugins/pstack/skills/poteto-mode/scripts/check-plan.mjs", + "plugins/pstack/skills/poteto-mode/scripts/worktree-audit.sh", + "plugins/pstack/skills/poteto-mode/scripts/package.json", + "plugins/pstack/skills/poteto-mode/scripts/bun.lock", + "plugins/pstack/skills/poteto-mode/scripts/check-plan.tsconfig.json", + "plugins/pstack/skills/show-me-your-work/scripts/log.sh" + ], + "nonRuntime": [ + "README.md", "AGENTS.md", "CLAUDE.md", "CHANGES.md", "LICENSE", "LICENSE.md", "UPSTREAM.md", + ".gitignore", ".mergify.yml", ".github/**", "docs/**", "assets/**", "tests/**", + "scripts/**", ".claude/skills/verify-open-pstack/**", ".agents/skills/verify-open-pstack", + "plugins/pstack/assets/**", "plugins/pstack/skills/poteto-mode/scripts/bootstrap.test.ts", + "plugins/pstack/skills/poteto-mode/scripts/check-plan.test.ts" + ] +} diff --git a/.claude/skills/verify-open-pstack/features/runner.md b/.claude/skills/verify-open-pstack/features/runner.md new file mode 100644 index 00000000..f2221c68 --- /dev/null +++ b/.claude/skills/verify-open-pstack/features/runner.md @@ -0,0 +1,17 @@ +# Runner and provider dispatch + +## Sub-features + +Strict argument validation, parent/provider routing, external child launch, prompt delivery, output and receipt persistence, and failure reporting; inspect the diff for which branches changed. + +## How to get to it (user POV) + +Invoke the installed parent workflow (`poteto-mode`, `arena`, or `swarm` as documented by the candidate) in the fresh isolated harness. Have that parent dispatch its documented external-provider lane. + +## Driving it with verify-open-pstack + +Read the candidate's `pstack-runner --help` and installed mapping through the native skill. Use a run-owned fixture and fresh output/receipt paths. Exercise all changed provider lanes without changing parent routing. Retain the native parent/child transcript, provider receipt, output, and concrete fixture effect. Confirm provider/model/effort/access mode match the requested route and output belongs to this invocation. Exercise changed invalid-input/failure branches as well. Keep direct runner tests in a separately labeled local-CLI artifact, never as the installed-harness result. + +## Gotchas + +A missing provider or unusable authentication is a failure, not a skip or weaker-model fallback. Do not add implicit timeouts. A model assertion that a child ran is insufficient; inspect the actual receipt and output. Never reuse output from another SHA or run. diff --git a/.claude/skills/verify-open-pstack/features/setup.md b/.claude/skills/verify-open-pstack/features/setup.md new file mode 100644 index 00000000..88c7db8c --- /dev/null +++ b/.claude/skills/verify-open-pstack/features/setup.md @@ -0,0 +1,17 @@ +# Setup and model configuration + +## Sub-features + +Installed setup invocation, harness-to-provider mapping, generated routing/model preferences, unavailable-provider handling, and changed configuration references. + +## How to get to it (user POV) + +Invoke `/pstack:setup-pstack` in the isolated Claude session, or `$setup-pstack` in the isolated Codex session. The installed instructions are the authority for the setup inputs. + +## Driving it with verify-open-pstack + +Use only the run-owned fixture workspace and isolated harness home. Exercise each changed setup branch with explicit requested model/provider inputs. Retain the native invocation transcript and generated configuration file (redacted), then invoke a consuming installed skill and observe it use the configured route. Check that all paths written remain in run-owned state. Record requested versus observed provider/model and changed failure branches. Capture separate local CLI output only as supplemental evidence. + +## Gotchas + +Do not copy a daily configuration to make setup pass. Stop if required provider authentication is unavailable. Generated configuration by itself does not prove the consuming surface. Credentials stay outside repository-controlled scripts; authenticate the isolated harness explicitly and redact secrets before selecting artifacts. diff --git a/.claude/skills/verify-open-pstack/features/shipped-tools.md b/.claude/skills/verify-open-pstack/features/shipped-tools.md new file mode 100644 index 00000000..c709f51b --- /dev/null +++ b/.claude/skills/verify-open-pstack/features/shipped-tools.md @@ -0,0 +1,17 @@ +# Shipped CLI tools from installed workflows + +## Sub-features + +`orch` state transitions, `watch-pr` observations, `check-plan` verdict boundaries, read-only worktree audit, and `show-me-your-work` logging. Exercise every tool and branch touched by selected paths; shared package changes select all tools. + +## How to get to it (user POV) + +Invoke the installed `poteto-mode` parent for orchestration/watch/plan/audit, or installed `show-me-your-work` for evidence logging, through each harness's native skill surface. + +## Driving it with verify-open-pstack + +Use the candidate's installed help and documented arguments. Create fixture state/output under the run root; use an explicitly named safe test PR where remote reads are required. Have the native installed workflow drive the changed tool, then inspect its real structured output and persisted state. For the audit assert no mutation; for logging check the written TSV and escaping; for orchestration check the stored transition; for watch/plan preserve the observed result and verdict. Retain native transcripts and these state artifacts. Label any direct helper invocation `local CLI` separately. Document every touched tool and changed branch in the operator-reviewed observation. + +## Gotchas + +A mock test is not live proof. Never run queue/merge actions or edit daily workflow state as a fixture. Do not infer a successful remote read from an empty response. Missing required access fails the feature; preserve its reason and draft status. diff --git a/.claude/skills/verify-open-pstack/features/skill-invocation.md b/.claude/skills/verify-open-pstack/features/skill-invocation.md new file mode 100644 index 00000000..dc7dbab5 --- /dev/null +++ b/.claude/skills/verify-open-pstack/features/skill-invocation.md @@ -0,0 +1,19 @@ +# Installed skill invocation + +## Sub-features + +Every registry skill is its own required feature. Include changed arguments, routing, consumed references, helper actions, and removal behavior. Principle leaves are model-invocable, not slash-menu entries. `project-skill` separately verifies this repository-local skill's discovery. + +## How to get to it (user POV) + +In the fresh isolated Claude session invoke `/pstack:`. In Codex use the installed skill selector or `$`. Invoke principle leaves through the model's native skill tool. For self-test invoke `/verify-open-pstack` in Claude and `$verify-open-pstack` in Codex. + +## Driving it with verify-open-pstack + +Launch the prepared harness with its recorded launcher. For each selected name, use a disposable fixture appropriate to its documented input, invoke the installed surface, and exercise every changed sub-feature. Observe native tool/skill loading and a concrete fixture effect, not the assistant saying it passed. Save the transcript and an artifact showing the actual result (diff, generated file, tool receipt, or UI capture). Include input, expected result, observed result, and each changed sub-feature in the operator review. + +For `project-skill`, invoke the discovered skill and ask it to run `verify.sh doctor` into a new child directory of this run's evidence root; forbid recursive `run`/publication. Preserve the native invocation transcript plus the resulting `doctor.json`, checking the executable path resolves to the pinned project skill. The Codex symlink and Claude canonical directory must both be discovered in fresh sessions. + +## Gotchas + +A generic prompt about a skill is not native invocation evidence. A list of discovered skills is not behavioral proof. If an invocation would merge, queue, release, or access daily state, use a safe disposable fixture or record failure; do not weaken the gate. Never use the child verifier to publish a status. diff --git a/.claude/skills/verify-open-pstack/package.json b/.claude/skills/verify-open-pstack/package.json new file mode 100644 index 00000000..5d275464 --- /dev/null +++ b/.claude/skills/verify-open-pstack/package.json @@ -0,0 +1,7 @@ +{ + "name": "@open-pstack/project-verification", + "private": true, + "type": "module", + "scripts": { "test": "bun test scripts", "typecheck": "tsc --noEmit --strict" }, + "devDependencies": { "bun-types": "1.4.0", "typescript": "5.9.3" } +} diff --git a/.claude/skills/verify-open-pstack/scripts/core.test.ts b/.claude/skills/verify-open-pstack/scripts/core.test.ts new file mode 100644 index 00000000..675a19c5 --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/core.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, test } from 'bun:test'; +import { execFileSync } from 'node:child_process'; +import { resolve } from 'node:path'; +import registryData from '../features/registry.json'; +import { classify, completeEvidence, matches, newReceipt, validateRegistry } from './core.ts'; +import { HARNESSES, type Receipt } from './types.ts'; + +export const SHA = 'a'.repeat(40), BASE = 'b'.repeat(40); +export const registry = validateRegistry(registryData); +export function receipt(selfTest = false): Receipt { + const r = newReceipt(90, SHA, BASE, selfTest, '/tmp/evidence'); + r.selection = classify([{ filename: 'README.md' }], registry); + return r; +} +export function evidence(r: Receipt): void { + for (const harness of HARNESSES) { + r.installations.push({ harness, sha: r.sha, cliVersion: 'test', pluginVersion: '1.5.0', + location: '/tmp/candidate/plugin', home: `/tmp/${harness}`, treeHash: 'c'.repeat(64) }); + for (const feature of [...r.selection.features, ...(r.selfTest ? ['project-skill'] : [])]) { + r.observations.push({ harness, feature, surface: 'native skill', action: 'invoke', observed: 'fixture changed', + transcript: '/tmp/log', transcriptHash: 'c'.repeat(64), reviewer: 'operator', + artifacts: [{ path: '/tmp/artifact', sha256: 'd'.repeat(64) }] }); + } + } +} + +describe('registry and ownership', () => { + test('all tracked plugin files are covered, including consumed markdown', () => { + const root = resolve(import.meta.dir, '../../../..'); + const paths = execFileSync('git', ['ls-files', '-z', 'plugins/pstack', '.claude-plugin', '.agents/plugins'], { cwd: root, encoding: 'utf8' }).split('\0').filter(Boolean); + expect(paths.length).toBeGreaterThan(100); + for (const filename of paths) expect(() => classify([{ filename }], registry)).not.toThrow(); + }); + test('validates external registry shape', () => { + for (const value of [null, {}, { ...registry, skills: ['../bad'] }, { ...registry, tools: ['x', 'x'] }, { ...registry, shared: [3] }]) { + expect(() => validateRegistry(value)).toThrow(); + } + }); + test('terminal prefix cannot match sibling directory', () => { + expect(matches('skills/x/file', 'skills/x/**')).toBe(true); + expect(matches('skills/xyz/file', 'skills/x/**')).toBe(false); + }); + test('markdown instructions are runtime', () => { + for (const filename of ['plugins/pstack/skills/architect/SKILL.md', 'plugins/pstack/skills/architect/references/guide.md']) { + const s = classify([{ filename }], registry); + expect(s.features).toContain('skill-invocation:architect'); + expect(s.noRuntime).toBe(false); + } + }); + test('shared references select every dependent skill and executable surface', () => { + const s = classify([{ filename: 'plugins/pstack/skills/poteto-mode/references/codex-tools.md' }], registry); + expect(s.skills).toEqual([...registry.skills].sort()); + for (const feature of ['setup', 'runner', 'shipped-tools']) expect(s.features).toContain(feature); + }); + test('rename and deletion preserve old runtime ownership', () => { + const s = classify([{ filename: 'docs/old.md', previous_filename: 'plugins/pstack/skills/architect/SKILL.md' }], registry); + expect(s.features).toContain('skill-invocation:architect'); + expect(classify([{ filename: 'plugins/pstack/skills/architect/SKILL.md' }], registry).noRuntime).toBe(false); + }); + test('unmapped paths and new executable leaves fail closed', () => { + for (const filename of ['plugins/pstack/new.ts', 'plugins/pstack/skills/architect/scripts/new.ts', '/absolute', '../escape', 'plugins/pstack/skills/new/SKILL.md']) { + expect(() => classify([{ filename }], registry)).toThrow(); + } + }); + test('docs and CI only select no harnesses', () => { + const s = classify([{ filename: 'README.md' }, { filename: '.github/workflows/ci.yml' }], registry); + expect(s.noRuntime).toBe(true); + expect(s.features).toEqual([]); + }); +}); + +describe('receipt and evidence boundaries', () => { + test('validates exact immutable SHAs and PR', () => { + for (const pr of [0, -1, 1.1, NaN]) expect(() => newReceipt(pr, SHA, BASE, false, '/tmp')).toThrow(); + for (const sha of ['main', 'a'.repeat(39), 'A'.repeat(40)]) expect(() => newReceipt(90, sha, BASE, false, '/tmp')).toThrow(); + }); + test('ordinary no-runtime has no install requirement', () => expect(() => completeEvidence(receipt())).not.toThrow()); + test('self-test is separate and required in both harnesses', () => { + const r = receipt(true); + expect(() => completeEvidence(r)).toThrow('Missing installation'); + evidence(r); + expect(() => completeEvidence(r)).not.toThrow(); + r.observations.pop(); + expect(() => completeEvidence(r)).toThrow('Missing/duplicate'); + }); + test('each selected skill must have its own observation', () => { + const r = receipt(); + r.selection = classify(registry.skills.slice(0, 2).map(s => ({ filename: `plugins/pstack/skills/${s}/SKILL.md` })), registry); + evidence(r); + expect(() => completeEvidence(r)).not.toThrow(); + r.observations.pop(); + expect(() => completeEvidence(r)).toThrow(); + }); + test('wrong SHA, duplicate and incomplete evidence cannot pass', () => { + for (const damage of [ + (r: Receipt) => { r.installations[0]!.sha = BASE; }, + (r: Receipt) => { r.observations.push(r.observations[0]!); }, + (r: Receipt) => { r.observations[0]!.transcriptHash = ''; }, + (r: Receipt) => { r.observations[0]!.observed = ''; }, + (r: Receipt) => { r.observations[0]!.artifacts = []; }, + (r: Receipt) => { r.failure = 'provider unavailable'; }, + ]) { + const r = receipt(true); evidence(r); damage(r); + expect(() => completeEvidence(r)).toThrow(); + } + }); +}); diff --git a/.claude/skills/verify-open-pstack/scripts/core.ts b/.claude/skills/verify-open-pstack/scripts/core.ts new file mode 100644 index 00000000..4bc3159a --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/core.ts @@ -0,0 +1,76 @@ +import { HARNESSES, REPO, type ChangedFile, type Receipt, type Registry, type Selection } from './types.ts'; + +export function validateRegistry(value: unknown): Registry { + if (!value || typeof value !== 'object') throw new Error('Invalid feature registry'); + const r = value as Registry; + for (const key of ['skills', 'shared', 'setup', 'runner', 'tools', 'nonRuntime'] as const) { + if (!Array.isArray(r[key]) || r[key].some(v => typeof v !== 'string' || !v || v.includes('..'))) { + throw new Error(`Invalid registry.${key}`); + } + if (new Set(r[key]).size !== r[key].length) throw new Error(`Duplicate registry.${key}`); + } + if (!r.skills.length || r.skills.some(v => !/^[a-z][a-z0-9-]+$/.test(v))) throw new Error('Invalid skills'); + return r; +} + +// Only a terminal /** is supported: unknown paths cannot match an accidental broad glob. +export function matches(path: string, pattern: string): boolean { + return pattern.endsWith('/**') ? path.startsWith(pattern.slice(0, -2)) : path === pattern; +} + +export function classify(files: ChangedFile[], registry: Registry): Selection { + const paths = [...new Set(files.flatMap(f => [f.filename, ...(f.previous_filename ? [f.previous_filename] : [])]))].sort(); + const skills = new Set(), features = new Set(); + for (const path of paths) { + if (!path || path.startsWith('/') || path.split('/').includes('..')) throw new Error(`Unsafe path: ${path}`); + let covered = false; + if (registry.shared.some(p => matches(path, p))) { + registry.skills.forEach(s => skills.add(s)); + ['setup', 'runner', 'shipped-tools'].forEach(f => features.add(f)); + covered = true; + } + for (const [key, feature] of [['setup', 'setup'], ['runner', 'runner'], ['tools', 'shipped-tools']] as const) { + if (registry[key].some(p => matches(path, p))) { features.add(feature); covered = true; } + } + const skill = /^plugins\/pstack\/skills\/([^/]+)\/(.+)$/.exec(path); + if (skill && registry.skills.includes(skill[1]!)) { + const leaf = skill[2]!; + if (leaf === 'SKILL.md' || leaf.startsWith('references/') || leaf.startsWith('playbooks/')) { + skills.add(skill[1]!); covered = true; + } + } + // Runtime ownership always takes precedence over documentation exemptions. + if (!covered && !registry.nonRuntime.some(p => matches(path, p))) throw new Error(`Unmapped path: ${path}`); + } + skills.forEach(s => features.add(`skill-invocation:${s}`)); + return { paths, skills: [...skills].sort(), features: [...features].sort(), noRuntime: !features.size }; +} + +export function newReceipt(pr: number, sha: string, base: string, selfTest: boolean, artifactRoot: string): Receipt { + if (!Number.isSafeInteger(pr) || pr < 1 || !/^[a-f0-9]{40}$/.test(sha) || !/^[a-f0-9]{40}$/.test(base)) throw new Error('Invalid PR/SHA'); + return { schema: 1, repo: REPO, pr, sha, base, selfTest, artifactRoot, phase: 'resolve', + selection: { paths: [], skills: [], features: [], noRuntime: false }, installations: [], observations: [], + started: new Date().toISOString(), cleanup: 'Run state retained; operator owns cleanup.' }; +} + +export function requiredFeatures(receipt: Receipt): string[] { + return [...receipt.selection.features, ...(receipt.selfTest ? ['project-skill'] : [])]; +} + +export function completeEvidence(receipt: Receipt): void { + const features = requiredFeatures(receipt); + if (!features.length && !receipt.selection.noRuntime) throw new Error('No classification'); + for (const harness of features.length ? HARNESSES : []) { + const installs = receipt.installations.filter(i => i.harness === harness && i.sha === receipt.sha); + if (installs.length !== 1 || !installs[0]!.location || !installs[0]!.treeHash) throw new Error(`Missing installation: ${harness}`); + for (const feature of features) { + const records = receipt.observations.filter(o => o.harness === harness && o.feature === feature); + if (records.length !== 1) throw new Error(`Missing/duplicate evidence: ${harness}/${feature}`); + const o = records[0]!; + if (o.reviewer !== 'operator' || !o.surface.trim() || !o.action.trim() || !o.observed.trim() || + !o.transcript || !/^[a-f0-9]{64}$/.test(o.transcriptHash) || !o.artifacts.length || + o.artifacts.some(a => !a.path || !/^[a-f0-9]{64}$/.test(a.sha256))) throw new Error(`Incomplete evidence: ${harness}/${feature}`); + } + } + if (receipt.failure) throw new Error(receipt.failure); +} diff --git a/.claude/skills/verify-open-pstack/scripts/types.ts b/.claude/skills/verify-open-pstack/scripts/types.ts new file mode 100644 index 00000000..76b55077 --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/types.ts @@ -0,0 +1,46 @@ +export const REPO = 'ericlitman/open-pstack'; +export const HARNESSES = ['claude', 'codex'] as const; +export type Harness = typeof HARNESSES[number]; +export type Phase = 'resolve' | 'classify' | 'prepare' | 'exercise' | 'publish' | 'ready' | 'failed' | 'head-moved'; +export interface Pull { + number: number; head: { sha: string }; base: { sha: string }; draft: boolean; + state: string; headRepo: string; +} +export interface ChangedFile { filename: string; previous_filename?: string } +export interface Registry { + skills: string[]; + shared: string[]; + setup: string[]; + runner: string[]; + tools: string[]; + nonRuntime: string[]; +} +export interface Selection { paths: string[]; skills: string[]; features: string[]; noRuntime: boolean } +export interface Observation { + harness: Harness; feature: string; surface: string; action: string; observed: string; + transcript: string; transcriptHash: string; reviewer: 'operator'; + artifacts: { path: string; sha256: string }[]; +} +export interface Installation { + harness: Harness; cliVersion: string; pluginVersion: string; sha: string; + location: string; treeHash: string; home: string; +} +export interface Receipt { + schema: 1; repo: string; pr: number; sha: string; base: string; phase: Phase; + selection: Selection; installations: Installation[]; observations: Observation[]; + selfTest: boolean; artifactRoot: string; failure?: string; + commentUrl?: string; status?: 'success' | 'failure'; madeReady?: boolean; + compensation?: string[]; cleanup: string; started: string; +} +export interface GitHub { + pull(pr: number): Promise; + files(pr: number): Promise; + comment(pr: number, body: string): Promise; + status(sha: string, state: 'success' | 'failure', target: string, description: string): Promise; + ready(pr: number): Promise; + draft(pr: number): Promise; +} +export interface Driver { + prepare(receipt: Receipt): Promise; + exercise(receipt: Receipt): Promise; +} diff --git a/.claude/skills/verify-open-pstack/tsconfig.json b/.claude/skills/verify-open-pstack/tsconfig.json new file mode 100644 index 00000000..4aba2486 --- /dev/null +++ b/.claude/skills/verify-open-pstack/tsconfig.json @@ -0,0 +1,8 @@ +{ + "compilerOptions": { + "target": "ESNext", "module": "ESNext", "moduleResolution": "bundler", + "allowImportingTsExtensions": true, "noEmit": true, "strict": true, + "types": ["bun-types"], "skipLibCheck": true + }, + "include": ["scripts/**/*.ts"] +} From d049c794ceddcbb8ab5e3f036791f3fa754b8f8a Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:34:48 +0000 Subject: [PATCH 02/20] docs(#90): record approved Mac isolation probe contract Use the existing operator probe while requiring installed-tree confirmation before accepting live evidence. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .../skills/verify-open-pstack/features/isolation.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .claude/skills/verify-open-pstack/features/isolation.md diff --git a/.claude/skills/verify-open-pstack/features/isolation.md b/.claude/skills/verify-open-pstack/features/isolation.md new file mode 100644 index 00000000..29f81531 --- /dev/null +++ b/.claude/skills/verify-open-pstack/features/isolation.md @@ -0,0 +1,12 @@ +# Isolation contract and approved operator probe + +The operator's read-only Mac probe is retained in issue #90 comment 5961645184; plan approval is comment 5961649164. On 2026-10-02 it reported Darwin, Claude Code 2.1.283, codex-cli 0.160.0, Bun 1.4.0, and gh 2.97.0. This is capability evidence, not proof of an installed candidate. + +Selected interfaces: +- Claude: a fresh `HOME` and `CLAUDE_CONFIG_DIR`, candidate `--plugin-dir`, explicit empty `--settings`, and empty `--setting-sources`. No daily plugin/configuration state is copied. +- Codex: a fresh `HOME` and `CODEX_HOME`; add the pinned checkout as a local marketplace with `codex plugin marketplace add --json`, then `codex plugin add pstack@open-pstack --json`. Start a new interactive task after installation. Do not use the daily profile. +- Both: a fresh candidate checkout, run-owned state, minimal inherited environment, interactive login inside isolated state or operator-provided provider credentials, and fresh native sessions. The publisher's GitHub credentials are never passed to the candidate or either harness. + +Phase 3 checks these CLI flags at runtime and verifies the installed plugin file tree against the pinned checkout. If either interface is absent or its installed tree cannot be established, stop. A help probe cannot substitute for actual installation/discovery. The build sandbox cannot supply the required Mac surface proof; delivery remains a draft for the operator's doctor and self-test. + +`doctor` records versions/help without installing or reading daily credentials. `run` uses the approved interfaces only after a passing doctor and an exact-head pin. Evidence remains outside the checkout. Cleanup is explicit operator removal of the named run-owned state after reviewing the receipt; never clean up a daily home. From fda23782d6211cdc33dc39850e1b5cdb442c87a2 Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:43:04 +0000 Subject: [PATCH 03/20] feat(#90): require isolated native harness evidence Keep publisher credentials out of candidate subprocesses and verify the Codex installed tree before accepting operator-reviewed surface evidence. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .../verify-open-pstack/features/isolation.md | 2 + .../verify-open-pstack/scripts/doctor.ts | 29 +++++ .../scripts/harness.test.ts | 70 +++++++++++ .../verify-open-pstack/scripts/harness.ts | 111 ++++++++++++++++++ .../skills/verify-open-pstack/scripts/io.ts | 64 ++++++++++ 5 files changed, 276 insertions(+) create mode 100644 .claude/skills/verify-open-pstack/scripts/doctor.ts create mode 100644 .claude/skills/verify-open-pstack/scripts/harness.test.ts create mode 100644 .claude/skills/verify-open-pstack/scripts/harness.ts create mode 100644 .claude/skills/verify-open-pstack/scripts/io.ts diff --git a/.claude/skills/verify-open-pstack/features/isolation.md b/.claude/skills/verify-open-pstack/features/isolation.md index 29f81531..be2a8e8b 100644 --- a/.claude/skills/verify-open-pstack/features/isolation.md +++ b/.claude/skills/verify-open-pstack/features/isolation.md @@ -7,6 +7,8 @@ Selected interfaces: - Codex: a fresh `HOME` and `CODEX_HOME`; add the pinned checkout as a local marketplace with `codex plugin marketplace add --json`, then `codex plugin add pstack@open-pstack --json`. Start a new interactive task after installation. Do not use the daily profile. - Both: a fresh candidate checkout, run-owned state, minimal inherited environment, interactive login inside isolated state or operator-provided provider credentials, and fresh native sessions. The publisher's GitHub credentials are never passed to the candidate or either harness. +The Codex JSON contract was checked against OpenAI's `codex-rs/cli/src/plugin_cmd.rs` at tag `rust-v0.160.0`: `JsonPluginAddOutput` uses camelCase `name`, `marketplaceName`, and `installedPath`; `JsonPluginListOutput.installed` contains `installed` and `enabled`. Unknown schemas fail closed rather than searching arbitrary caches for a plausible tree. + Phase 3 checks these CLI flags at runtime and verifies the installed plugin file tree against the pinned checkout. If either interface is absent or its installed tree cannot be established, stop. A help probe cannot substitute for actual installation/discovery. The build sandbox cannot supply the required Mac surface proof; delivery remains a draft for the operator's doctor and self-test. `doctor` records versions/help without installing or reading daily credentials. `run` uses the approved interfaces only after a passing doctor and an exact-head pin. Evidence remains outside the checkout. Cleanup is explicit operator removal of the named run-owned state after reviewing the receipt; never clean up a daily home. diff --git a/.claude/skills/verify-open-pstack/scripts/doctor.ts b/.claude/skills/verify-open-pstack/scripts/doctor.ts new file mode 100644 index 00000000..f2e1eacb --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/doctor.ts @@ -0,0 +1,29 @@ +import { mkdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { command, isolatedEnv, save, type Command } from './io.ts'; + +export async function doctor(root: string, run: Command = command, platform = process.platform): Promise { + const report: Record = { platform, date: new Date().toISOString(), skill: import.meta.dir + '/..', checks: {} }; + const checks = report.checks as Record; + try { + if (platform !== 'darwin') throw new Error('Live gate requires an operator Mac (Darwin)'); + const home = join(root, 'probe-home'); + await mkdir(join(home, 'tmp'), { recursive: true, mode: 0o700 }); + const env = isolatedEnv(home); + for (const binary of ['bun', 'git', 'claude', 'codex', 'gh']) checks[binary] = (await run([binary, '--version'], { env })).trim(); + const claude = await run(['claude', '--help'], { env }); + for (const flag of ['--plugin-dir', '--settings', '--setting-sources']) if (!claude.includes(flag)) throw new Error(`Claude isolation missing ${flag}`); + checks.claudeHelp = claude; + for (const args of [['codex', 'plugin', 'marketplace', 'add', '--help'], ['codex', 'plugin', 'add', '--help']]) { + const help = await run(args, { env }); + if (!help.includes('--json')) throw new Error('Codex local plugin installation interface unavailable'); + if (args.includes('marketplace') && !/local path|local or Git|local marketplace/i.test(help)) throw new Error('Codex local marketplace interface unavailable'); + checks[args.join(' ')] = help; + } + // PR resolution checks publisher authentication; a child surface doctor has no GitHub credentials. + report.result = 'pass'; + } catch (error) { + report.result = 'blocked'; report.reason = String(error); + throw error; + } finally { await save(join(root, 'doctor.json'), report); } +} diff --git a/.claude/skills/verify-open-pstack/scripts/harness.test.ts b/.claude/skills/verify-open-pstack/scripts/harness.test.ts new file mode 100644 index 00000000..08d6b894 --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/harness.test.ts @@ -0,0 +1,70 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { doctor } from './doctor.ts'; +import { codexInstallation, launch, verifyCodexEnabled } from './harness.ts'; +import { freshRoot, isolatedEnv, redact, retainedFile, treeHash, type Command } from './io.ts'; +const roots: string[] = []; +async function fixture(): Promise { const root = await mkdtemp(join(tmpdir(), 'pstack-test-')); roots.push(root); return root; } +afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); + +describe('isolated harness boundaries', () => { + test('candidate environment never carries publisher credentials or daily config', () => { + process.env.GH_TOKEN = 'test-publisher-token'; process.env.CODEX_HOME = '/daily/codex'; + process.env.CLAUDE_CONFIG_DIR = '/daily/claude'; + try { + for (const harness of ['claude', 'codex'] as const) { + const env = isolatedEnv('/run/home', harness); + expect(env.GH_TOKEN).toBeUndefined(); expect(env.GITHUB_TOKEN).toBeUndefined(); + expect(env.HOME).toBe('/run/home'); expect(env.GIT_CONFIG_GLOBAL).toBe('/dev/null'); + expect(JSON.stringify(env)).not.toContain('/daily/'); + } + expect(launch('claude', '/run/home', '/candidate')).toEqual(['claude', '--plugin-dir', '/candidate/plugins/pstack', '--settings', '/run/home/settings.json', '--setting-sources', '']); + expect(launch('codex', '/run/home', '/candidate')).toEqual(['codex']); + } finally { delete process.env.GH_TOKEN; delete process.env.CODEX_HOME; delete process.env.CLAUDE_CONFIG_DIR; } + }); + test('doctor blocks non-Mac and missing isolation interfaces, retaining reasons', async () => { + const root = await fixture(); + const run: Command = async args => args.includes('--version') ? 'version' : ''; + await expect(doctor(root, run, 'linux')).rejects.toThrow('operator Mac'); + expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('blocked'); + await expect(doctor(root, run, 'darwin')).rejects.toThrow('isolation missing'); + }); + test('doctor probes without installing or reading daily authentication', async () => { + const calls: string[][] = [], root = await fixture(); + const run: Command = async args => { calls.push(args); return args.includes('--version') ? 'version' : '--plugin-dir --settings --setting-sources --json local path'; }; + await doctor(root, run, 'darwin'); + expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('pass'); + expect(calls.every(c => c.includes('--help') || c.includes('--version'))).toBe(true); + }); + test('Codex exact installed tree and enabled listing are required', async () => { + const root = await fixture(), plugin = join(root, 'config/plugins/pstack'); + await mkdir(plugin, { recursive: true }); await writeFile(join(plugin, 'SKILL.md'), 'candidate'); + const hash = await treeHash(plugin), receipt = JSON.stringify({ name: 'pstack', marketplaceName: 'open-pstack', installedPath: plugin }); + expect(await codexInstallation(receipt, root, hash)).toBe(plugin); + await expect(codexInstallation(receipt, root, 'bad')).rejects.toThrow('differs'); + await expect(codexInstallation('{}', root, hash)).rejects.toThrow('Unrecognized'); + expect(() => verifyCodexEnabled(JSON.stringify({ installed: [{ name: 'pstack', marketplaceName: 'open-pstack', installed: true, enabled: true }] }))).not.toThrow(); + expect(() => verifyCodexEnabled(JSON.stringify({ installed: [{ name: 'pstack', marketplaceName: 'open-pstack', installed: true, enabled: false }] }))).toThrow('enabled'); + const outside = await fixture(); await writeFile(join(outside, 'SKILL.md'), 'candidate'); + await expect(codexInstallation(JSON.stringify({ name: 'pstack', marketplaceName: 'open-pstack', installedPath: outside }), root, hash)).rejects.toThrow('escaped'); + }); + test('plugin symlinks and evidence outside retained root are rejected', async () => { + const root = await fixture(), outside = await fixture(); + await writeFile(join(outside, 'secret'), 'outside'); await symlink(join(outside, 'secret'), join(root, 'escape')); + await expect(treeHash(root)).rejects.toThrow('symlink'); + await expect(retainedFile(root, 'escape')).rejects.toThrow('within output'); + await writeFile(join(root, 'reviewed.txt'), 'native surface'); + expect((await retainedFile(root, 'reviewed.txt')).sha256).toMatch(/^[a-f0-9]{64}$/); + await mkdir(join(root, 'state')); await writeFile(join(root, 'state/raw'), 'raw'); + await expect(retainedFile(root, 'state/raw')).rejects.toThrow('outside isolated state'); + }); + test('output must be fresh and outside the repository', async () => { + const root = await fixture(), repo = join(root, 'repo'); await mkdir(repo); + await expect(freshRoot(join(repo, 'output'), repo)).rejects.toThrow('outside'); + await expect(freshRoot(root, repo)).rejects.toThrow(); + expect(await freshRoot(join(root, 'output'), repo)).toBe(join(root, 'output')); + expect(redact('ghp_testsecret sk-providersecret Bearer abc')).toBe('[REDACTED] [REDACTED] [REDACTED]'); + }); +}); diff --git a/.claude/skills/verify-open-pstack/scripts/harness.ts b/.claude/skills/verify-open-pstack/scripts/harness.ts new file mode 100644 index 00000000..3dff467c --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/harness.ts @@ -0,0 +1,111 @@ +import { mkdir, readFile, realpath, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { createInterface } from 'node:readline/promises'; +import { requiredFeatures } from './core.ts'; +import { command, isolatedEnv, redact, retainedFile, save, treeHash, type Command } from './io.ts'; +import { doctor } from './doctor.ts'; +import { HARNESSES, REPO, type Driver, type Harness, type Installation, type Observation, type Receipt } from './types.ts'; + +export type Ask = (question: string) => Promise; +export const ask: Ask = async question => { + if (!process.stdin.isTTY) throw new Error('Operator review requires an interactive terminal'); + const reader = createInterface({ input: process.stdin, output: process.stdout }); + try { return (await reader.question(question + '\n> ')).trim(); } finally { reader.close(); } +}; +export function launch(harness: Harness, home: string, workspace: string): string[] { + return harness === 'claude' + ? ['claude', '--plugin-dir', join(workspace, 'plugins/pstack'), '--settings', join(home, 'settings.json'), '--setting-sources', ''] + : ['codex']; +} +export async function codexInstallation(output: string, home: string, expected: string): Promise { + const result = JSON.parse(output); + if (result.name !== 'pstack' || result.marketplaceName !== 'open-pstack' || typeof result.installedPath !== 'string') { + throw new Error('Unrecognized Codex installation receipt; isolation cannot be established'); + } + const path = await realpath(result.installedPath); + if (!path.startsWith(await realpath(home) + '/')) throw new Error('Codex installation escaped isolated home'); + if (await treeHash(path) !== expected) throw new Error('Codex installed tree differs from pinned candidate'); + return path; +} +export function verifyCodexEnabled(output: string): void { + const result = JSON.parse(output); + const plugins = Array.isArray(result.installed) ? result.installed.filter((p: Record) => p.name === 'pstack' && p.marketplaceName === 'open-pstack') : []; + if (plugins.length !== 1 || plugins[0].installed !== true || plugins[0].enabled !== true) throw new Error('Isolated Codex plugin is not installed and enabled'); +} +export class MacDriver implements Driver { + constructor(private run: Command = command, private review: Ask = ask) {} + async prepare(receipt: Receipt): Promise { + const root = receipt.artifactRoot; + await doctor(root, this.run); + const installs: Installation[] = []; + for (const harness of HARNESSES) { + const home = join(root, 'state', harness), workspace = join(home, 'workspace'); + await mkdir(join(home, 'tmp'), { recursive: true, mode: 0o700 }); + const env = isolatedEnv(home, harness); + await this.run(['git', 'clone', '--no-checkout', '--', `https://github.com/${REPO}.git`, workspace], { env }); + await this.run(['git', 'fetch', 'origin', receipt.sha], { cwd: workspace, env }); + await this.run(['git', 'checkout', '--detach', receipt.sha], { cwd: workspace, env }); + if ((await this.run(['git', 'rev-parse', 'HEAD'], { cwd: workspace, env })).trim() !== receipt.sha) throw new Error('Candidate checkout SHA mismatch'); + const candidate = join(workspace, 'plugins/pstack'), expected = await treeHash(candidate); + let location = candidate; + if (harness === 'claude') await writeFile(join(home, 'settings.json'), '{}\n', { mode: 0o600 }); + else { + const added = await this.run(['codex', 'plugin', 'marketplace', 'add', workspace, '--json'], { env, cwd: home }); + await save(join(root, 'codex-marketplace.json'), JSON.parse(added)); + const installed = await this.run(['codex', 'plugin', 'add', 'pstack@open-pstack', '--json'], { env, cwd: home }); + await save(join(root, 'codex-install.json'), JSON.parse(installed)); + location = await codexInstallation(installed, home, expected); + const listed = await this.run(['codex', 'plugin', 'list', '--marketplace', 'open-pstack', '--json'], { env, cwd: home }); + verifyCodexEnabled(listed); + await save(join(root, 'codex-plugins.json'), JSON.parse(listed)); + } + const manifest = JSON.parse(await readFile(join(location, harness === 'claude' ? '.claude-plugin/plugin.json' : '.codex-plugin/plugin.json'), 'utf8')); + installs.push({ harness, sha: receipt.sha, home, location, treeHash: expected, pluginVersion: manifest.version, + cliVersion: (await this.run([harness, '--version'], { env })).trim() }); + } + return installs; + } + async exercise(receipt: Receipt): Promise { + const observations: Observation[] = []; + for (const installation of receipt.installations) { + const { harness, home } = installation, workspace = join(home, 'workspace'); + const env = isolatedEnv(home, harness, true); + const request = { sha: receipt.sha, harness, workspace, features: requiredFeatures(receipt), + featureMap: join(workspace, '.claude/skills/verify-open-pstack/features'), + selfTest: receipt.selfTest ? 'Invoke the project skill natively; run only doctor with fresh output outside workspace. Do not invoke run or publish recursively.' : false }; + await save(join(receipt.artifactRoot, `${harness}-request.json`), request); + console.log(JSON.stringify(request, null, 2)); + const login = await this.review(`Authenticate only in this isolated ${harness} home if needed. Launch an unrecorded login session now? Type LOGIN or SKIP (API key already supplied).`); + if (login === 'LOGIN') await this.run(harness === 'claude' ? ['claude', 'auth', 'login'] : ['codex', 'login'], { cwd: workspace, env, interactive: true }); + else if (login !== 'SKIP') throw new Error('Expected LOGIN or SKIP'); + const raw = join(home, 'surface.raw'); + console.log('Exercise every requested feature from the native surface, using the maintained feature map. Exit when done.'); + await this.run(['script', '-q', raw, ...launch(harness, home, workspace)], { cwd: workspace, env, interactive: true }); + if (await treeHash(installation.location) !== installation.treeHash) throw new Error('Installed tree changed during exercise'); + const transcript = await retainedFile(receipt.artifactRoot, await this.review(`Copy/redact ${raw} into output (outside state), review it, and enter the reviewed transcript path:`)); + const text = await readFile(join(receipt.artifactRoot, transcript.path), 'utf8'); + if (redact(text) !== text) throw new Error('Transcript contains recognizable credentials; redact before accepting'); + for (const feature of requiredFeatures(receipt)) { + const surface = await this.review(`${harness}/${feature}: native surface/discovery entry point?`); + const action = await this.review('Concrete action exercised?'); + const observed = await this.review('Observed assertion/result (not the model\'s success claim)?'); + const artifacts = []; + for (const path of (await this.review('Reviewed artifact paths inside output, one or more separated by commas?')).split(',')) { + artifacts.push(await retainedFile(receipt.artifactRoot, path.trim())); + } + if (feature === 'project-skill') { + const doctors = await Promise.all(artifacts.map(a => readFile(join(receipt.artifactRoot, a.path), 'utf8'))); + if (!doctors.some(text => { try { const d = JSON.parse(text); return d.result === 'pass' && d.skill === join(workspace, '.claude/skills/verify-open-pstack/scripts/..'); } catch { return false; } })) { + throw new Error('Self-test requires the pinned project skill\'s passing child doctor.json'); + } + } + if (await this.review(`Operator: type PASS ${feature} only after reviewing the native transcript and artifacts; anything else fails.`) !== `PASS ${feature}`) { + throw new Error(`Operator rejected ${harness}/${feature}`); + } + observations.push({ harness, feature, surface: redact(surface), action: redact(action), observed: redact(observed), + reviewer: 'operator', transcript: transcript.path, transcriptHash: transcript.sha256, artifacts }); + } + } + return observations; + } +} diff --git a/.claude/skills/verify-open-pstack/scripts/io.ts b/.claude/skills/verify-open-pstack/scripts/io.ts new file mode 100644 index 00000000..0de6027d --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/io.ts @@ -0,0 +1,64 @@ +import { createHash } from 'node:crypto'; +import { lstat, mkdir, readFile, readdir, realpath, writeFile } from 'node:fs/promises'; +import { isAbsolute, join, relative, resolve } from 'node:path'; + +export type Command = (args: string[], options?: { cwd?: string; env?: Record; interactive?: boolean }) => Promise; +export const command: Command = async (args, options = {}) => { + const child = Bun.spawn(args, { cwd: options.cwd, env: options.env, + stdin: options.interactive ? 'inherit' : 'ignore', stdout: options.interactive ? 'inherit' : 'pipe', + stderr: options.interactive ? 'inherit' : 'pipe' }); + const [output, error] = options.interactive ? ['', ''] : await Promise.all([ + new Response(child.stdout).text(), new Response(child.stderr).text(), + ]); + if (await child.exited !== 0) throw new Error(`${args[0]} failed: ${redact(error).slice(0, 1500)}`); + return output; +}; +export function redact(text: string): string { + return text.replace(/(?:gh[pousr]_[\w]+|github_pat_[\w]+|sk-[\w-]+|Bearer\s+\S+)/gi, '[REDACTED]'); +} +export function isolatedEnv(home: string, harness?: 'claude' | 'codex', credentials = false): Record { + const env: Record = { PATH: (process.env.PATH ?? '').split(':').filter(isAbsolute).join(':'), + HOME: home, TMPDIR: join(home, 'tmp'), TERM: process.env.TERM ?? 'xterm-256color', + GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_SYSTEM: '/dev/null' }; + if (harness === 'claude') env.CLAUDE_CONFIG_DIR = join(home, 'config'); + if (harness === 'codex') env.CODEX_HOME = join(home, 'config'); + if (credentials) for (const name of ['ANTHROPIC_API_KEY', 'OPENAI_API_KEY']) { + if (process.env[name]) env[name] = process.env[name]!; + } + return env; +} +export async function save(path: string, value: unknown): Promise { + await writeFile(path, JSON.stringify(value, null, 2) + '\n', { mode: 0o600 }); +} +export async function freshRoot(path: string, repository: string): Promise { + if (!isAbsolute(path)) throw new Error('Output must be an absolute fresh directory outside the repository'); + const parent = await realpath(resolve(path, '..')); + const root = join(parent, path.split('/').at(-1)!); + const repo = await realpath(repository); + if (root === repo || root.startsWith(repo + '/')) throw new Error('Output must be outside the repository'); + await mkdir(root, { mode: 0o700 }); // EEXIST deliberately rejects reuse and symlinks. + return root; +} +export function sha256(data: string | Uint8Array): string { return createHash('sha256').update(data).digest('hex'); } +export async function treeHash(root: string): Promise { + const entries: string[] = []; + async function walk(dir: string): Promise { + for (const name of (await readdir(dir)).sort()) { + const path = join(dir, name), stat = await lstat(path); + if (stat.isSymbolicLink()) throw new Error(`Plugin tree symlink refused: ${path}`); + if (stat.isDirectory()) await walk(path); + else if (stat.isFile()) entries.push(`${relative(root, path)}\0${sha256(await readFile(path))}`); + else throw new Error(`Non-file in plugin tree: ${path}`); + } + } + await walk(root); + if (!entries.length) throw new Error('Empty plugin tree'); + return sha256(entries.join('\n')); +} +export async function retainedFile(root: string, path: string): Promise<{ path: string; sha256: string }> { + const actual = await realpath(isAbsolute(path) ? path : join(root, path)); + if (!actual.startsWith(root + '/') || actual.includes('/state/')) throw new Error('Evidence must be retained outside isolated state, within output'); + const stat = await lstat(actual); + if (!stat.isFile() || !stat.size) throw new Error('Evidence file must be a nonempty regular file'); + return { path: relative(root, actual), sha256: sha256(await readFile(actual)) }; +} From fe28f0bbc97bb171f93c1ed47075037b85b6007c Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:54:49 +0000 Subject: [PATCH 04/20] feat(#90): publish evidence only for the verified head Require native project-skill proof and compensate detected publication races before allowing draft readiness. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .claude/skills/verify-open-pstack/SKILL.md | 5 +- .../verify-open-pstack/features/README.md | 3 +- .../verify-open-pstack/features/registry.json | 3 +- .../verify-open-pstack/scripts/cli.test.ts | 7 ++ .../skills/verify-open-pstack/scripts/cli.ts | 45 +++++++ .../verify-open-pstack/scripts/core.test.ts | 10 +- .../skills/verify-open-pstack/scripts/core.ts | 9 +- .../verify-open-pstack/scripts/github.ts | 38 ++++++ .../verify-open-pstack/scripts/harness.ts | 2 +- .../skills/verify-open-pstack/scripts/io.ts | 5 +- .../verify-open-pstack/scripts/types.ts | 1 + .../verify-open-pstack/scripts/verify.sh | 8 ++ .../verify-open-pstack/scripts/verify.test.ts | 113 ++++++++++++++++++ .../verify-open-pstack/scripts/verify.ts | 67 +++++++++++ 14 files changed, 304 insertions(+), 12 deletions(-) create mode 100644 .claude/skills/verify-open-pstack/scripts/cli.test.ts create mode 100644 .claude/skills/verify-open-pstack/scripts/cli.ts create mode 100644 .claude/skills/verify-open-pstack/scripts/github.ts create mode 100755 .claude/skills/verify-open-pstack/scripts/verify.sh create mode 100644 .claude/skills/verify-open-pstack/scripts/verify.test.ts create mode 100644 .claude/skills/verify-open-pstack/scripts/verify.ts diff --git a/.claude/skills/verify-open-pstack/SKILL.md b/.claude/skills/verify-open-pstack/SKILL.md index 51252c85..4c5453f1 100644 --- a/.claude/skills/verify-open-pstack/SKILL.md +++ b/.claude/skills/verify-open-pstack/SKILL.md @@ -12,8 +12,9 @@ This is a repository-local, non-shipped skill shared with Codex through `.agents ```sh (cd .claude/skills/verify-open-pstack && bun install --frozen-lockfile) PR=123 # supplied delivery PR number -EVIDENCE="$(mktemp -d "${TMPDIR:-/tmp}/open-pstack-evidence.XXXXXX")" -.claude/skills/verify-open-pstack/scripts/verify.sh doctor --output "$EVIDENCE" +SESSION="$(mktemp -d "${TMPDIR:-/tmp}/open-pstack-evidence.XXXXXX")" +EVIDENCE="$SESSION/live" # must not exist yet +.claude/skills/verify-open-pstack/scripts/verify.sh doctor --output "$SESSION/probe" .claude/skills/verify-open-pstack/scripts/verify.sh run --pr "$PR" --self-test --output "$EVIDENCE" ``` diff --git a/.claude/skills/verify-open-pstack/features/README.md b/.claude/skills/verify-open-pstack/features/README.md index 4ce77077..b9db7e1c 100644 --- a/.claude/skills/verify-open-pstack/features/README.md +++ b/.claude/skills/verify-open-pstack/features/README.md @@ -9,6 +9,7 @@ | `setup` | Installed setup/model configuration | | `runner` | Installed parent invoking real configured provider lanes | | `tools` | Installed parent invoking the changed shipped CLI | +| `project` | Native discovery/invocation of this verifier in both harnesses | | `nonRuntime` | Explicit repository documentation/CI/assets/tests; no plugin harness launch | Skill-owned `SKILL.md`, `references/**`, and `playbooks/**` select their skill. Additional executable paths must be registered explicitly. Shared consumed mappings select every dependent skill conservatively. The coverage test inventories all tracked plugin files; new unmapped files block CI and runs. Deleted skill surfaces cannot silently disappear from evidence: choose a surviving documented entry point demonstrating the intended removal, or report blocked. @@ -20,4 +21,4 @@ The four documents below define sub-features, user entry points, driving instruc - `runner.md`: parent-to-child provider dispatch and receipts. - `shipped-tools.md`: orchestration, watch, plan checks, audit, and evidence logging. -An ordinary repository-only change is `no runtime change`. `--self-test` adds a separate `project-skill` exercise in each harness without changing that classification. It is mandatory for this skill's delivery PR. Plugin-runtime proof and local CLI proof remain separate. +Explicit non-runtime repository changes are `no runtime change`. Changes to this verifier always select `project-skill`; they cannot bypass native proof. `--self-test` also selects that exercise, deduplicated in each harness, and is mandatory for this skill's delivery PR. Plugin-runtime proof and local CLI proof remain separate. diff --git a/.claude/skills/verify-open-pstack/features/registry.json b/.claude/skills/verify-open-pstack/features/registry.json index 25123e3d..d3cf2883 100644 --- a/.claude/skills/verify-open-pstack/features/registry.json +++ b/.claude/skills/verify-open-pstack/features/registry.json @@ -39,10 +39,11 @@ "plugins/pstack/skills/poteto-mode/scripts/check-plan.tsconfig.json", "plugins/pstack/skills/show-me-your-work/scripts/log.sh" ], + "project": [".claude/skills/verify-open-pstack/**", ".agents/skills/verify-open-pstack"], "nonRuntime": [ "README.md", "AGENTS.md", "CLAUDE.md", "CHANGES.md", "LICENSE", "LICENSE.md", "UPSTREAM.md", ".gitignore", ".mergify.yml", ".github/**", "docs/**", "assets/**", "tests/**", - "scripts/**", ".claude/skills/verify-open-pstack/**", ".agents/skills/verify-open-pstack", + "scripts/**", "plugins/pstack/assets/**", "plugins/pstack/skills/poteto-mode/scripts/bootstrap.test.ts", "plugins/pstack/skills/poteto-mode/scripts/check-plan.test.ts" ] diff --git a/.claude/skills/verify-open-pstack/scripts/cli.test.ts b/.claude/skills/verify-open-pstack/scripts/cli.test.ts new file mode 100644 index 00000000..617fdf98 --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/cli.test.ts @@ -0,0 +1,7 @@ +import { expect, test } from 'bun:test'; +import { parse } from './cli.ts'; +test('CLI strictly validates run inputs and permits read-only child doctor', () => { + expect(parse(['doctor', '--output', '/fresh/probe']).mode).toBe('doctor'); + expect(parse(['run', '--pr', '123', '--self-test', '--output', '/fresh/run'])).toEqual({ mode: 'run', output: '/fresh/run', pr: 123, selfTest: true }); + for (const args of [[], ['wat'], ['run', '--pr', '0', '--output', '/tmp/a'], ['run', '--pr', '1.5', '--output', '/tmp/a'], ['doctor', '--pr', '1', '--output', '/tmp/a'], ['doctor', '--output', '/tmp/a', '--output', '/tmp/b'], ['run', '--pr', '1'], ['doctor', '--output', '--self-test'], ['doctor', '--output', '/tmp/a', '--publish']]) expect(() => parse(args)).toThrow(); +}); diff --git a/.claude/skills/verify-open-pstack/scripts/cli.ts b/.claude/skills/verify-open-pstack/scripts/cli.ts new file mode 100644 index 00000000..3ca1615c --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/cli.ts @@ -0,0 +1,45 @@ +import { readFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import { validateRegistry } from './core.ts'; +import { doctor } from './doctor.ts'; +import { Publisher } from './github.ts'; +import { MacDriver } from './harness.ts'; +import { freshRoot, redact, save } from './io.ts'; +import { verify } from './verify.ts'; +export function parse(args: string[]): { mode: 'doctor' | 'run'; output: string; pr: number; selfTest: boolean } { + const mode = args[0]; + if (mode !== 'doctor' && mode !== 'run') throw new Error('Usage: verify.sh doctor --output /fresh/path | run --pr NUMBER [--self-test] --output /fresh/path'); + let output = '', pr = 0, selfTest = false; + const seen = new Set(); + for (let i = 1; i < args.length; i++) { + const arg = args[i]!; + if (seen.has(arg)) throw new Error(`Duplicate option: ${arg}`); seen.add(arg); + if (arg === '--self-test' && mode === 'run') selfTest = true; + else if (arg === '--output' || arg === '--pr' && mode === 'run') { + const value = args[++i]; if (!value || value.startsWith('--')) throw new Error(`Missing value: ${arg}`); + if (arg === '--output') output = value; + else { if (!/^[1-9]\d*$/.test(value) || !Number.isSafeInteger(Number(value))) throw new Error('PR must be a positive integer'); pr = Number(value); } + } else throw new Error(`Unknown option: ${arg}`); + } + if (!output || mode === 'run' && !pr) throw new Error('Output and (for run) PR are required'); + return { mode, output, pr, selfTest }; +} +async function main(): Promise { + let root: string | undefined; + try { + const options = parse(process.argv.slice(2)), repository = resolve(import.meta.dir, '../../../..'); + root = await freshRoot(options.output, repository); + if (options.mode === 'doctor') await doctor(root); + else { + const registry = validateRegistry(JSON.parse(await readFile(join(import.meta.dir, '../features/registry.json'), 'utf8'))); + const receipt = await verify({ pr: options.pr, selfTest: options.selfTest, root, registry, github: new Publisher(), driver: new MacDriver(), persist: r => save(join(root!, 'receipt.json'), r) }); + console.log(`Pinned ${receipt.sha}: live-gate=${receipt.status}; evidence ${receipt.commentUrl}`); + } + console.log(`Retained evidence: ${root}`); + } catch (error) { + const failure = redact(error instanceof Error ? error.message : String(error)); + if (root) await save(join(root, 'failure.json'), { result: 'failed', reason: failure }); + console.error(failure); process.exitCode = 1; + } +} +if (import.meta.main) await main(); diff --git a/.claude/skills/verify-open-pstack/scripts/core.test.ts b/.claude/skills/verify-open-pstack/scripts/core.test.ts index 675a19c5..88700b96 100644 --- a/.claude/skills/verify-open-pstack/scripts/core.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/core.test.ts @@ -16,7 +16,7 @@ export function evidence(r: Receipt): void { for (const harness of HARNESSES) { r.installations.push({ harness, sha: r.sha, cliVersion: 'test', pluginVersion: '1.5.0', location: '/tmp/candidate/plugin', home: `/tmp/${harness}`, treeHash: 'c'.repeat(64) }); - for (const feature of [...r.selection.features, ...(r.selfTest ? ['project-skill'] : [])]) { + for (const feature of new Set([...r.selection.features, ...(r.selfTest ? ['project-skill'] : [])])) { r.observations.push({ harness, feature, surface: 'native skill', action: 'invoke', observed: 'fixture changed', transcript: '/tmp/log', transcriptHash: 'c'.repeat(64), reviewer: 'operator', artifacts: [{ path: '/tmp/artifact', sha256: 'd'.repeat(64) }] }); @@ -62,6 +62,14 @@ describe('registry and ownership', () => { expect(() => classify([{ filename }], registry)).toThrow(); } }); + test('project-local verification changes cannot bypass their native self-test', () => { + const r = receipt(true); + r.selection = classify([{ filename: '.claude/skills/verify-open-pstack/scripts/verify.ts' }], registry); + expect(r.selection.noRuntime).toBe(false); + expect(r.selection.features).toEqual(['project-skill']); + evidence(r); + expect(() => completeEvidence(r)).not.toThrow(); + }); test('docs and CI only select no harnesses', () => { const s = classify([{ filename: 'README.md' }, { filename: '.github/workflows/ci.yml' }], registry); expect(s.noRuntime).toBe(true); diff --git a/.claude/skills/verify-open-pstack/scripts/core.ts b/.claude/skills/verify-open-pstack/scripts/core.ts index 4bc3159a..6c85dc3f 100644 --- a/.claude/skills/verify-open-pstack/scripts/core.ts +++ b/.claude/skills/verify-open-pstack/scripts/core.ts @@ -3,7 +3,7 @@ import { HARNESSES, REPO, type ChangedFile, type Receipt, type Registry, type Se export function validateRegistry(value: unknown): Registry { if (!value || typeof value !== 'object') throw new Error('Invalid feature registry'); const r = value as Registry; - for (const key of ['skills', 'shared', 'setup', 'runner', 'tools', 'nonRuntime'] as const) { + for (const key of ['skills', 'shared', 'setup', 'runner', 'tools', 'project', 'nonRuntime'] as const) { if (!Array.isArray(r[key]) || r[key].some(v => typeof v !== 'string' || !v || v.includes('..'))) { throw new Error(`Invalid registry.${key}`); } @@ -29,7 +29,7 @@ export function classify(files: ChangedFile[], registry: Registry): Selection { ['setup', 'runner', 'shipped-tools'].forEach(f => features.add(f)); covered = true; } - for (const [key, feature] of [['setup', 'setup'], ['runner', 'runner'], ['tools', 'shipped-tools']] as const) { + for (const [key, feature] of [['setup', 'setup'], ['runner', 'runner'], ['tools', 'shipped-tools'], ['project', 'project-skill']] as const) { if (registry[key].some(p => matches(path, p))) { features.add(feature); covered = true; } } const skill = /^plugins\/pstack\/skills\/([^/]+)\/(.+)$/.exec(path); @@ -54,7 +54,7 @@ export function newReceipt(pr: number, sha: string, base: string, selfTest: bool } export function requiredFeatures(receipt: Receipt): string[] { - return [...receipt.selection.features, ...(receipt.selfTest ? ['project-skill'] : [])]; + return [...new Set([...receipt.selection.features, ...(receipt.selfTest ? ['project-skill'] : [])])]; } export function completeEvidence(receipt: Receipt): void { @@ -62,7 +62,7 @@ export function completeEvidence(receipt: Receipt): void { if (!features.length && !receipt.selection.noRuntime) throw new Error('No classification'); for (const harness of features.length ? HARNESSES : []) { const installs = receipt.installations.filter(i => i.harness === harness && i.sha === receipt.sha); - if (installs.length !== 1 || !installs[0]!.location || !installs[0]!.treeHash) throw new Error(`Missing installation: ${harness}`); + if (installs.length !== 1 || !installs[0]!.location || !/^[a-f0-9]{64}$/.test(installs[0]!.treeHash) || !installs[0]!.cliVersion || !installs[0]!.pluginVersion || !installs[0]!.home) throw new Error(`Missing installation: ${harness}`); for (const feature of features) { const records = receipt.observations.filter(o => o.harness === harness && o.feature === feature); if (records.length !== 1) throw new Error(`Missing/duplicate evidence: ${harness}/${feature}`); @@ -72,5 +72,6 @@ export function completeEvidence(receipt: Receipt): void { o.artifacts.some(a => !a.path || !/^[a-f0-9]{64}$/.test(a.sha256))) throw new Error(`Incomplete evidence: ${harness}/${feature}`); } } + if (receipt.installations.length !== (features.length ? HARNESSES.length : 0) || receipt.observations.length !== features.length * HARNESSES.length) throw new Error('Extra installation or evidence'); if (receipt.failure) throw new Error(receipt.failure); } diff --git a/.claude/skills/verify-open-pstack/scripts/github.ts b/.claude/skills/verify-open-pstack/scripts/github.ts new file mode 100644 index 00000000..8c57389c --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/github.ts @@ -0,0 +1,38 @@ +import { command, type Command } from './io.ts'; +import { REPO, type ChangedFile, type GitHub, type Pull } from './types.ts'; + +export class Publisher implements GitHub { + constructor(private run: Command = command) {} + private async api(path: string, args: string[] = []): Promise { + return JSON.parse(await this.run(['gh', 'api', `repos/${REPO}/${path}`, ...args])); + } + async pull(pr: number): Promise { + const data = await this.api(`pulls/${pr}`) as { number: number; head: { sha: string; repo?: { full_name: string } }; base: { sha: string }; draft: boolean; state: string }; + if (data.number !== pr || typeof data.draft !== 'boolean' || typeof data.state !== 'string') throw new Error('Invalid PR response'); + return { number: pr, head: { sha: data.head.sha }, base: { sha: data.base.sha }, draft: data.draft, state: data.state, headRepo: data.head.repo?.full_name ?? '' }; + } + async files(pr: number): Promise { + const metadata = await this.api(`pulls/${pr}`) as { changed_files: number }; + if (!Number.isInteger(metadata.changed_files) || metadata.changed_files > 3000) throw new Error('PR file count cannot be completely enumerated'); + const pages = await this.api(`pulls/${pr}/files?per_page=100`, ['--paginate', '--slurp']); + if (!Array.isArray(pages) || !pages.every(Array.isArray)) throw new Error('Invalid file pagination'); + const files: ChangedFile[] = pages.flat().map((file: { filename: string; previous_filename?: string }) => { + if (typeof file.filename !== 'string' || (file.previous_filename !== undefined && typeof file.previous_filename !== 'string')) throw new Error('Invalid changed file'); + return { filename: file.filename, ...(file.previous_filename ? { previous_filename: file.previous_filename } : {}) }; + }); + if (files.length !== metadata.changed_files || new Set(files.map(f => f.filename)).size !== files.length) throw new Error('Incomplete or duplicate changed-file response'); + return files; + } + async comment(pr: number, body: string): Promise { + if (body.length > 60000) throw new Error('Evidence exceeds comment budget'); + const result = await this.api(`issues/${pr}/comments`, ['--method', 'POST', '-f', `body=${body}`]) as { html_url: string }; + if (typeof result.html_url !== 'string' || !result.html_url.startsWith(`https://github.com/${REPO}/issues/`) && !result.html_url.startsWith(`https://github.com/${REPO}/pull/`)) throw new Error('Missing same-repository evidence comment URL'); + return result.html_url; + } + async status(sha: string, state: 'success' | 'failure', target: string, description: string): Promise { + if (!/^[a-f0-9]{40}$/.test(sha)) throw new Error('Invalid status SHA'); + await this.api(`statuses/${sha}`, ['--method', 'POST', '-f', 'context=live-gate', '-f', `state=${state}`, '-f', `target_url=${target}`, '-f', `description=${description.slice(0, 140)}`]); + } + async ready(pr: number): Promise { await this.run(['gh', 'pr', 'ready', String(pr), '--repo', REPO]); } + async draft(pr: number): Promise { await this.run(['gh', 'pr', 'ready', String(pr), '--undo', '--repo', REPO]); } +} diff --git a/.claude/skills/verify-open-pstack/scripts/harness.ts b/.claude/skills/verify-open-pstack/scripts/harness.ts index 3dff467c..cbddb65f 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.ts @@ -72,7 +72,7 @@ export class MacDriver implements Driver { const env = isolatedEnv(home, harness, true); const request = { sha: receipt.sha, harness, workspace, features: requiredFeatures(receipt), featureMap: join(workspace, '.claude/skills/verify-open-pstack/features'), - selfTest: receipt.selfTest ? 'Invoke the project skill natively; run only doctor with fresh output outside workspace. Do not invoke run or publish recursively.' : false }; + selfTest: requiredFeatures(receipt).includes('project-skill') ? 'Invoke the project skill natively; run only doctor with fresh output outside workspace. Do not invoke run or publish recursively.' : false }; await save(join(receipt.artifactRoot, `${harness}-request.json`), request); console.log(JSON.stringify(request, null, 2)); const login = await this.review(`Authenticate only in this isolated ${harness} home if needed. Launch an unrecorded login session now? Type LOGIN or SKIP (API key already supplied).`); diff --git a/.claude/skills/verify-open-pstack/scripts/io.ts b/.claude/skills/verify-open-pstack/scripts/io.ts index 0de6027d..03418be5 100644 --- a/.claude/skills/verify-open-pstack/scripts/io.ts +++ b/.claude/skills/verify-open-pstack/scripts/io.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { lstat, mkdir, readFile, readdir, realpath, writeFile } from 'node:fs/promises'; +import { lstat, mkdir, readFile, readdir, realpath, rename, writeFile } from 'node:fs/promises'; import { isAbsolute, join, relative, resolve } from 'node:path'; export type Command = (args: string[], options?: { cwd?: string; env?: Record; interactive?: boolean }) => Promise; @@ -28,7 +28,8 @@ export function isolatedEnv(home: string, harness?: 'claude' | 'codex', credenti return env; } export async function save(path: string, value: unknown): Promise { - await writeFile(path, JSON.stringify(value, null, 2) + '\n', { mode: 0o600 }); + await writeFile(path + '.tmp', redact(JSON.stringify(value, null, 2)) + '\n', { mode: 0o600 }); + await rename(path + '.tmp', path); } export async function freshRoot(path: string, repository: string): Promise { if (!isAbsolute(path)) throw new Error('Output must be an absolute fresh directory outside the repository'); diff --git a/.claude/skills/verify-open-pstack/scripts/types.ts b/.claude/skills/verify-open-pstack/scripts/types.ts index 76b55077..c310016e 100644 --- a/.claude/skills/verify-open-pstack/scripts/types.ts +++ b/.claude/skills/verify-open-pstack/scripts/types.ts @@ -13,6 +13,7 @@ export interface Registry { setup: string[]; runner: string[]; tools: string[]; + project: string[]; nonRuntime: string[]; } export interface Selection { paths: string[]; skills: string[]; features: string[]; noRuntime: boolean } diff --git a/.claude/skills/verify-open-pstack/scripts/verify.sh b/.claude/skills/verify-open-pstack/scripts/verify.sh new file mode 100755 index 00000000..793098cf --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/verify.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +command -v bun >/dev/null || { printf '%s\n' 'Bun is required; no fallback runtime.' >&2; exit 1; } +if [[ ! -d "$HERE/node_modules/typescript" ]]; then + (cd "$HERE" && bun install --frozen-lockfile --ignore-scripts) +fi +exec bun "$HERE/scripts/cli.ts" "$@" diff --git a/.claude/skills/verify-open-pstack/scripts/verify.test.ts b/.claude/skills/verify-open-pstack/scripts/verify.test.ts new file mode 100644 index 00000000..cfab1ba4 --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/verify.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, test } from 'bun:test'; +import data from '../features/registry.json'; +import { requiredFeatures, validateRegistry } from './core.ts'; +import { Publisher } from './github.ts'; +import { evidence, verify } from './verify.ts'; +import { HARNESSES, REPO, type Driver, type GitHub, type Pull, type Receipt } from './types.ts'; +const SHA = 'a'.repeat(40), BASE = 'b'.repeat(40), NEXT = 'c'.repeat(40), HASH = 'd'.repeat(64); +const URL = `https://github.com/${REPO}/pull/123#issuecomment-1`; +function fixture(runtime = false, selfTest = false) { + const calls: string[] = [], saved: Receipt[] = []; + const pull: Pull = { number: 123, head: { sha: SHA }, base: { sha: BASE }, draft: true, state: 'open', headRepo: REPO }; + let hook: (call: string) => void = () => {}; + const mark = (call: string) => { calls.push(call); hook(call); }; + const github: GitHub = { + async pull() { mark('pull'); return structuredClone(pull); }, + async files() { mark('files'); return [{ filename: runtime ? 'plugins/pstack/skills/architect/SKILL.md' : 'README.md' }]; }, + async comment(_pr, body) { mark('comment'); expect(body).toContain(SHA); return URL; }, + async status(sha, state, target) { mark(`status:${state}:${sha}`); expect(target).toBe(URL); }, + async ready() { mark('ready'); pull.draft = false; }, async draft() { mark('draft'); pull.draft = true; }, + }; + const driver: Driver = { + async prepare(r) { mark('prepare'); return HARNESSES.map(harness => ({ harness, sha: r.sha, cliVersion: 'test', pluginVersion: '1.5.0', treeHash: HASH, location: `/isolated/${harness}/plugin`, home: `/isolated/${harness}` })); }, + async exercise(r) { mark('exercise'); return HARNESSES.flatMap(harness => requiredFeatures(r).map(feature => ({ harness, feature, surface: 'native surface', action: 'invoke', observed: 'fixture changed', reviewer: 'operator' as const, transcript: 'retained.log', transcriptHash: HASH, artifacts: [{ path: 'fixture.json', sha256: HASH }] }))); }, + }; + const options = { pr: 123, root: '/tmp/evidence', selfTest, registry: validateRegistry(data), github, driver, persist: async (r: Receipt) => { saved.push(structuredClone(r)); } }; + return { options, calls, saved, pull, hook: (fn: typeof hook) => { hook = fn; } }; +} + +describe('exact-head publication state machine', () => { + test('docs-only publishes no-runtime evidence without either harness, then readies', async () => { + const f = fixture(), receipt = await verify(f.options); + expect(receipt.status).toBe('success'); expect(receipt.sha).toBe(SHA); + expect(evidence(receipt)).toContain('no runtime change'); + expect(f.calls).not.toContain('prepare'); expect(f.calls).not.toContain('exercise'); + expect(f.calls.indexOf('comment')).toBeLessThan(f.calls.indexOf(`status:success:${SHA}`)); + expect(f.calls.indexOf(`status:success:${SHA}`)).toBeLessThan(f.calls.indexOf('ready')); + }); + test('all changed surfaces and explicit project self-test run in both harnesses', async () => { + const f = fixture(true, true), r = await verify(f.options); + expect(r.observations).toHaveLength(4); expect(f.calls).toContain('prepare'); expect(f.calls).toContain('exercise'); + expect(evidence(r)).toContain('operator reviewed'); expect(evidence(r)).toContain(HASH); + }); + test('already-ready PR is never toggled ready', async () => { + const f = fixture(); f.pull.draft = false; await verify(f.options); expect(f.calls).not.toContain('ready'); + }); + test('closed or foreign PR never enters publication', async () => { + for (const field of ['state', 'headRepo'] as const) { + const f = fixture(); f.pull[field] = 'other'; + await expect(verify(f.options)).rejects.toThrow('open same-repository'); expect(f.calls).toEqual(['pull']); + } + }); + test('head movement during classify, prepare, exercise, or comment aborts without success or readiness', async () => { + for (const boundary of ['files', 'prepare', 'exercise', 'comment']) { + const f = fixture(true); f.hook(call => { if (call === boundary) f.pull.head.sha = NEXT; }); + await expect(verify(f.options)).rejects.toThrow('changed'); + expect(f.calls.some(c => c.startsWith('status:success'))).toBe(false); expect(f.calls).not.toContain('ready'); + expect(f.saved.at(-1)?.phase).toBe('head-moved'); + } + }); + test('head or base movement after status withdraws success only on original SHA', async () => { + for (const field of ['head', 'base'] as const) { + const f = fixture(); f.hook(call => { if (call === `status:success:${SHA}`) f.pull[field].sha = NEXT; }); + await expect(verify(f.options)).rejects.toThrow('changed'); + expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.calls.join()).not.toContain(`status:failure:${NEXT}`); + expect(f.calls).not.toContain('ready'); expect(f.saved.at(-1)?.status).toBe('failure'); + } + }); + test('post-ready head race restores draft and revokes the old status', async () => { + const f = fixture(); f.hook(call => { if (call === 'ready') f.pull.head.sha = NEXT; }); + await expect(verify(f.options)).rejects.toThrow('changed'); + expect(f.calls).toContain('draft'); expect(f.pull.draft).toBe(true); expect(f.saved.at(-1)?.madeReady).toBe(false); + }); + test('incomplete exercise records failure against pinned head and never readies', async () => { + const f = fixture(true); f.options.driver.exercise = async () => []; + await expect(verify(f.options)).rejects.toThrow('Missing/duplicate'); + expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.calls).not.toContain('ready'); + expect(f.saved.at(-1)?.failure).toContain('Missing/duplicate'); + }); + test('ambiguous success-write and ready-write failures are compensated', async () => { + for (const boundary of [`status:success:${SHA}`, 'ready']) { + const f = fixture(); f.hook(call => { if (call === boundary) throw new Error('response lost'); }); + await expect(verify(f.options)).rejects.toThrow('response lost'); + expect(f.calls).toContain(`status:failure:${SHA}`); + if (boundary === 'ready') expect(f.calls).toContain('draft'); + } + }); + test('failed compensation is retained, not reported as successful', async () => { + const f = fixture(); f.hook(call => { + if (call === `status:success:${SHA}`) f.pull.head.sha = NEXT; + if (call === `status:failure:${SHA}`) throw new Error('publisher denied'); + }); + await expect(verify(f.options)).rejects.toThrow('changed'); + expect(f.saved.at(-1)?.compensation?.join()).toContain('FAILED'); + }); +}); + +describe('publisher request boundaries', () => { + test('pagination must match complete PR file count', async () => { + for (const result of [[], [[{ filename: 'README.md' }, { filename: 'README.md' }]], {}]) { + const publisher = new Publisher(async args => JSON.stringify(args.some(a => a.includes('/files?')) ? result : { changed_files: 2 })); + await expect(publisher.files(123)).rejects.toThrow(); + } + const publisher = new Publisher(async args => JSON.stringify(args.some(a => a.includes('/files?')) ? [[{ filename: 'docs/new.md', previous_filename: 'plugins/pstack/skills/architect/SKILL.md' }], [{ filename: 'README.md' }]] : { changed_files: 2 })); + expect(await publisher.files(123)).toHaveLength(2); + await expect(new Publisher(async () => JSON.stringify({ changed_files: 3001 })).files(123)).rejects.toThrow('completely enumerated'); + }); + test('status targets the supplied exact SHA and evidence URL, never a branch', async () => { + const calls: string[][] = []; const publisher = new Publisher(async args => { calls.push(args); return '{}'; }); + await publisher.status(SHA, 'success', URL, 'passed'); + expect(calls[0]).toContain(`repos/${REPO}/statuses/${SHA}`); expect(calls[0]).toContain('context=live-gate'); expect(calls[0]).toContain(`target_url=${URL}`); + await expect(publisher.status('main', 'success', URL, 'passed')).rejects.toThrow('Invalid status SHA'); + }); +}); diff --git a/.claude/skills/verify-open-pstack/scripts/verify.ts b/.claude/skills/verify-open-pstack/scripts/verify.ts new file mode 100644 index 00000000..9ed3b182 --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/verify.ts @@ -0,0 +1,67 @@ +import { classify, completeEvidence, newReceipt } from './core.ts'; +import { redact } from './io.ts'; +import { REPO, type Driver, type GitHub, type Receipt, type Registry } from './types.ts'; + +class HeadMoved extends Error {} +const safe = (text: string): string => redact(text).replace(/[`<>\r\n]/g, ' ').slice(0, 2000); +export function evidence(receipt: Receipt): string { + const lines = ['## Live evidence: Open Pstack', `Pinned PR #${receipt.pr}: \`${receipt.sha}\` (base \`${receipt.base}\`).`, + `Result: ${receipt.failure ? `FAILED — ${safe(receipt.failure)}` : receipt.selection.noRuntime ? 'no runtime change' : 'all mapped features passed'}.`, + `Self-test: ${receipt.selfTest ? 'required in both harnesses' : 'not requested'}.`, + `Retained receipt: \`${safe(receipt.artifactRoot)}/receipt.json\`. Local CLI checks are not installed-harness evidence.`]; + for (const install of receipt.installations) lines.push(`- ${install.harness}: CLI ${safe(install.cliVersion)}, plugin ${safe(install.pluginVersion)}, SHA \`${install.sha}\`, tree \`${install.treeHash}\`, isolated \`${safe(install.location)}\`.`); + for (const record of receipt.observations) lines.push(`- ${record.harness} / ${safe(record.feature)}: ${safe(record.surface)}; action: ${safe(record.action)}; observed: ${safe(record.observed)}; operator reviewed.`, + ` Transcript \`${safe(record.transcript)}\` SHA-256 \`${record.transcriptHash}\`; artifacts ${record.artifacts.map(a => `\`${safe(a.path)}\` (\`${a.sha256}\`)`).join(', ')}.`); + lines.push('Every new head requires a fresh run, including queue-created heads. This helper never queues or merges.'); + return lines.join('\n'); +} +export async function verify(options: { pr: number; selfTest: boolean; root: string; registry: Registry; github: GitHub; driver: Driver; persist: (receipt: Receipt) => Promise }): Promise { + const { github, driver, persist } = options; + const initial = await github.pull(options.pr); + if (initial.state !== 'open' || initial.headRepo !== REPO) throw new Error('Require an open same-repository PR'); + const receipt = newReceipt(options.pr, initial.head.sha, initial.base.sha, options.selfTest, options.root); + let successAttempted = false, readyAttempted = false; + const current = async (): Promise => { + const pull = await github.pull(receipt.pr); + if (pull.head.sha !== receipt.sha || pull.base.sha !== receipt.base || pull.state !== 'open' || pull.headRepo !== REPO) throw new HeadMoved('PR head/base or open repository identity changed; run again'); + }; + const phase = async (next: Receipt['phase']): Promise => { receipt.phase = next; await persist(receipt); await current(); }; + try { + await phase('classify'); + receipt.selection = classify(await github.files(receipt.pr), options.registry); + await current(); await persist(receipt); + if (!receipt.selection.noRuntime || receipt.selfTest) { + await phase('prepare'); receipt.installations = await driver.prepare(receipt); await current(); await persist(receipt); + await phase('exercise'); receipt.observations = await driver.exercise(receipt); await current(); await persist(receipt); + } + completeEvidence(receipt); + await phase('publish'); + receipt.commentUrl = await github.comment(receipt.pr, evidence(receipt)); await persist(receipt); await current(); + successAttempted = true; + await github.status(receipt.sha, 'success', receipt.commentUrl, receipt.selection.noRuntime ? 'no runtime change' : 'All mapped features passed in both harnesses'); + receipt.status = 'success'; await persist(receipt); await current(); + if (initial.draft) { + await phase('ready'); readyAttempted = true; await github.ready(receipt.pr); receipt.madeReady = true; await persist(receipt); await current(); + } + await persist(receipt); return receipt; + } catch (error) { + receipt.failure = redact(error instanceof Error ? error.message : String(error)); + receipt.phase = error instanceof HeadMoved ? 'head-moved' : 'failed'; + receipt.compensation = []; + const recover = async (label: string, operation: () => Promise): Promise => { + try { await operation(); receipt.compensation!.push(`${label}: done`); } + catch (failure) { receipt.compensation!.push(`${label}: FAILED ${redact(String(failure))}`); } + }; + // Withdraw an attempted success on its original SHA even if the current head has moved. + if (successAttempted && receipt.commentUrl) await recover('withdraw success on pinned SHA', async () => { + await github.status(receipt.sha, 'failure', receipt.commentUrl!, 'Verification aborted; rerun required'); receipt.status = 'failure'; + }); + if (readyAttempted) await recover('restore draft', async () => { await github.draft(receipt.pr); receipt.madeReady = false; }); + if (!successAttempted) await recover('publish pinned failure', async () => { + await current(); + receipt.commentUrl = await github.comment(receipt.pr, evidence(receipt)); await current(); + await github.status(receipt.sha, 'failure', receipt.commentUrl, 'Verification failed; see retained evidence'); receipt.status = 'failure'; await current(); + }); + await persist(receipt); throw new Error(receipt.failure); + } +} From 20afcc2d0f9401fcebb778e692014cde6368b708 Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:58:48 +0000 Subject: [PATCH 05/20] test(#90): gate repository verifier without shipping it Keep independent helper checks and shared harness discovery outside plugin content, and require operator-owned isolated authentication. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .agents/skills/verify-open-pstack | 1 + .claude/skills/verify-open-pstack/SKILL.md | 2 +- .../scripts/harness.test.ts | 9 +++++- .../verify-open-pstack/scripts/harness.ts | 4 +-- .../skills/verify-open-pstack/scripts/io.ts | 5 +-- .gitignore | 3 ++ AGENTS.md | 2 ++ tests/skill-collision-repro.sh | 31 +++++++++++++++++++ 8 files changed, 49 insertions(+), 8 deletions(-) create mode 120000 .agents/skills/verify-open-pstack diff --git a/.agents/skills/verify-open-pstack b/.agents/skills/verify-open-pstack new file mode 120000 index 00000000..1f803db0 --- /dev/null +++ b/.agents/skills/verify-open-pstack @@ -0,0 +1 @@ +../../.claude/skills/verify-open-pstack \ No newline at end of file diff --git a/.claude/skills/verify-open-pstack/SKILL.md b/.claude/skills/verify-open-pstack/SKILL.md index 4c5453f1..847c6420 100644 --- a/.claude/skills/verify-open-pstack/SKILL.md +++ b/.claude/skills/verify-open-pstack/SKILL.md @@ -44,7 +44,7 @@ Retain `doctor.json`, `receipt.json`, installed provenance, redacted transcripts Only complete evidence publishes `live-gate=success` on the pinned SHA, targeting the evidence-comment URL; only then may this run mark a draft ready. Failure records its reason and, where possible, publishes failure against that same SHA. A changed head aborts; detected post-write races withdraw old-head success and restore draft if this run made it ready. GitHub mutations are not atomic; failed compensation is recorded prominently. Never attach the old run to a new head. -The builder opens this delivery as a draft and does **not** run publication. The Mac operator supplies live evidence. Workflow changes require explicit operator queue submission. If Mergify creates another head, rerun on it; never hand-post a replacement status or reuse an old receipt. Queue policy/recovery is outside this skill. +The builder opens this delivery as a draft and does **not** run publication. The Mac operator supplies live evidence. For this delivery, Mergify queues automatically once `verify`, `Unfret`, and `live-gate` pass and the PR is ready; no manual queue submission is required. If Mergify creates another head, rerun on it; never hand-post a replacement status or reuse an old receipt. Queue policy/recovery is outside this skill. ## Cleanup diff --git a/.claude/skills/verify-open-pstack/scripts/harness.test.ts b/.claude/skills/verify-open-pstack/scripts/harness.test.ts index 08d6b894..66904cb2 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.test.ts @@ -13,16 +13,23 @@ describe('isolated harness boundaries', () => { test('candidate environment never carries publisher credentials or daily config', () => { process.env.GH_TOKEN = 'test-publisher-token'; process.env.CODEX_HOME = '/daily/codex'; process.env.CLAUDE_CONFIG_DIR = '/daily/claude'; + const oldAnthropic = process.env.ANTHROPIC_API_KEY, oldOpenai = process.env.OPENAI_API_KEY; + process.env.ANTHROPIC_API_KEY = 'daily-anthropic'; process.env.OPENAI_API_KEY = 'daily-openai'; try { for (const harness of ['claude', 'codex'] as const) { const env = isolatedEnv('/run/home', harness); expect(env.GH_TOKEN).toBeUndefined(); expect(env.GITHUB_TOKEN).toBeUndefined(); + expect(env.ANTHROPIC_API_KEY).toBeUndefined(); expect(env.OPENAI_API_KEY).toBeUndefined(); expect(env.HOME).toBe('/run/home'); expect(env.GIT_CONFIG_GLOBAL).toBe('/dev/null'); expect(JSON.stringify(env)).not.toContain('/daily/'); } expect(launch('claude', '/run/home', '/candidate')).toEqual(['claude', '--plugin-dir', '/candidate/plugins/pstack', '--settings', '/run/home/settings.json', '--setting-sources', '']); expect(launch('codex', '/run/home', '/candidate')).toEqual(['codex']); - } finally { delete process.env.GH_TOKEN; delete process.env.CODEX_HOME; delete process.env.CLAUDE_CONFIG_DIR; } + } finally { + delete process.env.GH_TOKEN; delete process.env.CODEX_HOME; delete process.env.CLAUDE_CONFIG_DIR; + if (oldAnthropic === undefined) delete process.env.ANTHROPIC_API_KEY; else process.env.ANTHROPIC_API_KEY = oldAnthropic; + if (oldOpenai === undefined) delete process.env.OPENAI_API_KEY; else process.env.OPENAI_API_KEY = oldOpenai; + } }); test('doctor blocks non-Mac and missing isolation interfaces, retaining reasons', async () => { const root = await fixture(); diff --git a/.claude/skills/verify-open-pstack/scripts/harness.ts b/.claude/skills/verify-open-pstack/scripts/harness.ts index cbddb65f..4affefe1 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.ts @@ -69,13 +69,13 @@ export class MacDriver implements Driver { const observations: Observation[] = []; for (const installation of receipt.installations) { const { harness, home } = installation, workspace = join(home, 'workspace'); - const env = isolatedEnv(home, harness, true); + const env = isolatedEnv(home, harness); const request = { sha: receipt.sha, harness, workspace, features: requiredFeatures(receipt), featureMap: join(workspace, '.claude/skills/verify-open-pstack/features'), selfTest: requiredFeatures(receipt).includes('project-skill') ? 'Invoke the project skill natively; run only doctor with fresh output outside workspace. Do not invoke run or publish recursively.' : false }; await save(join(receipt.artifactRoot, `${harness}-request.json`), request); console.log(JSON.stringify(request, null, 2)); - const login = await this.review(`Authenticate only in this isolated ${harness} home if needed. Launch an unrecorded login session now? Type LOGIN or SKIP (API key already supplied).`); + const login = await this.review(`Authenticate only in this isolated ${harness} home if needed. Launch an unrecorded login session now? Type LOGIN or SKIP (already authenticated in this isolated home).`); if (login === 'LOGIN') await this.run(harness === 'claude' ? ['claude', 'auth', 'login'] : ['codex', 'login'], { cwd: workspace, env, interactive: true }); else if (login !== 'SKIP') throw new Error('Expected LOGIN or SKIP'); const raw = join(home, 'surface.raw'); diff --git a/.claude/skills/verify-open-pstack/scripts/io.ts b/.claude/skills/verify-open-pstack/scripts/io.ts index 03418be5..91c3c69f 100644 --- a/.claude/skills/verify-open-pstack/scripts/io.ts +++ b/.claude/skills/verify-open-pstack/scripts/io.ts @@ -16,15 +16,12 @@ export const command: Command = async (args, options = {}) => { export function redact(text: string): string { return text.replace(/(?:gh[pousr]_[\w]+|github_pat_[\w]+|sk-[\w-]+|Bearer\s+\S+)/gi, '[REDACTED]'); } -export function isolatedEnv(home: string, harness?: 'claude' | 'codex', credentials = false): Record { +export function isolatedEnv(home: string, harness?: 'claude' | 'codex'): Record { const env: Record = { PATH: (process.env.PATH ?? '').split(':').filter(isAbsolute).join(':'), HOME: home, TMPDIR: join(home, 'tmp'), TERM: process.env.TERM ?? 'xterm-256color', GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_SYSTEM: '/dev/null' }; if (harness === 'claude') env.CLAUDE_CONFIG_DIR = join(home, 'config'); if (harness === 'codex') env.CODEX_HOME = join(home, 'config'); - if (credentials) for (const name of ['ANTHROPIC_API_KEY', 'OPENAI_API_KEY']) { - if (process.env[name]) env[name] = process.env[name]!; - } return env; } export async function save(path: string, value: unknown): Promise { diff --git a/.gitignore b/.gitignore index 008e57b5..10fb2a92 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,6 @@ # bun deps for the vendored poteto-mode scripts (bootstrap.ts installs them on first run) plugins/pstack/skills/poteto-mode/scripts/node_modules/ + +# Non-shipped repository verification helper dependencies +.claude/skills/verify-open-pstack/node_modules/ diff --git a/AGENTS.md b/AGENTS.md index 6dc9c069..d4f5ed2a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,4 +8,6 @@ Before opening a pull request, run the Bun tests, strict typecheck, static invar Nothing merges, tags, releases, or rolls out until the exact candidate is installed and the changed behavior passes a live test from the real user surface in every affected harness. Unit tests, validators, source inspection, and self-reports do not satisfy this gate. Record the installed version, surface, action, and observed result in the pull request template. A pull request without that evidence remains a draft. +For Open Pstack PRs, use the shared repository-local `.claude/skills/verify-open-pstack/` skill (Codex: `.agents/skills/verify-open-pstack`). Run its independent Bun tests/typecheck separately from the shipped package. The Mac operator runs `scripts/verify.sh doctor`, then `scripts/verify.sh run --pr --self-test --output `; read the skill's Launch section for separate probe/live output directories. Builders leave delivery PRs draft and do not publish the gate. The helper requires isolated candidate installs in both harnesses, operator-reviewed native-surface evidence, and an exact-head `live-gate` targeting the evidence comment before readiness. Every new head, including a queue-created head, needs a fresh run; never hand-post a replacement status. This helper does not queue or merge. + Do not add an implicit runtime timeout or a weaker-model fallback. diff --git a/tests/skill-collision-repro.sh b/tests/skill-collision-repro.sh index 9a2ba488..b26d0c2a 100755 --- a/tests/skill-collision-repro.sh +++ b/tests/skill-collision-repro.sh @@ -380,6 +380,37 @@ else note "ok: codex logo path resolves" fi +verification="$repo/.claude/skills/verify-open-pstack" +verification_bad="" +if [ ! -L "$repo/.agents/skills/verify-open-pstack" ] || + [ "$(readlink "$repo/.agents/skills/verify-open-pstack" 2>/dev/null || true)" != "../../.claude/skills/verify-open-pstack" ]; then + verification_bad="Codex must link to the canonical Claude project skill" +fi +for section in Launch Doctor Drive Evidence Cleanup Helpers; do + grep -q "^## $section$" "$verification/SKILL.md" || verification_bad="$verification_bad missing $section;" +done +for file in features/registry.json features/README.md package.json bun.lock tsconfig.json; do + [ -f "$verification/$file" ] || verification_bad="$verification_bad missing $file;" +done +[ -x "$verification/scripts/verify.sh" ] || verification_bad="$verification_bad helper is not executable;" +if [ -e "$plugin/skills/verify-open-pstack" ] || [ -e "$plugin/commands/verify-open-pstack.md" ]; then + verification_bad="$verification_bad project verifier must not ship in the plugin;" +fi +if [ -n "$verification_bad" ]; then + note "FAIL: repository-local verification skill: $verification_bad" + fail=1 +else + note "ok: shared project verification skill has executable helper and maintained feature map" +fi + +note "Checking non-shipped verification helper" +( + cd "$verification" + bun install --frozen-lockfile + bun run test + bun run typecheck +) + if [ "${PSTACK_STATIC_ONLY:-0}" = "1" ]; then exit "$fail" fi From cfe38e7505b7ec78126eda784abf36974e76e7b5 Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:31:27 +0000 Subject: [PATCH 06/20] fix(#90): canonicalize child doctor provenance for self-test Compare real filesystem paths so doctor output and isolated workspace aliases agree. Regress with an actual doctor report, including wrong-workspace and blocked-report rejection. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .../verify-open-pstack/scripts/doctor.ts | 4 ++-- .../scripts/harness.test.ts | 22 +++++++++++++++++-- .../verify-open-pstack/scripts/harness.ts | 14 +++++++++--- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/.claude/skills/verify-open-pstack/scripts/doctor.ts b/.claude/skills/verify-open-pstack/scripts/doctor.ts index f2e1eacb..46b2a038 100644 --- a/.claude/skills/verify-open-pstack/scripts/doctor.ts +++ b/.claude/skills/verify-open-pstack/scripts/doctor.ts @@ -1,9 +1,9 @@ -import { mkdir } from 'node:fs/promises'; +import { mkdir, realpath } from 'node:fs/promises'; import { join } from 'node:path'; import { command, isolatedEnv, save, type Command } from './io.ts'; export async function doctor(root: string, run: Command = command, platform = process.platform): Promise { - const report: Record = { platform, date: new Date().toISOString(), skill: import.meta.dir + '/..', checks: {} }; + const report: Record = { platform, date: new Date().toISOString(), skill: await realpath(join(import.meta.dir, '..')), checks: {} }; const checks = report.checks as Record; try { if (platform !== 'darwin') throw new Error('Live gate requires an operator Mac (Darwin)'); diff --git a/.claude/skills/verify-open-pstack/scripts/harness.test.ts b/.claude/skills/verify-open-pstack/scripts/harness.test.ts index 66904cb2..cd8827d9 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.test.ts @@ -1,9 +1,9 @@ import { afterEach, describe, expect, test } from 'bun:test'; -import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from 'node:fs/promises'; +import { mkdtemp, mkdir, readFile, realpath, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { doctor } from './doctor.ts'; -import { codexInstallation, launch, verifyCodexEnabled } from './harness.ts'; +import { codexInstallation, launch, verifyCodexEnabled, verifyProjectDoctor } from './harness.ts'; import { freshRoot, isolatedEnv, redact, retainedFile, treeHash, type Command } from './io.ts'; const roots: string[] = []; async function fixture(): Promise { const root = await mkdtemp(join(tmpdir(), 'pstack-test-')); roots.push(root); return root; } @@ -45,6 +45,24 @@ describe('isolated harness boundaries', () => { expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('pass'); expect(calls.every(c => c.includes('--help') || c.includes('--version'))).toBe(true); }); + test('project self-test accepts real doctor output through canonical workspace paths', async () => { + const root = await fixture(), workspace = join(root, 'workspace'); + const skill = await realpath(join(import.meta.dir, '..')); + await mkdir(join(workspace, '.claude/skills'), { recursive: true }); + await symlink(skill, join(workspace, '.claude/skills/verify-open-pstack')); + const alias = join(root, 'workspace-alias'); await symlink(workspace, alias); + const run: Command = async args => args.includes('--version') ? 'version' : '--plugin-dir --settings --setting-sources --json local path'; + await doctor(root, run, 'darwin'); + const text = await readFile(join(root, 'doctor.json'), 'utf8'); + expect(JSON.parse(text).skill).toBe(skill); + await verifyProjectDoctor(['not JSON', text], workspace); + await verifyProjectDoctor([text], alias); + const other = join(root, 'other-workspace'); + await mkdir(join(other, '.claude/skills/verify-open-pstack'), { recursive: true }); + await expect(verifyProjectDoctor([text], other)).rejects.toThrow('passing child doctor'); + await expect(verifyProjectDoctor(['{}'], workspace)).rejects.toThrow('passing child doctor'); + await expect(verifyProjectDoctor([JSON.stringify({ ...JSON.parse(text), result: 'blocked' })], workspace)).rejects.toThrow('passing child doctor'); + }); test('Codex exact installed tree and enabled listing are required', async () => { const root = await fixture(), plugin = join(root, 'config/plugins/pstack'); await mkdir(plugin, { recursive: true }); await writeFile(join(plugin, 'SKILL.md'), 'candidate'); diff --git a/.claude/skills/verify-open-pstack/scripts/harness.ts b/.claude/skills/verify-open-pstack/scripts/harness.ts index 4affefe1..74c13464 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.ts @@ -32,6 +32,16 @@ export function verifyCodexEnabled(output: string): void { const plugins = Array.isArray(result.installed) ? result.installed.filter((p: Record) => p.name === 'pstack' && p.marketplaceName === 'open-pstack') : []; if (plugins.length !== 1 || plugins[0].installed !== true || plugins[0].enabled !== true) throw new Error('Isolated Codex plugin is not installed and enabled'); } +export async function verifyProjectDoctor(texts: string[], workspace: string): Promise { + const expected = await realpath(join(workspace, '.claude/skills/verify-open-pstack')); + for (const text of texts) { + try { + const d = JSON.parse(text); + if (d.result === 'pass' && typeof d.skill === 'string' && await realpath(d.skill) === expected) return; + } catch { /* Other reviewed artifacts need not be doctor reports. */ } + } + throw new Error('Self-test requires the pinned project skill\'s passing child doctor.json'); +} export class MacDriver implements Driver { constructor(private run: Command = command, private review: Ask = ask) {} async prepare(receipt: Receipt): Promise { @@ -95,9 +105,7 @@ export class MacDriver implements Driver { } if (feature === 'project-skill') { const doctors = await Promise.all(artifacts.map(a => readFile(join(receipt.artifactRoot, a.path), 'utf8'))); - if (!doctors.some(text => { try { const d = JSON.parse(text); return d.result === 'pass' && d.skill === join(workspace, '.claude/skills/verify-open-pstack/scripts/..'); } catch { return false; } })) { - throw new Error('Self-test requires the pinned project skill\'s passing child doctor.json'); - } + await verifyProjectDoctor(doctors, workspace); } if (await this.review(`Operator: type PASS ${feature} only after reviewing the native transcript and artifacts; anything else fails.`) !== `PASS ${feature}`) { throw new Error(`Operator rejected ${harness}/${feature}`); From 550a532b74d3fd04766f54416b7e22e535edf474 Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:07:07 +0000 Subject: [PATCH 07/20] fix(#90): create isolated config homes before harness startup Codex requires CODEX_HOME to exist before marketplace installation. Create private config directories for both harnesses and regression-test the complete preparation path before every isolated command. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .../scripts/harness.test.ts | 49 ++++++++++++++++++- .../verify-open-pstack/scripts/harness.ts | 1 + 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/.claude/skills/verify-open-pstack/scripts/harness.test.ts b/.claude/skills/verify-open-pstack/scripts/harness.test.ts index cd8827d9..109ace9e 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.test.ts @@ -1,9 +1,10 @@ import { afterEach, describe, expect, test } from 'bun:test'; -import { mkdtemp, mkdir, readFile, realpath, rm, symlink, writeFile } from 'node:fs/promises'; +import { cp, mkdtemp, mkdir, readFile, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { doctor } from './doctor.ts'; -import { codexInstallation, launch, verifyCodexEnabled, verifyProjectDoctor } from './harness.ts'; +import { codexInstallation, launch, MacDriver, verifyCodexEnabled, verifyProjectDoctor } from './harness.ts'; +import { newReceipt } from './core.ts'; import { freshRoot, isolatedEnv, redact, retainedFile, treeHash, type Command } from './io.ts'; const roots: string[] = []; async function fixture(): Promise { const root = await mkdtemp(join(tmpdir(), 'pstack-test-')); roots.push(root); return root; } @@ -63,6 +64,50 @@ describe('isolated harness boundaries', () => { await expect(verifyProjectDoctor(['{}'], workspace)).rejects.toThrow('passing child doctor'); await expect(verifyProjectDoctor([JSON.stringify({ ...JSON.parse(text), result: 'blocked' })], workspace)).rejects.toThrow('passing child doctor'); }); + test('prepare creates private configuration homes before any harness command', async () => { + const root = await fixture(), sha = 'a'.repeat(40), calls: string[][] = []; + const platform = Object.getOwnPropertyDescriptor(process, 'platform')!; + const run: Command = async (args, options = {}) => { + calls.push(args); + const env = options.env!; + const config = env.CODEX_HOME ?? env.CLAUDE_CONFIG_DIR; + if (config) { + const info = await stat(config); + expect(info.isDirectory()).toBe(true); + expect(info.mode & 0o777).toBe(0o700); + expect(config).toBe(join(env.HOME!, 'config')); + expect(env.GH_TOKEN).toBeUndefined(); + } + if (args[0] === 'git' && args[1] === 'clone') { + const plugin = join(args.at(-1)!, 'plugins/pstack'); + for (const manifest of ['.claude-plugin', '.codex-plugin']) { + await mkdir(join(plugin, manifest), { recursive: true }); + await writeFile(join(plugin, manifest, 'plugin.json'), JSON.stringify({ version: 'test' })); + } + } + if (args[0] === 'git' && args[1] === 'rev-parse') return sha; + if (args.includes('--version')) return 'version'; + if (args.includes('--help')) return '--plugin-dir --settings --setting-sources --json local path'; + if (args[0] === 'codex' && args[1] === 'plugin') { + expect(env.CODEX_HOME).toBeDefined(); + if (args[2] === 'marketplace') return '{}'; + if (args[2] === 'add') { + const installedPath = join(env.CODEX_HOME!, 'plugins/pstack'); + await cp(join(env.HOME!, 'workspace/plugins/pstack'), installedPath, { recursive: true }); + return JSON.stringify({ name: 'pstack', marketplaceName: 'open-pstack', installedPath }); + } + if (args[2] === 'list') return JSON.stringify({ installed: [{ name: 'pstack', marketplaceName: 'open-pstack', installed: true, enabled: true }] }); + } + return ''; + }; + try { + Object.defineProperty(process, 'platform', { ...platform, value: 'darwin' }); + const installs = await new MacDriver(run).prepare(newReceipt(111, sha, 'b'.repeat(40), true, root)); + expect(installs.map(i => i.harness)).toEqual(['claude', 'codex']); + expect(installs.every(i => i.sha === sha)).toBe(true); + expect(calls.some(c => c.join(' ') === 'codex plugin add pstack@open-pstack --json')).toBe(true); + } finally { Object.defineProperty(process, 'platform', platform); } + }); test('Codex exact installed tree and enabled listing are required', async () => { const root = await fixture(), plugin = join(root, 'config/plugins/pstack'); await mkdir(plugin, { recursive: true }); await writeFile(join(plugin, 'SKILL.md'), 'candidate'); diff --git a/.claude/skills/verify-open-pstack/scripts/harness.ts b/.claude/skills/verify-open-pstack/scripts/harness.ts index 74c13464..baf181ad 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.ts @@ -51,6 +51,7 @@ export class MacDriver implements Driver { for (const harness of HARNESSES) { const home = join(root, 'state', harness), workspace = join(home, 'workspace'); await mkdir(join(home, 'tmp'), { recursive: true, mode: 0o700 }); + await mkdir(join(home, 'config'), { recursive: true, mode: 0o700 }); const env = isolatedEnv(home, harness); await this.run(['git', 'clone', '--no-checkout', '--', `https://github.com/${REPO}.git`, workspace], { env }); await this.run(['git', 'fetch', 'origin', receipt.sha], { cwd: workspace, env }); From fb76b8c617cd74e79648c2023aea03d17b858fcb Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:09:56 +0000 Subject: [PATCH 08/20] test(#90): verify all isolated CLI startup paths exist Assert real home, temporary directory, working directory, and private Claude settings alongside config homes at each preparation command so command stubs cannot conceal missing startup paths. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .../skills/verify-open-pstack/scripts/harness.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.claude/skills/verify-open-pstack/scripts/harness.test.ts b/.claude/skills/verify-open-pstack/scripts/harness.test.ts index 109ace9e..4dff296e 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.test.ts @@ -71,6 +71,17 @@ describe('isolated harness boundaries', () => { calls.push(args); const env = options.env!; const config = env.CODEX_HOME ?? env.CLAUDE_CONFIG_DIR; + for (const path of [env.HOME!, env.TMPDIR!]) { + const info = await stat(path); + expect(info.isDirectory()).toBe(true); + expect(info.mode & 0o777).toBe(0o700); + } + if (options.cwd) expect((await stat(options.cwd)).isDirectory()).toBe(true); + if (args[0] === 'claude' && env.CLAUDE_CONFIG_DIR) { + const settings = join(env.HOME!, 'settings.json'); + expect(JSON.parse(await readFile(settings, 'utf8'))).toEqual({}); + expect((await stat(settings)).mode & 0o777).toBe(0o600); + } if (config) { const info = await stat(config); expect(info.isDirectory()).toBe(true); From 5da3aff61e462726554ac31e752ce2a586e24cff Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:22:36 +0000 Subject: [PATCH 09/20] fix(#90): preserve real HOME for macOS keychain lookup Isolate harness configuration and temporary files without replacing HOME, avoiding the operator-observed Keychain reset dialog. Preserve credential scrubbing and verify real HOME alongside private run-owned config paths. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .claude/skills/verify-open-pstack/SKILL.md | 4 ++-- .../verify-open-pstack/features/isolation.md | 6 +++--- .../verify-open-pstack/scripts/harness.test.ts | 15 ++++++++++----- .claude/skills/verify-open-pstack/scripts/io.ts | 4 +++- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.claude/skills/verify-open-pstack/SKILL.md b/.claude/skills/verify-open-pstack/SKILL.md index 847c6420..fd58e532 100644 --- a/.claude/skills/verify-open-pstack/SKILL.md +++ b/.claude/skills/verify-open-pstack/SKILL.md @@ -24,13 +24,13 @@ Omit `--self-test` for ordinary plugin verification; it is mandatory for this sk Requires Darwin, Bun, gh authentication, both harness CLIs, and operator access to every selected provider. `doctor` writes `doctor.json` outside the checkout. It checks the supported isolation flags but does not claim that installation or live behavior works. -The approved October 2 Mac probe (issue #90, comment 5961645184) reported Darwin, Claude Code 2.1.283, codex-cli 0.160.0, Bun 1.4.0, gh 2.97.0. Selected interfaces: Claude `--plugin-dir`, `--settings`, `--setting-sources`, `CLAUDE_CONFIG_DIR`; Codex local `plugin marketplace add`/`plugin add`, `CODEX_HOME`. Each harness also receives a run-owned `HOME`. Installation/provenance checks validate those choices during the actual run. Missing interfaces or authentication fail closed: never use a daily install. +The approved October 2 Mac probe (issue #90, comment 5961645184) reported Darwin, Claude Code 2.1.283, codex-cli 0.160.0, Bun 1.4.0, gh 2.97.0. Selected interfaces: Claude `--plugin-dir`, `--settings`, `--setting-sources`, `CLAUDE_CONFIG_DIR`; Codex local `plugin marketplace add`/`plugin add`, `CODEX_HOME`. Both harnesses preserve the operator's real `HOME`; only `CLAUDE_CONFIG_DIR`/`CODEX_HOME` and `TMPDIR` point into run-owned state. Never substitute a temporary `HOME`: the operator observed a macOS Keychain reset dialog with that setup. Never create, reset, or unlock any keychain. On the operator's Claude Code 2.1.283, the temporary config directory reported `loggedIn:false` and used a config-scoped `Claude Code-credentials-` item distinct from the daily credentials; this measured isolation must be confirmed by the operator's fresh live run. Installation/provenance checks validate those choices during the actual run. Missing interfaces or authentication fail closed: never use a daily install. ## Drive Read `features/README.md` and each selected feature document. Runtime markdown and consumed references are runtime. Shared paths select all consumers conservatively. `no runtime change` launches no harness unless the separate `--self-test` was requested. -The runner creates a detached exact-head candidate, separate homes and fixture workspaces, and executable launchers. Claude loads the candidate with its explicit plugin directory; Codex installs the candidate's local marketplace into its isolated home. The installed trees must match the candidate's content digest; a shared version string is insufficient. Fresh fixture sessions discover the canonical project skill and Codex link copied from this head. +The runner creates a detached exact-head candidate, separate run-owned config/state directories and fixture workspaces, and executable launchers. Process `HOME` remains the operator's real home; the run-owned paths referred to below as isolated homes are state roots, not replacements for `HOME`. Claude loads the candidate with its explicit plugin directory; Codex installs the candidate's local marketplace into its isolated home. The installed trees must match the candidate's content digest; a shared version string is insufficient. Fresh fixture sessions discover the canonical project skill and Codex link copied from this head. The operator authenticates only these isolated homes when needed, using the printed launcher with `auth login` (Claude) or `login` (Codex); do not copy daily auth/configuration. GitHub tokens and daily provider secrets are excluded from harness subprocess environments. A missing configured provider remains a feature failure. Never add a timeout or weaker-model fallback. diff --git a/.claude/skills/verify-open-pstack/features/isolation.md b/.claude/skills/verify-open-pstack/features/isolation.md index be2a8e8b..71c0a014 100644 --- a/.claude/skills/verify-open-pstack/features/isolation.md +++ b/.claude/skills/verify-open-pstack/features/isolation.md @@ -3,9 +3,9 @@ The operator's read-only Mac probe is retained in issue #90 comment 5961645184; plan approval is comment 5961649164. On 2026-10-02 it reported Darwin, Claude Code 2.1.283, codex-cli 0.160.0, Bun 1.4.0, and gh 2.97.0. This is capability evidence, not proof of an installed candidate. Selected interfaces: -- Claude: a fresh `HOME` and `CLAUDE_CONFIG_DIR`, candidate `--plugin-dir`, explicit empty `--settings`, and empty `--setting-sources`. No daily plugin/configuration state is copied. -- Codex: a fresh `HOME` and `CODEX_HOME`; add the pinned checkout as a local marketplace with `codex plugin marketplace add --json`, then `codex plugin add pstack@open-pstack --json`. Start a new interactive task after installation. Do not use the daily profile. -- Both: a fresh candidate checkout, run-owned state, minimal inherited environment, interactive login inside isolated state or operator-provided provider credentials, and fresh native sessions. The publisher's GitHub credentials are never passed to the candidate or either harness. +- Claude: the real `HOME` and a fresh run-owned `CLAUDE_CONFIG_DIR`, candidate `--plugin-dir`, explicit empty `--settings`, and empty `--setting-sources`. No daily plugin/configuration state is copied. +- Codex: the real `HOME` and a fresh run-owned `CODEX_HOME`; add the pinned checkout as a local marketplace with `codex plugin marketplace add --json`, then `codex plugin add pstack@open-pstack --json`. Start a new interactive task after installation. Do not use the daily profile. +- Both: a fresh candidate checkout, run-owned state and `TMPDIR`, minimal inherited environment, interactive login inside isolated config state, and fresh native sessions. The publisher's GitHub credentials and daily provider API keys are never passed to the candidate or either harness. Preserve the real `HOME` so macOS can find the login Keychain; never create, reset, or unlock any keychain. The operator measured config-scoped Claude OAuth items on 2.1.283 and observed `loggedIn:false` for a fresh config directory with real `HOME`; the new-head Mac run remains required to verify login and live behavior. Run-owned 'home' fields in receipts denote state roots, not process `HOME`. The Codex JSON contract was checked against OpenAI's `codex-rs/cli/src/plugin_cmd.rs` at tag `rust-v0.160.0`: `JsonPluginAddOutput` uses camelCase `name`, `marketplaceName`, and `installedPath`; `JsonPluginListOutput.installed` contains `installed` and `enabled`. Unknown schemas fail closed rather than searching arbitrary caches for a plausible tree. diff --git a/.claude/skills/verify-open-pstack/scripts/harness.test.ts b/.claude/skills/verify-open-pstack/scripts/harness.test.ts index 4dff296e..38a9bd5b 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.test.ts @@ -21,7 +21,9 @@ describe('isolated harness boundaries', () => { const env = isolatedEnv('/run/home', harness); expect(env.GH_TOKEN).toBeUndefined(); expect(env.GITHUB_TOKEN).toBeUndefined(); expect(env.ANTHROPIC_API_KEY).toBeUndefined(); expect(env.OPENAI_API_KEY).toBeUndefined(); - expect(env.HOME).toBe('/run/home'); expect(env.GIT_CONFIG_GLOBAL).toBe('/dev/null'); + expect(process.env.HOME).toBe(env.HOME); expect(env.GIT_CONFIG_GLOBAL).toBe('/dev/null'); + expect(env.TMPDIR).toBe('/run/home/tmp'); + expect(env[harness === 'claude' ? 'CLAUDE_CONFIG_DIR' : 'CODEX_HOME']).toBe('/run/home/config'); expect(JSON.stringify(env)).not.toContain('/daily/'); } expect(launch('claude', '/run/home', '/candidate')).toEqual(['claude', '--plugin-dir', '/candidate/plugins/pstack', '--settings', '/run/home/settings.json', '--setting-sources', '']); @@ -71,14 +73,16 @@ describe('isolated harness boundaries', () => { calls.push(args); const env = options.env!; const config = env.CODEX_HOME ?? env.CLAUDE_CONFIG_DIR; - for (const path of [env.HOME!, env.TMPDIR!]) { + expect(process.env.HOME).toBe(env.HOME); + expect((await stat(env.HOME!)).isDirectory()).toBe(true); + for (const path of [env.TMPDIR!]) { const info = await stat(path); expect(info.isDirectory()).toBe(true); expect(info.mode & 0o777).toBe(0o700); } if (options.cwd) expect((await stat(options.cwd)).isDirectory()).toBe(true); if (args[0] === 'claude' && env.CLAUDE_CONFIG_DIR) { - const settings = join(env.HOME!, 'settings.json'); + const settings = join(env.CLAUDE_CONFIG_DIR, '..', 'settings.json'); expect(JSON.parse(await readFile(settings, 'utf8'))).toEqual({}); expect((await stat(settings)).mode & 0o777).toBe(0o600); } @@ -86,7 +90,8 @@ describe('isolated harness boundaries', () => { const info = await stat(config); expect(info.isDirectory()).toBe(true); expect(info.mode & 0o777).toBe(0o700); - expect(config).toBe(join(env.HOME!, 'config')); + expect(config).toBe(join(env.TMPDIR!, '..', 'config')); + expect(config.startsWith(root + '/state/')).toBe(true); expect(env.GH_TOKEN).toBeUndefined(); } if (args[0] === 'git' && args[1] === 'clone') { @@ -104,7 +109,7 @@ describe('isolated harness boundaries', () => { if (args[2] === 'marketplace') return '{}'; if (args[2] === 'add') { const installedPath = join(env.CODEX_HOME!, 'plugins/pstack'); - await cp(join(env.HOME!, 'workspace/plugins/pstack'), installedPath, { recursive: true }); + await cp(join(env.CODEX_HOME!, '..', 'workspace/plugins/pstack'), installedPath, { recursive: true }); return JSON.stringify({ name: 'pstack', marketplaceName: 'open-pstack', installedPath }); } if (args[2] === 'list') return JSON.stringify({ installed: [{ name: 'pstack', marketplaceName: 'open-pstack', installed: true, enabled: true }] }); diff --git a/.claude/skills/verify-open-pstack/scripts/io.ts b/.claude/skills/verify-open-pstack/scripts/io.ts index 91c3c69f..ed8540b1 100644 --- a/.claude/skills/verify-open-pstack/scripts/io.ts +++ b/.claude/skills/verify-open-pstack/scripts/io.ts @@ -17,8 +17,10 @@ export function redact(text: string): string { return text.replace(/(?:gh[pousr]_[\w]+|github_pat_[\w]+|sk-[\w-]+|Bearer\s+\S+)/gi, '[REDACTED]'); } export function isolatedEnv(home: string, harness?: 'claude' | 'codex'): Record { + const realHome = process.env.HOME; + if (!realHome || !isAbsolute(realHome)) throw new Error('A real absolute HOME is required; never substitute a temporary home'); const env: Record = { PATH: (process.env.PATH ?? '').split(':').filter(isAbsolute).join(':'), - HOME: home, TMPDIR: join(home, 'tmp'), TERM: process.env.TERM ?? 'xterm-256color', + HOME: realHome, TMPDIR: join(home, 'tmp'), TERM: process.env.TERM ?? 'xterm-256color', GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_SYSTEM: '/dev/null' }; if (harness === 'claude') env.CLAUDE_CONFIG_DIR = join(home, 'config'); if (harness === 'codex') env.CODEX_HOME = join(home, 'config'); From faca0ff6d52bc0f7fa02c93555394f0694e52a33 Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:33:29 +0000 Subject: [PATCH 10/20] fix(#90): load candidate project skills and preserve account identity The operator measured missing skill discovery with empty setting sources and unknown Keychain account names without USER/LOGNAME. Load only project settings and preserve account identity without relaxing credential or config isolation. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .claude/skills/verify-open-pstack/SKILL.md | 2 +- .../verify-open-pstack/features/isolation.md | 4 ++-- .../scripts/harness.test.ts | 20 ++++++++++++++++++- .../verify-open-pstack/scripts/harness.ts | 2 +- .../skills/verify-open-pstack/scripts/io.ts | 3 +++ 5 files changed, 26 insertions(+), 5 deletions(-) diff --git a/.claude/skills/verify-open-pstack/SKILL.md b/.claude/skills/verify-open-pstack/SKILL.md index fd58e532..738c5c80 100644 --- a/.claude/skills/verify-open-pstack/SKILL.md +++ b/.claude/skills/verify-open-pstack/SKILL.md @@ -24,7 +24,7 @@ Omit `--self-test` for ordinary plugin verification; it is mandatory for this sk Requires Darwin, Bun, gh authentication, both harness CLIs, and operator access to every selected provider. `doctor` writes `doctor.json` outside the checkout. It checks the supported isolation flags but does not claim that installation or live behavior works. -The approved October 2 Mac probe (issue #90, comment 5961645184) reported Darwin, Claude Code 2.1.283, codex-cli 0.160.0, Bun 1.4.0, gh 2.97.0. Selected interfaces: Claude `--plugin-dir`, `--settings`, `--setting-sources`, `CLAUDE_CONFIG_DIR`; Codex local `plugin marketplace add`/`plugin add`, `CODEX_HOME`. Both harnesses preserve the operator's real `HOME`; only `CLAUDE_CONFIG_DIR`/`CODEX_HOME` and `TMPDIR` point into run-owned state. Never substitute a temporary `HOME`: the operator observed a macOS Keychain reset dialog with that setup. Never create, reset, or unlock any keychain. On the operator's Claude Code 2.1.283, the temporary config directory reported `loggedIn:false` and used a config-scoped `Claude Code-credentials-` item distinct from the daily credentials; this measured isolation must be confirmed by the operator's fresh live run. Installation/provenance checks validate those choices during the actual run. Missing interfaces or authentication fail closed: never use a daily install. +The approved October 2 Mac probe (issue #90, comment 5961645184) reported Darwin, Claude Code 2.1.283, codex-cli 0.160.0, Bun 1.4.0, gh 2.97.0. Selected interfaces: Claude `--plugin-dir`, `--settings`, `--setting-sources`, `CLAUDE_CONFIG_DIR`; Codex local `plugin marketplace add`/`plugin add`, `CODEX_HOME`. Claude uses `--setting-sources project` so the pinned checkout's project skills/settings load while user settings stay excluded. Both harnesses preserve the operator's real `HOME`, `USER`, and `LOGNAME` for Keychain location/account identity; only `CLAUDE_CONFIG_DIR`/`CODEX_HOME` and `TMPDIR` point into run-owned state. Never substitute a temporary `HOME`: the operator observed a macOS Keychain reset dialog with that setup. Never create, reset, or unlock any keychain. On the operator's Claude Code 2.1.283, the temporary config directory reported `loggedIn:false` and used a config-scoped `Claude Code-credentials-` item distinct from the daily credentials; this measured isolation must be confirmed by the operator's fresh live run. Installation/provenance checks validate those choices during the actual run. Missing interfaces or authentication fail closed: never use a daily install. ## Drive diff --git a/.claude/skills/verify-open-pstack/features/isolation.md b/.claude/skills/verify-open-pstack/features/isolation.md index 71c0a014..21ac1f40 100644 --- a/.claude/skills/verify-open-pstack/features/isolation.md +++ b/.claude/skills/verify-open-pstack/features/isolation.md @@ -3,9 +3,9 @@ The operator's read-only Mac probe is retained in issue #90 comment 5961645184; plan approval is comment 5961649164. On 2026-10-02 it reported Darwin, Claude Code 2.1.283, codex-cli 0.160.0, Bun 1.4.0, and gh 2.97.0. This is capability evidence, not proof of an installed candidate. Selected interfaces: -- Claude: the real `HOME` and a fresh run-owned `CLAUDE_CONFIG_DIR`, candidate `--plugin-dir`, explicit empty `--settings`, and empty `--setting-sources`. No daily plugin/configuration state is copied. +- Claude: the real `HOME` and a fresh run-owned `CLAUDE_CONFIG_DIR`, candidate `--plugin-dir`, explicit empty `--settings`, and `--setting-sources project` to load the pinned checkout's project settings and skills while excluding user settings. The operator measured project-skill discovery failing with an empty setting source and succeeding with `project` on Claude Code 2.1.283. No daily plugin/configuration state is copied. - Codex: the real `HOME` and a fresh run-owned `CODEX_HOME`; add the pinned checkout as a local marketplace with `codex plugin marketplace add --json`, then `codex plugin add pstack@open-pstack --json`. Start a new interactive task after installation. Do not use the daily profile. -- Both: a fresh candidate checkout, run-owned state and `TMPDIR`, minimal inherited environment, interactive login inside isolated config state, and fresh native sessions. The publisher's GitHub credentials and daily provider API keys are never passed to the candidate or either harness. Preserve the real `HOME` so macOS can find the login Keychain; never create, reset, or unlock any keychain. The operator measured config-scoped Claude OAuth items on 2.1.283 and observed `loggedIn:false` for a fresh config directory with real `HOME`; the new-head Mac run remains required to verify login and live behavior. Run-owned 'home' fields in receipts denote state roots, not process `HOME`. +- Both: a fresh candidate checkout, run-owned state and `TMPDIR`, minimal inherited environment, interactive login inside isolated config state, and fresh native sessions. The publisher's GitHub credentials and daily provider API keys are never passed to the candidate or either harness. Preserve the real `HOME`, `USER`, and `LOGNAME` so macOS can find the login Keychain and Claude uses the operator's account rather than `unknown`; never create, reset, or unlock any keychain. The operator measured config-scoped Claude OAuth items on 2.1.283 and observed `loggedIn:false` for a fresh config directory with real `HOME`; the new-head Mac run remains required to verify login and live behavior. Run-owned 'home' fields in receipts denote state roots, not process `HOME`. The Codex JSON contract was checked against OpenAI's `codex-rs/cli/src/plugin_cmd.rs` at tag `rust-v0.160.0`: `JsonPluginAddOutput` uses camelCase `name`, `marketplaceName`, and `installedPath`; `JsonPluginListOutput.installed` contains `installed` and `enabled`. Unknown schemas fail closed rather than searching arbitrary caches for a plausible tree. diff --git a/.claude/skills/verify-open-pstack/scripts/harness.test.ts b/.claude/skills/verify-open-pstack/scripts/harness.test.ts index 38a9bd5b..aa51ac41 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.test.ts @@ -26,7 +26,7 @@ describe('isolated harness boundaries', () => { expect(env[harness === 'claude' ? 'CLAUDE_CONFIG_DIR' : 'CODEX_HOME']).toBe('/run/home/config'); expect(JSON.stringify(env)).not.toContain('/daily/'); } - expect(launch('claude', '/run/home', '/candidate')).toEqual(['claude', '--plugin-dir', '/candidate/plugins/pstack', '--settings', '/run/home/settings.json', '--setting-sources', '']); + expect(launch('claude', '/run/home', '/candidate')).toEqual(['claude', '--plugin-dir', '/candidate/plugins/pstack', '--settings', '/run/home/settings.json', '--setting-sources', 'project']); expect(launch('codex', '/run/home', '/candidate')).toEqual(['codex']); } finally { delete process.env.GH_TOKEN; delete process.env.CODEX_HOME; delete process.env.CLAUDE_CONFIG_DIR; @@ -34,6 +34,24 @@ describe('isolated harness boundaries', () => { if (oldOpenai === undefined) delete process.env.OPENAI_API_KEY; else process.env.OPENAI_API_KEY = oldOpenai; } }); + test('candidate environment preserves real account names without inventing an account', () => { + const previous = { USER: process.env.USER, LOGNAME: process.env.LOGNAME }; + process.env.USER = 'operator-user'; process.env.LOGNAME = 'operator-login'; + try { + for (const harness of ['claude', 'codex'] as const) { + const env = isolatedEnv('/run/home', harness); + expect(env.USER).toBe('operator-user'); expect(env.LOGNAME).toBe('operator-login'); + expect(process.env.HOME).toBe(env.HOME); + } + delete process.env.USER; delete process.env.LOGNAME; + const env = isolatedEnv('/run/home', 'claude'); + expect(env.USER).toBeUndefined(); expect(env.LOGNAME).toBeUndefined(); + } finally { + for (const name of ['USER', 'LOGNAME'] as const) { + if (previous[name] === undefined) delete process.env[name]; else process.env[name] = previous[name]; + } + } + }); test('doctor blocks non-Mac and missing isolation interfaces, retaining reasons', async () => { const root = await fixture(); const run: Command = async args => args.includes('--version') ? 'version' : ''; diff --git a/.claude/skills/verify-open-pstack/scripts/harness.ts b/.claude/skills/verify-open-pstack/scripts/harness.ts index baf181ad..ae71b5c3 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.ts @@ -14,7 +14,7 @@ export const ask: Ask = async question => { }; export function launch(harness: Harness, home: string, workspace: string): string[] { return harness === 'claude' - ? ['claude', '--plugin-dir', join(workspace, 'plugins/pstack'), '--settings', join(home, 'settings.json'), '--setting-sources', ''] + ? ['claude', '--plugin-dir', join(workspace, 'plugins/pstack'), '--settings', join(home, 'settings.json'), '--setting-sources', 'project'] : ['codex']; } export async function codexInstallation(output: string, home: string, expected: string): Promise { diff --git a/.claude/skills/verify-open-pstack/scripts/io.ts b/.claude/skills/verify-open-pstack/scripts/io.ts index ed8540b1..e314c0df 100644 --- a/.claude/skills/verify-open-pstack/scripts/io.ts +++ b/.claude/skills/verify-open-pstack/scripts/io.ts @@ -22,6 +22,9 @@ export function isolatedEnv(home: string, harness?: 'claude' | 'codex'): Record< const env: Record = { PATH: (process.env.PATH ?? '').split(':').filter(isAbsolute).join(':'), HOME: realHome, TMPDIR: join(home, 'tmp'), TERM: process.env.TERM ?? 'xterm-256color', GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_SYSTEM: '/dev/null' }; + for (const name of ['USER', 'LOGNAME']) { + if (process.env[name] !== undefined) env[name] = process.env[name]; + } if (harness === 'claude') env.CLAUDE_CONFIG_DIR = join(home, 'config'); if (harness === 'codex') env.CODEX_HOME = join(home, 'config'); return env; From 980a30300e9d9628057f22070eb3a684949ad71b Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:01:09 +0000 Subject: [PATCH 11/20] fix(#90): contain candidate sessions and bind evidence to pinned source Address the Unfret review with disposable account credentials, OS-enforced candidate boundaries, immutable source and evidence checks, and publication and feature-coverage corrections. Keep the delivery ready for a new exact-head operator live run. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .claude/skills/verify-open-pstack/SKILL.md | 62 ++- .../verify-open-pstack/features/README.md | 4 +- .../verify-open-pstack/features/isolation.md | 44 +- .../verify-open-pstack/features/registry.json | 8 +- .../verify-open-pstack/features/runner.md | 2 +- .../verify-open-pstack/features/setup.md | 4 +- .../features/shipped-tools.md | 2 +- .../features/skill-invocation.md | 4 +- .../verify-open-pstack/scripts/cli.test.ts | 65 ++- .../skills/verify-open-pstack/scripts/cli.ts | 45 ++- .../verify-open-pstack/scripts/core.test.ts | 12 + .../verify-open-pstack/scripts/doctor.ts | 8 +- .../verify-open-pstack/scripts/github.ts | 51 ++- .../scripts/harness.test.ts | 158 ++++++-- .../verify-open-pstack/scripts/harness.ts | 88 ++-- .../verify-open-pstack/scripts/io.test.ts | 129 ++++++ .../skills/verify-open-pstack/scripts/io.ts | 86 +++- .../scripts/isolation.test.ts | 376 ++++++++++++++++++ .../verify-open-pstack/scripts/isolation.ts | 203 ++++++++++ .../scripts/provenance.test.ts | 177 +++++++++ .../verify-open-pstack/scripts/provenance.ts | 98 +++++ .../verify-open-pstack/scripts/types.ts | 9 +- .../verify-open-pstack/scripts/verify.test.ts | 146 ++++++- .../verify-open-pstack/scripts/verify.ts | 85 +++- AGENTS.md | 2 +- 25 files changed, 1669 insertions(+), 199 deletions(-) create mode 100644 .claude/skills/verify-open-pstack/scripts/io.test.ts create mode 100644 .claude/skills/verify-open-pstack/scripts/isolation.test.ts create mode 100644 .claude/skills/verify-open-pstack/scripts/isolation.ts create mode 100644 .claude/skills/verify-open-pstack/scripts/provenance.test.ts create mode 100644 .claude/skills/verify-open-pstack/scripts/provenance.ts diff --git a/.claude/skills/verify-open-pstack/SKILL.md b/.claude/skills/verify-open-pstack/SKILL.md index 738c5c80..854b309c 100644 --- a/.claude/skills/verify-open-pstack/SKILL.md +++ b/.claude/skills/verify-open-pstack/SKILL.md @@ -7,54 +7,76 @@ description: Verify an Open Pstack PR's exact head in isolated Claude Code and C ## Launch -This is a repository-local, non-shipped skill shared with Codex through `.agents/skills/verify-open-pstack`. Run from a trusted checkout on the operator's Mac. Never run a candidate's arbitrary scripts with GitHub credentials. Review this verifier before using it; it creates GitHub comments/statuses and can mark a draft ready. It never queues or merges. +This repository-local, non-shipped skill is shared with Codex through `.agents/skills/verify-open-pstack`. Run the reviewed verifier from a trusted checkout on the operator's Mac. GitHub publishing belongs exclusively to the trusted parent; never expose its credentials to candidate processes. The helper never queues or merges. ```sh (cd .claude/skills/verify-open-pstack && bun install --frozen-lockfile) -PR=123 # supplied delivery PR number +PR=111 # supplied delivery PR number +CLAUDE_ACCOUNT='selected-claude-account@example.com' # explicit caam vault identity +CODEX_ACCOUNT='selected-codex-account@example.com' # explicit caam vault identity SESSION="$(mktemp -d "${TMPDIR:-/tmp}/open-pstack-evidence.XXXXXX")" EVIDENCE="$SESSION/live" # must not exist yet .claude/skills/verify-open-pstack/scripts/verify.sh doctor --output "$SESSION/probe" -.claude/skills/verify-open-pstack/scripts/verify.sh run --pr "$PR" --self-test --output "$EVIDENCE" +.claude/skills/verify-open-pstack/scripts/verify.sh run --pr "$PR" --self-test \ + --claude-account "$CLAUDE_ACCOUNT" --codex-account "$CODEX_ACCOUNT" --output "$EVIDENCE" ``` -Omit `--self-test` for ordinary plugin verification; it is mandatory for this skill's own delivery. Use a new evidence directory for every run, even if the head is unchanged. A previous run cannot resume or transfer success. The helper pins the open, same-repository PR's 40-character head SHA, fetches that exact object, classifies old/new changed paths, and checks the head at every boundary and around publication. Unknown paths abort. +Replace both account placeholders with explicitly selected caam 0.1.22 vault identities; no default account or new authentication flow is allowed. Omit `--self-test` for ordinary plugin verification; it is mandatory for this skill's delivery. Use fresh evidence directories for every run. Pin the open, same-repository PR's exact head and base SHAs, classify changes from immutable Git objects, and recheck the PR at phase and publication boundaries. Unknown paths abort; previous evidence never transfers to a new head. ## Doctor -Requires Darwin, Bun, gh authentication, both harness CLIs, and operator access to every selected provider. `doctor` writes `doctor.json` outside the checkout. It checks the supported isolation flags but does not claim that installation or live behavior works. +The trusted-parent probe requires Darwin, Bun, gh authentication for publication, both harness CLIs, caam 0.1.22, and supported macOS `sandbox-exec` isolation. It records `doctor.json`; capability checks do not prove installation, authentication, sandbox enforcement, or live behavior. Missing capabilities fail closed. -The approved October 2 Mac probe (issue #90, comment 5961645184) reported Darwin, Claude Code 2.1.283, codex-cli 0.160.0, Bun 1.4.0, gh 2.97.0. Selected interfaces: Claude `--plugin-dir`, `--settings`, `--setting-sources`, `CLAUDE_CONFIG_DIR`; Codex local `plugin marketplace add`/`plugin add`, `CODEX_HOME`. Claude uses `--setting-sources project` so the pinned checkout's project skills/settings load while user settings stay excluded. Both harnesses preserve the operator's real `HOME`, `USER`, and `LOGNAME` for Keychain location/account identity; only `CLAUDE_CONFIG_DIR`/`CODEX_HOME` and `TMPDIR` point into run-owned state. Never substitute a temporary `HOME`: the operator observed a macOS Keychain reset dialog with that setup. Never create, reset, or unlock any keychain. On the operator's Claude Code 2.1.283, the temporary config directory reported `loggedIn:false` and used a config-scoped `Claude Code-credentials-` item distinct from the daily credentials; this measured isolation must be confirmed by the operator's fresh live run. Installation/provenance checks validate those choices during the actual run. Missing interfaces or authentication fail closed: never use a daily install. +Claude uses candidate `--plugin-dir`, empty `--settings`, `--setting-sources project`, and run-owned `CLAUDE_CONFIG_DIR`. Codex uses local marketplace/plugin installation and run-owned `CODEX_HOME`. Both sandboxed candidates use run-owned `HOME` and `TMPDIR`, retaining real `USER`/`LOGNAME` only for account identity. Both provider config variables must be isolated for cross-provider children. Do not link the real `Library` or Keychain into candidate state. Deny daily-home, GitHub/SSH/provider configuration, vault, and Keychain/securityd access with an OS-enforced boundary, not environment scrubbing alone. Never create, reset, unlock, or otherwise modify Keychain. + +The disposable credential contract replaces the earlier real-HOME/Keychain approach. The trusted parent validates each explicitly named vault identity and copies only `claude//.credentials.json` into `CLAUDE_CONFIG_DIR` and `codex//auth.json` into `CODEX_HOME`. It records the selected emails in the receipt, never modifies or activates the vault, and removes copied credential files in final cleanup. Candidates may use these disposable session credentials, but not publisher credentials or daily provider API keys. Missing credentials, identity mismatch, or quota failure stops the run; never log in or fall back to daily state. + +Operator-Mac handoff, from the trusted verifier checkout: + +```sh +PROOF="$(mktemp -d "${TMPDIR:-/tmp}/open-pstack-isolation-proof.XXXXXX")" +( + set -e -o pipefail + test "$(uname -s)" = Darwin + cd .claude/skills/verify-open-pstack + PSTACK_OPERATOR_SENTINELS=1 bun test --timeout 0 scripts/isolation.test.ts 2>&1 | + tee "$PROOF/isolation-proof.txt" +) +``` + +The explicit sentinel opt-in plants randomized, exclusive-create, non-secret files in the actual operator `HOME`, `~/.config/gh`, `~/.ssh`, `~/.claude`, `~/.codex`, and `~/Library/Keychains`; it never overwrites credential/SSH/Keychain files. Redirected directories fail closed. The trusted test parent removes only its own sentinel files and newly created empty directories in `finally`; if interrupted, inspect the printed cleanup-path list and remove only those named sentinel files, never whole daily directories. Require positive unsandboxed read controls and denied sandboxed reads/writes, aliases, and descendant reads; require positive unsandboxed Keychain-service lookup followed by denial of the same service. The real Mac test also proves pinned workspace/plugin write, unlink, rename, alias-write, and hardlink denial while the three approved dependency leaves remain writable. Preserve `$PROOF/isolation-proof.txt` and require both Mac tests to pass without skips. Linux skips and stubbed commands are not Mac enforcement proof. Then run Launch with fresh probe/live directories. Record that no Keychain dialog appeared and both native sessions made observable authenticated API requests; neither observation can be inferred from sentinel tests. Missing/pending adapter integration or failed proof blocks live verification. ## Drive -Read `features/README.md` and each selected feature document. Runtime markdown and consumed references are runtime. Shared paths select all consumers conservatively. `no runtime change` launches no harness unless the separate `--self-test` was requested. +Read `features/README.md` and every selected feature document. Runtime instructions, consumed references, and installed assets require live coverage. Shared paths select consumers conservatively. `no runtime change` launches no harness unless the separate self-test is requested. -The runner creates a detached exact-head candidate, separate run-owned config/state directories and fixture workspaces, and executable launchers. Process `HOME` remains the operator's real home; the run-owned paths referred to below as isolated homes are state roots, not replacements for `HOME`. Claude loads the candidate with its explicit plugin directory; Codex installs the candidate's local marketplace into its isolated home. The installed trees must match the candidate's content digest; a shared version string is insufficient. Fresh fixture sessions discover the canonical project skill and Codex link copied from this head. +Create detached exact-head candidate checkouts, run-owned homes/config roots and fixture workspaces. Claude loads the explicit candidate plugin; Codex installs its local marketplace. Verify installed plugin and canonical project-skill sources against immutable pinned Git provenance before and after exercise. Before native exercise, adapters must call `protectSources(profile, home, [workspace, installedPlugin])` and switch all candidate/descendant commands to the returned exercise policy outside candidate-writable state. That policy freezes the workspace and installed plugin, including ancestor rename/unlink, except `.claude/skills/verify-open-pstack/node_modules` and `plugins/pstack/skills/poteto-mode/scripts/node_modules` under the workspace and `skills/poteto-mode/scripts/node_modules` under the installed plugin. Reject source/ancestor symlinks rather than protecting a redirected path. Hash rechecks alone cannot prevent modify-execute-restore. Only those generated dependencies may be excluded; tracked source mutations or redirected skill aliases fail. Version strings alone are insufficient. -The operator authenticates only these isolated homes when needed, using the printed launcher with `auth login` (Claude) or `login` (Codex); do not copy daily auth/configuration. GitHub tokens and daily provider secrets are excluded from harness subprocess environments. A missing configured provider remains a feature failure. Never add a timeout or weaker-model fallback. +Every candidate command and external-provider descendant must remain inside the macOS sandbox, with both isolated provider configurations and no access to trusted-parent credentials. Setup exercises write only fixture run-owned homes, never literal daily user files. No candidate login, vault activation/writeback, implicit timeout, or weaker-model fallback is permitted. -For each selected feature in **each** harness, the runner opens a fresh native session with instructions from the feature map. Use a safe disposable fixture, exercise all changed sub-features, and observe actual tool calls and effects. Save and redact the native transcript and result artifacts outside the checkout. The runner asks for the surface, action, observed result, transcript path, and artifact paths, then requires explicit operator approval. Empty or missing evidence fails. A model's assertion that it passed does not count. Direct CLI proof is supplemental and must be labeled separately. +For every selected feature in both fresh native sessions, exercise changed sub-features and retain actual native tool calls and concrete fixture effects. Save redacted transcripts and artifacts outside isolated state. Require operator-reviewed surface, action, expected/observed result, and retained artifact paths. Model self-reports and direct CLI tests are not installed-harness evidence. -For `--self-test`, invoke this discovered project skill in both sessions and run `doctor` into a child evidence directory; save invocation traces and that output. Do not recursively run verification or publish from those sessions. This self-test cannot replace plugin-runtime exercises. +For `project-skill`, invoke `/verify-open-pstack` in Claude and `$verify-open-pstack` in Codex. Ask the discovered pinned skill to run `verify.sh doctor --candidate --output `. Candidate-mode doctor must not probe the vault or publisher authentication. Preserve the invocation and canonical-path doctor output; forbid recursive verification or publication. This does not replace required plugin exercises. ## Evidence -Retain `doctor.json`, `receipt.json`, installed provenance, redacted transcripts, artifacts, and cleanup outcome in `$EVIDENCE`. Artifact content is hashed before publication. The helper rechecks installation digests and artifact hashes after exercises. The PR comment includes `Live evidence:`, head SHA, installed version/location/tree, surface/action/observed result, artifact hashes, and failure or `no runtime change`. +Retain probe output, `receipt.json`, selected account emails, immutable source/installation provenance, redacted transcripts, artifacts, sandbox-denial proof, observable native API requests, and credential-cleanup outcome. Never retain copied credentials as evidence. Revalidate retained transcript/artifact hashes immediately before publication and around GitHub writes. Keep full detail in the receipt and bound the published summary. -Only complete evidence publishes `live-gate=success` on the pinned SHA, targeting the evidence-comment URL; only then may this run mark a draft ready. Failure records its reason and, where possible, publishes failure against that same SHA. A changed head aborts; detected post-write races withdraw old-head success and restore draft if this run made it ready. GitHub mutations are not atomic; failed compensation is recorded prominently. Never attach the old run to a new head. +The trusted parent publishes a pinned-SHA evidence comment and updates/reads back the PR template's `Live evidence:` block with installed version, surface, action, and observed result. Only complete evidence permits `live-gate=success` targeting that comment and subsequent draft-to-ready transition. Failure records its reason against the pinned SHA. Head movement aborts; revoke stale success on detected publication races and restore draft only if this run changed readiness. Preserve an already-ready PR on failure; record failed compensation prominently. -The builder opens this delivery as a draft and does **not** run publication. The Mac operator supplies live evidence. For this delivery, Mergify queues automatically once `verify`, `Unfret`, and `live-gate` pass and the PR is ready; no manual queue submission is required. If Mergify creates another head, rerun on it; never hand-post a replacement status or reuse an old receipt. Queue policy/recovery is outside this skill. +PR #111 is already ready and must not be re-drafted during this review round. Builders do not post the gate, queue, or merge. After Unfret passes, the operator performs fresh exact-head live proof. Mergify queues automatically once required checks pass and the PR is ready; no manual queue submission is required. A queue-created head needs a new run, never a hand-posted replacement status. ## Cleanup -The runner leaves all run-owned state and evidence intact; it does not delete daily state or kill unrelated processes. Quit spawned native sessions normally. After reviewing and archiving evidence, the operator may remove only this run's candidate/homes/workspaces. Record that action in the retained receipt's cleanup field; retain redacted transcripts and artifact hashes. No implicit cleanup or runtime timeout. +Final cleanup removes copied Claude `.credentials.json` and Codex `auth.json` files on success or failure, including disposable copies created for provider children. Record cleanup outcome without secrets; failed cleanup is a blocker requiring explicit remediation. Never modify the caam vault or daily files. Retain receipts, redacted transcripts, artifacts, and denial proof. Quit native sessions normally; after archiving evidence the operator may remove only named run-owned candidate/state/workspace paths. No unrelated process killing or implicit runtime timeout. ## Helpers -- `scripts/verify.sh doctor --output `: capability report, no installation/publication. -- `scripts/verify.sh run --pr [--self-test] --output `: supervised exact-head live verification and publication. Requires an interactive terminal for mapped exercises. -- `bun run test` and `bun run typecheck` in this skill directory: orchestration tests and strict types; not live proof. -- `features/registry.json`: maintained path ownership; unknown runtime paths block rather than guessing. +- `scripts/verify.sh doctor --output `: trusted-parent capability report, no installation/publication. +- `scripts/verify.sh doctor --candidate --output `: planned sandboxed self-test probe; no vault or publisher authentication probing. +- `scripts/verify.sh run --pr --claude-account --codex-account [--self-test] --output `: supervised exact-head verification; mapped exercises require an interactive terminal. +- `bun test scripts/isolation.test.ts` on the operator Mac: required real-filesystem and Keychain/securityd denial proof, not a substitute for live behavior. +- `bun run test` and `bun run typecheck`: independent helper tests and strict types, not live proof. +- `features/registry.json`: maintained path ownership; unknown runtime paths block. -Read receipts as data, never shell input. Do not put secrets in prompts/artifacts. All external commands use argument arrays; PR metadata is never evaluated as code. +Read receipts as data, never shell input. Do not put secrets in prompts or artifacts. diff --git a/.claude/skills/verify-open-pstack/features/README.md b/.claude/skills/verify-open-pstack/features/README.md index b9db7e1c..aaea8ba7 100644 --- a/.claude/skills/verify-open-pstack/features/README.md +++ b/.claude/skills/verify-open-pstack/features/README.md @@ -10,7 +10,9 @@ | `runner` | Installed parent invoking real configured provider lanes | | `tools` | Installed parent invoking the changed shipped CLI | | `project` | Native discovery/invocation of this verifier in both harnesses | -| `nonRuntime` | Explicit repository documentation/CI/assets/tests; no plugin harness launch | +| `nonRuntime` | Explicit repository documentation/CI/non-shipped assets/tests; no plugin harness launch | + +Shipped `plugins/pstack/assets/**` belongs to `shared`: even a logo-only change requires both installed harness surfaces, not the no-runtime lane. `bootstrap.ts` belongs to `tools`; exercise its actual consumers, `orch` and `watch-pr`, including dependency bootstrapping rather than the unrelated external runner. Skill-owned `SKILL.md`, `references/**`, and `playbooks/**` select their skill. Additional executable paths must be registered explicitly. Shared consumed mappings select every dependent skill conservatively. The coverage test inventories all tracked plugin files; new unmapped files block CI and runs. Deleted skill surfaces cannot silently disappear from evidence: choose a surviving documented entry point demonstrating the intended removal, or report blocked. diff --git a/.claude/skills/verify-open-pstack/features/isolation.md b/.claude/skills/verify-open-pstack/features/isolation.md index 21ac1f40..dc372a87 100644 --- a/.claude/skills/verify-open-pstack/features/isolation.md +++ b/.claude/skills/verify-open-pstack/features/isolation.md @@ -1,14 +1,40 @@ -# Isolation contract and approved operator probe +# Disposable credential isolation contract -The operator's read-only Mac probe is retained in issue #90 comment 5961645184; plan approval is comment 5961649164. On 2026-10-02 it reported Darwin, Claude Code 2.1.283, codex-cli 0.160.0, Bun 1.4.0, and gh 2.97.0. This is capability evidence, not proof of an installed candidate. +The October 2 probe (issue #90 comment 5961645184) reported Darwin, Claude Code 2.1.283, codex-cli 0.160.0, Bun 1.4.0, and gh 2.97.0. This is capability evidence, not candidate isolation proof. The October 3 operator-selected caam 0.1.22 contract (issue #112 comments 5968857062 and 5969996999) supersedes the real-HOME/Keychain login approach. -Selected interfaces: -- Claude: the real `HOME` and a fresh run-owned `CLAUDE_CONFIG_DIR`, candidate `--plugin-dir`, explicit empty `--settings`, and `--setting-sources project` to load the pinned checkout's project settings and skills while excluding user settings. The operator measured project-skill discovery failing with an empty setting source and succeeding with `project` on Claude Code 2.1.283. No daily plugin/configuration state is copied. -- Codex: the real `HOME` and a fresh run-owned `CODEX_HOME`; add the pinned checkout as a local marketplace with `codex plugin marketplace add --json`, then `codex plugin add pstack@open-pstack --json`. Start a new interactive task after installation. Do not use the daily profile. -- Both: a fresh candidate checkout, run-owned state and `TMPDIR`, minimal inherited environment, interactive login inside isolated config state, and fresh native sessions. The publisher's GitHub credentials and daily provider API keys are never passed to the candidate or either harness. Preserve the real `HOME`, `USER`, and `LOGNAME` so macOS can find the login Keychain and Claude uses the operator's account rather than `unknown`; never create, reset, or unlock any keychain. The operator measured config-scoped Claude OAuth items on 2.1.283 and observed `loggedIn:false` for a fresh config directory with real `HOME`; the new-head Mac run remains required to verify login and live behavior. Run-owned 'home' fields in receipts denote state roots, not process `HOME`. +## Trusted parent -The Codex JSON contract was checked against OpenAI's `codex-rs/cli/src/plugin_cmd.rs` at tag `rust-v0.160.0`: `JsonPluginAddOutput` uses camelCase `name`, `marketplaceName`, and `installedPath`; `JsonPluginListOutput.installed` contains `installed` and `enabled`. Unknown schemas fail closed rather than searching arbitrary caches for a plausible tree. +Require explicit `--claude-account EMAIL` and `--codex-account EMAIL`. Validate selected vault identities once per run; record their emails in receipts. Copy only `claude//.credentials.json` into run-owned `CLAUDE_CONFIG_DIR` and `codex//auth.json` into run-owned `CODEX_HOME`. Never activate accounts, modify/write back to the vault, copy daily configuration, or introduce login. Missing/mismatched credentials and quota exhaustion fail closed. GitHub authentication and publication remain exclusively in the trusted parent. -Phase 3 checks these CLI flags at runtime and verifies the installed plugin file tree against the pinned checkout. If either interface is absent or its installed tree cannot be established, stop. A help probe cannot substitute for actual installation/discovery. The build sandbox cannot supply the required Mac surface proof; delivery remains a draft for the operator's doctor and self-test. +## Candidate boundary -`doctor` records versions/help without installing or reading daily credentials. `run` uses the approved interfaces only after a passing doctor and an exact-head pin. Evidence remains outside the checkout. Cleanup is explicit operator removal of the named run-owned state after reviewing the receipt; never clean up a daily home. +- Use run-owned `HOME`, `TMPDIR`, `CLAUDE_CONFIG_DIR`, and `CODEX_HOME` for both parent harnesses and cross-provider descendants. Preserve real `USER`/`LOGNAME`; scrub publisher tokens and daily API credentials. +- Claude: candidate `--plugin-dir`, empty `--settings`, and `--setting-sources project`. This loads pinned project skills/settings while excluding user settings. +- Codex: pinned local marketplace/plugin installation, then a fresh native session. Validate installed identity, enabled state, contained path, and pinned source provenance; unknown schemas fail closed. +- Enforce candidate filesystem and credential isolation with macOS `sandbox-exec`, inherited by descendants. Deny daily home, `~/.config/gh`, `~/.ssh`, real `~/.claude`, real `~/.codex`, caam vault, and Keychain/securityd access. Do not link any real Library/Keychain path into run-owned state. Environment scrubbing alone is insufficient. +- Never invoke candidate login, create/reset/unlock Keychain, or use daily auth as fallback. Only copied disposable credentials may authenticate native API requests. + +Verify tracked plugin/project-skill provenance against immutable pinned Git objects before and after exercise. Before launching a native session, call `protectSources(profile, home, sources, run?)` with canonical workspace and installed-plugin roots, then use its returned exercise policy for every candidate command. The policy is an exclusive-create mode-0600 file beside the original policy, outside candidate-writable state. Source roots and all ancestors must be canonical directories, not symlinks. Source/plugin writes and ancestor rename/unlink are denied except workspace `.claude/skills/verify-open-pstack/node_modules`, workspace `plugins/pstack/skills/poteto-mode/scripts/node_modules`, and installed-plugin `skills/poteto-mode/scripts/node_modules`. No broader write allowance is added; the original daily-home/vault/Keychain denies remain in force. Generated directories are established before freezing their parents. Rehashes alone cannot prevent modify-execute-restore, so pending adapter integration is a blocker. Candidate self-test uses `doctor --candidate` without vault/publisher probing. + +## Operator-Mac proof + +From the trusted verifier checkout run: + +```sh +PROOF="$(mktemp -d "${TMPDIR:-/tmp}/open-pstack-isolation-proof.XXXXXX")" +( + set -e -o pipefail + test "$(uname -s)" = Darwin + cd .claude/skills/verify-open-pstack + PSTACK_OPERATOR_SENTINELS=1 bun test --timeout 0 scripts/isolation.test.ts 2>&1 | + tee "$PROOF/isolation-proof.txt" +) +``` + +The actual-HOME test requires explicit opt-in. It exclusively creates randomized `.pstack-denial-` non-secret files in the real daily home, `~/.config/gh`, `~/.ssh`, `~/.claude`, `~/.codex`, and `~/Library/Keychains`; no existing daily file or actual Keychain database is opened for writing. Noncanonical/symlinked daily directories fail before sentinel creation. It removes only its own files and directories it created that remain empty; cleanup paths are printed for safe recovery after interruption. Never recursively remove daily directories. + +Require both Mac tests to pass without skips. Positive unsandboxed reads prove sentinels exist; sandboxed direct reads, aliases, descendants, and writes must be denied and sentinel content preserved. Keychain Mach-service denial requires a reachable unsandboxed control for the same service; a missing service is not proof. Exercise-policy tests require protected source writes/unlinks/renames, ancestor moves, alias writes, hardlinks, and policy replacement to fail while all three generated dependency leaves remain writable. Record the emitted denial proof in `$PROOF/isolation-proof.txt` alongside the exact-head receipt. Linux skips or stubbed wrappers cannot establish Seatbelt enforcement. No Keychain dialog and authenticated API requests in both harnesses still require operator observation during native live sessions, not inference from these tests. Missing adapter integration or failed tests block delivery. After Unfret passes, use explicit-account Launch commands with fresh probe/live directories; leave PR #111 ready. + +## Cleanup + +Delete all copied session credential files in final cleanup on success or failure, including child-provider copies; record the outcome and block on cleanup failure. Keep redacted receipts/transcripts/artifacts and sentinel-denial proof. Never modify vault/daily files or delete evidence as credential cleanup. Subsequent operator removal is limited to named run-owned state. diff --git a/.claude/skills/verify-open-pstack/features/registry.json b/.claude/skills/verify-open-pstack/features/registry.json index d3cf2883..e987ca51 100644 --- a/.claude/skills/verify-open-pstack/features/registry.json +++ b/.claude/skills/verify-open-pstack/features/registry.json @@ -21,15 +21,15 @@ "shared": [ "plugins/pstack/.claude-plugin/plugin.json", "plugins/pstack/.codex-plugin/plugin.json", ".claude-plugin/marketplace.json", ".agents/plugins/marketplace.json", - "plugins/pstack/agents/**", "plugins/pstack/hooks/**", + "plugins/pstack/agents/**", "plugins/pstack/hooks/**", "plugins/pstack/assets/**", "plugins/pstack/skills/poteto-mode/references/**" ], "setup": ["plugins/pstack/skills/setup-pstack/SKILL.md"], "runner": [ - "plugins/pstack/skills/poteto-mode/scripts/runner/**", - "plugins/pstack/skills/poteto-mode/scripts/bootstrap.ts" + "plugins/pstack/skills/poteto-mode/scripts/runner/**" ], "tools": [ + "plugins/pstack/skills/poteto-mode/scripts/bootstrap.ts", "plugins/pstack/skills/poteto-mode/scripts/orch/**", "plugins/pstack/skills/poteto-mode/scripts/watch-pr/**", "plugins/pstack/skills/poteto-mode/scripts/check-plan.mjs", @@ -44,7 +44,7 @@ "README.md", "AGENTS.md", "CLAUDE.md", "CHANGES.md", "LICENSE", "LICENSE.md", "UPSTREAM.md", ".gitignore", ".mergify.yml", ".github/**", "docs/**", "assets/**", "tests/**", "scripts/**", - "plugins/pstack/assets/**", "plugins/pstack/skills/poteto-mode/scripts/bootstrap.test.ts", + "plugins/pstack/skills/poteto-mode/scripts/bootstrap.test.ts", "plugins/pstack/skills/poteto-mode/scripts/check-plan.test.ts" ] } diff --git a/.claude/skills/verify-open-pstack/features/runner.md b/.claude/skills/verify-open-pstack/features/runner.md index f2221c68..abb05929 100644 --- a/.claude/skills/verify-open-pstack/features/runner.md +++ b/.claude/skills/verify-open-pstack/features/runner.md @@ -14,4 +14,4 @@ Read the candidate's `pstack-runner --help` and installed mapping through the na ## Gotchas -A missing provider or unusable authentication is a failure, not a skip or weaker-model fallback. Do not add implicit timeouts. A model assertion that a child ran is insufficient; inspect the actual receipt and output. Never reuse output from another SHA or run. +Every external-provider child inherits the OS-enforced macOS sandbox, run-owned `HOME`/`TMPDIR`, and both run-owned `CLAUDE_CONFIG_DIR` and `CODEX_HOME`; never allow Claude→Codex or Codex→Claude to fall back to daily configuration. The trusted parent copies only the explicitly selected caam account credentials, records both emails, and deletes copied credentials in final cleanup. No candidate login, vault activation/writeback, Library/Keychain link, or Keychain/securityd access is allowed. Retain real-filesystem sentinel-denial evidence and observable native API requests for the dispatched lanes. A missing provider or unusable authentication is a failure, not a skip or weaker-model fallback. Do not add implicit timeouts. A model assertion that a child ran is insufficient; inspect the actual receipt and output. Never reuse output from another SHA or run. diff --git a/.claude/skills/verify-open-pstack/features/setup.md b/.claude/skills/verify-open-pstack/features/setup.md index 88c7db8c..514dafff 100644 --- a/.claude/skills/verify-open-pstack/features/setup.md +++ b/.claude/skills/verify-open-pstack/features/setup.md @@ -10,8 +10,8 @@ Invoke `/pstack:setup-pstack` in the isolated Claude session, or `$setup-pstack` ## Driving it with verify-open-pstack -Use only the run-owned fixture workspace and isolated harness home. Exercise each changed setup branch with explicit requested model/provider inputs. Retain the native invocation transcript and generated configuration file (redacted), then invoke a consuming installed skill and observe it use the configured route. Check that all paths written remain in run-owned state. Record requested versus observed provider/model and changed failure branches. Capture separate local CLI output only as supplemental evidence. +Use only the sandboxed run-owned fixture workspace and process `HOME`; both `CLAUDE_CONFIG_DIR` and `CODEX_HOME` must point into run-owned state. Literal `~/.claude` and `~/.codex` setup writes must resolve inside that fixture home, never the operator's daily home. Exercise each changed setup branch with explicit requested model/provider inputs. Retain the native invocation transcript and generated configuration file (redacted), then invoke a consuming installed skill and observe it use the configured route. Check that all paths written remain in run-owned state. Record requested versus observed provider/model and changed failure branches. Capture separate local CLI output only as supplemental evidence. ## Gotchas -Do not copy a daily configuration to make setup pass. Stop if required provider authentication is unavailable. Generated configuration by itself does not prove the consuming surface. Credentials stay outside repository-controlled scripts; authenticate the isolated harness explicitly and redact secrets before selecting artifacts. +Do not copy a daily configuration to make setup pass. Stop if required provider authentication is unavailable. Generated configuration by itself does not prove the consuming surface. The trusted parent requires explicit caam 0.1.22 account emails, validates vault identities, and copies only disposable Claude/Codex credential files into run-owned config roots. No candidate login or vault activation/writeback is allowed. Both sessions and provider children remain sandboxed against daily home/GitHub/SSH/provider files and Keychain/securityd, with no real Library/Keychain link. Retain sentinel-denial proof and observable native API requests; delete copied credentials in final cleanup while preserving redacted evidence. diff --git a/.claude/skills/verify-open-pstack/features/shipped-tools.md b/.claude/skills/verify-open-pstack/features/shipped-tools.md index c709f51b..41f00f83 100644 --- a/.claude/skills/verify-open-pstack/features/shipped-tools.md +++ b/.claude/skills/verify-open-pstack/features/shipped-tools.md @@ -14,4 +14,4 @@ Use the candidate's installed help and documented arguments. Create fixture stat ## Gotchas -A mock test is not live proof. Never run queue/merge actions or edit daily workflow state as a fixture. Do not infer a successful remote read from an empty response. Missing required access fails the feature; preserve its reason and draft status. +A mock test is not live proof. Never run queue/merge actions or edit daily workflow state as a fixture. Do not infer a successful remote read from an empty response. Candidate tools and descendants stay inside the macOS sandbox with run-owned `HOME`, both isolated provider configurations, and disposable explicitly selected caam session credentials. They cannot read daily home/GitHub/SSH/provider files or access Keychain/securityd; never pass publisher credentials to make a remote read succeed. GitHub evidence/status publication belongs only to the trusted parent. Missing required access fails the feature; preserve its reason without re-drafting an already-ready PR, including PR #111. diff --git a/.claude/skills/verify-open-pstack/features/skill-invocation.md b/.claude/skills/verify-open-pstack/features/skill-invocation.md index dc7dbab5..272b944c 100644 --- a/.claude/skills/verify-open-pstack/features/skill-invocation.md +++ b/.claude/skills/verify-open-pstack/features/skill-invocation.md @@ -10,9 +10,9 @@ In the fresh isolated Claude session invoke `/pstack:`. In Codex use the i ## Driving it with verify-open-pstack -Launch the prepared harness with its recorded launcher. For each selected name, use a disposable fixture appropriate to its documented input, invoke the installed surface, and exercise every changed sub-feature. Observe native tool/skill loading and a concrete fixture effect, not the assistant saying it passed. Save the transcript and an artifact showing the actual result (diff, generated file, tool receipt, or UI capture). Include input, expected result, observed result, and each changed sub-feature in the operator review. +Launch the prepared harness with its sandboxed recorded launcher: run-owned `HOME`/`TMPDIR`, both isolated provider config roots, and real `USER`/`LOGNAME`. Use only trusted-parent copies of the explicitly selected caam account credentials; no login, daily-state fallback, vault mutation, or Library/Keychain link. Deny daily-home/GitHub/SSH/provider and Keychain/securityd access for descendants too, and retain operator-Mac sentinel-denial proof plus observable native API requests. Final cleanup deletes copied credentials, not evidence. For each selected name, use a disposable fixture appropriate to its documented input, invoke the installed surface, and exercise every changed sub-feature. Observe native tool/skill loading and a concrete fixture effect, not the assistant saying it passed. Save the transcript and an artifact showing the actual result (diff, generated file, tool receipt, or UI capture). Include input, expected result, observed result, and each changed sub-feature in the operator review. -For `project-skill`, invoke the discovered skill and ask it to run `verify.sh doctor` into a new child directory of this run's evidence root; forbid recursive `run`/publication. Preserve the native invocation transcript plus the resulting `doctor.json`, checking the executable path resolves to the pinned project skill. The Codex symlink and Claude canonical directory must both be discovered in fresh sessions. +For `project-skill`, invoke the discovered skill and ask it to run planned `verify.sh doctor --candidate --output `; candidate mode must not probe the caam vault or publisher authentication. Forbid recursive `run`/publication. Preserve the native invocation transcript plus `doctor.json`, checking the canonical executable path and immutable source provenance belong to the pinned project skill. Revalidate source after exercise, excluding only explicitly permitted generated dependencies. The Codex symlink and Claude canonical directory must both be discovered in fresh sessions. If candidate mode is unavailable, stop; do not substitute trusted-parent doctor. ## Gotchas diff --git a/.claude/skills/verify-open-pstack/scripts/cli.test.ts b/.claude/skills/verify-open-pstack/scripts/cli.test.ts index 617fdf98..21b0409f 100644 --- a/.claude/skills/verify-open-pstack/scripts/cli.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/cli.test.ts @@ -1,7 +1,66 @@ import { expect, test } from 'bun:test'; import { parse } from './cli.ts'; +import { mkdtemp, readFile, realpath, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { doctor } from './doctor.ts'; +import type { Command } from './io.ts'; +const accounts = ['--claude-account', 'eric@litman.org', '--codex-account', 'eric@healthspanners.com']; +test('candidate mode is an explicit doctor-only flag', () => { + expect(parse(['doctor', '--candidate', '--output', '/fresh/probe'])).toEqual({ mode: 'doctor', output: '/fresh/probe', pr: 0, selfTest: false, candidate: true }); + expect(() => parse(['doctor', '--candidate', '--candidate', '--output', '/fresh/probe'])).toThrow('Duplicate option'); + expect(() => parse(['run', '--candidate', '--pr', '111', '--output', '/fresh/run', ...accounts])).toThrow('Unknown option'); +}); +test('candidate doctor probes only help and versions with private roots already present', async () => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-doctor-'))), calls: string[][] = []; + const run: Command = async (args, options = {}) => { + calls.push(args); + const env = options.env!; + for (const key of ['HOME', 'TMPDIR', 'CLAUDE_CONFIG_DIR', 'CODEX_HOME', 'XDG_CONFIG_HOME', 'XDG_CACHE_HOME', 'GH_CONFIG_DIR']) { + expect(env[key]!.startsWith(root + '/probe-home')).toBe(true); + const info = await stat(env[key]!); + expect(info.isDirectory()).toBe(true); + expect(info.mode & 0o777).toBe(0o700); + } + return args.includes('--version') ? 'version' : '--plugin-dir --settings --setting-sources --json local path'; + }; + try { + await doctor(root, run, 'darwin', true); + const report = JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')); + expect(report.candidate).toBe(true); expect(report.result).toBe('pass'); + expect(Object.keys(report.checks)).toEqual(['bun', 'git', 'claude', 'codex', 'claudeHelp', 'codex plugin marketplace add --help', 'codex plugin add --help']); + expect(calls.every(args => args.includes('--version') || args.includes('--help'))).toBe(true); + expect(calls.some(args => ['gh', 'caam', '/usr/bin/sandbox-exec'].includes(args[0]!))).toBe(false); + expect(calls.some(args => args.includes('auth') || args.includes('login'))).toBe(false); + await doctor(root, run, 'darwin'); + expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).candidate).toBe(false); + expect(calls.some(args => args[0] === 'gh' && args[1] === '--version')).toBe(true); + } finally { await rm(root, { recursive: true, force: true }); } +}); test('CLI strictly validates run inputs and permits read-only child doctor', () => { - expect(parse(['doctor', '--output', '/fresh/probe']).mode).toBe('doctor'); - expect(parse(['run', '--pr', '123', '--self-test', '--output', '/fresh/run'])).toEqual({ mode: 'run', output: '/fresh/run', pr: 123, selfTest: true }); - for (const args of [[], ['wat'], ['run', '--pr', '0', '--output', '/tmp/a'], ['run', '--pr', '1.5', '--output', '/tmp/a'], ['doctor', '--pr', '1', '--output', '/tmp/a'], ['doctor', '--output', '/tmp/a', '--output', '/tmp/b'], ['run', '--pr', '1'], ['doctor', '--output', '--self-test'], ['doctor', '--output', '/tmp/a', '--publish']]) expect(() => parse(args)).toThrow(); + expect(parse(['doctor', '--output', '/fresh/probe'])).toEqual({ mode: 'doctor', output: '/fresh/probe', pr: 0, selfTest: false, candidate: false }); + expect(parse(['run', '--pr', '123', '--self-test', '--output', '/fresh/run', ...accounts])).toEqual({ mode: 'run', output: '/fresh/run', pr: 123, selfTest: true, claudeAccount: 'eric@litman.org', codexAccount: 'eric@healthspanners.com' }); + for (const args of [[], ['wat'], ['run', '--pr', '0', '--output', '/tmp/a', ...accounts], ['run', '--pr', '1.5', '--output', '/tmp/a', ...accounts], ['doctor', '--pr', '1', '--output', '/tmp/a'], ['doctor', '--output', '/tmp/a', '--output', '/tmp/b'], ['run', '--pr', '1', ...accounts], ['doctor', '--output', '--self-test'], ['doctor', '--output', '/tmp/a', '--publish']]) expect(() => parse(args)).toThrow(); +}); +test('run requires both explicit account choices without defaults or doctor account options', () => { + const run = ['run', '--pr', '111', '--output', '/fresh/run']; + expect(() => parse(run)).toThrow('Run requires --claude-account and --codex-account'); + expect(() => parse([...run, ...accounts.slice(0, 2)])).toThrow('Run requires'); + expect(() => parse([...run, ...accounts.slice(2)])).toThrow('Run requires'); + for (const option of ['--claude-account', '--codex-account']) { + expect(() => parse(['doctor', '--output', '/fresh/probe', option, 'eric@litman.org'])).toThrow('Unknown option'); + expect(() => parse([...run, ...accounts, option, 'eric@mobilyze.com'])).toThrow('Duplicate option'); + expect(() => parse([...run, option])).toThrow('Missing value'); + expect(() => parse([...run, option, '--self-test'])).toThrow('Missing value'); + } +}); +test('account choices are bounded emails rather than vault paths or arbitrary strings', () => { + for (const option of ['--claude-account', '--codex-account']) { + const other = option === '--claude-account' ? '--codex-account' : '--claude-account'; + const args = ['run', '--pr', '111', '--output', '/fresh/run', other, 'eric@litman.org', option]; + for (const account of ['../eric@litman.org', 'claude/eric@litman.org', 'eric\\@litman.org', 'eric@../litman.org', 'eric@litman..org', 'eric', ' eric@litman.org', 'eric@litman.org\n', '@litman.org', 'eric@litman', 'x'.repeat(65) + '@litman.org', 'x@' + 'a.'.repeat(127) + 'org']) expect(() => parse([...args, account])).toThrow('Account must be a safe email'); + const result = parse([...args, 'eric+verification@litman.org']); + expect(result.mode).toBe('run'); + if (result.mode === 'run') expect(option === '--claude-account' ? result.claudeAccount : result.codexAccount).toBe('eric+verification@litman.org'); + } }); diff --git a/.claude/skills/verify-open-pstack/scripts/cli.ts b/.claude/skills/verify-open-pstack/scripts/cli.ts index 3ca1615c..c573942d 100644 --- a/.claude/skills/verify-open-pstack/scripts/cli.ts +++ b/.claude/skills/verify-open-pstack/scripts/cli.ts @@ -4,42 +4,65 @@ import { validateRegistry } from './core.ts'; import { doctor } from './doctor.ts'; import { Publisher } from './github.ts'; import { MacDriver } from './harness.ts'; -import { freshRoot, redact, save } from './io.ts'; +import { freshRoot, interruption, interruptCommands, redact, save } from './io.ts'; import { verify } from './verify.ts'; -export function parse(args: string[]): { mode: 'doctor' | 'run'; output: string; pr: number; selfTest: boolean } { +type Options = { mode: 'doctor'; output: string; pr: number; selfTest: boolean; candidate: boolean } + | { mode: 'run'; output: string; pr: number; selfTest: boolean; claudeAccount: string; codexAccount: string }; +export function parse(args: string[]): Options { const mode = args[0]; - if (mode !== 'doctor' && mode !== 'run') throw new Error('Usage: verify.sh doctor --output /fresh/path | run --pr NUMBER [--self-test] --output /fresh/path'); - let output = '', pr = 0, selfTest = false; + if (mode !== 'doctor' && mode !== 'run') throw new Error('Usage: verify.sh doctor [--candidate] --output /fresh/path | run --pr NUMBER --claude-account EMAIL --codex-account EMAIL [--self-test] --output /fresh/path'); + let output = '', pr = 0, selfTest = false, candidate = false, claudeAccount = '', codexAccount = ''; const seen = new Set(); for (let i = 1; i < args.length; i++) { const arg = args[i]!; if (seen.has(arg)) throw new Error(`Duplicate option: ${arg}`); seen.add(arg); if (arg === '--self-test' && mode === 'run') selfTest = true; - else if (arg === '--output' || arg === '--pr' && mode === 'run') { + else if (arg === '--candidate' && mode === 'doctor') candidate = true; + else if (arg === '--output' || mode === 'run' && ['--pr', '--claude-account', '--codex-account'].includes(arg)) { const value = args[++i]; if (!value || value.startsWith('--')) throw new Error(`Missing value: ${arg}`); if (arg === '--output') output = value; - else { if (!/^[1-9]\d*$/.test(value) || !Number.isSafeInteger(Number(value))) throw new Error('PR must be a positive integer'); pr = Number(value); } + else if (arg === '--pr') { if (!/^[1-9]\d*$/.test(value) || !Number.isSafeInteger(Number(value))) throw new Error('PR must be a positive integer'); pr = Number(value); } + else { + if (value.length > 254 || value.split('@')[0]!.length > 64 || value.includes('..') || !/^[A-Za-z0-9][A-Za-z0-9._+-]*@[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?)+$/.test(value)) throw new Error(`Account must be a safe email: ${arg}`); + if (arg === '--claude-account') claudeAccount = value; else codexAccount = value; + } } else throw new Error(`Unknown option: ${arg}`); } if (!output || mode === 'run' && !pr) throw new Error('Output and (for run) PR are required'); - return { mode, output, pr, selfTest }; + if (mode === 'run') { + if (!claudeAccount || !codexAccount) throw new Error('Run requires --claude-account and --codex-account'); + return { mode, output, pr, selfTest, claudeAccount, codexAccount }; + } + return { mode, output, pr, selfTest, candidate }; } async function main(): Promise { - let root: string | undefined; + let root: string | undefined, interrupted: 'SIGINT' | 'SIGTERM' | undefined; + const onInterrupt = (signal: 'SIGINT' | 'SIGTERM'): void => { + interrupted ??= signal; + process.exitCode = interrupted === 'SIGINT' ? 130 : 143; + void interruptCommands(signal).catch(error => { + console.error(redact(String(error))); process.exitCode = 1; + }); + }; + const onInt = (): void => onInterrupt('SIGINT'), onTerm = (): void => onInterrupt('SIGTERM'); + process.on('SIGINT', onInt); process.on('SIGTERM', onTerm); try { const options = parse(process.argv.slice(2)), repository = resolve(import.meta.dir, '../../../..'); root = await freshRoot(options.output, repository); - if (options.mode === 'doctor') await doctor(root); + if (options.mode === 'doctor') await doctor(root, undefined, undefined, options.candidate); else { const registry = validateRegistry(JSON.parse(await readFile(join(import.meta.dir, '../features/registry.json'), 'utf8'))); - const receipt = await verify({ pr: options.pr, selfTest: options.selfTest, root, registry, github: new Publisher(), driver: new MacDriver(), persist: r => save(join(root!, 'receipt.json'), r) }); + const receipt = await verify({ pr: options.pr, selfTest: options.selfTest, root, registry, github: new Publisher(), driver: new MacDriver(undefined, undefined, { claude: options.claudeAccount, codex: options.codexAccount }), persist: r => save(join(root!, 'receipt.json'), r) }); console.log(`Pinned ${receipt.sha}: live-gate=${receipt.status}; evidence ${receipt.commentUrl}`); } + interruption.signal.throwIfAborted(); console.log(`Retained evidence: ${root}`); } catch (error) { const failure = redact(error instanceof Error ? error.message : String(error)); if (root) await save(join(root, 'failure.json'), { result: 'failed', reason: failure }); - console.error(failure); process.exitCode = 1; + console.error(failure); process.exitCode = interrupted === 'SIGINT' ? 130 : interrupted === 'SIGTERM' ? 143 : 1; + } finally { + process.off('SIGINT', onInt); process.off('SIGTERM', onTerm); } } if (import.meta.main) await main(); diff --git a/.claude/skills/verify-open-pstack/scripts/core.test.ts b/.claude/skills/verify-open-pstack/scripts/core.test.ts index 88700b96..60c7c37d 100644 --- a/.claude/skills/verify-open-pstack/scripts/core.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/core.test.ts @@ -52,6 +52,18 @@ describe('registry and ownership', () => { expect(s.skills).toEqual([...registry.skills].sort()); for (const feature of ['setup', 'runner', 'shipped-tools']) expect(s.features).toContain(feature); }); + test('asset-only installed logo changes require real harness surfaces', () => { + const s = classify([{ filename: 'plugins/pstack/assets/logo.png' }], registry); + expect(s.noRuntime).toBe(false); + expect(s.skills).toEqual([...registry.skills].sort()); + for (const feature of ['setup', 'runner', 'shipped-tools']) expect(s.features).toContain(feature); + }); + test('bootstrap changes require its shipped tool consumers, not the unrelated runner', () => { + const s = classify([{ filename: 'plugins/pstack/skills/poteto-mode/scripts/bootstrap.ts' }], registry); + expect(s.noRuntime).toBe(false); + expect(s.features).toEqual(['shipped-tools']); + expect(s.features).not.toContain('runner'); + }); test('rename and deletion preserve old runtime ownership', () => { const s = classify([{ filename: 'docs/old.md', previous_filename: 'plugins/pstack/skills/architect/SKILL.md' }], registry); expect(s.features).toContain('skill-invocation:architect'); diff --git a/.claude/skills/verify-open-pstack/scripts/doctor.ts b/.claude/skills/verify-open-pstack/scripts/doctor.ts index 46b2a038..d82c28dc 100644 --- a/.claude/skills/verify-open-pstack/scripts/doctor.ts +++ b/.claude/skills/verify-open-pstack/scripts/doctor.ts @@ -2,15 +2,15 @@ import { mkdir, realpath } from 'node:fs/promises'; import { join } from 'node:path'; import { command, isolatedEnv, save, type Command } from './io.ts'; -export async function doctor(root: string, run: Command = command, platform = process.platform): Promise { - const report: Record = { platform, date: new Date().toISOString(), skill: await realpath(join(import.meta.dir, '..')), checks: {} }; +export async function doctor(root: string, run: Command = command, platform = process.platform, candidate = false): Promise { + const report: Record = { platform, candidate, date: new Date().toISOString(), skill: await realpath(join(import.meta.dir, '..')), checks: {} }; const checks = report.checks as Record; try { if (platform !== 'darwin') throw new Error('Live gate requires an operator Mac (Darwin)'); const home = join(root, 'probe-home'); - await mkdir(join(home, 'tmp'), { recursive: true, mode: 0o700 }); + for (const dir of ['tmp', '.claude', '.codex', '.config/gh', '.cache']) await mkdir(join(home, dir), { recursive: true, mode: 0o700 }); const env = isolatedEnv(home); - for (const binary of ['bun', 'git', 'claude', 'codex', 'gh']) checks[binary] = (await run([binary, '--version'], { env })).trim(); + for (const binary of candidate ? ['bun', 'git', 'claude', 'codex'] : ['bun', 'git', 'claude', 'codex', 'gh']) checks[binary] = (await run([binary, '--version'], { env })).trim(); const claude = await run(['claude', '--help'], { env }); for (const flag of ['--plugin-dir', '--settings', '--setting-sources']) if (!claude.includes(flag)) throw new Error(`Claude isolation missing ${flag}`); checks.claudeHelp = claude; diff --git a/.claude/skills/verify-open-pstack/scripts/github.ts b/.claude/skills/verify-open-pstack/scripts/github.ts index 8c57389c..abff9d43 100644 --- a/.claude/skills/verify-open-pstack/scripts/github.ts +++ b/.claude/skills/verify-open-pstack/scripts/github.ts @@ -1,27 +1,50 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { command, type Command } from './io.ts'; import { REPO, type ChangedFile, type GitHub, type Pull } from './types.ts'; +export function parseChangedFiles(output: string): ChangedFile[] { + if (output === '') return []; + if (!output.endsWith('\0')) throw new Error('Truncated changed-file diff'); + const fields = output.slice(0, -1).split('\0'), files: ChangedFile[] = []; + for (let i = 0; i < fields.length;) { + const status = fields[i++]; + if (!/^(?:[AMDT]|[RC]\d{1,3})$/.test(status)) throw new Error('Invalid changed-file status'); + const previous = /^[RC]/.test(status) ? fields[i++] : undefined, filename = fields[i++]; + if (!filename || previous === '') throw new Error('Incomplete changed-file diff'); + files.push({ filename, ...(previous !== undefined ? { previous_filename: previous } : {}) }); + } + if (new Set(files.map(f => f.filename)).size !== files.length) throw new Error('Duplicate changed-file diff'); + return files; +} export class Publisher implements GitHub { constructor(private run: Command = command) {} private async api(path: string, args: string[] = []): Promise { return JSON.parse(await this.run(['gh', 'api', `repos/${REPO}/${path}`, ...args])); } async pull(pr: number): Promise { - const data = await this.api(`pulls/${pr}`) as { number: number; head: { sha: string; repo?: { full_name: string } }; base: { sha: string }; draft: boolean; state: string }; - if (data.number !== pr || typeof data.draft !== 'boolean' || typeof data.state !== 'string') throw new Error('Invalid PR response'); - return { number: pr, head: { sha: data.head.sha }, base: { sha: data.base.sha }, draft: data.draft, state: data.state, headRepo: data.head.repo?.full_name ?? '' }; + const data = await this.api(`pulls/${pr}`) as { number: number; head: { sha: string; repo?: { full_name: string } }; base: { sha: string }; draft: boolean; state: string; body: string | null }; + if (data.number !== pr || typeof data.draft !== 'boolean' || typeof data.state !== 'string' || data.body !== null && typeof data.body !== 'string') throw new Error('Invalid PR response'); + return { number: pr, head: { sha: data.head.sha }, base: { sha: data.base.sha }, draft: data.draft, state: data.state, headRepo: data.head.repo?.full_name ?? '', body: data.body ?? '' }; + } + async files(base: string, head: string): Promise { + if (![base, head].every(sha => /^[a-f0-9]{40}$/.test(sha))) throw new Error('Invalid diff SHA'); + const repository = await mkdtemp(join(tmpdir(), 'open-pstack-diff-')); + const env = { PATH: process.env.PATH ?? '', GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_SYSTEM: '/dev/null', GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_COUNT: '0', GIT_TERMINAL_PROMPT: '0' }; + try { + await this.run(['git', 'init', '--bare', '--template=', repository], { env }); + await this.run(['git', 'fetch', '--no-tags', '--', `https://github.com/${REPO}.git`, base, head], { cwd: repository, env }); + for (const sha of [base, head]) { + if ((await this.run(['git', 'rev-parse', '--verify', `${sha}^{commit}`], { cwd: repository, env })).trim() !== sha) throw new Error('Pinned diff commit unavailable'); + } + const output = await this.run(['git', 'diff', '--no-ext-diff', '--no-textconv', '--name-status', '-z', '--find-renames', `${base}...${head}`, '--'], { cwd: repository, env }); + return parseChangedFiles(output); + } finally { await rm(repository, { recursive: true, force: true }); } } - async files(pr: number): Promise { - const metadata = await this.api(`pulls/${pr}`) as { changed_files: number }; - if (!Number.isInteger(metadata.changed_files) || metadata.changed_files > 3000) throw new Error('PR file count cannot be completely enumerated'); - const pages = await this.api(`pulls/${pr}/files?per_page=100`, ['--paginate', '--slurp']); - if (!Array.isArray(pages) || !pages.every(Array.isArray)) throw new Error('Invalid file pagination'); - const files: ChangedFile[] = pages.flat().map((file: { filename: string; previous_filename?: string }) => { - if (typeof file.filename !== 'string' || (file.previous_filename !== undefined && typeof file.previous_filename !== 'string')) throw new Error('Invalid changed file'); - return { filename: file.filename, ...(file.previous_filename ? { previous_filename: file.previous_filename } : {}) }; - }); - if (files.length !== metadata.changed_files || new Set(files.map(f => f.filename)).size !== files.length) throw new Error('Incomplete or duplicate changed-file response'); - return files; + async body(pr: number, body: string): Promise { + if (body.length > 60000) throw new Error('PR body exceeds evidence budget'); + await this.api(`pulls/${pr}`, ['--method', 'PATCH', '-f', `body=${body}`]); } async comment(pr: number, body: string): Promise { if (body.length > 60000) throw new Error('Evidence exceeds comment budget'); diff --git a/.claude/skills/verify-open-pstack/scripts/harness.test.ts b/.claude/skills/verify-open-pstack/scripts/harness.test.ts index aa51ac41..18a19c7b 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.test.ts @@ -5,9 +5,9 @@ import { join } from 'node:path'; import { doctor } from './doctor.ts'; import { codexInstallation, launch, MacDriver, verifyCodexEnabled, verifyProjectDoctor } from './harness.ts'; import { newReceipt } from './core.ts'; -import { freshRoot, isolatedEnv, redact, retainedFile, treeHash, type Command } from './io.ts'; +import { command, freshRoot, isolatedEnv, redact, retainedFile, treeHash, type Command } from './io.ts'; const roots: string[] = []; -async function fixture(): Promise { const root = await mkdtemp(join(tmpdir(), 'pstack-test-')); roots.push(root); return root; } +async function fixture(): Promise { const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-test-'))); roots.push(root); return root; } afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); describe('isolated harness boundaries', () => { @@ -21,9 +21,11 @@ describe('isolated harness boundaries', () => { const env = isolatedEnv('/run/home', harness); expect(env.GH_TOKEN).toBeUndefined(); expect(env.GITHUB_TOKEN).toBeUndefined(); expect(env.ANTHROPIC_API_KEY).toBeUndefined(); expect(env.OPENAI_API_KEY).toBeUndefined(); - expect(process.env.HOME).toBe(env.HOME); expect(env.GIT_CONFIG_GLOBAL).toBe('/dev/null'); + expect(env.HOME).toBe('/run/home'); expect(env.GIT_CONFIG_GLOBAL).toBe('/dev/null'); expect(env.TMPDIR).toBe('/run/home/tmp'); - expect(env[harness === 'claude' ? 'CLAUDE_CONFIG_DIR' : 'CODEX_HOME']).toBe('/run/home/config'); + expect(env.CLAUDE_CONFIG_DIR).toBe('/run/home/.claude'); + expect(env.CODEX_HOME).toBe('/run/home/.codex'); + expect(env.GH_CONFIG_DIR).toBe('/run/home/.config/gh'); expect(JSON.stringify(env)).not.toContain('/daily/'); } expect(launch('claude', '/run/home', '/candidate')).toEqual(['claude', '--plugin-dir', '/candidate/plugins/pstack', '--settings', '/run/home/settings.json', '--setting-sources', 'project']); @@ -41,7 +43,7 @@ describe('isolated harness boundaries', () => { for (const harness of ['claude', 'codex'] as const) { const env = isolatedEnv('/run/home', harness); expect(env.USER).toBe('operator-user'); expect(env.LOGNAME).toBe('operator-login'); - expect(process.env.HOME).toBe(env.HOME); + expect(env.HOME).toBe('/run/home'); } delete process.env.USER; delete process.env.LOGNAME; const env = isolatedEnv('/run/home', 'claude'); @@ -74,6 +76,9 @@ describe('isolated harness boundaries', () => { const alias = join(root, 'workspace-alias'); await symlink(workspace, alias); const run: Command = async args => args.includes('--version') ? 'version' : '--plugin-dir --settings --setting-sources --json local path'; await doctor(root, run, 'darwin'); + const parentText = await readFile(join(root, 'doctor.json'), 'utf8'); + await expect(verifyProjectDoctor([parentText], workspace)).rejects.toThrow('passing child doctor'); + await doctor(root, run, 'darwin', true); const text = await readFile(join(root, 'doctor.json'), 'utf8'); expect(JSON.parse(text).skill).toBe(skill); await verifyProjectDoctor(['not JSON', text], workspace); @@ -84,63 +89,130 @@ describe('isolated harness boundaries', () => { await expect(verifyProjectDoctor(['{}'], workspace)).rejects.toThrow('passing child doctor'); await expect(verifyProjectDoctor([JSON.stringify({ ...JSON.parse(text), result: 'blocked' })], workspace)).rejects.toThrow('passing child doctor'); }); - test('prepare creates private configuration homes before any harness command', async () => { - const root = await fixture(), sha = 'a'.repeat(40), calls: string[][] = []; - const platform = Object.getOwnPropertyDescriptor(process, 'platform')!; - const run: Command = async (args, options = {}) => { + test.each([false, true])('prepare uses real pinned files, disposable accounts and cleanup (install failure=%s)', async failInstall => { + const root = await fixture(), repository = join(root, 'repository'), calls: string[][] = []; + const accounts = { claude: 'claude@example.com', codex: 'codex@example.com' }; + const caam = join(root, 'caam'), vault = join(caam, 'data/vault'); + const credentials = { claude: JSON.stringify({ claudeAiOauth: { accessToken: 'fixture-claude' } }), codex: JSON.stringify({ tokens: { access_token: 'fixture-codex' } }) }; + for (const tool of ['claude', 'codex'] as const) { + const dir = join(vault, tool, accounts[tool]); await mkdir(dir, { recursive: true }); + await writeFile(join(dir, tool === 'claude' ? '.credentials.json' : 'auth.json'), credentials[tool]); + } + for (const manifest of ['.claude-plugin', '.codex-plugin']) { + const dir = join(repository, 'plugins/pstack', manifest); await mkdir(dir, { recursive: true }); + await writeFile(join(dir, 'plugin.json'), JSON.stringify({ version: 'test' })); + } + await mkdir(join(repository, '.claude/skills/verify-open-pstack'), { recursive: true }); + await writeFile(join(repository, '.claude/skills/verify-open-pstack/SKILL.md'), 'pinned project skill'); + await mkdir(join(repository, '.agents/skills'), { recursive: true }); + await symlink('../../.claude/skills/verify-open-pstack', join(repository, '.agents/skills/verify-open-pstack')); + await command(['git', 'init', repository]); + await command(['git', 'add', '.'], { cwd: repository }); + await command(['git', '-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.com', '-c', 'commit.gpgsign=false', 'commit', '-m', 'fixture'], { cwd: repository }); + const sha = (await command(['git', 'rev-parse', 'HEAD'], { cwd: repository })).trim(); + const platform = Object.getOwnPropertyDescriptor(process, 'platform')!, previousCaam = process.env.CAAM_HOME; + const run: Command = async (wrapped, options = {}) => { + const args = wrapped[0] === '/usr/bin/sandbox-exec' ? wrapped.slice(3) : wrapped; calls.push(args); - const env = options.env!; - const config = env.CODEX_HOME ?? env.CLAUDE_CONFIG_DIR; - expect(process.env.HOME).toBe(env.HOME); - expect((await stat(env.HOME!)).isDirectory()).toBe(true); - for (const path of [env.TMPDIR!]) { - const info = await stat(path); - expect(info.isDirectory()).toBe(true); - expect(info.mode & 0o777).toBe(0o700); + if (args[0] === 'caam') { + expect(wrapped.slice(0, 2)).toEqual(['/usr/bin/sandbox-exec', '-p']); + expect(wrapped[2]).toContain('(deny file-write*)'); + expect(options.env!.CAAM_HOME).toBe(caam); + return JSON.stringify({ profiles: ['claude', 'codex'].map(tool => ({ tool, name: accounts[tool as keyof typeof accounts], identity: { email: accounts[tool as keyof typeof accounts] } })) }); } - if (options.cwd) expect((await stat(options.cwd)).isDirectory()).toBe(true); - if (args[0] === 'claude' && env.CLAUDE_CONFIG_DIR) { - const settings = join(env.CLAUDE_CONFIG_DIR, '..', 'settings.json'); + const env = options.env!, candidate = env.HOME!.startsWith(join(root, 'state') + '/'); + if (candidate) { + expect(wrapped.slice(0, 2)).toEqual(['/usr/bin/sandbox-exec', '-f']); + const baseProfile = `${env.HOME!}.unit-test.sb`; + expect([baseProfile, `${baseProfile}.exercise.sb`]).toContain(wrapped[2]!); + for (const dir of [env.HOME!, env.TMPDIR!, env.CLAUDE_CONFIG_DIR!, env.CODEX_HOME!, env.GH_CONFIG_DIR!]) { + const info = await stat(dir); + expect(info.isDirectory()).toBe(true); expect(info.mode & 0o777).toBe(0o700); + } + expect(env.CLAUDE_CONFIG_DIR).toBe(join(env.HOME!, '.claude')); + expect(env.CODEX_HOME).toBe(join(env.HOME!, '.codex')); + expect(env.GH_TOKEN).toBeUndefined(); expect(env.CAAM_HOME).toBeUndefined(); + const settings = join(env.HOME!, 'settings.json'); expect(JSON.parse(await readFile(settings, 'utf8'))).toEqual({}); expect((await stat(settings)).mode & 0o777).toBe(0o600); } - if (config) { - const info = await stat(config); - expect(info.isDirectory()).toBe(true); - expect(info.mode & 0o777).toBe(0o700); - expect(config).toBe(join(env.TMPDIR!, '..', 'config')); - expect(config.startsWith(root + '/state/')).toBe(true); - expect(env.GH_TOKEN).toBeUndefined(); + if (options.cwd) expect((await stat(options.cwd)).isDirectory()).toBe(true); + if (args[0] === '/bin/rm') return command(args, options); + if (args[0] === '/usr/bin/true') return ''; + if (args[0] === '/usr/bin/script') { + expect(wrapped[2]).toBe(`${env.HOME!}.unit-test.sb.exercise.sb`); + expect(wrapped[2]!.startsWith(env.HOME! + '/')).toBe(false); + const policy = await readFile(wrapped[2]!, 'utf8'); + const workspace = join(env.HOME!, 'workspace'); + expect(policy).toContain(`(deny file-write* (require-all (subpath ${JSON.stringify(workspace)})`); + expect(policy).toContain(`(subpath ${JSON.stringify(join(workspace, '.claude/skills/verify-open-pstack/node_modules'))})`); + expect(policy).toContain(`(subpath ${JSON.stringify(join(workspace, 'plugins/pstack/skills/poteto-mode/scripts/node_modules'))})`); + if (args.includes('codex')) expect(policy).toContain(`(deny file-write* (require-all (subpath ${JSON.stringify(join(env.CODEX_HOME!, 'plugins/pstack'))})`); + expect(options.interactive).toBe(true); + throw new Error('fixture protected native surface reached'); } - if (args[0] === 'git' && args[1] === 'clone') { - const plugin = join(args.at(-1)!, 'plugins/pstack'); - for (const manifest of ['.claude-plugin', '.codex-plugin']) { - await mkdir(join(plugin, manifest), { recursive: true }); - await writeFile(join(plugin, manifest, 'plugin.json'), JSON.stringify({ version: 'test' })); - } + if (args[0] === 'git') { + const local = [...args]; + if (args[1] === 'clone') local[local.length - 2] = repository; + return command(local, options); } - if (args[0] === 'git' && args[1] === 'rev-parse') return sha; if (args.includes('--version')) return 'version'; if (args.includes('--help')) return '--plugin-dir --settings --setting-sources --json local path'; if (args[0] === 'codex' && args[1] === 'plugin') { - expect(env.CODEX_HOME).toBeDefined(); + for (const tool of ['claude', 'codex'] as const) { + const file = join(env[tool === 'claude' ? 'CLAUDE_CONFIG_DIR' : 'CODEX_HOME']!, tool === 'claude' ? '.credentials.json' : 'auth.json'); + expect(await readFile(file, 'utf8')).toBe(credentials[tool]); + expect((await stat(file)).mode & 0o777).toBe(0o600); + } + if (failInstall) throw new Error('fixture installation failed'); if (args[2] === 'marketplace') return '{}'; if (args[2] === 'add') { const installedPath = join(env.CODEX_HOME!, 'plugins/pstack'); - await cp(join(env.CODEX_HOME!, '..', 'workspace/plugins/pstack'), installedPath, { recursive: true }); + await cp(join(env.HOME!, 'workspace/plugins/pstack'), installedPath, { recursive: true }); return JSON.stringify({ name: 'pstack', marketplaceName: 'open-pstack', installedPath }); } if (args[2] === 'list') return JSON.stringify({ installed: [{ name: 'pstack', marketplaceName: 'open-pstack', installed: true, enabled: true }] }); } - return ''; + throw new Error(`Unexpected fixture command: ${args.join(' ')}`); + }; + // Unit adapter only: this wrapper does not enforce an OS boundary. The + // independent macOS isolation.test.ts lane executes real sandbox-exec. + const boundary = async (home: string): Promise => { + const profile = `${home}.unit-test.sb`; await writeFile(profile, 'unit adapter'); return profile; }; + const driver = new MacDriver(run, undefined, accounts, boundary); try { - Object.defineProperty(process, 'platform', { ...platform, value: 'darwin' }); - const installs = await new MacDriver(run).prepare(newReceipt(111, sha, 'b'.repeat(40), true, root)); - expect(installs.map(i => i.harness)).toEqual(['claude', 'codex']); - expect(installs.every(i => i.sha === sha)).toBe(true); - expect(calls.some(c => c.join(' ') === 'codex plugin add pstack@open-pstack --json')).toBe(true); - } finally { Object.defineProperty(process, 'platform', platform); } + Object.defineProperty(process, 'platform', { ...platform, value: 'darwin' }); process.env.CAAM_HOME = caam; + const preparation = driver.prepare(newReceipt(111, sha, sha, true, root)); + if (failInstall) await expect(preparation).rejects.toThrow('fixture installation failed'); + else { + const installs = await preparation; + expect(installs.map(i => i.harness)).toEqual(['claude', 'codex']); + expect(installs.map(i => i.account)).toEqual([accounts.claude, accounts.codex]); + expect(installs.every(i => i.sha === sha)).toBe(true); + expect(installs.every(i => /^[a-f0-9]{64}$/.test(i.sourceHash!))).toBe(true); + expect(installs[0]!.sourceHash).toBe(installs[1]!.sourceHash); + expect(calls.some(c => c.join(' ') === 'codex plugin add pstack@open-pstack --json')).toBe(true); + for (const installation of installs) { + const receipt = newReceipt(111, sha, sha, true, root); + receipt.installations = [installation]; + await expect(driver.exercise(receipt)).rejects.toThrow('fixture protected native surface reached'); + } + expect(calls.filter(c => c[0] === '/usr/bin/script')).toHaveLength(2); + } + } finally { + await driver.cleanup(); await driver.cleanup(); + Object.defineProperty(process, 'platform', platform); + if (previousCaam === undefined) delete process.env.CAAM_HOME; else process.env.CAAM_HOME = previousCaam; + } + for (const harness of ['claude', 'codex']) { + for (const tool of ['claude', 'codex'] as const) { + const filename = tool === 'claude' ? '.credentials.json' : 'auth.json'; + await expect(stat(join(root, 'state', harness, '.' + tool, filename))).rejects.toThrow('ENOENT'); + expect(await readFile(join(vault, tool, accounts[tool], filename), 'utf8')).toBe(credentials[tool]); + } + } + expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('pass'); }); test('Codex exact installed tree and enabled listing are required', async () => { const root = await fixture(), plugin = join(root, 'config/plugins/pstack'); diff --git a/.claude/skills/verify-open-pstack/scripts/harness.ts b/.claude/skills/verify-open-pstack/scripts/harness.ts index ae71b5c3..77032e5e 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.ts @@ -2,15 +2,17 @@ import { mkdir, readFile, realpath, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { createInterface } from 'node:readline/promises'; import { requiredFeatures } from './core.ts'; -import { command, isolatedEnv, redact, retainedFile, save, treeHash, type Command } from './io.ts'; +import { command, interruption, isolatedEnv, redact, retainedFile, save, treeHash, type Command } from './io.ts'; import { doctor } from './doctor.ts'; -import { HARNESSES, REPO, type Driver, type Harness, type Installation, type Observation, type Receipt } from './types.ts'; +import { copyCredentials, createSandbox, protectSources, removeCredentials, sandboxed, selectedAccounts, vaultRoot } from './isolation.ts'; +import { sourceDigest, sourceHash } from './provenance.ts'; +import { HARNESSES, REPO, type Accounts, type Driver, type Harness, type Installation, type Observation, type Receipt } from './types.ts'; export type Ask = (question: string) => Promise; export const ask: Ask = async question => { if (!process.stdin.isTTY) throw new Error('Operator review requires an interactive terminal'); const reader = createInterface({ input: process.stdin, output: process.stdout }); - try { return (await reader.question(question + '\n> ')).trim(); } finally { reader.close(); } + try { return (await reader.question(question + '\n> ', { signal: interruption.signal })).trim(); } finally { reader.close(); } }; export function launch(harness: Harness, home: string, workspace: string): string[] { return harness === 'claude' @@ -24,7 +26,7 @@ export async function codexInstallation(output: string, home: string, expected: } const path = await realpath(result.installedPath); if (!path.startsWith(await realpath(home) + '/')) throw new Error('Codex installation escaped isolated home'); - if (await treeHash(path) !== expected) throw new Error('Codex installed tree differs from pinned candidate'); + if (await treeHash(path, ['skills/poteto-mode/scripts/node_modules']) !== expected) throw new Error('Codex installed tree differs from pinned candidate'); return path; } export function verifyCodexEnabled(output: string): void { @@ -37,42 +39,72 @@ export async function verifyProjectDoctor(texts: string[], workspace: string): P for (const text of texts) { try { const d = JSON.parse(text); - if (d.result === 'pass' && typeof d.skill === 'string' && await realpath(d.skill) === expected) return; + if (d.result === 'pass' && d.candidate === true && typeof d.skill === 'string' && await realpath(d.skill) === expected) return; } catch { /* Other reviewed artifacts need not be doctor reports. */ } } throw new Error('Self-test requires the pinned project skill\'s passing child doctor.json'); } export class MacDriver implements Driver { - constructor(private run: Command = command, private review: Ask = ask) {} + private copied: string[] = []; + private profiles = new Map(); + constructor(private run: Command = command, private review: Ask = ask, private accounts?: Accounts, + private boundary: typeof createSandbox = createSandbox) {} + async cleanup(): Promise { await removeCredentials(this.copied, this.profiles, this.run); } + private candidate(home: string): Command { + const profile = this.profiles.get(home); + if (!profile) throw new Error('Candidate sandbox not established'); + return (args, options = {}) => this.run(sandboxed(profile, args), { ...options, env: isolatedEnv(home) }); + } async prepare(receipt: Receipt): Promise { const root = receipt.artifactRoot; await doctor(root, this.run); + if (!this.accounts) throw new Error('Explicit Claude and Codex caam accounts are required'); + // caam pre-run may migrate state. Enforce read-only access even in this + // trusted metadata operation: listing must never activate or write a vault. + const parentEnv: Record = {}; + for (const key of ['PATH', 'HOME', 'USER', 'LOGNAME', 'CAAM_HOME', 'XDG_CONFIG_HOME', 'XDG_DATA_HOME']) if (process.env[key]) parentEnv[key] = process.env[key]!; + const readOnly = '(version 1)(allow default)(deny file-write*)(allow file-write* (subpath "/dev"))'; + const accounts = selectedAccounts(await this.run(['/usr/bin/sandbox-exec', '-p', readOnly, 'caam', 'ls', '--json'], { env: parentEnv }), this.accounts); + const vault = await realpath(vaultRoot()); + // Fetch the immutable reference outside candidate-readable/writable state. + const referenceHome = join(root, 'source-home'), reference = join(referenceHome, 'workspace'); + await mkdir(join(referenceHome, 'tmp'), { recursive: true, mode: 0o700 }); + const trustedRun: Command = (args, options = {}) => this.run(args, { ...options, env: isolatedEnv(referenceHome) }); + await trustedRun(['git', 'clone', '--no-checkout', '--', `https://github.com/${REPO}.git`, reference]); + await trustedRun(['git', 'fetch', 'origin', receipt.sha], { cwd: reference }); + await trustedRun(['git', '-c', 'core.hooksPath=/dev/null', 'checkout', '--detach', receipt.sha], { cwd: reference }); + const pinnedSource = await sourceHash(reference, receipt.sha, trustedRun); const installs: Installation[] = []; for (const harness of HARNESSES) { const home = join(root, 'state', harness), workspace = join(home, 'workspace'); await mkdir(join(home, 'tmp'), { recursive: true, mode: 0o700 }); - await mkdir(join(home, 'config'), { recursive: true, mode: 0o700 }); + for (const dir of ['.claude', '.codex', '.config/gh', '.cache']) await mkdir(join(home, dir), { recursive: true, mode: 0o700 }); + await writeFile(join(home, 'settings.json'), '{}\n', { mode: 0o600 }); const env = isolatedEnv(home, harness); - await this.run(['git', 'clone', '--no-checkout', '--', `https://github.com/${REPO}.git`, workspace], { env }); - await this.run(['git', 'fetch', 'origin', receipt.sha], { cwd: workspace, env }); - await this.run(['git', 'checkout', '--detach', receipt.sha], { cwd: workspace, env }); - if ((await this.run(['git', 'rev-parse', 'HEAD'], { cwd: workspace, env })).trim() !== receipt.sha) throw new Error('Candidate checkout SHA mismatch'); - const candidate = join(workspace, 'plugins/pstack'), expected = await treeHash(candidate); + this.profiles.set(home, await this.boundary(home, this.run, vault)); + const candidateRun = this.candidate(home); + await candidateRun(['git', 'clone', '--no-checkout', '--', `https://github.com/${REPO}.git`, workspace], { env }); + await candidateRun(['git', 'fetch', 'origin', receipt.sha], { cwd: workspace, env }); + await candidateRun(['git', 'checkout', '--detach', receipt.sha], { cwd: workspace, env }); + if ((await candidateRun(['git', 'rev-parse', 'HEAD'], { cwd: workspace, env })).trim() !== receipt.sha) throw new Error('Candidate checkout SHA mismatch'); + const candidate = join(workspace, 'plugins/pstack'), expected = await treeHash(candidate, ['skills/poteto-mode/scripts/node_modules']); + if (await sourceDigest(workspace) !== pinnedSource) throw new Error('Candidate files differ from trusted pinned source'); + await copyCredentials(home, vault, accounts, this.copied); let location = candidate; - if (harness === 'claude') await writeFile(join(home, 'settings.json'), '{}\n', { mode: 0o600 }); - else { - const added = await this.run(['codex', 'plugin', 'marketplace', 'add', workspace, '--json'], { env, cwd: home }); + if (harness === 'codex') { + const added = await candidateRun(['codex', 'plugin', 'marketplace', 'add', workspace, '--json'], { env, cwd: home }); await save(join(root, 'codex-marketplace.json'), JSON.parse(added)); - const installed = await this.run(['codex', 'plugin', 'add', 'pstack@open-pstack', '--json'], { env, cwd: home }); + const installed = await candidateRun(['codex', 'plugin', 'add', 'pstack@open-pstack', '--json'], { env, cwd: home }); await save(join(root, 'codex-install.json'), JSON.parse(installed)); location = await codexInstallation(installed, home, expected); - const listed = await this.run(['codex', 'plugin', 'list', '--marketplace', 'open-pstack', '--json'], { env, cwd: home }); + const listed = await candidateRun(['codex', 'plugin', 'list', '--marketplace', 'open-pstack', '--json'], { env, cwd: home }); verifyCodexEnabled(listed); await save(join(root, 'codex-plugins.json'), JSON.parse(listed)); } const manifest = JSON.parse(await readFile(join(location, harness === 'claude' ? '.claude-plugin/plugin.json' : '.codex-plugin/plugin.json'), 'utf8')); - installs.push({ harness, sha: receipt.sha, home, location, treeHash: expected, pluginVersion: manifest.version, - cliVersion: (await this.run([harness, '--version'], { env })).trim() }); + if (await sourceDigest(workspace) !== pinnedSource) throw new Error('Pinned source changed during installation'); + installs.push({ harness, sha: receipt.sha, home, location, treeHash: expected, sourceHash: pinnedSource, account: accounts[harness], pluginVersion: manifest.version, + cliVersion: (await candidateRun([harness, '--version'], { env })).trim() }); } return installs; } @@ -80,19 +112,23 @@ export class MacDriver implements Driver { const observations: Observation[] = []; for (const installation of receipt.installations) { const { harness, home } = installation, workspace = join(home, 'workspace'); - const env = isolatedEnv(home, harness); + if (await sourceDigest(workspace) !== installation.sourceHash) throw new Error('Pinned source changed before exercise'); + const profile = this.profiles.get(home); + if (!profile) throw new Error('Candidate sandbox not established'); + this.profiles.set(home, await protectSources(profile, home, [workspace, installation.location], this.run)); + const env = isolatedEnv(home, harness), candidateRun = this.candidate(home); const request = { sha: receipt.sha, harness, workspace, features: requiredFeatures(receipt), featureMap: join(workspace, '.claude/skills/verify-open-pstack/features'), - selfTest: requiredFeatures(receipt).includes('project-skill') ? 'Invoke the project skill natively; run only doctor with fresh output outside workspace. Do not invoke run or publish recursively.' : false }; + selfTest: requiredFeatures(receipt).includes('project-skill') ? 'Invoke the project skill natively; run doctor --candidate --output "$HOME/self-test" within sandboxed state. Do not invoke run, vault listing, login, or publication recursively.' : false, + credentials: { claude: this.accounts?.claude, codex: this.accounts?.codex }, + isolation: 'Disposable file credentials only; no daily-home, Keychain, caam vault or publisher access. Exercise real requests; quota/auth failures fail closed.' }; await save(join(receipt.artifactRoot, `${harness}-request.json`), request); console.log(JSON.stringify(request, null, 2)); - const login = await this.review(`Authenticate only in this isolated ${harness} home if needed. Launch an unrecorded login session now? Type LOGIN or SKIP (already authenticated in this isolated home).`); - if (login === 'LOGIN') await this.run(harness === 'claude' ? ['claude', 'auth', 'login'] : ['codex', 'login'], { cwd: workspace, env, interactive: true }); - else if (login !== 'SKIP') throw new Error('Expected LOGIN or SKIP'); const raw = join(home, 'surface.raw'); console.log('Exercise every requested feature from the native surface, using the maintained feature map. Exit when done.'); - await this.run(['script', '-q', raw, ...launch(harness, home, workspace)], { cwd: workspace, env, interactive: true }); - if (await treeHash(installation.location) !== installation.treeHash) throw new Error('Installed tree changed during exercise'); + await candidateRun(['/usr/bin/script', '-q', raw, ...launch(harness, home, workspace)], { cwd: workspace, env, interactive: true }); + if (await treeHash(installation.location, ['skills/poteto-mode/scripts/node_modules']) !== installation.treeHash) throw new Error('Installed tree changed during exercise'); + if (await sourceDigest(workspace) !== installation.sourceHash) throw new Error('Pinned project/plugin source changed during exercise'); const transcript = await retainedFile(receipt.artifactRoot, await this.review(`Copy/redact ${raw} into output (outside state), review it, and enter the reviewed transcript path:`)); const text = await readFile(join(receipt.artifactRoot, transcript.path), 'utf8'); if (redact(text) !== text) throw new Error('Transcript contains recognizable credentials; redact before accepting'); diff --git a/.claude/skills/verify-open-pstack/scripts/io.test.ts b/.claude/skills/verify-open-pstack/scripts/io.test.ts new file mode 100644 index 00000000..2ccae866 --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/io.test.ts @@ -0,0 +1,129 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { chmod, mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { redact, retainedFile, save, treeHash } from './io.ts'; + +const roots: string[] = []; +async function fixture(): Promise { + const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-io-'))); + roots.push(root); + return root; +} +afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); + +describe('retained JSON and file boundaries', () => { + test('redacts string values before serialization without corrupting JSON punctuation or escapes', async () => { + const root = await fixture(), path = join(root, 'receipt.json'); + await save(path, { path: '/tmp/Bearer token', nested: ['Bearer secret",}', 'github_pat_sensitive'], text: 'Bearer secret, next\nquoted "value"', count: 2 }); + const json = await readFile(path, 'utf8'); + expect(JSON.parse(json)).toEqual({ path: '/tmp/[REDACTED]', nested: ['[REDACTED]",}', '[REDACTED]'], text: '[REDACTED], next\nquoted "value"', count: 2 }); + expect(json).not.toContain('secret'); + expect((await stat(path)).mode & 0o777).toBe(0o600); + expect(redact('Bearer token, next')).toBe('[REDACTED], next'); + }); + + test('permits evidence below an ancestor named state and canonicalizes output aliases', async () => { + const root = await fixture(), output = join(root, 'state', 'run'), alias = join(root, 'alias'); + await mkdir(output, { recursive: true }); + await writeFile(join(output, 'transcript.txt'), 'native surface evidence'); + await symlink(output, alias); + expect(await retainedFile(output, 'transcript.txt')).toEqual(await retainedFile(alias, 'transcript.txt')); + expect((await retainedFile(alias, 'transcript.txt')).path).toBe('transcript.txt'); + }); + + test('rejects this run state and aliases into it, but not another directory named state', async () => { + const root = await fixture(); + await mkdir(join(root, 'state')); + await mkdir(join(root, 'artifacts', 'state'), { recursive: true }); + await writeFile(join(root, 'state', 'secret.txt'), 'isolated authentication'); + await writeFile(join(root, 'artifacts', 'state', 'result.txt'), 'retained result'); + await symlink(join(root, 'state', 'secret.txt'), join(root, 'transcript.txt')); + await expect(retainedFile(root, 'state')).rejects.toThrow('outside isolated state'); + await expect(retainedFile(root, 'state/secret.txt')).rejects.toThrow('outside isolated state'); + await expect(retainedFile(root, 'transcript.txt')).rejects.toThrow('outside isolated state'); + expect((await retainedFile(root, 'artifacts/state/result.txt')).path).toBe('artifacts/state/result.txt'); + }); +}); + +describe('candidate tree digests', () => { + test('dependency bootstrap is excluded only at explicitly supplied generated directories', async () => { + const root = await fixture(), scripts = join(root, 'skills', 'poteto-mode', 'scripts'); + await mkdir(scripts, { recursive: true }); + await writeFile(join(scripts, 'bootstrap.ts'), 'export const candidate = 1;'); + const exclusions = ['skills/poteto-mode/scripts/node_modules']; + const pinned = await treeHash(root, exclusions); + const dependencies = join(scripts, 'node_modules'); + await mkdir(dependencies); + await writeFile(join(dependencies, 'generated.js'), 'generated dependency'); + await symlink(join(dependencies, 'generated.js'), join(dependencies, 'package')); + expect(await treeHash(root, exclusions)).toBe(pinned); + await expect(treeHash(root)).rejects.toThrow('symlink refused'); + await writeFile(join(scripts, 'bootstrap.ts'), 'export const candidate = 2;'); + expect(await treeHash(root, exclusions)).not.toBe(pinned); + await expect(treeHash(root, ['skills/poteto-mode/scripts'])).rejects.toThrow('Only explicit generated'); + await expect(treeHash(root, ['**/node_modules'])).rejects.toThrow('Only explicit generated'); + }); + + test('source executable permissions affect the hash and non-excluded symlinks are rejected', async () => { + const root = await fixture(), source = join(root, 'verify.sh'); + await writeFile(source, '#!/bin/sh\nexit 0\n', { mode: 0o644 }); + const pinned = await treeHash(root); + await chmod(source, 0o755); + expect(await treeHash(root)).not.toBe(pinned); + await symlink(source, join(root, 'alias.sh')); + await expect(treeHash(root)).rejects.toThrow('symlink refused'); + const alias = join(await fixture(), 'tree'); + await symlink(root, alias); + await expect(treeHash(alias)).rejects.toThrow('real directory'); + }); +}); + +describe('coordinated process interruption', () => { + test.each(['SIGINT', 'SIGTERM'] as const)('%s kills real process groups and permits only explicit cleanup commands', async signal => { + const root = await fixture(), home = join(root, 'candidate'), credential = join(home, '.claude/.credentials.json'); + const vault = join(root, 'vault.json'), report = join(root, 'report.json'), heartbeat = join(root, 'heartbeat'), ready = join(root, 'ready.json'); + await mkdir(join(home, '.claude'), { recursive: true, mode: 0o700 }); + await writeFile(credential, 'disposable token', { mode: 0o600 }); + await writeFile(vault, 'original vault sentinel', { mode: 0o600 }); + const grandchild = join(root, 'grandchild.ts'), dummy = join(root, 'dummy.ts'), worker = join(root, 'worker.ts'); + await writeFile(grandchild, `let counter = 0; await Bun.write(${JSON.stringify(heartbeat)}, String(counter));\nsetInterval(() => Bun.write(${JSON.stringify(heartbeat)}, String(++counter)), 10);\n`); + await writeFile(dummy, `const child = Bun.spawn([process.execPath, ${JSON.stringify(grandchild)}], { stdout: 'ignore', stderr: 'ignore' });\nwhile (!(await Bun.file(${JSON.stringify(heartbeat)}).exists())) await Bun.sleep(5);\nawait Bun.write(${JSON.stringify(ready)}, JSON.stringify({ pid: process.pid, grandchild: child.pid }));\nsetInterval(() => {}, 1000);\n`); + await writeFile(worker, `import { command, interruption, interruptCommands, save } from ${JSON.stringify(join(import.meta.dir, 'io.ts'))}; +process.on('SIGINT', () => { void interruptCommands('SIGINT'); }); +process.on('SIGTERM', () => { void interruptCommands('SIGTERM'); }); +let failure = '', rejected = false; +try { await command([process.execPath, ${JSON.stringify(dummy)}]); } +catch (error) { + failure = String(error); + try { await command(['/bin/true']); } catch { rejected = true; } +} finally { + await command(['/bin/rm', '-f', '--', ${JSON.stringify(credential)}], { allowInterrupted: true }); + await save(${JSON.stringify(report)}, { failure, rejected, aborted: interruption.signal.aborted }); + process.exitCode = 1; +} +`); + const child = Bun.spawn([process.execPath, worker], { stdout: 'pipe', stderr: 'pipe' }); + const stderr = new Response(child.stderr).text(), stdout = new Response(child.stdout).text(); + try { + for (let attempt = 0; attempt < 500 && !(await Bun.file(ready).exists()); attempt++) await Bun.sleep(5); + expect(await Bun.file(ready).exists()).toBe(true); + child.kill(signal); + expect(await child.exited).toBe(1); + expect(await stderr).toBe(''); expect(await stdout).toBe(''); + expect(JSON.parse(await readFile(report, 'utf8'))).toEqual({ failure: `Error: Interrupted by ${signal}`, rejected: true, aborted: true }); + await expect(stat(credential)).rejects.toThrow('ENOENT'); + expect(await readFile(vault, 'utf8')).toBe('original vault sentinel'); + const stopped = await readFile(heartbeat, 'utf8'); + await Bun.sleep(100); + expect(await readFile(heartbeat, 'utf8')).toBe(stopped); + } finally { + child.kill('SIGKILL'); + if (await Bun.file(ready).exists()) { + const { pid } = JSON.parse(await readFile(ready, 'utf8')); + try { process.kill(-pid, 'SIGKILL'); } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error; } + } + await child.exited; + } + }, 10000); +}); diff --git a/.claude/skills/verify-open-pstack/scripts/io.ts b/.claude/skills/verify-open-pstack/scripts/io.ts index e314c0df..35d9033e 100644 --- a/.claude/skills/verify-open-pstack/scripts/io.ts +++ b/.claude/skills/verify-open-pstack/scripts/io.ts @@ -2,35 +2,61 @@ import { createHash } from 'node:crypto'; import { lstat, mkdir, readFile, readdir, realpath, rename, writeFile } from 'node:fs/promises'; import { isAbsolute, join, relative, resolve } from 'node:path'; -export type Command = (args: string[], options?: { cwd?: string; env?: Record; interactive?: boolean }) => Promise; +export type Command = (args: string[], options?: { cwd?: string; env?: Record; interactive?: boolean; allowInterrupted?: boolean }) => Promise; +export const interruption = new AbortController(); +const activeCommands = new Map>(); +let interrupting: Promise | undefined; +export function interruptCommands(signal: 'SIGINT' | 'SIGTERM'): Promise { + if (interrupting) return interrupting; + interruption.abort(new Error(`Interrupted by ${signal}`)); + interrupting = (async () => { + const children = [...activeCommands]; + for (const [pid] of children) { + try { process.kill(-pid, 'SIGKILL'); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error; } + } + await Promise.all(children.map(([, exited]) => exited)); + })(); + return interrupting; +} export const command: Command = async (args, options = {}) => { - const child = Bun.spawn(args, { cwd: options.cwd, env: options.env, + if (!options.allowInterrupted) interruption.signal.throwIfAborted(); + const child = Bun.spawn(args, { cwd: options.cwd, env: options.env, detached: true, stdin: options.interactive ? 'inherit' : 'ignore', stdout: options.interactive ? 'inherit' : 'pipe', stderr: options.interactive ? 'inherit' : 'pipe' }); - const [output, error] = options.interactive ? ['', ''] : await Promise.all([ - new Response(child.stdout).text(), new Response(child.stderr).text(), - ]); - if (await child.exited !== 0) throw new Error(`${args[0]} failed: ${redact(error).slice(0, 1500)}`); - return output; + if (!options.allowInterrupted) activeCommands.set(child.pid, child.exited); + try { + const [output, error] = options.interactive ? ['', ''] : await Promise.all([ + new Response(child.stdout).text(), new Response(child.stderr).text(), + ]); + const code = await child.exited; + if (!options.allowInterrupted) interruption.signal.throwIfAborted(); + if (code !== 0) throw new Error(`${args[0]} failed: ${redact(error).slice(0, 1500)}`); + return output; + } finally { + activeCommands.delete(child.pid); + } }; export function redact(text: string): string { - return text.replace(/(?:gh[pousr]_[\w]+|github_pat_[\w]+|sk-[\w-]+|Bearer\s+\S+)/gi, '[REDACTED]'); + return text.replace(/(?:gh[pousr]_[\w]+|github_pat_[\w]+|sk-[\w-]+|Bearer[ \t]+[^\s"'<>()[\]{},;]+)/gi, '[REDACTED]'); } export function isolatedEnv(home: string, harness?: 'claude' | 'codex'): Record { - const realHome = process.env.HOME; - if (!realHome || !isAbsolute(realHome)) throw new Error('A real absolute HOME is required; never substitute a temporary home'); - const env: Record = { PATH: (process.env.PATH ?? '').split(':').filter(isAbsolute).join(':'), - HOME: realHome, TMPDIR: join(home, 'tmp'), TERM: process.env.TERM ?? 'xterm-256color', - GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_SYSTEM: '/dev/null' }; + if (!isAbsolute(home)) throw new Error('Candidate HOME must be an absolute run-owned directory'); + const env: Record = { PATH: [join(home, 'bin'), ...(process.env.PATH ?? '').split(':').filter(isAbsolute)].join(':'), + HOME: home, TMPDIR: join(home, 'tmp'), TERM: process.env.TERM ?? 'xterm-256color', + XDG_CONFIG_HOME: join(home, '.config'), XDG_CACHE_HOME: join(home, '.cache'), + GH_CONFIG_DIR: join(home, '.config/gh'), + GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_SYSTEM: '/dev/null', GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'credential.helper', GIT_CONFIG_VALUE_0: '', + CLAUDE_CONFIG_DIR: join(home, '.claude'), CODEX_HOME: join(home, '.codex') }; for (const name of ['USER', 'LOGNAME']) { if (process.env[name] !== undefined) env[name] = process.env[name]; } - if (harness === 'claude') env.CLAUDE_CONFIG_DIR = join(home, 'config'); - if (harness === 'codex') env.CODEX_HOME = join(home, 'config'); return env; } export async function save(path: string, value: unknown): Promise { - await writeFile(path + '.tmp', redact(JSON.stringify(value, null, 2)) + '\n', { mode: 0o600 }); + const json = JSON.stringify(value, (_key, item) => typeof item === 'string' ? redact(item) : item, 2); + await writeFile(path + '.tmp', json + '\n', { mode: 0o600 }); await rename(path + '.tmp', path); } export async function freshRoot(path: string, repository: string): Promise { @@ -43,14 +69,25 @@ export async function freshRoot(path: string, repository: string): Promise { +export async function treeHash(root: string, excludePaths: string[] = []): Promise { + const generated = new Set([ + 'skills/poteto-mode/scripts/node_modules', + 'plugins/pstack/skills/poteto-mode/scripts/node_modules', + '.claude/skills/verify-open-pstack/node_modules', + ]); + if (excludePaths.some(path => !generated.has(path))) throw new Error('Only explicit generated dependency directories may be excluded'); + const excluded = new Set(excludePaths); + const rootStat = await lstat(root); + if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) throw new Error('Plugin tree root must be a real directory'); const entries: string[] = []; async function walk(dir: string): Promise { for (const name of (await readdir(dir)).sort()) { - const path = join(dir, name), stat = await lstat(path); + const path = join(dir, name), entry = relative(root, path); + if (excluded.has(entry)) continue; + const stat = await lstat(path); if (stat.isSymbolicLink()) throw new Error(`Plugin tree symlink refused: ${path}`); if (stat.isDirectory()) await walk(path); - else if (stat.isFile()) entries.push(`${relative(root, path)}\0${sha256(await readFile(path))}`); + else if (stat.isFile()) entries.push(`${entry}\0${stat.mode & 0o111 ? '100755' : '100644'}\0${sha256(await readFile(path))}`); else throw new Error(`Non-file in plugin tree: ${path}`); } } @@ -59,9 +96,14 @@ export async function treeHash(root: string): Promise { return sha256(entries.join('\n')); } export async function retainedFile(root: string, path: string): Promise<{ path: string; sha256: string }> { - const actual = await realpath(isAbsolute(path) ? path : join(root, path)); - if (!actual.startsWith(root + '/') || actual.includes('/state/')) throw new Error('Evidence must be retained outside isolated state, within output'); + const output = await realpath(root); + const actual = await realpath(isAbsolute(path) ? path : join(output, path)); + const state = await realpath(join(output, 'state')).catch((error: NodeJS.ErrnoException) => { + if (error.code !== 'ENOENT') throw error; + return join(output, 'state'); + }); + if (!actual.startsWith(output + '/') || actual === state || actual.startsWith(state + '/')) throw new Error('Evidence must be retained outside isolated state, within output'); const stat = await lstat(actual); if (!stat.isFile() || !stat.size) throw new Error('Evidence file must be a nonempty regular file'); - return { path: relative(root, actual), sha256: sha256(await readFile(actual)) }; + return { path: relative(output, actual), sha256: sha256(await readFile(actual)) }; } diff --git a/.claude/skills/verify-open-pstack/scripts/isolation.test.ts b/.claude/skills/verify-open-pstack/scripts/isolation.test.ts new file mode 100644 index 00000000..2b1554f4 --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/isolation.test.ts @@ -0,0 +1,376 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { lstat, mkdir, mkdtemp, readFile, realpath, rm, rmdir, stat, symlink, writeFile } from 'node:fs/promises'; +import { randomUUID } from 'node:crypto'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { command, isolatedEnv } from './io.ts'; +import { copyCredentials, createSandbox, protectSources, removeCredentials, sandboxed, sandboxProfile, selectedAccounts } from './isolation.ts'; +import type { Accounts } from './types.ts'; + +const roots: string[] = []; +async function fixture(): Promise { + const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-isolation-'))); + roots.push(root); + return root; +} +afterEach(async () => { + for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); +}); +const accounts: Accounts = { claude: 'claude@example.com', codex: 'codex@example.com' }; +function profiles(claude = accounts.claude, codex = accounts.codex): string { + return JSON.stringify({ profiles: [ + { tool: 'claude', name: accounts.claude, identity: { email: claude } }, + { tool: 'codex', name: accounts.codex, identity: { email: codex } }, + ] }); +} +async function vault(root: string): Promise { + const base = join(root, 'vault'); + for (const tool of ['claude', 'codex'] as const) { + const path = join(base, tool, accounts[tool]); + await mkdir(path, { recursive: true, mode: 0o700 }); + await writeFile(join(path, tool === 'claude' ? '.credentials.json' : 'auth.json'), JSON.stringify( + tool === 'claude' ? { claudeAiOauth: { accessToken: 'selected-claude-token' } } : { tokens: { access_token: 'selected-codex-token' } }, + ), { mode: 0o600 }); + } + return base; +} + +describe('disposable credential boundary', () => { + test('both provider config roots stay in the candidate home, including external-provider children', () => { + const home = '/run/private-candidate'; + for (const tool of ['claude', 'codex'] as const) { + const env = isolatedEnv(home, tool); + expect(env.HOME).toBe(home); + expect(env.CLAUDE_CONFIG_DIR).toBe(join(home, '.claude')); + expect(env.CODEX_HOME).toBe(join(home, '.codex')); + expect(env.GH_CONFIG_DIR).toBe(join(home, '.config/gh')); + expect(env.TMPDIR).toBe(join(home, 'tmp')); + expect(env.GIT_CONFIG_VALUE_0).toBe(''); + expect(env.CAAM_HOME).toBeUndefined(); + expect(env.GH_TOKEN).toBeUndefined(); + expect(env.GITHUB_TOKEN).toBeUndefined(); + expect(env.ANTHROPIC_API_KEY).toBeUndefined(); + expect(env.OPENAI_API_KEY).toBeUndefined(); + expect(process.env.USER).toBe(env.USER); + expect(process.env.LOGNAME).toBe(env.LOGNAME); + } + }); + test('explicit vault profile identity must match each selected account', () => { + expect(selectedAccounts(profiles(), accounts)).toEqual(accounts); + expect(() => selectedAccounts(profiles('other@example.com'), accounts)).toThrow(); + expect(() => selectedAccounts(profiles(accounts.claude, 'other@example.com'), accounts)).toThrow(); + expect(() => selectedAccounts('{"profiles":[]}', accounts)).toThrow(); + expect(() => selectedAccounts('{}', accounts)).toThrow(); + expect(() => selectedAccounts('not JSON', accounts)).toThrow(); + expect(() => selectedAccounts(profiles(), { ...accounts, claude: '../escape@example.com' })).toThrow(); + const duplicate = JSON.parse(profiles()); duplicate.profiles.push(duplicate.profiles[0]); + expect(() => selectedAccounts(JSON.stringify(duplicate), accounts)).toThrow(); + }); + test('real credential files are private copied files, never vault links, and cleanup preserves the vault', async () => { + const root = await fixture(), base = await vault(root), home = join(root, 'candidate'); + const copied: string[] = []; + await copyCredentials(home, base, accounts, copied); + expect(copied).toHaveLength(2); + const originals = [join(base, 'claude', accounts.claude, '.credentials.json'), join(base, 'codex', accounts.codex, 'auth.json')]; + for (let i = 0; i < copied.length; i++) { + expect((await lstat(copied[i]!)).isSymbolicLink()).toBe(false); + expect((await stat(copied[i]!)).mode & 0o777).toBe(0o600); + expect((await stat(dirname(copied[i]!))).mode & 0o777).toBe(0o700); + expect(await readFile(copied[i]!, 'utf8')).toBe(await readFile(originals[i]!, 'utf8')); + } + const preserved = await Promise.all(originals.map(path => readFile(path, 'utf8'))); + await writeFile(copied[0]!, '{"refreshed":"candidate-only"}'); + expect(await readFile(originals[0]!, 'utf8')).toBe(preserved[0]); + await removeCredentials(copied); + await removeCredentials(copied); + for (const path of copied) expect(await Bun.file(path).exists()).toBe(false); + expect(await Promise.all(originals.map(path => readFile(path, 'utf8')))).toEqual(preserved); + }); + test('partial copy can be cleaned on failure without changing any vault file', async () => { + const root = await fixture(), base = await vault(root), copied: string[] = []; + const bad = join(base, 'codex', accounts.codex, 'auth.json'); + await writeFile(bad, '{"unknown":true}'); + await expect(copyCredentials(join(root, 'candidate'), base, accounts, copied)).rejects.toThrow('unknown format'); + expect(copied).toHaveLength(1); + await removeCredentials(copied); + expect(await Bun.file(copied[0]!).exists()).toBe(false); + expect(await readFile(bad, 'utf8')).toBe('{"unknown":true}'); + }); + test('vault symlinks and paths escaping the vault are not copied', async () => { + const root = await fixture(), base = await vault(root), copied: string[] = []; + const credential = join(base, 'claude', accounts.claude, '.credentials.json'); + const outside = join(root, 'outside-auth'); + await writeFile(outside, '{"claudeAiOauth":{"accessToken":"other-account"}}'); + await rm(credential); await symlink(outside, credential); + await expect(copyCredentials(join(root, 'candidate'), base, accounts, copied)).rejects.toThrow('contained regular file'); + expect(copied).toEqual([]); + }); + test('credential cleanup refuses an ancestor redirect without deleting the external sentinel', async () => { + const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; + await copyCredentials(home, base, accounts, copied); + const outside = join(root, 'external'), sentinel = join(outside, '.credentials.json'); + await mkdir(outside); await writeFile(sentinel, 'external-credential-sentinel'); + await rm(join(home, '.claude'), { recursive: true }); await symlink(outside, join(home, '.claude')); + await expect(removeCredentials(copied)).rejects.toThrow('redirected parent'); + expect(await readFile(sentinel, 'utf8')).toBe('external-credential-sentinel'); + const invoked: string[][] = []; + await removeCredentials(copied, new Map([[home, join(root, 'candidate.sb')]]), async args => { invoked.push(args); return ''; }); + expect(invoked).toEqual(copied.map(path => ['/usr/bin/sandbox-exec', '-f', join(root, 'candidate.sb'), '/bin/rm', '-f', '--', path])); + await expect(removeCredentials(copied, new Map())).rejects.toThrow('original candidate sandbox'); + expect(await readFile(sentinel, 'utf8')).toBe('external-credential-sentinel'); + }); + test('cleanup refuses leaf symlinks and copy records destinations before a failed write', async () => { + const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; + await mkdir(join(home, '.claude'), { recursive: true }); + const target = join(home, '.claude/.credentials.json'), sentinel = join(root, 'external-credential'); + await writeFile(sentinel, 'preserve'); await symlink(sentinel, target); + await expect(copyCredentials(home, base, accounts, copied)).rejects.toThrow(); + expect(copied).toEqual([target]); + await expect(removeCredentials(copied)).rejects.toThrow('nonregular file'); + expect(await readFile(sentinel, 'utf8')).toBe('preserve'); + }); + test('selected vault tool and profile directories cannot redirect to another account', async () => { + const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; + const original = join(base, 'claude', accounts.claude), other = join(base, 'claude/other@example.com'); + await mkdir(other); await writeFile(join(other, '.credentials.json'), '{"claudeAiOauth":{"accessToken":"wrong-account"}}'); + await rm(original, { recursive: true }); await symlink(other, original); + await expect(copyCredentials(home, base, accounts, copied)).rejects.toThrow('not symlinked'); + expect(copied).toEqual([]); + await rm(original); await rm(other, { recursive: true }); + const redirected = join(root, 'redirected-tool'); + await mkdir(join(redirected, accounts.claude), { recursive: true }); + await writeFile(join(redirected, accounts.claude, '.credentials.json'), '{"claudeAiOauth":{"accessToken":"wrong-account"}}'); + await rm(join(base, 'claude'), { recursive: true }); await symlink(redirected, join(base, 'claude')); + await expect(copyCredentials(home, base, accounts, copied)).rejects.toThrow('not symlinked'); + expect(copied).toEqual([]); + }); + test('the canonical vault deny overrides allowlisted files and candidate state cannot overlap it', async () => { + const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), alias = join(root, 'vault-alias'); + await mkdir(home); await symlink(base, alias); + const text = sandboxProfile(home, '/Users/operator', [join(base, 'claude', accounts.claude, '.credentials.json')], alias); + expect(text).toContain(`(deny file-read* file-write* (subpath ${JSON.stringify(base)}))`); + expect(text).not.toContain(`(subpath ${JSON.stringify(alias)})`); + await mkdir(join(base, 'nested')); + expect(() => sandboxProfile(join(base, 'nested'), '/Users/operator', [], base)).toThrow('must not overlap'); + expect(() => sandboxProfile(base, '/Users/operator', [], base)).toThrow('must not overlap'); + expect(() => sandboxProfile(root, '/Users/operator', [], base)).toThrow('must not overlap'); + }); + test('Seatbelt contract denies daily files, Keychain IPC, security execution, and arbitrary writes', () => { + const text = sandboxProfile('/private/run/candidate', '/Users/operator', ['/opt/homebrew/bin/bun']); + expect(text).toContain('(deny file-read* file-write* (subpath "/Users/operator"))'); + expect(text).toContain('(deny file-read* (require-not (require-any'); + expect(text).toContain('(deny file-write* (require-not (require-any'); + expect(text).toContain('(deny mach-lookup (require-not (require-any'); + expect(text).not.toContain('(global-name "com.apple.securityd")'); + expect(text).not.toContain('(global-name "com.apple.SecurityServer")'); + expect(text).not.toContain('(global-name "com.apple.SecurityAgent")'); + expect(text).toContain('(deny process-exec (literal "/usr/bin/security"))'); + expect(text).toContain('(deny process-info*'); + expect(() => sandboxProfile('/Users/operator/run', '/Users/operator', [])).toThrow('outside the operator home'); + expect(() => sandboxProfile('/Users/operator', '/Users/operator', [])).toThrow(); + expect(() => sandboxProfile('relative', '/Users/operator', [])).toThrow(); + }); +}); + +describe('exercise source protection', () => { + test('exercise policy stays outside candidate state with only three generated dependency exceptions', async () => { + const root = await fixture(), home = join(root, 'candidate'), workspace = join(home, 'workspace'), plugin = join(home, '.codex/plugins/pstack'); + await mkdir(workspace, { recursive: true }); await mkdir(plugin, { recursive: true }); + const profile = join(root, 'candidate.sb'); + await writeFile(profile, sandboxProfile(home, '/Users/operator', []), { mode: 0o600 }); + const original = await readFile(profile, 'utf8'); + const exercise = await protectSources(profile, home, [workspace, plugin], async () => ''); + const text = await readFile(exercise, 'utf8'); + expect(exercise.startsWith(home + '/')).toBe(false); + expect((await stat(exercise)).mode & 0o777).toBe(0o600); + expect(await readFile(profile, 'utf8')).toBe(original); + for (const path of [join(workspace, '.claude/skills/verify-open-pstack/node_modules'), join(workspace, 'plugins/pstack/skills/poteto-mode/scripts/node_modules'), join(plugin, 'skills/poteto-mode/scripts/node_modules')]) { + expect((await stat(path)).isDirectory()).toBe(true); + expect(text).toContain(`(subpath ${JSON.stringify(path)})`); + } + expect(text).toContain(`(deny file-write* (require-all (subpath ${JSON.stringify(workspace)})`); + expect(text).toContain(`(deny file-write* (require-all (subpath ${JSON.stringify(plugin)})`); + expect(text).toContain(`(literal ${JSON.stringify(join(home, '.codex'))})`); + await expect(protectSources(profile, home, [workspace], async () => '')).rejects.toThrow(); + }); + test('root, ancestor, dependency redirects, outside sources, and writable policies fail closed', async () => { + const root = await fixture(), home = join(root, 'candidate'), outside = join(root, 'outside'); + await mkdir(home); await mkdir(outside); + const profile = join(root, 'candidate.sb'); await writeFile(profile, 'trusted-policy'); + const alias = join(home, 'alias'); await symlink(outside, alias); + await expect(protectSources(profile, home, [alias], async () => '')).rejects.toThrow('Redirected source'); + await mkdir(join(outside, 'child')); await expect(protectSources(profile, home, [join(alias, 'child')], async () => '')).rejects.toThrow('Redirected source'); + await expect(protectSources(profile, home, [outside], async () => '')).rejects.toThrow('contained'); + await expect(protectSources(profile, home, [], async () => '')).rejects.toThrow('requires pinned'); + const workspace = join(home, 'workspace'); await mkdir(workspace); + await symlink(outside, join(workspace, '.claude')); + await expect(protectSources(profile, home, [workspace], async () => '')).rejects.toThrow('Redirected source'); + const writablePolicy = join(home, 'policy.sb'); await writeFile(writablePolicy, 'candidate-policy'); + await expect(protectSources(writablePolicy, home, [workspace], async () => '')).rejects.toThrow('outside candidate'); + const policyAlias = join(root, 'alias.sb'); await symlink(profile, policyAlias); + await expect(protectSources(policyAlias, home, [workspace], async () => '')).rejects.toThrow('regular trusted'); + expect(await readFile(profile, 'utf8')).toBe('trusted-policy'); + }); +}); + +// This is intentionally not a mocked-command test. Linux cannot prove Seatbelt +// enforcement; the operator must run this exact suite on the Mac before live proof. +const macTest = process.platform === 'darwin' ? test : test.skip; +macTest('real Mac sandbox denies daily-home and vault sentinels, writes, aliases, grandchildren, and Keychain IPC', async () => { + const root = await fixture(), daily = join(root, 'daily'), home = join(root, 'candidate'); + await mkdir(join(home, 'tmp'), { recursive: true, mode: 0o700 }); + const locations = [ + join(daily, 'home-sentinel'), join(daily, '.config/gh/hosts.yml'), join(daily, '.ssh/id_ed25519'), + join(daily, '.claude/.credentials.json'), join(daily, '.codex/auth.json'), + join(daily, 'Library/Keychains/login.keychain-db'), join(root, 'other-account-vault/auth.json'), + ]; + for (const path of locations) { + await mkdir(dirname(path), { recursive: true, mode: 0o700 }); + await writeFile(path, 'operator-sentinel-must-remain-private', { mode: 0o600 }); + } + const previousHome = process.env.HOME; + let profile: string; + const deniedVault = join(root, 'other-account-vault'); + try { process.env.HOME = daily; profile = await createSandbox(home, command, deniedVault); } + finally { + if (previousHome === undefined) delete process.env.HOME; else process.env.HOME = previousHome; + } + const options = { cwd: home, env: isolatedEnv(home) }; + const allowed = join(home, 'candidate-sentinel'); + await writeFile(allowed, 'candidate-state-allowed'); + expect(await command(sandboxed(profile, ['/bin/cat', allowed]), options)).toBe('candidate-state-allowed'); + expect(profile.startsWith(home + '/')).toBe(false); + expect((await stat(profile)).mode & 0o777).toBe(0o600); + const originalPolicy = await readFile(profile, 'utf8'); + expect(originalPolicy).toContain(`(deny file-read* file-write* (subpath ${JSON.stringify(deniedVault)}))`); + await expect(command(sandboxed(profile, ['/bin/sh', '-c', 'printf replacement > "$1"', 'policy', profile]), options)).rejects.toThrow(); + expect(await readFile(profile, 'utf8')).toBe(originalPolicy); + // This deliberately allowlists a vault file, then proves the final explicit + // vault denial overrides that allowance rather than relying on deny-default. + const overrideProfile = join(root, 'override.sb'), vaultSentinel = join(deniedVault, 'auth.json'); + await writeFile(overrideProfile, sandboxProfile(home, daily, [vaultSentinel], deniedVault), { mode: 0o600 }); + await expect(command(sandboxed(overrideProfile, ['/bin/cat', vaultSentinel]), options)).rejects.toThrow(); + await expect(command(sandboxed(overrideProfile, ['/bin/sh', '-c', 'printf overwritten > "$1"', 'vault', vaultSentinel]), options)).rejects.toThrow(); + expect(await readFile(vaultSentinel, 'utf8')).toBe('operator-sentinel-must-remain-private'); + // Production cleanup inherits the same policy even if candidates replace + // a configuration parent with a symlink to a vault credential directory. + const redirectedCredential = join(home, '.claude/.credentials.json'); + await writeFile(join(deniedVault, '.credentials.json'), 'preserve-external-credential'); + await symlink(deniedVault, join(home, '.claude')); + await expect(removeCredentials([redirectedCredential], new Map([[home, profile]]))).rejects.toThrow(); + expect(await readFile(join(deniedVault, '.credentials.json'), 'utf8')).toBe('preserve-external-credential'); + const proof = { platform: process.platform, denied: [] as string[], keychainServices: {} as Record }; + for (const [i, path] of locations.entries()) { + await expect(command(sandboxed(profile, ['/bin/cat', path]), options)).rejects.toThrow('failed'); + const alias = join(home, `alias-${i}`); await symlink(path, alias); + await expect(command(sandboxed(profile, ['/bin/cat', alias]), options)).rejects.toThrow('failed'); + // A shell's child inherits the OS policy even without a provider-specific env. + await expect(command(sandboxed(profile, ['/bin/sh', '-c', 'exec /bin/cat "$1"', 'child', path]), options)).rejects.toThrow('failed'); + await expect(command(sandboxed(profile, ['/bin/sh', '-c', 'printf overwritten > "$1"', 'setup', path]), options)).rejects.toThrow('failed'); + expect(await readFile(path, 'utf8')).toBe('operator-sentinel-must-remain-private'); + proof.denied.push(path); + } + await expect(command(sandboxed(profile, ['/usr/bin/security', 'list-keychains']), options)).rejects.toThrow(); + // Exercise actual Mach lookup, not just a security CLI deny rule. Compilation + // happens in the trusted test parent; candidate execution inherits Seatbelt. + const source = join(home, 'keychain-probe.c'), binary = join(home, 'keychain-probe'); + await writeFile(source, '#include \n#include \nint main(int n,char **v){mach_port_t p=0; kern_return_t r=bootstrap_look_up(bootstrap_port,v[1],&p); printf("%d\\n",r); return 0;}\n'); + await command(['/usr/bin/clang', source, '-o', binary], { cwd: home }); + let reachable = 0; + for (const service of ['com.apple.securityd.xpc', 'com.apple.securityd', 'com.apple.SecurityServer', 'com.apple.SecurityAgent']) { + // A missing service is not denial proof. Look up the exact same service + // first from the unsandboxed trusted parent without accessing any Keychain. + const unsandboxed = Number((await command([binary, service], { cwd: home })).trim()); + expect(Number.isFinite(unsandboxed)).toBe(true); + if (unsandboxed === 0) reachable++; + const result = Number((await command(sandboxed(profile, [binary, service]), options)).trim()); + expect(Number.isFinite(result)).toBe(true); expect(result).not.toBe(0); + proof.keychainServices[service] = { unsandboxed, sandboxed: result }; + } + expect(reachable).toBeGreaterThan(0); + console.log('Mac filesystem/Keychain denial proof:', JSON.stringify(proof)); +}, 0); + +// Explicit opt-in is required before planting new, randomized non-secret files +// in the operator's real HOME. Existing auth/SSH/Keychain files are never touched. +const operatorMacTest = process.platform === 'darwin' && process.env.PSTACK_OPERATOR_SENTINELS === '1' ? test : test.skip; +operatorMacTest('actual operator HOME denial and immutable exercise sources with writable dependency leaves', async () => { + const daily = await realpath(process.env.HOME!), root = await fixture(), home = join(root, 'candidate'); + const directories = [daily, join(daily, '.config/gh'), join(daily, '.ssh'), join(daily, '.claude'), join(daily, '.codex'), join(daily, 'Library/Keychains')]; + const createdDirectories: string[] = [], sentinels: string[] = [], proof: Record = { platform: process.platform, daily, denied: [] }; + async function ensureDirectory(path: string): Promise { + try { + const info = await lstat(path); + if (!info.isDirectory() || info.isSymbolicLink() || await realpath(path) !== path) throw new Error(`Refuse redirected daily sentinel directory: ${path}`); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + await ensureDirectory(dirname(path)); + await mkdir(path, { mode: 0o700 }); createdDirectories.push(path); + } + } + try { + for (const directory of directories) { + await ensureDirectory(directory); + const path = join(directory, `.pstack-denial-${randomUUID()}`); + await writeFile(path, 'non-secret-operator-denial-sentinel', { flag: 'wx', mode: 0o600 }); sentinels.push(path); + } + console.log('Operator sentinel cleanup paths:', JSON.stringify(sentinels)); + await mkdir(join(home, 'tmp'), { recursive: true, mode: 0o700 }); + const workspace = join(home, 'workspace'), plugin = join(home, '.codex/plugins/pstack'); + await mkdir(workspace); await mkdir(plugin, { recursive: true }); + const source = join(workspace, 'source.ts'), installed = join(plugin, 'SKILL.md'); + await writeFile(source, 'pinned-workspace-source'); await writeFile(installed, 'pinned-plugin-source'); + const profile = await createSandbox(home), exercise = await protectSources(profile, home, [workspace, plugin]); + const options = { cwd: home, env: isolatedEnv(home) }; + for (const [index, path] of sentinels.entries()) { + // Positive controls prove the file exists and is readable outside Seatbelt. + expect(await command(['/bin/cat', path])).toBe('non-secret-operator-denial-sentinel'); + await expect(command(sandboxed(exercise, ['/bin/cat', path]), options)).rejects.toThrow(); + const alias = join(home, `daily-alias-${index}`); await symlink(path, alias); + await expect(command(sandboxed(exercise, ['/bin/cat', alias]), options)).rejects.toThrow(); + await expect(command(sandboxed(exercise, ['/bin/sh', '-c', 'exec /bin/sh -c \'exec /bin/cat "$1"\' child "$1"', 'grandchild', path]), options)).rejects.toThrow(); + await expect(command(sandboxed(exercise, ['/bin/sh', '-c', 'printf overwritten > "$1"', 'write', path]), options)).rejects.toThrow(); + expect(await readFile(path, 'utf8')).toBe('non-secret-operator-denial-sentinel'); + } + proof.denied = [...sentinels]; + for (const path of [source, installed]) { + const original = await readFile(path, 'utf8'); + expect(await command(sandboxed(exercise, ['/bin/cat', path]), options)).toBe(original); + await expect(command(sandboxed(exercise, ['/bin/sh', '-c', 'printf altered > "$1"', 'write', path]), options)).rejects.toThrow(); + await expect(command(sandboxed(exercise, ['/bin/mv', path, join(home, 'relocated-source')]), options)).rejects.toThrow(); + await expect(command(sandboxed(exercise, ['/bin/rm', path]), options)).rejects.toThrow(); + await expect(command(sandboxed(exercise, ['/bin/ln', path, join(home, 'source-hardlink')]), options)).rejects.toThrow(); + const alias = join(home, `source-alias-${randomUUID()}`); await symlink(path, alias); + await expect(command(sandboxed(exercise, ['/bin/sh', '-c', 'printf altered > "$1"', 'write', alias]), options)).rejects.toThrow(); + expect(await readFile(path, 'utf8')).toBe(original); + } + for (const path of [workspace, dirname(plugin), plugin]) { + await expect(command(sandboxed(exercise, ['/bin/mv', path, join(home, `moved-${randomUUID()}`)]), options)).rejects.toThrow(); + } + for (const path of [join(workspace, '.claude/skills/verify-open-pstack/node_modules'), join(workspace, 'plugins/pstack/skills/poteto-mode/scripts/node_modules'), join(plugin, 'skills/poteto-mode/scripts/node_modules')]) { + const dependency = join(path, 'generated'); + await command(sandboxed(exercise, ['/bin/sh', '-c', 'printf generated > "$1"', 'bootstrap', dependency]), options); + expect(await readFile(dependency, 'utf8')).toBe('generated'); + } + await expect(command(sandboxed(exercise, ['/bin/sh', '-c', 'printf changed > "$1"', 'policy', exercise]), options)).rejects.toThrow(); + await expect(command(sandboxed(exercise, ['/usr/bin/security', 'list-keychains']), options)).rejects.toThrow(); + const probeSource = join(home, 'keychain-probe.c'), binary = join(home, 'keychain-probe'); + await writeFile(probeSource, '#include \n#include \nint main(int n,char **v){mach_port_t p=0; kern_return_t r=bootstrap_look_up(bootstrap_port,v[1],&p); printf("%d\\n",r); return 0;}\n'); + await command(['/usr/bin/clang', probeSource, '-o', binary], { cwd: home }); + const services: Record = {}; let reachable = 0; + for (const service of ['com.apple.securityd.xpc', 'com.apple.securityd', 'com.apple.SecurityServer', 'com.apple.SecurityAgent']) { + const unsandboxed = Number((await command([binary, service], { cwd: home })).trim()); + const result = Number((await command(sandboxed(exercise, [binary, service]), options)).trim()); + expect(Number.isFinite(unsandboxed)).toBe(true); expect(Number.isFinite(result)).toBe(true); expect(result).not.toBe(0); + if (unsandboxed === 0) reachable++; + services[service] = { unsandboxed, sandboxed: result }; + } + expect(reachable).toBeGreaterThan(0); proof.keychainServices = services; + proof.sourcesProtected = [workspace, plugin]; + } finally { + for (const path of sentinels) await rm(path); + // Remove only directories created by this test, and only if still empty. + for (const path of createdDirectories.reverse()) await rmdir(path); + } + console.log('Actual operator HOME/source denial proof:', JSON.stringify({ ...proof, sentinelsRemoved: true })); +}, 0); diff --git a/.claude/skills/verify-open-pstack/scripts/isolation.ts b/.claude/skills/verify-open-pstack/scripts/isolation.ts new file mode 100644 index 00000000..ba4dc6c5 --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/isolation.ts @@ -0,0 +1,203 @@ +import { realpathSync } from 'node:fs'; +import { chmod, copyFile, lstat, mkdir, readFile, realpath, rm, stat, writeFile } from 'node:fs/promises'; +import { basename, dirname, isAbsolute, join, resolve } from 'node:path'; +import { command, isolatedEnv, type Command } from './io.ts'; +import { HARNESSES, type Accounts } from './types.ts'; + +export function vaultRoot(): string { + if (process.env.CAAM_HOME) return join(process.env.CAAM_HOME, 'data/vault'); + if (process.env.XDG_DATA_HOME) return join(process.env.XDG_DATA_HOME, 'caam/vault'); + if (!process.env.HOME || !isAbsolute(process.env.HOME)) throw new Error('Trusted parent requires an absolute operator HOME'); + return join(process.env.HOME, '.local/share/caam/vault'); +} +export function selectedAccounts(text: string, accounts: Accounts): Accounts { + const result = JSON.parse(text); + if (!Array.isArray(result.profiles)) throw new Error('Unknown caam list schema; expected profiles'); + for (const tool of HARNESSES) { + const name = accounts[tool]; + if (!/^[A-Za-z0-9.!#$%&'*+=?^_`{|}~-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/.test(name) || name.length > 254) throw new Error(`Explicit safe ${tool} account email required`); + const matching = result.profiles.filter((p: Record) => p.tool === tool && p.name === name); + if (matching.length !== 1 || matching[0].identity?.email !== name) throw new Error(`caam ${tool}/${name} identity mismatch or missing; inspect caam list --json (never activate or repair automatically)`); + } + return accounts; +} +export async function copyCredentials(home: string, vault: string, accounts: Accounts, copied: string[]): Promise { + const base = await realpath(vault); + for (const tool of HARNESSES) { + const file = tool === 'claude' ? '.credentials.json' : 'auth.json'; + const profile = join(base, tool, accounts[tool]), source = join(profile, file); + for (const dir of [join(base, tool), profile]) { + const info = await lstat(dir); + if (!dir.startsWith(base + '/') || !info.isDirectory() || info.isSymbolicLink() || await realpath(dir) !== dir) throw new Error('Selected vault directories must be canonical, not symlinked'); + } + const stat = await lstat(source); + if (!stat.isFile() || stat.isSymbolicLink() || await realpath(source) !== source) throw new Error('Vault credentials must be a contained regular file'); + const bytes = await readFile(source); + const auth = JSON.parse(bytes.toString()); + if (tool === 'claude' ? typeof auth.claudeAiOauth?.accessToken !== 'string' : typeof auth.tokens?.access_token !== 'string') throw new Error(`${tool} vault requires file-backed OAuth credentials; unknown format blocks`); + const dir = join(home, tool === 'claude' ? '.claude' : '.codex'); + await mkdir(dir, { recursive: true, mode: 0o700 }); + const target = join(dir, file); + copied.push(target); + await writeFile(target, bytes, { mode: 0o600, flag: 'wx' }); + } +} +export async function removeCredentials(paths: string[], profiles?: Map, run: Command = command): Promise { + for (const path of paths) { + const home = dirname(dirname(path)); + if (profiles) { + const profile = profiles.get(home); + if (!profile) throw new Error('Credential cleanup requires the original candidate sandbox'); + await run(sandboxed(profile, ['/bin/rm', '-f', '--', path]), { env: isolatedEnv(home), cwd: home, allowInterrupted: true }); + } else { + // Trusted fixtures only. Production deletion is OS-contained against + // candidate-controlled ancestor replacement, including symlink races. + if (await realpath(dirname(path)) !== dirname(path)) throw new Error('Credential cleanup refuses redirected parent'); + const info = await lstat(path).catch((error: NodeJS.ErrnoException) => { + if (error.code !== 'ENOENT') throw error; + return undefined; + }); + if (info?.isSymbolicLink() || (info && !info.isFile())) throw new Error('Credential cleanup refuses nonregular file'); + await rm(path, { force: true }); + } + } +} +const quote = (value: string): string => JSON.stringify(value); +export function sandboxProfile(home: string, daily: string, executables: string[], deniedVaultRoot?: string): string { + if (!isAbsolute(home) || !isAbsolute(daily) || home === daily || home.startsWith(daily + '/')) throw new Error('Sandbox state must be outside the operator home'); + const vault = deniedVaultRoot === undefined ? undefined : realpathSync(deniedVaultRoot); + if (vault) { + const candidate = realpathSync(home); + if (candidate === vault || candidate.startsWith(vault + '/') || vault.startsWith(candidate + '/')) throw new Error('Candidate state and caam vault must not overlap'); + } + // File reads are allowlisted, not merely denied through HOME aliases. Mach + // services are allowlisted too: no securityd, SecurityAgent, or Keychain IPC. + return `(version 1) +(deny default) +(allow process-exec process-fork) +(allow pseudo-tty) +(allow file-ioctl (literal "/dev/ptmx") (regex #"^/dev/ttys")) +(allow signal (target same-sandbox)) +(allow process-info* (target same-sandbox)) +(allow sysctl-read) +(allow file-read-metadata) +(allow file-read* (subpath ${quote(home)}) + (subpath "/System") (subpath "/usr") (subpath "/bin") (subpath "/sbin") + (subpath "/opt/homebrew") (subpath "/Library/Apple") (subpath "/dev") + (subpath "/private/etc") (subpath "/private/var/db/dyld") + ${executables.map(path => `(literal ${quote(path)})`).join('\n ')}) +(allow file-write* (subpath ${quote(home)}) (subpath "/dev")) +(allow network-outbound (remote ip "*:*")) +(allow mach-lookup + (global-name "com.apple.trustd") (global-name "com.apple.trustd.agent") + (global-name "com.apple.mDNSResponder") (global-name "com.apple.system.notification_center") + (global-name "com.apple.SystemConfiguration.configd") (global-name "com.apple.bsd.dirhelper")) +(deny file-read* (require-not (require-any + (subpath ${quote(home)}) + (subpath "/System") (subpath "/usr") (subpath "/bin") (subpath "/sbin") + (subpath "/opt/homebrew") (subpath "/Library/Apple") (subpath "/dev") + (subpath "/private/etc") (subpath "/private/var/db/dyld") + ${executables.map(path => `(literal ${quote(path)})`).join('\n ')}))) +(deny file-read* file-write* (subpath ${quote(daily)})) +${vault ? `(deny file-read* file-write* (subpath ${quote(vault)}))` : ''} +(deny file-read* file-write* (subpath "/Library/Keychains") (subpath "/System/Library/Keychains")) +(deny file-write* (require-not (require-any (subpath ${quote(home)}) (subpath "/dev")))) +(deny mach-lookup (require-not (require-any + (global-name "com.apple.trustd") (global-name "com.apple.trustd.agent") + (global-name "com.apple.mDNSResponder") (global-name "com.apple.system.notification_center") + (global-name "com.apple.SystemConfiguration.configd") (global-name "com.apple.bsd.dirhelper")))) +(deny process-info* (require-not (target same-sandbox))) +(deny process-exec (literal "/usr/bin/security")) +(deny network-outbound (remote unix-socket (path-regex #".*"))) +(deny network-outbound (remote ip "localhost:*")) +`; +} +export async function createSandbox(home: string, run: Command = command, deniedVaultRoot?: string): Promise { + const daily = await realpath(process.env.HOME!); + const vault = deniedVaultRoot === undefined ? undefined : await realpath(deniedVaultRoot); + const candidate = await realpath(home); + // Validate boundaries before copying executables or writing the profile. + sandboxProfile(candidate, daily, [], vault); + const executables: string[] = []; + for (const name of ['claude', 'codex', 'bun', 'node', 'git']) { + const path = Bun.which(name); + if (!path && ['claude', 'codex', 'bun'].includes(name)) throw new Error(`Missing ${name}`); + if (path) { + const actual = await realpath(path); + if (actual.startsWith(daily + '/') || path.startsWith(daily + '/')) { + // Only standalone native binaries can be relocated without dependencies + // on daily state. Never grant a home-directory read exception. + const magic = (await readFile(actual)).subarray(0, 4).toString('hex'); + if (!['cffaedfe', 'cefaedfe', 'feedfacf', 'feedface', 'cafebabe', 'bebafeca'].includes(magic)) throw new Error(`Install standalone ${name} outside daily HOME; cannot safely relocate a script package`); + await mkdir(join(home, 'bin'), { recursive: true, mode: 0o700 }); + const target = join(home, 'bin', name); + await copyFile(actual, target); + await chmod(target, (await stat(actual)).mode & 0o777); + executables.push(target); + } else executables.push(path, actual); + } + } + // The policy stays outside candidate-writable state; later subprocesses must + // not load a profile that the candidate can replace between invocations. + const path = join(dirname(home), `${basename(home)}.sb`); + await writeFile(path, sandboxProfile(candidate, daily, executables, vault), { mode: 0o600, flag: 'wx' }); + await run(['/usr/bin/sandbox-exec', '-f', path, '/usr/bin/true'], { env: isolatedEnv(home), cwd: home }); + return path; +} +async function sourceDirectory(path: string): Promise { + if (!isAbsolute(path) || resolve(path) !== path) throw new Error('Source paths must be canonical absolute directories'); + let current = path; + for (;;) { + const info = await lstat(current); + if (info.isSymbolicLink() || !info.isDirectory()) throw new Error(`Redirected source directory: ${current}`); + if (dirname(current) === current) break; + current = dirname(current); + } + if (await realpath(path) !== path) throw new Error(`Redirected source directory: ${path}`); +} +export async function protectSources(profile: string, home: string, sources: string[], run: Command = command): Promise { + await sourceDirectory(home); + if (sources.length === 0) throw new Error('Exercise requires pinned source roots'); + await sourceDirectory(dirname(profile)); + if (!isAbsolute(profile) || resolve(profile) !== profile || profile === home || profile.startsWith(home + '/')) throw new Error('Exercise policy must be outside candidate state'); + const info = await lstat(profile); + if (!info.isFile() || info.isSymbolicLink()) throw new Error('Exercise policy must be a regular trusted file'); + const rules: string[] = []; + for (const source of new Set(sources)) { + await sourceDirectory(source); + if (!source.startsWith(home + '/')) throw new Error('Sources must be contained in candidate state'); + const workspace = source === join(home, 'workspace'); + const dependencies = workspace + ? [join(source, '.claude/skills/verify-open-pstack/node_modules'), join(source, 'plugins/pstack/skills/poteto-mode/scripts/node_modules')] + : [join(source, 'skills/poteto-mode/scripts/node_modules')]; + for (const dependency of dependencies) { + // Create the allowed leaves before freezing their parents; no redirected + // source ancestor may be used to create or exempt a dependency directory. + let ancestor = dirname(dependency); + for (;;) { + try { await lstat(ancestor); break; } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + ancestor = dirname(ancestor); + } + } + await sourceDirectory(ancestor); + await mkdir(dependency, { recursive: true, mode: 0o700 }); + await sourceDirectory(dependency); + } + rules.push(`(deny file-write* (require-all (subpath ${quote(source)}) (require-not (require-any ${dependencies.map(path => `(subpath ${quote(path)})`).join(' ')}))))`); + // Pin ancestors too: moving an installed-plugin parent must not relocate + // protected files outside the path-based policy between hash checks. + const pinned = new Set(); + for (const path of [source, ...dependencies.map(dirname)]) { + for (let current = path; current.startsWith(home + '/'); current = dirname(current)) pinned.add(current); + } + rules.push(`(deny file-write-unlink ${[...pinned].map(path => `(literal ${quote(path)})`).join(' ')})`); + } + const target = `${profile}.exercise.sb`; + await writeFile(target, (await readFile(profile, 'utf8')) + '\n' + rules.join('\n') + '\n', { mode: 0o600, flag: 'wx' }); + await run(sandboxed(target, ['/usr/bin/true']), { env: isolatedEnv(home), cwd: home }); + return target; +} +export function sandboxed(profile: string, args: string[]): string[] { + return ['/usr/bin/sandbox-exec', '-f', profile, ...args]; +} diff --git a/.claude/skills/verify-open-pstack/scripts/provenance.test.ts b/.claude/skills/verify-open-pstack/scripts/provenance.test.ts new file mode 100644 index 00000000..d8aae7c0 --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/provenance.test.ts @@ -0,0 +1,177 @@ +import { afterEach, expect, test } from 'bun:test'; +import { chmod, mkdtemp, mkdir, realpath, rename, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { command } from './io.ts'; +import { sourceDigest, sourceHash } from './provenance.ts'; + +const roots: string[] = []; +const project = '.claude/skills/verify-open-pstack'; +const plugin = 'plugins/pstack'; +const alias = '.agents/skills/verify-open-pstack'; +afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); +async function fixture(): Promise<{ root: string; sha: string }> { + const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-provenance-'))); roots.push(root); + await mkdir(join(root, project, 'scripts'), { recursive: true }); + await mkdir(join(root, plugin, 'skills/poteto-mode/scripts'), { recursive: true }); + await mkdir(join(root, '.agents/skills'), { recursive: true }); + await writeFile(join(root, project, 'SKILL.md'), 'native project skill\n'); + await writeFile(join(root, project, 'scripts/verify.sh'), '#!/bin/sh\ntrue\n', { mode: 0o755 }); + await writeFile(join(root, plugin, 'plugin.json'), '{"version":"1"}\n'); + await writeFile(join(root, plugin, 'skills/poteto-mode/scripts/bootstrap.ts'), 'bootstrap\n'); + await symlink('../../.claude/skills/verify-open-pstack', join(root, alias)); + for (const args of [ + ['git', 'init', '--quiet'], ['git', 'config', 'user.name', 'Provenance Test'], + ['git', 'config', 'user.email', 'test@example.invalid'], ['git', 'add', '.'], + ['git', '-c', 'commit.gpgsign=false', 'commit', '--quiet', '-m', 'fixture'], + ]) await command(args, { cwd: root }); + return { root, sha: (await command(['git', 'rev-parse', 'HEAD'], { cwd: root })).trim() }; +} + +test('real pinned source remains stable after both permitted dependency bootstraps, including symlinks', async () => { + const { root, sha } = await fixture(), before = await sourceHash(root, sha); + const dependencyTarget = join(root, 'generated-dependency'); + await mkdir(dependencyTarget); await writeFile(join(dependencyTarget, 'runtime.js'), 'generated\n'); + for (const path of [`${project}/node_modules`, `${plugin}/skills/poteto-mode/scripts/node_modules`]) await symlink(dependencyTarget, join(root, path)); + expect(await sourceHash(root, sha)).toBe(before); + await writeFile(join(dependencyTarget, 'runtime.js'), 'generated update\n'); + expect(await sourceHash(root, sha)).toBe(before); + expect(before).toMatch(/^[a-f0-9]{64}$/); + expect(await sourceDigest(root)).toBe(before); +}); + +test('canonical project code mutation and added source fail against pinned Git blobs', async () => { + const { root, sha } = await fixture(); + await writeFile(join(root, project, 'SKILL.md'), 'changed instructions\n'); + await expect(sourceHash(root, sha)).rejects.toThrow('Source differs from pinned Git tree'); + await writeFile(join(root, project, 'SKILL.md'), 'native project skill\n'); + await writeFile(join(root, project, 'scripts/new.ts'), 'untracked candidate code\n'); + await expect(sourceHash(root, sha)).rejects.toThrow('Source differs from pinned Git tree'); +}); + +test('plugin code mutation, executable-bit changes and missing source fail against the pinned tree', async () => { + const { root, sha } = await fixture(); + await writeFile(join(root, plugin, 'plugin.json'), '{}\n'); + await expect(sourceHash(root, sha)).rejects.toThrow('Source differs from pinned Git tree'); + await writeFile(join(root, plugin, 'plugin.json'), '{"version":"1"}\n'); + await command(['git', 'config', 'core.filemode', 'false'], { cwd: root }); + await chmod(join(root, project, 'scripts/verify.sh'), 0o644); + await expect(sourceHash(root, sha)).rejects.toThrow('Source differs from pinned Git tree'); + await chmod(join(root, project, 'scripts/verify.sh'), 0o755); + await rm(join(root, project, 'SKILL.md')); + await expect(sourceHash(root, sha)).rejects.toThrow('Source differs from pinned Git tree'); +}); + +test('Codex alias target must be exact and canonical even when alternate spelling resolves identically', async () => { + const { root, sha } = await fixture(); + await rm(join(root, alias)); + await symlink('../../.claude/skills/../skills/verify-open-pstack', join(root, alias)); + await expect(sourceHash(root, sha)).rejects.toThrow('Codex alias'); + await rm(join(root, alias)); + await mkdir(join(root, 'other-skill')); + await symlink('../../other-skill', join(root, alias)); + await expect(sourceHash(root, sha)).rejects.toThrow('Codex alias'); +}); + +test('generated exclusions are narrow and source symlinks are not accepted', async () => { + const { root, sha } = await fixture(); + await mkdir(join(root, project, 'scripts/node_modules')); + await writeFile(join(root, project, 'scripts/node_modules/hidden.ts'), 'unexpected\n'); + await expect(sourceHash(root, sha)).rejects.toThrow('Source differs from pinned Git tree'); + await rm(join(root, project, 'scripts/node_modules'), { recursive: true }); + await rm(join(root, project, 'SKILL.md')); + await symlink(join(root, plugin, 'plugin.json'), join(root, project, 'SKILL.md')); + await expect(sourceHash(root, sha)).rejects.toThrow('Source differs from pinned Git tree'); +}); + +test('filesystem-only rechecks cannot be spoofed by modified Git metadata or a forged Git executable', async () => { + const { root, sha } = await fixture(), pinned = await sourceHash(root, sha); + expect(await sourceDigest(root)).toBe(pinned); + await rm(join(root, '.git'), { recursive: true }); + await mkdir(join(root, '.git')); + await writeFile(join(root, '.git/HEAD'), 'forged pinned metadata\n'); + const bin = join(root, 'bin'), invoked = join(root, 'git-invoked'); + await mkdir(bin); + await writeFile(join(bin, 'git'), `#!/bin/sh\nprintf invoked > '${invoked}'\nprintf '${sha}\\n'\n`, { mode: 0o755 }); + const path = process.env.PATH; + try { + process.env.PATH = bin; + expect(await sourceDigest(root)).toBe(pinned); + await writeFile(join(root, project, 'SKILL.md'), 'forged project source\n'); + expect(await sourceDigest(root)).not.toBe(pinned); + expect(await Bun.file(invoked).exists()).toBe(false); + } finally { + if (path === undefined) delete process.env.PATH; + else process.env.PATH = path; + } +}); + +test('filesystem digest detects plugin/source additions, deletions, and executable mode changes', async () => { + const { root, sha } = await fixture(), pinned = await sourceHash(root, sha); + const manifest = join(root, plugin, 'plugin.json'); + await writeFile(manifest, 'changed plugin\n'); + expect(await sourceDigest(root)).not.toBe(pinned); + await writeFile(manifest, '{"version":"1"}\n'); + expect(await sourceDigest(root)).toBe(pinned); + const added = join(root, project, 'scripts/extra.ts'); + await writeFile(added, 'extra code\n'); + expect(await sourceDigest(root)).not.toBe(pinned); + await rm(added); + await chmod(join(root, project, 'scripts/verify.sh'), 0o644); + expect(await sourceDigest(root)).not.toBe(pinned); + await chmod(join(root, project, 'scripts/verify.sh'), 0o755); + expect(await sourceDigest(root)).toBe(pinned); + await rm(manifest); + expect(await sourceDigest(root)).not.toBe(pinned); +}); + +test('filesystem rechecks reject a candidate workspace redirected to an untouched pinned copy', async () => { + const { root, sha } = await fixture(), pinned = await sourceHash(root, sha); + const untouched = `${root}-untouched`; + await rename(root, untouched); roots.push(untouched); + await symlink(untouched, root); + expect(await sourceDigest(untouched)).toBe(pinned); + await expect(sourceDigest(root)).rejects.toThrow('workspace root must not be redirected'); +}); + +test('filesystem rechecks reject noncanonical aliases and source symlinks including ancestor directories', async () => { + const { root } = await fixture(); + await rm(join(root, alias)); + await symlink('../../.claude/skills/../skills/verify-open-pstack', join(root, alias)); + await expect(sourceDigest(root)).rejects.toThrow('Codex alias'); + await rm(join(root, alias)); + await symlink('../../.claude/skills/verify-open-pstack', join(root, alias)); + await rm(join(root, project, 'SKILL.md')); + await symlink(join(root, plugin, 'plugin.json'), join(root, project, 'SKILL.md')); + await expect(sourceDigest(root)).rejects.toThrow('source symlink'); + await rm(join(root, project, 'SKILL.md')); + await writeFile(join(root, project, 'SKILL.md'), 'native project skill\n'); + await rm(join(root, '.agents/skills'), { recursive: true }); + await mkdir(join(root, 'redirected-skills')); + await symlink('../../.claude/skills/verify-open-pstack', join(root, 'redirected-skills/verify-open-pstack')); + await symlink(join(root, 'redirected-skills'), join(root, '.agents/skills')); + await expect(sourceDigest(root)).rejects.toThrow('source directory'); +}); + +test('filesystem digest does not exclude dependency-like directories outside the two permitted paths', async () => { + const { root, sha } = await fixture(), pinned = await sourceHash(root, sha); + await mkdir(join(root, project, 'scripts/node_modules')); + await writeFile(join(root, project, 'scripts/node_modules/source.ts'), 'unexpected code\n'); + expect(await sourceDigest(root)).not.toBe(pinned); + await rm(join(root, project, 'scripts/node_modules/source.ts')); + await symlink(join(root, plugin, 'plugin.json'), join(root, project, 'scripts/node_modules/source.ts')); + await expect(sourceDigest(root)).rejects.toThrow('source symlink'); +}); + +test('HEAD must match exact pinned source and committed dependency directories cannot be excluded', async () => { + const { root, sha } = await fixture(); + await expect(sourceHash(root, 'a'.repeat(40))).rejects.toThrow('HEAD differs'); + await expect(sourceHash(root, 'HEAD')).rejects.toThrow('exact pinned SHA'); + await mkdir(join(root, project, 'node_modules')); + await writeFile(join(root, project, 'node_modules/committed.ts'), 'tracked runtime\n'); + await command(['git', 'add', '.'], { cwd: root }); + await command(['git', '-c', 'commit.gpgsign=false', 'commit', '--quiet', '-m', 'tracked dependencies'], { cwd: root }); + const changed = (await command(['git', 'rev-parse', 'HEAD'], { cwd: root })).trim(); + await expect(sourceHash(root, sha)).rejects.toThrow('HEAD differs'); + await expect(sourceHash(root, changed)).rejects.toThrow('Pinned source files missing'); +}); diff --git a/.claude/skills/verify-open-pstack/scripts/provenance.ts b/.claude/skills/verify-open-pstack/scripts/provenance.ts new file mode 100644 index 00000000..13933e8e --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/provenance.ts @@ -0,0 +1,98 @@ +import { createHash } from 'node:crypto'; +import { lstat, readFile, readlink, readdir, realpath } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import { command, sha256, treeHash, type Command } from './io.ts'; + +const plugin = 'plugins/pstack'; +const project = '.claude/skills/verify-open-pstack'; +const alias = '.agents/skills/verify-open-pstack'; +const aliasTarget = '../../.claude/skills/verify-open-pstack'; +const generated = new Set([ + 'plugins/pstack/skills/poteto-mode/scripts/node_modules', + '.claude/skills/verify-open-pstack/node_modules', +]); + +export async function sourceDigest(workspace: string): Promise { + const root = await realpath(workspace); + if (root !== resolve(workspace) || (await lstat(workspace)).isSymbolicLink()) throw new Error('Pinned workspace root must not be redirected'); + for (const path of ['plugins', plugin, '.claude', '.claude/skills', project, '.agents', '.agents/skills']) { + const stat = await lstat(join(root, path)); + if (stat.isSymbolicLink() || !stat.isDirectory()) throw new Error(`Pinned source directory refused: ${path}`); + } + const aliasPath = join(root, alias); + if (!(await lstat(aliasPath)).isSymbolicLink() || await readlink(aliasPath) !== aliasTarget || await realpath(aliasPath) !== join(root, project)) { + throw new Error('Codex alias does not resolve to the canonical pinned project skill'); + } + const entries: string[] = []; + async function walk(path: string): Promise { + if (generated.has(path)) return; + const actual = join(root, path), stat = await lstat(actual); + if (stat.isSymbolicLink()) throw new Error(`Pinned source symlink refused: ${path}`); + if (stat.isDirectory()) { + for (const name of (await readdir(actual)).sort()) await walk(`${path}/${name}`); + } else if (stat.isFile()) { + entries.push(`${path}\0${stat.mode & 0o111 ? '100755' : '100644'}\0${sha256(await readFile(actual))}`); + } else throw new Error(`Non-file in pinned source tree: ${path}`); + } + await walk(plugin); + await walk(project); + entries.push([alias, '120000', sha256(Buffer.from(await readlink(aliasPath)))].join('\0')); + return sha256(entries.sort().join('\n')); +} + +export async function sourceHash(workspace: string, sha: string, run: Command = command): Promise { + if (!/^[a-f0-9]{40}$/.test(sha)) throw new Error('Source provenance requires an exact pinned SHA'); + const git = ['git', '--no-replace-objects', '-c', 'core.fsmonitor=false', '-c', 'core.hooksPath=/dev/null']; + if ((await run([...git, 'rev-parse', 'HEAD'], { cwd: workspace })).trim() !== sha) { + throw new Error('Project source HEAD differs from pinned SHA'); + } + const aliasPath = join(workspace, alias); + if (!(await lstat(aliasPath)).isSymbolicLink() || await readlink(aliasPath) !== aliasTarget || await realpath(aliasPath) !== await realpath(join(workspace, project))) { + throw new Error('Codex alias does not resolve to the canonical pinned project skill'); + } + try { + await run([...git, 'diff', '--quiet', '--no-ext-diff', '--no-textconv', sha, '--', plugin, project, alias], { cwd: workspace }); + } catch { + throw new Error('Source differs from pinned Git tree'); + } + const tree = await run([...git, 'ls-tree', '-rz', '--full-tree', sha, '--', plugin, project, alias], { cwd: workspace }); + const expected = new Map(); + for (const entry of tree.split('\0').filter(Boolean)) { + const tab = entry.indexOf('\t'), fields = entry.slice(0, tab).split(' '), path = entry.slice(tab + 1); + if (tab < 0 || fields.length !== 3 || fields[1] !== 'blob' || !/^(100644|100755|120000)$/.test(fields[0]!) || !/^[a-f0-9]{40}$/.test(fields[2]!)) { + throw new Error('Unrecognized pinned source tree'); + } + if (path !== alias && !path.startsWith(plugin + '/') && !path.startsWith(project + '/')) throw new Error('Pinned source path escaped coverage'); + if (expected.has(path)) throw new Error('Duplicate pinned source path'); + expected.set(path, { mode: fields[0]!, blob: fields[2]! }); + } + if (!expected.has(alias) || ![...expected.keys()].some(path => path.startsWith(project + '/')) || ![...expected.keys()].some(path => path.startsWith(plugin + '/'))) { + throw new Error('Pinned source tree lacks plugin, canonical project skill, or Codex alias'); + } + const entries: string[] = []; + function accept(path: string, mode: string, bytes: Uint8Array): void { + const pinned = expected.get(path); + const blob = createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex'); + if (!pinned || pinned.mode !== mode || pinned.blob !== blob) throw new Error(`Source differs from pinned Git tree: ${path}`); + expected.delete(path); + entries.push(`${path}\0${mode}\0${sha256(bytes)}`); + } + async function walk(path: string): Promise { + // Only these two untracked dependency directories may vary after bootstrap. + if (generated.has(path)) return; + const actual = join(workspace, path), stat = await lstat(actual); + if (stat.isSymbolicLink()) throw new Error(`Pinned source symlink refused: ${path}`); + if (stat.isDirectory()) { + for (const name of (await readdir(actual)).sort()) await walk(`${path}/${name}`); + } else if (stat.isFile()) { + accept(path, stat.mode & 0o111 ? '100755' : '100644', await readFile(actual)); + } else throw new Error(`Non-file in pinned source tree: ${path}`); + } + const pluginHash = await treeHash(join(workspace, plugin), ['skills/poteto-mode/scripts/node_modules']); + await walk(plugin); + if (sha256(entries.map(entry => entry.slice(plugin.length + 1)).join('\n')) !== pluginHash) throw new Error('Plugin source changed while checking provenance'); + await walk(project); + accept(alias, '120000', Buffer.from(await readlink(aliasPath))); + if (expected.size) throw new Error(`Pinned source files missing: ${[...expected.keys()].join(', ')}`); + return sha256(entries.sort().join('\n')); +} diff --git a/.claude/skills/verify-open-pstack/scripts/types.ts b/.claude/skills/verify-open-pstack/scripts/types.ts index c310016e..550cfab3 100644 --- a/.claude/skills/verify-open-pstack/scripts/types.ts +++ b/.claude/skills/verify-open-pstack/scripts/types.ts @@ -1,10 +1,11 @@ export const REPO = 'ericlitman/open-pstack'; export const HARNESSES = ['claude', 'codex'] as const; export type Harness = typeof HARNESSES[number]; +export type Accounts = Record; export type Phase = 'resolve' | 'classify' | 'prepare' | 'exercise' | 'publish' | 'ready' | 'failed' | 'head-moved'; export interface Pull { number: number; head: { sha: string }; base: { sha: string }; draft: boolean; - state: string; headRepo: string; + state: string; headRepo: string; body: string; } export interface ChangedFile { filename: string; previous_filename?: string } export interface Registry { @@ -24,7 +25,7 @@ export interface Observation { } export interface Installation { harness: Harness; cliVersion: string; pluginVersion: string; sha: string; - location: string; treeHash: string; home: string; + location: string; treeHash: string; home: string; account?: string; sourceHash?: string; } export interface Receipt { schema: 1; repo: string; pr: number; sha: string; base: string; phase: Phase; @@ -35,7 +36,8 @@ export interface Receipt { } export interface GitHub { pull(pr: number): Promise; - files(pr: number): Promise; + files(base: string, head: string): Promise; + body(pr: number, body: string): Promise; comment(pr: number, body: string): Promise; status(sha: string, state: 'success' | 'failure', target: string, description: string): Promise; ready(pr: number): Promise; @@ -44,4 +46,5 @@ export interface GitHub { export interface Driver { prepare(receipt: Receipt): Promise; exercise(receipt: Receipt): Promise; + cleanup?(receipt: Receipt): Promise; } diff --git a/.claude/skills/verify-open-pstack/scripts/verify.test.ts b/.claude/skills/verify-open-pstack/scripts/verify.test.ts index cfab1ba4..fb488e33 100644 --- a/.claude/skills/verify-open-pstack/scripts/verify.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/verify.test.ts @@ -1,28 +1,39 @@ -import { describe, expect, test } from 'bun:test'; +import { afterEach, describe, expect, test } from 'bun:test'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import data from '../features/registry.json'; import { requiredFeatures, validateRegistry } from './core.ts'; -import { Publisher } from './github.ts'; -import { evidence, verify } from './verify.ts'; +import { parseChangedFiles, Publisher } from './github.ts'; +import { command, sha256 } from './io.ts'; +import { evidence, liveEvidenceBody, verify } from './verify.ts'; import { HARNESSES, REPO, type Driver, type GitHub, type Pull, type Receipt } from './types.ts'; const SHA = 'a'.repeat(40), BASE = 'b'.repeat(40), NEXT = 'c'.repeat(40), HASH = 'd'.repeat(64); const URL = `https://github.com/${REPO}/pull/123#issuecomment-1`; +const roots: string[] = []; +function temporary(): string { const root = realpathSync(mkdtempSync(join(tmpdir(), 'pstack-publish-test-'))); roots.push(root); return root; } +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); function fixture(runtime = false, selfTest = false) { + const root = temporary(); + writeFileSync(join(root, 'retained.log'), 'reviewed transcript'); writeFileSync(join(root, 'fixture.json'), 'reviewed artifact'); const calls: string[] = [], saved: Receipt[] = []; - const pull: Pull = { number: 123, head: { sha: SHA }, base: { sha: BASE }, draft: true, state: 'open', headRepo: REPO }; + const pull: Pull = { number: 123, head: { sha: SHA }, base: { sha: BASE }, draft: true, state: 'open', headRepo: REPO, body: '- [ ] The exact candidate is installed in every affected harness.\n- [ ] The changed behavior passes from each real user surface.\n- [ ] The installed version, action, and observed result appear below.\n\nLive evidence:\n\n_pending_\n\n## Operator notes\nRetain these notes.' }; let hook: (call: string) => void = () => {}; const mark = (call: string) => { calls.push(call); hook(call); }; const github: GitHub = { async pull() { mark('pull'); return structuredClone(pull); }, - async files() { mark('files'); return [{ filename: runtime ? 'plugins/pstack/skills/architect/SKILL.md' : 'README.md' }]; }, + async files(base, head) { expect(base).toBe(BASE); expect(head).toBe(SHA); mark('files'); return [{ filename: runtime ? 'plugins/pstack/skills/architect/SKILL.md' : 'README.md' }]; }, + async body(_pr, body) { mark('body'); pull.body = body; }, async comment(_pr, body) { mark('comment'); expect(body).toContain(SHA); return URL; }, async status(sha, state, target) { mark(`status:${state}:${sha}`); expect(target).toBe(URL); }, async ready() { mark('ready'); pull.draft = false; }, async draft() { mark('draft'); pull.draft = true; }, }; const driver: Driver = { async prepare(r) { mark('prepare'); return HARNESSES.map(harness => ({ harness, sha: r.sha, cliVersion: 'test', pluginVersion: '1.5.0', treeHash: HASH, location: `/isolated/${harness}/plugin`, home: `/isolated/${harness}` })); }, - async exercise(r) { mark('exercise'); return HARNESSES.flatMap(harness => requiredFeatures(r).map(feature => ({ harness, feature, surface: 'native surface', action: 'invoke', observed: 'fixture changed', reviewer: 'operator' as const, transcript: 'retained.log', transcriptHash: HASH, artifacts: [{ path: 'fixture.json', sha256: HASH }] }))); }, + async exercise(r) { mark('exercise'); return HARNESSES.flatMap(harness => requiredFeatures(r).map(feature => ({ harness, feature, surface: 'native surface', action: 'invoke', observed: 'fixture changed', reviewer: 'operator' as const, transcript: 'retained.log', transcriptHash: sha256('reviewed transcript'), artifacts: [{ path: 'fixture.json', sha256: sha256('reviewed artifact') }] }))); }, }; - const options = { pr: 123, root: '/tmp/evidence', selfTest, registry: validateRegistry(data), github, driver, persist: async (r: Receipt) => { saved.push(structuredClone(r)); } }; + const options = { pr: 123, root, selfTest, registry: validateRegistry(data), github, driver, persist: async (r: Receipt) => { saved.push(structuredClone(r)); } }; return { options, calls, saved, pull, hook: (fn: typeof hook) => { hook = fn; } }; } @@ -32,8 +43,12 @@ describe('exact-head publication state machine', () => { expect(receipt.status).toBe('success'); expect(receipt.sha).toBe(SHA); expect(evidence(receipt)).toContain('no runtime change'); expect(f.calls).not.toContain('prepare'); expect(f.calls).not.toContain('exercise'); - expect(f.calls.indexOf('comment')).toBeLessThan(f.calls.indexOf(`status:success:${SHA}`)); + expect(f.calls.indexOf('comment')).toBeLessThan(f.calls.indexOf('body')); + expect(f.calls.indexOf('body')).toBeLessThan(f.calls.indexOf(`status:success:${SHA}`)); expect(f.calls.indexOf(`status:success:${SHA}`)).toBeLessThan(f.calls.indexOf('ready')); + expect(f.pull.body).toContain(URL); expect(f.pull.body).toContain('Retain these notes.'); expect(f.pull.body).not.toContain('_pending_'); + expect(f.pull.body).toContain('- [x] The installed version, action, and observed result appear below.'); + expect(liveEvidenceBody(f.pull.body, receipt)).toBe(f.pull.body); }); test('all changed surfaces and explicit project self-test run in both harnesses', async () => { const f = fixture(true, true), r = await verify(f.options); @@ -43,6 +58,90 @@ describe('exact-head publication state machine', () => { test('already-ready PR is never toggled ready', async () => { const f = fixture(); f.pull.draft = false; await verify(f.options); expect(f.calls).not.toContain('ready'); }); + test('successful verification cleans credentials once before any publication', async () => { + const f = fixture(true); + f.options.driver.cleanup = async () => { f.calls.push('cleanup'); }; + const receipt = await verify(f.options); + expect(f.calls.filter(call => call === 'cleanup')).toHaveLength(1); + expect(f.calls.indexOf('exercise')).toBeLessThan(f.calls.indexOf('cleanup')); + expect(f.calls.indexOf('cleanup')).toBeLessThan(f.calls.indexOf('comment')); + expect(receipt.cleanup).toContain('removed'); + expect(receipt.status).toBe('success'); + }); + test('cleanup failure is retried and recorded before failed publication, preserving a ready PR', async () => { + const f = fixture(true); f.pull.draft = false; + f.options.driver.cleanup = async () => { f.calls.push('cleanup'); throw new Error('credential removal denied'); }; + const comment = f.options.github.comment; + f.options.github.comment = async (pr, body) => { + expect(f.saved.at(-1)?.phase).toBe('failed'); + expect(f.saved.at(-1)?.cleanup).toContain('FAILED'); + expect(f.saved.at(-1)?.failure).toContain('credential removal denied'); + expect(body).toContain('FAILED'); + return comment(pr, body); + }; + await expect(verify(f.options)).rejects.toThrow('credential removal denied'); + expect(f.calls.filter(call => call === 'cleanup')).toHaveLength(2); + expect(f.calls.lastIndexOf('cleanup')).toBeLessThan(f.calls.indexOf('comment')); + expect(f.calls).not.toContain(`status:success:${SHA}`); + expect(f.calls).toContain(`status:failure:${SHA}`); + expect(f.calls).not.toContain('ready'); expect(f.calls).not.toContain('draft'); + expect(f.pull.draft).toBe(false); + expect(f.saved.at(-1)?.compensation?.join()).toContain('remove copied session credentials: FAILED'); + }); + test('a successful cleanup retry still cannot publish success after the original cleanup failure', async () => { + const f = fixture(true); let attempts = 0; + f.options.driver.cleanup = async () => { if (++attempts === 1) throw new Error('temporary cleanup failure'); }; + await expect(verify(f.options)).rejects.toThrow('temporary cleanup failure'); + expect(attempts).toBe(2); expect(f.saved.at(-1)?.cleanup).toContain('removed'); + expect(f.calls).not.toContain(`status:success:${SHA}`); expect(f.calls).not.toContain('ready'); + }); + test('exercise failure cleans credentials before publishing its retained failure', async () => { + const f = fixture(true); f.pull.draft = false; + f.options.driver.exercise = async () => { f.calls.push('exercise'); throw new Error('surface failed'); }; + f.options.driver.cleanup = async () => { f.calls.push('cleanup'); }; + await expect(verify(f.options)).rejects.toThrow('surface failed'); + expect(f.calls.filter(call => call === 'cleanup')).toHaveLength(1); + expect(f.calls.indexOf('exercise')).toBeLessThan(f.calls.indexOf('cleanup')); + expect(f.calls.indexOf('cleanup')).toBeLessThan(f.calls.indexOf('comment')); + expect(f.saved.at(-1)?.cleanup).toContain('removed'); + expect(f.saved.at(-1)?.failure).toBe('surface failed'); + expect(f.pull.draft).toBe(false); expect(f.calls).not.toContain('draft'); + }); + test('publication failure does not repeat an already completed cleanup', async () => { + const f = fixture(true); + f.options.driver.cleanup = async () => { f.calls.push('cleanup'); }; + f.hook(call => { if (call === 'body') throw new Error('publisher failed'); }); + await expect(verify(f.options)).rejects.toThrow('publisher failed'); + expect(f.calls.filter(call => call === 'cleanup')).toHaveLength(1); + expect(f.saved.at(-1)?.cleanup).toContain('removed'); + }); + test('body readback must confirm installed evidence before success or readiness', async () => { + const f = fixture(); f.options.github.body = async () => {}; + await expect(verify(f.options)).rejects.toThrow('body update was not retained'); + expect(f.calls).not.toContain(`status:success:${SHA}`); expect(f.calls).not.toContain('ready'); + }); + test('already-ready PR stays ready when publication fails', async () => { + const f = fixture(); f.pull.draft = false; f.hook(call => { if (call === 'body') throw new Error('body denied'); }); + await expect(verify(f.options)).rejects.toThrow('body denied'); expect(f.calls).not.toContain('draft'); expect(f.pull.draft).toBe(false); + }); + test('transcript or artifact overwrite after acceptance prevents publication and readiness', async () => { + for (const file of ['retained.log', 'fixture.json']) { + for (const boundary of ['exercise', 'comment', 'body', `status:success:${SHA}`]) { + const f = fixture(true); f.hook(call => { if (call === boundary) writeFileSync(join(f.options.root, file), 'overwritten'); }); + await expect(verify(f.options)).rejects.toThrow('changed after acceptance'); expect(f.calls).not.toContain('ready'); + if (boundary === `status:success:${SHA}`) expect(f.calls).toContain(`status:failure:${SHA}`); + else expect(f.calls).not.toContain(`status:success:${SHA}`); + } + } + }); + test('broad evidence remains bounded for success, failure and PR body while retaining full receipt data', async () => { + const f = fixture(true), receipt = await verify(f.options); + const observation = receipt.observations[0]; + receipt.observations = Array.from({ length: 400 }, (_, index) => ({ ...observation, feature: `skill:${index}`, action: 'a'.repeat(3000), observed: 'r'.repeat(3000), artifacts: Array.from({ length: 20 }, (_, n) => ({ path: `artifact-${n}`, sha256: HASH })) })); + const comment = evidence(receipt); expect(comment.length).toBeLessThan(60000); expect(comment).toContain('additional reviewed observations'); expect(comment).toContain('Evidence-set SHA-256'); + expect(liveEvidenceBody(f.pull.body, receipt).length).toBeLessThan(60000); + receipt.failure = 'reason '.repeat(10000); expect(evidence(receipt).length).toBeLessThan(60000); expect(receipt.observations).toHaveLength(400); + }); test('closed or foreign PR never enters publication', async () => { for (const field of ['state', 'headRepo'] as const) { const f = fixture(); f.pull[field] = 'other'; @@ -50,7 +149,7 @@ describe('exact-head publication state machine', () => { } }); test('head movement during classify, prepare, exercise, or comment aborts without success or readiness', async () => { - for (const boundary of ['files', 'prepare', 'exercise', 'comment']) { + for (const boundary of ['files', 'prepare', 'exercise', 'comment', 'body']) { const f = fixture(true); f.hook(call => { if (call === boundary) f.pull.head.sha = NEXT; }); await expect(verify(f.options)).rejects.toThrow('changed'); expect(f.calls.some(c => c.startsWith('status:success'))).toBe(false); expect(f.calls).not.toContain('ready'); @@ -95,14 +194,27 @@ describe('exact-head publication state machine', () => { }); describe('publisher request boundaries', () => { - test('pagination must match complete PR file count', async () => { - for (const result of [[], [[{ filename: 'README.md' }, { filename: 'README.md' }]], {}]) { - const publisher = new Publisher(async args => JSON.stringify(args.some(a => a.includes('/files?')) ? result : { changed_files: 2 })); - await expect(publisher.files(123)).rejects.toThrow(); - } - const publisher = new Publisher(async args => JSON.stringify(args.some(a => a.includes('/files?')) ? [[{ filename: 'docs/new.md', previous_filename: 'plugins/pstack/skills/architect/SKILL.md' }], [{ filename: 'README.md' }]] : { changed_files: 2 })); - expect(await publisher.files(123)).toHaveLength(2); - await expect(new Publisher(async () => JSON.stringify({ changed_files: 3001 })).files(123)).rejects.toThrow('completely enumerated'); + test('NUL diff parser preserves rename provenance and rejects malformed results', () => { + expect(parseChangedFiles('R100\0plugins/pstack/skills/architect/SKILL.md\0docs/new.md\0M\0README.md\0')).toEqual([{ filename: 'docs/new.md', previous_filename: 'plugins/pstack/skills/architect/SKILL.md' }, { filename: 'README.md' }]); + expect(parseChangedFiles('')).toEqual([]); + for (const output of ['M\0README.md', 'R100\0old\0', 'M\0same\0M\0same\0', 'X\0file\0']) expect(() => parseChangedFiles(output)).toThrow(); + }); + test('fresh local git diff uses immutable base/head even when the source branch moves', async () => { + const repository = temporary(), env = { ...process.env, GIT_AUTHOR_NAME: 'test', GIT_AUTHOR_EMAIL: 'test@example.com', GIT_COMMITTER_NAME: 'test', GIT_COMMITTER_EMAIL: 'test@example.com' }; + const git = (...args: string[]) => execFileSync('git', args, { cwd: repository, env, encoding: 'utf8' }).trim(); + git('init', '-q'); writeFileSync(join(repository, 'README.md'), 'base'); git('add', '.'); git('commit', '-qm', 'base'); const base = git('rev-parse', 'HEAD'); + mkdirSync(join(repository, 'runtime')); writeFileSync(join(repository, 'runtime', 'skill.md'), 'immutable runtime'); git('add', '.'); git('commit', '-qm', 'runtime'); const head = git('rev-parse', 'HEAD'); + git('rm', '-q', 'runtime/skill.md'); git('commit', '-qm', 'later docs-only branch'); + const calls: string[][] = []; + const publisher = new Publisher(async (args, options) => { + calls.push(args); + if (args[0] === 'git' && args[1] === 'fetch') args = args.map(arg => arg === `https://github.com/${REPO}.git` ? repository : arg); + return command(args, options); + }); + expect(await publisher.files(base, head)).toEqual([{ filename: 'runtime/skill.md' }]); + expect(calls.some(args => args[0] === 'gh')).toBe(false); + expect(calls.at(-1)).toContain(`${base}...${head}`); + await expect(publisher.files('main', head)).rejects.toThrow('Invalid diff SHA'); }); test('status targets the supplied exact SHA and evidence URL, never a branch', async () => { const calls: string[][] = []; const publisher = new Publisher(async args => { calls.push(args); return '{}'; }); diff --git a/.claude/skills/verify-open-pstack/scripts/verify.ts b/.claude/skills/verify-open-pstack/scripts/verify.ts index 9ed3b182..ecc30314 100644 --- a/.claude/skills/verify-open-pstack/scripts/verify.ts +++ b/.claude/skills/verify-open-pstack/scripts/verify.ts @@ -1,47 +1,93 @@ import { classify, completeEvidence, newReceipt } from './core.ts'; -import { redact } from './io.ts'; -import { REPO, type Driver, type GitHub, type Receipt, type Registry } from './types.ts'; +import { redact, retainedFile, sha256 } from './io.ts'; +import { REPO, type Driver, type GitHub, type Pull, type Receipt, type Registry } from './types.ts'; class HeadMoved extends Error {} -const safe = (text: string): string => redact(text).replace(/[`<>\r\n]/g, ' ').slice(0, 2000); +const safe = (text: string, limit = 160): string => redact(text).replace(/[`<>\r\n]/g, ' ').slice(0, limit); export function evidence(receipt: Receipt): string { const lines = ['## Live evidence: Open Pstack', `Pinned PR #${receipt.pr}: \`${receipt.sha}\` (base \`${receipt.base}\`).`, - `Result: ${receipt.failure ? `FAILED — ${safe(receipt.failure)}` : receipt.selection.noRuntime ? 'no runtime change' : 'all mapped features passed'}.`, + `Result: ${receipt.failure ? `FAILED — ${safe(receipt.failure, 500)}` : receipt.selection.noRuntime ? 'no runtime change' : 'all mapped features passed'}.`, `Self-test: ${receipt.selfTest ? 'required in both harnesses' : 'not requested'}.`, - `Retained receipt: \`${safe(receipt.artifactRoot)}/receipt.json\`. Local CLI checks are not installed-harness evidence.`]; - for (const install of receipt.installations) lines.push(`- ${install.harness}: CLI ${safe(install.cliVersion)}, plugin ${safe(install.pluginVersion)}, SHA \`${install.sha}\`, tree \`${install.treeHash}\`, isolated \`${safe(install.location)}\`.`); - for (const record of receipt.observations) lines.push(`- ${record.harness} / ${safe(record.feature)}: ${safe(record.surface)}; action: ${safe(record.action)}; observed: ${safe(record.observed)}; operator reviewed.`, - ` Transcript \`${safe(record.transcript)}\` SHA-256 \`${record.transcriptHash}\`; artifacts ${record.artifacts.map(a => `\`${safe(a.path)}\` (\`${a.sha256}\`)`).join(', ')}.`); + `Retained receipt: \`${safe(receipt.artifactRoot, 500)}/receipt.json\`. Full observations and all transcript/artifact hashes are retained there. Local CLI checks are not installed-harness evidence.`, + `Evidence-set SHA-256: \`${sha256(JSON.stringify({ installations: receipt.installations, observations: receipt.observations }))}\`.`]; + for (const install of receipt.installations) lines.push(`- ${install.harness}: CLI ${safe(install.cliVersion)}, plugin ${safe(install.pluginVersion)}, SHA \`${install.sha}\`, tree \`${install.treeHash}\`.`); + let shown = 0; + for (const record of receipt.observations) { + const line = `- ${record.harness} / ${safe(record.feature, 72)}: ${safe(record.surface, 48)}; action: ${safe(record.action, 48)}; observed: ${safe(record.observed, 48)}; operator reviewed. Transcript SHA-256 \`${record.transcriptHash}\`; artifact-set SHA-256 \`${sha256(JSON.stringify(record.artifacts))}\`.`; + if (lines.join('\n').length + line.length > 48000) break; + lines.push(line); shown++; + } + if (shown < receipt.observations.length) lines.push(`${receipt.observations.length - shown} additional reviewed observations are in the retained receipt; see the evidence-set digest above.`); lines.push('Every new head requires a fresh run, including queue-created heads. This helper never queues or merges.'); return lines.join('\n'); } +export function liveEvidenceBody(body: string, receipt: Receipt): string { + const managed = /[\s\S]*?/; + const marker = /(?:^|\n)Live evidence:[^\n]*\n?/; + const match = marker.exec(body); + const summary = `\nLive evidence:\n\n${evidence(receipt)}\nEvidence comment: ${receipt.commentUrl}\n`; + if (managed.test(body)) body = body.replace(managed, () => summary); + else if (match) { + const start = match.index + (match[0].startsWith('\n') ? 1 : 0); + const remaining = body.slice(match.index + match[0].length), end = remaining.search(/^## /m); + body = body.slice(0, start) + summary + '\n\n' + (end < 0 ? '' : remaining.slice(end)); + } else body += '\n\n' + summary; + for (const text of ['The exact candidate is installed in every affected harness.', 'The changed behavior passes from each real user surface.', 'The installed version, action, and observed result appear below.']) { + body = body.replace(`- [ ] ${text}`, `- [x] ${text}${receipt.selection.noRuntime && !receipt.selfTest ? ' (No runtime change; harness exercise not required.)' : ''}`); + } + return body; +} +export async function revalidateEvidence(receipt: Receipt): Promise { + for (const record of receipt.observations) { + const transcript = await retainedFile(receipt.artifactRoot, record.transcript); + if (transcript.path !== record.transcript || transcript.sha256 !== record.transcriptHash) throw new Error('Reviewed transcript changed after acceptance'); + for (const artifact of record.artifacts) { + const actual = await retainedFile(receipt.artifactRoot, artifact.path); + if (actual.path !== artifact.path || actual.sha256 !== artifact.sha256) throw new Error('Reviewed artifact changed after acceptance'); + } + } +} export async function verify(options: { pr: number; selfTest: boolean; root: string; registry: Registry; github: GitHub; driver: Driver; persist: (receipt: Receipt) => Promise }): Promise { const { github, driver, persist } = options; const initial = await github.pull(options.pr); if (initial.state !== 'open' || initial.headRepo !== REPO) throw new Error('Require an open same-repository PR'); const receipt = newReceipt(options.pr, initial.head.sha, initial.base.sha, options.selfTest, options.root); - let successAttempted = false, readyAttempted = false; - const current = async (): Promise => { + let successAttempted = false, readyAttempted = false, cleanupComplete = false; + const cleanup = async (): Promise => { + if (cleanupComplete) return; + await driver.cleanup?.(receipt); + cleanupComplete = true; + receipt.cleanup = 'Copied session credential files removed; run-owned state and evidence retained'; + }; + const current = async (): Promise => { const pull = await github.pull(receipt.pr); if (pull.head.sha !== receipt.sha || pull.base.sha !== receipt.base || pull.state !== 'open' || pull.headRepo !== REPO) throw new HeadMoved('PR head/base or open repository identity changed; run again'); + return pull; }; const phase = async (next: Receipt['phase']): Promise => { receipt.phase = next; await persist(receipt); await current(); }; try { await phase('classify'); - receipt.selection = classify(await github.files(receipt.pr), options.registry); + receipt.selection = classify(await github.files(receipt.base, receipt.sha), options.registry); await current(); await persist(receipt); if (!receipt.selection.noRuntime || receipt.selfTest) { await phase('prepare'); receipt.installations = await driver.prepare(receipt); await current(); await persist(receipt); await phase('exercise'); receipt.observations = await driver.exercise(receipt); await current(); await persist(receipt); } + await cleanup(); completeEvidence(receipt); - await phase('publish'); - receipt.commentUrl = await github.comment(receipt.pr, evidence(receipt)); await persist(receipt); await current(); + await phase('publish'); await revalidateEvidence(receipt); + receipt.commentUrl = await github.comment(receipt.pr, evidence(receipt)); await persist(receipt); + const pull = await current(); await revalidateEvidence(receipt); + const body = liveEvidenceBody(pull.body, receipt); + await github.body(receipt.pr, body); + if ((await current()).body !== body) throw new Error('PR live-evidence body update was not retained'); + await revalidateEvidence(receipt); successAttempted = true; await github.status(receipt.sha, 'success', receipt.commentUrl, receipt.selection.noRuntime ? 'no runtime change' : 'All mapped features passed in both harnesses'); - receipt.status = 'success'; await persist(receipt); await current(); + receipt.status = 'success'; await persist(receipt); await current(); await revalidateEvidence(receipt); if (initial.draft) { - await phase('ready'); readyAttempted = true; await github.ready(receipt.pr); receipt.madeReady = true; await persist(receipt); await current(); + await phase('ready'); await revalidateEvidence(receipt); readyAttempted = true; + await github.ready(receipt.pr); receipt.madeReady = true; await persist(receipt); await current(); await revalidateEvidence(receipt); } await persist(receipt); return receipt; } catch (error) { @@ -52,6 +98,15 @@ export async function verify(options: { pr: number; selfTest: boolean; root: str try { await operation(); receipt.compensation!.push(`${label}: done`); } catch (failure) { receipt.compensation!.push(`${label}: FAILED ${redact(String(failure))}`); } }; + await recover('remove copied session credentials', async () => { + try { await cleanup(); } + catch (failure) { + receipt.cleanup = `FAILED ${redact(String(failure))}`; + receipt.failure += `; credential cleanup ${receipt.cleanup}`; + throw failure; + } + }); + await persist(receipt); // Withdraw an attempted success on its original SHA even if the current head has moved. if (successAttempted && receipt.commentUrl) await recover('withdraw success on pinned SHA', async () => { await github.status(receipt.sha, 'failure', receipt.commentUrl!, 'Verification aborted; rerun required'); receipt.status = 'failure'; diff --git a/AGENTS.md b/AGENTS.md index d4f5ed2a..dd852690 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,6 +8,6 @@ Before opening a pull request, run the Bun tests, strict typecheck, static invar Nothing merges, tags, releases, or rolls out until the exact candidate is installed and the changed behavior passes a live test from the real user surface in every affected harness. Unit tests, validators, source inspection, and self-reports do not satisfy this gate. Record the installed version, surface, action, and observed result in the pull request template. A pull request without that evidence remains a draft. -For Open Pstack PRs, use the shared repository-local `.claude/skills/verify-open-pstack/` skill (Codex: `.agents/skills/verify-open-pstack`). Run its independent Bun tests/typecheck separately from the shipped package. The Mac operator runs `scripts/verify.sh doctor`, then `scripts/verify.sh run --pr --self-test --output `; read the skill's Launch section for separate probe/live output directories. Builders leave delivery PRs draft and do not publish the gate. The helper requires isolated candidate installs in both harnesses, operator-reviewed native-surface evidence, and an exact-head `live-gate` targeting the evidence comment before readiness. Every new head, including a queue-created head, needs a fresh run; never hand-post a replacement status. This helper does not queue or merge. +For Open Pstack PRs, use the shared repository-local `.claude/skills/verify-open-pstack/` skill (Codex: `.agents/skills/verify-open-pstack`). Run its independent Bun tests/typecheck separately from the shipped package. The Mac operator runs `scripts/verify.sh doctor`, then `scripts/verify.sh run --pr --self-test --claude-account --codex-account --output `; read Launch for separate probe/live directories. The trusted parent validates the explicitly selected caam 0.1.22 vault identities, copies only disposable Claude `.credentials.json` and Codex `auth.json` into run-owned config roots, records account emails, and deletes copied credentials in final cleanup without vault activation/writeback. Sandboxed candidates and provider children use run-owned `HOME`/`TMPDIR` and both isolated provider config variables while retaining real `USER`/`LOGNAME`. macOS `sandbox-exec` must deny daily-home/GitHub/SSH/provider files and Keychain/securityd access; no real Library/Keychain link or candidate login is permitted. GitHub publication stays exclusively in the trusted parent. Setup uses fixture homes only; project self-test uses planned `doctor --candidate` without vault probing. Require operator-Mac `bun test scripts/isolation.test.ts`, real-filesystem sentinel denials, no Keychain dialog, and observable authenticated API requests from both native sessions; unavailable implementation or unproven enforcement blocks the run. Builders do not publish the gate, queue, or merge; new delivery PRs start draft, but already-ready PR #111 must not be re-drafted during review remediation. The helper requires exact-head installed-source provenance, operator-reviewed native evidence, revalidated artifact hashes, and a populated/read-back PR `Live evidence:` block before gate/readiness publication. Every new head, including a queue-created head, needs a fresh run after required reviews pass; never hand-post a replacement status. Preserve redacted evidence and record credential-cleanup outcome. Do not add an implicit runtime timeout or a weaker-model fallback. From fc72dbb0d2c02d740ed234a5999907a5b1c286d1 Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:09:35 +0000 Subject: [PATCH 12/20] fix(#90): allow root traversal after Seatbelt read denials Keep the literal root read exception last so native Mac sandbox startup survives the final deny rules; regression-test the ordering. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .../skills/verify-open-pstack/scripts/isolation.test.ts | 7 +++++++ .claude/skills/verify-open-pstack/scripts/isolation.ts | 2 ++ 2 files changed, 9 insertions(+) diff --git a/.claude/skills/verify-open-pstack/scripts/isolation.test.ts b/.claude/skills/verify-open-pstack/scripts/isolation.test.ts index 2b1554f4..d4d7cc2e 100644 --- a/.claude/skills/verify-open-pstack/scripts/isolation.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/isolation.test.ts @@ -155,6 +155,13 @@ describe('disposable credential boundary', () => { expect(() => sandboxProfile(base, '/Users/operator', [], base)).toThrow('must not overlap'); expect(() => sandboxProfile(root, '/Users/operator', [], base)).toThrow('must not overlap'); }); + test('root-read exception is the final Seatbelt rule after all read denials', () => { + const text = sandboxProfile('/private/run/candidate', '/Users/operator', ['/opt/homebrew/bin/bun']); + const rules = text.trim().split('\n').filter(line => !line.startsWith(';')); + expect(rules.at(-1)).toBe('(allow file-read* (literal "/"))'); + expect(text.lastIndexOf('(allow file-read* (literal "/"))')).toBeGreaterThan(text.lastIndexOf('(deny file-read*')); + expect(text.lastIndexOf('(allow file-read* (literal "/"))')).toBeGreaterThan(text.lastIndexOf('(deny network-outbound (remote ip "localhost:*"))')); + }); test('Seatbelt contract denies daily files, Keychain IPC, security execution, and arbitrary writes', () => { const text = sandboxProfile('/private/run/candidate', '/Users/operator', ['/opt/homebrew/bin/bun']); expect(text).toContain('(deny file-read* file-write* (subpath "/Users/operator"))'); diff --git a/.claude/skills/verify-open-pstack/scripts/isolation.ts b/.claude/skills/verify-open-pstack/scripts/isolation.ts index ba4dc6c5..5c2eea53 100644 --- a/.claude/skills/verify-open-pstack/scripts/isolation.ts +++ b/.claude/skills/verify-open-pstack/scripts/isolation.ts @@ -110,6 +110,8 @@ ${vault ? `(deny file-read* file-write* (subpath ${quote(vault)}))` : ''} (deny process-exec (literal "/usr/bin/security")) (deny network-outbound (remote unix-socket (path-regex #".*"))) (deny network-outbound (remote ip "localhost:*")) +; Keep this rule last: Seatbelt's later read denials otherwise override root traversal. +(allow file-read* (literal "/")) `; } export async function createSandbox(home: string, run: Command = command, deniedVaultRoot?: string): Promise { From 9e168eaa65d645a08e1519671b9175132577884f Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:03:44 +0000 Subject: [PATCH 13/20] fix(#90): close round-two credential and publication gaps Bind publishing to the exact candidate source, gate Mac isolation and native authentication evidence, remove relocated disposable credentials, and retain safe compensation and PR body boundaries. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .claude/skills/verify-open-pstack/SKILL.md | 2 + .../verify-open-pstack/features/README.md | 4 +- .../verify-open-pstack/features/isolation.md | 4 +- .../verify-open-pstack/features/registry.json | 10 +- .../verify-open-pstack/scripts/cli.test.ts | 91 +++++++++++++- .../skills/verify-open-pstack/scripts/cli.ts | 35 +++++- .../verify-open-pstack/scripts/core.test.ts | 18 ++- .../verify-open-pstack/scripts/github.ts | 8 +- .../scripts/harness.test.ts | 26 +++- .../verify-open-pstack/scripts/harness.ts | 24 ++++ .../verify-open-pstack/scripts/io.test.ts | 28 ++++- .../skills/verify-open-pstack/scripts/io.ts | 17 ++- .../scripts/isolation.test.ts | 72 +++++++++-- .../verify-open-pstack/scripts/isolation.ts | 67 ++++++---- .../verify-open-pstack/scripts/types.ts | 7 +- .../verify-open-pstack/scripts/verify.test.ts | 117 +++++++++++++++++- .../verify-open-pstack/scripts/verify.ts | 61 +++++++-- 17 files changed, 520 insertions(+), 71 deletions(-) diff --git a/.claude/skills/verify-open-pstack/SKILL.md b/.claude/skills/verify-open-pstack/SKILL.md index 854b309c..7a257157 100644 --- a/.claude/skills/verify-open-pstack/SKILL.md +++ b/.claude/skills/verify-open-pstack/SKILL.md @@ -44,6 +44,8 @@ PROOF="$(mktemp -d "${TMPDIR:-/tmp}/open-pstack-isolation-proof.XXXXXX")" ) ``` +Runtime `run` now executes this suite automatically in a separate trusted checkout of the pinned SHA, with `PSTACK_OPERATOR_SENTINELS=1`, **before copying disposable credentials**. It retains `mac-isolation-proof.txt` outside candidate state and requires both named Mac tests to report PASS with no skips/failures and a successful exit. The receipt binds its canonical path/hash and the exact SHA; missing or modified proof prevents success/readiness. The operator must review the result and explicitly confirm no Keychain dialog appeared. After **each** native session, review an observable authenticated API request/response in the retained transcript, describe the assertion, and confirm `PASS AUTH claude` or `PASS AUTH codex` plus no Keychain dialog. Model self-reports are not proof. The receipt retains these assertions and their reviewed transcript hashes, revalidated at publication boundaries. Fake unit-driver proof tests the gate only and does not establish OS enforcement. + The explicit sentinel opt-in plants randomized, exclusive-create, non-secret files in the actual operator `HOME`, `~/.config/gh`, `~/.ssh`, `~/.claude`, `~/.codex`, and `~/Library/Keychains`; it never overwrites credential/SSH/Keychain files. Redirected directories fail closed. The trusted test parent removes only its own sentinel files and newly created empty directories in `finally`; if interrupted, inspect the printed cleanup-path list and remove only those named sentinel files, never whole daily directories. Require positive unsandboxed read controls and denied sandboxed reads/writes, aliases, and descendant reads; require positive unsandboxed Keychain-service lookup followed by denial of the same service. The real Mac test also proves pinned workspace/plugin write, unlink, rename, alias-write, and hardlink denial while the three approved dependency leaves remain writable. Preserve `$PROOF/isolation-proof.txt` and require both Mac tests to pass without skips. Linux skips and stubbed commands are not Mac enforcement proof. Then run Launch with fresh probe/live directories. Record that no Keychain dialog appeared and both native sessions made observable authenticated API requests; neither observation can be inferred from sentinel tests. Missing/pending adapter integration or failed proof blocks live verification. ## Drive diff --git a/.claude/skills/verify-open-pstack/features/README.md b/.claude/skills/verify-open-pstack/features/README.md index aaea8ba7..7387b749 100644 --- a/.claude/skills/verify-open-pstack/features/README.md +++ b/.claude/skills/verify-open-pstack/features/README.md @@ -10,7 +10,7 @@ | `runner` | Installed parent invoking real configured provider lanes | | `tools` | Installed parent invoking the changed shipped CLI | | `project` | Native discovery/invocation of this verifier in both harnesses | -| `nonRuntime` | Explicit repository documentation/CI/non-shipped assets/tests; no plugin harness launch | +| `nonRuntime` | Explicit ordinary repository documentation/non-shipped assets/unit-test paths; no plugin harness launch | Shipped `plugins/pstack/assets/**` belongs to `shared`: even a logo-only change requires both installed harness surfaces, not the no-runtime lane. `bootstrap.ts` belongs to `tools`; exercise its actual consumers, `orch` and `watch-pr`, including dependency bootstrapping rather than the unrelated external runner. @@ -23,4 +23,6 @@ The four documents below define sub-features, user entry points, driving instruc - `runner.md`: parent-to-child provider dispatch and receipts. - `shipped-tools.md`: orchestration, watch, plan checks, audit, and evidence logging. +Consumed `AGENTS.md`/`CLAUDE.md` instructions and `tests/skill-collision-repro.sh` select `project-skill`. There are no blanket no-runtime exemptions for `.github/**`, `tests/**`, or `scripts/**`: unregistered instruction/enforcement paths fail closed until their runtime ownership is reviewed and mapped. + Explicit non-runtime repository changes are `no runtime change`. Changes to this verifier always select `project-skill`; they cannot bypass native proof. `--self-test` also selects that exercise, deduplicated in each harness, and is mandatory for this skill's delivery PR. Plugin-runtime proof and local CLI proof remain separate. diff --git a/.claude/skills/verify-open-pstack/features/isolation.md b/.claude/skills/verify-open-pstack/features/isolation.md index dc372a87..fcdc7b45 100644 --- a/.claude/skills/verify-open-pstack/features/isolation.md +++ b/.claude/skills/verify-open-pstack/features/isolation.md @@ -18,7 +18,9 @@ Verify tracked plugin/project-skill provenance against immutable pinned Git obje ## Operator-Mac proof -From the trusted verifier checkout run: +Runtime `run` automatically executes the pinned `scripts/isolation.test.ts` from a separate trusted exact-SHA checkout, with `PSTACK_OPERATOR_SENTINELS=1`, before copying disposable credentials. It retains `mac-isolation-proof.txt` outside candidate state, validates both named Mac tests passed without skips/failures and a successful exit, and requires explicit operator review of no Keychain dialog. The receipt binds SHA, canonical proof path/hash, and the review. Each native session then requires an operator-observed authenticated API request/response assertion bound to its reviewed transcript hash and `PASS AUTH ` confirming no Keychain dialog. Both harnesses are mandatory. Missing, mismatched, overwritten, skipped, or rejected proof blocks success/readiness at publication boundaries. Fake unit-driver proof exercises validation without claiming Mac enforcement. + +For a separate diagnostic check, from the trusted verifier checkout run: ```sh PROOF="$(mktemp -d "${TMPDIR:-/tmp}/open-pstack-isolation-proof.XXXXXX")" diff --git a/.claude/skills/verify-open-pstack/features/registry.json b/.claude/skills/verify-open-pstack/features/registry.json index e987ca51..7ee26fa3 100644 --- a/.claude/skills/verify-open-pstack/features/registry.json +++ b/.claude/skills/verify-open-pstack/features/registry.json @@ -39,11 +39,13 @@ "plugins/pstack/skills/poteto-mode/scripts/check-plan.tsconfig.json", "plugins/pstack/skills/show-me-your-work/scripts/log.sh" ], - "project": [".claude/skills/verify-open-pstack/**", ".agents/skills/verify-open-pstack"], + "project": [ + ".claude/skills/verify-open-pstack/**", ".agents/skills/verify-open-pstack", + "AGENTS.md", "CLAUDE.md", "tests/skill-collision-repro.sh" + ], "nonRuntime": [ - "README.md", "AGENTS.md", "CLAUDE.md", "CHANGES.md", "LICENSE", "LICENSE.md", "UPSTREAM.md", - ".gitignore", ".mergify.yml", ".github/**", "docs/**", "assets/**", "tests/**", - "scripts/**", + "README.md", "CHANGES.md", "LICENSE", "LICENSE.md", "UPSTREAM.md", + ".gitignore", ".mergify.yml", "docs/**", "assets/**", "plugins/pstack/skills/poteto-mode/scripts/bootstrap.test.ts", "plugins/pstack/skills/poteto-mode/scripts/check-plan.test.ts" ] diff --git a/.claude/skills/verify-open-pstack/scripts/cli.test.ts b/.claude/skills/verify-open-pstack/scripts/cli.test.ts index 21b0409f..154fb46c 100644 --- a/.claude/skills/verify-open-pstack/scripts/cli.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/cli.test.ts @@ -1,11 +1,96 @@ import { expect, test } from 'bun:test'; -import { parse } from './cli.ts'; -import { mkdtemp, readFile, realpath, rm, stat } from 'node:fs/promises'; +import { bindPublisher, parse } from './cli.ts'; +import { mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { doctor } from './doctor.ts'; -import type { Command } from './io.ts'; +import { command, type Command } from './io.ts'; +import type { GitHub, Pull } from './types.ts'; +import { REPO } from './types.ts'; +import { verify } from './verify.ts'; +import { validateRegistry } from './core.ts'; +import registry from '../features/registry.json'; const accounts = ['--claude-account', 'eric@litman.org', '--codex-account', 'eric@healthspanners.com']; +async function publisherFixture() { + const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-publisher-binding-'))); + const project = join(root, '.claude/skills/verify-open-pstack'); + await mkdir(join(project, 'scripts'), { recursive: true }); + await mkdir(join(project, 'features')); await mkdir(join(root, 'plugins/pstack'), { recursive: true }); + await mkdir(join(root, '.agents/skills'), { recursive: true }); + await symlink('../../.claude/skills/verify-open-pstack', join(root, '.agents/skills/verify-open-pstack')); + await writeFile(join(root, 'plugins/pstack/plugin.json'), '{}\n'); + for (const path of ['scripts/cli.ts', 'scripts/github.ts', 'scripts/verify.sh', 'features/registry.json']) await writeFile(join(project, path), 'pinned publisher source\n'); + for (const args of [ + ['git', 'init', '--quiet'], ['git', 'config', 'user.name', 'Publisher Test'], + ['git', 'config', 'user.email', 'publisher@example.invalid'], ['git', 'add', '.'], + ['git', '-c', 'commit.gpgsign=false', 'commit', '--quiet', '-m', 'pinned publisher'], + ]) await command(args, { cwd: root }); + const sha = (await command(['git', 'rev-parse', 'HEAD'], { cwd: root })).trim(); + const calls: string[] = [], url = `https://github.com/${REPO}/pull/111#issuecomment-1`; + const pull: Pull = { number: 111, head: { sha }, base: { sha }, state: 'open', draft: false, headRepo: REPO, body: '' }; + const github: GitHub = { + async pull() { calls.push('pull'); return structuredClone(pull); }, + async files() { calls.push('files'); return [{ filename: 'README.md' }]; }, + async body(_pr, body) { calls.push('body'); pull.body = body; }, + async comment() { calls.push('comment'); return url; }, + async status(_sha, state) { calls.push(`status:${state}`); }, + async ready() { calls.push('ready'); }, async draft() { calls.push('draft'); }, + }; + return { root, project, sha, github, calls, pull, url }; +} +test('publisher pin rejects an older local checkout before classification or publication', async () => { + const f = await publisherFixture(); + try { + f.pull.head.sha = 'a'.repeat(40); + await expect(bindPublisher(f.root, 111, f.github)).rejects.toThrow('HEAD differs from pinned SHA'); + expect(f.calls).toEqual(['pull']); + } finally { await rm(f.root, { recursive: true, force: true }); } +}); +test('publisher pin rejects modified registry, wrapper and code, plus untracked project source', async () => { + const f = await publisherFixture(); + try { + for (const path of ['features/registry.json', 'scripts/verify.sh', 'scripts/cli.ts', 'scripts/github.ts']) { + await writeFile(join(f.project, path), 'unapproved publisher logic\n'); + await expect(bindPublisher(f.root, 111, f.github)).rejects.toThrow('Source differs from pinned Git tree'); + await writeFile(join(f.project, path), 'pinned publisher source\n'); + } + await writeFile(join(f.project, 'scripts/untracked.ts'), 'untracked logic\n'); + await expect(bindPublisher(f.root, 111, f.github)).rejects.toThrow('Source differs from pinned Git tree'); + expect(f.calls.every(call => call === 'pull')).toBe(true); + } finally { await rm(f.root, { recursive: true, force: true }); } +}); +test('publisher source rechecks protect every success boundary but retain failure compensation', async () => { + const f = await publisherFixture(); + try { + const bound = await bindPublisher(f.root, 111, f.github); + await writeFile(join(f.project, 'scripts/github.ts'), 'changed after binding\n'); + for (const operation of [ + () => bound.files(f.sha, f.sha), () => bound.body(111, 'evidence'), + () => bound.comment(111, 'evidence'), () => bound.status(f.sha, 'success', f.url, 'verified'), () => bound.ready(111), + ]) await expect(operation()).rejects.toThrow('Source differs from pinned Git tree'); + expect(f.calls).toEqual(['pull']); + await bound.status(f.sha, 'failure', f.url, 'withdraw success'); await bound.draft(111); + expect(f.calls).toEqual(['pull', 'status:failure', 'draft']); + } finally { await rm(f.root, { recursive: true, force: true }); } +}); +test('publisher pin also gates docs-only classification and success with no harness fallback', async () => { + const f = await publisherFixture(); + try { + const github = await bindPublisher(f.root, 111, f.github); + const output = join(f.root, 'output'); await mkdir(output); + const receipt = await verify({ pr: 111, selfTest: false, root: output, registry: validateRegistry(registry), github, + driver: { async prepare() { throw new Error('Docs must not prepare harnesses'); }, async exercise() { throw new Error('Docs must not exercise harnesses'); } }, persist: async () => {} }); + expect(receipt.selection.noRuntime).toBe(true); expect(receipt.status).toBe('success'); expect(f.calls).toContain('status:success'); + await expect(github.files(f.sha, 'a'.repeat(40))).rejects.toThrow('classification SHAs'); + await expect(github.status('a'.repeat(40), 'success', f.url, 'wrong head')).rejects.toThrow('pinned SHA'); + await expect(github.body(112, 'wrong PR')).rejects.toThrow('one PR'); + await writeFile(join(f.root, 'README.md'), 'new docs head\n'); + await command(['git', 'add', 'README.md'], { cwd: f.root }); + await command(['git', '-c', 'commit.gpgsign=false', 'commit', '--quiet', '-m', 'new docs head'], { cwd: f.root }); + await expect(github.files(f.sha, f.sha)).rejects.toThrow('HEAD differs from pinned SHA'); + await expect(github.status(f.sha, 'success', f.url, 'stale checkout')).rejects.toThrow('HEAD differs from pinned SHA'); + } finally { await rm(f.root, { recursive: true, force: true }); } +}); test('candidate mode is an explicit doctor-only flag', () => { expect(parse(['doctor', '--candidate', '--output', '/fresh/probe'])).toEqual({ mode: 'doctor', output: '/fresh/probe', pr: 0, selfTest: false, candidate: true }); expect(() => parse(['doctor', '--candidate', '--candidate', '--output', '/fresh/probe'])).toThrow('Duplicate option'); diff --git a/.claude/skills/verify-open-pstack/scripts/cli.ts b/.claude/skills/verify-open-pstack/scripts/cli.ts index c573942d..3d4b3fa8 100644 --- a/.claude/skills/verify-open-pstack/scripts/cli.ts +++ b/.claude/skills/verify-open-pstack/scripts/cli.ts @@ -6,6 +6,38 @@ import { Publisher } from './github.ts'; import { MacDriver } from './harness.ts'; import { freshRoot, interruption, interruptCommands, redact, save } from './io.ts'; import { verify } from './verify.ts'; +import { sourceDigest, sourceHash } from './provenance.ts'; +import type { GitHub } from './types.ts'; + +export async function bindPublisher(repository: string, pr: number, github: GitHub): Promise { + const loadedSource = await sourceDigest(repository); + const pull = await github.pull(pr); + const pinnedSource = await sourceHash(repository, pull.head.sha); + if (pinnedSource !== loadedSource) throw new Error('Publisher source changed while binding the pinned head'); + const recheck = async (): Promise => { + if (await sourceHash(repository, pull.head.sha) !== pinnedSource) throw new Error('Publisher source changed after pinning'); + }; + const checkPr = (number: number): void => { + if (number !== pr) throw new Error('Publisher is bound to one PR'); + }; + return { + pull: number => { checkPr(number); return github.pull(number); }, + async files(base, head) { + if (base !== pull.base.sha || head !== pull.head.sha) throw new Error('Publisher pin differs from classification SHAs'); + await recheck(); return github.files(base, head); + }, + async body(number, body) { checkPr(number); await recheck(); await github.body(number, body); }, + async comment(number, body) { checkPr(number); await recheck(); return github.comment(number, body); }, + async status(sha, state, target, description) { + if (sha !== pull.head.sha) throw new Error('Publisher status differs from pinned SHA'); + // Failure compensation must remain possible even after source changes or interruption. + if (state === 'success') await recheck(); + await github.status(sha, state, target, description); + }, + async ready(number) { checkPr(number); await recheck(); await github.ready(number); }, + async draft(number) { checkPr(number); await github.draft(number); }, + }; +} type Options = { mode: 'doctor'; output: string; pr: number; selfTest: boolean; candidate: boolean } | { mode: 'run'; output: string; pr: number; selfTest: boolean; claudeAccount: string; codexAccount: string }; export function parse(args: string[]): Options { @@ -51,8 +83,9 @@ async function main(): Promise { root = await freshRoot(options.output, repository); if (options.mode === 'doctor') await doctor(root, undefined, undefined, options.candidate); else { + const github = await bindPublisher(repository, options.pr, new Publisher()); const registry = validateRegistry(JSON.parse(await readFile(join(import.meta.dir, '../features/registry.json'), 'utf8'))); - const receipt = await verify({ pr: options.pr, selfTest: options.selfTest, root, registry, github: new Publisher(), driver: new MacDriver(undefined, undefined, { claude: options.claudeAccount, codex: options.codexAccount }), persist: r => save(join(root!, 'receipt.json'), r) }); + const receipt = await verify({ pr: options.pr, selfTest: options.selfTest, root, registry, github, driver: new MacDriver(undefined, undefined, { claude: options.claudeAccount, codex: options.codexAccount }), persist: r => save(join(root!, 'receipt.json'), r) }); console.log(`Pinned ${receipt.sha}: live-gate=${receipt.status}; evidence ${receipt.commentUrl}`); } interruption.signal.throwIfAborted(); diff --git a/.claude/skills/verify-open-pstack/scripts/core.test.ts b/.claude/skills/verify-open-pstack/scripts/core.test.ts index 60c7c37d..9709fa76 100644 --- a/.claude/skills/verify-open-pstack/scripts/core.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/core.test.ts @@ -82,11 +82,25 @@ describe('registry and ownership', () => { evidence(r); expect(() => completeEvidence(r)).not.toThrow(); }); - test('docs and CI only select no harnesses', () => { - const s = classify([{ filename: 'README.md' }, { filename: '.github/workflows/ci.yml' }], registry); + test('ordinary repository documentation selects no harnesses', () => { + const s = classify([{ filename: 'README.md' }, { filename: 'docs/guide.md' }], registry); expect(s.noRuntime).toBe(true); expect(s.features).toEqual([]); }); + test('consumed instructions and known verifier enforcement require native project proof', () => { + for (const filename of ['AGENTS.md', 'CLAUDE.md', 'tests/skill-collision-repro.sh']) { + const s = classify([{ filename }], registry); + expect(s.noRuntime).toBe(false); + expect(s.features).toEqual(['project-skill']); + expect(classify([{ filename: 'docs/moved.md', previous_filename: filename }], registry).noRuntime).toBe(false); + } + }); + test('unregistered CI, tests, and scripts fail closed instead of bypassing runtime proof', () => { + for (const filename of ['.github/workflows/ci.yml', '.github/pull_request_template.md', 'tests/new-gate.sh', 'scripts/disable-proof.ts']) { + expect(() => classify([{ filename }], registry)).toThrow(); + expect(() => classify([{ filename: 'README.md', previous_filename: filename }], registry)).toThrow(); + } + }); }); describe('receipt and evidence boundaries', () => { diff --git a/.claude/skills/verify-open-pstack/scripts/github.ts b/.claude/skills/verify-open-pstack/scripts/github.ts index abff9d43..5c72af2f 100644 --- a/.claude/skills/verify-open-pstack/scripts/github.ts +++ b/.claude/skills/verify-open-pstack/scripts/github.ts @@ -20,8 +20,8 @@ export function parseChangedFiles(output: string): ChangedFile[] { } export class Publisher implements GitHub { constructor(private run: Command = command) {} - private async api(path: string, args: string[] = []): Promise { - return JSON.parse(await this.run(['gh', 'api', `repos/${REPO}/${path}`, ...args])); + private async api(path: string, args: string[] = [], allowInterrupted = false): Promise { + return JSON.parse(await this.run(['gh', 'api', `repos/${REPO}/${path}`, ...args], { allowInterrupted })); } async pull(pr: number): Promise { const data = await this.api(`pulls/${pr}`) as { number: number; head: { sha: string; repo?: { full_name: string } }; base: { sha: string }; draft: boolean; state: string; body: string | null }; @@ -54,8 +54,8 @@ export class Publisher implements GitHub { } async status(sha: string, state: 'success' | 'failure', target: string, description: string): Promise { if (!/^[a-f0-9]{40}$/.test(sha)) throw new Error('Invalid status SHA'); - await this.api(`statuses/${sha}`, ['--method', 'POST', '-f', 'context=live-gate', '-f', `state=${state}`, '-f', `target_url=${target}`, '-f', `description=${description.slice(0, 140)}`]); + await this.api(`statuses/${sha}`, ['--method', 'POST', '-f', 'context=live-gate', '-f', `state=${state}`, '-f', `target_url=${target}`, '-f', `description=${description.slice(0, 140)}`], state === 'failure'); } async ready(pr: number): Promise { await this.run(['gh', 'pr', 'ready', String(pr), '--repo', REPO]); } - async draft(pr: number): Promise { await this.run(['gh', 'pr', 'ready', String(pr), '--undo', '--repo', REPO]); } + async draft(pr: number): Promise { await this.run(['gh', 'pr', 'ready', String(pr), '--undo', '--repo', REPO], { allowInterrupted: true }); } } diff --git a/.claude/skills/verify-open-pstack/scripts/harness.test.ts b/.claude/skills/verify-open-pstack/scripts/harness.test.ts index 18a19c7b..9c066f4f 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.test.ts @@ -5,6 +5,7 @@ import { join } from 'node:path'; import { doctor } from './doctor.ts'; import { codexInstallation, launch, MacDriver, verifyCodexEnabled, verifyProjectDoctor } from './harness.ts'; import { newReceipt } from './core.ts'; +import { MAC_TESTS } from './verify.ts'; import { command, freshRoot, isolatedEnv, redact, retainedFile, treeHash, type Command } from './io.ts'; const roots: string[] = []; async function fixture(): Promise { const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-test-'))); roots.push(root); return root; } @@ -89,7 +90,8 @@ describe('isolated harness boundaries', () => { await expect(verifyProjectDoctor(['{}'], workspace)).rejects.toThrow('passing child doctor'); await expect(verifyProjectDoctor([JSON.stringify({ ...JSON.parse(text), result: 'blocked' })], workspace)).rejects.toThrow('passing child doctor'); }); - test.each([false, true])('prepare uses real pinned files, disposable accounts and cleanup (install failure=%s)', async failInstall => { + test.each([false, true, 'proof', 'review'])('prepare gates disposable credentials on pinned proof (failure=%s)', async failure => { + const failInstall = failure === true; const root = await fixture(), repository = join(root, 'repository'), calls: string[][] = []; const accounts = { claude: 'claude@example.com', codex: 'codex@example.com' }; const caam = join(root, 'caam'), vault = join(caam, 'data/vault'); @@ -114,6 +116,14 @@ describe('isolated harness boundaries', () => { const run: Command = async (wrapped, options = {}) => { const args = wrapped[0] === '/usr/bin/sandbox-exec' ? wrapped.slice(3) : wrapped; calls.push(args); + if (args[0] === '/bin/sh') { + expect(options.cwd).toBe(join(root, 'source-home/workspace/.claude/skills/verify-open-pstack')); + expect(options.env!.PSTACK_OPERATOR_SENTINELS).toBe('1'); expect(process.env.HOME).toBe(options.env!.HOME); + expect(args[2]).toContain('--timeout 0'); expect(options.env!.GH_TOKEN).toBeUndefined(); + for (const harness of ['claude', 'codex']) await expect(stat(join(root, 'state', harness))).rejects.toThrow('ENOENT'); + // Fake adapter output only; actual OS proof is required on the operator Mac. + return `${MAC_TESTS.map(name => `(pass) ${name} [1ms]`).join('\n')}\n 2 pass\n 0 fail\nPSTACK_MAC_TEST_EXIT=${failure === 'proof' ? '1' : '0'}\n`; + } if (args[0] === 'caam') { expect(wrapped.slice(0, 2)).toEqual(['/usr/bin/sandbox-exec', '-p']); expect(wrapped[2]).toContain('(deny file-write*)'); @@ -180,11 +190,15 @@ describe('isolated harness boundaries', () => { const boundary = async (home: string): Promise => { const profile = `${home}.unit-test.sb`; await writeFile(profile, 'unit adapter'); return profile; }; - const driver = new MacDriver(run, undefined, accounts, boundary); + const review = async () => failure === 'review' ? 'REJECT' : 'PASS MAC ISOLATION'; + const driver = new MacDriver(run, review, accounts, boundary); try { Object.defineProperty(process, 'platform', { ...platform, value: 'darwin' }); process.env.CAAM_HOME = caam; - const preparation = driver.prepare(newReceipt(111, sha, sha, true, root)); - if (failInstall) await expect(preparation).rejects.toThrow('fixture installation failed'); + const preparedReceipt = newReceipt(111, sha, sha, true, root); + const preparation = driver.prepare(preparedReceipt); + if (failure === 'proof') await expect(preparation).rejects.toThrow('Both pinned Mac'); + else if (failure === 'review') await expect(preparation).rejects.toThrow('Operator rejected Mac'); + else if (failInstall) await expect(preparation).rejects.toThrow('fixture installation failed'); else { const installs = await preparation; expect(installs.map(i => i.harness)).toEqual(['claude', 'codex']); @@ -195,7 +209,9 @@ describe('isolated harness boundaries', () => { expect(calls.some(c => c.join(' ') === 'codex plugin add pstack@open-pstack --json')).toBe(true); for (const installation of installs) { const receipt = newReceipt(111, sha, sha, true, root); - receipt.installations = [installation]; + receipt.installations = [installation]; receipt.macProof = preparedReceipt.macProof; + expect(receipt.macProof!.sha).toBe(sha); expect(receipt.macProof!.noKeychainDialog).toBe(true); + expect(receipt.macProof!.tests.sha256).toMatch(/^[a-f0-9]{64}$/); await expect(driver.exercise(receipt)).rejects.toThrow('fixture protected native surface reached'); } expect(calls.filter(c => c[0] === '/usr/bin/script')).toHaveLength(2); diff --git a/.claude/skills/verify-open-pstack/scripts/harness.ts b/.claude/skills/verify-open-pstack/scripts/harness.ts index 77032e5e..eb16eff8 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.ts @@ -6,6 +6,7 @@ import { command, interruption, isolatedEnv, redact, retainedFile, save, treeHas import { doctor } from './doctor.ts'; import { copyCredentials, createSandbox, protectSources, removeCredentials, sandboxed, selectedAccounts, vaultRoot } from './isolation.ts'; import { sourceDigest, sourceHash } from './provenance.ts'; +import { validateMacTests } from './verify.ts'; import { HARNESSES, REPO, type Accounts, type Driver, type Harness, type Installation, type Observation, type Receipt } from './types.ts'; export type Ask = (question: string) => Promise; @@ -74,6 +75,20 @@ export class MacDriver implements Driver { await trustedRun(['git', 'fetch', 'origin', receipt.sha], { cwd: reference }); await trustedRun(['git', '-c', 'core.hooksPath=/dev/null', 'checkout', '--detach', receipt.sha], { cwd: reference }); const pinnedSource = await sourceHash(reference, receipt.sha, trustedRun); + if (process.platform !== 'darwin') throw new Error('Live proof requires the operator Mac'); + // Run trusted pinned tests before exposing disposable credentials to candidates. + const proofOutput = await this.run(['/bin/sh', '-c', 'bun test --timeout 0 scripts/isolation.test.ts 2>&1; result=$?; printf "\\nPSTACK_MAC_TEST_EXIT=%s\\n" "$result"'], { + cwd: join(reference, '.claude/skills/verify-open-pstack'), + env: { ...parentEnv, PSTACK_OPERATOR_SENTINELS: '1', NO_COLOR: '1', FORCE_COLOR: '0' }, + }); + const proofPath = join(root, 'mac-isolation-proof.txt'); + await writeFile(proofPath, `Pinned SHA: ${receipt.sha}\n${redact(proofOutput)}`, { flag: 'wx', mode: 0o600 }); + const tests = await retainedFile(root, proofPath); + validateMacTests(await readFile(proofPath, 'utf8'), receipt.sha); + if (await this.review('Review mac-isolation-proof.txt: both Mac tests passed with no skips and no Keychain dialog appeared. Type PASS MAC ISOLATION to confirm:') !== 'PASS MAC ISOLATION') { + throw new Error('Operator rejected Mac isolation proof'); + } + receipt.macProof = { sha: receipt.sha, platform: 'darwin', reviewer: 'operator', tests, noKeychainDialog: true, authenticated: [] }; const installs: Installation[] = []; for (const harness of HARNESSES) { const home = join(root, 'state', harness), workspace = join(home, 'workspace'); @@ -110,6 +125,8 @@ export class MacDriver implements Driver { } async exercise(receipt: Receipt): Promise { const observations: Observation[] = []; + const proof = receipt.macProof; + if (!proof || proof.sha !== receipt.sha || proof.noKeychainDialog !== true) throw new Error('Required Mac isolation proof missing before native exercise'); for (const installation of receipt.installations) { const { harness, home } = installation, workspace = join(home, 'workspace'); if (await sourceDigest(workspace) !== installation.sourceHash) throw new Error('Pinned source changed before exercise'); @@ -132,6 +149,13 @@ export class MacDriver implements Driver { const transcript = await retainedFile(receipt.artifactRoot, await this.review(`Copy/redact ${raw} into output (outside state), review it, and enter the reviewed transcript path:`)); const text = await readFile(join(receipt.artifactRoot, transcript.path), 'utf8'); if (redact(text) !== text) throw new Error('Transcript contains recognizable credentials; redact before accepting'); + const authenticated = redact(await this.review(`${harness}: observed authenticated native API request and concrete response in this transcript (not a model self-report)?`)); + if (!authenticated.trim()) throw new Error('Authenticated native request assertion missing'); + if (await this.review(`Type PASS AUTH ${harness} only after reviewing that request and confirming no Keychain dialog appeared during this session:`) !== `PASS AUTH ${harness}`) { + proof.noKeychainDialog = false; + throw new Error(`Operator rejected ${harness} authentication/no-Keychain proof`); + } + proof.authenticated.push({ harness, observed: authenticated, transcript: transcript.path, transcriptHash: transcript.sha256 }); for (const feature of requiredFeatures(receipt)) { const surface = await this.review(`${harness}/${feature}: native surface/discovery entry point?`); const action = await this.review('Concrete action exercised?'); diff --git a/.claude/skills/verify-open-pstack/scripts/io.test.ts b/.claude/skills/verify-open-pstack/scripts/io.test.ts index 2ccae866..564f2e8d 100644 --- a/.claude/skills/verify-open-pstack/scripts/io.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/io.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, test } from 'bun:test'; import { chmod, mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { redact, retainedFile, save, treeHash } from './io.ts'; +import { redact, registerSecrets, retainedFile, save, treeHash } from './io.ts'; const roots: string[] = []; async function fixture(): Promise { @@ -17,12 +17,36 @@ describe('retained JSON and file boundaries', () => { const root = await fixture(), path = join(root, 'receipt.json'); await save(path, { path: '/tmp/Bearer token', nested: ['Bearer secret",}', 'github_pat_sensitive'], text: 'Bearer secret, next\nquoted "value"', count: 2 }); const json = await readFile(path, 'utf8'); - expect(JSON.parse(json)).toEqual({ path: '/tmp/[REDACTED]', nested: ['[REDACTED]",}', '[REDACTED]'], text: '[REDACTED], next\nquoted "value"', count: 2 }); + expect(JSON.parse(json)).toEqual({ path: '/tmp/Bearer token', nested: ['[REDACTED]",}', '[REDACTED]'], text: '[REDACTED], next\nquoted "value"', count: 2 }); expect(json).not.toContain('secret'); expect((await stat(path)).mode & 0o777).toBe(0o600); expect(redact('Bearer token, next')).toBe('[REDACTED], next'); }); + test('registered opaque access, refresh, and ID tokens cannot survive text or structured evidence', async () => { + const tokens = ['opaque-access.+/value', 'opaque-refresh:"quoted"', 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJvcGVyYXRvciJ9.signature']; + registerSecrets(tokens); + registerSecrets(['']); + for (const token of tokens) { + expect(redact(`prefix ${token} suffix ${token}`)).toBe('prefix [REDACTED] suffix [REDACTED]'); + } + const root = await fixture(), path = join(root, 'receipt.json'); + await save(path, { text: tokens.join('\n'), nested: tokens, path: `/tmp/${tokens[0]}/live` }); + const json = await readFile(path, 'utf8'); + for (const token of tokens) expect(json).not.toContain(token); + expect(JSON.parse(json)).toEqual({ text: tokens.map(() => '[REDACTED]').join('\n'), nested: tokens.map(() => '[REDACTED]'), path: '/tmp/[REDACTED]/live' }); + }); + + test('canonical structural paths retain token-like components without exempting free text', async () => { + const root = await fixture(), path = join(root, 'doctor.json'), canonical = '/tmp/sk-review/live'; + const evidence = { skill: `${canonical}/.claude/skills/verify-open-pstack`, workspace: canonical, + home: canonical, location: canonical, artifactRoot: canonical, featureMap: canonical, + transcript: `${canonical}/transcript.txt`, artifacts: [{ path: `${canonical}/artifact.txt` }], + selection: { paths: ['sk-review/SKILL.md', 'github_pat_reference.md'] }, checks: { explanation: 'sk-private' } }; + await save(path, evidence); + expect(JSON.parse(await readFile(path, 'utf8'))).toEqual({ ...evidence, checks: { explanation: '[REDACTED]' } }); + }); + test('permits evidence below an ancestor named state and canonicalizes output aliases', async () => { const root = await fixture(), output = join(root, 'state', 'run'), alias = join(root, 'alias'); await mkdir(output, { recursive: true }); diff --git a/.claude/skills/verify-open-pstack/scripts/io.ts b/.claude/skills/verify-open-pstack/scripts/io.ts index 35d9033e..fe0ea889 100644 --- a/.claude/skills/verify-open-pstack/scripts/io.ts +++ b/.claude/skills/verify-open-pstack/scripts/io.ts @@ -37,8 +37,16 @@ export const command: Command = async (args, options = {}) => { activeCommands.delete(child.pid); } }; +const secrets = new Set(); +export function registerSecrets(values: string[]): void { + for (const value of values) if (value) secrets.add(value); +} +function redactSecrets(text: string): string { + for (const secret of [...secrets].sort((a, b) => b.length - a.length)) text = text.split(secret).join('[REDACTED]'); + return text; +} export function redact(text: string): string { - return text.replace(/(?:gh[pousr]_[\w]+|github_pat_[\w]+|sk-[\w-]+|Bearer[ \t]+[^\s"'<>()[\]{},;]+)/gi, '[REDACTED]'); + return redactSecrets(text).replace(/(?:gh[pousr]_[\w]+|github_pat_[\w]+|sk-[\w-]+|Bearer[ \t]+[^\s"'<>()[\]{},;]+)/gi, '[REDACTED]'); } export function isolatedEnv(home: string, harness?: 'claude' | 'codex'): Record { if (!isAbsolute(home)) throw new Error('Candidate HOME must be an absolute run-owned directory'); @@ -55,7 +63,12 @@ export function isolatedEnv(home: string, harness?: 'claude' | 'codex'): Record< return env; } export async function save(path: string, value: unknown): Promise { - const json = JSON.stringify(value, (_key, item) => typeof item === 'string' ? redact(item) : item, 2); + const pathKeys = new Set(['path', 'paths', 'skill', 'workspace', 'featureMap', 'home', 'location', 'artifactRoot', 'transcript']); + const pathArrays = new WeakSet(); + const json = JSON.stringify(value, function (key, item) { + if (pathKeys.has(key) && Array.isArray(item)) pathArrays.add(item); + return typeof item === 'string' ? (pathKeys.has(key) || pathArrays.has(this) ? redactSecrets(item) : redact(item)) : item; + }, 2); await writeFile(path + '.tmp', json + '\n', { mode: 0o600 }); await rename(path + '.tmp', path); } diff --git a/.claude/skills/verify-open-pstack/scripts/isolation.test.ts b/.claude/skills/verify-open-pstack/scripts/isolation.test.ts index d4d7cc2e..bf34db27 100644 --- a/.claude/skills/verify-open-pstack/scripts/isolation.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/isolation.test.ts @@ -3,8 +3,8 @@ import { lstat, mkdir, mkdtemp, readFile, realpath, rm, rmdir, stat, symlink, wr import { randomUUID } from 'node:crypto'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; -import { command, isolatedEnv } from './io.ts'; -import { copyCredentials, createSandbox, protectSources, removeCredentials, sandboxed, sandboxProfile, selectedAccounts } from './isolation.ts'; +import { command, isolatedEnv, redact } from './io.ts'; +import { copyCredentials, createSandbox, protectSources, removeCredentials, resolveAppleGit, sandboxed, sandboxProfile, selectedAccounts } from './isolation.ts'; import type { Accounts } from './types.ts'; const roots: string[] = []; @@ -86,6 +86,31 @@ describe('disposable credential boundary', () => { for (const path of copied) expect(await Bun.file(path).exists()).toBe(false); expect(await Promise.all(originals.map(path => readFile(path, 'utf8')))).toEqual(preserved); }); + test('copy registers opaque OAuth access, refresh and ID tokens before candidate processing', async () => { + const root = await fixture(), base = await vault(root), copied: string[] = []; + const tokens = ['opaque.alpha.access.987', 'opaque-refresh-654', 'eyJhbGciOiJub25lIn0.eyJzdWIiOiJmaXh0dXJlIn0.sig', 'opaque.codex.access.321', 'opaque-codex-refresh-123']; + await writeFile(join(base, 'claude', accounts.claude, '.credentials.json'), JSON.stringify({ claudeAiOauth: { accessToken: tokens[0], refreshToken: tokens[1] } })); + await writeFile(join(base, 'codex', accounts.codex, 'auth.json'), JSON.stringify({ tokens: { id_token: tokens[2], access_token: tokens[3], refresh_token: tokens[4] } })); + await copyCredentials(join(root, 'candidate'), base, accounts, copied); + expect(redact(tokens.join(' '))).toBe(tokens.map(() => '[REDACTED]').join(' ')); + await removeCredentials(copied); + }); + test('cleanup removes relocated and refreshed credentials anywhere in candidate state while preserving external evidence', async () => { + const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; + await copyCredentials(home, base, accounts, copied); + const relocated = join(home, 'workspace/cache/relocated-oauth.json'); + await mkdir(dirname(relocated), { recursive: true }); + await writeFile(relocated, await readFile(copied[0]!)); + await writeFile(join(home, 'refreshed-token'), 'refreshed-disposable-token'); + await rm(copied[0]!); + const evidence = join(root, 'transcript.txt'); await writeFile(evidence, 'retained external evidence'); + const before = await readFile(join(base, 'claude', accounts.claude, '.credentials.json'), 'utf8'); + await removeCredentials(copied); + expect(await Bun.file(relocated).exists()).toBe(false); + expect(await Bun.file(join(home, 'refreshed-token')).exists()).toBe(false); + expect(await readFile(evidence, 'utf8')).toBe('retained external evidence'); + expect(await readFile(join(base, 'claude', accounts.claude, '.credentials.json'), 'utf8')).toBe(before); + }); test('partial copy can be cleaned on failure without changing any vault file', async () => { const root = await fixture(), base = await vault(root), copied: string[] = []; const bad = join(base, 'codex', accounts.codex, 'auth.json'); @@ -105,28 +130,29 @@ describe('disposable credential boundary', () => { await expect(copyCredentials(join(root, 'candidate'), base, accounts, copied)).rejects.toThrow('contained regular file'); expect(copied).toEqual([]); }); - test('credential cleanup refuses an ancestor redirect without deleting the external sentinel', async () => { + test('credential cleanup removes disposable state without following external configuration redirects', async () => { const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; await copyCredentials(home, base, accounts, copied); const outside = join(root, 'external'), sentinel = join(outside, '.credentials.json'); await mkdir(outside); await writeFile(sentinel, 'external-credential-sentinel'); await rm(join(home, '.claude'), { recursive: true }); await symlink(outside, join(home, '.claude')); - await expect(removeCredentials(copied)).rejects.toThrow('redirected parent'); - expect(await readFile(sentinel, 'utf8')).toBe('external-credential-sentinel'); const invoked: string[][] = []; await removeCredentials(copied, new Map([[home, join(root, 'candidate.sb')]]), async args => { invoked.push(args); return ''; }); - expect(invoked).toEqual(copied.map(path => ['/usr/bin/sandbox-exec', '-f', join(root, 'candidate.sb'), '/bin/rm', '-f', '--', path])); + expect(invoked).toEqual([['/usr/bin/sandbox-exec', '-f', join(root, 'candidate.sb'), '/bin/rm', '-rf', '--', home]]); await expect(removeCredentials(copied, new Map())).rejects.toThrow('original candidate sandbox'); + await removeCredentials(copied); + expect(await Bun.file(join(home, '.codex/auth.json')).exists()).toBe(false); expect(await readFile(sentinel, 'utf8')).toBe('external-credential-sentinel'); }); - test('cleanup refuses leaf symlinks and copy records destinations before a failed write', async () => { + test('cleanup removes leaf symlinks without following them and copy records destinations before a failed write', async () => { const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; await mkdir(join(home, '.claude'), { recursive: true }); const target = join(home, '.claude/.credentials.json'), sentinel = join(root, 'external-credential'); await writeFile(sentinel, 'preserve'); await symlink(sentinel, target); await expect(copyCredentials(home, base, accounts, copied)).rejects.toThrow(); expect(copied).toEqual([target]); - await expect(removeCredentials(copied)).rejects.toThrow('nonregular file'); + await removeCredentials(copied); + expect(await Bun.file(target).exists()).toBe(false); expect(await readFile(sentinel, 'utf8')).toBe('preserve'); }); test('selected vault tool and profile directories cannot redirect to another account', async () => { @@ -155,6 +181,28 @@ describe('disposable credential boundary', () => { expect(() => sandboxProfile(base, '/Users/operator', [], base)).toThrow('must not overlap'); expect(() => sandboxProfile(root, '/Users/operator', [], base)).toThrow('must not overlap'); }); + test('DNS permits only the macOS resolver socket after the general Unix-socket denial', () => { + const text = sandboxProfile('/private/run/candidate', '/Users/operator', []); + const exception = '(allow network-outbound (remote unix-socket (path "/private/var/run/mDNSResponder")))'; + expect(text).toContain(exception); + expect(text.indexOf(exception)).toBeGreaterThan(text.indexOf('(deny network-outbound (remote unix-socket')); + expect(text.indexOf(exception)).toBeLessThan(text.indexOf('(deny network-outbound (remote ip "localhost:*"))')); + expect(text).not.toContain('(allow network-outbound (remote unix-socket (path-regex'); + }); + test('Apple Git resolves beyond the shim and allows only its selected runtime', async () => { + const root = await fixture(), executable = join(root, 'Xcode.app/Contents/Developer/usr/bin/git'); + await mkdir(dirname(executable), { recursive: true }); await writeFile(executable, 'native-git-fixture'); + const calls: string[][] = []; + const selected = await resolveAppleGit('/usr/bin/git', async args => { calls.push(args); return executable + '\n'; }); + expect(calls).toEqual([['/usr/bin/xcrun', '--find', 'git']]); + expect(selected).toEqual({ executable, runtime: dirname(dirname(executable)) }); + const text = sandboxProfile('/private/run/candidate', '/Users/operator', [selected.executable], undefined, [selected.runtime!]); + expect(text.match(new RegExp('subpath ' + JSON.stringify(selected.runtime!).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'))).toHaveLength(2); + expect(text).not.toContain('(subpath "/Applications")'); + await expect(resolveAppleGit('/usr/bin/git', async () => '/usr/bin/git')).rejects.toThrow('beyond'); + await expect(resolveAppleGit('/usr/bin/git', async () => 'relative/git')).rejects.toThrow('absolute'); + expect(await resolveAppleGit(executable, async () => { throw new Error('Non-shim must not use xcrun'); })).toEqual({ executable }); + }); test('root-read exception is the final Seatbelt rule after all read denials', () => { const text = sandboxProfile('/private/run/candidate', '/Users/operator', ['/opt/homebrew/bin/bun']); const rules = text.trim().split('\n').filter(line => !line.startsWith(';')); @@ -243,6 +291,7 @@ macTest('real Mac sandbox denies daily-home and vault sentinels, writes, aliases if (previousHome === undefined) delete process.env.HOME; else process.env.HOME = previousHome; } const options = { cwd: home, env: isolatedEnv(home) }; + expect(await command(sandboxed(profile, ['git', '--version']), options)).toContain('git version'); const allowed = join(home, 'candidate-sentinel'); await writeFile(allowed, 'candidate-state-allowed'); expect(await command(sandboxed(profile, ['/bin/cat', allowed]), options)).toBe('candidate-state-allowed'); @@ -264,7 +313,9 @@ macTest('real Mac sandbox denies daily-home and vault sentinels, writes, aliases const redirectedCredential = join(home, '.claude/.credentials.json'); await writeFile(join(deniedVault, '.credentials.json'), 'preserve-external-credential'); await symlink(deniedVault, join(home, '.claude')); - await expect(removeCredentials([redirectedCredential], new Map([[home, profile]]))).rejects.toThrow(); + // Full-state cleanup runs after all probes below; it must not follow this + // redirect or leave tokens relocated outside their original config path. + await writeFile(join(home, 'relocated-token'), 'disposable-token'); expect(await readFile(join(deniedVault, '.credentials.json'), 'utf8')).toBe('preserve-external-credential'); const proof = { platform: process.platform, denied: [] as string[], keychainServices: {} as Record }; for (const [i, path] of locations.entries()) { @@ -295,6 +346,9 @@ macTest('real Mac sandbox denies daily-home and vault sentinels, writes, aliases proof.keychainServices[service] = { unsandboxed, sandboxed: result }; } expect(reachable).toBeGreaterThan(0); + await removeCredentials([redirectedCredential], new Map([[home, profile]])); + expect(await Bun.file(join(home, 'relocated-token')).exists()).toBe(false); + expect(await readFile(join(deniedVault, '.credentials.json'), 'utf8')).toBe('preserve-external-credential'); console.log('Mac filesystem/Keychain denial proof:', JSON.stringify(proof)); }, 0); diff --git a/.claude/skills/verify-open-pstack/scripts/isolation.ts b/.claude/skills/verify-open-pstack/scripts/isolation.ts index 5c2eea53..794f1d6c 100644 --- a/.claude/skills/verify-open-pstack/scripts/isolation.ts +++ b/.claude/skills/verify-open-pstack/scripts/isolation.ts @@ -1,7 +1,7 @@ import { realpathSync } from 'node:fs'; -import { chmod, copyFile, lstat, mkdir, readFile, realpath, rm, stat, writeFile } from 'node:fs/promises'; +import { chmod, copyFile, lstat, mkdir, readFile, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; import { basename, dirname, isAbsolute, join, resolve } from 'node:path'; -import { command, isolatedEnv, type Command } from './io.ts'; +import { command, isolatedEnv, registerSecrets, type Command } from './io.ts'; import { HARNESSES, type Accounts } from './types.ts'; export function vaultRoot(): string { @@ -35,6 +35,8 @@ export async function copyCredentials(home: string, vault: string, accounts: Acc const bytes = await readFile(source); const auth = JSON.parse(bytes.toString()); if (tool === 'claude' ? typeof auth.claudeAiOauth?.accessToken !== 'string' : typeof auth.tokens?.access_token !== 'string') throw new Error(`${tool} vault requires file-backed OAuth credentials; unknown format blocks`); + const tokens = tool === 'claude' ? auth.claudeAiOauth : auth.tokens; + registerSecrets(Object.entries(tokens).flatMap(([key, value]) => /token/i.test(key) && typeof value === 'string' ? [value] : [])); const dir = join(home, tool === 'claude' ? '.claude' : '.codex'); await mkdir(dir, { recursive: true, mode: 0o700 }); const target = join(dir, file); @@ -43,27 +45,30 @@ export async function copyCredentials(home: string, vault: string, accounts: Acc } } export async function removeCredentials(paths: string[], profiles?: Map, run: Command = command): Promise { - for (const path of paths) { - const home = dirname(dirname(path)); + for (const home of new Set(paths.map(path => dirname(dirname(path))))) { + await sourceDirectory(dirname(home)); + if (!isAbsolute(home) || resolve(home) !== home) throw new Error('Credential cleanup requires canonical candidate state'); + const info = await lstat(home).catch((error: NodeJS.ErrnoException) => { + if (error.code !== 'ENOENT') throw error; + return undefined; + }); + if (!info) continue; if (profiles) { const profile = profiles.get(home); if (!profile) throw new Error('Credential cleanup requires the original candidate sandbox'); - await run(sandboxed(profile, ['/bin/rm', '-f', '--', path]), { env: isolatedEnv(home), cwd: home, allowInterrupted: true }); + // Remove all disposable state, including renamed or refreshed tokens. + // Exercise-only write protection must not prevent contained cleanup. + const cleanupProfile = profile.endsWith('.exercise.sb') ? profile.slice(0, -'.exercise.sb'.length) : profile; + await run(sandboxed(cleanupProfile, ['/bin/rm', '-rf', '--', home]), { env: isolatedEnv(home), cwd: dirname(home), allowInterrupted: true }); } else { - // Trusted fixtures only. Production deletion is OS-contained against - // candidate-controlled ancestor replacement, including symlink races. - if (await realpath(dirname(path)) !== dirname(path)) throw new Error('Credential cleanup refuses redirected parent'); - const info = await lstat(path).catch((error: NodeJS.ErrnoException) => { - if (error.code !== 'ENOENT') throw error; - return undefined; - }); - if (info?.isSymbolicLink() || (info && !info.isFile())) throw new Error('Credential cleanup refuses nonregular file'); - await rm(path, { force: true }); + // Trusted fixtures only; production deletion remains OS-contained. + if (info.isSymbolicLink() || !info.isDirectory()) throw new Error('Credential cleanup refuses redirected candidate state'); + await rm(home, { recursive: true, force: true }); } } } const quote = (value: string): string => JSON.stringify(value); -export function sandboxProfile(home: string, daily: string, executables: string[], deniedVaultRoot?: string): string { +export function sandboxProfile(home: string, daily: string, executables: string[], deniedVaultRoot?: string, runtimes: string[] = []): string { if (!isAbsolute(home) || !isAbsolute(daily) || home === daily || home.startsWith(daily + '/')) throw new Error('Sandbox state must be outside the operator home'); const vault = deniedVaultRoot === undefined ? undefined : realpathSync(deniedVaultRoot); if (vault) { @@ -85,7 +90,8 @@ export function sandboxProfile(home: string, daily: string, executables: string[ (subpath "/System") (subpath "/usr") (subpath "/bin") (subpath "/sbin") (subpath "/opt/homebrew") (subpath "/Library/Apple") (subpath "/dev") (subpath "/private/etc") (subpath "/private/var/db/dyld") - ${executables.map(path => `(literal ${quote(path)})`).join('\n ')}) + ${executables.map(path => `(literal ${quote(path)})`).join('\n ')} + ${runtimes.map(path => `(subpath ${quote(path)})`).join('\n ')}) (allow file-write* (subpath ${quote(home)}) (subpath "/dev")) (allow network-outbound (remote ip "*:*")) (allow mach-lookup @@ -97,7 +103,8 @@ export function sandboxProfile(home: string, daily: string, executables: string[ (subpath "/System") (subpath "/usr") (subpath "/bin") (subpath "/sbin") (subpath "/opt/homebrew") (subpath "/Library/Apple") (subpath "/dev") (subpath "/private/etc") (subpath "/private/var/db/dyld") - ${executables.map(path => `(literal ${quote(path)})`).join('\n ')}))) + ${executables.map(path => `(literal ${quote(path)})`).join('\n ')} + ${runtimes.map(path => `(subpath ${quote(path)})`).join('\n ')}))) (deny file-read* file-write* (subpath ${quote(daily)})) ${vault ? `(deny file-read* file-write* (subpath ${quote(vault)}))` : ''} (deny file-read* file-write* (subpath "/Library/Keychains") (subpath "/System/Library/Keychains")) @@ -109,23 +116,35 @@ ${vault ? `(deny file-read* file-write* (subpath ${quote(vault)}))` : ''} (deny process-info* (require-not (target same-sandbox))) (deny process-exec (literal "/usr/bin/security")) (deny network-outbound (remote unix-socket (path-regex #".*"))) +(allow network-outbound (remote unix-socket (path "/private/var/run/mDNSResponder"))) (deny network-outbound (remote ip "localhost:*")) ; Keep this rule last: Seatbelt's later read denials otherwise override root traversal. (allow file-read* (literal "/")) `; } +export async function resolveAppleGit(path: string, run: Command = command): Promise<{ executable: string; runtime?: string }> { + const actual = await realpath(path); + if (actual !== '/usr/bin/git') return { executable: actual }; + const selected = (await run(['/usr/bin/xcrun', '--find', 'git'])).trim(); + if (!isAbsolute(selected) || basename(selected) !== 'git') throw new Error('Apple Git selection must be an absolute Git executable'); + const executable = await realpath(selected), runtime = dirname(dirname(executable)); + if (executable === actual || basename(dirname(executable)) !== 'bin' || basename(runtime) !== 'usr' || !(await stat(executable)).isFile()) throw new Error('Apple Git must resolve beyond its executable shim'); + return { executable, runtime }; +} export async function createSandbox(home: string, run: Command = command, deniedVaultRoot?: string): Promise { const daily = await realpath(process.env.HOME!); const vault = deniedVaultRoot === undefined ? undefined : await realpath(deniedVaultRoot); const candidate = await realpath(home); // Validate boundaries before copying executables or writing the profile. sandboxProfile(candidate, daily, [], vault); - const executables: string[] = []; + const executables: string[] = [], runtimes: string[] = []; for (const name of ['claude', 'codex', 'bun', 'node', 'git']) { const path = Bun.which(name); if (!path && ['claude', 'codex', 'bun'].includes(name)) throw new Error(`Missing ${name}`); if (path) { - const actual = await realpath(path); + const selected = name === 'git' ? await resolveAppleGit(path, run) : { executable: await realpath(path), runtime: undefined }; + const actual = selected.executable; + if (selected.runtime) runtimes.push(selected.runtime); if (actual.startsWith(daily + '/') || path.startsWith(daily + '/')) { // Only standalone native binaries can be relocated without dependencies // on daily state. Never grant a home-directory read exception. @@ -136,13 +155,19 @@ export async function createSandbox(home: string, run: Command = command, denied await copyFile(actual, target); await chmod(target, (await stat(actual)).mode & 0o777); executables.push(target); - } else executables.push(path, actual); + } else { + executables.push(actual); + if (selected.runtime) { + await mkdir(join(home, 'bin'), { recursive: true, mode: 0o700 }); + await symlink(actual, join(home, 'bin/git')); + } + } } } // The policy stays outside candidate-writable state; later subprocesses must // not load a profile that the candidate can replace between invocations. const path = join(dirname(home), `${basename(home)}.sb`); - await writeFile(path, sandboxProfile(candidate, daily, executables, vault), { mode: 0o600, flag: 'wx' }); + await writeFile(path, sandboxProfile(candidate, daily, executables, vault, runtimes), { mode: 0o600, flag: 'wx' }); await run(['/usr/bin/sandbox-exec', '-f', path, '/usr/bin/true'], { env: isolatedEnv(home), cwd: home }); return path; } diff --git a/.claude/skills/verify-open-pstack/scripts/types.ts b/.claude/skills/verify-open-pstack/scripts/types.ts index 550cfab3..f1ce8511 100644 --- a/.claude/skills/verify-open-pstack/scripts/types.ts +++ b/.claude/skills/verify-open-pstack/scripts/types.ts @@ -27,12 +27,17 @@ export interface Installation { harness: Harness; cliVersion: string; pluginVersion: string; sha: string; location: string; treeHash: string; home: string; account?: string; sourceHash?: string; } +export interface MacProof { + sha: string; platform: 'darwin'; reviewer: 'operator'; tests: { path: string; sha256: string }; + noKeychainDialog: boolean; + authenticated: { harness: Harness; observed: string; transcript: string; transcriptHash: string }[]; +} export interface Receipt { schema: 1; repo: string; pr: number; sha: string; base: string; phase: Phase; selection: Selection; installations: Installation[]; observations: Observation[]; selfTest: boolean; artifactRoot: string; failure?: string; commentUrl?: string; status?: 'success' | 'failure'; madeReady?: boolean; - compensation?: string[]; cleanup: string; started: string; + compensation?: string[]; cleanup: string; started: string; macProof?: MacProof; } export interface GitHub { pull(pr: number): Promise; diff --git a/.claude/skills/verify-open-pstack/scripts/verify.test.ts b/.claude/skills/verify-open-pstack/scripts/verify.test.ts index fb488e33..311fd014 100644 --- a/.claude/skills/verify-open-pstack/scripts/verify.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/verify.test.ts @@ -7,10 +7,12 @@ import data from '../features/registry.json'; import { requiredFeatures, validateRegistry } from './core.ts'; import { parseChangedFiles, Publisher } from './github.ts'; import { command, sha256 } from './io.ts'; -import { evidence, liveEvidenceBody, verify } from './verify.ts'; +import { evidence, liveEvidenceBody, MAC_TESTS, validateMacTests, verify } from './verify.ts'; import { HARNESSES, REPO, type Driver, type GitHub, type Pull, type Receipt } from './types.ts'; const SHA = 'a'.repeat(40), BASE = 'b'.repeat(40), NEXT = 'c'.repeat(40), HASH = 'd'.repeat(64); const URL = `https://github.com/${REPO}/pull/123#issuecomment-1`; +// Explicit fake-driver proof exercises the gate; it makes no OS-enforcement claim. +const testMacLog = (sha: string) => `Pinned SHA: ${sha}\n${MAC_TESTS.map(name => `(pass) ${name} [1ms]`).join('\n')}\n 2 pass\n 0 fail\nPSTACK_MAC_TEST_EXIT=0\n`; const roots: string[] = []; function temporary(): string { const root = realpathSync(mkdtempSync(join(tmpdir(), 'pstack-publish-test-'))); roots.push(root); return root; } afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); @@ -30,7 +32,11 @@ function fixture(runtime = false, selfTest = false) { async ready() { mark('ready'); pull.draft = false; }, async draft() { mark('draft'); pull.draft = true; }, }; const driver: Driver = { - async prepare(r) { mark('prepare'); return HARNESSES.map(harness => ({ harness, sha: r.sha, cliVersion: 'test', pluginVersion: '1.5.0', treeHash: HASH, location: `/isolated/${harness}/plugin`, home: `/isolated/${harness}` })); }, + async prepare(r) { + const log = testMacLog(r.sha); writeFileSync(join(root, 'mac-proof.log'), log); + r.macProof = { sha: r.sha, platform: 'darwin', reviewer: 'operator', tests: { path: 'mac-proof.log', sha256: sha256(log) }, noKeychainDialog: true, + authenticated: HARNESSES.map(harness => ({ harness, observed: 'unit-driver authenticated request assertion', transcript: 'retained.log', transcriptHash: sha256('reviewed transcript') })) }; + mark('prepare'); return HARNESSES.map(harness => ({ harness, sha: r.sha, cliVersion: 'test', pluginVersion: '1.5.0', treeHash: HASH, location: `/isolated/${harness}/plugin`, home: `/isolated/${harness}` })); }, async exercise(r) { mark('exercise'); return HARNESSES.flatMap(harness => requiredFeatures(r).map(feature => ({ harness, feature, surface: 'native surface', action: 'invoke', observed: 'fixture changed', reviewer: 'operator' as const, transcript: 'retained.log', transcriptHash: sha256('reviewed transcript'), artifacts: [{ path: 'fixture.json', sha256: sha256('reviewed artifact') }] }))); }, }; const options = { pr: 123, root, selfTest, registry: validateRegistry(data), github, driver, persist: async (r: Receipt) => { saved.push(structuredClone(r)); } }; @@ -38,6 +44,37 @@ function fixture(runtime = false, selfTest = false) { } describe('exact-head publication state machine', () => { + test('Mac test output requires both exact names, no skips/failures and successful process exit', () => { + expect(() => validateMacTests(testMacLog(SHA), SHA)).not.toThrow(); + for (const log of [testMacLog(NEXT), testMacLog(SHA).replace(MAC_TESTS[0], 'other test'), testMacLog(SHA).replace('(pass)', '(skip)'), + testMacLog(SHA).replace('0 fail', '1 fail'), testMacLog(SHA).replace('2 pass', '2 pass\n 1 skip'), testMacLog(SHA).replace('EXIT=0', 'EXIT=1')]) { + expect(() => validateMacTests(log, SHA)).toThrow('Both pinned Mac'); + } + }); + test.each(['missing', 'sha', 'keychain', 'authenticated', 'transcript', 'overwrite', 'skip'])('runtime Mac proof %s blocks success and readiness', async defect => { + const f = fixture(true); + const exercise = f.options.driver.exercise; + f.options.driver.exercise = async receipt => { + const observations = await exercise(receipt); + const proof = receipt.macProof!; + if (defect === 'missing') delete receipt.macProof; + if (defect === 'sha') proof.sha = NEXT; + if (defect === 'keychain') proof.noKeychainDialog = false; + if (defect === 'authenticated') proof.authenticated.pop(); + if (defect === 'transcript') proof.authenticated[0]!.transcriptHash = HASH; + if (defect === 'overwrite') writeFileSync(join(f.options.root, proof.tests.path), 'overwritten'); + if (defect === 'skip') { const log = testMacLog(SHA).replace('(pass)', '(skip)'); writeFileSync(join(f.options.root, proof.tests.path), log); proof.tests.sha256 = sha256(log); } + return observations; + }; + await expect(verify(f.options)).rejects.toThrow(); + expect(f.calls).not.toContain(`status:success:${SHA}`); expect(f.calls).not.toContain('ready'); + }); + test('proof mutation during publication withdraws exact-SHA success', async () => { + const f = fixture(true); + f.hook(call => { if (call === `status:success:${SHA}`) writeFileSync(join(f.options.root, 'mac-proof.log'), 'changed'); }); + await expect(verify(f.options)).rejects.toThrow('Mac isolation proof changed'); + expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.calls).not.toContain('ready'); + }); test('docs-only publishes no-runtime evidence without either harness, then readies', async () => { const f = fixture(), receipt = await verify(f.options); expect(receipt.status).toBe('success'); expect(receipt.sha).toBe(SHA); @@ -50,6 +87,18 @@ describe('exact-head publication state machine', () => { expect(f.pull.body).toContain('- [x] The installed version, action, and observed result appear below.'); expect(liveEvidenceBody(f.pull.body, receipt)).toBe(f.pull.body); }); + test('unmanaged evidence replacement preserves every following heading and operator note', async () => { + const receipt = await verify(fixture().options); + for (const suffix of ['# Release notes\nKeep these.', '## Notes\nKeep these.', '### Operator notes\nKeep these.', 'Ordinary operator text.\n\n## Later\nKeep these.', '---\nTrailing section.']) { + const before = `Summary is untouched.\n\nLive evidence:\n\n_pending_\n\n${suffix}`; + const after = liveEvidenceBody(before, receipt); + expect(after).toStartWith('Summary is untouched.\n\n'); + expect(after).toEndWith(suffix); expect(after).not.toContain('_pending_'); + expect(liveEvidenceBody(after, receipt)).toBe(after); + } + const ordinary = 'Live evidence:\nOperator-owned text immediately follows.\n# Closing notes'; + expect(liveEvidenceBody(ordinary, receipt)).toEndWith('Operator-owned text immediately follows.\n# Closing notes'); + }); test('all changed surfaces and explicit project self-test run in both harnesses', async () => { const f = fixture(true, true), r = await verify(f.options); expect(r.observations).toHaveLength(4); expect(f.calls).toContain('prepare'); expect(f.calls).toContain('exercise'); @@ -175,12 +224,47 @@ describe('exact-head publication state machine', () => { expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.calls).not.toContain('ready'); expect(f.saved.at(-1)?.failure).toContain('Missing/duplicate'); }); - test('ambiguous success-write and ready-write failures are compensated', async () => { + test('ambiguous publication responses revoke success without claiming a readiness transition', async () => { for (const boundary of [`status:success:${SHA}`, 'ready']) { const f = fixture(); f.hook(call => { if (call === boundary) throw new Error('response lost'); }); await expect(verify(f.options)).rejects.toThrow('response lost'); expect(f.calls).toContain(`status:failure:${SHA}`); - if (boundary === 'ready') expect(f.calls).toContain('draft'); + expect(f.calls).not.toContain('draft'); + } + }); + test('another actor making the PR ready during the run is not reverted after failure', async () => { + const f = fixture(); + f.hook(call => { if (call === `status:success:${SHA}`) f.pull.draft = false; }); + f.options.persist = async receipt => { + if (receipt.status === 'success') throw new Error('receipt denied'); + f.saved.push(structuredClone(receipt)); + }; + await expect(verify(f.options)).rejects.toThrow('receipt denied'); + expect(f.calls).not.toContain('ready'); expect(f.calls).not.toContain('draft'); expect(f.pull.draft).toBe(false); + }); + test('readiness is checked again after entering the ready phase', async () => { + const f = fixture(), persist = f.options.persist; + f.options.persist = async receipt => { await persist(receipt); if (receipt.phase === 'ready') f.pull.draft = false; }; + await verify(f.options); + expect(f.calls).not.toContain('ready'); expect(f.calls).not.toContain('draft'); expect(f.pull.draft).toBe(false); + }); + test('a failed or idempotent readiness call never rolls back another actor', async () => { + const f = fixture(); + f.options.github.ready = async () => { f.calls.push('ready'); f.pull.draft = false; throw new Error('already ready'); }; + await expect(verify(f.options)).rejects.toThrow('already ready'); + expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.calls).not.toContain('draft'); expect(f.pull.draft).toBe(false); + }); + test('receipt-write errors cannot prevent compensating an accepted success or ready write', async () => { + for (const boundary of [`status:success:${SHA}`, 'ready']) { + const f = fixture(); let deny = false; + f.hook(call => { if (call === boundary) { f.pull.base.sha = NEXT; deny = true; } }); + f.options.persist = async receipt => { + if (deny && ['head-moved', 'failed'].includes(receipt.phase)) throw new Error('ENOSPC'); + f.saved.push(structuredClone(receipt)); + }; + await expect(verify(f.options)).rejects.toThrow('changed'); + expect(f.calls).toContain(`status:failure:${SHA}`); + if (boundary === 'ready') { expect(f.calls).toContain('draft'); expect(f.pull.draft).toBe(true); } } }); test('failed compensation is retained, not reported as successful', async () => { @@ -216,6 +300,31 @@ describe('publisher request boundaries', () => { expect(calls.at(-1)).toContain(`${base}...${head}`); await expect(publisher.files('main', head)).rejects.toThrow('Invalid diff SHA'); }); + test('real interrupted command execution still permits only trusted rollback writes', () => { + for (const signal of ['SIGINT', 'SIGTERM']) { + const script = ` + import { command, interruptCommands } from ${JSON.stringify(new globalThis.URL('./io.ts', import.meta.url).pathname)}; + import { Publisher } from ${JSON.stringify(new globalThis.URL('./github.ts', import.meta.url).pathname)}; + await interruptCommands(${JSON.stringify(signal)}); + const executed = []; + const publisher = new Publisher(async (args, options) => { + const output = await command([process.execPath, '-e', 'console.log("{}")'], options); + executed.push(args); + return output; + }); + for (const ordinary of [() => publisher.ready(123), () => publisher.status(${JSON.stringify(SHA)}, 'success', ${JSON.stringify(URL)}, 'passed')]) { + try { await ordinary(); throw new Error('ordinary write escaped interruption'); } + catch (error) { if (!String(error).includes('Interrupted by')) throw error; } + } + await publisher.status(${JSON.stringify(SHA)}, 'failure', ${JSON.stringify(URL)}, 'withdraw'); + await publisher.draft(123); + console.log(JSON.stringify(executed)); + `; + const calls = JSON.parse(execFileSync(process.execPath, ['-e', script], { encoding: 'utf8' }).trim()) as string[][]; + expect(calls).toHaveLength(2); + expect(calls[0]).toContain('state=failure'); expect(calls[1]).toContain('--undo'); + } + }); test('status targets the supplied exact SHA and evidence URL, never a branch', async () => { const calls: string[][] = []; const publisher = new Publisher(async args => { calls.push(args); return '{}'; }); await publisher.status(SHA, 'success', URL, 'passed'); diff --git a/.claude/skills/verify-open-pstack/scripts/verify.ts b/.claude/skills/verify-open-pstack/scripts/verify.ts index ecc30314..ecf34001 100644 --- a/.claude/skills/verify-open-pstack/scripts/verify.ts +++ b/.claude/skills/verify-open-pstack/scripts/verify.ts @@ -1,4 +1,6 @@ import { classify, completeEvidence, newReceipt } from './core.ts'; +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; import { redact, retainedFile, sha256 } from './io.ts'; import { REPO, type Driver, type GitHub, type Pull, type Receipt, type Registry } from './types.ts'; @@ -9,7 +11,7 @@ export function evidence(receipt: Receipt): string { `Result: ${receipt.failure ? `FAILED — ${safe(receipt.failure, 500)}` : receipt.selection.noRuntime ? 'no runtime change' : 'all mapped features passed'}.`, `Self-test: ${receipt.selfTest ? 'required in both harnesses' : 'not requested'}.`, `Retained receipt: \`${safe(receipt.artifactRoot, 500)}/receipt.json\`. Full observations and all transcript/artifact hashes are retained there. Local CLI checks are not installed-harness evidence.`, - `Evidence-set SHA-256: \`${sha256(JSON.stringify({ installations: receipt.installations, observations: receipt.observations }))}\`.`]; + `Evidence-set SHA-256: \`${sha256(JSON.stringify({ installations: receipt.installations, observations: receipt.observations, macProof: receipt.macProof }))}\`.`]; for (const install of receipt.installations) lines.push(`- ${install.harness}: CLI ${safe(install.cliVersion)}, plugin ${safe(install.pluginVersion)}, SHA \`${install.sha}\`, tree \`${install.treeHash}\`.`); let shown = 0; for (const record of receipt.observations) { @@ -29,14 +31,46 @@ export function liveEvidenceBody(body: string, receipt: Receipt): string { if (managed.test(body)) body = body.replace(managed, () => summary); else if (match) { const start = match.index + (match[0].startsWith('\n') ? 1 : 0); - const remaining = body.slice(match.index + match[0].length), end = remaining.search(/^## /m); - body = body.slice(0, start) + summary + '\n\n' + (end < 0 ? '' : remaining.slice(end)); + // Only the template marker and its known placeholder belong to this helper. + const remaining = body.slice(match.index + match[0].length).replace(/^\n*_pending_\n?(?=\n|$)/, ''); + body = body.slice(0, start) + summary + '\n' + remaining; } else body += '\n\n' + summary; for (const text of ['The exact candidate is installed in every affected harness.', 'The changed behavior passes from each real user surface.', 'The installed version, action, and observed result appear below.']) { body = body.replace(`- [ ] ${text}`, `- [x] ${text}${receipt.selection.noRuntime && !receipt.selfTest ? ' (No runtime change; harness exercise not required.)' : ''}`); } return body; } +export const MAC_TESTS = [ + 'real Mac sandbox denies daily-home and vault sentinels, writes, aliases, grandchildren, and Keychain IPC', + 'actual operator HOME denial and immutable exercise sources with writable dependency leaves', +] as const; +export function validateMacTests(text: string, sha: string): void { + if (!text.startsWith(`Pinned SHA: ${sha}\n`) || !text.trimEnd().endsWith('PSTACK_MAC_TEST_EXIT=0') || + !/^\s*0 fail\s*$/m.test(text) || /^\s*[1-9]\d* skip\s*$/m.test(text) || /\((?:skip|fail)\)/.test(text) || + !MAC_TESTS.every(name => text.split('\n').some(line => line.startsWith(`(pass) ${name}`) && /^(?:\s+\[[^\]]+\])?\s*$/.test(line.slice(`(pass) ${name}`.length))))) { + throw new Error('Both pinned Mac isolation tests must pass without skips or failures'); + } +} +export async function revalidateMacProof(receipt: Receipt): Promise { + if (receipt.selection.noRuntime && !receipt.selfTest) return; + const proof = receipt.macProof; + if (!proof || proof.sha !== receipt.sha || proof.platform !== 'darwin' || proof.reviewer !== 'operator' || proof.noKeychainDialog !== true) { + throw new Error('Required exact-SHA operator-Mac proof or no-Keychain observation missing'); + } + const tests = await retainedFile(receipt.artifactRoot, proof.tests.path); + if (tests.path !== proof.tests.path || tests.sha256 !== proof.tests.sha256) throw new Error('Mac isolation proof changed after acceptance'); + validateMacTests(await readFile(join(receipt.artifactRoot, tests.path), 'utf8'), receipt.sha); + if (proof.authenticated.length !== 2) throw new Error('Authenticated native requests required in both harnesses'); + for (const harness of ['claude', 'codex'] as const) { + const requests = proof.authenticated.filter(record => record.harness === harness); + if (requests.length !== 1 || !requests[0]!.observed.trim()) throw new Error('Authenticated native request assertion missing'); + const request = requests[0]!, transcript = await retainedFile(receipt.artifactRoot, request.transcript); + if (transcript.path !== request.transcript || transcript.sha256 !== request.transcriptHash || + !receipt.observations.some(record => record.harness === harness && record.transcript === request.transcript && record.transcriptHash === request.transcriptHash)) { + throw new Error('Authenticated request is not bound to the reviewed native transcript'); + } + } +} export async function revalidateEvidence(receipt: Receipt): Promise { for (const record of receipt.observations) { const transcript = await retainedFile(receipt.artifactRoot, record.transcript); @@ -46,18 +80,19 @@ export async function revalidateEvidence(receipt: Receipt): Promise { if (actual.path !== artifact.path || actual.sha256 !== artifact.sha256) throw new Error('Reviewed artifact changed after acceptance'); } } + await revalidateMacProof(receipt); } export async function verify(options: { pr: number; selfTest: boolean; root: string; registry: Registry; github: GitHub; driver: Driver; persist: (receipt: Receipt) => Promise }): Promise { const { github, driver, persist } = options; const initial = await github.pull(options.pr); if (initial.state !== 'open' || initial.headRepo !== REPO) throw new Error('Require an open same-repository PR'); const receipt = newReceipt(options.pr, initial.head.sha, initial.base.sha, options.selfTest, options.root); - let successAttempted = false, readyAttempted = false, cleanupComplete = false; + let successAttempted = false, cleanupComplete = false; const cleanup = async (): Promise => { if (cleanupComplete) return; await driver.cleanup?.(receipt); cleanupComplete = true; - receipt.cleanup = 'Copied session credential files removed; run-owned state and evidence retained'; + receipt.cleanup = 'Disposable candidate homes removed, including relocated credentials; external evidence retained'; }; const current = async (): Promise => { const pull = await github.pull(receipt.pr); @@ -85,9 +120,11 @@ export async function verify(options: { pr: number; selfTest: boolean; root: str successAttempted = true; await github.status(receipt.sha, 'success', receipt.commentUrl, receipt.selection.noRuntime ? 'no runtime change' : 'All mapped features passed in both harnesses'); receipt.status = 'success'; await persist(receipt); await current(); await revalidateEvidence(receipt); - if (initial.draft) { - await phase('ready'); await revalidateEvidence(receipt); readyAttempted = true; - await github.ready(receipt.pr); receipt.madeReady = true; await persist(receipt); await current(); await revalidateEvidence(receipt); + if ((await current()).draft) { + await phase('ready'); await revalidateEvidence(receipt); + if ((await current()).draft) { + await github.ready(receipt.pr); receipt.madeReady = true; await persist(receipt); await current(); await revalidateEvidence(receipt); + } } await persist(receipt); return receipt; } catch (error) { @@ -106,17 +143,19 @@ export async function verify(options: { pr: number; selfTest: boolean; root: str throw failure; } }); - await persist(receipt); + await recover('persist failure receipt', async () => { await persist(receipt); }); // Withdraw an attempted success on its original SHA even if the current head has moved. if (successAttempted && receipt.commentUrl) await recover('withdraw success on pinned SHA', async () => { await github.status(receipt.sha, 'failure', receipt.commentUrl!, 'Verification aborted; rerun required'); receipt.status = 'failure'; }); - if (readyAttempted) await recover('restore draft', async () => { await github.draft(receipt.pr); receipt.madeReady = false; }); + // A failed/ambiguous readiness response does not establish ownership of that transition. + if (receipt.madeReady) await recover('restore draft', async () => { await github.draft(receipt.pr); receipt.madeReady = false; }); if (!successAttempted) await recover('publish pinned failure', async () => { await current(); receipt.commentUrl = await github.comment(receipt.pr, evidence(receipt)); await current(); await github.status(receipt.sha, 'failure', receipt.commentUrl, 'Verification failed; see retained evidence'); receipt.status = 'failure'; await current(); }); - await persist(receipt); throw new Error(receipt.failure); + await recover('persist compensation receipt', async () => { await persist(receipt); }); + throw new Error(receipt.failure); } } From 886d04a9c1814e1aeedf55645f8a19a5e45e3a7c Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:21:33 +0000 Subject: [PATCH 14/20] fix(#90): protect retained credentials and operator-owned PR state Bind publication to loaded code, reject credential-bearing evidence, and leave PR bodies and readiness reversals to operators. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .claude/skills/verify-open-pstack/SKILL.md | 2 +- .../verify-open-pstack/features/registry.json | 4 +- .../verify-open-pstack/scripts/cli.test.ts | 53 +++++++++++++----- .../skills/verify-open-pstack/scripts/cli.ts | 22 ++++++-- .../verify-open-pstack/scripts/core.test.ts | 2 +- .../verify-open-pstack/scripts/github.ts | 5 -- .../verify-open-pstack/scripts/harness.ts | 15 ++++- .../verify-open-pstack/scripts/io.test.ts | 11 ++++ .../skills/verify-open-pstack/scripts/io.ts | 9 ++- .../scripts/isolation.test.ts | 43 ++++++++++++++- .../verify-open-pstack/scripts/isolation.ts | 23 ++++++++ .../verify-open-pstack/scripts/types.ts | 4 +- .../verify-open-pstack/scripts/verify.test.ts | 55 +++++++++++-------- .../verify-open-pstack/scripts/verify.ts | 22 ++++---- 14 files changed, 198 insertions(+), 72 deletions(-) diff --git a/.claude/skills/verify-open-pstack/SKILL.md b/.claude/skills/verify-open-pstack/SKILL.md index 7a257157..d7a6220c 100644 --- a/.claude/skills/verify-open-pstack/SKILL.md +++ b/.claude/skills/verify-open-pstack/SKILL.md @@ -64,7 +64,7 @@ For `project-skill`, invoke `/verify-open-pstack` in Claude and `$verify-open-ps Retain probe output, `receipt.json`, selected account emails, immutable source/installation provenance, redacted transcripts, artifacts, sandbox-denial proof, observable native API requests, and credential-cleanup outcome. Never retain copied credentials as evidence. Revalidate retained transcript/artifact hashes immediately before publication and around GitHub writes. Keep full detail in the receipt and bound the published summary. -The trusted parent publishes a pinned-SHA evidence comment and updates/reads back the PR template's `Live evidence:` block with installed version, surface, action, and observed result. Only complete evidence permits `live-gate=success` targeting that comment and subsequent draft-to-ready transition. Failure records its reason against the pinned SHA. Head movement aborts; revoke stale success on detected publication races and restore draft only if this run changed readiness. Preserve an already-ready PR on failure; record failed compensation prominently. +The trusted parent publishes evidence only in its run's own pinned-SHA comment. It never writes the PR body; `receipt.json` retains `proposedTemplate`, including the proposed `Live evidence:` block with installed version, surface, action, and observed result, for the operator to apply without overwriting unrelated edits. Only complete evidence permits `live-gate=success` targeting that comment. After publication, the helper makes at most one forward, idempotent mark-ready call and makes no call when the PR is already ready. It never reverses, undoes, or compensates draft/ready state. Failure or head movement publishes `live-gate=failure` on the pinned SHA and leaves the PR's draft state untouched; record failed status compensation prominently. PR #111 is already ready and must not be re-drafted during this review round. Builders do not post the gate, queue, or merge. After Unfret passes, the operator performs fresh exact-head live proof. Mergify queues automatically once required checks pass and the PR is ready; no manual queue submission is required. A queue-created head needs a new run, never a hand-posted replacement status. diff --git a/.claude/skills/verify-open-pstack/features/registry.json b/.claude/skills/verify-open-pstack/features/registry.json index 7ee26fa3..e6421bdc 100644 --- a/.claude/skills/verify-open-pstack/features/registry.json +++ b/.claude/skills/verify-open-pstack/features/registry.json @@ -41,11 +41,11 @@ ], "project": [ ".claude/skills/verify-open-pstack/**", ".agents/skills/verify-open-pstack", - "AGENTS.md", "CLAUDE.md", "tests/skill-collision-repro.sh" + "AGENTS.md", "CLAUDE.md", "tests/skill-collision-repro.sh", ".mergify.yml" ], "nonRuntime": [ "README.md", "CHANGES.md", "LICENSE", "LICENSE.md", "UPSTREAM.md", - ".gitignore", ".mergify.yml", "docs/**", "assets/**", + ".gitignore", "docs/**", "assets/**", "plugins/pstack/skills/poteto-mode/scripts/bootstrap.test.ts", "plugins/pstack/skills/poteto-mode/scripts/check-plan.test.ts" ] diff --git a/.claude/skills/verify-open-pstack/scripts/cli.test.ts b/.claude/skills/verify-open-pstack/scripts/cli.test.ts index 154fb46c..b105f919 100644 --- a/.claude/skills/verify-open-pstack/scripts/cli.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/cli.test.ts @@ -1,6 +1,6 @@ import { expect, test } from 'bun:test'; -import { bindPublisher, parse } from './cli.ts'; -import { mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; +import { bindPublisher, parse, publisherCode } from './cli.ts'; +import { cp, mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { doctor } from './doctor.ts'; @@ -31,18 +31,42 @@ async function publisherFixture() { const github: GitHub = { async pull() { calls.push('pull'); return structuredClone(pull); }, async files() { calls.push('files'); return [{ filename: 'README.md' }]; }, - async body(_pr, body) { calls.push('body'); pull.body = body; }, async comment() { calls.push('comment'); return url; }, async status(_sha, state) { calls.push(`status:${state}`); }, - async ready() { calls.push('ready'); }, async draft() { calls.push('draft'); }, + async ready() { calls.push('ready'); }, }; - return { root, project, sha, github, calls, pull, url }; + return { root, project, sha, github, calls, pull, url, loadedCode: publisherCode(root) }; } +test('already-loaded publisher A cannot bind checkout B after asynchronous startup', async () => { + const f = await publisherFixture(); + try { + await cp(import.meta.dir, join(f.project, 'scripts'), { recursive: true }); + const cli = join(f.project, 'scripts/cli.ts'); + const source = await readFile(cli, 'utf8'); + await writeFile(cli, source + '\nexport const loadedRevision = "A";\n'); + await command(['git', 'add', '.'], { cwd: f.root }); + await command(['git', '-c', 'commit.gpgsign=false', 'commit', '--quiet', '-m', 'publisher A'], { cwd: f.root }); + const loaded = await import(cli) as typeof import('./cli.ts') & { loadedRevision: string }; + // Model freshRoot yielding after imports while another process advances the checkout. + await writeFile(cli, source + '\nexport const loadedRevision = "B";\n'); + await command(['git', 'add', '.'], { cwd: f.root }); + await command(['git', '-c', 'commit.gpgsign=false', 'commit', '--quiet', '-m', 'publisher B'], { cwd: f.root }); + f.pull.head.sha = f.pull.base.sha = (await command(['git', 'rev-parse', 'HEAD'], { cwd: f.root })).trim(); + expect(loaded.loadedRevision).toBe('A'); + await expect(loaded.bindPublisher(f.root, 111, f.github)).rejects.toThrow('Publisher source changed while binding'); + expect(f.calls).toEqual(['pull']); + const current = await import(cli + '?head=B') as typeof import('./cli.ts') & { loadedRevision: string }; + expect(current.loadedRevision).toBe('B'); + const bound = await current.bindPublisher(f.root, 111, f.github); + await bound.files(f.pull.base.sha, f.pull.head.sha); + expect(f.calls).toEqual(['pull', 'pull', 'files']); + } finally { await rm(f.root, { recursive: true, force: true }); } +}); test('publisher pin rejects an older local checkout before classification or publication', async () => { const f = await publisherFixture(); try { f.pull.head.sha = 'a'.repeat(40); - await expect(bindPublisher(f.root, 111, f.github)).rejects.toThrow('HEAD differs from pinned SHA'); + await expect(bindPublisher(f.root, 111, f.github, f.loadedCode)).rejects.toThrow('HEAD differs from pinned SHA'); expect(f.calls).toEqual(['pull']); } finally { await rm(f.root, { recursive: true, force: true }); } }); @@ -51,39 +75,40 @@ test('publisher pin rejects modified registry, wrapper and code, plus untracked try { for (const path of ['features/registry.json', 'scripts/verify.sh', 'scripts/cli.ts', 'scripts/github.ts']) { await writeFile(join(f.project, path), 'unapproved publisher logic\n'); - await expect(bindPublisher(f.root, 111, f.github)).rejects.toThrow('Source differs from pinned Git tree'); + await expect(bindPublisher(f.root, 111, f.github, f.loadedCode)).rejects.toThrow('Source differs from pinned Git tree'); await writeFile(join(f.project, path), 'pinned publisher source\n'); } await writeFile(join(f.project, 'scripts/untracked.ts'), 'untracked logic\n'); - await expect(bindPublisher(f.root, 111, f.github)).rejects.toThrow('Source differs from pinned Git tree'); + await expect(bindPublisher(f.root, 111, f.github, f.loadedCode)).rejects.toThrow('Source differs from pinned Git tree'); expect(f.calls.every(call => call === 'pull')).toBe(true); } finally { await rm(f.root, { recursive: true, force: true }); } }); test('publisher source rechecks protect every success boundary but retain failure compensation', async () => { const f = await publisherFixture(); try { - const bound = await bindPublisher(f.root, 111, f.github); + const bound = await bindPublisher(f.root, 111, f.github, f.loadedCode); await writeFile(join(f.project, 'scripts/github.ts'), 'changed after binding\n'); for (const operation of [ - () => bound.files(f.sha, f.sha), () => bound.body(111, 'evidence'), + () => bound.files(f.sha, f.sha), () => bound.comment(111, 'evidence'), () => bound.status(f.sha, 'success', f.url, 'verified'), () => bound.ready(111), ]) await expect(operation()).rejects.toThrow('Source differs from pinned Git tree'); expect(f.calls).toEqual(['pull']); - await bound.status(f.sha, 'failure', f.url, 'withdraw success'); await bound.draft(111); - expect(f.calls).toEqual(['pull', 'status:failure', 'draft']); + await bound.status(f.sha, 'failure', f.url, 'withdraw success'); + expect(f.calls).toEqual(['pull', 'status:failure']); + expect('body' in bound).toBe(false); expect('draft' in bound).toBe(false); } finally { await rm(f.root, { recursive: true, force: true }); } }); test('publisher pin also gates docs-only classification and success with no harness fallback', async () => { const f = await publisherFixture(); try { - const github = await bindPublisher(f.root, 111, f.github); + const github = await bindPublisher(f.root, 111, f.github, f.loadedCode); const output = join(f.root, 'output'); await mkdir(output); const receipt = await verify({ pr: 111, selfTest: false, root: output, registry: validateRegistry(registry), github, driver: { async prepare() { throw new Error('Docs must not prepare harnesses'); }, async exercise() { throw new Error('Docs must not exercise harnesses'); } }, persist: async () => {} }); expect(receipt.selection.noRuntime).toBe(true); expect(receipt.status).toBe('success'); expect(f.calls).toContain('status:success'); await expect(github.files(f.sha, 'a'.repeat(40))).rejects.toThrow('classification SHAs'); await expect(github.status('a'.repeat(40), 'success', f.url, 'wrong head')).rejects.toThrow('pinned SHA'); - await expect(github.body(112, 'wrong PR')).rejects.toThrow('one PR'); + await expect(github.comment(112, 'wrong PR')).rejects.toThrow('one PR'); await writeFile(join(f.root, 'README.md'), 'new docs head\n'); await command(['git', 'add', 'README.md'], { cwd: f.root }); await command(['git', '-c', 'commit.gpgsign=false', 'commit', '--quiet', '-m', 'new docs head'], { cwd: f.root }); diff --git a/.claude/skills/verify-open-pstack/scripts/cli.ts b/.claude/skills/verify-open-pstack/scripts/cli.ts index 3d4b3fa8..181fb0b6 100644 --- a/.claude/skills/verify-open-pstack/scripts/cli.ts +++ b/.claude/skills/verify-open-pstack/scripts/cli.ts @@ -1,4 +1,6 @@ import { readFile } from 'node:fs/promises'; +import { lstatSync, readFileSync, readdirSync } from 'node:fs'; +import { createHash } from 'node:crypto'; import { join, resolve } from 'node:path'; import { validateRegistry } from './core.ts'; import { doctor } from './doctor.ts'; @@ -9,11 +11,25 @@ import { verify } from './verify.ts'; import { sourceDigest, sourceHash } from './provenance.ts'; import type { GitHub } from './types.ts'; -export async function bindPublisher(repository: string, pr: number, github: GitHub): Promise { +export function publisherCode(repository: string): string { + const hash = createHash('sha256'); + const scripts = join(repository, '.claude/skills/verify-open-pstack/scripts'); + for (const name of readdirSync(scripts).sort()) { + if (!name.endsWith('.ts')) continue; + const path = join(scripts, name), stat = lstatSync(path); + if (!stat.isFile() || stat.isSymbolicLink()) throw new Error('Publisher module must be a regular file'); + hash.update(name + '\0').update(readFileSync(path)).update('\0'); + } + return hash.digest('hex'); +} +// Capture loaded module provenance synchronously, before main yields to freshRoot or GitHub. +const loadedCode = publisherCode(resolve(import.meta.dir, '../../../..')); + +export async function bindPublisher(repository: string, pr: number, github: GitHub, startupCode = loadedCode): Promise { const loadedSource = await sourceDigest(repository); const pull = await github.pull(pr); const pinnedSource = await sourceHash(repository, pull.head.sha); - if (pinnedSource !== loadedSource) throw new Error('Publisher source changed while binding the pinned head'); + if (pinnedSource !== loadedSource || publisherCode(repository) !== startupCode) throw new Error('Publisher source changed while binding the pinned head'); const recheck = async (): Promise => { if (await sourceHash(repository, pull.head.sha) !== pinnedSource) throw new Error('Publisher source changed after pinning'); }; @@ -26,7 +42,6 @@ export async function bindPublisher(repository: string, pr: number, github: GitH if (base !== pull.base.sha || head !== pull.head.sha) throw new Error('Publisher pin differs from classification SHAs'); await recheck(); return github.files(base, head); }, - async body(number, body) { checkPr(number); await recheck(); await github.body(number, body); }, async comment(number, body) { checkPr(number); await recheck(); return github.comment(number, body); }, async status(sha, state, target, description) { if (sha !== pull.head.sha) throw new Error('Publisher status differs from pinned SHA'); @@ -35,7 +50,6 @@ export async function bindPublisher(repository: string, pr: number, github: GitH await github.status(sha, state, target, description); }, async ready(number) { checkPr(number); await recheck(); await github.ready(number); }, - async draft(number) { checkPr(number); await github.draft(number); }, }; } type Options = { mode: 'doctor'; output: string; pr: number; selfTest: boolean; candidate: boolean } diff --git a/.claude/skills/verify-open-pstack/scripts/core.test.ts b/.claude/skills/verify-open-pstack/scripts/core.test.ts index 9709fa76..5d4912c3 100644 --- a/.claude/skills/verify-open-pstack/scripts/core.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/core.test.ts @@ -88,7 +88,7 @@ describe('registry and ownership', () => { expect(s.features).toEqual([]); }); test('consumed instructions and known verifier enforcement require native project proof', () => { - for (const filename of ['AGENTS.md', 'CLAUDE.md', 'tests/skill-collision-repro.sh']) { + for (const filename of ['AGENTS.md', 'CLAUDE.md', 'tests/skill-collision-repro.sh', '.mergify.yml']) { const s = classify([{ filename }], registry); expect(s.noRuntime).toBe(false); expect(s.features).toEqual(['project-skill']); diff --git a/.claude/skills/verify-open-pstack/scripts/github.ts b/.claude/skills/verify-open-pstack/scripts/github.ts index 5c72af2f..d83ad1ad 100644 --- a/.claude/skills/verify-open-pstack/scripts/github.ts +++ b/.claude/skills/verify-open-pstack/scripts/github.ts @@ -42,10 +42,6 @@ export class Publisher implements GitHub { return parseChangedFiles(output); } finally { await rm(repository, { recursive: true, force: true }); } } - async body(pr: number, body: string): Promise { - if (body.length > 60000) throw new Error('PR body exceeds evidence budget'); - await this.api(`pulls/${pr}`, ['--method', 'PATCH', '-f', `body=${body}`]); - } async comment(pr: number, body: string): Promise { if (body.length > 60000) throw new Error('Evidence exceeds comment budget'); const result = await this.api(`issues/${pr}/comments`, ['--method', 'POST', '-f', `body=${body}`]) as { html_url: string }; @@ -57,5 +53,4 @@ export class Publisher implements GitHub { await this.api(`statuses/${sha}`, ['--method', 'POST', '-f', 'context=live-gate', '-f', `state=${state}`, '-f', `target_url=${target}`, '-f', `description=${description.slice(0, 140)}`], state === 'failure'); } async ready(pr: number): Promise { await this.run(['gh', 'pr', 'ready', String(pr), '--repo', REPO]); } - async draft(pr: number): Promise { await this.run(['gh', 'pr', 'ready', String(pr), '--undo', '--repo', REPO], { allowInterrupted: true }); } } diff --git a/.claude/skills/verify-open-pstack/scripts/harness.ts b/.claude/skills/verify-open-pstack/scripts/harness.ts index eb16eff8..556649a8 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.ts @@ -4,7 +4,7 @@ import { createInterface } from 'node:readline/promises'; import { requiredFeatures } from './core.ts'; import { command, interruption, isolatedEnv, redact, retainedFile, save, treeHash, type Command } from './io.ts'; import { doctor } from './doctor.ts'; -import { copyCredentials, createSandbox, protectSources, removeCredentials, sandboxed, selectedAccounts, vaultRoot } from './isolation.ts'; +import { copyCredentials, createSandbox, protectSources, registerSessionCredentials, removeCredentials, sandboxed, selectedAccounts, vaultRoot } from './isolation.ts'; import { sourceDigest, sourceHash } from './provenance.ts'; import { validateMacTests } from './verify.ts'; import { HARNESSES, REPO, type Accounts, type Driver, type Harness, type Installation, type Observation, type Receipt } from './types.ts'; @@ -50,11 +50,20 @@ export class MacDriver implements Driver { private profiles = new Map(); constructor(private run: Command = command, private review: Ask = ask, private accounts?: Accounts, private boundary: typeof createSandbox = createSandbox) {} - async cleanup(): Promise { await removeCredentials(this.copied, this.profiles, this.run); } + async cleanup(): Promise { + try { + for (const home of this.profiles.keys()) await registerSessionCredentials(home); + } finally { + await removeCredentials(this.copied, this.profiles, this.run); + } + } private candidate(home: string): Command { const profile = this.profiles.get(home); if (!profile) throw new Error('Candidate sandbox not established'); - return (args, options = {}) => this.run(sandboxed(profile, args), { ...options, env: isolatedEnv(home) }); + return async (args, options = {}) => { + try { return await this.run(sandboxed(profile, args), { ...options, env: isolatedEnv(home) }); } + finally { await registerSessionCredentials(home); } + }; } async prepare(receipt: Receipt): Promise { const root = receipt.artifactRoot; diff --git a/.claude/skills/verify-open-pstack/scripts/io.test.ts b/.claude/skills/verify-open-pstack/scripts/io.test.ts index 564f2e8d..34abc557 100644 --- a/.claude/skills/verify-open-pstack/scripts/io.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/io.test.ts @@ -47,6 +47,17 @@ describe('retained JSON and file boundaries', () => { expect(JSON.parse(await readFile(path, 'utf8'))).toEqual({ ...evidence, checks: { explanation: '[REDACTED]' } }); }); + test.each(['text', 'binary'])('rejects and removes retained %s artifacts containing copied credentials', async kind => { + const root = await fixture(), path = join(root, 'artifact'); + const token = `known-copied-${kind}-opaque-credential`; + registerSecrets([token]); + await writeFile(path, kind === 'binary' ? Buffer.concat([Buffer.from([0, 255, 128]), Buffer.from(token), Buffer.from([0, 254])]) : `reviewed result ${token}`); + await expect(retainedFile(root, path)).rejects.toThrow('Evidence contains credentials'); + expect(await Bun.file(path).exists()).toBe(false); + await writeFile(path, kind === 'binary' ? Buffer.from([0, 255, 128, 0, 254]) : 'reviewed sanitized result'); + expect((await retainedFile(root, path)).path).toBe('artifact'); + }); + test('permits evidence below an ancestor named state and canonicalizes output aliases', async () => { const root = await fixture(), output = join(root, 'state', 'run'), alias = join(root, 'alias'); await mkdir(output, { recursive: true }); diff --git a/.claude/skills/verify-open-pstack/scripts/io.ts b/.claude/skills/verify-open-pstack/scripts/io.ts index fe0ea889..d0ae2da5 100644 --- a/.claude/skills/verify-open-pstack/scripts/io.ts +++ b/.claude/skills/verify-open-pstack/scripts/io.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { lstat, mkdir, readFile, readdir, realpath, rename, writeFile } from 'node:fs/promises'; +import { lstat, mkdir, readFile, readdir, realpath, rename, unlink, writeFile } from 'node:fs/promises'; import { isAbsolute, join, relative, resolve } from 'node:path'; export type Command = (args: string[], options?: { cwd?: string; env?: Record; interactive?: boolean; allowInterrupted?: boolean }) => Promise; @@ -118,5 +118,10 @@ export async function retainedFile(root: string, path: string): Promise<{ path: if (!actual.startsWith(output + '/') || actual === state || actual.startsWith(state + '/')) throw new Error('Evidence must be retained outside isolated state, within output'); const stat = await lstat(actual); if (!stat.isFile() || !stat.size) throw new Error('Evidence file must be a nonempty regular file'); - return { path: relative(output, actual), sha256: sha256(await readFile(actual)) }; + const bytes = await readFile(actual); + if ([...secrets].some(secret => bytes.includes(Buffer.from(secret))) || redact(bytes.toString('latin1')) !== bytes.toString('latin1')) { + await unlink(actual); + throw new Error('Evidence contains credentials; contaminated external file removed'); + } + return { path: relative(output, actual), sha256: sha256(bytes) }; } diff --git a/.claude/skills/verify-open-pstack/scripts/isolation.test.ts b/.claude/skills/verify-open-pstack/scripts/isolation.test.ts index bf34db27..675b2804 100644 --- a/.claude/skills/verify-open-pstack/scripts/isolation.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/isolation.test.ts @@ -3,8 +3,8 @@ import { lstat, mkdir, mkdtemp, readFile, realpath, rm, rmdir, stat, symlink, wr import { randomUUID } from 'node:crypto'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; -import { command, isolatedEnv, redact } from './io.ts'; -import { copyCredentials, createSandbox, protectSources, removeCredentials, resolveAppleGit, sandboxed, sandboxProfile, selectedAccounts } from './isolation.ts'; +import { command, isolatedEnv, redact, retainedFile } from './io.ts'; +import { copyCredentials, createSandbox, protectSources, registerSessionCredentials, removeCredentials, resolveAppleGit, sandboxed, sandboxProfile, selectedAccounts } from './isolation.ts'; import type { Accounts } from './types.ts'; const roots: string[] = []; @@ -95,6 +95,45 @@ describe('disposable credential boundary', () => { expect(redact(tokens.join(' '))).toBe(tokens.map(() => '[REDACTED]').join(' ')); await removeCredentials(copied); }); + test('real provider refreshes register opaque and JWT credentials before accepting evidence or cleanup', async () => { + const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; + await copyCredentials(home, base, accounts, copied); + const original = await Promise.all(copied.map(path => readFile(path, 'utf8'))); + const tokens = ['refreshed-claude-access.opaque', 'refreshed-claude-refresh.opaque', 'refreshed-codex-access.opaque', 'refreshed-codex-refresh.opaque', 'eyJhbGciOiJSUzI1NiJ9.refresh.fixtureSignature']; + await writeFile(copied[0]!, JSON.stringify({ claudeAiOauth: { accessToken: tokens[0], refreshToken: tokens[1] } })); + await writeFile(copied[1]!, JSON.stringify({ tokens: { access_token: tokens[2], refresh_token: tokens[3], id_token: tokens[4] } })); + await registerSessionCredentials(home); + expect(redact(tokens.join(' '))).toBe(tokens.map(() => '[REDACTED]').join(' ')); + for (const [i, token] of tokens.entries()) { + const evidence = join(root, `refreshed-artifact-${i}`); + await writeFile(evidence, Buffer.concat([Buffer.from([255, 0]), Buffer.from(token), Buffer.from([128, 0])])); + await expect(retainedFile(root, evidence)).rejects.toThrow('Evidence contains credentials'); + expect(await Bun.file(evidence).exists()).toBe(false); + } + await removeCredentials(copied); + await registerSessionCredentials(home); + expect(await Bun.file(home).exists()).toBe(false); + expect(original).toEqual(await Promise.all([ + readFile(join(base, 'claude', accounts.claude, '.credentials.json'), 'utf8'), + readFile(join(base, 'codex', accounts.codex, 'auth.json'), 'utf8'), + ])); + }); + test('session credential discovery refuses redirected directories and credential files', async () => { + const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; + await copyCredentials(home, base, accounts, copied); + const external = join(root, 'external'); + await mkdir(external); + await writeFile(join(external, '.credentials.json'), '{"claudeAiOauth":{"accessToken":"outside-fixture"}}'); + await rm(join(home, '.claude'), { recursive: true }); + await symlink(external, join(home, '.claude')); + await expect(registerSessionCredentials(home)).rejects.toThrow('redirected'); + await rm(join(home, '.claude')); + await mkdir(join(home, '.claude')); + await symlink(join(external, '.credentials.json'), copied[0]!); + await expect(registerSessionCredentials(home)).rejects.toThrow('regular files'); + expect(await readFile(join(external, '.credentials.json'), 'utf8')).toContain('outside-fixture'); + await removeCredentials(copied); + }); test('cleanup removes relocated and refreshed credentials anywhere in candidate state while preserving external evidence', async () => { const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; await copyCredentials(home, base, accounts, copied); diff --git a/.claude/skills/verify-open-pstack/scripts/isolation.ts b/.claude/skills/verify-open-pstack/scripts/isolation.ts index 794f1d6c..0f75ed45 100644 --- a/.claude/skills/verify-open-pstack/scripts/isolation.ts +++ b/.claude/skills/verify-open-pstack/scripts/isolation.ts @@ -44,6 +44,29 @@ export async function copyCredentials(home: string, vault: string, accounts: Acc await writeFile(target, bytes, { mode: 0o600, flag: 'wx' }); } } +export async function registerSessionCredentials(home: string): Promise { + const root = await realpath(home).catch((error: NodeJS.ErrnoException) => { + if (error.code !== 'ENOENT') throw error; + return undefined; + }); + if (root === undefined) return; + if (root !== home) throw new Error('Session credentials require canonical candidate state'); + for (const tool of HARNESSES) { + const dir = join(home, tool === 'claude' ? '.claude' : '.codex'); + if (await realpath(dir) !== dir) throw new Error('Session credential directory redirected'); + const file = join(dir, tool === 'claude' ? '.credentials.json' : 'auth.json'); + const info = await lstat(file).catch((error: NodeJS.ErrnoException) => { + if (error.code !== 'ENOENT') throw error; + return undefined; + }); + if (!info) continue; + if (!info.isFile() || info.isSymbolicLink()) throw new Error('Session credentials must be regular files'); + const auth = JSON.parse(await readFile(file, 'utf8')); + const tokens = tool === 'claude' ? auth.claudeAiOauth : auth.tokens; + if (!tokens || typeof tokens !== 'object') throw new Error('Unknown session OAuth credential format'); + registerSecrets(Object.entries(tokens).flatMap(([key, value]) => /token/i.test(key) && typeof value === 'string' ? [value] : [])); + } +} export async function removeCredentials(paths: string[], profiles?: Map, run: Command = command): Promise { for (const home of new Set(paths.map(path => dirname(dirname(path))))) { await sourceDirectory(dirname(home)); diff --git a/.claude/skills/verify-open-pstack/scripts/types.ts b/.claude/skills/verify-open-pstack/scripts/types.ts index f1ce8511..4c33f604 100644 --- a/.claude/skills/verify-open-pstack/scripts/types.ts +++ b/.claude/skills/verify-open-pstack/scripts/types.ts @@ -36,17 +36,15 @@ export interface Receipt { schema: 1; repo: string; pr: number; sha: string; base: string; phase: Phase; selection: Selection; installations: Installation[]; observations: Observation[]; selfTest: boolean; artifactRoot: string; failure?: string; - commentUrl?: string; status?: 'success' | 'failure'; madeReady?: boolean; + commentUrl?: string; status?: 'success' | 'failure'; madeReady?: boolean; proposedTemplate?: string; compensation?: string[]; cleanup: string; started: string; macProof?: MacProof; } export interface GitHub { pull(pr: number): Promise; files(base: string, head: string): Promise; - body(pr: number, body: string): Promise; comment(pr: number, body: string): Promise; status(sha: string, state: 'success' | 'failure', target: string, description: string): Promise; ready(pr: number): Promise; - draft(pr: number): Promise; } export interface Driver { prepare(receipt: Receipt): Promise; diff --git a/.claude/skills/verify-open-pstack/scripts/verify.test.ts b/.claude/skills/verify-open-pstack/scripts/verify.test.ts index 311fd014..2534eaab 100644 --- a/.claude/skills/verify-open-pstack/scripts/verify.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/verify.test.ts @@ -26,10 +26,9 @@ function fixture(runtime = false, selfTest = false) { const github: GitHub = { async pull() { mark('pull'); return structuredClone(pull); }, async files(base, head) { expect(base).toBe(BASE); expect(head).toBe(SHA); mark('files'); return [{ filename: runtime ? 'plugins/pstack/skills/architect/SKILL.md' : 'README.md' }]; }, - async body(_pr, body) { mark('body'); pull.body = body; }, async comment(_pr, body) { mark('comment'); expect(body).toContain(SHA); return URL; }, - async status(sha, state, target) { mark(`status:${state}:${sha}`); expect(target).toBe(URL); }, - async ready() { mark('ready'); pull.draft = false; }, async draft() { mark('draft'); pull.draft = true; }, + async status(sha, state, target) { mark(`status:${state}:${sha}`); expect([URL, `https://github.com/${REPO}/pull/123`]).toContain(target); }, + async ready() { mark('ready'); pull.draft = false; }, }; const driver: Driver = { async prepare(r) { @@ -80,12 +79,13 @@ describe('exact-head publication state machine', () => { expect(receipt.status).toBe('success'); expect(receipt.sha).toBe(SHA); expect(evidence(receipt)).toContain('no runtime change'); expect(f.calls).not.toContain('prepare'); expect(f.calls).not.toContain('exercise'); - expect(f.calls.indexOf('comment')).toBeLessThan(f.calls.indexOf('body')); - expect(f.calls.indexOf('body')).toBeLessThan(f.calls.indexOf(`status:success:${SHA}`)); + expect(f.calls.indexOf('comment')).toBeLessThan(f.calls.indexOf(`status:success:${SHA}`)); expect(f.calls.indexOf(`status:success:${SHA}`)).toBeLessThan(f.calls.indexOf('ready')); - expect(f.pull.body).toContain(URL); expect(f.pull.body).toContain('Retain these notes.'); expect(f.pull.body).not.toContain('_pending_'); - expect(f.pull.body).toContain('- [x] The installed version, action, and observed result appear below.'); - expect(liveEvidenceBody(f.pull.body, receipt)).toBe(f.pull.body); + expect(f.calls.filter(call => call === 'ready')).toHaveLength(1); expect(f.calls).not.toContain('draft'); + expect(f.calls).not.toContain('body'); + expect(f.pull.body).toContain('_pending_'); + expect(receipt.proposedTemplate).toContain(URL); + expect(receipt.proposedTemplate).toContain('Retain these notes.'); }); test('unmanaged evidence replacement preserves every following heading and operator note', async () => { const receipt = await verify(fixture().options); @@ -159,23 +159,28 @@ describe('exact-head publication state machine', () => { test('publication failure does not repeat an already completed cleanup', async () => { const f = fixture(true); f.options.driver.cleanup = async () => { f.calls.push('cleanup'); }; - f.hook(call => { if (call === 'body') throw new Error('publisher failed'); }); + f.hook(call => { if (call === 'comment') throw new Error('publisher failed'); }); await expect(verify(f.options)).rejects.toThrow('publisher failed'); expect(f.calls.filter(call => call === 'cleanup')).toHaveLength(1); expect(f.saved.at(-1)?.cleanup).toContain('removed'); }); - test('body readback must confirm installed evidence before success or readiness', async () => { - const f = fixture(); f.options.github.body = async () => {}; - await expect(verify(f.options)).rejects.toThrow('body update was not retained'); - expect(f.calls).not.toContain(`status:success:${SHA}`); expect(f.calls).not.toContain('ready'); + test('concurrent author body edits are untouched and the operator proposal is retained', async () => { + const f = fixture(), edited = '# Author revision\nUnrelated edits must survive.'; + expect('body' in f.options.github).toBe(false); + f.hook(call => { if (call === 'comment') f.pull.body = edited; }); + const receipt = await verify(f.options); + expect(f.pull.body).toBe(edited); expect(f.calls).not.toContain('body'); + expect(receipt.proposedTemplate).toContain('Live evidence:'); expect(receipt.proposedTemplate).toContain(URL); + expect(f.saved.at(-1)?.proposedTemplate).toBe(receipt.proposedTemplate); }); test('already-ready PR stays ready when publication fails', async () => { - const f = fixture(); f.pull.draft = false; f.hook(call => { if (call === 'body') throw new Error('body denied'); }); - await expect(verify(f.options)).rejects.toThrow('body denied'); expect(f.calls).not.toContain('draft'); expect(f.pull.draft).toBe(false); + const f = fixture(); f.pull.draft = false; f.hook(call => { if (call === 'comment') throw new Error('comment denied'); }); + await expect(verify(f.options)).rejects.toThrow('comment denied'); expect(f.calls).not.toContain('draft'); expect(f.pull.draft).toBe(false); + expect(f.calls).toContain(`status:failure:${SHA}`); }); test('transcript or artifact overwrite after acceptance prevents publication and readiness', async () => { for (const file of ['retained.log', 'fixture.json']) { - for (const boundary of ['exercise', 'comment', 'body', `status:success:${SHA}`]) { + for (const boundary of ['exercise', 'comment', `status:success:${SHA}`]) { const f = fixture(true); f.hook(call => { if (call === boundary) writeFileSync(join(f.options.root, file), 'overwritten'); }); await expect(verify(f.options)).rejects.toThrow('changed after acceptance'); expect(f.calls).not.toContain('ready'); if (boundary === `status:success:${SHA}`) expect(f.calls).toContain(`status:failure:${SHA}`); @@ -198,11 +203,12 @@ describe('exact-head publication state machine', () => { } }); test('head movement during classify, prepare, exercise, or comment aborts without success or readiness', async () => { - for (const boundary of ['files', 'prepare', 'exercise', 'comment', 'body']) { + for (const boundary of ['files', 'prepare', 'exercise', 'comment']) { const f = fixture(true); f.hook(call => { if (call === boundary) f.pull.head.sha = NEXT; }); await expect(verify(f.options)).rejects.toThrow('changed'); expect(f.calls.some(c => c.startsWith('status:success'))).toBe(false); expect(f.calls).not.toContain('ready'); expect(f.saved.at(-1)?.phase).toBe('head-moved'); + expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.calls).not.toContain('draft'); } }); test('head or base movement after status withdraws success only on original SHA', async () => { @@ -213,10 +219,11 @@ describe('exact-head publication state machine', () => { expect(f.calls).not.toContain('ready'); expect(f.saved.at(-1)?.status).toBe('failure'); } }); - test('post-ready head race restores draft and revokes the old status', async () => { + test('post-ready head race revokes pinned success without reversing readiness', async () => { const f = fixture(); f.hook(call => { if (call === 'ready') f.pull.head.sha = NEXT; }); await expect(verify(f.options)).rejects.toThrow('changed'); - expect(f.calls).toContain('draft'); expect(f.pull.draft).toBe(true); expect(f.saved.at(-1)?.madeReady).toBe(false); + expect(f.calls).not.toContain('draft'); expect(f.pull.draft).toBe(false); expect(f.saved.at(-1)?.madeReady).toBe(true); + expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.calls.filter(call => call === 'ready')).toHaveLength(1); }); test('incomplete exercise records failure against pinned head and never readies', async () => { const f = fixture(true); f.options.driver.exercise = async () => []; @@ -254,7 +261,7 @@ describe('exact-head publication state machine', () => { await expect(verify(f.options)).rejects.toThrow('already ready'); expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.calls).not.toContain('draft'); expect(f.pull.draft).toBe(false); }); - test('receipt-write errors cannot prevent compensating an accepted success or ready write', async () => { + test('receipt-write errors cannot prevent revoking pinned success and never reverse readiness', async () => { for (const boundary of [`status:success:${SHA}`, 'ready']) { const f = fixture(); let deny = false; f.hook(call => { if (call === boundary) { f.pull.base.sha = NEXT; deny = true; } }); @@ -264,7 +271,8 @@ describe('exact-head publication state machine', () => { }; await expect(verify(f.options)).rejects.toThrow('changed'); expect(f.calls).toContain(`status:failure:${SHA}`); - if (boundary === 'ready') { expect(f.calls).toContain('draft'); expect(f.pull.draft).toBe(true); } + expect(f.calls).not.toContain('draft'); + if (boundary === 'ready') expect(f.pull.draft).toBe(false); } }); test('failed compensation is retained, not reported as successful', async () => { @@ -317,12 +325,11 @@ describe('publisher request boundaries', () => { catch (error) { if (!String(error).includes('Interrupted by')) throw error; } } await publisher.status(${JSON.stringify(SHA)}, 'failure', ${JSON.stringify(URL)}, 'withdraw'); - await publisher.draft(123); console.log(JSON.stringify(executed)); `; const calls = JSON.parse(execFileSync(process.execPath, ['-e', script], { encoding: 'utf8' }).trim()) as string[][]; - expect(calls).toHaveLength(2); - expect(calls[0]).toContain('state=failure'); expect(calls[1]).toContain('--undo'); + expect(calls).toHaveLength(1); + expect(calls[0]).toContain('state=failure'); expect(calls.flat()).not.toContain('--undo'); } }); test('status targets the supplied exact SHA and evidence URL, never a branch', async () => { diff --git a/.claude/skills/verify-open-pstack/scripts/verify.ts b/.claude/skills/verify-open-pstack/scripts/verify.ts index ecf34001..ecb35fe0 100644 --- a/.claude/skills/verify-open-pstack/scripts/verify.ts +++ b/.claude/skills/verify-open-pstack/scripts/verify.ts @@ -113,10 +113,8 @@ export async function verify(options: { pr: number; selfTest: boolean; root: str await phase('publish'); await revalidateEvidence(receipt); receipt.commentUrl = await github.comment(receipt.pr, evidence(receipt)); await persist(receipt); const pull = await current(); await revalidateEvidence(receipt); - const body = liveEvidenceBody(pull.body, receipt); - await github.body(receipt.pr, body); - if ((await current()).body !== body) throw new Error('PR live-evidence body update was not retained'); - await revalidateEvidence(receipt); + receipt.proposedTemplate = liveEvidenceBody(pull.body, receipt); + await persist(receipt); await current(); await revalidateEvidence(receipt); successAttempted = true; await github.status(receipt.sha, 'success', receipt.commentUrl, receipt.selection.noRuntime ? 'no runtime change' : 'All mapped features passed in both harnesses'); receipt.status = 'success'; await persist(receipt); await current(); await revalidateEvidence(receipt); @@ -148,13 +146,15 @@ export async function verify(options: { pr: number; selfTest: boolean; root: str if (successAttempted && receipt.commentUrl) await recover('withdraw success on pinned SHA', async () => { await github.status(receipt.sha, 'failure', receipt.commentUrl!, 'Verification aborted; rerun required'); receipt.status = 'failure'; }); - // A failed/ambiguous readiness response does not establish ownership of that transition. - if (receipt.madeReady) await recover('restore draft', async () => { await github.draft(receipt.pr); receipt.madeReady = false; }); - if (!successAttempted) await recover('publish pinned failure', async () => { - await current(); - receipt.commentUrl = await github.comment(receipt.pr, evidence(receipt)); await current(); - await github.status(receipt.sha, 'failure', receipt.commentUrl, 'Verification failed; see retained evidence'); receipt.status = 'failure'; await current(); - }); + if (!successAttempted) { + await recover('publish failure evidence', async () => { + await current(); + receipt.commentUrl = await github.comment(receipt.pr, evidence(receipt)); + }); + await recover('publish pinned failure', async () => { + await github.status(receipt.sha, 'failure', receipt.commentUrl ?? `https://github.com/${REPO}/pull/${receipt.pr}`, 'Verification failed; see retained evidence'); receipt.status = 'failure'; + }); + } await recover('persist compensation receipt', async () => { await persist(receipt); }); throw new Error(receipt.failure); } From 6313b6c4995e2bea162b80f05ffc314337f5c56f Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 04:44:06 +0000 Subject: [PATCH 15/20] refactor(#90): adopt approved private-evidence verification contract Remove rejected PR mutation, sandbox, and publisher attestation mechanisms while preserving exact-head evidence, disposable credentials with selected-account caam health checks, private raw evidence, and the setup config-home blocker. Include the already-validated upstream tree without rewriting reviewed branch history. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .claude/skills/verify-open-pstack/SKILL.md | 90 +-- .../verify-open-pstack/features/README.md | 22 +- .../verify-open-pstack/features/assets.md | 19 + .../verify-open-pstack/features/isolation.md | 54 +- .../verify-open-pstack/features/registry.json | 7 +- .../verify-open-pstack/features/runner.md | 6 +- .../verify-open-pstack/features/setup.md | 12 +- .../features/shipped-tools.md | 6 +- .../features/skill-invocation.md | 10 +- .../verify-open-pstack/scripts/cli.test.ts | 132 +---- .../skills/verify-open-pstack/scripts/cli.ts | 71 +-- .../verify-open-pstack/scripts/core.test.ts | 76 ++- .../skills/verify-open-pstack/scripts/core.ts | 30 +- .../verify-open-pstack/scripts/doctor.test.ts | 65 +++ .../verify-open-pstack/scripts/doctor.ts | 30 +- .../verify-open-pstack/scripts/github.ts | 7 +- .../scripts/harness.test.ts | 226 ++++---- .../verify-open-pstack/scripts/harness.ts | 82 +-- .../verify-open-pstack/scripts/io.test.ts | 84 +-- .../skills/verify-open-pstack/scripts/io.ts | 44 +- .../scripts/isolation.test.ts | 538 ++++-------------- .../verify-open-pstack/scripts/isolation.ts | 294 +++------- .../verify-open-pstack/scripts/types.ts | 18 +- .../verify-open-pstack/scripts/verify.sh | 3 - .../verify-open-pstack/scripts/verify.test.ts | 356 ++++-------- .../verify-open-pstack/scripts/verify.ts | 195 +++---- .github/pull_request_template.md | 7 +- AGENTS.md | 4 +- docs/plans/2026-10-01-triage-and-roadmap.md | 12 +- .../references/provider-dispatch.md | 2 +- .../scripts/runner/commands.test.ts | 4 +- .../poteto-mode/scripts/runner/commands.ts | 5 +- .../poteto-mode/scripts/runner/run.test.ts | 21 + .../skills/poteto-mode/scripts/runner/run.ts | 7 +- 34 files changed, 1006 insertions(+), 1533 deletions(-) create mode 100644 .claude/skills/verify-open-pstack/features/assets.md create mode 100644 .claude/skills/verify-open-pstack/scripts/doctor.test.ts diff --git a/.claude/skills/verify-open-pstack/SKILL.md b/.claude/skills/verify-open-pstack/SKILL.md index d7a6220c..ea34b027 100644 --- a/.claude/skills/verify-open-pstack/SKILL.md +++ b/.claude/skills/verify-open-pstack/SKILL.md @@ -1,84 +1,98 @@ --- name: verify-open-pstack -description: Verify an Open Pstack PR's exact head in isolated Claude Code and Codex sessions, retain operator-reviewed real-surface evidence, and publish the live gate. +description: Verify an Open Pstack PR's exact candidate head in native Claude Code and Codex sessions, retain operator-reviewed real-surface evidence, and publish the exact-SHA live gate. --- # Verify Open Pstack ## Launch -This repository-local, non-shipped skill is shared with Codex through `.agents/skills/verify-open-pstack`. Run the reviewed verifier from a trusted checkout on the operator's Mac. GitHub publishing belongs exclusively to the trusted parent; never expose its credentials to candidate processes. The helper never queues or merges. +This repository-local, non-shipped skill is shared with Codex through `.agents/skills/verify-open-pstack`. Run the reviewed verifier from a trusted `main` checkout. The trusted parent owns GitHub reads and publication; candidate processes never receive publisher credentials. Its only GitHub writes are one structured evidence comment and `live-gate` status on the exact candidate SHA; it never queues or merges. + +Install development dependencies separately when testing this skill; `verify.sh` does not install at runtime: + +```sh +(cd .claude/skills/verify-open-pstack && bun install --frozen-lockfile --ignore-scripts) +``` + +Then run from the trusted publisher checkout: ```sh -(cd .claude/skills/verify-open-pstack && bun install --frozen-lockfile) PR=111 # supplied delivery PR number CLAUDE_ACCOUNT='selected-claude-account@example.com' # explicit caam vault identity CODEX_ACCOUNT='selected-codex-account@example.com' # explicit caam vault identity SESSION="$(mktemp -d "${TMPDIR:-/tmp}/open-pstack-evidence.XXXXXX")" EVIDENCE="$SESSION/live" # must not exist yet -.claude/skills/verify-open-pstack/scripts/verify.sh doctor --output "$SESSION/probe" +.claude/skills/verify-open-pstack/scripts/verify.sh doctor \ + --claude-account "$CLAUDE_ACCOUNT" --codex-account "$CODEX_ACCOUNT" --output "$SESSION/probe" .claude/skills/verify-open-pstack/scripts/verify.sh run --pr "$PR" --self-test \ --claude-account "$CLAUDE_ACCOUNT" --codex-account "$CODEX_ACCOUNT" --output "$EVIDENCE" ``` -Replace both account placeholders with explicitly selected caam 0.1.22 vault identities; no default account or new authentication flow is allowed. Omit `--self-test` for ordinary plugin verification; it is mandatory for this skill's delivery. Use fresh evidence directories for every run. Pin the open, same-repository PR's exact head and base SHAs, classify changes from immutable Git objects, and recheck the PR at phase and publication boundaries. Unknown paths abort; previous evidence never transfers to a new head. - -## Doctor +Replace both placeholders with explicitly selected caam 0.1.22 vault identities; never choose defaults or start a login flow. A selected identity may be active in the operator's daily Claude or Codex use. Omit `--self-test` for ordinary plugin verification; it is mandatory for this skill's delivery. -The trusted-parent probe requires Darwin, Bun, gh authentication for publication, both harness CLIs, caam 0.1.22, and supported macOS `sandbox-exec` isolation. It records `doctor.json`; capability checks do not prove installation, authentication, sandbox enforcement, or live behavior. Missing capabilities fail closed. +The trusted `main` publisher revision is recorded separately from the PR's candidate SHA. PR #111 bootstrap must use the maintainer-reviewed trusted publisher head, while all candidate installation, classification, evidence, comments, and `live-gate` status remain bound to PR #111's exact candidate SHA. Use a fresh private output directory for every run. The verifier requires an open same-repository PR, pins exact head and base SHAs, classifies immutable Git objects, and rechecks them at phase and publication boundaries. Unknown paths abort; evidence never transfers to another head. -Claude uses candidate `--plugin-dir`, empty `--settings`, `--setting-sources project`, and run-owned `CLAUDE_CONFIG_DIR`. Codex uses local marketplace/plugin installation and run-owned `CODEX_HOME`. Both sandboxed candidates use run-owned `HOME` and `TMPDIR`, retaining real `USER`/`LOGNAME` only for account identity. Both provider config variables must be isolated for cross-provider children. Do not link the real `Library` or Keychain into candidate state. Deny daily-home, GitHub/SSH/provider configuration, vault, and Keychain/securityd access with an OS-enforced boundary, not environment scrubbing alone. Never create, reset, unlock, or otherwise modify Keychain. - -The disposable credential contract replaces the earlier real-HOME/Keychain approach. The trusted parent validates each explicitly named vault identity and copies only `claude//.credentials.json` into `CLAUDE_CONFIG_DIR` and `codex//auth.json` into `CODEX_HOME`. It records the selected emails in the receipt, never modifies or activates the vault, and removes copied credential files in final cleanup. Candidates may use these disposable session credentials, but not publisher credentials or daily provider API keys. Missing credentials, identity mismatch, or quota failure stops the run; never log in or fall back to daily state. - -Operator-Mac handoff, from the trusted verifier checkout: +## Doctor -```sh -PROOF="$(mktemp -d "${TMPDIR:-/tmp}/open-pstack-isolation-proof.XXXXXX")" -( - set -e -o pipefail - test "$(uname -s)" = Darwin - cd .claude/skills/verify-open-pstack - PSTACK_OPERATOR_SENTINELS=1 bun test --timeout 0 scripts/isolation.test.ts 2>&1 | - tee "$PROOF/isolation-proof.txt" -) -``` +The trusted-parent doctor checks Bun, git, both native harness CLIs, and the capabilities needed by the run. For each explicitly selected identity it must run `caam limits --format json` (`` is `claude` or `codex`) and inspect the chosen `profile_name` for that provider. Selection is explicit regardless of whether the profile is active in daily use. An unauthorized or expired result fails closed and reports the exact manual repair command `caam add --no-activate --force`; the verifier must never execute that repair automatically. Candidate-mode doctor checks only candidate-safe capabilities and does not probe caam or publisher authentication. Capability checks do not prove installation, authentication, or live behavior; missing requirements fail closed. -Runtime `run` now executes this suite automatically in a separate trusted checkout of the pinned SHA, with `PSTACK_OPERATOR_SENTINELS=1`, **before copying disposable credentials**. It retains `mac-isolation-proof.txt` outside candidate state and requires both named Mac tests to report PASS with no skips/failures and a successful exit. The receipt binds its canonical path/hash and the exact SHA; missing or modified proof prevents success/readiness. The operator must review the result and explicitly confirm no Keychain dialog appeared. After **each** native session, review an observable authenticated API request/response in the retained transcript, describe the assertion, and confirm `PASS AUTH claude` or `PASS AUTH codex` plus no Keychain dialog. Model self-reports are not proof. The receipt retains these assertions and their reviewed transcript hashes, revalidated at publication boundaries. Fake unit-driver proof tests the gate only and does not establish OS enforcement. +Candidates retain the operator's real `HOME`, `USER`, and `LOGNAME`. `CLAUDE_CONFIG_DIR`, `CODEX_HOME`, `TMPDIR`, `GH_CONFIG_DIR`, and other XDG/config/cache locations are run-owned. Git configuration is disabled or run-owned. This separation is operational isolation, not an OS security boundary: the verifier makes no Seatbelt, source-protection, Keychain-denial, daily-home secrecy, or Mac filesystem-denial claim. -The explicit sentinel opt-in plants randomized, exclusive-create, non-secret files in the actual operator `HOME`, `~/.config/gh`, `~/.ssh`, `~/.claude`, `~/.codex`, and `~/Library/Keychains`; it never overwrites credential/SSH/Keychain files. Redirected directories fail closed. The trusted test parent removes only its own sentinel files and newly created empty directories in `finally`; if interrupted, inspect the printed cleanup-path list and remove only those named sentinel files, never whole daily directories. Require positive unsandboxed read controls and denied sandboxed reads/writes, aliases, and descendant reads; require positive unsandboxed Keychain-service lookup followed by denial of the same service. The real Mac test also proves pinned workspace/plugin write, unlink, rename, alias-write, and hardlink denial while the three approved dependency leaves remain writable. Preserve `$PROOF/isolation-proof.txt` and require both Mac tests to pass without skips. Linux skips and stubbed commands are not Mac enforcement proof. Then run Launch with fresh probe/live directories. Record that no Keychain dialog appeared and both native sessions made observable authenticated API requests; neither observation can be inferred from sentinel tests. Missing/pending adapter integration or failed proof blocks live verification. +The trusted parent validates each explicitly named caam identity. It copies only `claude//.credentials.json` into run-owned `CLAUDE_CONFIG_DIR` and `codex//auth.json` into run-owned `CODEX_HOME`. The disposable copies keep their refresh material so the native clients can refresh during the session; they are never written back to the vault. Missing files, unknown credential formats, identity mismatch, unusable authentication, or a `caam limits` `profile_name` that does not identify the selected provider profile fails closed. Never log in, repair credentials automatically, or fall back to daily provider configuration. ## Drive -Read `features/README.md` and every selected feature document. Runtime instructions, consumed references, and installed assets require live coverage. Shared paths select consumers conservatively. `no runtime change` launches no harness unless the separate self-test is requested. +Read `features/README.md` and every selected feature document. Runtime instructions, consumed references, shipped tools, and installed assets require coverage on their actual consumers. Shared paths select consumers conservatively. `no runtime change` launches no harness unless `--self-test` is requested. + +Create detached exact-head candidate checkouts, run-owned provider/config/temp roots, and fixture workspaces. Claude loads the explicit candidate plugin with candidate settings; Codex installs the pinned local marketplace/plugin. Verify installed plugin and project-skill sources against immutable candidate Git provenance before and after exercise. Version strings alone are insufficient. -Create detached exact-head candidate checkouts, run-owned homes/config roots and fixture workspaces. Claude loads the explicit candidate plugin; Codex installs its local marketplace. Verify installed plugin and canonical project-skill sources against immutable pinned Git provenance before and after exercise. Before native exercise, adapters must call `protectSources(profile, home, [workspace, installedPlugin])` and switch all candidate/descendant commands to the returned exercise policy outside candidate-writable state. That policy freezes the workspace and installed plugin, including ancestor rename/unlink, except `.claude/skills/verify-open-pstack/node_modules` and `plugins/pstack/skills/poteto-mode/scripts/node_modules` under the workspace and `skills/poteto-mode/scripts/node_modules` under the installed plugin. Reject source/ancestor symlinks rather than protecting a redirected path. Hash rechecks alone cannot prevent modify-execute-restore. Only those generated dependencies may be excluded; tracked source mutations or redirected skill aliases fail. Version strings alone are insufficient. +Every candidate command and external-provider descendant uses the recorded candidate environment. Candidate processes receive disposable provider credentials only, never GitHub publisher credentials. No candidate login, vault activation/writeback, implicit timeout, or weaker-model fallback is permitted. -Every candidate command and external-provider descendant must remain inside the macOS sandbox, with both isolated provider configurations and no access to trusted-parent credentials. Setup exercises write only fixture run-owned homes, never literal daily user files. No candidate login, vault activation/writeback, implicit timeout, or weaker-model fallback is permitted. +For every selected feature, exercise changed sub-features on each actual consuming native surface and retain real native tool calls and concrete fixture effects. Asset consumers come from the pinned plugin manifests; the current `plugins/pstack/assets/logo.png` is consumed by the Codex manifest only and therefore requires the Codex installed surface, not an invented Claude asset exercise. Save raw transcripts and artifacts in the private run root. Require operator-reviewed surface, action, expected/observed result, and retained artifact paths. Model self-reports and direct CLI tests are not installed-harness evidence. -For every selected feature in both fresh native sessions, exercise changed sub-features and retain actual native tool calls and concrete fixture effects. Save redacted transcripts and artifacts outside isolated state. Require operator-reviewed surface, action, expected/observed result, and retained artifact paths. Model self-reports and direct CLI tests are not installed-harness evidence. +Setup is currently fail-closed. Until setup-pstack config-home support in issue #120 merges, record exactly `setup exercise requires #120 (setup-pstack config-home)` and do not invoke setup against real `HOME`. -For `project-skill`, invoke `/verify-open-pstack` in Claude and `$verify-open-pstack` in Codex. Ask the discovered pinned skill to run `verify.sh doctor --candidate --output `. Candidate-mode doctor must not probe the vault or publisher authentication. Preserve the invocation and canonical-path doctor output; forbid recursive verification or publication. This does not replace required plugin exercises. +For `project-skill`, invoke `/verify-open-pstack` in Claude and `$verify-open-pstack` in Codex. Ask the discovered pinned skill to run `verify.sh doctor --candidate --output `. Preserve the invocation and canonical-path doctor output; forbid recursive verification or publication. This does not replace required plugin exercises. ## Evidence -Retain probe output, `receipt.json`, selected account emails, immutable source/installation provenance, redacted transcripts, artifacts, sandbox-denial proof, observable native API requests, and credential-cleanup outcome. Never retain copied credentials as evidence. Revalidate retained transcript/artifact hashes immediately before publication and around GitHub writes. Keep full detail in the receipt and bound the published summary. +Create the requested run root as a private mode-0700 directory. Keep raw receipts, transcripts, artifacts, selected account names, immutable source/installation provenance, and credential-cleanup outcome private in that root. Raw evidence may contain operational details and is not a public artifact. Keep it through merge; after the merged evidence is archived, only the operator deletes the named run root. Never retain copied credential files as evidence. -The trusted parent publishes evidence only in its run's own pinned-SHA comment. It never writes the PR body; `receipt.json` retains `proposedTemplate`, including the proposed `Live evidence:` block with installed version, surface, action, and observed result, for the operator to apply without overwriting unrelated edits. Only complete evidence permits `live-gate=success` targeting that comment. After publication, the helper makes at most one forward, idempotent mark-ready call and makes no call when the PR is already ready. It never reverses, undoes, or compensates draft/ready state. Failure or head movement publishes `live-gate=failure` on the pinned SHA and leaves the PR's draft state untouched; record failed status compensation prominently. +Before publication, revalidate retained transcript and artifact hashes. The trusted parent publishes one bounded, structured PR comment for the exact candidate SHA and sets `live-gate` on that same SHA with the comment URL as its target. It never edits the PR body, marks ready or draft, reverses a transition, or performs PR-state compensation. No other PR write is permitted. Head movement requires a fresh run and status on the new exact SHA. -PR #111 is already ready and must not be re-drafted during this review round. Builders do not post the gate, queue, or merge. After Unfret passes, the operator performs fresh exact-head live proof. Mergify queues automatically once required checks pass and the PR is ready; no manual queue submission is required. A queue-created head needs a new run, never a hand-posted replacement status. +The public-comment guard compares the rendered comment against the exact credential values registered from the copied known credential fields. It does not use token-shape heuristics and does not claim that arbitrary private raw evidence is secret-free. Unknown credential formats fail before publication. Failure details remain private; the public failure comment is bounded and the exact candidate SHA receives failure status only after disposable credential cleanup succeeds. ## Cleanup -Final cleanup removes copied Claude `.credentials.json` and Codex `auth.json` files on success or failure, including disposable copies created for provider children. Record cleanup outcome without secrets; failed cleanup is a blocker requiring explicit remediation. Never modify the caam vault or daily files. Retain receipts, redacted transcripts, artifacts, and denial proof. Quit native sessions normally; after archiving evidence the operator may remove only named run-owned candidate/state/workspace paths. No unrelated process killing or implicit runtime timeout. +Quit native sessions normally and remove only the copied disposable Claude `.credentials.json` and Codex `auth.json` files on success or failure, including refreshes written to those same run-owned paths. Preserve candidate state and raw evidence. Record cleanup outcome; failed credential cleanup blocks publication. Never modify the caam vault or daily provider files. Preserve the private mode-0700 raw evidence root through merge, archive it after merge, and let the operator delete only the named run root. Do not kill unrelated processes. ## Helpers - `scripts/verify.sh doctor --output `: trusted-parent capability report, no installation/publication. -- `scripts/verify.sh doctor --candidate --output `: planned sandboxed self-test probe; no vault or publisher authentication probing. +- `scripts/verify.sh doctor --candidate --output `: candidate-safe self-test probe; no vault or publisher authentication probing. - `scripts/verify.sh run --pr --claude-account --codex-account [--self-test] --output `: supervised exact-head verification; mapped exercises require an interactive terminal. -- `bun test scripts/isolation.test.ts` on the operator Mac: required real-filesystem and Keychain/securityd denial proof, not a substitute for live behavior. -- `bun run test` and `bun run typecheck`: independent helper tests and strict types, not live proof. +- `bun run test` and `bun run typecheck`: development checks after a separate frozen dependency install, not live proof. - `features/registry.json`: maintained path ownership; unknown runtime paths block. -Read receipts as data, never shell input. Do not put secrets in prompts or artifacts. +Read receipts as data, never shell input. Do not put credentials in prompts or public artifacts. + +## Scripts +- scripts/cli.test.ts +- scripts/cli.ts +- scripts/core.test.ts +- scripts/core.ts +- scripts/doctor.ts +- scripts/github.ts +- scripts/harness.test.ts +- scripts/harness.ts +- scripts/io.test.ts +- scripts/io.ts +- scripts/isolation.test.ts +- scripts/isolation.ts +- scripts/provenance.test.ts +- scripts/provenance.ts +- scripts/types.ts +- scripts/verify.sh +- scripts/verify.test.ts +- scripts/verify.ts diff --git a/.claude/skills/verify-open-pstack/features/README.md b/.claude/skills/verify-open-pstack/features/README.md index 7387b749..cc5a7a9f 100644 --- a/.claude/skills/verify-open-pstack/features/README.md +++ b/.claude/skills/verify-open-pstack/features/README.md @@ -1,28 +1,32 @@ # Maintained feature map -`registry.json` is the machine-readable ownership map. `core.ts` supports exact paths and terminal `/**` only. Unknown paths fail closed; rename classification includes both paths. Runtime markdown, references, playbooks, hooks, manifests, and scripts are runtime, even when the extension is `.md`. +`registry.json` is the machine-readable ownership map. `core.ts` supports exact paths and terminal `/**` only. Unknown paths fail closed; rename classification includes both paths. Runtime markdown, references, playbooks, hooks, manifests, and scripts are runtime even when their extension looks documentary. | Ownership | Required real surfaces | | --- | --- | | `skills` | One `skill-invocation:` exercise per selected skill, in both harnesses | -| `shared` | All listed skills, setup, runner, shipped tools in both harnesses | -| `setup` | Installed setup/model configuration | +| `shared` | All listed skills, setup, runner, and shipped tools in both harnesses | +| `assets` | Installed UI/resource surface for each actual consumer found in the pinned plugin manifests | +| `setup` | Installed setup/model configuration; currently the named #120 fail-closed result | | `runner` | Installed parent invoking real configured provider lanes | | `tools` | Installed parent invoking the changed shipped CLI | | `project` | Native discovery/invocation of this verifier in both harnesses | -| `nonRuntime` | Explicit ordinary repository documentation/non-shipped assets/unit-test paths; no plugin harness launch | +| `nonRuntime` | Explicit ordinary repository documentation, policy, non-shipped assets, and unit-test paths; no plugin harness launch | -Shipped `plugins/pstack/assets/**` belongs to `shared`: even a logo-only change requires both installed harness surfaces, not the no-runtime lane. `bootstrap.ts` belongs to `tools`; exercise its actual consumers, `orch` and `watch-pr`, including dependency bootstrapping rather than the unrelated external runner. +Asset ownership is consumer-driven, not blanket shared ownership. Inspect the exact candidate's plugin manifests. At present, `plugins/pstack/assets/logo.png` is referenced only by `plugins/pstack/.codex-plugin/plugin.json`, so a logo-only change selects the installed Codex asset surface and does not invent a Claude exercise. New shipped assets must be added to `assets` and must have a real manifest consumer; missing, ambiguous, or unknown consumption fails closed. -Skill-owned `SKILL.md`, `references/**`, and `playbooks/**` select their skill. Additional executable paths must be registered explicitly. Shared consumed mappings select every dependent skill conservatively. The coverage test inventories all tracked plugin files; new unmapped files block CI and runs. Deleted skill surfaces cannot silently disappear from evidence: choose a surviving documented entry point demonstrating the intended removal, or report blocked. +`bootstrap.ts` belongs to `tools`; exercise its actual consumers, `orch` and `watch-pr`, including dependency bootstrapping rather than the unrelated external runner. -The four documents below define sub-features, user entry points, driving instructions, and gotchas. Review each changed sub-feature in the diff; one aggregate observation cannot replace distinct selected skills. Record all changed sub-feature actions in the feature's transcript and artifact. Do not run destructive application work in the maintainer checkout. +Skill-owned `SKILL.md`, `references/**`, and `playbooks/**` select their skill. Additional executable paths must be registered explicitly. Shared consumed mappings select every dependent skill conservatively. The coverage test inventories tracked plugin files; new unmapped files block CI and runs. Deleted skill surfaces cannot silently disappear from evidence: choose a surviving documented entry point demonstrating the intended removal, or report blocked. + +The feature documents below define sub-features, user entry points, driving instructions, and gotchas. Review each changed sub-feature in the diff; one aggregate observation cannot replace distinct selected skills. Record all changed sub-feature actions in the feature's transcript and artifact. Do not run destructive application work in the maintainer checkout. - `skill-invocation.md`: installed native skills, principle leaves, and project-skill self-test. -- `setup.md`: setup/routing/model configuration. +- `setup.md`: setup/routing/model configuration and the #120 block. - `runner.md`: parent-to-child provider dispatch and receipts. - `shipped-tools.md`: orchestration, watch, plan checks, audit, and evidence logging. +- `assets.md`: manifest-declared installed assets on actual consuming harnesses. -Consumed `AGENTS.md`/`CLAUDE.md` instructions and `tests/skill-collision-repro.sh` select `project-skill`. There are no blanket no-runtime exemptions for `.github/**`, `tests/**`, or `scripts/**`: unregistered instruction/enforcement paths fail closed until their runtime ownership is reviewed and mapped. +Consumed `AGENTS.md`/`CLAUDE.md` instructions and `tests/skill-collision-repro.sh` select `project-skill`. `.mergify.yml` is explicitly `nonRuntime`. There are no blanket no-runtime exemptions for `.github/**`, `tests/**`, or `scripts/**`: unregistered instruction/enforcement paths fail closed until their ownership is reviewed and mapped. Explicit non-runtime repository changes are `no runtime change`. Changes to this verifier always select `project-skill`; they cannot bypass native proof. `--self-test` also selects that exercise, deduplicated in each harness, and is mandatory for this skill's delivery PR. Plugin-runtime proof and local CLI proof remain separate. diff --git a/.claude/skills/verify-open-pstack/features/assets.md b/.claude/skills/verify-open-pstack/features/assets.md new file mode 100644 index 00000000..d4da52ae --- /dev/null +++ b/.claude/skills/verify-open-pstack/features/assets.md @@ -0,0 +1,19 @@ +# Installed assets + +## Sub-features + +Manifest declaration, packaged presence, installed resolution, and the user-visible resource on every actual consuming harness. Determine consumers from the immutable candidate plugin manifests; do not assume all harnesses consume every asset. + +## How to get to it (user POV) + +Install the candidate plugin through each manifest that references the changed asset, then open or inspect the native installed surface that renders or exposes that resource. The current `plugins/pstack/assets/logo.png` is referenced by `plugins/pstack/.codex-plugin/plugin.json` at `interface.logo`; no Claude manifest references it. + +## Driving it with verify-open-pstack + +Inspect the pinned manifests and retain the exact asset-reference evidence. Confirm the referenced path resolves inside the pinned installed plugin, the installed bytes match candidate Git provenance, and the consuming native surface loads the resource. Retain a transcript plus a concrete UI capture or native inspection artifact. Record each actual consumer and why non-consumers were not selected. + +A logo-only change currently requires Codex installed-surface evidence only. If a manifest adds another consumer, exercise that consumer too. If a changed asset has no manifest reference, escapes the plugin, has ambiguous ownership, or cannot be observed on its declared native surface, fail closed rather than substituting unrelated skill, setup, runner, or tool exercises. + +## Gotchas + +Repository file existence alone is not installed behavior. Do not select every skill merely because an asset lives under `plugins/pstack`. Do not claim Claude coverage for the current logo when its manifest does not consume it. Direct image inspection may supplement, but cannot replace, the actual installed consumer observation. diff --git a/.claude/skills/verify-open-pstack/features/isolation.md b/.claude/skills/verify-open-pstack/features/isolation.md index fcdc7b45..c1aa3c3a 100644 --- a/.claude/skills/verify-open-pstack/features/isolation.md +++ b/.claude/skills/verify-open-pstack/features/isolation.md @@ -1,42 +1,42 @@ -# Disposable credential isolation contract +# Disposable credential and run-state contract -The October 2 probe (issue #90 comment 5961645184) reported Darwin, Claude Code 2.1.283, codex-cli 0.160.0, Bun 1.4.0, and gh 2.97.0. This is capability evidence, not candidate isolation proof. The October 3 operator-selected caam 0.1.22 contract (issue #112 comments 5968857062 and 5969996999) supersedes the real-HOME/Keychain login approach. +The verifier uses run-owned provider/configuration state and explicitly selected caam credentials copied for the run. This is operational state separation, not a Seatbelt or macOS filesystem security boundary. ## Trusted parent -Require explicit `--claude-account EMAIL` and `--codex-account EMAIL`. Validate selected vault identities once per run; record their emails in receipts. Copy only `claude//.credentials.json` into run-owned `CLAUDE_CONFIG_DIR` and `codex//auth.json` into run-owned `CODEX_HOME`. Never activate accounts, modify/write back to the vault, copy daily configuration, or introduce login. Missing/mismatched credentials and quota exhaustion fail closed. GitHub authentication and publication remain exclusively in the trusted parent. +Run the publisher from a maintainer-reviewed trusted `main` checkout. Record that publisher revision separately from the PR's exact candidate SHA; never treat publisher code provenance as candidate proof. For PR #111 bootstrap, use the reviewed trusted publisher head while pinning all candidate evidence and status to PR #111's reviewed exact head. -## Candidate boundary +Require explicit `--claude-account EMAIL` and `--codex-account EMAIL`; never infer or choose a default identity. A selected profile may be active in the operator's daily Claude or Codex use. Record selected account emails privately in the receipt. -- Use run-owned `HOME`, `TMPDIR`, `CLAUDE_CONFIG_DIR`, and `CODEX_HOME` for both parent harnesses and cross-provider descendants. Preserve real `USER`/`LOGNAME`; scrub publisher tokens and daily API credentials. -- Claude: candidate `--plugin-dir`, empty `--settings`, and `--setting-sources project`. This loads pinned project skills/settings while excluding user settings. -- Codex: pinned local marketplace/plugin installation, then a fresh native session. Validate installed identity, enabled state, contained path, and pinned source provenance; unknown schemas fail closed. -- Enforce candidate filesystem and credential isolation with macOS `sandbox-exec`, inherited by descendants. Deny daily home, `~/.config/gh`, `~/.ssh`, real `~/.claude`, real `~/.codex`, caam vault, and Keychain/securityd access. Do not link any real Library/Keychain path into run-owned state. Environment scrubbing alone is insufficient. -- Never invoke candidate login, create/reset/unlock Keychain, or use daily auth as fallback. Only copied disposable credentials may authenticate native API requests. +For each selection, trusted-parent doctor must run `caam limits --format json`, using `claude` or `codex` as ``, and inspect the chosen `profile_name` for that provider. A mismatched profile fails closed. An unauthorized or expired result also fails closed and reports the exact manual repair command `caam add --no-activate --force`. Never execute the repair automatically, switch identities, or fall back automatically. -Verify tracked plugin/project-skill provenance against immutable pinned Git objects before and after exercise. Before launching a native session, call `protectSources(profile, home, sources, run?)` with canonical workspace and installed-plugin roots, then use its returned exercise policy for every candidate command. The policy is an exclusive-create mode-0600 file beside the original policy, outside candidate-writable state. Source roots and all ancestors must be canonical directories, not symlinks. Source/plugin writes and ancestor rename/unlink are denied except workspace `.claude/skills/verify-open-pstack/node_modules`, workspace `plugins/pstack/skills/poteto-mode/scripts/node_modules`, and installed-plugin `skills/poteto-mode/scripts/node_modules`. No broader write allowance is added; the original daily-home/vault/Keychain denies remain in force. Generated directories are established before freezing their parents. Rehashes alone cannot prevent modify-execute-restore, so pending adapter integration is a blocker. Candidate self-test uses `doctor --candidate` without vault/publisher probing. +Copy only `claude//.credentials.json` into run-owned `CLAUDE_CONFIG_DIR` and `codex//auth.json` into run-owned `CODEX_HOME`. Copy the complete recognized credential file, including refresh material needed by the native client. Never write refreshed state back to the caam vault. Missing/mismatched files, unknown credential formats, or unusable authentication fail closed. GitHub authentication and publication remain exclusively in the trusted parent. -## Operator-Mac proof +## Candidate process state -Runtime `run` automatically executes the pinned `scripts/isolation.test.ts` from a separate trusted exact-SHA checkout, with `PSTACK_OPERATOR_SENTINELS=1`, before copying disposable credentials. It retains `mac-isolation-proof.txt` outside candidate state, validates both named Mac tests passed without skips/failures and a successful exit, and requires explicit operator review of no Keychain dialog. The receipt binds SHA, canonical proof path/hash, and the review. Each native session then requires an operator-observed authenticated API request/response assertion bound to its reviewed transcript hash and `PASS AUTH ` confirming no Keychain dialog. Both harnesses are mandatory. Missing, mismatched, overwritten, skipped, or rejected proof blocks success/readiness at publication boundaries. Fake unit-driver proof exercises validation without claiming Mac enforcement. +- Preserve the operator's real `HOME`, `USER`, and `LOGNAME` for native-client compatibility and identity. +- Set `CLAUDE_CONFIG_DIR`, `CODEX_HOME`, `TMPDIR`, `GH_CONFIG_DIR`, XDG config/cache, and other configurable provider/GitHub/temp locations to run-owned directories. +- Disable global/system git configuration and credential helpers for candidate commands. +- Claude uses the explicit candidate plugin, empty settings, and project setting sources. Codex uses pinned local marketplace/plugin installation. +- Provider descendants inherit both run-owned provider config roots. Never pass GitHub publisher credentials or daily provider API keys. +- Never invoke login, activate or modify the vault, or copy daily provider configuration as fallback. -For a separate diagnostic check, from the trusted verifier checkout run: +These controls do not deny a process access to real `HOME`, Keychain, or candidate sources. Do not claim Seatbelt enforcement, source write protection, Keychain/securityd denial, daily-home denial, or OS-backed secrecy. Provenance rechecks detect changed candidate/installed files but are not prevention. -```sh -PROOF="$(mktemp -d "${TMPDIR:-/tmp}/open-pstack-isolation-proof.XXXXXX")" -( - set -e -o pipefail - test "$(uname -s)" = Darwin - cd .claude/skills/verify-open-pstack - PSTACK_OPERATOR_SENTINELS=1 bun test --timeout 0 scripts/isolation.test.ts 2>&1 | - tee "$PROOF/isolation-proof.txt" -) -``` +## Setup boundary -The actual-HOME test requires explicit opt-in. It exclusively creates randomized `.pstack-denial-` non-secret files in the real daily home, `~/.config/gh`, `~/.ssh`, `~/.claude`, `~/.codex`, and `~/Library/Keychains`; no existing daily file or actual Keychain database is opened for writing. Noncanonical/symlinked daily directories fail before sentinel creation. It removes only its own files and directories it created that remain empty; cleanup paths are printed for safe recovery after interruption. Never recursively remove daily directories. +Because `HOME` remains real, setup must not be exercised until config-home support in issue #120 merges. Until then, setup selection fails closed with exactly: -Require both Mac tests to pass without skips. Positive unsandboxed reads prove sentinels exist; sandboxed direct reads, aliases, descendants, and writes must be denied and sentinel content preserved. Keychain Mach-service denial requires a reachable unsandboxed control for the same service; a missing service is not proof. Exercise-policy tests require protected source writes/unlinks/renames, ancestor moves, alias writes, hardlinks, and policy replacement to fail while all three generated dependency leaves remain writable. Record the emitted denial proof in `$PROOF/isolation-proof.txt` alongside the exact-head receipt. Linux skips or stubbed wrappers cannot establish Seatbelt enforcement. No Keychain dialog and authenticated API requests in both harnesses still require operator observation during native live sessions, not inference from these tests. Missing adapter integration or failed tests block delivery. After Unfret passes, use explicit-account Launch commands with fresh probe/live directories; leave PR #111 ready. +`setup exercise requires #120 (setup-pstack config-home)` -## Cleanup +Do not redirect `HOME`, allow setup to write daily files, or replace this result with a mock. -Delete all copied session credential files in final cleanup on success or failure, including child-provider copies; record the outcome and block on cleanup failure. Keep redacted receipts/transcripts/artifacts and sentinel-denial proof. Never modify vault/daily files or delete evidence as credential cleanup. Subsequent operator removal is limited to named run-owned state. +## Public comment check + +Register exact access/refresh/id-token values only from the copied, recognized credential fields. Immediately before a GitHub write, compare the rendered structured comment against those exact known values and refuse publication on a match. Do not use token-shape, prefix, entropy, regex, or other heuristic secrecy claims. Unknown credential layouts block before publication. Private raw evidence is not declared secret-free. + +## Private evidence and cleanup + +Create each output root fresh, outside the repository, with mode 0700; use private files beneath it. Retain raw transcripts, artifacts, receipts, and cleanup results in that private root through merge. Copied credential files are session state, not evidence: remove only the copied Claude `.credentials.json` and Codex `auth.json` files in run-owned config roots after native sessions on success or failure, including refreshes written to those same paths. Preserve candidate state and raw evidence. Cleanup failure blocks GitHub publication. + +After merge, archive the retained evidence and let the operator delete only the named run root. Never modify the caam vault or daily provider files, and never recursively delete an operator directory. diff --git a/.claude/skills/verify-open-pstack/features/registry.json b/.claude/skills/verify-open-pstack/features/registry.json index e6421bdc..ac211599 100644 --- a/.claude/skills/verify-open-pstack/features/registry.json +++ b/.claude/skills/verify-open-pstack/features/registry.json @@ -21,9 +21,10 @@ "shared": [ "plugins/pstack/.claude-plugin/plugin.json", "plugins/pstack/.codex-plugin/plugin.json", ".claude-plugin/marketplace.json", ".agents/plugins/marketplace.json", - "plugins/pstack/agents/**", "plugins/pstack/hooks/**", "plugins/pstack/assets/**", + "plugins/pstack/agents/**", "plugins/pstack/hooks/**", "plugins/pstack/skills/poteto-mode/references/**" ], + "assets": ["plugins/pstack/assets/**"], "setup": ["plugins/pstack/skills/setup-pstack/SKILL.md"], "runner": [ "plugins/pstack/skills/poteto-mode/scripts/runner/**" @@ -41,11 +42,11 @@ ], "project": [ ".claude/skills/verify-open-pstack/**", ".agents/skills/verify-open-pstack", - "AGENTS.md", "CLAUDE.md", "tests/skill-collision-repro.sh", ".mergify.yml" + "AGENTS.md", "CLAUDE.md", "tests/skill-collision-repro.sh" ], "nonRuntime": [ "README.md", "CHANGES.md", "LICENSE", "LICENSE.md", "UPSTREAM.md", - ".gitignore", "docs/**", "assets/**", + ".gitignore", ".mergify.yml", "docs/**", "assets/**", "plugins/pstack/skills/poteto-mode/scripts/bootstrap.test.ts", "plugins/pstack/skills/poteto-mode/scripts/check-plan.test.ts" ] diff --git a/.claude/skills/verify-open-pstack/features/runner.md b/.claude/skills/verify-open-pstack/features/runner.md index abb05929..071869b7 100644 --- a/.claude/skills/verify-open-pstack/features/runner.md +++ b/.claude/skills/verify-open-pstack/features/runner.md @@ -6,7 +6,7 @@ Strict argument validation, parent/provider routing, external child launch, prom ## How to get to it (user POV) -Invoke the installed parent workflow (`poteto-mode`, `arena`, or `swarm` as documented by the candidate) in the fresh isolated harness. Have that parent dispatch its documented external-provider lane. +Invoke the installed parent workflow (`poteto-mode`, `arena`, or `swarm` as documented by the candidate) in the fresh native harness. Have that parent dispatch its documented external-provider lane. ## Driving it with verify-open-pstack @@ -14,4 +14,6 @@ Read the candidate's `pstack-runner --help` and installed mapping through the na ## Gotchas -Every external-provider child inherits the OS-enforced macOS sandbox, run-owned `HOME`/`TMPDIR`, and both run-owned `CLAUDE_CONFIG_DIR` and `CODEX_HOME`; never allow Claude→Codex or Codex→Claude to fall back to daily configuration. The trusted parent copies only the explicitly selected caam account credentials, records both emails, and deletes copied credentials in final cleanup. No candidate login, vault activation/writeback, Library/Keychain link, or Keychain/securityd access is allowed. Retain real-filesystem sentinel-denial evidence and observable native API requests for the dispatched lanes. A missing provider or unusable authentication is a failure, not a skip or weaker-model fallback. Do not add implicit timeouts. A model assertion that a child ran is insufficient; inspect the actual receipt and output. Never reuse output from another SHA or run. +Every external-provider child retains real `HOME`, `USER`, and `LOGNAME` while inheriting run-owned `TMPDIR`, `GH_CONFIG_DIR`, XDG/config/cache paths, `CLAUDE_CONFIG_DIR`, and `CODEX_HOME`. This state separation is not an OS access-denial guarantee. Never let Claude→Codex or Codex→Claude fall back to daily provider configuration, and never pass publisher credentials. + +The trusted parent copies complete recognized credential files for explicitly selected caam accounts so native refresh can work, and removes only the copied credential files (including refreshes written to those paths) during cleanup without writing back to the vault or deleting raw evidence. A selected account may be active in daily use. Trusted-parent doctor runs `caam limits --format json` and inspects the chosen `profile_name` for each provider; a mismatch, missing provider, or unusable authentication is a failure, not a skip or weaker-model fallback. Unauthorized or expired results fail closed with the exact manual repair command `caam add --no-activate --force`; never execute it automatically. Do not add implicit timeouts. A model assertion that a child ran is insufficient; inspect the actual receipt and output. Never reuse output from another SHA or run. diff --git a/.claude/skills/verify-open-pstack/features/setup.md b/.claude/skills/verify-open-pstack/features/setup.md index 514dafff..d11132e4 100644 --- a/.claude/skills/verify-open-pstack/features/setup.md +++ b/.claude/skills/verify-open-pstack/features/setup.md @@ -6,12 +6,18 @@ Installed setup invocation, harness-to-provider mapping, generated routing/model ## How to get to it (user POV) -Invoke `/pstack:setup-pstack` in the isolated Claude session, or `$setup-pstack` in the isolated Codex session. The installed instructions are the authority for the setup inputs. +The intended installed entries are `/pstack:setup-pstack` in Claude and `$setup-pstack` in Codex. The installed instructions are the authority for setup inputs, but the verifier must not invoke them against the operator's real `HOME` before config-home support exists. ## Driving it with verify-open-pstack -Use only the sandboxed run-owned fixture workspace and process `HOME`; both `CLAUDE_CONFIG_DIR` and `CODEX_HOME` must point into run-owned state. Literal `~/.claude` and `~/.codex` setup writes must resolve inside that fixture home, never the operator's daily home. Exercise each changed setup branch with explicit requested model/provider inputs. Retain the native invocation transcript and generated configuration file (redacted), then invoke a consuming installed skill and observe it use the configured route. Check that all paths written remain in run-owned state. Record requested versus observed provider/model and changed failure branches. Capture separate local CLI output only as supplemental evidence. +Setup is mandatory fail-closed until setup-pstack config-home support in issue #120 merges. For every selected setup exercise, record exactly: + +`setup exercise requires #120 (setup-pstack config-home)` + +Stop that verification run. Do not invoke setup, redirect `HOME`, permit writes to daily paths, use a mock, or accept a generated file from a direct CLI test as substitute evidence. + +After #120 merges, use its explicit config-home to target run-owned state while retaining real `HOME`, `USER`, and `LOGNAME`. Exercise each changed setup branch with explicit requested model/provider inputs, retain the native invocation transcript and generated configuration file, and invoke a consuming installed skill to observe the configured route. Confirm all setup writes remain under the explicit run-owned config home. ## Gotchas -Do not copy a daily configuration to make setup pass. Stop if required provider authentication is unavailable. Generated configuration by itself does not prove the consuming surface. The trusted parent requires explicit caam 0.1.22 account emails, validates vault identities, and copies only disposable Claude/Codex credential files into run-owned config roots. No candidate login or vault activation/writeback is allowed. Both sessions and provider children remain sandboxed against daily home/GitHub/SSH/provider files and Keychain/securityd, with no real Library/Keychain link. Retain sentinel-denial proof and observable native API requests; delete copied credentials in final cleanup while preserving redacted evidence. +`CLAUDE_CONFIG_DIR`, `CODEX_HOME`, XDG/GitHub config, and temp paths are run-owned, but `HOME` is intentionally real. That is why setup cannot run safely before #120. Do not copy daily configuration, activate accounts, log in, or write back to caam. Verification accounts remain explicitly selected, but may be active in daily use. Trusted-parent doctor runs `caam limits --format json` and inspects the selected provider's `profile_name`; unauthorized or expired results fail closed with the exact manual repair command `caam add --no-activate --force`, which the verifier never executes automatically. Generated configuration by itself does not prove the consuming surface. diff --git a/.claude/skills/verify-open-pstack/features/shipped-tools.md b/.claude/skills/verify-open-pstack/features/shipped-tools.md index 41f00f83..36d51042 100644 --- a/.claude/skills/verify-open-pstack/features/shipped-tools.md +++ b/.claude/skills/verify-open-pstack/features/shipped-tools.md @@ -10,8 +10,10 @@ Invoke the installed `poteto-mode` parent for orchestration/watch/plan/audit, or ## Driving it with verify-open-pstack -Use the candidate's installed help and documented arguments. Create fixture state/output under the run root; use an explicitly named safe test PR where remote reads are required. Have the native installed workflow drive the changed tool, then inspect its real structured output and persisted state. For the audit assert no mutation; for logging check the written TSV and escaping; for orchestration check the stored transition; for watch/plan preserve the observed result and verdict. Retain native transcripts and these state artifacts. Label any direct helper invocation `local CLI` separately. Document every touched tool and changed branch in the operator-reviewed observation. +Use the candidate's installed help and documented arguments. Create fixture state/output under the private run root; use an explicitly named safe test PR where remote reads are required. Have the native installed workflow drive the changed tool, then inspect its real structured output and persisted state. For the audit assert no mutation; for logging check the written TSV and escaping; for orchestration check the stored transition; for watch/plan preserve the observed result and verdict. Retain native transcripts and these state artifacts. Label any direct helper invocation `local CLI` separately. Document every touched tool and changed branch in the operator-reviewed observation. + +`bootstrap.ts` changes require its real shipped-tool consumers, `orch` and `watch-pr`, including their clean dependency bootstrap path. They do not select the unrelated external runner. ## Gotchas -A mock test is not live proof. Never run queue/merge actions or edit daily workflow state as a fixture. Do not infer a successful remote read from an empty response. Candidate tools and descendants stay inside the macOS sandbox with run-owned `HOME`, both isolated provider configurations, and disposable explicitly selected caam session credentials. They cannot read daily home/GitHub/SSH/provider files or access Keychain/securityd; never pass publisher credentials to make a remote read succeed. GitHub evidence/status publication belongs only to the trusted parent. Missing required access fails the feature; preserve its reason without re-drafting an already-ready PR, including PR #111. +A mock test is not live proof. Never perform any PR write from a candidate process or edit daily workflow state as a fixture. Do not infer a successful remote read from an empty response. Candidate tools and descendants retain real `HOME`, `USER`, and `LOGNAME` but use run-owned provider/GitHub/XDG/config/cache/temp paths and explicitly selected caam credentials; the selected profiles may be active in daily use. They do not receive publisher credentials. Trusted-parent doctor runs `caam limits --format json` and inspects the chosen `profile_name` for each provider. Unauthorized or expired results fail closed with the exact manual repair command `caam add --no-activate --force`; never execute repair automatically. This does not claim OS denial of daily-home or Keychain access. Missing required access fails the feature; preserve its private reason and publish only the bounded structured exact-SHA result. diff --git a/.claude/skills/verify-open-pstack/features/skill-invocation.md b/.claude/skills/verify-open-pstack/features/skill-invocation.md index 272b944c..6d9b3ab0 100644 --- a/.claude/skills/verify-open-pstack/features/skill-invocation.md +++ b/.claude/skills/verify-open-pstack/features/skill-invocation.md @@ -6,14 +6,16 @@ Every registry skill is its own required feature. Include changed arguments, rou ## How to get to it (user POV) -In the fresh isolated Claude session invoke `/pstack:`. In Codex use the installed skill selector or `$`. Invoke principle leaves through the model's native skill tool. For self-test invoke `/verify-open-pstack` in Claude and `$verify-open-pstack` in Codex. +In a fresh Claude session invoke `/pstack:`. In Codex use the installed skill selector or `$`. Invoke principle leaves through the model's native skill tool. For self-test invoke `/verify-open-pstack` in Claude and `$verify-open-pstack` in Codex. ## Driving it with verify-open-pstack -Launch the prepared harness with its sandboxed recorded launcher: run-owned `HOME`/`TMPDIR`, both isolated provider config roots, and real `USER`/`LOGNAME`. Use only trusted-parent copies of the explicitly selected caam account credentials; no login, daily-state fallback, vault mutation, or Library/Keychain link. Deny daily-home/GitHub/SSH/provider and Keychain/securityd access for descendants too, and retain operator-Mac sentinel-denial proof plus observable native API requests. Final cleanup deletes copied credentials, not evidence. For each selected name, use a disposable fixture appropriate to its documented input, invoke the installed surface, and exercise every changed sub-feature. Observe native tool/skill loading and a concrete fixture effect, not the assistant saying it passed. Save the transcript and an artifact showing the actual result (diff, generated file, tool receipt, or UI capture). Include input, expected result, observed result, and each changed sub-feature in the operator review. +Launch the prepared harness with real `HOME`, `USER`, and `LOGNAME`; run-owned `TMPDIR`, GitHub/XDG/config/cache paths, `CLAUDE_CONFIG_DIR`, and `CODEX_HOME`; and no publisher credentials. Use trusted-parent copies of complete recognized credentials for explicitly selected caam accounts so native refresh can work; selected profiles may be active in daily use. Trusted-parent doctor must run `caam limits --format json` and inspect the chosen `profile_name` for each provider. Unauthorized or expired results fail closed with the exact manual repair command `caam add --no-activate --force`; the verifier never executes repair automatically. No login, daily-provider fallback, or vault writeback is allowed. -For `project-skill`, invoke the discovered skill and ask it to run planned `verify.sh doctor --candidate --output `; candidate mode must not probe the caam vault or publisher authentication. Forbid recursive `run`/publication. Preserve the native invocation transcript plus `doctor.json`, checking the canonical executable path and immutable source provenance belong to the pinned project skill. Revalidate source after exercise, excluding only explicitly permitted generated dependencies. The Codex symlink and Claude canonical directory must both be discovered in fresh sessions. If candidate mode is unavailable, stop; do not substitute trusted-parent doctor. +For each selected name, use a disposable fixture appropriate to its documented input, invoke the installed surface, and exercise every changed sub-feature. Observe native tool/skill loading and a concrete fixture effect, not the assistant saying it passed. Save the raw transcript and an artifact showing the actual result (diff, generated file, tool receipt, or UI capture) under the private mode-0700 evidence root. Include input, expected result, observed result, and each changed sub-feature in the operator review. + +For `project-skill`, invoke the discovered skill and ask it to run `verify.sh doctor --candidate --output `; candidate mode must not probe caam or publisher authentication. Forbid recursive `run` and publication. Preserve the native invocation transcript plus `doctor.json`, checking the canonical executable path and immutable source provenance belong to the pinned project skill. The Codex symlink and Claude canonical directory must both be discovered in fresh sessions. If candidate mode is unavailable, stop; do not substitute trusted-parent doctor. ## Gotchas -A generic prompt about a skill is not native invocation evidence. A list of discovered skills is not behavioral proof. If an invocation would merge, queue, release, or access daily state, use a safe disposable fixture or record failure; do not weaken the gate. Never use the child verifier to publish a status. +A generic prompt about a skill is not native invocation evidence. A list of discovered skills is not behavioral proof. Environment/config separation does not establish Seatbelt, daily-home, Keychain, or source-write denial, and no such claim belongs in evidence. If an invocation would merge, queue, release, mutate a PR, or write daily state, use a safe disposable fixture or record failure; do not weaken the gate. Never use the child verifier to publish a comment or status. diff --git a/.claude/skills/verify-open-pstack/scripts/cli.test.ts b/.claude/skills/verify-open-pstack/scripts/cli.test.ts index b105f919..7f3b34a6 100644 --- a/.claude/skills/verify-open-pstack/scripts/cli.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/cli.test.ts @@ -1,120 +1,33 @@ import { expect, test } from 'bun:test'; -import { bindPublisher, parse, publisherCode } from './cli.ts'; -import { cp, mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; +import { parse } from './cli.ts'; +import { mkdtemp, readFile, realpath, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { doctor } from './doctor.ts'; -import { command, type Command } from './io.ts'; -import type { GitHub, Pull } from './types.ts'; +import type { Command } from './io.ts'; +import type { GitHub } from './types.ts'; import { REPO } from './types.ts'; import { verify } from './verify.ts'; import { validateRegistry } from './core.ts'; import registry from '../features/registry.json'; const accounts = ['--claude-account', 'eric@litman.org', '--codex-account', 'eric@healthspanners.com']; -async function publisherFixture() { - const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-publisher-binding-'))); - const project = join(root, '.claude/skills/verify-open-pstack'); - await mkdir(join(project, 'scripts'), { recursive: true }); - await mkdir(join(project, 'features')); await mkdir(join(root, 'plugins/pstack'), { recursive: true }); - await mkdir(join(root, '.agents/skills'), { recursive: true }); - await symlink('../../.claude/skills/verify-open-pstack', join(root, '.agents/skills/verify-open-pstack')); - await writeFile(join(root, 'plugins/pstack/plugin.json'), '{}\n'); - for (const path of ['scripts/cli.ts', 'scripts/github.ts', 'scripts/verify.sh', 'features/registry.json']) await writeFile(join(project, path), 'pinned publisher source\n'); - for (const args of [ - ['git', 'init', '--quiet'], ['git', 'config', 'user.name', 'Publisher Test'], - ['git', 'config', 'user.email', 'publisher@example.invalid'], ['git', 'add', '.'], - ['git', '-c', 'commit.gpgsign=false', 'commit', '--quiet', '-m', 'pinned publisher'], - ]) await command(args, { cwd: root }); - const sha = (await command(['git', 'rev-parse', 'HEAD'], { cwd: root })).trim(); - const calls: string[] = [], url = `https://github.com/${REPO}/pull/111#issuecomment-1`; - const pull: Pull = { number: 111, head: { sha }, base: { sha }, state: 'open', draft: false, headRepo: REPO, body: '' }; + +test('publisher revision is recorded independently of candidate proof', async () => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-publisher-revision-'))); + const candidate = 'a'.repeat(40), base = 'b'.repeat(40), publisherRevision = 'c'.repeat(40); const github: GitHub = { - async pull() { calls.push('pull'); return structuredClone(pull); }, - async files() { calls.push('files'); return [{ filename: 'README.md' }]; }, - async comment() { calls.push('comment'); return url; }, - async status(_sha, state) { calls.push(`status:${state}`); }, - async ready() { calls.push('ready'); }, + async pull() { return { number: 111, head: { sha: candidate }, base: { sha: base }, state: 'open', headRepo: REPO }; }, + async files() { return [{ filename: 'README.md' }]; }, + async comment() { return `https://github.com/${REPO}/pull/111#issuecomment-1`; }, + async status() {}, }; - return { root, project, sha, github, calls, pull, url, loadedCode: publisherCode(root) }; -} -test('already-loaded publisher A cannot bind checkout B after asynchronous startup', async () => { - const f = await publisherFixture(); try { - await cp(import.meta.dir, join(f.project, 'scripts'), { recursive: true }); - const cli = join(f.project, 'scripts/cli.ts'); - const source = await readFile(cli, 'utf8'); - await writeFile(cli, source + '\nexport const loadedRevision = "A";\n'); - await command(['git', 'add', '.'], { cwd: f.root }); - await command(['git', '-c', 'commit.gpgsign=false', 'commit', '--quiet', '-m', 'publisher A'], { cwd: f.root }); - const loaded = await import(cli) as typeof import('./cli.ts') & { loadedRevision: string }; - // Model freshRoot yielding after imports while another process advances the checkout. - await writeFile(cli, source + '\nexport const loadedRevision = "B";\n'); - await command(['git', 'add', '.'], { cwd: f.root }); - await command(['git', '-c', 'commit.gpgsign=false', 'commit', '--quiet', '-m', 'publisher B'], { cwd: f.root }); - f.pull.head.sha = f.pull.base.sha = (await command(['git', 'rev-parse', 'HEAD'], { cwd: f.root })).trim(); - expect(loaded.loadedRevision).toBe('A'); - await expect(loaded.bindPublisher(f.root, 111, f.github)).rejects.toThrow('Publisher source changed while binding'); - expect(f.calls).toEqual(['pull']); - const current = await import(cli + '?head=B') as typeof import('./cli.ts') & { loadedRevision: string }; - expect(current.loadedRevision).toBe('B'); - const bound = await current.bindPublisher(f.root, 111, f.github); - await bound.files(f.pull.base.sha, f.pull.head.sha); - expect(f.calls).toEqual(['pull', 'pull', 'files']); - } finally { await rm(f.root, { recursive: true, force: true }); } -}); -test('publisher pin rejects an older local checkout before classification or publication', async () => { - const f = await publisherFixture(); - try { - f.pull.head.sha = 'a'.repeat(40); - await expect(bindPublisher(f.root, 111, f.github, f.loadedCode)).rejects.toThrow('HEAD differs from pinned SHA'); - expect(f.calls).toEqual(['pull']); - } finally { await rm(f.root, { recursive: true, force: true }); } -}); -test('publisher pin rejects modified registry, wrapper and code, plus untracked project source', async () => { - const f = await publisherFixture(); - try { - for (const path of ['features/registry.json', 'scripts/verify.sh', 'scripts/cli.ts', 'scripts/github.ts']) { - await writeFile(join(f.project, path), 'unapproved publisher logic\n'); - await expect(bindPublisher(f.root, 111, f.github, f.loadedCode)).rejects.toThrow('Source differs from pinned Git tree'); - await writeFile(join(f.project, path), 'pinned publisher source\n'); - } - await writeFile(join(f.project, 'scripts/untracked.ts'), 'untracked logic\n'); - await expect(bindPublisher(f.root, 111, f.github, f.loadedCode)).rejects.toThrow('Source differs from pinned Git tree'); - expect(f.calls.every(call => call === 'pull')).toBe(true); - } finally { await rm(f.root, { recursive: true, force: true }); } -}); -test('publisher source rechecks protect every success boundary but retain failure compensation', async () => { - const f = await publisherFixture(); - try { - const bound = await bindPublisher(f.root, 111, f.github, f.loadedCode); - await writeFile(join(f.project, 'scripts/github.ts'), 'changed after binding\n'); - for (const operation of [ - () => bound.files(f.sha, f.sha), - () => bound.comment(111, 'evidence'), () => bound.status(f.sha, 'success', f.url, 'verified'), () => bound.ready(111), - ]) await expect(operation()).rejects.toThrow('Source differs from pinned Git tree'); - expect(f.calls).toEqual(['pull']); - await bound.status(f.sha, 'failure', f.url, 'withdraw success'); - expect(f.calls).toEqual(['pull', 'status:failure']); - expect('body' in bound).toBe(false); expect('draft' in bound).toBe(false); - } finally { await rm(f.root, { recursive: true, force: true }); } -}); -test('publisher pin also gates docs-only classification and success with no harness fallback', async () => { - const f = await publisherFixture(); - try { - const github = await bindPublisher(f.root, 111, f.github, f.loadedCode); - const output = join(f.root, 'output'); await mkdir(output); - const receipt = await verify({ pr: 111, selfTest: false, root: output, registry: validateRegistry(registry), github, + const receipt = await verify({ pr: 111, selfTest: false, root, publisherRevision, registry: validateRegistry(registry), github, driver: { async prepare() { throw new Error('Docs must not prepare harnesses'); }, async exercise() { throw new Error('Docs must not exercise harnesses'); } }, persist: async () => {} }); - expect(receipt.selection.noRuntime).toBe(true); expect(receipt.status).toBe('success'); expect(f.calls).toContain('status:success'); - await expect(github.files(f.sha, 'a'.repeat(40))).rejects.toThrow('classification SHAs'); - await expect(github.status('a'.repeat(40), 'success', f.url, 'wrong head')).rejects.toThrow('pinned SHA'); - await expect(github.comment(112, 'wrong PR')).rejects.toThrow('one PR'); - await writeFile(join(f.root, 'README.md'), 'new docs head\n'); - await command(['git', 'add', 'README.md'], { cwd: f.root }); - await command(['git', '-c', 'commit.gpgsign=false', 'commit', '--quiet', '-m', 'new docs head'], { cwd: f.root }); - await expect(github.files(f.sha, f.sha)).rejects.toThrow('HEAD differs from pinned SHA'); - await expect(github.status(f.sha, 'success', f.url, 'stale checkout')).rejects.toThrow('HEAD differs from pinned SHA'); - } finally { await rm(f.root, { recursive: true, force: true }); } + expect(receipt.sha).toBe(candidate); + expect(receipt.publisherRevision).toBe(publisherRevision); + expect(receipt.publisherRevision).not.toBe(receipt.sha); + } finally { await rm(root, { recursive: true, force: true }); } }); test('candidate mode is an explicit doctor-only flag', () => { expect(parse(['doctor', '--candidate', '--output', '/fresh/probe'])).toEqual({ mode: 'doctor', output: '/fresh/probe', pr: 0, selfTest: false, candidate: true }); @@ -126,7 +39,10 @@ test('candidate doctor probes only help and versions with private roots already const run: Command = async (args, options = {}) => { calls.push(args); const env = options.env!; - for (const key of ['HOME', 'TMPDIR', 'CLAUDE_CONFIG_DIR', 'CODEX_HOME', 'XDG_CONFIG_HOME', 'XDG_CACHE_HOME', 'GH_CONFIG_DIR']) { + expect(env.HOME === process.env.HOME).toBe(true); + expect(env.USER === process.env.USER).toBe(true); + expect(env.LOGNAME === process.env.LOGNAME).toBe(true); + for (const key of ['TMPDIR', 'CLAUDE_CONFIG_DIR', 'CODEX_HOME', 'XDG_CONFIG_HOME', 'XDG_CACHE_HOME', 'GH_CONFIG_DIR']) { expect(env[key]!.startsWith(root + '/probe-home')).toBe(true); const info = await stat(env[key]!); expect(info.isDirectory()).toBe(true); @@ -152,13 +68,15 @@ test('CLI strictly validates run inputs and permits read-only child doctor', () expect(parse(['run', '--pr', '123', '--self-test', '--output', '/fresh/run', ...accounts])).toEqual({ mode: 'run', output: '/fresh/run', pr: 123, selfTest: true, claudeAccount: 'eric@litman.org', codexAccount: 'eric@healthspanners.com' }); for (const args of [[], ['wat'], ['run', '--pr', '0', '--output', '/tmp/a', ...accounts], ['run', '--pr', '1.5', '--output', '/tmp/a', ...accounts], ['doctor', '--pr', '1', '--output', '/tmp/a'], ['doctor', '--output', '/tmp/a', '--output', '/tmp/b'], ['run', '--pr', '1', ...accounts], ['doctor', '--output', '--self-test'], ['doctor', '--output', '/tmp/a', '--publish']]) expect(() => parse(args)).toThrow(); }); -test('run requires both explicit account choices without defaults or doctor account options', () => { +test('run requires both explicit accounts and parent doctor accepts paired account options', () => { const run = ['run', '--pr', '111', '--output', '/fresh/run']; expect(() => parse(run)).toThrow('Run requires --claude-account and --codex-account'); expect(() => parse([...run, ...accounts.slice(0, 2)])).toThrow('Run requires'); expect(() => parse([...run, ...accounts.slice(2)])).toThrow('Run requires'); + expect(parse(['doctor', '--output', '/fresh/probe', ...accounts])).toEqual({ mode: 'doctor', output: '/fresh/probe', pr: 0, selfTest: false, candidate: false, claudeAccount: 'eric@litman.org', codexAccount: 'eric@healthspanners.com' }); + expect(() => parse(['doctor', '--candidate', '--output', '/fresh/probe', ...accounts])).toThrow('cannot inspect caam credentials'); for (const option of ['--claude-account', '--codex-account']) { - expect(() => parse(['doctor', '--output', '/fresh/probe', option, 'eric@litman.org'])).toThrow('Unknown option'); + expect(() => parse(['doctor', '--output', '/fresh/probe', option, 'eric@litman.org'])).toThrow('require both'); expect(() => parse([...run, ...accounts, option, 'eric@mobilyze.com'])).toThrow('Duplicate option'); expect(() => parse([...run, option])).toThrow('Missing value'); expect(() => parse([...run, option, '--self-test'])).toThrow('Missing value'); diff --git a/.claude/skills/verify-open-pstack/scripts/cli.ts b/.claude/skills/verify-open-pstack/scripts/cli.ts index 181fb0b6..a7333b2a 100644 --- a/.claude/skills/verify-open-pstack/scripts/cli.ts +++ b/.claude/skills/verify-open-pstack/scripts/cli.ts @@ -1,58 +1,12 @@ import { readFile } from 'node:fs/promises'; -import { lstatSync, readFileSync, readdirSync } from 'node:fs'; -import { createHash } from 'node:crypto'; import { join, resolve } from 'node:path'; import { validateRegistry } from './core.ts'; import { doctor } from './doctor.ts'; import { Publisher } from './github.ts'; import { MacDriver } from './harness.ts'; -import { freshRoot, interruption, interruptCommands, redact, save } from './io.ts'; +import { command, freshRoot, interruption, interruptCommands, save } from './io.ts'; import { verify } from './verify.ts'; -import { sourceDigest, sourceHash } from './provenance.ts'; -import type { GitHub } from './types.ts'; - -export function publisherCode(repository: string): string { - const hash = createHash('sha256'); - const scripts = join(repository, '.claude/skills/verify-open-pstack/scripts'); - for (const name of readdirSync(scripts).sort()) { - if (!name.endsWith('.ts')) continue; - const path = join(scripts, name), stat = lstatSync(path); - if (!stat.isFile() || stat.isSymbolicLink()) throw new Error('Publisher module must be a regular file'); - hash.update(name + '\0').update(readFileSync(path)).update('\0'); - } - return hash.digest('hex'); -} -// Capture loaded module provenance synchronously, before main yields to freshRoot or GitHub. -const loadedCode = publisherCode(resolve(import.meta.dir, '../../../..')); - -export async function bindPublisher(repository: string, pr: number, github: GitHub, startupCode = loadedCode): Promise { - const loadedSource = await sourceDigest(repository); - const pull = await github.pull(pr); - const pinnedSource = await sourceHash(repository, pull.head.sha); - if (pinnedSource !== loadedSource || publisherCode(repository) !== startupCode) throw new Error('Publisher source changed while binding the pinned head'); - const recheck = async (): Promise => { - if (await sourceHash(repository, pull.head.sha) !== pinnedSource) throw new Error('Publisher source changed after pinning'); - }; - const checkPr = (number: number): void => { - if (number !== pr) throw new Error('Publisher is bound to one PR'); - }; - return { - pull: number => { checkPr(number); return github.pull(number); }, - async files(base, head) { - if (base !== pull.base.sha || head !== pull.head.sha) throw new Error('Publisher pin differs from classification SHAs'); - await recheck(); return github.files(base, head); - }, - async comment(number, body) { checkPr(number); await recheck(); return github.comment(number, body); }, - async status(sha, state, target, description) { - if (sha !== pull.head.sha) throw new Error('Publisher status differs from pinned SHA'); - // Failure compensation must remain possible even after source changes or interruption. - if (state === 'success') await recheck(); - await github.status(sha, state, target, description); - }, - async ready(number) { checkPr(number); await recheck(); await github.ready(number); }, - }; -} -type Options = { mode: 'doctor'; output: string; pr: number; selfTest: boolean; candidate: boolean } +type Options = { mode: 'doctor'; output: string; pr: number; selfTest: boolean; candidate: boolean; claudeAccount?: string; codexAccount?: string } | { mode: 'run'; output: string; pr: number; selfTest: boolean; claudeAccount: string; codexAccount: string }; export function parse(args: string[]): Options { const mode = args[0]; @@ -64,7 +18,7 @@ export function parse(args: string[]): Options { if (seen.has(arg)) throw new Error(`Duplicate option: ${arg}`); seen.add(arg); if (arg === '--self-test' && mode === 'run') selfTest = true; else if (arg === '--candidate' && mode === 'doctor') candidate = true; - else if (arg === '--output' || mode === 'run' && ['--pr', '--claude-account', '--codex-account'].includes(arg)) { + else if (['--output', '--claude-account', '--codex-account'].includes(arg) || mode === 'run' && arg === '--pr') { const value = args[++i]; if (!value || value.startsWith('--')) throw new Error(`Missing value: ${arg}`); if (arg === '--output') output = value; else if (arg === '--pr') { if (!/^[1-9]\d*$/.test(value) || !Number.isSafeInteger(Number(value))) throw new Error('PR must be a positive integer'); pr = Number(value); } @@ -79,15 +33,17 @@ export function parse(args: string[]): Options { if (!claudeAccount || !codexAccount) throw new Error('Run requires --claude-account and --codex-account'); return { mode, output, pr, selfTest, claudeAccount, codexAccount }; } - return { mode, output, pr, selfTest, candidate }; + if (Boolean(claudeAccount) !== Boolean(codexAccount)) throw new Error('Doctor account checks require both --claude-account and --codex-account'); + if (candidate && claudeAccount) throw new Error('Candidate doctor cannot inspect caam credentials'); + return { mode, output, pr, selfTest, candidate, ...(claudeAccount ? { claudeAccount, codexAccount } : {}) }; } async function main(): Promise { let root: string | undefined, interrupted: 'SIGINT' | 'SIGTERM' | undefined; const onInterrupt = (signal: 'SIGINT' | 'SIGTERM'): void => { interrupted ??= signal; process.exitCode = interrupted === 'SIGINT' ? 130 : 143; - void interruptCommands(signal).catch(error => { - console.error(redact(String(error))); process.exitCode = 1; + void interruptCommands(signal).catch(() => { + console.error('Failed to stop active verifier commands; inspect retained evidence.'); process.exitCode = 1; }); }; const onInt = (): void => onInterrupt('SIGINT'), onTerm = (): void => onInterrupt('SIGTERM'); @@ -95,19 +51,20 @@ async function main(): Promise { try { const options = parse(process.argv.slice(2)), repository = resolve(import.meta.dir, '../../../..'); root = await freshRoot(options.output, repository); - if (options.mode === 'doctor') await doctor(root, undefined, undefined, options.candidate); + if (options.mode === 'doctor') await doctor(root, undefined, undefined, options.candidate, options.claudeAccount && options.codexAccount ? { claude: options.claudeAccount, codex: options.codexAccount } : undefined); else { - const github = await bindPublisher(repository, options.pr, new Publisher()); + const publisherRevision = (await command(['git', 'rev-parse', 'HEAD'], { cwd: repository })).trim(); const registry = validateRegistry(JSON.parse(await readFile(join(import.meta.dir, '../features/registry.json'), 'utf8'))); - const receipt = await verify({ pr: options.pr, selfTest: options.selfTest, root, registry, github, driver: new MacDriver(undefined, undefined, { claude: options.claudeAccount, codex: options.codexAccount }), persist: r => save(join(root!, 'receipt.json'), r) }); + const receipt = await verify({ pr: options.pr, selfTest: options.selfTest, root, registry, github: new Publisher(), driver: new MacDriver(undefined, undefined, { claude: options.claudeAccount, codex: options.codexAccount }), persist: r => save(join(root!, 'receipt.json'), r), publisherRevision }); console.log(`Pinned ${receipt.sha}: live-gate=${receipt.status}; evidence ${receipt.commentUrl}`); } interruption.signal.throwIfAborted(); console.log(`Retained evidence: ${root}`); } catch (error) { - const failure = redact(error instanceof Error ? error.message : String(error)); + const failure = error instanceof Error ? error.message : String(error); if (root) await save(join(root, 'failure.json'), { result: 'failed', reason: failure }); - console.error(failure); process.exitCode = interrupted === 'SIGINT' ? 130 : interrupted === 'SIGTERM' ? 143 : 1; + console.error(root ? 'Verification failed; inspect retained failure.json.' : 'Verification failed before evidence could be retained.'); + process.exitCode = interrupted === 'SIGINT' ? 130 : interrupted === 'SIGTERM' ? 143 : 1; } finally { process.off('SIGINT', onInt); process.off('SIGTERM', onTerm); } diff --git a/.claude/skills/verify-open-pstack/scripts/core.test.ts b/.claude/skills/verify-open-pstack/scripts/core.test.ts index 5d4912c3..a3e2223e 100644 --- a/.claude/skills/verify-open-pstack/scripts/core.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/core.test.ts @@ -2,8 +2,10 @@ import { describe, expect, test } from 'bun:test'; import { execFileSync } from 'node:child_process'; import { resolve } from 'node:path'; import registryData from '../features/registry.json'; -import { classify, completeEvidence, matches, newReceipt, validateRegistry } from './core.ts'; -import { HARNESSES, type Receipt } from './types.ts'; +import { classify, completeEvidence, matches, newReceipt, requiredFeatures, requiredHarnesses, validateRegistry } from './core.ts'; +import { type Receipt } from './types.ts'; +import codexManifest from '../../../../plugins/pstack/.codex-plugin/plugin.json'; +import claudeManifest from '../../../../plugins/pstack/.claude-plugin/plugin.json'; export const SHA = 'a'.repeat(40), BASE = 'b'.repeat(40); export const registry = validateRegistry(registryData); @@ -13,10 +15,10 @@ export function receipt(selfTest = false): Receipt { return r; } export function evidence(r: Receipt): void { - for (const harness of HARNESSES) { + for (const harness of requiredHarnesses(r)) { r.installations.push({ harness, sha: r.sha, cliVersion: 'test', pluginVersion: '1.5.0', location: '/tmp/candidate/plugin', home: `/tmp/${harness}`, treeHash: 'c'.repeat(64) }); - for (const feature of new Set([...r.selection.features, ...(r.selfTest ? ['project-skill'] : [])])) { + for (const feature of requiredFeatures(r, harness)) { r.observations.push({ harness, feature, surface: 'native skill', action: 'invoke', observed: 'fixture changed', transcript: '/tmp/log', transcriptHash: 'c'.repeat(64), reviewer: 'operator', artifacts: [{ path: '/tmp/artifact', sha256: 'd'.repeat(64) }] }); @@ -32,7 +34,7 @@ describe('registry and ownership', () => { for (const filename of paths) expect(() => classify([{ filename }], registry)).not.toThrow(); }); test('validates external registry shape', () => { - for (const value of [null, {}, { ...registry, skills: ['../bad'] }, { ...registry, tools: ['x', 'x'] }, { ...registry, shared: [3] }]) { + for (const value of [null, {}, { ...registry, skills: ['../bad'] }, { ...registry, tools: ['x', 'x'] }, { ...registry, shared: [3] }, { ...registry, assets: undefined }, { ...registry, assets: ['../asset'] }, { ...registry, assets: ['x', 'x'] }]) { expect(() => validateRegistry(value)).toThrow(); } }); @@ -52,11 +54,50 @@ describe('registry and ownership', () => { expect(s.skills).toEqual([...registry.skills].sort()); for (const feature of ['setup', 'runner', 'shipped-tools']) expect(s.features).toContain(feature); }); - test('asset-only installed logo changes require real harness surfaces', () => { - const s = classify([{ filename: 'plugins/pstack/assets/logo.png' }], registry); - expect(s.noRuntime).toBe(false); - expect(s.skills).toEqual([...registry.skills].sort()); - for (const feature of ['setup', 'runner', 'shipped-tools']) expect(s.features).toContain(feature); + test('installed logo changes require only their actual Codex manifest consumer', () => { + expect(codexManifest.interface.logo).toBe('./assets/logo.png'); + expect(JSON.stringify(claudeManifest)).not.toContain('assets/logo.png'); + const r = receipt(); + r.selection = classify([{ filename: 'plugins/pstack/assets/logo.png' }], registry); + expect(r.selection.noRuntime).toBe(false); + expect(r.selection.skills).toEqual([]); + expect(r.selection.features).toEqual(['assets:codex']); + expect(requiredHarnesses(r)).toEqual(['codex']); + expect(requiredFeatures(r, 'claude')).toEqual([]); + expect(requiredFeatures(r, 'codex')).toEqual(['assets:codex']); + expect(() => completeEvidence(r)).toThrow('Missing installation: codex'); + evidence(r); + expect(r.installations.map(i => i.harness)).toEqual(['codex']); + expect(() => completeEvidence(r)).not.toThrow(); + r.observations.push({ ...r.observations[0]!, harness: 'claude' }); + expect(() => completeEvidence(r)).toThrow('Extra installation or evidence'); + }); + test('unknown assets fail closed rather than inheriting logo consumers', () => { + for (const filename of ['plugins/pstack/assets/new.png', 'plugins/pstack/assets/nested/logo.png']) { + expect(() => classify([{ filename }], registry)).toThrow('Unmapped asset consumer'); + expect(() => classify([{ filename: 'README.md', previous_filename: filename }], registry)).toThrow('Unmapped asset consumer'); + } + }); + test('manifest changes retain shared coverage in both harnesses', () => { + for (const filename of ['plugins/pstack/.claude-plugin/plugin.json', 'plugins/pstack/.codex-plugin/plugin.json']) { + const r = receipt(); + r.selection = classify([{ filename }], registry); + expect(requiredHarnesses(r)).toEqual(['claude', 'codex']); + expect(requiredFeatures(r, 'claude')).toContain('setup'); + expect(requiredFeatures(r, 'codex')).toContain('setup'); + } + }); + test('asset self-test requires both harnesses without fictitious Claude asset proof', () => { + const r = receipt(true); + r.selection = classify([{ filename: 'plugins/pstack/assets/logo.png' }], registry); + expect(requiredHarnesses(r)).toEqual(['claude', 'codex']); + expect(requiredFeatures(r, 'claude')).toEqual(['project-skill']); + expect(requiredFeatures(r, 'codex')).toEqual(['assets:codex', 'project-skill']); + evidence(r); + expect(r.observations).toHaveLength(3); + expect(() => completeEvidence(r)).not.toThrow(); + r.observations = r.observations.filter(o => o.harness !== 'claude'); + expect(() => completeEvidence(r)).toThrow('Missing/duplicate evidence: claude/project-skill'); }); test('bootstrap changes require its shipped tool consumers, not the unrelated runner', () => { const s = classify([{ filename: 'plugins/pstack/skills/poteto-mode/scripts/bootstrap.ts' }], registry); @@ -88,13 +129,22 @@ describe('registry and ownership', () => { expect(s.features).toEqual([]); }); test('consumed instructions and known verifier enforcement require native project proof', () => { - for (const filename of ['AGENTS.md', 'CLAUDE.md', 'tests/skill-collision-repro.sh', '.mergify.yml']) { + for (const filename of ['AGENTS.md', 'CLAUDE.md', 'tests/skill-collision-repro.sh']) { const s = classify([{ filename }], registry); expect(s.noRuntime).toBe(false); expect(s.features).toEqual(['project-skill']); expect(classify([{ filename: 'docs/moved.md', previous_filename: filename }], registry).noRuntime).toBe(false); } }); + test('Mergify enforcement is not plugin runtime, including renamed paths', () => { + for (const file of [{ filename: '.mergify.yml' }, { filename: 'docs/moved.md', previous_filename: '.mergify.yml' }]) { + const r = receipt(); + r.selection = classify([file], registry); + expect(r.selection.noRuntime).toBe(true); + expect(requiredHarnesses(r)).toEqual([]); + expect(() => completeEvidence(r)).not.toThrow(); + } + }); test('unregistered CI, tests, and scripts fail closed instead of bypassing runtime proof', () => { for (const filename of ['.github/workflows/ci.yml', '.github/pull_request_template.md', 'tests/new-gate.sh', 'scripts/disable-proof.ts']) { expect(() => classify([{ filename }], registry)).toThrow(); @@ -104,9 +154,11 @@ describe('registry and ownership', () => { }); describe('receipt and evidence boundaries', () => { - test('validates exact immutable SHAs and PR', () => { + test('validates exact immutable SHAs, PR, and optional publisher revision', () => { for (const pr of [0, -1, 1.1, NaN]) expect(() => newReceipt(pr, SHA, BASE, false, '/tmp')).toThrow(); for (const sha of ['main', 'a'.repeat(39), 'A'.repeat(40)]) expect(() => newReceipt(90, sha, BASE, false, '/tmp')).toThrow(); + expect(newReceipt(90, SHA, BASE, false, '/tmp', 'c'.repeat(40)).publisherRevision).toBe('c'.repeat(40)); + for (const revision of ['', 'main', 'c'.repeat(39), 'c'.repeat(64), 'C'.repeat(40)]) expect(() => newReceipt(90, SHA, BASE, false, '/tmp', revision)).toThrow('publisher revision'); }); test('ordinary no-runtime has no install requirement', () => expect(() => completeEvidence(receipt())).not.toThrow()); test('self-test is separate and required in both harnesses', () => { diff --git a/.claude/skills/verify-open-pstack/scripts/core.ts b/.claude/skills/verify-open-pstack/scripts/core.ts index 6c85dc3f..8bf4edac 100644 --- a/.claude/skills/verify-open-pstack/scripts/core.ts +++ b/.claude/skills/verify-open-pstack/scripts/core.ts @@ -1,9 +1,9 @@ -import { HARNESSES, REPO, type ChangedFile, type Receipt, type Registry, type Selection } from './types.ts'; +import { HARNESSES, REPO, type ChangedFile, type Harness, type Receipt, type Registry, type Selection } from './types.ts'; export function validateRegistry(value: unknown): Registry { if (!value || typeof value !== 'object') throw new Error('Invalid feature registry'); const r = value as Registry; - for (const key of ['skills', 'shared', 'setup', 'runner', 'tools', 'project', 'nonRuntime'] as const) { + for (const key of ['skills', 'shared', 'assets', 'setup', 'runner', 'tools', 'project', 'nonRuntime'] as const) { if (!Array.isArray(r[key]) || r[key].some(v => typeof v !== 'string' || !v || v.includes('..'))) { throw new Error(`Invalid registry.${key}`); } @@ -24,6 +24,10 @@ export function classify(files: ChangedFile[], registry: Registry): Selection { for (const path of paths) { if (!path || path.startsWith('/') || path.split('/').includes('..')) throw new Error(`Unsafe path: ${path}`); let covered = false; + if (registry.assets.some(p => matches(path, p))) { + if (path !== 'plugins/pstack/assets/logo.png') throw new Error(`Unmapped asset consumer: ${path}`); + features.add('assets:codex'); covered = true; + } if (registry.shared.some(p => matches(path, p))) { registry.skills.forEach(s => skills.add(s)); ['setup', 'runner', 'shipped-tools'].forEach(f => features.add(f)); @@ -46,24 +50,31 @@ export function classify(files: ChangedFile[], registry: Registry): Selection { return { paths, skills: [...skills].sort(), features: [...features].sort(), noRuntime: !features.size }; } -export function newReceipt(pr: number, sha: string, base: string, selfTest: boolean, artifactRoot: string): Receipt { +export function newReceipt(pr: number, sha: string, base: string, selfTest: boolean, artifactRoot: string, publisherRevision?: string): Receipt { if (!Number.isSafeInteger(pr) || pr < 1 || !/^[a-f0-9]{40}$/.test(sha) || !/^[a-f0-9]{40}$/.test(base)) throw new Error('Invalid PR/SHA'); - return { schema: 1, repo: REPO, pr, sha, base, selfTest, artifactRoot, phase: 'resolve', + if (publisherRevision !== undefined && !/^[a-f0-9]{40}$/.test(publisherRevision)) throw new Error('Invalid publisher revision'); + return { schema: 1, repo: REPO, pr, sha, base, selfTest, artifactRoot, ...(publisherRevision ? { publisherRevision } : {}), phase: 'resolve', selection: { paths: [], skills: [], features: [], noRuntime: false }, installations: [], observations: [], started: new Date().toISOString(), cleanup: 'Run state retained; operator owns cleanup.' }; } -export function requiredFeatures(receipt: Receipt): string[] { - return [...new Set([...receipt.selection.features, ...(receipt.selfTest ? ['project-skill'] : [])])]; +export function requiredFeatures(receipt: Receipt, harness?: Harness): string[] { + const features = [...new Set([...receipt.selection.features, ...(receipt.selfTest ? ['project-skill'] : [])])]; + return harness ? features.filter(feature => feature !== 'assets:codex' || harness === 'codex') : features; +} + +export function requiredHarnesses(receipt: Receipt): Harness[] { + return HARNESSES.filter(harness => requiredFeatures(receipt, harness).length > 0); } export function completeEvidence(receipt: Receipt): void { const features = requiredFeatures(receipt); if (!features.length && !receipt.selection.noRuntime) throw new Error('No classification'); - for (const harness of features.length ? HARNESSES : []) { + const harnesses = requiredHarnesses(receipt); + for (const harness of harnesses) { const installs = receipt.installations.filter(i => i.harness === harness && i.sha === receipt.sha); if (installs.length !== 1 || !installs[0]!.location || !/^[a-f0-9]{64}$/.test(installs[0]!.treeHash) || !installs[0]!.cliVersion || !installs[0]!.pluginVersion || !installs[0]!.home) throw new Error(`Missing installation: ${harness}`); - for (const feature of features) { + for (const feature of requiredFeatures(receipt, harness)) { const records = receipt.observations.filter(o => o.harness === harness && o.feature === feature); if (records.length !== 1) throw new Error(`Missing/duplicate evidence: ${harness}/${feature}`); const o = records[0]!; @@ -72,6 +83,7 @@ export function completeEvidence(receipt: Receipt): void { o.artifacts.some(a => !a.path || !/^[a-f0-9]{64}$/.test(a.sha256))) throw new Error(`Incomplete evidence: ${harness}/${feature}`); } } - if (receipt.installations.length !== (features.length ? HARNESSES.length : 0) || receipt.observations.length !== features.length * HARNESSES.length) throw new Error('Extra installation or evidence'); + const observationCount = harnesses.reduce((count, harness) => count + requiredFeatures(receipt, harness).length, 0); + if (receipt.installations.length !== harnesses.length || receipt.observations.length !== observationCount) throw new Error('Extra installation or evidence'); if (receipt.failure) throw new Error(receipt.failure); } diff --git a/.claude/skills/verify-open-pstack/scripts/doctor.test.ts b/.claude/skills/verify-open-pstack/scripts/doctor.test.ts new file mode 100644 index 00000000..3b748a23 --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/doctor.test.ts @@ -0,0 +1,65 @@ +import { afterEach, expect, test } from 'bun:test'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { doctor } from './doctor.ts'; +import type { Command } from './io.ts'; +import type { Harness } from './types.ts'; + +const roots: string[] = []; +const accounts = { claude: 'claude@example.com', codex: 'codex@example.com' }; +afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); +async function fixture(): Promise { + const root = await mkdtemp(join(tmpdir(), 'pstack-health-')); roots.push(root); return root; +} +function runner(calls: string[][], result: (tool: Harness) => string): Command { + return async args => { + calls.push(args); + if (args.includes('--version')) return 'version'; + if (args.includes('--help')) return '--plugin-dir --settings --setting-sources --json local path'; + if (args.join(' ') === 'caam ls --json') return JSON.stringify({ profiles: ['claude', 'codex'].map(tool => ({ tool, name: accounts[tool as Harness], identity: { email: accounts[tool as Harness] }, active: true })) }); + if (args[0] === 'caam' && args[1] === 'limits') return result(args[2] as Harness); + throw new Error(`Unexpected command: ${args.join(' ')}`); + }; +} +function healthy(tool: Harness): object[] { + return [{ provider: tool, profile_name: accounts[tool], usage: {} }, { provider: tool, profile_name: 'unselected@example.com', usage: { error: 'Unauthorized' } }]; +} + +test('doctor allows daily-active accounts and checks only selected provider/profile results', async () => { + const root = await fixture(), calls: string[][] = []; + await doctor(root, runner(calls, tool => JSON.stringify(healthy(tool))), 'darwin', false, accounts); + expect(calls.filter(args => args[0] === 'caam')).toEqual([ + ['caam', 'ls', '--json'], ['caam', 'limits', 'claude', '--format', 'json'], ['caam', 'limits', 'codex', '--format', 'json'], + ]); + expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('pass'); +}); + +for (const tool of ['claude', 'codex'] as const) { + test.each(['HTTP 401', 'Unauthorized', 'OAuth credential expired', 'invalid_grant'])('doctor gives manual %s repair for selected ' + tool + ' result', async error => { + const root = await fixture(), calls: string[][] = []; + const run = runner(calls, provider => JSON.stringify(provider === tool ? [{ provider, profile_name: accounts[provider], usage: { error } }] : healthy(provider))); + const restore = `caam add ${tool} ${accounts[tool]} --no-activate --force`; + await expect(doctor(root, run, 'darwin', false, accounts)).rejects.toThrow(restore); + const report = JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')); + expect(report.result).toBe('blocked'); expect(report.reason).toContain(restore); + expect(calls.some(args => ['add', 'activate', 'login'].includes(args[1]!))).toBe(false); + }); +} + +test('doctor catches command-level credential failures and expired credential-source state', async () => { + for (const failCommand of [false, true]) { + const root = await fixture(), calls: string[][] = []; + const run = runner(calls, tool => { + if (failCommand) throw new Error('caam limits: HTTP 401 Unauthorized'); + return JSON.stringify([{ provider: tool, profile_name: accounts[tool], credential_source: { state: 'expired' } }]); + }); + await expect(doctor(root, run, 'darwin', false, accounts)).rejects.toThrow('caam add claude claude@example.com --no-activate --force'); + } +}); + +test.each(['not JSON', '{}', '[]', '[{"provider":"codex","profile_name":"claude@example.com","usage":{}}]', '[{"provider":"claude","profile_name":"claude@example.com","usage":null}]', '[{"provider":"claude","profile_name":"claude@example.com","usage":{"error":"Network unavailable"}}]'])('doctor fails closed on unavailable or malformed selected health: %s', async output => { + const root = await fixture(), calls: string[][] = []; + await expect(doctor(root, runner(calls, () => output), 'darwin', false, accounts)).rejects.toThrow('health check failed'); + expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('blocked'); +}); diff --git a/.claude/skills/verify-open-pstack/scripts/doctor.ts b/.claude/skills/verify-open-pstack/scripts/doctor.ts index d82c28dc..b6ba8014 100644 --- a/.claude/skills/verify-open-pstack/scripts/doctor.ts +++ b/.claude/skills/verify-open-pstack/scripts/doctor.ts @@ -1,8 +1,10 @@ import { mkdir, realpath } from 'node:fs/promises'; import { join } from 'node:path'; import { command, isolatedEnv, save, type Command } from './io.ts'; +import { selectedAccounts } from './isolation.ts'; +import { HARNESSES, type Accounts } from './types.ts'; -export async function doctor(root: string, run: Command = command, platform = process.platform, candidate = false): Promise { +export async function doctor(root: string, run: Command = command, platform = process.platform, candidate = false, accounts?: Accounts): Promise { const report: Record = { platform, candidate, date: new Date().toISOString(), skill: await realpath(join(import.meta.dir, '..')), checks: {} }; const checks = report.checks as Record; try { @@ -20,6 +22,32 @@ export async function doctor(root: string, run: Command = command, platform = pr if (args.includes('marketplace') && !/local path|local or Git|local marketplace/i.test(help)) throw new Error('Codex local marketplace interface unavailable'); checks[args.join(' ')] = help; } + if (accounts) { + if (candidate) throw new Error('Candidate doctor cannot inspect caam credentials'); + const parentEnv: Record = {}; + for (const key of ['PATH', 'HOME', 'USER', 'LOGNAME', 'CAAM_HOME', 'XDG_CONFIG_HOME', 'XDG_DATA_HOME']) if (process.env[key]) parentEnv[key] = process.env[key]!; + selectedAccounts(await run(['caam', 'ls', '--json'], { env: parentEnv }), accounts); + for (const tool of HARNESSES) { + const restore = `caam add ${tool} ${accounts[tool]} --no-activate --force`; + try { + const output = await run(['caam', 'limits', tool, '--format', 'json'], { env: parentEnv }); + checks[`caam limits ${tool}`] = output; + const rows = JSON.parse(output); + if (!Array.isArray(rows)) throw new Error('Unknown caam limits schema; expected profile rows'); + const selected = rows.filter(row => row?.provider === tool && row?.profile_name === accounts[tool]); + if (selected.length !== 1) throw new Error(`Missing or ambiguous caam limits result for ${tool}/${accounts[tool]}`); + if (selected[0].credential_source?.state === 'expired') throw new Error('Credential copy expired'); + if (!selected[0].usage || typeof selected[0].usage !== 'object' || Array.isArray(selected[0].usage)) throw new Error('Missing caam limits usage result'); + const error = selected[0].usage.error; + if (error !== undefined && typeof error !== 'string') throw new Error('Unknown caam limits error schema'); + if (error) throw new Error(error); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + if (/unauthori[sz]ed|expired|\b401\b|invalid[_ -](?:token|grant)/i.test(reason)) throw new Error(`caam ${tool}/${accounts[tool]} copy is unauthorized or expired; restore with: ${restore}`); + throw new Error(`caam ${tool}/${accounts[tool]} copy health check failed: ${reason}`); + } + } + } // PR resolution checks publisher authentication; a child surface doctor has no GitHub credentials. report.result = 'pass'; } catch (error) { diff --git a/.claude/skills/verify-open-pstack/scripts/github.ts b/.claude/skills/verify-open-pstack/scripts/github.ts index d83ad1ad..a6d452c6 100644 --- a/.claude/skills/verify-open-pstack/scripts/github.ts +++ b/.claude/skills/verify-open-pstack/scripts/github.ts @@ -24,9 +24,9 @@ export class Publisher implements GitHub { return JSON.parse(await this.run(['gh', 'api', `repos/${REPO}/${path}`, ...args], { allowInterrupted })); } async pull(pr: number): Promise { - const data = await this.api(`pulls/${pr}`) as { number: number; head: { sha: string; repo?: { full_name: string } }; base: { sha: string }; draft: boolean; state: string; body: string | null }; - if (data.number !== pr || typeof data.draft !== 'boolean' || typeof data.state !== 'string' || data.body !== null && typeof data.body !== 'string') throw new Error('Invalid PR response'); - return { number: pr, head: { sha: data.head.sha }, base: { sha: data.base.sha }, draft: data.draft, state: data.state, headRepo: data.head.repo?.full_name ?? '', body: data.body ?? '' }; + const data = await this.api(`pulls/${pr}`) as { number: number; head: { sha: string; repo?: { full_name: string } }; base: { sha: string }; state: string }; + if (data.number !== pr || typeof data.state !== 'string') throw new Error('Invalid PR response'); + return { number: pr, head: { sha: data.head.sha }, base: { sha: data.base.sha }, state: data.state, headRepo: data.head.repo?.full_name ?? '' }; } async files(base: string, head: string): Promise { if (![base, head].every(sha => /^[a-f0-9]{40}$/.test(sha))) throw new Error('Invalid diff SHA'); @@ -52,5 +52,4 @@ export class Publisher implements GitHub { if (!/^[a-f0-9]{40}$/.test(sha)) throw new Error('Invalid status SHA'); await this.api(`statuses/${sha}`, ['--method', 'POST', '-f', 'context=live-gate', '-f', `state=${state}`, '-f', `target_url=${target}`, '-f', `description=${description.slice(0, 140)}`], state === 'failure'); } - async ready(pr: number): Promise { await this.run(['gh', 'pr', 'ready', String(pr), '--repo', REPO]); } } diff --git a/.claude/skills/verify-open-pstack/scripts/harness.test.ts b/.claude/skills/verify-open-pstack/scripts/harness.test.ts index 9c066f4f..bed2075a 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.test.ts @@ -5,53 +5,49 @@ import { join } from 'node:path'; import { doctor } from './doctor.ts'; import { codexInstallation, launch, MacDriver, verifyCodexEnabled, verifyProjectDoctor } from './harness.ts'; import { newReceipt } from './core.ts'; -import { MAC_TESTS } from './verify.ts'; -import { command, freshRoot, isolatedEnv, redact, retainedFile, treeHash, type Command } from './io.ts'; +import { command, freshRoot, isolatedEnv, retainedFile, treeHash, type Command } from './io.ts'; const roots: string[] = []; async function fixture(): Promise { const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-test-'))); roots.push(root); return root; } afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); describe('isolated harness boundaries', () => { - test('candidate environment never carries publisher credentials or daily config', () => { + test('candidate environment keeps operator identity while isolating provider configuration', () => { + const previous = { HOME: process.env.HOME, USER: process.env.USER, LOGNAME: process.env.LOGNAME, + GH_TOKEN: process.env.GH_TOKEN, CODEX_HOME: process.env.CODEX_HOME, CLAUDE_CONFIG_DIR: process.env.CLAUDE_CONFIG_DIR, + ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY, OPENAI_API_KEY: process.env.OPENAI_API_KEY }; + process.env.HOME = '/operator/home'; process.env.USER = 'operator-user'; process.env.LOGNAME = 'operator-login'; process.env.GH_TOKEN = 'test-publisher-token'; process.env.CODEX_HOME = '/daily/codex'; - process.env.CLAUDE_CONFIG_DIR = '/daily/claude'; - const oldAnthropic = process.env.ANTHROPIC_API_KEY, oldOpenai = process.env.OPENAI_API_KEY; - process.env.ANTHROPIC_API_KEY = 'daily-anthropic'; process.env.OPENAI_API_KEY = 'daily-openai'; + process.env.CLAUDE_CONFIG_DIR = '/daily/claude'; process.env.ANTHROPIC_API_KEY = 'daily-anthropic'; process.env.OPENAI_API_KEY = 'daily-openai'; try { for (const harness of ['claude', 'codex'] as const) { - const env = isolatedEnv('/run/home', harness); + const env = isolatedEnv('/run/state', harness); expect(env.GH_TOKEN).toBeUndefined(); expect(env.GITHUB_TOKEN).toBeUndefined(); expect(env.ANTHROPIC_API_KEY).toBeUndefined(); expect(env.OPENAI_API_KEY).toBeUndefined(); - expect(env.HOME).toBe('/run/home'); expect(env.GIT_CONFIG_GLOBAL).toBe('/dev/null'); - expect(env.TMPDIR).toBe('/run/home/tmp'); - expect(env.CLAUDE_CONFIG_DIR).toBe('/run/home/.claude'); - expect(env.CODEX_HOME).toBe('/run/home/.codex'); - expect(env.GH_CONFIG_DIR).toBe('/run/home/.config/gh'); - expect(JSON.stringify(env)).not.toContain('/daily/'); + expect(env.HOME).toBe('/operator/home'); expect(env.USER).toBe('operator-user'); expect(env.LOGNAME).toBe('operator-login'); + expect(env.GIT_CONFIG_GLOBAL).toBe('/dev/null'); expect(env.TMPDIR).toBe('/run/state/tmp'); + expect(env.CLAUDE_CONFIG_DIR).toBe('/run/state/.claude'); expect(env.CODEX_HOME).toBe('/run/state/.codex'); + expect(env.GH_CONFIG_DIR).toBe('/run/state/.config/gh'); expect(JSON.stringify(env)).not.toContain('/daily/'); } - expect(launch('claude', '/run/home', '/candidate')).toEqual(['claude', '--plugin-dir', '/candidate/plugins/pstack', '--settings', '/run/home/settings.json', '--setting-sources', 'project']); - expect(launch('codex', '/run/home', '/candidate')).toEqual(['codex']); + expect(launch('claude', '/run/state', '/candidate')).toEqual(['claude', '--plugin-dir', '/candidate/plugins/pstack', '--settings', '/run/state/settings.json', '--setting-sources', 'project']); + expect(launch('codex', '/run/state', '/candidate')).toEqual(['codex']); } finally { - delete process.env.GH_TOKEN; delete process.env.CODEX_HOME; delete process.env.CLAUDE_CONFIG_DIR; - if (oldAnthropic === undefined) delete process.env.ANTHROPIC_API_KEY; else process.env.ANTHROPIC_API_KEY = oldAnthropic; - if (oldOpenai === undefined) delete process.env.OPENAI_API_KEY; else process.env.OPENAI_API_KEY = oldOpenai; + for (const [name, value] of Object.entries(previous)) { + if (value === undefined) delete process.env[name]; else process.env[name] = value; + } } }); - test('candidate environment preserves real account names without inventing an account', () => { - const previous = { USER: process.env.USER, LOGNAME: process.env.LOGNAME }; - process.env.USER = 'operator-user'; process.env.LOGNAME = 'operator-login'; + test('candidate environment requires the real operator identity', () => { + const previous = { HOME: process.env.HOME, USER: process.env.USER, LOGNAME: process.env.LOGNAME }; try { - for (const harness of ['claude', 'codex'] as const) { - const env = isolatedEnv('/run/home', harness); - expect(env.USER).toBe('operator-user'); expect(env.LOGNAME).toBe('operator-login'); - expect(env.HOME).toBe('/run/home'); - } - delete process.env.USER; delete process.env.LOGNAME; - const env = isolatedEnv('/run/home', 'claude'); - expect(env.USER).toBeUndefined(); expect(env.LOGNAME).toBeUndefined(); + delete process.env.HOME; + expect(() => isolatedEnv('/run/state', 'claude')).toThrow('Real HOME'); + process.env.HOME = '/operator/home'; delete process.env.USER; + expect(() => isolatedEnv('/run/state', 'claude')).toThrow('USER and LOGNAME'); + process.env.USER = 'operator-user'; delete process.env.LOGNAME; + expect(() => isolatedEnv('/run/state', 'claude')).toThrow('USER and LOGNAME'); } finally { - for (const name of ['USER', 'LOGNAME'] as const) { - if (previous[name] === undefined) delete process.env[name]; else process.env[name] = previous[name]; + for (const [name, value] of Object.entries(previous)) { + if (value === undefined) delete process.env[name]; else process.env[name] = value; } } }); @@ -69,7 +65,7 @@ describe('isolated harness boundaries', () => { expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('pass'); expect(calls.every(c => c.includes('--help') || c.includes('--version'))).toBe(true); }); - test('project self-test accepts real doctor output through canonical workspace paths', async () => { + test('project self-test accepts real candidate doctor output through canonical workspace paths', async () => { const root = await fixture(), workspace = join(root, 'workspace'); const skill = await realpath(join(import.meta.dir, '..')); await mkdir(join(workspace, '.claude/skills'), { recursive: true }); @@ -81,7 +77,7 @@ describe('isolated harness boundaries', () => { await expect(verifyProjectDoctor([parentText], workspace)).rejects.toThrow('passing child doctor'); await doctor(root, run, 'darwin', true); const text = await readFile(join(root, 'doctor.json'), 'utf8'); - expect(JSON.parse(text).skill).toBe(skill); + expect(JSON.parse(text).candidate).toBe(true); expect(JSON.parse(text).skill).toBe(skill); await verifyProjectDoctor(['not JSON', text], workspace); await verifyProjectDoctor([text], alias); const other = join(root, 'other-workspace'); @@ -90,12 +86,54 @@ describe('isolated harness boundaries', () => { await expect(verifyProjectDoctor(['{}'], workspace)).rejects.toThrow('passing child doctor'); await expect(verifyProjectDoctor([JSON.stringify({ ...JSON.parse(text), result: 'blocked' })], workspace)).rejects.toThrow('passing child doctor'); }); - test.each([false, true, 'proof', 'review'])('prepare gates disposable credentials on pinned proof (failure=%s)', async failure => { - const failInstall = failure === true; - const root = await fixture(), repository = join(root, 'repository'), calls: string[][] = []; + test('setup fails closed before any prepare or exercise command', async () => { + const root = await fixture(), sha = '1'.repeat(40), calls: string[][] = []; + const receipt = newReceipt(120, sha, sha, false, root); + receipt.selection.features = ['setup']; + const driver = new MacDriver(async args => { calls.push(args); return ''; }, async () => '', { claude: 'claude@example.com', codex: 'codex@example.com' }); + const message = 'setup exercise requires #120 (setup-pstack config-home)'; + for (const operation of [() => driver.prepare(receipt), () => driver.exercise(receipt)]) { + let caught: unknown; + try { await operation(); } catch (error) { caught = error; } + expect(caught).toBeInstanceOf(Error); expect((caught as Error).message).toBe(message); expect(calls).toEqual([]); + } + }); + test.each(['claude', 'codex'] as const)('prepare refuses an unauthorized %s vault copy before candidate commands', async tool => { + const root = await fixture(), calls: string[][] = [], sha = '2'.repeat(40); + const accounts = { claude: 'claude@example.com', codex: 'codex@example.com' }; + const run: Command = async args => { + calls.push(args); + if (args.includes('--version')) return 'version'; + if (args.includes('--help')) return '--plugin-dir --settings --setting-sources --json local path'; + if (args[0] === 'caam' && args[1] === 'ls') return JSON.stringify({ profiles: [ + { tool: 'claude', name: accounts.claude, identity: { email: accounts.claude }, active: true }, + { tool: 'codex', name: accounts.codex, identity: { email: accounts.codex }, active: true }, + ] }); + if (args[0] === 'caam' && args[1] === 'limits') { + const provider = args[2] as keyof typeof accounts; + return JSON.stringify([{ provider, profile_name: accounts[provider], usage: { error: provider === tool ? 'HTTP 401 Unauthorized' : '' } }]); + } + throw new Error(`Unexpected fixture command: ${args.join(' ')}`); + }; + const platform = Object.getOwnPropertyDescriptor(process, 'platform')!; + try { + Object.defineProperty(process, 'platform', { ...platform, value: 'darwin' }); + await expect(new MacDriver(run, async () => '', accounts).prepare(newReceipt(111, sha, sha, true, root))).rejects.toThrow(`caam add ${tool} ${accounts[tool]} --no-activate --force`); + } finally { Object.defineProperty(process, 'platform', platform); } + expect(calls.some(c => c.join(' ') === 'caam ls --json')).toBe(true); + expect(calls.some(c => c.join(' ') === `caam limits ${tool} --format json`)).toBe(true); + expect(calls.some(c => c[0] === 'caam' && ['add', 'activate'].includes(c[1]!))).toBe(false); + expect(calls.some(c => c[0] === 'git' && !c.includes('--version'))).toBe(false); + expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('blocked'); + }); + test.each([false, true])('prepare uses pinned local git fixture and disposable config credentials (install failure=%s)', async failInstall => { + const root = await fixture(), repository = join(root, 'repository'), operatorHome = join(root, 'operator-home'), calls: string[][] = []; const accounts = { claude: 'claude@example.com', codex: 'codex@example.com' }; const caam = join(root, 'caam'), vault = join(caam, 'data/vault'); - const credentials = { claude: JSON.stringify({ claudeAiOauth: { accessToken: 'fixture-claude' } }), codex: JSON.stringify({ tokens: { access_token: 'fixture-codex' } }) }; + const credentials = { + claude: JSON.stringify({ claudeAiOauth: { accessToken: 'fixture-claude', refreshToken: 'fixture-claude-refresh' } }), + codex: JSON.stringify({ tokens: { access_token: 'fixture-codex', refresh_token: 'fixture-codex-refresh', id_token: 'fixture-codex-id' } }), + }; for (const tool of ['claude', 'codex'] as const) { const dir = join(vault, tool, accounts[tool]); await mkdir(dir, { recursive: true }); await writeFile(join(dir, tool === 'claude' ? '.credentials.json' : 'auth.json'), credentials[tool]); @@ -111,96 +149,70 @@ describe('isolated harness boundaries', () => { await command(['git', 'init', repository]); await command(['git', 'add', '.'], { cwd: repository }); await command(['git', '-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.com', '-c', 'commit.gpgsign=false', 'commit', '-m', 'fixture'], { cwd: repository }); - const sha = (await command(['git', 'rev-parse', 'HEAD'], { cwd: repository })).trim(); - const platform = Object.getOwnPropertyDescriptor(process, 'platform')!, previousCaam = process.env.CAAM_HOME; - const run: Command = async (wrapped, options = {}) => { - const args = wrapped[0] === '/usr/bin/sandbox-exec' ? wrapped.slice(3) : wrapped; + const sha = (await command(['git', 'rev-parse', 'HEAD'], { cwd: repository })).trim(), actualGit: Command = command; + await mkdir(operatorHome, { mode: 0o700 }); + const platform = Object.getOwnPropertyDescriptor(process, 'platform')!; + const previous = { CAAM_HOME: process.env.CAAM_HOME, HOME: process.env.HOME, USER: process.env.USER, LOGNAME: process.env.LOGNAME }; + const run: Command = async (args, options = {}) => { calls.push(args); - if (args[0] === '/bin/sh') { - expect(options.cwd).toBe(join(root, 'source-home/workspace/.claude/skills/verify-open-pstack')); - expect(options.env!.PSTACK_OPERATOR_SENTINELS).toBe('1'); expect(process.env.HOME).toBe(options.env!.HOME); - expect(args[2]).toContain('--timeout 0'); expect(options.env!.GH_TOKEN).toBeUndefined(); - for (const harness of ['claude', 'codex']) await expect(stat(join(root, 'state', harness))).rejects.toThrow('ENOENT'); - // Fake adapter output only; actual OS proof is required on the operator Mac. - return `${MAC_TESTS.map(name => `(pass) ${name} [1ms]`).join('\n')}\n 2 pass\n 0 fail\nPSTACK_MAC_TEST_EXIT=${failure === 'proof' ? '1' : '0'}\n`; - } if (args[0] === 'caam') { - expect(wrapped.slice(0, 2)).toEqual(['/usr/bin/sandbox-exec', '-p']); - expect(wrapped[2]).toContain('(deny file-write*)'); - expect(options.env!.CAAM_HOME).toBe(caam); - return JSON.stringify({ profiles: ['claude', 'codex'].map(tool => ({ tool, name: accounts[tool as keyof typeof accounts], identity: { email: accounts[tool as keyof typeof accounts] } })) }); + if (args[1] === 'ls') { + expect(args).toEqual(['caam', 'ls', '--json']); + return JSON.stringify({ profiles: ['claude', 'codex'].map(tool => ({ tool, name: accounts[tool as keyof typeof accounts], identity: { email: accounts[tool as keyof typeof accounts] }, active: true })) }); + } + const provider = args[2] as keyof typeof accounts; + expect(args).toEqual(['caam', 'limits', provider, '--format', 'json']); + return JSON.stringify([{ provider, profile_name: accounts[provider], usage: {} }]); } - const env = options.env!, candidate = env.HOME!.startsWith(join(root, 'state') + '/'); + const env = options.env, candidate = Boolean(env?.CLAUDE_CONFIG_DIR?.startsWith(join(root, 'state') + '/')); if (candidate) { - expect(wrapped.slice(0, 2)).toEqual(['/usr/bin/sandbox-exec', '-f']); - const baseProfile = `${env.HOME!}.unit-test.sb`; - expect([baseProfile, `${baseProfile}.exercise.sb`]).toContain(wrapped[2]!); - for (const dir of [env.HOME!, env.TMPDIR!, env.CLAUDE_CONFIG_DIR!, env.CODEX_HOME!, env.GH_CONFIG_DIR!]) { - const info = await stat(dir); - expect(info.isDirectory()).toBe(true); expect(info.mode & 0o777).toBe(0o700); + const stateHome = join(env!.CLAUDE_CONFIG_DIR!, '..'); + expect(env!.HOME).toBe(operatorHome); expect(env!.USER).toBe('operator-user'); expect(env!.LOGNAME).toBe('operator-login'); + expect(env!.CLAUDE_CONFIG_DIR).toBe(join(stateHome, '.claude')); expect(env!.CODEX_HOME).toBe(join(stateHome, '.codex')); + expect(env!.TMPDIR).toBe(join(stateHome, 'tmp')); expect(env!.GH_CONFIG_DIR).toBe(join(stateHome, '.config/gh')); + expect(env!.GH_TOKEN).toBeUndefined(); expect(env!.CAAM_HOME).toBeUndefined(); + for (const dir of [stateHome, env!.TMPDIR!, env!.CLAUDE_CONFIG_DIR!, env!.CODEX_HOME!, env!.GH_CONFIG_DIR!]) { + const info = await stat(dir); expect(info.isDirectory()).toBe(true); expect(info.mode & 0o777).toBe(0o700); } - expect(env.CLAUDE_CONFIG_DIR).toBe(join(env.HOME!, '.claude')); - expect(env.CODEX_HOME).toBe(join(env.HOME!, '.codex')); - expect(env.GH_TOKEN).toBeUndefined(); expect(env.CAAM_HOME).toBeUndefined(); - const settings = join(env.HOME!, 'settings.json'); - expect(JSON.parse(await readFile(settings, 'utf8'))).toEqual({}); + const settings = join(stateHome, 'settings.json'); expect(JSON.parse(await readFile(settings, 'utf8'))).toEqual({}); expect((await stat(settings)).mode & 0o777).toBe(0o600); } - if (options.cwd) expect((await stat(options.cwd)).isDirectory()).toBe(true); - if (args[0] === '/bin/rm') return command(args, options); - if (args[0] === '/usr/bin/true') return ''; if (args[0] === '/usr/bin/script') { - expect(wrapped[2]).toBe(`${env.HOME!}.unit-test.sb.exercise.sb`); - expect(wrapped[2]!.startsWith(env.HOME! + '/')).toBe(false); - const policy = await readFile(wrapped[2]!, 'utf8'); - const workspace = join(env.HOME!, 'workspace'); - expect(policy).toContain(`(deny file-write* (require-all (subpath ${JSON.stringify(workspace)})`); - expect(policy).toContain(`(subpath ${JSON.stringify(join(workspace, '.claude/skills/verify-open-pstack/node_modules'))})`); - expect(policy).toContain(`(subpath ${JSON.stringify(join(workspace, 'plugins/pstack/skills/poteto-mode/scripts/node_modules'))})`); - if (args.includes('codex')) expect(policy).toContain(`(deny file-write* (require-all (subpath ${JSON.stringify(join(env.CODEX_HOME!, 'plugins/pstack'))})`); - expect(options.interactive).toBe(true); - throw new Error('fixture protected native surface reached'); + expect(candidate).toBe(true); expect(options.interactive).toBe(true); + await writeFile(args[2]!, 'retained raw native evidence'); + throw new Error('fixture native surface reached'); } if (args[0] === 'git') { const local = [...args]; if (args[1] === 'clone') local[local.length - 2] = repository; - return command(local, options); + return actualGit(local, options); } if (args.includes('--version')) return 'version'; if (args.includes('--help')) return '--plugin-dir --settings --setting-sources --json local path'; if (args[0] === 'codex' && args[1] === 'plugin') { for (const tool of ['claude', 'codex'] as const) { - const file = join(env[tool === 'claude' ? 'CLAUDE_CONFIG_DIR' : 'CODEX_HOME']!, tool === 'claude' ? '.credentials.json' : 'auth.json'); - expect(await readFile(file, 'utf8')).toBe(credentials[tool]); - expect((await stat(file)).mode & 0o777).toBe(0o600); + const file = join(env![tool === 'claude' ? 'CLAUDE_CONFIG_DIR' : 'CODEX_HOME']!, tool === 'claude' ? '.credentials.json' : 'auth.json'); + expect(await readFile(file, 'utf8')).toBe(credentials[tool]); expect((await stat(file)).mode & 0o777).toBe(0o600); } if (failInstall) throw new Error('fixture installation failed'); if (args[2] === 'marketplace') return '{}'; if (args[2] === 'add') { - const installedPath = join(env.CODEX_HOME!, 'plugins/pstack'); - await cp(join(env.HOME!, 'workspace/plugins/pstack'), installedPath, { recursive: true }); + const installedPath = join(env!.CODEX_HOME!, 'plugins/pstack'); + await cp(join(join(env!.CLAUDE_CONFIG_DIR!, '..'), 'workspace/plugins/pstack'), installedPath, { recursive: true }); return JSON.stringify({ name: 'pstack', marketplaceName: 'open-pstack', installedPath }); } if (args[2] === 'list') return JSON.stringify({ installed: [{ name: 'pstack', marketplaceName: 'open-pstack', installed: true, enabled: true }] }); } throw new Error(`Unexpected fixture command: ${args.join(' ')}`); }; - // Unit adapter only: this wrapper does not enforce an OS boundary. The - // independent macOS isolation.test.ts lane executes real sandbox-exec. - const boundary = async (home: string): Promise => { - const profile = `${home}.unit-test.sb`; await writeFile(profile, 'unit adapter'); return profile; - }; - const review = async () => failure === 'review' ? 'REJECT' : 'PASS MAC ISOLATION'; - const driver = new MacDriver(run, review, accounts, boundary); + const driver = new MacDriver(run, async () => '', accounts); try { - Object.defineProperty(process, 'platform', { ...platform, value: 'darwin' }); process.env.CAAM_HOME = caam; - const preparedReceipt = newReceipt(111, sha, sha, true, root); - const preparation = driver.prepare(preparedReceipt); - if (failure === 'proof') await expect(preparation).rejects.toThrow('Both pinned Mac'); - else if (failure === 'review') await expect(preparation).rejects.toThrow('Operator rejected Mac'); - else if (failInstall) await expect(preparation).rejects.toThrow('fixture installation failed'); + Object.defineProperty(process, 'platform', { ...platform, value: 'darwin' }); + process.env.CAAM_HOME = caam; process.env.HOME = operatorHome; process.env.USER = 'operator-user'; process.env.LOGNAME = 'operator-login'; + const receipt = newReceipt(111, sha, sha, true, root), preparation = driver.prepare(receipt); + if (failInstall) await expect(preparation).rejects.toThrow('fixture installation failed'); else { - const installs = await preparation; + const installs = await preparation; receipt.installations = installs; expect(installs.map(i => i.harness)).toEqual(['claude', 'codex']); expect(installs.map(i => i.account)).toEqual([accounts.claude, accounts.codex]); expect(installs.every(i => i.sha === sha)).toBe(true); @@ -208,25 +220,28 @@ describe('isolated harness boundaries', () => { expect(installs[0]!.sourceHash).toBe(installs[1]!.sourceHash); expect(calls.some(c => c.join(' ') === 'codex plugin add pstack@open-pstack --json')).toBe(true); for (const installation of installs) { - const receipt = newReceipt(111, sha, sha, true, root); - receipt.installations = [installation]; receipt.macProof = preparedReceipt.macProof; - expect(receipt.macProof!.sha).toBe(sha); expect(receipt.macProof!.noKeychainDialog).toBe(true); - expect(receipt.macProof!.tests.sha256).toMatch(/^[a-f0-9]{64}$/); - await expect(driver.exercise(receipt)).rejects.toThrow('fixture protected native surface reached'); + const one = newReceipt(111, sha, sha, true, root); one.installations = [installation]; + await expect(driver.exercise(one)).rejects.toThrow('fixture native surface reached'); } expect(calls.filter(c => c[0] === '/usr/bin/script')).toHaveLength(2); } } finally { await driver.cleanup(); await driver.cleanup(); Object.defineProperty(process, 'platform', platform); - if (previousCaam === undefined) delete process.env.CAAM_HOME; else process.env.CAAM_HOME = previousCaam; + for (const [name, value] of Object.entries(previous)) { + if (value === undefined) delete process.env[name]; else process.env[name] = value; + } } + expect(calls.some(c => c.join(' ') === 'caam ls --json')).toBe(true); + expect(calls.some(c => c[0] === '/usr/bin/sandbox-exec')).toBe(false); for (const harness of ['claude', 'codex']) { + const home = join(root, 'state', harness); expect((await stat(home)).isDirectory()).toBe(true); for (const tool of ['claude', 'codex'] as const) { const filename = tool === 'claude' ? '.credentials.json' : 'auth.json'; - await expect(stat(join(root, 'state', harness, '.' + tool, filename))).rejects.toThrow('ENOENT'); + await expect(stat(join(home, '.' + tool, filename))).rejects.toThrow('ENOENT'); expect(await readFile(join(vault, tool, accounts[tool], filename), 'utf8')).toBe(credentials[tool]); } + if (!failInstall) expect(await readFile(join(home, 'surface.raw'), 'utf8')).toBe('retained raw native evidence'); } expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('pass'); }); @@ -257,6 +272,5 @@ describe('isolated harness boundaries', () => { await expect(freshRoot(join(repo, 'output'), repo)).rejects.toThrow('outside'); await expect(freshRoot(root, repo)).rejects.toThrow(); expect(await freshRoot(join(root, 'output'), repo)).toBe(join(root, 'output')); - expect(redact('ghp_testsecret sk-providersecret Bearer abc')).toBe('[REDACTED] [REDACTED] [REDACTED]'); }); }); diff --git a/.claude/skills/verify-open-pstack/scripts/harness.ts b/.claude/skills/verify-open-pstack/scripts/harness.ts index 556649a8..20229733 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.ts @@ -1,12 +1,11 @@ import { mkdir, readFile, realpath, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { createInterface } from 'node:readline/promises'; -import { requiredFeatures } from './core.ts'; -import { command, interruption, isolatedEnv, redact, retainedFile, save, treeHash, type Command } from './io.ts'; +import { requiredFeatures, requiredHarnesses } from './core.ts'; +import { command, interruption, isolatedEnv, retainedFile, save, treeHash, type Command } from './io.ts'; import { doctor } from './doctor.ts'; -import { copyCredentials, createSandbox, protectSources, registerSessionCredentials, removeCredentials, sandboxed, selectedAccounts, vaultRoot } from './isolation.ts'; +import { copyCredentials, removeCredentials, vaultRoot } from './isolation.ts'; import { sourceDigest, sourceHash } from './provenance.ts'; -import { validateMacTests } from './verify.ts'; import { HARNESSES, REPO, type Accounts, type Driver, type Harness, type Installation, type Observation, type Receipt } from './types.ts'; export type Ask = (question: string) => Promise; @@ -47,34 +46,22 @@ export async function verifyProjectDoctor(texts: string[], workspace: string): P } export class MacDriver implements Driver { private copied: string[] = []; - private profiles = new Map(); - constructor(private run: Command = command, private review: Ask = ask, private accounts?: Accounts, - private boundary: typeof createSandbox = createSandbox) {} - async cleanup(): Promise { - try { - for (const home of this.profiles.keys()) await registerSessionCredentials(home); - } finally { - await removeCredentials(this.copied, this.profiles, this.run); - } - } + constructor(private run: Command = command, private review: Ask = ask, private accounts?: Accounts) {} + async cleanup(): Promise { await removeCredentials(this.copied); } private candidate(home: string): Command { - const profile = this.profiles.get(home); - if (!profile) throw new Error('Candidate sandbox not established'); - return async (args, options = {}) => { - try { return await this.run(sandboxed(profile, args), { ...options, env: isolatedEnv(home) }); } - finally { await registerSessionCredentials(home); } - }; + return (args, options = {}) => this.run(args, { ...options, env: isolatedEnv(home) }); + } + private refuseSetup(receipt: Receipt): void { + if (requiredFeatures(receipt).some(feature => feature === 'setup' || feature === 'skill-invocation:setup-pstack')) { + throw new Error('setup exercise requires #120 (setup-pstack config-home)'); + } } async prepare(receipt: Receipt): Promise { + this.refuseSetup(receipt); const root = receipt.artifactRoot; - await doctor(root, this.run); if (!this.accounts) throw new Error('Explicit Claude and Codex caam accounts are required'); - // caam pre-run may migrate state. Enforce read-only access even in this - // trusted metadata operation: listing must never activate or write a vault. - const parentEnv: Record = {}; - for (const key of ['PATH', 'HOME', 'USER', 'LOGNAME', 'CAAM_HOME', 'XDG_CONFIG_HOME', 'XDG_DATA_HOME']) if (process.env[key]) parentEnv[key] = process.env[key]!; - const readOnly = '(version 1)(allow default)(deny file-write*)(allow file-write* (subpath "/dev"))'; - const accounts = selectedAccounts(await this.run(['/usr/bin/sandbox-exec', '-p', readOnly, 'caam', 'ls', '--json'], { env: parentEnv }), this.accounts); + await doctor(root, this.run, undefined, false, this.accounts); + const accounts = this.accounts; const vault = await realpath(vaultRoot()); // Fetch the immutable reference outside candidate-readable/writable state. const referenceHome = join(root, 'source-home'), reference = join(referenceHome, 'workspace'); @@ -85,27 +72,13 @@ export class MacDriver implements Driver { await trustedRun(['git', '-c', 'core.hooksPath=/dev/null', 'checkout', '--detach', receipt.sha], { cwd: reference }); const pinnedSource = await sourceHash(reference, receipt.sha, trustedRun); if (process.platform !== 'darwin') throw new Error('Live proof requires the operator Mac'); - // Run trusted pinned tests before exposing disposable credentials to candidates. - const proofOutput = await this.run(['/bin/sh', '-c', 'bun test --timeout 0 scripts/isolation.test.ts 2>&1; result=$?; printf "\\nPSTACK_MAC_TEST_EXIT=%s\\n" "$result"'], { - cwd: join(reference, '.claude/skills/verify-open-pstack'), - env: { ...parentEnv, PSTACK_OPERATOR_SENTINELS: '1', NO_COLOR: '1', FORCE_COLOR: '0' }, - }); - const proofPath = join(root, 'mac-isolation-proof.txt'); - await writeFile(proofPath, `Pinned SHA: ${receipt.sha}\n${redact(proofOutput)}`, { flag: 'wx', mode: 0o600 }); - const tests = await retainedFile(root, proofPath); - validateMacTests(await readFile(proofPath, 'utf8'), receipt.sha); - if (await this.review('Review mac-isolation-proof.txt: both Mac tests passed with no skips and no Keychain dialog appeared. Type PASS MAC ISOLATION to confirm:') !== 'PASS MAC ISOLATION') { - throw new Error('Operator rejected Mac isolation proof'); - } - receipt.macProof = { sha: receipt.sha, platform: 'darwin', reviewer: 'operator', tests, noKeychainDialog: true, authenticated: [] }; const installs: Installation[] = []; - for (const harness of HARNESSES) { + for (const harness of requiredHarnesses(receipt)) { const home = join(root, 'state', harness), workspace = join(home, 'workspace'); await mkdir(join(home, 'tmp'), { recursive: true, mode: 0o700 }); for (const dir of ['.claude', '.codex', '.config/gh', '.cache']) await mkdir(join(home, dir), { recursive: true, mode: 0o700 }); await writeFile(join(home, 'settings.json'), '{}\n', { mode: 0o600 }); const env = isolatedEnv(home, harness); - this.profiles.set(home, await this.boundary(home, this.run, vault)); const candidateRun = this.candidate(home); await candidateRun(['git', 'clone', '--no-checkout', '--', `https://github.com/${REPO}.git`, workspace], { env }); await candidateRun(['git', 'fetch', 'origin', receipt.sha], { cwd: workspace, env }); @@ -133,21 +106,17 @@ export class MacDriver implements Driver { return installs; } async exercise(receipt: Receipt): Promise { + this.refuseSetup(receipt); const observations: Observation[] = []; - const proof = receipt.macProof; - if (!proof || proof.sha !== receipt.sha || proof.noKeychainDialog !== true) throw new Error('Required Mac isolation proof missing before native exercise'); for (const installation of receipt.installations) { const { harness, home } = installation, workspace = join(home, 'workspace'); if (await sourceDigest(workspace) !== installation.sourceHash) throw new Error('Pinned source changed before exercise'); - const profile = this.profiles.get(home); - if (!profile) throw new Error('Candidate sandbox not established'); - this.profiles.set(home, await protectSources(profile, home, [workspace, installation.location], this.run)); const env = isolatedEnv(home, harness), candidateRun = this.candidate(home); const request = { sha: receipt.sha, harness, workspace, features: requiredFeatures(receipt), featureMap: join(workspace, '.claude/skills/verify-open-pstack/features'), - selfTest: requiredFeatures(receipt).includes('project-skill') ? 'Invoke the project skill natively; run doctor --candidate --output "$HOME/self-test" within sandboxed state. Do not invoke run, vault listing, login, or publication recursively.' : false, + selfTest: requiredFeatures(receipt).includes('project-skill') ? `Invoke the candidate project skill natively; run doctor --candidate --output "${join(home, 'self-test')}". Do not invoke run, vault listing, login, or publication recursively.` : false, credentials: { claude: this.accounts?.claude, codex: this.accounts?.codex }, - isolation: 'Disposable file credentials only; no daily-home, Keychain, caam vault or publisher access. Exercise real requests; quota/auth failures fail closed.' }; + isolation: 'Real HOME/USER; both provider config roots and temporary state are run-owned. Use only selected disposable file credentials, never activate vault accounts or log in. Exercise real requests; quota/auth failures fail closed.' }; await save(join(receipt.artifactRoot, `${harness}-request.json`), request); console.log(JSON.stringify(request, null, 2)); const raw = join(home, 'surface.raw'); @@ -155,17 +124,8 @@ export class MacDriver implements Driver { await candidateRun(['/usr/bin/script', '-q', raw, ...launch(harness, home, workspace)], { cwd: workspace, env, interactive: true }); if (await treeHash(installation.location, ['skills/poteto-mode/scripts/node_modules']) !== installation.treeHash) throw new Error('Installed tree changed during exercise'); if (await sourceDigest(workspace) !== installation.sourceHash) throw new Error('Pinned project/plugin source changed during exercise'); - const transcript = await retainedFile(receipt.artifactRoot, await this.review(`Copy/redact ${raw} into output (outside state), review it, and enter the reviewed transcript path:`)); - const text = await readFile(join(receipt.artifactRoot, transcript.path), 'utf8'); - if (redact(text) !== text) throw new Error('Transcript contains recognizable credentials; redact before accepting'); - const authenticated = redact(await this.review(`${harness}: observed authenticated native API request and concrete response in this transcript (not a model self-report)?`)); - if (!authenticated.trim()) throw new Error('Authenticated native request assertion missing'); - if (await this.review(`Type PASS AUTH ${harness} only after reviewing that request and confirming no Keychain dialog appeared during this session:`) !== `PASS AUTH ${harness}`) { - proof.noKeychainDialog = false; - throw new Error(`Operator rejected ${harness} authentication/no-Keychain proof`); - } - proof.authenticated.push({ harness, observed: authenticated, transcript: transcript.path, transcriptHash: transcript.sha256 }); - for (const feature of requiredFeatures(receipt)) { + const transcript = await retainedFile(receipt.artifactRoot, await this.review(`Retain a private copy of ${raw} inside output (outside state), review it, and enter its path. Do not publish raw content:`)); + for (const feature of requiredFeatures(receipt, harness)) { const surface = await this.review(`${harness}/${feature}: native surface/discovery entry point?`); const action = await this.review('Concrete action exercised?'); const observed = await this.review('Observed assertion/result (not the model\'s success claim)?'); @@ -180,7 +140,7 @@ export class MacDriver implements Driver { if (await this.review(`Operator: type PASS ${feature} only after reviewing the native transcript and artifacts; anything else fails.`) !== `PASS ${feature}`) { throw new Error(`Operator rejected ${harness}/${feature}`); } - observations.push({ harness, feature, surface: redact(surface), action: redact(action), observed: redact(observed), + observations.push({ harness, feature, surface, action, observed, reviewer: 'operator', transcript: transcript.path, transcriptHash: transcript.sha256, artifacts }); } } diff --git a/.claude/skills/verify-open-pstack/scripts/io.test.ts b/.claude/skills/verify-open-pstack/scripts/io.test.ts index 34abc557..79e7c1af 100644 --- a/.claude/skills/verify-open-pstack/scripts/io.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/io.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, test } from 'bun:test'; import { chmod, mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { redact, registerSecrets, retainedFile, save, treeHash } from './io.ts'; +import { assertNoKnownSecrets, command, isolatedEnv, registerSecrets, retainedFile, save, sha256, treeHash } from './io.ts'; const roots: string[] = []; async function fixture(): Promise { @@ -12,50 +12,62 @@ async function fixture(): Promise { } afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); -describe('retained JSON and file boundaries', () => { - test('redacts string values before serialization without corrupting JSON punctuation or escapes', async () => { +describe('private values and retained file boundaries', () => { + test('saves canonical private JSON unchanged with private permissions', async () => { const root = await fixture(), path = join(root, 'receipt.json'); - await save(path, { path: '/tmp/Bearer token', nested: ['Bearer secret",}', 'github_pat_sensitive'], text: 'Bearer secret, next\nquoted "value"', count: 2 }); - const json = await readFile(path, 'utf8'); - expect(JSON.parse(json)).toEqual({ path: '/tmp/Bearer token', nested: ['[REDACTED]",}', '[REDACTED]'], text: '[REDACTED], next\nquoted "value"', count: 2 }); - expect(json).not.toContain('secret'); + const evidence = { workspace: '/tmp/sk-review/live', nested: ['Bearer private",}', 'github_pat_sensitive'], + checks: { explanation: 'sk-private' }, count: 2 }; + await save(path, evidence); + expect(await readFile(path, 'utf8')).toBe(JSON.stringify(evidence, null, 2) + '\n'); expect((await stat(path)).mode & 0o777).toBe(0o600); - expect(redact('Bearer token, next')).toBe('[REDACTED], next'); }); - test('registered opaque access, refresh, and ID tokens cannot survive text or structured evidence', async () => { - const tokens = ['opaque-access.+/value', 'opaque-refresh:"quoted"', 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJvcGVyYXRvciJ9.signature']; - registerSecrets(tokens); - registerSecrets(['']); - for (const token of tokens) { - expect(redact(`prefix ${token} suffix ${token}`)).toBe('prefix [REDACTED] suffix [REDACTED]'); - } - const root = await fixture(), path = join(root, 'receipt.json'); - await save(path, { text: tokens.join('\n'), nested: tokens, path: `/tmp/${tokens[0]}/live` }); - const json = await readFile(path, 'utf8'); - for (const token of tokens) expect(json).not.toContain(token); - expect(JSON.parse(json)).toEqual({ text: tokens.map(() => '[REDACTED]').join('\n'), nested: tokens.map(() => '[REDACTED]'), path: '/tmp/[REDACTED]/live' }); + test('keeps private command errors unchanged', async () => { + let failure: unknown; + try { await command(['/bin/sh', '-c', "printf '%s' 'private sk-review diagnostic' >&2; exit 7"]); } + catch (error) { failure = error; } + expect(String(failure)).toBe('Error: /bin/sh failed: private sk-review diagnostic'); }); - test('canonical structural paths retain token-like components without exempting free text', async () => { - const root = await fixture(), path = join(root, 'doctor.json'), canonical = '/tmp/sk-review/live'; - const evidence = { skill: `${canonical}/.claude/skills/verify-open-pstack`, workspace: canonical, - home: canonical, location: canonical, artifactRoot: canonical, featureMap: canonical, - transcript: `${canonical}/transcript.txt`, artifacts: [{ path: `${canonical}/artifact.txt` }], - selection: { paths: ['sk-review/SKILL.md', 'github_pat_reference.md'] }, checks: { explanation: 'sk-private' } }; - await save(path, evidence); - expect(JSON.parse(await readFile(path, 'utf8'))).toEqual({ ...evidence, checks: { explanation: '[REDACTED]' } }); + test('refuses only exact copied secrets in comments with a generic error', () => { + const token = 'opaque-copied-token.+/value'; + registerSecrets([token, '']); + expect(() => assertNoKnownSecrets(`review ${token} result`)).toThrow('Comment contains copied credentials'); + try { assertNoKnownSecrets(token); } catch (error) { expect(String(error)).not.toContain(token); } + expect(() => assertNoKnownSecrets('review opaque-copied-token result')).not.toThrow(); + expect(() => assertNoKnownSecrets('review sk-private and sk-review result')).not.toThrow(); + }); + + test('uses real account metadata, run-owned config paths, and normal PATH', () => { + const previous = { HOME: process.env.HOME, USER: process.env.USER, LOGNAME: process.env.LOGNAME, PATH: process.env.PATH }; + process.env.HOME = '/Users/operator'; process.env.USER = 'operator-user'; process.env.LOGNAME = 'operator-login'; + process.env.PATH = '/usr/local/bin:/usr/bin:/bin'; + try { + expect(() => isolatedEnv('relative/home')).toThrow('absolute run-owned'); + const env = isolatedEnv('/run/home', 'claude'); + expect(env).toMatchObject({ HOME: '/Users/operator', USER: 'operator-user', LOGNAME: 'operator-login', + PATH: '/usr/local/bin:/usr/bin:/bin', TMPDIR: '/run/home/tmp', XDG_CONFIG_HOME: '/run/home/.config', + XDG_CACHE_HOME: '/run/home/.cache', GH_CONFIG_DIR: '/run/home/.config/gh', + CLAUDE_CONFIG_DIR: '/run/home/.claude', CODEX_HOME: '/run/home/.codex' }); + expect(env.PATH).not.toContain('/run/home/bin'); + process.env.HOME = 'relative/home'; + expect(() => isolatedEnv('/run/home')).toThrow('Real HOME must be absolute'); + process.env.HOME = '/Users/operator'; delete process.env.USER; + expect(() => isolatedEnv('/run/home', 'codex')).toThrow('Real USER and LOGNAME are required'); + } finally { + for (const name of ['HOME', 'USER', 'LOGNAME', 'PATH'] as const) { + if (previous[name] === undefined) delete process.env[name]; else process.env[name] = previous[name]; + } + } }); - test.each(['text', 'binary'])('rejects and removes retained %s artifacts containing copied credentials', async kind => { - const root = await fixture(), path = join(root, 'artifact'); - const token = `known-copied-${kind}-opaque-credential`; + test('retains binary artifacts unchanged without inspecting copied secrets', async () => { + const root = await fixture(), path = join(root, 'artifact.bin'), token = 'known-copied-binary-opaque-credential'; + const bytes = Buffer.concat([Buffer.from([0, 255, 128]), Buffer.from(token), Buffer.from([0, 254])]); registerSecrets([token]); - await writeFile(path, kind === 'binary' ? Buffer.concat([Buffer.from([0, 255, 128]), Buffer.from(token), Buffer.from([0, 254])]) : `reviewed result ${token}`); - await expect(retainedFile(root, path)).rejects.toThrow('Evidence contains credentials'); - expect(await Bun.file(path).exists()).toBe(false); - await writeFile(path, kind === 'binary' ? Buffer.from([0, 255, 128, 0, 254]) : 'reviewed sanitized result'); - expect((await retainedFile(root, path)).path).toBe('artifact'); + await writeFile(path, bytes); + expect(await retainedFile(root, path)).toEqual({ path: 'artifact.bin', sha256: sha256(bytes) }); + expect(Buffer.from(await readFile(path))).toEqual(bytes); }); test('permits evidence below an ancestor named state and canonicalizes output aliases', async () => { diff --git a/.claude/skills/verify-open-pstack/scripts/io.ts b/.claude/skills/verify-open-pstack/scripts/io.ts index d0ae2da5..92716d34 100644 --- a/.claude/skills/verify-open-pstack/scripts/io.ts +++ b/.claude/skills/verify-open-pstack/scripts/io.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { lstat, mkdir, readFile, readdir, realpath, rename, unlink, writeFile } from 'node:fs/promises'; +import { lstat, mkdir, readFile, readdir, realpath, rename, writeFile } from 'node:fs/promises'; import { isAbsolute, join, relative, resolve } from 'node:path'; export type Command = (args: string[], options?: { cwd?: string; env?: Record; interactive?: boolean; allowInterrupted?: boolean }) => Promise; @@ -31,7 +31,7 @@ export const command: Command = async (args, options = {}) => { ]); const code = await child.exited; if (!options.allowInterrupted) interruption.signal.throwIfAborted(); - if (code !== 0) throw new Error(`${args[0]} failed: ${redact(error).slice(0, 1500)}`); + if (code !== 0) throw new Error(`${args[0]} failed: ${error}`); return output; } finally { activeCommands.delete(child.pid); @@ -41,34 +41,24 @@ const secrets = new Set(); export function registerSecrets(values: string[]): void { for (const value of values) if (value) secrets.add(value); } -function redactSecrets(text: string): string { - for (const secret of [...secrets].sort((a, b) => b.length - a.length)) text = text.split(secret).join('[REDACTED]'); - return text; +export function assertNoKnownSecrets(body: string): void { + if ([...secrets].some(secret => body.includes(secret))) throw new Error('Comment contains copied credentials'); } -export function redact(text: string): string { - return redactSecrets(text).replace(/(?:gh[pousr]_[\w]+|github_pat_[\w]+|sk-[\w-]+|Bearer[ \t]+[^\s"'<>()[\]{},;]+)/gi, '[REDACTED]'); -} -export function isolatedEnv(home: string, harness?: 'claude' | 'codex'): Record { - if (!isAbsolute(home)) throw new Error('Candidate HOME must be an absolute run-owned directory'); - const env: Record = { PATH: [join(home, 'bin'), ...(process.env.PATH ?? '').split(':').filter(isAbsolute)].join(':'), - HOME: home, TMPDIR: join(home, 'tmp'), TERM: process.env.TERM ?? 'xterm-256color', - XDG_CONFIG_HOME: join(home, '.config'), XDG_CACHE_HOME: join(home, '.cache'), - GH_CONFIG_DIR: join(home, '.config/gh'), +export function isolatedEnv(state: string, _harness?: 'claude' | 'codex'): Record { + if (!isAbsolute(state)) throw new Error('Candidate state must be an absolute run-owned directory'); + const { HOME, USER, LOGNAME } = process.env; + if (!HOME || !isAbsolute(HOME)) throw new Error('Real HOME must be absolute'); + if (!USER || !LOGNAME) throw new Error('Real USER and LOGNAME are required'); + return { PATH: process.env.PATH ?? '', HOME, USER, LOGNAME, + TMPDIR: join(state, 'tmp'), TERM: process.env.TERM ?? 'xterm-256color', + XDG_CONFIG_HOME: join(state, '.config'), XDG_CACHE_HOME: join(state, '.cache'), + GH_CONFIG_DIR: join(state, '.config/gh'), GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_SYSTEM: '/dev/null', GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'credential.helper', GIT_CONFIG_VALUE_0: '', - CLAUDE_CONFIG_DIR: join(home, '.claude'), CODEX_HOME: join(home, '.codex') }; - for (const name of ['USER', 'LOGNAME']) { - if (process.env[name] !== undefined) env[name] = process.env[name]; - } - return env; + CLAUDE_CONFIG_DIR: join(state, '.claude'), CODEX_HOME: join(state, '.codex') }; } export async function save(path: string, value: unknown): Promise { - const pathKeys = new Set(['path', 'paths', 'skill', 'workspace', 'featureMap', 'home', 'location', 'artifactRoot', 'transcript']); - const pathArrays = new WeakSet(); - const json = JSON.stringify(value, function (key, item) { - if (pathKeys.has(key) && Array.isArray(item)) pathArrays.add(item); - return typeof item === 'string' ? (pathKeys.has(key) || pathArrays.has(this) ? redactSecrets(item) : redact(item)) : item; - }, 2); + const json = JSON.stringify(value, null, 2); await writeFile(path + '.tmp', json + '\n', { mode: 0o600 }); await rename(path + '.tmp', path); } @@ -119,9 +109,5 @@ export async function retainedFile(root: string, path: string): Promise<{ path: const stat = await lstat(actual); if (!stat.isFile() || !stat.size) throw new Error('Evidence file must be a nonempty regular file'); const bytes = await readFile(actual); - if ([...secrets].some(secret => bytes.includes(Buffer.from(secret))) || redact(bytes.toString('latin1')) !== bytes.toString('latin1')) { - await unlink(actual); - throw new Error('Evidence contains credentials; contaminated external file removed'); - } return { path: relative(output, actual), sha256: sha256(bytes) }; } diff --git a/.claude/skills/verify-open-pstack/scripts/isolation.test.ts b/.claude/skills/verify-open-pstack/scripts/isolation.test.ts index 675b2804..78f56c2a 100644 --- a/.claude/skills/verify-open-pstack/scripts/isolation.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/isolation.test.ts @@ -1,10 +1,9 @@ import { afterEach, describe, expect, test } from 'bun:test'; -import { lstat, mkdir, mkdtemp, readFile, realpath, rm, rmdir, stat, symlink, writeFile } from 'node:fs/promises'; -import { randomUUID } from 'node:crypto'; +import { lstat, mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; -import { command, isolatedEnv, redact, retainedFile } from './io.ts'; -import { copyCredentials, createSandbox, protectSources, registerSessionCredentials, removeCredentials, resolveAppleGit, sandboxed, sandboxProfile, selectedAccounts } from './isolation.ts'; +import { assertNoKnownSecrets, isolatedEnv } from './io.ts'; +import { copyCredentials, removeCredentials, selectedAccounts, vaultRoot } from './isolation.ts'; import type { Accounts } from './types.ts'; const roots: string[] = []; @@ -16,11 +15,12 @@ async function fixture(): Promise { afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); + const accounts: Accounts = { claude: 'claude@example.com', codex: 'codex@example.com' }; -function profiles(claude = accounts.claude, codex = accounts.codex): string { +function profiles(active?: 'claude' | 'codex' | 'both'): string { return JSON.stringify({ profiles: [ - { tool: 'claude', name: accounts.claude, identity: { email: claude } }, - { tool: 'codex', name: accounts.codex, identity: { email: codex } }, + { tool: 'claude', name: accounts.claude, active: active === 'claude' || active === 'both', identity: { email: accounts.claude } }, + { tool: 'codex', name: accounts.codex, active: active === 'codex' || active === 'both', identity: { email: accounts.codex } }, ] }); } async function vault(root: string): Promise { @@ -29,448 +29,162 @@ async function vault(root: string): Promise { const path = join(base, tool, accounts[tool]); await mkdir(path, { recursive: true, mode: 0o700 }); await writeFile(join(path, tool === 'claude' ? '.credentials.json' : 'auth.json'), JSON.stringify( - tool === 'claude' ? { claudeAiOauth: { accessToken: 'selected-claude-token' } } : { tokens: { access_token: 'selected-codex-token' } }, + tool === 'claude' + ? { claudeAiOauth: { accessToken: 'selected-claude-access', refreshToken: 'selected-claude-refresh' }, account: 'claude-evidence' } + : { tokens: { access_token: 'selected-codex-access', refresh_token: 'selected-codex-refresh', id_token: 'selected-codex-id' }, account_id: 'codex-evidence' }, ), { mode: 0o600 }); } return base; } -describe('disposable credential boundary', () => { - test('both provider config roots stay in the candidate home, including external-provider children', () => { - const home = '/run/private-candidate'; - for (const tool of ['claude', 'codex'] as const) { - const env = isolatedEnv(home, tool); - expect(env.HOME).toBe(home); - expect(env.CLAUDE_CONFIG_DIR).toBe(join(home, '.claude')); - expect(env.CODEX_HOME).toBe(join(home, '.codex')); - expect(env.GH_CONFIG_DIR).toBe(join(home, '.config/gh')); - expect(env.TMPDIR).toBe(join(home, 'tmp')); - expect(env.GIT_CONFIG_VALUE_0).toBe(''); - expect(env.CAAM_HOME).toBeUndefined(); - expect(env.GH_TOKEN).toBeUndefined(); - expect(env.GITHUB_TOKEN).toBeUndefined(); - expect(env.ANTHROPIC_API_KEY).toBeUndefined(); - expect(env.OPENAI_API_KEY).toBeUndefined(); - expect(process.env.USER).toBe(env.USER); - expect(process.env.LOGNAME).toBe(env.LOGNAME); +describe('disposable credential isolation', () => { + test('vault root follows caam precedence and requires an absolute operator home', () => { + const previous = { CAAM_HOME: process.env.CAAM_HOME, XDG_DATA_HOME: process.env.XDG_DATA_HOME, HOME: process.env.HOME }; + try { + process.env.CAAM_HOME = '/operator/caam'; process.env.XDG_DATA_HOME = '/operator/data'; process.env.HOME = '/operator/home'; + expect(vaultRoot()).toBe('/operator/caam/data/vault'); + delete process.env.CAAM_HOME; + expect(vaultRoot()).toBe('/operator/data/caam/vault'); + delete process.env.XDG_DATA_HOME; + expect(vaultRoot()).toBe('/operator/home/.local/share/caam/vault'); + process.env.HOME = 'relative'; + expect(() => vaultRoot()).toThrow('absolute operator HOME'); + } finally { + for (const [name, value] of Object.entries(previous)) { + if (value === undefined) delete process.env[name]; else process.env[name] = value; + } } }); - test('explicit vault profile identity must match each selected account', () => { + + test('selected accounts match exact identities regardless of daily activity and reject mismatches', () => { expect(selectedAccounts(profiles(), accounts)).toEqual(accounts); - expect(() => selectedAccounts(profiles('other@example.com'), accounts)).toThrow(); - expect(() => selectedAccounts(profiles(accounts.claude, 'other@example.com'), accounts)).toThrow(); + expect(selectedAccounts(profiles('both'), accounts)).toEqual(accounts); + + const wrongIdentity = JSON.parse(profiles('both')); wrongIdentity.profiles[0].identity.email = 'other@example.com'; + expect(() => selectedAccounts(JSON.stringify(wrongIdentity), accounts)).toThrow(`caam claude/${accounts.claude} identity mismatch`); + const duplicate = JSON.parse(profiles()); duplicate.profiles.push(duplicate.profiles[0]); + expect(() => selectedAccounts(JSON.stringify(duplicate), accounts)).toThrow(); expect(() => selectedAccounts('{"profiles":[]}', accounts)).toThrow(); expect(() => selectedAccounts('{}', accounts)).toThrow(); expect(() => selectedAccounts('not JSON', accounts)).toThrow(); - expect(() => selectedAccounts(profiles(), { ...accounts, claude: '../escape@example.com' })).toThrow(); - const duplicate = JSON.parse(profiles()); duplicate.profiles.push(duplicate.profiles[0]); - expect(() => selectedAccounts(JSON.stringify(duplicate), accounts)).toThrow(); + expect(() => selectedAccounts(profiles(), { ...accounts, claude: '../escape@example.com' })).toThrow('safe claude account'); }); - test('real credential files are private copied files, never vault links, and cleanup preserves the vault', async () => { - const root = await fixture(), base = await vault(root), home = join(root, 'candidate'); - const copied: string[] = []; - await copyCredentials(home, base, accounts, copied); - expect(copied).toHaveLength(2); + + test('copies complete refresh-capable credentials to isolated config roots with private modes and leaves the vault unchanged', async () => { + const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; const originals = [join(base, 'claude', accounts.claude, '.credentials.json'), join(base, 'codex', accounts.codex, 'auth.json')]; - for (let i = 0; i < copied.length; i++) { - expect((await lstat(copied[i]!)).isSymbolicLink()).toBe(false); - expect((await stat(copied[i]!)).mode & 0o777).toBe(0o600); - expect((await stat(dirname(copied[i]!))).mode & 0o777).toBe(0o700); - expect(await readFile(copied[i]!, 'utf8')).toBe(await readFile(originals[i]!, 'utf8')); + const before = await Promise.all(originals.map(path => readFile(path))); + + await copyCredentials(home, base, accounts, copied); + + const env = isolatedEnv(home); + expect(copied).toEqual([join(env.CLAUDE_CONFIG_DIR!, '.credentials.json'), join(env.CODEX_HOME!, 'auth.json')]); + for (let index = 0; index < copied.length; index++) { + expect((await lstat(copied[index]!)).isSymbolicLink()).toBe(false); + expect((await stat(copied[index]!)).mode & 0o777).toBe(0o600); + expect((await stat(dirname(copied[index]!))).mode & 0o777).toBe(0o700); + expect(await readFile(copied[index]!)).toEqual(before[index]!); } - const preserved = await Promise.all(originals.map(path => readFile(path, 'utf8'))); - await writeFile(copied[0]!, '{"refreshed":"candidate-only"}'); - expect(await readFile(originals[0]!, 'utf8')).toBe(preserved[0]); - await removeCredentials(copied); - await removeCredentials(copied); - for (const path of copied) expect(await Bun.file(path).exists()).toBe(false); - expect(await Promise.all(originals.map(path => readFile(path, 'utf8')))).toEqual(preserved); + expect(JSON.parse(await readFile(copied[0]!, 'utf8')).claudeAiOauth.refreshToken).toBe('selected-claude-refresh'); + expect(JSON.parse(await readFile(copied[1]!, 'utf8')).tokens.refresh_token).toBe('selected-codex-refresh'); + expect(await Promise.all(originals.map(path => readFile(path)))).toEqual(before); }); - test('copy registers opaque OAuth access, refresh and ID tokens before candidate processing', async () => { + + test('registers only exact copied credential token fields for parent comment-body redaction', async () => { const root = await fixture(), base = await vault(root), copied: string[] = []; - const tokens = ['opaque.alpha.access.987', 'opaque-refresh-654', 'eyJhbGciOiJub25lIn0.eyJzdWIiOiJmaXh0dXJlIn0.sig', 'opaque.codex.access.321', 'opaque-codex-refresh-123']; - await writeFile(join(base, 'claude', accounts.claude, '.credentials.json'), JSON.stringify({ claudeAiOauth: { accessToken: tokens[0], refreshToken: tokens[1] } })); - await writeFile(join(base, 'codex', accounts.codex, 'auth.json'), JSON.stringify({ tokens: { id_token: tokens[2], access_token: tokens[3], refresh_token: tokens[4] } })); + const exact = ['comment-claude-access-7', 'comment-claude-refresh-7', 'comment-codex-access-7', 'comment-codex-refresh-7', 'comment-codex-id-7']; + const ordinary = ['visible-token-shaped-claude-field-7', 'visible-token-shaped-codex-field-7']; + await writeFile(join(base, 'claude', accounts.claude, '.credentials.json'), JSON.stringify({ + claudeAiOauth: { accessToken: exact[0], refreshToken: exact[1], sessionTokenHint: ordinary[0] }, + })); + await writeFile(join(base, 'codex', accounts.codex, 'auth.json'), JSON.stringify({ + tokens: { access_token: exact[2], refresh_token: exact[3], id_token: exact[4], token_hint: ordinary[1] }, + })); + await copyCredentials(join(root, 'candidate'), base, accounts, copied); - expect(redact(tokens.join(' '))).toBe(tokens.map(() => '[REDACTED]').join(' ')); - await removeCredentials(copied); + + for (const secret of exact) expect(() => assertNoKnownSecrets(secret)).toThrow('Comment contains copied credentials'); + expect(() => assertNoKnownSecrets(ordinary.join(' '))).not.toThrow(); }); - test('real provider refreshes register opaque and JWT credentials before accepting evidence or cleanup', async () => { + + test('cleanup removes only copied files while preserving candidate state, raw evidence, config directories, and vault files', async () => { const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; await copyCredentials(home, base, accounts, copied); - const original = await Promise.all(copied.map(path => readFile(path, 'utf8'))); - const tokens = ['refreshed-claude-access.opaque', 'refreshed-claude-refresh.opaque', 'refreshed-codex-access.opaque', 'refreshed-codex-refresh.opaque', 'eyJhbGciOiJSUzI1NiJ9.refresh.fixtureSignature']; - await writeFile(copied[0]!, JSON.stringify({ claudeAiOauth: { accessToken: tokens[0], refreshToken: tokens[1] } })); - await writeFile(copied[1]!, JSON.stringify({ tokens: { access_token: tokens[2], refresh_token: tokens[3], id_token: tokens[4] } })); - await registerSessionCredentials(home); - expect(redact(tokens.join(' '))).toBe(tokens.map(() => '[REDACTED]').join(' ')); - for (const [i, token] of tokens.entries()) { - const evidence = join(root, `refreshed-artifact-${i}`); - await writeFile(evidence, Buffer.concat([Buffer.from([255, 0]), Buffer.from(token), Buffer.from([128, 0])])); - await expect(retainedFile(root, evidence)).rejects.toThrow('Evidence contains credentials'); - expect(await Bun.file(evidence).exists()).toBe(false); - } + const raw = join(home, 'surface.raw'), artifact = join(home, 'reviewed-transcript.txt'), state = join(home, '.claude/session-state.json'); + await writeFile(raw, 'private raw transcript'); + await writeFile(artifact, 'private reviewed evidence'); + await writeFile(state, 'candidate session state'); + const originals = [join(base, 'claude', accounts.claude, '.credentials.json'), join(base, 'codex', accounts.codex, 'auth.json')]; + const before = await Promise.all(originals.map(path => readFile(path, 'utf8'))); + await removeCredentials(copied); - await registerSessionCredentials(home); - expect(await Bun.file(home).exists()).toBe(false); - expect(original).toEqual(await Promise.all([ - readFile(join(base, 'claude', accounts.claude, '.credentials.json'), 'utf8'), - readFile(join(base, 'codex', accounts.codex, 'auth.json'), 'utf8'), - ])); - }); - test('session credential discovery refuses redirected directories and credential files', async () => { - const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; - await copyCredentials(home, base, accounts, copied); - const external = join(root, 'external'); - await mkdir(external); - await writeFile(join(external, '.credentials.json'), '{"claudeAiOauth":{"accessToken":"outside-fixture"}}'); - await rm(join(home, '.claude'), { recursive: true }); - await symlink(external, join(home, '.claude')); - await expect(registerSessionCredentials(home)).rejects.toThrow('redirected'); - await rm(join(home, '.claude')); - await mkdir(join(home, '.claude')); - await symlink(join(external, '.credentials.json'), copied[0]!); - await expect(registerSessionCredentials(home)).rejects.toThrow('regular files'); - expect(await readFile(join(external, '.credentials.json'), 'utf8')).toContain('outside-fixture'); await removeCredentials(copied); + + for (const path of copied) expect(await Bun.file(path).exists()).toBe(false); + expect(await readFile(raw, 'utf8')).toBe('private raw transcript'); + expect(await readFile(artifact, 'utf8')).toBe('private reviewed evidence'); + expect(await readFile(state, 'utf8')).toBe('candidate session state'); + expect((await stat(join(home, '.claude'))).isDirectory()).toBe(true); + expect((await stat(join(home, '.codex'))).isDirectory()).toBe(true); + expect(await Promise.all(originals.map(path => readFile(path, 'utf8')))).toEqual(before); }); - test('cleanup removes relocated and refreshed credentials anywhere in candidate state while preserving external evidence', async () => { + + test('partial copy cleanup removes the copied file without deleting candidate evidence or changing the vault', async () => { const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; - await copyCredentials(home, base, accounts, copied); - const relocated = join(home, 'workspace/cache/relocated-oauth.json'); - await mkdir(dirname(relocated), { recursive: true }); - await writeFile(relocated, await readFile(copied[0]!)); - await writeFile(join(home, 'refreshed-token'), 'refreshed-disposable-token'); - await rm(copied[0]!); - const evidence = join(root, 'transcript.txt'); await writeFile(evidence, 'retained external evidence'); - const before = await readFile(join(base, 'claude', accounts.claude, '.credentials.json'), 'utf8'); - await removeCredentials(copied); - expect(await Bun.file(relocated).exists()).toBe(false); - expect(await Bun.file(join(home, 'refreshed-token')).exists()).toBe(false); - expect(await readFile(evidence, 'utf8')).toBe('retained external evidence'); - expect(await readFile(join(base, 'claude', accounts.claude, '.credentials.json'), 'utf8')).toBe(before); - }); - test('partial copy can be cleaned on failure without changing any vault file', async () => { - const root = await fixture(), base = await vault(root), copied: string[] = []; const bad = join(base, 'codex', accounts.codex, 'auth.json'); await writeFile(bad, '{"unknown":true}'); - await expect(copyCredentials(join(root, 'candidate'), base, accounts, copied)).rejects.toThrow('unknown format'); + await expect(copyCredentials(home, base, accounts, copied)).rejects.toThrow('unknown format'); expect(copied).toHaveLength(1); + const evidence = join(home, 'prepare-failure.txt'); await writeFile(evidence, 'retain failed-run evidence'); + await removeCredentials(copied); + expect(await Bun.file(copied[0]!).exists()).toBe(false); + expect(await readFile(evidence, 'utf8')).toBe('retain failed-run evidence'); expect(await readFile(bad, 'utf8')).toBe('{"unknown":true}'); }); - test('vault symlinks and paths escaping the vault are not copied', async () => { - const root = await fixture(), base = await vault(root), copied: string[] = []; - const credential = join(base, 'claude', accounts.claude, '.credentials.json'); - const outside = join(root, 'outside-auth'); - await writeFile(outside, '{"claudeAiOauth":{"accessToken":"other-account"}}'); + + test('vault credential files and selected account directories must be canonical regular sources', async () => { + const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; + const credential = join(base, 'claude', accounts.claude, '.credentials.json'), outside = join(root, 'outside-auth'); + await writeFile(outside, '{"claudeAiOauth":{"accessToken":"external"}}'); await rm(credential); await symlink(outside, credential); - await expect(copyCredentials(join(root, 'candidate'), base, accounts, copied)).rejects.toThrow('contained regular file'); + await expect(copyCredentials(home, base, accounts, copied)).rejects.toThrow('contained regular file'); + expect(copied).toEqual([]); + + await rm(join(base, 'claude', accounts.claude), { recursive: true }); + await symlink(join(base, 'codex', accounts.codex), join(base, 'claude', accounts.claude)); + await expect(copyCredentials(home, base, accounts, copied)).rejects.toThrow('not symlinked'); + expect(await readFile(outside, 'utf8')).toContain('external'); + }); + + test('candidate config-directory redirects fail closed without touching external sentinels', async () => { + const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), external = join(root, 'external'); + await mkdir(home); await mkdir(external); + const sentinel = join(external, '.credentials.json'); await writeFile(sentinel, 'external sentinel'); + await symlink(external, join(home, '.claude')); + const copied: string[] = []; + + await expect(copyCredentials(home, base, accounts, copied)).rejects.toThrow('redirected'); + expect(await readFile(sentinel, 'utf8')).toBe('external sentinel'); expect(copied).toEqual([]); + + await expect(removeCredentials([join(home, '.claude/.credentials.json')])).rejects.toThrow('redirected candidate credential directories'); + expect(await readFile(sentinel, 'utf8')).toBe('external sentinel'); + expect((await stat(home)).isDirectory()).toBe(true); }); - test('credential cleanup removes disposable state without following external configuration redirects', async () => { + + test('cleanup unlinks a copied-file leaf redirect without following it or removing candidate state', async () => { const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; await copyCredentials(home, base, accounts, copied); - const outside = join(root, 'external'), sentinel = join(outside, '.credentials.json'); - await mkdir(outside); await writeFile(sentinel, 'external-credential-sentinel'); - await rm(join(home, '.claude'), { recursive: true }); await symlink(outside, join(home, '.claude')); - const invoked: string[][] = []; - await removeCredentials(copied, new Map([[home, join(root, 'candidate.sb')]]), async args => { invoked.push(args); return ''; }); - expect(invoked).toEqual([['/usr/bin/sandbox-exec', '-f', join(root, 'candidate.sb'), '/bin/rm', '-rf', '--', home]]); - await expect(removeCredentials(copied, new Map())).rejects.toThrow('original candidate sandbox'); - await removeCredentials(copied); - expect(await Bun.file(join(home, '.codex/auth.json')).exists()).toBe(false); - expect(await readFile(sentinel, 'utf8')).toBe('external-credential-sentinel'); - }); - test('cleanup removes leaf symlinks without following them and copy records destinations before a failed write', async () => { - const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; - await mkdir(join(home, '.claude'), { recursive: true }); - const target = join(home, '.claude/.credentials.json'), sentinel = join(root, 'external-credential'); - await writeFile(sentinel, 'preserve'); await symlink(sentinel, target); - await expect(copyCredentials(home, base, accounts, copied)).rejects.toThrow(); - expect(copied).toEqual([target]); + const external = join(root, 'external-credential'); await writeFile(external, 'external sentinel'); + await rm(copied[0]!); await symlink(external, copied[0]!); + const state = join(home, 'state.json'); await writeFile(state, 'preserved'); + await removeCredentials(copied); - expect(await Bun.file(target).exists()).toBe(false); - expect(await readFile(sentinel, 'utf8')).toBe('preserve'); - }); - test('selected vault tool and profile directories cannot redirect to another account', async () => { - const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; - const original = join(base, 'claude', accounts.claude), other = join(base, 'claude/other@example.com'); - await mkdir(other); await writeFile(join(other, '.credentials.json'), '{"claudeAiOauth":{"accessToken":"wrong-account"}}'); - await rm(original, { recursive: true }); await symlink(other, original); - await expect(copyCredentials(home, base, accounts, copied)).rejects.toThrow('not symlinked'); - expect(copied).toEqual([]); - await rm(original); await rm(other, { recursive: true }); - const redirected = join(root, 'redirected-tool'); - await mkdir(join(redirected, accounts.claude), { recursive: true }); - await writeFile(join(redirected, accounts.claude, '.credentials.json'), '{"claudeAiOauth":{"accessToken":"wrong-account"}}'); - await rm(join(base, 'claude'), { recursive: true }); await symlink(redirected, join(base, 'claude')); - await expect(copyCredentials(home, base, accounts, copied)).rejects.toThrow('not symlinked'); - expect(copied).toEqual([]); - }); - test('the canonical vault deny overrides allowlisted files and candidate state cannot overlap it', async () => { - const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), alias = join(root, 'vault-alias'); - await mkdir(home); await symlink(base, alias); - const text = sandboxProfile(home, '/Users/operator', [join(base, 'claude', accounts.claude, '.credentials.json')], alias); - expect(text).toContain(`(deny file-read* file-write* (subpath ${JSON.stringify(base)}))`); - expect(text).not.toContain(`(subpath ${JSON.stringify(alias)})`); - await mkdir(join(base, 'nested')); - expect(() => sandboxProfile(join(base, 'nested'), '/Users/operator', [], base)).toThrow('must not overlap'); - expect(() => sandboxProfile(base, '/Users/operator', [], base)).toThrow('must not overlap'); - expect(() => sandboxProfile(root, '/Users/operator', [], base)).toThrow('must not overlap'); - }); - test('DNS permits only the macOS resolver socket after the general Unix-socket denial', () => { - const text = sandboxProfile('/private/run/candidate', '/Users/operator', []); - const exception = '(allow network-outbound (remote unix-socket (path "/private/var/run/mDNSResponder")))'; - expect(text).toContain(exception); - expect(text.indexOf(exception)).toBeGreaterThan(text.indexOf('(deny network-outbound (remote unix-socket')); - expect(text.indexOf(exception)).toBeLessThan(text.indexOf('(deny network-outbound (remote ip "localhost:*"))')); - expect(text).not.toContain('(allow network-outbound (remote unix-socket (path-regex'); - }); - test('Apple Git resolves beyond the shim and allows only its selected runtime', async () => { - const root = await fixture(), executable = join(root, 'Xcode.app/Contents/Developer/usr/bin/git'); - await mkdir(dirname(executable), { recursive: true }); await writeFile(executable, 'native-git-fixture'); - const calls: string[][] = []; - const selected = await resolveAppleGit('/usr/bin/git', async args => { calls.push(args); return executable + '\n'; }); - expect(calls).toEqual([['/usr/bin/xcrun', '--find', 'git']]); - expect(selected).toEqual({ executable, runtime: dirname(dirname(executable)) }); - const text = sandboxProfile('/private/run/candidate', '/Users/operator', [selected.executable], undefined, [selected.runtime!]); - expect(text.match(new RegExp('subpath ' + JSON.stringify(selected.runtime!).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'))).toHaveLength(2); - expect(text).not.toContain('(subpath "/Applications")'); - await expect(resolveAppleGit('/usr/bin/git', async () => '/usr/bin/git')).rejects.toThrow('beyond'); - await expect(resolveAppleGit('/usr/bin/git', async () => 'relative/git')).rejects.toThrow('absolute'); - expect(await resolveAppleGit(executable, async () => { throw new Error('Non-shim must not use xcrun'); })).toEqual({ executable }); - }); - test('root-read exception is the final Seatbelt rule after all read denials', () => { - const text = sandboxProfile('/private/run/candidate', '/Users/operator', ['/opt/homebrew/bin/bun']); - const rules = text.trim().split('\n').filter(line => !line.startsWith(';')); - expect(rules.at(-1)).toBe('(allow file-read* (literal "/"))'); - expect(text.lastIndexOf('(allow file-read* (literal "/"))')).toBeGreaterThan(text.lastIndexOf('(deny file-read*')); - expect(text.lastIndexOf('(allow file-read* (literal "/"))')).toBeGreaterThan(text.lastIndexOf('(deny network-outbound (remote ip "localhost:*"))')); - }); - test('Seatbelt contract denies daily files, Keychain IPC, security execution, and arbitrary writes', () => { - const text = sandboxProfile('/private/run/candidate', '/Users/operator', ['/opt/homebrew/bin/bun']); - expect(text).toContain('(deny file-read* file-write* (subpath "/Users/operator"))'); - expect(text).toContain('(deny file-read* (require-not (require-any'); - expect(text).toContain('(deny file-write* (require-not (require-any'); - expect(text).toContain('(deny mach-lookup (require-not (require-any'); - expect(text).not.toContain('(global-name "com.apple.securityd")'); - expect(text).not.toContain('(global-name "com.apple.SecurityServer")'); - expect(text).not.toContain('(global-name "com.apple.SecurityAgent")'); - expect(text).toContain('(deny process-exec (literal "/usr/bin/security"))'); - expect(text).toContain('(deny process-info*'); - expect(() => sandboxProfile('/Users/operator/run', '/Users/operator', [])).toThrow('outside the operator home'); - expect(() => sandboxProfile('/Users/operator', '/Users/operator', [])).toThrow(); - expect(() => sandboxProfile('relative', '/Users/operator', [])).toThrow(); - }); -}); -describe('exercise source protection', () => { - test('exercise policy stays outside candidate state with only three generated dependency exceptions', async () => { - const root = await fixture(), home = join(root, 'candidate'), workspace = join(home, 'workspace'), plugin = join(home, '.codex/plugins/pstack'); - await mkdir(workspace, { recursive: true }); await mkdir(plugin, { recursive: true }); - const profile = join(root, 'candidate.sb'); - await writeFile(profile, sandboxProfile(home, '/Users/operator', []), { mode: 0o600 }); - const original = await readFile(profile, 'utf8'); - const exercise = await protectSources(profile, home, [workspace, plugin], async () => ''); - const text = await readFile(exercise, 'utf8'); - expect(exercise.startsWith(home + '/')).toBe(false); - expect((await stat(exercise)).mode & 0o777).toBe(0o600); - expect(await readFile(profile, 'utf8')).toBe(original); - for (const path of [join(workspace, '.claude/skills/verify-open-pstack/node_modules'), join(workspace, 'plugins/pstack/skills/poteto-mode/scripts/node_modules'), join(plugin, 'skills/poteto-mode/scripts/node_modules')]) { - expect((await stat(path)).isDirectory()).toBe(true); - expect(text).toContain(`(subpath ${JSON.stringify(path)})`); - } - expect(text).toContain(`(deny file-write* (require-all (subpath ${JSON.stringify(workspace)})`); - expect(text).toContain(`(deny file-write* (require-all (subpath ${JSON.stringify(plugin)})`); - expect(text).toContain(`(literal ${JSON.stringify(join(home, '.codex'))})`); - await expect(protectSources(profile, home, [workspace], async () => '')).rejects.toThrow(); - }); - test('root, ancestor, dependency redirects, outside sources, and writable policies fail closed', async () => { - const root = await fixture(), home = join(root, 'candidate'), outside = join(root, 'outside'); - await mkdir(home); await mkdir(outside); - const profile = join(root, 'candidate.sb'); await writeFile(profile, 'trusted-policy'); - const alias = join(home, 'alias'); await symlink(outside, alias); - await expect(protectSources(profile, home, [alias], async () => '')).rejects.toThrow('Redirected source'); - await mkdir(join(outside, 'child')); await expect(protectSources(profile, home, [join(alias, 'child')], async () => '')).rejects.toThrow('Redirected source'); - await expect(protectSources(profile, home, [outside], async () => '')).rejects.toThrow('contained'); - await expect(protectSources(profile, home, [], async () => '')).rejects.toThrow('requires pinned'); - const workspace = join(home, 'workspace'); await mkdir(workspace); - await symlink(outside, join(workspace, '.claude')); - await expect(protectSources(profile, home, [workspace], async () => '')).rejects.toThrow('Redirected source'); - const writablePolicy = join(home, 'policy.sb'); await writeFile(writablePolicy, 'candidate-policy'); - await expect(protectSources(writablePolicy, home, [workspace], async () => '')).rejects.toThrow('outside candidate'); - const policyAlias = join(root, 'alias.sb'); await symlink(profile, policyAlias); - await expect(protectSources(policyAlias, home, [workspace], async () => '')).rejects.toThrow('regular trusted'); - expect(await readFile(profile, 'utf8')).toBe('trusted-policy'); + expect(await Bun.file(copied[0]!).exists()).toBe(false); + expect(await readFile(external, 'utf8')).toBe('external sentinel'); + expect(await readFile(state, 'utf8')).toBe('preserved'); }); }); - -// This is intentionally not a mocked-command test. Linux cannot prove Seatbelt -// enforcement; the operator must run this exact suite on the Mac before live proof. -const macTest = process.platform === 'darwin' ? test : test.skip; -macTest('real Mac sandbox denies daily-home and vault sentinels, writes, aliases, grandchildren, and Keychain IPC', async () => { - const root = await fixture(), daily = join(root, 'daily'), home = join(root, 'candidate'); - await mkdir(join(home, 'tmp'), { recursive: true, mode: 0o700 }); - const locations = [ - join(daily, 'home-sentinel'), join(daily, '.config/gh/hosts.yml'), join(daily, '.ssh/id_ed25519'), - join(daily, '.claude/.credentials.json'), join(daily, '.codex/auth.json'), - join(daily, 'Library/Keychains/login.keychain-db'), join(root, 'other-account-vault/auth.json'), - ]; - for (const path of locations) { - await mkdir(dirname(path), { recursive: true, mode: 0o700 }); - await writeFile(path, 'operator-sentinel-must-remain-private', { mode: 0o600 }); - } - const previousHome = process.env.HOME; - let profile: string; - const deniedVault = join(root, 'other-account-vault'); - try { process.env.HOME = daily; profile = await createSandbox(home, command, deniedVault); } - finally { - if (previousHome === undefined) delete process.env.HOME; else process.env.HOME = previousHome; - } - const options = { cwd: home, env: isolatedEnv(home) }; - expect(await command(sandboxed(profile, ['git', '--version']), options)).toContain('git version'); - const allowed = join(home, 'candidate-sentinel'); - await writeFile(allowed, 'candidate-state-allowed'); - expect(await command(sandboxed(profile, ['/bin/cat', allowed]), options)).toBe('candidate-state-allowed'); - expect(profile.startsWith(home + '/')).toBe(false); - expect((await stat(profile)).mode & 0o777).toBe(0o600); - const originalPolicy = await readFile(profile, 'utf8'); - expect(originalPolicy).toContain(`(deny file-read* file-write* (subpath ${JSON.stringify(deniedVault)}))`); - await expect(command(sandboxed(profile, ['/bin/sh', '-c', 'printf replacement > "$1"', 'policy', profile]), options)).rejects.toThrow(); - expect(await readFile(profile, 'utf8')).toBe(originalPolicy); - // This deliberately allowlists a vault file, then proves the final explicit - // vault denial overrides that allowance rather than relying on deny-default. - const overrideProfile = join(root, 'override.sb'), vaultSentinel = join(deniedVault, 'auth.json'); - await writeFile(overrideProfile, sandboxProfile(home, daily, [vaultSentinel], deniedVault), { mode: 0o600 }); - await expect(command(sandboxed(overrideProfile, ['/bin/cat', vaultSentinel]), options)).rejects.toThrow(); - await expect(command(sandboxed(overrideProfile, ['/bin/sh', '-c', 'printf overwritten > "$1"', 'vault', vaultSentinel]), options)).rejects.toThrow(); - expect(await readFile(vaultSentinel, 'utf8')).toBe('operator-sentinel-must-remain-private'); - // Production cleanup inherits the same policy even if candidates replace - // a configuration parent with a symlink to a vault credential directory. - const redirectedCredential = join(home, '.claude/.credentials.json'); - await writeFile(join(deniedVault, '.credentials.json'), 'preserve-external-credential'); - await symlink(deniedVault, join(home, '.claude')); - // Full-state cleanup runs after all probes below; it must not follow this - // redirect or leave tokens relocated outside their original config path. - await writeFile(join(home, 'relocated-token'), 'disposable-token'); - expect(await readFile(join(deniedVault, '.credentials.json'), 'utf8')).toBe('preserve-external-credential'); - const proof = { platform: process.platform, denied: [] as string[], keychainServices: {} as Record }; - for (const [i, path] of locations.entries()) { - await expect(command(sandboxed(profile, ['/bin/cat', path]), options)).rejects.toThrow('failed'); - const alias = join(home, `alias-${i}`); await symlink(path, alias); - await expect(command(sandboxed(profile, ['/bin/cat', alias]), options)).rejects.toThrow('failed'); - // A shell's child inherits the OS policy even without a provider-specific env. - await expect(command(sandboxed(profile, ['/bin/sh', '-c', 'exec /bin/cat "$1"', 'child', path]), options)).rejects.toThrow('failed'); - await expect(command(sandboxed(profile, ['/bin/sh', '-c', 'printf overwritten > "$1"', 'setup', path]), options)).rejects.toThrow('failed'); - expect(await readFile(path, 'utf8')).toBe('operator-sentinel-must-remain-private'); - proof.denied.push(path); - } - await expect(command(sandboxed(profile, ['/usr/bin/security', 'list-keychains']), options)).rejects.toThrow(); - // Exercise actual Mach lookup, not just a security CLI deny rule. Compilation - // happens in the trusted test parent; candidate execution inherits Seatbelt. - const source = join(home, 'keychain-probe.c'), binary = join(home, 'keychain-probe'); - await writeFile(source, '#include \n#include \nint main(int n,char **v){mach_port_t p=0; kern_return_t r=bootstrap_look_up(bootstrap_port,v[1],&p); printf("%d\\n",r); return 0;}\n'); - await command(['/usr/bin/clang', source, '-o', binary], { cwd: home }); - let reachable = 0; - for (const service of ['com.apple.securityd.xpc', 'com.apple.securityd', 'com.apple.SecurityServer', 'com.apple.SecurityAgent']) { - // A missing service is not denial proof. Look up the exact same service - // first from the unsandboxed trusted parent without accessing any Keychain. - const unsandboxed = Number((await command([binary, service], { cwd: home })).trim()); - expect(Number.isFinite(unsandboxed)).toBe(true); - if (unsandboxed === 0) reachable++; - const result = Number((await command(sandboxed(profile, [binary, service]), options)).trim()); - expect(Number.isFinite(result)).toBe(true); expect(result).not.toBe(0); - proof.keychainServices[service] = { unsandboxed, sandboxed: result }; - } - expect(reachable).toBeGreaterThan(0); - await removeCredentials([redirectedCredential], new Map([[home, profile]])); - expect(await Bun.file(join(home, 'relocated-token')).exists()).toBe(false); - expect(await readFile(join(deniedVault, '.credentials.json'), 'utf8')).toBe('preserve-external-credential'); - console.log('Mac filesystem/Keychain denial proof:', JSON.stringify(proof)); -}, 0); - -// Explicit opt-in is required before planting new, randomized non-secret files -// in the operator's real HOME. Existing auth/SSH/Keychain files are never touched. -const operatorMacTest = process.platform === 'darwin' && process.env.PSTACK_OPERATOR_SENTINELS === '1' ? test : test.skip; -operatorMacTest('actual operator HOME denial and immutable exercise sources with writable dependency leaves', async () => { - const daily = await realpath(process.env.HOME!), root = await fixture(), home = join(root, 'candidate'); - const directories = [daily, join(daily, '.config/gh'), join(daily, '.ssh'), join(daily, '.claude'), join(daily, '.codex'), join(daily, 'Library/Keychains')]; - const createdDirectories: string[] = [], sentinels: string[] = [], proof: Record = { platform: process.platform, daily, denied: [] }; - async function ensureDirectory(path: string): Promise { - try { - const info = await lstat(path); - if (!info.isDirectory() || info.isSymbolicLink() || await realpath(path) !== path) throw new Error(`Refuse redirected daily sentinel directory: ${path}`); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - await ensureDirectory(dirname(path)); - await mkdir(path, { mode: 0o700 }); createdDirectories.push(path); - } - } - try { - for (const directory of directories) { - await ensureDirectory(directory); - const path = join(directory, `.pstack-denial-${randomUUID()}`); - await writeFile(path, 'non-secret-operator-denial-sentinel', { flag: 'wx', mode: 0o600 }); sentinels.push(path); - } - console.log('Operator sentinel cleanup paths:', JSON.stringify(sentinels)); - await mkdir(join(home, 'tmp'), { recursive: true, mode: 0o700 }); - const workspace = join(home, 'workspace'), plugin = join(home, '.codex/plugins/pstack'); - await mkdir(workspace); await mkdir(plugin, { recursive: true }); - const source = join(workspace, 'source.ts'), installed = join(plugin, 'SKILL.md'); - await writeFile(source, 'pinned-workspace-source'); await writeFile(installed, 'pinned-plugin-source'); - const profile = await createSandbox(home), exercise = await protectSources(profile, home, [workspace, plugin]); - const options = { cwd: home, env: isolatedEnv(home) }; - for (const [index, path] of sentinels.entries()) { - // Positive controls prove the file exists and is readable outside Seatbelt. - expect(await command(['/bin/cat', path])).toBe('non-secret-operator-denial-sentinel'); - await expect(command(sandboxed(exercise, ['/bin/cat', path]), options)).rejects.toThrow(); - const alias = join(home, `daily-alias-${index}`); await symlink(path, alias); - await expect(command(sandboxed(exercise, ['/bin/cat', alias]), options)).rejects.toThrow(); - await expect(command(sandboxed(exercise, ['/bin/sh', '-c', 'exec /bin/sh -c \'exec /bin/cat "$1"\' child "$1"', 'grandchild', path]), options)).rejects.toThrow(); - await expect(command(sandboxed(exercise, ['/bin/sh', '-c', 'printf overwritten > "$1"', 'write', path]), options)).rejects.toThrow(); - expect(await readFile(path, 'utf8')).toBe('non-secret-operator-denial-sentinel'); - } - proof.denied = [...sentinels]; - for (const path of [source, installed]) { - const original = await readFile(path, 'utf8'); - expect(await command(sandboxed(exercise, ['/bin/cat', path]), options)).toBe(original); - await expect(command(sandboxed(exercise, ['/bin/sh', '-c', 'printf altered > "$1"', 'write', path]), options)).rejects.toThrow(); - await expect(command(sandboxed(exercise, ['/bin/mv', path, join(home, 'relocated-source')]), options)).rejects.toThrow(); - await expect(command(sandboxed(exercise, ['/bin/rm', path]), options)).rejects.toThrow(); - await expect(command(sandboxed(exercise, ['/bin/ln', path, join(home, 'source-hardlink')]), options)).rejects.toThrow(); - const alias = join(home, `source-alias-${randomUUID()}`); await symlink(path, alias); - await expect(command(sandboxed(exercise, ['/bin/sh', '-c', 'printf altered > "$1"', 'write', alias]), options)).rejects.toThrow(); - expect(await readFile(path, 'utf8')).toBe(original); - } - for (const path of [workspace, dirname(plugin), plugin]) { - await expect(command(sandboxed(exercise, ['/bin/mv', path, join(home, `moved-${randomUUID()}`)]), options)).rejects.toThrow(); - } - for (const path of [join(workspace, '.claude/skills/verify-open-pstack/node_modules'), join(workspace, 'plugins/pstack/skills/poteto-mode/scripts/node_modules'), join(plugin, 'skills/poteto-mode/scripts/node_modules')]) { - const dependency = join(path, 'generated'); - await command(sandboxed(exercise, ['/bin/sh', '-c', 'printf generated > "$1"', 'bootstrap', dependency]), options); - expect(await readFile(dependency, 'utf8')).toBe('generated'); - } - await expect(command(sandboxed(exercise, ['/bin/sh', '-c', 'printf changed > "$1"', 'policy', exercise]), options)).rejects.toThrow(); - await expect(command(sandboxed(exercise, ['/usr/bin/security', 'list-keychains']), options)).rejects.toThrow(); - const probeSource = join(home, 'keychain-probe.c'), binary = join(home, 'keychain-probe'); - await writeFile(probeSource, '#include \n#include \nint main(int n,char **v){mach_port_t p=0; kern_return_t r=bootstrap_look_up(bootstrap_port,v[1],&p); printf("%d\\n",r); return 0;}\n'); - await command(['/usr/bin/clang', probeSource, '-o', binary], { cwd: home }); - const services: Record = {}; let reachable = 0; - for (const service of ['com.apple.securityd.xpc', 'com.apple.securityd', 'com.apple.SecurityServer', 'com.apple.SecurityAgent']) { - const unsandboxed = Number((await command([binary, service], { cwd: home })).trim()); - const result = Number((await command(sandboxed(exercise, [binary, service]), options)).trim()); - expect(Number.isFinite(unsandboxed)).toBe(true); expect(Number.isFinite(result)).toBe(true); expect(result).not.toBe(0); - if (unsandboxed === 0) reachable++; - services[service] = { unsandboxed, sandboxed: result }; - } - expect(reachable).toBeGreaterThan(0); proof.keychainServices = services; - proof.sourcesProtected = [workspace, plugin]; - } finally { - for (const path of sentinels) await rm(path); - // Remove only directories created by this test, and only if still empty. - for (const path of createdDirectories.reverse()) await rmdir(path); - } - console.log('Actual operator HOME/source denial proof:', JSON.stringify({ ...proof, sentinelsRemoved: true })); -}, 0); diff --git a/.claude/skills/verify-open-pstack/scripts/isolation.ts b/.claude/skills/verify-open-pstack/scripts/isolation.ts index 0f75ed45..ab97742f 100644 --- a/.claude/skills/verify-open-pstack/scripts/isolation.ts +++ b/.claude/skills/verify-open-pstack/scripts/isolation.ts @@ -1,8 +1,7 @@ -import { realpathSync } from 'node:fs'; -import { chmod, copyFile, lstat, mkdir, readFile, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; -import { basename, dirname, isAbsolute, join, resolve } from 'node:path'; -import { command, isolatedEnv, registerSecrets, type Command } from './io.ts'; -import { HARNESSES, type Accounts } from './types.ts'; +import { chmod, lstat, mkdir, readFile, realpath, unlink, writeFile } from 'node:fs/promises'; +import { isAbsolute, join, resolve } from 'node:path'; +import { isolatedEnv, registerSecrets } from './io.ts'; +import { HARNESSES, type Accounts, type Harness } from './types.ts'; export function vaultRoot(): string { if (process.env.CAAM_HOME) return join(process.env.CAAM_HOME, 'data/vault'); @@ -10,244 +9,101 @@ export function vaultRoot(): string { if (!process.env.HOME || !isAbsolute(process.env.HOME)) throw new Error('Trusted parent requires an absolute operator HOME'); return join(process.env.HOME, '.local/share/caam/vault'); } + export function selectedAccounts(text: string, accounts: Accounts): Accounts { const result = JSON.parse(text); if (!Array.isArray(result.profiles)) throw new Error('Unknown caam list schema; expected profiles'); for (const tool of HARNESSES) { const name = accounts[tool]; if (!/^[A-Za-z0-9.!#$%&'*+=?^_`{|}~-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/.test(name) || name.length > 254) throw new Error(`Explicit safe ${tool} account email required`); - const matching = result.profiles.filter((p: Record) => p.tool === tool && p.name === name); - if (matching.length !== 1 || matching[0].identity?.email !== name) throw new Error(`caam ${tool}/${name} identity mismatch or missing; inspect caam list --json (never activate or repair automatically)`); + const matching = result.profiles.filter((profile: Record) => profile.tool === tool && profile.name === name); + if (matching.length !== 1 || (matching[0].identity as Record | undefined)?.email !== name) { + throw new Error(`caam ${tool}/${name} identity mismatch or missing; inspect caam list --json (never activate or repair automatically)`); + } } return accounts; } + +function credentialLocation(home: string, tool: Harness): { directory: string; path: string } { + const env = isolatedEnv(home); + const directory = tool === 'claude' ? env.CLAUDE_CONFIG_DIR : env.CODEX_HOME; + if (!directory) throw new Error(`Isolated ${tool} configuration directory is missing`); + return { directory, path: join(directory, tool === 'claude' ? '.credentials.json' : 'auth.json') }; +} + +async function canonicalDirectory(path: string, message: string): Promise { + const info = await lstat(path); + if (!info.isDirectory() || info.isSymbolicLink() || await realpath(path) !== path) throw new Error(message); +} + +function credentialTokens(tool: Harness, auth: Record): string[] { + const fields = (tool === 'claude' ? auth.claudeAiOauth : auth.tokens) as Record | undefined; + const access = tool === 'claude' ? fields?.accessToken : fields?.access_token; + if (typeof access !== 'string') throw new Error(`${tool} vault requires file-backed OAuth credentials; unknown format blocks`); + const names = tool === 'claude' ? ['accessToken', 'refreshToken'] : ['access_token', 'refresh_token', 'id_token']; + return names.flatMap(name => typeof fields?.[name] === 'string' ? [fields[name] as string] : []); +} + export async function copyCredentials(home: string, vault: string, accounts: Accounts, copied: string[]): Promise { + if (!isAbsolute(home) || resolve(home) !== home) throw new Error('Credential copy requires canonical candidate state'); const base = await realpath(vault); + await mkdir(home, { recursive: true, mode: 0o700 }); + await canonicalDirectory(home, 'Credential copy refuses redirected candidate state'); + for (const tool of HARNESSES) { const file = tool === 'claude' ? '.credentials.json' : 'auth.json'; const profile = join(base, tool, accounts[tool]), source = join(profile, file); - for (const dir of [join(base, tool), profile]) { - const info = await lstat(dir); - if (!dir.startsWith(base + '/') || !info.isDirectory() || info.isSymbolicLink() || await realpath(dir) !== dir) throw new Error('Selected vault directories must be canonical, not symlinked'); + for (const directory of [join(base, tool), profile]) { + const info = await lstat(directory); + if (!directory.startsWith(base + '/') || !info.isDirectory() || info.isSymbolicLink() || await realpath(directory) !== directory) { + throw new Error('Selected vault directories must be canonical, not symlinked'); + } } - const stat = await lstat(source); - if (!stat.isFile() || stat.isSymbolicLink() || await realpath(source) !== source) throw new Error('Vault credentials must be a contained regular file'); + const sourceInfo = await lstat(source); + if (!sourceInfo.isFile() || sourceInfo.isSymbolicLink() || await realpath(source) !== source) throw new Error('Vault credentials must be a contained regular file'); const bytes = await readFile(source); - const auth = JSON.parse(bytes.toString()); - if (tool === 'claude' ? typeof auth.claudeAiOauth?.accessToken !== 'string' : typeof auth.tokens?.access_token !== 'string') throw new Error(`${tool} vault requires file-backed OAuth credentials; unknown format blocks`); - const tokens = tool === 'claude' ? auth.claudeAiOauth : auth.tokens; - registerSecrets(Object.entries(tokens).flatMap(([key, value]) => /token/i.test(key) && typeof value === 'string' ? [value] : [])); - const dir = join(home, tool === 'claude' ? '.claude' : '.codex'); - await mkdir(dir, { recursive: true, mode: 0o700 }); - const target = join(dir, file); - copied.push(target); - await writeFile(target, bytes, { mode: 0o600, flag: 'wx' }); + const auth = JSON.parse(bytes.toString()) as Record; + registerSecrets(credentialTokens(tool, auth)); + + const target = credentialLocation(home, tool); + await mkdir(target.directory, { recursive: true, mode: 0o700 }); + await canonicalDirectory(target.directory, 'Candidate credential directory redirected'); + await chmod(target.directory, 0o700); + copied.push(target.path); + await writeFile(target.path, bytes, { mode: 0o600, flag: 'wx' }); } } -export async function registerSessionCredentials(home: string): Promise { - const root = await realpath(home).catch((error: NodeJS.ErrnoException) => { - if (error.code !== 'ENOENT') throw error; - return undefined; - }); - if (root === undefined) return; - if (root !== home) throw new Error('Session credentials require canonical candidate state'); - for (const tool of HARNESSES) { - const dir = join(home, tool === 'claude' ? '.claude' : '.codex'); - if (await realpath(dir) !== dir) throw new Error('Session credential directory redirected'); - const file = join(dir, tool === 'claude' ? '.credentials.json' : 'auth.json'); - const info = await lstat(file).catch((error: NodeJS.ErrnoException) => { + +export async function removeCredentials(paths: string[]): Promise { + for (const path of new Set(paths)) { + if (!isAbsolute(path) || resolve(path) !== path) throw new Error('Credential cleanup requires canonical copied paths'); + const home = resolve(path, '../..'); + const expected = HARNESSES.map(tool => credentialLocation(home, tool).path); + if (!expected.includes(path)) throw new Error('Credential cleanup only removes copied provider credential files'); + + const homeInfo = await lstat(home).catch((error: NodeJS.ErrnoException) => { if (error.code !== 'ENOENT') throw error; return undefined; }); - if (!info) continue; - if (!info.isFile() || info.isSymbolicLink()) throw new Error('Session credentials must be regular files'); - const auth = JSON.parse(await readFile(file, 'utf8')); - const tokens = tool === 'claude' ? auth.claudeAiOauth : auth.tokens; - if (!tokens || typeof tokens !== 'object') throw new Error('Unknown session OAuth credential format'); - registerSecrets(Object.entries(tokens).flatMap(([key, value]) => /token/i.test(key) && typeof value === 'string' ? [value] : [])); - } -} -export async function removeCredentials(paths: string[], profiles?: Map, run: Command = command): Promise { - for (const home of new Set(paths.map(path => dirname(dirname(path))))) { - await sourceDirectory(dirname(home)); - if (!isAbsolute(home) || resolve(home) !== home) throw new Error('Credential cleanup requires canonical candidate state'); - const info = await lstat(home).catch((error: NodeJS.ErrnoException) => { + if (!homeInfo) continue; + if (!homeInfo.isDirectory() || homeInfo.isSymbolicLink() || await realpath(home) !== home) throw new Error('Credential cleanup refuses redirected candidate state'); + + const directory = resolve(path, '..'); + const directoryInfo = await lstat(directory).catch((error: NodeJS.ErrnoException) => { if (error.code !== 'ENOENT') throw error; return undefined; }); - if (!info) continue; - if (profiles) { - const profile = profiles.get(home); - if (!profile) throw new Error('Credential cleanup requires the original candidate sandbox'); - // Remove all disposable state, including renamed or refreshed tokens. - // Exercise-only write protection must not prevent contained cleanup. - const cleanupProfile = profile.endsWith('.exercise.sb') ? profile.slice(0, -'.exercise.sb'.length) : profile; - await run(sandboxed(cleanupProfile, ['/bin/rm', '-rf', '--', home]), { env: isolatedEnv(home), cwd: dirname(home), allowInterrupted: true }); - } else { - // Trusted fixtures only; production deletion remains OS-contained. - if (info.isSymbolicLink() || !info.isDirectory()) throw new Error('Credential cleanup refuses redirected candidate state'); - await rm(home, { recursive: true, force: true }); + if (!directoryInfo) continue; + if (!directoryInfo.isDirectory() || directoryInfo.isSymbolicLink() || await realpath(directory) !== directory) { + throw new Error('Credential cleanup refuses redirected candidate credential directories'); } + + const info = await lstat(path).catch((error: NodeJS.ErrnoException) => { + if (error.code !== 'ENOENT') throw error; + return undefined; + }); + if (!info) continue; + if (!info.isFile() && !info.isSymbolicLink()) throw new Error('Credential cleanup only removes regular credential files or their leaf links'); + await unlink(path); } } -const quote = (value: string): string => JSON.stringify(value); -export function sandboxProfile(home: string, daily: string, executables: string[], deniedVaultRoot?: string, runtimes: string[] = []): string { - if (!isAbsolute(home) || !isAbsolute(daily) || home === daily || home.startsWith(daily + '/')) throw new Error('Sandbox state must be outside the operator home'); - const vault = deniedVaultRoot === undefined ? undefined : realpathSync(deniedVaultRoot); - if (vault) { - const candidate = realpathSync(home); - if (candidate === vault || candidate.startsWith(vault + '/') || vault.startsWith(candidate + '/')) throw new Error('Candidate state and caam vault must not overlap'); - } - // File reads are allowlisted, not merely denied through HOME aliases. Mach - // services are allowlisted too: no securityd, SecurityAgent, or Keychain IPC. - return `(version 1) -(deny default) -(allow process-exec process-fork) -(allow pseudo-tty) -(allow file-ioctl (literal "/dev/ptmx") (regex #"^/dev/ttys")) -(allow signal (target same-sandbox)) -(allow process-info* (target same-sandbox)) -(allow sysctl-read) -(allow file-read-metadata) -(allow file-read* (subpath ${quote(home)}) - (subpath "/System") (subpath "/usr") (subpath "/bin") (subpath "/sbin") - (subpath "/opt/homebrew") (subpath "/Library/Apple") (subpath "/dev") - (subpath "/private/etc") (subpath "/private/var/db/dyld") - ${executables.map(path => `(literal ${quote(path)})`).join('\n ')} - ${runtimes.map(path => `(subpath ${quote(path)})`).join('\n ')}) -(allow file-write* (subpath ${quote(home)}) (subpath "/dev")) -(allow network-outbound (remote ip "*:*")) -(allow mach-lookup - (global-name "com.apple.trustd") (global-name "com.apple.trustd.agent") - (global-name "com.apple.mDNSResponder") (global-name "com.apple.system.notification_center") - (global-name "com.apple.SystemConfiguration.configd") (global-name "com.apple.bsd.dirhelper")) -(deny file-read* (require-not (require-any - (subpath ${quote(home)}) - (subpath "/System") (subpath "/usr") (subpath "/bin") (subpath "/sbin") - (subpath "/opt/homebrew") (subpath "/Library/Apple") (subpath "/dev") - (subpath "/private/etc") (subpath "/private/var/db/dyld") - ${executables.map(path => `(literal ${quote(path)})`).join('\n ')} - ${runtimes.map(path => `(subpath ${quote(path)})`).join('\n ')}))) -(deny file-read* file-write* (subpath ${quote(daily)})) -${vault ? `(deny file-read* file-write* (subpath ${quote(vault)}))` : ''} -(deny file-read* file-write* (subpath "/Library/Keychains") (subpath "/System/Library/Keychains")) -(deny file-write* (require-not (require-any (subpath ${quote(home)}) (subpath "/dev")))) -(deny mach-lookup (require-not (require-any - (global-name "com.apple.trustd") (global-name "com.apple.trustd.agent") - (global-name "com.apple.mDNSResponder") (global-name "com.apple.system.notification_center") - (global-name "com.apple.SystemConfiguration.configd") (global-name "com.apple.bsd.dirhelper")))) -(deny process-info* (require-not (target same-sandbox))) -(deny process-exec (literal "/usr/bin/security")) -(deny network-outbound (remote unix-socket (path-regex #".*"))) -(allow network-outbound (remote unix-socket (path "/private/var/run/mDNSResponder"))) -(deny network-outbound (remote ip "localhost:*")) -; Keep this rule last: Seatbelt's later read denials otherwise override root traversal. -(allow file-read* (literal "/")) -`; -} -export async function resolveAppleGit(path: string, run: Command = command): Promise<{ executable: string; runtime?: string }> { - const actual = await realpath(path); - if (actual !== '/usr/bin/git') return { executable: actual }; - const selected = (await run(['/usr/bin/xcrun', '--find', 'git'])).trim(); - if (!isAbsolute(selected) || basename(selected) !== 'git') throw new Error('Apple Git selection must be an absolute Git executable'); - const executable = await realpath(selected), runtime = dirname(dirname(executable)); - if (executable === actual || basename(dirname(executable)) !== 'bin' || basename(runtime) !== 'usr' || !(await stat(executable)).isFile()) throw new Error('Apple Git must resolve beyond its executable shim'); - return { executable, runtime }; -} -export async function createSandbox(home: string, run: Command = command, deniedVaultRoot?: string): Promise { - const daily = await realpath(process.env.HOME!); - const vault = deniedVaultRoot === undefined ? undefined : await realpath(deniedVaultRoot); - const candidate = await realpath(home); - // Validate boundaries before copying executables or writing the profile. - sandboxProfile(candidate, daily, [], vault); - const executables: string[] = [], runtimes: string[] = []; - for (const name of ['claude', 'codex', 'bun', 'node', 'git']) { - const path = Bun.which(name); - if (!path && ['claude', 'codex', 'bun'].includes(name)) throw new Error(`Missing ${name}`); - if (path) { - const selected = name === 'git' ? await resolveAppleGit(path, run) : { executable: await realpath(path), runtime: undefined }; - const actual = selected.executable; - if (selected.runtime) runtimes.push(selected.runtime); - if (actual.startsWith(daily + '/') || path.startsWith(daily + '/')) { - // Only standalone native binaries can be relocated without dependencies - // on daily state. Never grant a home-directory read exception. - const magic = (await readFile(actual)).subarray(0, 4).toString('hex'); - if (!['cffaedfe', 'cefaedfe', 'feedfacf', 'feedface', 'cafebabe', 'bebafeca'].includes(magic)) throw new Error(`Install standalone ${name} outside daily HOME; cannot safely relocate a script package`); - await mkdir(join(home, 'bin'), { recursive: true, mode: 0o700 }); - const target = join(home, 'bin', name); - await copyFile(actual, target); - await chmod(target, (await stat(actual)).mode & 0o777); - executables.push(target); - } else { - executables.push(actual); - if (selected.runtime) { - await mkdir(join(home, 'bin'), { recursive: true, mode: 0o700 }); - await symlink(actual, join(home, 'bin/git')); - } - } - } - } - // The policy stays outside candidate-writable state; later subprocesses must - // not load a profile that the candidate can replace between invocations. - const path = join(dirname(home), `${basename(home)}.sb`); - await writeFile(path, sandboxProfile(candidate, daily, executables, vault, runtimes), { mode: 0o600, flag: 'wx' }); - await run(['/usr/bin/sandbox-exec', '-f', path, '/usr/bin/true'], { env: isolatedEnv(home), cwd: home }); - return path; -} -async function sourceDirectory(path: string): Promise { - if (!isAbsolute(path) || resolve(path) !== path) throw new Error('Source paths must be canonical absolute directories'); - let current = path; - for (;;) { - const info = await lstat(current); - if (info.isSymbolicLink() || !info.isDirectory()) throw new Error(`Redirected source directory: ${current}`); - if (dirname(current) === current) break; - current = dirname(current); - } - if (await realpath(path) !== path) throw new Error(`Redirected source directory: ${path}`); -} -export async function protectSources(profile: string, home: string, sources: string[], run: Command = command): Promise { - await sourceDirectory(home); - if (sources.length === 0) throw new Error('Exercise requires pinned source roots'); - await sourceDirectory(dirname(profile)); - if (!isAbsolute(profile) || resolve(profile) !== profile || profile === home || profile.startsWith(home + '/')) throw new Error('Exercise policy must be outside candidate state'); - const info = await lstat(profile); - if (!info.isFile() || info.isSymbolicLink()) throw new Error('Exercise policy must be a regular trusted file'); - const rules: string[] = []; - for (const source of new Set(sources)) { - await sourceDirectory(source); - if (!source.startsWith(home + '/')) throw new Error('Sources must be contained in candidate state'); - const workspace = source === join(home, 'workspace'); - const dependencies = workspace - ? [join(source, '.claude/skills/verify-open-pstack/node_modules'), join(source, 'plugins/pstack/skills/poteto-mode/scripts/node_modules')] - : [join(source, 'skills/poteto-mode/scripts/node_modules')]; - for (const dependency of dependencies) { - // Create the allowed leaves before freezing their parents; no redirected - // source ancestor may be used to create or exempt a dependency directory. - let ancestor = dirname(dependency); - for (;;) { - try { await lstat(ancestor); break; } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - ancestor = dirname(ancestor); - } - } - await sourceDirectory(ancestor); - await mkdir(dependency, { recursive: true, mode: 0o700 }); - await sourceDirectory(dependency); - } - rules.push(`(deny file-write* (require-all (subpath ${quote(source)}) (require-not (require-any ${dependencies.map(path => `(subpath ${quote(path)})`).join(' ')}))))`); - // Pin ancestors too: moving an installed-plugin parent must not relocate - // protected files outside the path-based policy between hash checks. - const pinned = new Set(); - for (const path of [source, ...dependencies.map(dirname)]) { - for (let current = path; current.startsWith(home + '/'); current = dirname(current)) pinned.add(current); - } - rules.push(`(deny file-write-unlink ${[...pinned].map(path => `(literal ${quote(path)})`).join(' ')})`); - } - const target = `${profile}.exercise.sb`; - await writeFile(target, (await readFile(profile, 'utf8')) + '\n' + rules.join('\n') + '\n', { mode: 0o600, flag: 'wx' }); - await run(sandboxed(target, ['/usr/bin/true']), { env: isolatedEnv(home), cwd: home }); - return target; -} -export function sandboxed(profile: string, args: string[]): string[] { - return ['/usr/bin/sandbox-exec', '-f', profile, ...args]; -} diff --git a/.claude/skills/verify-open-pstack/scripts/types.ts b/.claude/skills/verify-open-pstack/scripts/types.ts index 4c33f604..423af112 100644 --- a/.claude/skills/verify-open-pstack/scripts/types.ts +++ b/.claude/skills/verify-open-pstack/scripts/types.ts @@ -2,15 +2,16 @@ export const REPO = 'ericlitman/open-pstack'; export const HARNESSES = ['claude', 'codex'] as const; export type Harness = typeof HARNESSES[number]; export type Accounts = Record; -export type Phase = 'resolve' | 'classify' | 'prepare' | 'exercise' | 'publish' | 'ready' | 'failed' | 'head-moved'; +export type Phase = 'resolve' | 'classify' | 'prepare' | 'exercise' | 'publish' | 'failed' | 'head-moved'; export interface Pull { - number: number; head: { sha: string }; base: { sha: string }; draft: boolean; - state: string; headRepo: string; body: string; + number: number; head: { sha: string }; base: { sha: string }; + state: string; headRepo: string; } export interface ChangedFile { filename: string; previous_filename?: string } export interface Registry { skills: string[]; shared: string[]; + assets: string[]; setup: string[]; runner: string[]; tools: string[]; @@ -27,24 +28,17 @@ export interface Installation { harness: Harness; cliVersion: string; pluginVersion: string; sha: string; location: string; treeHash: string; home: string; account?: string; sourceHash?: string; } -export interface MacProof { - sha: string; platform: 'darwin'; reviewer: 'operator'; tests: { path: string; sha256: string }; - noKeychainDialog: boolean; - authenticated: { harness: Harness; observed: string; transcript: string; transcriptHash: string }[]; -} export interface Receipt { schema: 1; repo: string; pr: number; sha: string; base: string; phase: Phase; selection: Selection; installations: Installation[]; observations: Observation[]; - selfTest: boolean; artifactRoot: string; failure?: string; - commentUrl?: string; status?: 'success' | 'failure'; madeReady?: boolean; proposedTemplate?: string; - compensation?: string[]; cleanup: string; started: string; macProof?: MacProof; + selfTest: boolean; artifactRoot: string; publisherRevision?: string; failure?: string; + commentUrl?: string; status?: 'success' | 'failure'; cleanup: string; started: string; } export interface GitHub { pull(pr: number): Promise; files(base: string, head: string): Promise; comment(pr: number, body: string): Promise; status(sha: string, state: 'success' | 'failure', target: string, description: string): Promise; - ready(pr: number): Promise; } export interface Driver { prepare(receipt: Receipt): Promise; diff --git a/.claude/skills/verify-open-pstack/scripts/verify.sh b/.claude/skills/verify-open-pstack/scripts/verify.sh index 793098cf..4cfacbf8 100755 --- a/.claude/skills/verify-open-pstack/scripts/verify.sh +++ b/.claude/skills/verify-open-pstack/scripts/verify.sh @@ -2,7 +2,4 @@ set -euo pipefail HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" command -v bun >/dev/null || { printf '%s\n' 'Bun is required; no fallback runtime.' >&2; exit 1; } -if [[ ! -d "$HERE/node_modules/typescript" ]]; then - (cd "$HERE" && bun install --frozen-lockfile --ignore-scripts) -fi exec bun "$HERE/scripts/cli.ts" "$@" diff --git a/.claude/skills/verify-open-pstack/scripts/verify.test.ts b/.claude/skills/verify-open-pstack/scripts/verify.test.ts index 2534eaab..f8e7cf06 100644 --- a/.claude/skills/verify-open-pstack/scripts/verify.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/verify.test.ts @@ -7,281 +7,146 @@ import data from '../features/registry.json'; import { requiredFeatures, validateRegistry } from './core.ts'; import { parseChangedFiles, Publisher } from './github.ts'; import { command, sha256 } from './io.ts'; -import { evidence, liveEvidenceBody, MAC_TESTS, validateMacTests, verify } from './verify.ts'; +import { evidence, verify } from './verify.ts'; import { HARNESSES, REPO, type Driver, type GitHub, type Pull, type Receipt } from './types.ts'; -const SHA = 'a'.repeat(40), BASE = 'b'.repeat(40), NEXT = 'c'.repeat(40), HASH = 'd'.repeat(64); + +const SHA = 'a'.repeat(40), BASE = 'b'.repeat(40), NEXT = 'c'.repeat(40), HASH = 'd'.repeat(64), REVISION = 'e'.repeat(40); const URL = `https://github.com/${REPO}/pull/123#issuecomment-1`; -// Explicit fake-driver proof exercises the gate; it makes no OS-enforcement claim. -const testMacLog = (sha: string) => `Pinned SHA: ${sha}\n${MAC_TESTS.map(name => `(pass) ${name} [1ms]`).join('\n')}\n 2 pass\n 0 fail\nPSTACK_MAC_TEST_EXIT=0\n`; const roots: string[] = []; function temporary(): string { const root = realpathSync(mkdtempSync(join(tmpdir(), 'pstack-publish-test-'))); roots.push(root); return root; } afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + function fixture(runtime = false, selfTest = false) { const root = temporary(); - writeFileSync(join(root, 'retained.log'), 'reviewed transcript'); writeFileSync(join(root, 'fixture.json'), 'reviewed artifact'); - const calls: string[] = [], saved: Receipt[] = []; - const pull: Pull = { number: 123, head: { sha: SHA }, base: { sha: BASE }, draft: true, state: 'open', headRepo: REPO, body: '- [ ] The exact candidate is installed in every affected harness.\n- [ ] The changed behavior passes from each real user surface.\n- [ ] The installed version, action, and observed result appear below.\n\nLive evidence:\n\n_pending_\n\n## Operator notes\nRetain these notes.' }; + writeFileSync(join(root, 'retained.log'), 'reviewed transcript'); + writeFileSync(join(root, 'fixture.json'), 'reviewed artifact'); + const calls: string[] = [], saved: Receipt[] = [], comments: string[] = []; + const pull: Pull = { number: 123, head: { sha: SHA }, base: { sha: BASE }, state: 'open', headRepo: REPO }; let hook: (call: string) => void = () => {}; const mark = (call: string) => { calls.push(call); hook(call); }; const github: GitHub = { async pull() { mark('pull'); return structuredClone(pull); }, async files(base, head) { expect(base).toBe(BASE); expect(head).toBe(SHA); mark('files'); return [{ filename: runtime ? 'plugins/pstack/skills/architect/SKILL.md' : 'README.md' }]; }, - async comment(_pr, body) { mark('comment'); expect(body).toContain(SHA); return URL; }, - async status(sha, state, target) { mark(`status:${state}:${sha}`); expect([URL, `https://github.com/${REPO}/pull/123`]).toContain(target); }, - async ready() { mark('ready'); pull.draft = false; }, + async comment(_pr, body) { mark('comment'); comments.push(body); expect(body).toContain(SHA); return URL; }, + async status(sha, state, target) { mark(`status:${state}:${sha}`); expect(target).toBe(state === 'success' || comments.length ? URL : `https://github.com/${REPO}/pull/123`); }, }; const driver: Driver = { - async prepare(r) { - const log = testMacLog(r.sha); writeFileSync(join(root, 'mac-proof.log'), log); - r.macProof = { sha: r.sha, platform: 'darwin', reviewer: 'operator', tests: { path: 'mac-proof.log', sha256: sha256(log) }, noKeychainDialog: true, - authenticated: HARNESSES.map(harness => ({ harness, observed: 'unit-driver authenticated request assertion', transcript: 'retained.log', transcriptHash: sha256('reviewed transcript') })) }; - mark('prepare'); return HARNESSES.map(harness => ({ harness, sha: r.sha, cliVersion: 'test', pluginVersion: '1.5.0', treeHash: HASH, location: `/isolated/${harness}/plugin`, home: `/isolated/${harness}` })); }, + async prepare(r) { mark('prepare'); return HARNESSES.map(harness => ({ harness, sha: r.sha, cliVersion: 'test', pluginVersion: '1.5.0', treeHash: HASH, location: `/isolated/${harness}/plugin`, home: `/isolated/${harness}` })); }, async exercise(r) { mark('exercise'); return HARNESSES.flatMap(harness => requiredFeatures(r).map(feature => ({ harness, feature, surface: 'native surface', action: 'invoke', observed: 'fixture changed', reviewer: 'operator' as const, transcript: 'retained.log', transcriptHash: sha256('reviewed transcript'), artifacts: [{ path: 'fixture.json', sha256: sha256('reviewed artifact') }] }))); }, }; - const options = { pr: 123, root, selfTest, registry: validateRegistry(data), github, driver, persist: async (r: Receipt) => { saved.push(structuredClone(r)); } }; - return { options, calls, saved, pull, hook: (fn: typeof hook) => { hook = fn; } }; + const options = { pr: 123, root, selfTest, publisherRevision: REVISION, registry: validateRegistry(data), github, driver, persist: async (r: Receipt) => { saved.push(structuredClone(r)); } }; + return { options, calls, saved, comments, pull, hook: (fn: typeof hook) => { hook = fn; } }; } -describe('exact-head publication state machine', () => { - test('Mac test output requires both exact names, no skips/failures and successful process exit', () => { - expect(() => validateMacTests(testMacLog(SHA), SHA)).not.toThrow(); - for (const log of [testMacLog(NEXT), testMacLog(SHA).replace(MAC_TESTS[0], 'other test'), testMacLog(SHA).replace('(pass)', '(skip)'), - testMacLog(SHA).replace('0 fail', '1 fail'), testMacLog(SHA).replace('2 pass', '2 pass\n 1 skip'), testMacLog(SHA).replace('EXIT=0', 'EXIT=1')]) { - expect(() => validateMacTests(log, SHA)).toThrow('Both pinned Mac'); - } - }); - test.each(['missing', 'sha', 'keychain', 'authenticated', 'transcript', 'overwrite', 'skip'])('runtime Mac proof %s blocks success and readiness', async defect => { - const f = fixture(true); - const exercise = f.options.driver.exercise; - f.options.driver.exercise = async receipt => { - const observations = await exercise(receipt); - const proof = receipt.macProof!; - if (defect === 'missing') delete receipt.macProof; - if (defect === 'sha') proof.sha = NEXT; - if (defect === 'keychain') proof.noKeychainDialog = false; - if (defect === 'authenticated') proof.authenticated.pop(); - if (defect === 'transcript') proof.authenticated[0]!.transcriptHash = HASH; - if (defect === 'overwrite') writeFileSync(join(f.options.root, proof.tests.path), 'overwritten'); - if (defect === 'skip') { const log = testMacLog(SHA).replace('(pass)', '(skip)'); writeFileSync(join(f.options.root, proof.tests.path), log); proof.tests.sha256 = sha256(log); } - return observations; - }; - await expect(verify(f.options)).rejects.toThrow(); - expect(f.calls).not.toContain(`status:success:${SHA}`); expect(f.calls).not.toContain('ready'); - }); - test('proof mutation during publication withdraws exact-SHA success', async () => { - const f = fixture(true); - f.hook(call => { if (call === `status:success:${SHA}`) writeFileSync(join(f.options.root, 'mac-proof.log'), 'changed'); }); - await expect(verify(f.options)).rejects.toThrow('Mac isolation proof changed'); - expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.calls).not.toContain('ready'); - }); - test('docs-only publishes no-runtime evidence without either harness, then readies', async () => { +describe('exact-head evidence publication', () => { + test('docs-only publishes one no-runtime comment and exact-SHA live-gate without harnesses or PR mutation', async () => { const f = fixture(), receipt = await verify(f.options); - expect(receipt.status).toBe('success'); expect(receipt.sha).toBe(SHA); - expect(evidence(receipt)).toContain('no runtime change'); + expect(receipt.status).toBe('success'); expect(receipt.sha).toBe(SHA); expect(receipt.publisherRevision).toBe(REVISION); expect(f.calls).not.toContain('prepare'); expect(f.calls).not.toContain('exercise'); - expect(f.calls.indexOf('comment')).toBeLessThan(f.calls.indexOf(`status:success:${SHA}`)); - expect(f.calls.indexOf(`status:success:${SHA}`)).toBeLessThan(f.calls.indexOf('ready')); - expect(f.calls.filter(call => call === 'ready')).toHaveLength(1); expect(f.calls).not.toContain('draft'); - expect(f.calls).not.toContain('body'); - expect(f.pull.body).toContain('_pending_'); - expect(receipt.proposedTemplate).toContain(URL); - expect(receipt.proposedTemplate).toContain('Retain these notes.'); + expect(f.calls.filter(call => call === 'comment')).toHaveLength(1); + expect(f.calls).toContain(`status:success:${SHA}`); + expect(f.comments[0]).toContain('PASSED — no runtime change'); expect(f.comments[0]).toContain(REVISION); + expect(Object.keys(receipt)).not.toContain('proposedTemplate'); expect(Object.keys(receipt)).not.toContain('madeReady'); expect(Object.keys(receipt)).not.toContain('compensation'); }); - test('unmanaged evidence replacement preserves every following heading and operator note', async () => { - const receipt = await verify(fixture().options); - for (const suffix of ['# Release notes\nKeep these.', '## Notes\nKeep these.', '### Operator notes\nKeep these.', 'Ordinary operator text.\n\n## Later\nKeep these.', '---\nTrailing section.']) { - const before = `Summary is untouched.\n\nLive evidence:\n\n_pending_\n\n${suffix}`; - const after = liveEvidenceBody(before, receipt); - expect(after).toStartWith('Summary is untouched.\n\n'); - expect(after).toEndWith(suffix); expect(after).not.toContain('_pending_'); - expect(liveEvidenceBody(after, receipt)).toBe(after); + + test('runtime selection and explicit self-test exercise every feature in both harnesses', async () => { + const f = fixture(true, true), receipt = await verify(f.options); + expect(requiredFeatures(receipt).sort()).toEqual(['project-skill', 'skill-invocation:architect']); + expect(receipt.installations.map(record => record.harness).sort()).toEqual([...HARNESSES]); + expect(receipt.observations).toHaveLength(4); + for (const feature of requiredFeatures(receipt)) for (const harness of HARNESSES) { + expect(receipt.observations.filter(record => record.feature === feature && record.harness === harness)).toHaveLength(1); } - const ordinary = 'Live evidence:\nOperator-owned text immediately follows.\n# Closing notes'; - expect(liveEvidenceBody(ordinary, receipt)).toEndWith('Operator-owned text immediately follows.\n# Closing notes'); + expect(f.comments).toHaveLength(1); expect(f.comments[0]).toContain('operator reviewed'); }); - test('all changed surfaces and explicit project self-test run in both harnesses', async () => { - const f = fixture(true, true), r = await verify(f.options); - expect(r.observations).toHaveLength(4); expect(f.calls).toContain('prepare'); expect(f.calls).toContain('exercise'); - expect(evidence(r)).toContain('operator reviewed'); expect(evidence(r)).toContain(HASH); - }); - test('already-ready PR is never toggled ready', async () => { - const f = fixture(); f.pull.draft = false; await verify(f.options); expect(f.calls).not.toContain('ready'); + + test('self-test alone runs project proof in both harnesses', async () => { + const receipt = await verify(fixture(false, true).options); + expect(receipt.selection.noRuntime).toBe(true); + expect(receipt.observations.map(record => `${record.harness}/${record.feature}`).sort()).toEqual(['claude/project-skill', 'codex/project-skill']); }); - test('successful verification cleans credentials once before any publication', async () => { - const f = fixture(true); - f.options.driver.cleanup = async () => { f.calls.push('cleanup'); }; + + test('successful verification cleans disposable credentials before any publication', async () => { + const f = fixture(true); f.options.driver.cleanup = async () => { f.calls.push('cleanup'); }; const receipt = await verify(f.options); expect(f.calls.filter(call => call === 'cleanup')).toHaveLength(1); expect(f.calls.indexOf('exercise')).toBeLessThan(f.calls.indexOf('cleanup')); expect(f.calls.indexOf('cleanup')).toBeLessThan(f.calls.indexOf('comment')); expect(receipt.cleanup).toContain('removed'); - expect(receipt.status).toBe('success'); }); - test('cleanup failure is retried and recorded before failed publication, preserving a ready PR', async () => { - const f = fixture(true); f.pull.draft = false; - f.options.driver.cleanup = async () => { f.calls.push('cleanup'); throw new Error('credential removal denied'); }; - const comment = f.options.github.comment; - f.options.github.comment = async (pr, body) => { - expect(f.saved.at(-1)?.phase).toBe('failed'); - expect(f.saved.at(-1)?.cleanup).toContain('FAILED'); - expect(f.saved.at(-1)?.failure).toContain('credential removal denied'); - expect(body).toContain('FAILED'); - return comment(pr, body); - }; - await expect(verify(f.options)).rejects.toThrow('credential removal denied'); + + test('persistent cleanup failure prevents every public write and remains private', async () => { + const f = fixture(true); f.options.driver.cleanup = async () => { f.calls.push('cleanup'); throw new Error('RAW_CREDENTIAL_CLEANUP_EXCEPTION'); }; + await expect(verify(f.options)).rejects.toThrow('RAW_CREDENTIAL_CLEANUP_EXCEPTION'); expect(f.calls.filter(call => call === 'cleanup')).toHaveLength(2); - expect(f.calls.lastIndexOf('cleanup')).toBeLessThan(f.calls.indexOf('comment')); - expect(f.calls).not.toContain(`status:success:${SHA}`); - expect(f.calls).toContain(`status:failure:${SHA}`); - expect(f.calls).not.toContain('ready'); expect(f.calls).not.toContain('draft'); - expect(f.pull.draft).toBe(false); - expect(f.saved.at(-1)?.compensation?.join()).toContain('remove copied session credentials: FAILED'); + expect(f.calls).not.toContain('comment'); expect(f.calls.some(call => call.startsWith('status:'))).toBe(false); + expect(f.saved.at(-1)?.failure).toContain('RAW_CREDENTIAL_CLEANUP_EXCEPTION'); }); - test('a successful cleanup retry still cannot publish success after the original cleanup failure', async () => { + + test('a successful cleanup retry permits one generic failure publication', async () => { const f = fixture(true); let attempts = 0; - f.options.driver.cleanup = async () => { if (++attempts === 1) throw new Error('temporary cleanup failure'); }; - await expect(verify(f.options)).rejects.toThrow('temporary cleanup failure'); - expect(attempts).toBe(2); expect(f.saved.at(-1)?.cleanup).toContain('removed'); - expect(f.calls).not.toContain(`status:success:${SHA}`); expect(f.calls).not.toContain('ready'); - }); - test('exercise failure cleans credentials before publishing its retained failure', async () => { - const f = fixture(true); f.pull.draft = false; - f.options.driver.exercise = async () => { f.calls.push('exercise'); throw new Error('surface failed'); }; - f.options.driver.cleanup = async () => { f.calls.push('cleanup'); }; - await expect(verify(f.options)).rejects.toThrow('surface failed'); - expect(f.calls.filter(call => call === 'cleanup')).toHaveLength(1); - expect(f.calls.indexOf('exercise')).toBeLessThan(f.calls.indexOf('cleanup')); - expect(f.calls.indexOf('cleanup')).toBeLessThan(f.calls.indexOf('comment')); - expect(f.saved.at(-1)?.cleanup).toContain('removed'); - expect(f.saved.at(-1)?.failure).toBe('surface failed'); - expect(f.pull.draft).toBe(false); expect(f.calls).not.toContain('draft'); + f.options.driver.cleanup = async () => { f.calls.push('cleanup'); if (++attempts === 1) throw new Error('RAW_FIRST_CLEANUP_FAILURE'); }; + await expect(verify(f.options)).rejects.toThrow('RAW_FIRST_CLEANUP_FAILURE'); + expect(f.calls.filter(call => call === 'cleanup')).toHaveLength(2); expect(f.comments).toHaveLength(1); + expect(f.comments[0]).toContain('FAILED — verification did not complete'); expect(f.comments[0]).not.toContain('RAW_FIRST_CLEANUP_FAILURE'); + expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.saved.at(-1)?.failure).toContain('RAW_FIRST_CLEANUP_FAILURE'); }); - test('publication failure does not repeat an already completed cleanup', async () => { + + test('failure publication is structured and excludes raw exceptions, transcripts, and artifact paths', async () => { const f = fixture(true); + f.options.driver.exercise = async () => { f.calls.push('exercise'); throw new Error('RAW_PROVIDER_EXCEPTION secret detail'); }; f.options.driver.cleanup = async () => { f.calls.push('cleanup'); }; - f.hook(call => { if (call === 'comment') throw new Error('publisher failed'); }); - await expect(verify(f.options)).rejects.toThrow('publisher failed'); - expect(f.calls.filter(call => call === 'cleanup')).toHaveLength(1); - expect(f.saved.at(-1)?.cleanup).toContain('removed'); - }); - test('concurrent author body edits are untouched and the operator proposal is retained', async () => { - const f = fixture(), edited = '# Author revision\nUnrelated edits must survive.'; - expect('body' in f.options.github).toBe(false); - f.hook(call => { if (call === 'comment') f.pull.body = edited; }); - const receipt = await verify(f.options); - expect(f.pull.body).toBe(edited); expect(f.calls).not.toContain('body'); - expect(receipt.proposedTemplate).toContain('Live evidence:'); expect(receipt.proposedTemplate).toContain(URL); - expect(f.saved.at(-1)?.proposedTemplate).toBe(receipt.proposedTemplate); + await expect(verify(f.options)).rejects.toThrow('RAW_PROVIDER_EXCEPTION'); + expect(f.comments).toHaveLength(1); + const body = f.comments[0]!; + expect(body).toContain('## Live evidence: Open Pstack'); expect(body).toContain(`Candidate SHA: \`${SHA}\``); + for (const forbidden of ['RAW_PROVIDER_EXCEPTION', 'secret detail', 'reviewed transcript', 'retained.log', 'fixture.json']) expect(body).not.toContain(forbidden); + expect(f.saved.at(-1)?.failure).toContain('RAW_PROVIDER_EXCEPTION secret detail'); }); - test('already-ready PR stays ready when publication fails', async () => { - const f = fixture(); f.pull.draft = false; f.hook(call => { if (call === 'comment') throw new Error('comment denied'); }); - await expect(verify(f.options)).rejects.toThrow('comment denied'); expect(f.calls).not.toContain('draft'); expect(f.pull.draft).toBe(false); - expect(f.calls).toContain(`status:failure:${SHA}`); + + test('an ambiguous comment response is never retried', async () => { + const f = fixture(true); f.options.driver.cleanup = async () => { f.calls.push('cleanup'); }; + f.hook(call => { if (call === 'comment') throw new Error('response lost'); }); + await expect(verify(f.options)).rejects.toThrow('response lost'); + expect(f.calls.filter(call => call === 'comment')).toHaveLength(1); + expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.calls).not.toContain(`status:success:${SHA}`); }); - test('transcript or artifact overwrite after acceptance prevents publication and readiness', async () => { - for (const file of ['retained.log', 'fixture.json']) { - for (const boundary of ['exercise', 'comment', `status:success:${SHA}`]) { - const f = fixture(true); f.hook(call => { if (call === boundary) writeFileSync(join(f.options.root, file), 'overwritten'); }); - await expect(verify(f.options)).rejects.toThrow('changed after acceptance'); expect(f.calls).not.toContain('ready'); - if (boundary === `status:success:${SHA}`) expect(f.calls).toContain(`status:failure:${SHA}`); - else expect(f.calls).not.toContain(`status:success:${SHA}`); - } + + test('transcript or artifact overwrite at publication boundaries revokes or blocks exact-SHA success', async () => { + for (const file of ['retained.log', 'fixture.json']) for (const boundary of ['comment', `status:success:${SHA}`]) { + const f = fixture(true); f.options.driver.cleanup = async () => { f.calls.push('cleanup'); }; + f.hook(call => { if (call === boundary) writeFileSync(join(f.options.root, file), 'overwritten'); }); + await expect(verify(f.options)).rejects.toThrow('changed after acceptance'); + expect(f.comments).toHaveLength(1); expect(f.calls).toContain(`status:failure:${SHA}`); + if (boundary === 'comment') expect(f.calls).not.toContain(`status:success:${SHA}`); } }); - test('broad evidence remains bounded for success, failure and PR body while retaining full receipt data', async () => { - const f = fixture(true), receipt = await verify(f.options); - const observation = receipt.observations[0]; - receipt.observations = Array.from({ length: 400 }, (_, index) => ({ ...observation, feature: `skill:${index}`, action: 'a'.repeat(3000), observed: 'r'.repeat(3000), artifacts: Array.from({ length: 20 }, (_, n) => ({ path: `artifact-${n}`, sha256: HASH })) })); - const comment = evidence(receipt); expect(comment.length).toBeLessThan(60000); expect(comment).toContain('additional reviewed observations'); expect(comment).toContain('Evidence-set SHA-256'); - expect(liveEvidenceBody(f.pull.body, receipt).length).toBeLessThan(60000); - receipt.failure = 'reason '.repeat(10000); expect(evidence(receipt).length).toBeLessThan(60000); expect(receipt.observations).toHaveLength(400); + + test('head/base races never publish success and failure gate stays on the pinned candidate', async () => { + for (const boundary of ['files', 'prepare', 'exercise', 'comment', `status:success:${SHA}`]) { + const f = fixture(true); f.options.driver.cleanup = async () => { f.calls.push('cleanup'); }; + f.hook(call => { if (call === boundary) f.pull.head.sha = NEXT; }); + await expect(verify(f.options)).rejects.toThrow('changed'); + if (boundary !== `status:success:${SHA}`) expect(f.calls).not.toContain(`status:success:${SHA}`); + expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.calls.join()).not.toContain(`status:failure:${NEXT}`); + expect(f.comments.length).toBeLessThanOrEqual(1); + } }); + test('closed or foreign PR never enters publication', async () => { for (const field of ['state', 'headRepo'] as const) { const f = fixture(); f.pull[field] = 'other'; await expect(verify(f.options)).rejects.toThrow('open same-repository'); expect(f.calls).toEqual(['pull']); } }); - test('head movement during classify, prepare, exercise, or comment aborts without success or readiness', async () => { - for (const boundary of ['files', 'prepare', 'exercise', 'comment']) { - const f = fixture(true); f.hook(call => { if (call === boundary) f.pull.head.sha = NEXT; }); - await expect(verify(f.options)).rejects.toThrow('changed'); - expect(f.calls.some(c => c.startsWith('status:success'))).toBe(false); expect(f.calls).not.toContain('ready'); - expect(f.saved.at(-1)?.phase).toBe('head-moved'); - expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.calls).not.toContain('draft'); - } - }); - test('head or base movement after status withdraws success only on original SHA', async () => { - for (const field of ['head', 'base'] as const) { - const f = fixture(); f.hook(call => { if (call === `status:success:${SHA}`) f.pull[field].sha = NEXT; }); - await expect(verify(f.options)).rejects.toThrow('changed'); - expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.calls.join()).not.toContain(`status:failure:${NEXT}`); - expect(f.calls).not.toContain('ready'); expect(f.saved.at(-1)?.status).toBe('failure'); - } - }); - test('post-ready head race revokes pinned success without reversing readiness', async () => { - const f = fixture(); f.hook(call => { if (call === 'ready') f.pull.head.sha = NEXT; }); - await expect(verify(f.options)).rejects.toThrow('changed'); - expect(f.calls).not.toContain('draft'); expect(f.pull.draft).toBe(false); expect(f.saved.at(-1)?.madeReady).toBe(true); - expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.calls.filter(call => call === 'ready')).toHaveLength(1); - }); - test('incomplete exercise records failure against pinned head and never readies', async () => { - const f = fixture(true); f.options.driver.exercise = async () => []; - await expect(verify(f.options)).rejects.toThrow('Missing/duplicate'); - expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.calls).not.toContain('ready'); - expect(f.saved.at(-1)?.failure).toContain('Missing/duplicate'); - }); - test('ambiguous publication responses revoke success without claiming a readiness transition', async () => { - for (const boundary of [`status:success:${SHA}`, 'ready']) { - const f = fixture(); f.hook(call => { if (call === boundary) throw new Error('response lost'); }); - await expect(verify(f.options)).rejects.toThrow('response lost'); - expect(f.calls).toContain(`status:failure:${SHA}`); - expect(f.calls).not.toContain('draft'); - } - }); - test('another actor making the PR ready during the run is not reverted after failure', async () => { - const f = fixture(); - f.hook(call => { if (call === `status:success:${SHA}`) f.pull.draft = false; }); - f.options.persist = async receipt => { - if (receipt.status === 'success') throw new Error('receipt denied'); - f.saved.push(structuredClone(receipt)); - }; - await expect(verify(f.options)).rejects.toThrow('receipt denied'); - expect(f.calls).not.toContain('ready'); expect(f.calls).not.toContain('draft'); expect(f.pull.draft).toBe(false); - }); - test('readiness is checked again after entering the ready phase', async () => { - const f = fixture(), persist = f.options.persist; - f.options.persist = async receipt => { await persist(receipt); if (receipt.phase === 'ready') f.pull.draft = false; }; - await verify(f.options); - expect(f.calls).not.toContain('ready'); expect(f.calls).not.toContain('draft'); expect(f.pull.draft).toBe(false); - }); - test('a failed or idempotent readiness call never rolls back another actor', async () => { - const f = fixture(); - f.options.github.ready = async () => { f.calls.push('ready'); f.pull.draft = false; throw new Error('already ready'); }; - await expect(verify(f.options)).rejects.toThrow('already ready'); - expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.calls).not.toContain('draft'); expect(f.pull.draft).toBe(false); - }); - test('receipt-write errors cannot prevent revoking pinned success and never reverse readiness', async () => { - for (const boundary of [`status:success:${SHA}`, 'ready']) { - const f = fixture(); let deny = false; - f.hook(call => { if (call === boundary) { f.pull.base.sha = NEXT; deny = true; } }); - f.options.persist = async receipt => { - if (deny && ['head-moved', 'failed'].includes(receipt.phase)) throw new Error('ENOSPC'); - f.saved.push(structuredClone(receipt)); - }; - await expect(verify(f.options)).rejects.toThrow('changed'); - expect(f.calls).toContain(`status:failure:${SHA}`); - expect(f.calls).not.toContain('draft'); - if (boundary === 'ready') expect(f.pull.draft).toBe(false); - } - }); - test('failed compensation is retained, not reported as successful', async () => { - const f = fixture(); f.hook(call => { - if (call === `status:success:${SHA}`) f.pull.head.sha = NEXT; - if (call === `status:failure:${SHA}`) throw new Error('publisher denied'); - }); - await expect(verify(f.options)).rejects.toThrow('changed'); - expect(f.saved.at(-1)?.compensation?.join()).toContain('FAILED'); + + test('large evidence remains bounded and exposes no raw transcript content', async () => { + const receipt = await verify(fixture(true).options), observation = receipt.observations[0]!; + receipt.observations = Array.from({ length: 400 }, (_, index) => ({ ...observation, feature: `skill:${index}`, action: 'a'.repeat(3000), observed: 'r'.repeat(3000), artifacts: Array.from({ length: 20 }, (_, n) => ({ path: `artifact-${n}`, sha256: HASH })) })); + const comment = evidence(receipt); + expect(comment.length).toBeLessThan(60000); expect(comment).toContain('Additional observations'); expect(comment).toContain('Evidence-set SHA-256'); + expect(comment).not.toContain('reviewed transcript'); expect(comment).not.toContain('retained.log'); + receipt.failure = 'RAW '.repeat(10000); expect(evidence(receipt).length).toBeLessThan(60000); expect(evidence(receipt)).not.toContain('RAW'); }); }); @@ -291,49 +156,40 @@ describe('publisher request boundaries', () => { expect(parseChangedFiles('')).toEqual([]); for (const output of ['M\0README.md', 'R100\0old\0', 'M\0same\0M\0same\0', 'X\0file\0']) expect(() => parseChangedFiles(output)).toThrow(); }); - test('fresh local git diff uses immutable base/head even when the source branch moves', async () => { + + test('fresh local git diff uses immutable base/head even when a source branch moves', async () => { const repository = temporary(), env = { ...process.env, GIT_AUTHOR_NAME: 'test', GIT_AUTHOR_EMAIL: 'test@example.com', GIT_COMMITTER_NAME: 'test', GIT_COMMITTER_EMAIL: 'test@example.com' }; const git = (...args: string[]) => execFileSync('git', args, { cwd: repository, env, encoding: 'utf8' }).trim(); git('init', '-q'); writeFileSync(join(repository, 'README.md'), 'base'); git('add', '.'); git('commit', '-qm', 'base'); const base = git('rev-parse', 'HEAD'); mkdirSync(join(repository, 'runtime')); writeFileSync(join(repository, 'runtime', 'skill.md'), 'immutable runtime'); git('add', '.'); git('commit', '-qm', 'runtime'); const head = git('rev-parse', 'HEAD'); git('rm', '-q', 'runtime/skill.md'); git('commit', '-qm', 'later docs-only branch'); const calls: string[][] = []; - const publisher = new Publisher(async (args, options) => { - calls.push(args); - if (args[0] === 'git' && args[1] === 'fetch') args = args.map(arg => arg === `https://github.com/${REPO}.git` ? repository : arg); - return command(args, options); - }); + const publisher = new Publisher(async (args, options) => { calls.push(args); if (args[0] === 'git' && args[1] === 'fetch') args = args.map(arg => arg === `https://github.com/${REPO}.git` ? repository : arg); return command(args, options); }); expect(await publisher.files(base, head)).toEqual([{ filename: 'runtime/skill.md' }]); - expect(calls.some(args => args[0] === 'gh')).toBe(false); - expect(calls.at(-1)).toContain(`${base}...${head}`); + expect(calls.some(args => args[0] === 'gh')).toBe(false); expect(calls.at(-1)).toContain(`${base}...${head}`); await expect(publisher.files('main', head)).rejects.toThrow('Invalid diff SHA'); }); - test('real interrupted command execution still permits only trusted rollback writes', () => { + + test('interruption blocks ordinary status writes but permits a pinned failure gate', () => { for (const signal of ['SIGINT', 'SIGTERM']) { const script = ` import { command, interruptCommands } from ${JSON.stringify(new globalThis.URL('./io.ts', import.meta.url).pathname)}; import { Publisher } from ${JSON.stringify(new globalThis.URL('./github.ts', import.meta.url).pathname)}; await interruptCommands(${JSON.stringify(signal)}); const executed = []; - const publisher = new Publisher(async (args, options) => { - const output = await command([process.execPath, '-e', 'console.log("{}")'], options); - executed.push(args); - return output; - }); - for (const ordinary of [() => publisher.ready(123), () => publisher.status(${JSON.stringify(SHA)}, 'success', ${JSON.stringify(URL)}, 'passed')]) { - try { await ordinary(); throw new Error('ordinary write escaped interruption'); } - catch (error) { if (!String(error).includes('Interrupted by')) throw error; } - } - await publisher.status(${JSON.stringify(SHA)}, 'failure', ${JSON.stringify(URL)}, 'withdraw'); + const publisher = new Publisher(async (args, options) => { const output = await command([process.execPath, '-e', 'console.log("{}")'], options); executed.push(args); return output; }); + try { await publisher.status(${JSON.stringify(SHA)}, 'success', ${JSON.stringify(URL)}, 'passed'); throw new Error('ordinary write escaped interruption'); } + catch (error) { if (!String(error).includes('Interrupted by')) throw error; } + await publisher.status(${JSON.stringify(SHA)}, 'failure', ${JSON.stringify(URL)}, 'failed'); console.log(JSON.stringify(executed)); `; const calls = JSON.parse(execFileSync(process.execPath, ['-e', script], { encoding: 'utf8' }).trim()) as string[][]; - expect(calls).toHaveLength(1); - expect(calls[0]).toContain('state=failure'); expect(calls.flat()).not.toContain('--undo'); + expect(calls).toHaveLength(1); expect(calls[0]).toContain('state=failure'); } }); + test('status targets the supplied exact SHA and evidence URL, never a branch', async () => { - const calls: string[][] = []; const publisher = new Publisher(async args => { calls.push(args); return '{}'; }); + const calls: string[][] = [], publisher = new Publisher(async args => { calls.push(args); return '{}'; }); await publisher.status(SHA, 'success', URL, 'passed'); expect(calls[0]).toContain(`repos/${REPO}/statuses/${SHA}`); expect(calls[0]).toContain('context=live-gate'); expect(calls[0]).toContain(`target_url=${URL}`); await expect(publisher.status('main', 'success', URL, 'passed')).rejects.toThrow('Invalid status SHA'); diff --git a/.claude/skills/verify-open-pstack/scripts/verify.ts b/.claude/skills/verify-open-pstack/scripts/verify.ts index ecb35fe0..04186e0f 100644 --- a/.claude/skills/verify-open-pstack/scripts/verify.ts +++ b/.claude/skills/verify-open-pstack/scripts/verify.ts @@ -1,76 +1,39 @@ import { classify, completeEvidence, newReceipt } from './core.ts'; -import { readFile } from 'node:fs/promises'; -import { join } from 'node:path'; -import { redact, retainedFile, sha256 } from './io.ts'; +import { assertNoKnownSecrets, retainedFile, sha256 } from './io.ts'; import { REPO, type Driver, type GitHub, type Pull, type Receipt, type Registry } from './types.ts'; class HeadMoved extends Error {} -const safe = (text: string, limit = 160): string => redact(text).replace(/[`<>\r\n]/g, ' ').slice(0, limit); +// Omit unsafe/oversized values whole: truncating or rewriting one could hide an exact registered secret. +const safe = (text: string, limit = 160): string => text.length <= limit && /^[\x20-\x7e]*$/.test(text) && !/[`<>]/.test(text) ? text : '[value omitted]'; + +/** Build the single structured public record for this run. Failure details remain private in receipt.json. */ export function evidence(receipt: Receipt): string { - const lines = ['## Live evidence: Open Pstack', `Pinned PR #${receipt.pr}: \`${receipt.sha}\` (base \`${receipt.base}\`).`, - `Result: ${receipt.failure ? `FAILED — ${safe(receipt.failure, 500)}` : receipt.selection.noRuntime ? 'no runtime change' : 'all mapped features passed'}.`, - `Self-test: ${receipt.selfTest ? 'required in both harnesses' : 'not requested'}.`, - `Retained receipt: \`${safe(receipt.artifactRoot, 500)}/receipt.json\`. Full observations and all transcript/artifact hashes are retained there. Local CLI checks are not installed-harness evidence.`, - `Evidence-set SHA-256: \`${sha256(JSON.stringify({ installations: receipt.installations, observations: receipt.observations, macProof: receipt.macProof }))}\`.`]; - for (const install of receipt.installations) lines.push(`- ${install.harness}: CLI ${safe(install.cliVersion)}, plugin ${safe(install.pluginVersion)}, SHA \`${install.sha}\`, tree \`${install.treeHash}\`.`); - let shown = 0; - for (const record of receipt.observations) { - const line = `- ${record.harness} / ${safe(record.feature, 72)}: ${safe(record.surface, 48)}; action: ${safe(record.action, 48)}; observed: ${safe(record.observed, 48)}; operator reviewed. Transcript SHA-256 \`${record.transcriptHash}\`; artifact-set SHA-256 \`${sha256(JSON.stringify(record.artifacts))}\`.`; - if (lines.join('\n').length + line.length > 48000) break; - lines.push(line); shown++; - } - if (shown < receipt.observations.length) lines.push(`${receipt.observations.length - shown} additional reviewed observations are in the retained receipt; see the evidence-set digest above.`); - lines.push('Every new head requires a fresh run, including queue-created heads. This helper never queues or merges.'); - return lines.join('\n'); -} -export function liveEvidenceBody(body: string, receipt: Receipt): string { - const managed = /[\s\S]*?/; - const marker = /(?:^|\n)Live evidence:[^\n]*\n?/; - const match = marker.exec(body); - const summary = `\nLive evidence:\n\n${evidence(receipt)}\nEvidence comment: ${receipt.commentUrl}\n`; - if (managed.test(body)) body = body.replace(managed, () => summary); - else if (match) { - const start = match.index + (match[0].startsWith('\n') ? 1 : 0); - // Only the template marker and its known placeholder belong to this helper. - const remaining = body.slice(match.index + match[0].length).replace(/^\n*_pending_\n?(?=\n|$)/, ''); - body = body.slice(0, start) + summary + '\n' + remaining; - } else body += '\n\n' + summary; - for (const text of ['The exact candidate is installed in every affected harness.', 'The changed behavior passes from each real user surface.', 'The installed version, action, and observed result appear below.']) { - body = body.replace(`- [ ] ${text}`, `- [x] ${text}${receipt.selection.noRuntime && !receipt.selfTest ? ' (No runtime change; harness exercise not required.)' : ''}`); - } - return body; -} -export const MAC_TESTS = [ - 'real Mac sandbox denies daily-home and vault sentinels, writes, aliases, grandchildren, and Keychain IPC', - 'actual operator HOME denial and immutable exercise sources with writable dependency leaves', -] as const; -export function validateMacTests(text: string, sha: string): void { - if (!text.startsWith(`Pinned SHA: ${sha}\n`) || !text.trimEnd().endsWith('PSTACK_MAC_TEST_EXIT=0') || - !/^\s*0 fail\s*$/m.test(text) || /^\s*[1-9]\d* skip\s*$/m.test(text) || /\((?:skip|fail)\)/.test(text) || - !MAC_TESTS.every(name => text.split('\n').some(line => line.startsWith(`(pass) ${name}`) && /^(?:\s+\[[^\]]+\])?\s*$/.test(line.slice(`(pass) ${name}`.length))))) { - throw new Error('Both pinned Mac isolation tests must pass without skips or failures'); - } -} -export async function revalidateMacProof(receipt: Receipt): Promise { - if (receipt.selection.noRuntime && !receipt.selfTest) return; - const proof = receipt.macProof; - if (!proof || proof.sha !== receipt.sha || proof.platform !== 'darwin' || proof.reviewer !== 'operator' || proof.noKeychainDialog !== true) { - throw new Error('Required exact-SHA operator-Mac proof or no-Keychain observation missing'); - } - const tests = await retainedFile(receipt.artifactRoot, proof.tests.path); - if (tests.path !== proof.tests.path || tests.sha256 !== proof.tests.sha256) throw new Error('Mac isolation proof changed after acceptance'); - validateMacTests(await readFile(join(receipt.artifactRoot, tests.path), 'utf8'), receipt.sha); - if (proof.authenticated.length !== 2) throw new Error('Authenticated native requests required in both harnesses'); - for (const harness of ['claude', 'codex'] as const) { - const requests = proof.authenticated.filter(record => record.harness === harness); - if (requests.length !== 1 || !requests[0]!.observed.trim()) throw new Error('Authenticated native request assertion missing'); - const request = requests[0]!, transcript = await retainedFile(receipt.artifactRoot, request.transcript); - if (transcript.path !== request.transcript || transcript.sha256 !== request.transcriptHash || - !receipt.observations.some(record => record.harness === harness && record.transcript === request.transcript && record.transcriptHash === request.transcriptHash)) { - throw new Error('Authenticated request is not bound to the reviewed native transcript'); + const failed = Boolean(receipt.failure); + const lines = [ + '## Live evidence: Open Pstack', + `- Candidate SHA: \`${receipt.sha}\``, + `- Base SHA: \`${receipt.base}\``, + `- Result: ${failed ? 'FAILED — verification did not complete; rerun required' : receipt.selection.noRuntime && !receipt.selfTest ? 'PASSED — no runtime change' : 'PASSED — all mapped features verified'}`, + `- Self-test: ${receipt.selfTest ? failed ? 'required in both harnesses' : 'completed in both harnesses' : 'not requested'}`, + `- Publisher revision: ${receipt.publisherRevision ? `\`${receipt.publisherRevision}\`` : 'not supplied'}`, + `- Evidence-set SHA-256: \`${sha256(JSON.stringify({ installations: receipt.installations, observations: receipt.observations }))}\``, + ]; + if (!failed) { + for (const install of receipt.installations) { + lines.push(`- Installation (${install.harness}): CLI ${safe(install.cliVersion)}, plugin ${safe(install.pluginVersion)}, candidate \`${install.sha}\`, tree \`${install.treeHash}\`.`); + } + let shown = 0; + for (const record of receipt.observations) { + const line = `- Observation (${record.harness} / ${safe(record.feature, 72)}): surface ${safe(record.surface, 48)}; action ${safe(record.action, 48)}; result ${safe(record.observed, 48)}; operator reviewed; transcript SHA-256 \`${record.transcriptHash}\`; artifact-set SHA-256 \`${sha256(JSON.stringify(record.artifacts))}\`.`; + if (lines.join('\n').length + line.length > 48000) break; + lines.push(line); shown++; } + if (shown < receipt.observations.length) lines.push(`- Additional observations: ${receipt.observations.length - shown} retained privately; bound by the evidence-set digest.`); } + lines.push('- Policy: every new head requires a fresh run; this verifier does not change PR state, queue, or merge.'); + return lines.join('\n'); } + export async function revalidateEvidence(receipt: Receipt): Promise { for (const record of receipt.observations) { const transcript = await retainedFile(receipt.artifactRoot, record.transcript); @@ -80,26 +43,51 @@ export async function revalidateEvidence(receipt: Receipt): Promise { if (actual.path !== artifact.path || actual.sha256 !== artifact.sha256) throw new Error('Reviewed artifact changed after acceptance'); } } - await revalidateMacProof(receipt); } -export async function verify(options: { pr: number; selfTest: boolean; root: string; registry: Registry; github: GitHub; driver: Driver; persist: (receipt: Receipt) => Promise }): Promise { + +type VerifyOptions = { + pr: number; + selfTest: boolean; + root: string; + registry: Registry; + github: GitHub; + driver: Driver; + persist: (receipt: Receipt) => Promise; + publisherRevision?: string; +}; + +export async function verify(options: VerifyOptions): Promise { const { github, driver, persist } = options; const initial = await github.pull(options.pr); if (initial.state !== 'open' || initial.headRepo !== REPO) throw new Error('Require an open same-repository PR'); - const receipt = newReceipt(options.pr, initial.head.sha, initial.base.sha, options.selfTest, options.root); - let successAttempted = false, cleanupComplete = false; + const receipt = newReceipt(options.pr, initial.head.sha, initial.base.sha, options.selfTest, options.root, options.publisherRevision); + let successAttempted = false, commentAttempted = false, cleanupComplete = false; const cleanup = async (): Promise => { if (cleanupComplete) return; await driver.cleanup?.(receipt); cleanupComplete = true; - receipt.cleanup = 'Disposable candidate homes removed, including relocated credentials; external evidence retained'; + receipt.cleanup = 'Copied disposable credentials removed; candidate state and private raw evidence retained'; }; const current = async (): Promise => { const pull = await github.pull(receipt.pr); if (pull.head.sha !== receipt.sha || pull.base.sha !== receipt.base || pull.state !== 'open' || pull.headRepo !== REPO) throw new HeadMoved('PR head/base or open repository identity changed; run again'); return pull; }; - const phase = async (next: Receipt['phase']): Promise => { receipt.phase = next; await persist(receipt); await current(); }; + const phase = async (next: Receipt['phase']): Promise => { + receipt.phase = next; + await persist(receipt); + await current(); + }; + const publishComment = async (): Promise => { + const body = evidence(receipt); + assertNoKnownSecrets(body); + commentAttempted = true; + receipt.commentUrl = await github.comment(receipt.pr, body); + await persist(receipt); + }; + const appendFailure = (label: string, error: unknown): void => { + receipt.failure = `${receipt.failure}; ${label}: ${error instanceof Error ? error.message : String(error)}`; + }; try { await phase('classify'); receipt.selection = classify(await github.files(receipt.base, receipt.sha), options.registry); @@ -110,52 +98,37 @@ export async function verify(options: { pr: number; selfTest: boolean; root: str } await cleanup(); completeEvidence(receipt); - await phase('publish'); await revalidateEvidence(receipt); - receipt.commentUrl = await github.comment(receipt.pr, evidence(receipt)); await persist(receipt); - const pull = await current(); await revalidateEvidence(receipt); - receipt.proposedTemplate = liveEvidenceBody(pull.body, receipt); - await persist(receipt); await current(); await revalidateEvidence(receipt); + await phase('publish'); await revalidateEvidence(receipt); await current(); + await publishComment(); + await current(); await revalidateEvidence(receipt); successAttempted = true; - await github.status(receipt.sha, 'success', receipt.commentUrl, receipt.selection.noRuntime ? 'no runtime change' : 'All mapped features passed in both harnesses'); + await github.status(receipt.sha, 'success', receipt.commentUrl!, receipt.selection.noRuntime && !receipt.selfTest ? 'No runtime change' : 'All mapped features passed in both harnesses'); receipt.status = 'success'; await persist(receipt); await current(); await revalidateEvidence(receipt); - if ((await current()).draft) { - await phase('ready'); await revalidateEvidence(receipt); - if ((await current()).draft) { - await github.ready(receipt.pr); receipt.madeReady = true; await persist(receipt); await current(); await revalidateEvidence(receipt); - } - } - await persist(receipt); return receipt; + return receipt; } catch (error) { - receipt.failure = redact(error instanceof Error ? error.message : String(error)); + receipt.failure = error instanceof Error ? error.message : String(error); receipt.phase = error instanceof HeadMoved ? 'head-moved' : 'failed'; - receipt.compensation = []; - const recover = async (label: string, operation: () => Promise): Promise => { - try { await operation(); receipt.compensation!.push(`${label}: done`); } - catch (failure) { receipt.compensation!.push(`${label}: FAILED ${redact(String(failure))}`); } - }; - await recover('remove copied session credentials', async () => { - try { await cleanup(); } - catch (failure) { - receipt.cleanup = `FAILED ${redact(String(failure))}`; - receipt.failure += `; credential cleanup ${receipt.cleanup}`; - throw failure; + try { await cleanup(); } + catch (cleanupError) { + receipt.cleanup = `FAILED: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`; + appendFailure('credential cleanup', cleanupError); + } + try { await persist(receipt); } + catch (persistError) { appendFailure('persist failure receipt', persistError); } + + // Publication is forbidden while disposable credential cleanup remains incomplete. + if (cleanupComplete) { + if (!commentAttempted) { + try { await current(); await publishComment(); } + catch (publicationError) { appendFailure('publish failure evidence', publicationError); } } - }); - await recover('persist failure receipt', async () => { await persist(receipt); }); - // Withdraw an attempted success on its original SHA even if the current head has moved. - if (successAttempted && receipt.commentUrl) await recover('withdraw success on pinned SHA', async () => { - await github.status(receipt.sha, 'failure', receipt.commentUrl!, 'Verification aborted; rerun required'); receipt.status = 'failure'; - }); - if (!successAttempted) { - await recover('publish failure evidence', async () => { - await current(); - receipt.commentUrl = await github.comment(receipt.pr, evidence(receipt)); - }); - await recover('publish pinned failure', async () => { - await github.status(receipt.sha, 'failure', receipt.commentUrl ?? `https://github.com/${REPO}/pull/${receipt.pr}`, 'Verification failed; see retained evidence'); receipt.status = 'failure'; - }); + try { + await github.status(receipt.sha, 'failure', receipt.commentUrl ?? `https://github.com/${REPO}/pull/${receipt.pr}`, successAttempted ? 'Verification aborted; rerun required' : 'Verification failed; rerun required'); + receipt.status = 'failure'; + } catch (statusError) { appendFailure('publish failure live-gate', statusError); } } - await recover('persist compensation receipt', async () => { await persist(receipt); }); + try { await persist(receipt); } + catch (persistError) { appendFailure('persist final failure receipt', persistError); } throw new Error(receipt.failure); } } diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f6e5b835..212ae453 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -8,9 +8,6 @@ Closes # - [ ] Bun tests, strict typecheck, static invariants, and plugin validation pass. - [ ] The exact candidate is installed in every affected harness. - [ ] The changed behavior passes from each real user surface. -- [ ] The installed version, action, and observed result appear below. +- [ ] The `live-gate` status on the final head links the evidence comment (installed version, surface, action, observed result). -Live evidence: - - -A pull request without live evidence remains a draft. Do not merge, tag, release, or roll it out. +Mergify does not queue a pull request without `live-gate` on its exact head. Do not merge, tag, release, or roll out without it. diff --git a/AGENTS.md b/AGENTS.md index dd852690..c0a3fac3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,8 +6,8 @@ Cursor's `cursor/plugins/pstack` tree is the content upstream. Keep one shared s Before opening a pull request, run the Bun tests, strict typecheck, static invariants, and plugin validation. -Nothing merges, tags, releases, or rolls out until the exact candidate is installed and the changed behavior passes a live test from the real user surface in every affected harness. Unit tests, validators, source inspection, and self-reports do not satisfy this gate. Record the installed version, surface, action, and observed result in the pull request template. A pull request without that evidence remains a draft. +Nothing merges, tags, releases, or rolls out until the exact candidate is installed and the changed behavior passes a live test from the real user surface in every affected harness. Unit tests, validators, source inspection, and self-reports do not satisfy this gate. The record is the `live-gate` commit status on that exact head, which links an evidence comment naming the installed version, surface, action, and observed result. Mergify does not queue a pull request without it. A pull request may be ready for review before its live test, but every new head, including a queue-created head, needs its own. -For Open Pstack PRs, use the shared repository-local `.claude/skills/verify-open-pstack/` skill (Codex: `.agents/skills/verify-open-pstack`). Run its independent Bun tests/typecheck separately from the shipped package. The Mac operator runs `scripts/verify.sh doctor`, then `scripts/verify.sh run --pr --self-test --claude-account --codex-account --output `; read Launch for separate probe/live directories. The trusted parent validates the explicitly selected caam 0.1.22 vault identities, copies only disposable Claude `.credentials.json` and Codex `auth.json` into run-owned config roots, records account emails, and deletes copied credentials in final cleanup without vault activation/writeback. Sandboxed candidates and provider children use run-owned `HOME`/`TMPDIR` and both isolated provider config variables while retaining real `USER`/`LOGNAME`. macOS `sandbox-exec` must deny daily-home/GitHub/SSH/provider files and Keychain/securityd access; no real Library/Keychain link or candidate login is permitted. GitHub publication stays exclusively in the trusted parent. Setup uses fixture homes only; project self-test uses planned `doctor --candidate` without vault probing. Require operator-Mac `bun test scripts/isolation.test.ts`, real-filesystem sentinel denials, no Keychain dialog, and observable authenticated API requests from both native sessions; unavailable implementation or unproven enforcement blocks the run. Builders do not publish the gate, queue, or merge; new delivery PRs start draft, but already-ready PR #111 must not be re-drafted during review remediation. The helper requires exact-head installed-source provenance, operator-reviewed native evidence, revalidated artifact hashes, and a populated/read-back PR `Live evidence:` block before gate/readiness publication. Every new head, including a queue-created head, needs a fresh run after required reviews pass; never hand-post a replacement status. Preserve redacted evidence and record credential-cleanup outcome. +For Open Pstack PRs, use the shared repository-local `.claude/skills/verify-open-pstack/` skill (Codex: `.agents/skills/verify-open-pstack`). Run the reviewed verifier from trusted `main`, recording the publisher revision separately from the candidate SHA. Read Launch for the explicit inactive caam account options and separate private probe/live directories. Candidate processes and provider children retain real `HOME`/`USER`/`LOGNAME` with both provider config roots and temporary state run-owned; full disposable credential copies retain native refresh material and are never written back to the vault. Cleanup removes only copied credential files and preserves mode-0700 raw evidence through merge. Setup coverage fails closed with `setup exercise requires #120 (setup-pstack config-home)` until #120 merges. Publication is one structured evidence comment plus exact-SHA `live-gate`, with exact copied-secret checks on the public comment only. No PR-body, ready/draft, reversal, or compensation writes, publisher-code attestation, or OS filesystem-denial claims are permitted. Builders do not publish the gate, queue, or merge. Do not add an implicit runtime timeout or a weaker-model fallback. diff --git a/docs/plans/2026-10-01-triage-and-roadmap.md b/docs/plans/2026-10-01-triage-and-roadmap.md index fedcb8f8..3ba650f1 100644 --- a/docs/plans/2026-10-01-triage-and-roadmap.md +++ b/docs/plans/2026-10-01-triage-and-roadmap.md @@ -10,7 +10,7 @@ Snapshot: `main` at `1c91e65` (Open Pstack 1.5.0, Cursor pstack 0.15.5). Cursor |---|---| | #100 / PR #101 Mergify queue | Add a required `live-gate` status first (section 6). The draft convention alone can't hold the queue, because Unfret reviews only ready PRs. Then land it, record the first queue-merged PR on #100, and close MASTRA-455, which duplicates it. | | factory-run GitHub sources (MASTRA-737) | `begin` requires a Linear issue (`factory-ops.mjs` around line 1463), and `closeout` throws `Delivery item has no Linear issue source`. open-pstack cards have been GitHub-sourced since MASTRA-450. Add a GitHub issue source to `begin` (post prior findings as an issue comment) and to `closeout` (confirm the merged PR closed the issue, or close it with the evidence link). | -| Live-gate skill (#90, reopened) | #90 was closed as completed, but there's no project verification skill on `main` or in the main checkout. Factory sandboxes have no `bun`, `claude`, `codex`, or `grok` (see the factory triage comments), so the AGENTS.md installed-candidate gate has to run on the Mac. The skill should install a PR head into Claude Code and Codex, run the changed behavior, and write the PR template's evidence block. | +| Live-gate skill (#90, reopened) | #90 was closed as completed, but there's no project verification skill on `main` or in the main checkout. Factory sandboxes have no `bun`, `claude`, `codex`, or `grok` (see the factory triage comments), so the AGENTS.md installed-candidate gate has to run on the Mac. The skill should install a PR head into Claude Code and Codex, run the changed behavior, then post one evidence comment and the exact-SHA `live-gate` status, changing nothing else on the PR (see section 7). | | Factory reachability | Resolved: `FACTORY_API_URL=https://studio2.tail062eee.ts.net:8444` reaches the API. The open-pstack project is `9af4ac1d-64a8-49d7-9d84-aa8d0e43c347`. | | MASTRA-453 | #34's Build session is parked on an unresolved `request_access`. Cancel that run when #34 is planned again. | @@ -133,9 +133,9 @@ Each issue goes through these steps: 2. Claude moves the card to Planning with `factory-run`. 3. The planner posts a plan. 4. `factory-adjudicate` judges the plan with a different model. -5. Build runs on the Factory branch. Before opening the PR, the builder runs the checks AGENTS.md requires: Bun tests, strict typecheck, static invariants, and plugin validation. Then it opens the PR as a draft. If the build host lacks Bun, as the triage sandbox did, fix the host before the first run. Don't skip the checks. -6. Claude runs the live gate on the Mac (A3) against the draft head, records the evidence in the PR, and sets the `live-gate` status on that exact head. Only then is the PR marked ready, as AGENTS.md requires. -7. Unfret reviews the ready PR. The builder fixes the findings. Each new head needs a fresh `live-gate` status, because the status is bound to the head SHA. +5. Build runs on the Factory branch. Before opening the PR, the builder runs the checks AGENTS.md requires: Bun tests, strict typecheck, static invariants, and plugin validation. Then it opens the PR, ready for review. If the build host lacks Bun, as the triage sandbox did, fix the host before the first run. Don't skip the checks. +6. Unfret reviews the PR. The builder fixes the findings until Unfret passes. +7. Claude runs the live gate (A3) on that final head. The verifier posts one evidence comment and the `live-gate` status on that exact head, and changes nothing else on the PR. Any later head, including a queue-created head, needs a fresh run. 8. Mergify queues the PR once `verify`, `Unfret`, and `live-gate` pass on the same head. 9. The builder verifies the merged result and moves the card to Done. 10. `closeout` closes the GitHub issue (A2) and records findings. @@ -165,3 +165,7 @@ Waves: - `begin` and `closeout` accept only Linear sources (MASTRA-737). This blocks every open-pstack run. - #101 gap: Unfret doesn't review drafts, so a PR has to be non-draft to get `Unfret`. Once it's non-draft, #101 queues it when `verify` and `Unfret` pass, which can happen before the live test. Fix requirement: the commit that merges must be the commit the installed-harness test ran on, and the queue must never wait on a check that nothing can post. The likely shape, to be confirmed in the #101 session against Mergify's docs: in-place queue checks (empty `merge_conditions`, `batch_size: 1`, so Mergify creates no draft batch PR), and `queue_conditions` that require `-draft`, `verify`, `Unfret`, `live-gate` (a commit status that the #90 skill posts on the exact head), and the branch being up to date with `main`. With the merge-commit method and an up-to-date branch, the merged tree is the tested tree. Any `main` change forces a rebase, which creates a new head that needs fresh `verify`, `Unfret`, and `live-gate`. Corrections are posted on the #101 Greptile thread. - Order: MASTRA-737, then the live-gate check in #101 and its merge, then #90. After that, Wave 1. Until #90 exists, the operator posts `live-gate` by hand on each PR head, after running the installed-harness test, with a link to the evidence. #101 is merged that way as well. + +## 7. Gate change (2026-10-03) + +AGENTS.md now treats the exact-head `live-gate` status and its linked evidence comment as the live-evidence record, in place of the PR template and the draft rule. Mergify's required `live-gate` check enforces it. The change came out of the #111 step-back: every way the verifier changed draft, ready, or body state produced new review findings, and draft state was no longer a merge control. The verifier (#90) posts only the comment and the status. #106's runbook follows this order: review first, then the live gate on the final head. diff --git a/plugins/pstack/skills/poteto-mode/references/provider-dispatch.md b/plugins/pstack/skills/poteto-mode/references/provider-dispatch.md index e0d62a2a..005c5620 100644 --- a/plugins/pstack/skills/poteto-mode/references/provider-dispatch.md +++ b/plugins/pstack/skills/poteto-mode/references/provider-dispatch.md @@ -80,7 +80,7 @@ Start the background process, continue launching the other lanes, then drain the The runner and its preflight have no implicit timeout. Do not invent a duration from role, mode, or a convenient round number; real implementation lanes can run for 90 minutes or much longer. Pass `--timeout` only when the user, an external service deadline, or a measured task contract supplies a real bound. That value starts at wrapper entry, before module loading and argument parsing, and remains one absolute deadline across setup, preflight, model execution, and output capture. It is never a fresh allowance per child, and long waits are armed in runtime-safe chunks without shortening the supplied deadline. Otherwise supervise liveness through the retained background task/session handle and cancel manually only on evidence that the run is dead. Cancel through that retained handle so the runner receives SIGINT or SIGTERM, sends it to an active child when one remains, stops waiting on inherited output pipes, removes the empty output reservation, and writes a `cancelled` receipt. Preserve that receipt; a retry is a new attempt with new unique output and receipt paths. Unchanged running state is not a dropout, and Claude's ten-minute foreground ceiling is never a reason to terminate a healthy lane. -Read-only mode maps to Claude plan mode with project-only settings and an explicit tool list, Codex's read-only sandbox, and Grok plan mode plus its `read-only` sandbox and read-oriented tool list. Grok's built-in read-only profile deliberately keeps its own state and system temporary directories writable, so point a read-only Grok lane at the actual checkout rather than a worktree under `/tmp`, `/var/tmp`, or the host's temporary directory. `isolated-write` maps to Claude `acceptEdits` with project-only settings, Codex `workspace-write`, and Grok `acceptEdits` plus its `workspace` sandbox and write-capable tool list. Give every writer only a dedicated worktree or output directory. Never route a writer into the primary checkout. +Read-only mode maps to Claude plan mode with project-only settings and an explicit tool list, Codex's read-only sandbox, and Grok auto mode plus its `read-only` sandbox and read-oriented tool list. Headless Grok cancels the whole turn on a permission prompt, which its `plan` mode raises for any shell command outside its built-in read-only list; auto mode reports a blocked call to the model instead. Grok's built-in read-only profile deliberately keeps its own state and system temporary directories writable, so point a read-only Grok lane at the actual checkout rather than a worktree under `/tmp`, `/var/tmp`, or the host's temporary directory. `isolated-write` maps to Claude `acceptEdits` with project-only settings, Codex `workspace-write`, and Grok auto mode plus its `workspace` sandbox and write-capable tool list. Grok uses auto mode in both access modes because `acceptEdits` raises the same turn-cancelling prompt for a shell command outside Grok's built-in list. Give every writer only a dedicated worktree or output directory. Never route a writer into the primary checkout. Every concurrent external lane needs distinct prompt, output, and receipt paths. The launcher reserves output and receipt paths exclusively and refuses to overwrite them. diff --git a/plugins/pstack/skills/poteto-mode/scripts/runner/commands.test.ts b/plugins/pstack/skills/poteto-mode/scripts/runner/commands.test.ts index ea697ff2..a1574538 100644 --- a/plugins/pstack/skills/poteto-mode/scripts/runner/commands.test.ts +++ b/plugins/pstack/skills/poteto-mode/scripts/runner/commands.test.ts @@ -96,7 +96,7 @@ describe("invocationCommand", () => { "--reasoning-effort", "xhigh", "--permission-mode", - "plan", + "auto", "--sandbox", "read-only", "--tools", @@ -124,7 +124,7 @@ describe("invocationCommand", () => { expect(grok.args).toEqual( expect.arrayContaining([ "--permission-mode", - "acceptEdits", + "auto", "--sandbox", "workspace", "--tools", diff --git a/plugins/pstack/skills/poteto-mode/scripts/runner/commands.ts b/plugins/pstack/skills/poteto-mode/scripts/runner/commands.ts index 5f2b10c6..6edb4693 100644 --- a/plugins/pstack/skills/poteto-mode/scripts/runner/commands.ts +++ b/plugins/pstack/skills/poteto-mode/scripts/runner/commands.ts @@ -128,8 +128,11 @@ export function invocationCommand(options: RunnerOptions): CommandSpec { options.model, "--reasoning-effort", options.effort, + // Headless Grok cancels the whole turn on a permission prompt, in both access modes. + // Auto mode reports a blocked call to the model instead; the sandbox still confines + // writes (read-only, or workspace for isolated-write). "--permission-mode", - permissionMode(options.mode), + "auto", "--sandbox", grokSandbox(options.mode), "--tools", diff --git a/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts b/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts index 20743b52..c114d102 100644 --- a/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts +++ b/plugins/pstack/skills/poteto-mode/scripts/runner/run.test.ts @@ -120,6 +120,9 @@ if (name === "claude") { console.log(JSON.stringify({type:"thread.started",thread_id:"o1"})); console.log(JSON.stringify({type:"item.completed",item:{type:"agent_message",text:"CODEX_OK"}})); console.log(JSON.stringify({type:"turn.completed",usage:{input_tokens:20,cached_input_tokens:5,output_tokens:3,reasoning_output_tokens:1}})); +} else if (process.env.FAKE_GROK_ERROR_RESULT === "1") { + console.log(JSON.stringify({type:"system",subtype:"init",skills:Array(500).fill("skill-name")})); + console.log(JSON.stringify({type:"result",subtype:"error_during_execution",is_error:true,stop_reason:"cancelled",errors:["cancelled"]})); } else { console.log(JSON.stringify({type:"assistant",message:{content:[{type:"text",text:"progress"}]}})); console.log(JSON.stringify({type:"result",subtype:"success",is_error:false,result:"GROK_OK",session_id:"g1",usage:{input_tokens:30,output_tokens:4,total_tokens:34},total_cost_usd:0.02,modelUsage:{[model + "-build"]:{}}})); @@ -244,6 +247,7 @@ beforeEach(() => { delete process.env.FAKE_DESCENDANT_HOLDS_PIPES_MS; delete process.env.FAKE_DESCENDANT_PID_PATH; delete process.env.FAKE_SELF_SIGNAL; + delete process.env.FAKE_GROK_ERROR_RESULT; }); afterEach(() => { @@ -268,6 +272,7 @@ afterEach(() => { delete process.env.FAKE_DESCENDANT_HOLDS_PIPES_MS; delete process.env.FAKE_DESCENDANT_PID_PATH; delete process.env.FAKE_SELF_SIGNAL; + delete process.env.FAKE_GROK_ERROR_RESULT; rmSync(scratch, { recursive: true, force: true }); }); @@ -307,6 +312,22 @@ describe("runLane", () => { }); }); + it("keeps Grok's terminal error result in malformed-output evidence", async () => { + process.env.FAKE_GROK_ERROR_RESULT = "1"; + const input = options("grok", "grok-error-result"); + const result = await runLane(input); + expect(result.exitCode).toBe(65); + expect(existsSync(input.outputPath)).toBe(false); + const recorded = receipt(input.receiptPath); + expect(recorded).toMatchObject({ + status: "malformed-output", + exitCode: 0, + error: { message: "grok reported an error result" }, + }); + expect(recorded.error?.evidence).toContain('"stop_reason":"cancelled","errors":["cancelled"]'); + expect(recorded.error?.evidence.length).toBeLessThanOrEqual(4_000); + }); + it("classifies an unavailable model without falling back", async () => { process.env.FAKE_INVALID_MODEL = "1"; const input = options("codex"); diff --git a/plugins/pstack/skills/poteto-mode/scripts/runner/run.ts b/plugins/pstack/skills/poteto-mode/scripts/runner/run.ts index 054564a4..aa887f00 100644 --- a/plugins/pstack/skills/poteto-mode/scripts/runner/run.ts +++ b/plugins/pstack/skills/poteto-mode/scripts/runner/run.ts @@ -52,6 +52,11 @@ function evidence(value: string): string { return value.trim().slice(0, ERROR_EVIDENCE_LIMIT); } +// A provider's terminal event ends its stream, so malformed output keeps the tail. +function trailingEvidence(value: string): string { + return value.trim().slice(-ERROR_EVIDENCE_LIMIT); +} + function removeIfExists(path: string): void { if (existsSync(path)) unlinkSync(path); } @@ -839,7 +844,7 @@ async function executeLane( costUsd: null, error: { message, - evidence: evidence(`${result.stderr}\n${result.stdout}`), + evidence: trailingEvidence(`${result.stderr}\n${result.stdout}`), }, }); } From 28d3d6cdf6a29e0450383bf1d2ea7e4e06713d1b Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 05:26:57 +0000 Subject: [PATCH 16/20] fix(#90): support native credential sources for external contributors Replace required account-vault infrastructure with explicit authenticated Claude and Codex configuration directories, native status checks, and credential-file-only candidate copies. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .claude/skills/verify-open-pstack/SKILL.md | 26 +-- .../verify-open-pstack/features/isolation.md | 12 +- .../verify-open-pstack/features/runner.md | 2 +- .../verify-open-pstack/features/setup.md | 2 +- .../features/shipped-tools.md | 2 +- .../features/skill-invocation.md | 4 +- .../verify-open-pstack/scripts/cli.test.ts | 63 +++--- .../skills/verify-open-pstack/scripts/cli.ts | 28 +-- .../verify-open-pstack/scripts/doctor.test.ts | 111 ++++++---- .../verify-open-pstack/scripts/doctor.ts | 40 ++-- .../scripts/harness.test.ts | 108 ++++++---- .../verify-open-pstack/scripts/harness.ts | 27 ++- .../scripts/isolation.test.ts | 193 ++++++++---------- .../verify-open-pstack/scripts/isolation.ts | 68 +++--- .../verify-open-pstack/scripts/types.ts | 4 +- AGENTS.md | 2 +- 16 files changed, 360 insertions(+), 332 deletions(-) diff --git a/.claude/skills/verify-open-pstack/SKILL.md b/.claude/skills/verify-open-pstack/SKILL.md index ea34b027..d3d6724f 100644 --- a/.claude/skills/verify-open-pstack/SKILL.md +++ b/.claude/skills/verify-open-pstack/SKILL.md @@ -19,27 +19,27 @@ Then run from the trusted publisher checkout: ```sh PR=111 # supplied delivery PR number -CLAUDE_ACCOUNT='selected-claude-account@example.com' # explicit caam vault identity -CODEX_ACCOUNT='selected-codex-account@example.com' # explicit caam vault identity +CLAUDE_SOURCE='/absolute/path/to/authenticated/claude-config' +CODEX_SOURCE='/absolute/path/to/authenticated/codex-home' SESSION="$(mktemp -d "${TMPDIR:-/tmp}/open-pstack-evidence.XXXXXX")" EVIDENCE="$SESSION/live" # must not exist yet .claude/skills/verify-open-pstack/scripts/verify.sh doctor \ - --claude-account "$CLAUDE_ACCOUNT" --codex-account "$CODEX_ACCOUNT" --output "$SESSION/probe" + --claude-config "$CLAUDE_SOURCE" --codex-home "$CODEX_SOURCE" --output "$SESSION/probe" .claude/skills/verify-open-pstack/scripts/verify.sh run --pr "$PR" --self-test \ - --claude-account "$CLAUDE_ACCOUNT" --codex-account "$CODEX_ACCOUNT" --output "$EVIDENCE" + --claude-config "$CLAUDE_SOURCE" --codex-home "$CODEX_SOURCE" --output "$EVIDENCE" ``` -Replace both placeholders with explicitly selected caam 0.1.22 vault identities; never choose defaults or start a login flow. A selected identity may be active in the operator's daily Claude or Codex use. Omit `--self-test` for ordinary plugin verification; it is mandatory for this skill's delivery. +Replace both placeholders with existing operator-pre-authenticated normal-login configuration directories. Selection is explicit; a selected account may be active in daily use. No profile manager or maintainer-specific infrastructure is required. Authenticate those directories manually before verification; the verifier never starts a login flow. Omit `--self-test` for ordinary plugin verification; it is mandatory for this skill's delivery. The trusted `main` publisher revision is recorded separately from the PR's candidate SHA. PR #111 bootstrap must use the maintainer-reviewed trusted publisher head, while all candidate installation, classification, evidence, comments, and `live-gate` status remain bound to PR #111's exact candidate SHA. Use a fresh private output directory for every run. The verifier requires an open same-repository PR, pins exact head and base SHAs, classifies immutable Git objects, and rechecks them at phase and publication boundaries. Unknown paths abort; evidence never transfers to another head. ## Doctor -The trusted-parent doctor checks Bun, git, both native harness CLIs, and the capabilities needed by the run. For each explicitly selected identity it must run `caam limits --format json` (`` is `claude` or `codex`) and inspect the chosen `profile_name` for that provider. Selection is explicit regardless of whether the profile is active in daily use. An unauthorized or expired result fails closed and reports the exact manual repair command `caam add --no-activate --force`; the verifier must never execute that repair automatically. Candidate-mode doctor checks only candidate-safe capabilities and does not probe caam or publisher authentication. Capability checks do not prove installation, authentication, or live behavior; missing requirements fail closed. +The trusted-parent doctor checks Bun, git, both native harness CLIs, and the capabilities needed by the run. It requires both supplied source directories to exist and validates their authentication with `CLAUDE_CONFIG_DIR= claude auth status` and `CODEX_HOME= codex login status`. Unauthorized, expired, missing, or unusable authentication fails closed with the corresponding manual login command: `CLAUDE_CONFIG_DIR= claude auth login` or `CODEX_HOME= codex login`. The verifier never executes login or repairs credentials automatically. Candidate-mode doctor checks only candidate-safe capabilities and does not inspect source credentials or publisher authentication. Capability and authentication checks do not prove installation or live behavior; missing requirements fail closed. Candidates retain the operator's real `HOME`, `USER`, and `LOGNAME`. `CLAUDE_CONFIG_DIR`, `CODEX_HOME`, `TMPDIR`, `GH_CONFIG_DIR`, and other XDG/config/cache locations are run-owned. Git configuration is disabled or run-owned. This separation is operational isolation, not an OS security boundary: the verifier makes no Seatbelt, source-protection, Keychain-denial, daily-home secrecy, or Mac filesystem-denial claim. -The trusted parent validates each explicitly named caam identity. It copies only `claude//.credentials.json` into run-owned `CLAUDE_CONFIG_DIR` and `codex//auth.json` into run-owned `CODEX_HOME`. The disposable copies keep their refresh material so the native clients can refresh during the session; they are never written back to the vault. Missing files, unknown credential formats, identity mismatch, unusable authentication, or a `caam limits` `profile_name` that does not identify the selected provider profile fails closed. Never log in, repair credentials automatically, or fall back to daily provider configuration. +The trusted parent copies only `.credentials.json` from `--claude-config ` into run-owned `CLAUDE_CONFIG_DIR` and `auth.json` from `--codex-home ` into run-owned `CODEX_HOME`. It does not copy settings or other source-directory state. Complete recognized credential files retain their refresh material so native clients can refresh during the session; refreshed copies are never written back to the source directories. Missing files, unknown credential formats, or unusable authentication fail closed. Never log in, repair credentials automatically, or fall back to daily provider configuration. ## Drive @@ -47,7 +47,7 @@ Read `features/README.md` and every selected feature document. Runtime instructi Create detached exact-head candidate checkouts, run-owned provider/config/temp roots, and fixture workspaces. Claude loads the explicit candidate plugin with candidate settings; Codex installs the pinned local marketplace/plugin. Verify installed plugin and project-skill sources against immutable candidate Git provenance before and after exercise. Version strings alone are insufficient. -Every candidate command and external-provider descendant uses the recorded candidate environment. Candidate processes receive disposable provider credentials only, never GitHub publisher credentials. No candidate login, vault activation/writeback, implicit timeout, or weaker-model fallback is permitted. +Every candidate command and external-provider descendant uses the recorded candidate environment. Candidate processes receive disposable provider credentials only, never GitHub publisher credentials. No candidate login, source-credential writeback, implicit timeout, or weaker-model fallback is permitted. For every selected feature, exercise changed sub-features on each actual consuming native surface and retain real native tool calls and concrete fixture effects. Asset consumers come from the pinned plugin manifests; the current `plugins/pstack/assets/logo.png` is consumed by the Codex manifest only and therefore requires the Codex installed surface, not an invented Claude asset exercise. Save raw transcripts and artifacts in the private run root. Require operator-reviewed surface, action, expected/observed result, and retained artifact paths. Model self-reports and direct CLI tests are not installed-harness evidence. @@ -57,7 +57,7 @@ For `project-skill`, invoke `/verify-open-pstack` in Claude and `$verify-open-ps ## Evidence -Create the requested run root as a private mode-0700 directory. Keep raw receipts, transcripts, artifacts, selected account names, immutable source/installation provenance, and credential-cleanup outcome private in that root. Raw evidence may contain operational details and is not a public artifact. Keep it through merge; after the merged evidence is archived, only the operator deletes the named run root. Never retain copied credential files as evidence. +Create the requested run root as a private mode-0700 directory. Keep raw receipts, transcripts, artifacts, selected credential-source paths, immutable source/installation provenance, and credential-cleanup outcome private in that root. Raw evidence may contain operational details and is not a public artifact. Keep it through merge; after the merged evidence is archived, only the operator deletes the named run root. Never retain copied credential files as evidence. Before publication, revalidate retained transcript and artifact hashes. The trusted parent publishes one bounded, structured PR comment for the exact candidate SHA and sets `live-gate` on that same SHA with the comment URL as its target. It never edits the PR body, marks ready or draft, reverses a transition, or performs PR-state compensation. No other PR write is permitted. Head movement requires a fresh run and status on the new exact SHA. @@ -65,13 +65,13 @@ The public-comment guard compares the rendered comment against the exact credent ## Cleanup -Quit native sessions normally and remove only the copied disposable Claude `.credentials.json` and Codex `auth.json` files on success or failure, including refreshes written to those same run-owned paths. Preserve candidate state and raw evidence. Record cleanup outcome; failed credential cleanup blocks publication. Never modify the caam vault or daily provider files. Preserve the private mode-0700 raw evidence root through merge, archive it after merge, and let the operator delete only the named run root. Do not kill unrelated processes. +Quit native sessions normally and remove only the copied disposable Claude `.credentials.json` and Codex `auth.json` files on success or failure, including refreshes written to those same run-owned paths. Preserve candidate state and raw evidence. Record cleanup outcome; failed credential cleanup blocks publication. Never modify the supplied source credential files or daily provider files. Preserve the private mode-0700 raw evidence root through merge, archive it after merge, and let the operator delete only the named run root. Do not kill unrelated processes. ## Helpers -- `scripts/verify.sh doctor --output `: trusted-parent capability report, no installation/publication. -- `scripts/verify.sh doctor --candidate --output `: candidate-safe self-test probe; no vault or publisher authentication probing. -- `scripts/verify.sh run --pr --claude-account --codex-account [--self-test] --output `: supervised exact-head verification; mapped exercises require an interactive terminal. +- `scripts/verify.sh doctor --claude-config --codex-home --output `: trusted-parent capability and source-authentication report, no installation/publication. +- `scripts/verify.sh doctor --candidate --output `: candidate-safe self-test probe; no source-credential or publisher authentication probing. +- `scripts/verify.sh run --pr --claude-config --codex-home [--self-test] --output `: supervised exact-head verification; mapped exercises require an interactive terminal. - `bun run test` and `bun run typecheck`: development checks after a separate frozen dependency install, not live proof. - `features/registry.json`: maintained path ownership; unknown runtime paths block. diff --git a/.claude/skills/verify-open-pstack/features/isolation.md b/.claude/skills/verify-open-pstack/features/isolation.md index c1aa3c3a..c20a67ea 100644 --- a/.claude/skills/verify-open-pstack/features/isolation.md +++ b/.claude/skills/verify-open-pstack/features/isolation.md @@ -1,16 +1,16 @@ # Disposable credential and run-state contract -The verifier uses run-owned provider/configuration state and explicitly selected caam credentials copied for the run. This is operational state separation, not a Seatbelt or macOS filesystem security boundary. +The verifier uses run-owned provider/configuration state and credential files copied from explicitly supplied operator-pre-authenticated normal-login directories. No profile manager or maintainer-specific infrastructure is required. This is operational state separation, not a Seatbelt or macOS filesystem security boundary. ## Trusted parent Run the publisher from a maintainer-reviewed trusted `main` checkout. Record that publisher revision separately from the PR's exact candidate SHA; never treat publisher code provenance as candidate proof. For PR #111 bootstrap, use the reviewed trusted publisher head while pinning all candidate evidence and status to PR #111's reviewed exact head. -Require explicit `--claude-account EMAIL` and `--codex-account EMAIL`; never infer or choose a default identity. A selected profile may be active in the operator's daily Claude or Codex use. Record selected account emails privately in the receipt. +Require explicit `--claude-config ` and `--codex-home ` pointing to existing operator-pre-authenticated normal-login configuration directories; never infer source directories or choose defaults. A selected account may be active in daily use. Record source paths privately in the receipt. -For each selection, trusted-parent doctor must run `caam limits --format json`, using `claude` or `codex` as ``, and inspect the chosen `profile_name` for that provider. A mismatched profile fails closed. An unauthorized or expired result also fails closed and reports the exact manual repair command `caam add --no-activate --force`. Never execute the repair automatically, switch identities, or fall back automatically. +Trusted-parent doctor validates those source directories with `CLAUDE_CONFIG_DIR= claude auth status` and `CODEX_HOME= codex login status`. Missing or unusable authentication, including unauthorized or expired credentials, fails closed with the corresponding manual remediation: `CLAUDE_CONFIG_DIR= claude auth login` or `CODEX_HOME= codex login`. Never execute login automatically, switch identities, or fall back automatically. -Copy only `claude//.credentials.json` into run-owned `CLAUDE_CONFIG_DIR` and `codex//auth.json` into run-owned `CODEX_HOME`. Copy the complete recognized credential file, including refresh material needed by the native client. Never write refreshed state back to the caam vault. Missing/mismatched files, unknown credential formats, or unusable authentication fail closed. GitHub authentication and publication remain exclusively in the trusted parent. +Copy only the source Claude `.credentials.json` into run-owned `CLAUDE_CONFIG_DIR` and source Codex `auth.json` into run-owned `CODEX_HOME`. Copy the complete recognized credential file, including refresh material needed by the native client, but no other configuration or source state. Never write refreshed state back to source directories. Missing credential files, unknown credential formats, or unusable authentication fail closed. GitHub authentication and publication remain exclusively in the trusted parent. ## Candidate process state @@ -19,7 +19,7 @@ Copy only `claude//.credentials.json` into run-owned `CLAUDE_CONFIG_DIR - Disable global/system git configuration and credential helpers for candidate commands. - Claude uses the explicit candidate plugin, empty settings, and project setting sources. Codex uses pinned local marketplace/plugin installation. - Provider descendants inherit both run-owned provider config roots. Never pass GitHub publisher credentials or daily provider API keys. -- Never invoke login, activate or modify the vault, or copy daily provider configuration as fallback. +- Never invoke login, modify source credentials, or copy daily provider configuration as fallback. These controls do not deny a process access to real `HOME`, Keychain, or candidate sources. Do not claim Seatbelt enforcement, source write protection, Keychain/securityd denial, daily-home denial, or OS-backed secrecy. Provenance rechecks detect changed candidate/installed files but are not prevention. @@ -39,4 +39,4 @@ Register exact access/refresh/id-token values only from the copied, recognized c Create each output root fresh, outside the repository, with mode 0700; use private files beneath it. Retain raw transcripts, artifacts, receipts, and cleanup results in that private root through merge. Copied credential files are session state, not evidence: remove only the copied Claude `.credentials.json` and Codex `auth.json` files in run-owned config roots after native sessions on success or failure, including refreshes written to those same paths. Preserve candidate state and raw evidence. Cleanup failure blocks GitHub publication. -After merge, archive the retained evidence and let the operator delete only the named run root. Never modify the caam vault or daily provider files, and never recursively delete an operator directory. +After merge, archive the retained evidence and let the operator delete only the named run root. Never modify supplied source credential files or daily provider files, and never recursively delete an operator directory. diff --git a/.claude/skills/verify-open-pstack/features/runner.md b/.claude/skills/verify-open-pstack/features/runner.md index 071869b7..aecdb348 100644 --- a/.claude/skills/verify-open-pstack/features/runner.md +++ b/.claude/skills/verify-open-pstack/features/runner.md @@ -16,4 +16,4 @@ Read the candidate's `pstack-runner --help` and installed mapping through the na Every external-provider child retains real `HOME`, `USER`, and `LOGNAME` while inheriting run-owned `TMPDIR`, `GH_CONFIG_DIR`, XDG/config/cache paths, `CLAUDE_CONFIG_DIR`, and `CODEX_HOME`. This state separation is not an OS access-denial guarantee. Never let Claude→Codex or Codex→Claude fall back to daily provider configuration, and never pass publisher credentials. -The trusted parent copies complete recognized credential files for explicitly selected caam accounts so native refresh can work, and removes only the copied credential files (including refreshes written to those paths) during cleanup without writing back to the vault or deleting raw evidence. A selected account may be active in daily use. Trusted-parent doctor runs `caam limits --format json` and inspects the chosen `profile_name` for each provider; a mismatch, missing provider, or unusable authentication is a failure, not a skip or weaker-model fallback. Unauthorized or expired results fail closed with the exact manual repair command `caam add --no-activate --force`; never execute it automatically. Do not add implicit timeouts. A model assertion that a child ran is insufficient; inspect the actual receipt and output. Never reuse output from another SHA or run. +Supply operator-pre-authenticated normal-login directories with `--claude-config ` and `--codex-home `. The trusted parent copies only complete recognized `.credentials.json` and `auth.json` files so native refresh can work, and removes only those copied credential files (including refreshes written to those paths) during cleanup without writing back to source directories or deleting raw evidence. A selected account may be active in daily use. Trusted-parent doctor validates the supplied directories with `claude auth status` under the source `CLAUDE_CONFIG_DIR` and `codex login status` under the source `CODEX_HOME`; missing or unusable authentication is a failure, not a skip or weaker-model fallback. Authentication failures report `CLAUDE_CONFIG_DIR= claude auth login` or `CODEX_HOME= codex login`; never execute login automatically. Do not add implicit timeouts. A model assertion that a child ran is insufficient; inspect the actual receipt and output. Never reuse output from another SHA or run. diff --git a/.claude/skills/verify-open-pstack/features/setup.md b/.claude/skills/verify-open-pstack/features/setup.md index d11132e4..6fb2556d 100644 --- a/.claude/skills/verify-open-pstack/features/setup.md +++ b/.claude/skills/verify-open-pstack/features/setup.md @@ -20,4 +20,4 @@ After #120 merges, use its explicit config-home to target run-owned state while ## Gotchas -`CLAUDE_CONFIG_DIR`, `CODEX_HOME`, XDG/GitHub config, and temp paths are run-owned, but `HOME` is intentionally real. That is why setup cannot run safely before #120. Do not copy daily configuration, activate accounts, log in, or write back to caam. Verification accounts remain explicitly selected, but may be active in daily use. Trusted-parent doctor runs `caam limits --format json` and inspects the selected provider's `profile_name`; unauthorized or expired results fail closed with the exact manual repair command `caam add --no-activate --force`, which the verifier never executes automatically. Generated configuration by itself does not prove the consuming surface. +`CLAUDE_CONFIG_DIR`, `CODEX_HOME`, XDG/GitHub config, and temp paths are run-owned, but `HOME` is intentionally real. That is why setup cannot run safely before #120. Supply operator-pre-authenticated normal-login directories with `--claude-config ` and `--codex-home `; selected accounts may be active in daily use. Copy only `.credentials.json` and `auth.json` into run-owned provider roots, preserving refresh material. Do not copy other daily configuration, log in, or write credentials back to source directories. Trusted-parent doctor validates sources with `claude auth status` under the supplied `CLAUDE_CONFIG_DIR` and `codex login status` under the supplied `CODEX_HOME`; authentication failures fail closed with `CLAUDE_CONFIG_DIR= claude auth login` or `CODEX_HOME= codex login`, which the verifier never executes automatically. Generated configuration by itself does not prove the consuming surface. diff --git a/.claude/skills/verify-open-pstack/features/shipped-tools.md b/.claude/skills/verify-open-pstack/features/shipped-tools.md index 36d51042..e67b1dec 100644 --- a/.claude/skills/verify-open-pstack/features/shipped-tools.md +++ b/.claude/skills/verify-open-pstack/features/shipped-tools.md @@ -16,4 +16,4 @@ Use the candidate's installed help and documented arguments. Create fixture stat ## Gotchas -A mock test is not live proof. Never perform any PR write from a candidate process or edit daily workflow state as a fixture. Do not infer a successful remote read from an empty response. Candidate tools and descendants retain real `HOME`, `USER`, and `LOGNAME` but use run-owned provider/GitHub/XDG/config/cache/temp paths and explicitly selected caam credentials; the selected profiles may be active in daily use. They do not receive publisher credentials. Trusted-parent doctor runs `caam limits --format json` and inspects the chosen `profile_name` for each provider. Unauthorized or expired results fail closed with the exact manual repair command `caam add --no-activate --force`; never execute repair automatically. This does not claim OS denial of daily-home or Keychain access. Missing required access fails the feature; preserve its private reason and publish only the bounded structured exact-SHA result. +A mock test is not live proof. Never perform any PR write from a candidate process or edit daily workflow state as a fixture. Do not infer a successful remote read from an empty response. Candidate tools and descendants retain real `HOME`, `USER`, and `LOGNAME` but use run-owned provider/GitHub/XDG/config/cache/temp paths. Supply operator-pre-authenticated normal-login directories with `--claude-config ` and `--codex-home `; selected accounts may be active in daily use. Only complete recognized `.credentials.json` and `auth.json` files are copied, preserving refresh material without source writeback. Candidates do not receive publisher credentials. Trusted-parent doctor validates sources with `claude auth status` under the supplied `CLAUDE_CONFIG_DIR` and `codex login status` under the supplied `CODEX_HOME`. Authentication failures fail closed with `CLAUDE_CONFIG_DIR= claude auth login` or `CODEX_HOME= codex login`; never execute login automatically. This does not claim OS denial of daily-home or Keychain access. Missing required access fails the feature; preserve its private reason and publish only the bounded structured exact-SHA result. diff --git a/.claude/skills/verify-open-pstack/features/skill-invocation.md b/.claude/skills/verify-open-pstack/features/skill-invocation.md index 6d9b3ab0..a5b790da 100644 --- a/.claude/skills/verify-open-pstack/features/skill-invocation.md +++ b/.claude/skills/verify-open-pstack/features/skill-invocation.md @@ -10,11 +10,11 @@ In a fresh Claude session invoke `/pstack:`. In Codex use the installed sk ## Driving it with verify-open-pstack -Launch the prepared harness with real `HOME`, `USER`, and `LOGNAME`; run-owned `TMPDIR`, GitHub/XDG/config/cache paths, `CLAUDE_CONFIG_DIR`, and `CODEX_HOME`; and no publisher credentials. Use trusted-parent copies of complete recognized credentials for explicitly selected caam accounts so native refresh can work; selected profiles may be active in daily use. Trusted-parent doctor must run `caam limits --format json` and inspect the chosen `profile_name` for each provider. Unauthorized or expired results fail closed with the exact manual repair command `caam add --no-activate --force`; the verifier never executes repair automatically. No login, daily-provider fallback, or vault writeback is allowed. +Launch the prepared harness with real `HOME`, `USER`, and `LOGNAME`; run-owned `TMPDIR`, GitHub/XDG/config/cache paths, `CLAUDE_CONFIG_DIR`, and `CODEX_HOME`; and no publisher credentials. Supply operator-pre-authenticated normal-login directories with `--claude-config ` and `--codex-home `; selected accounts may be active in daily use. Use trusted-parent copies of only complete recognized `.credentials.json` and `auth.json` files, retaining refresh material. Trusted-parent doctor validates sources with `claude auth status` under the supplied `CLAUDE_CONFIG_DIR` and `codex login status` under the supplied `CODEX_HOME`. Authentication failures fail closed with `CLAUDE_CONFIG_DIR= claude auth login` or `CODEX_HOME= codex login`; the verifier never executes login automatically. No candidate login, daily-provider fallback, or source-credential writeback is allowed. For each selected name, use a disposable fixture appropriate to its documented input, invoke the installed surface, and exercise every changed sub-feature. Observe native tool/skill loading and a concrete fixture effect, not the assistant saying it passed. Save the raw transcript and an artifact showing the actual result (diff, generated file, tool receipt, or UI capture) under the private mode-0700 evidence root. Include input, expected result, observed result, and each changed sub-feature in the operator review. -For `project-skill`, invoke the discovered skill and ask it to run `verify.sh doctor --candidate --output `; candidate mode must not probe caam or publisher authentication. Forbid recursive `run` and publication. Preserve the native invocation transcript plus `doctor.json`, checking the canonical executable path and immutable source provenance belong to the pinned project skill. The Codex symlink and Claude canonical directory must both be discovered in fresh sessions. If candidate mode is unavailable, stop; do not substitute trusted-parent doctor. +For `project-skill`, invoke the discovered skill and ask it to run `verify.sh doctor --candidate --output `; candidate mode must not inspect source credentials or probe publisher authentication. Forbid recursive `run` and publication. Preserve the native invocation transcript plus `doctor.json`, checking the canonical executable path and immutable source provenance belong to the pinned project skill. The Codex symlink and Claude canonical directory must both be discovered in fresh sessions. If candidate mode is unavailable, stop; do not substitute trusted-parent doctor. ## Gotchas diff --git a/.claude/skills/verify-open-pstack/scripts/cli.test.ts b/.claude/skills/verify-open-pstack/scripts/cli.test.ts index 7f3b34a6..c109092d 100644 --- a/.claude/skills/verify-open-pstack/scripts/cli.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/cli.test.ts @@ -10,7 +10,7 @@ import { REPO } from './types.ts'; import { verify } from './verify.ts'; import { validateRegistry } from './core.ts'; import registry from '../features/registry.json'; -const accounts = ['--claude-account', 'eric@litman.org', '--codex-account', 'eric@healthspanners.com']; +const sources = ['--claude-config', '/operator/claude config', '--codex-home', '/operator/codex home']; test('publisher revision is recorded independently of candidate proof', async () => { const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-publisher-revision-'))); @@ -32,7 +32,7 @@ test('publisher revision is recorded independently of candidate proof', async () test('candidate mode is an explicit doctor-only flag', () => { expect(parse(['doctor', '--candidate', '--output', '/fresh/probe'])).toEqual({ mode: 'doctor', output: '/fresh/probe', pr: 0, selfTest: false, candidate: true }); expect(() => parse(['doctor', '--candidate', '--candidate', '--output', '/fresh/probe'])).toThrow('Duplicate option'); - expect(() => parse(['run', '--candidate', '--pr', '111', '--output', '/fresh/run', ...accounts])).toThrow('Unknown option'); + expect(() => parse(['run', '--candidate', '--pr', '111', '--output', '/fresh/run', ...sources])).toThrow('Unknown option'); }); test('candidate doctor probes only help and versions with private roots already present', async () => { const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-doctor-'))), calls: string[][] = []; @@ -56,39 +56,48 @@ test('candidate doctor probes only help and versions with private roots already expect(report.candidate).toBe(true); expect(report.result).toBe('pass'); expect(Object.keys(report.checks)).toEqual(['bun', 'git', 'claude', 'codex', 'claudeHelp', 'codex plugin marketplace add --help', 'codex plugin add --help']); expect(calls.every(args => args.includes('--version') || args.includes('--help'))).toBe(true); - expect(calls.some(args => ['gh', 'caam', '/usr/bin/sandbox-exec'].includes(args[0]!))).toBe(false); + expect(calls.some(args => args[0] === 'gh')).toBe(false); expect(calls.some(args => args.includes('auth') || args.includes('login'))).toBe(false); - await doctor(root, run, 'darwin'); - expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).candidate).toBe(false); - expect(calls.some(args => args[0] === 'gh' && args[1] === '--version')).toBe(true); } finally { await rm(root, { recursive: true, force: true }); } }); test('CLI strictly validates run inputs and permits read-only child doctor', () => { - expect(parse(['doctor', '--output', '/fresh/probe'])).toEqual({ mode: 'doctor', output: '/fresh/probe', pr: 0, selfTest: false, candidate: false }); - expect(parse(['run', '--pr', '123', '--self-test', '--output', '/fresh/run', ...accounts])).toEqual({ mode: 'run', output: '/fresh/run', pr: 123, selfTest: true, claudeAccount: 'eric@litman.org', codexAccount: 'eric@healthspanners.com' }); - for (const args of [[], ['wat'], ['run', '--pr', '0', '--output', '/tmp/a', ...accounts], ['run', '--pr', '1.5', '--output', '/tmp/a', ...accounts], ['doctor', '--pr', '1', '--output', '/tmp/a'], ['doctor', '--output', '/tmp/a', '--output', '/tmp/b'], ['run', '--pr', '1', ...accounts], ['doctor', '--output', '--self-test'], ['doctor', '--output', '/tmp/a', '--publish']]) expect(() => parse(args)).toThrow(); + expect(parse(['doctor', '--candidate', '--output', '/fresh/probe'])).toEqual({ mode: 'doctor', output: '/fresh/probe', pr: 0, selfTest: false, candidate: true }); + expect(parse(['run', '--pr', '123', '--self-test', '--output', '/fresh/run', ...sources])).toEqual({ mode: 'run', output: '/fresh/run', pr: 123, selfTest: true, claudeConfig: sources[1], codexHome: sources[3] }); + for (const args of [[], ['wat'], ['run', '--pr', '0', '--output', '/tmp/a', ...sources], ['run', '--pr', '1.5', '--output', '/tmp/a', ...sources], ['run', '--pr', '9007199254740992', '--output', '/tmp/a', ...sources], ['doctor', '--pr', '1', '--output', '/tmp/a'], ['doctor', '--output', '/tmp/a', '--output', '/tmp/b'], ['run', '--pr', '1', ...sources], ['doctor', '--output', '--self-test'], ['doctor', '--output', '/tmp/a', '--publish']]) expect(() => parse(args)).toThrow(); }); -test('run requires both explicit accounts and parent doctor accepts paired account options', () => { +test('run and parent doctor require paired credential directories; candidate doctor rejects sources', () => { const run = ['run', '--pr', '111', '--output', '/fresh/run']; - expect(() => parse(run)).toThrow('Run requires --claude-account and --codex-account'); - expect(() => parse([...run, ...accounts.slice(0, 2)])).toThrow('Run requires'); - expect(() => parse([...run, ...accounts.slice(2)])).toThrow('Run requires'); - expect(parse(['doctor', '--output', '/fresh/probe', ...accounts])).toEqual({ mode: 'doctor', output: '/fresh/probe', pr: 0, selfTest: false, candidate: false, claudeAccount: 'eric@litman.org', codexAccount: 'eric@healthspanners.com' }); - expect(() => parse(['doctor', '--candidate', '--output', '/fresh/probe', ...accounts])).toThrow('cannot inspect caam credentials'); - for (const option of ['--claude-account', '--codex-account']) { - expect(() => parse(['doctor', '--output', '/fresh/probe', option, 'eric@litman.org'])).toThrow('require both'); - expect(() => parse([...run, ...accounts, option, 'eric@mobilyze.com'])).toThrow('Duplicate option'); - expect(() => parse([...run, option])).toThrow('Missing value'); - expect(() => parse([...run, option, '--self-test'])).toThrow('Missing value'); + const parent = ['doctor', '--output', '/fresh/probe']; + expect(() => parse(run)).toThrow('Run requires --claude-config and --codex-home'); + expect(() => parse(parent)).toThrow('Doctor requires both --claude-config and --codex-home'); + expect(() => parse([...run, ...sources.slice(0, 2)])).toThrow('Run requires'); + expect(() => parse([...run, ...sources.slice(2)])).toThrow('Run requires'); + expect(parse([...parent, ...sources])).toEqual({ mode: 'doctor', output: '/fresh/probe', pr: 0, selfTest: false, candidate: false, claudeConfig: sources[1], codexHome: sources[3] }); + expect(() => parse([...parent, '--candidate', ...sources])).toThrow('Candidate doctor cannot inspect source credentials'); + for (const option of ['--claude-config', '--codex-home']) { + expect(() => parse([...parent, option, '/operator/config'])).toThrow('requires both'); + expect(() => parse([...parent, '--candidate', option, '/operator/config'])).toThrow('Candidate doctor cannot inspect source credentials'); + for (const mode of [run, parent]) { + expect(() => parse([...mode, ...sources, option, '/other/config'])).toThrow('Duplicate option'); + expect(() => parse([...mode, option])).toThrow('Missing value'); + expect(() => parse([...mode, option, '--output'])).toThrow('Missing value'); + expect(() => parse([...mode, option, ''])).toThrow('Missing value'); + } } }); -test('account choices are bounded emails rather than vault paths or arbitrary strings', () => { +test('credential options accept directory paths and reject NUL paths and obsolete account options', () => { + for (const option of ['--claude-config', '--codex-home']) { + const other = option === '--claude-config' ? '--codex-home' : '--claude-config'; + for (const mode of [['run', '--pr', '111'], ['doctor']]) { + const args = [...mode, '--output', '/fresh/run', other, '/operator/config', option]; + expect(() => parse([...args, '/operator/config\0suffix'])).toThrow(`Invalid credential directory: ${option}`); + for (const directory of ['/operator/config with spaces', './preauthenticated-config']) { + const result = parse([...args, directory]); + expect(option === '--claude-config' ? result.claudeConfig : result.codexHome).toBe(directory); + } + } + } for (const option of ['--claude-account', '--codex-account']) { - const other = option === '--claude-account' ? '--codex-account' : '--claude-account'; - const args = ['run', '--pr', '111', '--output', '/fresh/run', other, 'eric@litman.org', option]; - for (const account of ['../eric@litman.org', 'claude/eric@litman.org', 'eric\\@litman.org', 'eric@../litman.org', 'eric@litman..org', 'eric', ' eric@litman.org', 'eric@litman.org\n', '@litman.org', 'eric@litman', 'x'.repeat(65) + '@litman.org', 'x@' + 'a.'.repeat(127) + 'org']) expect(() => parse([...args, account])).toThrow('Account must be a safe email'); - const result = parse([...args, 'eric+verification@litman.org']); - expect(result.mode).toBe('run'); - if (result.mode === 'run') expect(option === '--claude-account' ? result.claudeAccount : result.codexAccount).toBe('eric+verification@litman.org'); + expect(() => parse(['run', '--pr', '111', '--output', '/fresh/run', ...sources, option, 'person@example.com'])).toThrow(`Unknown option: ${option}`); } }); diff --git a/.claude/skills/verify-open-pstack/scripts/cli.ts b/.claude/skills/verify-open-pstack/scripts/cli.ts index a7333b2a..a46c7d66 100644 --- a/.claude/skills/verify-open-pstack/scripts/cli.ts +++ b/.claude/skills/verify-open-pstack/scripts/cli.ts @@ -6,36 +6,36 @@ import { Publisher } from './github.ts'; import { MacDriver } from './harness.ts'; import { command, freshRoot, interruption, interruptCommands, save } from './io.ts'; import { verify } from './verify.ts'; -type Options = { mode: 'doctor'; output: string; pr: number; selfTest: boolean; candidate: boolean; claudeAccount?: string; codexAccount?: string } - | { mode: 'run'; output: string; pr: number; selfTest: boolean; claudeAccount: string; codexAccount: string }; +type Options = { mode: 'doctor'; output: string; pr: number; selfTest: boolean; candidate: boolean; claudeConfig?: string; codexHome?: string } + | { mode: 'run'; output: string; pr: number; selfTest: boolean; claudeConfig: string; codexHome: string }; export function parse(args: string[]): Options { const mode = args[0]; - if (mode !== 'doctor' && mode !== 'run') throw new Error('Usage: verify.sh doctor [--candidate] --output /fresh/path | run --pr NUMBER --claude-account EMAIL --codex-account EMAIL [--self-test] --output /fresh/path'); - let output = '', pr = 0, selfTest = false, candidate = false, claudeAccount = '', codexAccount = ''; + if (mode !== 'doctor' && mode !== 'run') throw new Error('Usage: verify.sh doctor (--candidate | --claude-config DIR --codex-home DIR) --output /fresh/path | run --pr NUMBER --claude-config DIR --codex-home DIR [--self-test] --output /fresh/path'); + let output = '', pr = 0, selfTest = false, candidate = false, claudeConfig = '', codexHome = ''; const seen = new Set(); for (let i = 1; i < args.length; i++) { const arg = args[i]!; if (seen.has(arg)) throw new Error(`Duplicate option: ${arg}`); seen.add(arg); if (arg === '--self-test' && mode === 'run') selfTest = true; else if (arg === '--candidate' && mode === 'doctor') candidate = true; - else if (['--output', '--claude-account', '--codex-account'].includes(arg) || mode === 'run' && arg === '--pr') { + else if (['--output', '--claude-config', '--codex-home'].includes(arg) || mode === 'run' && arg === '--pr') { const value = args[++i]; if (!value || value.startsWith('--')) throw new Error(`Missing value: ${arg}`); if (arg === '--output') output = value; else if (arg === '--pr') { if (!/^[1-9]\d*$/.test(value) || !Number.isSafeInteger(Number(value))) throw new Error('PR must be a positive integer'); pr = Number(value); } else { - if (value.length > 254 || value.split('@')[0]!.length > 64 || value.includes('..') || !/^[A-Za-z0-9][A-Za-z0-9._+-]*@[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?)+$/.test(value)) throw new Error(`Account must be a safe email: ${arg}`); - if (arg === '--claude-account') claudeAccount = value; else codexAccount = value; + if (value.includes('\0')) throw new Error(`Invalid credential directory: ${arg}`); + if (arg === '--claude-config') claudeConfig = value; else codexHome = value; } } else throw new Error(`Unknown option: ${arg}`); } if (!output || mode === 'run' && !pr) throw new Error('Output and (for run) PR are required'); if (mode === 'run') { - if (!claudeAccount || !codexAccount) throw new Error('Run requires --claude-account and --codex-account'); - return { mode, output, pr, selfTest, claudeAccount, codexAccount }; + if (!claudeConfig || !codexHome) throw new Error('Run requires --claude-config and --codex-home'); + return { mode, output, pr, selfTest, claudeConfig, codexHome }; } - if (Boolean(claudeAccount) !== Boolean(codexAccount)) throw new Error('Doctor account checks require both --claude-account and --codex-account'); - if (candidate && claudeAccount) throw new Error('Candidate doctor cannot inspect caam credentials'); - return { mode, output, pr, selfTest, candidate, ...(claudeAccount ? { claudeAccount, codexAccount } : {}) }; + if (candidate && (claudeConfig || codexHome)) throw new Error('Candidate doctor cannot inspect source credentials'); + if (!candidate && (!claudeConfig || !codexHome)) throw new Error('Doctor requires both --claude-config and --codex-home'); + return { mode, output, pr, selfTest, candidate, ...(claudeConfig ? { claudeConfig, codexHome } : {}) }; } async function main(): Promise { let root: string | undefined, interrupted: 'SIGINT' | 'SIGTERM' | undefined; @@ -51,11 +51,11 @@ async function main(): Promise { try { const options = parse(process.argv.slice(2)), repository = resolve(import.meta.dir, '../../../..'); root = await freshRoot(options.output, repository); - if (options.mode === 'doctor') await doctor(root, undefined, undefined, options.candidate, options.claudeAccount && options.codexAccount ? { claude: options.claudeAccount, codex: options.codexAccount } : undefined); + if (options.mode === 'doctor') await doctor(root, undefined, undefined, options.candidate, options.claudeConfig && options.codexHome ? { claude: options.claudeConfig, codex: options.codexHome } : undefined); else { const publisherRevision = (await command(['git', 'rev-parse', 'HEAD'], { cwd: repository })).trim(); const registry = validateRegistry(JSON.parse(await readFile(join(import.meta.dir, '../features/registry.json'), 'utf8'))); - const receipt = await verify({ pr: options.pr, selfTest: options.selfTest, root, registry, github: new Publisher(), driver: new MacDriver(undefined, undefined, { claude: options.claudeAccount, codex: options.codexAccount }), persist: r => save(join(root!, 'receipt.json'), r), publisherRevision }); + const receipt = await verify({ pr: options.pr, selfTest: options.selfTest, root, registry, github: new Publisher(), driver: new MacDriver(undefined, undefined, { claude: options.claudeConfig, codex: options.codexHome }), persist: r => save(join(root!, 'receipt.json'), r), publisherRevision }); console.log(`Pinned ${receipt.sha}: live-gate=${receipt.status}; evidence ${receipt.commentUrl}`); } interruption.signal.throwIfAborted(); diff --git a/.claude/skills/verify-open-pstack/scripts/doctor.test.ts b/.claude/skills/verify-open-pstack/scripts/doctor.test.ts index 3b748a23..258ddb97 100644 --- a/.claude/skills/verify-open-pstack/scripts/doctor.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/doctor.test.ts @@ -1,65 +1,98 @@ import { afterEach, expect, test } from 'bun:test'; -import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { doctor } from './doctor.ts'; +import { loginFix } from './isolation.ts'; import type { Command } from './io.ts'; -import type { Harness } from './types.ts'; +import type { CredentialSources, Harness } from './types.ts'; const roots: string[] = []; -const accounts = { claude: 'claude@example.com', codex: 'codex@example.com' }; afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); -async function fixture(): Promise { - const root = await mkdtemp(join(tmpdir(), 'pstack-health-')); roots.push(root); return root; +async function fixture(): Promise<{ root: string; sources: CredentialSources }> { + const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-native-status-'))); roots.push(root); + const sources = { claude: join(root, 'Claude source config'), codex: join(root, 'Codex source home') }; + for (const tool of ['claude', 'codex'] as const) { + await mkdir(sources[tool], { mode: 0o700 }); + await writeFile(join(sources[tool], tool === 'claude' ? '.credentials.json' : 'auth.json'), JSON.stringify( + tool === 'claude' ? { claudeAiOauth: { accessToken: 'doctor-claude-access', refreshToken: 'doctor-claude-refresh' } } : { tokens: { access_token: 'doctor-codex-access', refresh_token: 'doctor-codex-refresh', id_token: 'doctor-codex-id' } }, + )); + } + return { root, sources }; } -function runner(calls: string[][], result: (tool: Harness) => string): Command { - return async args => { +function runner(calls: string[][], result: (tool: Harness, options: Parameters[1]) => string = tool => tool === 'claude' ? '{"loggedIn":true}' : ''): Command { + return async (args, options) => { calls.push(args); if (args.includes('--version')) return 'version'; if (args.includes('--help')) return '--plugin-dir --settings --setting-sources --json local path'; - if (args.join(' ') === 'caam ls --json') return JSON.stringify({ profiles: ['claude', 'codex'].map(tool => ({ tool, name: accounts[tool as Harness], identity: { email: accounts[tool as Harness] }, active: true })) }); - if (args[0] === 'caam' && args[1] === 'limits') return result(args[2] as Harness); + if (args.join(' ') === 'claude auth status --json' || args.join(' ') === 'codex login status') return result(args[0] as Harness, options); throw new Error(`Unexpected command: ${args.join(' ')}`); }; } -function healthy(tool: Harness): object[] { - return [{ provider: tool, profile_name: accounts[tool], usage: {} }, { provider: tool, profile_name: 'unselected@example.com', usage: { error: 'Unauthorized' } }]; -} +function statusCalls(calls: string[][]): string[][] { return calls.filter(args => args.includes('status')); } -test('doctor allows daily-active accounts and checks only selected provider/profile results', async () => { - const root = await fixture(), calls: string[][] = []; - await doctor(root, runner(calls, tool => JSON.stringify(healthy(tool))), 'darwin', false, accounts); - expect(calls.filter(args => args[0] === 'caam')).toEqual([ - ['caam', 'ls', '--json'], ['caam', 'limits', 'claude', '--format', 'json'], ['caam', 'limits', 'codex', '--format', 'json'], - ]); +test('doctor validates explicit native config directories independently of profile infrastructure', async () => { + const { root, sources } = await fixture(), calls: string[][] = []; + const before = await Promise.all([readFile(join(sources.claude, '.credentials.json')), readFile(join(sources.codex, 'auth.json'))]); + await doctor(root, runner(calls, (tool, options) => { + expect(options?.env?.CLAUDE_CONFIG_DIR).toBe(sources.claude); + expect(options?.env?.CODEX_HOME).toBe(sources.codex); + expect(options?.env?.HOME === process.env.HOME).toBe(true); + expect(options?.env?.USER === process.env.USER).toBe(true); + expect(options?.env?.LOGNAME === process.env.LOGNAME).toBe(true); + for (const key of ['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'CLAUDE_CODE_OAUTH_TOKEN', 'GH_TOKEN', 'GITHUB_TOKEN']) expect(options?.env?.[key]).toBeUndefined(); + return tool === 'claude' ? '{"loggedIn":true}' : ''; + }), 'darwin', false, sources); + expect(statusCalls(calls)).toEqual([['claude', 'auth', 'status', '--json'], ['codex', 'login', 'status']]); + expect(calls.some(args => args[0] === 'caam' || args.includes('login') && !args.includes('status'))).toBe(false); expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('pass'); + expect(await Promise.all([readFile(join(sources.claude, '.credentials.json')), readFile(join(sources.codex, 'auth.json'))])).toEqual(before); + expect(await Bun.file(join(root, 'probe-home/.claude/.credentials.json')).exists()).toBe(false); + expect(await Bun.file(join(root, 'probe-home/.codex/auth.json')).exists()).toBe(false); +}); + +test.each(['{"loggedIn":false}', '{}', '[]', 'null', 'not JSON', '{"loggedIn":"true"}'])('doctor fails closed for negative or malformed Claude status: %s', async status => { + const { root, sources } = await fixture(), calls: string[][] = []; + await expect(doctor(root, runner(calls, () => status), 'darwin', false, sources)).rejects.toThrow(loginFix('claude', sources.claude)); + expect(statusCalls(calls)).toEqual([['claude', 'auth', 'status', '--json']]); + const report = JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')); + expect(report.result).toBe('blocked'); expect(report.reason).toContain(loginFix('claude', sources.claude)); }); for (const tool of ['claude', 'codex'] as const) { - test.each(['HTTP 401', 'Unauthorized', 'OAuth credential expired', 'invalid_grant'])('doctor gives manual %s repair for selected ' + tool + ' result', async error => { - const root = await fixture(), calls: string[][] = []; - const run = runner(calls, provider => JSON.stringify(provider === tool ? [{ provider, profile_name: accounts[provider], usage: { error } }] : healthy(provider))); - const restore = `caam add ${tool} ${accounts[tool]} --no-activate --force`; - await expect(doctor(root, run, 'darwin', false, accounts)).rejects.toThrow(restore); - const report = JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')); - expect(report.result).toBe('blocked'); expect(report.reason).toContain(restore); - expect(calls.some(args => ['add', 'activate', 'login'].includes(args[1]!))).toBe(false); + test.each(['Not logged in', 'Unauthorized', 'Credentials expired', 'Command unavailable'])('doctor gives generic manual login guidance for ' + tool + ': %s', async error => { + const { root, sources } = await fixture(), calls: string[][] = []; + const run = runner(calls, provider => { + if (provider === tool) throw new Error(error); + return '{"loggedIn":true}'; + }); + await expect(doctor(root, run, 'darwin', false, sources)).rejects.toThrow(loginFix(tool, sources[tool])); + expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('blocked'); + expect(calls.some(args => args[0] === 'caam' || args.includes('login') && !args.includes('status'))).toBe(false); }); } -test('doctor catches command-level credential failures and expired credential-source state', async () => { - for (const failCommand of [false, true]) { - const root = await fixture(), calls: string[][] = []; - const run = runner(calls, tool => { - if (failCommand) throw new Error('caam limits: HTTP 401 Unauthorized'); - return JSON.stringify([{ provider: tool, profile_name: accounts[tool], credential_source: { state: 'expired' } }]); - }); - await expect(doctor(root, run, 'darwin', false, accounts)).rejects.toThrow('caam add claude claude@example.com --no-activate --force'); - } +test('Codex login status accepts an empty stdout on successful exit', async () => { + const { root, sources } = await fixture(), calls: string[][] = []; + await doctor(root, runner(calls), 'darwin', false, sources); + expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).checks['codex login status']).toBe(''); +}); + +test('parent doctor requires sources, and missing directories or files block before native commands', async () => { + const { root, sources } = await fixture(), calls: string[][] = []; + await expect(doctor(root, runner(calls), 'darwin')).rejects.toThrow('--claude-config and --codex-home '); + const missing = { ...sources, claude: join(root, 'missing source') }; + await expect(doctor(root, runner(calls), 'darwin', false, missing)).rejects.toThrow(loginFix('claude', missing.claude)); + await rm(join(sources.codex, 'auth.json')); + await expect(doctor(root, runner(calls), 'darwin', false, sources)).rejects.toThrow(loginFix('codex', sources.codex)); + expect(calls).toEqual([]); }); -test.each(['not JSON', '{}', '[]', '[{"provider":"codex","profile_name":"claude@example.com","usage":{}}]', '[{"provider":"claude","profile_name":"claude@example.com","usage":null}]', '[{"provider":"claude","profile_name":"claude@example.com","usage":{"error":"Network unavailable"}}]'])('doctor fails closed on unavailable or malformed selected health: %s', async output => { - const root = await fixture(), calls: string[][] = []; - await expect(doctor(root, runner(calls, () => output), 'darwin', false, accounts)).rejects.toThrow('health check failed'); - expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('blocked'); +test('candidate doctor is capabilities-only and refuses credential-source inspection', async () => { + const { root, sources } = await fixture(), calls: string[][] = []; + await doctor(root, runner(calls), 'darwin', true); + expect(statusCalls(calls)).toEqual([]); expect(calls.some(args => args[0] === 'gh')).toBe(false); + const report = JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')); + expect(report.result).toBe('pass'); expect(report.candidate).toBe(true); + await expect(doctor(root, runner(calls), 'darwin', true, sources)).rejects.toThrow('cannot inspect source credentials'); }); diff --git a/.claude/skills/verify-open-pstack/scripts/doctor.ts b/.claude/skills/verify-open-pstack/scripts/doctor.ts index b6ba8014..fa865096 100644 --- a/.claude/skills/verify-open-pstack/scripts/doctor.ts +++ b/.claude/skills/verify-open-pstack/scripts/doctor.ts @@ -1,14 +1,17 @@ import { mkdir, realpath } from 'node:fs/promises'; import { join } from 'node:path'; import { command, isolatedEnv, save, type Command } from './io.ts'; -import { selectedAccounts } from './isolation.ts'; -import { HARNESSES, type Accounts } from './types.ts'; +import { credentialSources, loginFix } from './isolation.ts'; +import { HARNESSES, type CredentialSources } from './types.ts'; -export async function doctor(root: string, run: Command = command, platform = process.platform, candidate = false, accounts?: Accounts): Promise { +export async function doctor(root: string, run: Command = command, platform = process.platform, candidate = false, sources?: CredentialSources): Promise { const report: Record = { platform, candidate, date: new Date().toISOString(), skill: await realpath(join(import.meta.dir, '..')), checks: {} }; const checks = report.checks as Record; try { if (platform !== 'darwin') throw new Error('Live gate requires an operator Mac (Darwin)'); + if (candidate && sources) throw new Error('Candidate doctor cannot inspect source credentials'); + if (!candidate && !sources) throw new Error('Doctor requires --claude-config and --codex-home '); + const canonical = sources ? await credentialSources(sources) : undefined; const home = join(root, 'probe-home'); for (const dir of ['tmp', '.claude', '.codex', '.config/gh', '.cache']) await mkdir(join(home, dir), { recursive: true, mode: 0o700 }); const env = isolatedEnv(home); @@ -22,33 +25,20 @@ export async function doctor(root: string, run: Command = command, platform = pr if (args.includes('marketplace') && !/local path|local or Git|local marketplace/i.test(help)) throw new Error('Codex local marketplace interface unavailable'); checks[args.join(' ')] = help; } - if (accounts) { - if (candidate) throw new Error('Candidate doctor cannot inspect caam credentials'); - const parentEnv: Record = {}; - for (const key of ['PATH', 'HOME', 'USER', 'LOGNAME', 'CAAM_HOME', 'XDG_CONFIG_HOME', 'XDG_DATA_HOME']) if (process.env[key]) parentEnv[key] = process.env[key]!; - selectedAccounts(await run(['caam', 'ls', '--json'], { env: parentEnv }), accounts); + if (canonical) { + const authEnv = { ...env, CLAUDE_CONFIG_DIR: canonical.claude, CODEX_HOME: canonical.codex }; for (const tool of HARNESSES) { - const restore = `caam add ${tool} ${accounts[tool]} --no-activate --force`; + const args = tool === 'claude' ? ['claude', 'auth', 'status', '--json'] : ['codex', 'login', 'status']; try { - const output = await run(['caam', 'limits', tool, '--format', 'json'], { env: parentEnv }); - checks[`caam limits ${tool}`] = output; - const rows = JSON.parse(output); - if (!Array.isArray(rows)) throw new Error('Unknown caam limits schema; expected profile rows'); - const selected = rows.filter(row => row?.provider === tool && row?.profile_name === accounts[tool]); - if (selected.length !== 1) throw new Error(`Missing or ambiguous caam limits result for ${tool}/${accounts[tool]}`); - if (selected[0].credential_source?.state === 'expired') throw new Error('Credential copy expired'); - if (!selected[0].usage || typeof selected[0].usage !== 'object' || Array.isArray(selected[0].usage)) throw new Error('Missing caam limits usage result'); - const error = selected[0].usage.error; - if (error !== undefined && typeof error !== 'string') throw new Error('Unknown caam limits error schema'); - if (error) throw new Error(error); - } catch (error) { - const reason = error instanceof Error ? error.message : String(error); - if (/unauthori[sz]ed|expired|\b401\b|invalid[_ -](?:token|grant)/i.test(reason)) throw new Error(`caam ${tool}/${accounts[tool]} copy is unauthorized or expired; restore with: ${restore}`); - throw new Error(`caam ${tool}/${accounts[tool]} copy health check failed: ${reason}`); + const output = await run(args, { env: authEnv }); + checks[args.join(' ')] = output; + if (tool === 'claude' && JSON.parse(output)?.loggedIn !== true) throw new Error('Not logged in'); + } catch { + throw new Error(`${tool} source authentication status failed; log in once with: ${loginFix(tool, canonical[tool])}`); } } } - // PR resolution checks publisher authentication; a child surface doctor has no GitHub credentials. + // Status checks report cached login state; authenticated native requests still need live proof. report.result = 'pass'; } catch (error) { report.result = 'blocked'; report.reason = String(error); diff --git a/.claude/skills/verify-open-pstack/scripts/harness.test.ts b/.claude/skills/verify-open-pstack/scripts/harness.test.ts index bed2075a..e63828e9 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.test.ts @@ -9,6 +9,19 @@ import { command, freshRoot, isolatedEnv, retainedFile, treeHash, type Command } const roots: string[] = []; async function fixture(): Promise { const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-test-'))); roots.push(root); return root; } afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); +async function sourceFixture(root: string) { + const sources = { claude: join(root, 'source-claude'), codex: join(root, 'source-codex') }; + const credentials = { + claude: JSON.stringify({ claudeAiOauth: { accessToken: 'fixture-claude', refreshToken: 'fixture-claude-refresh' } }), + codex: JSON.stringify({ tokens: { access_token: 'fixture-codex', refresh_token: 'fixture-codex-refresh', id_token: 'fixture-codex-id' } }), + }; + for (const tool of ['claude', 'codex'] as const) { + await mkdir(sources[tool], { mode: 0o700 }); + await writeFile(join(sources[tool], tool === 'claude' ? '.credentials.json' : 'auth.json'), credentials[tool]); + await writeFile(join(sources[tool], 'source-only.txt'), 'must not be copied'); + } + return { sources, credentials }; +} describe('isolated harness boundaries', () => { test('candidate environment keeps operator identity while isolating provider configuration', () => { @@ -56,12 +69,12 @@ describe('isolated harness boundaries', () => { const run: Command = async args => args.includes('--version') ? 'version' : ''; await expect(doctor(root, run, 'linux')).rejects.toThrow('operator Mac'); expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('blocked'); - await expect(doctor(root, run, 'darwin')).rejects.toThrow('isolation missing'); + await expect(doctor(root, run, 'darwin', true)).rejects.toThrow('isolation missing'); }); test('doctor probes without installing or reading daily authentication', async () => { const calls: string[][] = [], root = await fixture(); const run: Command = async args => { calls.push(args); return args.includes('--version') ? 'version' : '--plugin-dir --settings --setting-sources --json local path'; }; - await doctor(root, run, 'darwin'); + await doctor(root, run, 'darwin', true); expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('pass'); expect(calls.every(c => c.includes('--help') || c.includes('--version'))).toBe(true); }); @@ -72,7 +85,9 @@ describe('isolated harness boundaries', () => { await symlink(skill, join(workspace, '.claude/skills/verify-open-pstack')); const alias = join(root, 'workspace-alias'); await symlink(workspace, alias); const run: Command = async args => args.includes('--version') ? 'version' : '--plugin-dir --settings --setting-sources --json local path'; - await doctor(root, run, 'darwin'); + const { sources } = await sourceFixture(root); + const parentRun: Command = async args => args[1] === 'auth' ? JSON.stringify({ loggedIn: true }) : args[1] === 'login' ? 'Logged in using ChatGPT' : run(args); + await doctor(root, parentRun, 'darwin', false, sources); const parentText = await readFile(join(root, 'doctor.json'), 'utf8'); await expect(verifyProjectDoctor([parentText], workspace)).rejects.toThrow('passing child doctor'); await doctor(root, run, 'darwin', true); @@ -90,7 +105,7 @@ describe('isolated harness boundaries', () => { const root = await fixture(), sha = '1'.repeat(40), calls: string[][] = []; const receipt = newReceipt(120, sha, sha, false, root); receipt.selection.features = ['setup']; - const driver = new MacDriver(async args => { calls.push(args); return ''; }, async () => '', { claude: 'claude@example.com', codex: 'codex@example.com' }); + const driver = new MacDriver(async args => { calls.push(args); return ''; }, async () => '', { claude: join(root, 'source-claude'), codex: join(root, 'source-codex') }); const message = 'setup exercise requires #120 (setup-pstack config-home)'; for (const operation of [() => driver.prepare(receipt), () => driver.exercise(receipt)]) { let caught: unknown; @@ -98,46 +113,44 @@ describe('isolated harness boundaries', () => { expect(caught).toBeInstanceOf(Error); expect((caught as Error).message).toBe(message); expect(calls).toEqual([]); } }); - test.each(['claude', 'codex'] as const)('prepare refuses an unauthorized %s vault copy before candidate commands', async tool => { + test.each(['claude', 'codex'] as const)('prepare refuses an unauthenticated %s source before candidate commands', async tool => { const root = await fixture(), calls: string[][] = [], sha = '2'.repeat(40); - const accounts = { claude: 'claude@example.com', codex: 'codex@example.com' }; - const run: Command = async args => { + const { sources, credentials } = await sourceFixture(root); + const run: Command = async (args, options = {}) => { calls.push(args); if (args.includes('--version')) return 'version'; if (args.includes('--help')) return '--plugin-dir --settings --setting-sources --json local path'; - if (args[0] === 'caam' && args[1] === 'ls') return JSON.stringify({ profiles: [ - { tool: 'claude', name: accounts.claude, identity: { email: accounts.claude }, active: true }, - { tool: 'codex', name: accounts.codex, identity: { email: accounts.codex }, active: true }, - ] }); - if (args[0] === 'caam' && args[1] === 'limits') { - const provider = args[2] as keyof typeof accounts; - return JSON.stringify([{ provider, profile_name: accounts[provider], usage: { error: provider === tool ? 'HTTP 401 Unauthorized' : '' } }]); + if (args[1] === 'auth' || args[1] === 'login') { + expect(options.env!.CLAUDE_CONFIG_DIR).toBe(sources.claude); + expect(options.env!.CODEX_HOME).toBe(sources.codex); + if (args[0] === 'claude') { + expect(args).toEqual(['claude', 'auth', 'status', '--json']); + return JSON.stringify({ loggedIn: tool !== 'claude' }); + } + expect(args).toEqual(['codex', 'login', 'status']); + if (tool === 'codex') throw new Error('Not logged in'); + return 'Logged in using ChatGPT'; } throw new Error(`Unexpected fixture command: ${args.join(' ')}`); }; const platform = Object.getOwnPropertyDescriptor(process, 'platform')!; + const remediation = tool === 'claude' ? `CLAUDE_CONFIG_DIR='${sources.claude}' claude auth login` : `CODEX_HOME='${sources.codex}' codex login`; try { Object.defineProperty(process, 'platform', { ...platform, value: 'darwin' }); - await expect(new MacDriver(run, async () => '', accounts).prepare(newReceipt(111, sha, sha, true, root))).rejects.toThrow(`caam add ${tool} ${accounts[tool]} --no-activate --force`); + await expect(new MacDriver(run, async () => '', sources).prepare(newReceipt(111, sha, sha, true, root))).rejects.toThrow(remediation); } finally { Object.defineProperty(process, 'platform', platform); } - expect(calls.some(c => c.join(' ') === 'caam ls --json')).toBe(true); - expect(calls.some(c => c.join(' ') === `caam limits ${tool} --format json`)).toBe(true); - expect(calls.some(c => c[0] === 'caam' && ['add', 'activate'].includes(c[1]!))).toBe(false); + expect(calls.some(c => c.join(' ') === (tool === 'claude' ? 'claude auth status --json' : 'codex login status'))).toBe(true); + expect(calls.some(c => c.includes('login') && c.join(' ') !== 'codex login status')).toBe(false); expect(calls.some(c => c[0] === 'git' && !c.includes('--version'))).toBe(false); + await expect(stat(join(root, 'state'))).rejects.toThrow('ENOENT'); + for (const provider of ['claude', 'codex'] as const) { + expect(await readFile(join(sources[provider], provider === 'claude' ? '.credentials.json' : 'auth.json'), 'utf8')).toBe(credentials[provider]); + } expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('blocked'); }); test.each([false, true])('prepare uses pinned local git fixture and disposable config credentials (install failure=%s)', async failInstall => { const root = await fixture(), repository = join(root, 'repository'), operatorHome = join(root, 'operator-home'), calls: string[][] = []; - const accounts = { claude: 'claude@example.com', codex: 'codex@example.com' }; - const caam = join(root, 'caam'), vault = join(caam, 'data/vault'); - const credentials = { - claude: JSON.stringify({ claudeAiOauth: { accessToken: 'fixture-claude', refreshToken: 'fixture-claude-refresh' } }), - codex: JSON.stringify({ tokens: { access_token: 'fixture-codex', refresh_token: 'fixture-codex-refresh', id_token: 'fixture-codex-id' } }), - }; - for (const tool of ['claude', 'codex'] as const) { - const dir = join(vault, tool, accounts[tool]); await mkdir(dir, { recursive: true }); - await writeFile(join(dir, tool === 'claude' ? '.credentials.json' : 'auth.json'), credentials[tool]); - } + const { sources, credentials } = await sourceFixture(root); for (const manifest of ['.claude-plugin', '.codex-plugin']) { const dir = join(repository, 'plugins/pstack', manifest); await mkdir(dir, { recursive: true }); await writeFile(join(dir, 'plugin.json'), JSON.stringify({ version: 'test' })); @@ -152,17 +165,18 @@ describe('isolated harness boundaries', () => { const sha = (await command(['git', 'rev-parse', 'HEAD'], { cwd: repository })).trim(), actualGit: Command = command; await mkdir(operatorHome, { mode: 0o700 }); const platform = Object.getOwnPropertyDescriptor(process, 'platform')!; - const previous = { CAAM_HOME: process.env.CAAM_HOME, HOME: process.env.HOME, USER: process.env.USER, LOGNAME: process.env.LOGNAME }; + const previous = { HOME: process.env.HOME, USER: process.env.USER, LOGNAME: process.env.LOGNAME }; const run: Command = async (args, options = {}) => { calls.push(args); - if (args[0] === 'caam') { - if (args[1] === 'ls') { - expect(args).toEqual(['caam', 'ls', '--json']); - return JSON.stringify({ profiles: ['claude', 'codex'].map(tool => ({ tool, name: accounts[tool as keyof typeof accounts], identity: { email: accounts[tool as keyof typeof accounts] }, active: true })) }); + if (args[1] === 'auth' || args[1] === 'login') { + expect(options.env!.CLAUDE_CONFIG_DIR).toBe(sources.claude); + expect(options.env!.CODEX_HOME).toBe(sources.codex); + if (args[0] === 'claude') { + expect(args).toEqual(['claude', 'auth', 'status', '--json']); + return JSON.stringify({ loggedIn: true }); } - const provider = args[2] as keyof typeof accounts; - expect(args).toEqual(['caam', 'limits', provider, '--format', 'json']); - return JSON.stringify([{ provider, profile_name: accounts[provider], usage: {} }]); + expect(args).toEqual(['codex', 'login', 'status']); + return 'Logged in using ChatGPT'; } const env = options.env, candidate = Boolean(env?.CLAUDE_CONFIG_DIR?.startsWith(join(root, 'state') + '/')); if (candidate) { @@ -170,7 +184,7 @@ describe('isolated harness boundaries', () => { expect(env!.HOME).toBe(operatorHome); expect(env!.USER).toBe('operator-user'); expect(env!.LOGNAME).toBe('operator-login'); expect(env!.CLAUDE_CONFIG_DIR).toBe(join(stateHome, '.claude')); expect(env!.CODEX_HOME).toBe(join(stateHome, '.codex')); expect(env!.TMPDIR).toBe(join(stateHome, 'tmp')); expect(env!.GH_CONFIG_DIR).toBe(join(stateHome, '.config/gh')); - expect(env!.GH_TOKEN).toBeUndefined(); expect(env!.CAAM_HOME).toBeUndefined(); + expect(env!.GH_TOKEN).toBeUndefined(); expect(env!.GITHUB_TOKEN).toBeUndefined(); for (const dir of [stateHome, env!.TMPDIR!, env!.CLAUDE_CONFIG_DIR!, env!.CODEX_HOME!, env!.GH_CONFIG_DIR!]) { const info = await stat(dir); expect(info.isDirectory()).toBe(true); expect(info.mode & 0o777).toBe(0o700); } @@ -205,16 +219,16 @@ describe('isolated harness boundaries', () => { } throw new Error(`Unexpected fixture command: ${args.join(' ')}`); }; - const driver = new MacDriver(run, async () => '', accounts); + const driver = new MacDriver(run, async () => '', sources); try { Object.defineProperty(process, 'platform', { ...platform, value: 'darwin' }); - process.env.CAAM_HOME = caam; process.env.HOME = operatorHome; process.env.USER = 'operator-user'; process.env.LOGNAME = 'operator-login'; + process.env.HOME = operatorHome; process.env.USER = 'operator-user'; process.env.LOGNAME = 'operator-login'; const receipt = newReceipt(111, sha, sha, true, root), preparation = driver.prepare(receipt); if (failInstall) await expect(preparation).rejects.toThrow('fixture installation failed'); else { const installs = await preparation; receipt.installations = installs; expect(installs.map(i => i.harness)).toEqual(['claude', 'codex']); - expect(installs.map(i => i.account)).toEqual([accounts.claude, accounts.codex]); + expect(installs.map(i => i.credentialSource)).toEqual([sources.claude, sources.codex]); expect(installs.every(i => i.sha === sha)).toBe(true); expect(installs.every(i => /^[a-f0-9]{64}$/.test(i.sourceHash!))).toBe(true); expect(installs[0]!.sourceHash).toBe(installs[1]!.sourceHash); @@ -226,20 +240,30 @@ describe('isolated harness boundaries', () => { expect(calls.filter(c => c[0] === '/usr/bin/script')).toHaveLength(2); } } finally { + await mkdir(join(root, 'artifacts'), { mode: 0o700 }); + await writeFile(join(root, 'artifacts/private.txt'), credentials.codex, { mode: 0o600 }); + for (const harness of ['claude', 'codex']) await writeFile(join(root, 'state', harness, 'native-state.db'), 'retained candidate state'); await driver.cleanup(); await driver.cleanup(); Object.defineProperty(process, 'platform', platform); for (const [name, value] of Object.entries(previous)) { if (value === undefined) delete process.env[name]; else process.env[name] = value; } } - expect(calls.some(c => c.join(' ') === 'caam ls --json')).toBe(true); + expect(calls.filter(c => c.join(' ') === 'claude auth status --json')).toHaveLength(1); + expect(calls.filter(c => c.join(' ') === 'codex login status')).toHaveLength(1); expect(calls.some(c => c[0] === '/usr/bin/sandbox-exec')).toBe(false); + expect(await readFile(join(root, 'artifacts/private.txt'), 'utf8')).toBe(credentials.codex); + expect((await stat(join(root, 'artifacts'))).mode & 0o777).toBe(0o700); + expect((await stat(join(root, 'artifacts/private.txt'))).mode & 0o777).toBe(0o600); for (const harness of ['claude', 'codex']) { const home = join(root, 'state', harness); expect((await stat(home)).isDirectory()).toBe(true); + expect(await readFile(join(home, 'native-state.db'), 'utf8')).toBe('retained candidate state'); for (const tool of ['claude', 'codex'] as const) { const filename = tool === 'claude' ? '.credentials.json' : 'auth.json'; await expect(stat(join(home, '.' + tool, filename))).rejects.toThrow('ENOENT'); - expect(await readFile(join(vault, tool, accounts[tool], filename), 'utf8')).toBe(credentials[tool]); + expect(await readFile(join(sources[tool], filename), 'utf8')).toBe(credentials[tool]); + expect(await readFile(join(sources[tool], 'source-only.txt'), 'utf8')).toBe('must not be copied'); + await expect(stat(join(home, '.' + tool, 'source-only.txt'))).rejects.toThrow('ENOENT'); } if (!failInstall) expect(await readFile(join(home, 'surface.raw'), 'utf8')).toBe('retained raw native evidence'); } diff --git a/.claude/skills/verify-open-pstack/scripts/harness.ts b/.claude/skills/verify-open-pstack/scripts/harness.ts index 20229733..404484f4 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.ts @@ -4,9 +4,9 @@ import { createInterface } from 'node:readline/promises'; import { requiredFeatures, requiredHarnesses } from './core.ts'; import { command, interruption, isolatedEnv, retainedFile, save, treeHash, type Command } from './io.ts'; import { doctor } from './doctor.ts'; -import { copyCredentials, removeCredentials, vaultRoot } from './isolation.ts'; +import { copyCredentials, credentialSources, removeCredentials } from './isolation.ts'; import { sourceDigest, sourceHash } from './provenance.ts'; -import { HARNESSES, REPO, type Accounts, type Driver, type Harness, type Installation, type Observation, type Receipt } from './types.ts'; +import { HARNESSES, REPO, type CredentialSources, type Driver, type Harness, type Installation, type Observation, type Receipt } from './types.ts'; export type Ask = (question: string) => Promise; export const ask: Ask = async question => { @@ -46,7 +46,7 @@ export async function verifyProjectDoctor(texts: string[], workspace: string): P } export class MacDriver implements Driver { private copied: string[] = []; - constructor(private run: Command = command, private review: Ask = ask, private accounts?: Accounts) {} + constructor(private run: Command = command, private review: Ask = ask, private sources?: CredentialSources) {} async cleanup(): Promise { await removeCredentials(this.copied); } private candidate(home: string): Command { return (args, options = {}) => this.run(args, { ...options, env: isolatedEnv(home) }); @@ -59,11 +59,10 @@ export class MacDriver implements Driver { async prepare(receipt: Receipt): Promise { this.refuseSetup(receipt); const root = receipt.artifactRoot; - if (!this.accounts) throw new Error('Explicit Claude and Codex caam accounts are required'); - await doctor(root, this.run, undefined, false, this.accounts); - const accounts = this.accounts; - const vault = await realpath(vaultRoot()); - // Fetch the immutable reference outside candidate-readable/writable state. + if (!this.sources) throw new Error('Explicit --claude-config and --codex-home credential directories are required'); + await doctor(root, this.run, undefined, false, this.sources); + const sources = await credentialSources(this.sources); + // Fetch a separate immutable reference for pinned-source comparisons. const referenceHome = join(root, 'source-home'), reference = join(referenceHome, 'workspace'); await mkdir(join(referenceHome, 'tmp'), { recursive: true, mode: 0o700 }); const trustedRun: Command = (args, options = {}) => this.run(args, { ...options, env: isolatedEnv(referenceHome) }); @@ -86,7 +85,7 @@ export class MacDriver implements Driver { if ((await candidateRun(['git', 'rev-parse', 'HEAD'], { cwd: workspace, env })).trim() !== receipt.sha) throw new Error('Candidate checkout SHA mismatch'); const candidate = join(workspace, 'plugins/pstack'), expected = await treeHash(candidate, ['skills/poteto-mode/scripts/node_modules']); if (await sourceDigest(workspace) !== pinnedSource) throw new Error('Candidate files differ from trusted pinned source'); - await copyCredentials(home, vault, accounts, this.copied); + await copyCredentials(home, sources, this.copied); let location = candidate; if (harness === 'codex') { const added = await candidateRun(['codex', 'plugin', 'marketplace', 'add', workspace, '--json'], { env, cwd: home }); @@ -100,7 +99,7 @@ export class MacDriver implements Driver { } const manifest = JSON.parse(await readFile(join(location, harness === 'claude' ? '.claude-plugin/plugin.json' : '.codex-plugin/plugin.json'), 'utf8')); if (await sourceDigest(workspace) !== pinnedSource) throw new Error('Pinned source changed during installation'); - installs.push({ harness, sha: receipt.sha, home, location, treeHash: expected, sourceHash: pinnedSource, account: accounts[harness], pluginVersion: manifest.version, + installs.push({ harness, sha: receipt.sha, home, location, treeHash: expected, sourceHash: pinnedSource, credentialSource: sources[harness], pluginVersion: manifest.version, cliVersion: (await candidateRun([harness, '--version'], { env })).trim() }); } return installs; @@ -112,11 +111,11 @@ export class MacDriver implements Driver { const { harness, home } = installation, workspace = join(home, 'workspace'); if (await sourceDigest(workspace) !== installation.sourceHash) throw new Error('Pinned source changed before exercise'); const env = isolatedEnv(home, harness), candidateRun = this.candidate(home); - const request = { sha: receipt.sha, harness, workspace, features: requiredFeatures(receipt), + const request = { sha: receipt.sha, harness, workspace, features: requiredFeatures(receipt, harness), featureMap: join(workspace, '.claude/skills/verify-open-pstack/features'), - selfTest: requiredFeatures(receipt).includes('project-skill') ? `Invoke the candidate project skill natively; run doctor --candidate --output "${join(home, 'self-test')}". Do not invoke run, vault listing, login, or publication recursively.` : false, - credentials: { claude: this.accounts?.claude, codex: this.accounts?.codex }, - isolation: 'Real HOME/USER; both provider config roots and temporary state are run-owned. Use only selected disposable file credentials, never activate vault accounts or log in. Exercise real requests; quota/auth failures fail closed.' }; + selfTest: requiredFeatures(receipt, harness).includes('project-skill') ? `Invoke the candidate project skill natively; run doctor --candidate --output "${join(home, 'self-test')}". Do not invoke run, login, or publication recursively.` : false, + credentials: { claude: env.CLAUDE_CONFIG_DIR, codex: env.CODEX_HOME }, + isolation: 'Real HOME/USER; both provider config roots and temporary state are run-owned. Use only copied disposable file credentials, never log in or write back to the credential sources. Exercise real requests; quota/auth failures fail closed.' }; await save(join(receipt.artifactRoot, `${harness}-request.json`), request); console.log(JSON.stringify(request, null, 2)); const raw = join(home, 'surface.raw'); diff --git a/.claude/skills/verify-open-pstack/scripts/isolation.test.ts b/.claude/skills/verify-open-pstack/scripts/isolation.test.ts index 78f56c2a..44f1102c 100644 --- a/.claude/skills/verify-open-pstack/scripts/isolation.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/isolation.test.ts @@ -1,10 +1,10 @@ import { afterEach, describe, expect, test } from 'bun:test'; -import { lstat, mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; +import { lstat, mkdir, mkdtemp, readFile, readdir, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { assertNoKnownSecrets, isolatedEnv } from './io.ts'; -import { copyCredentials, removeCredentials, selectedAccounts, vaultRoot } from './isolation.ts'; -import type { Accounts } from './types.ts'; +import { copyCredentials, credentialSources, loginFix, removeCredentials } from './isolation.ts'; +import type { CredentialSources } from './types.ts'; const roots: string[] = []; async function fixture(): Promise { @@ -12,71 +12,46 @@ async function fixture(): Promise { roots.push(root); return root; } -afterEach(async () => { - for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); -}); - -const accounts: Accounts = { claude: 'claude@example.com', codex: 'codex@example.com' }; -function profiles(active?: 'claude' | 'codex' | 'both'): string { - return JSON.stringify({ profiles: [ - { tool: 'claude', name: accounts.claude, active: active === 'claude' || active === 'both', identity: { email: accounts.claude } }, - { tool: 'codex', name: accounts.codex, active: active === 'codex' || active === 'both', identity: { email: accounts.codex } }, - ] }); -} -async function vault(root: string): Promise { - const base = join(root, 'vault'); +afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); +async function sources(root: string): Promise { + const directories = { claude: join(root, 'Claude logged in'), codex: join(root, 'Codex logged in') }; for (const tool of ['claude', 'codex'] as const) { - const path = join(base, tool, accounts[tool]); - await mkdir(path, { recursive: true, mode: 0o700 }); - await writeFile(join(path, tool === 'claude' ? '.credentials.json' : 'auth.json'), JSON.stringify( + await mkdir(directories[tool], { mode: 0o700 }); + await writeFile(join(directories[tool], tool === 'claude' ? '.credentials.json' : 'auth.json'), JSON.stringify( tool === 'claude' ? { claudeAiOauth: { accessToken: 'selected-claude-access', refreshToken: 'selected-claude-refresh' }, account: 'claude-evidence' } : { tokens: { access_token: 'selected-codex-access', refresh_token: 'selected-codex-refresh', id_token: 'selected-codex-id' }, account_id: 'codex-evidence' }, ), { mode: 0o600 }); + await writeFile(join(directories[tool], 'settings.json'), '{"source-only":true}'); } - return base; + return directories; } - -describe('disposable credential isolation', () => { - test('vault root follows caam precedence and requires an absolute operator home', () => { - const previous = { CAAM_HOME: process.env.CAAM_HOME, XDG_DATA_HOME: process.env.XDG_DATA_HOME, HOME: process.env.HOME }; - try { - process.env.CAAM_HOME = '/operator/caam'; process.env.XDG_DATA_HOME = '/operator/data'; process.env.HOME = '/operator/home'; - expect(vaultRoot()).toBe('/operator/caam/data/vault'); - delete process.env.CAAM_HOME; - expect(vaultRoot()).toBe('/operator/data/caam/vault'); - delete process.env.XDG_DATA_HOME; - expect(vaultRoot()).toBe('/operator/home/.local/share/caam/vault'); - process.env.HOME = 'relative'; - expect(() => vaultRoot()).toThrow('absolute operator HOME'); - } finally { - for (const [name, value] of Object.entries(previous)) { - if (value === undefined) delete process.env[name]; else process.env[name] = value; - } - } +function files(source: CredentialSources): string[] { return [join(source.claude, '.credentials.json'), join(source.codex, 'auth.json')]; } + +describe('generic disposable credential sources', () => { + test('validates and canonicalizes explicitly selected normal-login directories with spaces', async () => { + const root = await fixture(), source = await sources(root); + expect(await credentialSources({ claude: join(source.claude, '..', 'Claude logged in'), codex: source.codex })).toEqual(source); + const missing = { ...source, codex: join(root, 'not logged in') }; + await expect(credentialSources(missing)).rejects.toThrow(loginFix('codex', missing.codex)); + await rm(files(source)[0]!); + await expect(credentialSources(source)).rejects.toThrow(loginFix('claude', source.claude)); }); - test('selected accounts match exact identities regardless of daily activity and reject mismatches', () => { - expect(selectedAccounts(profiles(), accounts)).toEqual(accounts); - expect(selectedAccounts(profiles('both'), accounts)).toEqual(accounts); - - const wrongIdentity = JSON.parse(profiles('both')); wrongIdentity.profiles[0].identity.email = 'other@example.com'; - expect(() => selectedAccounts(JSON.stringify(wrongIdentity), accounts)).toThrow(`caam claude/${accounts.claude} identity mismatch`); - const duplicate = JSON.parse(profiles()); duplicate.profiles.push(duplicate.profiles[0]); - expect(() => selectedAccounts(JSON.stringify(duplicate), accounts)).toThrow(); - expect(() => selectedAccounts('{"profiles":[]}', accounts)).toThrow(); - expect(() => selectedAccounts('{}', accounts)).toThrow(); - expect(() => selectedAccounts('not JSON', accounts)).toThrow(); - expect(() => selectedAccounts(profiles(), { ...accounts, claude: '../escape@example.com' })).toThrow('safe claude account'); + test('login guidance quotes directories safely without activating or repairing anything', async () => { + const { command } = await import('./io.ts'); + const root = await fixture(), path = join(root, "dir's $(touch unexpected) ; config"); + const fix = loginFix('claude', path); + const assignment = fix.slice('CLAUDE_CONFIG_DIR='.length, -' claude auth login'.length); + expect(await command(['/bin/sh', '-c', `printf '%s' ${assignment}`], { cwd: root })).toBe(path); + expect(await Bun.file(join(root, 'unexpected')).exists()).toBe(false); + expect(loginFix('codex', '/normal/codex')).toBe("CODEX_HOME='/normal/codex' codex login"); }); - test('copies complete refresh-capable credentials to isolated config roots with private modes and leaves the vault unchanged', async () => { - const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; - const originals = [join(base, 'claude', accounts.claude, '.credentials.json'), join(base, 'codex', accounts.codex, 'auth.json')]; - const before = await Promise.all(originals.map(path => readFile(path))); - - await copyCredentials(home, base, accounts, copied); - + test('copies only complete refresh-capable credential files with private modes and leaves source configuration unchanged', async () => { + const root = await fixture(), source = await sources(root), home = join(root, 'candidate'), copied: string[] = []; + const originals = files(source), before = await Promise.all(originals.map(path => readFile(path))); + await copyCredentials(home, source, copied); const env = isolatedEnv(home); expect(copied).toEqual([join(env.CLAUDE_CONFIG_DIR!, '.credentials.json'), join(env.CODEX_HOME!, 'auth.json')]); for (let index = 0; index < copied.length; index++) { @@ -84,107 +59,103 @@ describe('disposable credential isolation', () => { expect((await stat(copied[index]!)).mode & 0o777).toBe(0o600); expect((await stat(dirname(copied[index]!))).mode & 0o777).toBe(0o700); expect(await readFile(copied[index]!)).toEqual(before[index]!); + expect(await readdir(dirname(copied[index]!))).toEqual([index === 0 ? '.credentials.json' : 'auth.json']); } expect(JSON.parse(await readFile(copied[0]!, 'utf8')).claudeAiOauth.refreshToken).toBe('selected-claude-refresh'); expect(JSON.parse(await readFile(copied[1]!, 'utf8')).tokens.refresh_token).toBe('selected-codex-refresh'); expect(await Promise.all(originals.map(path => readFile(path)))).toEqual(before); + for (const directory of Object.values(source)) expect(await readFile(join(directory, 'settings.json'), 'utf8')).toBe('{"source-only":true}'); }); - test('registers only exact copied credential token fields for parent comment-body redaction', async () => { - const root = await fixture(), base = await vault(root), copied: string[] = []; + test('registers only exact copied OAuth credential fields for public comment checks', async () => { + const root = await fixture(), source = await sources(root), copied: string[] = []; const exact = ['comment-claude-access-7', 'comment-claude-refresh-7', 'comment-codex-access-7', 'comment-codex-refresh-7', 'comment-codex-id-7']; const ordinary = ['visible-token-shaped-claude-field-7', 'visible-token-shaped-codex-field-7']; - await writeFile(join(base, 'claude', accounts.claude, '.credentials.json'), JSON.stringify({ - claudeAiOauth: { accessToken: exact[0], refreshToken: exact[1], sessionTokenHint: ordinary[0] }, - })); - await writeFile(join(base, 'codex', accounts.codex, 'auth.json'), JSON.stringify({ - tokens: { access_token: exact[2], refresh_token: exact[3], id_token: exact[4], token_hint: ordinary[1] }, - })); - - await copyCredentials(join(root, 'candidate'), base, accounts, copied); - + await writeFile(files(source)[0]!, JSON.stringify({ claudeAiOauth: { accessToken: exact[0], refreshToken: exact[1], sessionTokenHint: ordinary[0] } })); + await writeFile(files(source)[1]!, JSON.stringify({ tokens: { access_token: exact[2], refresh_token: exact[3], id_token: exact[4], token_hint: ordinary[1] } })); + await copyCredentials(join(root, 'candidate'), source, copied); for (const secret of exact) expect(() => assertNoKnownSecrets(secret)).toThrow('Comment contains copied credentials'); expect(() => assertNoKnownSecrets(ordinary.join(' '))).not.toThrow(); }); - test('cleanup removes only copied files while preserving candidate state, raw evidence, config directories, and vault files', async () => { - const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; - await copyCredentials(home, base, accounts, copied); - const raw = join(home, 'surface.raw'), artifact = join(home, 'reviewed-transcript.txt'), state = join(home, '.claude/session-state.json'); - await writeFile(raw, 'private raw transcript'); - await writeFile(artifact, 'private reviewed evidence'); - await writeFile(state, 'candidate session state'); - const originals = [join(base, 'claude', accounts.claude, '.credentials.json'), join(base, 'codex', accounts.codex, 'auth.json')]; - const before = await Promise.all(originals.map(path => readFile(path, 'utf8'))); - - await removeCredentials(copied); - await removeCredentials(copied); + test('supports native Codex auth.json API-key credentials without discarding fields', async () => { + const root = await fixture(), source = await sources(root), home = join(root, 'candidate'), copied: string[] = []; + const auth = { OPENAI_API_KEY: 'exact-codex-api-key-7', auth_mode: 'apikey', last_refresh: null }; + await writeFile(files(source)[1]!, JSON.stringify(auth)); + await copyCredentials(home, source, copied); + expect(JSON.parse(await readFile(copied[1]!, 'utf8'))).toEqual(auth); + expect(() => assertNoKnownSecrets(auth.OPENAI_API_KEY)).toThrow('Comment contains copied credentials'); + }); + test('cleanup removes only copied files while preserving candidate state, private raw evidence, config directories, and source credentials', async () => { + const root = await fixture(), source = await sources(root), home = join(root, 'candidate'), copied: string[] = []; + await copyCredentials(home, source, copied); + const raw = join(home, 'surface.raw'), artifact = join(home, 'reviewed-transcript.txt'), state = join(home, '.claude/session-state.json'); + await writeFile(raw, 'private raw transcript'); await writeFile(artifact, 'private reviewed evidence'); await writeFile(state, 'candidate session state'); + const before = await Promise.all(files(source).map(path => readFile(path, 'utf8'))); + await removeCredentials(copied); await removeCredentials(copied); for (const path of copied) expect(await Bun.file(path).exists()).toBe(false); expect(await readFile(raw, 'utf8')).toBe('private raw transcript'); expect(await readFile(artifact, 'utf8')).toBe('private reviewed evidence'); expect(await readFile(state, 'utf8')).toBe('candidate session state'); expect((await stat(join(home, '.claude'))).isDirectory()).toBe(true); expect((await stat(join(home, '.codex'))).isDirectory()).toBe(true); - expect(await Promise.all(originals.map(path => readFile(path, 'utf8')))).toEqual(before); + expect(await Promise.all(files(source).map(path => readFile(path, 'utf8')))).toEqual(before); }); - test('partial copy cleanup removes the copied file without deleting candidate evidence or changing the vault', async () => { - const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; - const bad = join(base, 'codex', accounts.codex, 'auth.json'); - await writeFile(bad, '{"unknown":true}'); - await expect(copyCredentials(home, base, accounts, copied)).rejects.toThrow('unknown format'); + test('partial copy cleanup retains failed-run evidence and malformed source files', async () => { + const root = await fixture(), source = await sources(root), home = join(root, 'candidate'), copied: string[] = []; + const bad = files(source)[1]!; await writeFile(bad, '{"unknown":true}'); + await expect(copyCredentials(home, source, copied)).rejects.toThrow(loginFix('codex', source.codex)); expect(copied).toHaveLength(1); const evidence = join(home, 'prepare-failure.txt'); await writeFile(evidence, 'retain failed-run evidence'); - await removeCredentials(copied); - expect(await Bun.file(copied[0]!).exists()).toBe(false); expect(await readFile(evidence, 'utf8')).toBe('retain failed-run evidence'); expect(await readFile(bad, 'utf8')).toBe('{"unknown":true}'); }); - test('vault credential files and selected account directories must be canonical regular sources', async () => { - const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; - const credential = join(base, 'claude', accounts.claude, '.credentials.json'), outside = join(root, 'outside-auth'); + test('source credentials and selected directories must be canonical regular sources', async () => { + const root = await fixture(), source = await sources(root), home = join(root, 'candidate'), copied: string[] = []; + const credential = files(source)[0]!, outside = join(root, 'outside-auth'); await writeFile(outside, '{"claudeAiOauth":{"accessToken":"external"}}'); await rm(credential); await symlink(outside, credential); - await expect(copyCredentials(home, base, accounts, copied)).rejects.toThrow('contained regular file'); + await expect(copyCredentials(home, source, copied)).rejects.toThrow('contained regular file'); expect(copied).toEqual([]); - - await rm(join(base, 'claude', accounts.claude), { recursive: true }); - await symlink(join(base, 'codex', accounts.codex), join(base, 'claude', accounts.claude)); - await expect(copyCredentials(home, base, accounts, copied)).rejects.toThrow('not symlinked'); + await rm(source.claude, { recursive: true }); await symlink(source.codex, source.claude); + await expect(copyCredentials(home, source, copied)).rejects.toThrow('not symlinked'); expect(await readFile(outside, 'utf8')).toContain('external'); }); test('candidate config-directory redirects fail closed without touching external sentinels', async () => { - const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), external = join(root, 'external'); + const root = await fixture(), source = await sources(root), home = join(root, 'candidate'), external = join(root, 'external'); await mkdir(home); await mkdir(external); const sentinel = join(external, '.credentials.json'); await writeFile(sentinel, 'external sentinel'); await symlink(external, join(home, '.claude')); const copied: string[] = []; - - await expect(copyCredentials(home, base, accounts, copied)).rejects.toThrow('redirected'); - expect(await readFile(sentinel, 'utf8')).toBe('external sentinel'); - expect(copied).toEqual([]); - + await expect(copyCredentials(home, source, copied)).rejects.toThrow('redirected'); + expect(await readFile(sentinel, 'utf8')).toBe('external sentinel'); expect(copied).toEqual([]); await expect(removeCredentials([join(home, '.claude/.credentials.json')])).rejects.toThrow('redirected candidate credential directories'); - expect(await readFile(sentinel, 'utf8')).toBe('external sentinel'); - expect((await stat(home)).isDirectory()).toBe(true); + expect(await readFile(sentinel, 'utf8')).toBe('external sentinel'); expect((await stat(home)).isDirectory()).toBe(true); }); - test('cleanup unlinks a copied-file leaf redirect without following it or removing candidate state', async () => { - const root = await fixture(), base = await vault(root), home = join(root, 'candidate'), copied: string[] = []; - await copyCredentials(home, base, accounts, copied); + test('cleanup unlinks copied-file leaf redirects without following them or removing candidate state', async () => { + const root = await fixture(), source = await sources(root), home = join(root, 'candidate'), copied: string[] = []; + await copyCredentials(home, source, copied); const external = join(root, 'external-credential'); await writeFile(external, 'external sentinel'); await rm(copied[0]!); await symlink(external, copied[0]!); const state = join(home, 'state.json'); await writeFile(state, 'preserved'); - await removeCredentials(copied); - expect(await Bun.file(copied[0]!).exists()).toBe(false); - expect(await readFile(external, 'utf8')).toBe('external sentinel'); - expect(await readFile(state, 'utf8')).toBe('preserved'); + expect(await readFile(external, 'utf8')).toBe('external sentinel'); expect(await readFile(state, 'utf8')).toBe('preserved'); + }); + + test('an existing destination is not added to cleanup or deleted on copy failure', async () => { + const root = await fixture(), source = await sources(root), home = join(root, 'candidate'), copied: string[] = []; + await mkdir(join(home, '.claude'), { recursive: true }); + const existing = join(home, '.claude/.credentials.json'); await writeFile(existing, 'existing credential sentinel'); + await expect(copyCredentials(home, source, copied)).rejects.toThrow(); + expect(copied).toEqual([]); await removeCredentials(copied); + expect(await readFile(existing, 'utf8')).toBe('existing credential sentinel'); }); }); diff --git a/.claude/skills/verify-open-pstack/scripts/isolation.ts b/.claude/skills/verify-open-pstack/scripts/isolation.ts index ab97742f..b24bee34 100644 --- a/.claude/skills/verify-open-pstack/scripts/isolation.ts +++ b/.claude/skills/verify-open-pstack/scripts/isolation.ts @@ -1,27 +1,31 @@ -import { chmod, lstat, mkdir, readFile, realpath, unlink, writeFile } from 'node:fs/promises'; +import { chmod, lstat, mkdir, open, readFile, realpath, unlink } from 'node:fs/promises'; import { isAbsolute, join, resolve } from 'node:path'; import { isolatedEnv, registerSecrets } from './io.ts'; -import { HARNESSES, type Accounts, type Harness } from './types.ts'; +import { HARNESSES, type CredentialSources, type Harness } from './types.ts'; -export function vaultRoot(): string { - if (process.env.CAAM_HOME) return join(process.env.CAAM_HOME, 'data/vault'); - if (process.env.XDG_DATA_HOME) return join(process.env.XDG_DATA_HOME, 'caam/vault'); - if (!process.env.HOME || !isAbsolute(process.env.HOME)) throw new Error('Trusted parent requires an absolute operator HOME'); - return join(process.env.HOME, '.local/share/caam/vault'); +export function loginFix(tool: Harness, directory: string): string { + const quoted = `'${directory.replaceAll("'", "'\\''")}'`; + return tool === 'claude' ? `CLAUDE_CONFIG_DIR=${quoted} claude auth login` : `CODEX_HOME=${quoted} codex login`; } -export function selectedAccounts(text: string, accounts: Accounts): Accounts { - const result = JSON.parse(text); - if (!Array.isArray(result.profiles)) throw new Error('Unknown caam list schema; expected profiles'); +export async function credentialSources(sources: CredentialSources): Promise { + const canonical = {} as CredentialSources; for (const tool of HARNESSES) { - const name = accounts[tool]; - if (!/^[A-Za-z0-9.!#$%&'*+=?^_`{|}~-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/.test(name) || name.length > 254) throw new Error(`Explicit safe ${tool} account email required`); - const matching = result.profiles.filter((profile: Record) => profile.tool === tool && profile.name === name); - if (matching.length !== 1 || (matching[0].identity as Record | undefined)?.email !== name) { - throw new Error(`caam ${tool}/${name} identity mismatch or missing; inspect caam list --json (never activate or repair automatically)`); + const directory = sources[tool]; + try { + if (typeof directory !== 'string' || !directory || directory.includes('\0')) throw new Error('explicit source directory required'); + const path = resolve(directory), info = await lstat(path); + if (!info.isDirectory() || info.isSymbolicLink()) throw new Error('source directory must be a directory, not symlinked'); + canonical[tool] = await realpath(path); + const source = join(canonical[tool], tool === 'claude' ? '.credentials.json' : 'auth.json'); + const sourceInfo = await lstat(source); + if (!sourceInfo.isFile() || sourceInfo.isSymbolicLink() || await realpath(source) !== source) throw new Error('credentials must be a contained regular file'); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + throw new Error(`${tool} credential source unavailable: ${reason}; log in once with: ${loginFix(tool, directory ?? '')}`); } } - return accounts; + return canonical; } function credentialLocation(home: string, tool: Harness): { directory: string; path: string } { @@ -39,38 +43,36 @@ async function canonicalDirectory(path: string, message: string): Promise function credentialTokens(tool: Harness, auth: Record): string[] { const fields = (tool === 'claude' ? auth.claudeAiOauth : auth.tokens) as Record | undefined; const access = tool === 'claude' ? fields?.accessToken : fields?.access_token; - if (typeof access !== 'string') throw new Error(`${tool} vault requires file-backed OAuth credentials; unknown format blocks`); + const apiKey = tool === 'codex' && typeof auth.OPENAI_API_KEY === 'string' && auth.OPENAI_API_KEY ? auth.OPENAI_API_KEY : undefined; + if ((typeof access !== 'string' || !access) && !apiKey) throw new Error(`${tool} requires file-backed native credentials; unknown format blocks`); const names = tool === 'claude' ? ['accessToken', 'refreshToken'] : ['access_token', 'refresh_token', 'id_token']; - return names.flatMap(name => typeof fields?.[name] === 'string' ? [fields[name] as string] : []); + return [...(apiKey ? [apiKey] : []), ...names.flatMap(name => typeof fields?.[name] === 'string' ? [fields[name] as string] : [])]; } -export async function copyCredentials(home: string, vault: string, accounts: Accounts, copied: string[]): Promise { +export async function copyCredentials(home: string, sources: CredentialSources, copied: string[]): Promise { if (!isAbsolute(home) || resolve(home) !== home) throw new Error('Credential copy requires canonical candidate state'); - const base = await realpath(vault); + const canonical = await credentialSources(sources); await mkdir(home, { recursive: true, mode: 0o700 }); await canonicalDirectory(home, 'Credential copy refuses redirected candidate state'); for (const tool of HARNESSES) { - const file = tool === 'claude' ? '.credentials.json' : 'auth.json'; - const profile = join(base, tool, accounts[tool]), source = join(profile, file); - for (const directory of [join(base, tool), profile]) { - const info = await lstat(directory); - if (!directory.startsWith(base + '/') || !info.isDirectory() || info.isSymbolicLink() || await realpath(directory) !== directory) { - throw new Error('Selected vault directories must be canonical, not symlinked'); - } - } - const sourceInfo = await lstat(source); - if (!sourceInfo.isFile() || sourceInfo.isSymbolicLink() || await realpath(source) !== source) throw new Error('Vault credentials must be a contained regular file'); + const source = join(canonical[tool], tool === 'claude' ? '.credentials.json' : 'auth.json'); const bytes = await readFile(source); - const auth = JSON.parse(bytes.toString()) as Record; - registerSecrets(credentialTokens(tool, auth)); + try { + const auth = JSON.parse(bytes.toString()) as Record; + registerSecrets(credentialTokens(tool, auth)); + } catch { + throw new Error(`${tool} credential source has unsupported file-backed credentials; log in once with: ${loginFix(tool, canonical[tool])}`); + } const target = credentialLocation(home, tool); await mkdir(target.directory, { recursive: true, mode: 0o700 }); await canonicalDirectory(target.directory, 'Candidate credential directory redirected'); await chmod(target.directory, 0o700); + const file = await open(target.path, 'wx', 0o600); copied.push(target.path); - await writeFile(target.path, bytes, { mode: 0o600, flag: 'wx' }); + try { await file.writeFile(bytes); } + finally { await file.close(); } } } diff --git a/.claude/skills/verify-open-pstack/scripts/types.ts b/.claude/skills/verify-open-pstack/scripts/types.ts index 423af112..ac16e94b 100644 --- a/.claude/skills/verify-open-pstack/scripts/types.ts +++ b/.claude/skills/verify-open-pstack/scripts/types.ts @@ -1,7 +1,7 @@ export const REPO = 'ericlitman/open-pstack'; export const HARNESSES = ['claude', 'codex'] as const; export type Harness = typeof HARNESSES[number]; -export type Accounts = Record; +export type CredentialSources = Record; export type Phase = 'resolve' | 'classify' | 'prepare' | 'exercise' | 'publish' | 'failed' | 'head-moved'; export interface Pull { number: number; head: { sha: string }; base: { sha: string }; @@ -26,7 +26,7 @@ export interface Observation { } export interface Installation { harness: Harness; cliVersion: string; pluginVersion: string; sha: string; - location: string; treeHash: string; home: string; account?: string; sourceHash?: string; + location: string; treeHash: string; home: string; credentialSource?: string; sourceHash?: string; } export interface Receipt { schema: 1; repo: string; pr: number; sha: string; base: string; phase: Phase; diff --git a/AGENTS.md b/AGENTS.md index bb9b3820..c6c5edc4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,6 +8,6 @@ Before opening a pull request, run the Bun tests, strict typecheck, static invar Nothing merges, tags, releases, or rolls out until the exact candidate is installed and the changed behavior passes a live test from the real user surface in every affected harness. Unit tests, validators, source inspection, and self-reports do not satisfy this gate. The record is the `live-gate` commit status on that exact head, which links an evidence comment naming the installed version, surface, action, and observed result. Mergify does not queue a pull request without it. A pull request may be ready for review before its live test, but every new head, including a queue-created head, needs its own. -For Open Pstack PRs, use the shared repository-local `.claude/skills/verify-open-pstack/` skill (Codex: `.agents/skills/verify-open-pstack`). Run the reviewed verifier from trusted `main`, recording the publisher revision separately from the candidate SHA. Read Launch for the explicit caam account options and separate private probe/live directories. Daily-active accounts are allowed; doctor checks each selected profile with `caam limits --format json` and reports `caam add --no-activate --force` for unauthorized or expired credentials without repairing them automatically. Candidate processes and provider children retain real `HOME`/`USER`/`LOGNAME` with both provider config roots and temporary state run-owned; full disposable credential copies retain native refresh material and are never written back to the vault. Cleanup removes only copied credential files and preserves mode-0700 raw evidence through merge. Setup coverage fails closed with `setup exercise requires #120 (setup-pstack config-home)` until #120 merges. Publication is one structured evidence comment plus exact-SHA `live-gate`, with exact copied-secret checks on the public comment only. No PR-body, ready/draft, reversal, or compensation writes, publisher-code attestation, or OS filesystem-denial claims are permitted. Builders do not publish the gate, queue, or merge. +For Open Pstack PRs, use the shared repository-local `.claude/skills/verify-open-pstack/` skill (Codex: `.agents/skills/verify-open-pstack`). Run the reviewed verifier from trusted `main`, recording the publisher revision separately from the candidate SHA. Read Launch for the explicit `--claude-config ` and `--codex-home ` options and separate private probe/live directories. Supply operator-pre-authenticated normal-login configuration directories; daily-active accounts are allowed. Doctor validates the source directories with `claude auth status` and `codex login status`. Authentication failures report `CLAUDE_CONFIG_DIR= claude auth login` or `CODEX_HOME= codex login` without logging in automatically. Candidate processes and provider children retain real `HOME`/`USER`/`LOGNAME` with both provider config roots and temporary state run-owned; only `.credentials.json` and `auth.json` are copied, retaining native refresh material, and are never written back to the source directories. Cleanup removes only copied credential files and preserves mode-0700 raw evidence through merge. Setup coverage fails closed with `setup exercise requires #120 (setup-pstack config-home)` until #120 merges. Publication is one structured evidence comment plus exact-SHA `live-gate`, with exact copied-secret checks on the public comment only. No PR-body, ready/draft, reversal, or compensation writes, publisher-code attestation, or OS filesystem-denial claims are permitted. Builders do not publish the gate, queue, or merge. Do not add an implicit runtime timeout or a weaker-model fallback. From b8d62368b8439c863e564a7ac6911ec530520382 Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 05:43:16 +0000 Subject: [PATCH 17/20] fix(#90): preserve operator evidence and exercise candidate verifier Re-prompt unpublishable observation values and require verifier-changing PRs to run their reviewed candidate publisher before preparing native sessions. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .claude/skills/verify-open-pstack/SKILL.md | 4 +-- .../scripts/harness.test.ts | 26 +++++++++++++++++++ .../verify-open-pstack/scripts/harness.ts | 9 ++++--- .../verify-open-pstack/scripts/verify.test.ts | 21 +++++++++++++++ .../verify-open-pstack/scripts/verify.ts | 13 +++++++--- AGENTS.md | 2 +- 6 files changed, 65 insertions(+), 10 deletions(-) diff --git a/.claude/skills/verify-open-pstack/SKILL.md b/.claude/skills/verify-open-pstack/SKILL.md index d3d6724f..36976a29 100644 --- a/.claude/skills/verify-open-pstack/SKILL.md +++ b/.claude/skills/verify-open-pstack/SKILL.md @@ -7,7 +7,7 @@ description: Verify an Open Pstack PR's exact candidate head in native Claude Co ## Launch -This repository-local, non-shipped skill is shared with Codex through `.agents/skills/verify-open-pstack`. Run the reviewed verifier from a trusted `main` checkout. The trusted parent owns GitHub reads and publication; candidate processes never receive publisher credentials. Its only GitHub writes are one structured evidence comment and `live-gate` status on the exact candidate SHA; it never queues or merges. +This repository-local, non-shipped skill is shared with Codex through `.agents/skills/verify-open-pstack`. Verifier-changing PRs run from a checkout of their reviewed candidate head; all other PRs run from trusted `main`. The trusted parent owns GitHub reads and publication; candidate processes never receive publisher credentials. Its only GitHub writes are one structured evidence comment and `live-gate` status on the exact candidate SHA; it never queues or merges. Install development dependencies separately when testing this skill; `verify.sh` does not install at runtime: @@ -31,7 +31,7 @@ EVIDENCE="$SESSION/live" # must not exist yet Replace both placeholders with existing operator-pre-authenticated normal-login configuration directories. Selection is explicit; a selected account may be active in daily use. No profile manager or maintainer-specific infrastructure is required. Authenticate those directories manually before verification; the verifier never starts a login flow. Omit `--self-test` for ordinary plugin verification; it is mandatory for this skill's delivery. -The trusted `main` publisher revision is recorded separately from the PR's candidate SHA. PR #111 bootstrap must use the maintainer-reviewed trusted publisher head, while all candidate installation, classification, evidence, comments, and `live-gate` status remain bound to PR #111's exact candidate SHA. Use a fresh private output directory for every run. The verifier requires an open same-repository PR, pins exact head and base SHAs, classifies immutable Git objects, and rechecks them at phase and publication boundaries. Unknown paths abort; evidence never transfers to another head. +The publisher revision is recorded separately from the candidate SHA and must equal it when classification selects `project-skill`. Run those PRs from their reviewed candidate head; run all other PRs from trusted `main`. Use a fresh private output directory for every run. The verifier requires an open same-repository PR, pins exact head and base SHAs, classifies immutable Git objects, and rechecks them at phase and publication boundaries. Unknown paths abort; evidence never transfers to another head. ## Doctor diff --git a/.claude/skills/verify-open-pstack/scripts/harness.test.ts b/.claude/skills/verify-open-pstack/scripts/harness.test.ts index e63828e9..7143447d 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.test.ts @@ -5,6 +5,8 @@ import { join } from 'node:path'; import { doctor } from './doctor.ts'; import { codexInstallation, launch, MacDriver, verifyCodexEnabled, verifyProjectDoctor } from './harness.ts'; import { newReceipt } from './core.ts'; +import { sourceDigest } from './provenance.ts'; +import { evidence } from './verify.ts'; import { command, freshRoot, isolatedEnv, retainedFile, treeHash, type Command } from './io.ts'; const roots: string[] = []; async function fixture(): Promise { const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-test-'))); roots.push(root); return root; } @@ -269,6 +271,30 @@ describe('isolated harness boundaries', () => { } expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('pass'); }); + test('exercise re-prompts unsafe evidence and publishes accepted answers verbatim', async () => { + const root = await fixture(), home = join(root, 'state/claude'), workspace = join(home, 'workspace'); + for (const path of ['plugins/pstack', '.claude/skills/verify-open-pstack', '.agents/skills']) await mkdir(join(workspace, path), { recursive: true }); + await symlink('../../.claude/skills/verify-open-pstack', join(workspace, '.agents/skills/verify-open-pstack')); + await writeFile(join(workspace, 'plugins/pstack/SKILL.md'), 'candidate plugin'); + await writeFile(join(root, 'transcript.txt'), 'reviewed native transcript'); + await writeFile(join(root, 'artifact.txt'), 'reviewed effect'); + const receipt = newReceipt(111, 'a'.repeat(40), 'b'.repeat(40), false, root); + receipt.selection = { paths: [], skills: ['architect'], features: ['skill-invocation:architect'], noRuntime: false }; + receipt.installations = [{ harness: 'claude', home, location: join(workspace, 'plugins/pstack'), sha: receipt.sha, cliVersion: 'test', pluginVersion: 'test', treeHash: await treeHash(join(workspace, 'plugins/pstack')), sourceHash: await sourceDigest(workspace) }]; + const long = 's'.repeat(54), answers = ['transcript.txt', long, '`unsafe`', long, '✓', long, 'artifact.txt', 'PASS skill-invocation:architect']; + const questions: string[] = []; + const driver = new MacDriver(async () => '', async question => { questions.push(question); return answers.shift()!; }); + receipt.observations = await driver.exercise(receipt); + expect(receipt.observations[0]!.surface).toBe(long); + expect(questions.filter(question => question.startsWith('Concrete action'))).toHaveLength(2); + expect(questions.filter(question => question.startsWith('Observed assertion'))).toHaveLength(2); + for (const question of questions.slice(1, 6)) expect(question).toContain('printable ASCII, no backtick, < or >, and at most 160 characters'); + const comment = evidence(receipt); + expect(comment).toContain(`surface ${long}; action ${long}; result ${long};`); + expect(comment).not.toContain('[value omitted]'); + expect(answers).toHaveLength(0); + }); + test('Codex exact installed tree and enabled listing are required', async () => { const root = await fixture(), plugin = join(root, 'config/plugins/pstack'); await mkdir(plugin, { recursive: true }); await writeFile(join(plugin, 'SKILL.md'), 'candidate'); diff --git a/.claude/skills/verify-open-pstack/scripts/harness.ts b/.claude/skills/verify-open-pstack/scripts/harness.ts index 404484f4..45c4c559 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.ts @@ -6,6 +6,7 @@ import { command, interruption, isolatedEnv, retainedFile, save, treeHash, type import { doctor } from './doctor.ts'; import { copyCredentials, credentialSources, removeCredentials } from './isolation.ts'; import { sourceDigest, sourceHash } from './provenance.ts'; +import { EVIDENCE_LIMIT, publishable } from './verify.ts'; import { HARNESSES, REPO, type CredentialSources, type Driver, type Harness, type Installation, type Observation, type Receipt } from './types.ts'; export type Ask = (question: string) => Promise; @@ -125,9 +126,11 @@ export class MacDriver implements Driver { if (await sourceDigest(workspace) !== installation.sourceHash) throw new Error('Pinned project/plugin source changed during exercise'); const transcript = await retainedFile(receipt.artifactRoot, await this.review(`Retain a private copy of ${raw} inside output (outside state), review it, and enter its path. Do not publish raw content:`)); for (const feature of requiredFeatures(receipt, harness)) { - const surface = await this.review(`${harness}/${feature}: native surface/discovery entry point?`); - const action = await this.review('Concrete action exercised?'); - const observed = await this.review('Observed assertion/result (not the model\'s success claim)?'); + const answers = { surface: '', action: '', observed: '' }; + for (const [field, question] of [['surface', `${harness}/${feature}: native surface/discovery entry point?`], ['action', 'Concrete action exercised?'], ['observed', 'Observed assertion/result (not the model\'s success claim)?']] as const) { + do { answers[field] = await this.review(`${question} Use printable ASCII, no backtick, < or >, and at most ${EVIDENCE_LIMIT} characters.`); } while (!publishable(answers[field])); + } + const { surface, action, observed } = answers; const artifacts = []; for (const path of (await this.review('Reviewed artifact paths inside output, one or more separated by commas?')).split(',')) { artifacts.push(await retainedFile(receipt.artifactRoot, path.trim())); diff --git a/.claude/skills/verify-open-pstack/scripts/verify.test.ts b/.claude/skills/verify-open-pstack/scripts/verify.test.ts index f8e7cf06..8a57d78c 100644 --- a/.claude/skills/verify-open-pstack/scripts/verify.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/verify.test.ts @@ -66,6 +66,27 @@ describe('exact-head evidence publication', () => { expect(receipt.observations.map(record => `${record.harness}/${record.feature}`).sort()).toEqual(['claude/project-skill', 'codex/project-skill']); }); + test('verifier changes require the candidate publisher before any driver call or GitHub write', async () => { + for (const revision of [REVISION, SHA]) { + const f = fixture(); f.options.publisherRevision = revision; + f.options.github.files = async () => [{ filename: '.claude/skills/verify-open-pstack/scripts/verify.ts' }]; + f.options.driver.cleanup = async () => { f.calls.push('cleanup'); }; + if (revision !== SHA) { + await expect(verify(f.options)).rejects.toThrow('rerun from that checkout'); + expect(f.calls.some(call => ['prepare', 'exercise', 'cleanup', 'comment'].includes(call) || call.startsWith('status:'))).toBe(false); + } else { + expect((await verify(f.options)).status).toBe('success'); + expect(f.calls).toContain('prepare'); expect(f.calls).toContain('exercise'); + } + } + }); + + test('non-verifier changes do not require a candidate publisher revision', async () => { + const f = fixture(true); + expect((await verify(f.options)).status).toBe('success'); + expect(f.calls).toContain('prepare'); + }); + test('successful verification cleans disposable credentials before any publication', async () => { const f = fixture(true); f.options.driver.cleanup = async () => { f.calls.push('cleanup'); }; const receipt = await verify(f.options); diff --git a/.claude/skills/verify-open-pstack/scripts/verify.ts b/.claude/skills/verify-open-pstack/scripts/verify.ts index 04186e0f..618b0a99 100644 --- a/.claude/skills/verify-open-pstack/scripts/verify.ts +++ b/.claude/skills/verify-open-pstack/scripts/verify.ts @@ -4,7 +4,9 @@ import { REPO, type Driver, type GitHub, type Pull, type Receipt, type Registry class HeadMoved extends Error {} // Omit unsafe/oversized values whole: truncating or rewriting one could hide an exact registered secret. -const safe = (text: string, limit = 160): string => text.length <= limit && /^[\x20-\x7e]*$/.test(text) && !/[`<>]/.test(text) ? text : '[value omitted]'; +export const EVIDENCE_LIMIT = 160; +export const publishable = (text: string, limit = EVIDENCE_LIMIT): boolean => text.length <= limit && /^[\x20-\x7e]*$/.test(text) && !/[`<>]/.test(text); +const safe = (text: string, limit = EVIDENCE_LIMIT): string => publishable(text, limit) ? text : '[value omitted]'; /** Build the single structured public record for this run. Failure details remain private in receipt.json. */ export function evidence(receipt: Receipt): string { @@ -24,7 +26,7 @@ export function evidence(receipt: Receipt): string { } let shown = 0; for (const record of receipt.observations) { - const line = `- Observation (${record.harness} / ${safe(record.feature, 72)}): surface ${safe(record.surface, 48)}; action ${safe(record.action, 48)}; result ${safe(record.observed, 48)}; operator reviewed; transcript SHA-256 \`${record.transcriptHash}\`; artifact-set SHA-256 \`${sha256(JSON.stringify(record.artifacts))}\`.`; + const line = `- Observation (${record.harness} / ${safe(record.feature, 72)}): surface ${safe(record.surface)}; action ${safe(record.action)}; result ${safe(record.observed)}; operator reviewed; transcript SHA-256 \`${record.transcriptHash}\`; artifact-set SHA-256 \`${sha256(JSON.stringify(record.artifacts))}\`.`; if (lines.join('\n').length + line.length > 48000) break; lines.push(line); shown++; } @@ -88,9 +90,12 @@ export async function verify(options: VerifyOptions): Promise { const appendFailure = (label: string, error: unknown): void => { receipt.failure = `${receipt.failure}; ${label}: ${error instanceof Error ? error.message : String(error)}`; }; + await phase('classify'); + receipt.selection = classify(await github.files(receipt.base, receipt.sha), options.registry); + if (receipt.selection.features.includes('project-skill') && receipt.publisherRevision !== receipt.sha) { + throw new Error('Verifier-changing PRs must run from a checkout of the candidate head; rerun from that checkout'); + } try { - await phase('classify'); - receipt.selection = classify(await github.files(receipt.base, receipt.sha), options.registry); await current(); await persist(receipt); if (!receipt.selection.noRuntime || receipt.selfTest) { await phase('prepare'); receipt.installations = await driver.prepare(receipt); await current(); await persist(receipt); diff --git a/AGENTS.md b/AGENTS.md index c6c5edc4..d05622ec 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,6 +8,6 @@ Before opening a pull request, run the Bun tests, strict typecheck, static invar Nothing merges, tags, releases, or rolls out until the exact candidate is installed and the changed behavior passes a live test from the real user surface in every affected harness. Unit tests, validators, source inspection, and self-reports do not satisfy this gate. The record is the `live-gate` commit status on that exact head, which links an evidence comment naming the installed version, surface, action, and observed result. Mergify does not queue a pull request without it. A pull request may be ready for review before its live test, but every new head, including a queue-created head, needs its own. -For Open Pstack PRs, use the shared repository-local `.claude/skills/verify-open-pstack/` skill (Codex: `.agents/skills/verify-open-pstack`). Run the reviewed verifier from trusted `main`, recording the publisher revision separately from the candidate SHA. Read Launch for the explicit `--claude-config ` and `--codex-home ` options and separate private probe/live directories. Supply operator-pre-authenticated normal-login configuration directories; daily-active accounts are allowed. Doctor validates the source directories with `claude auth status` and `codex login status`. Authentication failures report `CLAUDE_CONFIG_DIR= claude auth login` or `CODEX_HOME= codex login` without logging in automatically. Candidate processes and provider children retain real `HOME`/`USER`/`LOGNAME` with both provider config roots and temporary state run-owned; only `.credentials.json` and `auth.json` are copied, retaining native refresh material, and are never written back to the source directories. Cleanup removes only copied credential files and preserves mode-0700 raw evidence through merge. Setup coverage fails closed with `setup exercise requires #120 (setup-pstack config-home)` until #120 merges. Publication is one structured evidence comment plus exact-SHA `live-gate`, with exact copied-secret checks on the public comment only. No PR-body, ready/draft, reversal, or compensation writes, publisher-code attestation, or OS filesystem-denial claims are permitted. Builders do not publish the gate, queue, or merge. +For Open Pstack PRs, use the shared repository-local `.claude/skills/verify-open-pstack/` skill (Codex: `.agents/skills/verify-open-pstack`). Verifier-changing PRs run from a checkout of their reviewed candidate head, with the publisher revision equal to the candidate SHA; all other PRs run from trusted `main`. Record the publisher revision separately from the candidate SHA. Read Launch for the explicit `--claude-config ` and `--codex-home ` options and separate private probe/live directories. Supply operator-pre-authenticated normal-login configuration directories; daily-active accounts are allowed. Doctor validates the source directories with `claude auth status` and `codex login status`. Authentication failures report `CLAUDE_CONFIG_DIR= claude auth login` or `CODEX_HOME= codex login` without logging in automatically. Candidate processes and provider children retain real `HOME`/`USER`/`LOGNAME` with both provider config roots and temporary state run-owned; only `.credentials.json` and `auth.json` are copied, retaining native refresh material, and are never written back to the source directories. Cleanup removes only copied credential files and preserves mode-0700 raw evidence through merge. Setup coverage fails closed with `setup exercise requires #120 (setup-pstack config-home)` until #120 merges. Publication is one structured evidence comment plus exact-SHA `live-gate`, with exact copied-secret checks on the public comment only. No PR-body, ready/draft, reversal, or compensation writes, publisher-code attestation, or OS filesystem-denial claims are permitted. Builders do not publish the gate, queue, or merge. Do not add an implicit runtime timeout or a weaker-model fallback. From c2bb2d7653d9c605cc936d7d20a052f14cf38a4f Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 12:20:11 +0000 Subject: [PATCH 18/20] fix(#90): reuse native logins and restore setup changes Remove copied credential handling and source flags so verification reuses existing native logins. Preserve setup files byte-for-byte and prohibit publication after restoration failures. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .claude/skills/verify-open-pstack/SKILL.md | 34 ++- .../verify-open-pstack/features/README.md | 5 +- .../verify-open-pstack/features/isolation.md | 41 ++-- .../verify-open-pstack/features/runner.md | 4 +- .../verify-open-pstack/features/setup.md | 12 +- .../features/shipped-tools.md | 2 +- .../features/skill-invocation.md | 2 +- .../verify-open-pstack/scripts/cli.test.ts | 55 +---- .../skills/verify-open-pstack/scripts/cli.ts | 29 +-- .../verify-open-pstack/scripts/doctor.test.ts | 109 ++------ .../verify-open-pstack/scripts/doctor.ts | 25 +- .../scripts/harness.test.ts | 177 ++++++------- .../verify-open-pstack/scripts/harness.ts | 55 +++-- .../verify-open-pstack/scripts/io.test.ts | 27 +- .../skills/verify-open-pstack/scripts/io.ts | 10 +- .../scripts/isolation.test.ts | 232 ++++++++---------- .../verify-open-pstack/scripts/isolation.ts | 142 ++++------- .../verify-open-pstack/scripts/types.ts | 3 +- .../verify-open-pstack/scripts/verify.test.ts | 15 +- .../verify-open-pstack/scripts/verify.ts | 18 +- AGENTS.md | 2 +- 21 files changed, 380 insertions(+), 619 deletions(-) diff --git a/.claude/skills/verify-open-pstack/SKILL.md b/.claude/skills/verify-open-pstack/SKILL.md index 36976a29..771085bc 100644 --- a/.claude/skills/verify-open-pstack/SKILL.md +++ b/.claude/skills/verify-open-pstack/SKILL.md @@ -19,59 +19,55 @@ Then run from the trusted publisher checkout: ```sh PR=111 # supplied delivery PR number -CLAUDE_SOURCE='/absolute/path/to/authenticated/claude-config' -CODEX_SOURCE='/absolute/path/to/authenticated/codex-home' SESSION="$(mktemp -d "${TMPDIR:-/tmp}/open-pstack-evidence.XXXXXX")" EVIDENCE="$SESSION/live" # must not exist yet -.claude/skills/verify-open-pstack/scripts/verify.sh doctor \ - --claude-config "$CLAUDE_SOURCE" --codex-home "$CODEX_SOURCE" --output "$SESSION/probe" -.claude/skills/verify-open-pstack/scripts/verify.sh run --pr "$PR" --self-test \ - --claude-config "$CLAUDE_SOURCE" --codex-home "$CODEX_SOURCE" --output "$EVIDENCE" +.claude/skills/verify-open-pstack/scripts/verify.sh doctor --output "$SESSION/probe" +.claude/skills/verify-open-pstack/scripts/verify.sh run --pr "$PR" --self-test --output "$EVIDENCE" ``` -Replace both placeholders with existing operator-pre-authenticated normal-login configuration directories. Selection is explicit; a selected account may be active in daily use. No profile manager or maintainer-specific infrastructure is required. Authenticate those directories manually before verification; the verifier never starts a login flow. Omit `--self-test` for ordinary plugin verification; it is mandatory for this skill's delivery. +Reuse the Claude Code and Codex logins you already have; no extra login or credential source flags. Omit `--self-test` for ordinary plugin verification; it is mandatory for this skill's delivery. The publisher revision is recorded separately from the candidate SHA and must equal it when classification selects `project-skill`. Run those PRs from their reviewed candidate head; run all other PRs from trusted `main`. Use a fresh private output directory for every run. The verifier requires an open same-repository PR, pins exact head and base SHAs, classifies immutable Git objects, and rechecks them at phase and publication boundaries. Unknown paths abort; evidence never transfers to another head. ## Doctor -The trusted-parent doctor checks Bun, git, both native harness CLIs, and the capabilities needed by the run. It requires both supplied source directories to exist and validates their authentication with `CLAUDE_CONFIG_DIR= claude auth status` and `CODEX_HOME= codex login status`. Unauthorized, expired, missing, or unusable authentication fails closed with the corresponding manual login command: `CLAUDE_CONFIG_DIR= claude auth login` or `CODEX_HOME= codex login`. The verifier never executes login or repairs credentials automatically. Candidate-mode doctor checks only candidate-safe capabilities and does not inspect source credentials or publisher authentication. Capability and authentication checks do not prove installation or live behavior; missing requirements fail closed. +The parent doctor checks Bun, git, both native harness CLIs, and required capabilities. There are no credential-source flags or source-directory authentication checks. Candidate-mode doctor checks only candidate-safe capabilities, not publisher authentication. Capability checks do not prove installation or live behavior; missing requirements fail closed. -Candidates retain the operator's real `HOME`, `USER`, and `LOGNAME`. `CLAUDE_CONFIG_DIR`, `CODEX_HOME`, `TMPDIR`, `GH_CONFIG_DIR`, and other XDG/config/cache locations are run-owned. Git configuration is disabled or run-owned. This separation is operational isolation, not an OS security boundary: the verifier makes no Seatbelt, source-protection, Keychain-denial, daily-home secrecy, or Mac filesystem-denial claim. +Candidates retain real `HOME`, `USER`, and `LOGNAME`. Claude uses normal operator config/login. Codex uses a run-owned `CODEX_HOME` with `auth.json` symlinked to the operator's `${CODEX_HOME:-$HOME/.codex}/auth.json`, resolving the daily home before overriding the environment. If that source file is absent, stop with a plain message that this tool needs file-based Codex auth; do not start login. Refresh writes through the symlink to the single existing login file. -The trusted parent copies only `.credentials.json` from `--claude-config ` into run-owned `CLAUDE_CONFIG_DIR` and `auth.json` from `--codex-home ` into run-owned `CODEX_HOME`. It does not copy settings or other source-directory state. Complete recognized credential files retain their refresh material so native clients can refresh during the session; refreshed copies are never written back to the source directories. Missing files, unknown credential formats, or unusable authentication fail closed. Never log in, repair credentials automatically, or fall back to daily provider configuration. +Temporary, GitHub, and fixture state remain run-owned; Git configuration is disabled or run-owned. This is operational state separation, not an OS security boundary. Do not read, parse, or copy credential contents, register tokens, or check public comments against credentials. No extra login directories are needed. ## Drive Read `features/README.md` and every selected feature document. Runtime instructions, consumed references, shipped tools, and installed assets require coverage on their actual consumers. Shared paths select consumers conservatively. `no runtime change` launches no harness unless `--self-test` is requested. -Create detached exact-head candidate checkouts, run-owned provider/config/temp roots, and fixture workspaces. Claude loads the explicit candidate plugin with candidate settings; Codex installs the pinned local marketplace/plugin. Verify installed plugin and project-skill sources against immutable candidate Git provenance before and after exercise. Version strings alone are insufficient. +Create detached exact-head candidate checkouts and run-owned temporary/fixture state. Launch Claude with normal operator config/login and session-only `claude --plugin-dir /plugins/pstack --settings '{"enabledPlugins":{"pstack@open-pstack":false}}'`. Codex installs the exact-head marketplace/plugin in its run-owned home with `codex plugin marketplace add … --ref ` and `codex plugin add`. Verify candidate plugin and project-skill sources against immutable Git provenance before and after exercise; version strings alone are insufficient. -Every candidate command and external-provider descendant uses the recorded candidate environment. Candidate processes receive disposable provider credentials only, never GitHub publisher credentials. No candidate login, source-credential writeback, implicit timeout, or weaker-model fallback is permitted. +Candidate commands and provider descendants use the recorded native-login arrangement, never GitHub publisher credentials. No extra login, implicit timeout, or weaker-model fallback is permitted. If Claude's plugin flags fail to exclude installed pstack or Codex replaces the auth symlink instead of writing through it, stop and reopen the design. For every selected feature, exercise changed sub-features on each actual consuming native surface and retain real native tool calls and concrete fixture effects. Asset consumers come from the pinned plugin manifests; the current `plugins/pstack/assets/logo.png` is consumed by the Codex manifest only and therefore requires the Codex installed surface, not an invented Claude asset exercise. Save raw transcripts and artifacts in the private run root. Require operator-reviewed surface, action, expected/observed result, and retained artifact paths. Model self-reports and direct CLI tests are not installed-harness evidence. -Setup is currently fail-closed. Until setup-pstack config-home support in issue #120 merges, record exactly `setup exercise requires #120 (setup-pstack config-home)` and do not invoke setup against real `HOME`. +For setup only, snapshot real `${CLAUDE_CONFIG_DIR:-$HOME/.claude}/pstack-models.md` and `CLAUDE.md`, including original absence, before exercise. Restore exact bytes or absence afterwards on success or failure and verify restoration before any publication. Restoration failure stops the run and publishes nothing. For `project-skill`, invoke `/verify-open-pstack` in Claude and `$verify-open-pstack` in Codex. Ask the discovered pinned skill to run `verify.sh doctor --candidate --output `. Preserve the invocation and canonical-path doctor output; forbid recursive verification or publication. This does not replace required plugin exercises. ## Evidence -Create the requested run root as a private mode-0700 directory. Keep raw receipts, transcripts, artifacts, selected credential-source paths, immutable source/installation provenance, and credential-cleanup outcome private in that root. Raw evidence may contain operational details and is not a public artifact. Keep it through merge; after the merged evidence is archived, only the operator deletes the named run root. Never retain copied credential files as evidence. +Create a fresh private mode-0700 run root. Keep raw receipts, transcripts, artifacts, immutable source/installation provenance, and setup restoration results private through merge. Raw evidence is not a public artifact; after archival, only the operator deletes the named run root. Before publication, revalidate retained transcript and artifact hashes. The trusted parent publishes one bounded, structured PR comment for the exact candidate SHA and sets `live-gate` on that same SHA with the comment URL as its target. It never edits the PR body, marks ready or draft, reverses a transition, or performs PR-state compensation. No other PR write is permitted. Head movement requires a fresh run and status on the new exact SHA. -The public-comment guard compares the rendered comment against the exact credential values registered from the copied known credential fields. It does not use token-shape heuristics and does not claim that arbitrary private raw evidence is secret-free. Unknown credential formats fail before publication. Failure details remain private; the public failure comment is bounded and the exact candidate SHA receives failure status only after disposable credential cleanup succeeds. +Re-prompt operator `surface`, `action`, and `observed` values until each is printable ASCII without backtick or angle brackets and at most 160 characters. Publish accepted values verbatim using the shared predicate. Other unsafe fields remain omitted whole. Failure details remain private; never publish raw evidence or claim it is secret-free. Failed setup restoration forbids all publication. ## Cleanup -Quit native sessions normally and remove only the copied disposable Claude `.credentials.json` and Codex `auth.json` files on success or failure, including refreshes written to those same run-owned paths. Preserve candidate state and raw evidence. Record cleanup outcome; failed credential cleanup blocks publication. Never modify the supplied source credential files or daily provider files. Preserve the private mode-0700 raw evidence root through merge, archive it after merge, and let the operator delete only the named run root. Do not kill unrelated processes. +Quit native sessions normally. Verify any setup restoration before publication. Removing the Codex run home removes only its auth symlink, never the operator's target file. Preserve private raw evidence through merge, archive it afterwards, and let the operator delete only the named run root. No credential-copy cleanup or separate `~/.pstack-verify` login directories are needed. Do not kill unrelated processes. ## Helpers -- `scripts/verify.sh doctor --claude-config --codex-home --output `: trusted-parent capability and source-authentication report, no installation/publication. -- `scripts/verify.sh doctor --candidate --output `: candidate-safe self-test probe; no source-credential or publisher authentication probing. -- `scripts/verify.sh run --pr --claude-config --codex-home [--self-test] --output `: supervised exact-head verification; mapped exercises require an interactive terminal. +- `scripts/verify.sh doctor --output `: parent capability report, no installation/publication. +- `scripts/verify.sh doctor --candidate --output `: candidate-safe self-test probe; no publisher authentication probing. +- `scripts/verify.sh run --pr [--self-test] --output `: supervised exact-head verification; mapped exercises require an interactive terminal. - `bun run test` and `bun run typecheck`: development checks after a separate frozen dependency install, not live proof. - `features/registry.json`: maintained path ownership; unknown runtime paths block. diff --git a/.claude/skills/verify-open-pstack/features/README.md b/.claude/skills/verify-open-pstack/features/README.md index cc5a7a9f..d3d00939 100644 --- a/.claude/skills/verify-open-pstack/features/README.md +++ b/.claude/skills/verify-open-pstack/features/README.md @@ -7,7 +7,7 @@ | `skills` | One `skill-invocation:` exercise per selected skill, in both harnesses | | `shared` | All listed skills, setup, runner, and shipped tools in both harnesses | | `assets` | Installed UI/resource surface for each actual consumer found in the pinned plugin manifests | -| `setup` | Installed setup/model configuration; currently the named #120 fail-closed result | +| `setup` | Installed setup/model configuration with byte-exact daily Claude file restoration before publication | | `runner` | Installed parent invoking real configured provider lanes | | `tools` | Installed parent invoking the changed shipped CLI | | `project` | Native discovery/invocation of this verifier in both harnesses | @@ -22,7 +22,8 @@ Skill-owned `SKILL.md`, `references/**`, and `playbooks/**` select their skill. The feature documents below define sub-features, user entry points, driving instructions, and gotchas. Review each changed sub-feature in the diff; one aggregate observation cannot replace distinct selected skills. Record all changed sub-feature actions in the feature's transcript and artifact. Do not run destructive application work in the maintainer checkout. - `skill-invocation.md`: installed native skills, principle leaves, and project-skill self-test. -- `setup.md`: setup/routing/model configuration and the #120 block. +- `setup.md`: setup/routing/model configuration and byte-exact restoration. +- `isolation.md`: existing native logins, Codex auth symlink, and private run state. - `runner.md`: parent-to-child provider dispatch and receipts. - `shipped-tools.md`: orchestration, watch, plan checks, audit, and evidence logging. - `assets.md`: manifest-declared installed assets on actual consuming harnesses. diff --git a/.claude/skills/verify-open-pstack/features/isolation.md b/.claude/skills/verify-open-pstack/features/isolation.md index c20a67ea..257ce32c 100644 --- a/.claude/skills/verify-open-pstack/features/isolation.md +++ b/.claude/skills/verify-open-pstack/features/isolation.md @@ -1,42 +1,31 @@ -# Disposable credential and run-state contract +# Native login and run-state contract -The verifier uses run-owned provider/configuration state and credential files copied from explicitly supplied operator-pre-authenticated normal-login directories. No profile manager or maintainer-specific infrastructure is required. This is operational state separation, not a Seatbelt or macOS filesystem security boundary. +Reuse the operator's existing Claude Code and Codex logins. No extra login, credential source flags, credential reading/copying/parsing, token registration, or exact-secret public-comment check. This is operational state separation, not an OS security boundary. ## Trusted parent -Run the publisher from a maintainer-reviewed trusted `main` checkout. Record that publisher revision separately from the PR's exact candidate SHA; never treat publisher code provenance as candidate proof. For PR #111 bootstrap, use the reviewed trusted publisher head while pinning all candidate evidence and status to PR #111's reviewed exact head. - -Require explicit `--claude-config ` and `--codex-home ` pointing to existing operator-pre-authenticated normal-login configuration directories; never infer source directories or choose defaults. A selected account may be active in daily use. Record source paths privately in the receipt. - -Trusted-parent doctor validates those source directories with `CLAUDE_CONFIG_DIR= claude auth status` and `CODEX_HOME= codex login status`. Missing or unusable authentication, including unauthorized or expired credentials, fails closed with the corresponding manual remediation: `CLAUDE_CONFIG_DIR= claude auth login` or `CODEX_HOME= codex login`. Never execute login automatically, switch identities, or fall back automatically. - -Copy only the source Claude `.credentials.json` into run-owned `CLAUDE_CONFIG_DIR` and source Codex `auth.json` into run-owned `CODEX_HOME`. Copy the complete recognized credential file, including refresh material needed by the native client, but no other configuration or source state. Never write refreshed state back to source directories. Missing credential files, unknown credential formats, or unusable authentication fail closed. GitHub authentication and publication remain exclusively in the trusted parent. +Verifier-changing PRs run from their reviewed candidate head, with publisher revision equal to candidate SHA; all other PRs run from trusted `main`. Record publisher revision separately. GitHub authentication and publication remain exclusively in the parent; candidate processes never receive publisher credentials. ## Candidate process state -- Preserve the operator's real `HOME`, `USER`, and `LOGNAME` for native-client compatibility and identity. -- Set `CLAUDE_CONFIG_DIR`, `CODEX_HOME`, `TMPDIR`, `GH_CONFIG_DIR`, XDG config/cache, and other configurable provider/GitHub/temp locations to run-owned directories. -- Disable global/system git configuration and credential helpers for candidate commands. -- Claude uses the explicit candidate plugin, empty settings, and project setting sources. Codex uses pinned local marketplace/plugin installation. -- Provider descendants inherit both run-owned provider config roots. Never pass GitHub publisher credentials or daily provider API keys. -- Never invoke login, modify source credentials, or copy daily provider configuration as fallback. +- Preserve real `HOME`, `USER`, and `LOGNAME`. +- Claude uses normal operator login/config with `claude --plugin-dir /plugins/pstack --settings '{"enabledPlugins":{"pstack@open-pstack":false}}'`. These flags load only the candidate for that session, without persistent installation or settings changes. +- Codex installs the exact-head marketplace with `codex plugin marketplace add … --ref ` and `codex plugin add` in a run-owned `CODEX_HOME`. Symlink its `auth.json` to the operator's `${CODEX_HOME:-$HOME/.codex}/auth.json`, resolving the daily home before overriding the environment. +- If the daily home has no `auth.json`, fail closed: this tool needs file-based Codex auth. Do not read credentials, copy them, or start login. Native refresh writes through the symlink to the single existing login file. +- Keep temporary, GitHub, and fixture state run-owned; disable global/system Git configuration and credential helpers for candidate commands. Provider descendants use the same native-login arrangement, never publisher credentials. -These controls do not deny a process access to real `HOME`, Keychain, or candidate sources. Do not claim Seatbelt enforcement, source write protection, Keychain/securityd denial, daily-home denial, or OS-backed secrecy. Provenance rechecks detect changed candidate/installed files but are not prevention. +Do not claim Seatbelt enforcement, source-write protection, Keychain denial, daily-home denial, or OS-backed secrecy. Provenance rechecks detect changed candidate/installed files but are not prevention. ## Setup boundary -Because `HOME` remains real, setup must not be exercised until config-home support in issue #120 merges. Until then, setup selection fails closed with exactly: - -`setup exercise requires #120 (setup-pstack config-home)` +For setup only, snapshot `${CLAUDE_CONFIG_DIR:-$HOME/.claude}/pstack-models.md` and `CLAUDE.md` before exercise, including original absence. Restore exact bytes or absence afterwards, on success or failure, and verify restoration before any publication. Failed restoration stops the run and publishes nothing. Do not substitute a mock or generated configuration alone for installed-surface evidence. -Do not redirect `HOME`, allow setup to write daily files, or replace this result with a mock. - -## Public comment check +## Private evidence and cleanup -Register exact access/refresh/id-token values only from the copied, recognized credential fields. Immediately before a GitHub write, compare the rendered structured comment against those exact known values and refuse publication on a match. Do not use token-shape, prefix, entropy, regex, or other heuristic secrecy claims. Unknown credential layouts block before publication. Private raw evidence is not declared secret-free. +Use a fresh external mode-0700 run root. Retain raw transcripts, artifacts, receipts, and restoration results privately through merge. Quit native sessions normally. Removing the Codex run home removes its auth symlink, never the operator's target file. No copied-credential cleanup or separate `~/.pstack-verify` login directories are needed. -## Private evidence and cleanup +After merge, archive retained evidence and let the operator delete only the named run root. Never recursively delete an operator directory or kill unrelated processes. -Create each output root fresh, outside the repository, with mode 0700; use private files beneath it. Retain raw transcripts, artifacts, receipts, and cleanup results in that private root through merge. Copied credential files are session state, not evidence: remove only the copied Claude `.credentials.json` and Codex `auth.json` files in run-owned config roots after native sessions on success or failure, including refreshes written to those same paths. Preserve candidate state and raw evidence. Cleanup failure blocks GitHub publication. +## Native verification -After merge, archive the retained evidence and let the operator delete only the named run root. Never modify supplied source credential files or daily provider files, and never recursively delete an operator directory. +Confirm a real Mac session loads only the candidate in Claude, Codex authenticates through the symlink, daily Codex still works after any refresh, and setup restoration is byte-exact. If Codex replaces the symlink or installed pstack leaks into Claude's candidate session, stop and reopen the design rather than improvise. diff --git a/.claude/skills/verify-open-pstack/features/runner.md b/.claude/skills/verify-open-pstack/features/runner.md index aecdb348..c92d34a2 100644 --- a/.claude/skills/verify-open-pstack/features/runner.md +++ b/.claude/skills/verify-open-pstack/features/runner.md @@ -14,6 +14,6 @@ Read the candidate's `pstack-runner --help` and installed mapping through the na ## Gotchas -Every external-provider child retains real `HOME`, `USER`, and `LOGNAME` while inheriting run-owned `TMPDIR`, `GH_CONFIG_DIR`, XDG/config/cache paths, `CLAUDE_CONFIG_DIR`, and `CODEX_HOME`. This state separation is not an OS access-denial guarantee. Never let Claude→Codex or Codex→Claude fall back to daily provider configuration, and never pass publisher credentials. +Provider children retain real `HOME`, `USER`, and `LOGNAME` and the native-login arrangement in `isolation.md`: normal Claude config/login with session-only candidate plugin flags, and run-owned Codex config with auth symlinked to the daily login. Temporary and GitHub state remain run-owned; never pass publisher credentials. No credential reading/copying, source flags, or extra login is required. Missing file-based Codex auth fails closed. State separation is not an OS access-denial guarantee. -Supply operator-pre-authenticated normal-login directories with `--claude-config ` and `--codex-home `. The trusted parent copies only complete recognized `.credentials.json` and `auth.json` files so native refresh can work, and removes only those copied credential files (including refreshes written to those paths) during cleanup without writing back to source directories or deleting raw evidence. A selected account may be active in daily use. Trusted-parent doctor validates the supplied directories with `claude auth status` under the source `CLAUDE_CONFIG_DIR` and `codex login status` under the source `CODEX_HOME`; missing or unusable authentication is a failure, not a skip or weaker-model fallback. Authentication failures report `CLAUDE_CONFIG_DIR= claude auth login` or `CODEX_HOME= codex login`; never execute login automatically. Do not add implicit timeouts. A model assertion that a child ran is insufficient; inspect the actual receipt and output. Never reuse output from another SHA or run. +Do not add implicit timeouts or weaker-model fallbacks. A model assertion that a child ran is insufficient; inspect the actual receipt and output. Never reuse output from another SHA or run. diff --git a/.claude/skills/verify-open-pstack/features/setup.md b/.claude/skills/verify-open-pstack/features/setup.md index 6fb2556d..b209df9d 100644 --- a/.claude/skills/verify-open-pstack/features/setup.md +++ b/.claude/skills/verify-open-pstack/features/setup.md @@ -6,18 +6,14 @@ Installed setup invocation, harness-to-provider mapping, generated routing/model ## How to get to it (user POV) -The intended installed entries are `/pstack:setup-pstack` in Claude and `$setup-pstack` in Codex. The installed instructions are the authority for setup inputs, but the verifier must not invoke them against the operator's real `HOME` before config-home support exists. +Invoke `/pstack:setup-pstack` in Claude or `$setup-pstack` in Codex. The installed instructions are the authority for setup inputs. ## Driving it with verify-open-pstack -Setup is mandatory fail-closed until setup-pstack config-home support in issue #120 merges. For every selected setup exercise, record exactly: +For setup only, snapshot the operator's real `${CLAUDE_CONFIG_DIR:-$HOME/.claude}/pstack-models.md` and `CLAUDE.md` before invocation, preserving exact bytes and whether each file existed. Exercise every changed setup branch with explicit model/provider inputs. Retain the native transcript and generated configuration privately, then invoke a consuming installed skill to observe the configured route. -`setup exercise requires #120 (setup-pstack config-home)` - -Stop that verification run. Do not invoke setup, redirect `HOME`, permit writes to daily paths, use a mock, or accept a generated file from a direct CLI test as substitute evidence. - -After #120 merges, use its explicit config-home to target run-owned state while retaining real `HOME`, `USER`, and `LOGNAME`. Exercise each changed setup branch with explicit requested model/provider inputs, retain the native invocation transcript and generated configuration file, and invoke a consuming installed skill to observe the configured route. Confirm all setup writes remain under the explicit run-owned config home. +Restore both files byte for byte afterwards, including restoring original absence, on success or failure. Verify restoration before any evidence comment or status publication. Restoration failure stops the run and publishes nothing. Retain snapshot and restoration results in the private mode-0700 evidence root through merge. ## Gotchas -`CLAUDE_CONFIG_DIR`, `CODEX_HOME`, XDG/GitHub config, and temp paths are run-owned, but `HOME` is intentionally real. That is why setup cannot run safely before #120. Supply operator-pre-authenticated normal-login directories with `--claude-config ` and `--codex-home `; selected accounts may be active in daily use. Copy only `.credentials.json` and `auth.json` into run-owned provider roots, preserving refresh material. Do not copy other daily configuration, log in, or write credentials back to source directories. Trusted-parent doctor validates sources with `claude auth status` under the supplied `CLAUDE_CONFIG_DIR` and `codex login status` under the supplied `CODEX_HOME`; authentication failures fail closed with `CLAUDE_CONFIG_DIR= claude auth login` or `CODEX_HOME= codex login`, which the verifier never executes automatically. Generated configuration by itself does not prove the consuming surface. +Claude uses the operator's normal config/login; Codex uses a run-owned home with auth symlinked to the daily login. Do not redirect `HOME`, copy credentials, start login, or use a mock as setup proof. Generated configuration alone does not prove the consuming surface. Setup restoration is the only permitted temporary change to these daily Claude files. diff --git a/.claude/skills/verify-open-pstack/features/shipped-tools.md b/.claude/skills/verify-open-pstack/features/shipped-tools.md index e67b1dec..b3cf91fd 100644 --- a/.claude/skills/verify-open-pstack/features/shipped-tools.md +++ b/.claude/skills/verify-open-pstack/features/shipped-tools.md @@ -16,4 +16,4 @@ Use the candidate's installed help and documented arguments. Create fixture stat ## Gotchas -A mock test is not live proof. Never perform any PR write from a candidate process or edit daily workflow state as a fixture. Do not infer a successful remote read from an empty response. Candidate tools and descendants retain real `HOME`, `USER`, and `LOGNAME` but use run-owned provider/GitHub/XDG/config/cache/temp paths. Supply operator-pre-authenticated normal-login directories with `--claude-config ` and `--codex-home `; selected accounts may be active in daily use. Only complete recognized `.credentials.json` and `auth.json` files are copied, preserving refresh material without source writeback. Candidates do not receive publisher credentials. Trusted-parent doctor validates sources with `claude auth status` under the supplied `CLAUDE_CONFIG_DIR` and `codex login status` under the supplied `CODEX_HOME`. Authentication failures fail closed with `CLAUDE_CONFIG_DIR= claude auth login` or `CODEX_HOME= codex login`; never execute login automatically. This does not claim OS denial of daily-home or Keychain access. Missing required access fails the feature; preserve its private reason and publish only the bounded structured exact-SHA result. +A mock test is not live proof. Never perform PR writes from a candidate process or edit daily workflow state as a fixture. Do not infer a successful remote read from an empty response. Candidates retain real identity and reuse normal Claude login/config and Codex's symlinked daily auth as described in `isolation.md`; GitHub and temporary state remain run-owned. No credential contents are read or copied and no extra login is started. Candidates never receive publisher credentials. This does not claim OS denial of daily-home or Keychain access. Missing access fails the feature; preserve its private reason and publish only the bounded exact-SHA result, never after failed setup restoration. diff --git a/.claude/skills/verify-open-pstack/features/skill-invocation.md b/.claude/skills/verify-open-pstack/features/skill-invocation.md index a5b790da..4291a0f3 100644 --- a/.claude/skills/verify-open-pstack/features/skill-invocation.md +++ b/.claude/skills/verify-open-pstack/features/skill-invocation.md @@ -10,7 +10,7 @@ In a fresh Claude session invoke `/pstack:`. In Codex use the installed sk ## Driving it with verify-open-pstack -Launch the prepared harness with real `HOME`, `USER`, and `LOGNAME`; run-owned `TMPDIR`, GitHub/XDG/config/cache paths, `CLAUDE_CONFIG_DIR`, and `CODEX_HOME`; and no publisher credentials. Supply operator-pre-authenticated normal-login directories with `--claude-config ` and `--codex-home `; selected accounts may be active in daily use. Use trusted-parent copies of only complete recognized `.credentials.json` and `auth.json` files, retaining refresh material. Trusted-parent doctor validates sources with `claude auth status` under the supplied `CLAUDE_CONFIG_DIR` and `codex login status` under the supplied `CODEX_HOME`. Authentication failures fail closed with `CLAUDE_CONFIG_DIR= claude auth login` or `CODEX_HOME= codex login`; the verifier never executes login automatically. No candidate login, daily-provider fallback, or source-credential writeback is allowed. +Launch with real `HOME`, `USER`, and `LOGNAME`, run-owned temporary/GitHub state, and no publisher credentials. Claude uses normal operator login/config with `claude --plugin-dir /plugins/pstack --settings '{"enabledPlugins":{"pstack@open-pstack":false}}'`. Codex installs the exact-head candidate in a run-owned `CODEX_HOME` with `auth.json` symlinked to the daily `${CODEX_HOME:-$HOME/.codex}/auth.json`. Missing file-based auth fails closed. No source flags, credential reading/copying, or extra login; see `isolation.md`. For each selected name, use a disposable fixture appropriate to its documented input, invoke the installed surface, and exercise every changed sub-feature. Observe native tool/skill loading and a concrete fixture effect, not the assistant saying it passed. Save the raw transcript and an artifact showing the actual result (diff, generated file, tool receipt, or UI capture) under the private mode-0700 evidence root. Include input, expected result, observed result, and each changed sub-feature in the operator review. diff --git a/.claude/skills/verify-open-pstack/scripts/cli.test.ts b/.claude/skills/verify-open-pstack/scripts/cli.test.ts index c109092d..ea32f423 100644 --- a/.claude/skills/verify-open-pstack/scripts/cli.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/cli.test.ts @@ -10,7 +10,6 @@ import { REPO } from './types.ts'; import { verify } from './verify.ts'; import { validateRegistry } from './core.ts'; import registry from '../features/registry.json'; -const sources = ['--claude-config', '/operator/claude config', '--codex-home', '/operator/codex home']; test('publisher revision is recorded independently of candidate proof', async () => { const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-publisher-revision-'))); @@ -32,7 +31,7 @@ test('publisher revision is recorded independently of candidate proof', async () test('candidate mode is an explicit doctor-only flag', () => { expect(parse(['doctor', '--candidate', '--output', '/fresh/probe'])).toEqual({ mode: 'doctor', output: '/fresh/probe', pr: 0, selfTest: false, candidate: true }); expect(() => parse(['doctor', '--candidate', '--candidate', '--output', '/fresh/probe'])).toThrow('Duplicate option'); - expect(() => parse(['run', '--candidate', '--pr', '111', '--output', '/fresh/run', ...sources])).toThrow('Unknown option'); + expect(() => parse(['run', '--candidate', '--pr', '111', '--output', '/fresh/run'])).toThrow('Unknown option'); }); test('candidate doctor probes only help and versions with private roots already present', async () => { const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-doctor-'))), calls: string[][] = []; @@ -42,7 +41,8 @@ test('candidate doctor probes only help and versions with private roots already expect(env.HOME === process.env.HOME).toBe(true); expect(env.USER === process.env.USER).toBe(true); expect(env.LOGNAME === process.env.LOGNAME).toBe(true); - for (const key of ['TMPDIR', 'CLAUDE_CONFIG_DIR', 'CODEX_HOME', 'XDG_CONFIG_HOME', 'XDG_CACHE_HOME', 'GH_CONFIG_DIR']) { + expect(env.CLAUDE_CONFIG_DIR === process.env.CLAUDE_CONFIG_DIR).toBe(true); + for (const key of ['TMPDIR', 'CODEX_HOME', 'XDG_CONFIG_HOME', 'XDG_CACHE_HOME', 'GH_CONFIG_DIR']) { expect(env[key]!.startsWith(root + '/probe-home')).toBe(true); const info = await stat(env[key]!); expect(info.isDirectory()).toBe(true); @@ -54,50 +54,19 @@ test('candidate doctor probes only help and versions with private roots already await doctor(root, run, 'darwin', true); const report = JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')); expect(report.candidate).toBe(true); expect(report.result).toBe('pass'); - expect(Object.keys(report.checks)).toEqual(['bun', 'git', 'claude', 'codex', 'claudeHelp', 'codex plugin marketplace add --help', 'codex plugin add --help']); expect(calls.every(args => args.includes('--version') || args.includes('--help'))).toBe(true); - expect(calls.some(args => args[0] === 'gh')).toBe(false); - expect(calls.some(args => args.includes('auth') || args.includes('login'))).toBe(false); + expect(calls.some(args => args[0] === 'gh' || args.includes('auth') || args.includes('login'))).toBe(false); } finally { await rm(root, { recursive: true, force: true }); } }); -test('CLI strictly validates run inputs and permits read-only child doctor', () => { - expect(parse(['doctor', '--candidate', '--output', '/fresh/probe'])).toEqual({ mode: 'doctor', output: '/fresh/probe', pr: 0, selfTest: false, candidate: true }); - expect(parse(['run', '--pr', '123', '--self-test', '--output', '/fresh/run', ...sources])).toEqual({ mode: 'run', output: '/fresh/run', pr: 123, selfTest: true, claudeConfig: sources[1], codexHome: sources[3] }); - for (const args of [[], ['wat'], ['run', '--pr', '0', '--output', '/tmp/a', ...sources], ['run', '--pr', '1.5', '--output', '/tmp/a', ...sources], ['run', '--pr', '9007199254740992', '--output', '/tmp/a', ...sources], ['doctor', '--pr', '1', '--output', '/tmp/a'], ['doctor', '--output', '/tmp/a', '--output', '/tmp/b'], ['run', '--pr', '1', ...sources], ['doctor', '--output', '--self-test'], ['doctor', '--output', '/tmp/a', '--publish']]) expect(() => parse(args)).toThrow(); +test('run and parent doctor need no credential flags', () => { + expect(parse(['doctor', '--output', '/fresh/probe'])).toEqual({ mode: 'doctor', output: '/fresh/probe', pr: 0, selfTest: false, candidate: false }); + expect(parse(['run', '--pr', '123', '--self-test', '--output', '/fresh/run'])).toEqual({ mode: 'run', output: '/fresh/run', pr: 123, selfTest: true }); + for (const args of [[], ['wat'], ['run', '--pr', '0', '--output', '/tmp/a'], ['run', '--pr', '1.5', '--output', '/tmp/a'], ['run', '--pr', '9007199254740992', '--output', '/tmp/a'], ['doctor', '--pr', '1', '--output', '/tmp/a'], ['doctor', '--output', '/tmp/a', '--output', '/tmp/b'], ['run', '--pr', '1'], ['doctor', '--output', '--self-test'], ['doctor', '--output', '/tmp/a', '--publish']]) expect(() => parse(args)).toThrow(); }); -test('run and parent doctor require paired credential directories; candidate doctor rejects sources', () => { - const run = ['run', '--pr', '111', '--output', '/fresh/run']; - const parent = ['doctor', '--output', '/fresh/probe']; - expect(() => parse(run)).toThrow('Run requires --claude-config and --codex-home'); - expect(() => parse(parent)).toThrow('Doctor requires both --claude-config and --codex-home'); - expect(() => parse([...run, ...sources.slice(0, 2)])).toThrow('Run requires'); - expect(() => parse([...run, ...sources.slice(2)])).toThrow('Run requires'); - expect(parse([...parent, ...sources])).toEqual({ mode: 'doctor', output: '/fresh/probe', pr: 0, selfTest: false, candidate: false, claudeConfig: sources[1], codexHome: sources[3] }); - expect(() => parse([...parent, '--candidate', ...sources])).toThrow('Candidate doctor cannot inspect source credentials'); - for (const option of ['--claude-config', '--codex-home']) { - expect(() => parse([...parent, option, '/operator/config'])).toThrow('requires both'); - expect(() => parse([...parent, '--candidate', option, '/operator/config'])).toThrow('Candidate doctor cannot inspect source credentials'); - for (const mode of [run, parent]) { - expect(() => parse([...mode, ...sources, option, '/other/config'])).toThrow('Duplicate option'); - expect(() => parse([...mode, option])).toThrow('Missing value'); - expect(() => parse([...mode, option, '--output'])).toThrow('Missing value'); - expect(() => parse([...mode, option, ''])).toThrow('Missing value'); +test('credential-directory and account options are removed', () => { + for (const option of ['--claude-config', '--codex-home', '--claude-account', '--codex-account']) { + for (const args of [['run', '--pr', '111'], ['doctor'], ['doctor', '--candidate']]) { + expect(() => parse([...args, '--output', '/fresh/run', option, '/operator/config'])).toThrow(`Unknown option: ${option}`); } } }); -test('credential options accept directory paths and reject NUL paths and obsolete account options', () => { - for (const option of ['--claude-config', '--codex-home']) { - const other = option === '--claude-config' ? '--codex-home' : '--claude-config'; - for (const mode of [['run', '--pr', '111'], ['doctor']]) { - const args = [...mode, '--output', '/fresh/run', other, '/operator/config', option]; - expect(() => parse([...args, '/operator/config\0suffix'])).toThrow(`Invalid credential directory: ${option}`); - for (const directory of ['/operator/config with spaces', './preauthenticated-config']) { - const result = parse([...args, directory]); - expect(option === '--claude-config' ? result.claudeConfig : result.codexHome).toBe(directory); - } - } - } - for (const option of ['--claude-account', '--codex-account']) { - expect(() => parse(['run', '--pr', '111', '--output', '/fresh/run', ...sources, option, 'person@example.com'])).toThrow(`Unknown option: ${option}`); - } -}); diff --git a/.claude/skills/verify-open-pstack/scripts/cli.ts b/.claude/skills/verify-open-pstack/scripts/cli.ts index a46c7d66..145ddc7c 100644 --- a/.claude/skills/verify-open-pstack/scripts/cli.ts +++ b/.claude/skills/verify-open-pstack/scripts/cli.ts @@ -6,36 +6,27 @@ import { Publisher } from './github.ts'; import { MacDriver } from './harness.ts'; import { command, freshRoot, interruption, interruptCommands, save } from './io.ts'; import { verify } from './verify.ts'; -type Options = { mode: 'doctor'; output: string; pr: number; selfTest: boolean; candidate: boolean; claudeConfig?: string; codexHome?: string } - | { mode: 'run'; output: string; pr: number; selfTest: boolean; claudeConfig: string; codexHome: string }; +type Options = { mode: 'doctor'; output: string; pr: number; selfTest: boolean; candidate: boolean } + | { mode: 'run'; output: string; pr: number; selfTest: boolean }; export function parse(args: string[]): Options { const mode = args[0]; - if (mode !== 'doctor' && mode !== 'run') throw new Error('Usage: verify.sh doctor (--candidate | --claude-config DIR --codex-home DIR) --output /fresh/path | run --pr NUMBER --claude-config DIR --codex-home DIR [--self-test] --output /fresh/path'); - let output = '', pr = 0, selfTest = false, candidate = false, claudeConfig = '', codexHome = ''; + if (mode !== 'doctor' && mode !== 'run') throw new Error('Usage: verify.sh doctor [--candidate] --output /fresh/path | run --pr NUMBER [--self-test] --output /fresh/path'); + let output = '', pr = 0, selfTest = false, candidate = false; const seen = new Set(); for (let i = 1; i < args.length; i++) { const arg = args[i]!; if (seen.has(arg)) throw new Error(`Duplicate option: ${arg}`); seen.add(arg); if (arg === '--self-test' && mode === 'run') selfTest = true; else if (arg === '--candidate' && mode === 'doctor') candidate = true; - else if (['--output', '--claude-config', '--codex-home'].includes(arg) || mode === 'run' && arg === '--pr') { + else if (arg === '--output' || mode === 'run' && arg === '--pr') { const value = args[++i]; if (!value || value.startsWith('--')) throw new Error(`Missing value: ${arg}`); if (arg === '--output') output = value; - else if (arg === '--pr') { if (!/^[1-9]\d*$/.test(value) || !Number.isSafeInteger(Number(value))) throw new Error('PR must be a positive integer'); pr = Number(value); } - else { - if (value.includes('\0')) throw new Error(`Invalid credential directory: ${arg}`); - if (arg === '--claude-config') claudeConfig = value; else codexHome = value; - } + else { if (!/^[1-9]\d*$/.test(value) || !Number.isSafeInteger(Number(value))) throw new Error('PR must be a positive integer'); pr = Number(value); } } else throw new Error(`Unknown option: ${arg}`); } if (!output || mode === 'run' && !pr) throw new Error('Output and (for run) PR are required'); - if (mode === 'run') { - if (!claudeConfig || !codexHome) throw new Error('Run requires --claude-config and --codex-home'); - return { mode, output, pr, selfTest, claudeConfig, codexHome }; - } - if (candidate && (claudeConfig || codexHome)) throw new Error('Candidate doctor cannot inspect source credentials'); - if (!candidate && (!claudeConfig || !codexHome)) throw new Error('Doctor requires both --claude-config and --codex-home'); - return { mode, output, pr, selfTest, candidate, ...(claudeConfig ? { claudeConfig, codexHome } : {}) }; + if (mode === 'run') return { mode, output, pr, selfTest }; + return { mode, output, pr, selfTest, candidate }; } async function main(): Promise { let root: string | undefined, interrupted: 'SIGINT' | 'SIGTERM' | undefined; @@ -51,11 +42,11 @@ async function main(): Promise { try { const options = parse(process.argv.slice(2)), repository = resolve(import.meta.dir, '../../../..'); root = await freshRoot(options.output, repository); - if (options.mode === 'doctor') await doctor(root, undefined, undefined, options.candidate, options.claudeConfig && options.codexHome ? { claude: options.claudeConfig, codex: options.codexHome } : undefined); + if (options.mode === 'doctor') await doctor(root, undefined, undefined, options.candidate); else { const publisherRevision = (await command(['git', 'rev-parse', 'HEAD'], { cwd: repository })).trim(); const registry = validateRegistry(JSON.parse(await readFile(join(import.meta.dir, '../features/registry.json'), 'utf8'))); - const receipt = await verify({ pr: options.pr, selfTest: options.selfTest, root, registry, github: new Publisher(), driver: new MacDriver(undefined, undefined, { claude: options.claudeConfig, codex: options.codexHome }), persist: r => save(join(root!, 'receipt.json'), r), publisherRevision }); + const receipt = await verify({ pr: options.pr, selfTest: options.selfTest, root, registry, github: new Publisher(), driver: new MacDriver(), persist: r => save(join(root!, 'receipt.json'), r), publisherRevision }); console.log(`Pinned ${receipt.sha}: live-gate=${receipt.status}; evidence ${receipt.commentUrl}`); } interruption.signal.throwIfAborted(); diff --git a/.claude/skills/verify-open-pstack/scripts/doctor.test.ts b/.claude/skills/verify-open-pstack/scripts/doctor.test.ts index 258ddb97..f04411ca 100644 --- a/.claude/skills/verify-open-pstack/scripts/doctor.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/doctor.test.ts @@ -1,98 +1,39 @@ import { afterEach, expect, test } from 'bun:test'; -import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, readFile, realpath, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { doctor } from './doctor.ts'; -import { loginFix } from './isolation.ts'; import type { Command } from './io.ts'; -import type { CredentialSources, Harness } from './types.ts'; const roots: string[] = []; afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); -async function fixture(): Promise<{ root: string; sources: CredentialSources }> { - const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-native-status-'))); roots.push(root); - const sources = { claude: join(root, 'Claude source config'), codex: join(root, 'Codex source home') }; - for (const tool of ['claude', 'codex'] as const) { - await mkdir(sources[tool], { mode: 0o700 }); - await writeFile(join(sources[tool], tool === 'claude' ? '.credentials.json' : 'auth.json'), JSON.stringify( - tool === 'claude' ? { claudeAiOauth: { accessToken: 'doctor-claude-access', refreshToken: 'doctor-claude-refresh' } } : { tokens: { access_token: 'doctor-codex-access', refresh_token: 'doctor-codex-refresh', id_token: 'doctor-codex-id' } }, - )); - } - return { root, sources }; +async function fixture(): Promise { + const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-doctor-'))); roots.push(root); + return root; } -function runner(calls: string[][], result: (tool: Harness, options: Parameters[1]) => string = tool => tool === 'claude' ? '{"loggedIn":true}' : ''): Command { - return async (args, options) => { - calls.push(args); - if (args.includes('--version')) return 'version'; - if (args.includes('--help')) return '--plugin-dir --settings --setting-sources --json local path'; - if (args.join(' ') === 'claude auth status --json' || args.join(' ') === 'codex login status') return result(args[0] as Harness, options); - throw new Error(`Unexpected command: ${args.join(' ')}`); - }; -} -function statusCalls(calls: string[][]): string[][] { return calls.filter(args => args.includes('status')); } - -test('doctor validates explicit native config directories independently of profile infrastructure', async () => { - const { root, sources } = await fixture(), calls: string[][] = []; - const before = await Promise.all([readFile(join(sources.claude, '.credentials.json')), readFile(join(sources.codex, 'auth.json'))]); - await doctor(root, runner(calls, (tool, options) => { - expect(options?.env?.CLAUDE_CONFIG_DIR).toBe(sources.claude); - expect(options?.env?.CODEX_HOME).toBe(sources.codex); - expect(options?.env?.HOME === process.env.HOME).toBe(true); - expect(options?.env?.USER === process.env.USER).toBe(true); - expect(options?.env?.LOGNAME === process.env.LOGNAME).toBe(true); - for (const key of ['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'CLAUDE_CODE_OAUTH_TOKEN', 'GH_TOKEN', 'GITHUB_TOKEN']) expect(options?.env?.[key]).toBeUndefined(); - return tool === 'claude' ? '{"loggedIn":true}' : ''; - }), 'darwin', false, sources); - expect(statusCalls(calls)).toEqual([['claude', 'auth', 'status', '--json'], ['codex', 'login', 'status']]); - expect(calls.some(args => args[0] === 'caam' || args.includes('login') && !args.includes('status'))).toBe(false); - expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('pass'); - expect(await Promise.all([readFile(join(sources.claude, '.credentials.json')), readFile(join(sources.codex, 'auth.json'))])).toEqual(before); - expect(await Bun.file(join(root, 'probe-home/.claude/.credentials.json')).exists()).toBe(false); - expect(await Bun.file(join(root, 'probe-home/.codex/auth.json')).exists()).toBe(false); -}); - -test.each(['{"loggedIn":false}', '{}', '[]', 'null', 'not JSON', '{"loggedIn":"true"}'])('doctor fails closed for negative or malformed Claude status: %s', async status => { - const { root, sources } = await fixture(), calls: string[][] = []; - await expect(doctor(root, runner(calls, () => status), 'darwin', false, sources)).rejects.toThrow(loginFix('claude', sources.claude)); - expect(statusCalls(calls)).toEqual([['claude', 'auth', 'status', '--json']]); - const report = JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')); - expect(report.result).toBe('blocked'); expect(report.reason).toContain(loginFix('claude', sources.claude)); -}); - -for (const tool of ['claude', 'codex'] as const) { - test.each(['Not logged in', 'Unauthorized', 'Credentials expired', 'Command unavailable'])('doctor gives generic manual login guidance for ' + tool + ': %s', async error => { - const { root, sources } = await fixture(), calls: string[][] = []; - const run = runner(calls, provider => { - if (provider === tool) throw new Error(error); - return '{"loggedIn":true}'; - }); - await expect(doctor(root, run, 'darwin', false, sources)).rejects.toThrow(loginFix(tool, sources[tool])); - expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('blocked'); - expect(calls.some(args => args[0] === 'caam' || args.includes('login') && !args.includes('status'))).toBe(false); +for (const candidate of [false, true]) { + test(`doctor is capabilities-only (candidate=${candidate})`, async () => { + const root = await fixture(), calls: string[][] = []; + const run: Command = async args => { + calls.push(args); + if (args.includes('--version')) return 'version'; + if (args.includes('--help')) return '--plugin-dir --settings --setting-sources --json local path'; + throw new Error(`Unexpected command: ${args.join(' ')}`); + }; + await doctor(root, run, 'darwin', candidate); + expect(calls.every(args => args.includes('--version') || args.includes('--help'))).toBe(true); + expect(calls.some(args => args.includes('auth') || args.includes('login') || args.includes('status'))).toBe(false); + expect(calls.some(args => args[0] === 'gh')).toBe(!candidate); + expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('pass'); }); } - -test('Codex login status accepts an empty stdout on successful exit', async () => { - const { root, sources } = await fixture(), calls: string[][] = []; - await doctor(root, runner(calls), 'darwin', false, sources); - expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).checks['codex login status']).toBe(''); +test('doctor records blocked capability checks', async () => { + const root = await fixture(); + await expect(doctor(root, async () => { throw new Error('Missing binary'); }, 'darwin')).rejects.toThrow('Missing binary'); + expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('blocked'); }); - -test('parent doctor requires sources, and missing directories or files block before native commands', async () => { - const { root, sources } = await fixture(), calls: string[][] = []; - await expect(doctor(root, runner(calls), 'darwin')).rejects.toThrow('--claude-config and --codex-home '); - const missing = { ...sources, claude: join(root, 'missing source') }; - await expect(doctor(root, runner(calls), 'darwin', false, missing)).rejects.toThrow(loginFix('claude', missing.claude)); - await rm(join(sources.codex, 'auth.json')); - await expect(doctor(root, runner(calls), 'darwin', false, sources)).rejects.toThrow(loginFix('codex', sources.codex)); +test('doctor refuses non-Mac live verification before commands', async () => { + const root = await fixture(), calls: string[][] = []; + await expect(doctor(root, async args => { calls.push(args); return ''; }, 'linux')).rejects.toThrow('operator Mac'); expect(calls).toEqual([]); }); - -test('candidate doctor is capabilities-only and refuses credential-source inspection', async () => { - const { root, sources } = await fixture(), calls: string[][] = []; - await doctor(root, runner(calls), 'darwin', true); - expect(statusCalls(calls)).toEqual([]); expect(calls.some(args => args[0] === 'gh')).toBe(false); - const report = JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')); - expect(report.result).toBe('pass'); expect(report.candidate).toBe(true); - await expect(doctor(root, runner(calls), 'darwin', true, sources)).rejects.toThrow('cannot inspect source credentials'); -}); diff --git a/.claude/skills/verify-open-pstack/scripts/doctor.ts b/.claude/skills/verify-open-pstack/scripts/doctor.ts index fa865096..4881304b 100644 --- a/.claude/skills/verify-open-pstack/scripts/doctor.ts +++ b/.claude/skills/verify-open-pstack/scripts/doctor.ts @@ -1,23 +1,18 @@ import { mkdir, realpath } from 'node:fs/promises'; import { join } from 'node:path'; import { command, isolatedEnv, save, type Command } from './io.ts'; -import { credentialSources, loginFix } from './isolation.ts'; -import { HARNESSES, type CredentialSources } from './types.ts'; -export async function doctor(root: string, run: Command = command, platform = process.platform, candidate = false, sources?: CredentialSources): Promise { +export async function doctor(root: string, run: Command = command, platform = process.platform, candidate = false): Promise { const report: Record = { platform, candidate, date: new Date().toISOString(), skill: await realpath(join(import.meta.dir, '..')), checks: {} }; const checks = report.checks as Record; try { if (platform !== 'darwin') throw new Error('Live gate requires an operator Mac (Darwin)'); - if (candidate && sources) throw new Error('Candidate doctor cannot inspect source credentials'); - if (!candidate && !sources) throw new Error('Doctor requires --claude-config and --codex-home '); - const canonical = sources ? await credentialSources(sources) : undefined; const home = join(root, 'probe-home'); - for (const dir of ['tmp', '.claude', '.codex', '.config/gh', '.cache']) await mkdir(join(home, dir), { recursive: true, mode: 0o700 }); + for (const dir of ['tmp', '.codex', '.config/gh', '.cache']) await mkdir(join(home, dir), { recursive: true, mode: 0o700 }); const env = isolatedEnv(home); for (const binary of candidate ? ['bun', 'git', 'claude', 'codex'] : ['bun', 'git', 'claude', 'codex', 'gh']) checks[binary] = (await run([binary, '--version'], { env })).trim(); const claude = await run(['claude', '--help'], { env }); - for (const flag of ['--plugin-dir', '--settings', '--setting-sources']) if (!claude.includes(flag)) throw new Error(`Claude isolation missing ${flag}`); + for (const flag of ['--plugin-dir', '--settings']) if (!claude.includes(flag)) throw new Error(`Claude isolation missing ${flag}`); checks.claudeHelp = claude; for (const args of [['codex', 'plugin', 'marketplace', 'add', '--help'], ['codex', 'plugin', 'add', '--help']]) { const help = await run(args, { env }); @@ -25,20 +20,6 @@ export async function doctor(root: string, run: Command = command, platform = pr if (args.includes('marketplace') && !/local path|local or Git|local marketplace/i.test(help)) throw new Error('Codex local marketplace interface unavailable'); checks[args.join(' ')] = help; } - if (canonical) { - const authEnv = { ...env, CLAUDE_CONFIG_DIR: canonical.claude, CODEX_HOME: canonical.codex }; - for (const tool of HARNESSES) { - const args = tool === 'claude' ? ['claude', 'auth', 'status', '--json'] : ['codex', 'login', 'status']; - try { - const output = await run(args, { env: authEnv }); - checks[args.join(' ')] = output; - if (tool === 'claude' && JSON.parse(output)?.loggedIn !== true) throw new Error('Not logged in'); - } catch { - throw new Error(`${tool} source authentication status failed; log in once with: ${loginFix(tool, canonical[tool])}`); - } - } - } - // Status checks report cached login state; authenticated native requests still need live proof. report.result = 'pass'; } catch (error) { report.result = 'blocked'; report.reason = String(error); diff --git a/.claude/skills/verify-open-pstack/scripts/harness.test.ts b/.claude/skills/verify-open-pstack/scripts/harness.test.ts index 7143447d..e934e093 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, test } from 'bun:test'; -import { cp, mkdtemp, mkdir, readFile, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; +import { cp, lstat, mkdtemp, mkdir, readFile, readlink, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { doctor } from './doctor.ts'; @@ -10,19 +10,21 @@ import { evidence } from './verify.ts'; import { command, freshRoot, isolatedEnv, retainedFile, treeHash, type Command } from './io.ts'; const roots: string[] = []; async function fixture(): Promise { const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-test-'))); roots.push(root); return root; } -afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); -async function sourceFixture(root: string) { - const sources = { claude: join(root, 'source-claude'), codex: join(root, 'source-codex') }; - const credentials = { - claude: JSON.stringify({ claudeAiOauth: { accessToken: 'fixture-claude', refreshToken: 'fixture-claude-refresh' } }), - codex: JSON.stringify({ tokens: { access_token: 'fixture-codex', refresh_token: 'fixture-codex-refresh', id_token: 'fixture-codex-id' } }), - }; - for (const tool of ['claude', 'codex'] as const) { - await mkdir(sources[tool], { mode: 0o700 }); - await writeFile(join(sources[tool], tool === 'claude' ? '.credentials.json' : 'auth.json'), credentials[tool]); - await writeFile(join(sources[tool], 'source-only.txt'), 'must not be copied'); +const originalEnv = { HOME: process.env.HOME, USER: process.env.USER, LOGNAME: process.env.LOGNAME, + CLAUDE_CONFIG_DIR: process.env.CLAUDE_CONFIG_DIR, CODEX_HOME: process.env.CODEX_HOME }; +afterEach(async () => { + for (const [name, value] of Object.entries(originalEnv)) { + if (value === undefined) delete process.env[name]; else process.env[name] = value; } - return { sources, credentials }; + for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); +}); +async function loginFixture(root: string): Promise { + const home = join(root, 'daily-codex'); + await mkdir(home, { mode: 0o700 }); + await writeFile(join(home, 'auth.json'), 'opaque existing login'); + await writeFile(join(home, 'daily-state'), 'do not touch'); + process.env.CODEX_HOME = home; + return join(home, 'auth.json'); } describe('isolated harness boundaries', () => { @@ -40,10 +42,14 @@ describe('isolated harness boundaries', () => { expect(env.ANTHROPIC_API_KEY).toBeUndefined(); expect(env.OPENAI_API_KEY).toBeUndefined(); expect(env.HOME).toBe('/operator/home'); expect(env.USER).toBe('operator-user'); expect(env.LOGNAME).toBe('operator-login'); expect(env.GIT_CONFIG_GLOBAL).toBe('/dev/null'); expect(env.TMPDIR).toBe('/run/state/tmp'); - expect(env.CLAUDE_CONFIG_DIR).toBe('/run/state/.claude'); expect(env.CODEX_HOME).toBe('/run/state/.codex'); - expect(env.GH_CONFIG_DIR).toBe('/run/state/.config/gh'); expect(JSON.stringify(env)).not.toContain('/daily/'); + expect(env.CLAUDE_CONFIG_DIR).toBe('/daily/claude'); expect(env.CODEX_HOME).toBe('/run/state/.codex'); + expect(env.GH_CONFIG_DIR).toBe('/run/state/.config/gh'); + expect(JSON.stringify(env)).not.toContain('/daily/codex'); } - expect(launch('claude', '/run/state', '/candidate')).toEqual(['claude', '--plugin-dir', '/candidate/plugins/pstack', '--settings', '/run/state/settings.json', '--setting-sources', 'project']); + expect(launch('claude', '/run/state', '/candidate')).toEqual(['claude', '--plugin-dir', '/candidate/plugins/pstack', '--settings', '{"enabledPlugins":{"pstack@open-pstack":false}}']); + expect(launch('claude', '/run/state', '/candidate').join(' ')).not.toContain('/run/state'); + delete process.env.CLAUDE_CONFIG_DIR; + expect(isolatedEnv('/run/state', 'claude').CLAUDE_CONFIG_DIR).toBeUndefined(); expect(launch('codex', '/run/state', '/candidate')).toEqual(['codex']); } finally { for (const [name, value] of Object.entries(previous)) { @@ -87,9 +93,7 @@ describe('isolated harness boundaries', () => { await symlink(skill, join(workspace, '.claude/skills/verify-open-pstack')); const alias = join(root, 'workspace-alias'); await symlink(workspace, alias); const run: Command = async args => args.includes('--version') ? 'version' : '--plugin-dir --settings --setting-sources --json local path'; - const { sources } = await sourceFixture(root); - const parentRun: Command = async args => args[1] === 'auth' ? JSON.stringify({ loggedIn: true }) : args[1] === 'login' ? 'Logged in using ChatGPT' : run(args); - await doctor(root, parentRun, 'darwin', false, sources); + await doctor(root, run, 'darwin', false); const parentText = await readFile(join(root, 'doctor.json'), 'utf8'); await expect(verifyProjectDoctor([parentText], workspace)).rejects.toThrow('passing child doctor'); await doctor(root, run, 'darwin', true); @@ -103,56 +107,10 @@ describe('isolated harness boundaries', () => { await expect(verifyProjectDoctor(['{}'], workspace)).rejects.toThrow('passing child doctor'); await expect(verifyProjectDoctor([JSON.stringify({ ...JSON.parse(text), result: 'blocked' })], workspace)).rejects.toThrow('passing child doctor'); }); - test('setup fails closed before any prepare or exercise command', async () => { - const root = await fixture(), sha = '1'.repeat(40), calls: string[][] = []; - const receipt = newReceipt(120, sha, sha, false, root); - receipt.selection.features = ['setup']; - const driver = new MacDriver(async args => { calls.push(args); return ''; }, async () => '', { claude: join(root, 'source-claude'), codex: join(root, 'source-codex') }); - const message = 'setup exercise requires #120 (setup-pstack config-home)'; - for (const operation of [() => driver.prepare(receipt), () => driver.exercise(receipt)]) { - let caught: unknown; - try { await operation(); } catch (error) { caught = error; } - expect(caught).toBeInstanceOf(Error); expect((caught as Error).message).toBe(message); expect(calls).toEqual([]); - } - }); - test.each(['claude', 'codex'] as const)('prepare refuses an unauthenticated %s source before candidate commands', async tool => { - const root = await fixture(), calls: string[][] = [], sha = '2'.repeat(40); - const { sources, credentials } = await sourceFixture(root); - const run: Command = async (args, options = {}) => { - calls.push(args); - if (args.includes('--version')) return 'version'; - if (args.includes('--help')) return '--plugin-dir --settings --setting-sources --json local path'; - if (args[1] === 'auth' || args[1] === 'login') { - expect(options.env!.CLAUDE_CONFIG_DIR).toBe(sources.claude); - expect(options.env!.CODEX_HOME).toBe(sources.codex); - if (args[0] === 'claude') { - expect(args).toEqual(['claude', 'auth', 'status', '--json']); - return JSON.stringify({ loggedIn: tool !== 'claude' }); - } - expect(args).toEqual(['codex', 'login', 'status']); - if (tool === 'codex') throw new Error('Not logged in'); - return 'Logged in using ChatGPT'; - } - throw new Error(`Unexpected fixture command: ${args.join(' ')}`); - }; - const platform = Object.getOwnPropertyDescriptor(process, 'platform')!; - const remediation = tool === 'claude' ? `CLAUDE_CONFIG_DIR='${sources.claude}' claude auth login` : `CODEX_HOME='${sources.codex}' codex login`; - try { - Object.defineProperty(process, 'platform', { ...platform, value: 'darwin' }); - await expect(new MacDriver(run, async () => '', sources).prepare(newReceipt(111, sha, sha, true, root))).rejects.toThrow(remediation); - } finally { Object.defineProperty(process, 'platform', platform); } - expect(calls.some(c => c.join(' ') === (tool === 'claude' ? 'claude auth status --json' : 'codex login status'))).toBe(true); - expect(calls.some(c => c.includes('login') && c.join(' ') !== 'codex login status')).toBe(false); - expect(calls.some(c => c[0] === 'git' && !c.includes('--version'))).toBe(false); - await expect(stat(join(root, 'state'))).rejects.toThrow('ENOENT'); - for (const provider of ['claude', 'codex'] as const) { - expect(await readFile(join(sources[provider], provider === 'claude' ? '.credentials.json' : 'auth.json'), 'utf8')).toBe(credentials[provider]); - } - expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('blocked'); - }); - test.each([false, true])('prepare uses pinned local git fixture and disposable config credentials (install failure=%s)', async failInstall => { + + test.each([false, true])('prepare pins local Git candidate and links existing Codex login (install failure=%s)', async failInstall => { const root = await fixture(), repository = join(root, 'repository'), operatorHome = join(root, 'operator-home'), calls: string[][] = []; - const { sources, credentials } = await sourceFixture(root); + const auth = await loginFixture(root); for (const manifest of ['.claude-plugin', '.codex-plugin']) { const dir = join(repository, 'plugins/pstack', manifest); await mkdir(dir, { recursive: true }); await writeFile(join(dir, 'plugin.json'), JSON.stringify({ version: 'test' })); @@ -170,28 +128,18 @@ describe('isolated harness boundaries', () => { const previous = { HOME: process.env.HOME, USER: process.env.USER, LOGNAME: process.env.LOGNAME }; const run: Command = async (args, options = {}) => { calls.push(args); - if (args[1] === 'auth' || args[1] === 'login') { - expect(options.env!.CLAUDE_CONFIG_DIR).toBe(sources.claude); - expect(options.env!.CODEX_HOME).toBe(sources.codex); - if (args[0] === 'claude') { - expect(args).toEqual(['claude', 'auth', 'status', '--json']); - return JSON.stringify({ loggedIn: true }); - } - expect(args).toEqual(['codex', 'login', 'status']); - return 'Logged in using ChatGPT'; - } - const env = options.env, candidate = Boolean(env?.CLAUDE_CONFIG_DIR?.startsWith(join(root, 'state') + '/')); + const env = options.env, candidate = Boolean(env?.CODEX_HOME?.startsWith(join(root, 'state') + '/')); if (candidate) { - const stateHome = join(env!.CLAUDE_CONFIG_DIR!, '..'); + const stateHome = join(env!.CODEX_HOME!, '..'); expect(env!.HOME).toBe(operatorHome); expect(env!.USER).toBe('operator-user'); expect(env!.LOGNAME).toBe('operator-login'); - expect(env!.CLAUDE_CONFIG_DIR).toBe(join(stateHome, '.claude')); expect(env!.CODEX_HOME).toBe(join(stateHome, '.codex')); + expect(env!.CLAUDE_CONFIG_DIR).toBe(join(operatorHome, 'claude-config')); expect(env!.CODEX_HOME).toBe(join(stateHome, '.codex')); expect(env!.TMPDIR).toBe(join(stateHome, 'tmp')); expect(env!.GH_CONFIG_DIR).toBe(join(stateHome, '.config/gh')); expect(env!.GH_TOKEN).toBeUndefined(); expect(env!.GITHUB_TOKEN).toBeUndefined(); - for (const dir of [stateHome, env!.TMPDIR!, env!.CLAUDE_CONFIG_DIR!, env!.CODEX_HOME!, env!.GH_CONFIG_DIR!]) { + for (const dir of [stateHome, env!.TMPDIR!, env!.CODEX_HOME!, env!.GH_CONFIG_DIR!]) { const info = await stat(dir); expect(info.isDirectory()).toBe(true); expect(info.mode & 0o777).toBe(0o700); } - const settings = join(stateHome, 'settings.json'); expect(JSON.parse(await readFile(settings, 'utf8'))).toEqual({}); - expect((await stat(settings)).mode & 0o777).toBe(0o600); + await expect(stat(join(stateHome, '.claude'))).rejects.toThrow('ENOENT'); + await expect(stat(join(stateHome, 'settings.json'))).rejects.toThrow('ENOENT'); } if (args[0] === '/usr/bin/script') { expect(candidate).toBe(true); expect(options.interactive).toBe(true); @@ -206,31 +154,30 @@ describe('isolated harness boundaries', () => { if (args.includes('--version')) return 'version'; if (args.includes('--help')) return '--plugin-dir --settings --setting-sources --json local path'; if (args[0] === 'codex' && args[1] === 'plugin') { - for (const tool of ['claude', 'codex'] as const) { - const file = join(env![tool === 'claude' ? 'CLAUDE_CONFIG_DIR' : 'CODEX_HOME']!, tool === 'claude' ? '.credentials.json' : 'auth.json'); - expect(await readFile(file, 'utf8')).toBe(credentials[tool]); expect((await stat(file)).mode & 0o777).toBe(0o600); - } + const linked = join(env!.CODEX_HOME!, 'auth.json'); + expect((await lstat(linked)).isSymbolicLink()).toBe(true); + expect(await readlink(linked)).toBe(auth); if (failInstall) throw new Error('fixture installation failed'); if (args[2] === 'marketplace') return '{}'; if (args[2] === 'add') { const installedPath = join(env!.CODEX_HOME!, 'plugins/pstack'); - await cp(join(join(env!.CLAUDE_CONFIG_DIR!, '..'), 'workspace/plugins/pstack'), installedPath, { recursive: true }); + await cp(join(env!.CODEX_HOME!, '../workspace/plugins/pstack'), installedPath, { recursive: true }); return JSON.stringify({ name: 'pstack', marketplaceName: 'open-pstack', installedPath }); } if (args[2] === 'list') return JSON.stringify({ installed: [{ name: 'pstack', marketplaceName: 'open-pstack', installed: true, enabled: true }] }); } throw new Error(`Unexpected fixture command: ${args.join(' ')}`); }; - const driver = new MacDriver(run, async () => '', sources); + const driver = new MacDriver(run, async () => ''); try { Object.defineProperty(process, 'platform', { ...platform, value: 'darwin' }); process.env.HOME = operatorHome; process.env.USER = 'operator-user'; process.env.LOGNAME = 'operator-login'; + process.env.CLAUDE_CONFIG_DIR = join(operatorHome, 'claude-config'); const receipt = newReceipt(111, sha, sha, true, root), preparation = driver.prepare(receipt); if (failInstall) await expect(preparation).rejects.toThrow('fixture installation failed'); else { const installs = await preparation; receipt.installations = installs; expect(installs.map(i => i.harness)).toEqual(['claude', 'codex']); - expect(installs.map(i => i.credentialSource)).toEqual([sources.claude, sources.codex]); expect(installs.every(i => i.sha === sha)).toBe(true); expect(installs.every(i => /^[a-f0-9]{64}$/.test(i.sourceHash!))).toBe(true); expect(installs[0]!.sourceHash).toBe(installs[1]!.sourceHash); @@ -243,7 +190,7 @@ describe('isolated harness boundaries', () => { } } finally { await mkdir(join(root, 'artifacts'), { mode: 0o700 }); - await writeFile(join(root, 'artifacts/private.txt'), credentials.codex, { mode: 0o600 }); + await writeFile(join(root, 'artifacts/private.txt'), 'private raw evidence', { mode: 0o600 }); for (const harness of ['claude', 'codex']) await writeFile(join(root, 'state', harness, 'native-state.db'), 'retained candidate state'); await driver.cleanup(); await driver.cleanup(); Object.defineProperty(process, 'platform', platform); @@ -251,26 +198,52 @@ describe('isolated harness boundaries', () => { if (value === undefined) delete process.env[name]; else process.env[name] = value; } } - expect(calls.filter(c => c.join(' ') === 'claude auth status --json')).toHaveLength(1); - expect(calls.filter(c => c.join(' ') === 'codex login status')).toHaveLength(1); + expect(calls.some(c => c.includes('auth') || c.includes('login'))).toBe(false); expect(calls.some(c => c[0] === '/usr/bin/sandbox-exec')).toBe(false); - expect(await readFile(join(root, 'artifacts/private.txt'), 'utf8')).toBe(credentials.codex); + expect(await readFile(join(root, 'artifacts/private.txt'), 'utf8')).toBe('private raw evidence'); + expect(await readFile(auth, 'utf8')).toBe('opaque existing login'); + expect(await readFile(join(auth, '../daily-state'), 'utf8')).toBe('do not touch'); expect((await stat(join(root, 'artifacts'))).mode & 0o777).toBe(0o700); expect((await stat(join(root, 'artifacts/private.txt'))).mode & 0o777).toBe(0o600); for (const harness of ['claude', 'codex']) { const home = join(root, 'state', harness); expect((await stat(home)).isDirectory()).toBe(true); expect(await readFile(join(home, 'native-state.db'), 'utf8')).toBe('retained candidate state'); - for (const tool of ['claude', 'codex'] as const) { - const filename = tool === 'claude' ? '.credentials.json' : 'auth.json'; - await expect(stat(join(home, '.' + tool, filename))).rejects.toThrow('ENOENT'); - expect(await readFile(join(sources[tool], filename), 'utf8')).toBe(credentials[tool]); - expect(await readFile(join(sources[tool], 'source-only.txt'), 'utf8')).toBe('must not be copied'); - await expect(stat(join(home, '.' + tool, 'source-only.txt'))).rejects.toThrow('ENOENT'); - } + if (harness === 'codex') await expect(stat(join(home, '.codex'))).rejects.toThrow('ENOENT'); + await expect(stat(join(home, '.claude'))).rejects.toThrow('ENOENT'); if (!failInstall) expect(await readFile(join(home, 'surface.raw'), 'utf8')).toBe('retained raw native evidence'); } expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('pass'); }); + test.each([false, true])('setup interruption restores exact bytes or forbids publication (restore failure=%s)', async failRestore => { + const root = await fixture(), home = join(root, 'state/claude'), workspace = join(home, 'workspace'); + const config = join(root, 'daily-claude'); process.env.CLAUDE_CONFIG_DIR = config; + for (const path of ['plugins/pstack', '.claude/skills/verify-open-pstack', '.agents/skills']) await mkdir(join(workspace, path), { recursive: true }); + await symlink('../../.claude/skills/verify-open-pstack', join(workspace, '.agents/skills/verify-open-pstack')); + await writeFile(join(workspace, 'plugins/pstack/SKILL.md'), 'candidate setup'); + await mkdir(config); + const original = Buffer.from([0, 13, 10, 255, 97]); + await writeFile(join(config, 'CLAUDE.md'), original); + const receipt = newReceipt(111, 'a'.repeat(40), 'b'.repeat(40), false, root); + receipt.selection = { paths: [], skills: [], features: ['setup'], noRuntime: false }; + receipt.installations = [{ harness: 'claude', home, location: join(workspace, 'plugins/pstack'), sha: receipt.sha, cliVersion: 'test', pluginVersion: 'test', treeHash: await treeHash(join(workspace, 'plugins/pstack')), sourceHash: await sourceDigest(workspace) }]; + const driver = new MacDriver(async args => { + expect(args[0]).toBe('/usr/bin/script'); + await writeFile(join(config, 'CLAUDE.md'), 'changed by setup'); + await writeFile(join(config, 'pstack-models.md'), 'created by setup'); + if (failRestore) { await rm(join(config, 'CLAUDE.md')); await mkdir(join(config, 'CLAUDE.md')); } + throw new Error('interrupted setup'); + }, async () => ''); + await expect(driver.exercise(receipt)).rejects.toThrow(failRestore ? 'Setup restoration requires a regular file' : 'interrupted setup'); + if (failRestore) { + await expect(driver.cleanup()).rejects.toThrow('Setup restoration requires a regular file'); + await rm(join(config, 'CLAUDE.md'), { recursive: true }); + await expect(driver.cleanup()).rejects.toThrow('Setup restoration failed; publication forbidden'); + return; + } + await driver.cleanup(); + expect(await readFile(join(config, 'CLAUDE.md'))).toEqual(original); + await expect(stat(join(config, 'pstack-models.md'))).rejects.toThrow('ENOENT'); + }); test('exercise re-prompts unsafe evidence and publishes accepted answers verbatim', async () => { const root = await fixture(), home = join(root, 'state/claude'), workspace = join(home, 'workspace'); for (const path of ['plugins/pstack', '.claude/skills/verify-open-pstack', '.agents/skills']) await mkdir(join(workspace, path), { recursive: true }); diff --git a/.claude/skills/verify-open-pstack/scripts/harness.ts b/.claude/skills/verify-open-pstack/scripts/harness.ts index 45c4c559..7f5a3015 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.ts @@ -1,13 +1,13 @@ -import { mkdir, readFile, realpath, writeFile } from 'node:fs/promises'; +import { mkdir, readFile, realpath } from 'node:fs/promises'; import { join } from 'node:path'; import { createInterface } from 'node:readline/promises'; import { requiredFeatures, requiredHarnesses } from './core.ts'; import { command, interruption, isolatedEnv, retainedFile, save, treeHash, type Command } from './io.ts'; import { doctor } from './doctor.ts'; -import { copyCredentials, credentialSources, removeCredentials } from './isolation.ts'; +import { assertCodexAuth, linkCodexAuth, removeCodexHome, restoreSetup, snapshotSetup, type SetupSnapshot } from './isolation.ts'; import { sourceDigest, sourceHash } from './provenance.ts'; import { EVIDENCE_LIMIT, publishable } from './verify.ts'; -import { HARNESSES, REPO, type CredentialSources, type Driver, type Harness, type Installation, type Observation, type Receipt } from './types.ts'; +import { HARNESSES, REPO, type Driver, type Harness, type Installation, type Observation, type Receipt } from './types.ts'; export type Ask = (question: string) => Promise; export const ask: Ask = async question => { @@ -17,7 +17,7 @@ export const ask: Ask = async question => { }; export function launch(harness: Harness, home: string, workspace: string): string[] { return harness === 'claude' - ? ['claude', '--plugin-dir', join(workspace, 'plugins/pstack'), '--settings', join(home, 'settings.json'), '--setting-sources', 'project'] + ? ['claude', '--plugin-dir', join(workspace, 'plugins/pstack'), '--settings', '{"enabledPlugins":{"pstack@open-pstack":false}}'] : ['codex']; } export async function codexInstallation(output: string, home: string, expected: string): Promise { @@ -46,23 +46,22 @@ export async function verifyProjectDoctor(texts: string[], workspace: string): P throw new Error('Self-test requires the pinned project skill\'s passing child doctor.json'); } export class MacDriver implements Driver { - private copied: string[] = []; - constructor(private run: Command = command, private review: Ask = ask, private sources?: CredentialSources) {} - async cleanup(): Promise { await removeCredentials(this.copied); } + private codexHomes: string[] = []; + private setup?: SetupSnapshot; + private restorationFailed = false; + constructor(private run: Command = command, private review: Ask = ask) {} + async cleanup(): Promise { + if (this.setup) await restoreSetup(this.setup); + if (this.restorationFailed) throw new Error('Setup restoration failed; publication forbidden'); + for (const home of this.codexHomes) await removeCodexHome(home); + this.codexHomes = []; + } private candidate(home: string): Command { return (args, options = {}) => this.run(args, { ...options, env: isolatedEnv(home) }); } - private refuseSetup(receipt: Receipt): void { - if (requiredFeatures(receipt).some(feature => feature === 'setup' || feature === 'skill-invocation:setup-pstack')) { - throw new Error('setup exercise requires #120 (setup-pstack config-home)'); - } - } async prepare(receipt: Receipt): Promise { - this.refuseSetup(receipt); const root = receipt.artifactRoot; - if (!this.sources) throw new Error('Explicit --claude-config and --codex-home credential directories are required'); - await doctor(root, this.run, undefined, false, this.sources); - const sources = await credentialSources(this.sources); + await doctor(root, this.run); // Fetch a separate immutable reference for pinned-source comparisons. const referenceHome = join(root, 'source-home'), reference = join(referenceHome, 'workspace'); await mkdir(join(referenceHome, 'tmp'), { recursive: true, mode: 0o700 }); @@ -76,8 +75,7 @@ export class MacDriver implements Driver { for (const harness of requiredHarnesses(receipt)) { const home = join(root, 'state', harness), workspace = join(home, 'workspace'); await mkdir(join(home, 'tmp'), { recursive: true, mode: 0o700 }); - for (const dir of ['.claude', '.codex', '.config/gh', '.cache']) await mkdir(join(home, dir), { recursive: true, mode: 0o700 }); - await writeFile(join(home, 'settings.json'), '{}\n', { mode: 0o600 }); + for (const dir of ['.codex', '.config/gh', '.cache']) await mkdir(join(home, dir), { recursive: true, mode: 0o700 }); const env = isolatedEnv(home, harness); const candidateRun = this.candidate(home); await candidateRun(['git', 'clone', '--no-checkout', '--', `https://github.com/${REPO}.git`, workspace], { env }); @@ -86,9 +84,10 @@ export class MacDriver implements Driver { if ((await candidateRun(['git', 'rev-parse', 'HEAD'], { cwd: workspace, env })).trim() !== receipt.sha) throw new Error('Candidate checkout SHA mismatch'); const candidate = join(workspace, 'plugins/pstack'), expected = await treeHash(candidate, ['skills/poteto-mode/scripts/node_modules']); if (await sourceDigest(workspace) !== pinnedSource) throw new Error('Candidate files differ from trusted pinned source'); - await copyCredentials(home, sources, this.copied); let location = candidate; if (harness === 'codex') { + await linkCodexAuth(home); + this.codexHomes.push(home); const added = await candidateRun(['codex', 'plugin', 'marketplace', 'add', workspace, '--json'], { env, cwd: home }); await save(join(root, 'codex-marketplace.json'), JSON.parse(added)); const installed = await candidateRun(['codex', 'plugin', 'add', 'pstack@open-pstack', '--json'], { env, cwd: home }); @@ -100,13 +99,12 @@ export class MacDriver implements Driver { } const manifest = JSON.parse(await readFile(join(location, harness === 'claude' ? '.claude-plugin/plugin.json' : '.codex-plugin/plugin.json'), 'utf8')); if (await sourceDigest(workspace) !== pinnedSource) throw new Error('Pinned source changed during installation'); - installs.push({ harness, sha: receipt.sha, home, location, treeHash: expected, sourceHash: pinnedSource, credentialSource: sources[harness], pluginVersion: manifest.version, + installs.push({ harness, sha: receipt.sha, home, location, treeHash: expected, sourceHash: pinnedSource, pluginVersion: manifest.version, cliVersion: (await candidateRun([harness, '--version'], { env })).trim() }); } return installs; } async exercise(receipt: Receipt): Promise { - this.refuseSetup(receipt); const observations: Observation[] = []; for (const installation of receipt.installations) { const { harness, home } = installation, workspace = join(home, 'workspace'); @@ -115,13 +113,22 @@ export class MacDriver implements Driver { const request = { sha: receipt.sha, harness, workspace, features: requiredFeatures(receipt, harness), featureMap: join(workspace, '.claude/skills/verify-open-pstack/features'), selfTest: requiredFeatures(receipt, harness).includes('project-skill') ? `Invoke the candidate project skill natively; run doctor --candidate --output "${join(home, 'self-test')}". Do not invoke run, login, or publication recursively.` : false, - credentials: { claude: env.CLAUDE_CONFIG_DIR, codex: env.CODEX_HOME }, - isolation: 'Real HOME/USER; both provider config roots and temporary state are run-owned. Use only copied disposable file credentials, never log in or write back to the credential sources. Exercise real requests; quota/auth failures fail closed.' }; + isolation: 'Existing operator login; Claude uses session-only candidate plugin settings. Codex uses run-owned configuration with a symlink to existing auth.json. Do not log in. Exercise real requests; quota/auth failures fail closed.' }; await save(join(receipt.artifactRoot, `${harness}-request.json`), request); console.log(JSON.stringify(request, null, 2)); const raw = join(home, 'surface.raw'); console.log('Exercise every requested feature from the native surface, using the maintained feature map. Exit when done.'); - await candidateRun(['/usr/bin/script', '-q', raw, ...launch(harness, home, workspace)], { cwd: workspace, env, interactive: true }); + const setup = requiredFeatures(receipt, harness).some(feature => feature === 'setup' || feature === 'skill-invocation:setup-pstack'); + if (setup) this.setup = await snapshotSetup(); + try { + await candidateRun(['/usr/bin/script', '-q', raw, ...launch(harness, home, workspace)], { cwd: workspace, env, interactive: true }); + } finally { + if (this.setup) { + try { await restoreSetup(this.setup); this.setup = undefined; } + catch (error) { this.restorationFailed = true; throw error; } + } + } + if (harness === 'codex') await assertCodexAuth(home); if (await treeHash(installation.location, ['skills/poteto-mode/scripts/node_modules']) !== installation.treeHash) throw new Error('Installed tree changed during exercise'); if (await sourceDigest(workspace) !== installation.sourceHash) throw new Error('Pinned project/plugin source changed during exercise'); const transcript = await retainedFile(receipt.artifactRoot, await this.review(`Retain a private copy of ${raw} inside output (outside state), review it, and enter its path. Do not publish raw content:`)); diff --git a/.claude/skills/verify-open-pstack/scripts/io.test.ts b/.claude/skills/verify-open-pstack/scripts/io.test.ts index 79e7c1af..7a04301c 100644 --- a/.claude/skills/verify-open-pstack/scripts/io.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/io.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, test } from 'bun:test'; import { chmod, mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { assertNoKnownSecrets, command, isolatedEnv, registerSecrets, retainedFile, save, sha256, treeHash } from './io.ts'; +import { command, isolatedEnv, retainedFile, save, sha256, treeHash } from './io.ts'; const roots: string[] = []; async function fixture(): Promise { @@ -29,17 +29,8 @@ describe('private values and retained file boundaries', () => { expect(String(failure)).toBe('Error: /bin/sh failed: private sk-review diagnostic'); }); - test('refuses only exact copied secrets in comments with a generic error', () => { - const token = 'opaque-copied-token.+/value'; - registerSecrets([token, '']); - expect(() => assertNoKnownSecrets(`review ${token} result`)).toThrow('Comment contains copied credentials'); - try { assertNoKnownSecrets(token); } catch (error) { expect(String(error)).not.toContain(token); } - expect(() => assertNoKnownSecrets('review opaque-copied-token result')).not.toThrow(); - expect(() => assertNoKnownSecrets('review sk-private and sk-review result')).not.toThrow(); - }); - test('uses real account metadata, run-owned config paths, and normal PATH', () => { - const previous = { HOME: process.env.HOME, USER: process.env.USER, LOGNAME: process.env.LOGNAME, PATH: process.env.PATH }; + const previous = { HOME: process.env.HOME, USER: process.env.USER, LOGNAME: process.env.LOGNAME, PATH: process.env.PATH, CLAUDE_CONFIG_DIR: process.env.CLAUDE_CONFIG_DIR }; process.env.HOME = '/Users/operator'; process.env.USER = 'operator-user'; process.env.LOGNAME = 'operator-login'; process.env.PATH = '/usr/local/bin:/usr/bin:/bin'; try { @@ -48,14 +39,23 @@ describe('private values and retained file boundaries', () => { expect(env).toMatchObject({ HOME: '/Users/operator', USER: 'operator-user', LOGNAME: 'operator-login', PATH: '/usr/local/bin:/usr/bin:/bin', TMPDIR: '/run/home/tmp', XDG_CONFIG_HOME: '/run/home/.config', XDG_CACHE_HOME: '/run/home/.cache', GH_CONFIG_DIR: '/run/home/.config/gh', - CLAUDE_CONFIG_DIR: '/run/home/.claude', CODEX_HOME: '/run/home/.codex' }); + CODEX_HOME: '/run/home/.codex' }); + delete process.env.CLAUDE_CONFIG_DIR; + expect(isolatedEnv('/run/home').CLAUDE_CONFIG_DIR).toBeUndefined(); + process.env.CLAUDE_CONFIG_DIR = '/Users/operator/custom-claude'; + for (const harness of ['claude', 'codex'] as const) { + const inherited = isolatedEnv('/run/home', harness); + expect(inherited.CLAUDE_CONFIG_DIR).toBe('/Users/operator/custom-claude'); + expect(inherited.CODEX_HOME).toBe('/run/home/.codex'); + for (const key of ['GH_TOKEN', 'GITHUB_TOKEN', 'OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'CLAUDE_CODE_OAUTH_TOKEN']) expect(inherited[key]).toBeUndefined(); + } expect(env.PATH).not.toContain('/run/home/bin'); process.env.HOME = 'relative/home'; expect(() => isolatedEnv('/run/home')).toThrow('Real HOME must be absolute'); process.env.HOME = '/Users/operator'; delete process.env.USER; expect(() => isolatedEnv('/run/home', 'codex')).toThrow('Real USER and LOGNAME are required'); } finally { - for (const name of ['HOME', 'USER', 'LOGNAME', 'PATH'] as const) { + for (const name of ['HOME', 'USER', 'LOGNAME', 'PATH', 'CLAUDE_CONFIG_DIR'] as const) { if (previous[name] === undefined) delete process.env[name]; else process.env[name] = previous[name]; } } @@ -64,7 +64,6 @@ describe('private values and retained file boundaries', () => { test('retains binary artifacts unchanged without inspecting copied secrets', async () => { const root = await fixture(), path = join(root, 'artifact.bin'), token = 'known-copied-binary-opaque-credential'; const bytes = Buffer.concat([Buffer.from([0, 255, 128]), Buffer.from(token), Buffer.from([0, 254])]); - registerSecrets([token]); await writeFile(path, bytes); expect(await retainedFile(root, path)).toEqual({ path: 'artifact.bin', sha256: sha256(bytes) }); expect(Buffer.from(await readFile(path))).toEqual(bytes); diff --git a/.claude/skills/verify-open-pstack/scripts/io.ts b/.claude/skills/verify-open-pstack/scripts/io.ts index 92716d34..2434dc77 100644 --- a/.claude/skills/verify-open-pstack/scripts/io.ts +++ b/.claude/skills/verify-open-pstack/scripts/io.ts @@ -37,13 +37,6 @@ export const command: Command = async (args, options = {}) => { activeCommands.delete(child.pid); } }; -const secrets = new Set(); -export function registerSecrets(values: string[]): void { - for (const value of values) if (value) secrets.add(value); -} -export function assertNoKnownSecrets(body: string): void { - if ([...secrets].some(secret => body.includes(secret))) throw new Error('Comment contains copied credentials'); -} export function isolatedEnv(state: string, _harness?: 'claude' | 'codex'): Record { if (!isAbsolute(state)) throw new Error('Candidate state must be an absolute run-owned directory'); const { HOME, USER, LOGNAME } = process.env; @@ -55,7 +48,8 @@ export function isolatedEnv(state: string, _harness?: 'claude' | 'codex'): Recor GH_CONFIG_DIR: join(state, '.config/gh'), GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_SYSTEM: '/dev/null', GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'credential.helper', GIT_CONFIG_VALUE_0: '', - CLAUDE_CONFIG_DIR: join(state, '.claude'), CODEX_HOME: join(state, '.codex') }; + ...(process.env.CLAUDE_CONFIG_DIR !== undefined ? { CLAUDE_CONFIG_DIR: process.env.CLAUDE_CONFIG_DIR } : {}), + CODEX_HOME: join(state, '.codex') }; } export async function save(path: string, value: unknown): Promise { const json = JSON.stringify(value, null, 2); diff --git a/.claude/skills/verify-open-pstack/scripts/isolation.test.ts b/.claude/skills/verify-open-pstack/scripts/isolation.test.ts index 44f1102c..226bf381 100644 --- a/.claude/skills/verify-open-pstack/scripts/isolation.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/isolation.test.ts @@ -1,161 +1,123 @@ import { afterEach, describe, expect, test } from 'bun:test'; -import { lstat, mkdir, mkdtemp, readFile, readdir, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; +import { lstat, mkdir, mkdtemp, readFile, readlink, realpath, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; -import { assertNoKnownSecrets, isolatedEnv } from './io.ts'; -import { copyCredentials, credentialSources, loginFix, removeCredentials } from './isolation.ts'; -import type { CredentialSources } from './types.ts'; +import { join } from 'node:path'; +import { assertCodexAuth, linkCodexAuth, removeCodexHome, restoreSetup, snapshotSetup } from './isolation.ts'; const roots: string[] = []; -async function fixture(): Promise { +const environment = { HOME: process.env.HOME, CODEX_HOME: process.env.CODEX_HOME, CLAUDE_CONFIG_DIR: process.env.CLAUDE_CONFIG_DIR }; +afterEach(async () => { + for (const [name, value] of Object.entries(environment)) { + if (value === undefined) delete process.env[name]; else process.env[name] = value; + } + for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); +}); +async function fixture() { const root = await realpath(await mkdtemp(join(tmpdir(), 'pstack-isolation-'))); roots.push(root); - return root; + process.env.HOME = root; + process.env.CODEX_HOME = join(root, 'daily-codex'); + process.env.CLAUDE_CONFIG_DIR = join(root, 'daily-claude'); + const home = join(root, 'run'); + for (const directory of [home, process.env.CODEX_HOME, process.env.CLAUDE_CONFIG_DIR]) await mkdir(directory, { mode: 0o700 }); + return { root, home, auth: join(process.env.CODEX_HOME, 'auth.json'), claude: process.env.CLAUDE_CONFIG_DIR }; } -afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); -async function sources(root: string): Promise { - const directories = { claude: join(root, 'Claude logged in'), codex: join(root, 'Codex logged in') }; - for (const tool of ['claude', 'codex'] as const) { - await mkdir(directories[tool], { mode: 0o700 }); - await writeFile(join(directories[tool], tool === 'claude' ? '.credentials.json' : 'auth.json'), JSON.stringify( - tool === 'claude' - ? { claudeAiOauth: { accessToken: 'selected-claude-access', refreshToken: 'selected-claude-refresh' }, account: 'claude-evidence' } - : { tokens: { access_token: 'selected-codex-access', refresh_token: 'selected-codex-refresh', id_token: 'selected-codex-id' }, account_id: 'codex-evidence' }, - ), { mode: 0o600 }); - await writeFile(join(directories[tool], 'settings.json'), '{"source-only":true}'); - } - return directories; -} -function files(source: CredentialSources): string[] { return [join(source.claude, '.credentials.json'), join(source.codex, 'auth.json')]; } -describe('generic disposable credential sources', () => { - test('validates and canonicalizes explicitly selected normal-login directories with spaces', async () => { - const root = await fixture(), source = await sources(root); - expect(await credentialSources({ claude: join(source.claude, '..', 'Claude logged in'), codex: source.codex })).toEqual(source); - const missing = { ...source, codex: join(root, 'not logged in') }; - await expect(credentialSources(missing)).rejects.toThrow(loginFix('codex', missing.codex)); - await rm(files(source)[0]!); - await expect(credentialSources(source)).rejects.toThrow(loginFix('claude', source.claude)); +describe('existing native login', () => { + test('links opaque auth without parsing or copying it and removes only run-owned Codex state', async () => { + const { home, auth } = await fixture(); + await writeFile(auth, 'opaque non-JSON authentication'); + await linkCodexAuth(home); + const link = join(home, '.codex/auth.json'); + expect((await lstat(link)).isSymbolicLink()).toBe(true); + expect(await readlink(link)).toBe(auth); + await assertCodexAuth(home); + await writeFile(join(home, '.codex/state.db'), 'candidate state'); + await writeFile(join(home, 'surface.raw'), 'private evidence'); + await removeCodexHome(home); + expect(await Bun.file(link).exists()).toBe(false); + expect(await readFile(auth, 'utf8')).toBe('opaque non-JSON authentication'); + expect(await readFile(join(home, 'surface.raw'), 'utf8')).toBe('private evidence'); }); - test('login guidance quotes directories safely without activating or repairing anything', async () => { - const { command } = await import('./io.ts'); - const root = await fixture(), path = join(root, "dir's $(touch unexpected) ; config"); - const fix = loginFix('claude', path); - const assignment = fix.slice('CLAUDE_CONFIG_DIR='.length, -' claude auth login'.length); - expect(await command(['/bin/sh', '-c', `printf '%s' ${assignment}`], { cwd: root })).toBe(path); - expect(await Bun.file(join(root, 'unexpected')).exists()).toBe(false); - expect(loginFix('codex', '/normal/codex')).toBe("CODEX_HOME='/normal/codex' codex login"); + test('defaults to the real HOME Codex login', async () => { + const { root, home } = await fixture(); + delete process.env.CODEX_HOME; + await mkdir(join(root, '.codex')); + await writeFile(join(root, '.codex/auth.json'), 'opaque'); + await linkCodexAuth(home); + expect(await readlink(join(home, '.codex/auth.json'))).toBe(join(root, '.codex/auth.json')); }); - test('copies only complete refresh-capable credential files with private modes and leaves source configuration unchanged', async () => { - const root = await fixture(), source = await sources(root), home = join(root, 'candidate'), copied: string[] = []; - const originals = files(source), before = await Promise.all(originals.map(path => readFile(path))); - await copyCredentials(home, source, copied); - const env = isolatedEnv(home); - expect(copied).toEqual([join(env.CLAUDE_CONFIG_DIR!, '.credentials.json'), join(env.CODEX_HOME!, 'auth.json')]); - for (let index = 0; index < copied.length; index++) { - expect((await lstat(copied[index]!)).isSymbolicLink()).toBe(false); - expect((await stat(copied[index]!)).mode & 0o777).toBe(0o600); - expect((await stat(dirname(copied[index]!))).mode & 0o777).toBe(0o700); - expect(await readFile(copied[index]!)).toEqual(before[index]!); - expect(await readdir(dirname(copied[index]!))).toEqual([index === 0 ? '.credentials.json' : 'auth.json']); - } - expect(JSON.parse(await readFile(copied[0]!, 'utf8')).claudeAiOauth.refreshToken).toBe('selected-claude-refresh'); - expect(JSON.parse(await readFile(copied[1]!, 'utf8')).tokens.refresh_token).toBe('selected-codex-refresh'); - expect(await Promise.all(originals.map(path => readFile(path)))).toEqual(before); - for (const directory of Object.values(source)) expect(await readFile(join(directory, 'settings.json'), 'utf8')).toBe('{"source-only":true}'); + test('missing or non-file source auth refuses keyring-only authentication', async () => { + const { home, auth } = await fixture(); + await expect(linkCodexAuth(home)).rejects.toThrow('existing file-based authentication'); + await mkdir(auth); + await expect(linkCodexAuth(home)).rejects.toThrow('keyring-only authentication'); }); - test('registers only exact copied OAuth credential fields for public comment checks', async () => { - const root = await fixture(), source = await sources(root), copied: string[] = []; - const exact = ['comment-claude-access-7', 'comment-claude-refresh-7', 'comment-codex-access-7', 'comment-codex-refresh-7', 'comment-codex-id-7']; - const ordinary = ['visible-token-shaped-claude-field-7', 'visible-token-shaped-codex-field-7']; - await writeFile(files(source)[0]!, JSON.stringify({ claudeAiOauth: { accessToken: exact[0], refreshToken: exact[1], sessionTokenHint: ordinary[0] } })); - await writeFile(files(source)[1]!, JSON.stringify({ tokens: { access_token: exact[2], refresh_token: exact[3], id_token: exact[4], token_hint: ordinary[1] } })); - await copyCredentials(join(root, 'candidate'), source, copied); - for (const secret of exact) expect(() => assertNoKnownSecrets(secret)).toThrow('Comment contains copied credentials'); - expect(() => assertNoKnownSecrets(ordinary.join(' '))).not.toThrow(); + test('replaced auth blocks cleanup/publication and preserves the operator source', async () => { + const { home, auth } = await fixture(); + await writeFile(auth, 'operator login'); + await linkCodexAuth(home); + const link = join(home, '.codex/auth.json'); + await rm(link); await writeFile(link, 'replacement'); + await expect(assertCodexAuth(home)).rejects.toThrow('stop and report before publication'); + await expect(removeCodexHome(home)).rejects.toThrow('Codex replaced or redirected auth.json'); + expect(await readFile(auth, 'utf8')).toBe('operator login'); + expect(await readFile(link, 'utf8')).toBe('replacement'); }); - test('supports native Codex auth.json API-key credentials without discarding fields', async () => { - const root = await fixture(), source = await sources(root), home = join(root, 'candidate'), copied: string[] = []; - const auth = { OPENAI_API_KEY: 'exact-codex-api-key-7', auth_mode: 'apikey', last_refresh: null }; - await writeFile(files(source)[1]!, JSON.stringify(auth)); - await copyCredentials(home, source, copied); - expect(JSON.parse(await readFile(copied[1]!, 'utf8'))).toEqual(auth); - expect(() => assertNoKnownSecrets(auth.OPENAI_API_KEY)).toThrow('Comment contains copied credentials'); - }); - - test('cleanup removes only copied files while preserving candidate state, private raw evidence, config directories, and source credentials', async () => { - const root = await fixture(), source = await sources(root), home = join(root, 'candidate'), copied: string[] = []; - await copyCredentials(home, source, copied); - const raw = join(home, 'surface.raw'), artifact = join(home, 'reviewed-transcript.txt'), state = join(home, '.claude/session-state.json'); - await writeFile(raw, 'private raw transcript'); await writeFile(artifact, 'private reviewed evidence'); await writeFile(state, 'candidate session state'); - const before = await Promise.all(files(source).map(path => readFile(path, 'utf8'))); - await removeCredentials(copied); await removeCredentials(copied); - for (const path of copied) expect(await Bun.file(path).exists()).toBe(false); - expect(await readFile(raw, 'utf8')).toBe('private raw transcript'); - expect(await readFile(artifact, 'utf8')).toBe('private reviewed evidence'); - expect(await readFile(state, 'utf8')).toBe('candidate session state'); - expect((await stat(join(home, '.claude'))).isDirectory()).toBe(true); - expect((await stat(join(home, '.codex'))).isDirectory()).toBe(true); - expect(await Promise.all(files(source).map(path => readFile(path, 'utf8')))).toEqual(before); - }); - - test('partial copy cleanup retains failed-run evidence and malformed source files', async () => { - const root = await fixture(), source = await sources(root), home = join(root, 'candidate'), copied: string[] = []; - const bad = files(source)[1]!; await writeFile(bad, '{"unknown":true}'); - await expect(copyCredentials(home, source, copied)).rejects.toThrow(loginFix('codex', source.codex)); - expect(copied).toHaveLength(1); - const evidence = join(home, 'prepare-failure.txt'); await writeFile(evidence, 'retain failed-run evidence'); - await removeCredentials(copied); - expect(await Bun.file(copied[0]!).exists()).toBe(false); - expect(await readFile(evidence, 'utf8')).toBe('retain failed-run evidence'); - expect(await readFile(bad, 'utf8')).toBe('{"unknown":true}'); + test('redirected run-owned Codex directory is refused without touching the source', async () => { + const { home, auth } = await fixture(); + await writeFile(auth, 'operator login'); + await symlink(process.env.CODEX_HOME!, join(home, '.codex')); + await expect(linkCodexAuth(home)).rejects.toThrow('Codex state directory redirected'); + expect(await readFile(auth, 'utf8')).toBe('operator login'); }); +}); - test('source credentials and selected directories must be canonical regular sources', async () => { - const root = await fixture(), source = await sources(root), home = join(root, 'candidate'), copied: string[] = []; - const credential = files(source)[0]!, outside = join(root, 'outside-auth'); - await writeFile(outside, '{"claudeAiOauth":{"accessToken":"external"}}'); - await rm(credential); await symlink(outside, credential); - await expect(copyCredentials(home, source, copied)).rejects.toThrow('contained regular file'); - expect(copied).toEqual([]); - await rm(source.claude, { recursive: true }); await symlink(source.codex, source.claude); - await expect(copyCredentials(home, source, copied)).rejects.toThrow('not symlinked'); - expect(await readFile(outside, 'utf8')).toContain('external'); +describe('setup-only snapshot and restoration', () => { + test('restores both files byte-exact including binary bytes and restores absence', async () => { + const { claude } = await fixture(); + const models = join(claude, 'pstack-models.md'), instructions = join(claude, 'CLAUDE.md'); + const original = Buffer.from([0, 255, 13, 10, 65]); + await writeFile(models, original); + const snapshot = await snapshotSetup(); + await writeFile(models, 'setup mutation'); await writeFile(instructions, 'new setup instructions'); + await restoreSetup(snapshot); + expect(await readFile(models)).toEqual(original); + expect(await Bun.file(instructions).exists()).toBe(false); }); - test('candidate config-directory redirects fail closed without touching external sentinels', async () => { - const root = await fixture(), source = await sources(root), home = join(root, 'candidate'), external = join(root, 'external'); - await mkdir(home); await mkdir(external); - const sentinel = join(external, '.credentials.json'); await writeFile(sentinel, 'external sentinel'); - await symlink(external, join(home, '.claude')); - const copied: string[] = []; - await expect(copyCredentials(home, source, copied)).rejects.toThrow('redirected'); - expect(await readFile(sentinel, 'utf8')).toBe('external sentinel'); expect(copied).toEqual([]); - await expect(removeCredentials([join(home, '.claude/.credentials.json')])).rejects.toThrow('redirected candidate credential directories'); - expect(await readFile(sentinel, 'utf8')).toBe('external sentinel'); expect((await stat(home)).isDirectory()).toBe(true); + test('defaults to HOME/.claude and restores deleted existing files', async () => { + const { root } = await fixture(); + delete process.env.CLAUDE_CONFIG_DIR; + const directory = join(root, '.claude'); await mkdir(directory); + const path = join(directory, 'CLAUDE.md'); await writeFile(path, 'original\r\n'); + const snapshot = await snapshotSetup(); + await rm(path); + await restoreSetup(snapshot); + expect(await readFile(path, 'utf8')).toBe('original\r\n'); }); - test('cleanup unlinks copied-file leaf redirects without following them or removing candidate state', async () => { - const root = await fixture(), source = await sources(root), home = join(root, 'candidate'), copied: string[] = []; - await copyCredentials(home, source, copied); - const external = join(root, 'external-credential'); await writeFile(external, 'external sentinel'); - await rm(copied[0]!); await symlink(external, copied[0]!); - const state = join(home, 'state.json'); await writeFile(state, 'preserved'); - await removeCredentials(copied); - expect(await Bun.file(copied[0]!).exists()).toBe(false); - expect(await readFile(external, 'utf8')).toBe('external sentinel'); expect(await readFile(state, 'utf8')).toBe('preserved'); + test('rejects symlink snapshots and failed restoration without following redirected files', async () => { + const { root, claude } = await fixture(); + const path = join(claude, 'pstack-models.md'), external = join(root, 'external'); + await writeFile(external, 'untouched'); + await symlink(external, path); + await expect(snapshotSetup()).rejects.toThrow('regular file'); + await rm(path); await writeFile(path, 'original'); + const snapshot = await snapshotSetup(); + await rm(path); await symlink(external, path); + await expect(restoreSetup(snapshot)).rejects.toThrow('regular file'); + expect(await readFile(external, 'utf8')).toBe('untouched'); }); - test('an existing destination is not added to cleanup or deleted on copy failure', async () => { - const root = await fixture(), source = await sources(root), home = join(root, 'candidate'), copied: string[] = []; - await mkdir(join(home, '.claude'), { recursive: true }); - const existing = join(home, '.claude/.credentials.json'); await writeFile(existing, 'existing credential sentinel'); - await expect(copyCredentials(home, source, copied)).rejects.toThrow(); - expect(copied).toEqual([]); await removeCredentials(copied); - expect(await readFile(existing, 'utf8')).toBe('existing credential sentinel'); + test('non-file setup mutations fail restoration instead of publishing success', async () => { + const { claude } = await fixture(); + const snapshot = await snapshotSetup(); + await mkdir(join(claude, 'CLAUDE.md')); + await expect(restoreSetup(snapshot)).rejects.toThrow('regular file'); }); }); diff --git a/.claude/skills/verify-open-pstack/scripts/isolation.ts b/.claude/skills/verify-open-pstack/scripts/isolation.ts index b24bee34..cb4ef31e 100644 --- a/.claude/skills/verify-open-pstack/scripts/isolation.ts +++ b/.claude/skills/verify-open-pstack/scripts/isolation.ts @@ -1,111 +1,73 @@ -import { chmod, lstat, mkdir, open, readFile, realpath, unlink } from 'node:fs/promises'; +import { lstat, mkdir, readFile, readlink, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; import { isAbsolute, join, resolve } from 'node:path'; -import { isolatedEnv, registerSecrets } from './io.ts'; -import { HARNESSES, type CredentialSources, type Harness } from './types.ts'; -export function loginFix(tool: Harness, directory: string): string { - const quoted = `'${directory.replaceAll("'", "'\\''")}'`; - return tool === 'claude' ? `CLAUDE_CONFIG_DIR=${quoted} claude auth login` : `CODEX_HOME=${quoted} codex login`; +function codexAuth(): string { + return resolve(process.env.CODEX_HOME ?? join(process.env.HOME!, '.codex'), 'auth.json'); } -export async function credentialSources(sources: CredentialSources): Promise { - const canonical = {} as CredentialSources; - for (const tool of HARNESSES) { - const directory = sources[tool]; - try { - if (typeof directory !== 'string' || !directory || directory.includes('\0')) throw new Error('explicit source directory required'); - const path = resolve(directory), info = await lstat(path); - if (!info.isDirectory() || info.isSymbolicLink()) throw new Error('source directory must be a directory, not symlinked'); - canonical[tool] = await realpath(path); - const source = join(canonical[tool], tool === 'claude' ? '.credentials.json' : 'auth.json'); - const sourceInfo = await lstat(source); - if (!sourceInfo.isFile() || sourceInfo.isSymbolicLink() || await realpath(source) !== source) throw new Error('credentials must be a contained regular file'); - } catch (error) { - const reason = error instanceof Error ? error.message : String(error); - throw new Error(`${tool} credential source unavailable: ${reason}; log in once with: ${loginFix(tool, directory ?? '')}`); - } - } - return canonical; +async function codexDirectory(home: string): Promise { + if (!isAbsolute(home) || await realpath(home) !== home) throw new Error('Codex state must be a canonical run-owned directory'); + const directory = join(home, '.codex'); + const info = await lstat(directory); + if (!info.isDirectory() || info.isSymbolicLink() || await realpath(directory) !== directory) throw new Error('Codex state directory redirected'); + return directory; } -function credentialLocation(home: string, tool: Harness): { directory: string; path: string } { - const env = isolatedEnv(home); - const directory = tool === 'claude' ? env.CLAUDE_CONFIG_DIR : env.CODEX_HOME; - if (!directory) throw new Error(`Isolated ${tool} configuration directory is missing`); - return { directory, path: join(directory, tool === 'claude' ? '.credentials.json' : 'auth.json') }; +export async function linkCodexAuth(home: string): Promise { + const source = codexAuth(); + const info = await stat(source).catch((error: NodeJS.ErrnoException) => { + if (error.code !== 'ENOENT') throw error; + return undefined; + }); + if (!info?.isFile()) throw new Error('Codex verification requires existing file-based authentication (auth.json); keyring-only authentication is not supported'); + await mkdir(join(home, '.codex'), { recursive: true, mode: 0o700 }); + await symlink(source, join(await codexDirectory(home), 'auth.json')); } -async function canonicalDirectory(path: string, message: string): Promise { +export async function assertCodexAuth(home: string): Promise { + const path = join(await codexDirectory(home), 'auth.json'); const info = await lstat(path); - if (!info.isDirectory() || info.isSymbolicLink() || await realpath(path) !== path) throw new Error(message); + if (!info.isSymbolicLink() || await readlink(path) !== codexAuth()) { + throw new Error('Codex replaced or redirected auth.json instead of writing through its symlink; stop and report before publication'); + } } -function credentialTokens(tool: Harness, auth: Record): string[] { - const fields = (tool === 'claude' ? auth.claudeAiOauth : auth.tokens) as Record | undefined; - const access = tool === 'claude' ? fields?.accessToken : fields?.access_token; - const apiKey = tool === 'codex' && typeof auth.OPENAI_API_KEY === 'string' && auth.OPENAI_API_KEY ? auth.OPENAI_API_KEY : undefined; - if ((typeof access !== 'string' || !access) && !apiKey) throw new Error(`${tool} requires file-backed native credentials; unknown format blocks`); - const names = tool === 'claude' ? ['accessToken', 'refreshToken'] : ['access_token', 'refresh_token', 'id_token']; - return [...(apiKey ? [apiKey] : []), ...names.flatMap(name => typeof fields?.[name] === 'string' ? [fields[name] as string] : [])]; +export async function removeCodexHome(home: string): Promise { + await assertCodexAuth(home); + await rm(await codexDirectory(home), { recursive: true }); } -export async function copyCredentials(home: string, sources: CredentialSources, copied: string[]): Promise { - if (!isAbsolute(home) || resolve(home) !== home) throw new Error('Credential copy requires canonical candidate state'); - const canonical = await credentialSources(sources); - await mkdir(home, { recursive: true, mode: 0o700 }); - await canonicalDirectory(home, 'Credential copy refuses redirected candidate state'); +export type SetupSnapshot = { path: string; bytes?: Buffer; mode?: number }[]; - for (const tool of HARNESSES) { - const source = join(canonical[tool], tool === 'claude' ? '.credentials.json' : 'auth.json'); - const bytes = await readFile(source); - try { - const auth = JSON.parse(bytes.toString()) as Record; - registerSecrets(credentialTokens(tool, auth)); - } catch { - throw new Error(`${tool} credential source has unsupported file-backed credentials; log in once with: ${loginFix(tool, canonical[tool])}`); - } +async function setupFile(path: string) { + const info = await lstat(path).catch((error: NodeJS.ErrnoException) => { + if (error.code !== 'ENOENT') throw error; + return undefined; + }); + if (info && (!info.isFile() || info.isSymbolicLink())) throw new Error(`Setup restoration requires a regular file: ${path}`); + return info; +} - const target = credentialLocation(home, tool); - await mkdir(target.directory, { recursive: true, mode: 0o700 }); - await canonicalDirectory(target.directory, 'Candidate credential directory redirected'); - await chmod(target.directory, 0o700); - const file = await open(target.path, 'wx', 0o600); - copied.push(target.path); - try { await file.writeFile(bytes); } - finally { await file.close(); } +export async function snapshotSetup(): Promise { + const directory = process.env.CLAUDE_CONFIG_DIR ?? join(process.env.HOME!, '.claude'); + const snapshot: SetupSnapshot = []; + for (const name of ['pstack-models.md', 'CLAUDE.md']) { + const path = join(directory, name), info = await setupFile(path); + snapshot.push(info ? { path, bytes: await readFile(path), mode: info.mode & 0o777 } : { path }); } + return snapshot; } -export async function removeCredentials(paths: string[]): Promise { - for (const path of new Set(paths)) { - if (!isAbsolute(path) || resolve(path) !== path) throw new Error('Credential cleanup requires canonical copied paths'); - const home = resolve(path, '../..'); - const expected = HARNESSES.map(tool => credentialLocation(home, tool).path); - if (!expected.includes(path)) throw new Error('Credential cleanup only removes copied provider credential files'); - - const homeInfo = await lstat(home).catch((error: NodeJS.ErrnoException) => { - if (error.code !== 'ENOENT') throw error; - return undefined; - }); - if (!homeInfo) continue; - if (!homeInfo.isDirectory() || homeInfo.isSymbolicLink() || await realpath(home) !== home) throw new Error('Credential cleanup refuses redirected candidate state'); - - const directory = resolve(path, '..'); - const directoryInfo = await lstat(directory).catch((error: NodeJS.ErrnoException) => { - if (error.code !== 'ENOENT') throw error; - return undefined; - }); - if (!directoryInfo) continue; - if (!directoryInfo.isDirectory() || directoryInfo.isSymbolicLink() || await realpath(directory) !== directory) { - throw new Error('Credential cleanup refuses redirected candidate credential directories'); +export async function restoreSetup(snapshot: SetupSnapshot): Promise { + for (const { path, bytes, mode } of snapshot) { + await setupFile(path); + if (bytes === undefined) await rm(path, { force: true }); + else await writeFile(path, bytes, { mode }); + } + for (const { path, bytes } of snapshot) { + const info = await setupFile(path); + if (bytes === undefined ? info !== undefined : !info || !(await readFile(path)).equals(bytes)) { + throw new Error(`Setup restoration verification failed: ${path}`); } - - const info = await lstat(path).catch((error: NodeJS.ErrnoException) => { - if (error.code !== 'ENOENT') throw error; - return undefined; - }); - if (!info) continue; - if (!info.isFile() && !info.isSymbolicLink()) throw new Error('Credential cleanup only removes regular credential files or their leaf links'); - await unlink(path); } } diff --git a/.claude/skills/verify-open-pstack/scripts/types.ts b/.claude/skills/verify-open-pstack/scripts/types.ts index ac16e94b..d360948f 100644 --- a/.claude/skills/verify-open-pstack/scripts/types.ts +++ b/.claude/skills/verify-open-pstack/scripts/types.ts @@ -1,7 +1,6 @@ export const REPO = 'ericlitman/open-pstack'; export const HARNESSES = ['claude', 'codex'] as const; export type Harness = typeof HARNESSES[number]; -export type CredentialSources = Record; export type Phase = 'resolve' | 'classify' | 'prepare' | 'exercise' | 'publish' | 'failed' | 'head-moved'; export interface Pull { number: number; head: { sha: string }; base: { sha: string }; @@ -26,7 +25,7 @@ export interface Observation { } export interface Installation { harness: Harness; cliVersion: string; pluginVersion: string; sha: string; - location: string; treeHash: string; home: string; credentialSource?: string; sourceHash?: string; + location: string; treeHash: string; home: string; sourceHash?: string; } export interface Receipt { schema: 1; repo: string; pr: number; sha: string; base: string; phase: Phase; diff --git a/.claude/skills/verify-open-pstack/scripts/verify.test.ts b/.claude/skills/verify-open-pstack/scripts/verify.test.ts index 8a57d78c..c730f70f 100644 --- a/.claude/skills/verify-open-pstack/scripts/verify.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/verify.test.ts @@ -87,7 +87,7 @@ describe('exact-head evidence publication', () => { expect(f.calls).toContain('prepare'); }); - test('successful verification cleans disposable credentials before any publication', async () => { + test('successful verification restores setup and removes Codex run state before any publication', async () => { const f = fixture(true); f.options.driver.cleanup = async () => { f.calls.push('cleanup'); }; const receipt = await verify(f.options); expect(f.calls.filter(call => call === 'cleanup')).toHaveLength(1); @@ -104,13 +104,14 @@ describe('exact-head evidence publication', () => { expect(f.saved.at(-1)?.failure).toContain('RAW_CREDENTIAL_CLEANUP_EXCEPTION'); }); - test('a successful cleanup retry permits one generic failure publication', async () => { + test('failed setup restoration forbids publication even if cleanup retry succeeds', async () => { const f = fixture(true); let attempts = 0; - f.options.driver.cleanup = async () => { f.calls.push('cleanup'); if (++attempts === 1) throw new Error('RAW_FIRST_CLEANUP_FAILURE'); }; - await expect(verify(f.options)).rejects.toThrow('RAW_FIRST_CLEANUP_FAILURE'); - expect(f.calls.filter(call => call === 'cleanup')).toHaveLength(2); expect(f.comments).toHaveLength(1); - expect(f.comments[0]).toContain('FAILED — verification did not complete'); expect(f.comments[0]).not.toContain('RAW_FIRST_CLEANUP_FAILURE'); - expect(f.calls).toContain(`status:failure:${SHA}`); expect(f.saved.at(-1)?.failure).toContain('RAW_FIRST_CLEANUP_FAILURE'); + f.options.driver.cleanup = async () => { f.calls.push('cleanup'); if (++attempts === 1) throw new Error('Setup restoration failed'); }; + await expect(verify(f.options)).rejects.toThrow('Setup restoration failed'); + expect(f.calls.filter(call => call === 'cleanup')).toHaveLength(2); + expect(f.comments).toHaveLength(0); + expect(f.calls.some(call => call.startsWith('status:'))).toBe(false); + expect(f.saved.at(-1)?.failure).toContain('Setup restoration failed'); }); test('failure publication is structured and excludes raw exceptions, transcripts, and artifact paths', async () => { diff --git a/.claude/skills/verify-open-pstack/scripts/verify.ts b/.claude/skills/verify-open-pstack/scripts/verify.ts index 618b0a99..d250d18b 100644 --- a/.claude/skills/verify-open-pstack/scripts/verify.ts +++ b/.claude/skills/verify-open-pstack/scripts/verify.ts @@ -1,9 +1,9 @@ import { classify, completeEvidence, newReceipt } from './core.ts'; -import { assertNoKnownSecrets, retainedFile, sha256 } from './io.ts'; +import { retainedFile, sha256 } from './io.ts'; import { REPO, type Driver, type GitHub, type Pull, type Receipt, type Registry } from './types.ts'; class HeadMoved extends Error {} -// Omit unsafe/oversized values whole: truncating or rewriting one could hide an exact registered secret. +// Omit unsafe or oversized values whole rather than rewriting operator evidence. export const EVIDENCE_LIMIT = 160; export const publishable = (text: string, limit = EVIDENCE_LIMIT): boolean => text.length <= limit && /^[\x20-\x7e]*$/.test(text) && !/[`<>]/.test(text); const safe = (text: string, limit = EVIDENCE_LIMIT): string => publishable(text, limit) ? text : '[value omitted]'; @@ -63,12 +63,13 @@ export async function verify(options: VerifyOptions): Promise { const initial = await github.pull(options.pr); if (initial.state !== 'open' || initial.headRepo !== REPO) throw new Error('Require an open same-repository PR'); const receipt = newReceipt(options.pr, initial.head.sha, initial.base.sha, options.selfTest, options.root, options.publisherRevision); - let successAttempted = false, commentAttempted = false, cleanupComplete = false; + let successAttempted = false, commentAttempted = false, cleanupComplete = false, cleanupFailed = false; const cleanup = async (): Promise => { if (cleanupComplete) return; - await driver.cleanup?.(receipt); + try { await driver.cleanup?.(receipt); } + catch (error) { cleanupFailed = true; throw error; } cleanupComplete = true; - receipt.cleanup = 'Copied disposable credentials removed; candidate state and private raw evidence retained'; + receipt.cleanup = 'Setup files restored and verified; run-owned Codex home removed; private raw evidence retained'; }; const current = async (): Promise => { const pull = await github.pull(receipt.pr); @@ -82,7 +83,6 @@ export async function verify(options: VerifyOptions): Promise { }; const publishComment = async (): Promise => { const body = evidence(receipt); - assertNoKnownSecrets(body); commentAttempted = true; receipt.commentUrl = await github.comment(receipt.pr, body); await persist(receipt); @@ -116,13 +116,13 @@ export async function verify(options: VerifyOptions): Promise { try { await cleanup(); } catch (cleanupError) { receipt.cleanup = `FAILED: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`; - appendFailure('credential cleanup', cleanupError); + appendFailure('setup restoration or Codex cleanup', cleanupError); } try { await persist(receipt); } catch (persistError) { appendFailure('persist failure receipt', persistError); } - // Publication is forbidden while disposable credential cleanup remains incomplete. - if (cleanupComplete) { + // Failed setup restoration or Codex auth-link cleanup forbids all publication. + if (cleanupComplete && !cleanupFailed) { if (!commentAttempted) { try { await current(); await publishComment(); } catch (publicationError) { appendFailure('publish failure evidence', publicationError); } diff --git a/AGENTS.md b/AGENTS.md index d05622ec..6a284340 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,6 +8,6 @@ Before opening a pull request, run the Bun tests, strict typecheck, static invar Nothing merges, tags, releases, or rolls out until the exact candidate is installed and the changed behavior passes a live test from the real user surface in every affected harness. Unit tests, validators, source inspection, and self-reports do not satisfy this gate. The record is the `live-gate` commit status on that exact head, which links an evidence comment naming the installed version, surface, action, and observed result. Mergify does not queue a pull request without it. A pull request may be ready for review before its live test, but every new head, including a queue-created head, needs its own. -For Open Pstack PRs, use the shared repository-local `.claude/skills/verify-open-pstack/` skill (Codex: `.agents/skills/verify-open-pstack`). Verifier-changing PRs run from a checkout of their reviewed candidate head, with the publisher revision equal to the candidate SHA; all other PRs run from trusted `main`. Record the publisher revision separately from the candidate SHA. Read Launch for the explicit `--claude-config ` and `--codex-home ` options and separate private probe/live directories. Supply operator-pre-authenticated normal-login configuration directories; daily-active accounts are allowed. Doctor validates the source directories with `claude auth status` and `codex login status`. Authentication failures report `CLAUDE_CONFIG_DIR= claude auth login` or `CODEX_HOME= codex login` without logging in automatically. Candidate processes and provider children retain real `HOME`/`USER`/`LOGNAME` with both provider config roots and temporary state run-owned; only `.credentials.json` and `auth.json` are copied, retaining native refresh material, and are never written back to the source directories. Cleanup removes only copied credential files and preserves mode-0700 raw evidence through merge. Setup coverage fails closed with `setup exercise requires #120 (setup-pstack config-home)` until #120 merges. Publication is one structured evidence comment plus exact-SHA `live-gate`, with exact copied-secret checks on the public comment only. No PR-body, ready/draft, reversal, or compensation writes, publisher-code attestation, or OS filesystem-denial claims are permitted. Builders do not publish the gate, queue, or merge. +For Open Pstack PRs, use the shared repository-local `.claude/skills/verify-open-pstack/` skill (Codex: `.agents/skills/verify-open-pstack`). Verifier-changing PRs run from a checkout of their reviewed candidate head, with the publisher revision equal to the candidate SHA; all other PRs run from trusted `main`. Record the publisher revision separately from the candidate SHA. Reuse the operator's existing logins without credential source flags, extra logins, or credential reading/copying. Claude uses normal operator config with session-only `--plugin-dir /plugins/pstack --settings '{"enabledPlugins":{"pstack@open-pstack":false}}'`. Codex installs the exact-head candidate in a run-owned `CODEX_HOME`, with `auth.json` symlinked to the operator's `${CODEX_HOME:-$HOME/.codex}/auth.json`; absent file-based auth fails closed. Removing the run home removes only the symlink, never its target. Setup alone snapshots real `${CLAUDE_CONFIG_DIR:-$HOME/.claude}/pstack-models.md` and `CLAUDE.md`, restores their exact bytes or original absence afterwards, and verifies restoration before any publication; restoration failure publishes nothing. Preserve private mode-0700 raw evidence through merge. Publication is one structured evidence comment plus exact-SHA `live-gate`; no credential parsing, token registration, or exact-secret comment checks. No PR-body, ready/draft, reversal, or compensation writes, publisher-code attestation, or OS filesystem-denial claims are permitted. Builders do not publish the gate, queue, or merge. Do not add an implicit runtime timeout or a weaker-model fallback. From 32c66f4cc2a833814323e4baa98d7c115ac4b45d Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 13:32:05 +0000 Subject: [PATCH 19/20] fix(#90): authenticate Codex children from either harness Link existing Codex authentication into every candidate home so Claude setup and runner exercises can dispatch Codex children. Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .claude/skills/verify-open-pstack/scripts/harness.test.ts | 5 ++++- .claude/skills/verify-open-pstack/scripts/harness.ts | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.claude/skills/verify-open-pstack/scripts/harness.test.ts b/.claude/skills/verify-open-pstack/scripts/harness.test.ts index e934e093..4dcb3b74 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.test.ts @@ -143,6 +143,9 @@ describe('isolated harness boundaries', () => { } if (args[0] === '/usr/bin/script') { expect(candidate).toBe(true); expect(options.interactive).toBe(true); + const linked = join(env!.CODEX_HOME!, 'auth.json'); + expect((await lstat(linked)).isSymbolicLink()).toBe(true); + expect(await readlink(linked)).toBe(auth); await writeFile(args[2]!, 'retained raw native evidence'); throw new Error('fixture native surface reached'); } @@ -208,7 +211,7 @@ describe('isolated harness boundaries', () => { for (const harness of ['claude', 'codex']) { const home = join(root, 'state', harness); expect((await stat(home)).isDirectory()).toBe(true); expect(await readFile(join(home, 'native-state.db'), 'utf8')).toBe('retained candidate state'); - if (harness === 'codex') await expect(stat(join(home, '.codex'))).rejects.toThrow('ENOENT'); + await expect(stat(join(home, '.codex'))).rejects.toThrow('ENOENT'); await expect(stat(join(home, '.claude'))).rejects.toThrow('ENOENT'); if (!failInstall) expect(await readFile(join(home, 'surface.raw'), 'utf8')).toBe('retained raw native evidence'); } diff --git a/.claude/skills/verify-open-pstack/scripts/harness.ts b/.claude/skills/verify-open-pstack/scripts/harness.ts index 7f5a3015..2a4e3634 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.ts @@ -85,9 +85,9 @@ export class MacDriver implements Driver { const candidate = join(workspace, 'plugins/pstack'), expected = await treeHash(candidate, ['skills/poteto-mode/scripts/node_modules']); if (await sourceDigest(workspace) !== pinnedSource) throw new Error('Candidate files differ from trusted pinned source'); let location = candidate; + await linkCodexAuth(home); + this.codexHomes.push(home); if (harness === 'codex') { - await linkCodexAuth(home); - this.codexHomes.push(home); const added = await candidateRun(['codex', 'plugin', 'marketplace', 'add', workspace, '--json'], { env, cwd: home }); await save(join(root, 'codex-marketplace.json'), JSON.parse(added)); const installed = await candidateRun(['codex', 'plugin', 'add', 'pstack@open-pstack', '--json'], { env, cwd: home }); From 7c0318af277eb679edbd51515ee02fd2a2eca34d Mon Sep 17 00:00:00 2001 From: "mastra-platform[bot]" <284800079+mastra-platform[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 14:45:05 +0000 Subject: [PATCH 20/20] fix(#90): snapshot and restore every setup target in both harnesses Co-Authored-By: mastra-platform[bot] <284800079+mastra-platform[bot]@users.noreply.github.com> --- .../scripts/harness.test.ts | 27 ++++++++++++------- .../scripts/isolation.test.ts | 20 ++++++++++++++ .../verify-open-pstack/scripts/isolation.ts | 11 +++++--- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/.claude/skills/verify-open-pstack/scripts/harness.test.ts b/.claude/skills/verify-open-pstack/scripts/harness.test.ts index 4dcb3b74..4242f5b8 100644 --- a/.claude/skills/verify-open-pstack/scripts/harness.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/harness.test.ts @@ -217,35 +217,42 @@ describe('isolated harness boundaries', () => { } expect(JSON.parse(await readFile(join(root, 'doctor.json'), 'utf8')).result).toBe('pass'); }); - test.each([false, true])('setup interruption restores exact bytes or forbids publication (restore failure=%s)', async failRestore => { - const root = await fixture(), home = join(root, 'state/claude'), workspace = join(home, 'workspace'); - const config = join(root, 'daily-claude'); process.env.CLAUDE_CONFIG_DIR = config; + test.each([['claude', false], ['claude', true], ['codex', false], ['codex', true]] as const)('%s setup interruption restores exact bytes or forbids publication (restore failure=%s)', async (harness, failRestore) => { + const root = await fixture(), home = join(root, 'state', harness), workspace = join(home, 'workspace'); + process.env.HOME = root; + const config = join(root, harness === 'claude' ? 'daily-claude' : '.codex'); process.env.CLAUDE_CONFIG_DIR = join(root, 'daily-claude'); + const instructions = harness === 'claude' ? 'CLAUDE.md' : 'AGENTS.md'; for (const path of ['plugins/pstack', '.claude/skills/verify-open-pstack', '.agents/skills']) await mkdir(join(workspace, path), { recursive: true }); await symlink('../../.claude/skills/verify-open-pstack', join(workspace, '.agents/skills/verify-open-pstack')); await writeFile(join(workspace, 'plugins/pstack/SKILL.md'), 'candidate setup'); await mkdir(config); const original = Buffer.from([0, 13, 10, 255, 97]); - await writeFile(join(config, 'CLAUDE.md'), original); + await writeFile(join(config, instructions), original); + if (harness === 'codex') await writeFile(join(config, 'pstack-models.md'), original); const receipt = newReceipt(111, 'a'.repeat(40), 'b'.repeat(40), false, root); receipt.selection = { paths: [], skills: [], features: ['setup'], noRuntime: false }; - receipt.installations = [{ harness: 'claude', home, location: join(workspace, 'plugins/pstack'), sha: receipt.sha, cliVersion: 'test', pluginVersion: 'test', treeHash: await treeHash(join(workspace, 'plugins/pstack')), sourceHash: await sourceDigest(workspace) }]; + receipt.installations = [{ harness, home, location: join(workspace, 'plugins/pstack'), sha: receipt.sha, cliVersion: 'test', pluginVersion: 'test', treeHash: await treeHash(join(workspace, 'plugins/pstack')), sourceHash: await sourceDigest(workspace) }]; const driver = new MacDriver(async args => { expect(args[0]).toBe('/usr/bin/script'); - await writeFile(join(config, 'CLAUDE.md'), 'changed by setup'); + await writeFile(join(config, instructions), 'changed by setup'); await writeFile(join(config, 'pstack-models.md'), 'created by setup'); - if (failRestore) { await rm(join(config, 'CLAUDE.md')); await mkdir(join(config, 'CLAUDE.md')); } + if (harness === 'codex') await writeFile(join(config, 'CLAUDE.md'), 'created by setup'); + if (failRestore) { await rm(join(config, instructions)); await mkdir(join(config, instructions)); } throw new Error('interrupted setup'); }, async () => ''); await expect(driver.exercise(receipt)).rejects.toThrow(failRestore ? 'Setup restoration requires a regular file' : 'interrupted setup'); if (failRestore) { await expect(driver.cleanup()).rejects.toThrow('Setup restoration requires a regular file'); - await rm(join(config, 'CLAUDE.md'), { recursive: true }); + await rm(join(config, instructions), { recursive: true }); await expect(driver.cleanup()).rejects.toThrow('Setup restoration failed; publication forbidden'); return; } await driver.cleanup(); - expect(await readFile(join(config, 'CLAUDE.md'))).toEqual(original); - await expect(stat(join(config, 'pstack-models.md'))).rejects.toThrow('ENOENT'); + expect(await readFile(join(config, instructions))).toEqual(original); + if (harness === 'codex') { + expect(await readFile(join(config, 'pstack-models.md'))).toEqual(original); + await expect(stat(join(config, 'CLAUDE.md'))).rejects.toThrow('ENOENT'); + } else await expect(stat(join(config, 'pstack-models.md'))).rejects.toThrow('ENOENT'); }); test('exercise re-prompts unsafe evidence and publishes accepted answers verbatim', async () => { const root = await fixture(), home = join(root, 'state/claude'), workspace = join(home, 'workspace'); diff --git a/.claude/skills/verify-open-pstack/scripts/isolation.test.ts b/.claude/skills/verify-open-pstack/scripts/isolation.test.ts index 226bf381..1260e270 100644 --- a/.claude/skills/verify-open-pstack/scripts/isolation.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/isolation.test.ts @@ -90,6 +90,26 @@ describe('setup-only snapshot and restoration', () => { expect(await Bun.file(instructions).exists()).toBe(false); }); + test('restores all default and custom setup targets, including absence', async () => { + const { root, claude } = await fixture(); + const directories = [join(root, '.claude'), join(root, '.codex'), claude, process.env.CODEX_HOME!]; + const original = Buffer.from([0, 255, 13, 10, 65]); + for (const directory of directories) { + await mkdir(directory, { recursive: true }); + for (const name of ['pstack-models.md', 'CLAUDE.md']) await writeFile(join(directory, name), original); + } + const snapshot = await snapshotSetup(); + expect(snapshot).toHaveLength(12); + for (const { path } of snapshot) await writeFile(path, 'setup mutation'); + await restoreSetup(snapshot); + for (const directory of directories) { + for (const name of ['pstack-models.md', 'CLAUDE.md']) expect(await readFile(join(directory, name))).toEqual(original); + expect(await Bun.file(join(directory, 'AGENTS.md')).exists()).toBe(false); + } + process.env.CLAUDE_CONFIG_DIR = directories[0]; process.env.CODEX_HOME = directories[1]; + expect(await snapshotSetup()).toHaveLength(6); + }); + test('defaults to HOME/.claude and restores deleted existing files', async () => { const { root } = await fixture(); delete process.env.CLAUDE_CONFIG_DIR; diff --git a/.claude/skills/verify-open-pstack/scripts/isolation.ts b/.claude/skills/verify-open-pstack/scripts/isolation.ts index cb4ef31e..94af84a0 100644 --- a/.claude/skills/verify-open-pstack/scripts/isolation.ts +++ b/.claude/skills/verify-open-pstack/scripts/isolation.ts @@ -49,11 +49,14 @@ async function setupFile(path: string) { } export async function snapshotSetup(): Promise { - const directory = process.env.CLAUDE_CONFIG_DIR ?? join(process.env.HOME!, '.claude'); + const directories = new Set([join(process.env.HOME!, '.claude'), join(process.env.HOME!, '.codex'), + process.env.CLAUDE_CONFIG_DIR, process.env.CODEX_HOME].filter((directory): directory is string => !!directory).map(directory => resolve(directory))); const snapshot: SetupSnapshot = []; - for (const name of ['pstack-models.md', 'CLAUDE.md']) { - const path = join(directory, name), info = await setupFile(path); - snapshot.push(info ? { path, bytes: await readFile(path), mode: info.mode & 0o777 } : { path }); + for (const directory of directories) { + for (const name of ['pstack-models.md', 'CLAUDE.md', 'AGENTS.md']) { + const path = join(directory, name), info = await setupFile(path); + snapshot.push(info ? { path, bytes: await readFile(path), mode: info.mode & 0o777 } : { path }); + } } return snapshot; }