diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9145e25 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,21 @@ +# Generated from operatorstack/intelligence-flow. +name: Verify Boatstack distribution + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - run: python3 -m unittest discover -s tests -v + - run: python3 -m compileall -q boatstack diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml new file mode 100644 index 0000000..94b9f3b --- /dev/null +++ b/.github/workflows/sync-upstream.yml @@ -0,0 +1,69 @@ +# Generated from operatorstack/intelligence-flow. +name: Sync from Intelligence Flow + +on: + schedule: + - cron: "17 */6 * * *" + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +concurrency: + group: sync-intelligence-flow + cancel-in-progress: false + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Check out Boatstack + uses: actions/checkout@v4 + with: + path: boatstack-repo + - name: Check out Intelligence Flow + uses: actions/checkout@v4 + with: + repository: operatorstack/intelligence-flow + ref: main + path: intelligence-flow + - name: Generate projection + id: generate + shell: bash + run: | + source_commit="$(git -C intelligence-flow log -1 --format=%H -- examples/12-product-engineering-loop)" + python3 intelligence-flow/examples/12-product-engineering-loop/scripts/build_boatstack.py \ + --repo boatstack-repo \ + --source-commit "$source_commit" \ + --write + echo "source_commit=$source_commit" >> "$GITHUB_OUTPUT" + - name: Open generated pull request + env: + GH_TOKEN: ${{ github.token }} + SOURCE_COMMIT: ${{ steps.generate.outputs.source_commit }} + shell: bash + run: | + cd boatstack-repo + if [[ -z "$(git status --porcelain)" ]]; then + echo "Boatstack already matches Intelligence Flow." + exit 0 + fi + short="${SOURCE_COMMIT:0:12}" + branch="sync/intelligence-flow-$short" + existing="$(gh pr list --head "$branch" --state open --json url --jq '.[0].url')" + if [[ -n "$existing" ]]; then + echo "Upstream PR already open: $existing" + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git switch -c "$branch" + git add -A + git commit -m "Sync Boatstack from Intelligence Flow $short" + git push --set-upstream origin "$branch" + gh pr create \ + --base main \ + --head "$branch" \ + --title "Sync Boatstack from Intelligence Flow $short" \ + --body "Generated from operatorstack/intelligence-flow@$SOURCE_COMMIT. Review provenance, tests, examples, and context-cost changes before merging." diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..734c757 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.py[cod] +.DS_Store +.venv/ +venv/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..6b2ff4d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,9 @@ + + +# Contributing + +Boatstack is a generated distribution. Propose changes to workflow semantics, templates, evidence rules, or generated presentation in [Intelligence Flow](https://github.com/operatorstack/intelligence-flow/tree/aae685d2513cd25537284e4e68177411ace7ac9a/examples/12-product-engineering-loop). + +The Boatstack repository receives those changes through a generated pull request. Review the PR's `UPSTREAM.json`, tests, adapter diff, and context-size change; do not hand-edit generated output on `main`. + +Repository-specific examples and outcome reports can be proposed upstream as new evidence. A failure becomes a durable move only after its mechanism and non-regression gate are documented. diff --git a/README.md b/README.md index 1911f2c..7d5cabb 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,145 @@ -# boatstack -Plan the route. Prove the work. Ship. + + +# Boatstack + +**Plan the route. Prove the work. Ship.** + +Boatstack is loop engineering for coding agents: a model-neutral path from a product request to an explicitly approved, tested, reviewed pull request. Its behavior is generated from [Intelligence Flow at `aae685d2513cd25537284e4e68177411ace7ac9a`](https://github.com/operatorstack/intelligence-flow/tree/aae685d2513cd25537284e4e68177411ace7ac9a/examples/12-product-engineering-loop). + +It is not a claim that a longer prompt writes better code. Here is what the loop actually does. + +## One request, as executable state + +Start with ordinary product intent: + +```text +Add machine-readable JSON output to the diagram printer while preserving the current text output. +``` + +`/auto-plan` inspects the smallest relevant code boundary and makes contract choices visible: + +```text +Q1 Public API? sibling serializeFlowGraph() | change printFlowGraph() +Q2 Stability? versioned schema | internal object dump +Q3 Run data? compact overlay | entire execution trace +``` + +The accepted answers become observable criteria and tasks—not hidden assumptions: + +```json +{ + "acceptance_criteria": [ + {"id": "AC-1", "text": "Return parseable schema-versioned graph JSON."}, + {"id": "AC-4", "text": "Keep existing ASCII output byte-compatible."} + ], + "tasks": [{ + "id": "T-3", + "acceptance_criteria": ["AC-1", "AC-4"], + "validation": [ + "pnpm exec tsx examples/05-diagram-printer/json-check.ts", + "diff -u expected-output.txt actual-output.txt" + ] + }] +} +``` + +The compiler refuses a criterion with no task or verification. Then `/plan-gate` requires a named human and binds approval to content hashes: + +```bash +python3 boatstack/scripts/compile_plan.py \ + --plan .product-loop/features/diagram-json/plan.json \ + --out-dir .product-loop/features/diagram-json/compiled + +python3 boatstack/scripts/approve_plan.py \ + --spec .product-loop/features/diagram-json/spec.md \ + --plan .product-loop/features/diagram-json/plan.json \ + --tasks .product-loop/features/diagram-json/compiled/tasks.json \ + --approved-by "Boateng Opoku-Yeboah" \ + --output .product-loop/features/diagram-json/plan.lock.json +``` + +Build work checks that lock first: + +```console +$ python3 boatstack/scripts/approve_plan.py ... --check +PASS: approved plan lock matches the current artifacts + +# after plan.json changes +$ python3 boatstack/scripts/approve_plan.py ... --check +BLOCKED: stale or invalid plan lock: plan +``` + +That is the approval boundary in code: conversation cannot silently turn a draft into permission to build. + +See the complete, linked [worked example](examples/diagram-json/README.md). + +## Install into a repository + +```bash +git clone https://github.com/operatorstack/boatstack.git && cd boatstack +cp project.example.json /path/to/product/.boatstack-project.json +# Replace the example paths and commands with facts from the product repository. + +python3 boatstack/scripts/export_repo.py \ + --repo /path/to/product \ + --config /path/to/product/.boatstack-project.json \ + --adapter-name boatstack + +# Review the dry run, then materialize it on a branch. +python3 boatstack/scripts/export_repo.py \ + --repo /path/to/product \ + --config /path/to/product/.boatstack-project.json \ + --adapter-name boatstack \ + --write +``` + +The exporter creates one canonical `.product-loop/` runtime and thin adapters for: + +```text +.cursor/commands/{auto-plan,plan-gate,build,test-gate,review,ship,retro}.md +.cursor/rules/boatstack.mdc +.agents/skills/boatstack/SKILL.md +.claude/skills/boatstack/SKILL.md +.github/PULL_REQUEST_TEMPLATE/boatstack.md +``` + +It refuses to overwrite user-owned host files. Run the same export with `--check` in CI to detect drift. + +## Why “loop engineering” + +A coding model is one operator inside a controlled path: + +```text +intent -> questions -> spec -> plan -> human approval -> build + -> test evidence -> review evidence -> PR -> failure analysis + ^ | + +--- promoted moves ---+ +``` + +- **Optimization:** select the smallest context and ceremony that preserve the required quality and evidence constraints. +- **Control:** represent state explicitly, gate transitions, verify outputs, preserve known-good progress, and feed observed failures into separately tested improvements. +- **Model neutrality:** route on ambiguity, risk, convergence, tool results, and evidence—not model brand, price, or a guessed capability tier. + +The full mapping from equations to files and checks is in [Loop engineering](docs/loop-engineering.md). + +## Evidence, with boundaries + +The rules were informed by a mechanically audited local corpus of **4,016 benchmark trial results** and **3,985 signal streams**, plus two real product-repository studies. For example: + +| Observed failure | Encoded move | +|---|---| +| Restarting discarded partial progress | Preserve known-good state; repair locally | +| Structured-output errors hid useful work | Validate and perform bounded same-step repair | +| Stronger verification wording regressed | Treat self-review as evidence, not the oracle | +| Blind context trimming lost accuracy | Select relevant context without deleting required state | +| A development-slice gain did not transfer | Require representative gates before promoting a move | + +Read the [research and design record](docs/research-and-design.md) and [corpus audit](docs/benchmark-corpus-audit.md). This evidence motivates the loop; it does not prove that every future feature or model will improve. + +## Context has a budget + +The three canonical runtime references currently total approximately **3371 estimated tokens** using `ceil(characters / 4)`. That is a stable compactness signal, not provider billing. Host adapters stay thin and load the operation-specific slice on demand. + +## Status + +Boatstack is an alpha research distribution. It can generate host adapters, compile traceable task/test artifacts, hash-lock explicit approval, detect stale plans, and preserve provenance. The next proof boundary is a paired feature-development evaluation against a plain host harness. diff --git a/UPSTREAM.json b/UPSTREAM.json new file mode 100644 index 0000000..e65b74b --- /dev/null +++ b/UPSTREAM.json @@ -0,0 +1,59 @@ +{ + "canonical_context": { + "characters": 13481, + "estimated_tokens": 3371, + "estimator": "ceil(total characters / 4); compactness signal, not provider billing", + "files": [ + "product-engineering-loop/references/workflow.md", + "product-engineering-loop/references/artifacts.md", + "product-engineering-loop/references/failure-moves.md" + ] + }, + "files": { + ".github/workflows/ci.yml": "9480a65a3a4d24b42f7854566ad4a55100b7f2c2b25bffc6bb6b368ba0848104", + ".github/workflows/sync-upstream.yml": "f8c84e316e296ac5928bc0222cef64848ea0fda6846a42f540b0747e8c4eb5f7", + ".gitignore": "94fa252979321511b0ce5fa598f71905f6dc29c5bef6660ff4498a5a39c167ba", + "CONTRIBUTING.md": "217090f78f44da12890d7ad4aa555f1233c64bfe2119877d497e7c13fb4b3f91", + "README.md": "adc08d8b50f731f5f9336bc4eaca3ce0a4ebb64c75e94aa6e6f0dabfe26e028c", + "boatstack/SKILL.md": "335731973cea2c5d0eb67b9d3870cc332490d1188e3943bb185ab78ab4b4b886", + "boatstack/agents/openai.yaml": "8429c65868025e798d345cc2a9bd78f2bc3280ee982f8f7874504395d6d68368", + "boatstack/assets/templates/adr.md": "c577a3c1c1319061f61deb053597e6e853657022185fe28b8f733327e2a78565", + "boatstack/assets/templates/evidence.md": "12dac552bc5373ab443367d5797f41988f14284bcf46d16dfd72015cfddf9ad1", + "boatstack/assets/templates/feature-spec.md": "c7e007cc4295ed4c599642c0587021ef978e729cf0946f6bf3a6c4f01d366ad4", + "boatstack/assets/templates/gaps.md": "911cc2f086104d35071b952950c2ec44258641419f10b2355c594f33eb492cbe", + "boatstack/assets/templates/move.md": "91bfd9a9b9426ac023eb88fd19f4f638190481c1855f1239acc73830528e50f0", + "boatstack/assets/templates/plan-lock.json": "3e44dea05419cf198ee8112e9b9fdff92287edc2480a03fca026560fe929d468", + "boatstack/assets/templates/plan.json": "803907480dd150da36337f3ecf46e3617f3e26ace9be282e540032983cb77e86", + "boatstack/assets/templates/questions.md": "86c9bcf51172fe222b7b28bffccaf3da3b1ea0633c7a2348272fdbbd8eea6740", + "boatstack/assets/templates/test-plan.md": "6db8a9f27dd171fb80222a501cae50eb051e7278c04703fa43b5ff86dd4d2df4", + "boatstack/references/artifacts.md": "caaa7337674bf707a53f0854c7d95e58333ac566f2bc67f0d77533230796221a", + "boatstack/references/failure-moves.md": "2d7d3988c70718e9cc02104f9899a00208173e2f654d1046edd22079f4d46f41", + "boatstack/references/portability.md": "5490a045526c4cd6fd52bcddeb0039119208478fb17656cd2f6b3d5f71698ce6", + "boatstack/references/workflow.md": "2c2343b3ef3dd7684dc6a027da4c8e5b1ce927cf3979aca32abea0d0d2028ee7", + "boatstack/scripts/approve_plan.py": "92cb14cf0703bd25d053f8939575ab651a0274418d9ec85f3827f4c23c30001c", + "boatstack/scripts/compile_plan.py": "523befa52993f5606a5ed7a91678459254ba6cfa074e9aafa9f4010876332968", + "boatstack/scripts/export_repo.py": "42444369b2b4d8430b5347761626ac7725aae25aa4726b5bdca5325a4fc80ad0", + "docs/benchmark-corpus-audit.md": "f2d206fe8579a514f9da82b2c96c19b343ac004be67617e1bd34f0f8e0e5e6c6", + "docs/benchmark-submission-audit.md": "9518abdd17690729c6423f87cab20418ed47b0915b5faa44b9ef975e9e9c3b79", + "docs/loop-engineering.md": "2cddb0aa963f15149c69102a8215401d32848d228acbf55c3df20db27b654a34", + "docs/research-and-design.md": "543836387090f8dc01381b1e46d1c6760bcbf8119b4004ece8d6f6f68d08db4f", + "examples/diagram-json/README.md": "51871b16438cbef2bbdf5077dda0b5b06e77cbe76882d34e4a05c17c8f13a2b3", + "examples/diagram-json/compiled/evidence.md": "1ba1c989ade070a8ef9a508fbd788d100d7292f2dbacbb2bce895468019f619d", + "examples/diagram-json/compiled/tasks.json": "d66d693df1ba7dd34f65ce93afea54006563c14d642a1bf0d1d9311b3fcfb37b", + "examples/diagram-json/compiled/test-matrix.json": "0497cf73f84515cfc493e4904eda4c2be6c0621fc0a11a3b1349803b5acf91cb", + "examples/diagram-json/plan.json": "d1208003042a9d10f5efb010fc32fc7ac7bdefa427938260586e90daa0cb4414", + "examples/diagram-json/plan.lock.json": "e6f5a6a7d3898eacaeacb1f38d84f34b6c7bee25b1d1eb79cb7ca3157cb897ea", + "examples/diagram-json/questions.md": "1a0050041cac0a8d53e6ebfe04cbec4a298cdc8c50efeeb6fa15aeb663c5ec76", + "examples/diagram-json/request.md": "0808fc41c36779c404f4a3a121167da6e76cac56df526e70f9ed6d3e0d4c02ed", + "examples/diagram-json/spec.md": "a943c81cf2a88d23d5b300e6b9dc1dafc80923a9b6b9ab5297a67b4e2054b9d5", + "project.example.json": "2054228f4c824d43385b7732e9f38f17739900d3cec6567bc23c5fe6c890d1be", + "tests/test_boatstack.py": "9d61e552a196b9c9fba8bd175b7c3ae1fcfead1f61eb6396df065477086369e1" + }, + "generator": "operatorstack/intelligence-flow:boatstack-distribution", + "schema_version": 1, + "source": { + "commit": "aae685d2513cd25537284e4e68177411ace7ac9a", + "path": "examples/12-product-engineering-loop", + "repository": "operatorstack/intelligence-flow" + } +} diff --git a/boatstack/SKILL.md b/boatstack/SKILL.md new file mode 100644 index 0000000..12b8d23 --- /dev/null +++ b/boatstack/SKILL.md @@ -0,0 +1,153 @@ +--- +name: boatstack +description: Turn a product request into a question-led, specification-first implementation with test, review, and ship gates, then learn from the evidence without silently changing project rules. Use when planning or building a feature, creating an implementation PR, reviewing work against product intent, diagnosing repeated coding-agent failures, or exporting the same engineering loop to Cursor, Claude Code, Codex, and GitHub. +--- + +# Boatstack + +Build the smallest complete product slice that can be independently verified. Keep the workflow model-neutral: project facts and gate evidence are canonical; host-specific prompts are adapters. + +## Start by selecting the operation + +Map the request to one operation: + +- `init`: inspect a repository and create or update `.product-loop/project.json`. +- `auto-plan`: turn product intent into a reviewable draft feature package. +- `plan-gate`: present the draft for explicit human acceptance, then freeze its approved contents and generate the executable package. +- `build`: implement approved tasks in bounded, reversible slices. +- `test-gate`: test requirements and relevant regressions using independent evidence. +- `review-gate`: review the diff against the spec, project invariants, risks, and known gaps. +- `ship-gate`: prepare a reviewable PR with evidence, rollback notes, and explicit gaps. +- `retro`: classify failures, propose a harness move, and gate it before promotion. +- `export`: generate thin Cursor, Claude Code, Codex, and GitHub adapters. + +For the full state machine, read [workflow.md](references/workflow.md). For artifact meanings and templates, read [artifacts.md](references/artifacts.md). + +## Bound the outcome + +For ordinary feature work, define one bounded outcome: + +1. one product domain; +2. one input/output contract; +3. one user-visible goal; +4. one next operator; +5. one verification boundary. + +Because this workflow is also a reusable product, maintain delivery and improvement as separate paths: + +- **Delivery path:** intent -> questions -> spec -> plan -> code -> gates -> PR. +- **Improvement path:** traces -> failure classification -> proposed move -> paired evaluation -> promote/reject. + +Never mix benchmark observations or speculative harness changes into the delivery path during an active feature. The improvement path may propose an experiment; only a passed promotion gate changes the canonical loop. + +## Initialize from repository evidence + +Inspect only the minimal relevant code and documentation. Look for: + +- `AGENTS.md`, `CLAUDE.md`, `.cursor/rules`, constitutions, architecture docs, ADRs, prior feature specs, and open gap ledgers; +- entry points, schemas, public interfaces, decision-making functions, validators, tests, CI, deployment, and rollback paths; +- recent PRs touching the same domain; +- commands that actually build, lint, type-check, and test the affected slice. + +Do not scan the entire repository by default. Record discovered paths and commands in `.product-loop/project.json`; preserve existing host configuration rather than replacing it. + +## Run `auto-plan` + +1. Write the bounded outcome definition before proposing architecture. +2. Separate facts, decisions, unknowns, and safely deferrable gaps. +3. Answer discoverable code questions by inspection. +4. Ask the developer only questions whose answers materially change behavior, contracts, risk, or acceptance. Ask 1-3 concise questions at a time, give 2-3 mutually exclusive choices, recommend one, and explain the impact. +5. Record answers and provenance in the question ledger. +6. Create the feature spec: problem, users, outcomes, non-goals, acceptance criteria, invariants, interfaces, failure behavior, observability, rollout, and rollback. +7. Run product, design, engineering, and developer-experience reviews only when applicable. If gstack is installed, its review skills can implement these lenses; do not require it. +8. If Spec Kit is installed, use its constitution/specify/clarify/plan/tasks/analyze/checklist flow as an artifact generator. The canonical artifact contract remains authoritative. +9. End with a **draft**, never an implied approval. Do not generate executable task state or start implementation from `auto-plan` alone. + +Do not treat an ADR as general project context. ADRs record accepted durable decisions. Use a question ledger for unknowns and a gap ledger for known divergence. + +## Run `plan-gate` + +1. Present the draft spec, plan, open decisions, accepted assumptions, gaps, risks, and proposed verification in a reviewable form. +2. Ask the developer to approve it or request changes. Silence and continued conversation are not approval. +3. On changes, return to `auto-plan`, preserve the feedback in the question/decision ledger, and issue a new draft. +4. On explicit approval, deterministically compile the already-approved structured plan into the task graph, requirement-test traceability rows, evidence skeleton, and expected gate commands. Do not add semantics during compilation. +5. Calculate content hashes and write `plan.lock.json` with the approver, timestamp, source commit, spec hash, plan hash, and task-graph hash. +6. If the spec or plan changes later, invalidate the lock and return to this gate. + +`build` must refuse to run when the plan lock is absent, stale, or does not match the approved artifacts. + +The reference implementation performs the post-approval materialization and lock in this order: + +```bash +python3 .product-loop/tools/compile_plan.py \ + --plan .product-loop/features//plan.json \ + --out-dir .product-loop/features//compiled + +python3 .product-loop/tools/approve_plan.py \ + --spec .product-loop/features//spec.md \ + --plan .product-loop/features//plan.json \ + --tasks .product-loop/features//compiled/tasks.json \ + --approved-by "" \ + --output .product-loop/features//plan.lock.json +``` + +The first command validates and compiles already-approved semantics; it must not invent new tasks or acceptance criteria. + +## Build without erasing evidence + +- Work from approved tasks and acceptance criteria. +- Preserve the last known-good state; repair locally instead of restarting a near-correct implementation. +- Re-scope context at task boundaries. Include relevant source, interfaces, invariants, and tests—not arbitrary history. +- Stop and ask when implementation exposes a new product decision or a high-impact irreversible choice. +- Log deviations from the plan. Update the spec when product intent changes; add an ADR only when a durable architectural decision changes. +- Do not repeat the same failed tactic more than twice without re-diagnosing the failure class. + +Do not branch the workflow on model brand, price, or a guessed capability tier. Branch only on observable work state: unresolved ambiguity, risk, convergence, repeated tactics, tool results, test fidelity, and gate evidence. A repository may choose any implementation model; the contract and gates stay the same. + +## Enforce the gates + +### Test gate + +- Derive tests from acceptance criteria and affected contracts, not only from the implementation. +- Run existing relevant tests plus targeted new tests, linters, type checks, builds, and runtime checks. +- Treat model-authored tests and same-model self-review as evidence, not ground truth. +- Validate that tests load and exercise the intended interface. For high-risk code, add an independent oracle such as contract fixtures, mutation testing, differential checks, staging verification, or human acceptance. +- A failing check blocks the gate. A skipped check must include a reason and risk owner. + +### Review gate + +- Review the actual diff, not the intended plan alone. +- Check spec traceability, invariants, data/security/tenancy boundaries, failure behavior, backward compatibility, migrations, observability, tests, docs, and gaps. +- Use an independent reviewer for high-risk changes, repeated failures, or when the existing review evidence is circular. +- Convert actionable findings into tasks. Do not pass while critical findings are open. + +### Ship gate + +- Require a clean, intentional diff; passing required checks; a filled evidence ledger; explicit known gaps; and rollout/rollback notes. +- Create a PR, but keep merge and deploy as separate authorized actions. +- Never hide failed experiments, skipped checks, or `PASS_WITH_GAPS` behind a green summary. + +Gate statuses are `PASS`, `PASS_WITH_GAPS`, and `BLOCKED`. Critical safety, correctness, or product-acceptance gaps always produce `BLOCKED`. + +## Learn without overfitting + +Read [failure-moves.md](references/failure-moves.md) before proposing a loop change. + +1. Classify the observed failure below the surface symptom. +2. State a mechanism and the exact failure population the move targets. +3. Estimate cost, risk, and possible regressions. +4. Run a cheap smoke test, then a paired representative evaluation. +5. Keep a holdout or independent acceptance boundary. +6. Promote only a clear non-regressing result; otherwise record `REJECT` or `WASH`. + +More steps, more context, stronger wording, more tests, or more retries are not improvements by themselves. Preserve negative results in the move ledger. + +## Export host adapters + +Read [portability.md](references/portability.md), then use: + +```bash +python3 boatstack/scripts/export_repo.py --adapter-name boatstack --repo /path/to/repo --config /path/to/project.json --write +``` + +Run with `--check` in CI to detect drift. The exporter writes generated files only and refuses to overwrite user-owned files. Review the generated diff in a branch and ship it through a PR. diff --git a/boatstack/agents/openai.yaml b/boatstack/agents/openai.yaml new file mode 100644 index 0000000..c14214d --- /dev/null +++ b/boatstack/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Boatstack" + short_description: "Plan, build, verify, review, and ship with evidence." + default_prompt: "Use $boatstack to turn this product request into a question-led, evidence-gated implementation loop." diff --git a/boatstack/assets/templates/adr.md b/boatstack/assets/templates/adr.md new file mode 100644 index 0000000..8391efb --- /dev/null +++ b/boatstack/assets/templates/adr.md @@ -0,0 +1,17 @@ +# ADR : + +- Status: proposed +- Date: +- Supersedes: + +## Context and forces + +## Decision + +## Alternatives considered + +## Consequences and risks + +## Verification + +## Supersession rule diff --git a/boatstack/assets/templates/evidence.md b/boatstack/assets/templates/evidence.md new file mode 100644 index 0000000..b5c2308 --- /dev/null +++ b/boatstack/assets/templates/evidence.md @@ -0,0 +1,24 @@ +# Evidence ledger: + +- Canonical loop version: +- Config hash: +- Approved plan hash: +- Plan approver: +- Implementation commit: +- Gate status: `BLOCKED` + +## Acceptance evidence + +| Acceptance criterion | Evidence | Result | +|---|---|---| + +## Commands and checks + +| Check | Command | Result | Evidence/link | +|---|---|---|---| + +## Review findings + +## Known gaps + +## Rollout and rollback diff --git a/boatstack/assets/templates/feature-spec.md b/boatstack/assets/templates/feature-spec.md new file mode 100644 index 0000000..5078443 --- /dev/null +++ b/boatstack/assets/templates/feature-spec.md @@ -0,0 +1,31 @@ +# Feature spec: + +## Outcome boundary + +- Domain: +- Actor: +- Input: +- Output: +- User-visible goal: +- Next operator: +- Verification boundary: + +## Problem and outcome + +## Non-goals + +## Scenarios + +## Acceptance criteria + +## Interfaces and data + +## Invariants and trust boundaries + +## Failure and recovery behavior + +## Observability + +## Rollout and rollback + +## Linked questions, ADRs, and gaps diff --git a/boatstack/assets/templates/gaps.md b/boatstack/assets/templates/gaps.md new file mode 100644 index 0000000..a8ed5a7 --- /dev/null +++ b/boatstack/assets/templates/gaps.md @@ -0,0 +1,4 @@ +# Gap ledger: + +| ID | Expected | Actual | Impact/severity | Why deferred | Owner | Revisit trigger | Acceptance criteria | Ship blocking? | +|---|---|---|---|---|---|---|---|---| diff --git a/boatstack/assets/templates/move.md b/boatstack/assets/templates/move.md new file mode 100644 index 0000000..7a42960 --- /dev/null +++ b/boatstack/assets/templates/move.md @@ -0,0 +1,18 @@ +# Loop move: + +- Target failure: +- Population: +- Mechanism: +- Minimal change: +- Expected effect: +- Cost: +- Regression risks: +- Smoke test: +- Paired evaluation: +- Holdout: +- Rollback/default: +- Decision: `PROPOSED` + +## Evidence + +## Result and interpretation diff --git a/boatstack/assets/templates/plan-lock.json b/boatstack/assets/templates/plan-lock.json new file mode 100644 index 0000000..7b99ae5 --- /dev/null +++ b/boatstack/assets/templates/plan-lock.json @@ -0,0 +1,12 @@ +{ + "schema_version": 1, + "status": "APPROVED", + "approved_by": "", + "approved_at": "", + "source_commit": "", + "spec_sha256": "", + "plan_sha256": "", + "task_graph_sha256": "", + "invalidated_at": null, + "invalidation_reason": null +} diff --git a/boatstack/assets/templates/plan.json b/boatstack/assets/templates/plan.json new file mode 100644 index 0000000..652463d --- /dev/null +++ b/boatstack/assets/templates/plan.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "feature_id": "", + "spec_path": "", + "acceptance_criteria": [ + { + "id": "AC-1", + "text": "" + } + ], + "tasks": [ + { + "id": "T-1", + "title": "", + "depends_on": [], + "acceptance_criteria": ["AC-1"], + "validation": [""], + "rollback_boundary": "" + } + ] +} diff --git a/boatstack/assets/templates/questions.md b/boatstack/assets/templates/questions.md new file mode 100644 index 0000000..03355d4 --- /dev/null +++ b/boatstack/assets/templates/questions.md @@ -0,0 +1,4 @@ +# Question ledger: + +| ID | Question | Why it matters | Options | Recommendation | Answer | Source | Status/expiry | +|---|---|---|---|---|---|---|---| diff --git a/boatstack/assets/templates/test-plan.md b/boatstack/assets/templates/test-plan.md new file mode 100644 index 0000000..86db3a4 --- /dev/null +++ b/boatstack/assets/templates/test-plan.md @@ -0,0 +1,10 @@ +# Test plan: + +| Requirement/risk | Check | Oracle/source | Command or procedure | Result/evidence | Owner | +|---|---|---|---|---|---| + +## Required regression checks + +## Independent verification + +## Skipped checks and accepted risk diff --git a/boatstack/references/artifacts.md b/boatstack/references/artifacts.md new file mode 100644 index 0000000..b03bcd1 --- /dev/null +++ b/boatstack/references/artifacts.md @@ -0,0 +1,62 @@ +# Artifact contract + +Artifacts separate facts, decisions, unknowns, incompleteness, and evidence. Combining them into one context document makes stale assumptions difficult to detect. + +| Artifact | Purpose | Create or update when | +|---|---|---| +| Project constitution | Stable principles and non-negotiable invariants | A rule should govern most future work | +| Repository map | Minimal entry points, interfaces, commands, and verification boundaries | The relevant architecture or tooling changes | +| Feature brief/spec | Product intent, outcomes, scenarios, acceptance criteria, non-goals | A product slice is proposed or its intent changes | +| Question ledger | Unknowns, choices, human answers, provenance, expiry | The repo cannot answer a material question | +| ADR | Accepted durable architecture decision and rationale | A meaningful architecture choice is accepted | +| Plan/tasks | Dependency-ordered implementation operations and checks | A spec is resolved enough to build | +| Test plan | Requirement-to-evidence mapping and oracle independence | Planning and after discovered failure modes | +| Gap ledger | Known divergence between desired and current state | Work is deferred, partial, incompatible, or intentionally absent | +| Risk/threat note | Assets, actors, trust boundaries, abuse/failure paths | Security, data, tenancy, billing, auth, or destructive paths change | +| Runbook | Deploy, observe, recover, and roll back | Operational behavior changes | +| Evidence ledger | Commands, results, review evidence, screenshots, CI and runtime links | Every gate | +| Move ledger | Failure class, intervention, prediction, paired result, decision | Improving the loop itself | + +## ADR boundary + +An ADR is not a dump of all project context. It records one durable decision: + +- status: proposed, accepted, superseded, or rejected; +- context and forces; +- decision; +- alternatives; +- consequences and risks; +- verification and supersession rule. + +Unknowns stay in the question ledger. Known incomplete work stays in the gap ledger. Temporary implementation detail stays in the plan or PR. + +## Gap boundary + +A gap is an explicit difference between the accepted target and the current implementation. Record: + +- expected state and actual state; +- impact and severity; +- reason it remains; +- owner; +- trigger or deadline for revisiting; +- affected acceptance criteria; +- whether it blocks ship. + +`PASS_WITH_GAPS` is allowed only if project policy permits it and no gap is critical. + +## Provenance + +Every material statement should indicate whether it came from: + +- repository evidence; +- runtime evidence; +- a human answer; +- an accepted ADR; +- an assumption; +- an external source. + +Generated artifacts include the canonical loop version and config hash. Human edits to generated adapters are drift and should be moved into project-owned context or canonical source. + +## Templates + +Copy only the templates required for the current slice from `assets/templates/`. Do not create empty ceremony. The feature spec, question ledger, test plan, gap ledger, and evidence ledger are the usual minimum for material product work. diff --git a/boatstack/references/failure-moves.md b/boatstack/references/failure-moves.md new file mode 100644 index 0000000..c925ac0 --- /dev/null +++ b/boatstack/references/failure-moves.md @@ -0,0 +1,50 @@ +# Failure taxonomy and move catalog + +Select a move only after locating the failure below its surface symptom. “Timed out,” “tests failed,” and “the agent got confused” are starting observations, not diagnoses. + +| Failure class | Evidence | Candidate moves | Main regression risk | +|---|---|---|---| +| Unknown requirement | Plausible implementations disagree on product behavior | Ask a targeted human question; record answer and expiry | Invented requirements or stalled delivery | +| Context miss | Relevant interface/invariant existed but was not loaded | Reload minimal relevant context; add routing reference | Blind truncation removes useful state | +| Protocol malformed | Invalid JSON/schema/tool call despite recoverable intent | Parse repair; schema validation; constrained retry | Retrying semantic errors as syntax | +| Tool/transport | API, shell, network, or environment failure | Classify retryability; bounded retry; fallback; resume | Duplicate side effects or retry storms | +| Step/budget exhaustion | Progress is still converging at cap | Continue from checkpoint; conditional budget increase | More time converts timeout into wrong answer or thrash | +| Thrashing | Repeated actions without new evidence | Stop after repeated tactic; re-diagnose; stronger planner | Spending more tokens on the same loop | +| Implementation correctness | Independent tests fail the contract | Local repair from failing evidence; narrower task | Rebuilding and losing near-correct work | +| Test fidelity | Tests pass wrong code or reject correct code | Contract fixtures; collect/load gate; mutation/differential/human oracle | Treating more model-authored tests as truth | +| Review miss | Defect found after same-agent review | Independent reviewer; risk checklist; mechanical enforcement | Expensive review everywhere | +| Scope drift | Diff no longer maps to approved outcomes | Re-scope; split PR; update spec with approval | Hiding product changes in implementation | +| Security/tenancy | Trust boundary or data scope violated | Specialist review; invariant test; deny-by-default guard | Generic prompt mistaken for enforcement | +| Integration/deploy | Local pass but runtime fails | Environment parity; canary; health checks; rollback | Treating staging as identical to production | +| Documentation drift | Durable behavior and docs disagree | Update source-of-truth artifact; drift check | Growing instructions with unverified rules | + +## Lessons encoded from the benchmark campaign + +- **Parse repair is a protocol move.** It can recover malformed completion without pretending to improve reasoning. +- **More steps are conditional.** Qwen experiments reduced step exhaustion but largely converted it into confident wrong answers. Increase budget only when trajectories show continuing progress. +- **Strict self-checking is not monotonic.** A stricter prompt caused collateral rework and regression. Preserve a known-good snapshot and require an oracle with fidelity to the real goal. +- **Self-authored tests are scaffolding before they are truth.** Spec-first helped a development slice but its frozen oracle agreed poorly with the hidden grader and did not transfer to the full board. +- **Development promotion is not product promotion.** A +7 point development result became a statistical wash on the full distribution. Representative evaluation and holdout remain mandatory. +- **Do not discard near-correct work.** Repair attempts can wash or regress, so retain prior evidence and compare states. +- **Model changes relocate the bottleneck.** The same harness exposed different binding modes on Gemini and Qwen. Route moves by measured failure population, not by a universal “best loop.” + +## Move proposal schema + +Before experimenting, record: + +```yaml +id: stable-move-name +target_failure: one-class +population: observable predicate selecting affected runs +mechanism: why this intervention should change the outcome +change: one minimal behavioral delta +expected_effect: directional metric prediction +cost: latency, tokens, money, and human attention +risks: plausible regressions and affected populations +smoke: cheapest mechanism check +evaluation: paired sample, representative distribution, holdout +rollback: identity/default behavior +decision: PROPOSED | PROMOTE | REJECT | WASH +``` + +Never promote from an unpaired anecdote, a mid-run aggregate with mismatched coverage, or a metric produced solely by the model being evaluated. diff --git a/boatstack/references/portability.md b/boatstack/references/portability.md new file mode 100644 index 0000000..1cfee27 --- /dev/null +++ b/boatstack/references/portability.md @@ -0,0 +1,65 @@ +# Harness-neutral portability + +## Canonical package and adapters + +The source of truth is `.product-loop/`: + +- `project.json`: repo-specific facts and policy; +- `workflow.md`: state machine and gate semantics; +- `artifacts.md`: document contract; +- `failure-moves.md`: failure taxonomy and experimental rules; +- `templates/`: artifact templates; +- `generated.lock.json`: generator version, config hash, and generated file list. + +Host-specific files are compiled adapters: + +- Cursor: `.cursor/rules/product-engineering-loop.mdc` and `.cursor/commands/*.md`; +- Claude Code: `.claude/skills/product-engineering-loop/SKILL.md`; +- Codex: `.agents/skills/product-engineering-loop/SKILL.md`; +- GitHub: `.github/PULL_REQUEST_TEMPLATE/product-engineering-loop.md`. + +Adapters point to the canonical package; they do not copy its full reasoning. This keeps behavior consistent while letting each host expose its native invocation surface. + +## Repository ownership + +The exporter must not replace: + +- `AGENTS.md`; +- `CLAUDE.md`; +- existing Cursor rules or commands; +- CI or PR templates with the same path; +- any file without the generated marker. + +If a collision exists, stop and show the conflict. A human may move durable content into `.product-loop/project.json`, choose another adapter path, or explicitly reconcile it in a PR. + +## Export PR contract + +An installation or update PR should show: + +- canonical loop version and config hash; +- host adapters added or changed; +- project context paths and real verification commands; +- existing instructions left untouched; +- collisions or unsupported host features; +- dry-run/check output; +- rollout and removal steps. + +Generated output is reviewable code. Do not auto-merge it simply because generation succeeded. + +## Host notes + +### Cursor + +Use project rules in `.cursor/rules/*.mdc`; `.cursorrules` is legacy. Use `.cursor/commands/*.md` for the named workflow commands. Keep the rule short and point it to `.product-loop/` artifacts. Cursor CLI also reads `AGENTS.md` and `CLAUDE.md`, so avoid duplicating those files into the generated rule. + +### Claude Code + +Use a project skill under `.claude/skills/`. Keep `CLAUDE.md` as project-owned durable context. If using the Agent SDK in automation, explicitly enable project setting sources when repository instructions are required; do not assume the SDK loads filesystem settings by default. + +### Codex + +Use a repo skill under `.agents/skills/`. Keep `AGENTS.md` concise for persistent repo conventions and route task-specific workflow detail into the skill and `.product-loop/` references. + +### GitHub + +The generated PR template collects evidence; branch protection and CI remain the enforcement layer. A future exporter can generate opt-in CI, but it must use commands from `project.json` and never invent repository checks. diff --git a/boatstack/references/workflow.md b/boatstack/references/workflow.md new file mode 100644 index 0000000..eb95c0d --- /dev/null +++ b/boatstack/references/workflow.md @@ -0,0 +1,167 @@ +# Canonical workflow + +## State machine + +```text +INTENT + -> PROJECT + -> QUESTIONS + -> SPEC + -> PLAN + -> PLAN_GATE + -> PLAN_LOCKED + -> BUILD + -> TEST_GATE + -> REVIEW_GATE + -> SHIP_GATE + -> PR_OPEN + -> RETRO +``` + +Each transition emits an artifact and evidence. A host adapter may change how a command is invoked, but it must not skip a transition or redefine a gate. + +## State contracts + +### `INTENT -> PROJECT` + +Define the request as: + +- domain; +- affected actor; +- input and output; +- user-visible outcome; +- next operator; +- verification boundary. + +Reject a scope definition that combines unrelated domains or cannot name an observable outcome. + +### `PROJECT -> QUESTIONS` + +Inspect the minimal code paths and durable project context. Classify every uncertainty: + +- **discoverable fact:** answer through repository or runtime inspection; +- **product decision:** ask the developer or stakeholder; +- **technical decision:** propose options and record the accepted rationale; +- **deferrable gap:** record it with impact and trigger; +- **irrelevant:** exclude it from the slice. + +Questions are required when different answers change an external contract, data model, safety boundary, user experience, acceptance criterion, or irreversible implementation choice. + +### `QUESTIONS -> SPEC` + +The spec must contain: + +- problem and target user; +- desired outcome and metrics; +- non-goals; +- user stories or scenarios; +- acceptance criteria; +- current and proposed interfaces; +- invariants and trust boundaries; +- failure, empty, loading, and recovery behavior; +- observability; +- migration, rollout, and rollback; +- linked questions, ADRs, and gaps. + +Do not encode guessed answers as facts. Mark a reversible assumption and give it an expiry trigger. + +### `SPEC -> PLAN` + +Create tasks in dependency order. Each task names: + +- files or components likely affected; +- contract or acceptance criteria served; +- validation command or evidence; +- rollback boundary; +- unknowns that would stop implementation. + +Run only relevant review lenses: + +- product/taste: value, scope, user journey, non-goals; +- design: states, accessibility, responsive behavior, content; +- engineering: boundaries, data flow, state, failure modes, security, migrations; +- developer experience: APIs, naming, discoverability, operability. + +If gstack is installed, its review skills can execute these lenses. If Spec Kit is installed, it can generate and cross-check the spec, plan, tasks, and checklists. Their output is normalized into this artifact contract. + +### `PLAN -> PLAN_GATE` + +Present the full draft and require an explicit human `approve` or a change request. Do not interpret silence, a new implementation question, or a tool permission as plan approval. + +### `PLAN_GATE -> PLAN_LOCKED` + +After approval, deterministically: + +1. hash the approved spec and plan; +2. compile the approved structured plan into the task graph, requirement-test traceability rows, evidence skeleton, and expected gate commands without adding semantics; +3. record approver, timestamp, source commit, and all artifact hashes in `plan.lock.json`; +4. verify every task maps to at least one acceptance criterion or declared enabling dependency. + +Any later change to the approved spec or plan invalidates the lock and returns the feature to `PLAN_GATE`. + +### `PLAN_LOCKED -> BUILD` + +Implement one coherent task slice at a time. After each slice: + +1. run the cheapest relevant check; +2. compare the diff to the task contract; +3. preserve the known-good state; +4. record deviations or new unknowns; +5. continue, ask, or re-plan explicitly. + +### `BUILD -> TEST_GATE` + +Create requirement-to-evidence traceability. Use this evidence ladder: + +1. syntax, schema, and load/collect checks; +2. unit and contract tests; +3. integration and end-to-end tests; +4. differential, property, or mutation checks where useful; +5. staging/runtime verification; +6. human acceptance for product behavior. + +The riskier the slice, the less acceptable same-model, self-authored tests are as the only oracle. + +### `TEST_GATE -> REVIEW_GATE` + +Review only after required mechanical checks pass, unless reviewing a failure is the goal. The reviewer inspects the actual diff and reports findings by severity with file/line evidence, consequence, and correction. + +### `REVIEW_GATE -> SHIP_GATE` + +Require: + +- all critical findings resolved; +- acceptance criteria traced to evidence; +- required commands passed; +- docs and durable decisions updated; +- gaps explicit; +- deployment and rollback understood; +- secrets and unintended artifacts excluded. + +### `SHIP_GATE -> PR_OPEN` + +Create a PR with the feature spec, decision links, test evidence, review findings, gaps, rollout, and rollback. Opening a PR does not authorize merge or deployment. + +### `PR_OPEN -> RETRO` + +Record unexpected friction and outcomes. A retro may propose a loop move, but it may not mutate durable instructions automatically. + +## Gate semantics + +- `PASS`: required evidence is present; no gate-blocking gap remains. +- `PASS_WITH_GAPS`: no critical gap remains; each accepted gap has impact, owner, and trigger. +- `BLOCKED`: required evidence failed or a critical unknown/gap remains. + +## State routing + +The workflow never branches on model provider, model name, price, or presumed capability. Route only from observed state: + +- unresolved product choice -> ask the human; +- undiscovered code fact -> inspect the minimal relevant slice; +- high-risk boundary -> require independent evidence and the configured reviewer; +- repeated tactic without new evidence -> stop and re-diagnose; +- converging work at a budget boundary -> resume from checkpoint if policy permits; +- weak or circular oracle -> add an independent verification source; +- changed approved intent -> invalidate the plan lock and return to `PLAN_GATE`. + +The same state contract applies whether the repository uses a local model, a cheap API model, or a frontier model. diff --git a/boatstack/scripts/approve_plan.py b/boatstack/scripts/approve_plan.py new file mode 100644 index 0000000..6bfcd08 --- /dev/null +++ b/boatstack/scripts/approve_plan.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Create or verify a human-approved, hash-addressed plan lock.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +from datetime import datetime, timezone +from pathlib import Path + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def git_commit(cwd: Path) -> str: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=cwd, text=True, capture_output=True + ) + return result.stdout.strip() if result.returncode == 0 else "unknown" + + +def expected(args: argparse.Namespace) -> dict[str, object]: + approved_at = args.approved_at + if not approved_at: + approved_at = datetime.now(timezone.utc).replace(microsecond=0).isoformat() + return { + "schema_version": 1, + "status": "APPROVED", + "approved_by": args.approved_by, + "approved_at": approved_at, + "source_commit": args.source_commit or git_commit(args.spec.parent), + "spec_path": str(args.spec), + "spec_sha256": sha256(args.spec), + "plan_path": str(args.plan), + "plan_sha256": sha256(args.plan), + "task_graph_path": str(args.tasks), + "task_graph_sha256": sha256(args.tasks), + "invalidated_at": None, + "invalidation_reason": None, + } + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Create a plan lock only after a human explicitly approves the draft." + ) + parser.add_argument("--spec", type=Path, required=True) + parser.add_argument("--plan", type=Path, required=True) + parser.add_argument("--tasks", type=Path, required=True) + parser.add_argument("--approved-by") + parser.add_argument("--approved-at") + parser.add_argument("--source-commit") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + + for path in [args.spec, args.plan, args.tasks]: + if path is not None and not path.is_file(): + parser.error(f"required approved artifact does not exist: {path}") + + if args.check: + if not args.output.is_file(): + print(f"BLOCKED: plan lock is missing: {args.output}") + return 1 + try: + lock = json.loads(args.output.read_text()) + except (OSError, ValueError, TypeError) as exc: + print(f"BLOCKED: plan lock is unreadable: {exc}") + return 1 + mismatches = [] + for label, path in [("spec", args.spec), ("plan", args.plan), ("task_graph", args.tasks)]: + expected_hash = sha256(path) + if lock.get(f"{label}_sha256") != expected_hash: + mismatches.append(label) + if lock.get("status") != "APPROVED" or lock.get("invalidated_at"): + mismatches.append("status") + if not lock.get("approved_by"): + mismatches.append("approver") + if mismatches: + print("BLOCKED: stale or invalid plan lock: " + ", ".join(mismatches)) + return 1 + print("PASS: approved plan lock matches the current artifacts") + return 0 + + if not args.approved_by or not args.approved_by.strip(): + parser.error("--approved-by must name the human who explicitly approved the plan") + lock = expected(args) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(lock, indent=2, sort_keys=True) + "\n") + print(f"wrote approved plan lock: {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/boatstack/scripts/compile_plan.py b/boatstack/scripts/compile_plan.py new file mode 100644 index 0000000..1ee4338 --- /dev/null +++ b/boatstack/scripts/compile_plan.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Validate an approved structured plan and compile executable gate artifacts.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def fail(message: str) -> None: + raise ValueError(message) + + +def validate(plan: dict) -> None: + if plan.get("schema_version") != 1: + fail("schema_version must be 1") + if not plan.get("feature_id"): + fail("feature_id is required") + criteria = plan.get("acceptance_criteria") + tasks = plan.get("tasks") + if not isinstance(criteria, list) or not criteria: + fail("at least one acceptance criterion is required") + if not isinstance(tasks, list) or not tasks: + fail("at least one task is required") + + criterion_ids = [item.get("id") for item in criteria if isinstance(item, dict)] + task_ids = [item.get("id") for item in tasks if isinstance(item, dict)] + if len(criterion_ids) != len(criteria) or None in criterion_ids or len(set(criterion_ids)) != len(criterion_ids): + fail("acceptance criterion ids must be present and unique") + if len(task_ids) != len(tasks) or None in task_ids or len(set(task_ids)) != len(task_ids): + fail("task ids must be present and unique") + + known_criteria = set(criterion_ids) + known_tasks = set(task_ids) + covered: set[str] = set() + graph: dict[str, list[str]] = {} + for task in tasks: + task_id = task["id"] + dependencies = task.get("depends_on") or [] + mapped = task.get("acceptance_criteria") or [] + validations = task.get("validation") or [] + if task_id in dependencies: + fail(f"task {task_id} cannot depend on itself") + unknown_dependencies = set(dependencies) - known_tasks + if unknown_dependencies: + fail(f"task {task_id} has unknown dependencies: {sorted(unknown_dependencies)}") + unknown_criteria = set(mapped) - known_criteria + if unknown_criteria: + fail(f"task {task_id} maps unknown criteria: {sorted(unknown_criteria)}") + if not mapped and not task.get("enabling_reason"): + fail(f"task {task_id} must map acceptance criteria or state an enabling_reason") + if not isinstance(validations, list) or not validations: + fail(f"task {task_id} requires at least one validation command or procedure") + covered.update(mapped) + graph[task_id] = list(dependencies) + + uncovered = known_criteria - covered + if uncovered: + fail(f"uncovered acceptance criteria: {sorted(uncovered)}") + + visiting: set[str] = set() + visited: set[str] = set() + + def visit(task_id: str) -> None: + if task_id in visiting: + fail(f"task dependency cycle includes {task_id}") + if task_id in visited: + return + visiting.add(task_id) + for dependency in graph[task_id]: + visit(dependency) + visiting.remove(task_id) + visited.add(task_id) + + for task_id in task_ids: + visit(task_id) + + +def compile_artifacts(plan: dict) -> tuple[dict, dict, str]: + criteria = {item["id"]: item for item in plan["acceptance_criteria"]} + task_graph = { + "schema_version": 1, + "feature_id": plan["feature_id"], + "source_plan_status": "HUMAN_APPROVED", + "tasks": plan["tasks"], + } + rows = [] + for criterion_id, criterion in criteria.items(): + serving = [task for task in plan["tasks"] if criterion_id in (task.get("acceptance_criteria") or [])] + validations = [] + for task in serving: + for check in task.get("validation") or []: + validations.append({"task_id": task["id"], "check": check}) + rows.append({ + "criterion_id": criterion_id, + "criterion": criterion.get("text", ""), + "tasks": [task["id"] for task in serving], + "validations": validations, + "result": "BLOCKED", + "evidence": None, + }) + test_matrix = { + "schema_version": 1, + "feature_id": plan["feature_id"], + "requirements": rows, + } + evidence_lines = [ + f"# Evidence ledger: {plan['feature_id']}", + "", + "- Approved plan lock: pending", + "- Test gate: `BLOCKED`", + "- Review gate: `BLOCKED`", + "- Ship gate: `BLOCKED`", + "", + "## Acceptance evidence", + "", + "| Criterion | Tasks | Result | Evidence |", + "|---|---|---|---|", + ] + for row in rows: + evidence_lines.append( + f"| {row['criterion_id']}: {row['criterion']} | {', '.join(row['tasks'])} | `BLOCKED` | |" + ) + evidence_lines.extend(["", "## Commands and checks", "", "## Review findings", "", "## Known gaps", "", "## Rollout and rollback", ""]) + return task_graph, test_matrix, "\n".join(evidence_lines) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--plan", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + args = parser.parse_args() + try: + plan = json.loads(args.plan.read_text()) + validate(plan) + task_graph, test_matrix, evidence = compile_artifacts(plan) + except (OSError, ValueError, json.JSONDecodeError) as exc: + print(f"BLOCKED: invalid approved plan: {exc}") + return 1 + args.out_dir.mkdir(parents=True, exist_ok=True) + (args.out_dir / "tasks.json").write_text(json.dumps(task_graph, indent=2, sort_keys=True) + "\n") + (args.out_dir / "test-matrix.json").write_text(json.dumps(test_matrix, indent=2, sort_keys=True) + "\n") + (args.out_dir / "evidence.md").write_text(evidence) + print(f"PASS: compiled approved plan into {args.out_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/boatstack/scripts/export_repo.py b/boatstack/scripts/export_repo.py new file mode 100644 index 0000000..afee25b --- /dev/null +++ b/boatstack/scripts/export_repo.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +"""Export the canonical product loop into thin repo-specific host adapters.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from pathlib import Path + + +VERSION = "0.1.0" +GENERATOR = "product-engineering-loop-exporter" +ALLOWED_ADAPTERS = {"cursor", "claude", "codex", "github"} +MARKER = "Generated by product-engineering-loop exporter. Do not edit; change canonical source or project.json." +ADAPTER_NAME = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def generated_markdown(body: str) -> bytes: + return f"\n\n{body.rstrip()}\n".encode() + + +def generated_frontmatter(body: str) -> bytes: + if not body.startswith("---\n"): + raise ValueError("frontmatter adapter must start with ---") + closing = body.find("\n---\n", 4) + if closing < 0: + raise ValueError("frontmatter adapter is missing its closing ---") + insert_at = closing + len("\n---\n") + marked = body[:insert_at] + f"\n\n" + body[insert_at:] + return (marked.rstrip() + "\n").encode() + + +def generated_script(body: str) -> bytes: + lines = body.splitlines() + if lines and lines[0].startswith("#!"): + lines.insert(1, f"# {MARKER}") + else: + lines.insert(0, f"# {MARKER}") + return ("\n".join(lines).rstrip() + "\n").encode() + + +def generated_json(value: dict) -> bytes: + data = dict(value) + data["_generated_by"] = GENERATOR + data["_loop_version"] = VERSION + return (json.dumps(data, indent=2, sort_keys=True) + "\n").encode() + + +def load_config(path: Path) -> dict: + data = json.loads(path.read_text()) + if data.get("schema_version") != 1: + raise ValueError("project config schema_version must be 1") + project = data.get("project") + if not isinstance(project, dict) or not project.get("name"): + raise ValueError("project.name is required") + commands = project.get("commands") + if not isinstance(commands, dict) or not commands.get("test"): + raise ValueError("project.commands.test is required; the exporter will not invent it") + adapters = set(data.get("adapters") or ALLOWED_ADAPTERS) + unknown = adapters - ALLOWED_ADAPTERS + if unknown: + raise ValueError("unsupported adapters: " + ", ".join(sorted(unknown))) + return data + + +def command_body(operation: str, extra: str) -> str: + return f"""# {operation} + +Run the `{operation}` operation from `@.product-loop/workflow.md`. + +Read `@.product-loop/project.json`, `@.product-loop/artifacts.md`, and only the minimal repository context relevant to the current feature. {extra} + +Use the gate semantics in the canonical workflow. Do not redefine them in this adapter. +""" + + +def build_files( + config_path: Path, + config: dict, + skill_root: Path, + adapters: set[str], + adapter_name: str = "product-engineering-loop", +) -> dict[Path, bytes]: + files: dict[Path, bytes] = {} + files[Path(".product-loop/project.json")] = generated_json(config) + for name in ["workflow.md", "artifacts.md", "failure-moves.md"]: + files[Path(".product-loop") / name] = generated_markdown( + (skill_root / "references" / name).read_text() + ) + for template in sorted((skill_root / "assets" / "templates").glob("*")): + if template.suffix == ".json": + value = json.loads(template.read_text()) + files[Path(".product-loop/templates") / template.name] = generated_json(value) + else: + files[Path(".product-loop/templates") / template.name] = generated_markdown(template.read_text()) + files[Path(".product-loop/tools/approve_plan.py")] = generated_script( + (skill_root / "scripts" / "approve_plan.py").read_text() + ) + files[Path(".product-loop/tools/compile_plan.py")] = generated_script( + (skill_root / "scripts" / "compile_plan.py").read_text() + ) + + operations = { + "auto-plan": "Produce a draft only. Do not implement and do not imply the user accepted it.", + "plan-gate": "Require explicit human approval. Only then run `.product-loop/tools/compile_plan.py` and `.product-loop/tools/approve_plan.py` to create the executable task/evidence package and lock.", + "build": "Before editing, locate the feature spec, plan, compiled tasks, and plan lock; run `.product-loop/tools/approve_plan.py --check` against them. Stop if it reports `BLOCKED`.", + "test-gate": "Build a requirement-to-evidence matrix and treat self-authored tests as evidence rather than the sole oracle.", + "review-gate": "Review the actual diff against approved intent, invariants, risks, gaps, and test evidence.", + "ship-gate": "Prepare a PR only; do not merge or deploy without separate authorization.", + "review": "Alias of `review-gate`: review the actual diff against approved intent, invariants, risks, gaps, and test evidence.", + "ship": "Alias of `ship-gate`: prepare a PR only; do not merge or deploy without separate authorization.", + "retro": "Classify evidence and propose a move; never promote it or change durable rules without a paired gate.", + } + + if "cursor" in adapters: + rule = """--- +description: Use the canonical product engineering loop for planning, approved implementation, evidence gates, PR preparation, and loop retrospectives. +globs: +alwaysApply: false +--- + +The source of truth is @.product-loop/workflow.md and @.product-loop/project.json. +Use @.product-loop/artifacts.md for document meanings and @.product-loop/failure-moves.md for retrospectives. +Do not start build work until the explicit plan gate has produced a valid plan lock. +Do not branch behavior on model name, provider, or price; branch on observed work state and evidence. +""" + files[Path(f".cursor/rules/{adapter_name}.mdc")] = generated_frontmatter(rule) + for operation, extra in operations.items(): + files[Path(f".cursor/commands/{operation}.md")] = generated_markdown(command_body(operation, extra)) + + adapter_skill = f"""--- +name: {adapter_name} +description: Run the repository's canonical question-led product engineering loop for planning, explicit plan approval, implementation, test/review/ship gates, and evidence-based retrospectives. +--- + +# Product Engineering Loop Adapter + +Read `.product-loop/project.json` and `.product-loop/workflow.md`. The requested operation is supplied by the user; valid operations are `auto-plan`, `plan-gate`, `build`, `test-gate`, `review-gate`/`review`, `ship-gate`/`ship`, and `retro`. + +Use `.product-loop/artifacts.md` for document boundaries and `.product-loop/failure-moves.md` for improvement experiments. Do not implement from an unapproved or stale plan. Do not branch on model identity; use observable state and gate evidence. +""" + if "claude" in adapters: + files[Path(f".claude/skills/{adapter_name}/SKILL.md")] = generated_frontmatter(adapter_skill) + if "codex" in adapters: + files[Path(f".agents/skills/{adapter_name}/SKILL.md")] = generated_frontmatter(adapter_skill) + + if "github" in adapters: + pr = """# Product-loop PR + +## Approved intent + +- Feature spec: +- Approved plan hash: +- Human approver: +- Linked ADRs/questions: + +## Outcome + +- User-visible change: +- Non-goals preserved: + +## Gate evidence + +- Test gate: `BLOCKED` +- Review gate: `BLOCKED` +- Ship gate: `BLOCKED` +- Evidence ledger: + +## Known gaps + +- Gap ledger: +- `PASS_WITH_GAPS` rationale, owner, and revisit trigger: + +## Rollout and rollback + +- Rollout: +- Observability: +- Rollback: + +## Generated adapter update + +- Canonical loop version: +- Config hash: +- Export check: +""" + files[Path(f".github/PULL_REQUEST_TEMPLATE/{adapter_name}.md")] = generated_markdown(pr) + + lock_entries = { + str(path): sha256_bytes(content) for path, content in sorted(files.items(), key=lambda item: str(item[0])) + } + lock = { + "schema_version": 1, + "generator": GENERATOR, + "loop_version": VERSION, + "config_source": config_path.name, + "config_sha256": sha256_bytes(config_path.read_bytes()), + "adapters": sorted(adapters), + "files": lock_entries, + } + files[Path(".product-loop/generated.lock.json")] = generated_json(lock) + return files + + +def owned(content: bytes, path: Path) -> bool: + if MARKER.encode() in content: + return True + if path.suffix == ".json": + try: + return json.loads(content).get("_generated_by") == GENERATOR + except (ValueError, AttributeError): + return False + return False + + +def write_files(repo: Path, files: dict[Path, bytes]) -> int: + collisions = [] + for relative, content in files.items(): + target = repo / relative + if target.exists() and target.read_bytes() != content and not owned(target.read_bytes(), relative): + collisions.append(str(relative)) + if collisions: + print("BLOCKED: refusing to overwrite user-owned files:") + for collision in collisions: + print(f" {collision}") + return 2 + for relative, content in files.items(): + target = repo / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + print(f"wrote {len(files)} generated files to {repo}") + return 0 + + +def check_files(repo: Path, files: dict[Path, bytes]) -> int: + problems = [] + for relative, content in files.items(): + target = repo / relative + if not target.exists(): + problems.append(f"missing {relative}") + elif target.read_bytes() != content: + problems.append(f"drift {relative}") + if problems: + print("BLOCKED: generated output is stale") + for problem in problems: + print(f" {problem}") + return 1 + print(f"PASS: {len(files)} generated files match canonical source and config") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--adapters", help="comma-separated override") + parser.add_argument( + "--adapter-name", + default="product-engineering-loop", + help="kebab-case name for generated host adapter files (default: product-engineering-loop)", + ) + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--write", action="store_true") + mode.add_argument("--check", action="store_true") + args = parser.parse_args() + + repo = args.repo.resolve() + config_path = args.config.resolve() + if not repo.is_dir(): + parser.error(f"repo does not exist: {repo}") + try: + config = load_config(config_path) + adapters = set(args.adapters.split(",")) if args.adapters else set(config.get("adapters") or ALLOWED_ADAPTERS) + unknown = adapters - ALLOWED_ADAPTERS + if unknown: + raise ValueError("unsupported adapters: " + ", ".join(sorted(unknown))) + if not ADAPTER_NAME.fullmatch(args.adapter_name): + raise ValueError("adapter name must be a lowercase kebab-case slug") + except (OSError, ValueError, json.JSONDecodeError) as exc: + parser.error(str(exc)) + + skill_root = Path(__file__).resolve().parent.parent + files = build_files(config_path, config, skill_root, adapters, args.adapter_name) + if args.check: + return check_files(repo, files) + if args.write: + return write_files(repo, files) + print(f"dry run: would generate {len(files)} files in {repo}") + for relative in sorted(files, key=str): + print(f" {relative}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/benchmark-corpus-audit.md b/docs/benchmark-corpus-audit.md new file mode 100644 index 0000000..e38baf3 --- /dev/null +++ b/docs/benchmark-corpus-audit.md @@ -0,0 +1,39 @@ +# Local Terminal-Bench corpus audit + +> Generated by `scripts/audit_terminal_bench.py`. This proves aggregate read coverage of every local trial result and available signal stream; it does not claim every transcript was manually interpreted. + +## Coverage + +- Per-trial results read: **3571** +- Run/corpus groups: **19** +- Signal streams read: **3540** (99.13% coverage) +- Aggregate Harbor result files excluded: **18** +- Unreadable trial results: **0** + +## Groups + +| Corpus | Run | Model | Trials | Pass | Partial | Fail | Errors | Signals | Parse errors | Command timeouts | +|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:| +| jobs-cloud | 2026-06-09__16-53-01 | gemini/gemini-3.1-pro-preview | 445 | 300 | 0 | 136 | 9 | 445 | 27 | 154 | +| jobs-fullrun-2.1 | 2026-06-10__15-35-20 | gemini/gemini-3.1-pro-preview | 445 | 291 | 0 | 144 | 10 | 444 | 55 | 159 | +| jobs-gate | 2026-06-10__01-44-32 | gemini/gemini-3.1-pro-preview | 225 | 84 | 0 | 132 | 9 | 225 | 19 | 93 | +| jobs-gate | 2026-06-10__06-50-59 | gemini/gemini-3.1-pro-preview | 225 | 79 | 0 | 136 | 10 | 224 | 19 | 250 | +| jobs-gate | 2026-06-10__06-51-19 | gemini/gemini-3.1-pro-preview | 225 | 67 | 0 | 151 | 7 | 224 | 12 | 200 | +| jobs-gate | 2026-06-10__09-38-07 | gemini/gemini-3.1-pro-preview | 221 | 85 | 0 | 127 | 9 | 221 | 29 | 130 | +| jobs-gate | 2026-06-10__11-15-41 | gemini/gemini-3.1-pro-preview | 225 | 84 | 0 | 131 | 10 | 225 | 20 | 119 | +| jobs-gate | 2026-06-10__13-36-04 | gemini/gemini-3.1-pro-preview | 225 | 83 | 0 | 81 | 61 | 198 | 13 | 91 | +| jobs-rerun | 2026-06-09__22-01-02 | gemini/gemini-3.1-pro-preview | 163 | 57 | 0 | 98 | 8 | 162 | 17 | 67 | +| qwen-floor | 2026-06-11__16-36-32 | vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas | 100 | 18 | 0 | 80 | 2 | 100 | 0 | 32 | +| qwen-floor-board | 2026-06-12__07-56-37 | vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas | 90 | 24 | 0 | 63 | 3 | 90 | 5 | 18 | +| qwen-mutation-trial | 2026-06-12__02-23-37 | vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas | 8 | 3 | 0 | 5 | 0 | 8 | 0 | 0 | +| qwen-raise40 | 2026-06-11__19-12-16 | vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas | 99 | 17 | 0 | 79 | 3 | 99 | 1 | 24 | +| qwen-spec-first | 2026-06-11__23-54-32 | vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas | 99 | 25 | 0 | 69 | 5 | 99 | 1 | 22 | +| qwen-spec-first-board | 2026-06-12__05-38-49 | vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas | 194 | 46 | 0 | 142 | 6 | 194 | 2 | 62 | +| qwen-spec-first-s30 | 2026-06-11__23-07-24 | vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas | 39 | 5 | 0 | 34 | 0 | 39 | 0 | 4 | +| qwen-verify-repair | 2026-06-11__22-05-07 | vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas | 99 | 18 | 0 | 80 | 1 | 99 | 1 | 27 | +| submission-2.0 | 2026-06-10__18-19-50 | gemini/gemini-3.1-pro-preview | 432 | 264 | 0 | 151 | 17 | 432 | 35 | 234 | +| submission-2.0 | 2026-06-10__21-03-33 | gemini/gemini-3.1-pro-preview | 12 | 5 | 0 | 7 | 0 | 12 | 1 | 4 | + +## Interpretation boundary + +The generated JSON contains outcome, terminal-reason, protocol-error, timeout, and selected loop-event aggregates for every group. Semantic claims about mechanisms still require the paired gates, experiment log, and representative trajectory inspection; aggregate coverage alone cannot establish causality. diff --git a/docs/benchmark-submission-audit.md b/docs/benchmark-submission-audit.md new file mode 100644 index 0000000..467d17b --- /dev/null +++ b/docs/benchmark-submission-audit.md @@ -0,0 +1,21 @@ +# Local Terminal-Bench corpus audit + +> Generated by `scripts/audit_terminal_bench.py`. This proves aggregate read coverage of every local trial result and available signal stream; it does not claim every transcript was manually interpreted. + +## Coverage + +- Per-trial results read: **445** +- Run/corpus groups: **1** +- Signal streams read: **445** (100.00% coverage) +- Aggregate Harbor result files excluded: **1** +- Unreadable trial results: **0** + +## Groups + +| Corpus | Run | Model | Trials | Pass | Partial | Fail | Errors | Signals | Parse errors | Command timeouts | +|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:| +| 2026-07-15__18-08-50-submission | 2026-07-15__18-08-50 | gemini/gemini-3.1-pro-preview | 445 | 296 | 0 | 147 | 2 | 445 | 77 | 112 | + +## Interpretation boundary + +The generated JSON contains outcome, terminal-reason, protocol-error, timeout, and selected loop-event aggregates for every group. Semantic claims about mechanisms still require the paired gates, experiment log, and representative trajectory inspection; aggregate coverage alone cannot establish causality. diff --git a/docs/loop-engineering.md b/docs/loop-engineering.md new file mode 100644 index 0000000..a156518 --- /dev/null +++ b/docs/loop-engineering.md @@ -0,0 +1,92 @@ + + +# Loop engineering + +Boatstack treats software delivery as a feedback system around a coding model. The model matters, but it is not the whole system. + +## The minimal state + +For current repository state `x_t`, select only the task-relevant slice: + +```text +z_t = P_s(x_t) +``` + +Canonicalize that slice into one domain, one contract, one outcome, and one next operator: + +```text +r_t = R(z_t) +u_t = f(r_t) +x_(t+1) = V(u_t, acceptance, invariants) +``` + +In the repository, those terms are not decorative notation: + +| Term | Boatstack artifact or check | +|---|---| +| `P_s` | context paths in `project.json` plus the current feature boundary | +| `R` | question ledger, feature spec, acceptance criteria, structured plan | +| `f` | one operation: auto-plan, plan-gate, build, test, review, or ship | +| `V` | requirement/test matrix, command evidence, review findings, plan hashes | +| `x_(t+1)` | locked plan, bounded diff, gate result, PR, or recorded gap | + +ZCA creates immediate value by reducing a vague feature request to one verifiable slice. For a shipped SDK, API, or CLI, Boatstack uses two slices: the implementation boundary and one representative consumer path. + +## Optimization is constrained, not blind compression + +Boatstack aims to minimize the cost of context and ceremony subject to an accepted outcome: + +```text +minimize C(context) + C(ceremony) + C(rework) +subject to acceptance criteria pass + project invariants hold + required evidence exists + approval is current +``` + +That is why context trimming is not automatically an optimization. If removing state increases rework or false acceptance, total cost rises. The canonical runtime references are approximately **3371 estimated tokens**, while host adapters point to one operation at a time. + +## Control appears at state transitions + +The plan gate is a concrete controller boundary: + +```python +if sha256(current_plan) != approval["plan_sha256"]: + return "BLOCKED: plan changed after approval" +``` + +The plan compiler is another: + +```python +uncovered = acceptance_ids - task_acceptance_ids +if uncovered: + raise ValueError(f"uncovered acceptance criteria: {sorted(uncovered)}") +``` + +The test gate maps each claim to evidence instead of asking the implementer whether it feels finished: + +```text +AC-4: ASCII output stays byte-compatible + -> diff -u expected-output.txt actual-output.txt + -> PASS | FAIL | BLOCKED +``` + +These boundaries do not guarantee correct software. They make missing authority, missing coverage, stale state, and failed evidence observable before shipping. + +## There are two loops + +Delivery and loop improvement remain separate: + +```text +delivery: intent -> approved plan -> code -> gates -> PR +improvement: failure evidence -> mechanism -> candidate move -> paired gate + -> promote | wash | reject +``` + +A failed task can suggest a better move, but one anecdote cannot silently rewrite every future prompt. Promotion requires a representative comparison and a non-regression boundary. + +## What is evidence-backed + +The current moves were derived from the Intelligence Flow benchmark corpus and product-repository studies. The generated source commit is [`aae685d2513cd25537284e4e68177411ace7ac9a`](https://github.com/operatorstack/intelligence-flow/tree/aae685d2513cd25537284e4e68177411ace7ac9a/examples/12-product-engineering-loop). + +The evidence supports specific failure mechanisms and guardrails. It does not establish that Boatstack is optimal, that control-theory notation proves software quality, or that one workflow dominates every team. Those are evaluation questions, so the distribution preserves measurements, provenance, gaps, and negative results. diff --git a/docs/research-and-design.md b/docs/research-and-design.md new file mode 100644 index 0000000..f199b27 --- /dev/null +++ b/docs/research-and-design.md @@ -0,0 +1,160 @@ +# Research and design: a harness-neutral product engineering loop + +## Outcome + +The proposed product is not a large prompt and not a Codex-, Cursor-, Claude-, or model-specific harness. It is a versioned engineering protocol with: + +1. canonical project facts, artifacts, states, gates, and evidence under `.product-loop/`; +2. thin generated adapters for each host; +3. a human approval boundary between planning and executable work; +4. a separate evidence-gated loop for improving the protocol itself. + +The initial implementation is in [`product-engineering-loop/`](product-engineering-loop/). Its exporter generates Cursor rules/commands, Claude Code and Codex skills, and a GitHub PR template from one source. + +The public [Boatstack](https://github.com/operatorstack/boatstack) repository is a compiled distribution, not a second source of workflow truth. `scripts/build_boatstack.py` projects this package into a branded README, loop-engineering explanation, worked example, tests, and installable skill; `UPSTREAM.json` binds every generated file to its Intelligence Flow commit. A Boatstack-owned scheduled workflow polls this public source and proposes changes by PR. + +## Outcome sizing and where value emerges + +For a feature, the minimal outcome definition is: + +```text +one domain + one contract + one outcome + one next operator + one verifier +``` + +Because the loop may become a shipped product, it keeps delivery and improvement as separate paths: + +- **Delivery path:** developer intent -> questions -> draft -> human approval -> deterministic materialization -> build -> test -> review -> PR. +- **Improvement path:** run evidence -> failure mode -> proposed move -> paired representative gate -> promote/reject/wash. + +Value emerges twice. The delivery path reduces assumption-driven code and produces reviewable evidence immediately. The improvement path lets recurring failures compound into better tooling without allowing one anecdote to pollute every future repository. + +## Commands and the approval boundary + +```text +Cursor/GitHub intent + -> /auto-plan draft spec + structured plan; no code + -> /plan-gate explicit human approve/change request + after approval: compile tasks/test matrix/evidence + hash lock + -> /build refuses absent or stale lock + -> /test-gate requirement-derived independent evidence + -> /review diff + intent + invariant + risk + gap review + -> /ship PR preparation, not merge/deploy + -> /retro propose a loop move; never silently promote it +``` + +`/auto-plan` cannot infer acceptance from silence. `/plan-gate` records the approver and hashes the spec, plan, and compiled task graph. Any semantic edit invalidates the lock and returns to approval. This turns the developer's agreement into a machine-checkable state transition instead of conversational memory. + +## Why the workflow has no model conditions + +The benchmark evidence shows that the binding failure mode changes by task, distribution, and intervention. It does not justify hardcoding “cheap model workflow” and “strong model workflow.” A model name, provider, or price is not an observed failure state. + +The loop therefore branches only on: + +- unknown versus discoverable information; +- risk and reversibility; +- convergence versus thrashing; +- protocol/tool outcomes; +- test-oracle fidelity; +- repeated tactics; +- gate evidence. + +This still allows a repository owner to choose any model or routing service. It means the engineering contract stays identical and performance differences become measurable rather than baked into prompts. + +## Benchmark corpus coverage + +The evidence was audited in two reproducible passes: + +- [`BENCHMARK_CORPUS_AUDIT.md`](BENCHMARK_CORPUS_AUDIT.md): **3,571** historical per-trial results across 19 run/corpus groups; 3,540 signal streams; no unreadable results. +- [`BENCHMARK_SUBMISSION_2_1_AUDIT.md`](BENCHMARK_SUBMISSION_2_1_AUDIT.md): all **445** July Terminal-Bench 2.1 submission trials and all 445 signal streams. + +Combined mechanical coverage is **4,016 trial results** and **3,985 signal streams**. The JSON companions preserve group-level outcomes, terminal reasons, protocol errors, timeouts, and loop-event aggregates. + +This is not a claim that every transcript was manually read. Mechanism conclusions come from the preregistered comparisons, paired gates, experiment log, and representative trajectory inspections. The audit proves that every locally present result was included in aggregate coverage. + +Four run IDs mentioned in the historical notes do not have raw corpora locally: + +- `2026-06-10__03-21-28`: two-arm correctness-relief candidate; summary preserved in `IMPROVEMENTS.md`; +- `2026-06-10__05-59-14`: nine-trial retry-verified smoke; full table preserved in `RESEARCH_LOG.md` E8; +- `2026-06-10__15-05-24`: incomplete spend-cap false start, explicitly excluded from inference; +- `2026-06-11__15-09-20`: early Qwen probe; summary preserved in `ZERO_TO_QWEN.md`. + +Those are **summary-only evidence** in this design. They are not represented as newly re-derived raw results. + +## What the Terminal-Bench data actually encodes + +| Observation | Evidence | Coding-loop rule | +|---|---|---| +| Fatal command timeouts hid capability | Non-fatal timeout handling moved the original score from roughly 54% to 65.4% | Tool failures become observations when safely recoverable; external timeout remains authoritative | +| Malformed structured responses were recoverable | July screen repaired 56/63 malformed responses; full run repaired 490/567 exposures | Validate schemas and attempt bounded same-step parse repair; do not label protocol failure as reasoning failure | +| More verification wording did not create truth | Verify-before-finish and same-model repair variants washed | Self-review is evidence, never the sole oracle | +| Restarting destroyed partial progress | Failed retry-verified restarts retained roughly 53% mean fractional progress | Preserve last known-good state; use targeted repair and compare snapshots | +| Blind context trimming regressed | Windowing lost 7.2 binary points against the floor in its gate | Project minimal relevant context, but never discard state merely to reduce tokens | +| More steps changed the label, not correctness | Qwen 30->40 reduced exhaustion but converted failures into near misses/wrong answers | Increase budget only when trajectory evidence shows convergence; stop thrashing | +| Strict self-checking caused collateral damage | Strictness was a certified loss against its non-strict base | Stronger instructions are not monotonic; protect known-good output and verify against an independent contract | +| Spec-first helped a development slice for the wrong reason | Qwen dev gate gained 7 points, while the frozen test oracle had about 47% fidelity | Specs/tests can scaffold understanding, but model-authored tests do not become ground truth | +| The development result did not transfer | Spec-first was a statistical wash on the full board | Promotion requires representative distribution/holdout, not only a tuned dev slice | +| Model change relocated the bottleneck | Same harness exposed near-miss dominance on Gemini and step exhaustion on Qwen | Diagnose the active population each time; do not encode model-specific recipes | +| Mid-run aggregates changed direction | Qwen board interpretation moved as task coverage deepened | Compare paired completed coverage and uncertainty, not early aggregate rank | + +Sources: [`RESEARCH_LOG.md`](../11-harbor-submit/RESEARCH_LOG.md), [`EXPERIMENT_GEMINI20_2026-07-15.md`](../11-harbor-submit/EXPERIMENT_GEMINI20_2026-07-15.md), [`ZERO_TO_QWEN.md`](../11-harbor-submit/ZERO_TO_QWEN.md), and [`docs/12-self-verification-fidelity.md`](../../docs/12-self-verification-fidelity.md). + +## What two example repositories add + +Terminal-Bench supplies failure mechanics; the product repositories supply real engineering context. + +The first example repository demonstrates: + +- durable non-negotiables for tenancy, evidence handling, audit logs, and resource caps; +- deterministic core state machines with an LLM behind a narrow adapter port; +- a docs/spec PR before the code PR for uncertain contracts; +- typed input slots instead of hardcoded client details; +- acceptance tests plus explicitly parked work. + +The second example repository demonstrates: + +- durable decision notes with insight, rationale, decision, risks, and open questions; +- known divergences kept visible rather than implied complete; +- mechanical CI rules derived from recurring real-world failure patterns; +- test plans spanning unit/build/staging/fail-soft behavior. + +This is why ADRs are only one artifact. The loop also needs a question ledger, feature spec, gap ledger, test plan, risk note, evidence ledger, and runbook when relevant. + +## What is adopted from gstack and Spec Kit + +From [gstack](https://github.com/garrytan/gstack/blob/main/docs/skills.md): forcing questions before planning; product, design, engineering, and developer-experience review lenses; plan artifacts as deliverables; review readiness; and a structured ship preflight. Its [`/autoplan`](https://github.com/garrytan/gstack/blob/main/autoplan/SKILL.md) is an integration option, not the canonical source. + +From [GitHub Spec Kit](https://github.com/github/spec-kit): constitution, specify, clarify, plan, tasks, analyze, checklist, implement, and converge stages. Spec Kit can generate artifacts, but `.product-loop/` normalizes their meaning and preserves the explicit human plan gate. + +The loop does not adopt a universal “boil the ocean” policy. Completeness is required for the approved outcome; unrelated architecture remains outside its boundary. + +## Host portability + +- [Cursor project rules](https://docs.cursor.com/context/rules) live in `.cursor/rules`; project commands live in [`.cursor/commands`](https://docs.cursor.com/en/agent/chat/commands). Cursor is a first-class exported surface. +- Claude Code receives a project skill while `CLAUDE.md` stays repository-owned. +- Codex receives a repo skill under `.agents/skills`; [OpenAI recommends](https://learn.chatgpt.com/docs/customization/overview) keeping durable `AGENTS.md` guidance small and workflows in reusable skills. +- GitHub receives a PR template that exposes approved-plan hashes, gate status, gaps, evidence, rollout, and rollback. + +The exporter refuses to overwrite any non-generated file. Its lock records canonical version, config hash, adapters, and output hashes so a PR can show exactly what changed. + +## Evaluation of the finished loop + +The best public primary benchmark is [FeatureBench](https://github.com/LiberCoders/FeatureBench), because it targets complex feature development and provides a 100-instance fast split plus agent integrations. Evaluate the same model and tasks with: + +```text +plain host harness vs product engineering loop +``` + +Measure resolved rate, regression rate, tokens/cost, elapsed time, question count, plan revisions, stale-lock blocks, test-oracle independence, review findings, and ship-gate false accepts. + +Add a private held-out feature set drawn from the two example repositories for the parts public executable benchmarks do not score: whether the right human questions were asked, durable decisions and gaps were classified correctly, project invariants were preserved, and the PR was actually usable. SWE-bench can be a supplemental bug-fix check, but it is less aligned with feature/product work. Do not use another Terminal-Bench run as the primary validation for this product. + +## Open questions before private-repo productization + +1. Should the canonical artifact store stay as versioned files or become a small local database with generated Markdown views? +2. Which project facts may be inferred during `init`, and which must always be human-confirmed? +3. What is the minimum independent oracle required at each risk level? +4. Should adapter updates be generated locally, by a GitHub App, or both? +5. How should private traces be redacted before entering the improvement corpus? +6. What promotion sample size/noise band should the product default to outside benchmarks? + +The next valuable step is to install the exporter into a clean fixture repository, forward-test `/auto-plan -> /plan-gate -> /build` on one real feature, and only then apply it to the two example repositories. diff --git a/examples/diagram-json/README.md b/examples/diagram-json/README.md new file mode 100644 index 0000000..3f26bec --- /dev/null +++ b/examples/diagram-json/README.md @@ -0,0 +1,111 @@ +# Worked example: JSON output for diagrams + +This is a worked demonstration of the product engineering loop. The feature is +intentionally small and uses code already in this repository: + +> Add machine-readable JSON output to the diagram printer while preserving the +> current text output. + +No product code is changed by this example. The named approval below is a +simulated walkthrough record, not authorization to implement or ship the +feature. + +## What a developer does + +First, install the loop adapters in a repository and open the coding agent's +plan mode. The developer can type the product request in ordinary language: + +```text +Add machine-readable JSON output to the diagram printer while preserving the +current text output. +``` + +Then run `/auto-plan`. In this repository the agent should inspect only: + +- `src/diagram.ts` for the current contract and rendering behavior; +- `src/index.ts` for the public export boundary; +- `examples/05-diagram-printer/` for executable examples and regression output; +- `package.json` and TypeScript configs for real validation commands. + +The result is a draft, not code: + +- [product request](request.md) +- [question and decision ledger](questions.md) +- [feature specification](spec.md) +- [structured plan](plan.json) + +## The missing human step + +The agent presents the draft with the three contract decisions in +`questions.md`. A representative exchange is: + +```text +Agent: The draft is ready. The recommended contract is an additive serializer, +a versioned public schema, and a compact run overlay. No code has been changed. +Approve this plan or tell me what to revise. + +Example Maintainer: Approve this demonstration plan. +``` + +Only after that explicit answer does `/plan-gate` compile and lock the plan: + +```bash +python3 ../../boatstack/scripts/compile_plan.py \ + --plan plan.json \ + --out-dir compiled + +python3 ../../boatstack/scripts/approve_plan.py \ + --spec spec.md \ + --plan plan.json \ + --tasks compiled/tasks.json \ + --approved-by "Example Maintainer (simulated walkthrough)" \ + --output plan.lock.json +``` + +That produces: + +- [compiled task graph](compiled/tasks.json) +- [requirement-to-test matrix](compiled/test-matrix.json) +- [evidence ledger](compiled/evidence.md) +- [content-addressed plan lock](plan.lock.json) + +The lock is the deterministic boundary between agreement and implementation. +Editing `spec.md`, `plan.json`, or the compiled task graph makes its check fail. + +```bash +python3 ../../boatstack/scripts/approve_plan.py \ + --spec spec.md \ + --plan plan.json \ + --tasks compiled/tasks.json \ + --output plan.lock.json \ + --check +``` + +## What happens next in a real feature + +The remaining commands consume the same canonical artifacts regardless of the +coding model or host: + +1. `/build` verifies the lock, implements one task at a time, and stops for any + newly discovered product decision. +2. `/test-gate` runs the matrix and attaches command output or fixture evidence + to each acceptance criterion. +3. `/review` examines the actual diff for compatibility, schema stability, + unhandled failures, and evidence gaps. +4. `/ship` prepares a PR containing the spec, decisions, evidence, gaps, + rollout, and rollback. It does not merge or deploy. +5. `/retro` classifies unexpected failures and proposes a separately evaluated + loop improvement instead of silently rewriting the workflow. + +In Cursor these appear as generated slash commands. Claude Code and Codex get +thin skill adapters, while GitHub receives a matching PR template. The workflow +itself remains in `.product-loop/`, so the contract is not hardcoded to a host. + +## Why this example has immediate value + +The one-sentence request is reduced to one public API boundary and five +observable acceptance criteria. The human makes the three decisions that would +otherwise be guessed; everything after approval becomes traceable and +machine-checkable. That is the useful unit: not a large process framework, but +a small feature package that can prove what was agreed, built, tested, reviewed, +and shipped. diff --git a/examples/diagram-json/compiled/evidence.md b/examples/diagram-json/compiled/evidence.md new file mode 100644 index 0000000..1c0e81b --- /dev/null +++ b/examples/diagram-json/compiled/evidence.md @@ -0,0 +1,24 @@ +# Evidence ledger: diagram-json-v1 + +- Approved plan lock: pending +- Test gate: `BLOCKED` +- Review gate: `BLOCKED` +- Ship gate: `BLOCKED` + +## Acceptance evidence + +| Criterion | Tasks | Result | Evidence | +|---|---|---|---| +| AC-1: The public serializer returns parseable schema-versioned graph JSON. | T-1, T-3 | `BLOCKED` | | +| AC-2: Serialization is deterministic and preserves ordered graph data. | T-1, T-3 | `BLOCKED` | | +| AC-3: Run serialization exposes only the compact signal and bottleneck overlay. | T-1, T-3 | `BLOCKED` | | +| AC-4: Existing ASCII output remains byte-compatible. | T-3 | `BLOCKED` | | +| AC-5: The API and schema types are publicly exported and documented. | T-2, T-3 | `BLOCKED` | | + +## Commands and checks + +## Review findings + +## Known gaps + +## Rollout and rollback diff --git a/examples/diagram-json/compiled/tasks.json b/examples/diagram-json/compiled/tasks.json new file mode 100644 index 0000000..f829f26 --- /dev/null +++ b/examples/diagram-json/compiled/tasks.json @@ -0,0 +1,60 @@ +{ + "feature_id": "diagram-json-v1", + "schema_version": 1, + "source_plan_status": "HUMAN_APPROVED", + "tasks": [ + { + "acceptance_criteria": [ + "AC-1", + "AC-2", + "AC-3" + ], + "depends_on": [], + "id": "T-1", + "rollback_boundary": "Revert the serializer and schema types in src/diagram.ts without touching the ASCII renderer.", + "title": "Define the v1 schema and pure serializer at the diagram boundary", + "validation": [ + "pnpm typecheck", + "pnpm exec tsx examples/05-diagram-printer/json-check.ts" + ] + }, + { + "acceptance_criteria": [ + "AC-5" + ], + "depends_on": [ + "T-1" + ], + "id": "T-2", + "rollback_boundary": "Remove the new src/index.ts exports and JSON documentation together.", + "title": "Expose and document the additive public contract", + "validation": [ + "pnpm typecheck", + "pnpm build" + ] + }, + { + "acceptance_criteria": [ + "AC-1", + "AC-2", + "AC-3", + "AC-4", + "AC-5" + ], + "depends_on": [ + "T-1", + "T-2" + ], + "id": "T-3", + "rollback_boundary": "Revert the JSON fixture/check and documentation; retain the pre-feature expected ASCII fixture.", + "title": "Add contract fixtures and prove text-renderer compatibility", + "validation": [ + "pnpm exec tsx examples/05-diagram-printer/json-check.ts", + "pnpm example:diagram", + "diff -u examples/05-diagram-printer/expected-output.txt <(pnpm --silent example:diagram)", + "pnpm typecheck", + "pnpm build" + ] + } + ] +} diff --git a/examples/diagram-json/compiled/test-matrix.json b/examples/diagram-json/compiled/test-matrix.json new file mode 100644 index 0000000..4817763 --- /dev/null +++ b/examples/diagram-json/compiled/test-matrix.json @@ -0,0 +1,197 @@ +{ + "feature_id": "diagram-json-v1", + "requirements": [ + { + "criterion": "The public serializer returns parseable schema-versioned graph JSON.", + "criterion_id": "AC-1", + "evidence": null, + "result": "BLOCKED", + "tasks": [ + "T-1", + "T-3" + ], + "validations": [ + { + "check": "pnpm typecheck", + "task_id": "T-1" + }, + { + "check": "pnpm exec tsx examples/05-diagram-printer/json-check.ts", + "task_id": "T-1" + }, + { + "check": "pnpm exec tsx examples/05-diagram-printer/json-check.ts", + "task_id": "T-3" + }, + { + "check": "pnpm example:diagram", + "task_id": "T-3" + }, + { + "check": "diff -u examples/05-diagram-printer/expected-output.txt <(pnpm --silent example:diagram)", + "task_id": "T-3" + }, + { + "check": "pnpm typecheck", + "task_id": "T-3" + }, + { + "check": "pnpm build", + "task_id": "T-3" + } + ] + }, + { + "criterion": "Serialization is deterministic and preserves ordered graph data.", + "criterion_id": "AC-2", + "evidence": null, + "result": "BLOCKED", + "tasks": [ + "T-1", + "T-3" + ], + "validations": [ + { + "check": "pnpm typecheck", + "task_id": "T-1" + }, + { + "check": "pnpm exec tsx examples/05-diagram-printer/json-check.ts", + "task_id": "T-1" + }, + { + "check": "pnpm exec tsx examples/05-diagram-printer/json-check.ts", + "task_id": "T-3" + }, + { + "check": "pnpm example:diagram", + "task_id": "T-3" + }, + { + "check": "diff -u examples/05-diagram-printer/expected-output.txt <(pnpm --silent example:diagram)", + "task_id": "T-3" + }, + { + "check": "pnpm typecheck", + "task_id": "T-3" + }, + { + "check": "pnpm build", + "task_id": "T-3" + } + ] + }, + { + "criterion": "Run serialization exposes only the compact signal and bottleneck overlay.", + "criterion_id": "AC-3", + "evidence": null, + "result": "BLOCKED", + "tasks": [ + "T-1", + "T-3" + ], + "validations": [ + { + "check": "pnpm typecheck", + "task_id": "T-1" + }, + { + "check": "pnpm exec tsx examples/05-diagram-printer/json-check.ts", + "task_id": "T-1" + }, + { + "check": "pnpm exec tsx examples/05-diagram-printer/json-check.ts", + "task_id": "T-3" + }, + { + "check": "pnpm example:diagram", + "task_id": "T-3" + }, + { + "check": "diff -u examples/05-diagram-printer/expected-output.txt <(pnpm --silent example:diagram)", + "task_id": "T-3" + }, + { + "check": "pnpm typecheck", + "task_id": "T-3" + }, + { + "check": "pnpm build", + "task_id": "T-3" + } + ] + }, + { + "criterion": "Existing ASCII output remains byte-compatible.", + "criterion_id": "AC-4", + "evidence": null, + "result": "BLOCKED", + "tasks": [ + "T-3" + ], + "validations": [ + { + "check": "pnpm exec tsx examples/05-diagram-printer/json-check.ts", + "task_id": "T-3" + }, + { + "check": "pnpm example:diagram", + "task_id": "T-3" + }, + { + "check": "diff -u examples/05-diagram-printer/expected-output.txt <(pnpm --silent example:diagram)", + "task_id": "T-3" + }, + { + "check": "pnpm typecheck", + "task_id": "T-3" + }, + { + "check": "pnpm build", + "task_id": "T-3" + } + ] + }, + { + "criterion": "The API and schema types are publicly exported and documented.", + "criterion_id": "AC-5", + "evidence": null, + "result": "BLOCKED", + "tasks": [ + "T-2", + "T-3" + ], + "validations": [ + { + "check": "pnpm typecheck", + "task_id": "T-2" + }, + { + "check": "pnpm build", + "task_id": "T-2" + }, + { + "check": "pnpm exec tsx examples/05-diagram-printer/json-check.ts", + "task_id": "T-3" + }, + { + "check": "pnpm example:diagram", + "task_id": "T-3" + }, + { + "check": "diff -u examples/05-diagram-printer/expected-output.txt <(pnpm --silent example:diagram)", + "task_id": "T-3" + }, + { + "check": "pnpm typecheck", + "task_id": "T-3" + }, + { + "check": "pnpm build", + "task_id": "T-3" + } + ] + } + ], + "schema_version": 1 +} diff --git a/examples/diagram-json/plan.json b/examples/diagram-json/plan.json new file mode 100644 index 0000000..3ccb58c --- /dev/null +++ b/examples/diagram-json/plan.json @@ -0,0 +1,82 @@ +{ + "schema_version": 1, + "feature_id": "diagram-json-v1", + "spec_path": "examples/diagram-json/spec.md", + "acceptance_criteria": [ + { + "id": "AC-1", + "text": "The public serializer returns parseable schema-versioned graph JSON." + }, + { + "id": "AC-2", + "text": "Serialization is deterministic and preserves ordered graph data." + }, + { + "id": "AC-3", + "text": "Run serialization exposes only the compact signal and bottleneck overlay." + }, + { + "id": "AC-4", + "text": "Existing ASCII output remains byte-compatible." + }, + { + "id": "AC-5", + "text": "The API and schema types are publicly exported and documented." + } + ], + "tasks": [ + { + "id": "T-1", + "title": "Define the v1 schema and pure serializer at the diagram boundary", + "depends_on": [], + "acceptance_criteria": [ + "AC-1", + "AC-2", + "AC-3" + ], + "validation": [ + "pnpm typecheck", + "pnpm exec tsx examples/05-diagram-printer/json-check.ts" + ], + "rollback_boundary": "Revert the serializer and schema types in src/diagram.ts without touching the ASCII renderer." + }, + { + "id": "T-2", + "title": "Expose and document the additive public contract", + "depends_on": [ + "T-1" + ], + "acceptance_criteria": [ + "AC-5" + ], + "validation": [ + "pnpm typecheck", + "pnpm build" + ], + "rollback_boundary": "Remove the new src/index.ts exports and JSON documentation together." + }, + { + "id": "T-3", + "title": "Add contract fixtures and prove text-renderer compatibility", + "depends_on": [ + "T-1", + "T-2" + ], + "acceptance_criteria": [ + "AC-1", + "AC-2", + "AC-3", + "AC-4", + "AC-5" + ], + "validation": [ + "pnpm exec tsx examples/05-diagram-printer/json-check.ts", + "pnpm example:diagram", + "diff -u examples/05-diagram-printer/expected-output.txt <(pnpm --silent example:diagram)", + "pnpm typecheck", + "pnpm build" + ], + "rollback_boundary": "Revert the JSON fixture/check and documentation; retain the pre-feature expected ASCII fixture." + } + ] +} diff --git a/examples/diagram-json/plan.lock.json b/examples/diagram-json/plan.lock.json new file mode 100644 index 0000000..d6874c7 --- /dev/null +++ b/examples/diagram-json/plan.lock.json @@ -0,0 +1,15 @@ +{ + "approved_at": "2026-07-16T12:00:00+00:00", + "approved_by": "Example Maintainer (simulated walkthrough)", + "invalidated_at": null, + "invalidation_reason": null, + "plan_path": "examples/diagram-json/plan.json", + "plan_sha256": "d1208003042a9d10f5efb010fc32fc7ac7bdefa427938260586e90daa0cb4414", + "schema_version": 1, + "source_commit": "aae685d2513cd25537284e4e68177411ace7ac9a", + "spec_path": "examples/diagram-json/spec.md", + "spec_sha256": "a943c81cf2a88d23d5b300e6b9dc1dafc80923a9b6b9ab5297a67b4e2054b9d5", + "status": "APPROVED", + "task_graph_path": "examples/diagram-json/compiled/tasks.json", + "task_graph_sha256": "d66d693df1ba7dd34f65ce93afea54006563c14d642a1bf0d1d9311b3fcfb37b" +} diff --git a/examples/diagram-json/questions.md b/examples/diagram-json/questions.md new file mode 100644 index 0000000..1b2e847 --- /dev/null +++ b/examples/diagram-json/questions.md @@ -0,0 +1,11 @@ +# Question ledger: diagram JSON output + +| ID | Question | Why it matters | Options | Recommendation | Answer | Source | Status/expiry | +|---|---|---|---|---|---|---|---| +| Q-1 | How should JSON enter the public API? | Changing `printFlowGraph` to return multiple shapes would weaken its existing string contract. | Add `serializeFlowGraph`; add a `format` option to `printFlowGraph`; replace the current return type. | Add the sibling `serializeFlowGraph` function. | Add the sibling function. | Simulated maintainer decision | Accepted for demo | +| Q-2 | Is the JSON an internal object dump or a supported contract? | Consumers need to know whether field changes are breaking. | Versioned minimal schema; undocumented internal dump. | Publish a minimal schema with `schemaVersion: 1`. | Use the versioned minimal schema. | Simulated maintainer decision | Accepted for demo | +| Q-3 | What run data belongs in v1? | Serializing the entire execution trace expands scope and can leak unrelated internals. | Edge signal overlay plus optional bottleneck; entire `FlowRun`; topology only. | Include the compact overlay and bottleneck already exposed by the text renderer. | Use the compact overlay, excluding the raw trace. | Simulated maintainer decision | Accepted for demo | + +Discoverable facts were answered from `src/diagram.ts`, `src/index.ts`, +`examples/05-diagram-printer/`, `package.json`, and the TypeScript configs. No +other product choice is hidden as an assumption in this draft. diff --git a/examples/diagram-json/request.md b/examples/diagram-json/request.md new file mode 100644 index 0000000..69c848c --- /dev/null +++ b/examples/diagram-json/request.md @@ -0,0 +1,14 @@ +# Product request + +Add machine-readable JSON output to the diagram printer while preserving the +current text output. + +## Initial projection + +- Domain: diagram serialization +- Actor: a library integrator consuming harness diagrams +- Input: an existing `DiagramGraph` and optional `FlowRun` +- Output: a stable JSON document +- User-visible goal: consume the same diagram data in scripts and tools +- Next operator: an application parses and stores or renders the JSON +- Verification boundary: public contract tests plus byte-compatible ASCII output diff --git a/examples/diagram-json/spec.md b/examples/diagram-json/spec.md new file mode 100644 index 0000000..b50f12b --- /dev/null +++ b/examples/diagram-json/spec.md @@ -0,0 +1,97 @@ +# Feature spec: versioned JSON diagram output + +## Outcome boundary + +- Domain: diagram serialization +- Actor: a library integrator consuming harness diagrams +- Input: `DiagramGraph`, optional `FlowRun`, and diagram options +- Output: a deterministic JSON string with `schemaVersion: 1` +- User-visible goal: use harness diagrams in scripts and non-terminal tools +- Next operator: a JSON parser, store, or renderer +- Verification boundary: public contract fixture and unchanged ASCII fixture + +## Problem and outcome + +`printFlowGraph` makes the harness structure inspectable in a terminal, but a +consumer cannot safely parse that presentation text. Add an independent public +serializer so machines can consume the topology and the same compact run +overlay without changing the text-rendering contract. + +## Non-goals + +- Replacing or redesigning the ASCII renderer. +- Serializing the complete `FlowRun` or private implementation state. +- Adding a command-line interface, network endpoint, or UI. +- Supporting arbitrary schema versions in this feature. + +## Scenarios + +1. A consumer serializes a static graph and parses its nodes and edges. +2. A consumer provides a run and receives edge signals and a bottleneck summary. +3. An existing caller continues to receive identical text from `printFlowGraph`. + +## Acceptance criteria + +- **AC-1:** `serializeFlowGraph(graph)` returns parseable JSON containing + `schemaVersion: 1`, graph identity, nodes, and edges. +- **AC-2:** repeated calls with the same inputs produce byte-identical output, + preserve node/edge order, and omit absent optional fields. +- **AC-3:** with a run, the document includes the compact edge-signal overlay + and optional bottleneck summary, respects the existing cost visibility option, + and does not expose the raw trace. +- **AC-4:** all existing `printFlowGraph` outputs remain byte-compatible with + `examples/05-diagram-printer/expected-output.txt`. +- **AC-5:** the serializer and its public schema types are exported from + `src/index.ts` and demonstrated in the diagram example documentation. + +## Interfaces and data + +Add the following sibling API rather than changing `printFlowGraph`: + +```ts +serializeFlowGraph( + graph: DiagramGraph, + run?: FlowRun, + options?: DiagramOptions, +): string +``` + +The v1 document has a top-level `schemaVersion`, graph identity, ordered `nodes` +and `edges`, plus optional `signals` and `bottleneck` fields. Public TypeScript +types define the exact schema. Serialization uses a documented, fixed JSON +format so fixture comparison is meaningful. + +## Invariants and trust boundaries + +- `printFlowGraph` retains its current signature and behavior. +- Graph and run inputs are not mutated. +- Only documented diagram fields and compact derived measurements cross the + serializer boundary. +- The output contains no raw prompts, model responses, environment values, or + execution trace. + +## Failure and recovery behavior + +Invalid graph references keep the current diagram-layer behavior; this feature +does not introduce a second graph validator. JSON serialization errors propagate +to the caller. A schema change requires a new schema version rather than a +silent v1 mutation. + +## Observability + +No telemetry is added. Contract fixtures, TypeScript checks, and executable +examples provide evidence at the library boundary. + +## Rollout and rollback + +This is an additive API with no migration. Rollback removes the serializer, +schema types, fixture, and documentation as one commit while leaving the text +renderer untouched. + +## Linked questions, ADRs, and gaps + +- Decisions: [Q-1 through Q-3](questions.md). +- ADR: not required; the change is additive and local to the existing diagram + boundary. +- Known gap: schema evolution beyond v1 is deliberately deferred until a real + consumer requires it. diff --git a/project.example.json b/project.example.json new file mode 100644 index 0000000..3252080 --- /dev/null +++ b/project.example.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "project": { + "name": "example-product", + "default_branch": "main", + "context": [ + "README.md", + "AGENTS.md", + "docs/architecture/", + "docs/decisions/" + ], + "commands": { + "build": "npm run build", + "lint": "npm run lint", + "test": "npm test", + "typecheck": "npm run typecheck" + }, + "high_risk_paths": [ + "migrations/**", + "auth/**", + "billing/**" + ] + }, + "workflow": { + "human_plan_approval": true, + "independent_review_for_high_risk": true, + "allow_pass_with_gaps": true + }, + "adapters": ["cursor", "claude", "codex", "github"] +} diff --git a/tests/test_boatstack.py b/tests/test_boatstack.py new file mode 100644 index 0000000..7e190d3 --- /dev/null +++ b/tests/test_boatstack.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SKILL = ROOT / "boatstack" +EXPORTER = SKILL / "scripts" / "export_repo.py" +COMPILER = SKILL / "scripts" / "compile_plan.py" +APPROVER = SKILL / "scripts" / "approve_plan.py" +CONFIG = ROOT / "project.example.json" + + +class BoatstackDistributionTests(unittest.TestCase): + def run_script(self, *args: object, expected: int = 0) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + [sys.executable, *map(str, args)], text=True, capture_output=True + ) + self.assertEqual(result.returncode, expected, result.stdout + result.stderr) + return result + + def test_branded_multi_host_export_and_drift_check(self) -> None: + with tempfile.TemporaryDirectory() as temp: + repo = Path(temp) + arguments = [ + EXPORTER, + "--repo", repo, + "--config", CONFIG, + "--adapter-name", "boatstack", + ] + self.run_script(*arguments, "--write") + result = self.run_script(*arguments, "--check") + self.assertIn("PASS", result.stdout) + self.assertTrue((repo / ".cursor/commands/plan-gate.md").is_file()) + self.assertTrue((repo / ".agents/skills/boatstack/SKILL.md").is_file()) + self.assertTrue((repo / ".claude/skills/boatstack/SKILL.md").is_file()) + + def test_compiler_and_hash_lock_block_stale_plan(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + spec = root / "spec.md" + plan = root / "plan.json" + compiled = root / "compiled" + lock = root / "plan.lock.json" + spec.write_text("# Accepted spec\n") + plan.write_text(json.dumps({ + "schema_version": 1, + "feature_id": "feature-one", + "acceptance_criteria": [{"id": "AC-1", "text": "observable result"}], + "tasks": [{ + "id": "T-1", + "title": "implement result", + "depends_on": [], + "acceptance_criteria": ["AC-1"], + "validation": ["python3 -m unittest"], + }], + })) + self.run_script(COMPILER, "--plan", plan, "--out-dir", compiled) + tasks = compiled / "tasks.json" + self.run_script( + APPROVER, + "--spec", spec, + "--plan", plan, + "--tasks", tasks, + "--approved-by", "Test Human", + "--approved-at", "2026-07-16T12:00:00+00:00", + "--source-commit", "test", + "--output", lock, + ) + self.run_script( + APPROVER, + "--spec", spec, + "--plan", plan, + "--tasks", tasks, + "--output", lock, + "--check", + ) + plan.write_text(plan.read_text() + "\n") + blocked = self.run_script( + APPROVER, + "--spec", spec, + "--plan", plan, + "--tasks", tasks, + "--output", lock, + "--check", + expected=1, + ) + self.assertIn("stale", blocked.stdout) + + def test_uncovered_acceptance_criterion_is_not_compiled(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + plan = root / "plan.json" + plan.write_text(json.dumps({ + "schema_version": 1, + "feature_id": "invalid", + "acceptance_criteria": [ + {"id": "AC-1", "text": "covered"}, + {"id": "AC-2", "text": "not covered"}, + ], + "tasks": [{ + "id": "T-1", + "depends_on": [], + "acceptance_criteria": ["AC-1"], + "validation": ["python3 -m unittest"], + }], + })) + result = self.run_script( + COMPILER, "--plan", plan, "--out-dir", root / "compiled", expected=1 + ) + self.assertIn("uncovered acceptance criteria", result.stdout) + + +if __name__ == "__main__": + unittest.main()