From 8de0d2c1310b7bd498ab0e02ecb9dc2a817dd88b Mon Sep 17 00:00:00 2001 From: bigboateng Date: Thu, 30 Jul 2026 11:28:44 +0100 Subject: [PATCH] feat(boatstack): capture visual evidence automatically during ship MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the plan declares relevant visual scenarios and the repository registers a capture command, pr-context/check-pr/publish-pr now run the harness themselves whenever evidence is missing or stale — capture is a delivery property, not a prescribed agent step. One hook in PreparePRContext covers all three verbs because publish re-derives the context and staleness co-varies with the fingerprinted product diff. - harness failure degrades to the recorded NOT_VERIFIED gap with a bounded pr_visual_evidence_capture_detail (never a context error, and deliberately outside the context fingerprint) - no registered command → exactly the prior prescribed path - a harness that dirties the working tree is refused with the violating paths (capture contract: write only BOATSTACK_CAPTURE_OUTPUT) - the require publication block is now a calm denial (workflow-visual-evidence-missing) whose solution set enumerates the recovery ladder: capture-evidence, capability-register, provision-capability, record-pr-visual-evidence, planning-write - ship-gate skill text: capturing is no longer the agent's step; its job is the privacy review of the captured PNGs Disclosure-Reviewed: reviewed — public-safe only, private facet kept out of this commit --- .../2026-07-30-auto-capture-on-ship.md | 3 + .../product-engineering-loop/capability.go | 2 +- .../product-engineering-loop/denial.go | 14 +++ .../denial_solutions.go | 41 +++++++ .../denial_solutions_conformance_test.go | 1 + .../product-engineering-loop/export.go | 2 +- .../product-engineering-loop/pr.go | 91 +++++++++++++- .../product-engineering-loop/pr_test.go | 115 ++++++++++++++++++ 8 files changed, 264 insertions(+), 5 deletions(-) create mode 100644 labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-30-auto-capture-on-ship.md diff --git a/labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-30-auto-capture-on-ship.md b/labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-30-auto-capture-on-ship.md new file mode 100644 index 000000000..42d4dc014 --- /dev/null +++ b/labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-30-auto-capture-on-ship.md @@ -0,0 +1,3 @@ +### Boatstack captures visual evidence itself during ship + +When a plan declares relevant visual scenarios and the repository registers a capture command (`project.commands.visual`), `pr-context`, `check-pr`, and `publish-pr` now run the capture harness automatically whenever evidence is missing or stale. The agent no longer has to be told to take or attach screenshots — its remaining step is the privacy review of the captured PNGs. A harness failure records a bounded gap in `pr_visual_evidence_capture_detail` and keeps today's `suggest` behavior; under `require`, the publication block is now a calm denial that names the full recovery ladder (`capture-evidence`, `capability-register`, `provision-capability`, `record-pr-visual-evidence`, or a `not_relevant` plan decision). A capture command that modifies the working tree is refused with the exact violating paths. Repositories without a registered command keep the prior prescribed path exactly. diff --git a/labs/12-product-engineering-loop/product-engineering-loop/capability.go b/labs/12-product-engineering-loop/product-engineering-loop/capability.go index b192c5e7d..8d4688b0d 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/capability.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/capability.go @@ -41,7 +41,7 @@ var capabilityRegistry = map[string]Capability{ "visual": { Name: "visual", CommandAliases: []string{"visual", "screenshot", "e2e"}, - AdmittedStages: []string{"BUILD", "TEST_PASSED"}, + AdmittedStages: []string{"BUILD", "TEST_PASSED", "REVIEW_PASSED", "PR_PREVIEW"}, RetryClass: "IDEMPOTENT_EXTERNAL", }, } diff --git a/labs/12-product-engineering-loop/product-engineering-loop/denial.go b/labs/12-product-engineering-loop/product-engineering-loop/denial.go index a537745d0..d661b04e1 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/denial.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/denial.go @@ -534,6 +534,20 @@ func denialFor(host string, finding SafetyFinding) Denial { d.Reassurance = "No push or pull request was made." return d + case "workflow-visual-evidence-missing": + target := "this delivery" + if finding.BlockingFeature != "" { + target = fmt.Sprintf("feature %q", finding.BlockingFeature) + } + d.Qualifier = "visual evidence is owed" + d.Detail = "PR publication is blocked until required visual evidence is current for " + target + "." + if reason := strings.TrimSpace(finding.Reason); reason != "" { + d.Detail += " Automatic capture reported: " + reason + "." + } + d.Detail += " Boatstack captures the plan's approved scenarios itself once a repository command is registered; declare pr_visual_evidence not_relevant (with a reason) only for a genuinely nonvisual change." + d.Reassurance = "No pull request was created or updated." + return d + case "operation-in-flight": d.Severity = SeverityAdvisory d.Qualifier = "already supervised" diff --git a/labs/12-product-engineering-loop/product-engineering-loop/denial_solutions.go b/labs/12-product-engineering-loop/product-engineering-loop/denial_solutions.go index 3eaf32fb9..518f2b3db 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/denial_solutions.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/denial_solutions.go @@ -81,6 +81,47 @@ func enumerateDenialSolutions(repo, host string, finding SafetyFinding) Solution appendObserveOption(&set, repo, "", "delivery.next") return set + case finding.Category == "workflow-visual-evidence-missing": + // Every rung of the recovery ladder, in preference order: run the + // registered harness, register or provision a harness, record + // externally captured evidence, or amend the plan's relevance — + // the only escape for a genuinely nonvisual change. + captureArgs := repoFlagArgs(repo) + captureOwed := []string{"--feature"} + if feature := strings.TrimSpace(finding.BlockingFeature); feature != "" { + captureArgs = append(captureArgs, "--feature", feature) + captureOwed = nil + } + appendSolution(&set, PrescribedCommand{ + Verb: "capture-evidence", Args: captureArgs, + RequiresHumanInput: captureOwed, AutoDerivable: len(captureOwed) == 0, + Transition: denialMarker("capture-evidence"), + }) + appendSolution(&set, PrescribedCommand{ + Verb: "capability-register", Args: append(repoFlagArgs(repo), "--capability", "visual"), + RequiresHumanInput: []string{"--command"}, + Transition: denialMarker("capability-register"), + }) + appendSolution(&set, PrescribedCommand{ + Verb: "provision-capability", Args: append(repoFlagArgs(repo), "--capability", "visual"), + AutoDerivable: true, + Transition: denialMarker("provision-capability"), + }) + appendSolution(&set, PrescribedCommand{ + Verb: "record-pr-visual-evidence", Args: repoFlagArgs(repo), + RequiresHumanInput: []string{"--manifest"}, + Transition: denialMarker("record-pr-visual-evidence"), + }) + if feature := strings.TrimSpace(finding.BlockingFeature); feature != "" { + // The artifact name and its Markdown (stdin) are authored content — owed. + appendSolution(&set, PrescribedCommand{ + Verb: "planning-write", Args: append(repoFlagArgs(repo), "--feature", feature), + RequiresHumanInput: []string{"--artifact"}, + Transition: denialMarker("planning-write"), + }) + } + return set + case strings.HasPrefix(finding.Category, "operation-"): // Observation-only by design: inspect the durable operation state before // any retry (the observed-effect discipline). diff --git a/labs/12-product-engineering-loop/product-engineering-loop/denial_solutions_conformance_test.go b/labs/12-product-engineering-loop/product-engineering-loop/denial_solutions_conformance_test.go index dfee3c8f4..a765741b5 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/denial_solutions_conformance_test.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/denial_solutions_conformance_test.go @@ -27,6 +27,7 @@ var denialCategoryInventory = []SafetyFinding{ {Category: "workflow-phase-bypass", Source: "planning-state", WorkflowStage: "DRAFT_PLAN", NextOperation: "plan-gate", BlockingFeature: "demo"}, {Category: "workflow-phase-bypass", Source: "planning-state", WorkflowStage: "NOT_STARTED", NextOperation: "planning-write", AttemptedPath: ".product-loop/features/demo/plan.md"}, {Category: "workflow-publication-bypass", BlockingFeature: "demo", BlockingSlice: "s1", Source: "tool-input"}, + {Category: "workflow-visual-evidence-missing", BlockingFeature: "demo", Source: "publication"}, {Category: "operation-in-flight", OperationID: "op_1", OperationState: "RUNNING", Source: "operation-state"}, {Category: "operation-already-succeeded", OperationID: "op_2", OperationState: "SUCCEEDED", Source: "operation-state"}, {Category: "operation-reconciliation-required", OperationID: "op_3", Source: "operation-state"}, diff --git a/labs/12-product-engineering-loop/product-engineering-loop/export.go b/labs/12-product-engineering-loop/product-engineering-loop/export.go index b642a2f2c..4119e0e7a 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/export.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/export.go @@ -322,7 +322,7 @@ func BuildExportBundle(configPath string, config ProjectConfig, rawConfig []byte "repair": "First run recovery-status --repo . with the user's exact free-form requested change, its observed source stage, bounded evidence when available, and --json. This resolver covers both active and current-branch published deliveries. On repair_active, read delivery-status, the current plan lock and acceptance criteria, the actual diff, and current receipts; classify the request and invoke record-change before any product edit. On draft_corrective_child, invoke record-change on the published parent, preserve its lock, receipts, slices, and publication evidence, and automatically prepare the suggested one-slice child plan with parent_delivery, exact correction, inherited intent, observed failure, returned existing_diff_sha256 and existing_changed_paths, verification requirements, and the resolved PR destination. Lead with The PR needs a corrective delivery. I prepared it for your approval. Then pause at the normal fingerprinted plan approval boundary; never reuse the parent's approval. An open PR reuses its verified head branch and is updated after fresh gates and publication confirmation. A merged or closed PR uses a fresh branch and PR; when a fingerprinted correction diff already exists, leave the original worktree untouched and transfer that exact reviewed diff into the fresh child only after approval. PUBLISHED_UNKNOWN may be drafted but its destination remains blocking at publication. Stop on BLOCKED and ask one targeted feature question using the returned blockers. If no managed target exists, continue ordinary conversation. Never discard pre-existing correction edits, edit runtime state directly, or bypass test, review, and ship gates. Never ask the user to repeat a denied push or PR mutation. If Cursor reports MainThreadShellExec not initialized, make Developer: Reload Window the one recovery action because Boatstack's hook did not start; reserve reinstall guidance for Boatstack runtime integrity errors.", "test-gate": "Read delivery-status and test only the active delivery slice. Run the internal repository safety check, build a requirement-to-evidence matrix, and treat self-authored tests as evidence rather than the sole oracle. If the active slice contains a systemic_boundary task, the evidence must prove the verification_oracle actively blocked or normalized a violation attempt (negative test). External writes require immutable target identity, transactional or fix-forward failure behavior, and an independent safety oracle. For relevant PR visual scenarios, use repository-owned capture first, then the host browser against the existing development server, one supplied launch instruction, or an approved machine-only runtime. Do not edit repository dependencies or configuration for capture. Review the exact PNGs for secrets and private data and import their temporary manifest with record-pr-visual-evidence. Commit the intentional slice product and evidence diff, then record-delivery-gate for the active feature and slice with --gate test and PASS or PASS_WITH_GAPS. Editing evidence Markdown alone never passes the gate. On pass respond Tests passed and make Run /review-gate the one next action. On failure respond Testing found a problem and make the required non-destructive repair the one next action.", "review-gate": "Read delivery-status and review the active slice's actual diff against authorized intent, invariants, risks, gaps, and test evidence. Run the internal repository safety check. Executable destructive capability is blocking even when ordinary tests pass. When workflow.maintain_changelog is true, verify the new CHANGELOG.md Unreleased entry accurately describes the actual reader-visible impact. When workflow.independent_review_for_high_risk is true and changed paths match project.high_risk_paths, use a human peer or separate agent and pass --reviewer-identity plus --review-method human_peer or separate_agent. On pass invoke record-delivery-gate for the same feature and slice with --gate review; it must reject changed or untested diffs, disallowed gaps, missing reviewer provenance, and malformed required changelog evidence. Then respond Review passed and make Run /ship-gate the one next action. When blocked respond Changes required and make the highest-priority blocking repair the one next action.", - "ship-gate": "Prepare a reviewer-ready PR only; do not merge or deploy without separate authorization. Require the current managed feature approval, lock, test evidence, review evidence, and a passing repository safety scan, and commit the intentional product/artifact diff before projection. Internally run pr-context --repo . --feature in json and template formats, project the approved intent, actual committed diff, decisions, evidence, gaps, rollout, rollback, safety outcome, and operator-only recovery boundary into its required pr.md path, then run check-pr --repo . --preview . Generate a clear, product-focused PR title that describes the user value or system outcome rather than listing technical components (do not use sequence prefixes like 'PR 1'). Always include why, what changed, review order, evidence, gaps/risks, rollout/rollback, and collapsed provenance. When PR visual evidence is relevant or unresolved, show the exact fingerprinted local PNGs and public-repository warning, render the structural Visual evidence section, and treat o or u as authorization for the exact PR package plus one Boatstack-owned evidence comment. Use a signed-in host browser to upload or update that comment when available and record the observed PR and comment URLs with record-pr-visual-publication; otherwise expose the local paths for manual attachment. Suggest records a visible gap; require blocks completed publication. Preserve an opened PR and fix forward from visual_pending after attachment failure. Add security/privacy, migration, or operations sections only when relevant. Show the exact title and rendered body before any GitHub mutation. If PR_ACTION is open, respond PR ready and render the one next action as: Reply `o` to open PR. If update, render: Reply `u` to update PR. If manual, preserve the preview and give one manual publication action. Continue accepting the full replies open PR and update PR for compatibility without advertising them. Only after the matching state-scoped shortcut or compatible full reply: commit only the reviewed pr.md, rerun check-pr and require the same preview fingerprint (PREVIEW_FINGERPRINT), then run publish-pr with --action open or update and that fingerprint. The publisher performs a non-force push and rechecks context before GitHub mutation. If the diff or evidence changes, regenerate instead. If a required check fails on the base branch too, record the evidence and recommend a separate repair PR. Never edit unrelated code in this approved feature branch; a policy-approved bypass requires explicit human authorization. After publication respond PR opened with the link and make Review the PR the one next action; never imply merge authorization. If publish-pr returns UPDATE_AVAILABLE, keep Review the PR as the only next action and append a collapsed update notice saying no files changed and /boatstack-update may be run from the clean default branch after this feature PR merges. Do not check for releases before successful publication.", + "ship-gate": "Prepare a reviewer-ready PR only; do not merge or deploy without separate authorization. Require the current managed feature approval, lock, test evidence, review evidence, and a passing repository safety scan, and commit the intentional product/artifact diff before projection. Internally run pr-context --repo . --feature in json and template formats, project the approved intent, actual committed diff, decisions, evidence, gaps, rollout, rollback, safety outcome, and operator-only recovery boundary into its required pr.md path, then run check-pr --repo . --preview . Generate a clear, product-focused PR title that describes the user value or system outcome rather than listing technical components (do not use sequence prefixes like 'PR 1'). Always include why, what changed, review order, evidence, gaps/risks, rollout/rollback, and collapsed provenance. When PR visual evidence is relevant, pr-context runs the registered repository capture command itself whenever evidence is missing or stale, so capturing is not your step: review the exact fingerprinted local PNGs for secrets and private data, show the public-repository warning, render the structural Visual evidence section, and treat o or u as authorization for the exact PR package plus one Boatstack-owned evidence comment. Fall back to manual capture (host browser, capture-evidence, record-pr-visual-evidence) only when the context reports the capture capability unavailable or names a harness failure in pr_visual_evidence_capture_detail. Use a signed-in host browser to upload or update that comment when available and record the observed PR and comment URLs with record-pr-visual-publication; otherwise expose the local paths for manual attachment. Suggest records a visible gap; require blocks completed publication. Preserve an opened PR and fix forward from visual_pending after attachment failure. Add security/privacy, migration, or operations sections only when relevant. Show the exact title and rendered body before any GitHub mutation. If PR_ACTION is open, respond PR ready and render the one next action as: Reply `o` to open PR. If update, render: Reply `u` to update PR. If manual, preserve the preview and give one manual publication action. Continue accepting the full replies open PR and update PR for compatibility without advertising them. Only after the matching state-scoped shortcut or compatible full reply: commit only the reviewed pr.md, rerun check-pr and require the same preview fingerprint (PREVIEW_FINGERPRINT), then run publish-pr with --action open or update and that fingerprint. The publisher performs a non-force push and rechecks context before GitHub mutation. If the diff or evidence changes, regenerate instead. If a required check fails on the base branch too, record the evidence and recommend a separate repair PR. Never edit unrelated code in this approved feature branch; a policy-approved bypass requires explicit human authorization. After publication respond PR opened with the link and make Review the PR the one next action; never imply merge authorization. If publish-pr returns UPDATE_AVAILABLE, keep Review the PR as the only next action and append a collapsed update notice saying no files changed and /boatstack-update may be run from the clean default branch after this feature PR merges. Do not check for releases before successful publication.", "boatstack-update": "Prepare a visible Boatstack infrastructure update; never mix it into product work or merge it. Force check-update with the current helper when available, but treat it and doctor as diagnostics rather than repair authority. If the installed helper is unavailable, resolve the latest stable tag from the official GitHub release endpoint and continue with the checksum-verified target installer; never require the broken helper to fix itself. If current, respond Boatstack is current with No action required. Before mutation fetch the default ref, then require the current default branch whose HEAD equals origin/ and no product or user-owned edits; otherwise respond Update postponed and give one recovery action. Ensure no update PR or branch already exists and create chore/update-boatstack-v. Fetch the installer from that exact release tag: it must checksum-verify the target helper before consulting installed state. Run it with BOATSTACK_MODE=update, BOATSTACK_VERSION=, BOATSTACK_REPO=, and BOATSTACK_YES=1. Exact installed hook and generated-state migrations are automatic. If the verified target helper reports REPAIR_AVAILABLE, show repair-status, state that the repair remains in this update PR, and ask whether the user wants the exact update rerun with BOATSTACK_REPAIR=1; do not infer that authority from the update request. Never offer repair for user-owned, mixed, malformed, symlinked, product, network, or authentication failures. Downgrades additionally require separately requested BOATSTACK_ALLOW_DOWNGRADE=1. The verified update preserves configuration, adapters, integrations, and unrelated host settings, writes any repair backup to Git-common state, runs doctor, and touches only Boatstack infrastructure. After installation use prepare-update-pr --repo . --version --json. Show version and repair provenance, release notes and link, integration state, title, body, changed paths, checksums, rollout, and rollback. Respond Boatstack update ready and render the one next action as: Reply `o` to open update PR. Continue accepting the full reply open update PR for compatibility without advertising it. Only the matching state-scoped shortcut or compatible full reply authorizes publish-update-pr --repo . --preview --preview-fingerprint . The deterministic publisher stages only previewed paths, reuses an existing update commit, pushes normally, reconciles the exact branch and PR after an interrupted response, and opens at most one reviewer-ready PR. Never stage, commit, push, or open the update PR through free-form terminal calls. If GitHub auth is unavailable, preserve the branch and give one manual publication action. If operation-status reports EXECUTING, wait; if it reports RECONCILE_REQUIRED, reconcile instead of repeating publication. After publication respond Update PR opened with the link and make Review the PR the one next action. On one collision or health failure, respond Update needs attention and make addressing that named problem the one next action. Never merge automatically.", "review": "Alias of review-gate: review the actual diff against approved intent, invariants, risks, gaps, and test evidence. Use Review passed or Changes required and the same single-action routing as review-gate.", "ship": "Alias of ship-gate: prepare and preview the exact reviewer-ready title and body before any GitHub mutation. Require the state-scoped reply o to open or u to update the PR before publication, recheck the preview against current evidence, and never merge or deploy. Keep pre-existing unrelated failures out of the approved feature branch. Use PR ready before confirmation or PR opened after publication.", diff --git a/labs/12-product-engineering-loop/product-engineering-loop/pr.go b/labs/12-product-engineering-loop/product-engineering-loop/pr.go index f08b95ee0..9e934d828 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/pr.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/pr.go @@ -24,6 +24,9 @@ type PRContextOptions struct { Feature string SliceID string Base string + // CaptureRunner overrides the harness executor for automatic visual + // evidence capture (tests); nil uses the repository-command runner. + CaptureRunner CaptureRunner } type PRSource struct { @@ -61,6 +64,10 @@ type PRContext struct { PRVisualEvidenceFingerprint string `json:"pr_visual_evidence_fingerprint"` PRVisualEvidenceRelevance string `json:"pr_visual_evidence_relevance"` PRVisualEvidenceSource string `json:"pr_visual_evidence_source"` + // PRVisualEvidenceCaptureDetail explains why automatic capture could not + // produce current evidence. Deliberately outside the context fingerprint: + // a flaky harness message must not destabilize preview equality. + PRVisualEvidenceCaptureDetail string `json:"pr_visual_evidence_capture_detail,omitempty"` PRVisualEvidence *PRVisualEvidenceManifest `json:"pr_visual_evidence,omitempty"` Sources []PRSource `json:"sources,omitempty"` PreviewPath string `json:"preview_path"` @@ -109,6 +116,73 @@ func planVisualDecision(repo, feature string) (string, string, []PRVisualScenari return relevance, "managed-plan", scenarios, nil } +// ensureCurrentPRVisualEvidence runs the registered capture harness when ship +// preparation finds declared-relevant visual evidence missing or stale, so +// capture is a delivery property, never a prescribed agent step. It returns a +// bounded detail string when automatic capture could not produce current +// evidence (no repository command registered, or the harness failed after its +// supervised attempts) — resolution then proceeds exactly as before this hook +// existed. The returned error is reserved for a harness contract violation: +// the capture command dirtied the working tree. +func ensureCurrentPRVisualEvidence(repo string, config ProjectConfig, mode, feature, base, diffHash string, runner CaptureRunner) (string, error) { + if mode != "managed" || normalizedPRVisualEvidencePolicy(config.Workflow.PRVisualEvidence) == "off" { + return "", nil + } + relevance, _, scenarios, err := planVisualDecision(repo, feature) + if err != nil || relevance != "relevant" || len(scenarios) == 0 { + return "", nil + } + key, err := visualEvidenceKey("managed", feature, "") + if err != nil { + return "", nil + } + if loaded, loadErr := LoadPRVisualEvidence(repo, key); loadErr == nil && loaded.Status == "PASS" && loaded.ProductDiffSHA256 == diffHash { + return "", nil + } + resolution, err := ResolveCapability("visual", config) + if err != nil || resolution.Kind != "repository-command" { + // The agent-mediated capture rungs (host browser, supplied launch) + // are deliberately not automated here; without a repository-owned + // command the prescribed path stays exactly as it was. + return "no visual capture capability is registered; register one with capability-register --capability visual --command ", nil + } + dirtyBefore, err := dirtyPaths(repo) + if err != nil { + return "", err + } + _, captureErr := CaptureEvidence(CaptureEvidenceOptions{Repo: repo, Capability: "visual", Feature: feature, Base: base, Runner: runner}) + dirtyAfter, dirtyErr := dirtyPaths(repo) + if dirtyErr == nil { + known := make(map[string]bool, len(dirtyBefore)) + for _, path := range dirtyBefore { + known[path] = true + } + var introduced []string + for _, path := range dirtyAfter { + if !known[path] { + introduced = append(introduced, path) + } + } + if len(introduced) > 0 { + return "", fmt.Errorf("the visual capture harness modified the working tree (%s); the capture contract allows writing only the file named by BOATSTACK_CAPTURE_OUTPUT — revert these paths and fix the registered command", strings.Join(introduced, ", ")) + } + } + if captureErr != nil { + return boundedCaptureDetail(captureErr.Error()), nil + } + return "", nil +} + +// boundedCaptureDetail folds a harness error into a single bounded line so a +// flaky harness cannot flood context JSON or denial text. +func boundedCaptureDetail(detail string) string { + detail = strings.Join(strings.Fields(detail), " ") + if len(detail) > 300 { + detail = detail[:300] + "…" + } + return detail +} + func resolvePRVisualEvidence(repo string, config ProjectConfig, mode, feature, head, diffHash string) (string, string, int, string, string, string, *PRVisualEvidenceManifest, error) { policy := normalizedPRVisualEvidencePolicy(config.Workflow.PRVisualEvidence) relevance, source := "unresolved", "agent-proposed" @@ -657,6 +731,10 @@ func PreparePRContext(options PRContextOptions) (PRContext, error) { if err != nil { return PRContext{}, err } + captureDetail, err := ensureCurrentPRVisualEvidence(repo, config, mode, options.Feature, base, SHA256Bytes(diff), options.CaptureRunner) + if err != nil { + return PRContext{}, err + } visualPolicy, visualStatus, visualCount, visualFingerprint, visualRelevance, visualSource, visualManifest, err := resolvePRVisualEvidence( repo, config, mode, options.Feature, head, SHA256Bytes(diff), ) @@ -684,8 +762,9 @@ func PreparePRContext(options PRContextOptions) (PRContext, error) { PRVisualEvidencePolicy: visualPolicy, PRVisualEvidenceStatus: visualStatus, PRVisualEvidenceCount: visualCount, PRVisualEvidenceFingerprint: visualFingerprint, PRVisualEvidenceRelevance: visualRelevance, PRVisualEvidenceSource: visualSource, - PRVisualEvidence: visualManifest, - PreviewPath: previewPath, + PRVisualEvidenceCaptureDetail: captureDetail, + PRVisualEvidence: visualManifest, + PreviewPath: previewPath, }, nil } @@ -1043,7 +1122,13 @@ func PublishPR(options PRPublishOptions) (string, error) { return "", fmt.Errorf("publication action must be open or update") } if context.PRVisualEvidencePolicy == "require" && context.PRVisualEvidenceStatus != "PASS" { - return "", fmt.Errorf("PR publication is blocked until required visual evidence is current") + finding := SafetyFinding{ + Category: "workflow-visual-evidence-missing", + Source: "publication", + BlockingFeature: context.Feature, + Reason: context.PRVisualEvidenceCaptureDetail, + } + return "", fmt.Errorf("%s", denialWithOptions(repo, "", finding).Render(RenderPlain)) } dirty, err := dirtyPaths(repo) if err != nil { diff --git a/labs/12-product-engineering-loop/product-engineering-loop/pr_test.go b/labs/12-product-engineering-loop/product-engineering-loop/pr_test.go index 2d963d463..fe40ea689 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/pr_test.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/pr_test.go @@ -2,6 +2,7 @@ package boatstack import ( "encoding/json" + "errors" "os" "os/exec" "path/filepath" @@ -744,6 +745,120 @@ func TestProductDiffChangeInvalidatesPassVisualEvidence(t *testing.T) { } } +// Invariant: declared-relevant visual evidence is captured by ship itself, +// never prescribed to the agent, and repeat context preparation is a no-op +// while the product diff is unchanged. +func TestPreparePRContextAutoCapturesRelevantVisualEvidence(t *testing.T) { + repo := prTestRepoConfigured(t, func(config *ProjectConfig) { + config.Workflow.PRVisualEvidence = "suggest" + config.Project.Commands["visual"] = "repo-owned-harness" + }) + activateManagedFeature(t, repo, "reviewer-ready") + runner := &stubCaptureRunner{write: func(request CaptureRequest) error { + writeTestPNG(t, request.OutputPath) + return nil + }} + context, err := PreparePRContext(PRContextOptions{Repo: repo, Feature: "reviewer-ready", CaptureRunner: runner}) + if err != nil { + t.Fatal(err) + } + if context.PRVisualEvidenceStatus != "PASS" || context.PRVisualEvidenceCount != 1 { + t.Fatalf("ship preparation did not capture declared evidence itself: %#v", context) + } + if context.PRVisualEvidenceCaptureDetail != "" { + t.Fatalf("successful auto-capture reported a gap: %s", context.PRVisualEvidenceCaptureDetail) + } + if runner.calls != 1 { + t.Fatalf("expected one harness run for one scenario, got %d", runner.calls) + } + again, err := PreparePRContext(PRContextOptions{Repo: repo, Feature: "reviewer-ready", CaptureRunner: runner}) + if err != nil { + t.Fatal(err) + } + if runner.calls != 1 { + t.Fatalf("auto-capture re-ran the harness for an unchanged product diff: %d calls", runner.calls) + } + if again.PRVisualEvidenceFingerprint != context.PRVisualEvidenceFingerprint { + t.Fatal("repeat context preparation destabilized the visual fingerprint") + } +} + +// Invariant: a failing harness degrades to exactly today's suggest behavior — +// a visible NOT_VERIFIED gap with a bounded detail, never a context error. +func TestAutoCaptureFailureDegradesToTodayUnderSuggest(t *testing.T) { + repo := prTestRepoConfigured(t, func(config *ProjectConfig) { + config.Workflow.PRVisualEvidence = "suggest" + config.Project.Commands["visual"] = "repo-owned-harness" + }) + activateManagedFeature(t, repo, "reviewer-ready") + runner := &stubCaptureRunner{write: func(request CaptureRequest) error { + return errors.New("dev server unreachable") + }} + context, err := PreparePRContext(PRContextOptions{Repo: repo, Feature: "reviewer-ready", CaptureRunner: runner}) + if err != nil { + t.Fatal(err) + } + if context.PRVisualEvidenceStatus != "NOT_VERIFIED" { + t.Fatalf("harness failure should leave the recorded gap, got %s", context.PRVisualEvidenceStatus) + } + if !strings.Contains(context.PRVisualEvidenceCaptureDetail, "dev server unreachable") { + t.Fatalf("capture detail lost the harness failure: %q", context.PRVisualEvidenceCaptureDetail) + } +} + +// Invariant (zero-value): without a registered repository command the runner +// is never consulted and behavior is exactly the prior prescribed path. +func TestAutoCaptureUnavailableCapabilityFallsBackToPrescribedPath(t *testing.T) { + repo := prTestRepoConfigured(t, func(config *ProjectConfig) { + config.Workflow.PRVisualEvidence = "suggest" + }) + activateManagedFeature(t, repo, "reviewer-ready") + runner := &stubCaptureRunner{write: func(request CaptureRequest) error { + t.Fatal("runner must not run without a registered repository command") + return nil + }} + context, err := PreparePRContext(PRContextOptions{Repo: repo, Feature: "reviewer-ready", CaptureRunner: runner}) + if err != nil { + t.Fatal(err) + } + if runner.calls != 0 || context.PRVisualEvidenceStatus != "NOT_VERIFIED" { + t.Fatalf("unavailable capability did not fall back cleanly: calls=%d status=%s", runner.calls, context.PRVisualEvidenceStatus) + } + if !strings.Contains(context.PRVisualEvidenceCaptureDetail, "capability-register") { + t.Fatalf("capture detail does not name the provisioning verb: %q", context.PRVisualEvidenceCaptureDetail) + } +} + +// Invariant: the require block is a calm denial carrying the computed +// solution set, so a blocked publication names its own recovery ladder. +func TestRequiredVisualEvidenceDenialCarriesSolutionSet(t *testing.T) { + repo := prTestRepoConfigured(t, func(config *ProjectConfig) { + config.Workflow.PRVisualEvidence = "require" + }) + activateManagedFeature(t, repo, "reviewer-ready") + context, err := PreparePRContext(PRContextOptions{Repo: repo, Feature: "reviewer-ready"}) + if err != nil { + t.Fatal(err) + } + if context.PRVisualEvidenceStatus != "BLOCKED" { + t.Fatalf("require with no evidence should block, got %s", context.PRVisualEvidenceStatus) + } + previewPath := writePreview(t, repo, context, "Require visual review evidence", visualEvidenceBody(managedPRBody(), context.PRVisualEvidenceStatus)) + preview, _, err := CheckPRPreview(repo, previewPath) + if err != nil { + t.Fatal(err) + } + _, err = PublishPR(PRPublishOptions{Repo: repo, PreviewPath: previewPath, ExpectedFingerprint: preview.Fingerprint, Action: "open"}) + if err == nil { + t.Fatal("required visual evidence did not block publication") + } + for _, needle := range []string{"required visual evidence", "capture-evidence", "reviewer-ready"} { + if !strings.Contains(err.Error(), needle) { + t.Fatalf("denial is missing %q:\n%s", needle, err.Error()) + } + } +} + func TestPublishPRRequiresExactConfirmationAndUsesBodyWithoutFrontmatter(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("fake gh fixture uses a POSIX shell; publication behavior is covered by cross-platform pure-Go checks")