diff --git a/labs/12-product-engineering-loop/boatstack-distribution/CONFIGURATION.md b/labs/12-product-engineering-loop/boatstack-distribution/CONFIGURATION.md index a316d2129..ecd7a510b 100644 --- a/labs/12-product-engineering-loop/boatstack-distribution/CONFIGURATION.md +++ b/labs/12-product-engineering-loop/boatstack-distribution/CONFIGURATION.md @@ -52,7 +52,7 @@ failed, or stale results. | Permit visible verification gaps | `workflow.allow_pass_with_gaps` | `false` rejects `PASS_WITH_GAPS` at delivery and PR gates; `true` retains the gaps as evidence. | | Maintain reader-facing history | `workflow.maintain_changelog` | Managed delivery and Boatstack-prepared PRs require a categorized `CHANGELOG.md` entry. | | Check for a systemic boundary | `workflow.boundary_analysis` | Planning guidance asks whether the request is a local symptom before scope expands. | -| Add frontend PR screenshots | `workflow.pr_visual_evidence` | `suggest` exposes missing screenshots as a gap; `require` blocks completed publication. | +| Add frontend PR screenshots | `workflow.pr_visual_evidence` | `suggest` exposes missing screenshots as a gap; `require` blocks completed publication. A plan that approves visual scenarios lifts `suggest` to require semantics for that feature; `off` and a per-feature `not_relevant` decision (with a reason) are the escapes. Boatstack captures registered scenarios automatically during ship. | | Render screenshots inline on a private PR | `workflow.visual_evidence_publish.*` | `mode: external-host` uploads the captured PNGs to an anonymous expiring host so the comment renders inline even on a private repo; opt-in, never automatic. | | Ignore old ambiguous deliveries | `workflow.ignored_deliveries` | Listed feature slugs are excluded from delivery-ambiguity resolution so past work stops blocking new work; new, unlisted ambiguous deliveries still pause. | | Pursue the PR to merge, not just to open | `delivery.terminal` | `merged` keeps the read-only flow advisors naming post-publish steps (watch checks, route corrections) until the PR is observed merged; the default `published` ends the flow when the PR is open, exactly as before. | diff --git a/labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-30-plan-approved-scenarios-escalate-to-require.md b/labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-30-plan-approved-scenarios-escalate-to-require.md new file mode 100644 index 000000000..27fc32e16 --- /dev/null +++ b/labs/12-product-engineering-loop/boatstack-distribution/release-notes/2026-07-30-plan-approved-scenarios-escalate-to-require.md @@ -0,0 +1,3 @@ +### Plan-approved visual scenarios now ship with require semantics + +When the approved plan declares `pr_visual_evidence` relevance `relevant` with scenarios, the configured `suggest` policy escalates to require semantics for that feature: publication blocks until current PASS evidence exists, even when no capture capability is registered yet. A plan that promises pixels can no longer ship with a `NOT_VERIFIED` gap. The escapes stay explicit: `off` globally, or a per-feature `not_relevant` decision with a reason for genuinely nonvisual changes. Because Boatstack now captures registered scenarios automatically during ship, provisioned repositories will not notice the escalation. One-time effect after upgrading: an existing `suggest` preview for a feature with declared scenarios reports a changed context fingerprint — regenerate the preview with `pr-context` before publishing. diff --git a/labs/12-product-engineering-loop/product-engineering-loop/capture.go b/labs/12-product-engineering-loop/product-engineering-loop/capture.go index 2c6414772..c3481d285 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/capture.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/capture.go @@ -163,7 +163,10 @@ func CaptureEvidence(options CaptureEvidenceOptions) (PRVisualEvidenceManifest, manifest := PRVisualEvidenceManifest{ Key: key, - Policy: config.Workflow.PRVisualEvidence, + // The manifest records the configured policy verbatim (informational); + // the effective policy — including plan-escalated require semantics — + // is re-derived by resolvePRVisualEvidence at every decode. + Policy: config.Workflow.PRVisualEvidence, Relevance: relevance, RelevanceSource: source, Status: "PASS", diff --git a/labs/12-product-engineering-loop/product-engineering-loop/capture_test.go b/labs/12-product-engineering-loop/product-engineering-loop/capture_test.go index 9b02af440..183e8dfdf 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/capture_test.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/capture_test.go @@ -112,7 +112,7 @@ func TestCaptureEvidenceProducesManifestTrustedByPRContext(t *testing.T) { if err != nil { t.Fatal(err) } - _, status, count, _, _, _, resolved, err := resolvePRVisualEvidence(repo, config, "managed", "reviewer-ready", head, diffHash) + _, status, count, _, _, _, _, resolved, err := resolvePRVisualEvidence(repo, config, "managed", "reviewer-ready", head, diffHash) if err != nil { t.Fatal(err) } diff --git a/labs/12-product-engineering-loop/product-engineering-loop/denial.go b/labs/12-product-engineering-loop/product-engineering-loop/denial.go index d661b04e1..8981cc129 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/denial.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/denial.go @@ -541,6 +541,9 @@ func denialFor(host string, finding SafetyFinding) Denial { } d.Qualifier = "visual evidence is owed" d.Detail = "PR publication is blocked until required visual evidence is current for " + target + "." + if finding.PolicySource == "plan-escalated" { + d.Detail += " The approved plan declares visual scenarios, so the configured suggest policy ships with require semantics for this feature." + } if reason := strings.TrimSpace(finding.Reason); reason != "" { d.Detail += " Automatic capture reported: " + reason + "." } diff --git a/labs/12-product-engineering-loop/product-engineering-loop/pr.go b/labs/12-product-engineering-loop/product-engineering-loop/pr.go index 9e934d828..c3ef0c278 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/pr.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/pr.go @@ -64,6 +64,9 @@ type PRContext struct { PRVisualEvidenceFingerprint string `json:"pr_visual_evidence_fingerprint"` PRVisualEvidenceRelevance string `json:"pr_visual_evidence_relevance"` PRVisualEvidenceSource string `json:"pr_visual_evidence_source"` + // PRVisualEvidencePolicySource is "configured", or "plan-escalated" when + // a plan-approved visual decision lifts suggest to require semantics. + PRVisualEvidencePolicySource string `json:"pr_visual_evidence_policy_source,omitempty"` // PRVisualEvidenceCaptureDetail explains why automatic capture could not // produce current evidence. Deliberately outside the context fingerprint: // a flaky harness message must not destabilize preview equality. @@ -183,7 +186,7 @@ func boundedCaptureDetail(detail string) string { return detail } -func resolvePRVisualEvidence(repo string, config ProjectConfig, mode, feature, head, diffHash string) (string, string, int, string, string, string, *PRVisualEvidenceManifest, error) { +func resolvePRVisualEvidence(repo string, config ProjectConfig, mode, feature, head, diffHash string) (string, string, int, string, string, string, string, *PRVisualEvidenceManifest, error) { policy := normalizedPRVisualEvidencePolicy(config.Workflow.PRVisualEvidence) relevance, source := "unresolved", "agent-proposed" var scenarios []PRVisualScenario @@ -191,12 +194,20 @@ func resolvePRVisualEvidence(repo string, config ProjectConfig, mode, feature, h var err error relevance, source, scenarios, err = planVisualDecision(repo, feature) if err != nil { - return "", "", 0, "", "", "", nil, err + return "", "", 0, "", "", "", "", nil, err } } + // The effective policy is what everything downstream (coercion, preview + // frontmatter, the publication block) reads. Escalation keys on the + // plan's own decision, before any manifest overrides it. + policySource := "configured" + if visualEscalationApplies(policy, relevance, len(scenarios)) { + policy = "require" + policySource = "plan-escalated" + } key, err := visualEvidenceKey(mode, feature, head) if err != nil { - return "", "", 0, "", "", "", nil, err + return "", "", 0, "", "", "", "", nil, err } status := "NOT_APPLICABLE" var manifest *PRVisualEvidenceManifest @@ -235,9 +246,9 @@ func resolvePRVisualEvidence(repo string, config ProjectConfig, mode, feature, h } raw, err := MarshalJSON(payload) if err != nil { - return "", "", 0, "", "", "", nil, err + return "", "", 0, "", "", "", "", nil, err } - return policy, status, count, SHA256Bytes(raw), relevance, source, manifest, nil + return policy, status, count, SHA256Bytes(raw), relevance, source, policySource, manifest, nil } type PRPublishOptions struct { @@ -735,7 +746,7 @@ func PreparePRContext(options PRContextOptions) (PRContext, error) { if err != nil { return PRContext{}, err } - visualPolicy, visualStatus, visualCount, visualFingerprint, visualRelevance, visualSource, visualManifest, err := resolvePRVisualEvidence( + visualPolicy, visualStatus, visualCount, visualFingerprint, visualRelevance, visualSource, visualPolicySource, visualManifest, err := resolvePRVisualEvidence( repo, config, mode, options.Feature, head, SHA256Bytes(diff), ) if err != nil { @@ -762,6 +773,7 @@ func PreparePRContext(options PRContextOptions) (PRContext, error) { PRVisualEvidencePolicy: visualPolicy, PRVisualEvidenceStatus: visualStatus, PRVisualEvidenceCount: visualCount, PRVisualEvidenceFingerprint: visualFingerprint, PRVisualEvidenceRelevance: visualRelevance, PRVisualEvidenceSource: visualSource, + PRVisualEvidencePolicySource: visualPolicySource, PRVisualEvidenceCaptureDetail: captureDetail, PRVisualEvidence: visualManifest, PreviewPath: previewPath, @@ -1127,6 +1139,7 @@ func PublishPR(options PRPublishOptions) (string, error) { Source: "publication", BlockingFeature: context.Feature, Reason: context.PRVisualEvidenceCaptureDetail, + PolicySource: context.PRVisualEvidencePolicySource, } return "", fmt.Errorf("%s", denialWithOptions(repo, "", finding).Render(RenderPlain)) } diff --git a/labs/12-product-engineering-loop/product-engineering-loop/pr_test.go b/labs/12-product-engineering-loop/product-engineering-loop/pr_test.go index fe40ea689..10c9cc923 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/pr_test.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/pr_test.go @@ -620,9 +620,14 @@ func TestManagedPRVisualEvidenceUsesApprovedPlanScenarios(t *testing.T) { if err != nil { t.Fatal(err) } - if context.Mode != "managed" || context.PRVisualEvidenceRelevance != "relevant" || context.PRVisualEvidenceSource != "managed-plan" || context.PRVisualEvidenceStatus != "NOT_VERIFIED" { + // A plan-approved visual decision escalates suggest to require semantics, + // so missing evidence is BLOCKED here, not a shippable NOT_VERIFIED gap. + if context.Mode != "managed" || context.PRVisualEvidenceRelevance != "relevant" || context.PRVisualEvidenceSource != "managed-plan" || context.PRVisualEvidenceStatus != "BLOCKED" { t.Fatalf("managed visual decision was not projected: %#v", context) } + if context.PRVisualEvidencePolicy != "require" || context.PRVisualEvidencePolicySource != "plan-escalated" { + t.Fatalf("plan-approved scenarios did not escalate suggest: %#v", context) + } previewPath := writePreview(t, repo, context, "Expose approved visual review scenario", visualEvidenceBody(managedPRBody(), context.PRVisualEvidenceStatus)) if _, _, err := CheckPRPreview(repo, previewPath); err != nil { t.Fatal(err) @@ -728,15 +733,17 @@ func TestProductDiffChangeInvalidatesPassVisualEvidence(t *testing.T) { t.Fatal(err) } changedDiff := strings.Repeat("c", 64) - _, status, _, _, _, _, _, err := resolvePRVisualEvidence(repo, config, "managed", "reviewer-ready", context.HeadBranch, changedDiff) + // The plan declares relevant scenarios, so suggest ships with require + // semantics: stale evidence is BLOCKED, never a shippable gap. + _, status, _, _, _, _, _, _, err := resolvePRVisualEvidence(repo, config, "managed", "reviewer-ready", context.HeadBranch, changedDiff) if err != nil { t.Fatal(err) } - if status != "NOT_VERIFIED" { - t.Fatalf("product change did not stale the evidence: %s", status) + if status != "BLOCKED" { + t.Fatalf("product change did not stale the evidence to a block: %s", status) } config.Workflow.PRVisualEvidence = "require" - _, status, _, _, _, _, _, err = resolvePRVisualEvidence(repo, config, "managed", "reviewer-ready", context.HeadBranch, changedDiff) + _, status, _, _, _, _, _, _, err = resolvePRVisualEvidence(repo, config, "managed", "reviewer-ready", context.HeadBranch, changedDiff) if err != nil { t.Fatal(err) } @@ -745,6 +752,92 @@ func TestProductDiffChangeInvalidatesPassVisualEvidence(t *testing.T) { } } +// Invariant: a plan that promises pixels cannot ship without them — suggest +// escalates to require semantics for plan-approved scenarios even when no +// capture capability is registered, and the denial says why. +func TestSuggestEscalatesToRequireWhenPlanDeclaresScenarios(t *testing.T) { + repo := prTestRepoConfigured(t, func(config *ProjectConfig) { + config.Workflow.PRVisualEvidence = "suggest" + }) + activateManagedFeature(t, repo, "reviewer-ready") + context, err := PreparePRContext(PRContextOptions{Repo: repo, Feature: "reviewer-ready"}) + if err != nil { + t.Fatal(err) + } + if context.PRVisualEvidencePolicy != "require" || context.PRVisualEvidencePolicySource != "plan-escalated" || context.PRVisualEvidenceStatus != "BLOCKED" { + t.Fatalf("plan-approved scenarios did not escalate: %#v", context) + } + previewPath := writePreview(t, repo, context, "Escalate approved visual scenarios", visualEvidenceBody(managedPRBody(), context.PRVisualEvidenceStatus)) + preview, _, err := CheckPRPreview(repo, previewPath) + if err != nil { + t.Fatal(err) + } + _, err = PublishPR(PRPublishOptions{Repo: repo, PreviewPath: previewPath, ExpectedFingerprint: preview.Fingerprint, Action: "open"}) + if err == nil { + t.Fatal("escalated require did not block publication") + } + for _, needle := range []string{"required visual evidence", "require semantics", "capability-register"} { + if !strings.Contains(err.Error(), needle) { + t.Fatalf("escalated denial is missing %q:\n%s", needle, err.Error()) + } + } +} + +// The escalation predicate's full semantics table: `off` is the global +// opt-out, not_relevant the per-feature escape, and configured require needs +// no escalation. Capability availability is deliberately absent — a missing +// harness is a provisioning gap, not a license to ship unverified. +func TestVisualEscalationPredicate(t *testing.T) { + cases := []struct { + policy, relevance string + scenarios int + want bool + }{ + {"suggest", "relevant", 1, true}, + {"suggest", "relevant", 3, true}, + {"suggest", "relevant", 0, false}, + {"suggest", "not_relevant", 0, false}, + {"suggest", "unresolved", 1, false}, + {"require", "relevant", 1, false}, + {"off", "relevant", 1, false}, + } + for _, c := range cases { + if got := visualEscalationApplies(c.policy, c.relevance, c.scenarios); got != c.want { + t.Errorf("visualEscalationApplies(%s, %s, %d) = %v, want %v", c.policy, c.relevance, c.scenarios, got, c.want) + } + } +} + +// Invariant: a not_relevant plan decision (with its reason) keeps the +// configured suggest semantics — the per-feature escape for nonvisual changes. +func TestNotRelevantPlanKeepsSuggestSemantics(t *testing.T) { + repo := prTestRepoConfigured(t, func(config *ProjectConfig) { + config.Workflow.PRVisualEvidence = "suggest" + }) + directory := filepath.Join(repo, ".product-loop", "features", "log-rotation") + if err := os.MkdirAll(directory, 0o755); err != nil { + t.Fatal(err) + } + plan := validPlan() + plan["feature_id"] = "log-rotation" + plan["pr_visual_evidence"] = map[string]any{ + "relevance": "not_relevant", + "reason": "backend log rotation has no reviewer-visible surface", + } + writeMarkdownPlan(t, filepath.Join(directory, "plan.md"), plan, true) + config, _, err := LoadConfig(filepath.Join(repo, ".product-loop", "project.json")) + if err != nil { + t.Fatal(err) + } + policy, status, _, _, _, _, policySource, _, err := resolvePRVisualEvidence(repo, config, "managed", "log-rotation", "feat/log-rotation", strings.Repeat("d", 64)) + if err != nil { + t.Fatal(err) + } + if policy != "suggest" || policySource != "configured" || status != "NOT_APPLICABLE" { + t.Fatalf("not_relevant did not keep suggest semantics: policy=%s source=%s status=%s", policy, policySource, status) + } +} + // Invariant: declared-relevant visual evidence is captured by ship itself, // never prescribed to the agent, and repeat context preparation is a no-op // while the product diff is unchanged. @@ -783,9 +876,9 @@ func TestPreparePRContextAutoCapturesRelevantVisualEvidence(t *testing.T) { } } -// Invariant: a failing harness degrades to exactly today's suggest behavior — -// a visible NOT_VERIFIED gap with a bounded detail, never a context error. -func TestAutoCaptureFailureDegradesToTodayUnderSuggest(t *testing.T) { +// Invariant: a failing harness never errors context preparation — it records +// a bounded detail, and the plan-escalated require semantics hold the block. +func TestAutoCaptureFailureRecordsBoundedDetail(t *testing.T) { repo := prTestRepoConfigured(t, func(config *ProjectConfig) { config.Workflow.PRVisualEvidence = "suggest" config.Project.Commands["visual"] = "repo-owned-harness" @@ -798,8 +891,8 @@ func TestAutoCaptureFailureDegradesToTodayUnderSuggest(t *testing.T) { if err != nil { t.Fatal(err) } - if context.PRVisualEvidenceStatus != "NOT_VERIFIED" { - t.Fatalf("harness failure should leave the recorded gap, got %s", context.PRVisualEvidenceStatus) + if context.PRVisualEvidenceStatus != "BLOCKED" { + t.Fatalf("escalated require should block on a failed capture, got %s", context.PRVisualEvidenceStatus) } if !strings.Contains(context.PRVisualEvidenceCaptureDetail, "dev server unreachable") { t.Fatalf("capture detail lost the harness failure: %q", context.PRVisualEvidenceCaptureDetail) @@ -821,7 +914,7 @@ func TestAutoCaptureUnavailableCapabilityFallsBackToPrescribedPath(t *testing.T) if err != nil { t.Fatal(err) } - if runner.calls != 0 || context.PRVisualEvidenceStatus != "NOT_VERIFIED" { + if runner.calls != 0 || context.PRVisualEvidenceStatus != "BLOCKED" { t.Fatalf("unavailable capability did not fall back cleanly: calls=%d status=%s", runner.calls, context.PRVisualEvidenceStatus) } if !strings.Contains(context.PRVisualEvidenceCaptureDetail, "capability-register") { diff --git a/labs/12-product-engineering-loop/product-engineering-loop/references/config-schema.md b/labs/12-product-engineering-loop/product-engineering-loop/references/config-schema.md index 2d29ce72e..e5df48667 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/references/config-schema.md +++ b/labs/12-product-engineering-loop/product-engineering-loop/references/config-schema.md @@ -79,7 +79,7 @@ This is the exhaustive serialization contract, not a list of recommended user ed - `allow_pass_with_gaps` (boolean, optional): Deterministic gate control. `false` rejects `PASS_WITH_GAPS`; `true` preserves explicit gaps. - `maintain_changelog` (boolean, optional): Whether a reader-visible `CHANGELOG.md` entry is required for each delivery slice. - `boundary_analysis` (boolean, optional): Agent-mediated planning guidance that presents local repair versus programmatic enforcement as a material product decision. -- `pr_visual_evidence` (string, optional): `off`, `suggest`, or `require`. Omission is `off`. Relevant PRs use machine-local PNG evidence without committing media to Git; `suggest` records missing evidence as a visible gap and `require` blocks completed publication. +- `pr_visual_evidence` (string, optional): `off`, `suggest`, or `require`. Omission is `off`. Relevant PRs use machine-local PNG evidence without committing media to Git; `suggest` records missing evidence as a visible gap and `require` blocks completed publication. When the approved plan declares `relevance: relevant` with scenarios, `suggest` ships with require semantics for that feature (a plan that promises pixels cannot ship without them) — even when no capture capability is registered yet. The two escapes are `off` (global) and a `not_relevant` plan decision with a reason (per feature, for genuinely nonvisual changes). Boatstack runs a registered capture command (`project.commands.visual`) automatically during ship, so under normal provisioning the escalation is invisible. - `visual_evidence_publish` (object, optional): Agent-mediated publish control for how captured PNG bytes reach the pull-request comment. Omission keeps the default: commit the bytes to a public Boatstack-owned evidence branch and render them inline, but only for a **public** GitHub origin (a private origin falls back to manual attachment). Fields: - `mode` (string, optional): `external-host` opts the repository — including a **private** one — into uploading the exact PNG bytes to an anonymous expiring host so the comment renders inline anywhere. It is **never auto-selected** because it publishes screenshot bytes to a third party; only this explicit value turns it on. Empty keeps the default public-branch behavior. - `host` (string, optional): `litterbox` (default) or `catbox`. Only meaningful when `mode` is `external-host`. `litterbox` auto-expires uploads; `catbox` is permanent. diff --git a/labs/12-product-engineering-loop/product-engineering-loop/safety.go b/labs/12-product-engineering-loop/product-engineering-loop/safety.go index 6d15550c0..e301fa634 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/safety.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/safety.go @@ -26,6 +26,9 @@ type SafetyFinding struct { AttemptedPath string `json:"attempted_path,omitempty"` OperationID string `json:"operation_id,omitempty"` OperationState string `json:"operation_state,omitempty"` + // PolicySource explains a policy-derived denial: "plan-escalated" when a + // plan-approved visual decision lifts suggest to require semantics. + PolicySource string `json:"policy_source,omitempty"` AttemptNumber int `json:"attempt_number,omitempty"` ReconciliationRequired bool `json:"reconciliation_required,omitempty"` // RepeatCount is how many times this same denial (category at stage) has diff --git a/labs/12-product-engineering-loop/product-engineering-loop/visual_evidence.go b/labs/12-product-engineering-loop/product-engineering-loop/visual_evidence.go index 1f22727df..9fc9bfe70 100644 --- a/labs/12-product-engineering-loop/product-engineering-loop/visual_evidence.go +++ b/labs/12-product-engineering-loop/product-engineering-loop/visual_evidence.go @@ -130,6 +130,19 @@ func normalizedPRVisualEvidencePolicy(value string) string { return value } +// visualEscalationApplies decides when the configured suggest policy ships +// with require semantics: the approved plan declares visual relevance with +// concrete scenarios. A plan that promises pixels cannot ship without them. +// The escalation is deliberately independent of capture-capability +// availability — a missing harness is a provisioning gap the publication +// denial names, never a license to ship unverified. `off` remains the global +// opt-out (the predicate never fires) and a not_relevant plan decision (with +// its reason) remains the per-feature escape for genuinely nonvisual changes. +// control-law: plan-approved-scenarios-imply-require +func visualEscalationApplies(configured, relevance string, scenarioCount int) bool { + return configured == "suggest" && relevance == "relevant" && scenarioCount > 0 +} + func visualEvidenceKey(mode, feature, head string) (string, error) { key := feature if mode == "ad-hoc" {