From 441dc65f0af1b578c886edd6e025ba2fad442446 Mon Sep 17 00:00:00 2001 From: Pengfei Hu Date: Mon, 6 Jul 2026 21:36:19 -0700 Subject: [PATCH] Add the Claude Code plugin marketplace; fix silently-broken skill frontmatter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The repo now doubles as a Claude Code plugin marketplace — the symmetric counterpart of the Codex marketplace under .agents/plugins/: - .claude-plugin/marketplace.json (marketplace: agents-shipgate) with one skill-only plugin entry sourcing ./plugins/claude-code. - plugins/claude-code/: plugin.json + the /shipgate command + the agents-shipgate skill, both byte-identical copies of the canonical sources (.claude/commands/shipgate.md, skills/agents-shipgate/). Skill-only by design, like the Codex plugin: no hooks, no MCP config, no scanner binary — hooks stay on the explicit install-hooks CLI path where the agents-shipgate binary must exist. - tests/test_claude_code_plugin_package.py mirrors the Codex package contract: installable manifest, versions pinned to __version__ (so release bumps cannot strand the marketplace), skill-only shape, byte-identity with canonical sources, and frontmatter-parses tests. Fixed in passing, caught by `claude plugin validate`: the canonical SKILL.md and /shipgate command shipped YAML frontmatter whose description contains an unquoted ":" — YAML parse fails and Claude Code loads the file with silently-empty metadata, which kills description-based skill auto-triggering for every existing install. Both are now quoted; all byte-identical copies synced (adoption-kits/claude-code-skill SKILL.md included, with EXPECTED_CLAUDE_CODE_SKILL_RENDER_SHA256 updated and the outgoing hash appended to the kit's prior_render_sha256 per the kit-change protocol); the new frontmatter test prevents regression. Wiring: README Claude Code section, docs/agents/use-with-claude-code.md (marketplace subsection incl. team pre-provisioning via extraKnownMarketplaces), CHANGELOG. `claude plugin validate` passes at both marketplace and plugin level; full CI-style suite + static-only lint + ruff green; self-verify routes the agent-instructions changes to human review (SHIP-VERIFY-TRUST-ROOT-TOUCHED) — which this PR is. Co-Authored-By: Claude Fable 5 --- .claude-plugin/marketplace.json | 33 ++++ .claude/commands/shipgate.md | 2 +- CHANGELOG.md | 15 ++ README.md | 14 ++ .../.agents-shipgate-kit-metadata.json | 3 +- adoption-kits/claude-code-skill/SKILL.md | 2 +- docs/agents/use-with-claude-code.md | 21 +++ .../claude-code/.claude-plugin/plugin.json | 21 +++ plugins/claude-code/commands/shipgate.md | 81 +++++++++ .../skills/agents-shipgate/SKILL.md | 110 +++++++++++++ .../ci-recipes/advisory-pr-comment.yml | 26 +++ .../prompts/add-shipgate-to-repo.md | 155 ++++++++++++++++++ .../prompts/decide-shipgate-relevance.md | 151 +++++++++++++++++ .../prompts/explain-finding-to-user.md | 89 ++++++++++ .../prompts/fix-top-finding.md | 72 ++++++++ .../prompts/recommend-fixes.md | 82 +++++++++ .../prompts/stabilize-strict-mode.md | 77 +++++++++ .../prompts/triage-false-positive.md | 90 ++++++++++ .../prompts/upgrade-shipgate-version.md | 73 +++++++++ .../prompts/verify-agent-diff.md | 96 +++++++++++ skills/agents-shipgate/SKILL.md | 2 +- tests/test_agent_instructions_renderers.py | 2 +- tests/test_claude_code_plugin_package.py | 116 +++++++++++++ 23 files changed, 1328 insertions(+), 5 deletions(-) create mode 100644 .claude-plugin/marketplace.json create mode 100644 plugins/claude-code/.claude-plugin/plugin.json create mode 100644 plugins/claude-code/commands/shipgate.md create mode 100644 plugins/claude-code/skills/agents-shipgate/SKILL.md create mode 100644 plugins/claude-code/skills/agents-shipgate/ci-recipes/advisory-pr-comment.yml create mode 100644 plugins/claude-code/skills/agents-shipgate/prompts/add-shipgate-to-repo.md create mode 100644 plugins/claude-code/skills/agents-shipgate/prompts/decide-shipgate-relevance.md create mode 100644 plugins/claude-code/skills/agents-shipgate/prompts/explain-finding-to-user.md create mode 100644 plugins/claude-code/skills/agents-shipgate/prompts/fix-top-finding.md create mode 100644 plugins/claude-code/skills/agents-shipgate/prompts/recommend-fixes.md create mode 100644 plugins/claude-code/skills/agents-shipgate/prompts/stabilize-strict-mode.md create mode 100644 plugins/claude-code/skills/agents-shipgate/prompts/triage-false-positive.md create mode 100644 plugins/claude-code/skills/agents-shipgate/prompts/upgrade-shipgate-version.md create mode 100644 plugins/claude-code/skills/agents-shipgate/prompts/verify-agent-diff.md create mode 100644 tests/test_claude_code_plugin_package.py diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 00000000..284317c5 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,33 @@ +{ + "name": "agents-shipgate", + "owner": { + "name": "Three Moons Lab", + "email": "help@threemoonslab.com" + }, + "description": "Agents Shipgate — the deterministic merge gate for AI-generated agent capability changes. Skill-only plugin: workflows for shipgate check, agents-shipgate verify, and shipgate audit --host; the scanner runs through the agents-shipgate CLI installed locally.", + "plugins": [ + { + "name": "agents-shipgate", + "source": "./plugins/claude-code", + "description": "Run Agents Shipgate Tool-Use Readiness workflows from Claude Code: detect whether Shipgate applies, bootstrap the advisory gate, verify agent-capability PRs, and triage findings without fabricating human approval. Requires the agents-shipgate CLI (pipx install agents-shipgate, contract v10 or newer).", + "version": "0.14.0", + "author": { + "name": "Three Moons Lab", + "url": "https://threemoonslab.com/" + }, + "homepage": "https://threemoonslab.com/", + "repository": "https://github.com/ThreeMoonsLab/agents-shipgate", + "license": "Apache-2.0", + "keywords": [ + "agents", + "claude-code", + "tool-use-readiness", + "release-gate", + "mcp-security", + "policy-as-code" + ], + "category": "security", + "strict": true + } + ] +} diff --git a/.claude/commands/shipgate.md b/.claude/commands/shipgate.md index a048a42c..68233663 100644 --- a/.claude/commands/shipgate.md +++ b/.claude/commands/shipgate.md @@ -1,5 +1,5 @@ --- -description: Run the prominent Agents Shipgate flows: check, verify, or audit --host +description: "Run the prominent Agents Shipgate flows: check, verify, or audit --host" --- Arguments: `$ARGUMENTS` diff --git a/CHANGELOG.md b/CHANGELOG.md index 803d5e07..0e86e521 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ ## Unreleased +- **Claude Code plugin marketplace.** The repo now doubles as a Claude Code + plugin marketplace (`/plugin marketplace add ThreeMoonsLab/agents-shipgate`, + then `/plugin install agents-shipgate@agents-shipgate`) — the symmetric + counterpart of the existing Codex marketplace. The plugin is skill-only + (the auto-triggering skill + the namespaced `/agents-shipgate:shipgate` + command); the scanner stays in the separately installed CLI and hooks stay + on the explicit `install-hooks` path. Byte-identity with the canonical + skill/command sources is test-pinned. Fixed in passing (caught by + `claude plugin validate`): the canonical `SKILL.md` and `/shipgate` + command shipped YAML frontmatter with an unquoted `:` in `description`, + which Claude Code loads as silently-empty metadata — breaking + description-based skill auto-triggering for every existing install. Both + are now quoted, all byte-identical copies synced, and a regression test + parses the frontmatter. + - **Contract v10 (additive): machine-readable `verify_required` on the Codex boundary result.** `shipgate check` already escalated to `warn` and routed to `verify` when a diff touched a tool surface it cannot gate; that deferral diff --git a/README.md b/README.md index 72c48dfc..5e15a593 100644 --- a/README.md +++ b/README.md @@ -311,6 +311,20 @@ agent mode auto-enables, so a zero-flag `agents-shipgate verify` prints the compact agent result. Slash command, skill internals, and manual paths: [`docs/agents/use-with-claude-code.md`](docs/agents/use-with-claude-code.md). +Prefer a plugin over a committed kit? This repo is also a Claude Code plugin +marketplace — the skill-only symmetric counterpart of the Codex plugin below +(workflows, not the scanner binary; install the CLI separately): + +```bash +/plugin marketplace add ThreeMoonsLab/agents-shipgate +/plugin install agents-shipgate@agents-shipgate +``` + +The plugin ships the auto-triggering `agents-shipgate` skill and the +`/agents-shipgate:shipgate` command (plugin commands are namespaced). It does +not ship hooks — install those explicitly with `agents-shipgate install-hooks +--target claude-code --write`, which requires the CLI on `PATH`. + **Codex** — install the skill-only plugin from this repo's marketplace, or write the repo-scoped kit directly: diff --git a/adoption-kits/claude-code-skill/.agents-shipgate-kit-metadata.json b/adoption-kits/claude-code-skill/.agents-shipgate-kit-metadata.json index 3c9e9a38..56f2b4fa 100644 --- a/adoption-kits/claude-code-skill/.agents-shipgate-kit-metadata.json +++ b/adoption-kits/claude-code-skill/.agents-shipgate-kit-metadata.json @@ -19,7 +19,8 @@ "398e88622bf73b524f91405ffc5dbccde651c6a9cb7c2df035ab01d39a964e4f", "9860b9246057289450b425daa212f248be8082327101f2a5e6a355a266f779c1", "5062b30ee84b3871c6532ce0e6c7ad41ac0203a1bd29be51ac3342e01fcd09cb", - "6031a900efa636677786ceda881af0aa4630500683ee1845f4dd1439fdbe2e70" + "6031a900efa636677786ceda881af0aa4630500683ee1845f4dd1439fdbe2e70", + "1b4470b625110acdc6f1e3930b7e6e6f43653c493e9a44c7ccde0dd9c88b09bb" ], "prompts/add-shipgate-to-repo.md": [ "ea3c37cfbbd42c40d164abfe21d468a3a5550d5384125f94a53c947dea6b4b2a", diff --git a/adoption-kits/claude-code-skill/SKILL.md b/adoption-kits/claude-code-skill/SKILL.md index 371c6479..f2dd010e 100644 --- a/adoption-kits/claude-code-skill/SKILL.md +++ b/adoption-kits/claude-code-skill/SKILL.md @@ -1,6 +1,6 @@ --- name: agents-shipgate -description: Run prominent Agents Shipgate flows when a change touches what an AI agent can do: `shipgate check`, `agents-shipgate verify`, or `shipgate audit --host`. Use after adding or modifying MCP servers or tools, tool/function definitions (@tool, @function_tool), OpenAPI specs that describe agent tools, agent prompts, permission scopes, approval or confirmation policies, agent CI workflows, or shipgate.yaml — and before creating a PR for any such change. Also use to verify agent-related PRs, fix or triage Shipgate findings, add Shipgate to CI, or interpret Shipgate verifier/report artifacts. Triggers on phrases like "add shipgate", "verify this agent PR", "merge verdict", "release readiness for my agent", "tool-use readiness", "shipgate check", "agents-shipgate verify", "audit host grants", "shipgate.yaml", "agents-shipgate-reports/verifier.json", "agents-shipgate-reports/report.json", "fix shipgate finding". +description: 'Run prominent Agents Shipgate flows when a change touches what an AI agent can do: `shipgate check`, `agents-shipgate verify`, or `shipgate audit --host`. Use after adding or modifying MCP servers or tools, tool/function definitions (@tool, @function_tool), OpenAPI specs that describe agent tools, agent prompts, permission scopes, approval or confirmation policies, agent CI workflows, or shipgate.yaml — and before creating a PR for any such change. Also use to verify agent-related PRs, fix or triage Shipgate findings, add Shipgate to CI, or interpret Shipgate verifier/report artifacts. Triggers on phrases like "add shipgate", "verify this agent PR", "merge verdict", "release readiness for my agent", "tool-use readiness", "shipgate check", "agents-shipgate verify", "audit host grants", "shipgate.yaml", "agents-shipgate-reports/verifier.json", "agents-shipgate-reports/report.json", "fix shipgate finding".' --- # agents-shipgate skill diff --git a/docs/agents/use-with-claude-code.md b/docs/agents/use-with-claude-code.md index e7e6cce8..7f557cba 100644 --- a/docs/agents/use-with-claude-code.md +++ b/docs/agents/use-with-claude-code.md @@ -35,6 +35,27 @@ when those files exist: agents-shipgate init --workspace . --write --claude-code ``` +### Plugin marketplace (no committed files) + +Prefer not to commit skill files into the repo? This repository is a Claude +Code plugin marketplace; the plugin is **skill-only** — it supplies the same +auto-triggering skill plus the slash command, and updates with `/plugin +update` instead of re-running `init`: + +```text +/plugin marketplace add ThreeMoonsLab/agents-shipgate +/plugin install agents-shipgate@agents-shipgate +``` + +Plugin commands are namespaced: the command installs as +`/agents-shipgate:shipgate` (the committed-kit path keeps plain `/shipgate`). +The plugin does not ship hooks or the scanner — install the CLI (`pipx +install agents-shipgate`, contract v10 or newer) and add hooks explicitly +with `agents-shipgate install-hooks --target claude-code --write`. To +pre-provision the marketplace for a whole team, add it to +`.claude/settings.json` under `extraKnownMarketplaces` and enable +`"agents-shipgate@agents-shipgate"` in `enabledPlugins`. + To install the surfaces manually (no CLI), from the root of the project where you want `/shipgate` and the skill available: diff --git a/plugins/claude-code/.claude-plugin/plugin.json b/plugins/claude-code/.claude-plugin/plugin.json new file mode 100644 index 00000000..f6b28da2 --- /dev/null +++ b/plugins/claude-code/.claude-plugin/plugin.json @@ -0,0 +1,21 @@ +{ + "name": "agents-shipgate", + "version": "0.14.0", + "description": "Run Agents Shipgate Tool-Use Readiness workflows from Claude Code. Skill-only: the plugin supplies the agents-shipgate skill and the /agents-shipgate:shipgate command; the scanner runs through the agents-shipgate CLI installed in the local environment (pipx install agents-shipgate, contract v10 or newer). For the deterministic hooks (trigger check after edits, full verify at Stop), run: agents-shipgate install-hooks --target claude-code --write.", + "author": { + "name": "Three Moons Lab", + "email": "help@threemoonslab.com", + "url": "https://threemoonslab.com/" + }, + "homepage": "https://threemoonslab.com/", + "repository": "https://github.com/ThreeMoonsLab/agents-shipgate", + "license": "Apache-2.0", + "keywords": [ + "agents", + "claude-code", + "tool-use-readiness", + "release-gate", + "mcp-security", + "policy-as-code" + ] +} diff --git a/plugins/claude-code/commands/shipgate.md b/plugins/claude-code/commands/shipgate.md new file mode 100644 index 00000000..68233663 --- /dev/null +++ b/plugins/claude-code/commands/shipgate.md @@ -0,0 +1,81 @@ +--- +description: "Run the prominent Agents Shipgate flows: check, verify, or audit --host" +--- + +Arguments: `$ARGUMENTS` + +Agents Shipgate is the deterministic merge gate for AI-generated agent +capability changes. + +If the arguments include `audit`, run the host-grant audit flow. If they include +`check`, run the local boundary check. Otherwise run the verifier flow. The +supporting adoption and scan commands remain available, but this slash command +should lead with only the prominent flows: `shipgate check`, `agents-shipgate verify`, +and `shipgate audit --host`. + +The canonical, self-contained verifier instructions live in the bundled prompt +files. For verifier runs, read `prompts/verify-agent-diff.md`. Try these paths +in order; use the first that exists: + +1. `.claude/skills/agents-shipgate/prompts/.md` — bundled with the `agents-shipgate` skill if installed in this project. +2. `prompts/.md` — present when this repo is a clone of `agents-shipgate` itself. +3. `https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/prompts/.md` — last-resort fetch. + +Prominent commands: + +```bash +AGENTS_SHIPGATE_AGENT_MODE=1 shipgate check \ + --agent claude-code --workspace . --format codex-boundary-json +AGENTS_SHIPGATE_AGENT_MODE=1 agents-shipgate verify \ + --workspace . --config shipgate.yaml \ + --base origin/main --head HEAD \ + --ci-mode advisory --format json +AGENTS_SHIPGATE_AGENT_MODE=1 shipgate audit --host \ + --json --out agents-shipgate-reports/host-grants.json +``` + +For local uncommitted work, omit `--base`/`--head` so uncommitted edits are +scanned. For committed PR/CI refs, make the base ref available first because +`verify` never fetches. + +Required behavior (do not skip): + +1. Set `AGENTS_SHIPGATE_AGENT_MODE=1` for every CLI call so errors emit a `next_action` JSON line on stderr. +2. Run `agents-shipgate contract --json` when available and use it to verify the installed CLI's schema versions and gating signal. +3. For verifier runs, parse `agents-shipgate-reports/agent-handoff.json` first, + then `verifier.json`, `verify-run.json`, and + `report.json.release_decision.decision` as the release gate. +4. For check runs, parse stdout as `shipgate.codex_boundary_result/v1` and + switch on `decision`, `completion_allowed`, `must_stop`, `first_next_action`, + `human_review`, and `repair`. +5. For host audits, parse `agents-shipgate-reports/host-grants.json` when + `--out` is used, or stdout when running JSON-only. +6. Do **not** bypass the verifier by suppressing findings, lowering severity, + expanding baselines or waivers, removing Shipgate CI, or weakening agent + instructions. Verify-mode `SHIP-VERIFY-*` checks route those trust-root edits + to human review. +7. Add `agents-shipgate-reports/` to `.gitignore` if it is not already. + +Report back: `release_decision.decision` and `reason`, `merge_verdict`, +`can_merge_without_human`, blocker / review-item counts, top 3 active findings +by severity, `verifier_summary` trust-root flags when present, capability +change highlights, and one suggested next step. + +## Ongoing PRs + +For an ongoing PR that changes agent tools, MCP exports, OpenAPI specs, prompts, +permissions, policies, CI gates, or `shipgate.yaml`, run the verifier: + +```bash +agents-shipgate verify --base origin/main --head HEAD --json +``` + +Read `agents-shipgate-reports/agent-handoff.json` first and lead with +`gate.merge_verdict` (a deterministic projection of `release_decision.decision`, +which remains the gate in `report.json`), then the authoritative substrate +`agents-shipgate-reports/verifier.json` and supporting/provisional +`capability_review.top_changes[]`. Do not claim completion when +`merge_verdict` is `blocked`, `insufficient_evidence`, or +`human_review_required` unless the user accepted the human-review requirement, and +never weaken `shipgate.yaml`, Shipgate CI, `AGENTS.md`, policies, baselines, or +waivers to make Shipgate pass. diff --git a/plugins/claude-code/skills/agents-shipgate/SKILL.md b/plugins/claude-code/skills/agents-shipgate/SKILL.md new file mode 100644 index 00000000..f2dd010e --- /dev/null +++ b/plugins/claude-code/skills/agents-shipgate/SKILL.md @@ -0,0 +1,110 @@ +--- +name: agents-shipgate +description: 'Run prominent Agents Shipgate flows when a change touches what an AI agent can do: `shipgate check`, `agents-shipgate verify`, or `shipgate audit --host`. Use after adding or modifying MCP servers or tools, tool/function definitions (@tool, @function_tool), OpenAPI specs that describe agent tools, agent prompts, permission scopes, approval or confirmation policies, agent CI workflows, or shipgate.yaml — and before creating a PR for any such change. Also use to verify agent-related PRs, fix or triage Shipgate findings, add Shipgate to CI, or interpret Shipgate verifier/report artifacts. Triggers on phrases like "add shipgate", "verify this agent PR", "merge verdict", "release readiness for my agent", "tool-use readiness", "shipgate check", "agents-shipgate verify", "audit host grants", "shipgate.yaml", "agents-shipgate-reports/verifier.json", "agents-shipgate-reports/report.json", "fix shipgate finding".' +--- + +# agents-shipgate skill + +`agents-shipgate` is the deterministic merge gate for AI-generated agent capability changes — a local-first, static Tool-Use Readiness review. It analyzes `shipgate.yaml` plus tool sources (MCP exports, OpenAPI specs, OpenAI Agents SDK Python files, Anthropic Messages API artifacts, Google ADK files, LangChain/LangGraph files, CrewAI files, OpenAI API artifacts, Codex repo config, Codex plugin packages and marketplaces, n8n workflow JSON) and emits deterministic verifier artifacts, findings, Markdown, JSON, and SARIF. + +It does **not** run agents, call tools, invoke LLMs, connect to MCP servers, or send telemetry by default. Static analysis only; audited exceptions are pinned in `tests/test_adapter_static_only.py::ALLOWED_EXCEPTIONS`. + +> The skill name is intentionally `agents-shipgate` (not `shipgate`) so it does not collide with the `/shipgate` slash command shipped at `.claude/commands/shipgate.md` — Claude Code lets a skill with the same name preempt a command, which would bypass the bootstrap flow the slash command is meant to deliver. + +## When to use this skill + +- The current diff adds or modifies MCP server or tool definitions, `@tool`/`@function_tool` decorators, OpenAPI specs describing agent tools, agent prompts, permission scopes, approval/confirmation policies, or agent CI workflows — run the verifier before reporting the change complete or creating a PR. +- The user asks to add Tool-Use Readiness or pre-merge checks to an agent project. +- The user asks whether an AI-generated agent PR can merge. +- The repo already has `shipgate.yaml`, `agents-shipgate-reports/verifier.json`, or `agents-shipgate-reports/report.json`. +- The user asks to fix, triage, suppress, or explain a Shipgate finding. +- The user wants to add Shipgate to CI (GitHub Actions, GitLab CI, CircleCI). + +## When NOT to use this skill + +- Generic linting / type checking — use the project's existing tooling. +- Runtime monitoring, evals, or behavioral testing — Shipgate is static-only. +- LLM output quality assessment — out of scope. +- Editing `agents-shipgate`'s own check implementations — that's upstream-repo work, not user-repo work. + +## How to act + +Pick the matching task and follow the linked recipe verbatim. Recipes are bundled inside this skill so behavior is pinned to the installed version and works offline. Each prompt is self-contained: install commands, exit codes, and `AGENTS_SHIPGATE_AGENT_MODE=1` error handling are in the prompt itself. + +| Task | Recipe | +|---|---| +| Decide whether Shipgate should run at all (apply `docs/triggers.json` against the PR) | [`prompts/decide-shipgate-relevance.md`](prompts/decide-shipgate-relevance.md) | +| Bootstrap a repo (install, init, scan, report) | [`prompts/add-shipgate-to-repo.md`](prompts/add-shipgate-to-repo.md) | +| Verify an agent-related PR or local diff before finishing | [`prompts/verify-agent-diff.md`](prompts/verify-agent-diff.md) | +| Add Shipgate to CI for the first time (advisory, PR comment) | See "First-time CI setup" below; copy [`ci-recipes/advisory-pr-comment.yml`](ci-recipes/advisory-pr-comment.yml) | +| Fix the highest-severity finding | [`prompts/fix-top-finding.md`](prompts/fix-top-finding.md) | +| Recommend fixes across all active findings | [`prompts/recommend-fixes.md`](prompts/recommend-fixes.md) | +| Explain a single finding in user-facing prose (3–5 sentences for a PR comment / chat reply) | [`prompts/explain-finding-to-user.md`](prompts/explain-finding-to-user.md); pair with `agents-shipgate explain-finding --from agents-shipgate-reports/report.json --json` | +| Triage a suspected false positive | [`prompts/triage-false-positive.md`](prompts/triage-false-positive.md) | +| Promote advisory CI to strict CI (assumes advisory is already running) | [`prompts/stabilize-strict-mode.md`](prompts/stabilize-strict-mode.md) | +| Upgrade agents-shipgate version | [`prompts/upgrade-shipgate-version.md`](prompts/upgrade-shipgate-version.md) | + +Always: + +1. Set `AGENTS_SHIPGATE_AGENT_MODE=1` so errors emit a `next_action` JSON line on stderr (auto-enabled inside Claude Code via the harness's `CLAUDECODE=1` env var, and Cursor via `CURSOR_TRACE_ID`). +2. For local agent control, run `shipgate check --agent claude-code --workspace . --format codex-boundary-json` and read the stdout `shipgate.codex_boundary_result/v1` object. Switch on `decision`; follow `first_next_action`, `repair`, and `human_review`. +3. For verifier runs, parse `agents-shipgate-reports/verifier.json` first, then `agents-shipgate-reports/verify-run.json`: + `merge_verdict`, `can_merge_without_human`, `first_next_action`, + `fix_task`, and `capability_review.top_changes`. Then parse + `agents-shipgate-reports/report.json.release_decision.decision`; it is the + release gate. +4. Before editing `shipgate.yaml`, Shipgate CI, AGENTS/CLAUDE/Cursor rules, policy packs, baselines, waivers, suppressions, Codex hooks/config, Codex plugin manifests, `.mcp.json`, `.app.json`, or `SKILL.md`, plan to run `agents-shipgate verify` before completion and route trust-root review to a human when the verifier requires it. +5. Before finishing an agent-related diff, run `shipgate check --agent claude-code --workspace . --format codex-boundary-json`. For committed PR/CI verification, run `agents-shipgate verify --workspace . --config shipgate.yaml --base origin/main --head HEAD --ci-mode advisory --format json` after making the base ref available. `verify` never fetches. For host grants, run `shipgate audit --host --json --out agents-shipgate-reports/host-grants.json`. +6. Do not bypass the verifier by suppressing findings, lowering severity, expanding baselines or waivers, removing Shipgate CI, or weakening agent instructions; verify-mode `SHIP-VERIFY-*` checks make those trust-root edits release-visible. +7. Confirm with the user before any command that writes files (`init --write`, `baseline save`). + +## First-time CI setup (advisory) + +If the user has no Shipgate CI yet, default to **advisory** mode — never strict, never with a baseline. The promotion path comes later, only after findings have been reviewed. + +1. Confirm the repo has `shipgate.yaml` and a clean local scan (`agents-shipgate scan -c shipgate.yaml --ci-mode advisory` exits `0`). If not, run the bootstrap recipe first. +2. Create `.github/workflows/agents-shipgate.yml` from [`ci-recipes/advisory-pr-comment.yml`](ci-recipes/advisory-pr-comment.yml). It runs on every pull request, posts a summary comment, uploads the report as an artifact, and never fails the job. +3. Confirm `permissions: pull-requests: write` is acceptable to the user before committing — required for the PR comment. +4. Push and open a test PR. Verify the agents-shipgate comment appears. +5. **Stop here.** Promotion to strict mode is a separate task — only run [`prompts/stabilize-strict-mode.md`](prompts/stabilize-strict-mode.md) after the user has reviewed the advisory output and decided which findings they accept. + +For non-GitHub CI (GitLab, CircleCI, Jenkins, Azure Pipelines, Buildkite, Bitbucket, pre-commit) refer to https://github.com/ThreeMoonsLab/agents-shipgate/tree/main/examples or `docs/integrations.md` in the upstream repo. Always start in advisory mode. + +## Stable contracts (rely on these) + +- **CLI surface** follows the current 0.x contract line — see https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/STABILITY.md. +- **Installed CLI contract**: when available, run `agents-shipgate contract --json` to verify local schema versions, capability/research surfaces, `release_decision.decision`, and manual-review signal fields. Older installs should use [`docs/agent-contract-current.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md) or upgrade before automating against the local contract command. +- **Verifier JSON**: `verifier_schema_version: "0.1"`. Read `merge_verdict`, `can_merge_without_human`, `first_next_action`, `fix_task`, `capability_review.top_changes`, `trust_root_touched`, and `policy_weakened` before summarizing an AI-generated PR. `merge_verdict` is a deterministic projection; the gate remains `report.json.release_decision.decision`. +- **Verify run JSON**: `verify-run.json` uses `schema_version: "shipgate.verify_run/v1"` and records stable run identity, subject refs, input hashes, outcome, and artifact hashes. It is the reproducibility artifact for `verify`; do not treat it as a second gate. +- **Report JSON**: `report_schema_version: "0.28"`. Read `release_decision.decision` first for release gating; use `agent_summary` / `findings[].agent_action` for agent routing and `reviewer_summary` for the human-review entry point. v0.28 moves policy-pack owner/reviewer/approval routing metadata to `findings[].policy_routing` so `Finding.evidence` stays deterministic match/gating evidence; routing metadata is non-enforcing and does not affect fingerprints, suppressions, baselines, `blocks_release`, or release decisions. v0.27 added policy-pack distribution metadata (`loaded_policy_packs[].{source,sha256,sha256_status,owner}`) for organization audit while preserving gate behavior. v0.26 added structured evidence gaps (`release_decision.evidence_coverage.evidence_gaps[]` — one actionable remediation row per low-confidence tool or source warning) plus the advisory `suggested-inventory.json` skeleton written next to `report.json`; when the decision is `insufficient_evidence`, follow each gap's `next_action` instead of guessing. v0.25 added opt-in capability-linked local trace/provenance evidence (`capability_runtime_evidence`, `findings[].capability_trace_refs`, and mirrored `ReleaseDecisionItem.capability_trace_refs`) while preserving fingerprints, baselines, and gate behavior. v0.24 added capability-native policy evidence (`findings[].capability_refs`, optional `findings[].capability_policy_evidence`, and mirrored `ReleaseDecisionItem.capability_refs`). v0.23 added semantic metadata to `capability_change` members while preserving existing buckets. v0.22 added the verifier-cycle blocks `capability_change`, `protected_surface_changes`, `effective_policy`, `human_ack`, and `verifier_summary` — all reviewer-facing projections that never gate independently (`release_decision.decision` stays the only gate). `reviewer_summary`, `verifier_summary`, runtime trace/evidence fields, and non-gating capability diff projections are supporting/provisional explanatory context. To remove heuristic findings from the active gate, rerun scan with `--no-heuristics`; filtered findings remain in `findings[]` with `suppressed=true`, and `heuristics_filter` records `enabled`, `excluded_provenance_kinds`, `filtered_finding_count`, and `filtered_by_kind`; `runtime_trace` findings are not filtered as heuristics. To inspect provenance without changing gate behavior, use `agents-shipgate findings --from agents-shipgate-reports/report.json --provenance-kind keyword_heuristic,regex_heuristic,runtime_trace --json`. Do not gate on `summary.status`; it is legacy and baseline-blind. The full field list lives in [`docs/agent-contract-current.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md#read-these-first-for-release-gating), and reports validate against [`docs/report-schema.v0.28.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/report-schema.v0.28.json). +- **Release Evidence Packet**: `agents-shipgate-reports/packet.{md,json,html}` (and `packet.pdf` with the `[pdf]` extras) is emitted alongside the report by default as a supporting/provisional reviewer artifact. The packet has fixed reviewer sections governed by [`docs/packet-schema.v0.7.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/packet-schema.v0.7.json) (latest; v0.7 adds capability-linked local trace evidence summary and trace refs under `human_in_the_loop`). See [STABILITY.md §Release Evidence Packet](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/STABILITY.md#release-evidence-packet-v07). Use the packet for reviewer-shaped output; use the report for finding details. +- **Capability standard**: `agents-shipgate capability export` emits a stable static capability lock (`capability_lock_schema_version: "0.2"`) and `agents-shipgate capability diff` emits a stable semantic diff (`capability_lock_diff_schema_version: "0.3"`). These artifacts are supporting/provisional, non-gating, exclude runtime trace evidence, and are documented in [`docs/capability-standard.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/capability-standard.md). +- **Governance benchmark**: `benchmark/agent-pr-governance/cases.yaml` and `scripts/run_governance_benchmark.py` are the stable research benchmark substrate (`governance_benchmark_result_schema_version: "0.2"`), not a release gate. See [`docs/governance-benchmark.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/governance-benchmark.md). +- **Single source of truth for the contract**: [`docs/agent-contract-current.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md). When the schema bumps, that file updates first. +- **Exit codes**: `0` pass, `2` config error, `3` parse error, `4` other error, `20` strict-mode gate failure. +- **Check IDs** (e.g. `SHIP-POLICY-APPROVAL-MISSING`) are stable; new ones may be added but existing ones will not be renamed or repurposed. + +## Boundaries (do not violate) + +- Do not claim a finding is fixed without re-running `agents-shipgate verify` and reporting the new merge verdict and release decision. +- Do not silently suppress findings — `checks.ignore` requires a `reason` and the manifest validator rejects empty reasons. +- Do not commit `agents-shipgate-reports/` — it's regenerated each run; add it to `.gitignore`. +- Do not run `agents-shipgate baseline save` until the user has reviewed the initial findings; baselining ratchets in noise. +- Do not enable strict CI as the first CI step. Always start advisory. +- Do not modify checks in `agents-shipgate`'s own source — that's upstream repo work. +- Do not weaken Shipgate trust roots to make a verifier run pass. Policy, baseline, + waiver, CI, trigger-catalog, and agent-instruction changes require human review. + +## If something errors out + +Set `AGENTS_SHIPGATE_AGENT_MODE=1` and re-run. The CLI appends a JSON line to stderr with `{error, message, next_action}`. Follow the `next_action`. The error kinds emitted by the current CLI: + +| Error kind | Fix | +|---|---| +| `config_error` | Manifest is missing, malformed, or fails validation. Common cause: no `shipgate.yaml` yet — run `agents-shipgate init --workspace . --write`. | +| `config_already_exists` | `init --write` was run with an existing `shipgate.yaml`. Edit the file in place or remove it before re-running. | +| `input_parse_error` | A file referenced from the manifest (`tool_sources[].path`, baseline, policy pack) is missing, malformed, or resolves outside the manifest directory. Correct the path. | +| `unknown_check_id` | The check ID passed to `explain` does not exist. Run `agents-shipgate list-checks --json` to enumerate. | +| `other_error` / `internal_error` | Unexpected failure. Re-run with `--verbose` and include the output if filing an issue. | + +For deeper troubleshooting see https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/troubleshooting.md. diff --git a/plugins/claude-code/skills/agents-shipgate/ci-recipes/advisory-pr-comment.yml b/plugins/claude-code/skills/agents-shipgate/ci-recipes/advisory-pr-comment.yml new file mode 100644 index 00000000..fb5f5e8d --- /dev/null +++ b/plugins/claude-code/skills/agents-shipgate/ci-recipes/advisory-pr-comment.yml @@ -0,0 +1,26 @@ +# Advisory PR comment. +# Recommended starting point — runs the scanner on every PR, posts a summary +# comment, uploads the report as an artifact, and never fails the job. +name: Agents Shipgate (advisory) + +on: + pull_request: + +permissions: + contents: read + pull-requests: write + +jobs: + shipgate: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: ThreeMoonsLab/agents-shipgate@v0.14.0 + with: + ci_mode: advisory + diff_base: target + pr_comment: 'true' + shipgate_version: '0.14.0' diff --git a/plugins/claude-code/skills/agents-shipgate/prompts/add-shipgate-to-repo.md b/plugins/claude-code/skills/agents-shipgate/prompts/add-shipgate-to-repo.md new file mode 100644 index 00000000..c13c707e --- /dev/null +++ b/plugins/claude-code/skills/agents-shipgate/prompts/add-shipgate-to-repo.md @@ -0,0 +1,155 @@ +# Prompt · Add Agents Shipgate to a repo + +You are working in a repo that may contain an AI agent — likely one of: an MCP server tool list (`*mcp*.json` or `.agents-shipgate/*.json`), an OpenAPI spec the agent calls, a Codex plugin package (`.codex-plugin/plugin.json`) or marketplace (`.agents/plugins/marketplace.json`), a Python file with `@function_tool` / `@tool` decorators (OpenAI Agents SDK, LangChain, CrewAI), a Google ADK agent in `agent.py`, an Anthropic Messages API artifact set under `prompts/`/`tools/anthropic-tools.json`/`policies/anthropic-policy.yaml`, or an OpenAI API artifact set under `prompts/`/`tools/openai-tools.json`/`openai-config.json`. + +Your job is to drive the first-adoption helper flow end-to-end in one +tool-using turn, which adds the deterministic merge gate for AI-generated agent +capability changes — a local-first, static Tool-Use Readiness review. Ongoing +agent-related PRs should use `agents-shipgate verify` after this adoption step. + +## Your task + +1. **Install the tool - pin the version so a stale build can't shadow it.** This flow uses the current verifier, agent-handoff, primary-command, and Codex-boundary contracts and requires **contract v9 or newer**; an older copy lingering on `PATH` may lack the command or schema fields this prompt expects. Prefer a **pinned, zero-install** runner that fetches the exact version every time instead of trusting whatever is already on `PATH`. **Pin it into one variable and use that for every step below**, so no single command can fall through to a stale binary: + ```bash + SG="uvx agents-shipgate@0.14.0" # uv: ephemeral, always the pinned build + # or: SG="pipx run agents-shipgate==0.14.0" + $SG --version # confirm the pinned runner resolves + ``` + Every step below calls `$SG …`; e.g. `$SG verify --preview --json` runs the verify preview through the pinned runner, never a `PATH` copy. + + If you would rather install onto `PATH`, pin the floor and **fail loudly when it resolves older** — a plain `pipx install agents-shipgate` is a no-op when an older build already exists — then set `SG=agents-shipgate`: + ```bash + python -m pip install -U --pre agents-shipgate + agents-shipgate contract --json # STOP if this reports contract_version < 9 - re-run pinned via uvx agents-shipgate@0.14.0 + SG=agents-shipgate # only after the line above confirms contract v9+ + ``` + +2. **Sanity-check the install** before touching the user's code: + ```bash + $SG self-check --json + ``` + Confirm `"ready": true`. If not, surface the failure to the user. + + When available, verify the installed CLI contract locally: + ```bash + $SG contract --json + ``` + Read `report_schema_version`, `packet_schema_version`, `gating_signal`, and + `manual_review_signals[]`; prefer these local values over stale docs. If the + command is not recognized on an older install, continue after `self-check` + using [`docs/agent-contract-current.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md) + and upgrade before relying on local contract verification in automation. + +3. **Preview the verify flow:** + ```bash + $SG verify --preview --json + ``` + Read the response and next action. Preview is the first-look verify entry + point: it does not require a manifest, does not scan, and tells you whether + to configure Shipgate, skip, or run the full verifier. + + If preview exposes trigger/detection metadata, stop only when all relevance + signals are absent and the user did not explicitly request Shipgate. + Otherwise proceed. MCP/OpenAPI tool-surface repos and Codex plugin package + repos can be valid Shipgate targets even when Python framework detection + would classify `is_agent_project: false`; look for `suggested_sources` and + `codex_plugin_candidates` when those fields are present. + +4. **Generate a starter manifest + GitHub Actions workflow:** + ```bash + $SG init --workspace . --write --ci --json + ``` + The `--json` form returns: + - `manifest_status`: `"written"` | `"skipped_existing"` | `"not_attempted"` + - `workflow.status` (with `--ci`): `"written"` | `"skipped_existing_target"` | `"skipped_cross_reference"` + - `placeholders[]` — entries the template intentionally left as `CHANGE_ME` because no high-confidence signal was available + - `auto_detected.agent_name` — the value the manifest carries (`null` when the template fell back to `CHANGE_ME`) + + `--ci` writes `.github/workflows/agents-shipgate.yml` orthogonally to `--write`. Each gets its own overwrite-refusal check; existing workflows that already call `ThreeMoonsLab/agents-shipgate` skip with a distinct `cross_reference_path`. + +5. **Replace placeholders.** Walk `placeholders[]` from the JSON output. On a fresh workspace the template typically leaves two: + - `agent.name: CHANGE_ME` — replace with the agent's actual role (no strong `Agent(name="…")` literal was found in the source). + - `agent.declared_purpose[]: CHANGE_ME` — replace with a one-line description of what the agent should do (auto-init can't infer this; the schema requires a non-empty value). + + Read the agent's prompt or main file to derive both. Skipping this leaves an invalid adoption artifact — the manifest validates but downstream consumers see meaningless defaults. + +6. **Run the scan with patch suggestions:** + ```bash + $SG scan -c shipgate.yaml --suggest-patches --format json --ci-mode advisory + ``` + The report lands at `agents-shipgate-reports/report.json`. The supporting Release Evidence Packet lands at `agents-shipgate-reports/packet.{md,json,html}`. Parse `report.json`; Codex plugin facts, when present, live under `codex_plugin_surface`. + + **Read these first for release gating (v0.8+):** + - `release_decision.decision` ∈ `{"blocked", "review_required", "insufficient_evidence", "passed"}` — baseline-aware. This is the gating signal. `insufficient_evidence` (v0.14+) fires when evidence coverage is degraded past threshold; treat unknown future values as `review_required`. + - `release_decision.{reason, blockers, review_items, fail_policy.would_fail_ci}` + + **Read these for release review (v0.9+):** + - `capability_facts[]`, `declared_intentions[]`, `misalignments[]`, `release_consequence`, `suggested_scenarios[]` + + **Per-finding fields:** + - `check_id`, `severity`, `category`, `tool_name`, `recommendation`, `suppressed` + - `autofix_safe`, `requires_human_review`, `suggested_patch_kind`, `docs_url` (v0.7+) + - `patches[]` (only with `--suggest-patches`) — each has `kind` ∈ `{set_pointer, append_pointer, remove_pointer, manual}` plus `confidence` + `target_file` + etc. for non-manual kinds. + + **Top-level:** `manifest_dir` (absolute path of the manifest's directory — used by `apply-patches` for the containment check). `summary.{status, critical_count, high_count, medium_count}` is preserved for v0.7 callers and is baseline-blind — do not gate on `summary.status` for new consumers. Full contract: [`docs/agent-contract-current.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/agent-contract-current.md). + +7. **Apply the safe patches:** + ```bash + $SG apply-patches --from agents-shipgate-reports/report.json --confidence high --apply --json + ``` + Default `--confidence high` only mutates patches whose `confidence` field is `"high"`. Today that's the 3 stale-manifest removals. Scope-coverage appends ship at `medium` and require explicit `--confidence medium` to apply. ManualPatches are never auto-applied. + + **Decision tree** for walking the report: + ``` + for finding in active_findings: + if finding.suggested_patch_kind in ("manual", "none"): + surface_to_user(finding) # Surface; do NOT auto-apply. + continue + if finding.autofix_safe is True: + plan_to_apply(finding) # Will be applied at --confidence high. + continue + surface_for_medium_review(finding) # Medium-confidence — opt-in only. + ``` + + Trace findings (`SHIP-API-TRACE-{APPROVAL,CONFIRMATION}-MISSING`) are permanent ManualPatch by policy. Implement the runtime gate; never edit the trace recording — that patches the evidence, not the agent. See [`docs/autofix-policy.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/autofix-policy.md) for the full classification. + +8. **Add `agents-shipgate-reports/` to `.gitignore`** if it isn't already. The reports are scan artifacts, not source. + +9. **Report back to the user**: + - `release_decision.decision` and `release_decision.reason` (the gating signal — baseline-aware, v0.8+) + - Blocker / review-item counts (`len(release_decision.blockers)` / `len(release_decision.review_items)`) + - The path to the supporting Release Evidence Packet (`agents-shipgate-reports/packet.md`) for reviewer-shaped output + - The top 3 active critical/high findings (use `report.json`, not stdout) + - Which patches were applied (count from `apply-patches --json` output's `files`) + - Any check IDs the user should investigate first — link to `docs_url` from the finding for full rationale, or use `$SG explain --json` for the same content via CLI + +## What to do if the scan errors out + +Re-run the failing `$SG …` command with `AGENTS_SHIPGATE_AGENT_MODE=1` set. The CLI will append a JSON line to stderr with `{error, message, next_action}`. Follow the `next_action`. + +Common errors and fixes: + +| Error | Fix | +|---|---| +| `Config file not found: shipgate.yaml` | Run `$SG init --workspace . --write` first | +| `Input path '...' resolves outside manifest directory` | The declared `tool_sources[].path` is outside the manifest dir. Move the spec inside the tree, symlink it, or copy it | +| `Invalid shipgate.yaml: ... Did you mean X?` | A field is at the wrong nesting level; move it as suggested | +| `Containment violation` (apply-patches exit 5) | A patch's `target_file` resolved outside `report.manifest_dir`. Re-run scan to refresh; never patch arbitrary system files | + +## What NOT to do + +- Do **not** commit `agents-shipgate-reports/` — it's regenerated each run. +- Do **not** run `$SG baseline save` until the user has reviewed the initial findings. Baselining ratchets in noise that strict CI will silently ignore. The right time to baseline is **after** the user has decided which findings they accept. +- Do **not** suppress findings without a real `reason` — the manifest validator rejects empty reasons, and the `reason` field is the audit trail when someone asks "why is this OK?" +- Do **not** use `risk_overrides.tools.{tool}.remove_tags` to silence a finding without checking whether the heuristic is actually wrong. Prefer `checks.ignore` with a reason. +- Do **not** edit a trace recording to flip `approved` or `confirmed` — implement the runtime gate instead. + +## Verification before reporting success + +- `agents-shipgate-reports/report.json` exists and parses as JSON +- `report.json` carries `report_schema_version: "0.11"` (or higher) and a non-empty `manifest_dir` +- `report.json` carries a non-null `release_decision.decision` — this is the field to surface to the user +- `shipgate.yaml` has no `CHANGE_ME` values (comments containing the literal `CHANGE_ME` are informational and OK) +- `.gitignore` contains `agents-shipgate-reports/` (or equivalent) +- If `--ci` ran with `workflow.status: "written"`: `.github/workflows/agents-shipgate.yml` exists and references `ThreeMoonsLab/agents-shipgate@v…` +- The user knows the top 3 findings and at least one suggested next step diff --git a/plugins/claude-code/skills/agents-shipgate/prompts/decide-shipgate-relevance.md b/plugins/claude-code/skills/agents-shipgate/prompts/decide-shipgate-relevance.md new file mode 100644 index 00000000..2e647122 --- /dev/null +++ b/plugins/claude-code/skills/agents-shipgate/prompts/decide-shipgate-relevance.md @@ -0,0 +1,151 @@ +# Prompt · Decide whether Agents Shipgate is relevant + +You are working in a repo or reviewing a PR and need to decide whether +to propose Agents Shipgate as the next step. The other prompts in +[`prompts/`](https://github.com/ThreeMoonsLab/agents-shipgate/tree/main/prompts) +assume relevance is already established — this one runs **before** that +decision and tells you yes or no with a rationale. + +The decision is fully data-driven: it does not depend on prose-reading. +[`docs/triggers.json`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/triggers.json) is the machine-readable +mirror of the AGENTS.md trigger table; you fetch (or read) it and apply +the rules to the changed file list. + +## Your task + +1. **Identify the changed file set.** Repo-relative, forward slashes: + - PR context: `git diff --name-only origin/main...HEAD` + - Working tree: `git status --short` (uncommitted) + - User-pasted diff: parse `diff --git a/ b/` headers + +2. **Fetch the trigger catalog.** Either: + - **Local repo** (already adopted Shipgate): read `docs/triggers.json` directly. + - **Remote** (target repo without Shipgate): fetch + `https://raw.githubusercontent.com/ThreeMoonsLab/agents-shipgate/main/docs/triggers.json`. + - The catalog has `schema_version: "0.1"` and is stable for `0.x`. + +3. **Apply the rules.** Two equivalent options: + + **Option A — read the JSON yourself.** Walk `rules[]`. For each rule, + evaluate `rule.when` against the changed file list **and** the unified + diff body — several rules use `diff_contains` predicates (e.g. + `@function_tool`) that a path-only listing cannot satisfy. The + predicate vocabulary is documented in `triggers.json` under + `predicate_vocabulary`; the action precedence is in + `action_precedence`. See the decision tree below. + + **Option B — call the bundled evaluator** (when Shipgate is installed). + Use the `--git-diff` flag so paths AND diff body come from git in one + call; piping `git diff --name-only` alone causes `diff_contains` rules + (decorators, framework tokens, Action URL) to silently never fire: + ```bash + agents-shipgate self-check --json # confirm install + python -m agents_shipgate.triggers \ + --git-diff origin/main...HEAD --json + ``` + For uncommitted changes pass `--git-diff` with no revspec — that + runs `git diff HEAD` (covers BOTH staged and unstaged tracked + changes) plus `git ls-files --others --exclude-standard` to add + untracked file paths. Untracked files contribute paths only; their + content is not in `diff_text`, so `diff_contains` rules won't fire + on a brand-new file until you `git add` it (or pass `--diff-text` + manually). If your repo already has a manifest, also pass + `--manifest-present` so the `force_run` rule can fire. + The output shape is `{run_shipgate, dry_run_recommended, + matched_rules, stop_conditions_fired, rationale, schema_version}`. + +4. **Emit the decision.** Always reply in this exact JSON shape so + downstream automation can parse you: + + ```json + { + "run_shipgate": true, + "dry_run_recommended": false, + "matched_rules": [ + {"id": "TRIGGER-MCP-EXPORT-CHANGED", "action": "run_shipgate"} + ], + "rationale": "PR adds tools/mcp-export.json; MCP exports are a tool surface." + } + ``` + +5. **Then act.** + - If `run_shipgate: true`: run verify preview through a **version-pinned, + zero-install** runner so a stale copy already on `PATH` can't answer — + `uvx agents-shipgate@0.14.0 verify --preview --json` (or + `pipx run agents-shipgate==0.14.0 ...`). Only fall back to a bare + `agents-shipgate verify --preview --json` once `agents-shipgate --version` + confirms contract v9 or newer. Then follow + [`prompts/add-shipgate-to-repo.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/prompts/add-shipgate-to-repo.md) + for the first-adoption helper flow, or point the user at the GitHub + Action (`ThreeMoonsLab/agents-shipgate@v0.14.0`) if they prefer CI. + - If `run_shipgate: false` and `dry_run_recommended: true`: propose + a non-mutating scan only — never propose `init --write` based on a + dry-run match alone. Phrase it as "X may have shifted the tool + surface; want me to run a read-only scan against the existing + manifest?" If there's no existing manifest, surface the + `matched_rules` and let the user choose. + - If `run_shipgate: false` and `dry_run_recommended: false`: **do + not propose Shipgate.** Recommend whatever the actual review need + is (lint, type check, unit test, security scan). Mentioning + Shipgate when no rule matched is noise. + +## Decision tree (when reading `triggers.json` by hand) + +``` +For each changed file path AND the unified diff body: + For each rule in triggers.rules: + if rule.when matches → record (rule.id, rule.action) + +stop_fired := every clause in triggers.stop_conditions holds + (requires running detect first; if you haven't, treat as false) + +# Action precedence (highest first), see triggers.json:action_precedence: +if stop_fired: → run = false +elif any action == "force_run": → run = true (manifest present) +elif any action == "skip_shipgate": → run = false (skip beats run) +elif any action == "run_shipgate": → run = true +elif any action == "dry_run": → run = false, dry_run_recommended = true +else: → run = false (no rules matched) +``` + +Why `skip_shipgate` beats `run_shipgate`: a brittle `diff_contains` match +(e.g. `@tool` mentioned in README prose) should not override the explicit +"this is a docs-only PR with no tool surface impact" signal. + +Why `force_run` overrides `skip_shipgate`: an existing `shipgate.yaml` is +the operational opt-in; even a docs-only PR in such a repo gets scanned +because the cost is low (advisory) and tool-adjacent prose changes can +matter. + +## What NOT to do + +- Do **not** propose Shipgate based on filename guesses ("looks like an + AI agent"). The trigger catalog is the source of truth — if no rule + matches, the answer is no. +- Do **not** silently fall back to "yes, run it" when you can't fetch + `triggers.json`. Surface the fetch failure to the user and ask. +- Do **not** invent rule IDs in the output. Every entry in + `matched_rules` must come from `triggers.json`. +- Do **not** treat the **negative control** ("update docs only") as a + reason to propose Shipgate. The `TRIGGER-DOCS-ONLY-NEGATIVE` rule + fires `skip_shipgate` for a reason — and it covers test-only PRs + too, not just `*.md`. +- Do **not** propose `agents-shipgate init --write` on a `dry_run`-only + match. `dry_run_recommended: true` justifies a non-mutating `scan` + against an existing manifest, nothing more. +- Do **not** rely on bare `--git-diff` for brand-new untracked files + to fire `diff_contains` rules. Bare flag covers tracked changes + (staged + unstaged) and untracked file *paths*, but not untracked + file *content*. `git add` first, or pass `--diff-text` explicitly. + +## Verification before reporting + +- Output is valid JSON with the keys `run_shipgate`, + `dry_run_recommended`, `matched_rules`, `rationale`. +- Every `matched_rules[].id` exists in the loaded `triggers.json`. +- If `run_shipgate: true`, the next-step command is named. +- If `run_shipgate: false` AND `dry_run_recommended: true`, exactly + one Shipgate command appears (a non-mutating `scan` against an + existing manifest) — never `init --write`. +- If `run_shipgate: false` AND `dry_run_recommended: false`, no + Shipgate command appears anywhere in your reply. diff --git a/plugins/claude-code/skills/agents-shipgate/prompts/explain-finding-to-user.md b/plugins/claude-code/skills/agents-shipgate/prompts/explain-finding-to-user.md new file mode 100644 index 00000000..13145f6b --- /dev/null +++ b/plugins/claude-code/skills/agents-shipgate/prompts/explain-finding-to-user.md @@ -0,0 +1,89 @@ +# Prompt · Explain a single Agents Shipgate finding to a user + +You need to translate one `report.json` finding into prose for a human +who has never read the Shipgate docs. Companion to `agents-shipgate +explain-finding `, which gives you the structured payload +you'll quote. + +This is for the moment when an agent has run a scan, identified the top +finding (via `agent_summary.first_recommended_action.why` or by walking +`findings[]`), and now has to summarize it for a PR comment, chat +reply, or commit message. The user shouldn't have to follow a doc link +to understand what's going on. + +## Your task + +1. **Get the fingerprint.** Read it from `agent_summary.first_recommended_action.why` if that names a `check_id` and tool, then look up the matching `findings[].fingerprint` in `report.json`. Otherwise pick the highest-severity active finding (`critical > high > medium > low`) and read `fingerprint` directly from that entry. + +2. **Run `explain-finding` to get the structured payload.** + ```bash + agents-shipgate explain-finding \ + --from agents-shipgate-reports/report.json --json + ``` + The output carries: + - `check_id`, `title`, `severity`, `category` — what the check is. + - `tool_name`, `tool_id` — the affected tool (may be null for manifest-level checks). + - `evidence` — the structured evidence the check captured. + - `recommendation` — the check author's verbatim suggested fix. + - `agent_action` — `auto_apply | propose_patch_for_review | escalate_to_human | suppress_with_reason | informational`. + - `metadata` — full `CheckMetadata` (rationale, fires_when, evidence_fields, docs_url) when the check is in the catalog. + - `explanation` — a deterministic 3–5-sentence prose summary you can quote verbatim or rewrite. + +3. **Write the prose for the user.** Three to five sentences, in this order: + 1. **What.** Name the check (`check_id` is fine), the affected tool (`tool_name`), and the severity in one sentence. If the check has no `tool_name`, name what the check examined (e.g. "the manifest", "permissions"). + 2. **Why it matters.** Pull from `metadata.rationale` or `metadata.fires_when`. If neither exists, paraphrase the `recommendation`. Avoid verbatim verbose catalog text — translate "limited automation review" into plain English. + 3. **What you'll do (or want).** Map `agent_action` to a concrete next step: + - `auto_apply`: "I can apply the fix automatically — say yes and I'll run `apply-patches --confidence high --apply`." + - `propose_patch_for_review`: "There's a suggested patch but the confidence is medium/low (or there's a manual sibling). Want me to show the diff before applying?" + - `escalate_to_human`: "There's no automatic fix. Here's the recommended remediation: [paraphrase recommendation]. Want me to draft the change for you to review?" + - `suppress_with_reason`: "If you want to accept this risk, I can add a suppression with reason. What should the reason say?" + - `informational`: "No action needed; flagging for awareness." + 4. *(Optional)* **Where to learn more.** If `metadata.docs_url` exists, link it. + 5. *(Optional)* **Suppression status.** If `suppressed` is true, mention that — otherwise omit. + +4. **Cite evidence sparingly.** Only quote a specific evidence value when it makes the explanation concrete (e.g. naming the broken parameter, the file path from `source.ref`). Do not dump the whole `evidence` dict. + +5. **Format for the surface.** PR comments and chat support markdown — use a code span for `check_id` and `tool_name`. Plain text emails should drop the backticks but keep the structure. + +## Example + +Input (from `explain-finding fp_f092940f62fbb012 --from ... --json`): +```json +{ + "check_id": "SHIP-POLICY-APPROVAL-MISSING", + "severity": "critical", + "tool_name": "stripe.create_refund", + "agent_action": "escalate_to_human", + "recommendation": "Declare an approval policy or remove the tool.", + "metadata": { + "rationale": "High-risk actions need explicit approval before promotion.", + "fires_when": "Financial/destructive risk exists without approval policy." + } +} +``` + +Good prose for a PR comment: + +> The Tool-Use Readiness scan flagged a critical issue: `stripe.create_refund` doesn't declare an approval policy in `shipgate.yaml`. High-risk actions like refunds need an explicit human approval gate before they can ship — without one, an agent could trigger a refund on its own without review. There's no automatic fix here. The right remediation is to either add `policies.require_approval_for_tools: [stripe.create_refund]` (with a reviewer-visible approval trace) or remove the tool from this release surface. Want me to draft the manifest change for you? + +Bad prose for the same input: + +> Finding `fp_f092940f62fbb012`: `SHIP-POLICY-APPROVAL-MISSING` fired with severity `critical` on `stripe.create_refund`. autofix_safe=false, requires_human_review=true. evidence: risk_tags=[financial_action, destructive]. recommendation: "Declare an approval policy or remove the tool." + +The bad version is true but unreadable — it dumps the JSON instead of translating it. + +## What NOT to do + +- Do **not** quote the structured `explanation` field verbatim if it's robotic. It's a deterministic baseline; rewrite for tone when needed. +- Do **not** fabricate consequences. If the check's `rationale` doesn't say "could trigger a refund," don't say it. Stay grounded in catalog text. +- Do **not** propose `apply-patches` for `escalate_to_human` findings — the user has to decide on the fix manually. +- Do **not** propose adding a `checks.ignore` entry as the default response. Suppression is a real choice, but it's the last resort and needs an audit-trail-quality reason. Use the [`triage-false-positive.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/prompts/triage-false-positive.md) prompt for that workflow. +- Do **not** include the fingerprint string in the user-facing prose unless they specifically asked for it. Fingerprints are agent-to-agent identifiers, not human-friendly labels. + +## Verification before sending the message + +- The user-facing prose names the affected tool (or what the check examined) at least once. +- The severity is mentioned somewhere (a word like "critical" or "medium-severity" — not just the JSON token). +- The action sentence matches the finding's `agent_action`. If the message says "I'll apply this automatically," `agent_action` must be `auto_apply`. +- No raw JSON dumps in the prose — translate, don't quote. +- If `metadata.docs_url` exists, include it (or link text equivalent). diff --git a/plugins/claude-code/skills/agents-shipgate/prompts/fix-top-finding.md b/plugins/claude-code/skills/agents-shipgate/prompts/fix-top-finding.md new file mode 100644 index 00000000..e2a45c8f --- /dev/null +++ b/plugins/claude-code/skills/agents-shipgate/prompts/fix-top-finding.md @@ -0,0 +1,72 @@ +# Prompt · Fix the top Agents Shipgate finding + +You are working in a repo with `shipgate.yaml` already in place. Run a scan and fix the highest-severity unsuppressed finding. + +## Your task + +1. **Run a scan and locate the top finding.** + ```bash + agents-shipgate scan -c shipgate.yaml --ci-mode advisory + ``` + Read `agents-shipgate-reports/report.json`. For v0.12+ reports the easy path is `agent_summary.first_recommended_action.why` — for most `blocked`/`review_required` verdicts it names the top finding's `check_id` and `tool_name` directly. Three exceptions to expect: + + - **`insufficient_evidence` verdict** (v0.14+; the scan saw too many low-confidence tools or 4+ source warnings to gate release). There is no specific finding to fix; the action's `why` describes the evidence situation and recommends gathering deeper sources (MCP/OpenAPI inputs, eval traces, additional source files). Follow that guidance instead of looking for a top finding. + - **Evidence-coverage-driven `review_required`** (sub-threshold low-confidence/static evidence; no specific finding to fix). The action's `why` describes the evidence situation and recommends gathering MCP/OpenAPI inputs or eval traces — there is no `check_id` to parse out. If you see "low-confidence evidence" or "static-only" in the why-text, follow that guidance instead of looking for a top finding. + - **`auto_appliable_patches > 0`**. The action proposes `apply-patches`; the why-text names the apply-patches command, not a specific finding. Walk `findings[]` for the actual top entry. + + Fall back to picking the entry with the highest severity (`critical > high > medium > low > info`) and `"suppressed": false` whenever the action doesn't name a finding directly. + +2. **Look up the check definition.** + ```bash + agents-shipgate explain --json + ``` + This returns the `CheckMetadata` with `description`, `rationale`, `fires_when`, `evidence_fields`, `recommendation`. + +3. **Diagnose the fix.** There are exactly four legitimate responses to a finding. v0.12+ reports project the routing via `agent_action`: + + | Response | When | `agent_action` (v0.12+) | + |---|---|---| + | **Add the missing policy / scope / annotation** to `shipgate.yaml` | The check is correct; the manifest just hadn't declared the safeguard yet | `propose_patch_for_review` (a `set_pointer`/`append_pointer` patch is attached) or `escalate_to_human` (no patch — you write the entry by hand) | + | **Override the heuristic** via `risk_overrides.tools.{tool}.{tags,remove_tags}` | The risk classification is wrong (e.g. a GET endpoint that picked up the `destructive` tag because of a misleading operationId) | `escalate_to_human` | + | **Suppress the finding** via `checks.ignore` with a `reason` | The check is correct but you've decided to accept the risk explicitly (e.g. "tool deprecated 2026-Q2") | `escalate_to_human` (the future `suppress_with_reason` value is reserved for checks that pre-classify themselves as suppressible) | + | **Fix the underlying tool definition** | The tool spec itself is wrong (missing description, broad scope, free-form action field) | `escalate_to_human` | + +4. **Apply the fix.** Edit either `shipgate.yaml` or the tool source file. Do not delete tools wholesale to silence findings. + +5. **Re-scan and confirm the count went down.** + ```bash + agents-shipgate scan -c shipgate.yaml --ci-mode advisory + ``` + The previously-failing fingerprint should be gone from `report.json`. + +6. **Report back**: + - What was the original finding (check ID, tool, severity) + - Which of the four response types you used + - The diff to `shipgate.yaml` (or other file) you applied + - The new finding count + +## Common fixes by check ID + +| Check | Typical fix | +|---|---| +| `SHIP-POLICY-APPROVAL-MISSING` | Add the tool to `policies.require_approval_for_tools` with a reason | +| `SHIP-POLICY-CONFIRMATION-MISSING` | Add the tool to `policies.require_confirmation_for_tools` | +| `SHIP-SIDEFX-IDEMPOTENCY-MISSING` | Add an `idempotency_key` parameter, set `idempotentHint: true` annotation, or list under `policies.require_idempotency_for_tools` | +| `SHIP-AUTH-MISSING-SCOPE` | Declare the scope on the tool (in OpenAPI security or MCP metadata) and in `permissions.scopes` | +| `SHIP-AUTH-MANIFEST-BROAD-SCOPE` | Replace `*` / `admin` with the specific operation scope(s) | +| `SHIP-DOC-MISSING-DESCRIPTION` | Add a 20+ char description to the tool definition | +| `SHIP-SCHEMA-BROAD-FREE-TEXT` | Constrain the parameter with an enum, structured schema, or narrower fields | +| `SHIP-SCHEMA-MISSING-BOUNDS` | Add `maximum` to the numeric parameter | +| `SHIP-INVENTORY-LOW-CONFIDENCE-PRODUCTION-SURFACE` | Declare the tools through MCP/OpenAPI for higher-confidence inventory; or move target to staging | + +## What NOT to do + +- Do not blanket-suppress an entire check. Suppressions are per-tool unless the check is genuinely irrelevant for this repo. +- Do not write `reason: "false positive"` without explanation. Reviewers should be able to read the reason and understand the decision in 60 seconds. +- Do not edit `agents-shipgate-reports/`. It's regenerated each run. + +## Verification + +- The previously-failing finding's fingerprint is no longer present in `report.json` +- The fix is committed in a single, focused diff (manifest change + reason) +- If you used `checks.ignore`, the `reason` is concrete (a date, a ticket link, or "tool deprecated; see roadmap") diff --git a/plugins/claude-code/skills/agents-shipgate/prompts/recommend-fixes.md b/plugins/claude-code/skills/agents-shipgate/prompts/recommend-fixes.md new file mode 100644 index 00000000..abe986bb --- /dev/null +++ b/plugins/claude-code/skills/agents-shipgate/prompts/recommend-fixes.md @@ -0,0 +1,82 @@ +# Prompt · Recommend fixes for active Agents Shipgate findings + +You are working in a repo with `shipgate.yaml` already in place and want a coordinated remediation pass across **all** active findings — not just the top one. Walk every finding, classify it against the current autofix policy, and surface targeted fix recommendations. Apply only the safe, high-confidence patches (after preview + explicit confirmation); leave the rest for human review with concrete advice. + +## Your task + +1. **Always run a fresh v0.8+ scan with patches.** Do not reuse a stale report — earlier scans may be pre-v0.7 (no remediation fields), pre-v0.8 (no `release_decision`), or may lack `patches[]` (no `--suggest-patches`). Set `AGENTS_SHIPGATE_AGENT_MODE=1` so errors emit a `next_action` JSON line on stderr. + ```bash + AGENTS_SHIPGATE_AGENT_MODE=1 agents-shipgate scan -c shipgate.yaml \ + --suggest-patches --format json --ci-mode advisory + ``` + Read `agents-shipgate-reports/report.json`. Verify `report_schema_version` is `"0.8"` or higher. Filter `findings[]` to entries with `"suppressed": false`. + +2. **Bucket each active finding into one of four classes.** Read `agent_action` (v0.12+; deterministic projection of patches/autofix/human-review fields) to bucket each active finding directly. If `agent_action` is missing (older v0.11 or earlier reports), fall back to the legacy three-field check shown in the right column. The buckets correspond to [`docs/autofix-policy.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/docs/autofix-policy.md): + + | Bucket | `agent_action` (v0.12+) | Legacy fallback (v0.11 or earlier) | Example check IDs | + |---|---|---|---| + | **A. Safe auto-fix** | `auto_apply` | `autofix_safe == true` | `SHIP-MANIFEST-STALE-{SUPPRESSION,POLICY,RISK-OVERRIDE}` when the match is unique | + | **B. Medium-confidence config fix** | `propose_patch_for_review` | `autofix_safe == false` AND `suggested_patch_kind` ∈ `{set_pointer, append_pointer, remove_pointer}` | `SHIP-AUTH-SCOPE-COVERAGE-MISSING` | + | **C. Manual** | `escalate_to_human` (with `suggested_patch_kind == "manual"`) | `suggested_patch_kind == "manual"` | Documentation, schema bounds, owner gaps, ADK/LangChain/CrewAI metadata, and the never-auto-fix trace findings | + | **D. No patch emitted** | `escalate_to_human` (with `suggested_patch_kind == "none"`) | `suggested_patch_kind == "none"` | The generator emitted nothing — but the finding can still be high/critical (e.g. low-confidence inventory). Treat as **human triage**, not informational. | + | (skip) | `informational` | `suppressed == true` | Already-suppressed findings; show counts only. | + + For one-fetch counts read the top-level `agent_summary` block (v0.12+): + `agent_summary.auto_appliable_patches` is the bucket-A count, and + `agent_summary.needs_human_review` is buckets B + C + D combined + (every active finding the user must weigh in on before applying — + medium/low-confidence patches AND escalations). To split bucket B + from bucket C+D you have to walk `findings[].agent_action` — + agent_summary deliberately does not disaggregate them, since the + distinction is an implementation detail of the patch-confidence + policy rather than a release-gate signal. Use + `agent_summary.first_recommended_action.command` as your default + suggestion when bucket A is non-empty. + +3. **Build a recommendation card per finding.** For each, present: + - `check_id`, `title`, `severity`, `tool_name`, `confidence` + - The verbatim `recommendation` string (per-finding fix text from the check author) + - `docs_url` as a markdown link (when non-null) + - **Concrete fix step** — branch on patch kind, since the patch shapes differ: + - `set_pointer` / `append_pointer`: show `target_file`, `pointer`, `value`, `confidence`, `rationale` + - `remove_pointer`: show `target_file`, `pointer`, `confidence`, `rationale` + - `manual`: show `instructions` verbatim. `ManualPatch` has only `kind` and `instructions` — do NOT try to read `target_file`/`pointer`/`value`; they don't exist. + - No patches (bucket D): use `evidence` and `source` to make `recommendation` concrete — quote the offending parameter name, the file path from `source.ref`, the manifest key. Generic advice is not acceptable here. + +4. **Present the prioritised plan.** Severity-ordered (critical → high → medium → low → info), grouped by bucket within each severity tier. Show counts per bucket up front. For low/info findings in bucket D, summary-link via `docs_url` rather than full cards — avoid wall-of-text. + +5. **Decision points — ask the user explicitly. Always preview before mutating.** + - **Bucket A (safe auto-fix).** First run a **dry-run** (omit `--apply`): + ```bash + agents-shipgate apply-patches \ + --from agents-shipgate-reports/report.json \ + --confidence high + ``` + Show the user the planned file diffs. Only after explicit confirmation, re-run with `--apply --json`. Never silently apply. + - **Bucket B (medium-confidence config).** Surface the patches with their `pointer` and `value`. Tell the user the opt-in command (`apply-patches --confidence medium`) and that they must read the appended values first — scope strings can encode policy choices. Do not apply on the user's behalf in this recipe. + - **Bucket C (manual).** Ask whether to walk through them now or defer. For deep dive on a single finding, cross-link to [`fix-top-finding.md`](fix-top-finding.md). Never edit a trace recording to silence `SHIP-API-TRACE-{APPROVAL,CONFIRMATION}-MISSING` — that patches the evidence, not the agent. Implement the runtime gate instead. + - **Bucket D (no patch).** Ask whether to walk through them — these need diagnosis, not patch application. Cross-link to [`fix-top-finding.md`](fix-top-finding.md); the four-response decision tree (add policy / override / suppress / fix tool spec) applies. + +6. **Re-scan after applying any Bucket A patches.** Show the diff in `summary.{critical_count, high_count, medium_count}`. Confirm the previously-fixed fingerprints are gone from `report.json`. + +7. **Report back**: + - Counts per bucket (A/B/C/D) and per severity + - What was applied (from `apply-patches --apply --json` output's `files`) + - What remains, with one clear next action per remaining bucket + - Any cross-links the user should follow ([`fix-top-finding.md`](fix-top-finding.md), [`triage-false-positive.md`](triage-false-positive.md)) + +## What NOT to do + +- Do **not** run `apply-patches --apply` without showing the dry-run preview first AND getting explicit user confirmation, even when `autofix_safe == true`. +- Do **not** apply `--confidence medium` patches in this recipe. They are opt-in only and require the user to read the appended values. +- Do **not** edit a trace recording to silence `SHIP-API-TRACE-{APPROVAL,CONFIRMATION}-MISSING`. Trace findings are class-four "never auto-fix" per the autofix policy. Implement the runtime approval/confirmation gate. +- Do **not** recommend `checks.ignore` as a fix here. That's the [`triage-false-positive.md`](triage-false-positive.md) workflow's job — cross-link to it. +- Do **not** claim a finding is fixed without re-running `agents-shipgate scan` and showing the diff in counts. +- Do **not** invent recommendations not grounded in `recommendation`, `evidence`, `patches[].instructions`, or `docs_url`. Use evidence to make advice concrete; do not replace check-author guidance with a guess. + +## Verification + +- A fresh `report.json` exists, validates as `report_schema_version: "0.8"` (or higher; v0.12+ exposes `agent_action` and `agent_summary`), and was generated with `--suggest-patches`. +- Each presented card cites a concrete location: `target_file` + `pointer` for non-manual patches, `instructions` verbatim for manual patches, file path + parameter name from `evidence`/`source` for bucket D. +- If Bucket A patches were applied: re-scan shows lower active counts AND the previously-failing fingerprints are absent from the new `report.json`. +- If only B/C/D were surfaced: counts are unchanged (expected); the user has a clear list of next actions. diff --git a/plugins/claude-code/skills/agents-shipgate/prompts/stabilize-strict-mode.md b/plugins/claude-code/skills/agents-shipgate/prompts/stabilize-strict-mode.md new file mode 100644 index 00000000..45cccd27 --- /dev/null +++ b/plugins/claude-code/skills/agents-shipgate/prompts/stabilize-strict-mode.md @@ -0,0 +1,77 @@ +# Prompt · Stabilize Agents Shipgate strict mode + +The user has Agents Shipgate running in **advisory** mode and wants to graduate to **strict** mode (CI fails on findings) without surprising contributors. + +## The pattern + +1. Run a fresh scan and inventory the active findings. +2. Tune `risk_overrides` and `checks.ignore` for genuine false positives, with reasons. +3. Save a baseline of everything that's left. +4. Switch CI to strict mode with the baseline applied — only NEW findings fail. +5. Pick a severity threshold; usually start with `critical`, raise to `[critical, high]` later. + +## Your task + +1. **Inventory current findings.** + ```bash + agents-shipgate scan -c shipgate.yaml --ci-mode advisory + ``` + Look at `agents-shipgate-reports/report.json` `summary.critical_count`, `high_count`, `medium_count`. If the active list is small (< 20 unique check IDs), consider just fixing them rather than baselining. + +2. **Tune false positives.** For each unique check ID, decide: + - True positive that should be fixed → use the `fix-top-finding.md` prompt to apply a real fix. + - True positive that the team explicitly accepts (deprecated tool, known limitation) → add to `checks.ignore` with a real `reason`. + - False positive (heuristic misfire) → use `risk_overrides.tools.{tool}.remove_tags` or add tags via `risk_overrides.tools.{tool}.tags`. + +3. **Save the baseline:** + ```bash + agents-shipgate baseline save -c shipgate.yaml \ + --out .agents-shipgate/baseline.json + ``` + +4. **Commit the baseline:** + ```bash + git add .agents-shipgate/baseline.json + git commit -m "Baseline shipgate findings ($N criticals, $M highs)" + ``` + +5. **Update the CI workflow.** Replace the existing advisory step with strict + baseline. Use [`examples/github-actions/03-strict-with-baseline.yml`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/examples/github-actions/03-strict-with-baseline.yml) as the template: + ```yaml + - uses: ThreeMoonsLab/agents-shipgate@v0.14.0 + with: + shipgate_version: '0.14.0' + ci_mode: strict + fail_on: critical + baseline: .agents-shipgate/baseline.json + pr_comment: 'true' + ``` + +6. **Verify the gate fires correctly.** In a throwaway branch, deliberately introduce a new finding (e.g. add a wildcard scope) and confirm CI fails. Revert before merging. + +## When to refresh the baseline + +| Situation | Action | +|---|---| +| Found a false positive after baselining | Add a `checks.ignore` entry; do **not** re-baseline | +| Fixed several findings | Re-baseline so resolved ones disappear: `agents-shipgate baseline save ...` | +| Upgraded shipgate to a version with new checks | New check IDs surface as new findings; fix or suppress, then re-baseline | +| Added new tools that have no policy yet | Each new tool's findings are `new` and will fail; fix or accept, then re-baseline | + +Re-baselining is just running `baseline save` again. Diff the new file vs the old in code review so the team sees what's been accepted. + +## Promotion to `[critical, high]` + +After a sprint or two of strict-on-critical, the active high-severity list usually compresses enough to flip on. Update `fail_on: critical,high` and re-baseline. + +## What NOT to do + +- Do **not** baseline in your first run as a "shortcut to make CI green." That hides the existing risk surface from review. +- Do **not** baseline findings that have a real fix — fix them first, baseline only what you're explicitly accepting. +- Do **not** write `--fail-on critical,high` without a baseline if the repo has many existing high findings; CI will fail on day one and contributors will mute the workflow. + +## Verification + +- `.agents-shipgate/baseline.json` is committed and contains `findings[]` +- CI workflow uses `ci_mode: strict` and `baseline: .agents-shipgate/baseline.json` +- A test PR that adds a deliberate new critical finding fails CI +- A test PR that doesn't change the tool surface passes CI diff --git a/plugins/claude-code/skills/agents-shipgate/prompts/triage-false-positive.md b/plugins/claude-code/skills/agents-shipgate/prompts/triage-false-positive.md new file mode 100644 index 00000000..db39be5a --- /dev/null +++ b/plugins/claude-code/skills/agents-shipgate/prompts/triage-false-positive.md @@ -0,0 +1,90 @@ +# Prompt · Triage a suspected Agents Shipgate false positive + +The user thinks a specific finding is wrong. You need to decide whether to override the heuristic, suppress the finding, or convince the user that the check is correct. + +## Your task + +1. **Read the full finding.** From `agents-shipgate-reports/report.json`: + ```json + { + "id": "fp_...", + "check_id": "SHIP-...", + "tool_name": "...", + "severity": "...", + "evidence": { ... }, + "recommendation": "..." + } + ``` + And the check definition: + ```bash + agents-shipgate explain --json + ``` + +2. **Read the actual tool definition.** Look up the OpenAPI / MCP / SDK source: + - For OpenAPI: open the spec at the path given in `findings[].source.ref` + - For MCP: open the JSON file + - For SDK: open the `.py` file at the line given in `source.location` + +3. **Apply the decision tree:** + + ``` + Is the heuristic wrong about the tool? + (e.g. "destructive" tag on a GET; "financial_action" tag on a non-financial scope) + → YES: override via risk_overrides.tools.{tool}.remove_tags + → NO: continue + + Is the check fundamentally inapplicable to this tool? + (e.g. SHIP-DOC-MISSING-DESCRIPTION on an internal-only tool slated for removal) + → YES: suppress via checks.ignore with a concrete reason + → NO: continue + + The check is correct. Fix the tool definition. + → use the fix-top-finding.md prompt + ``` + +## Override vs suppress — which to use + +| Use `risk_overrides` when | Use `checks.ignore` when | +|---|---| +| The risk **classification** is wrong | The classification is right but the team accepts the risk | +| You want to remove a tag (e.g. `remove_tags: [destructive]`) | You want to suppress one specific finding | +| The fix benefits all checks that consume that tag | The acceptance is per-check, per-tool | +| Example: a `get_records` GET picks up `destructive` from substring "destroy" | Example: a documented internal-only tool with no description | + +**Rule of thumb:** if the fix would silence multiple findings naturally, use `risk_overrides`. If you want to acknowledge one specific finding by name, use `checks.ignore`. + +## Required: a concrete `reason` + +Both `checks.ignore` entries and `risk_overrides` entries take a `reason`. Empty reasons fail manifest validation. Good reasons answer "why is this OK?" in a way a future reviewer can verify: + +| Bad reason | Better reason | +|---|---| +| `false positive` | `GET endpoint; "destroy" appears in operationId only because it returns destroy-status` | +| `not applicable` | `Tool deprecated 2026-Q2; deletion tracked in JIRA-1234` | +| `team decision` | `Reviewed by platform-eng 2026-04-10; see ADR-007` | + +## Re-run and confirm + +After editing the manifest: + +```bash +agents-shipgate scan -c shipgate.yaml --ci-mode advisory +``` + +The previously-failing fingerprint should be gone (overridden) or marked `"suppressed": true` (suppressed) in `report.json`. + +## When the heuristic is genuinely buggy + +If you've found a real classifier bug — the kind that affects many users, not just this tool — file an issue tagged `false-positive` at https://github.com/ThreeMoonsLab/agents-shipgate/issues with: + +- The check ID +- A minimal reproduction (manifest fragment + tool source) +- The current behavior vs. expected behavior + +The risk classifier in `core/risk_hints.py` improves through reports. + +## Verification + +- The decision (override / suppress / fix) is documented in the manifest with a reason. +- The previously-failing fingerprint is gone or `"suppressed": true` in the next scan. +- The `reason` would be understandable to a reviewer who hasn't seen the finding. diff --git a/plugins/claude-code/skills/agents-shipgate/prompts/upgrade-shipgate-version.md b/plugins/claude-code/skills/agents-shipgate/prompts/upgrade-shipgate-version.md new file mode 100644 index 00000000..0fdec8d8 --- /dev/null +++ b/plugins/claude-code/skills/agents-shipgate/prompts/upgrade-shipgate-version.md @@ -0,0 +1,73 @@ +# Prompt · Upgrade Agents Shipgate version + +Bump the agents-shipgate version pinned in CI and the development environment. + +## Your task + +1. **Read the changelog** for the gap between the current and target version: + - https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/CHANGELOG.md + - Specifically look for entries under "Breaking changes" and "New checks added". + +2. **Update the pin in three places** (in this order): + + a. **`pyproject.toml`** (if the project depends on shipgate as a dev dep): + ```toml + [project.optional-dependencies] + dev = ["agents-shipgate==", ...] + ``` + + b. **CI workflow** at `.github/workflows/shipgate.yml`: + ```yaml + - uses: ThreeMoonsLab/agents-shipgate@v + with: + shipgate_version: '' + ``` + + c. **Pre-commit config** at `.pre-commit-config.yaml` (if present): + ```yaml + repos: + - repo: https://github.com/ThreeMoonsLab/agents-shipgate + rev: v + ``` + +3. **Run a local scan** with the new version: + ```bash + pipx upgrade agents-shipgate + agents-shipgate --version # confirm the new version is in PATH + agents-shipgate scan -c shipgate.yaml --ci-mode advisory + ``` + +4. **Compare the new finding count to the baseline.** If `report.json` shows new finding fingerprints (any with `"baseline_status": "new"`): + - These are usually new checks added in the upgrade. Read the changelog "New checks added" section. + - For each new check ID, decide: fix, override, or suppress (see [`triage-false-positive.md`](triage-false-positive.md)). + +5. **Re-baseline if the new findings are accepted:** + ```bash + agents-shipgate baseline save -c shipgate.yaml \ + --out .agents-shipgate/baseline.json + ``` + +6. **Commit** the version bumps + the new baseline (if regenerated) in one PR. Title: `Upgrade agents-shipgate v → v`. + +## Stability guarantees + +Per [`STABILITY.md`](https://github.com/ThreeMoonsLab/agents-shipgate/blob/main/STABILITY.md), within `0.x`: + +- Existing check IDs do not change names or fingerprint algorithms. +- Existing CLI flags do not break. +- The JSON report's stable fields persist. + +So a `0.2.x → 0.3.x` upgrade should not silently break existing suppressions or baselines. If it does, that's a stability bug — file an issue. + +## What may legitimately change + +- Risk-classifier keyword sets (false-positive tuning). Use `risk_overrides` to pin specific behavior. +- New checks fire (additive). Triage with the prompts above. +- Markdown report layout (parse `report.json` instead). + +## Verification + +- `agents-shipgate --version` reflects the new version +- CI workflow uses the new version +- A scan completes without error +- The baseline file (if used) is up to date diff --git a/plugins/claude-code/skills/agents-shipgate/prompts/verify-agent-diff.md b/plugins/claude-code/skills/agents-shipgate/prompts/verify-agent-diff.md new file mode 100644 index 00000000..18a77ff4 --- /dev/null +++ b/plugins/claude-code/skills/agents-shipgate/prompts/verify-agent-diff.md @@ -0,0 +1,96 @@ +# Prompt · Verify an agent-related diff + +You are reviewing or finishing a change to a tool-using AI agent. Use Agents +Shipgate as the deterministic verifier for the diff before you report that the +work is complete. + +## Your task + +1. **Set agent mode for structured recovery hints.** + ```bash + export AGENTS_SHIPGATE_AGENT_MODE=1 + ``` + +2. **Use verify preview only when relevance or setup is unclear.** + ```bash + agents-shipgate verify --preview --json + ``` + Preview is a lightweight verify entry point: no manifest required, no scan, + exit 0. It tells you whether to configure Shipgate, skip, or run the full + verifier. If the repo already has `shipgate.yaml`, proceed to full verify. + +3. **Treat protected-surface edits as verifier-owned review.** + Before editing `shipgate.yaml`, Shipgate CI, AGENTS/CLAUDE/Cursor rules, + policy packs, baselines, waivers, suppressions, Codex hooks/config, Codex + plugin manifests, `.mcp.json`, `.app.json`, or `SKILL.md`, do not + self-approve the trust-root change. Run full verify before reporting + completion and route human review when the verifier requires it. + +4. **Run the verifier.** + For local uncommitted work, omit `--head` and omit `--base` so the + checked-out working tree is scanned, including uncommitted edits: + ```bash + agents-shipgate verify --workspace . --config shipgate.yaml \ + --ci-mode advisory --format json + ``` + For committed PR or CI verification, pass the head ref explicitly: + ```bash + agents-shipgate verify --workspace . --config shipgate.yaml \ + --base origin/main --head HEAD --ci-mode advisory --format json + ``` + `verify` never fetches. If you pass `--base` and that ref is missing, + `verify` exits 2 with an unknown merge verdict instead of producing a + head-only pass. Fetch the base ref for committed PR/CI verification, or + omit `--base` for local working-tree verification. + +5. **Read JSON, not Markdown.** + - `agents-shipgate-reports/verifier.json` is the PR/controller artifact. + - Lead with `merge_verdict`, `applicability`, and `agent_controller`, then + inspect `first_next_action.actor` and `fix_task.safe_to_attempt`. + - `agents-shipgate-reports/report.json` is the release-gate artifact. + - `release_decision.decision` is the only gate signal. + - `capability_review.top_changes[]` and `verifier_summary` are + supporting/provisional composition summaries; verdict-like values mirror + `release_decision.decision` and never gate independently. + +6. **Do not bypass the verifier.** Do not suppress findings, lower severity, + expand baselines or waivers, remove Shipgate CI, or soften agent + instructions to make the run pass. Those trust-root edits are protected by + `SHIP-VERIFY-*` findings and require human review. + +7. **Report back with:** + - `merge_verdict` and `headline` from `verifier.json` + - `capability_review.top_changes[]` + - `first_next_action.actor` and `fix_task.safe_to_attempt` + - `release_decision.decision` and `release_decision.reason` + - blocker count and review-item count + - `verifier_summary.protected_surface_touched` + - `verifier_summary.policy_weakened` + - top `verifier_summary.top_reason_codes[]` + - whether `verifier.json.base_status` was `succeeded`, `cache_hit`, or a + degraded status + - the next safe action from `agent_summary.first_recommended_action` + +## What NOT to do + +- Do not claim the diff is verified until `agents-shipgate verify` has run or + `agents-shipgate verify --preview --json` has returned a clear skip verdict. +- Do not claim completion when `merge_verdict` is `blocked`, + `insufficient_evidence`, or `human_review_required` unless the user + explicitly accepts human review. +- Do not use `summary.status` for gating; it is legacy and baseline-blind. +- Do not invent approval, confirmation, idempotency, prohibited-action, + broad-scope, human acknowledgement, or runtime trace evidence. +- Do not commit `agents-shipgate-reports/`. + +## Verification + +- `agents-shipgate-reports/report.json` exists and parses. +- `agents-shipgate-reports/verifier.json` exists and parses. +- `verifier.json.merge_verdict` is surfaced to the user. +- `applicability` and `agent_controller` are considered before generic findings. +- `capability_review.top_changes[]` is treated as supporting/provisional review + context. +- `report.json.release_decision.decision` is surfaced to the user. +- If `verifier_summary.protected_surface_touched` or `policy_weakened` is true, + the response names the human-review requirement. diff --git a/skills/agents-shipgate/SKILL.md b/skills/agents-shipgate/SKILL.md index 371c6479..f2dd010e 100644 --- a/skills/agents-shipgate/SKILL.md +++ b/skills/agents-shipgate/SKILL.md @@ -1,6 +1,6 @@ --- name: agents-shipgate -description: Run prominent Agents Shipgate flows when a change touches what an AI agent can do: `shipgate check`, `agents-shipgate verify`, or `shipgate audit --host`. Use after adding or modifying MCP servers or tools, tool/function definitions (@tool, @function_tool), OpenAPI specs that describe agent tools, agent prompts, permission scopes, approval or confirmation policies, agent CI workflows, or shipgate.yaml — and before creating a PR for any such change. Also use to verify agent-related PRs, fix or triage Shipgate findings, add Shipgate to CI, or interpret Shipgate verifier/report artifacts. Triggers on phrases like "add shipgate", "verify this agent PR", "merge verdict", "release readiness for my agent", "tool-use readiness", "shipgate check", "agents-shipgate verify", "audit host grants", "shipgate.yaml", "agents-shipgate-reports/verifier.json", "agents-shipgate-reports/report.json", "fix shipgate finding". +description: 'Run prominent Agents Shipgate flows when a change touches what an AI agent can do: `shipgate check`, `agents-shipgate verify`, or `shipgate audit --host`. Use after adding or modifying MCP servers or tools, tool/function definitions (@tool, @function_tool), OpenAPI specs that describe agent tools, agent prompts, permission scopes, approval or confirmation policies, agent CI workflows, or shipgate.yaml — and before creating a PR for any such change. Also use to verify agent-related PRs, fix or triage Shipgate findings, add Shipgate to CI, or interpret Shipgate verifier/report artifacts. Triggers on phrases like "add shipgate", "verify this agent PR", "merge verdict", "release readiness for my agent", "tool-use readiness", "shipgate check", "agents-shipgate verify", "audit host grants", "shipgate.yaml", "agents-shipgate-reports/verifier.json", "agents-shipgate-reports/report.json", "fix shipgate finding".' --- # agents-shipgate skill diff --git a/tests/test_agent_instructions_renderers.py b/tests/test_agent_instructions_renderers.py index 86487562..e0a9a017 100644 --- a/tests/test_agent_instructions_renderers.py +++ b/tests/test_agent_instructions_renderers.py @@ -45,7 +45,7 @@ REPO_ROOT = Path(__file__).resolve().parent.parent EXPECTED_CLAUDE_CODE_SKILL_RENDER_SHA256 = { ".claude/skills/agents-shipgate/SKILL.md": ( - "1b4470b625110acdc6f1e3930b7e6e6f43653c493e9a44c7ccde0dd9c88b09bb" + "52607a7b0ed5b83f4512618bbb9f277b77649943e66aa120dcf38aca7bfe1e16" ), ".claude/skills/agents-shipgate/ci-recipes/advisory-pr-comment.yml": ( "82957a521b5914b3e678e6b76e7088306559c8ad6bcf7c2dce7fb1e822b6bec6" diff --git a/tests/test_claude_code_plugin_package.py b/tests/test_claude_code_plugin_package.py new file mode 100644 index 00000000..3b19273c --- /dev/null +++ b/tests/test_claude_code_plugin_package.py @@ -0,0 +1,116 @@ +"""Claude Code plugin marketplace package contract. + +The repo doubles as a Claude Code plugin marketplace +(``/plugin marketplace add ThreeMoonsLab/agents-shipgate``), the symmetric +counterpart of the Codex marketplace under ``.agents/plugins/`` +(tests/test_codex_plugin_launch_package.py). Same discipline: the plugin is +skill-only (workflows, not the scanner binary), its skill and command are +byte-identical copies of the canonical sources, and every version literal +moves with ``__version__`` on release. + +The frontmatter tests exist because ``claude plugin validate`` caught the +canonical SKILL.md and /shipgate command shipping YAML frontmatter that +failed to parse (unquoted ``:`` in ``description``) — Claude Code loads such +files with silently-empty metadata, which kills description-based skill +auto-triggering. That must never regress. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import yaml + +from agents_shipgate import __version__ + +REPO_ROOT = Path(__file__).resolve().parent.parent +MARKETPLACE_PATH = REPO_ROOT / ".claude-plugin" / "marketplace.json" +PLUGIN_ROOT = REPO_ROOT / "plugins" / "claude-code" +CANONICAL_SKILL = REPO_ROOT / "skills" / "agents-shipgate" +CANONICAL_COMMAND = REPO_ROOT / ".claude" / "commands" / "shipgate.md" + + +def _frontmatter(path: Path) -> dict: + text = path.read_text(encoding="utf-8") + assert text.startswith("---\n"), f"{path} has no frontmatter" + block = text.split("---\n", 2)[1] + parsed = yaml.safe_load(block) + assert isinstance(parsed, dict), ( + f"{path} frontmatter failed to parse as a YAML mapping — Claude Code " + "loads it with silently-empty metadata (the validate finding this " + "suite exists to prevent)." + ) + return parsed + + +def test_marketplace_manifest_is_installable() -> None: + marketplace = json.loads(MARKETPLACE_PATH.read_text(encoding="utf-8")) + assert marketplace["name"] == "agents-shipgate" + assert marketplace["owner"]["name"] == "Three Moons Lab" + [entry] = marketplace["plugins"] + assert entry["name"] == "agents-shipgate" + assert entry["source"] == "./plugins/claude-code" + assert (REPO_ROOT / "plugins" / "claude-code" / ".claude-plugin" / "plugin.json").is_file() + assert entry["version"] == __version__, ( + "marketplace plugin entry version must move with __version__ on " + "every release bump." + ) + assert entry["strict"] is True + + +def test_plugin_manifest_is_skill_only_and_versioned() -> None: + manifest = json.loads( + (PLUGIN_ROOT / ".claude-plugin" / "plugin.json").read_text(encoding="utf-8") + ) + assert manifest["name"] == "agents-shipgate" + assert manifest["version"] == __version__, ( + "plugin.json version must move with __version__ on every release bump." + ) + # Skill-only, like the Codex plugin: no hooks, MCP servers, agents, or + # bin/ shipped from the marketplace. Hooks stay on the explicit + # install-hooks CLI path where the agents-shipgate binary must exist. + for key in ("hooks", "mcpServers", "agents", "lspServers"): + assert key not in manifest, f"plugin must stay skill-only; found {key!r}" + assert not (PLUGIN_ROOT / "hooks").exists() + assert not (PLUGIN_ROOT / ".mcp.json").exists() + assert not (PLUGIN_ROOT / "agents").exists() + + +def test_plugin_skill_is_byte_identical_to_canonical() -> None: + plugin_skill = PLUGIN_ROOT / "skills" / "agents-shipgate" + canonical_files = sorted( + p.relative_to(CANONICAL_SKILL) for p in CANONICAL_SKILL.rglob("*") if p.is_file() + ) + plugin_files = sorted( + p.relative_to(plugin_skill) for p in plugin_skill.rglob("*") if p.is_file() + ) + assert canonical_files == plugin_files, ( + "plugin skill file set diverged from skills/agents-shipgate/ — " + "copy the canonical tree wholesale." + ) + for rel in canonical_files: + assert (plugin_skill / rel).read_bytes() == (CANONICAL_SKILL / rel).read_bytes(), ( + f"plugins/claude-code/skills/agents-shipgate/{rel} diverged from " + f"the canonical skills/agents-shipgate/{rel}; sync the copies in " + "the same commit." + ) + + +def test_plugin_command_is_byte_identical_to_canonical() -> None: + plugin_command = PLUGIN_ROOT / "commands" / "shipgate.md" + assert plugin_command.read_bytes() == CANONICAL_COMMAND.read_bytes(), ( + "plugins/claude-code/commands/shipgate.md diverged from " + ".claude/commands/shipgate.md (the renderer-pinned canonical source)." + ) + + +def test_skill_and_command_frontmatter_parse_with_metadata() -> None: + skill_fm = _frontmatter(PLUGIN_ROOT / "skills" / "agents-shipgate" / "SKILL.md") + assert skill_fm.get("name") == "agents-shipgate" + assert isinstance(skill_fm.get("description"), str) and skill_fm["description"], ( + "skill description is the auto-trigger surface; it must survive " + "YAML parsing." + ) + command_fm = _frontmatter(PLUGIN_ROOT / "commands" / "shipgate.md") + assert isinstance(command_fm.get("description"), str) and command_fm["description"]