diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c88fe5f..e149e69 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,6 +6,8 @@ - [ ] The repo URL is public and reachable. - [ ] The entry has a specific category. +- [ ] Required fields, category slug, artifact type, maturity, and labels match the [catalog schema reference](../docs/catalog-schema.md). +- [ ] Source evidence is captured in the PR body using the worksheet in the [catalog schema reference](../docs/catalog-schema.md#evidence-capture-worksheet). - [ ] The `risk_notes` field explains what could go wrong. - [ ] The `operator_note` field explains why an infrastructure operator should care. - [ ] Labels match the observed behavior, not marketing claims. @@ -21,11 +23,12 @@ ## Validation ```bash -python scripts/validate_repos_yaml.py -python scripts/sync_readme_counts.py --check -python -m pytest -q -python scripts/run_mock_eval_scenarios.py -python scripts/audit_github_repos.py --workers 12 --fail-on-unreachable +python3 scripts/validate_repos_yaml.py +python3 scripts/sync_readme_counts.py --check +python3 scripts/sync_catalog_json.py --check +python3 -m pytest -q +python3 scripts/run_mock_eval_scenarios.py +python3 scripts/audit_github_repos.py --workers 12 --fail-on-unreachable ``` - [ ] Relevant commands above pass locally, or this PR explains why a command is not applicable. diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d0b2b2..1f51a36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,139 @@ documented here. The format is based on ### Added +- **CI/CD and release automation guidance.** Added schema-reference guidance and + regression coverage for reviewing release agents against pipeline triggers, + artifact publication, protected environments, scoped CI tokens, log/artifact + exposure, approval records, rollback evidence, and public-safe fixture examples. +- **PR validation checklist coverage.** Added regression tests for the pull + request template's catalog schema links and local validation command block, and + updated contributor guidance to require `python3` plus generated-sidecar checks. +- **Policy-as-code and guardrail evaluation guidance.** Added schema-reference + guidance and regression coverage for reviewing policy agents against read-only + recommendations, exception approval, admission-controller changes, enforcement + paths, artifact hashes, audit evidence, remediation risks, and public-safe + synthetic policy examples. +- **Cost, quota, and FinOps impact guidance.** Added schema-reference guidance + and regression coverage for reviewing cost and FinOps agents against billing + lookup, budget/quota/resource mutations, optimization tradeoffs, sensitive cost + exports, approval evidence, rollback, and public-safe synthetic billing + examples. +- **Secrets and identity operations guidance.** Added schema-reference guidance + and regression coverage for reviewing secrets-manager, identity-platform, and + access-review agents against secret reveal/rotation/revocation, principal and + policy changes, redaction, audit evidence, expiration, rollback, and public-safe + synthetic identity examples. +- **MLOps model operation and evaluation guidance.** Added schema-reference + guidance and regression coverage for reviewing MLOps agents against experiment + lookup, model training, registry promotion, endpoint deployment, feature-store + writes, prompt/eval data exposure, rollback, audit evidence, and public-safe + synthetic model-operation examples. +- **Data platform operations and data movement guidance.** Added + schema-reference guidance and regression coverage for reviewing data-platform + agents against metadata lookup, production queries, writes/backfills/deletes, + dataset scope, masking, retention, audit evidence, rollback, and public-safe + synthetic warehouse examples. +- **Compliance evidence and audit export guidance.** Added schema-reference + guidance and regression coverage for reviewing compliance/GRC agents against + control scope, evidence stores, read-only audit exports, remediation actions, + actor identity, retention boundaries, audit trails, and public-safe synthetic + compliance examples. +- **Incident automation and escalation guidance.** Added schema-reference + guidance and regression coverage for scoring incident, on-call, and runbook + agents by paging, alert-silence, escalation, remediation, audit, rollback, and + public-safe incident-evidence boundaries. +- **Approval evidence and change-control guidance.** Added schema-reference + guidance and regression coverage for checking durable approval artifacts, + actor identity, target resource context, change tickets, environment gates, + break-glass or auto-approval risks, and post-change audit/rollback evidence + before assigning approval or evidence labels to write-capable agents. +- **Runtime isolation boundary guidance.** Added schema-reference guidance and + regression coverage for checking where agents and MCP servers execute, + including sandboxes, disposable containers, ephemeral CI runners, host + filesystem mounts, Docker sockets, kubeconfig contexts, browser profiles, + outbound egress, and public-safe fixture evidence. +- **Dependency and supply-chain boundary guidance.** Added schema-reference + guidance and regression coverage for reviewing package, container, CI action, + plugin, generated-code, curl-to-shell, Docker socket, version-pinning, signing, + checksum, and rollback/uninstall risks before raising catalog maturity. +- **Credential lifecycle and revocation guidance.** Added schema-reference + guidance and regression coverage for checking whether agent and MCP credentials + can be safely issued, rotated, revoked, audited, disconnected from webhooks or + integrations, and attributed to scoped bot/service identities instead of broad + human administrator access. +- **Telemetry and retention boundary guidance.** Added schema-reference guidance + and regression coverage for checking prompt/tool-output telemetry, hosted logs, + retention/deletion/export controls, redaction, local logging, data residency, + and whether evidence signals are durable and safe to share. +- **Tool permission and consent boundary guidance.** Added schema-reference + guidance and regression coverage for reviewing default tool permissions, + destructive-tool separation, per-tool allowlists, scoped runner identities, + revocable scopes, enforced approval gates, and most-privileged-tool scoring. +- **Evaluation environment boundary guidance.** Added schema-reference guidance + and regression coverage for using sandbox projects, test tenants, fixture + repositories, read-only workspaces, narrow OAuth scopes, limited egress, and + redacted public-safe evidence before raising production-adjacent maturity. +- **Agent instruction-boundary review guidance.** Added schema-reference guidance + and regression coverage for treating repository content, logs, tickets, + generated plans, and third-party MCP metadata as untrusted data rather than + executable instructions when scoring DevOps agents and MCP servers. +- **Public-safe catalog metadata rules.** Added schema-reference guidance and + regression coverage that keeps tokens, customer data, tenant URLs, private + hostnames, production prompts, and private evidence out of catalog metadata, + README rows, generated reports, and PR notes. +- **Catalog PR review checklist.** Added schema-reference guidance and regression + coverage for reviewing catalog pull requests against coherent scope, + reproducible source evidence, safety-score alignment, README/generated-sidecar + sync, no-secret hygiene, and recorded validation commands. +- **Hosted MCP credential-boundary guidance.** Added schema-reference guidance and + regression coverage for checking hosted MCP authentication modes, OAuth or token + scopes, remote data-handling signals, read-only endpoints, and no-secret review + practices before cataloging vendor-hosted MCP endpoints. +- **Catalog safety-score evidence rules.** Added schema-reference guidance and + regression coverage for assigning action level, approval, evidence tracing, + and labels from inspected tool-surface evidence instead of broad category + assumptions or marketing language. +- **Catalog external-signal guidance.** Added schema-reference guidance and + regression coverage for using broad MCP indexes, registry mirrors, popularity + dashboards, and third-party evaluations as discovery prompts rather than + acceptance evidence or replacements for first-party source verification. +- **Catalog change decision guide.** Added schema-reference guidance and + regression coverage for choosing whether a catalog PR should refresh, replace, + add, downgrade, or remove a row based on canonical-source and operator-safety + evidence. +- **Catalog deprecation/removal guidance.** Added schema-reference guidance and + regression coverage for refreshing, downgrading, replacing, or removing + archived, deprecated, unreachable, unsafe, or superseded catalog entries + without preserving obsolete rows just to maintain counts. +- **Catalog risk-note writing guide.** Added contributor guidance and regression + coverage for writing `risk_notes` as concrete operator warnings that name + credential boundaries, write/telemetry risks, dry-run-first controls, approval + expectations, and missing evidence. +- **Catalog freshness audit guidance.** The schema reference now tells + contributors when and how to run the GitHub metadata audit, what report files + it produces, how to treat stale or archived warnings, and when manual + non-GitHub reachability checks are still required. +- **Catalog schema reference.** Added a validator-backed reference for required + `data/repos.yaml` fields, allowed category slugs, artifact types, maturity + values, evaluation labels, score-to-label invariants, and pre-submit commands + so contributors can classify entries consistently before CI runs. +- **Catalog entry template.** Added a minimal `data/repos.yaml` entry template + and review checklist to the schema reference so daily catalog additions start + with safe defaults, top-level list placement, credential-scoping notes, and + label-to-score consistency reminders. +- **Catalog source verification checklist.** Added reproducible pre-submit checks + for reachability, freshness, tool surface, credential boundaries, and safety + signals so catalog reviewers can validate entries from public evidence before + trusting external indexes or marketing copy. +- **Catalog evidence capture worksheet.** Added a PR-ready source-evidence note + template with harmless GitHub metadata checks and no-secret reminders so + reviewers can reproduce catalog claims without exposing credentials. +- **Catalog identity rules.** Documented unique `name` and `url` expectations, + canonical source selection, single-row use-case handling, and the narrow case + where separate documentation and runnable artifact rows can coexist. +- **README synchronization rules.** Documented the README surfaces that must stay + aligned with `data/repos.yaml`, including Recently added, catalog section + tables, quick picks, top-picks guidance, and the README count check. - **Operator safety checklist.** Added a practical preflight runbook for evaluating DevOps agents and MCP servers with read-only-first credentials, domain-specific credential boundaries, no-secret-in-context handling, @@ -23,15 +156,24 @@ documented here. The format is based on ### Changed +- **Guarded the operator safety checklist.** Added regression coverage so the + read-only-first, no-secret-in-context, dry-run/proposal, approval, blast-radius, + audit-evidence, and go/no-go guidance stays linked from the main entry points + and remains present during future documentation edits. +- **Clarified catalog provenance classification.** The catalog schema reference now + defines when `official-*` categories are appropriate versus `community-*` + categories, and regression tests keep that contributor guidance present so + ecosystem-adjacent tools are not misclassified as official sources. - **Expanded the agent scorecard safety review.** The reusable scorecard now captures least-privilege credential scope, no-secret-in-context checks, redaction expectations, dry-run/preview commands, approval records, and audit artifacts, plus an explicit production-readiness decision, before recommending production-adjacent use. - **Expanded pull request safety checklist.** The PR template now asks - contributors to confirm no-secret-in-context handling, least-privilege - credential guidance, approval gates, dry-run/preview behavior, audit evidence, - rollback expectations, and telemetry/external API disclosure before review. + contributors to confirm schema-reference alignment, source-evidence capture, + no-secret-in-context handling, least-privilege credential guidance, approval + gates, dry-run/preview behavior, audit evidence, rollback expectations, and + telemetry/external API disclosure before review. - **Hardened catalog schema validation.** Required string fields now reject blank values, and `labels` / `use_cases` must contain at least one non-empty string item so incomplete catalog rows fail locally before reaching README generation diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d407b3f..c6e3f94 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,6 +14,7 @@ Thanks for helping make this a practical operator-grade index instead of a hype - Prefer entries that can be evaluated without real cloud credentials. - For write-capable, credentialed, or production-adjacent entries, use the [operator safety checklist](docs/operator-safety-checklist.md) to confirm domain-specific least-privilege credential boundaries, no-secret-in-context handling, dry-run/proposal behavior, approval gates, blast-radius limits, and audit evidence. - PRs should update [data/repos.yaml](data/repos.yaml) and [README.md](README.md) when the public index changes. +- Use the [catalog schema reference](docs/catalog-schema.md) when choosing category slugs, artifact types, maturity values, and evaluation labels. ## Entry checklist @@ -31,14 +32,15 @@ Thanks for helping make this a practical operator-grade index instead of a hype ## Local validation ```bash -python -m venv .venv +python3 -m venv .venv source .venv/bin/activate python -m pip install -e ".[dev]" -python scripts/validate_repos_yaml.py -python scripts/sync_readme_counts.py -pytest -q +python3 scripts/validate_repos_yaml.py +python3 scripts/sync_readme_counts.py --check +python3 scripts/sync_catalog_json.py --check +python3 -m pytest -q ``` -`sync_readme_counts.py` refreshes the entry/category counts in the README intro from `data/repos.yaml`, so you never edit those numbers by hand. +`sync_readme_counts.py` refreshes the entry/category counts in the README intro from `data/repos.yaml`, and `sync_catalog_json.py --check` verifies the skill-installer catalog sidecar stays aligned, so you never edit generated discovery metadata by hand. -For a deeper freshness check before substantial catalog work, run `python scripts/audit_github_repos.py --stale-days 365`; it writes JSON and Markdown reports under `reports/` and warns on unreachable, archived, private, language-drifted, or stale GitHub repositories. +For a deeper freshness check before substantial catalog work, run `python3 scripts/audit_github_repos.py --stale-days 365`; it writes JSON and Markdown reports under `reports/` and warns on unreachable, archived, private, language-drifted, or stale GitHub repositories. diff --git a/README.md b/README.md index fe6b8f6..a3d453d 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Most agent lists stop at discovery. This one is built for operators: ## Contents - [Evaluation labels](#evaluation-labels) +- [Catalog schema reference](#catalog-schema-reference) - [Compliance evidence checklist](#compliance-evidence-checklist) - [Operator safety checklist](#operator-safety-checklist) - [Top picks by use case](#top-picks-by-use-case) @@ -41,6 +42,10 @@ Most agent lists stop at discovery. This one is built for operators: Labels are shorthand for structured fields recorded on every entry in [data/repos.yaml](data/repos.yaml) — [how entries are scored](docs/scoring.md) explains each field and how it is verified. +## Catalog schema reference + +Catalog entries are validated against a curator-owned schema for required fields, allowed category slugs, artifact types, maturity values, evaluation labels, absolute `https://` URLs, duplicate names/URLs, and score-to-label consistency. See the [catalog schema reference](docs/catalog-schema.md) before adding or reclassifying entries. + ## Compliance evidence checklist Production-adjacent agent runs need a reviewable evidence packet, not just a chat transcript. Use the [compliance evidence checklist](docs/compliance-evidence.md) to capture request context, identity and data boundaries, redacted tool calls, approval records, validation output, and follow-ups for MCP servers, skills, incident copilots, Terraform reviewers, and other DevOps agents. diff --git a/docs/catalog-schema.md b/docs/catalog-schema.md new file mode 100644 index 0000000..f1860dd --- /dev/null +++ b/docs/catalog-schema.md @@ -0,0 +1,818 @@ +# Catalog schema reference + +`data/repos.yaml` is the structured source of truth for the public catalog. Each entry is a top-level YAML list item with the required fields below. Keep this file and `scripts/validate_repos_yaml.py` aligned whenever a new category, artifact type, or scoring value is introduced. + +## Required fields + +Every catalog entry must include: + +- `name` — stable display name, usually `owner/repo` for GitHub projects or a concise documentation title for hosted/vendor docs. +- `url` — absolute `https://` URL. Relative links, bare hostnames, and `http://` URLs are rejected. +- `category` — one of the allowed catalog section slugs below. +- `type` — one of the allowed artifact kinds below. +- `framework` — implementation framework or `unknown` when not applicable. +- `primary_language` — main language or content surface, such as `Go`, `Python`, `TypeScript`, `Documentation`, or `unknown`. +- `cloud_provider` — provider scope, such as `aws`, `azure`, `gcp`, `multi-cloud`, `kubernetes`, or `none`. +- `use_cases` — non-empty list of concrete operator use cases. +- `action_level` — one of `read-only`, `proposal`, `write-capable`, or `unknown`. +- `human_approval` — `true`, `false`, or `unknown`. +- `evidence_tracing` — one of `yes`, `partial`, `none`, or `unknown`. Quote `"yes"` in YAML so it stays a string. +- `maturity` — one of the allowed maturity values below. +- `risk_notes` — non-empty blast-radius and credential-risk note. +- `operator_note` — non-empty reason an infrastructure operator should care. +- `labels` — non-empty list using only the README evaluation labels below. + +## Catalog change decision guide + +Use the smallest safe catalog change that keeps operator trust high: + +1. **Refresh** an existing row when the source is still canonical but metadata, + scoring, README wording, or safety notes are stale. +2. **Replace** a row when vendor or upstream docs point to a maintained official + successor for a deprecated, archived, or superseded artifact. +3. **Add** a row only after verifying the source is a distinct operator-facing + artifact with non-duplicative use cases and enough public evidence to score it. +4. **Downgrade** maturity, action level, or labels when freshness, approval, + tracing, credential, or blast-radius signals are weaker than the current row + claims. +5. **Remove** a row when it is unreachable, unsafe, obsolete, duplicated, or lacks + clear operator value after checking for a current official successor. + +Document the chosen path in the pull request body with the evidence worksheet +below, and keep `risk_notes`, labels, and README discovery surfaces aligned with +that decision. + +## Catalog PR review checklist + +Before merging a catalog change, reviewers should be able to trace every changed +row from public source evidence to README presentation and safety scoring: + +- Scope: the pull request changes one coherent catalog, schema, safety, or docs + concern rather than bundling unrelated vendor additions and tooling changes. +- Source evidence: each added, refreshed, replaced, downgraded, or removed row + includes a reproducible evidence worksheet with canonical source, reachability, + freshness, tool surface, credential boundary, and safety-signal notes. +- Safety scoring: `action_level`, `human_approval`, `evidence_tracing`, + `maturity`, `risk_notes`, and labels match the most privileged documented tool + capability, not the safest hoped-for use case. +- Discovery sync: README Recently added, catalog section tables, quick picks, + top picks, generated sidecars, and entry counts are updated when + `data/repos.yaml` changes. +- Secrets hygiene: PR text, screenshots, catalog metadata, and generated reports + contain no tokens, tenant URLs, private hostnames, customer data, or copied + production prompts. +- Verification: the PR records the local commands run, including schema + validation, tests, README count checks, sidecar sync when available, and + whitespace checks. + +## Identity and duplicate rules + +Catalog identity is intentionally strict so generated README tables, audits, and +review discussions can point to a single stable row: + +- `name` must be unique across the whole flat list. Prefer `owner/repo` for + GitHub-backed projects so the display name stays stable even if the README + section changes. +- `url` must be unique across the whole flat list and must be the canonical + operator-facing source for the artifact being cataloged. +- Do not add a second row for the same artifact only to represent another use + case or README section. Capture additional operator use cases in `use_cases` + and choose the best single `category`. +- Separate documentation and runnable surfaces may have distinct rows only when + they are genuinely different artifacts, such as vendor docs plus a separate + installable `mcp-server` repository. + +## README synchronization rules + +When `data/repos.yaml` changes, update every README surface that helps operators +discover or verify the row. At minimum, run `python3 scripts/sync_readme_counts.py` +so the intro count stays aligned with the YAML source of truth. For new catalog +entries, also update these README sections in the same pull request: + +- `## Recently added` — prepend a dated row for the new or refreshed entry. +- The matching catalog section table for the entry's `category`. +- The intro quick-pick table when the category already has a representative row. +- `## Top picks by use case` only when the entry is genuinely a top recommendation + for that operator workflow; do not add weak entries just to fill the table. + +Run `python3 scripts/sync_readme_counts.py --check` before review so README count +drift fails locally rather than in CI. + +## Allowed categories + +Category slugs define the curated README sections and are validated in CI. The prefix is also a provenance signal: + +- `official-*` categories are for first-party vendor, CNCF/Kubernetes SIG, foundation, or upstream project-governed sources where maintainership is clear from the repository owner or official documentation. +- `community-*` categories are for useful third-party tools, discovery lists, or skill collections that are not governed by the vendor/project whose platform they operate. + +Do not classify a repo as official just because it integrates with an official API, appears in a third-party list, or uses a vendor name. When provenance is unclear, prefer a community category or hold the entry until a first-party source confirms ownership. + +- `community-agent-skills` +- `community-discovery` +- `community-mcp-servers` +- `official-agent-frameworks` +- `official-agent-security-tools` +- `official-agent-skills` +- `official-browser-automation-mcp-servers` +- `official-ci-cd-mcp-servers` +- `official-cloud-agent-toolkits` +- `official-cloud-mcp-servers` +- `official-cloud-security-mcp-servers` +- `official-cloudops-agent-samples` +- `official-data-platform-mcp-servers` +- `official-devops-mcp-platforms` +- `official-devops-mcp-servers` +- `official-diagramming-mcp-tools` +- `official-finops-mcp-servers` +- `official-gitops-mcp-servers` +- `official-iac-mcp-servers` +- `official-mcp-reference-implementations` +- `official-mcp-registry` +- `official-mcp-sdks` +- `official-platform-agent-toolkits` +- `official-security-mcp-servers` +- `official-sre-mcp-servers` + +Add a new category only when the existing taxonomy cannot describe the entry clearly, and update all of these together: + +1. `scripts/validate_repos_yaml.py` +2. `tests/test_repos_yaml.py` +3. `README.md` catalog section and quick-pick references when applicable +4. This schema reference + +## Allowed artifact types + +Use the narrowest type that describes the actual operator surface: + +- `agent-framework` +- `agent-plugin` +- `agent-security-scanner` +- `agent-template` +- `agent-toolkit` +- `curated-list` +- `documentation` +- `hosted-mcp-server` +- `mcp-operator` +- `mcp-plugin` +- `mcp-registry` +- `mcp-server` +- `mcp-server-catalog` +- `mcp-server-collection` +- `mcp-server-plugin` +- `reference-architecture` +- `reference-implementations` +- `registry` +- `sdk` +- `security-guidance` +- `security-tool` +- `skill` +- `skill-library` + +Docs pages and runnable servers are distinct artifacts. A vendor may have both a `documentation` entry and a separate `mcp-server` or `hosted-mcp-server` entry when both are useful to operators. + +## Minimal entry template + +Start new entries from this shape, then replace every placeholder with verified +facts from the official repository or documentation. `data/repos.yaml` is a flat +list, so add this as a new top-level list item rather than nesting it under a +category key. + +```yaml +- name: owner/repo-or-doc-name + url: https://example.com/official-source + category: official-devops-mcp-servers + type: mcp-server + framework: unknown + primary_language: unknown + cloud_provider: none + use_cases: + - Short operator task this tool supports + action_level: read-only + human_approval: unknown + evidence_tracing: unknown + maturity: production-adjacent + risk_notes: Verify credential scope, write tools, telemetry, and audit behavior before production-adjacent use. + operator_note: Explain why a DevOps, SRE, platform, cloud, security, or MLOps operator should evaluate it. + labels: + - mcp +``` + +Template review checklist: + +- Replace placeholder `unknown` values whenever official docs expose a more + specific framework, language, credential, approval, or tracing signal. +- Use `action_level: proposal` for dry-run or plan-generating tools, and + `action_level: write-capable` plus the `write` label when any tool can mutate + infrastructure, code, tickets, cloud resources, or production data. +- Prefer read-only, scoped, or test credentials in `risk_notes`; do not paste + secrets, tokens, customer data, or private endpoints into catalog metadata. +- Add `approval` only with `human_approval: true`, and add `evidence` only when + official docs or code show audit logs, traces, citations, run artifacts, or + similarly durable evidence. + +## Source verification checklist + +Before adding or refreshing a catalog row, collect harmless public evidence for +the operator-facing surface rather than relying on marketing copy or a broad MCP +index. A reviewer should be able to reproduce these checks without secrets: + +- Reachability: confirm the repository or documentation URL returns successfully + and is the canonical upstream, vendor, foundation, or community project page. +- Freshness: for GitHub projects, check that the repository is not archived and + has recent enough activity for the maturity claim; otherwise explain the stale + or archival signal in `risk_notes`. +- Tool surface: verify whether the artifact is a runnable `mcp-server`, a hosted + endpoint, an SDK, documentation, a skill, or only a curated list, then set + `type` to the narrowest matching value. +- Credential boundary: identify the least-privilege credential mode an operator + can use for evaluation, or state clearly when the project only documents broad + credentials or leaves credential scope unspecified. +- Safety signals: map observed dry-run behavior, approval gates, evidence or + audit artifacts, telemetry, and write capability back to `action_level`, + `human_approval`, `evidence_tracing`, `risk_notes`, and `labels`. + +### Evaluation environment boundary guidance + +Catalog review should start in a disposable environment that cannot mutate or +expose production systems. Before treating an agent, MCP server, or toolkit as +production-adjacent, verify and document the evaluation boundary: + +- Use sandbox cloud projects, test tenants, fixture repositories, disposable + issue trackers, demo clusters, or read-only workspaces before connecting real + infrastructure. +- Prefer read-only credentials and narrow OAuth scopes first; move to scoped + write credentials only when a documented dry-run, preview, or approval flow has + already been inspected. +- Keep network egress, webhook targets, CI/CD triggers, and external API access + limited to test systems while validating tool behavior. +- Confirm logs, traces, exported reports, and screenshots are redacted and public + safe before referencing them in `risk_notes`, README rows, or pull request + evidence. +- If a tool cannot be evaluated safely without production credentials or customer + data, do not raise its maturity score; record the evaluation blocker in + `risk_notes` instead. + +### Runtime isolation boundary guidance + +Catalog reviewers should also verify where the agent or MCP server executes. A +safe credential story is not enough when the runtime can see the operator's host, +cluster, browser session, or CI runner filesystem: + +- Prefer local sandboxes, disposable containers, ephemeral CI runners, dev + clusters, or test workstations before running an agent on laptops or shared + runners that contain production credentials. +- Check whether the tool needs host filesystem mounts, shell access, Docker + socket access, kubeconfig contexts, browser profiles, SSH agents, package + managers, or cloud CLIs that can inherit ambient credentials. +- Limit outbound network egress and webhook callbacks to known test endpoints + while evaluating tools that can execute commands, fetch plugins, open browser + sessions, or call external APIs. +- Treat missing sandboxing, namespace, filesystem, process, or egress controls as + maturity blockers for write-capable artifacts; name the runtime boundary in + `risk_notes` instead of assuming the host environment is safe. +- Capture evidence with synthetic fixture repositories, demo clusters, and public + test data only; do not publish local paths, usernames, private runner labels, + kubeconfig context names, or internal hostnames. + +### Hosted MCP credential boundary guidance + +Hosted MCP servers deserve extra credential scrutiny because the runnable surface +is remote and may proxy operator data through a vendor-controlled endpoint. Before +cataloging a `hosted-mcp-server`, verify and document: + +- Authentication mode: OAuth, API key, bearer token, SSO, unauthenticated public + endpoint, or unknown. Prefer OAuth or scoped tokens over long-lived broad keys. +- Scope boundary: read-only endpoint, documented OAuth scopes, project or tenant + scoping, sandbox workspace, or unknown scope. If only broad account access is + documented, say so in `risk_notes`. +- Data handling: whether prompts, tool arguments, logs, traces, or retrieved + records leave the operator's environment, and whether vendor docs describe + retention, audit logs, or telemetry controls. +- Evaluation guardrail: start with disposable workspaces, test tenants, or + read-only scopes, and never paste tokens, customer data, tenant URLs, or private + hostnames into `data/repos.yaml`, README rows, screenshots, or PR comments. + +For hosted endpoints that expose both read-only and write-capable tool groups, +score the row by the most privileged documented capability unless the catalog row +explicitly points to a separate read-only endpoint or mode. + +### Tool permission and consent boundary guidance + +Agent and MCP catalog rows should reflect the permissions an operator must grant +before the artifact can act, not just the friendly demo path. During review, +inspect the documented tool list, manifest, CLI flags, or host configuration and +record how consent is enforced: + +- Identify the default permission posture: disabled until explicitly enabled, + read-only by default, allowlisted per tool, or broadly enabled once connected. +- Check whether destructive tools such as deploy, delete, merge, rotate, trigger, + remediate, or shell execution can be separated from lookup and planning tools. +- Prefer entries that support per-tool allowlists, scoped runner identities, + workspace or namespace boundaries, and revocable OAuth scopes or tokens. +- Treat "user can review output" as weaker than an enforced approval gate; only + set `human_approval: true` when the tool or host blocks mutation until explicit + operator consent is recorded. +- Note missing permission separation, broad host access, or undocumented consent + behavior in `risk_notes`, and score by the most privileged tool exposed by the + configured artifact. + +### Approval evidence and change-control guidance + +For write-capable agents and MCP servers, `human_approval: true` should mean an +operator can verify who approved which action, against which target, and with what +rollback or change-control context. Before giving approval credit, inspect whether +the artifact or host workflow documents durable approval evidence: + +- Identify the approval surface: pull request review, chat approval, ticket state, + deployment environment gate, policy decision, signed plan, or explicit CLI/UI + confirmation. +- Check that approval records include actor identity, timestamp, requested action, + target environment or resource, diff/plan or command summary, and outcome. +- Prefer workflows that bind approvals to scoped test repositories, sandbox + projects, change tickets, or deployment environments instead of broad one-time + consent for future writes. +- Treat break-glass or auto-approval modes as production risks; mention required + compensating controls such as post-change audit review, rollback plan, or + limited-duration access in `risk_notes`. +- Do not mark `approval` or `evidence` labels solely because a demo asks the user + to press enter; require a durable review artifact, audit log, ticket, or run + record that can be inspected after the action. + +### Incident automation and escalation guidance + +Incident, on-call, and runbook agents can page people, silence alerts, modify +incident state, run remediation commands, or publish stakeholder updates. Score +these rows by the operational blast radius of their most privileged documented +workflow: + +- Separate notification, summarization, and read-only timeline tools from actions + that acknowledge, silence, escalate, reassign, resolve, or trigger remediation. +- Prefer dry-run runbooks, proposed remediation steps, test services, demo alerts, + and sandbox incident rooms before granting access to live on-call rotations or + production paging policies. +- Check whether escalation changes and alert silences capture actor identity, + incident ID, service or environment, duration, reason, rollback path, and audit + evidence. +- Treat auto-remediation, bulk alert suppression, broad paging-policy access, or + undocumented escalation controls as write-capable production risks and call them + out in `risk_notes`. +- Keep incident examples public-safe: use synthetic incidents, fixture alerts, + redacted timelines, and placeholders such as `` instead of real + outage details, customer impact, phone numbers, or private escalation targets. + +### Compliance evidence and audit export guidance + +Compliance, GRC, and audit-assistance agents can collect policy evidence, inspect +control status, open remediation tickets, or export reports that may expose +regulated environment details. Score these rows by the evidence and control data +they can access or change: + +- Verify the artifact documents which compliance frameworks, controls, evidence + stores, tickets, cloud accounts, repositories, or identity systems it can read + before treating it as production-adjacent. +- Separate evidence collection and gap summaries from actions that change control + status, approve exceptions, create attestations, assign findings, or trigger + remediation workflows. +- Prefer sandbox frameworks, fixture evidence folders, demo cloud accounts, + read-only audit exports, and redacted reports before connecting real compliance + workspaces or customer environments. +- Check whether exported evidence includes actor identity, source system, control + ID, resource scope, timestamp, hash or version, retention boundary, and audit + trail so `evidence_tracing` claims are reproducible. +- Keep compliance examples public-safe: use synthetic controls, placeholder + auditor names, ``, ``, and redacted finding + IDs instead of customer evidence, private policies, legal opinions, or regulated + data. + +### Policy-as-code and guardrail evaluation guidance + +Policy-as-code agents can inspect, generate, waive, or enforce controls across +repositories, CI/CD pipelines, infrastructure plans, Kubernetes admission, +identity policy, and cloud organizations. Score these rows by the strongest policy +decision or enforcement path they can influence: + +- Separate read-only policy lookup, drift summaries, and recommendation generation + from actions that approve exceptions, suppress findings, change deny rules, + update admission controllers, merge policy bundles, or trigger enforcement. +- Prefer fixture repositories, sample Terraform plans, demo clusters, sandbox + cloud accounts, and read-only policy engines before connecting production CI, + admission webhooks, organization policies, or compliance systems. +- Check whether policy decisions capture actor identity, resource scope, rule or + control ID, input artifact hash, exception reason, expiration, and audit trail + before using `approval` or `evidence` labels. +- Treat automatic remediation, broad exception creation, silent policy downgrade, + or undocumented enforcement order as write-capable risks and explain them in + `risk_notes`. +- Keep examples public-safe: use ``, ``, + ``, and synthetic violation reports instead of customer policy + bundles, production resource IDs, private compliance mappings, or real drift + exports. + +### Data platform operations and data movement guidance + +Data platform agents and MCP servers may inspect schemas, query production data, +move records between systems, update pipelines, or expose warehouse metadata. +Score these rows by the most sensitive dataset or mutation path they can reach: + +- Separate metadata discovery, lineage lookup, and read-only query helpers from + tools that run writes, backfills, deletes, schema migrations, access grants, or + pipeline triggers. +- Prefer sample databases, masked datasets, test warehouses, read-only roles, and + narrow project or dataset scopes before granting access to production + warehouses, queues, buckets, or streaming topics. +- Check whether query logs, exported rows, embeddings, lineage graphs, and cached + results can contain PII, secrets, customer data, or regulated records, then + document masking, retention, and deletion boundaries in `risk_notes`. +- Verify whether data mutations capture actor identity, query or job ID, dataset + and table scope, row-count or partition impact, timestamp, rollback plan, and + audit evidence before using `approval` or `evidence` labels. +- Keep data examples public-safe: use fixture schemas, synthetic rows, + ``, ``, and redacted query plans instead of real + table names, customer identifiers, exported records, or private lineage graphs. + +### Cost, quota, and FinOps impact guidance + +Cost and FinOps agents may estimate spend, inspect invoices, change budgets, +trigger optimization actions, or resize infrastructure. Score these rows by the +highest-cost action or financial data surface they can reach: + +- Separate read-only billing lookup, forecast summaries, and recommendation + generation from actions that change budgets, reservations, quotas, instance + sizes, autoscaling policy, storage class, retention, or resource lifecycle. +- Prefer sandbox billing exports, read-only cost explorer roles, limited accounts, + fixture usage data, and explicit budget thresholds before connecting production + cloud organizations, chargeback systems, or procurement workflows. +- Check whether cost reports, invoices, tags, account names, project IDs, and usage + exports can reveal customer, revenue, contract, or business-sensitive data; + document masking, retention, and access boundaries in `risk_notes`. +- Verify whether optimization actions capture actor identity, account or project + scope, resource IDs, estimated savings, performance or availability tradeoffs, + rollback plan, timestamp, and audit evidence before using `approval` or + `evidence` labels. +- Keep examples public-safe: use ``, ``, + ``, synthetic account names, and redacted cost figures + instead of real invoices, account IDs, customer names, or contract terms. + +### MLOps model operation and evaluation guidance + +MLOps agents and MCP servers may inspect experiments, fine-tune models, deploy +endpoints, change feature pipelines, or export evaluation artifacts. Score these +rows by the most privileged model, data, or deployment workflow they document: + +- Separate read-only experiment lookup, model-card summarization, and evaluation + report generation from actions that start training jobs, register models, + approve promotions, deploy endpoints, update feature stores, or trigger batch + inference. +- Prefer toy datasets, masked evaluation sets, sandbox model registries, test + inference endpoints, and read-only tracking credentials before connecting + production training clusters, registries, feature stores, or customer prompts. +- Check whether prompts, labels, embeddings, model outputs, checkpoints, + fine-tuning datasets, and eval traces can expose PII, secrets, licensed data, or + unreleased model behavior; document retention, deletion, export, and redaction + boundaries in `risk_notes`. +- Verify whether model promotions, endpoint deployments, feature-pipeline writes, + and training runs capture actor identity, model or run ID, dataset version, + evaluation threshold, rollback path, timestamp, and audit evidence before using + `approval` or `evidence` labels. +- Keep MLOps examples public-safe: use ``, ``, + ``, synthetic prompts, and redacted scorecards instead of real + customer data, proprietary checkpoints, private model names, or production eval + logs. + +### Secrets and identity operations guidance + +Secrets managers, identity platforms, and access-review agents can expose or +change some of the highest-impact credentials in an operator environment. Score +these rows by the most privileged secret, principal, or policy workflow they can +reach: + +- Separate read-only lookup, inventory, posture summaries, and access-review + recommendations from actions that create, rotate, revoke, delete, reveal, or + grant secrets, keys, tokens, users, groups, roles, policies, and bindings. +- Prefer sandbox vaults, test directories, read-only identity scopes, short-lived + credentials, and fixture principals before connecting production secrets + stores, SSO tenants, cloud IAM, Kubernetes service accounts, or CI/CD secrets. +- Check whether command output, traces, screenshots, reports, and model context + can contain secret values, token metadata, principal names, group membership, + policy documents, or access paths; document redaction, masking, retention, and + audit boundaries in `risk_notes`. +- Verify whether identity or secret mutations capture actor identity, target + principal or secret path, policy diff, reason, ticket/change ID, timestamp, + expiration, rollback or revocation path, and audit evidence before using + `approval` or `evidence` labels. +- Keep examples public-safe: use ``, ``, + ``, redacted policy snippets, and synthetic access-review + findings instead of real usernames, tenant IDs, vault paths, token prefixes, or + production IAM policies. + +### CI/CD and release automation guidance + +CI/CD, deployment, and release-engineering agents can change source control, +trigger pipelines, publish artifacts, deploy environments, or alter release +gates. Score these rows by the highest-impact pipeline or release action they can +perform: + +- Separate read-only build lookup, log summarization, and deployment status tools + from actions that rerun jobs, cancel pipelines, approve environments, merge + pull requests, publish packages, tag releases, or deploy/rollback workloads. +- Prefer fixture repositories, non-production runners, protected test + environments, dry-run workflows, and scoped CI tokens before granting access to + production pipelines, package registries, deployment keys, or release branches. +- Check whether pipeline logs, artifacts, environment variables, dependency + reports, and deployment manifests can expose secrets, customer data, internal + hostnames, or unreleased product details; document masking, retention, and + artifact access boundaries in `risk_notes`. +- Verify whether release actions capture actor identity, repository or environment + scope, commit SHA, workflow run ID, artifact digest, approval record, rollback + path, timestamp, and audit evidence before using `approval` or `evidence` + labels. +- Keep examples public-safe: use ``, ``, + ``, synthetic pipeline logs, and redacted artifact names + instead of production release IDs, private package names, runner labels, or + deployment URLs. + +### Credential lifecycle and revocation guidance + +Catalog reviewers should check not only which credentials an artifact needs, but +also how safely those credentials can be issued, rotated, revoked, and audited +after evaluation. Before raising maturity for an agent, MCP server, or toolkit, +look for lifecycle controls that keep abandoned experiments from becoming +long-lived production access paths: + +- Prefer OAuth apps, short-lived tokens, workload identity, or scoped service + accounts over shared personal access tokens and static cloud keys. +- Verify that setup docs describe how to rotate or revoke credentials, remove + installed apps, disable webhooks, and disconnect CI/CD or chat integrations. +- Check whether credential use is attributable to a bot, service account, + workspace, project, tenant, repository, or environment rather than a broad + human administrator identity. +- Treat missing revocation, rotation, expiration, or audit guidance as a safety + gap in `risk_notes`, especially for write-capable tools and hosted MCP servers. +- Do not catalog example credential values, tenant-specific callback URLs, app + client secrets, or private installation IDs; describe the public lifecycle + control instead. + +### Telemetry and retention boundary guidance + +Agents and MCP servers often observe prompts, tool arguments, command output, +repository contents, cloud inventory, tickets, traces, and incident notes. Before +raising maturity or evidence scores, verify where that data can flow: + +- Identify whether telemetry, analytics, hosted logs, traces, prompt capture, + crash reports, or model-evaluation uploads are enabled by default, opt-in, + opt-out, self-hosted, or undocumented. +- Check documented retention, deletion, export, tenant-isolation, and regional + processing controls before treating vendor-hosted evidence as production safe. +- Prefer projects that let operators disable telemetry, redact sensitive fields, + keep logs local, or route audit evidence to operator-controlled storage. +- Treat command output, infrastructure inventories, stack traces, and ticket text + as potentially sensitive even when credentials are redacted. +- Record unknown retention, broad vendor-side logging, missing redaction controls, + or unclear data residency in `risk_notes`; do not use `evidence` labels for + telemetry-only signals unless the evidence is durable, reviewable, and safe to + share. + +### Dependency and supply-chain boundary guidance + +Agent toolkits and MCP servers often ask operators to install packages, container +images, browser extensions, CI actions, plugins, or generated code before a tool +can run. Review the install path as part of safety scoring, not as a separate +afterthought: + +- Prefer artifacts that are published from official package registries, + first-party container registries, signed releases, or reproducible source builds + with clear version tags. +- Check whether setup requires curl-to-shell installers, privileged Docker socket + access, host filesystem mounts, broad CI secrets, or dynamically downloaded + plugins, and call out those risks in `risk_notes`. +- Pin versions for evaluation where possible, record the inspected release or + commit in PR evidence, and avoid cataloging instructions that require running + unpinned install scripts against production workstations or runners. +- Treat generated code, third-party plugins, and transitive tool downloads as + untrusted until reviewed; evaluate them in disposable environments before + granting repository, cloud, cluster, or identity permissions. +- Do not raise maturity solely because a project has many stars or package + downloads; use provenance, release freshness, least-privilege install docs, + signing or checksum signals, and safe rollback/uninstall guidance. + +### Public-safe metadata rules + +Catalog metadata, README rows, screenshots, generated reports, and pull request +notes must be safe to publish. Treat every catalog review as public by default: + +- Do not include API tokens, OAuth refresh tokens, bearer tokens, service-account + keys, kubeconfigs, SSH keys, webhook secrets, session cookies, or one-time auth + codes. +- Do not paste customer data, production prompts, tenant-specific URLs, private + hostnames, internal repository paths, account IDs, project IDs, cluster names, + database names, or log snippets that could identify a real environment. +- Use generic placeholders such as ``, ``, + ``, or `` when explaining credential or + environment boundaries. +- If source evidence requires a private console, API response, or screenshot, + summarize the public documentation claim instead and keep private evidence out + of the catalog PR. +- When a tool's safety posture depends on secret scanning or redaction features, + mention the documented control in `risk_notes` without copying example secrets + or live configuration values. + +### Agent instruction-boundary review + +DevOps agents and MCP servers can surface untrusted text from repositories, +issues, run logs, cloud resources, tickets, dashboards, database rows, and web +pages. Catalog reviewers should check whether a tool documents prompt-injection or +tool-output trust-boundary controls before marking it production-adjacent: + +- Treat remote content, README snippets, generated plans, command output, logs, + incident notes, and third-party MCP registry metadata as data, not executable + instructions for the reviewing agent. +- Prefer tools that separate system/developer instructions from retrieved context, + quote or cite external evidence, and avoid automatically following instructions + embedded in tool results, comments, or logs. +- For write-capable agents, require dry-run/proposal mode and explicit human + approval before applying changes derived from untrusted content. +- Note missing instruction-boundary, sandboxing, allowlist, or confirmation + behavior in `risk_notes` rather than assuming the host agent will contain the + risk. +- Keep any prompt-injection examples synthetic and public-safe; do not paste real + production prompts, incident transcripts, customer tickets, or sensitive logs + into catalog metadata or PR evidence. + +### Safety-score evidence rules + +Safety scores must be backed by the tool surface you inspected, not by broad +category assumptions or marketing language. Use conservative values when the +source is unclear: + +- Set `action_level: read-only` only when the documented tools expose lookup, + search, describe, or export behavior without mutation; set `proposal` when the + artifact produces plans, diffs, previews, or recommendations that require a + separate apply step. +- Set `action_level: write-capable` whenever any documented tool can create, + update, delete, trigger, deploy, merge, rotate, acknowledge, remediate, or run + commands against source control, cloud resources, clusters, CI/CD, identity, + secrets, databases, observability, or incident systems. +- Set `human_approval: true` only when official docs, code, or examples show an + explicit approval gate before the write-capable operation. A general statement + that operators should review output is not enough. +- Set `evidence_tracing: "yes"` or `partial` only when the source documents + durable traces such as audit logs, citations, run artifacts, change records, + request IDs, or exported reports. If evidence is not documented, use `unknown` + or `none` and explain the gap in `risk_notes`. +- Keep labels synchronized with these structured fields: `write` for + write-capable tools, `approval` for explicit approval gates, and `evidence` for + documented tracing or audit artifacts. + +### External index and evaluation signals + +Broad MCP indexes, registry mirrors, popularity dashboards, and third-party +evaluation scorecards are useful discovery inputs, but they are not catalog +acceptance evidence by themselves. Treat external signals as review prompts: + +- Use external indexes to find candidates, duplicates, aliases, or missing + official documentation, then verify every catalog claim against first-party + vendor, foundation, CNCF/Kubernetes SIG, or upstream project sources before + editing `data/repos.yaml`. +- Keep imported rankings, popularity metrics, automated evaluations, and broad + MCP index scores in generated sidecars or review notes rather than copying + them into safety-scored catalog fields. +- Do not let third-party scores override the local rubric for `action_level`, + `human_approval`, `evidence_tracing`, `maturity`, `risk_notes`, or labels. +- When external sources disagree with first-party docs, document the conflict in + the pull request evidence worksheet and prefer the canonical upstream source + for catalog metadata. + +### Risk notes writing guide + +Write `risk_notes` as a short operator warning, not a marketing summary. Good +risk notes answer what could go wrong during evaluation and what guardrail should +be used first: + +- Name the credential boundary: read-only token, scoped test account, sandbox + cloud project, limited Kubernetes namespace, or unknown scope. +- Identify write capability, destructive actions, external API calls, telemetry, + or data exfiltration paths when the tool can reach infrastructure or sensitive + systems. +- Prefer dry-run, proposal, preview, or plan-only workflows before write-capable + execution, and mention required human approval when official docs or code show + an approval gate. +- Call out missing evidence honestly: use `unknown` scoring or note when audit + logs, traces, citations, or run artifacts are not documented. + +Example: + +```yaml +risk_notes: Use a read-only GitHub token for evaluation; write-capable issue and pull-request tools require scoped test repositories, explicit approval, and audit-log review before production use. +``` + +### Automated GitHub freshness audit + +For substantial catalog edits, run the lightweight GitHub metadata audit before +review so stale or unreachable repositories are visible next to the proposed +metadata change: + +```bash +python3 scripts/audit_github_repos.py --stale-days 365 +``` + +The command writes `reports/github-repo-audit.json` and +`reports/github-repo-audit.md` with reachability, archived/private status, +primary-language drift, last push time, and stale-repository warnings for GitHub +URLs in `data/repos.yaml`. Treat the reports as curator evidence, not generated +catalog source: summarize relevant warnings in the pull request body, update +`risk_notes` when an entry is stale or archived, and do not commit the reports +unless a reviewer explicitly asks for a point-in-time audit artifact. + +For non-GitHub documentation and hosted MCP endpoints, record a separate manual +reachability check because the GitHub audit intentionally skips those URLs. + +### Deprecation and removal handling + +When a cataloged project becomes archived, deprecated, unreachable, or materially +less safe than its current score suggests, prefer a traceable refresh over a +silent delete: + +- Replace the row with the current official successor when vendor or upstream + documentation points to a maintained repository, hosted endpoint, or docs page. +- Keep a stale row only when it still has operator value, lower the maturity or + action score as needed, and explain the archived, deprecated, or unsupported + status in `risk_notes`. +- Remove a row when the source is unreachable, unsafe, abandoned without clear + operator value, or superseded by a better canonical entry; mention the removal + in `CHANGELOG.md` unless it is part of routine duplicate cleanup. +- Never preserve an obsolete entry just to maintain README counts. Run + `python3 scripts/sync_readme_counts.py` after removals or replacements. + +### Evidence capture worksheet + +Paste a short evidence note into the pull request body when catalog metadata +changes. Keep it factual, reproducible, and free of secrets: + +```markdown +Source evidence: +- Canonical source: +- Reachability check: +- Freshness signal: +- Tool surface: +- Credential boundary: +- Safety signals: +``` + +Suggested harmless commands for GitHub-backed entries: + +```bash +gh repo view OWNER/REPO --json nameWithOwner,isArchived,pushedAt,defaultBranchRef,licenseInfo,url,repositoryTopics +gh api repos/OWNER/REPO/contents/README.md --jq .download_url +``` + +For non-GitHub documentation, record the canonical URL, HTTP status, redirected +URL if any, and content type. Do not include access tokens, private tenant URLs, +internal hostnames, customer data, or screenshots that expose credentials. + +## Maturity values + +- `production-adjacent` — official or mature enough to evaluate near production, but not a production-readiness guarantee. +- `active-oss` — active open-source project with useful operator value. +- `prototype` — useful but early, experimental, or lower-confidence. +- `curated-list` — index or registry rather than a runnable tool. +- `skill-library` — installable agent-skill collection. +- `unknown` — insufficient evidence; prefer avoiding this for new entries unless the operator value is clear. + +## Evaluation labels and consistency rules + +Labels are the README-facing shorthand for structured safety fields: + +- `prod` — production-adjacent maturity. +- `prototype` — prototype maturity. +- `mcp` — MCP/server integration. +- `approval` — `human_approval: true`. +- `evidence` — `evidence_tracing: "yes"` or `partial`. +- `write` — `action_level: write-capable`. + +Validator-enforced invariants: + +- `write` requires `action_level: write-capable`, and write-capable entries must include `write`. +- `approval` requires `human_approval: true`, and human-approval entries must include `approval`. +- `evidence_tracing: "yes"` requires `evidence`; `evidence` cannot be used with `none` or `unknown`. +- `prod` and `prototype` are mutually exclusive. +- `maturity: prototype` requires `prototype` and cannot use `prod`. + +## Pre-submit commands + +Run these before opening a catalog PR: + +```bash +python3 scripts/sync_readme_counts.py +python3 scripts/sync_catalog_json.py +python3 scripts/validate_repos_yaml.py +python3 -m pytest -q +git diff --check +``` + +If `pytest` is unavailable, create a local virtual environment and install dev dependencies first: + +```bash +python3 -m venv .venv +. .venv/bin/activate +python -m pip install -e '.[dev]' +``` diff --git a/docs/safety-model.md b/docs/safety-model.md index 20f7f6a..aaa8b7c 100644 --- a/docs/safety-model.md +++ b/docs/safety-model.md @@ -4,6 +4,8 @@ Infrastructure agents need a stricter safety bar than generic chat assistants. T For a practical preflight before enabling a cataloged MCP server or agent against real infrastructure, use the [operator safety checklist](operator-safety-checklist.md). +For curator-facing schema rules that keep `data/repos.yaml` and README labels aligned, see the [catalog schema reference](catalog-schema.md). + ## Principles ### Read-only first diff --git a/docs/scoring.md b/docs/scoring.md index a4071e8..1ed59ce 100644 --- a/docs/scoring.md +++ b/docs/scoring.md @@ -2,6 +2,8 @@ Most agent lists stop at discovery. Infrastructure teams need more: whether a tool can perform write actions, whether it has approval gates, whether it preserves evidence, and whether it is stable enough to depend on. Every entry in [data/repos.yaml](../data/repos.yaml) records five dimensions, assessed from project documentation and exposed tool surfaces rather than marketing claims. +For the full validator-backed field list, category slugs, artifact types, maturity values, labels, and pre-submit commands, see the [catalog schema reference](catalog-schema.md). + ## The five fields | Field | Question it answers | How it is verified | diff --git a/tests/test_catalog_schema_reference.py b/tests/test_catalog_schema_reference.py new file mode 100644 index 0000000..b2c77ef --- /dev/null +++ b/tests/test_catalog_schema_reference.py @@ -0,0 +1,547 @@ +from pathlib import Path + +from scripts.validate_repos_yaml import ( + ALLOWED_ACTION_LEVELS, + ALLOWED_CATEGORIES, + ALLOWED_EVIDENCE_TRACING, + ALLOWED_HUMAN_APPROVAL, + ALLOWED_LABELS, + ALLOWED_MATURITY, + ALLOWED_TYPES, + REQUIRED_FIELDS, +) + +SCHEMA_DOC = Path("docs/catalog-schema.md") + + +def _schema_text() -> str: + return SCHEMA_DOC.read_text(encoding="utf-8") + + +def test_catalog_schema_reference_lists_required_fields(): + text = _schema_text() + + for field in REQUIRED_FIELDS: + assert f"`{field}`" in text + + +def test_catalog_schema_reference_documents_catalog_change_decision_guide(): + text = _schema_text() + + assert "## Catalog change decision guide" in text + assert "smallest safe catalog change" in text + assert "**Refresh**" in text + assert "**Replace**" in text + assert "**Add**" in text + assert "**Downgrade**" in text + assert "**Remove**" in text + assert "evidence worksheet" in text + + +def test_catalog_schema_reference_includes_catalog_pr_review_checklist(): + text = _schema_text() + + assert "## Catalog PR review checklist" in text + assert "one coherent catalog, schema, safety, or docs" in text + assert "reproducible evidence worksheet" in text + assert "most privileged documented tool" in text + assert "generated sidecars" in text + assert "no tokens, tenant URLs" in text + assert "local commands run" in text + + +def test_catalog_schema_reference_documents_identity_and_duplicate_rules(): + text = _schema_text() + + assert "## Identity and duplicate rules" in text + assert "`name` must be unique" in text + assert "`url` must be unique" in text + assert "canonical" in text + assert "Do not add a second row for the same artifact" in text + assert "Separate documentation and runnable surfaces" in text + + +def test_catalog_schema_reference_documents_readme_synchronization_rules(): + text = _schema_text() + + assert "## README synchronization rules" in text + assert "python3 scripts/sync_readme_counts.py" in text + assert "python3 scripts/sync_readme_counts.py --check" in text + assert "## Recently added" in text + assert "matching catalog section table" in text + assert "intro quick-pick table" in text + assert "## Top picks by use case" in text + + +def test_catalog_schema_reference_lists_allowed_categories(): + text = _schema_text() + + for category in ALLOWED_CATEGORIES: + assert f"`{category}`" in text + + +def test_catalog_schema_reference_documents_category_provenance(): + text = _schema_text() + + assert "`official-*`" in text + assert "first-party vendor" in text + assert "`community-*`" in text + assert "not governed by the vendor/project" in text + + +def test_catalog_schema_reference_includes_minimal_entry_template(): + text = _schema_text() + + assert "## Minimal entry template" in text + assert "add this as a new top-level list item" in text + assert "```yaml" in text + assert "- name: owner/repo-or-doc-name" in text + assert "action_level: read-only" in text + assert "labels:" in text + assert "- mcp" in text + assert "Template review checklist" in text + assert "do not paste" in text + + +def test_catalog_schema_reference_includes_source_verification_checklist(): + text = _schema_text() + + assert "## Source verification checklist" in text + assert "without secrets" in text + assert "Reachability" in text + assert "Freshness" in text + assert "not archived" in text + assert "Tool surface" in text + assert "Credential boundary" in text + assert "Safety signals" in text + + +def test_catalog_schema_reference_includes_safety_score_evidence_rules(): + text = _schema_text() + + assert "### Safety-score evidence rules" in text + assert "category assumptions or marketing language" in text + assert "`action_level: read-only`" in text + assert "`action_level: write-capable`" in text + assert "create," in text + assert "rotate, acknowledge, remediate" in text + assert "`human_approval: true` only when" in text + assert "explicit approval gate" in text + assert "`evidence_tracing: \"yes\"` or `partial`" in text + assert "durable traces" in text + assert "Keep labels synchronized" in text + + +def test_catalog_schema_reference_includes_evaluation_environment_boundary_guidance(): + text = _schema_text() + + assert "### Evaluation environment boundary guidance" in text + assert "disposable environment" in text + assert "sandbox cloud projects" in text + assert "test tenants" in text + assert "fixture repositories" in text + assert "read-only credentials and narrow OAuth scopes" in text + assert "network egress" in text + assert "webhook targets" in text + assert "redacted and public" in text + assert "without production credentials or customer" in text + + +def test_catalog_schema_reference_includes_runtime_isolation_boundary_guidance(): + text = _schema_text() + + assert "### Runtime isolation boundary guidance" in text + assert "where the agent or MCP server executes" in text + assert "local sandboxes" in text + assert "disposable containers" in text + assert "ephemeral CI runners" in text + assert "host filesystem mounts" in text + assert "Docker\n socket access" in text + assert "kubeconfig contexts" in text + assert "outbound network egress" in text + assert "maturity blockers for write-capable artifacts" in text + assert "synthetic fixture repositories" in text + assert "do not publish local paths" in text + + +def test_catalog_schema_reference_includes_hosted_mcp_credential_boundary_guidance(): + text = _schema_text() + + assert "### Hosted MCP credential boundary guidance" in text + assert "`hosted-mcp-server`" in text + assert "Authentication mode" in text + assert "OAuth" in text + assert "Scope boundary" in text + assert "read-only endpoint" in text + assert "Data handling" in text + assert "prompts, tool arguments, logs, traces" in text + assert "disposable workspaces" in text + assert "never paste tokens" in text + assert "most privileged documented capability" in text + + +def test_catalog_schema_reference_includes_tool_permission_and_consent_boundary_guidance(): + text = _schema_text() + + assert "### Tool permission and consent boundary guidance" in text + assert "documented tool list" in text + assert "default permission posture" in text + assert "read-only by default" in text + assert "destructive tools such as deploy" in text + assert "per-tool allowlists" in text + assert "scoped runner identities" in text + assert "enforced approval gate" in text + assert "most privileged tool exposed" in text + + +def test_catalog_schema_reference_includes_approval_evidence_change_control_guidance(): + text = _schema_text() + + assert "### Approval evidence and change-control guidance" in text + assert "who approved which action" in text + assert "pull request review" in text + assert "chat approval" in text + assert "ticket state" in text + assert "deployment environment gate" in text + assert "actor identity" in text + assert "target environment or resource" in text + assert "break-glass or auto-approval modes" in text + assert "durable review artifact" in text + + +def test_catalog_schema_reference_includes_incident_automation_escalation_guidance(): + text = _schema_text() + + assert "### Incident automation and escalation guidance" in text + assert "Incident, on-call, and runbook agents" in text + assert "page people" in text + assert "silence alerts" in text + assert "read-only timeline tools" in text + assert "acknowledge, silence, escalate" in text + assert "dry-run runbooks" in text + assert "sandbox incident rooms" in text + assert "incident ID" in text + assert "rollback path" in text + assert "auto-remediation" in text + assert "" in text + + +def test_catalog_schema_reference_includes_compliance_evidence_audit_export_guidance(): + text = _schema_text() + + assert "### Compliance evidence and audit export guidance" in text + assert "Compliance, GRC, and audit-assistance agents" in text + assert "control status" in text + assert "evidence collection and gap summaries" in text + assert "approve exceptions" in text + assert "fixture evidence folders" in text + assert "read-only audit exports" in text + assert "actor identity" in text + assert "control\n ID" in text + assert "retention boundary" in text + assert "" in text + assert "" in text + + +def test_catalog_schema_reference_includes_data_platform_operations_guidance(): + text = _schema_text() + + assert "### Data platform operations and data movement guidance" in text + assert "Data platform agents and MCP servers" in text + assert "schemas, query production data" in text + assert "metadata discovery, lineage lookup" in text + assert "writes, backfills, deletes" in text + assert "sample databases" in text + assert "masked datasets" in text + assert "read-only roles" in text + assert "query logs, exported rows, embeddings" in text + assert "PII, secrets, customer data" in text + assert "query or job ID" in text + assert "row-count or partition impact" in text + assert "" in text + assert "" in text + + +def test_catalog_schema_reference_includes_cost_quota_finops_guidance(): + text = _schema_text() + + assert "### Cost, quota, and FinOps impact guidance" in text + assert "Cost and FinOps agents" in text + assert "estimate spend" in text + assert "change budgets" in text + assert "read-only billing lookup" in text + assert "reservations, quotas" in text + assert "read-only cost explorer roles" in text + assert "explicit budget thresholds" in text + assert "cost reports, invoices" in text + assert "business-sensitive data" in text + assert "estimated savings" in text + assert "performance or availability tradeoffs" in text + assert "" in text + assert "" in text + assert "" in text + + +def test_catalog_schema_reference_includes_policy_as_code_guidance(): + text = _schema_text() + + assert "### Policy-as-code and guardrail evaluation guidance" in text + assert "Policy-as-code agents" in text + assert "strongest policy" in text + assert "approve exceptions" in text + assert "update admission controllers" in text + assert "fixture repositories" in text + assert "sample Terraform plans" in text + assert "demo clusters" in text + assert "actor identity" in text + assert "rule or\n control ID" in text + assert "input artifact hash" in text + assert "automatic remediation" in text + assert "broad exception creation" in text + assert "" in text + assert "" in text + assert "" in text + + +def test_catalog_schema_reference_includes_mlops_model_operation_guidance(): + text = _schema_text() + + assert "### MLOps model operation and evaluation guidance" in text + assert "MLOps agents and MCP servers" in text + assert "fine-tune models" in text + assert "deploy endpoints" in text + assert "read-only experiment lookup" in text + assert "start training jobs" in text + assert "sandbox model registries" in text + assert "test\n inference endpoints" in text + assert "prompts, labels, embeddings" in text + assert "dataset version" in text + assert "evaluation threshold" in text + assert "" in text + assert "" in text + assert "" in text + + +def test_catalog_schema_reference_includes_secrets_identity_operations_guidance(): + text = _schema_text() + + assert "### Secrets and identity operations guidance" in text + assert "Secrets managers, identity platforms" in text + assert "read-only lookup" in text + assert "create, rotate, revoke, delete" in text + assert "sandbox vaults" in text + assert "test directories" in text + assert "read-only identity scopes" in text + assert "Kubernetes service accounts" in text + assert "secret values, token metadata" in text + assert "policy diff" in text + assert "ticket/change ID" in text + assert "rollback or revocation path" in text + assert "" in text + assert "" in text + assert "" in text + + +def test_catalog_schema_reference_includes_ci_cd_release_automation_guidance(): + text = _schema_text() + + assert "### CI/CD and release automation guidance" in text + assert "CI/CD, deployment, and release-engineering agents" in text + assert "trigger pipelines" in text + assert "publish artifacts" in text + assert "read-only build lookup" in text + assert "rerun jobs" in text + assert "approve environments" in text + assert "merge\n pull requests" in text + assert "fixture repositories" in text + assert "non-production runners" in text + assert "scoped CI tokens" in text + assert "pipeline logs, artifacts" in text + assert "commit SHA" in text + assert "workflow run ID" in text + assert "artifact digest" in text + assert "" in text + assert "" in text + assert "" in text + + +def test_catalog_schema_reference_includes_credential_lifecycle_guidance(): + text = _schema_text() + + assert "### Credential lifecycle and revocation guidance" in text + assert "issued, rotated, revoked, and audited" in text + assert "OAuth apps, short-lived tokens" in text + assert "scoped service" in text + assert "rotate or revoke credentials" in text + assert "disable webhooks" in text + assert "bot, service account" in text + assert "missing revocation, rotation, expiration" in text + assert "hosted MCP servers" in text + assert "Do not catalog example credential values" in text + + +def test_catalog_schema_reference_includes_telemetry_and_retention_boundary_guidance(): + text = _schema_text() + + assert "### Telemetry and retention boundary guidance" in text + assert "prompts, tool arguments, command output" in text + assert "telemetry, analytics, hosted logs" in text + assert "enabled by default, opt-in" in text + assert "retention, deletion, export" in text + assert "disable telemetry" in text + assert "operator-controlled storage" in text + assert "unknown retention" in text + assert "telemetry-only signals" in text + + +def test_catalog_schema_reference_includes_dependency_supply_chain_boundary_guidance(): + text = _schema_text() + + assert "### Dependency and supply-chain boundary guidance" in text + assert "packages, container" in text + assert "official package registries" in text + assert "first-party container registries" in text + assert "curl-to-shell installers" in text + assert "privileged Docker socket" in text + assert "Pin versions for evaluation" in text + assert "transitive tool downloads as" in text + assert "signing or checksum signals" in text + assert "safe rollback/uninstall guidance" in text + + +def test_catalog_schema_reference_includes_public_safe_metadata_rules(): + text = _schema_text() + + assert "### Public-safe metadata rules" in text + assert "safe to publish" in text + assert "API tokens" in text + assert "service-account" in text + assert "customer data" in text + assert "tenant-specific URLs" in text + assert "" in text + assert "" in text + assert "keep private evidence out" in text + assert "without copying example secrets" in text + + +def test_catalog_schema_reference_includes_agent_instruction_boundary_review(): + text = _schema_text() + + assert "### Agent instruction-boundary review" in text + assert "prompt-injection" in text + assert "tool-output trust-boundary" in text + assert "as data, not executable" in text + assert "system/developer instructions" in text + assert "embedded in tool results" in text + assert "dry-run/proposal mode" in text + assert "instruction-boundary" in text + assert "prompt-injection examples synthetic" in text + + +def test_catalog_schema_reference_includes_external_signal_guidance(): + text = _schema_text() + + assert "### External index and evaluation signals" in text + assert "not catalog" in text + assert "acceptance evidence by themselves" in text + assert "review prompts" in text + assert "first-party" in text + assert "generated sidecars" in text + assert "Do not let third-party scores override the local rubric" in text + assert "pull request evidence worksheet" in text + + +def test_catalog_schema_reference_includes_risk_notes_writing_guide(): + text = _schema_text() + + assert "### Risk notes writing guide" in text + assert "short operator warning" in text + assert "credential boundary" in text + assert "write capability" in text + assert "telemetry" in text + assert "dry-run, proposal, preview, or plan-only" in text + assert "missing evidence" in text + assert "risk_notes: Use a read-only GitHub token" in text + + +def test_catalog_schema_reference_includes_github_freshness_audit_guidance(): + text = _schema_text() + + assert "### Automated GitHub freshness audit" in text + assert "python3 scripts/audit_github_repos.py --stale-days 365" in text + assert "reports/github-repo-audit.json" in text + assert "reports/github-repo-audit.md" in text + assert "reachability, archived/private status" in text + assert "do not commit the reports" in text + assert "non-GitHub documentation and hosted MCP endpoints" in text + + +def test_catalog_schema_reference_includes_deprecation_and_removal_guidance(): + text = _schema_text() + + assert "### Deprecation and removal handling" in text + assert "archived, deprecated, unreachable" in text + assert "current official successor" in text + assert "lower the maturity or" in text + assert "explain the archived, deprecated, or unsupported" in text + assert "Remove a row when the source is unreachable" in text + assert "Never preserve an obsolete entry just to maintain README counts" in text + + +def test_catalog_schema_reference_includes_evidence_capture_worksheet(): + text = _schema_text() + + assert "### Evidence capture worksheet" in text + assert "pull request body" in text + assert "Source evidence:" in text + assert "Canonical source" in text + assert "Reachability check" in text + assert "Credential boundary" in text + assert "gh repo view OWNER/REPO --json" in text + assert "Do not include access tokens" in text + + +def test_catalog_schema_reference_lists_allowed_action_levels(): + text = _schema_text() + + for action_level in ALLOWED_ACTION_LEVELS: + assert f"`{action_level}`" in text + + +def test_catalog_schema_reference_lists_allowed_human_approval_values(): + text = _schema_text() + + expected_values = { + "true" if value is True else "false" if value is False else value + for value in ALLOWED_HUMAN_APPROVAL + } + for human_approval in expected_values: + assert f"`{human_approval}`" in text + + +def test_catalog_schema_reference_lists_allowed_evidence_tracing_values(): + text = _schema_text() + + for evidence_tracing in ALLOWED_EVIDENCE_TRACING: + assert f"`{evidence_tracing}`" in text + + +def test_catalog_schema_reference_lists_allowed_artifact_types(): + text = _schema_text() + + for artifact_type in ALLOWED_TYPES: + assert f"`{artifact_type}`" in text + + +def test_catalog_schema_reference_lists_allowed_maturity_values(): + text = _schema_text() + + for maturity in ALLOWED_MATURITY: + assert f"`{maturity}`" in text + + +def test_catalog_schema_reference_lists_allowed_evaluation_labels(): + text = _schema_text() + + for label in ALLOWED_LABELS: + assert f"`{label}`" in text diff --git a/tests/test_operator_safety_checklist.py b/tests/test_operator_safety_checklist.py new file mode 100644 index 0000000..29a41fb --- /dev/null +++ b/tests/test_operator_safety_checklist.py @@ -0,0 +1,57 @@ +from pathlib import Path + +README = Path("README.md") +CONTRIBUTING = Path("CONTRIBUTING.md") +SAFETY_MODEL = Path("docs/safety-model.md") +OPERATOR_CHECKLIST = Path("docs/operator-safety-checklist.md") +SCORECARD = Path("templates/agent-scorecard.md") + + +def _text(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def test_operator_safety_checklist_covers_core_controls(): + text = _text(OPERATOR_CHECKLIST) + + required_phrases = [ + "Start read-only", + "Keep secrets out of model context", + "Require dry-run or proposal mode before writes", + "Define the human approval gate", + "Limit blast radius", + "Capture evidence and audit logs", + "Production-adjacent go/no-go", + "read-only token", + "namespace-scoped role", + "plan-only workspace token", + "approval record and approver", + ] + for phrase in required_phrases: + assert phrase in text + + +def test_operator_safety_checklist_is_linked_from_entry_points(): + expected_links = [ + "docs/operator-safety-checklist.md", + "operator-safety-checklist.md", + ] + + assert expected_links[0] in _text(README) + assert expected_links[0] in _text(CONTRIBUTING) + assert expected_links[1] in _text(SAFETY_MODEL) + + +def test_agent_scorecard_preserves_safety_preflight_fields(): + text = _text(SCORECARD) + + for phrase in [ + "Credential and context boundary", + "Least-privilege scope to grant", + "Secrets excluded from model context", + "Dry-run or preview command", + "Approval recorded in", + "Audit artifacts", + "Production-readiness decision", + ]: + assert phrase in text diff --git a/tests/test_pr_template.py b/tests/test_pr_template.py new file mode 100644 index 0000000..d99ef21 --- /dev/null +++ b/tests/test_pr_template.py @@ -0,0 +1,32 @@ +from pathlib import Path + +PR_TEMPLATE = Path(".github/PULL_REQUEST_TEMPLATE.md") + + +def _template_text() -> str: + return PR_TEMPLATE.read_text(encoding="utf-8") + + +def test_pr_template_points_catalog_submitters_to_schema_reference(): + text = _template_text() + + assert "catalog schema reference" in text + assert "docs/catalog-schema.md" in text + assert "docs/catalog-schema.md#evidence-capture-worksheet" in text + assert "data/repos.yaml" in text + assert "README.md" in text + + +def test_pr_template_validation_block_includes_core_local_checks(): + text = _template_text() + + expected_commands = ( + "python3 scripts/validate_repos_yaml.py", + "python3 scripts/sync_readme_counts.py --check", + "python3 scripts/sync_catalog_json.py --check", + "python3 -m pytest -q", + "python3 scripts/run_mock_eval_scenarios.py", + "python3 scripts/audit_github_repos.py --workers 12 --fail-on-unreachable", + ) + for command in expected_commands: + assert command in text