From 1fca3a88425f6609b5aef93725a5b1e2ae54e249 Mon Sep 17 00:00:00 2001 From: morpheuslord Date: Fri, 9 Oct 2026 22:27:33 +0530 Subject: [PATCH 01/21] Replace the one-page handbook with a multi-page docs site site/build.py builds the GitHub Pages site from the Modernist design: tabs for guides, CLI, Python API, MCP server, reference and changelog, each with its own sidebar. CLI options come from the click commands, API pages from docstrings, config pages from config.yaml and the CloudGConfig models, and the long docs/*.md references are split per section. New hand-written guides, CLI and API pages carry runnable examples, editor and terminal styled code blocks with linked tabs, and mermaid diagrams. Search, dark mode and redirects from the old index.html#anchor links work as static files. The Pages workflow now builds the site. --- .github/workflows/pages.yml | 62 +- CHANGELOG.md | 8 + docs/index.html | 887 -------------------- site/AUTHORING.md | 190 +++++ site/build.py | 940 ++++++++++++++++++++++ site/content/api/cloudgconfig.md | 222 +++++ site/content/api/cloudgengine.md | 266 ++++++ site/content/api/cloudontology.md | 269 +++++++ site/content/api/dependencygraph.md | 204 +++++ site/content/api/entry-points.md | 303 +++++++ site/content/api/event-hooks.md | 257 ++++++ site/content/api/findingsnormaliser.md | 179 ++++ site/content/api/graphbuilder.md | 218 +++++ site/content/api/inventorymapper.md | 145 ++++ site/content/api/inventoryresult.md | 150 ++++ site/content/api/mcp-layer.md | 263 ++++++ site/content/api/models.md | 223 +++++ site/content/api/parse-report.md | 144 ++++ site/content/api/ragexporter.md | 249 ++++++ site/content/api/reachabilityanalyzer.md | 255 ++++++ site/content/api/relationshiplinker.md | 204 +++++ site/content/api/resilience.md | 192 +++++ site/content/api/results.md | 203 +++++ site/content/cli/auth-flags.md | 177 ++++ site/content/cli/collect.md | 93 +++ site/content/cli/deps.md | 147 ++++ site/content/cli/ingest.md | 150 ++++ site/content/cli/map.md | 196 +++++ site/content/cli/mcp-call.md | 149 ++++ site/content/cli/mcp-config.md | 142 ++++ site/content/cli/mcp-prompts.md | 118 +++ site/content/cli/mcp-read.md | 109 +++ site/content/cli/mcp-resources.md | 100 +++ site/content/cli/mcp-serve.md | 182 +++++ site/content/cli/mcp-tools.md | 108 +++ site/content/cli/output-flags.md | 141 ++++ site/content/cli/provider-flags.md | 137 ++++ site/content/cli/report.md | 126 +++ site/content/cli/run.md | 190 +++++ site/content/cli/scan.md | 102 +++ site/content/guides/authentication.md | 329 ++++++++ site/content/guides/aws-organizations.md | 394 +++++++++ site/content/guides/ci-automation.md | 400 +++++++++ site/content/guides/compliance.md | 328 ++++++++ site/content/guides/dependencies.md | 321 ++++++++ site/content/guides/docker.md | 359 +++++++++ site/content/guides/graph-ontology-rag.md | 455 +++++++++++ site/content/guides/ingesting.md | 427 ++++++++++ site/content/guides/installation.md | 245 ++++++ site/content/guides/introduction.md | 128 +++ site/content/guides/inventory-mapping.md | 343 ++++++++ site/content/guides/pipeline.md | 356 ++++++++ site/content/guides/python-recipes.md | 565 +++++++++++++ site/content/guides/quickstart.md | 161 ++++ site/content/guides/rate-limits.md | 299 +++++++ site/content/guides/reports.md | 306 +++++++ site/content/guides/running-scanners.md | 299 +++++++ site/content/mcp/connect.md | 329 ++++++++ site/content/mcp/index.md | 264 ++++++ site/content/reference/output-files.md | 280 +++++++ site/data/config-notes.yaml | 60 ++ site/data/nav.yaml | 195 +++++ site/data/stats.yaml | 19 + site/docsgen/__init__.py | 1 + site/docsgen/gen_api.py | 319 ++++++++ site/docsgen/gen_changelog.py | 51 ++ site/docsgen/gen_cli.py | 176 ++++ site/docsgen/gen_config.py | 184 +++++ site/docsgen/highlight.py | 246 ++++++ site/docsgen/icons.py | 63 ++ site/docsgen/md.py | 330 ++++++++ site/static/css/docs.css | 731 +++++++++++++++++ site/static/img/favicon.svg | 1 + site/static/img/logo.svg | 1 + site/static/js/docs.js | 521 ++++++++++++ site/templates/404.html | 22 + site/templates/_api_index.html | 11 + site/templates/_api_object.html | 68 ++ site/templates/_cli_index.html | 23 + site/templates/_config.html | 20 + site/templates/_hero_graph.html | 51 ++ site/templates/_logo_mark.html | 1 + site/templates/_options.html | 17 + site/templates/base.html | 90 +++ site/templates/changelog.html | 36 + site/templates/home.html | 88 ++ site/templates/page.html | 87 ++ 87 files changed, 17498 insertions(+), 902 deletions(-) delete mode 100644 docs/index.html create mode 100644 site/AUTHORING.md create mode 100644 site/build.py create mode 100644 site/content/api/cloudgconfig.md create mode 100644 site/content/api/cloudgengine.md create mode 100644 site/content/api/cloudontology.md create mode 100644 site/content/api/dependencygraph.md create mode 100644 site/content/api/entry-points.md create mode 100644 site/content/api/event-hooks.md create mode 100644 site/content/api/findingsnormaliser.md create mode 100644 site/content/api/graphbuilder.md create mode 100644 site/content/api/inventorymapper.md create mode 100644 site/content/api/inventoryresult.md create mode 100644 site/content/api/mcp-layer.md create mode 100644 site/content/api/models.md create mode 100644 site/content/api/parse-report.md create mode 100644 site/content/api/ragexporter.md create mode 100644 site/content/api/reachabilityanalyzer.md create mode 100644 site/content/api/relationshiplinker.md create mode 100644 site/content/api/resilience.md create mode 100644 site/content/api/results.md create mode 100644 site/content/cli/auth-flags.md create mode 100644 site/content/cli/collect.md create mode 100644 site/content/cli/deps.md create mode 100644 site/content/cli/ingest.md create mode 100644 site/content/cli/map.md create mode 100644 site/content/cli/mcp-call.md create mode 100644 site/content/cli/mcp-config.md create mode 100644 site/content/cli/mcp-prompts.md create mode 100644 site/content/cli/mcp-read.md create mode 100644 site/content/cli/mcp-resources.md create mode 100644 site/content/cli/mcp-serve.md create mode 100644 site/content/cli/mcp-tools.md create mode 100644 site/content/cli/output-flags.md create mode 100644 site/content/cli/provider-flags.md create mode 100644 site/content/cli/report.md create mode 100644 site/content/cli/run.md create mode 100644 site/content/cli/scan.md create mode 100644 site/content/guides/authentication.md create mode 100644 site/content/guides/aws-organizations.md create mode 100644 site/content/guides/ci-automation.md create mode 100644 site/content/guides/compliance.md create mode 100644 site/content/guides/dependencies.md create mode 100644 site/content/guides/docker.md create mode 100644 site/content/guides/graph-ontology-rag.md create mode 100644 site/content/guides/ingesting.md create mode 100644 site/content/guides/installation.md create mode 100644 site/content/guides/introduction.md create mode 100644 site/content/guides/inventory-mapping.md create mode 100644 site/content/guides/pipeline.md create mode 100644 site/content/guides/python-recipes.md create mode 100644 site/content/guides/quickstart.md create mode 100644 site/content/guides/rate-limits.md create mode 100644 site/content/guides/reports.md create mode 100644 site/content/guides/running-scanners.md create mode 100644 site/content/mcp/connect.md create mode 100644 site/content/mcp/index.md create mode 100644 site/content/reference/output-files.md create mode 100644 site/data/config-notes.yaml create mode 100644 site/data/nav.yaml create mode 100644 site/data/stats.yaml create mode 100644 site/docsgen/__init__.py create mode 100644 site/docsgen/gen_api.py create mode 100644 site/docsgen/gen_changelog.py create mode 100644 site/docsgen/gen_cli.py create mode 100644 site/docsgen/gen_config.py create mode 100644 site/docsgen/highlight.py create mode 100644 site/docsgen/icons.py create mode 100644 site/docsgen/md.py create mode 100644 site/static/css/docs.css create mode 100644 site/static/img/favicon.svg create mode 100644 site/static/img/logo.svg create mode 100644 site/static/js/docs.js create mode 100644 site/templates/404.html create mode 100644 site/templates/_api_index.html create mode 100644 site/templates/_api_object.html create mode 100644 site/templates/_cli_index.html create mode 100644 site/templates/_config.html create mode 100644 site/templates/_hero_graph.html create mode 100644 site/templates/_logo_mark.html create mode 100644 site/templates/_options.html create mode 100644 site/templates/base.html create mode 100644 site/templates/changelog.html create mode 100644 site/templates/home.html create mode 100644 site/templates/page.html diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 659e9b5..d796f4d 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -1,6 +1,8 @@ name: Docs -# Deploys docs/ to GitHub Pages on every push to main that touches it. +# Builds the documentation site (site/build.py) and deploys it to GitHub Pages +# on every push to main that touches the docs, the content they are generated +# from (click commands, docstrings, config.yaml, CHANGELOG.md) or the site itself. # # One-time setup: under Settings -> Pages, set "Build and deployment" # source to "GitHub Actions". The GITHUB_TOKEN cannot create the Pages @@ -12,6 +14,16 @@ on: branches: [main] paths: - "docs/**" + - "site/**" + - "cloudg/**" + - "config.yaml" + - "CHANGELOG.md" + - "pyproject.toml" + - ".github/workflows/pages.yml" + pull_request: + paths: + - "docs/**" + - "site/**" - ".github/workflows/pages.yml" workflow_dispatch: @@ -19,11 +31,42 @@ permissions: contents: read concurrency: - group: pages + group: pages-${{ github.ref }} cancel-in-progress: true jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 # tags, so source links point at the release tag + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + # The CLI and API reference pages are generated by importing cloudg, + # so the package is installed; its core dependencies already include + # everything the site generator needs (markdown-it-py, Pygments, Jinja2). + - name: Install cloudg + run: pip install . + + - name: Build the site + run: python site/build.py --strict + + - uses: actions/upload-pages-artifact@v3 + if: github.event_name != 'pull_request' + with: + path: site/dist/ + # The artifact name carries the run attempt: on a re-run, the previous + # attempt's "github-pages" artifact still exists in the run, and + # deploy-pages refuses when it finds two artifacts with the same name. + name: github-pages-${{ github.run_attempt }} + deploy: + if: github.event_name != 'pull_request' + needs: build runs-on: ubuntu-latest permissions: pages: write @@ -32,20 +75,9 @@ jobs: name: github-pages url: ${{ steps.deployment.outputs.page_url }} steps: - - uses: actions/checkout@v7 - - - uses: actions/configure-pages@v6 - - # The artifact name carries the run attempt: on a re-run, the previous - # attempt's "github-pages" artifact still exists in the run, and - # deploy-pages refuses when it finds two artifacts with the same name - # ("Multiple artifacts named github-pages were unexpectedly found"). - - uses: actions/upload-pages-artifact@v5 - with: - path: docs/ - name: github-pages-${{ github.run_attempt }} + - uses: actions/configure-pages@v5 - id: deployment - uses: actions/deploy-pages@v5 + uses: actions/deploy-pages@v4 with: artifact_name: github-pages-${{ github.run_attempt }} diff --git a/CHANGELOG.md b/CHANGELOG.md index b2fd5af..f642cbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ Notable changes per release. Patch releases are folded into the major entry they belong to. +## Unreleased + +Documentation: + +- The handbook at morpheuslord.github.io/cloudg is now a site of separate pages instead of one long `docs/index.html`. Guides, the CLI, the Python API, the MCP server, the reference documents and this changelog each have their own tab and sidebar. `site/build.py` builds it: the CLI options tables come from the click definitions, the API pages from the docstrings, the config pages from `config.yaml` and the `CloudGConfig` models, and the long `docs/*.md` references are split into one page per section. Writing rules for new pages are in `site/AUTHORING.md`. +- New guides cover installation, the pipeline phase by phase, authentication for each provider, AWS Organizations, scanners and ingest input formats, compliance mapping, reports, the graph, ontology and RAG exports, rate limits, Docker, CI and Python recipes. Code samples are complete scripts, and processes are drawn as diagrams. +- Search (Ctrl K or /), a dark theme, linked code tabs and copy buttons work without a server. Links to the old `index.html#anchor` sections redirect to the new pages. + ## 0.6.0 (2026-10-09) Two additions and a round of graph fixes. cloudg is now a Model Context Protocol server, so AI agents can query an inventory map, its graph, findings, compliance and ontology under an access and privacy policy. Every cloud API call now goes through a shared rate limiter that slows down when a provider throttles, so an organization-wide map (or an agent asking for one too often) no longer fails or hammers the APIs. diff --git a/docs/index.html b/docs/index.html deleted file mode 100644 index 0c47b06..0000000 --- a/docs/index.html +++ /dev/null @@ -1,887 +0,0 @@ -cloudg Handbook - - - -
- - -
-
- -
cloud graphing · AWS / Azure / GCP
-

Map the cloud. Scan it. Merge everything.

-

cloudg collects your infrastructure into a graph, fans out to the security scanners you already trust, and merges every finding against 28 compliance frameworks. It can also skip the scanning entirely: aggregate outputs you already have, or map the complete inventory, every service interlinked, with no scanner involved.

-
- GitHub - PyPI - Markdown reference - Changelog - Report viewer -
-
- -
-

Installation

- 
# core: graph engine, normaliser, renderers, ingest
-pip install cloudg
-
-# cloud SDK extras, pulled in only for collection
-pip install cloudg[aws]        # boto3, aioboto3
-pip install cloudg[azure]      # Azure management SDKs
-pip install cloudg[gcp]        # google-cloud-asset, google-auth
-pip install cloudg[mcp]        # the official MCP SDK, see MCP server
-pip install cloudg[all]        # all of the above
-

The core package alone ships the graph engine, ontology builder, findings normaliser, report renderers and cloudg ingest. That is enough to aggregate existing scan results on a machine with no cloud access at all.

-
-

The scanners are not Python dependencies. Prowler, ScoutSuite, Checkov and Trivy are separate executables. Install whichever subset you want; cloudg checks PATH at run time and skips anything missing with a warning. install.sh / install.bat set up all of them, and the Docker image bundles the four binaries.

-
-

Requires Python 3.11 or newer.

-
- -
-

The pipeline

- 
collect -> graph -> scan -> normalise -> report
-                \-> ontology / RAG / terraform
-

Collection runs all configured providers concurrently with asyncio, iterating accounts and regions per provider. Regions auto-discover with --regions all. Assets and network edges go into a directed NetworkX graph: BFS from the internet node finds exposed resources, and blast-radius scoring estimates what an attacker could reach from each node.

-

The enabled scanners run in parallel over the same inventory, each in its own thread with its own timeout. Their findings, the graph's reachability findings and the built-in IAM linter's results all flow into the normaliser, which deduplicates within and across scanners, rescores against CVSS, and maps everything to compliance controls.

-

Every phase degrades gracefully. A missing binary, an unreachable provider or a failed exporter logs a warning; nothing else stops.

-

Alongside the pipeline sits an independent function: inventory mapping. It runs no scanners at all. It deep-collects everything deployed or default across every service, links it into one map, and lets scanner findings merge into that map later.

-
- -
-

cloudg run

-

The full pipeline: collect, graph, scan, normalise, report.

- 
cloudg run -p aws --regions us-east-1
-cloudg run -p all --regions all
-cloudg run -p aws --scanners prowler,trivy
-cloudg run -p aws --regions us-east-1 --terraform
-
- - - - - - - - - - -
FlagMeaning
-p, --provideraws, azure, gcp or all; repeatable
--regionsall for auto-discovery, or a comma-separated list
--scannerssubset of prowler,scoutsuite,checkov,trivy,iam; defaults to scanners.enabled
--iac-dirdirectory for the IaC scanners (Checkov, Trivy fs)
--imagescomma-separated container images for Trivy
--ontology / --no-ontologyRDF export, on by default
--rag-export / --no-rag-exportRAG chunks, on by default
--terraform / --no-terraformTerraform recreation, off by default
-o, --outputoutput directory, ./reports by default
-

When no IaC directory is configured but --terraform is on, the IaC scanners target the generated Terraform recreation of the live infrastructure. Checkov then audits your actual cloud, not whatever directory cloudg happens to run from. There is deliberately no fallback to .: a zero-finding scan of an unrelated directory reads like a clean bill of health, and cloudg refuses to produce one.

-
- -
-

collect & scan

-

The phases alone. collect writes assets and edges without running a scanner; scan runs the selected scanners concurrently and writes raw-findings.json, no collection.

- 
cloudg collect -p aws --profile prod --region eu-west-1
-cloudg scan -p aws --scanners prowler,checkov --iac-dir ./terraform
-
- -
-

cloudg map

-

Scanner-independent inventory mapping: what exists, and how is it wired together. What is wrong stays the scanners' business. No scanner runs and none needs to be installed; read credentials are enough.

- 
cloudg map -p aws --regions all
-cloudg map -p aws --org --regions all                        # every account in the Organization / Control Tower
-cloudg map -p aws --org --ou Workloads --exclude-account 111122223333
-cloudg map -p aws --services containers,serverless,security   # narrow the run
-cloudg map -p all --regions all
-cloudg map -p aws --findings ./reports/raw-findings.json      # overlay scanner output
-

Coverage comes in four layers:

-
- - - - - -
LayerWhat it adds
Deep collectorsgrouped into families selectable with --services: network (VPC fabric, transit gateway attachments, VPC endpoints, ALB/NLB listeners and target groups), compute (EC2, Auto Scaling, launch templates), containers (ECR, ECS services and task definitions, EKS nodegroups, Fargate, addons, access entries, pod identity), kubernetes (workloads, services, ingresses and service accounts inside EKS), serverless (Lambda triggers and URLs, API Gateway, Step Functions), integration (SQS, SNS, EventBridge, Kinesis), storage/data, identity (the IAM graph), security (GuardDuty, Security Hub, Inspector coverage, Macie, Config, Access Analyzer, Detective, WAF, Network Firewall, Shield), logging, dns, iac (CloudFormation stacks and what they manage). Account-wide services run once per account; disabled security services show as gaps.
Catch-all sweepsthe AWS Resource Groups Tagging API, Azure Resource Manager's full resources.list(), and GCP Cloud Asset Inventory enumerate resources in scope, so services without a dedicated collector still land on the map; a duplicate of a dedicated collector's asset is dropped and the richer one wins
Relationship linkerresolves what each asset declares it talks to into typed edges: INVOKES, USES_IMAGE, ASSUMES_ROLE, IAM_TRUST, GRANTS_ACCESS, PROTECTS, MONITORS, MANAGES, GOVERNS, LOGS_TO, ROUTE, LOAD_BALANCER_TARGET, CONTAINS, each with a fine-grained ontology relationship. Resolution is scoped to account and region; references into unmapped accounts become external account nodes
Account hierarchyaccount nodes containing their top-level resources; with --org, under the OU tree next to SCPs, the Control Tower landing zone (governed regions, log archive and audit accounts) and enabled controls
-

Outputs: inventory-map.json (assets, interconnections, unresolved references, summary), inventory-map.graphml, inventory-graph.json, inventory-dependencies.json (shared dependencies, blast radius, cross-account edges, security coverage) and, with --org, inventory-organization.json. With --findings, additionally asset-map.json (per-asset risk, riskiest first) and compliance-map.json (framework → affected assets). The map never depends on the scanners; findings merge in whenever they exist. Every file is specified field by field in the inventory reference.

-
-

Control Tower: run from the management account. Member accounts are reached through AWSControlTowerExecution by default, which is an administrator role; deploy a read-only role (SecurityAudit + ViewOnlyAccess) with a StackSet and pass --org-role for production use.

-
-
-

The same capability is library API: engine.map_inventory(), or cloudg.inventory.InventoryMapper, RelationshipLinker, DependencyGraph and discover_organization standalone. See Python API.

-
-
- -
-

cloudg deps

-

Interdependency queries over a saved map, no cloud access needed. Each edge reads "dependent needs dependency": a function needs its role, image, key and the queue that triggers it; an ALB needs the WAF protecting it; a resource needs the stack managing it.

- 
cloudg deps arn:aws:iam::123456789012:role/app-role         # both directions
-cloudg deps jobs-queue --direction down --depth 5            # blast radius
-cloudg deps --map ./reports                                  # shared dependencies, blast radius, cross-account edges
-cloudg deps my-function --direction up --json
-
- -
-

cloudg ingest

-

Aggregation without execution. Feed cloudg the native output files of scans that already ran (in CI, on another host, on a schedule) and it runs the normalisation half of the pipeline: cross-scanner dedupe via the check-equivalence rulesets, compliance mapping and report generation. No scanner executes and no cloud credentials are needed.

- 
cloudg ingest \
-  --prowler ./prowler-output/ \
-  --scoutsuite ./scoutsuite-report/ \
-  --checkov ./results_json.json \
-  --trivy ./trivy-image.json --trivy ./trivy-fs.json \
-  -o ./reports
-

Every input flag is repeatable and any combination or subset of tools works: one tool alone, or all four together. The exact formats are pinned down under Input requirements.

-
-

Ingest runs no collection, so outputs that need live assets (topology, ontology, RAG, Terraform) stay empty. With credentials available, combine both through the API: collect() + ingest_reports() + analyze().

-
-
- -
-

cloudg report

-

Re-render reports from a previous run's findings.json. No cloud, no scanners. --format takes html, json, svg or all.

- 
cloudg report -i ./reports/findings.json --format html
-
- -
-

Input requirements

-

What each ingest path expects, and the command that produces it.

- -

Prowler

- 
prowler aws -M json-asff -o ./prowler-output
-

cloudg accepts the output directory or any single file from it, as a JSON array or JSONL. The fields read from each ASFF finding:

- 
{
-  "Id": "prowler-aws-s3_bucket_default_encryption-123456789012-eu-west-1-...",
-  "Title": "S3 bucket default encryption",
-  "Severity": {"Label": "HIGH"},
-  "Resources": [{"Id": "arn:aws:s3:::my-bucket"}],
-  "Compliance": {"Status": "FAILED", "RelatedRequirements": ["CIS 2.1.1"]},
-  "Remediation": {"Recommendation": {"Text": "..."}}
-}
-

Findings with a PASSED status are skipped. The check name embedded in Id becomes the dedupe key.

- -

ScoutSuite

- 
scout aws --report-dir ./scoutsuite-report --no-browser
-

Accepts the report directory (searched recursively for scoutsuite_results*.js) or the file itself. The file is JavaScript, a JSON object assigned to a variable; cloudg strips the assignment and parses the JSON. Each entry under services.<service>.findings with flagged_items > 0 yields one finding per flagged item. level maps to severity: danger is CRITICAL, warning HIGH, caution MEDIUM.

- -

Checkov

- 
checkov -d ./iac --output json > results_json.json
-

Accepts the JSON file or a directory containing results_json.json. Both output shapes parse: a single {check_type, results} object, or a list of them when several frameworks ran. Findings come from results.failed_checks[], reading check_id, check_name, severity, resource, file_path and guideline.

- -

Trivy

- 
trivy image --format json myrepo/app:latest > trivy-image.json
-trivy fs --format json --scanners vuln,misconfig,secret ./iac > trivy-fs.json
-

Accepts a single JSON file or a directory of them. The top-level ArtifactType decides the parser: container_image goes through the image parser (vulnerabilities and secrets), everything else through the filesystem parser, which also handles misconfigurations. ArtifactName becomes the resource identifier, and the CVSS v3 score is read when present.

-
- -
-

Output contract

-
- - - - - - - - - - -
FileContents
report.htmlinteractive report: D3 topology, findings table, compliance matrix; self-contained, works offline
findings.jsonthe machine-readable result, structure below; round-trips through cloudg report -i
raw-findings.jsonpre-normalisation findings from scan and ingest
topology.svg / .graphml / -cytoscape.jsonthe graph in three formats
ontology.ttl / .jsonldRDF ontology, 64 inferred relation types, SPARQL-queryable
rag_chunks.jsonlretrieval-ready chunks, one JSON object per line
terraform/*.tf.jsonTerraform recreation of live infrastructure with an import.sh
inventory-map.json / .graphml, inventory-graph.jsonscanner-independent inventory map: assets, interconnections, summary (cloudg map)
asset-map.json, compliance-map.jsoninventory overlaid with scanner findings (cloudg map --findings)
- 
{
-  "metadata":   {"scan_id", "provider", "account_id", "region", "started_at", "completed_at"},
-  "summary":    {"total_assets", "total_findings", "severity_breakdown", "compliance_frameworks"},
-  "assets":     [CloudAsset, ...],
-  "findings":   [Finding, ...],
-  "compliance": [ComplianceResult, ...],
-  "graph":      {"nodes": [], "links": []}
-}
-
- -
-

Data models

-

Everything flowing through cloudg is a Pydantic v2 model from cloudg.schema.models. All models allow extra fields, so scanner-specific attributes survive serialisation.

-

Finding

-
- - - - - - - - - - - -
FieldTypeNotes
idstra random UUID; reachability findings use a UUID5 of the rule and the asset's ARN (or asset ID), so they keep their id from scan to scan
resource_id, resource_arnstrinternal ID and cloud-native identifier
severitySeverityCRITICAL, HIGH, MEDIUM, LOW, INFO
title, descriptionstrscanner tags are stripped for cross-scanner comparison
evidence, remediationstr | None
source_toolstr"prowler"; becomes "prowler, checkov" after a merge
source_finding_idstr | Nonethe scanner's own check or CVE ID; drives dedupe
compliance_frameworkslist[str]unioned on merge
cvss_scorefloat | None0.0 to 10.0
risk_scorecomputedseverity base averaged with CVSS when present
-

The rest

-

CloudAsset carries arn (the stable identifier; id is regenerated each run), name, a 156-value asset_type taxonomy (EC2, LAMBDA_FUNCTION, CONTAINER_REGISTRY, K8S_WORKLOAD, IAM_ROLE, THREAT_DETECTOR, ORG_UNIT, ...), provider, region, account_id, tags, metadata (with declared relations and aliases on inventory assets) and is_internet_exposed. NetworkEdge is a directed edge with an edge_type (network: SECURITY_GROUP_RULE, NACL_RULE, ROUTE, PEERING, LOAD_BALANCER_TARGET, INTERNET_EXPOSED; structure: CONTAINS, ATTACHED_TO, REFERENCES; identity: IAM_TRUST, IAM_POLICY_ATTACHMENT, ASSUMES_ROLE, GRANTS_ACCESS; workload: INVOKES, USES_IMAGE, LOGS_TO; security and governance: PROTECTS, MONITORS, MANAGES, GOVERNS), a fine-grained ontology relationship, properties, and ports, protocol and CIDR for network rules. ComplianceResult is one control-level verdict with framework, control_id, status and the finding_ids behind it. ScanResult is the aggregate container the normaliser returns and the renderers consume, with a computed summary.

-
- -
-

Inventory deep dives

-

The inventory mapper is documented in three further markdown documents, for people consuming its output and people extending it.

-
- - - - -
DocumentWhat is in it
Inventory referenceThe structure of every return value and exported file, field by field, with annotated examples from a real run: CloudAsset, NetworkEdge, identifier formats, edge direction and dependency semantics, the relation object, InventoryResult, every summary key and how it is computed, coverage records and service names, unresolved references, dependency trees, shared dependencies, blast radius, cross-account edges, security coverage, inventory-map.json, inventory-graph.json, GraphML attributes, inventory-organization.json, organization map nodes, asset-map.json, compliance-map.json and cloudg deps --json.
Inventory catalogAll 156 asset types by category; for each, the AWS ARN, CloudFormation, Azure ARM and GCP Cloud Asset Inventory types mapped to it, the metadata keys and value types it carries, the relations its collectors declare, and the edges entering and leaving it; the full relationship matrix and relationship vocabulary.
Inventory internalsArchitecture and data flow, a code map of every module, a mapping run step by step, the AWS deep collector (mixins, task registry, all 133 tasks, helpers, Cloud Control and Tagging sweeps, Kubernetes, security service gaps), Organizations and Control Tower discovery, the linker's index and resolution rules, the Azure and GCP extractor frameworks with every registered type, dependency semantics, catalogs, recipes for new collectors, extractors and asset types, security rules, code quality limits and testing.
-
- -
-

Configuration

-

config.yaml mirrors every CLI flag and adds persistent settings. Everything is optional. The shipped config.yaml documents each field inline; the structure, with defaults:

- 
providers: [aws]
-
-aws:
-  regions: [us-east-1]          # or [all]
-  accounts: []                  # multi-account fan-out
-  role_name: null
-  role_arn: null
-  external_id: null
-  web_identity_token_file: null
-
-scanners:
-  enabled: [prowler, scoutsuite, checkov, trivy, iam]
-  timeout_seconds: 3600
-  iac_directories: []
-  trivy_images: []
-  prowler_extra_args: []        # passthrough, per scanner
-
-inventory: { tagging_sweep: true, link_references: true, services: [all], kubernetes: true }  # cloudg map
-ontology:  { enabled: true, export_formats: [turtle, json-ld] }
-rag:       { enabled: true, chunk_strategy: hybrid, max_chunk_tokens: 2000 }
-terraform: { enabled: false }
-rulesets:  { rules_dir: null }  # defaults to the packaged cloudg/rules/
-ratelimit: { enabled: true, aws: {}, azure: {}, gcp: {}, live_cooldown_seconds: 120 }  # see Rate limits
-concurrency_limit: 5
-

In code, the same structure is CloudGConfig:

- 
from cloudg import CloudGConfig, load_config
-
-config = load_config("config.yaml")
-config = CloudGConfig(providers=["aws"])
-config.scanners.enabled = ["prowler", "iam"]
-

aws.max_retries and aws.retry_mode drive botocore's retries since 0.6.0 (they used to be ignored). botocore counts retries after the first attempt, so the default of 10 allows up to 11 attempts per call.

-
- -
-

Rate limits

-

A map of an AWS Organization sends thousands of read-only calls into every member account, and an AI agent can ask for another live map a minute after the last one. Since 0.6.0 every cloud API call goes through cloudg.resilience, one governor shared by every client in the process:

-
    -
  • Token buckets per account x region x service, per API action for EC2, per subscription and resource provider on Azure, per project or organization and RPC on GCP. The defaults sit at or below each provider's published limits: 20 rps per AWS service and per EC2 action, 20 rps per Azure subscription, 3 rps per user for Resource Graph, 1.5 rps for Cloud Asset ListAssets.
    • -
  • A throttled scope is halved for every client using it and paused for the server's Retry-After, then recovers by 5% of its ceiling every five seconds without throttling.
    • -
  • At most 128 requests are in flight per AWS account, 16 per Azure subscription or GCP project (max_concurrency).
    • -
  • A service, or an EC2 action, still failing after its retries five times in a row is skipped for 60 seconds by a circuit breaker; then one probe call goes through. A throttled DescribeSubnets does not block DescribeVpcs.
    • -
  • Retries use decorrelated jitter and never undercut Retry-After. botocore, azure-core and google-api-core keep doing their own retrying, configured by cloudg and reporting back to it. Once the provider's shared retry budget is spent, every call still gets its first two retries and no more.
    • -
-

Throttling never fails a map. A service that stayed throttled shows up in coverage as FAILED or PARTIAL with an error starting throttled:, and the run's numbers land in InventoryResult.throttling, in summary["throttling"] and in inventory-map.json:

- 
aws/123456789012/us-east-1/ec2: throttled 11x, slowed to 5.0 rps, 10 retries, 1 calls gave up
-

Tune it under ratelimit; unset limits keep the built-ins and service overrides merge into them:

- 
ratelimit:
-  aws:
-    account_max_rps: 40                  # all services of one account together
-    services:
-      resourcegroupstaggingapi: { max_rps: 1, burst: 2 }
-      ec2.DescribeImageAttribute: { max_rps: 5, burst: 10 }
-  live_cooldown_seconds: 600            # MCP live collections, per scope
-

A misspelled key under ratelimit is logged as unknown and ignored. ratelimit.enabled: false turns it all off; botocore's own retries remain. The rate limits and throttling guide covers the architecture, the per-provider wiring, the built-in limits with their sources, every key, a throttled run's output, the Python API (call_with_resilience, @resilient, stats_scope, LiveOperationGuard) and tuning.

-
- -
-

Python API

-

CloudGEngine in cloudg.api is the integration entry point: async-first, sync wrappers included, built for embedding in larger systems (SIEM pipelines, orchestration platforms, command extensions such as hol-guard).

- 
from cloudg import CloudGConfig, CloudGEngine
-
-engine = CloudGEngine(CloudGConfig(providers=["aws"]))
-engine.on_finding = lambda f: siem.send(f.model_dump(mode="json"))
-engine.on_error = lambda phase, exc: alerting.notify(phase, exc)
-
-result = engine.run_pipeline_sync()
-print(result.to_summary())
-
- - - - - - - - - -
MethodReturnsDoes
collect()CollectionResultmulti-provider asset collection: assets, edges, coverage
scan(assets, edges)list[Finding]enabled scanners in parallel, plus reachability and IAM linting
ingest_reports(reports)list[Finding]parse existing scanner outputs; nothing executes
normalise_findings(findings)ScanResultdedupe, cross-scanner merge, CVSS rescoring, compliance mapping
analyze(assets, edges, findings)AnalysisResultgraph, ontology, RAG chunks, Terraform, attack paths
run_pipeline()PipelineResulteverything, end to end; run_pipeline_sync() for non-async callers
run_from_reports(reports)PipelineResultingest, normalise, report in one call; sync twin available
map_inventory()InventoryResultscanner-independent inventory mapping: deep collection, catch-all sweeps, relationship linking; sync twin available
-

Event hooks: on_phase_start(phase), on_collection_complete(result), on_finding(finding), on_scan_complete(findings), on_analysis_complete(result), on_error(phase, exc). Exceptions raised inside a hook are swallowed, so a broken callback cannot take the pipeline down. PipelineResult carries the data, report paths, coverage and a to_summary() dict ready to serialise.

- -

Inventory mapping from code

- 
from cloudg.inventory import InventoryMapper, RelationshipLinker
-
-mapper = InventoryMapper(config)
-inventory = mapper.map_inventory_sync()                 # no scanners
-inventory.export("./reports")                          # map + graphml + graph json
-
-asset_map = mapper.build_asset_map(inventory, findings) # asset -> risk
-compliance = mapper.build_compliance_map(inventory, findings)
-
-edges = RelationshipLinker(assets).link()               # works on ANY CloudAsset list
- -

The deeper toolkit

-

The CLI is a thin layer; the modules the pipeline uses internally are public, importable API. The markdown reference documents each in full.

-
- - - - - - - - - - -
ModuleFrom code
cloudg.inventoryInventoryMapper, RelationshipLinker, and the three deep collectors, all standalone
cloudg.graph.builderGraphBuilder: attack paths, lateral movement, centrality / blast-radius metrics, D3 / Cytoscape / GraphML export
cloudg.graph.ontologyCloudOntology: 64 typed RDF relations, SPARQL-queryable, Turtle / JSON-LD
cloudg.graph.rag_exportRAGExporter: retrieval-ready JSONL chunks of the infrastructure for LLM pipelines
cloudg.renderers.terraform_exportTerraformExporter: .tf.json recreation plus import.sh, with a coverage preview()
cloudg.normaliserFindingsNormaliser: dedupe, cross-scanner merge, CVSS rescore, compliance mapping on any findings list
cloudg.ingestparse_report / ingest_reports: every scanner's parser, no engine, no credentials
cloudg.mcpCloudGMCPLayer: the MCP layer, in-process or mounted into your own MCP server; see MCP server
cloudg.resiliencecall_with_resilience, @resilient, get_governor, stats_scope, LiveOperationGuard: shared rate limits, breakers and retries for your own cloud calls; see Rate limits
-
- -
-

Integration recipes

- -

Aggregate existing scan results, get reports

- 
engine = CloudGEngine(CloudGConfig())
-result = engine.run_from_reports_sync(
-    {"prowler": ["./prowler-output/"], "trivy": ["./trivy-image.json"]},
-    output_dir="./reports",
-)
-print(result.total_findings, result.severity_breakdown)
-print(result.report_paths["html"])
- -

Parse first, decide later

- 
from cloudg.ingest import ingest_reports
-
-findings = ingest_reports({"prowler": ["./out/"], "checkov": ["results_json.json"]})
-scan_result = engine.normalise_findings(findings)
-
-for c in scan_result.compliance:
-    if c.status.value == "FAIL":
-        print(c.framework, c.control_id, len(c.finding_ids))
-

The per-scanner parsers are also public: TrivyScanner.parse_report("./trivy.json") and friends, one tool with no engine.

- -

Map the inventory, overlay findings later

- 
inventory = engine.map_inventory_sync(output_dir="./reports")   # no scanners
-print(inventory.summary["assets_by_service"])   # what exists
-print(inventory.summary["internet_exposed"])    # what faces the internet
-
-# hours or days later, when scanner output exists:
-findings = engine.ingest_reports({"prowler": ["./ci-prowler-output/"]})
-InventoryMapper(config).export_merged(inventory, findings, "./reports")
-# -> asset-map.json, compliance-map.json
- -

Combine live collection with ingested reports

- 
collection = await engine.collect()
-ingested = engine.ingest_reports({"prowler": ["./ci-prowler-output/"]})
-scan_result = engine.normalise_findings(ingested, assets=collection.assets)
-await engine.analyze(collection.assets, collection.edges, scan_result.findings)
- -

Gate a CI job on severity

- 
result = CloudGEngine(CloudGConfig()).run_from_reports_sync(
-    {"checkov": ["results_json.json"]}
-)
-if result.severity_breakdown.get("CRITICAL", 0):
-    sys.exit(1)
-
- -
-

Plugins

-

Custom collectors and scanners register through setuptools entry points; no core changes needed. PluginRegistry discovers them at startup and falls back to the built-ins for any name not overridden.

- 
# my_package/scanner.py
-from cloudg.schema.models import Finding, Severity
-
-class MyScanner:
-    def run(self) -> list[Finding]:
-        return [Finding(
-            resource_id="arn:aws:s3:::example",
-            severity=Severity.HIGH,
-            title="Example finding",
-            description="What was found and why it matters",
-            source_tool="myscanner",
-            source_finding_id="MY_CHECK_001",
-        )]
- 
[project.entry-points."cloudg.scanners"]
-myscanner = "my_package.scanner:MyScanner"
-
-[project.entry-points."cloudg.collectors"]
-mycloud = "my_package.collector:MyCollector"
-

After pip install, the name works in scanners.enabled and --scanners. If your scanner emits a stable check ID in source_finding_id, add it to cloudg/rules/check_equivalence.yaml so its findings merge with equivalent checks from other tools.

-
- -
-

Authentication

-

Every provider supports several methods, resolved in fixed priority order, so the same config works on a laptop, in CI and on cloud compute.

-
- - - - -
ProviderMethods, most explicit first
AWSdirect keys; OIDC web identity federation (--aws-role-arn with --aws-web-identity-token-file, the GitHub Actions and EKS pattern); named profile including SSO; the default chain. STS assumption layers on top with --aws-external-id for the third-party auditor pattern, and multi-account fan-out uses accounts plus role_name.
Azureworkload identity federation; service principal with secret or certificate; managed identity; the DefaultAzureCredential chain, which covers az login.
GCPcredentials file (service account key or workload identity federation config); application default credentials. --gcp-impersonate-sa layers impersonation on either.
-
- -
-

Docker

- 
docker pull ghcr.io/morpheuslord/cloudg:latest
-docker compose run --rm cloudg run -p aws --regions us-east-1
-

The image bundles all four scanner binaries, so it is the shortest path to the full pipeline.

-
- -
-

MCP server

-

cloudg 0.6.0 is also a Model Context Protocol server. An AI agent (Claude Desktop, Claude Code, Cursor, VS Code or your own) can load an inventory map or a findings report, search and inspect assets, walk the relationship graph, ask for attack paths and blast radius, review findings and compliance gaps, query the ontology and, where the policy allows it, start live collections.

-

Everything sits in one class, cloudg.mcp.CloudGMCPLayer. Whichever way a request arrives (stdio, HTTP, an MCP server you mounted cloudg into, or a direct Python call), it passes through the same policy: access rules, rate limits, redaction, pseudonymisation, projection and annotation.

-
- - - - -
ModeEntry pointUse it when
Standalone servercloudg mcp serveyou want cloudg as its own MCP server for a desktop client or a gateway
Mountedlayer.register_into(server)you already run an MCP server (official mcp SDK 1.x or 2.x, or fastmcp 2, 3 or 4)
In-processawait layer.call_tool(...)you are writing Python and do not need MCP on the wire
- 
pip install cloudg            # the layer, the CLI and the dependency-free native server
-pip install "cloudg[mcp]"     # + the official mcp SDK: sdk flavor and register_into(...)
-pip install fastmcp           # only for the fastmcp flavor or mounting into fastmcp
-

The full guide is MCP.md; the catalog is in MCP_TOOLS.md, policies and transforms in MCP_PRIVACY.md, and the design in MCP_INTERNALS.md.

-
- -
-

Serving it

- 
# stdio: the desktop client launches the server
-cloudg mcp serve --dataset prod=./reports/inventory-map.json
-
-# the snippet to paste into claude-desktop, claude-code, cursor or vscode
-cloudg mcp config --client claude-desktop --dataset prod=./reports/inventory-map.json
-
-# streamable HTTP on 127.0.0.1:8765/mcp, bearer token mapped to a role
-cloudg mcp serve --transport http --auth-token env:CLOUDG_MCP_TOKEN:analyst
-

cloudg-mcp and python -m cloudg.mcp run the same commands. Transports: stdio, streamable HTTP and the deprecated SSE endpoints. The server speaks both protocol eras on one object: the handshake versions 2024-11-05, 2025-03-26, 2025-06-18 and 2025-11-25, and the stateless 2026-07-28. --flavor chooses the implementation: native (standard library only), sdk (the official SDK's low-level server), fastmcp, or auto, which uses the SDK when it is installed.

-
-

HTTP binds to loopback by default and checks Origin and Host. Before binding anything else, set --auth-token, --allowed-host and --allowed-origin.

-
-
- -
-

Embedding it

-

Mount every tool, resource and prompt into a server you already run. The host keeps its own tools, and names cloudg does not own still reach the host's handlers. Mount after the host has registered them.

- 
from mcp.server.mcpserver import MCPServer      # mcp 2.x; FastMCP on mcp 1.x works the same
-from cloudg.mcp import CloudGMCPLayer
-
-server = MCPServer("platform-tools")
-layer = CloudGMCPLayer(policy="standard")
-layer.workspace.load("inventory/inventory-map.json", "prod")
-layer.register_into(server, prefix="cloudg_")
-

Or call it in-process, no MCP on the wire:

- 
layer = CloudGMCPLayer(policy="strict")
-await layer.call_tool("load_dataset", {"path": "inventory/inventory-map.json", "name": "prod"})
-res = await layer.call_tool("find_assets", {"internet_exposed": True})
-res.structured["items"]                     # identifiers pseudonymised under strict
-

The mcp extra pins mcp>=1.30,<3; this release was verified with mcp 1.30.0 and 2.3.0, and fastmcp 2.14.7, 3.4.8 and 4.0.11. Mounting into MCPServer or FastMCP (and into fastmcp) goes through private attributes that a later SDK release can rename, so pin the minor version you tested for that path. The low-level Server path and the native flavor do not depend on them.

-
- -
-

Policies & privacy

-

--policy takes a built-in profile, a YAML or JSON file, or inline JSON. The default is standard.

-
- - - - - - - - -
ProfileIn shortTools (local user)
openno restrictions and no transforms, for trusted local use74
standardsecrets redacted, private keys dropped, credential ids masked, attacker-controllable text fenced, secrets in arguments refused; reveal_token only for admin and privacy-admin73
strictas standard, plus pseudonymised identifiers, IPs, e-mails and tag values, rate limits on every tool, and no live, export or restricted tools61
read_onlyas standard, without cloud access, processes or file writes66
airgappedas standard, without cloud access or processes69
auditas read_only, every call logged, risk annotations inline66
soc-analystrole-based example: analysts see pseudonyms, the lead role sees clear values62; analyst 60, lead 70
-

Pseudonyms round-trip: an agent can pass a pseudonymised ARN back as an argument and the input pipeline reverses it. Set CLOUDG_MCP_VAULT_KEY to keep them stable across restarts; a policy with vault.path also saves the vault when the server shuts down. reveal_token, for the roles allowed to use it, turns a pseudonym or an alias back into the real value. Policies extend each other and carry per-role rules, sensitivity ceilings, per-principal rate limits and transform pipelines; the details are in MCP_PRIVACY.md.

-

The four live tools (map_inventory, collect_assets, run_scanners, run_pipeline) use the server's own cloud credentials, whoever calls. They check those credentials first (preflight, at most 25 seconds), then run under the live operation guard: identical concurrent calls share one run, a scope collected in the last two minutes is refused with retry_after_seconds and a loaded live dataset of the same providers to use instead, and at most one live operation runs per scope and two overall. admin and operator can pass force=true to skip the cooldown. Each live tool's answer reports any throttling during its run. rate_limit_status and cloudg://ratelimit show the state; see Rate limits.

-
- -
-

Tool catalog

-

74 tools, 13 resources, 11 resource templates and 11 prompts; the CLI adds a cloudg://metrics resource. Every primitive, with its arguments and result shape, is in MCP_TOOLS.md. The cloudg://docs resources serve the markdown reference and the inventory reference; the wheel ships both inside the package, so they work from a plain pip install.

-
- - - - - - - - - - - -
CategoryToolsWhat they do
graph17neighbours, paths, exposure, attack and lateral paths, dependencies, blast radius, centrality, sub-graphs
inventory11search, inspect and aggregate assets, accounts, regions, tags, coverage, organization
findings10browse, summarise, prioritise, suppress and ingest security findings
workspace8load, select, snapshot, diff and unload datasets
ontology6RDF ontology, read-only SPARQL, semantic neighbourhoods, RAG chunks
compliance5framework posture, controls and gaps
live5the four live tools and rate_limit_status
privacy5policy, pseudonym vault and data-handling controls
meta4server capabilities and cloudg's vocabulary
export3Terraform recreation, reports and ontology files
-

The prompts are packaged workflows: security_posture_review, executive_summary, attack_surface_report, investigate_asset, blast_radius_assessment, change_impact_analysis, cross_account_trust_review, compliance_gap_analysis, remediation_plan, incident_triage and drift_review.

-
- -
-

MCP CLI

-
- - - - - - -
CommandDoes
cloudg mcp serverun the server: --transport, --host, --port, --path, --flavor, --auth-token, --allowed-origin, --allowed-host, --cache-ttl, --max-concurrency, --log-level
cloudg mcp tools / resources / promptswhat a principal sees under a policy (--role, --principal-id; --json for the exact wire definitions)
cloudg mcp call TOOL --args JSONcall a tool in-process through the full policy; exit status 1 on a tool error
cloudg mcp read URIread a resource in-process
cloudg mcp configprint a client configuration snippet
-

Every command but config takes --policy, --dataset [NAME=]PATH (repeatable), --prefix, --include-category / --exclude-category, --include-tool / --exclude-tool, --read-only, --audit-log FILE, --timeout and --registry MODULE:ATTR.

-
- -
-

Changelog

-

Every release, newest first. The full detail lives in CHANGELOG.md.

- -
- 0.6.0 2026-10-09 -

MCP server and throttling resilience

-

cloudg is now a Model Context Protocol server. cloudg mcp serve (also cloudg-mcp) exposes 74 tools, 13 resources, 11 resource templates and 11 prompts over stdio or streamable HTTP, on both MCP protocol eras (2024-11-05 to 2025-11-25, and 2026-07-28), with a dependency-free native server or the official SDK. The same layer mounts into an existing MCP SDK or fastmcp server, or runs in-process. Seven policy profiles control access and transform the data on its way out: redaction, pseudonyms that round-trip, fencing of attacker-controllable text, projection and annotation. Live collections check credentials first and run under a guard with single-flight, cooldowns and concurrency caps.

-

Every cloud API call now goes through cloudg.resilience: shared adaptive token buckets per account, region and service, concurrency caps, Retry-After, circuit breakers per service (per action for EC2) and retries bounded by a shared budget. Throttled services are recorded in coverage as throttled: ... instead of failing the map, and InventoryResult.throttling reports what happened. New ratelimit config section; aws.max_retries and aws.retry_mode are now honoured. New mcp extra (mcp>=1.30,<3), included in all.

-

The graph analysis reports different results for the same estate. Internet exposure follows network-flow edges only (rules, routes, peering, load balancer targets, attachments to security groups, VPC and subnet containment), so an IAM grant or an API Gateway invoking a Lambda function no longer makes a resource internet-exposed. Entry points are 0.0.0.0/0, ::/0 and the Azure Internet, Any and * sources. Sensitive ports are matched as numbers, so all-traffic rules, Azure * and comma lists report SSH and RDP, 2200-2300 no longer counts as port 22, and egress rules report nothing. Security groups, NSGs, NACLs and target groups are no longer reported as unexpected exposed resources. Finding ids are a hash of the rule and the asset's ARN, the same on every scan. GraphBuilder merges parallel rules between the same two nodes instead of keeping only the last one, which had hidden SSH open to the world behind a later rule for port 443. AWS security group rule edges point at the security group asset. In the ontology, containment and protection relations now point the right way and match both endpoint types, a generic CONTAINS relation brings the total to 64, port relations are read from the parsed ranges, and ENCRYPTED_BY_KMS links an asset to its key. The ontology and RAG exports attach findings to assets by ID, ARN, unique name or unique ARN tail.

-
- -
- 0.5.3 2026-10-04 -

URL sanitization fix

-

A security fix for CodeQL "incomplete URL substring sanitization" alerts. The GCP relation extractors identified a resource's API service with string prefix tests on host-like literals (cloudkms.googleapis.com/, //iam.googleapis.com/, //sqladmin.googleapis.com/, //container.googleapis.com/). They now parse the service out of the name with anchored regular expressions. Behaviour is unchanged, which was checked against the previous implementation on about 92,000 generated inputs.

-
- -
- 0.5.2 2026-10-04 -

Inventory documentation

-

The inventory mapper is documented in full: the inventory reference specifies every return structure and exported file field by field with annotated examples, the inventory catalog covers all 156 asset types with their metadata and relationships, and inventory internals explains how the mapper works and how to extend it. No code changes.

-
- -
- 0.5.1 2026-10-04 -

Deep inventory: interdependencies, every account, every service

-

The inventory map now follows workloads into containers and Kubernetes (ECR, ECS services and task definitions, EKS nodegroups, Fargate, addons, access entries, pod identity, and the Deployments, Services, Ingresses and ServiceAccounts inside clusters), through serverless and event wiring (Lambda triggers and URLs, API Gateway, SQS, SNS, EventBridge, Step Functions, Kinesis), across the full IAM graph and Identity Center, and into the security services and scanners watching the estate, including where they are not enabled. AWS now has 133 collectors plus a Cloud Control API sweep that lists every listable resource type, tagged or not; Azure is read through Azure Resource Graph across every subscription and management group; GCP through Cloud Asset Inventory across the whole organization.

-

cloudg map --org maps every account of an AWS Organization / Control Tower landing zone (OU tree, SCPs, landing zone, governed regions, controls). Edges are typed (INVOKES, USES_IMAGE, ASSUMES_ROLE, GRANTS_ACCESS, PROTECTS, MONITORS, MANAGES, GOVERNS, LOGS_TO) and cloudg deps answers what an asset needs and what breaks with it. Type maps moved into YAML catalogs. Fixes include global services duplicated per region, a single Azure NSG rule discarding a whole subscription, a GCP state-field crash that truncated projects, and CodeQL / Codacy security findings.

-
- -
- 0.5.0 2026-09-30 -

Inventory mapping

-

A scanner-independent inventory mapper: cloudg map and CloudGEngine.map_inventory() deep-collect everything deployed or default across AWS, Azure and GCP (network fabric included, with catch-all sweeps so every service appears) and link it into one map. Scanner findings merge in afterwards as asset-map.json and compliance-map.json. The relationship linker, deep collectors and mapper are all public API.

-
- -
- 0.4.1 2026-09-29 -

Security hardening

-

A pass over the Checkov and Codacy findings: the Docker image runs as a non-root user with a healthcheck, the installers no longer pipe downloads straight into a shell, third-party GitHub Actions are pinned to commit SHAs, plugin loading validates module:Class paths, swallowed errors are logged, and log messages no longer place credential-like words next to runtime values.

-
- -
- 0.4.0 2026-09-29 -

Ingest mode

-

cloudg now works entirely from scanner outputs you already have: the cloudg ingest command, public parse_report() parsers on all four scanner wrappers, and ingest_reports() / normalise_findings() / run_from_reports() on the engine. No cloud credentials, no scanner binaries. Plus this documentation and a least-privilege fix to the publish workflow.

-
- -
- 0.3.0 2026-09-26 -

The cloudg rebrand

-

The project, package, CLI, Docker image, installers and CI became cloudg. Multi-cloud authentication (OIDC federation, STS assumption, managed identity, impersonation), compliance rulesets generated from Prowler's public data (28 frameworks, 4,166 controls, 10,236 check mappings), uv packaging, PyPI trusted publishing, GHCR image, MIT license. Patches folded in: 0.3.1 stopped the IaC scanners from silently scanning the working directory; 0.3.2 rekeyed dedupe on scanner check IDs with the cross-scanner equivalence map and moved the CLI to a Rich UI layer.

-
-
- -
-

cloudg is MIT-licensed. Found a gap in this handbook? Open an issue.

-
-
-
diff --git a/site/AUTHORING.md b/site/AUTHORING.md new file mode 100644 index 0000000..4fdc01e --- /dev/null +++ b/site/AUTHORING.md @@ -0,0 +1,190 @@ +# Writing pages for the cloudg docs site + +The site is built by `site/build.py` from three kinds of input: + +- hand-written pages in `site/content/` (guides, CLI examples, API prose) +- generated reference data (click commands, docstrings, `config.yaml`, `CHANGELOG.md`) +- the long reference documents in `docs/*.md`, which are split at their `##` headings + +Build it with: + +```bash +.venv/bin/python site/build.py # writes site/dist +.venv/bin/python site/build.py --serve # builds, then serves on http://127.0.0.1:8000/cloudg/ +``` + +## Style rules + +- No em dashes anywhere, and no `--` or spaced en dash standing in for one. Use a comma, a colon, parentheses or two sentences. +- Straight quotes only. No emoji. +- Sentence-case headings. +- Run every page you write through the `/human` skill's scanner before committing it. +- Every code example must work as written against the current release. Check signatures in the source; run the Python ones when you can (the sample scanner output inlined in `tests/test_ingest.py` is handy for ingest examples). + +## Front matter + +Every file starts with YAML front matter. + +```yaml +--- +title: Inventory mapping +lede: "`cloudg map` asks a different question from `cloudg run`: what exists, and how is it wired together?" +meta: # optional strip under the lede (guides only) + - [Command, "`cloudg map`"] + - [Providers, "AWS, Azure, GCP"] +source: docs/DOCUMENTATION.md # optional: what "Edit on GitHub" opens. Defaults to this file. +since: "0.5.1" # optional: shown as "Last changed in v0.5.1" +--- +``` + +`lede` and `meta` values are inline markdown. + +## Links + +Write site links as root-relative paths without the base: `/guides/quickstart/`, `/cli/map/#options`, `/reference/config/ratelimit/#aws.retry_budget`. The build adds the base path. The full URL map is in `site/data/nav.yaml`. + +## Markdown extensions + +Standard CommonMark plus GFM tables. On top of that: + +### Headings + +`##` and `###` feed the "On this page" list. Anchors are generated from the text; force one with a trailing `{#my-anchor}`. Never skip a level. + +### Code blocks + +The info string takes a language and optional attributes: + +````markdown +```python title="map_and_overlay.py" hl="4-6" +from cloudg import CloudGConfig, CloudGEngine +... +``` +```` + +- `title` shows as the file name in the editor bar. Use a real file name for code people should save (`deploy.py`, `config.yaml`, `.github/workflows/cloudg.yml`). +- `hl` highlights lines: `hl="3"`, `hl="2,5-7"`. +- `bash` and `shell` blocks render as a terminal. Each command line gets a `$` prompt automatically; comment lines (`# ...`) are left out when the reader presses Copy. Put each command on its own line, or continue it with a trailing `\`. +- `console` is for a captured session with output. Write the prompt yourself (`$ cloudg deps ...`) and the output below it. +- Other languages (`python`, `yaml`, `json`, `toml`, `dockerfile`, `ini`, `text`) get line numbers. + +### Code tabs + +Consecutive fenced blocks that each carry a `tab` attribute become one tabbed block. Tabs with the same label are linked across the page, and the reader's choice is remembered. + +````markdown +```bash tab="CLI" +cloudg map -p aws --regions all +``` + +```python tab="Python" title="map.py" +from cloudg import CloudGConfig, CloudGEngine +... +``` + +```bash tab="Docker" +docker run --rm -v ~/.aws:/home/cloudg/.aws:ro -v "$PWD/reports:/app/reports" cloudg map -p aws --regions all +``` +```` + +Use the labels `CLI`, `Python`, `Docker` for ways to do one thing, and `pip`, `uv`, `Docker` for installs. + +### Diagrams + +A `mermaid` block renders as a diagram. Add a caption with `caption="..."`. + +````markdown +```mermaid caption="What cloudg run does, phase by phase" +flowchart LR + A[Collect] --> B[Graph] --> C[Scan] --> D[Normalise] --> E[Report] +``` +```` + +Keep diagrams small enough to read at 680 px: up to about 12 nodes. Prefer `flowchart LR` or `TD`, `sequenceDiagram` for request flows and `stateDiagram-v2` for state machines. Don't set colours or `classDef` styles; the theme handles it. Quote labels that contain punctuation: `A["cloudg map --org"]`. + +### Callouts + +```markdown +:::note +The map doesn't depend on the scanners. +::: + +:::warning Admin rights +AWSControlTowerExecution has admin rights. +::: +``` + +Kinds: `note`, `tip`, `warning`, `danger`. Text after the kind replaces the default label. + +### Steps + +Numbered steps, one `###` per step. Use four colons on the outer fence when a callout sits inside. + +```markdown +::::steps +### Install +... +:::tip +... +::: +### Authenticate +... +:::: +``` + +### Numbered cells + +Two to four short items side by side, one `###` per cell. + +```markdown +:::cells +### Deep collectors +ECS, EKS, Lambda triggers... +### Breadth sweep +On AWS, a Cloud Control sweep... +::: +``` + +### Link cards + +A list of links rendered as a grid of cards. The text after the link is the description. + +```markdown +:::links +- [Inventory mapping](/guides/inventory-mapping/) What exists and how it is wired. +- [cloudg deps](/cli/deps/) Blast radius for one asset. +::: +``` + +## Page kinds + +### Guides: `site/content/guides/.md` + +Front matter plus a body. The body starts with prose or an `##`, never with an `#` (the title comes from front matter). + +### CLI commands: `site/content/cli/.md` + +The generator writes the usage line and the options table from the click command. Your file adds: + +```yaml +--- +command: map # or "mcp serve" +lede: Builds the scanner-independent inventory map. +intro: | # optional markdown shown above the usage line + ... +--- +``` + +The body follows the options table. Typical sections: `## Examples`, `## Output files`, `## Exit codes`, `## Notes`. Pages without a `command` key (the shared flag pages) are plain pages. + +### Python API: `site/content/api/.md` + +```yaml +--- +object: cloudg.api.CloudGEngine # import path; omit for a plain page +members: [run_pipeline, run_pipeline_sync, collect, map_inventory] # optional order/filter +lede: Runs the whole pipeline from Python, or any one phase of it. +--- +``` + +The generator writes the kind tag, source link, signature and a ruled row per member (signature, docstring, parameter table). Your body follows the members. Typical sections: `## Examples`, `## Notes`. diff --git a/site/build.py b/site/build.py new file mode 100644 index 0000000..17e33e4 --- /dev/null +++ b/site/build.py @@ -0,0 +1,940 @@ +#!/usr/bin/env python3 +"""Build the cloudg documentation site into site/dist. + + python site/build.py # build + python site/build.py --serve # build, then serve on http://127.0.0.1:8000/cloudg/ + python site/build.py --strict # fail on broken internal links (used in CI) + python site/build.py --base / # build for serving at the domain root + +See site/AUTHORING.md for the page format. +""" + +from __future__ import annotations + +import argparse +import functools +import http.server +import json +import os +import posixpath +import re +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from html import escape +from pathlib import Path +from urllib.parse import urlsplit + +SITE = Path(__file__).resolve().parent +REPO = SITE.parent +sys.path.insert(0, str(SITE)) +sys.path.insert(0, str(REPO)) + +import yaml # noqa: E402 +from jinja2 import Environment, FileSystemLoader, select_autoescape # noqa: E402 +from markupsafe import Markup # noqa: E402 +from markdown_it import MarkdownIt # noqa: E402 + +from docsgen import gen_api, gen_changelog, gen_cli, gen_config # noqa: E402 +from docsgen.highlight import render_code_block # noqa: E402 +from docsgen.icons import icon # noqa: E402 +from docsgen.md import Heading, Renderer, github_slug, render_inline, split_front_matter, strip_tags # noqa: E402 + +GITHUB = "https://github.com/morpheuslord/cloudg" +PYPI = "https://pypi.org/project/cloudg/" +SPLIT_H3_BYTES = 40_000 +_FENCE = re.compile(r"^\s{0,3}(`{3,}|~{3,})(.*)$") +_NUM_PREFIX = re.compile(r"^\d+(\.\d+)*\.?\s+") + + +# --------------------------------------------------------------------------- +# Page model +# --------------------------------------------------------------------------- + + +@dataclass +class Page: + url: str # "guides/quickstart"; "" for home + title: str + tab: str = "" + group: str = "" + kind: str = "guide" + search_type: str = "Guides" + source: str = "" # repo-relative file the content comes from + source_line: int = 0 + nav_title: str = "" + in_sidebar: bool = True + level: int = 1 + mono_title: bool = False + # filled in by the renderers + body: str = "" + headings: list = field(default_factory=list) + lede: str = "" + meta: list = field(default_factory=list) + since: str = "" + crumbs: list = field(default_factory=list) + extra: dict = field(default_factory=dict) + toc_label: str = "On this page" + # inputs for deferred rendering + loader: object = None + prev: "Page | None" = None + next: "Page | None" = None + description: str = "" + + +class Site: + def __init__(self, base: str, strict: bool): + self.base = "/" + base.strip("/") + "/" if base.strip("/") else "/" + self.strict = strict + self.nav = yaml.safe_load((SITE / "data" / "nav.yaml").read_text()) + self.pages: list[Page] = [] + self.by_url: dict[str, Page] = {} + self.tabs: list[dict] = [] + self.sidebars: dict[str, list[dict]] = {} + self.anchor_map: dict[tuple[str, str], str] = {} # (doc basename, anchor) -> url#anchor + self.doc_first_page: dict[str, str] = {} + self.broken: list[tuple[str, str]] = [] + self.version = self._version() + self.git_ref = f"v{self.version}" if self._tag_exists(f"v{self.version}") else "main" + self.env = Environment( + loader=FileSystemLoader(str(SITE / "templates")), + autoescape=select_autoescape(["html"]), + trim_blocks=True, + lstrip_blocks=True, + ) + self.env.globals.update(icon=icon, site=self) + self.env.filters["wbr"] = lambda text, sep: Markup( + str(escape(text)).replace(str(escape(sep)), str(escape(sep)) + "") + ) + self._plain_md = MarkdownIt("commonmark") + + # -- helpers --------------------------------------------------------------- + + @staticmethod + def _version() -> str: + m = re.search(r'^version\s*=\s*"([^"]+)"', (REPO / "pyproject.toml").read_text(), re.M) + return m.group(1) if m else "0.0.0" + + @staticmethod + def _tag_exists(tag: str) -> bool: + try: + out = subprocess.run(["git", "-C", str(REPO), "tag", "--list", tag], capture_output=True, text=True) + return out.stdout.strip() == tag + except OSError: + return False + + def href(self, url: str, anchor: str = "") -> str: + path = self.base + (url.strip("/") + "/" if url.strip("/") else "") + return path + (f"#{anchor}" if anchor else "") + + def asset(self, path: str) -> str: + return self.base + path.lstrip("/") + + def blob(self, path: str, line: int = 0, ref: str | None = None) -> str: + url = f"{GITHUB}/blob/{ref or 'main'}/{path}" + return url + (f"#L{line}" if line else "") + + def edit_url(self, page: Page) -> str: + if not page.source: + return "" + if page.source_line: + return self.blob(page.source, page.source_line, self.git_ref) + return f"{GITHUB}/edit/main/{page.source}" + + def add(self, page: Page) -> Page: + if page.url in self.by_url: + raise SystemExit(f"duplicate page url: {page.url}") + self.pages.append(page) + self.by_url[page.url] = page + return page + + def plain(self, md_inline: str) -> str: + return strip_tags(self._plain_md.renderInline(md_inline)).strip() + + # -- link resolution ------------------------------------------------------- + + def resolver(self, page: Page, doc: str | None = None): + """Return a function that rewrites hrefs found on ``page``.""" + + def resolve(href: str) -> str: + if not href or href.startswith(("http://", "https://", "mailto:", "data:")): + return href + if href.startswith("//"): + return href + if href.startswith("/"): + path, _, anchor = href.partition("#") + url = path.strip("/") + self._check(page, url, anchor) + return self.href(url, anchor) + if href.startswith("#"): + if doc: + target = self.anchor_map.get((doc, href[1:])) + if target: + url, _, anchor = target.partition("#") + if url == page.url: + return "#" + anchor if anchor else "#" + return self.href(url, anchor) + return href + # relative file links from the docs/*.md sources + path, _, anchor = href.partition("#") + base_dir = posixpath.dirname(page.source) if page.source else "docs" + target = posixpath.normpath(posixpath.join(base_dir, path)) + name = posixpath.basename(target) + if name in self.doc_first_page and target.startswith("docs/"): + if anchor and (name, anchor) in self.anchor_map: + url, _, a = self.anchor_map[(name, anchor)].partition("#") + return self.href(url, a) + return self.href(self.doc_first_page[name], "") + if target == "CHANGELOG.md": + return self.href("changelog", anchor) + if target == "config.yaml": + return self.href("reference/config/providers") + if target.startswith(".."): + return href + return self.blob(target) + (f"#{anchor}" if anchor else "") + + return resolve + + def _check(self, page: Page, url: str, anchor: str) -> None: + self.broken_candidates.append((page.url, url, anchor)) + + broken_candidates: list = [] + + # -- page discovery ---------------------------------------------------------- + + def discover(self) -> None: + self.broken_candidates = [] + home = self.add(Page("", "cloudg documentation", kind="home", search_type="Guides", in_sidebar=False)) + home.source = "site/templates/home.html" + for tab in self.nav["tabs"]: + self.tabs.append({"id": tab["id"], "label": tab["label"], "root": tab["root"]}) + groups = [] + for gi, group in enumerate(tab.get("groups", []), start=1): + items = [] + if "split" in group: + items = self._discover_split(tab["id"], group) + elif group.get("generated") == "config": + items = self._discover_config(tab["id"], group) + else: + for item in group["items"]: + items.append(self._discover_item(tab["id"], group["title"], item)) + groups.append({"title": group["title"], "num": f"{gi:02d}", "pages": items}) + self.sidebars[tab["id"]] = groups + self.add( + Page("changelog", "Changelog", tab="changelog", kind="changelog", search_type="Changelog", + source="CHANGELOG.md", in_sidebar=False) + ) + + def _discover_item(self, tab: str, group: str, item: dict) -> Page: + title = item["title"] + if "generated" in item: + page = Page(item["path"], title, tab=tab, group=group, kind=item["generated"]) + page.search_type = {"cli-index": "CLI", "api-index": "Python API"}.get(item["generated"], "Guides") + return self.add(page) + rel = item["page"] + url = item.get("path", rel) + file = SITE / "content" / f"{rel}.md" + kind = {"cli": "cli", "api": "api", "mcp": "guide", "reference": "guide"}.get(tab, "guide") + search_type = {"guides": "Guides", "cli": "CLI", "api": "Python API", "mcp": "MCP", "reference": "Reference"}[tab] + page = Page(url, title, tab=tab, group=group, kind=kind, search_type=search_type, + source=str(file.relative_to(REPO)), nav_title=title) + page.loader = file + return self.add(page) + + def _discover_split(self, tab: str, group: dict) -> list[Page]: + doc_path = REPO / group["split"] + name = doc_path.name + prefix = group["prefix"] + skip = set(group.get("skip", [])) + text = doc_path.read_text() + lines = text.split("\n") + heads = _scan_headings(lines) + h1 = next((h for h in heads if h[0] == 1), None) + h2s = [h for h in heads if h[0] == 2] + search_type = "MCP" if tab == "mcp" else "Reference" + pages: list[Page] = [] + + # Overview page from the text above the first H2 + first_h2 = h2s[0][2] if h2s else len(lines) + intro_start = h1[2] + 1 if h1 else 0 + intro = "\n".join(lines[intro_start:first_h2]).strip() + overview = Page(prefix, self.plain(h1[1]) if h1 else group["title"], tab=tab, group=group["title"], + kind="split-index", search_type=search_type, source=group["split"], nav_title="Overview") + overview.extra = {"doc": name, "md": intro, "children": [], "pre_slugs": []} + pages.append(self.add(overview)) + self.doc_first_page[name] = prefix + + slug_counts: dict[str, int] = {} + pending: list[str] = [] # slugs consumed by titles and skipped sections before the next page renders + + def gh_unique(slug: str) -> str: + if slug in slug_counts: + slug_counts[slug] += 1 + return f"{slug}-{slug_counts[slug]}" + slug_counts[slug] = 0 + return slug + + def consume(slug: str) -> str: + s = gh_unique(slug) + pending.append(slug) + return s + + shared: dict[str, int] = {} + overview.extra["shared_slugs"] = shared + + # anchors for headings above the first H2 belong to the overview + for lvl, txt, idx in heads: + if idx >= first_h2: + break + self.anchor_map[(name, gh_unique(github_slug(self.plain(txt))))] = prefix + + bounds = [(h, h2s[i + 1][2] if i + 1 < len(h2s) else len(lines)) for i, h in enumerate(h2s)] + for (lvl, txt, start), end in bounds: + title = self.plain(txt) + sec_heads = [h for h in heads if start <= h[2] < end] + if title in skip: + for h in sec_heads: + consume(github_slug(self.plain(h[1]))) + continue + slug = page_slug(title) + url = f"{prefix}/{slug}" + body_lines = lines[start + 1 : end] + size = sum(len(x) + 1 for x in body_lines) + h3s = [h for h in sec_heads if h[0] == 3] + page = Page(url, _NUM_PREFIX.sub("", title), tab=tab, group=group["title"], kind="split", + search_type=search_type, source=group["split"], source_line=0) + number = _NUM_PREFIX.match(title) + page.extra = {"doc": name, "number": number.group(0).strip() if number else "", "shared_slugs": shared} + self.add(page) + pages.append(page) + overview.extra["children"].append(page) + title_anchor = consume(github_slug(title)) + page.extra["title_anchor"] = title_anchor + page.extra["pre_slugs"] = list(pending) + pending.clear() + self.anchor_map[(name, title_anchor)] = url + if size > SPLIT_H3_BYTES and len(h3s) >= 3: + first_h3 = h3s[0][2] + page.extra.update(md="\n".join(lines[start + 1 : first_h3]), children=[], heading_base=-1) + for h in sec_heads: + if h[2] < first_h3 and h[0] > 2: + self.anchor_map[(name, gh_unique(github_slug(self.plain(h[1]))))] = url + sub_bounds = [(h, h3s[i + 1][2] if i + 1 < len(h3s) else end) for i, h in enumerate(h3s)] + for (_, stxt, sstart), send in sub_bounds: + stitle = self.plain(stxt) + sslug = page_slug(stitle) + surl = f"{url}/{sslug}" + n = 2 + while surl in self.by_url: + surl = f"{url}/{sslug}-{n}" + n += 1 + sub = Page(surl, stitle, tab=tab, group=group["title"], kind="split", search_type=search_type, + source=group["split"], in_sidebar=False, level=2) + sub.extra = {"doc": name, "md": "\n".join(lines[sstart + 1 : send]), "heading_base": -2, + "parent": page, "shared_slugs": shared} + self.add(sub) + pages.append(sub) + page.extra["children"].append(sub) + sub_anchor = consume(github_slug(stitle)) + sub.extra["title_anchor"] = sub_anchor + sub.extra["pre_slugs"] = list(pending) + pending.clear() + self.anchor_map[(name, sub_anchor)] = surl + for h in heads: + if sstart < h[2] < send: + self.anchor_map[(name, gh_unique(github_slug(self.plain(h[1]))))] = f"{surl}" + else: + page.extra.update(md="\n".join(body_lines), heading_base=-1) + for h in sec_heads[1:]: + self.anchor_map[(name, gh_unique(github_slug(self.plain(h[1]))))] = url + # store anchors with a fragment so links land on the heading itself + for key, url in list(self.anchor_map.items()): + if key[0] == name and "#" not in url: + self.anchor_map[key] = f"{url}#{key[1]}" + return pages + + def _discover_config(self, tab: str, group: dict) -> list[Page]: + sections = gen_config.parse(REPO / "config.yaml") + _merge_model_descriptions(sections) + overrides = yaml.safe_load((SITE / "data" / "config-notes.yaml").read_text()) or {} + pages = [] + for s in sections: + for k in s.keys: + note = overrides.get(f"{s.name}.{k.path}") + if note: + k.description = note + url = f"{group['prefix']}/{s.name}" + page = Page(url, s.name, tab=tab, group=group["title"], kind="config", search_type="Config", + source="config.yaml", mono_title=True) + page.extra = {"section": s, "intro": overrides.get(s.name, "")} + pages.append(self.add(page)) + return pages + + # -- rendering ------------------------------------------------------------- + + def render_all(self) -> None: + cli = gen_cli.collect(REPO) + self.cli = cli + for page in self.pages: + page.crumbs = self._crumbs(page) + kind = page.kind + if kind == "home": + continue + if kind in ("guide",): + self._render_markdown_page(page) + elif kind == "cli": + self._render_cli_page(page, cli) + elif kind == "api": + self._render_api_page(page) + elif kind in ("split", "split-index"): + self._render_split_page(page) + elif kind == "config": + self._render_config_page(page) + elif kind == "changelog": + self._render_changelog(page) + elif kind == "cli-index": + self._render_cli_index(page, cli) + elif kind == "api-index": + self._render_api_index(page) + self._link_prev_next() + + def _crumbs(self, page: Page) -> list[str]: + tab = next((t["label"] for t in self.tabs if t["id"] == page.tab), "") + out = [tab] if tab else [] + if page.group and page.group != tab: + out.append(page.group) + parent = page.extra.get("parent") if page.extra else None + if parent: + out.append(parent.title) + return out + + def _load_md(self, page: Page) -> tuple[dict, str]: + file: Path = page.loader # type: ignore[assignment] + if not file.exists(): + print(f"warning: missing content file {file.relative_to(REPO)}", file=sys.stderr) + return {"title": page.title, "lede": "This page hasn't been written yet."}, "" + return split_front_matter(file.read_text()) + + def _apply_front(self, page: Page, meta: dict) -> None: + res = self.resolver(page) + if meta.get("title"): + page.title = meta["title"] + if meta.get("lede"): + page.lede = render_inline(str(meta["lede"]), res) + page.description = strip_tags(page.lede) + if meta.get("description"): + page.description = meta["description"] + page.meta = [(label, render_inline(str(value), res)) for label, value in meta.get("meta", [])] + page.since = str(meta.get("since", "")) or "" + if meta.get("source"): + page.source = meta["source"] + + def _render_markdown_page(self, page: Page) -> None: + meta, body = self._load_md(page) + self._apply_front(page, meta) + r = Renderer(self.resolver(page)).render(body) + page.body, page.headings = r.html, r.headings + + def _render_split_page(self, page: Page) -> None: + doc = page.extra["doc"] + md = page.extra.get("md", "") + renderer = Renderer(self.resolver(page, doc), heading_base=page.extra.get("heading_base", -1), + slugs=page.extra.get("shared_slugs")) + for slug in page.extra.get("pre_slugs", []): + renderer.register(slug) + r = renderer.render(md) + body = r.html + children = page.extra.get("children") or [] + if children: + cards = "".join( + f'' + f'{escape((c.extra.get("number", "") + " " if c.extra.get("number") else "") + c.title)}' + f'{icon("arrow-right", 16)}{escape(c.description)}' + for c in children + ) + label = "Sections" if page.kind == "split-index" else "In this section" + body += f'

{label}

' + r.headings.append(type(r.headings[0])(2, "sections", label, label) if r.headings else None) + r.headings = [h for h in r.headings if h is not None] + if not r.headings: + r.headings = [Heading(2, "sections", label, label)] + page.body, page.headings = body, r.headings + if not page.description: + first = re.search(r"

(.*?)

", r.html, re.S) + if first: + page.description = _truncate(strip_tags(first.group(1)), 160) + if page.kind == "split" and page.extra.get("number"): + page.extra["kicker"] = f"Section {page.extra['number'].rstrip('.')}" + + def _render_cli_page(self, page: Page, cli: dict) -> None: + meta, body = self._load_md(page) + self._apply_front(page, meta) + page.mono_title = bool(meta.get("command")) + res = self.resolver(page) + r = Renderer(res) + parts = [] + command = meta.get("command") + if command: + cmd = cli.get(command) + if cmd is None: + raise SystemExit(f"{page.source}: unknown command {command!r}") + page.title = f"cloudg {command}" + if not page.lede: + page.lede = escape(cmd.short_help) + if cmd.source_file: + page.source, page.source_line = cmd.source_file, cmd.source_line + page.meta = page.meta or [] + if meta.get("intro"): + parts.append(r.render(meta["intro"]).html) + r.headings.append(Heading(2, "usage", "Usage", "Usage")) + parts.append( + f'
{escape(cmd.usage)}
' + ) + r.headings.append(Heading(2, "options", "Options", "Options")) + parts.append(self.env.get_template("_options.html").render(cmd=cmd, page=page)) + page.extra["command"] = cmd + parts.append(r.render(body).html) + page.body, page.headings = "".join(parts), r.headings + + def _render_cli_index(self, page: Page, cli: dict) -> None: + page.title = "CLI reference" + page.lede = "Every cloudg command and the flags it takes. The tables are generated from the click definitions, so they match --help." + page.source = "cloudg/cli.py" + rows = [] + for group in self.sidebars["cli"][1:]: + for p in group["pages"]: + cmd_name = p.title.replace("cloudg ", "", 1) if p.title.startswith("cloudg ") else None + cmd = cli.get(cmd_name) if cmd_name else None + rows.append((group["title"], p, cmd)) + page.extra = {"rows": rows, "root": cli["__root__"]} + page.headings = [Heading(2, "commands", "Commands", "Commands"), Heading(2, "global-options", "Global options", "Global options")] + page.body = self.env.get_template("_cli_index.html").render(page=page) + page.mono_title = False + + def _render_api_page(self, page: Page) -> None: + meta, body = self._load_md(page) + self._apply_front(page, meta) + res = self.resolver(page) + r = Renderer(res) + objects = meta.get("object") + parts = [] + if objects: + paths = objects if isinstance(objects, list) else [objects] + members = meta.get("members") + loaded = [] + for p in paths: + m = members.get(p) if isinstance(members, dict) else (members if len(paths) == 1 else None) + try: + loaded.append(gen_api.load(p, REPO, m)) + except Exception as exc: # noqa: BLE001 + raise SystemExit(f"{page.source}: cannot load {p}: {exc}") from exc + first = loaded[0] + page.mono_title = len(loaded) == 1 and not meta.get("title") + if page.mono_title: + page.title = first.name + if len(loaded) == 1: + page.extra.update(kind_tag=first.kind, module=first.module, src=first) + page.source, page.source_line = first.source_file, first.source_line + if not page.lede and first.doc.summary: + page.lede = render_inline(first.doc.summary, res) + multi = len(loaded) > 1 + for obj in loaded: + if multi: + r.headings.append(Heading(2, obj.name.lower(), obj.name, obj.name)) + else: + members_title = "Methods" if first.kind == "class" else "Members" + for title, hid in (("Fields", "fields"), ("Values", "values"), (members_title, "members")): + if (title == "Fields" and obj.fields) or (title == "Values" and obj.enum_values) or ( + title == members_title and obj.members + ): + r.headings.append(Heading(2, hid, title, title)) + for mem in obj.members: + r.headings.append(Heading(3, f"{obj.name}.{mem.name}", mem.name, mem.name)) + parts.append( + self.env.get_template("_api_object.html").render( + obj=obj, multi=multi, page=page, md=lambda t: Renderer(res).render(t).html, + inline=lambda t: render_inline(t, res), code=render_code_block, + blob=lambda line: self.blob(obj.source_file, line, self.git_ref), + ) + ) + rb = r.render(body) + parts.append(rb.html) + page.body, page.headings = "".join(parts), r.headings + + def _render_api_index(self, page: Page) -> None: + page.title = "Python API" + page.lede = ( + "cloudg is a library first. The CLI is a thin layer over CloudGEngine, so anything a command" + " does you can do from Python, one phase at a time if you like." + ) + page.source = "cloudg/__init__.py" + groups = [] + for group in self.sidebars["api"][1:]: + items = [] + for p in group["pages"]: + items.append(p) + groups.append((group["title"], items)) + page.extra = {"groups": groups} + md_file = SITE / "content" / "api" / "index.md" + intro_html = "" + headings = [] + if md_file.exists(): + meta, body = split_front_matter(md_file.read_text()) + if meta.get("lede"): + page.lede = render_inline(meta["lede"], self.resolver(page)) + rr = Renderer(self.resolver(page)).render(body) + intro_html, headings = rr.html, rr.headings + page.headings = headings + [Heading(2, "reference", "Reference", "Reference")] + page.body = intro_html + self.env.get_template("_api_index.html").render(page=page) + + def _render_config_page(self, page: Page) -> None: + s = page.extra["section"] + res = self.resolver(page) + page.title = s.name + intro = page.extra.get("intro") or s.intro + rest = "" + if len(intro) > 240: + first, sep, rest = intro.partition(". ") + intro = first + ("." if sep else "") + page.lede = render_inline(intro, res) if intro else "" + page.description = strip_tags(page.lede) + page.toc_label = "Sections" + page.headings = [Heading(2, k.path, k.path, k.path) for k in s.keys if "." not in k.path] + yaml_lines = _highlight_yaml_lines(s.lines) + page.body = (f"

{render_inline(rest, res)}

" if rest else "") + self.env.get_template("_config.html").render( + s=s, page=page, yaml_lines=yaml_lines, inline=lambda t: render_inline(t, res) + ) + page.source_line = _line_of(REPO / "config.yaml", f"{s.name}:") if s.name != "general" else 0 + if page.source_line: + page.source = "config.yaml" + + def _render_changelog(self, page: Page) -> None: + head, releases = gen_changelog.parse(REPO / "CHANGELOG.md") + res = self.resolver(page) + page.lede = render_inline(head, res) if head else "" + rendered = [] + for rel in releases: + rr = Renderer(res, heading_base=1) + intro = rr.render(rel.intro).html if rel.intro else "" + blocks = [(label, Renderer(res, heading_base=1).render(text).html) for label, text in rel.blocks] + rendered.append({"rel": rel, "intro": intro, "blocks": blocks}) + page.extra = {"releases": rendered} + self.releases = releases + page.headings = [Heading(2, r.anchor, r.version, r.version) for r in releases] + + def _link_prev_next(self) -> None: + for tab, groups in self.sidebars.items(): + seq: list[Page] = [] + for g in groups: + for p in g["pages"]: + seq.append(p) + for i, p in enumerate(seq): + p.prev = seq[i - 1] if i > 0 else None + p.next = seq[i + 1] if i + 1 < len(seq) else None + + # -- output ---------------------------------------------------------------- + + def write(self, out: Path) -> None: + if out.exists(): + shutil.rmtree(out) + out.mkdir(parents=True) + shutil.copytree(SITE / "static", out / "static") + for extra in ("viewer.html",): + src = REPO / "docs" / extra + if src.exists(): + shutil.copy2(src, out / extra) + if (REPO / "docs" / "assets").is_dir(): + shutil.copytree(REPO / "docs" / "assets", out / "assets") + stats = self._stats() + redirects = {k: self.href(v.split("#")[0], v.partition("#")[2]) for k, v in self.nav["redirects"].items()} + versions = self._versions() + (out / "versions.json").write_text(json.dumps(versions, indent=2)) + for page in self.pages: + tpl = { + "home": "home.html", + "changelog": "changelog.html", + }.get(page.kind, "page.html") + html = self.env.get_template(tpl).render( + page=page, stats=stats, redirects=redirects, versions=versions, + sidebar=self.sidebars.get(page.tab, []), tabs=self.tabs, + ) + dest = out / page.url / "index.html" if page.url else out / "index.html" + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(html) + nf = Page("404", "Page not found", kind="404", in_sidebar=False) + (out / "404.html").write_text( + self.env.get_template("404.html").render(page=nf, redirects=redirects, versions=versions, tabs=self.tabs, + sidebar=[]) + ) + (out / "search-index.json").write_text(json.dumps(self._search_index(), separators=(",", ":"))) + (out / ".nojekyll").write_text("") + + def _stats(self) -> dict: + import glob + + controls = 0 + for f in glob.glob(str(REPO / "cloudg" / "rules" / "frameworks" / "*.yaml")): + data = yaml.safe_load(Path(f).read_text()) or {} + controls += len(data.get("controls") or []) + try: + from cloudg.schema.models import AssetType + + asset_types = len(AssetType) + except Exception: # noqa: BLE001 + asset_types = 0 + tools = 0 + try: + from cloudg.mcp import default_registry + + tools = len(default_registry().tools()) + except Exception: # noqa: BLE001 + pass + data = yaml.safe_load((SITE / "data" / "stats.yaml").read_text()) + computed = {"controls": controls, "asset_types": asset_types, "mcp_tools": tools} + out = [] + for item in data["stats"]: + value = computed.get(item.get("compute", ""), 0) or item.get("value", 0) + out.append({"value": f"{value:,}", "label": item["label"], "href": item.get("href", "")}) + return {"items": out, "commands": len([k for k, v in self.cli.items() if not v.group and k != "__root__"])} + + def _versions(self) -> list[dict]: + out = [] + releases = getattr(self, "releases", []) + for r in releases: + if not re.match(r"^\d", r.version): + continue + latest = r.version == self.version + out.append({ + "version": r.version, + "date": r.date, + "latest": latest, + "url": self.href("") if latest else self.href("changelog", r.anchor), + }) + out.append({"version": "main", "date": "unreleased", "latest": False, "url": f"{GITHUB}/tree/main"}) + return out + + def _search_index(self) -> list[dict]: + entries = [] + for page in self.pages: + if page.kind in ("home",): + continue + chunks = _sections(page.body, page.headings) + title = page.title + entries.append({ + "t": title, "u": self.href(page.url), "k": page.search_type, "h": "", + "x": _truncate(strip_tags(page.lede) + " " + chunks[0][1] if chunks else strip_tags(page.lede), 600), + "g": page.group, "m": int(page.mono_title), + }) + for hid, text, htext in chunks[1:]: + entries.append({ + "t": title, "u": self.href(page.url, hid), "k": page.search_type, "h": htext, + "x": _truncate(text, 600), "g": page.group, + "m": int(page.mono_title or page.kind == "config"), + }) + if page.kind == "config": + s = page.extra["section"] + for k in s.keys: + entries.append({ + "t": f"{s.name}.{k.path}", "u": self.href(page.url, k.path), "k": "Config", "h": "", + "x": f"{k.type} · {k.default or 'none'} · {k.description}", "g": page.group, "m": 1, + }) + return entries + + def check_links(self) -> list[str]: + problems = [] + ids_cache: dict[str, set[str]] = {} + for src, url, anchor in self.broken_candidates: + if url not in self.by_url: + problems.append(f"{src or 'home'}: link to missing page /{url}/") + continue + if anchor: + if url not in ids_cache: + ids_cache[url] = set(re.findall(r'id="([^"]+)"', self.by_url[url].body)) + ids_cache[url].update(h.id for h in self.by_url[url].headings) + if anchor not in ids_cache[url]: + problems.append(f"{src or 'home'}: link to missing anchor /{url}/#{anchor}") + return sorted(set(problems)) + + +# --------------------------------------------------------------------------- +# small utilities +# --------------------------------------------------------------------------- + + +def _scan_headings(lines: list[str]) -> list[tuple[int, str, int]]: + out = [] + fence = None + for i, line in enumerate(lines): + fm = _FENCE.match(line) + if fence is None and fm: + fence = fm.group(1) + continue + if fence is not None: + if fm and fm.group(1)[0] == fence[0] and len(fm.group(1)) >= len(fence) and not fm.group(2).strip(): + fence = None + continue + m = re.match(r"^(#{1,6})\s+(.*?)\s*#*\s*$", line) + if m: + out.append((len(m.group(1)), m.group(2), i)) + return out + + +def page_slug(title: str) -> str: + """URL segment for a split page: number prefix dropped, punctuation turned into hyphens.""" + title = _NUM_PREFIX.sub("", title).lower() + return re.sub(r"[^a-z0-9]+", "-", title).strip("-") or "section" + + +def _truncate(text: str, n: int) -> str: + text = " ".join(text.split()) + return text if len(text) <= n else text[: n - 1].rsplit(" ", 1)[0] + "…" + + +def _sections(html: str, headings: list) -> list[tuple[str, str, str]]: + """Split rendered HTML at h2/h3 ids into (id, text, heading) chunks.""" + ids = {h.id: h.text for h in headings if h.level in (2, 3)} + html = re.sub(r']*>#', "", html) + parts = re.split(r'(", " ", content, flags=re.S)) + out.append((hid, text, ids[hid])) + else: + prev = out[-1] + out[-1] = (prev[0], prev[1] + " " + strip_tags(content), prev[2]) + i += 3 + return out + + +def _line_of(path: Path, needle: str) -> int: + for i, line in enumerate(path.read_text().split("\n"), start=1): + if line.startswith(needle): + return i + return 0 + + +def _highlight_yaml_lines(lines: list[str]) -> list[str]: + from docsgen.highlight import tokenize_lines + + toks = tokenize_lines("\n".join(lines) + "\n", "yaml") + toks += [""] * (len(lines) - len(toks)) + return toks + + +def _merge_model_descriptions(sections) -> None: + """Prefer pydantic Field descriptions from CloudGConfig; add keys the example file omits.""" + try: + from cloudg.config import CloudGConfig + except Exception: # noqa: BLE001 + return + + def model_of(annotation): + if hasattr(annotation, "model_fields"): + return annotation + for arg in getattr(annotation, "__args__", ()) or (): + if hasattr(arg, "model_fields"): + return arg + return None + + for s in sections: + top = CloudGConfig.model_fields.get(s.name) + model = model_of(top.annotation) if top else None + if s.name == "general": + model = CloudGConfig + if model is None: + continue + for k in s.keys: + cur = model + finfo = None + for part in k.path.split("."): + if cur is None or part not in cur.model_fields: + finfo = None + break + finfo = cur.model_fields[part] + cur = model_of(finfo.annotation) + if finfo is not None and finfo.description: + desc = finfo.description.strip() + if not k.description or len(desc) > len(k.description): + k.description = desc + known = {k.path for k in s.keys} + for fname, finfo in model.model_fields.items(): + if s.name == "general" and model_of(finfo.annotation) is not None: + continue + if s.name == "general" and fname in ("providers",): + continue + if fname not in known and model_of(finfo.annotation) is None and not any(p.startswith(fname + ".") for p in known): + if s.name == "general": + continue + default = "" if finfo.default_factory else repr(finfo.default) + if default in ("None",): + default = "null" + s.keys.append(gen_config.Key(fname, gen_api._ann(finfo.annotation), default, + (finfo.description or "") + " Not in the example file.", -1)) + + +# --------------------------------------------------------------------------- +# entry point +# --------------------------------------------------------------------------- + + +def serve(out: Path, base: str, port: int) -> None: + class Handler(http.server.SimpleHTTPRequestHandler): + def end_headers(self): + self.send_header("Cache-Control", "no-store") + super().end_headers() + + def translate_path(self, path): # noqa: D401 + parsed = urlsplit(path).path + if base != "/" and parsed.startswith(base): + parsed = "/" + parsed[len(base):] + return str(out) + parsed + + def send_error(self, code, message=None, explain=None): + if code == 404 and (out / "404.html").exists(): + body = (out / "404.html").read_bytes() + self.send_response(404) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return + super().send_error(code, message, explain) + + handler = functools.partial(Handler, directory=str(out)) + with http.server.ThreadingHTTPServer(("127.0.0.1", port), handler) as httpd: + print(f"serving http://127.0.0.1:{port}{base}") + httpd.serve_forever() + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--out", default=str(SITE / "dist")) + ap.add_argument("--base", default=os.environ.get("DOCS_BASE", "/cloudg/")) + ap.add_argument("--strict", action="store_true", help="exit non-zero on broken internal links") + ap.add_argument("--serve", action="store_true") + ap.add_argument("--port", type=int, default=8000) + args = ap.parse_args() + + site = Site(args.base, args.strict) + site.discover() + site.render_all() + out = Path(args.out) + site.write(out) + problems = site.check_links() + for p in problems: + print(f"link: {p}", file=sys.stderr) + print(f"built {len(site.pages)} pages into {out} (base {site.base})") + if problems and args.strict: + return 1 + if args.serve: + serve(out, site.base, args.port) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/site/content/api/cloudgconfig.md b/site/content/api/cloudgconfig.md new file mode 100644 index 0000000..985ac80 --- /dev/null +++ b/site/content/api/cloudgconfig.md @@ -0,0 +1,222 @@ +--- +object: [cloudg.config.CloudGConfig, cloudg.config.load_config] +members: + cloudg.config.CloudGConfig: [] +lede: "`CloudGConfig` is the pydantic model behind `config.yaml`. Build it in code, load it from a file, or do both and override a few fields." +--- + +## Two ways to get a config + +`CloudGConfig()` with no arguments is a working configuration: AWS only, `us-east-1`, all five scanners enabled, ontology and RAG export on, Terraform export off. Every field has a default, so you only set what differs. + +In code, pass keyword arguments or assign to the nested models after construction: + +```python +from cloudg import CloudGConfig + +config = CloudGConfig(providers=["aws"]) +config.aws.regions = ["eu-west-1", "eu-central-1"] +config.scanners.enabled = ["prowler", "iam"] +``` + +From a file, `load_config(path)` reads the YAML with `yaml.safe_load` and hands the dict to `CloudGConfig.model_validate()`. The file has the same shape as the model: top-level keys are the attributes below, nested keys are the fields of each section. The generated [config.yaml reference](/reference/config/providers/) documents every key with its default. + +```mermaid caption="What load_config does with the path it gets" +flowchart TD + P["load_config(path)"] --> N{"path is None?"} + N -->|yes| D["CloudGConfig() defaults"] + N -->|no| E{"file exists?"} + E -->|no| W["log a warning"] --> D + E -->|yes| Y["yaml.safe_load"] + Y --> V["CloudGConfig.model_validate"] + V --> U["warn about unknown ratelimit keys"] + U --> C["CloudGConfig"] + V -->|invalid value| X["pydantic ValidationError"] +``` + +Two branches in that diagram return defaults instead of failing: a `None` path and a path that does not exist. A typo in the file name therefore gives you a run against `us-east-1` with every scanner enabled, and the only sign is a `WARNING cloudg.config: Config file ... not found, using defaults` line. If that matters, check `Path(path).is_file()` before calling, or use the strict loader in the examples. An empty file also gives defaults. The CLI calls the same function with the `-c` value and does not look for a `config.yaml` in the working directory on its own. + +### The nested sections + +Each section is its own pydantic model in `cloudg.config`. You can import them to build a section in one go, which also validates it at construction time. + +| Attribute | Model | What it controls | +|---|---|---| +| `aws` | `AWSConfig` | regions, accounts, credentials (keys, profile, role, OIDC token file), botocore retries | +| `aws.organization` | `AWSOrganizationConfig` | AWS Organizations and Control Tower discovery for `cloudg map` | +| `azure` | `AzureConfig` | subscriptions, tenant, service principal or managed identity, locations, management groups | +| `gcp` | `GCPConfig` | projects or organization, credentials file, impersonation, Cloud Asset Inventory scope and skipped types | +| `scanners` | `ScannerConfig` | which scanners run, their extra arguments, Checkov frameworks, Trivy images, IaC directories | +| `inventory` | `InventoryConfig` | the deep inventory collectors: sweeps, service families, Kubernetes, IAM resource edges | +| `graph` | `GraphConfig` | graph engine options | +| `ontology` | `OntologyConfig` | ontology on or off, export formats | +| `rag` | `RAGConfig` | RAG export on or off, chunk size | +| `terraform` | `TerraformConfig` | Terraform recreation on or off, output directory | +| `report` | `ReportConfig` | report formats and directory used by the CLI | +| `rulesets` | `RulesetConfig` | the YAML rules directory used by the normaliser | +| `ratelimit` | `RateLimitConfig` | throttling: per-provider `ProviderRateLimitConfig`, per-service `ServiceRateLimitConfig`, live-run guard | + +Three plain fields sit at the top level too: `log_file` and `verbose` (read by the CLI only) and `concurrency_limit` (1 to 50, default 5). The field description calls it "max concurrent API calls", but what it bounds is the number of collection units running at once: one AWS account in one region, one Azure subscription or one GCP project. It applies to `collect()` and `map_inventory()` alike. + +The old single-provider form still works. `CloudGConfig(provider="azure")` sets `providers` to `["azure"]`, as long as `providers` was left at its default. `provider` itself is excluded from `model_dump()`. + +### Unknown keys + +Every section ignores keys it does not define. Only `ratelimit` says so: unknown keys there, under `ratelimit.` and under `ratelimit..services.`, produce a warning such as `Unknown config key ratelimit.aws.max_rsp is ignored`. A misspelt key anywhere else (`aws.regoins`, `scanners.enable`) is dropped without a message, and the field keeps its default. The [strict loader](#reject-unknown-keys) below closes that gap. + +### Environment variables + +`CloudGConfig` reads nothing from the environment. Credentials are different: when a credential field is empty, the code that builds the cloud client falls back to the variables each SDK documents. + +| Field left empty | Falls back to | +|---|---| +| `aws.web_identity_token_file` | `AWS_WEB_IDENTITY_TOKEN_FILE` | +| `aws.*` keys and `aws.profile` | the boto3 default chain (`AWS_ACCESS_KEY_ID`, `AWS_PROFILE`, instance and task roles, SSO cache) | +| `azure.tenant_id`, `azure.client_id` | `AZURE_TENANT_ID`, `AZURE_CLIENT_ID` | +| `azure.client_secret`, `azure.federated_token_file` | `AZURE_CLIENT_SECRET`, `AZURE_FEDERATED_TOKEN_FILE` | +| every Azure credential | `DefaultAzureCredential` | +| `gcp.credentials_file` | application default credentials, which read `GOOGLE_APPLICATION_CREDENTIALS` | + +Two variables change cloudg behaviour outside the config: `CLOUDG_CATALOG_DIR` overlays the inventory reference catalogs, and `CLOUDG_CACHE_DIR` moves the Cloud Control schema cache (default `~/.cache/cloudg`). [Authentication](/guides/authentication/) covers the full credential order for each provider. + +## Examples + +### Build a multi-cloud config in code + +Secrets come from the environment; nothing secret is written into the script. + +```python title="multicloud_config.py" +import os + +from cloudg import CloudGConfig, CloudGEngine +from cloudg.config import AWSConfig, AzureConfig, GCPConfig, ScannerConfig + +config = CloudGConfig( + providers=["aws", "azure", "gcp"], + aws=AWSConfig( + regions=["eu-west-1", "eu-central-1"], + role_arn="arn:aws:iam::123456789012:role/cloudg-readonly", + external_id=os.environ.get("CLOUDG_EXTERNAL_ID"), + ), + azure=AzureConfig( + subscription_ids=["00000000-0000-0000-0000-000000000000"], + tenant_id=os.environ.get("AZURE_TENANT_ID"), + client_id=os.environ.get("AZURE_CLIENT_ID"), + client_secret=os.environ.get("AZURE_CLIENT_SECRET"), + regions=["westeurope", "northeurope"], + ), + gcp=GCPConfig(project_ids=["orders-prod", "orders-staging"]), + scanners=ScannerConfig(enabled=["prowler", "checkov"], iac_directories=["./terraform"]), +) +config.ontology.export_formats = ["turtle"] +config.rag.enabled = False + +# Only what differs from the defaults, without the client secret +print(config.model_dump(mode="json", exclude_defaults=True, exclude={"azure": {"client_secret"}})) +engine = CloudGEngine(config) +``` + +### Load config.yaml and override in code + +```yaml title="config.yaml" +providers: [aws, gcp] +aws: + regions: [eu-west-1, eu-central-1] + profile: security-audit +gcp: + organization_id: "123456789012" +scanners: + enabled: [prowler, checkov] + iac_directories: [./terraform] +``` + +```python title="load_and_override.py" +import sys + +from cloudg import CloudGEngine, load_config + +config = load_config("config.yaml") +if len(sys.argv) > 1: + config.aws.regions = sys.argv[1].split(",") # e.g. python load_and_override.py us-east-1 + +print(config.providers, config.aws.regions, config.gcp.organization_id) +engine = CloudGEngine(config) +``` + +Quote numeric IDs such as `organization_id` in YAML. Unquoted, `123456789012` is an integer and pydantic rejects it for a string field. + +### Reject unknown keys + +This loader fails on a missing file and on any key the models do not define, before cloudg sees the config. + +```python title="strict_config.py" +import sys +from pathlib import Path + +import yaml +from pydantic import BaseModel + +from cloudg import CloudGConfig + + +def unknown_keys(data: object, model: type[BaseModel], path: str = "") -> list[str]: + """Dotted paths of the keys in `data` that `model` does not define.""" + if not isinstance(data, dict): + return [] + found = [] + for key, value in data.items(): + field = model.model_fields.get(key) + if field is None: + found.append(path + key) + elif isinstance(field.annotation, type) and issubclass(field.annotation, BaseModel): + found += unknown_keys(value, field.annotation, f"{path}{key}.") + return found + + +def load_strict(path: str) -> CloudGConfig: + file = Path(path) + if not file.is_file(): + raise FileNotFoundError(path) # load_config() would quietly use defaults + data = yaml.safe_load(file.read_text()) or {} + bad = unknown_keys(data, CloudGConfig) + if bad: + raise ValueError("unknown config keys: " + ", ".join(bad)) + return CloudGConfig.model_validate(data) + + +if __name__ == "__main__": + try: + config = load_strict(sys.argv[1] if len(sys.argv) > 1 else "config.yaml") + except (FileNotFoundError, ValueError) as exc: + sys.exit(f"config error: {exc}") + print("providers:", config.providers, "aws regions:", config.aws.regions) +``` + +```console +$ python strict_config.py config.yaml +config error: unknown config keys: aws.regoins, ratelimit.aws.max_rsp +``` + +## Notes + +Validation runs when the model is built, not when you assign to it. `config.aws.regions = "eu-west-1"` is accepted and leaves a string where a list belongs, and the collectors then misbehave. Assign the right types, or build the section with its model (`AWSConfig(regions=[...])`) so pydantic checks it. + +The patterns are enforced at construction: `aws.retry_mode` must be `legacy`, `standard` or `adaptive`; `gcp.collection_scope` must be `auto`, `organization` or `project`; `rag.chunk_strategy` must be one of `entity`, `community`, `relation_group`, `hybrid`. Bounded integers (`aws.max_retries` 1 to 30, `concurrency_limit` 1 to 50, `scanners.timeout_seconds` at least 60) raise a `ValidationError` when out of range. + +`rulesets.rules_dir` must be a string. Writing `rules_dir: null` in YAML fails validation; leave the key out to get the rules shipped inside the package. + +`model_dump()` includes credential fields such as `aws.secret_access_key` and `azure.client_secret` in plain text when they are set. Exclude them before logging a config: `config.model_dump(exclude={"aws": {"secret_access_key", "session_token"}, "azure": {"client_secret"}})`. + +Some keys are accepted but not read by the engine in 0.6.0: `graph.persist_graphml`, `graph.max_nodes_warn`, `graph.export_cytoscape`, `ontology.include_raw_metadata`, `rag.chunk_strategy`, `report.inline_js` and `rulesets.load_external`. `graph.compute_attack_paths`, `report.formats` and `report.output_dir` are read by the CLI commands only. `CloudGEngine.analyze()` computes attack paths whatever `graph.compute_attack_paths` says. + +`InventoryMapper` deep-copies the config before a run, so organization discovery (which rewrites `aws.accounts`, `aws.role_name` and possibly `aws.regions`) never changes the object you passed in. + +## Related + +:::links +- [config.yaml reference](/reference/config/providers/) Every key, its type and default. +- [Rate limit settings](/reference/config/ratelimit/) The `ratelimit` section in full. +- [Authentication](/guides/authentication/) Credential order for AWS, Azure and GCP. +- [CloudGEngine](/api/cloudgengine/) What runs on the config. +- [Rate limits and throttling](/guides/rate-limits/) When to change the `ratelimit` defaults. +::: diff --git a/site/content/api/cloudgengine.md b/site/content/api/cloudgengine.md new file mode 100644 index 0000000..f6aa53e --- /dev/null +++ b/site/content/api/cloudgengine.md @@ -0,0 +1,266 @@ +--- +object: cloudg.api.CloudGEngine +members: [run_pipeline, run_pipeline_sync, collect, scan, analyze, ingest_reports, normalise_findings, run_from_reports, run_from_reports_sync, map_inventory, map_inventory_sync] +lede: "The engine behind `cloudg run`, `cloudg ingest` and `cloudg map`: call the whole pipeline in one line, or drive each phase yourself." +--- + +## Which method to call + +`CloudGEngine` takes one argument, a [`CloudGConfig`](/api/cloudgconfig/), and keeps it on `engine.config`. Everything else is a method call. The methods fall into three groups, and they share very little state: the engine holds the config and the six [event hooks](/api/event-hooks/), nothing more. You can call `collect()` twice, or call `analyze()` on assets you built by hand, and nothing on the engine gets in the way. + +```mermaid caption="What each entry point runs" +flowchart LR + RP["run_pipeline()"] --> C["collect()"] + RP --> S["scan()"] + RP --> A["analyze()"] + RP --> N["FindingsNormaliser"] + RP --> R["findings.json, report.html"] + RFR["run_from_reports()"] --> I["ingest_reports()"] + RFR --> NF["normalise_findings()"] + NF --> N + RFR --> R + MI["map_inventory()"] --> IM["InventoryMapper"] +``` + +| You have | Call | Cloud access | Scanner binaries | +|---|---|---|---| +| Credentials, and want everything `cloudg run` produces | `run_pipeline()` | yes | the ones in `scanners.enabled` | +| Prowler, ScoutSuite, Checkov or Trivy output from somewhere else | `run_from_reports()` | no | no | +| Credentials, and only want to know what exists and how it connects | `map_inventory()` | yes | no | +| Your own assets, or a need to stop between phases | `collect()`, `scan()`, `analyze()`, `normalise_findings()` | for `collect()` and the cloud scanners | for `scan()` | + +The `*_sync` variants wrap the async method in `asyncio.run()`. Use them from plain scripts. Inside a running event loop (a Jupyter notebook, say) `asyncio.run()` refuses to start, so `await` the async method instead. + +### What run_pipeline does, phase by phase + +`run_pipeline(output_dir)` creates `output_dir`, then: + +1. `collect()` runs the collectors for every provider in `config.providers`, across `aws.regions`, `aws.accounts`, the Azure subscriptions and the GCP projects you configured. +2. `scan()` builds a graph of the collected assets and runs the reachability analysis on it, then starts every scanner in `scanners.enabled` in a thread pool: Prowler and ScoutSuite once per provider, Checkov once per IaC directory, Trivy once over `scanners.trivy_images`. The IAM policy linter runs whenever there are assets. +3. `analyze()` rebuilds the graph and records node and edge counts, reachability findings and lateral movement paths. It then writes the ontology (`ontology.enabled`), the RAG chunks (`rag.enabled`) and the Terraform recreation (`terraform.enabled`). +4. The scanner findings and the reachability findings go through [`FindingsNormaliser`](/api/findingsnormaliser/): deduplication, cross-scanner merging, CVSS rescoring and compliance mapping. The reachability analysis ran in both step 2 and step 3. Both runs produce the same check on the same resource, so the deduplication pass folds the copies into one. +5. The JSON and HTML reports are written to `output_dir/findings.json` and `output_dir/report.html`, with the D3 graph embedded. + +Everything ends up in one [`PipelineResult`](/api/results/). + +### IaC targets for Checkov + +`scan()` picks the directories Checkov scans in this order: the `iac_dir` argument, then `scanners.iac_directories`, then a Terraform recreation of the live assets (generated on the spot when `terraform.enabled` is true). When none of these exists Checkov is skipped with a warning. cloudg no longer falls back to the current directory, because scanning whatever happens to be in your working directory says nothing about the cloud. The same logic is public as `cloudg.api.resolve_iac_dirs(iac_dir, config_dirs, terraform_dir=None)`, which returns the directories together with their source: `"cli"`, `"config"`, `"terraform"`, or `None` when nothing qualifies. + +## Examples + +### Full pipeline + +This needs read access to the account (any method from [Authentication](/guides/authentication/) works) and the scanner binaries you enable. Missing binaries are skipped. + +```python title="nightly_pipeline.py" +import asyncio +import json + +from cloudg import CloudGConfig, CloudGEngine + + +async def main() -> None: + config = CloudGConfig(providers=["aws"]) + config.aws.profile = "security-audit" + config.aws.regions = ["eu-west-1", "eu-central-1"] + config.scanners.enabled = ["prowler", "trivy"] + config.scanners.trivy_images = [ + "123456789012.dkr.ecr.eu-west-1.amazonaws.com/orders-worker:latest" + ] + + engine = CloudGEngine(config) + result = await engine.run_pipeline(output_dir="./reports") + + print(json.dumps(result.to_summary(), indent=2)) + print(result.report_paths) # {'json': .../findings.json, 'html': .../report.html} + + # Collection failures don't raise; they are recorded per service + for cov in result.coverage: + if cov.failed_services: + print("incomplete:", cov.to_summary()) + + +asyncio.run(main()) +``` + +### Ingest only + +No credentials and no scanners: point the engine at reports produced elsewhere. Keys are tool names (`prowler`, `scoutsuite`, `checkov`, `trivy`); values are lists of files or directories. + +```python title="ingest_only.py" +from cloudg import CloudGConfig, CloudGEngine + +engine = CloudGEngine(CloudGConfig()) +engine.on_phase_start = lambda phase: print("phase:", phase) + +result = engine.run_from_reports_sync( + { + "prowler": ["./prowler-output/"], + "checkov": ["./results_json.json"], + "trivy": ["./trivy.json"], + }, + output_dir="./reports", +) + +print(result.total_findings, result.severity_breakdown) +print(result.report_paths) +for f in result.findings: + print(f.severity.value, f.risk_score, f.source_tool, f.title) +``` + +With one failing Prowler check, one Checkov failure and one Trivy CVE, it prints: + +```console +$ python ingest_only.py +phase: ingest +phase: normalisation +phase: reporting +3 {'CRITICAL': 1, 'HIGH': 2} +{'json': PosixPath('reports/findings.json'), 'html': PosixPath('reports/report.html')} +CRITICAL 9.5 trivy [Trivy] CVE-2024-0001: openssl (alpine) +HIGH 7.5 prowler S3 bucket default encryption +HIGH 7.5 checkov [Checkov/terraform] Ensure S3 bucket has server-side encryption enabled +``` + +The result has no assets, so `graph_nodes`, `ontology_triples` and `attack_paths` stay empty. To get them as well, collect live assets and pass the ingested findings to `analyze()` (next examples). + +### Map only + +`map_inventory()` runs the deep inventory collectors and the [relationship linker](/api/relationshiplinker/). No scanner is involved. Passing `output_dir` writes the map files; passing `findings` as well writes `asset-map.json` and `compliance-map.json`. + +```python title="map_only.py" +from cloudg import CloudGConfig, CloudGEngine + +config = CloudGConfig(providers=["aws"]) +config.aws.regions = ["ALL"] +config.inventory.exclude_services = ["kubernetes"] + +engine = CloudGEngine(config) +inventory = engine.map_inventory_sync(output_dir="./inventory", tagging_sweep=False) + +summary = inventory.summary +print(summary["total_assets"], "assets,", summary["total_edges"], "edges") +print("internet exposed:", summary["internet_exposed"]) +print("top services:", list(summary["assets_by_service"].items())[:5]) +``` + +The return value is an [`InventoryResult`](/api/inventoryresult/). Unlike `collect()`, this method re-raises when the mapping fails, after calling `on_error("inventory_mapping", exc)`. + +### Phases by hand + +Driving the phases yourself lets you swap one out. This script runs offline: two hand-built assets stand in for `collect()`, an existing Trivy report stands in for `scan()`, and the graph, ontology and RAG outputs are produced from them. + +```python title="phases_offline.py" +import asyncio + +from cloudg import CloudGConfig, CloudGEngine +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType, NetworkEdge + +# Two assets and two edges, as a collector or a CMDB export would supply them +sg = CloudAsset( + arn="arn:aws:ec2:eu-west-1:123456789012:security-group/sg-0a1b2c3d4e5f60718", + name="bastion-sg", + asset_type=AssetType.SECURITY_GROUP, + provider=CloudProvider.AWS, + region="eu-west-1", + account_id="123456789012", +) +vm = CloudAsset( + arn="arn:aws:ec2:eu-west-1:123456789012:instance/i-0b1c2d3e4f5061728", + name="bastion", + asset_type=AssetType.EC2, + provider=CloudProvider.AWS, + region="eu-west-1", + account_id="123456789012", +) +edges = [ + NetworkEdge( + source_id="0.0.0.0/0", + target_id=sg.id, + edge_type=EdgeType.SECURITY_GROUP_RULE, + cidr="0.0.0.0/0", + port_range="22", + protocol="TCP", + ), + NetworkEdge(source_id=sg.id, target_id=vm.id, edge_type=EdgeType.ATTACHED_TO), +] + + +async def main() -> None: + engine = CloudGEngine(CloudGConfig(providers=["aws"])) + engine.on_phase_start = lambda phase: print("phase:", phase) + + findings = engine.ingest_reports({"trivy": ["./trivy.json"]}) + analysis = await engine.analyze([sg, vm], edges, findings, output_dir="./reports") + scan_result = engine.normalise_findings( + findings + analysis.reachability_findings, assets=[sg, vm] + ) + + print(analysis.graph_nodes, "nodes,", analysis.graph_edges, "edges") + for f in analysis.reachability_findings: + print(f.severity.value, f.title) + print("ontology:", analysis.ontology_triples, "triples, last file", analysis.ontology_path) + print("RAG chunks:", analysis.rag_chunks_path) + print(scan_result.summary["severity_breakdown"]) + + +asyncio.run(main()) +``` + +```console +$ python phases_offline.py +phase: ingest +phase: analysis +3 nodes, 2 edges +CRITICAL Security group allows SSH (port 22) from 0.0.0.0/0 +ontology: 670 triples, last file reports/ontology.jsonld +RAG chunks: reports/rag_chunks.jsonl +{'CRITICAL': 2} +``` + +The graph has three nodes for two assets: `0.0.0.0/0` becomes a node of its own. With credentials, replace the hand-built lists with `collection = await engine.collect()` and pass `collection.assets` and `collection.edges`, and replace the ingest with `await engine.scan(...)` if you want the scanners to run. + +### Hooks + +Hooks are plain attributes. Set them before calling a method; each one is optional. + +```python title="hooks.py" +from cloudg import CloudGConfig, CloudGEngine + +engine = CloudGEngine(CloudGConfig()) +engine.on_phase_start = lambda phase: print(f"-> {phase}") +engine.on_finding = lambda f: print(f" {f.severity.value:8} {f.title}") +engine.on_error = lambda phase, exc: print(f"!! {phase}: {exc}") + +engine.run_from_reports_sync({"trivy": ["./trivy.json"]}, output_dir="./reports") +``` + +[Event hooks](/api/event-hooks/) lists every hook, when it fires and what it receives, with longer examples. + +## Notes + +`PipelineResult.errors` is not the full error list. It only holds normalisation and reporting failures. A provider that could not be collected shows up as a `FAILED` entry in `result.coverage`; a scanner that crashed, a graph that could not be built, an ontology or RAG export that failed are reported through `on_error` and the `cloudg.api` logger only. If you need to know that a run was complete, set `on_error` and check `coverage`. + +`collect()` never raises for a collection problem. When the collector itself throws, the engine calls `on_error("collection", exc)` and returns an empty `CollectionResult`, and `run_pipeline()` carries on with no assets. + +`on_finding` and `on_scan_complete` fire after `scan()` or `ingest_reports()`, before normalisation. The findings they receive have not been deduplicated or rescored yet. Normalisation then changes those same objects in place (severity, `source_tool`, `compliance_frameworks`). For the final list, read `result.findings` after the run. + +`map_inventory(findings=...)` only uses `findings` when `output_dir` is set too. Without it the findings are ignored. To overlay findings on a map you already hold, call [`InventoryMapper.export_merged()`](/api/inventorymapper/) yourself. + +`terraform.output_dir` defaults to `./reports/terraform`, relative to the current directory, and it is not derived from the `output_dir` you pass to `run_pipeline()` or `analyze()`. Set both when you move the reports somewhere else. + +`scanners.timeout_seconds` does not stop a slow scanner in 0.6.0. The engine passes it to `future.result()` only after `as_completed()` has handed over a finished future. The limits that do apply are the subprocess timeouts inside the scanner wrappers: 3600 seconds for Prowler and ScoutSuite, 1800 for Checkov and for each Trivy image. + +The IAM linter runs whenever `scan()` has assets, whether or not `iam` is in `scanners.enabled`. + +## Related + +:::links +- [Event hooks](/api/event-hooks/) The six callbacks and when each one fires. +- [Result objects](/api/results/) PipelineResult, CollectionResult and AnalysisResult. +- [CloudGConfig](/api/cloudgconfig/) Building the config the engine runs on. +- [The pipeline](/guides/pipeline/) The same phases from the CLI side. +- [Python recipes](/guides/python-recipes/) Longer integration examples. +- [cloudg run](/cli/run/) The command that wraps `run_pipeline()`. +::: diff --git a/site/content/api/cloudontology.md b/site/content/api/cloudontology.md new file mode 100644 index 0000000..bc3b0bc --- /dev/null +++ b/site/content/api/cloudontology.md @@ -0,0 +1,269 @@ +--- +title: CloudOntology +object: cloudg.graph.ontology.CloudOntology +members: [build, graph, triple_count, query, query_internet_exposed, query_by_relation_group, query_asset_neighbourhood, query_compliance_gaps, save, to_turtle, to_jsonld, to_rdfxml, to_ntriples, stats] +lede: Writes assets, edges and findings into an rdflib graph with 64 typed relations, so you can export it as RDF and query it with SPARQL. +since: "0.6.0" +--- + +The ontology is the semantic view of an inventory. Where [GraphBuilder](/api/graphbuilder/) keeps the collector's edge types (`SECURITY_GROUP_RULE`, `ATTACHED_TO`), `CloudOntology` reads each edge together with its endpoint types, ports and CIDR and turns it into relations that say what the edge means: `INTERNET_REACHABLE`, `ONLY_SSH`, `PROTECTED_BY_SG`, `VPC_CONTAINS_SUBNET`. `cloudg run` builds it after the scanners finish, so findings are part of the graph, and writes `ontology.ttl` and `ontology.jsonld` by default. + +## What build() writes + +```mermaid caption="Where the triples come from" +flowchart LR + S["Schema: classes and properties"] --> G["rdflib Graph"] + A["Assets"] --> I["Individuals with data properties"] --> G + E["Edges"] --> R["infer_relations()"] --> G + E --> M["EdgeMetadata blank nodes"] --> G + A --> AR["Asset-level relations"] --> G + F["Findings"] --> X["AssetIndex resolves the asset"] --> G +``` + +Three namespaces are bound on the graph, so they appear as prefixes in every serialisation and in your SPARQL: + +| Prefix | IRI | Holds | +|---|---|---| +| `cm:` | `https://cloudg.io/ontology#` | classes such as `cm:RelationalDatabase` and `cm:SecurityFinding` | +| `cmr:` | `https://cloudg.io/resource/` | individuals, keyed by asset id (`cmr:orders-db`), plus `cmr:finding_`, `cmr:compliance_` and tag nodes | +| `cmp:` | `https://cloudg.io/property/` | the 64 relations and the data properties (`cmp:hasName`, `cmp:hasARN`) | + +In order, `build()` adds: + +1. The schema. An OWL class for every asset type (EC2, VIRTUAL_MACHINE and GCE_INSTANCE all map to `cm:ComputeInstance`; types without a hand-picked name get the CamelCase of the enum name), each a subclass of `cm:CloudResource`. Every `RelationType` becomes an `owl:ObjectProperty` with an `rdfs:label` and a `cm:relationGroup` literal. This part alone is 636 triples, so an empty inventory still produces a sizeable file. +2. One individual per asset: `rdf:type`, `cmp:hasName`, `cmp:hasProvider`, `cmp:hasRegion`, `cmp:isInternetExposed` (an `xsd:boolean`), and `cmp:hasARN` and `cmp:hasAccountId` when set. Each tag adds a `cmp:TAGGED_WITH` link to a `cm:TagValue` node named `tag__`. +3. For each edge, the relations from `infer_relations()`, as `source relation target`. Endpoints that are not assets (`0.0.0.0/0`) become `cm:CloudResource` individuals named after the id. An edge with a CIDR, port or protocol also gets a blank node of type `cm:EdgeMetadata` with `cm:fromNode`, `cm:toNode`, `cmp:hasCIDR`, `cmp:hasPort` and `cmp:hasProtocol`. +4. Asset-level relations from metadata: `OWNED_BY`, `COST_ALLOCATED_TO` and `MONITORED_BY` from tags such as `owner`, `team`, `costcenter` and `monitoring`; `VPC_CONTAINS_SUBNET` and `CONTAINS` from a VPC to the assets that share its `vpc_id`; `ENCRYPTED_BY_KMS` from an encrypted asset to its key; `ROTATES_SECRET`; and `PROTECTED_BY_SG` from an EC2 instance's `security_groups` metadata. +5. Each finding as a `cm:SecurityFinding` with `cmp:hasName` (the title), `cmp:hasSeverity` and `cmp:hasRiskScore`, linked by `cmp:FINDING_AFFECTS` to the asset it resolves to. Each compliance framework on the finding becomes `cmr:compliance_` with a `cmp:COMPLIANCE_GOVERNS` link to the same asset. + +Findings are matched to assets through `AssetIndex`: by asset id, then ARN (from `resource_id` or `resource_arn`), then a unique name, then a unique ARN tail. Scanner findings that name a resource by ARN therefore attach to the real asset instead of a dangling `cmr:arn:aws:...` node. Only when nothing matches does the raw `resource_id` become the object. + +### Edge types and the relations they produce + +| Edge | Relations | +|---|---| +| `SECURITY_GROUP_RULE` | `INGRESS_ALLOWED` or `EGRESS_ALLOWED`; `INTERNET_REACHABLE` when the source is `0.0.0.0/0`, `::/0` or an Azure `Internet`, `Any` or `*` tag; `CIDR_RESTRICTED` for any other CIDR; then `ALL_TRAFFIC` for an all-ports rule, else `ONLY_SSH`, `ONLY_HTTP`, `ONLY_HTTPS`, `ONLY_RDP` for each of 22, 80, 443, 3389 inside the ranges, else `PORT_RESTRICTED` | +| `ATTACHED_TO` | `PROTECTED_BY_SG` when the target is a security group or NSG, `PROTECTED_BY_NACL` for a NACL, `DEPENDS_ON` otherwise | +| `CONTAINS` | `VPC_CONTAINS_SUBNET`, `SUBNET_CONTAINS_INSTANCE`, `CLUSTER_CONTAINS_SERVICE` or `ORG_CONTAINS_ACCOUNT` when both endpoint types fit, plain `CONTAINS` otherwise | +| `IAM_TRUST` | `ROLE_ASSUMES_ROLE`, plus `CROSS_ACCOUNT_TRUST` when the two accounts differ | +| `IAM_POLICY_ATTACHMENT` | `USER_HAS_POLICY`, `ROLE_HAS_POLICY` or `GROUP_HAS_POLICY` by target type | +| `LOAD_BALANCER_TARGET` | `LB_TARGETS_INSTANCE` and `LOAD_BALANCED_BY` | +| `PEERING`, `ROUTE`, `INTERNET_EXPOSED` | `VPC_PEERED`, `TRANSIT_ROUTED`, `INTERNET_REACHABLE` | +| `INVOKES`, `LOGS_TO` | the relation of the same name | +| `USES_IMAGE`, `ASSUMES_ROLE` | `RUNS_ON` | +| `GRANTS_ACCESS`, `PROTECTS`, `MONITORS`, `MANAGES`, `GOVERNS`, `REFERENCES` | `POLICY_ALLOWS_ACTION`, `PROTECTED_BY_WAF`, `MONITORED_BY`, `OWNED_BY`, `COMPLIANCE_GOVERNS`, `DEPENDS_ON` | +| `NACL_RULE` | nothing; only the `EdgeMetadata` node | + +A `relationship` declared on the edge (any `RelationType` name, as the linker writes them) changes this. On `CONTAINS`, `ATTACHED_TO` and the typed inventory edges it replaces the inferred relation. On the network and IAM edges it comes first and the inferred ones follow. + +### The 64 relation types + +| Group | Relations | +|---|---| +| `NETWORK` (16) | `INGRESS_ALLOWED`, `INGRESS_DENIED`, `EGRESS_ALLOWED`, `EGRESS_DENIED`, `ONLY_HTTP`, `ONLY_HTTPS`, `ONLY_SSH`, `ONLY_RDP`, `ALL_TRAFFIC`, `PORT_RESTRICTED`, `CIDR_RESTRICTED`, `INTERNET_REACHABLE`, `VPC_PEERED`, `TRANSIT_ROUTED`, `NAT_TRANSLATED`, `DNS_RESOLVED` | +| `CONTAINMENT` (8) | `CONTAINS`, `VPC_CONTAINS_SUBNET`, `SUBNET_CONTAINS_INSTANCE`, `REGION_CONTAINS_VPC`, `ACCOUNT_CONTAINS_REGION`, `ORG_CONTAINS_ACCOUNT`, `CLUSTER_CONTAINS_SERVICE`, `LB_TARGETS_INSTANCE` | +| `IAM` (10) | `ROLE_ASSUMES_ROLE`, `USER_HAS_POLICY`, `ROLE_HAS_POLICY`, `GROUP_HAS_POLICY`, `POLICY_ALLOWS_ACTION`, `POLICY_DENIES_ACTION`, `CROSS_ACCOUNT_TRUST`, `SERVICE_LINKED_ROLE`, `PERMISSION_BOUNDARY_LIMITS`, `SCP_RESTRICTS` | +| `DATA_FLOW` (9) | `READS_FROM`, `WRITES_TO`, `ENCRYPTS_WITH`, `DECRYPTS_WITH`, `LOGS_TO`, `STREAMS_TO`, `REPLICATES_TO`, `BACKUP_TO`, `CACHE_FOR` | +| `SECURITY` (9) | `PROTECTED_BY_SG`, `PROTECTED_BY_NACL`, `PROTECTED_BY_WAF`, `ENCRYPTED_BY_KMS`, `ROTATES_SECRET`, `CERTIFICATE_SECURES`, `FINDING_AFFECTS`, `VULNERABILITY_EXPLOITS`, `COMPLIANCE_GOVERNS` | +| `COMPUTE` (8) | `RUNS_ON`, `TRIGGERED_BY`, `INVOKES`, `SCALES_WITH`, `LOAD_BALANCED_BY`, `SCHEDULED_BY`, `DEPENDS_ON`, `SERVES_TRAFFIC_TO` | +| `GOVERNANCE` (4) | `TAGGED_WITH`, `COST_ALLOCATED_TO`, `OWNED_BY`, `MONITORED_BY` | + +Several of these, `TRIGGERED_BY` and `READS_FROM` among them, are never inferred from edges and appear only when a collector declares them as an edge's `relationship`. `RelationType`, `RelationGroup`, `get_relation_group()` and `get_relations_for_group()` are importable from `cloudg.graph.ontology`. + +### Serialisation formats + +| `fmt` passed to `save()` | rdflib format | File `cloudg run` writes | +|---|---|---| +| `turtle`, `ttl` | Turtle | `ontology.ttl` | +| `json-ld`, `jsonld` | JSON-LD | `ontology.jsonld` | +| `xml`, `rdfxml` | RDF/XML | `ontology.rdf` | +| `nt`, `ntriples` | N-Triples | `ontology.nt` | + +`save()` falls back to Turtle for any other value instead of raising, and creates missing parent directories. `cloudg run` writes one file per entry in `ontology.export_formats` (default `["turtle", "json-ld"]`), and `--no-ontology` or `ontology.enabled: false` skips the phase. + +Use the first spelling in each row in `config.yaml`. The file name is picked from a separate table that only knows `turtle`, `json-ld`, `xml` and `nt`, so `jsonld` writes JSON-LD into `ontology.ttl`. `CloudGEngine` has the same table without `nt`, so N-Triples from the Python engine also land in `ontology.ttl`, replacing the Turtle file when both formats are configured. + +## Examples + +### Build, query and save + +Four assets, two edges and one scanner finding that names the database by ARN. The query lists every relation in the graph with readable names on both ends, skipping the data properties. + +```python title="ontology_demo.py" +"""Build the RDF ontology for a small estate, save it and query it with SPARQL.""" + +from pathlib import Path + +from rdflib import Graph + +from cloudg.graph.ontology import CloudOntology, RelationGroup +from cloudg.schema.models import ( + AssetType, + CloudAsset, + CloudProvider, + EdgeType, + Finding, + NetworkEdge, + Severity, +) + +ACCOUNT = "123456789012" +REGION = "eu-west-1" +KEY_ARN = f"arn:aws:kms:{REGION}:{ACCOUNT}:key/1111aaaa-22bb-33cc-44dd-555555eeeeee" + + +def asset(asset_id: str, asset_type: AssetType, arn: str, **extra) -> CloudAsset: + return CloudAsset( + id=asset_id, + name=asset_id, + asset_type=asset_type, + provider=CloudProvider.AWS, + region=REGION, + account_id=ACCOUNT, + arn=arn, + **extra, + ) + + +assets = [ + asset("sg-web", AssetType.SECURITY_GROUP, f"arn:aws:ec2:{REGION}:{ACCOUNT}:security-group/sg-web"), + asset("web-1", AssetType.EC2, f"arn:aws:ec2:{REGION}:{ACCOUNT}:instance/i-0web1", + tags={"owner": "payments"}), + asset("orders-key", AssetType.KMS_KEY, KEY_ARN), + asset("orders-db", AssetType.RDS_INSTANCE, f"arn:aws:rds:{REGION}:{ACCOUNT}:db:orders-db", + metadata={"storage_encrypted": True, "kms_key_id": KEY_ARN}), +] + +edges = [ + NetworkEdge(source_id="0.0.0.0/0", target_id="sg-web", edge_type=EdgeType.SECURITY_GROUP_RULE, + port_range="22", protocol="TCP", cidr="0.0.0.0/0", direction="ingress"), + NetworkEdge(source_id="web-1", target_id="sg-web", edge_type=EdgeType.ATTACHED_TO), +] + +# A scanner finding that names the database by ARN, not by asset id +findings = [ + Finding( + resource_id=f"arn:aws:rds:{REGION}:{ACCOUNT}:db:orders-db", + severity=Severity.HIGH, + title="RDS instance has no deletion protection", + description="Deletion protection is disabled.", + source_tool="prowler", + compliance_frameworks=["CIS"], + ) +] + +onto = CloudOntology() +onto.build(assets, edges, findings) +stats = onto.stats() +print(stats["total_triples"], "triples,", stats["classes_used"], "classes,", + stats["individuals"], "individuals") +print("groups:", dict(sorted(stats["relation_group_counts"].items()))) + +# The graph binds the cm:, cmr: and cmp: prefixes, so queries can use them +rows = onto.query(""" + SELECT ?subject ?relation ?object WHERE { + ?s ?p ?o . + FILTER(STRSTARTS(STR(?p), STR(cmp:))) + FILTER(!STRSTARTS(STR(?p), STR(cmp:has))) + FILTER(?p != cmp:isInternetExposed) + ?s cmp:hasName ?subject . + OPTIONAL { ?o cmp:hasName ?oname } + BIND(STRAFTER(STR(?p), STR(cmp:)) AS ?relation) + BIND(COALESCE(?oname, STR(?o)) AS ?object) + } ORDER BY ?subject ?relation +""") +for row in rows: + print(f" {row['subject']} {row['relation']} {row['object']}") + +print("internet reachable:", [r["name"] for r in onto.query_internet_exposed()]) +print("security group:", len(onto.query_by_relation_group(RelationGroup.SECURITY)), "triples") +print("compliance gaps:", [(r["severity"], r["resourceName"]) for r in onto.query_compliance_gaps()]) + +out = Path("ontology-out") +for fmt, name in [("turtle", "ontology.ttl"), ("json-ld", "ontology.jsonld"), + ("xml", "ontology.rdf"), ("nt", "ontology.nt")]: + onto.save(out / name, fmt=fmt) + +# Any RDF tool can read the files back; here rdflib itself +reloaded = Graph().parse(out / "ontology.ttl", format="turtle") +print("reloaded", len(reloaded), "triples from", sorted(p.name for p in out.iterdir())) +``` + +```console +$ python ontology_demo.py +689 triples, 9 classes, 9 individuals +groups: {'GOVERNANCE': 2, 'NETWORK': 3, 'SECURITY': 4} + 0.0.0.0/0 INGRESS_ALLOWED sg-web + 0.0.0.0/0 INTERNET_REACHABLE sg-web + 0.0.0.0/0 ONLY_SSH sg-web + RDS instance has no deletion protection FINDING_AFFECTS orders-db + orders-db ENCRYPTED_BY_KMS orders-key + web-1 OWNED_BY https://cloudg.io/resource/tag_payments + web-1 PROTECTED_BY_SG sg-web + web-1 TAGGED_WITH https://cloudg.io/resource/tag_owner_payments +internet reachable: ['sg-web'] +security group: 4 triples +compliance gaps: [('HIGH', 'orders-db')] +reloaded 689 triples from ['ontology.jsonld', 'ontology.nt', 'ontology.rdf', 'ontology.ttl'] +``` + +Things to notice: + +- 636 of the 689 triples are schema. The `stats()` counts cover classes and individuals actually used. +- `ENCRYPTED_BY_KMS` points at `orders-key` because the database's `kms_key_id` matched the key asset's ARN. With no collected key, the object would be the raw key reference. +- The finding attached to `orders-db` although its `resource_id` was an ARN. +- `query_internet_exposed()` lists the objects of `INTERNET_REACHABLE` triples, which here is the security group. It reads the rule edges, not the reachability walk; for "which workloads can the internet reach" use [ReachabilityAnalyzer](/api/reachabilityanalyzer/). +- The `OWNED_BY` object is a tag node (`tag_payments`) with no `cmp:hasName`, so the query printed its IRI. + +Saving N-Triples prints a harmless `UserWarning: NTSerializer always uses UTF-8 encoding` from rdflib. + +### Query a saved file without cloudg + +The exported files are plain RDF. This needs only rdflib and the `ontology.ttl` from the previous example (or from `cloudg run`). + +```python title="query_saved.py" +from rdflib import Graph, Namespace + +CMP = Namespace("https://cloudg.io/property/") + +g = Graph().parse("ontology-out/ontology.ttl") +rows = g.query( + """ + SELECT ?resource ?finding ?severity WHERE { + ?f cmp:FINDING_AFFECTS ?r ; + cmp:hasName ?finding ; + cmp:hasSeverity ?severity . + ?r cmp:hasName ?resource . + } + """, + initNs={"cmp": CMP}, +) +for resource, finding, severity in rows: + print(f"{severity}: {resource}: {finding}") +``` + +```console +$ python query_saved.py +HIGH: orders-db: RDS instance has no deletion protection +``` + +The same query works against `ontology.jsonld` with `Graph().parse("ontology.jsonld", format="json-ld")`, and the Turtle file loads unchanged into a triple store such as GraphDB or Jena Fuseki. + +## Notes + +- `query()` returns every value as a string, including literals such as `"false"` and `"9.5"`. An `OPTIONAL` variable that stays unbound comes back as the string `"None"`, not as Python `None`. +- `query_asset_neighbourhood(asset_id)` puts the id into the query as `cmr:`. That works for ids made of letters, digits, `-` and `_`, such as the UUIDs collectors assign, but an id containing `/` or `:` makes the query fail to parse; write the query yourself with the full IRI in angle brackets. The one-hop result includes data properties (`hasName`, `hasRegion`) next to the relations. +- In 0.6.0, `query_asset_neighbourhood()` with `hops` greater than 1 raises a pyparsing `ParseException`: the generated `{1,n}` path quantifier is not SPARQL 1.1, and rdflib rejects it. For a multi-hop walk, use a SPARQL property path such as `cmr:web-1 (cmp:PROTECTED_BY_SG|cmp:CONTAINS)+ ?n` through `query()`, or the MCP `ontology_neighbourhood` tool. +- The tag node name is `tag__` with spaces and `/` replaced by `_`. Other characters pass through into the IRI as they are. +- Each `build()` call adds to the same graph. Create a new `CloudOntology` per inventory. +- The JSON-LD output uses rdflib's serialiser without a framing context, so it is a flat list of nodes. + +## Related + +:::links +- [Graph, ontology and RAG](/guides/graph-ontology-rag/) When to use which view of the inventory. +- [RAGExporter](/api/ragexporter/) Relation-group chunks built from the same inference rules. +- [GraphBuilder](/api/graphbuilder/) The NetworkX view of the same edges. +- [cloudg run](/cli/run/) The `--ontology/--no-ontology` switch and the files it writes. +- [Output files](/reference/output-files/) Every file the pipeline writes, ontology included. +::: diff --git a/site/content/api/dependencygraph.md b/site/content/api/dependencygraph.md new file mode 100644 index 0000000..dc88fe1 --- /dev/null +++ b/site/content/api/dependencygraph.md @@ -0,0 +1,204 @@ +--- +object: [cloudg.inventory.dependencies.DependencyGraph, cloudg.inventory.dependencies.AssetIndex] +members: + cloudg.inventory.dependencies.DependencyGraph: [find, depends_on, dependents, direct_counts, tree, shared_dependencies, blast_radius] + cloudg.inventory.dependencies.AssetIndex: [add, find, resolve, resolve_finding] +lede: "What does this asset need, and what breaks if it goes away? `DependencyGraph` answers both over any set of assets and edges; `AssetIndex` is the matcher that finds the asset a reference or a finding is about." +--- + +## From edges to dependencies + +Edges in an inventory map point in their natural direction: a function `ASSUMES_ROLE` a role, a queue `INVOKES` a function, a WAF `PROTECTS` an API. That direction is not always the direction of dependency. The function needs its role, so the arrow and the dependency agree. The function also needs the queue that triggers it, but the `INVOKES` arrow points from the queue to the function. + +`DependencyGraph(assets, edges, include_hierarchy=False)` turns every edge into one `dependent -> dependency` arrow, using a fixed table (`cloudg.inventory.dependencies.DEPENDENCY_DIRECTION`): + +| Direction | Edge types | Reads as | +|---|---|---| +| forward: the source depends on the target | `REFERENCES`, `ATTACHED_TO`, `ROUTE`, `PEERING`, `USES_IMAGE`, `ASSUMES_ROLE`, `LOGS_TO`, `LOAD_BALANCER_TARGET`, `GRANTS_ACCESS`, `IAM_POLICY_ATTACHMENT`, `IAM_TRUST` | a function depends on its role, its image, its key, its log group | +| reverse: the target depends on the source | `CONTAINS`, `INVOKES`, `PROTECTS`, `MONITORS`, `MANAGES`, `GOVERNS` | a function depends on the queue that triggers it, an API on the WAF in front of it, a resource on the stack that manages it | +| ignored | `SECURITY_GROUP_RULE`, `NACL_RULE`, `INTERNET_EXPOSED` | network rules are not dependencies | + +```mermaid caption="Natural edges on the left, dependency arrows on the right" +flowchart LR + subgraph natural ["inventory edges"] + Q1["queue"] -->|INVOKES| F1["function"] + F1 -->|ASSUMES_ROLE| R1["role"] + end + subgraph deps ["dependency view"] + F2["function"] -->|needs| Q2["queue"] + F2 -->|needs| R2["role"] + end +``` + +On that view, `depends_on(x)` walks upstream (everything `x` needs) and `dependents(x)` walks downstream (everything that needs `x`, its blast radius). Both are breadth-first, visit each asset once, stop at `max_depth` (default 10) and return `DependencyLink` records with `asset_id`, `via_edge`, `relationship`, `depth` and `parent_id`, the asset one hop closer to the start. + +Edges whose source or target is not in `assets` are skipped. With `include_hierarchy=False`, `CONTAINS` edges into `ORGANIZATION`, `ORG_UNIT` and `CLOUD_ACCOUNT` nodes are skipped too, because otherwise every account would top every blast radius list. `InventoryResult.dependency_graph()` builds the graph with this default. + +This is an availability and change-impact view: what stops working when something is deleted, disabled or misconfigured. Compromise spreads differently. A stolen function credential exposes the role it assumes, which is the forward direction of `ASSUMES_ROLE` in the map itself, not a dependent of the role. For attack paths use the graph analysis in [`CloudGEngine.analyze()`](/api/cloudgengine/) or walk the identity edges directly. + +### The reports + +`tree(asset_id, direction="both", max_depth=3)` nests the two walks into a dict: `asset`, plus `depends_on` and/or `dependents` (pick with `"up"`, `"down"` or `"both"`), each node carrying `id`, `name`, `arn`, `type`, `account_id`, `region`, `via`, `relationship` and `children`. It is what `cloudg deps` prints. + +`shared_dependencies(top=25)` ranks assets by how many distinct assets depend on them directly, ignoring `CONTAINS` and `PEERING` and anything with fewer than two dependents. One KMS key behind forty resources shows up here. + +`blast_radius(candidates=None, top=25)` ranks assets by the size of their full transitive dependent set and adds `accounts_affected` and `internet_exposed_dependents`. Without `candidates` it looks at the `4 × top` assets with the most direct dependents, because walking from every asset in a large map is slow. Pass `candidates` (asset ids) to score exactly the assets you care about. + +`find(ref)` looks up an asset by id, ARN, unique name or unique ARN tail and returns the `CloudAsset`, or `None`. It builds an `AssetIndex` on first use. + +### AssetIndex + +`AssetIndex(assets)` is the one matcher the ontology, the RAG export and the dependency graph share for "which asset is this string about". It tries, in order: + +1. the internal `id` +2. the ARN or resource ID, or an alias registered with `add(..., aliases=[...])`; the first asset indexed under it wins +3. the name, if exactly one asset has it +4. the ARN tail after the last `/`, if exactly one asset has it +5. the ARN resource part and its last `:` or `/` segment (`db:orders`, `orders`), opt-in with `arn_parts=True` +6. the name ignoring case, opt-in with `casefold=True` + +Ambiguous names and tails never match at their own tier. Tiers 5 and 6 are also skipped for a reference that is the exact name of more than one asset. `resolve_finding(finding)` tries both `finding.resource_id` and `finding.resource_arn` at every tier, which matters because scanners often put an ARN or a display name in `resource_id`. + +## Examples + +### Blast radius of a KMS key + +Runs offline on six linked assets. + +```python title="blast_radius.py" +from cloudg.inventory import RelationshipLinker +from cloudg.inventory.dependencies import DependencyGraph +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider + +ACCOUNT = "123456789012" +REGION = "eu-west-1" +KEY = f"arn:aws:kms:{REGION}:{ACCOUNT}:key/1111aaaa-22bb-33cc-44dd-555555eeeeee" +ROLE = f"arn:aws:iam::{ACCOUNT}:role/orders-worker-role" +QUEUE = f"arn:aws:sqs:{REGION}:{ACCOUNT}:orders-inbound" +TABLE = f"arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/orders" +FUNCTION = f"arn:aws:lambda:{REGION}:{ACCOUNT}:function:orders-worker" +API = f"arn:aws:apigateway:{REGION}::/restapis/a1b2c3d4e5" + + +def aws(arn, name, asset_type, *relations, region=REGION, exposed=False): + return CloudAsset(arn=arn, name=name, asset_type=asset_type, provider=CloudProvider.AWS, + region=region, account_id=ACCOUNT, is_internet_exposed=exposed, + metadata={"relations": list(relations)}) + + +def rel(target, edge, relationship=None, reverse=False): + return {"target": target, "edge": edge, "relationship": relationship, "reverse": reverse} + + +assets = [ + aws(KEY, "orders-key", AssetType.KMS_KEY), + aws(ROLE, "orders-worker-role", AssetType.IAM_ROLE, region="global"), + aws(QUEUE, "orders-inbound", AssetType.MESSAGE_QUEUE, rel(KEY, "REFERENCES", "ENCRYPTED_BY_KMS")), + aws(TABLE, "orders", AssetType.DYNAMODB_TABLE, rel(KEY, "REFERENCES", "ENCRYPTED_BY_KMS")), + aws(FUNCTION, "orders-worker", AssetType.LAMBDA_FUNCTION, + rel(ROLE, "ASSUMES_ROLE", "RUNS_ON"), + rel(TABLE, "REFERENCES", "WRITES_TO"), + rel(QUEUE, "INVOKES", "TRIGGERED_BY", reverse=True)), + aws(API, "orders-api", AssetType.API_GATEWAY, rel(FUNCTION, "INVOKES"), exposed=True), +] +edges = RelationshipLinker(assets).link() +graph = DependencyGraph(assets, edges) + +key = graph.find("orders-key") # ID, ARN, unique name or unique ARN tail +print("If", key.name, "is disabled:") +for link in graph.dependents(key.id): + asset = graph.assets[link.asset_id] + parent = graph.assets[link.parent_id].name + print(f" depth {link.depth}: {asset.name} ({asset.asset_type.value}) needs {parent} via {link.via_edge}") + +worker = graph.find(FUNCTION) +print("orders-worker needs:", [graph.assets[l.asset_id].name for l in graph.depends_on(worker.id)]) +print("direct (needs, needed by):", graph.direct_counts(worker.id)) + +for row in graph.blast_radius(top=3): + print(f"{row['name']:20} {row['transitive_dependents']} dependents, " + f"{row['internet_exposed_dependents']} internet-exposed") +``` + +```console +$ python blast_radius.py +If orders-key is disabled: + depth 1: orders-inbound (MESSAGE_QUEUE) needs orders-key via REFERENCES + depth 1: orders (DYNAMODB_TABLE) needs orders-key via REFERENCES + depth 2: orders-worker (LAMBDA_FUNCTION) needs orders-inbound via INVOKES +orders-worker needs: ['orders-worker-role', 'orders', 'orders-inbound', 'orders-api', 'orders-key'] +direct (needs, needed by): (4, 0) +orders-key 3 dependents, 0 internet-exposed +orders-api 1 dependents, 0 internet-exposed +orders 1 dependents, 0 internet-exposed +``` + +Two results are worth a second look. The function appears at depth 2 under the key, reached through the queue rather than through the table, because breadth-first search records the first path it finds. And `orders-api` is in the function's `depends_on` list: `INVOKES` is a reverse edge, so the function depends on whatever invokes it. Read that as "the function receives no traffic if the API goes away". + +### Match findings to assets + +```python title="match_findings.py" +from cloudg.inventory.dependencies import AssetIndex +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, Finding, Severity + + +def asset(arn, name, asset_type): + return CloudAsset(arn=arn, name=name, asset_type=asset_type, + provider=CloudProvider.AWS, region="eu-west-1", account_id="123456789012") + + +assets = [ + asset("arn:aws:rds:eu-west-1:123456789012:db:orders", "orders", AssetType.RDS_INSTANCE), + asset("arn:aws:dynamodb:eu-west-1:123456789012:table/orders", "orders", AssetType.DYNAMODB_TABLE), + asset("arn:aws:lambda:eu-west-1:123456789012:function:Api-Handler", "Api-Handler", + AssetType.LAMBDA_FUNCTION), +] +index = AssetIndex(assets) +names = {a.id: f"{a.asset_type.value}:{a.name}" for a in assets} + + +def finding(resource_id, resource_arn=None): + return Finding(resource_id=resource_id, resource_arn=resource_arn, severity=Severity.MEDIUM, + title="example", description="example", source_tool="custom") + + +cases = [ + finding("x", "arn:aws:rds:eu-west-1:123456789012:db:orders"), # exact ARN + finding("orders"), # name shared by two assets + finding("db:orders"), # ARN resource part + finding("api-handler"), # wrong letter case +] +for f in cases: + default = index.resolve_finding(f) + loose = index.resolve_finding(f, arn_parts=True, casefold=True) + print(f"{f.resource_arn or f.resource_id:48} default={names.get(default)} loose={names.get(loose)}") +``` + +```console +$ python match_findings.py +arn:aws:rds:eu-west-1:123456789012:db:orders default=RDS_INSTANCE:orders loose=RDS_INSTANCE:orders +orders default=DYNAMODB_TABLE:orders loose=DYNAMODB_TABLE:orders +db:orders default=None loose=RDS_INSTANCE:orders +api-handler default=None loose=LAMBDA_FUNCTION:Api-Handler +``` + +The second line shows the limit of tier 3. `orders` is the name of two assets, so the name tier refuses it, but the tail tier still matches: only the DynamoDB ARN ends in `/orders` (the RDS ARN uses `:`). If a reference can be a bare name in an estate where names repeat, check the asset type of what you got back. + +## Notes + +`DependencyGraph` keeps references to the assets you pass (`graph.assets` maps id to `CloudAsset`) and copies nothing. Build a new one after the map changes; there is no way to add edges to an existing graph. + +`direct_counts()` counts edges, not assets. Two parallel edges between the same pair count twice. `shared_dependencies()` counts distinct assets. + +`blast_radius()` drops candidates with no dependents at all, so the list can be shorter than `top`. + +The module also has two functions that work on plain asset and edge lists: `cross_account_edges(assets, edges)` (every edge between two different accounts, hierarchy containment excluded) and `security_coverage(assets, edges)` (security services by account and region, the gaps, workloads without vulnerability scanning, internet-facing entry points without a WAF). `InventoryResult.analysis()` calls both. + +## Related + +:::links +- [Dependencies and blast radius](/guides/dependencies/) The guide, with `cloudg deps`. +- [cloudg deps](/cli/deps/) The CLI front end to `tree()`. +- [InventoryResult](/api/inventoryresult/) `dependency_graph()` and `analysis()`. +- [RelationshipLinker](/api/relationshiplinker/) Where the edges come from. +::: diff --git a/site/content/api/entry-points.md b/site/content/api/entry-points.md new file mode 100644 index 0000000..c4c1c1f --- /dev/null +++ b/site/content/api/entry-points.md @@ -0,0 +1,303 @@ +--- +title: Entry points +object: cloudg.registry.PluginRegistry +members: [get_collector, get_scanner, list_collectors, list_scanners] +lede: Ship a collector or a scanner as its own package, and `PluginRegistry` finds it through Python entry points. +--- + +cloudg looks for plugins in two entry point groups, `cloudg.collectors` and `cloudg.scanners`. A package that declares an entry point in one of them becomes visible to `PluginRegistry` as soon as it is installed in the same environment, with no change to cloudg. cloudg registers its own built-ins the same way in its `pyproject.toml`: + +```toml title="pyproject.toml" +[project.entry-points."cloudg.collectors"] +aws = "cloudg.collectors.aws:AsyncAWSCollector" +azure = "cloudg.collectors.azure:AzureCollector" +gcp = "cloudg.collectors.gcp:GCPCollector" + +[project.entry-points."cloudg.scanners"] +prowler = "cloudg.scanners.prowler:ProwlerScanner" +scoutsuite = "cloudg.scanners.scoutsuite:ScoutSuiteScanner" +checkov = "cloudg.scanners.checkov:CheckovScanner" +trivy = "cloudg.scanners.trivy:TrivyScanner" +iam = "cloudg.scanners.iam_linter:IAMLinter" +``` + +:::warning The CLI does not load plugins in 0.6.0 +`cloudg run`, `cloudg scan` and `cloudg map` choose their collectors and scanners by name in their own code and never ask `PluginRegistry`. A plugin scanner's name in `--scanners` or `scanners.enabled` is silently ignored, and `-p` accepts only `aws`, `azure` and `gcp`. Drive plugins from Python, as in the example below, and hand the results to the normaliser and renderers. The handbook's statement that an installed scanner "is available in `scanners.enabled` and `--scanners`" does not match the code. +::: + +## How discovery works + +Discovery is lazy and happens once per `PluginRegistry` instance, on the first `get_*` or `list_*` call. + +```mermaid caption="What the first get or list call does" +flowchart TD + A["First get_* or list_* call"] --> B["Read entry points in cloudg.collectors"] + B --> C{"ep.load() works?"} + C -->|yes| D["Register under the entry point name"] + C -->|no| E["Log a warning, skip it"] + D --> F["Same for cloudg.scanners"] + E --> F + F --> G{"Built-in name still missing?"} + G -->|yes| H["Import it from the fallback table"] + G -->|no| I["Cached for this registry"] + H --> I +``` + +The fallback table in `cloudg/registry.py` maps the same eight names as the entry points above to the same classes. It matters when cloudg runs from a source checkout that was never installed, so its own entry point metadata is missing; in a normal install the built-ins arrive through their entry points and the fallback adds nothing. A built-in whose import fails (a missing optional SDK, say) is left out quietly, at debug log level. + +What the registry returns is the class itself. It does not instantiate anything and does not check the class's interface, so constructor arguments are between you and the plugin. + +Two more behaviours to know before you name a plugin: + +- Don't reuse a built-in name. cloudg's own `prowler` entry point and yours would both be loaded under the same key, and whichever `importlib.metadata` lists last wins. That order is not something you control. +- A registry instance never looks again. After installing a plugin into a running process, create a new `PluginRegistry`. + +## Writing a plugin + +### The interfaces + +Collectors subclass `cloudg.collectors.base.BaseCollector`, an abstract class with two coroutines to implement and one convenience method: + +| Method | Returns | Notes | +|---|---|---| +| `async collect()` | `list[CloudAsset]` | abstract | +| `async collect_edges()` | `list[NetworkEdge]` | abstract | +| `async run()` | `(assets, edges)` | calls the two above in order | + +Scanners have no base class in cloudg. The built-ins share an informal shape, and following it keeps your scanner interchangeable with them: a static `is_available()` that says whether the tool can run, `run()` returning `list[Finding]`, and a class method `parse_report(path)` that turns saved output back into findings (what `cloudg ingest` relies on for the built-ins). `IAMLinter` and `TrivyScanner` deviate from it (`analyze_policies(assets)`, `scan_images(images)`), so code that drives scanners generically should not assume more than the shape you define. + +Every asset needs a `provider` from `CloudProvider`, which has three values: `AWS`, `AZURE` and `GCP`. A collector for another platform has to file its assets under one of them. + +### A complete plugin package + +The package below adds a `cmdb` collector, which reads assets from a CMDB export file instead of a cloud API, and a `tagpolicy` scanner that flags assets without an `owner` tag. Both run offline. + +```text +cloudg-cmdb/ +├── pyproject.toml +└── src/ + └── cloudg_cmdb/ + ├── __init__.py + ├── collector.py + └── scanner.py +``` + +```toml title="cloudg-cmdb/pyproject.toml" +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "cloudg-cmdb" +version = "0.1.0" +description = "cloudg plugins: a CMDB export collector and a tag policy scanner" +requires-python = ">=3.11" +dependencies = ["cloudg>=0.6.0"] + +[project.entry-points."cloudg.collectors"] +cmdb = "cloudg_cmdb.collector:CMDBCollector" + +[project.entry-points."cloudg.scanners"] +tagpolicy = "cloudg_cmdb.scanner:TagPolicyScanner" +``` + +```python title="cloudg-cmdb/src/cloudg_cmdb/__init__.py" +"""cloudg plugins for a CMDB export.""" +``` + +The collector does not build edges between assets itself. It declares what each asset talks to in `metadata["relations"]` and lets cloudg's `RelationshipLinker` resolve those references into typed edges, the way the built-in collectors do. + +```python title="cloudg-cmdb/src/cloudg_cmdb/collector.py" +"""Collector that reads assets from a CMDB export instead of a cloud API.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from cloudg.collectors.base import BaseCollector +from cloudg.inventory.linker import RelationshipLinker +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, NetworkEdge + + +class CMDBCollector(BaseCollector): + """Turns a CMDB JSON export into CloudAssets and linked edges.""" + + def __init__(self, export_path: str | Path) -> None: + self._path = Path(export_path) + self._assets: list[CloudAsset] | None = None + + async def collect(self) -> list[CloudAsset]: + rows = await asyncio.to_thread(self._read) + self._assets = [ + CloudAsset( + name=row["name"], + arn=row["arn"], + asset_type=AssetType(row["type"]), + provider=CloudProvider(row.get("provider", "AWS")), + region=row.get("region", "global"), + account_id=row.get("account"), + tags=row.get("tags", {}), + metadata={ + # The linker turns these into typed edges + "relations": [ + {"target": target, "edge": edge} + for edge, targets in row.get("links", {}).items() + for target in targets + ], + }, + ) + for row in rows + ] + return self._assets + + async def collect_edges(self) -> list[NetworkEdge]: + if self._assets is None: + await self.collect() + return RelationshipLinker(self._assets or []).link() + + def _read(self) -> list[dict]: + return json.loads(self._path.read_text())["assets"] +``` + +```python title="cloudg-cmdb/src/cloudg_cmdb/scanner.py" +"""Scanner that flags assets missing required tags.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from cloudg.schema.models import CloudAsset, Finding, Severity + +CHECK_ID = "TAGPOLICY_001" + + +class TagPolicyScanner: + """Follows the built-in scanners' shape: is_available, run, parse_report.""" + + def __init__(self, assets: list[CloudAsset], required: tuple[str, ...] = ("owner",)) -> None: + self._assets = assets + self._required = required + + @staticmethod + def is_available() -> bool: + return True # pure Python, nothing to install + + def run(self) -> list[Finding]: + findings = [] + for asset in self._assets: + missing = [t for t in self._required if t not in asset.tags] + if missing: + findings.append( + Finding( + resource_id=asset.id, + resource_arn=asset.arn, + severity=Severity.LOW, + title=f"Missing required tags: {', '.join(missing)}", + description=f"{asset.name} has no {', '.join(missing)} tag.", + remediation="Tag the resource with an owning team.", + source_tool="tagpolicy", + source_finding_id=CHECK_ID, + ) + ) + return findings + + @classmethod + def parse_report(cls, path: str) -> list[Finding]: + """Read back findings this scanner wrote earlier as JSON.""" + return [Finding.model_validate(item) for item in json.loads(Path(path).read_text())] +``` + +Install it into the environment cloudg runs in, editable while you work on it: + +```bash tab="pip" +pip install -e ./cloudg-cmdb +``` + +```bash tab="uv" +uv pip install -e ./cloudg-cmdb +``` + +### Use the plugins + +A CMDB export with three assets, one of which declares that it runs as a role: + +```json title="cmdb-export.json" +{ + "assets": [ + {"name": "billing-vm", "type": "EC2", "region": "eu-west-1", "account": "123456789012", + "arn": "arn:aws:ec2:eu-west-1:123456789012:instance/i-0billing", + "tags": {"owner": "billing"}, + "links": {"ASSUMES_ROLE": ["arn:aws:iam::123456789012:role/billing-app"]}}, + {"name": "billing-app", "type": "IAM_ROLE", "region": "global", "account": "123456789012", + "arn": "arn:aws:iam::123456789012:role/billing-app"}, + {"name": "invoices", "type": "S3_BUCKET", "region": "eu-west-1", "account": "123456789012", + "arn": "arn:aws:s3:::invoices"} + ] +} +``` + +The driver finds both plugins by name, collects, scans, runs the findings through the same normaliser `cloudg run` uses, and writes a `findings.json`: + +```python title="run_plugins.py" +"""Discover the plugins, collect, scan, normalise and write findings.json.""" + +import asyncio + +from cloudg.normaliser import FindingsNormaliser +from cloudg.registry import PluginRegistry +from cloudg.renderers.json_export import JSONExporter + +registry = PluginRegistry() +print("collectors:", registry.list_collectors()) +print("scanners:", registry.list_scanners()) + +collector = registry.get_collector("cmdb")("cmdb-export.json") +assets, edges = asyncio.run(collector.run()) +for e in edges: + print("edge:", e.edge_type.value, e.description or "") + +scanner = registry.get_scanner("tagpolicy")(assets, required=("owner",)) +findings = scanner.run() if scanner.is_available() else [] +result = FindingsNormaliser().normalise(findings, assets=assets) +result.edges = edges +for f in result.findings: + print(f"{f.severity.value:4} {f.title} ({f.resource_arn})") + +path = JSONExporter(output_dir="reports").export(result) +print("wrote", path) +``` + +```console +$ python run_plugins.py +collectors: ['cmdb', 'aws', 'azure', 'gcp'] +scanners: ['tagpolicy', 'checkov', 'iam', 'prowler', 'scoutsuite', 'trivy'] +edge: ASSUMES_ROLE +LOW Missing required tags: owner (arn:aws:iam::123456789012:role/billing-app) +LOW Missing required tags: owner (arn:aws:s3:::invoices) +wrote reports/findings.json +$ cloudg report -i reports/findings.json -o reports/html --format html +``` + +The list order follows entry point discovery, so yours may differ. The last command renders `report.html` from the plugin's findings; `cloudg report` reads any `findings.json` in this shape, whoever wrote it. + +To merge your scanner's findings with an equivalent check from Prowler, Checkov or Trivy, add your `source_tool` and `source_finding_id` to a `check_equivalence.yaml` and point the normaliser at it with `FindingsNormaliser(rules_dir=...)` (or `rulesets.rules_dir` in `config.yaml`). That directory replaces the bundled `cloudg/rules` rather than adding to it, so start from a copy of it. + +## Notes + +- A plugin that fails to import is logged as a warning (`Failed to load scanner plugin ...`) and skipped; the others still load. It goes to the `cloudg.registry` logger, so configure `logging` in your script to see it. +- `get_collector()` and `get_scanner()` raise `KeyError` for an unknown name, and the message lists the names that are registered. +- The relation format the linker reads (`target`, `edge`, `relationship`, `reverse`, `properties`) is specified in section 7 of the inventory reference, and the [RelationshipLinker](/api/relationshiplinker/) page shows the linker itself. Unknown `edge` values are linked as `REFERENCES`. +- Assets from a plugin collector work with everything downstream: [GraphBuilder](/api/graphbuilder/), the reachability analysis, the ontology, and `InventoryResult(assets=..., edges=...).export(dir)` for an `inventory-map.json` that `cloudg deps` can read. + +## Related + +:::links +- [FindingsNormaliser](/api/findingsnormaliser/) Dedupe, scoring and compliance mapping for any findings list. +- [Data models](/api/models/) `CloudAsset`, `NetworkEdge` and `Finding`, field by field. +- [RelationshipLinker](/api/relationshiplinker/) How declared relations become edges. +- [Python recipes](/guides/python-recipes/) More ways to combine the pipeline phases from code. +- [cloudg report](/cli/report/) Render HTML and SVG from a `findings.json`. +::: diff --git a/site/content/api/event-hooks.md b/site/content/api/event-hooks.md new file mode 100644 index 0000000..26351ba --- /dev/null +++ b/site/content/api/event-hooks.md @@ -0,0 +1,257 @@ +--- +title: Event hooks +lede: "Six optional callbacks on `CloudGEngine` report phase changes, findings, results and failures while a run is in progress. They are plain attributes: assign a function, and the engine calls it." +--- + +## The six hooks + +Every hook starts as `None` in `CloudGEngine.__init__`. Assign a callable before you call a method, and reassign or reset it to `None` between runs as you like. The callback types are defined in `cloudg.api` as `typing.Callable` aliases, so you can annotate your own handlers with them. + +| Hook | Type alias | Signature | Fires when | +|---|---|---|---| +| `on_phase_start` | `OnPhaseStart` | `(phase: str) -> None` | a phase begins: `collection`, `inventory_mapping`, `scanning`, `ingest`, `analysis`, `normalisation`, `reporting` | +| `on_collection_complete` | `OnCollectionComplete` | `(result: CollectionResult) -> None` | `collect()` finished without the collector raising | +| `on_finding` | `OnFinding` | `(finding: Finding) -> None` | once per finding, at the end of `scan()` or `ingest_reports()` | +| `on_scan_complete` | `OnScanComplete` | `(findings: list[Finding]) -> None` | right after the last `on_finding` call of `scan()` or `ingest_reports()` | +| `on_analysis_complete` | `OnAnalysisComplete` | `(result: AnalysisResult) -> None` | at the end of every `analyze()`, including one where every step failed | +| `on_error` | `OnError` | `(phase: str, exc: Exception) -> None` | a phase, a scanner or an analysis step failed and the engine carried on | + +```python +from cloudg.api import OnError, OnFinding + +def forward(finding) -> None: ... +def alert(phase: str, exc: Exception) -> None: ... + +on_finding: OnFinding = forward +on_error: OnError = alert +``` + +## When each one fires + +The diagram follows `run_pipeline()`, the one method that can call all six. + +```mermaid caption="Hook calls during run_pipeline()" +sequenceDiagram + participant App as Your code + participant E as CloudGEngine + App->>E: run_pipeline() + E->>App: on_phase_start("collection") + E->>App: on_collection_complete(CollectionResult) + E->>App: on_phase_start("scanning") + Note over E: scanners run in a thread pool + E->>App: on_error("prowler-aws", exc), per failed scanner + E->>App: on_finding(Finding), once per finding + E->>App: on_scan_complete(list of Finding) + E->>App: on_phase_start("analysis") + E->>App: on_analysis_complete(AnalysisResult) + E->>App: on_phase_start("normalisation") + E->>App: on_phase_start("reporting") + E-->>App: PipelineResult +``` + +Other entry points use a subset: + +| Method | Phases announced | Other hooks it can call | +|---|---|---| +| `run_pipeline()` | collection, scanning, analysis, normalisation, reporting | all six | +| `run_from_reports()` | ingest, normalisation, reporting | `on_finding`, `on_scan_complete`, `on_error` | +| `collect()` | collection | `on_collection_complete`, `on_error` | +| `scan()` | scanning | `on_finding`, `on_scan_complete`, `on_error` | +| `ingest_reports()` | ingest | `on_finding`, `on_scan_complete` | +| `analyze()` | analysis | `on_analysis_complete`, `on_error` | +| `map_inventory()` | inventory_mapping | `on_error`, then the exception is raised again | +| `normalise_findings()` | none | none | + +`InventoryMapper`, `FindingsNormaliser` and the other classes you can use without the engine have no hooks at all. + +### What the phase argument of on_error can be + +`on_error` receives a phase name that tells you which part failed, which is more specific than the names `on_phase_start` announces: + +| `phase` | Raised by | +|---|---| +| `collection` | the multi-provider collector itself (a single provider that fails is recorded in `coverage` instead) | +| `inventory_mapping` | `map_inventory()` | +| `graph_analysis` | the reachability pass at the start of `scan()` | +| `terraform` | generating the Terraform recreation, in `scan()` (for Checkov) or in `analyze()` | +| `prowler-`, `scoutsuite-` | one scanner run per provider, such as `prowler-aws` | +| `checkov-` | one Checkov run per IaC directory | +| `trivy`, `iam` | the Trivy image scan, the IAM policy linter | +| `graph_build`, `ontology`, `rag_export` | the steps of `analyze()` | +| `normalisation`, `reporting` | the last two pipeline phases; these also land in `PipelineResult.errors` | + +### Timing + +All hooks run synchronously, on the thread that runs the engine method, and the engine waits for each one to return. A slow callback slows the run down by exactly that much. + +`on_finding` is not a live feed from the scanners. `scan()` waits for every scanner in its thread pool to finish, then calls `on_finding` for each collected finding in one burst, then `on_scan_complete`. The findings at that point are raw: not yet deduplicated, rescored or mapped to compliance frameworks. Normalisation later updates those same objects in place, so a finding you stored by reference may change severity after your hook saw it. If you forward findings, serialise them in the hook (`finding.model_dump(mode="json")`) to keep what you were given, or forward `result.findings` after the run to get the final list. + +### Exceptions inside a hook + +The engine wraps every hook call in `try/except Exception`. An exception your callback raises is logged and dropped, and the run continues. For `on_finding` that happens per finding, so a handler that fails on every finding is called, and fails, once for each of them. + +The log record is a DEBUG message, `Event hook raised; ignoring`, with the traceback attached, on the `cloudg.api` logger (or `cloudg.api_scanners` for `on_finding` and `on_scan_complete` during a scan). At the default log level you will not see it. While you develop a hook, turn it on: + +```python +import logging + +logging.basicConfig() +logging.getLogger("cloudg.api").setLevel(logging.DEBUG) +logging.getLogger("cloudg.api_scanners").setLevel(logging.DEBUG) +``` + +This also means raising from a hook cannot stop a run. The fail-fast example below shows what to do instead. + +## Examples + +### Progress bar with rich + +`rich` is already a cloudg dependency. Each phase start advances the bar past the previous phase; `console.log` prints above the bar without breaking it. + +```python title="pipeline_progress.py" +from rich.progress import Progress + +from cloudg import CloudGConfig, CloudGEngine + +PHASES = ["collection", "scanning", "analysis", "normalisation", "reporting"] + +engine = CloudGEngine(CloudGConfig(providers=["aws"])) + +with Progress() as progress: + task = progress.add_task("starting", total=len(PHASES)) + seen = [] + + def on_phase_start(phase): + if seen: # the previous phase has finished + progress.advance(task) + seen.append(phase) + progress.update(task, description=phase) + + engine.on_phase_start = on_phase_start + engine.on_collection_complete = lambda c: progress.console.log( + f"collected {len(c.assets)} assets, {len(c.edges)} edges in {c.duration_ms} ms" + ) + engine.on_scan_complete = lambda findings: progress.console.log(f"{len(findings)} raw findings") + engine.on_error = lambda phase, exc: progress.console.log(f"[red]{phase} failed:[/] {exc}") + + result = engine.run_pipeline_sync(output_dir="./reports") + progress.update(task, completed=len(PHASES), description="done") + +print(result.to_summary()) +``` + +For `run_from_reports()` use `PHASES = ["ingest", "normalisation", "reporting"]`. + +### Forward findings to a SIEM + +The hook only puts each finding on a queue; a background thread does the writing, so a slow sink never holds up the engine. Here the sink is a JSON Lines spool file that a log shipper tails. Swap `writer()` for an HTTP client if your SIEM takes events directly. This example runs offline against existing Prowler and Trivy output. + +```python title="siem_forward.py" +import json +import queue +import threading + +from cloudg import CloudGConfig, CloudGEngine + +SPOOL = "./cloudg-findings.jsonl" # tailed by the SIEM agent (Fluent Bit, Splunk UF, ...) +STOP = object() +outbox: queue.Queue = queue.Queue() + + +def writer() -> None: + with open(SPOOL, "a", encoding="utf-8") as spool: + while (item := outbox.get()) is not STOP: + spool.write(json.dumps(item) + "\n") + spool.flush() + + +thread = threading.Thread(target=writer, daemon=True) +thread.start() + +engine = CloudGEngine(CloudGConfig()) +engine.on_finding = lambda f: outbox.put({"kind": "finding", **f.model_dump(mode="json")}) +engine.on_error = lambda phase, exc: outbox.put( + {"kind": "error", "phase": phase, "error": f"{type(exc).__name__}: {exc}"} +) + +result = engine.run_from_reports_sync( + {"prowler": ["./prowler-output/"], "trivy": ["./trivy.json"]}, + output_dir="./reports", +) + +outbox.put(STOP) +thread.join(timeout=30) +print(f"forwarded {result.total_findings} findings to {SPOOL}") +``` + +Each line is one finding as `Finding.model_dump(mode="json")` produces it (see [Data models](/api/models/) for the fields), plus `"kind": "finding"`. + +### Fail fast + +Raising inside `on_error` does nothing, because the engine swallows it. Record the failure instead, and check between phases you drive yourself. `run_pipeline()` has no such checkpoints, so this uses the individual phase methods. + +```python title="fail_fast.py" +import asyncio +import sys + +from cloudg import CloudGConfig, CloudGEngine + + +class PhaseFailed(RuntimeError): + pass + + +async def main() -> int: + engine = CloudGEngine(CloudGConfig(providers=["aws"])) + failures: list[tuple[str, Exception]] = [] + engine.on_error = lambda phase, exc: failures.append((phase, exc)) + + def checkpoint(step: str) -> None: + if failures: + phase, exc = failures[0] + raise PhaseFailed(f"{phase} failed during {step}: {exc}") + + collection = await engine.collect() + checkpoint("collection") + broken = [c.to_summary() for c in collection.coverage if c.failed_services] + if broken or not collection.assets: + raise PhaseFailed(f"collection incomplete: {broken or 'no assets'}") + + findings = await engine.scan(collection.assets, collection.edges, output_dir="./reports") + checkpoint("scanning") + + analysis = await engine.analyze(collection.assets, collection.edges, findings, "./reports") + checkpoint("analysis") + + scan_result = engine.normalise_findings( + findings + analysis.reachability_findings, assets=collection.assets + ) + print(scan_result.summary) + return 0 + + +try: + sys.exit(asyncio.run(main())) +except PhaseFailed as exc: + print(f"cloudg: {exc}", file=sys.stderr) + sys.exit(2) +``` + +A failed provider never reaches `on_error`; it is a `FAILED` entry in `collection.coverage`. That is why the script checks coverage as well as the failure list after collecting. On a failure the script prints one line in the form `cloudg: failed during : ` to stderr and exits with status 2, so a CI job or a scheduler sees the run as failed. + +## Notes + +Hooks are per engine instance. Two engines in one process do not share them, and there is no global registry. + +`on_collection_complete` does not fire when `collect()` fails as a whole; you get `on_error("collection", exc)` and an empty `CollectionResult` instead. `on_analysis_complete` fires even when every analysis step failed, with whatever was filled in. + +Nothing fires for findings produced by `analyze()`. Its reachability findings reach you through `AnalysisResult.reachability_findings` and, after `run_pipeline()`, through `PipelineResult.findings`. + +## Related + +:::links +- [CloudGEngine](/api/cloudgengine/) The methods that call the hooks. +- [Result objects](/api/results/) What `on_collection_complete` and `on_analysis_complete` receive. +- [Data models](/api/models/) The `Finding` that `on_finding` receives. +- [CI and automation](/guides/ci-automation/) Running cloudg unattended. +::: diff --git a/site/content/api/findingsnormaliser.md b/site/content/api/findingsnormaliser.md new file mode 100644 index 0000000..2807344 --- /dev/null +++ b/site/content/api/findingsnormaliser.md @@ -0,0 +1,179 @@ +--- +object: cloudg.normaliser.FindingsNormaliser +members: [normalise] +lede: "Takes findings from any number of scanners and returns one `ScanResult`: duplicates merged, severities adjusted by CVSS, findings sorted, and compliance controls mapped." +--- + +## What normalise does + +`FindingsNormaliser(rules_dir=None)` loads its rules once, at construction: every `*.yaml` file under `rules_dir` (searched recursively) as a compliance ruleset, and `check_equivalence.yaml` from the top of `rules_dir` as the cross-scanner equivalence map. With `rules_dir=None` it uses the directory shipped inside the package, `cloudg/rules/`, which holds CIS, NIST 800-53, PCI DSS, SOC 2, ISO 27001, HIPAA and GDPR rulesets plus the per-provider framework mappings under `frameworks/`. + +`normalise(*finding_lists, assets=None)` accepts any number of lists, one per source, and runs them through one pass: + +```mermaid caption="FindingsNormaliser.normalise()" +flowchart TD + A["all lists concatenated"] --> B["pass 1: dedupe within a scanner"] + B --> C["pass 2: merge across scanners"] + C --> D["CVSS adjusts severity"] + D --> E["compliance frameworks added to each finding"] + E --> F["sort CRITICAL to INFO"] + F --> G["one ComplianceResult per framework control"] + G --> H["ScanResult"] +``` + +### Deduplication + +The first pass works inside one scanner. Findings with the same `source_tool`, the same check ID and the same resource (`resource_arn`, or `resource_id` when there is no ARN) are one finding reported twice, for example once per output format. The check ID is the scanner's own: the check name embedded in a Prowler ASFF `Id` (`s3_bucket_default_encryption` out of `prowler-aws-s3_bucket_default_encryption-123456789012-...`), or `source_finding_id` as it is for Checkov, Trivy and ScoutSuite. A finding without a check ID is keyed on its normalised title instead, so two different checks that happen to share a generic title are not collapsed. + +The second pass works across scanners, and it is deliberately strict. Two findings from different tools merge only when they are about the same resource, their titles are equal after normalisation (lower case, leading `[Checkov/terraform]`-style tags removed, punctuation collapsed), and both check IDs map to the same canonical ID in `check_equivalence.yaml`. Findings without check IDs merge with each other on title alone. A finding with a check ID never merges with one without. When in doubt, cloudg keeps both: a visible duplicate is better than a scanner's result quietly disappearing. + +A merge keeps the finding with the higher severity, joins the tool names into `source_tool` (`"trivy, prowler"`) and unions `compliance_frameworks`. The survivor keeps its own `id`, title and `source_finding_id`. + +### Scoring + +A `cvss_score` of 9.0 or more sets the severity to `CRITICAL`. A score from 7.0 to 8.9 raises anything below `HIGH` to `HIGH`. Severity never goes down. `risk_score` on the finding is computed from severity and CVSS whenever you read it; see [Data models](/api/models/). + +### Compliance mapping + +Frameworks are added to `finding.compliance_frameworks` from four sources, in this order: + +1. what the scanner reported itself (Prowler's `RelatedRequirements`, for example), already on the finding when it arrives +2. an exact match of the check ID against the `checks` lists in the rulesets, which also yields the real control ID and title +3. only if the finding still has no framework: the `patterns` regexes in the rulesets, matched against title and description +4. only if that found nothing either: a small built-in regex table for CIS, NIST 800-53, PCI DSS, GDPR, SOC 2 and HIPAA + +Then each (framework, control) pair becomes one `ComplianceResult` with the ids of the findings behind it. For exact matches the control ID and title come from the ruleset. Otherwise the control ID is built from the finding: `/CKV_...` for Checkov, `/CVE-...` for Trivy CVEs, a Prowler-derived ID, or `-aggregate` when nothing better exists. Pattern matches always land in the aggregate control. + +Every `ComplianceResult` this produces has status `FAIL`, because it is built from failing findings. Controls without findings are not listed, so the absence of a control does not mean it passed. + +## Examples + +### What merging and scoring do + +Five findings about one bucket and one image, constructed by hand so each rule shows. Runs offline. + +```python title="normalise_demo.py" +from cloudg.normaliser import FindingsNormaliser +from cloudg.schema.models import Finding, Severity + +BUCKET = "arn:aws:s3:::orders-archive" + + +def finding(tool, check, title, severity, **extra): + return Finding(resource_id=BUCKET, resource_arn=BUCKET, source_tool=tool, + source_finding_id=check, title=title, description=title, + severity=severity, **extra) + + +prowler = [ + finding("prowler", "prowler-aws-s3_bucket_default_encryption-123456789012-eu-west-1-orders-archive", + "S3 bucket default encryption", Severity.MEDIUM), + # the same check reported twice (two output formats, say): a within-scanner duplicate + finding("prowler", "prowler-aws-s3_bucket_default_encryption-123456789012-eu-west-1-orders-archive", + "S3 bucket default encryption", Severity.MEDIUM), +] +trivy = [ + # equivalent check (AVD-AWS-0088), same resource, same normalised title + finding("trivy", "AVD-AWS-0088", "[Trivy] S3 bucket default encryption", Severity.HIGH), + # a CVE with a CVSS score of 9.8 reported as MEDIUM + finding("trivy", "CVE-2024-0001", "openssl: buffer overflow", Severity.MEDIUM, cvss_score=9.8), +] +untagged = [finding("custom", None, "CloudTrail logging disabled in eu-west-1", Severity.LOW)] + +result = FindingsNormaliser().normalise(prowler, trivy, untagged) +for f in result.findings: + print(f"{f.severity.value:8} {f.risk_score:4} {f.source_tool:16} {f.title} {f.compliance_frameworks[:4]}") +print(len(result.compliance), "compliance results") +``` + +```console +$ python normalise_demo.py +CRITICAL 9.7 trivy openssl: buffer overflow ['HIPAA', 'NIST-800-53', 'PCI-DSS', 'SOC2'] +HIGH 7.5 trivy, prowler [Trivy] S3 bucket default encryption ['CIS-AWS', 'GDPR', 'ISO-27001', 'SOC2'] +LOW 2.5 custom CloudTrail logging disabled in eu-west-1 ['GDPR', 'HIPAA', 'ISO-27001', 'NIST-800-53'] +11 compliance results +``` + +Five findings in, three out. The two Prowler copies collapsed in pass 1. The result merged with the Trivy check in pass 2, because `s3_bucket_default_encryption` and `AVD-AWS-0088` share the canonical ID `aws-s3-default-encryption` and the titles match once `[Trivy]` is dropped. The Trivy finding survived because it was `HIGH`. The CVE went from `MEDIUM` to `CRITICAL` on its CVSS score. + +### Add your own framework + +`rules_dir` replaces the packaged rules; it does not extend them. To keep the built-in rulesets and the equivalence map, copy them and add your file to the copy: + +```python title="custom_rules.py" +import shutil +from pathlib import Path + +import cloudg +from cloudg import CloudGConfig, CloudGEngine +from cloudg.ingest import parse_report + +# Start from the rules cloudg ships, then add one file of your own +rules = Path("./rules") +shutil.copytree(Path(cloudg.__file__).parent / "rules", rules, dirs_exist_ok=True) +(rules / "acme_baseline.yaml").write_text( + """\ +framework: ACME-BASELINE +controls: + - id: ACME-ENC-01 + title: Data at rest is encrypted with a managed key + checks: [s3_bucket_default_encryption, CKV_AWS_19, AVD-AWS-0088] + - id: ACME-LOG-02 + title: Audit logging is enabled + patterns: ["cloudtrail", "audit log"] +""" +) + +config = CloudGConfig() +config.rulesets.rules_dir = str(rules) +engine = CloudGEngine(config) + +findings = parse_report("prowler", "./prowler-output/") + parse_report("checkov", "./results_json.json") +scan_result = engine.normalise_findings(findings) + +for c in scan_result.compliance: + if c.framework == "ACME-BASELINE": + print(c.control_id, "|", c.control_title, "|", c.status.value, len(c.finding_ids), "findings") +``` + +```console +$ python custom_rules.py +ACME-ENC-01 | Data at rest is encrypted with a managed key | FAIL 2 findings +``` + +A ruleset file has a `framework` name (the file name is used when it is missing) and a list of `controls`, each with an `id`, an optional `title`, and `checks` (exact scanner check IDs), `patterns` (regexes) or both. Several files may name the same framework; their controls are combined. `CloudGEngine.normalise_findings()` builds the normaliser from `config.rulesets.rules_dir`, so setting the config is enough; standalone, pass `FindingsNormaliser(rules_dir="./rules")`. + +To teach the merger that two checks are the same, add an entry to `check_equivalence.yaml` in your copy: + +```yaml title="rules/check_equivalence.yaml" +equivalences: + - id: aws-s3-default-encryption + title: S3 bucket default encryption at rest + checks: + prowler: [s3_bucket_default_encryption] + checkov: [CKV_AWS_19] + trivy: [AVD-AWS-0088] +``` + +Scanner names are the lower-case `source_tool` values. + +## Notes + +`normalise()` changes the findings you pass in. Severity, `source_tool` and `compliance_frameworks` are updated on the original objects, and the duplicates that lost a merge are simply left out of the result. Pass copies (`[f.model_copy(deep=True) for f in findings]`) if you need the originals untouched. + +The merge keeps the survivor's `source_finding_id`, and the exact check-ID mapping runs after the merge. When a Trivy finding survives over a Prowler one, the Prowler check name is no longer there to look up, and the controls it would have matched are not added. Frameworks the scanners reported natively are kept, because those are unioned during the merge. + +A `rules_dir` that does not exist is not an error. The normaliser then has no rulesets and no equivalence map, and only the built-in regex table maps frameworks. Check the path when compliance results look thin. + +Merging across scanners needs the same resource identifier on both sides. Checkov reports Terraform addresses (`aws_s3_bucket.data`) and Prowler reports ARNs, so an equivalent Checkov and Prowler finding normally stay two findings. + +The normaliser keeps the last result on the instance, but nothing reads it back; treat `normalise()` as a function and use its return value. One instance can normalise any number of batches. + +## Related + +:::links +- [Compliance frameworks](/guides/compliance/) Which frameworks cloudg maps and how to read the results. +- [parse_report](/api/parse-report/) Getting findings out of scanner output. +- [Data models](/api/models/) Finding, ComplianceResult and ScanResult. +- [Ingesting existing output](/guides/ingesting/) The CLI path through the same normaliser. +::: diff --git a/site/content/api/graphbuilder.md b/site/content/api/graphbuilder.md new file mode 100644 index 0000000..1305e51 --- /dev/null +++ b/site/content/api/graphbuilder.md @@ -0,0 +1,218 @@ +--- +title: GraphBuilder +object: cloudg.graph.builder.GraphBuilder +members: [build, graph, find_attack_paths, find_lateral_movement_paths, compute_centrality, subgraph, to_d3_json, to_cytoscape_json, to_json_str, save_graphml, load_graphml] +lede: Turns a list of `CloudAsset` and `NetworkEdge` objects into a NetworkX `DiGraph`, then searches, scores and exports it. +since: "0.6.0" +--- + +`GraphBuilder` is the first thing `cloudg run` does after collection, and `InventoryResult.export()` uses it to write `inventory-map.graphml` and `inventory-graph.json`. It needs no credentials and no network: hand it assets and edges from a collector, a saved `inventory-map.json` or your own code, and it gives you a plain `networkx.DiGraph` you can keep working on with the whole NetworkX library. + +## What build() does + +Each asset becomes a node keyed by `asset.id`. The node carries `name`, `asset_type`, `provider`, `region`, `arn`, `account_id`, `tags` (the tag dict serialised as a JSON string, so GraphML can store it) and `is_internet_exposed`. Metadata and `raw_data` stay behind. + +Each edge then joins `source_id` to `target_id`. Collectors write edges whose endpoints are not assets at all: a CIDR such as `0.0.0.0/0`, an Azure service tag or an unresolved reference. For those, `build()` creates a placeholder node with `asset_type` and `provider` set to `EXTERNAL` and `is_external=True`. + +```mermaid caption="How build() places one NetworkEdge" +flowchart TD + A["NetworkEdge"] --> B{"Endpoints are nodes?"} + B -->|no| C["Add EXTERNAL placeholder"] --> D + B -->|yes| D{"Edge already on this pair?"} + D -->|no| E["Add edge"] + D -->|yes| F{"Rule edge, same CIDR and direction?"} + F -->|yes| G["Merge ports, protocols, descriptions"] + F -->|no| H["Replace attributes"] +``` + +A `DiGraph` holds one edge per ordered pair. Collectors often emit several rules for one pair (two ingress rules from `0.0.0.0/0` to the same security group, for 22 and 443), so since 0.6.0 `SECURITY_GROUP_RULE`, `NACL_RULE` and `INTERNET_EXPOSED` edges on the same pair are merged when their `cidr` and `direction` match: + +| Attribute | After merging rules for 22/TCP and 443/TCP | After merging 22/TCP and an ICMP rule | +|---|---|---| +| `port_range` | `"22,443"` | `"22"` (ICMP adds no ports) | +| `protocol` | `"TCP"` | `"TCP,ICMP"` | +| `description` | both descriptions joined with `"; "` | the same | + +An empty port list on a TCP, UDP or all-protocol filter rule (GCP writes these) means every port, so it adds `0-65535` to the union. With mixed protocols the merged `port_range` says which ports are open from the source, but not which protocol each one is open for. Any other edge type that lands on an occupied pair replaces the earlier edge's attributes, which is what NetworkX does by default. Count edges in the `NetworkEdge` list, not in the graph, when you need the collector's numbers. + +`NetworkEdge.ports` and `properties` are not copied onto the graph. Every edge attribute is a string (`""` when unset), which keeps the graph GraphML-safe. + +## Examples + +### Build, search and export a small estate + +The script builds eight assets and eleven edges by hand, so it runs offline. It also groups the nodes into communities with python-louvain, which cloudg installs; the [RAG exporter](/api/ragexporter/) runs the same `best_partition` call for its community chunks. + +```python title="build_graph.py" +"""Build a graph from a hand-made estate and look at it from a few angles.""" + +import json +from pathlib import Path + +import community as community_louvain # python-louvain, a cloudg dependency + +from cloudg.graph.builder import GraphBuilder +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType, NetworkEdge + +ACCOUNT = "123456789012" + + +def asset(asset_id: str, name: str, asset_type: AssetType, arn_suffix: str, **extra) -> CloudAsset: + return CloudAsset( + id=asset_id, + name=name, + asset_type=asset_type, + provider=CloudProvider.AWS, + region="eu-west-1", + account_id=ACCOUNT, + arn=f"arn:aws:{arn_suffix}", + **extra, + ) + + +assets = [ + asset("sg-alb", "sg-alb", AssetType.SECURITY_GROUP, f"ec2:eu-west-1:{ACCOUNT}:security-group/sg-alb"), + asset("sg-web", "sg-web", AssetType.SECURITY_GROUP, f"ec2:eu-west-1:{ACCOUNT}:security-group/sg-web"), + asset("alb", "web-alb", AssetType.LOAD_BALANCER, f"elasticloadbalancing:eu-west-1:{ACCOUNT}:loadbalancer/app/web-alb/1"), + asset("tg", "web-tg", AssetType.TARGET_GROUP, f"elasticloadbalancing:eu-west-1:{ACCOUNT}:targetgroup/web-tg/1"), + asset("web-1", "web-1", AssetType.EC2, f"ec2:eu-west-1:{ACCOUNT}:instance/i-0web1", is_internet_exposed=True), + asset("web-2", "web-2", AssetType.EC2, f"ec2:eu-west-1:{ACCOUNT}:instance/i-0web2"), + asset("app-role", "app-role", AssetType.IAM_ROLE, f"iam::{ACCOUNT}:role/app-role"), + asset("orders-db", "orders-db", AssetType.RDS_INSTANCE, f"rds:eu-west-1:{ACCOUNT}:db:orders-db"), +] + + +def rule(source: str, target: str, port: str, cidr: str = "", description: str = "") -> NetworkEdge: + return NetworkEdge( + source_id=source, + target_id=target, + edge_type=EdgeType.SECURITY_GROUP_RULE, + port_range=port, + protocol="TCP", + cidr=cidr or None, + direction="ingress", + description=description or None, + ) + + +edges = [ + # Two rules on the same pair: GraphBuilder merges them into one edge + rule("0.0.0.0/0", "sg-alb", "443", "0.0.0.0/0", "HTTPS from anywhere"), + rule("0.0.0.0/0", "sg-alb", "80", "0.0.0.0/0", "HTTP redirect"), + rule("sg-alb", "sg-web", "8080", description="ALB to web tier"), + NetworkEdge(source_id="alb", target_id="sg-alb", edge_type=EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), + NetworkEdge(source_id="web-1", target_id="sg-web", edge_type=EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), + NetworkEdge(source_id="web-2", target_id="sg-web", edge_type=EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), + NetworkEdge(source_id="alb", target_id="tg", edge_type=EdgeType.LOAD_BALANCER_TARGET), + NetworkEdge(source_id="tg", target_id="web-1", edge_type=EdgeType.LOAD_BALANCER_TARGET), + NetworkEdge(source_id="tg", target_id="web-2", edge_type=EdgeType.LOAD_BALANCER_TARGET), + NetworkEdge(source_id="web-1", target_id="app-role", edge_type=EdgeType.IAM_TRUST), + NetworkEdge(source_id="app-role", target_id="orders-db", edge_type=EdgeType.GRANTS_ACCESS), +] + +builder = GraphBuilder() +graph = builder.build(assets, edges) +print(f"{graph.number_of_nodes()} nodes, {graph.number_of_edges()} edges from {len(edges)} NetworkEdges") + +merged = graph.edges["0.0.0.0/0", "sg-alb"] +print("merged rule:", merged["port_range"], "|", merged["description"]) +print("placeholder:", graph.nodes["0.0.0.0/0"]["asset_type"], graph.nodes["0.0.0.0/0"]["is_external"]) + + +def names(path: list[str]) -> str: + return " -> ".join(graph.nodes[n]["name"] for n in path) + + +# Paths follow edges in their stored direction +for path in builder.find_attack_paths("0.0.0.0/0", "sg-web"): + print("internet path:", names(path)) +for path in builder.find_attack_paths("web-1", "orders-db"): + print("identity path:", names(path)) +for path in builder.find_lateral_movement_paths(): + print("lateral path:", names(path)) + +centrality = builder.compute_centrality() +top = sorted(centrality.items(), key=lambda kv: kv[1]["betweenness"], reverse=True)[:3] +for node_id, scores in top: + print(f"betweenness {scores['betweenness']:.3f} {graph.nodes[node_id]['name']}") + +# Louvain works on undirected graphs; the RAG exporter does the same +partition = community_louvain.best_partition(graph.to_undirected(), random_state=42) +groups: dict[int, list[str]] = {} +for node_id, community_id in partition.items(): + groups.setdefault(community_id, []).append(graph.nodes[node_id]["name"]) +for community_id, members in sorted(groups.items()): + print(f"community {community_id}: {sorted(members)}") + +out = Path("graph-out") +builder.save_graphml(out / "estate.graphml") +(out / "estate-d3.json").write_text(builder.to_json_str()) +(out / "estate-cytoscape.json").write_text(json.dumps(builder.to_cytoscape_json(), indent=2)) +print("wrote", sorted(p.name for p in out.iterdir())) +``` + +Output with cloudg 0.6.0 and python-louvain 0.16: + +```console +$ python build_graph.py +9 nodes, 10 edges from 11 NetworkEdges +merged rule: 443,80 | HTTPS from anywhere; HTTP redirect +placeholder: EXTERNAL True +internet path: 0.0.0.0/0 -> sg-alb -> sg-web +identity path: web-1 -> app-role -> orders-db +lateral path: web-1 -> app-role +betweenness 0.080 web-1 +betweenness 0.071 web-tg +betweenness 0.054 app-role +community 0: ['0.0.0.0/0', 'sg-alb'] +community 1: ['sg-web', 'web-2'] +community 2: ['app-role', 'orders-db', 'web-1'] +community 3: ['web-alb', 'web-tg'] +wrote ['estate-cytoscape.json', 'estate-d3.json', 'estate.graphml'] +``` + +Eight assets plus the `0.0.0.0/0` placeholder make nine nodes, and the two rules into `sg-alb` became one edge. There is no internet path to `web-1` here even though the load balancer targets it: `alb -> sg-alb` is an `ATTACHED_TO` edge pointing from the resource to its group, and `find_attack_paths` walks edges only in the direction they are stored. The [reachability analysis](/api/reachabilityanalyzer/) knows which way traffic actually moves; walk `network_flow_graph(graph)` instead when you want flow-aware paths. + +Leave out `random_state` and Louvain may number and split the communities differently from one run to the next. + +### Graph a saved inventory map + +`cloudg map` already writes `inventory-map.graphml`, but you can rebuild the graph from `inventory-map.json` to filter it first. `InventoryResult.load()` accepts the file or the directory holding it. + +```python title="graph_from_map.py" +from cloudg.graph.builder import GraphBuilder +from cloudg.inventory import InventoryResult + +result = InventoryResult.load("./reports") +aws_eu = [a for a in result.assets if a.region.startswith("eu-")] +keep = {a.id for a in aws_eu} +edges = [e for e in result.edges if e.source_id in keep or e.target_id in keep] + +builder = GraphBuilder() +graph = builder.build(aws_eu, edges) +builder.save_graphml("./reports/eu-only.graphml") +print(graph.number_of_nodes(), "nodes,", graph.number_of_edges(), "edges") +``` + +Edges whose other end lies outside the filter bring that end back as an `EXTERNAL` placeholder, because `build()` adds a node for every endpoint it sees. + +## Notes + +- `build()` clears the graph first, so one builder can be reused, but `builder.graph` always holds the last build. +- Past 10,000 assets `build()` logs a warning about memory. The threshold is fixed in code; the `graph.max_nodes_warn` key in `config.yaml` is not read by the builder. +- `find_attack_paths()` runs `nx.all_simple_paths` with a `cutoff` of `max_depth` (default 10). On a large, dense graph the number of simple paths explodes; pick specific endpoints and keep the depth low. An unknown node id gives `[]`, not an exception. +- `find_lateral_movement_paths()` collects paths of at most 8 hops from every node with `is_internet_exposed` or `is_external` set to the target of each `IAM_TRUST` edge, over every edge type, and stops after about 100 paths. Every placeholder counts as a starting point, private CIDRs included. `cloudg run` calls it when `graph.compute_attack_paths` is on and prints only the count. +- `compute_centrality()` returns normalised degree, in-degree, out-degree and betweenness per node. Betweenness is exact (`nx.betweenness_centrality`), so it is slow on graphs of tens of thousands of nodes. +- `subgraph(node_ids)` returns an independent copy, so changes to it do not touch `builder.graph`. +- `save_graphml()` creates missing parent directories and writes with `nx.write_graphml_xml`. `load_graphml()` reads back through `nx.read_graphml`, which returns a `DiGraph` with string attributes; booleans such as `is_internet_exposed` come back as Python booleans because GraphML stores their type. +- The D3 export (`to_d3_json`, `to_json_str`) has `nodes` and `links`. Cytoscape's has `elements.nodes` and `elements.edges` with a smaller attribute set (no ARN or CIDR). The field reference for the D3 form is [inventory-graph.json](/reference/inventory/inventory-graph-json/). + +## Related + +:::links +- [ReachabilityAnalyzer](/api/reachabilityanalyzer/) Internet exposure, flow-aware walks and blast radius over this graph. +- [CloudOntology](/api/cloudontology/) The same assets and edges as RDF triples. +- [RAGExporter](/api/ragexporter/) Chunks built from the graph's communities. +- [Graph, ontology and RAG](/guides/graph-ontology-rag/) How the three analyses fit together. +- [inventory-map.graphml](/reference/inventory/inventory-map-graphml/) Node and edge attributes, field by field. +::: diff --git a/site/content/api/inventorymapper.md b/site/content/api/inventorymapper.md new file mode 100644 index 0000000..828d512 --- /dev/null +++ b/site/content/api/inventorymapper.md @@ -0,0 +1,145 @@ +--- +object: cloudg.inventory.InventoryMapper +members: [map_inventory, map_inventory_sync, build_asset_map, build_compliance_map, export_merged] +lede: "Maps what is deployed and how it connects, with no scanner involved, and overlays scanner findings on that map whenever they turn up." +--- + +## What a mapping run does + +`InventoryMapper(config, tagging_sweep=None)` stores the config and reads the `inventory` section from it. `tagging_sweep` overrides `config.inventory.tagging_sweep` for this mapper only. Nothing touches the cloud until you call `map_inventory()`. + +```mermaid caption="InventoryMapper.map_inventory(), step by step" +flowchart TD + A["deep copy of the config"] --> B{"aws.organization.enabled?"} + B -->|yes| C["discover the org, pick accounts and member role"] + B -->|no| D + C --> D["Azure management groups, GCP org and folders"] + D --> E["deep collectors per account and region"] + E --> F["deduplicate assets seen twice"] + F --> G["RelationshipLinker: typed edges"] + G --> H["account hierarchy nodes"] + H --> I["InventoryResult"] +``` + +In more detail: + +1. The config is deep-copied. Organization discovery rewrites `aws.accounts`, `aws.role_name` and, with `aws.regions: [ALL]` and Control Tower, `aws.regions`. Those changes stay inside the run; the object you passed in is never modified. +2. With `aws.organization.enabled`, the mapper reads the OU tree, the member accounts, SCPs and, if present, the Control Tower landing zone. The selected accounts become the collection targets, reached by assuming `aws.organization.role_name` (falling back to `aws.role_name`, then `AWSControlTowerExecution`). The topology is kept on `mapper.organization` after the run. When discovery fails, the error is logged, an `organizations` coverage record is marked `FAILED`, and only the caller's account is mapped. +3. With Azure in `providers` and `azure.map_management_groups` on, the management group and Azure Policy hierarchy is read. With GCP, `gcp.organization_id` set and `gcp.map_hierarchy` on, the organization, folders, org policies and VPC Service Controls perimeters are read. +4. The deep collectors run per account and region (AWS), per subscription (Azure) and per project or organization (GCP), with the `inventory` options applied: service families, exclusions, Kubernetes, the tagging and Cloud Control sweeps, IAM resource edges and the rest. +5. Assets that share an ARN (the same resource seen from two regions or from a collector and a sweep) collapse into one. The detailed copy wins over a sweep or placeholder copy, and their declared relations are merged. +6. The [`RelationshipLinker`](/api/relationshiplinker/) resolves declared relations and applies its provider rules, with `inventory.link_references` deciding whether the generic reference scan runs. Placeholder `CLOUD_ACCOUNT` nodes are added for accounts that were referenced but not mapped. +7. With `inventory.account_hierarchy` on (the default), account nodes are added and linked to the top-level resources they contain. + +Cloud API throttling does not fail a map. Calls are rate limited and retried according to `config.ratelimit`; a service that is still throttled after its retries is recorded as `FAILED` or `PARTIAL` in `coverage`, and the run's throttling telemetry lands in `result.throttling`, with each message also logged as a warning. If the code that calls `map_inventory()` already runs inside a `cloudg.resilience.stats_scope()`, the mapper reports into that scope. + +`map_inventory()` returns an [`InventoryResult`](/api/inventoryresult/). Calling the engine's `map_inventory()` instead gets you the same result plus an `on_phase_start("inventory_mapping")` hook and optional export. + +## Merging findings into a map + +The last three methods never call a cloud API or a scanner. They take a finished `InventoryResult` and a list of findings from anywhere: `CloudGEngine.scan()`, `ingest_reports()`, a saved findings file. + +`build_asset_map(result, findings)` returns one entry per asset in the map: `id`, `arn`, `name`, `type`, `provider`, `region`, `account_id`, `internet_exposed`, `finding_count`, `severity_breakdown` and `finding_ids`, sorted with the most findings first, plus `total_assets` and `assets_with_findings` at the top level. A finding belongs to an asset when its `resource_arn` or its `resource_id` equals the asset's `id`, `arn` or `name`. + +`build_compliance_map(result, findings)` groups by compliance framework. Each framework gets `findings`, `severity_breakdown`, `affected_assets` (the asset's ARN, or the finding's own `resource_arn` when no asset matched) and `affected_asset_count`. It reads `finding.compliance_frameworks` as it is, so run the findings through [`FindingsNormaliser`](/api/findingsnormaliser/) first if you want the framework mapping cloudg adds on top of what the scanner reported. + +`export_merged(result, findings, output_dir)` writes both: `asset-map.json` and `compliance-map.json`, and returns their paths under the keys `asset_map` and `compliance_map`. + +## Examples + +### Map an AWS organization + +Run this from the management account or a delegated administrator. The member role must exist in every target account; a read-only role deployed with a StackSet is the usual choice. + +```python title="map_org.py" +from cloudg import CloudGConfig +from cloudg.inventory import InventoryMapper + +config = CloudGConfig(providers=["aws"]) +config.aws.regions = ["ALL"] # Control Tower governed regions, if any +config.aws.organization.enabled = True +config.aws.organization.role_name = "cloudg-readonly" +config.aws.organization.include_ous = ["Workloads"] # nested OUs included +config.aws.organization.exclude_accounts = ["111111111111"] + +mapper = InventoryMapper(config) +inventory = mapper.map_inventory_sync() + +topology = mapper.organization # OrganizationTopology, or None +if topology is not None: + print("governed regions:", topology.governed_regions) + +summary = inventory.summary +print(summary["accounts"], "accounts,", summary["total_assets"], "assets") +print("cross-account edges:", summary["cross_account_edges"]) +print("security service gaps:", summary["security_service_gaps"]) + +for cov in inventory.coverage: + if cov.failed_services: + print("failed:", cov.to_summary()["failures"]) + +inventory.export("./inventory") +``` + +[AWS Organizations](/guides/aws-organizations/) covers the role, the account selection and Control Tower in detail. + +### Overlay findings that arrive later + +Mapping and scanning often run on different schedules or different hosts. This script runs offline: it loads a map exported earlier, ingests Prowler and Trivy output, normalises it, and writes the merged maps next to the inventory. + +```python title="overlay_findings.py" +from cloudg import CloudGConfig, InventoryMapper, InventoryResult +from cloudg.ingest import ingest_reports +from cloudg.normaliser import FindingsNormaliser + +# A map written earlier by `cloudg map -o ./inventory` or InventoryResult.export() +inventory = InventoryResult.load("./inventory") + +# Scanner output that arrived later, normalised so compliance mapping is filled in +raw = ingest_reports({"prowler": ["./prowler-output/"], "trivy": ["./trivy.json"]}) +findings = FindingsNormaliser().normalise(raw).findings + +mapper = InventoryMapper(CloudGConfig(providers=["aws"])) +paths = mapper.export_merged(inventory, findings, "./inventory") +print({name: str(path) for name, path in paths.items()}) + +compliance = mapper.build_compliance_map(inventory, findings) +for framework, entry in list(compliance["frameworks"].items())[:3]: + print(framework, entry["findings"], entry["affected_assets"]) + +asset_map = mapper.build_asset_map(inventory, findings) +print(asset_map["total_assets"], "assets,", asset_map["assets_with_findings"], "with findings") +``` + +```console +$ python overlay_findings.py +{'asset_map': 'inventory/asset-map.json', 'compliance_map': 'inventory/compliance-map.json'} +CIS 1 ['arn:aws:s3:::my-bucket'] +CVE 1 ['myrepo/app:latest'] +GDPR-AWS 1 ['arn:aws:s3:::my-bucket'] +6 assets, 0 with findings +``` + +The map in this run holds six assets of an orders service, and the findings are about a bucket and a container image that are not in it. `asset-map.json` therefore shows no asset with findings, while `compliance-map.json` still lists both findings under their own identifiers. A finding about something the map contains is counted against that asset. + +The mapper needs a config only to exist; the merge methods ignore it. `InventoryMapper(CloudGConfig())` is enough here. + +## Notes + +Asset matching in `build_asset_map()` includes the asset's `name`, and it checks `resource_id` even when the finding carries a `resource_arn` that points elsewhere. Two assets with the same name (an SQS queue and a DynamoDB table both called `orders`) can therefore both receive a finding meant for one of them. `build_compliance_map()` resolves by ARN first and does not have this problem. When names repeat in your estate, prefer findings with `resource_arn` set and check `asset-map.json` for duplicates. + +`map_inventory()` and the sync wrapper keep no state between calls except `mapper.organization`, which is overwritten by every run. One mapper can map several times. + +`map_inventory_sync()` uses `asyncio.run()`. In a notebook or any code that already runs an event loop, `await mapper.map_inventory()` instead. + +`InventoryResult.coverage` is not written to any export file. Read it from the returned object if you want the failure list; the CLI prints the failed collectors after a `cloudg map` run. + +## Related + +:::links +- [InventoryResult](/api/inventoryresult/) What `map_inventory()` returns. +- [RelationshipLinker](/api/relationshiplinker/) The linking step on its own. +- [DependencyGraph](/api/dependencygraph/) Blast radius and dependency walks over the map. +- [Inventory mapping](/guides/inventory-mapping/) The guide, with the CLI side. +- [cloudg map](/cli/map/) The command that wraps this class. +::: diff --git a/site/content/api/inventoryresult.md b/site/content/api/inventoryresult.md new file mode 100644 index 0000000..3af0988 --- /dev/null +++ b/site/content/api/inventoryresult.md @@ -0,0 +1,150 @@ +--- +object: cloudg.inventory.InventoryResult +members: [summary, dependency_graph, analysis, export, load] +lede: "The inventory map as one object: linked assets and edges, a summary computed on demand, the dependency analysis, and the files it saves to and loads from." +--- + +## Where it comes from + +`InventoryResult` is a dataclass defined in `cloudg/inventory/mapper_result.py` and importable from `cloudg`, `cloudg.inventory` and `cloudg.inventory.mapper`. You get one from `InventoryMapper.map_inventory()`, from `CloudGEngine.map_inventory()`, or from `InventoryResult.load()` on a map saved earlier. You can also build one yourself from any list of `CloudAsset` and `NetworkEdge` objects; every field has a default, and the methods work the same on a hand-built result. + +```mermaid caption="An inventory map's round trip through the file system" +flowchart LR + M["map_inventory()"] --> R["InventoryResult"] + Y["your own assets and edges"] --> R + R -->|"export(dir)"| F["inventory-map.json and friends"] + F -->|"InventoryResult.load(dir)"| L["InventoryResult without coverage, raw_data, duration"] + R -->|"dependency_graph()"| G["DependencyGraph"] + R -->|"analysis()"| A["dependency and coverage report"] +``` + +## The parts that need explaining + +`summary` is a property, not a stored field, and it is recomputed from `assets` and `edges` on every access. On a map with tens of thousands of assets that is a noticeable amount of work, so read it once into a variable. Its keys are counts and breakdowns: `total_assets`, `total_edges`, `providers`, `assets_by_type`, `assets_by_service`, `assets_by_region`, `assets_by_account`, `edges_by_type`, `edges_by_relationship`, `unlinked_assets`, `internet_exposed`, `accounts`, `cross_account_edges`, `external_accounts`, `security_service_gaps`, `unresolved_references`, and `organization` and `throttling` when those exist. The [summary reference](/reference/inventory/summary/) says how each one is computed. `assets_by_service` is derived from the identifier: the service field of an ARN, the provider namespace of an Azure ID, the API host of a GCP name. + +`dependency_graph(include_hierarchy=False)` builds a new [`DependencyGraph`](/api/dependencygraph/) on each call. With the default, `CONTAINS` edges into organization, OU and account nodes are left out, so "everything in this account depends on the account" does not dominate every blast radius. + +`analysis(top=25)` runs the dependency graph and the coverage checks and returns a dict with five keys: `shared_dependencies` (assets many others directly depend on), `largest_blast_radius`, `cross_account_edges` (one entry per edge between two accounts, with `external` true when one side is an unmapped account), `security_coverage` (which security services run in which account and region, the gaps, workloads no vulnerability scanner covers, internet-facing entry points without a WAF) and `unresolved_references`. This is what `export()` saves as `inventory-dependencies.json`. + +`export(output_dir)` creates the directory and writes `inventory-map.json` (summary, providers, regions, assets, edges, unresolved references and throttling), `inventory-map.graphml`, `inventory-graph.json` (D3 format, for viewers), `inventory-dependencies.json` and, when an organization was mapped, `inventory-organization.json`. It returns the paths keyed `map`, `graphml`, `graph`, `dependencies` and `organization`. JSON is written with `indent=2` and `default=str`, so datetimes come out as strings. + +`load(path)` takes `inventory-map.json` or the directory that holds it. It re-validates every asset and edge into models and reads `inventory-organization.json` from the same directory when it exists. Some data does not survive the trip: + +| Field | After `load()` | +|---|---| +| `coverage` | empty: coverage is never written to a file | +| `duration_ms` | `0` | +| `CloudAsset.raw_data` | empty on every asset, because the model excludes it from serialisation | +| everything else | as exported | + +The missing `raw_data` matters if you link a loaded map again: a few linker rules (instance profiles, Lambda execution roles taken from the raw `Role` field) read the raw payload, so they only fire on a fresh collection. Declared relations in `metadata["relations"]` are kept and still link. + +## Examples + +### Build, export, reload, analyse + +Runs offline. Four assets of an orders service are linked by the [`RelationshipLinker`](/api/relationshiplinker/), one of them points at a role in an account that was never mapped, and the result goes through a full save and load. + +```python title="inventory_roundtrip.py" +from cloudg import InventoryResult +from cloudg.inventory import RelationshipLinker +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider + +ACCOUNT = "123456789012" +REGION = "eu-west-1" +KEY = f"arn:aws:kms:{REGION}:{ACCOUNT}:key/1111aaaa-22bb-33cc-44dd-555555eeeeee" +QUEUE = f"arn:aws:sqs:{REGION}:{ACCOUNT}:orders-inbound" +TABLE = f"arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/orders" +FUNCTION = f"arn:aws:lambda:{REGION}:{ACCOUNT}:function:orders-worker" + + +def aws(arn, name, asset_type, **metadata): + return CloudAsset(arn=arn, name=name, asset_type=asset_type, provider=CloudProvider.AWS, + region=REGION, account_id=ACCOUNT, metadata=metadata) + + +assets = [ + aws(KEY, "orders-key", AssetType.KMS_KEY), + aws(QUEUE, "orders-inbound", AssetType.MESSAGE_QUEUE, kms_key_id=KEY), + aws(TABLE, "orders", AssetType.DYNAMODB_TABLE, kms_key_id=KEY), + aws(FUNCTION, "orders-worker", AssetType.LAMBDA_FUNCTION, kms_key_id=KEY, relations=[ + {"target": QUEUE, "edge": "INVOKES", "relationship": "TRIGGERED_BY", "reverse": True}, + {"target": TABLE, "edge": "REFERENCES", "relationship": "WRITES_TO"}, + {"target": "arn:aws:iam::999999999999:role/partner-reader", "edge": "ASSUMES_ROLE"}, + ]), +] + +linker = RelationshipLinker(assets) +edges = linker.link() +inventory = InventoryResult( + assets=assets + linker.external_assets, + edges=edges, + providers=["aws"], + regions={"aws": [REGION]}, + unresolved_references=linker.unresolved, +) + +paths = inventory.export("./inventory") +print(sorted(paths)) + +again = InventoryResult.load("./inventory") # the directory or inventory-map.json +s = again.summary +print(s["total_assets"], s["total_edges"], s["edges_by_type"]) +print("external accounts:", s["external_accounts"], "cross-account edges:", s["cross_account_edges"]) + +analysis = again.analysis(top=5) +print(list(analysis)) +print([(d["name"], d["direct_dependents"]) for d in analysis["shared_dependencies"]]) +print(analysis["cross_account_edges"]) +``` + +```console +$ python inventory_roundtrip.py +['dependencies', 'graph', 'graphml', 'map'] +5 6 {'INVOKES': 1, 'REFERENCES': 4, 'ASSUMES_ROLE': 1} +external accounts: 1 cross-account edges: 1 +['shared_dependencies', 'largest_blast_radius', 'cross_account_edges', 'security_coverage', 'unresolved_references'] +[('orders-key', 3)] +[{'source': 'arn:aws:lambda:eu-west-1:123456789012:function:orders-worker', 'source_account': '123456789012', 'target': 'arn:aws:iam::999999999999:root', 'target_account': '999999999999', 'edge_type': 'ASSUMES_ROLE', 'relationship': None, 'external': True}] +``` + +Five assets, not four: the role in account `999999999999` was not mapped, so the linker created an external `CLOUD_ACCOUNT` placeholder (ARN `arn:aws:iam::999999999999:root`) and pointed the `ASSUMES_ROLE` edge at it. The three `kms_key_id` values became `REFERENCES` edges to the key, which is why the key tops `shared_dependencies`. + +### Answer a question from a saved map + +Any map exported by `cloudg map -o ./inventory` loads the same way. No credentials are needed. + +```python title="whats_exposed.py" +from cloudg import InventoryResult + +inventory = InventoryResult.load("./inventory") +summary = inventory.summary + +print(f"{summary['internet_exposed']} of {summary['total_assets']} assets face the internet") +for asset in inventory.assets: + if asset.is_internet_exposed: + print(f" {asset.asset_type.value:16} {asset.account_id or '-':14} {asset.display_id}") + +print("unresolved references:", summary["unresolved_references"]) +for ref in inventory.unresolved_references[:10]: + print(f" {ref['source_name']} -> {ref['target']} ({ref['edge_type']})") +``` + +## Notes + +`unresolved_references` holds at most 2000 entries per run. A count of exactly 2000 in the summary means the cap was hit and more are missing. + +`providers` comes back from `load()` even for maps written by older cloudg versions, which kept it only inside `summary`. + +`export()` overwrites files of the same name without asking. Export each run into its own directory if you want to compare maps over time. + +Building an `InventoryResult` yourself does not deduplicate or link anything. Pass assets that are already unique by ARN and edges you produced (with the linker or otherwise), or the summary counts will include the duplicates. + +## Related + +:::links +- [InventoryMapper](/api/inventorymapper/) The class that produces this result. +- [DependencyGraph](/api/dependencygraph/) What `dependency_graph()` returns. +- [InventoryResult reference](/reference/inventory/inventoryresult/) Every field and file, specified. +- [Output files](/reference/output-files/) All files cloudg writes, side by side. +::: diff --git a/site/content/api/mcp-layer.md b/site/content/api/mcp-layer.md new file mode 100644 index 0000000..50c015a --- /dev/null +++ b/site/content/api/mcp-layer.md @@ -0,0 +1,263 @@ +--- +title: CloudGMCPLayer +object: + - cloudg.mcp.CloudGMCPLayer + - cloudg.mcp.register_into + - cloudg.mcp.build_server + - cloudg.mcp.export_definitions +members: + cloudg.mcp.CloudGMCPLayer: [call_tool, read_resource, get_prompt, complete, list_tools, list_resources, list_resource_templates, list_prompts, tools_wire, get_tool, register_into, serve, use, set_prefix, on_change, notify_change, context] +lede: The whole cloudg MCP catalog as one Python object you can call in-process, mount into your own MCP server, or export as framework-free definitions. +source: docs/MCP.md +since: "0.6.0" +--- + +`cloudg mcp serve` is a thin wrapper around this class. Everything the server does, from the 74 tools to the policy that hides, redacts and pseudonymises, lives in `CloudGMCPLayer`, so you can embed the same catalog wherever your agent or server already runs. Importing `cloudg.mcp` pulls in nothing beyond cloudg's own dependencies; the official `mcp` SDK (the `cloudg[mcp]` extra) or `fastmcp` is imported only when you mount into one of them. + +There are three ways in, and all of them end at the same four methods: + +| You have | Use | You get | +|---|---|---| +| Python code, an agent loop, a test | `await layer.call_tool(...)`, `read_resource`, `get_prompt`, `complete` | `ToolResult`, resource contents, `PromptResult` objects | +| An MCP server built on the SDK or fastmcp | `layer.register_into(server)` | cloudg's tools, resources and prompts next to the host's own | +| A gateway or framework that speaks neither | `export_definitions(layer)` | MCP wire dicts plus async handlers, for one principal | + +```mermaid caption="Every route goes through the same layer methods" +flowchart LR + A["Your code"] --> L["layer.call_tool / read_resource / get_prompt / complete"] + B["SDK or fastmcp server via register_into"] --> L + C["export_definitions handlers"] --> L + D["cloudg mcp serve"] --> L + L --> V{"Visible to this principal?"} + V -->|no| N["NotFoundError, as if it did not exist"] + V -->|yes| M["Middleware, outermost first"] + M --> P["Policy check, input transforms"] + P --> H["Handler"] + H --> O["Output transforms"] +``` + +A tool the policy hides from the caller is indistinguishable from one that does not exist. Every other failure of a tool call (bad arguments, access denied, a handler exception, a timeout) comes back as a `ToolResult` with `is_error=True` that a model can read and recover from, rather than as an exception. Resources and prompts raise `MCPLayerError` subclasses instead. + +## Building a layer + +`CloudGMCPLayer(config=None, *, options=None, **kwargs)` takes every setting as a keyword or as one `LayerOptions` object (`from cloudg.mcp.layer import LayerOptions`); keywords next to `options` override its fields, and an unknown keyword raises `TypeError`. The settings people change most: + +| Keyword | Default | Effect | +|---|---|---| +| `workspace` | `Workspace(config)` | The datasets the tools query, and the directories they may read and write. Pass your own to control both | +| `policy` | `$CLOUDG_MCP_POLICY`, else `"standard"` | A profile name, a policy file path, inline JSON, a dict or a `Policy` | +| `prefix` | `""` | Prepended to tool and prompt names (`cloudg_find_assets`); up to 64 of `A-Za-z0-9_.-` | +| `include_categories`, `exclude_tools` and the other filters | none | Build a filtered copy of the registry | +| `middleware` | `()` | Middleware callables, outermost first; `layer.use(mw)` appends one | +| `default_timeout` | 300 seconds | Per tool call; a tool's own `timeout_seconds` wins | +| `max_output_chars` | 200,000 | Cap on a result's text block | + +The datasets come from the workspace. `workspace.load(path, name)` reads an `inventory-map.json`, a `findings.json` or a directory holding them, but only under the workspace's `allowed_roots`; with none given, those are the working directory and the report directory. A relative path is resolved against the first allowed root, not the current directory, so with `allowed_roots=["./reports"]` you load `"inventory-map.json"`, not `"./reports/inventory-map.json"`. `workspace.add_inventory(result, name)` adds an `InventoryResult` you already have in memory, with no files involved. + +### Policies + +The seven built-in profiles are `open`, `standard`, `strict`, `read_only`, `airgapped`, `audit` and `soc-analyst` (`cloudg.mcp.available_profiles()` lists them). The policy decides which tools each principal sees, whether calls are rate limited, and which transforms run over arguments and results: redaction, pseudonymisation of ARNs and account ids, projection, annotation. A call made without a `principal` runs as `Principal.local()` (id `local`, roles `default` and `local`), which is what a desktop client on stdio gets too. Pass `principal=Principal(id=..., roles={...})` to see what a remote caller would. + +The same layer object can only have one policy. To serve two audiences from one process, build two layers over one shared `Workspace`, as the first example does. + +## Examples + +### Call the catalog in-process + +No SDK, no server and no files: the inventory is built in memory, and the script calls tools and reads a resource as the local user, then again under the `strict` profile as a remote guest. An audit log middleware records each call with its argument values hashed. + +```python title="mcp_inprocess.py" +"""Call the cloudg MCP layer in-process, as two different principals.""" + +import asyncio + +from cloudg.inventory import InventoryResult +from cloudg.mcp import CloudGMCPLayer, Principal, Workspace +from cloudg.mcp.middleware import AuditLogMiddleware +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType, NetworkEdge + +ACCOUNT = "123456789012" + + +def asset(asset_id: str, asset_type: AssetType, arn: str, **extra) -> CloudAsset: + return CloudAsset(id=asset_id, name=asset_id, asset_type=asset_type, provider=CloudProvider.AWS, + region="eu-west-1", account_id=ACCOUNT, arn=arn, **extra) + + +inventory = InventoryResult( + assets=[ + asset("sg-web", AssetType.SECURITY_GROUP, f"arn:aws:ec2:eu-west-1:{ACCOUNT}:security-group/sg-web"), + asset("web-1", AssetType.EC2, f"arn:aws:ec2:eu-west-1:{ACCOUNT}:instance/i-0web1", + is_internet_exposed=True), + asset("orders-db", AssetType.RDS_INSTANCE, f"arn:aws:rds:eu-west-1:{ACCOUNT}:db:orders-db"), + ], + edges=[ + NetworkEdge(source_id="0.0.0.0/0", target_id="sg-web", edge_type=EdgeType.SECURITY_GROUP_RULE, + port_range="22", protocol="TCP", cidr="0.0.0.0/0", direction="ingress"), + NetworkEdge(source_id="web-1", target_id="sg-web", edge_type=EdgeType.ATTACHED_TO), + ], + providers=["aws"], +) + + +async def main() -> None: + workspace = Workspace(allowed_roots=["."]) + workspace.add_inventory(inventory, "demo") + audit = AuditLogMiddleware("mcp-audit.jsonl") # argument values are hashed + layer = CloudGMCPLayer(policy="standard", workspace=workspace, middleware=[audit]) + + print(len(layer.list_tools()), "tools visible to the local user") + + result = await layer.call_tool("count_assets", {"group_by": "type"}) + print("count_assets:", result.structured["groups"]) + + result = await layer.call_tool("find_assets", {"internet_exposed": True}) + print("exposed:", [(a["name"], a["arn"]) for a in result.structured["items"]]) + + contents = await layer.read_resource("cloudg://datasets/demo/summary") + print("summary resource:", contents[0].mime_type, len(contents[0].text), "chars") + + # A tool error comes back as an isError result, not an exception + result = await layer.call_tool("get_asset", {"ref": "no-such-asset"}) + print("missing asset:", result.is_error, result.content[0].text) + + # The same layer under a stricter policy, for a remote caller + strict = CloudGMCPLayer(policy="strict", workspace=workspace) + guest = Principal(id="guest", roles={"default"}) + print(len(strict.list_tools(guest)), "tools visible to guest under strict") + result = await strict.call_tool("find_assets", {"internet_exposed": True}, principal=guest) + print("guest sees:", [(a["name"], a["arn"]) for a in result.structured["items"]]) + + audit.close() + with open("mcp-audit.jsonl") as fh: + print("audit:", fh.readline().strip()) + + +asyncio.run(main()) +``` + +```console +$ python mcp_inprocess.py +73 tools visible to the local user +count_assets: {'EC2': 1, 'RDS_INSTANCE': 1, 'SECURITY_GROUP': 1} +exposed: [('web-1', 'arn:aws:ec2:eu-west-1:123456789012:instance/i-0web1')] +summary resource: application/json 734 chars +missing asset: True No asset matches the reference in this dataset. Use find_assets(query=...) to search by name, ARN or tag. +61 tools visible to guest under strict +guest sees: [('res-528c2a8001', 'arn:aws:ec2:eu-west-1:724607639135:instance/res-8a639d2144')] +audit: {"argument_hashes":{"group_by":"4653d2e6f4d67e7d"},"argument_keys":["group_by"],"duration_ms":1.59,"event":"mcp.call","kind":"tool","name":"count_assets","outcome":"ok","principal":{"id":"local","roles":["default","local"]},"request_id":null,"sensitivity":"internal","transforms":{...},"ts":"2026-10-09T16:42:45.339+00:00"} +``` + +The `transforms` object in the audit line is shortened here. What the run shows: + +- Under `standard`, the local user sees 73 of the 74 tools. `reveal_token`, which reverses pseudonyms, stays hidden. +- `strict` hides 13 tools from the guest, among them the live tools (`map_inventory`, `run_pipeline`), the file exports and `sparql_query`, and it replaces names, account ids and ARN resource parts with format-preserving pseudonyms. The pseudonyms change on every run unless `CLOUDG_MCP_VAULT_KEY` is set; with the key they are stable across restarts. +- `get_asset` for a missing reference did not raise. The error text is fixed and does not echo the value the caller sent. + +### Mount into an MCP server you already run + +With mcp 2.x installed (`pip install "cloudg[mcp]"`), `register_into` adds the catalog to an `MCPServer` next to its own tools. Mount after the host has registered its own handlers. The second half of the script uses `export_definitions`, which needs no framework at all. + +```python title="mcp_embed.py" +"""Mount cloudg's catalog next to a host server's own tool (mcp 2.x).""" + +import asyncio + +from mcp import Client +from mcp.server.mcpserver import MCPServer + +from cloudg.inventory import InventoryResult +from cloudg.mcp import CloudGMCPLayer, Workspace, export_definitions +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider + +server = MCPServer("platform-tools") + + +@server.tool() +def ticket_count(team: str) -> int: + """How many open tickets a team has (the host's own tool).""" + return 7 + + +inventory = InventoryResult( + assets=[ + CloudAsset(id=f"vm-{i}", name=f"vm-{i}", asset_type=AssetType.EC2, + provider=CloudProvider.AWS, region="eu-west-1", account_id="123456789012") + for i in range(3) + ], + providers=["aws"], +) +workspace = Workspace(allowed_roots=["."]) +workspace.add_inventory(inventory, "demo") + +layer = CloudGMCPLayer(policy="standard", workspace=workspace) +binding = layer.register_into(server, prefix="cloudg_") # mount after the host's own tools + + +async def main() -> None: + async with Client(server) as client: + names = [t.name for t in (await client.list_tools()).tools] + print(len(names), "tools:", names[:3], "...") + res = await client.call_tool("cloudg_count_assets", {"group_by": "provider"}) + print("cloudg tool:", res.structured_content["groups"]) + res = await client.call_tool("ticket_count", {"team": "sec"}) + print("host tool:", res.content[0].text) + + # The same catalog without any MCP framework + defs = export_definitions(layer) + print(sorted(defs.handlers)) + reply = await defs.dispatch("tools/call", {"name": "cloudg_count_assets", + "arguments": {"group_by": "type"}}) + print("dispatch:", reply["structuredContent"]["groups"], reply["isError"]) + + +asyncio.run(main()) +``` + +```console +$ python mcp_embed.py +74 tools: ['ticket_count', 'cloudg_workspace_status', 'cloudg_list_datasets'] ... +cloudg tool: {'AWS': 3} +host tool: 7 +['completion/complete', 'prompts/get', 'prompts/list', 'resources/list', 'resources/read', 'resources/templates/list', 'tools/call', 'tools/list'] +dispatch: {'EC2': 3} False +``` + +The host's `ticket_count` plus cloudg's 73 visible tools make 74. Run the server with its own runner (`server.run()`, `server.streamable_http_app()`); cloudg adds no transport of its own here. Verified with mcp 2.3.0; `register_into` also accepts an mcp 1.x `FastMCP`, a low-level `Server` and a `fastmcp.FastMCP` (2.x to 4.x). + +With a prefix set, only prefixed names resolve, in-process too: `layer.call_tool("count_assets")` on this layer raises `NotFoundError`, and `cloudg_count_assets` works. That is what keeps cloudg from answering for a host tool that happens to share a name. Resource URIs are never prefixed; they already start with `cloudg://`. + +### Run it as its own server from Python + +`layer.serve()` is the same server `cloudg mcp serve` starts, with your layer: + +```python title="serve_layer.py" +from cloudg.mcp import CloudGMCPLayer, Workspace + +workspace = Workspace(allowed_roots=["./reports"]) +workspace.load("inventory-map.json", "prod") # relative to the first allowed root +layer = CloudGMCPLayer(policy="strict", workspace=workspace, prefix="cloudg_") +layer.serve("stdio") # blocks until the client disconnects +``` + +For HTTP, `layer.serve("http", host="127.0.0.1", port=8765, auth_tokens=["env:MCP_TOKEN:analyst:ci"])` takes the same options as the CLI. The token spec format and the other transport settings are on the [cloudg mcp serve](/cli/mcp-serve/) page. + +## Notes + +- `register_into(server, prefix=...)` changes the layer's own prefix. Mounting one layer into two servers with different prefixes leaves both on the last one, with a warning; build one layer per prefix. +- On the low-level SDK servers, `register_into` wraps whichever handlers exist when it runs, so a host handler registered afterwards replaces cloudg's. Mount last. +- `cloudg.mcp.build_server(layer, flavor)` takes only the flavor (`auto`, `sdk`/`lowlevel`, `mcpserver`, `fastmcp`, `native`). `cloudg.mcp.adapters.build_server` also takes `name`, `principal_resolver`, `include` and native server keywords. +- `export_definitions(layer, principal)` snapshots the definitions for one principal. Its handlers dispatch through the layer, so the policy still applies per call; pass `principal=` to `dispatch()` to call as someone else. +- `cloudg.mcp.adapters` also has `to_openai_tools(layer)`, `to_anthropic_tools(layer)` and `to_langchain_tools(layer)` for function-calling APIs. Execute the model's tool calls with `await layer.call_tool(name, arguments, principal=...)`. +- Sync tool handlers run in a worker thread, so a slow graph build does not block the event loop. `notify_change()` is safe to call from those threads. +- A policy with `vault.path` set persists its pseudonym vault: `serve()` saves it when serving ends, and any process saves a changed vault at interpreter exit. Call `layer.policy.save_vault()` to save at a moment you choose, for example before a long-running worker is killed. + +## Related + +:::links +- [MCP server overview](/mcp/) What the catalog covers and how clients connect. +- [In-process use](/mcp/server/in-process-use/) Every layer method and the `ToolContext` handlers receive. +- [Built-in profiles](/mcp/privacy/built-in-profiles/) What each of the seven policies allows and transforms. +- [Tool index](/mcp/tools/tool-index/) All 74 tools by category. +- [cloudg mcp serve](/cli/mcp-serve/) The same layer behind stdio or HTTP. +::: diff --git a/site/content/api/models.md b/site/content/api/models.md new file mode 100644 index 0000000..b0fc811 --- /dev/null +++ b/site/content/api/models.md @@ -0,0 +1,223 @@ +--- +object: [cloudg.schema.models.Finding, cloudg.schema.models.CloudAsset, cloudg.schema.models.NetworkEdge, cloudg.schema.models.ComplianceResult, cloudg.schema.models.ScanResult, cloudg.schema.models.Severity, cloudg.schema.models.CloudProvider, cloudg.schema.models.ComplianceStatus] +members: + cloudg.schema.models.Finding: [risk_score] + cloudg.schema.models.CloudAsset: [display_id] + cloudg.schema.models.NetworkEdge: [] + cloudg.schema.models.ComplianceResult: [] + cloudg.schema.models.ScanResult: [summary] + cloudg.schema.models.Severity: [] + cloudg.schema.models.CloudProvider: [] + cloudg.schema.models.ComplianceStatus: [] +lede: "The pydantic v2 models every part of cloudg exchanges: assets, the edges between them, findings, compliance results, and the `ScanResult` that holds them together." +--- + +## How the models fit together + +Every collector produces `CloudAsset` and `NetworkEdge` objects, every scanner parser produces `Finding` objects, and the normaliser bundles findings and their `ComplianceResult` entries into a `ScanResult`. The links between them are plain string ids, not object references: + +```mermaid caption="The id fields that connect the models" +flowchart LR + SR["ScanResult"] -->|assets| CA["CloudAsset"] + SR -->|edges| NE["NetworkEdge"] + SR -->|findings| F["Finding"] + SR -->|compliance| CR["ComplianceResult"] + NE -->|"source_id, target_id"| CA + F -->|"resource_id, resource_arn"| CA + CR -->|finding_ids| F +``` + +Those references are not checked. An edge may point at an id no asset has (cloudg's own security group rules use the CIDR `0.0.0.0/0` as `source_id`, and the graph builder turns it into a node of its own), and a finding's `resource_id` is often an ARN or a Terraform address instead of an asset id. [`AssetIndex`](/api/dependencygraph/) is the matcher cloudg uses to resolve such references. + +All five models are configured with `extra="allow"`: a field the model does not define is kept on the instance and written back out by `model_dump()`. That makes it easy to carry your own data (`CloudAsset(..., owner="team-orders")`) through cloudg's pipeline, and it also means a typo in a keyword argument is not an error. + +## Enums and the big taxonomies + +`Severity` has five members: `CRITICAL`, `HIGH`, `MEDIUM`, `LOW`, `INFO`. The normaliser sorts in that order. `CloudProvider` is `AWS`, `AZURE` or `GCP`, upper case, unlike `CloudGConfig.providers`, which uses lower-case strings. `ComplianceStatus` is `PASS`, `FAIL`, `NOT_APPLICABLE` or `MANUAL`. + +All of them are `str` enums, so `Severity.HIGH == "HIGH"` is true, and pydantic accepts the plain string wherever the enum is expected. Values outside the enum raise a `ValidationError`. + +`AssetType` has 156 values, from `EC2` and `S3_BUCKET` to `K8S_WORKLOAD`, `LANDING_ZONE` and `AI_GUARDRAIL`, plus `OTHER` for anything without a better fit. The [asset types by category](/reference/catalog/asset-types-by-category/) page lists them all, grouped, with the metadata each type carries. + +`EdgeType` has 20 values. Eleven describe network and identity structure (`SECURITY_GROUP_RULE`, `NACL_RULE`, `ROUTE`, `IAM_TRUST`, `IAM_POLICY_ATTACHMENT`, `CONTAINS`, `PEERING`, `LOAD_BALANCER_TARGET`, `INTERNET_EXPOSED`, `ATTACHED_TO`, `REFERENCES`) and nine are the typed inventory relationships (`INVOKES`, `USES_IMAGE`, `ASSUMES_ROLE`, `GRANTS_ACCESS`, `LOGS_TO`, `PROTECTS`, `MONITORS`, `MANAGES`, `GOVERNS`), always read "source verb target". The finer `relationship` label on an edge comes from a separate vocabulary, documented under [relationship vocabulary](/reference/catalog/relationship-vocabulary/). + +## Serialised forms + +`model_dump(mode="json")` gives the shape cloudg writes to its JSON files. These two were produced by the script in the examples; only the random `id` values will differ on your machine. + +```json title="CloudAsset" +{ + "id": "5a9456d0-7a28-4be8-b9c3-68b0675fc9b1", + "arn": "arn:aws:s3:::orders-archive", + "name": "orders-archive", + "asset_type": "S3_BUCKET", + "provider": "AWS", + "region": "eu-west-1", + "account_id": "123456789012", + "tags": { + "team": "orders" + }, + "metadata": { + "versioning": "Enabled", + "kms_key_id": null + }, + "collected_at": "2026-10-09T08:30:00", + "is_internet_exposed": false, + "display_id": "arn:aws:s3:::orders-archive" +} +``` + +`raw_data` is missing on purpose: the field is declared with `exclude=True`, so the provider's raw API response never leaves the process. `display_id` is a computed field, the ARN when there is one and the internal `id` otherwise. + +```json title="Finding" +{ + "id": "dd17c2f7-6f95-47c4-960f-5d5b455c5035", + "resource_id": "5a9456d0-7a28-4be8-b9c3-68b0675fc9b1", + "resource_arn": "arn:aws:s3:::orders-archive", + "severity": "HIGH", + "title": "S3 bucket default encryption", + "description": "Bucket has no default encryption", + "evidence": null, + "remediation": "Enable default encryption", + "source_tool": "prowler", + "source_finding_id": "prowler-aws-s3_bucket_default_encryption-123456789012-eu-west-1-orders-archive", + "compliance_frameworks": [ + "CIS" + ], + "cvss_score": 7.2, + "detected_at": "2026-10-09T08:31:00", + "is_suppressed": false, + "risk_score": 7.3 +} +``` + +`risk_score` is computed on every read. Each severity has a base value (CRITICAL 9.5, HIGH 7.5, MEDIUM 5.0, LOW 2.5, INFO 0.5). Without a `cvss_score` the risk score is that base; with one, it is the mean of the base and the CVSS score, rounded to one decimal. Here that is (7.5 + 7.2) / 2 = 7.35, which Python's `round()` turns into 7.3. `cvss_score` itself must lie between 0 and 10. + +`source_tool` holds one tool name, or several joined with `", "` after the normaliser merged findings from different scanners (`"trivy, prowler"`). + +## Examples + +### Build, inspect and round-trip the models + +Runs offline. + +```python title="models_tour.py" +import json +from datetime import datetime + +from cloudg.schema.models import ( + AssetType, CloudAsset, CloudProvider, ComplianceResult, ComplianceStatus, + EdgeType, Finding, NetworkEdge, ScanResult, Severity, +) + +bucket = CloudAsset( + arn="arn:aws:s3:::orders-archive", + name="orders-archive", + asset_type=AssetType.S3_BUCKET, + provider=CloudProvider.AWS, + region="eu-west-1", + account_id="123456789012", + tags={"team": "orders"}, + metadata={"versioning": "Enabled", "kms_key_id": None}, + raw_data={"Name": "orders-archive"}, # never serialised + collected_at=datetime(2026, 10, 9, 8, 30), +) +public = NetworkEdge( + source_id="0.0.0.0/0", # a CIDR, not an asset id + target_id=bucket.id, + edge_type=EdgeType.INTERNET_EXPOSED, + port_range="443", + protocol="TCP", + cidr="0.0.0.0/0", +) +finding = Finding( + resource_id=bucket.id, + resource_arn=bucket.arn, + severity=Severity.HIGH, + title="S3 bucket default encryption", + description="Bucket has no default encryption", + remediation="Enable default encryption", + source_tool="prowler", + source_finding_id="prowler-aws-s3_bucket_default_encryption-123456789012-eu-west-1-orders-archive", + compliance_frameworks=["CIS"], + cvss_score=7.2, + detected_at=datetime(2026, 10, 9, 8, 31), +) +control = ComplianceResult( + framework="CIS", + control_id="CIS/s3_bucket_default_encryption", + status=ComplianceStatus.FAIL, + finding_ids=[finding.id], + resource_arn=bucket.arn, +) +result = ScanResult(assets=[bucket], findings=[finding], edges=[public], compliance=[control]) + +print(json.dumps(bucket.model_dump(mode="json"), indent=2)) +print(json.dumps(finding.model_dump(mode="json"), indent=2)) +print(result.summary) +print(finding.risk_score, bucket.display_id) + +# Round trip: computed fields are dumped, so drop them before validating again +data = finding.model_dump(mode="json") +data.pop("risk_score") +assert Finding.model_validate(data) == finding +print("round trip ok") +``` + +The last lines of its output: + +```console +$ python models_tour.py | tail -3 +{'total_assets': 1, 'total_findings': 1, 'total_edges': 1, 'severity_breakdown': {'HIGH': 1}, 'compliance_frameworks': ['CIS']} +7.3 arn:aws:s3:::orders-archive +round trip ok +``` + +`ScanResult.summary` is computed too: totals of assets, findings and edges, the severity breakdown (only severities that occur), and the distinct compliance frameworks, in no particular order. + +### Read a findings file back + +```python title="load_findings.py" +import json + +from cloudg.schema.models import Finding, Severity + +COMPUTED = {"risk_score"} + +with open("./reports/raw-findings.json") as f: + raw = json.load(f) + +findings = [ + Finding.model_validate({k: v for k, v in item.items() if k not in COMPUTED}) + for item in raw +] +urgent = [f for f in findings if f.severity in (Severity.CRITICAL, Severity.HIGH)] +for f in sorted(urgent, key=lambda f: f.risk_score, reverse=True): + print(f"{f.risk_score:4} {f.severity.value:8} {f.resource_arn or f.resource_id} {f.title}") +``` + +[parse_report](/api/parse-report/) shows how to write such a file from scanner output. + +## Notes + +Ids are random. `CloudAsset.id`, `Finding.id` and the others default to a fresh `uuid4` on every construction, so the same bucket collected twice gets two different ids. Use `arn` (or `display_id`) to match assets across runs, and `source_tool` plus `source_finding_id` plus the resource to match findings. + +Computed fields come back as extras. Because of `extra="allow"`, validating a dump that still contains `display_id`, `risk_score` or `summary` does not fail; the value is stored as an extra attribute and ignored, and the property keeps computing the real one. `InventoryResult.load()` strips `display_id` for this reason, and the `load_findings.py` example does the same for `risk_score`. + +Timestamps are naive. `collected_at`, `detected_at`, `started_at` and `completed_at` default to `datetime.utcnow()`, which has no timezone, and they serialise without an offset (`2026-10-09T08:30:00`). Treat them as UTC. + +Assignment is not validated. `finding.severity = "SEVERE"` succeeds, and the plain string then fails later in code that reads `finding.severity.value`, such as `ScanResult.summary`, `PipelineResult.severity_breakdown` and the reports. Build a new model, or use `Finding.model_validate({**finding.model_dump(), "severity": ...})`, when values come from outside. + +`is_suppressed` is honoured by the MCP findings tools, which hide suppressed findings unless asked. The normaliser, the reports and `PipelineResult.severity_breakdown` count them like any other finding. + +`NetworkEdge.direction` defaults to `"ingress"` and `ports` to an empty list; most collectors fill `port_range` (`"80-443"`, `"22"`) instead of `ports`. Parallel security group rules between the same two nodes are separate edges in the data and are merged into one edge only inside the graph builder. + +## Related + +:::links +- [Asset types by category](/reference/catalog/asset-types-by-category/) All 156 `AssetType` values, grouped. +- [Relationship vocabulary](/reference/catalog/relationship-vocabulary/) The `relationship` labels on edges. +- [Result objects](/api/results/) The dataclasses that carry these models out of the engine. +- [FindingsNormaliser](/api/findingsnormaliser/) Where findings get merged, rescored and mapped. +- [Output files](/reference/output-files/) The JSON files these models are written to. +::: diff --git a/site/content/api/parse-report.md b/site/content/api/parse-report.md new file mode 100644 index 0000000..330dcbd --- /dev/null +++ b/site/content/api/parse-report.md @@ -0,0 +1,144 @@ +--- +object: [cloudg.ingest.parse_report, cloudg.ingest.ingest_reports] +lede: "Turn the native output of Prowler, ScoutSuite, Checkov and Trivy into cloudg `Finding` objects. No engine, no credentials, no scanner binaries." +--- + +## Two functions, one dispatcher + +`cloudg.ingest` is a small module with two public functions and one constant. `SUPPORTED_TOOLS` is the tuple `("prowler", "scoutsuite", "checkov", "trivy")`. + +`parse_report(tool, path)` handles one report. It lower-cases and strips the tool name, checks that the path exists, and hands the path to the `parse_report` classmethod of the matching scanner wrapper: `ProwlerScanner`, `ScoutSuiteScanner`, `CheckovScanner` or `TrivyScanner` in `cloudg.scanners`. Those classmethods are public too, if you prefer to call one directly. + +`ingest_reports(reports)` takes a dict of tool name to a list of paths and calls `parse_report` for each pair, collecting everything into one list. It is what `CloudGEngine.ingest_reports()` and `cloudg ingest` use underneath. + +```mermaid caption="How ingest_reports handles each path" +flowchart TD + R["reports dict"] --> L["for each tool, for each path"] + L --> P["parse_report(tool, path)"] + P -->|"unknown tool"| V["ValueError"] + P -->|"path missing"| N["FileNotFoundError"] + P --> S["ScannerClass.parse_report(path)"] + S --> F["list of Finding"] + V --> K["logged at ERROR, path skipped"] + N --> K + F --> C["combined list"] +``` + +`parse_report` raises both errors. `ingest_reports` catches exactly those two, logs them on the `cloudg.ingest` logger at ERROR level, and carries on with the next path. A file that exists but does not parse (truncated JSON, the wrong tool's output) is handled one level down: the scanner wrapper logs a warning and returns what it could read, often an empty list. Neither function raises for bad content. + +## What each tool accepts + +| Tool | A file | A directory | Produce it with | +|---|---|---|---| +| `prowler` | an ASFF JSON array or JSONL file | searched recursively for `*.json` | `prowler aws -M json-asff -o ./prowler-output` | +| `scoutsuite` | `scoutsuite_results*.js` | searched recursively for `scoutsuite_results*.js` | `scout aws --report-dir ./scoutsuite-report --no-browser` | +| `checkov` | the JSON report | `results_json.json` anywhere below it, else every `*.json` | `checkov -d ./iac --output json > results_json.json` | +| `trivy` | an `image` or `fs` JSON report | every `*.json` below it | `trivy image --format json myrepo/app:latest > trivy-image.json` | + +A few rules per tool decide what becomes a finding. Prowler checks with `Compliance.Status` of `PASSED` are skipped, and the check name inside the ASFF `Id` becomes the deduplication key. ScoutSuite yields one finding per flagged item, with `danger`, `warning` and `caution` mapped to CRITICAL, HIGH and MEDIUM. Checkov findings come from `results.failed_checks`, whether the report is one object or a list of them (several frameworks). Trivy reports are split by their `ArtifactType`: `container_image` reports give vulnerability and secret findings, anything else also gives misconfigurations. [Input requirements](/guides/ingesting/#input-requirements) lists every field read. + +The findings come back raw. Their severities are what the scanner said, duplicates are still there, and only scanner-native compliance tags are set. Pass them through [`FindingsNormaliser`](/api/findingsnormaliser/) (or `CloudGEngine.normalise_findings()`) before you count or report on them. + +## Examples + +### Parse first, decide later + +Runs offline against one Prowler directory, one Checkov file and one Trivy file. Logging is on so the skipped paths are visible. + +```python title="parse_first.py" +import logging + +from cloudg.ingest import SUPPORTED_TOOLS, ingest_reports, parse_report +from cloudg.normaliser import FindingsNormaliser + +logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") + +print(SUPPORTED_TOOLS) +findings = parse_report("prowler", "./prowler-output/") +findings += parse_report("checkov", "./results_json.json") +print(len(findings), "findings from two reports") + +# A typo'd tool name or a missing path is skipped by ingest_reports, not raised +more = ingest_reports({"trivy": ["./trivy.json", "./missing.json"], "trivvy": ["./trivy.json"]}) +print(len(more), "findings from the batch") + +try: + parse_report("grype", "./trivy.json") +except ValueError as exc: + print("ValueError:", exc) + +scan_result = FindingsNormaliser().normalise(findings, more) +print(scan_result.summary["severity_breakdown"], len(scan_result.compliance), "control results") +``` + +```console +$ python parse_first.py +INFO cloudg.scanners.prowler: Parsed 1 findings from Prowler +INFO cloudg.scanners.checkov: Parsed 1 findings from Checkov +INFO cloudg.scanners.trivy: Parsed 1 findings from Trivy for myrepo/app:latest +INFO cloudg.ingest: [trivy] 1 findings ingested from ./trivy.json +ERROR cloudg.ingest: Skipping trivy report ./missing.json: Report path does not exist: ./missing.json +ERROR cloudg.ingest: Skipping trivvy report ./trivy.json: Unsupported tool 'trivvy'. Supported: prowler, scoutsuite, checkov, trivy +('prowler', 'scoutsuite', 'checkov', 'trivy') +2 findings from two reports +1 findings from the batch +ValueError: Unsupported tool 'grype'. Supported: prowler, scoutsuite, checkov, trivy +{'CRITICAL': 1, 'HIGH': 2} 31 control results +``` + +The log lines appear first because logging writes to stderr and `print` to stdout. Without `logging.basicConfig()`, the two ERROR lines still reach stderr through Python's last-resort handler, but the INFO lines do not. + +### Save findings for cloudg map + +`cloudg map --findings` reads a JSON list of findings. This writes one from scanner output collected in CI, so a later map run on another host can overlay it. + +```python title="export_findings.py" +import json +from pathlib import Path + +from cloudg.ingest import ingest_reports + +findings = ingest_reports( + { + "prowler": ["./prowler-output/"], + "checkov": ["./results_json.json"], + "trivy": ["./trivy.json"], + } +) +out = Path("./reports/raw-findings.json") +out.parent.mkdir(parents=True, exist_ok=True) +out.write_text(json.dumps([f.model_dump(mode="json") for f in findings], indent=2)) +print(f"wrote {len(findings)} findings to {out}") +``` + +```bash +cloudg map -p aws --regions all -o ./reports --findings ./reports/raw-findings.json +``` + +### One tool, no dispatcher + +```python title="trivy_only.py" +from cloudg.scanners.trivy import TrivyScanner + +for f in TrivyScanner.parse_report("./trivy.json"): + print(f.severity.value, f.source_finding_id, f.resource_id, f.cvss_score) +``` + +## Notes + +A misspelt tool name in `ingest_reports` costs you that tool's findings and one ERROR log line, nothing more. If you build the dict from user input, validate the keys against `SUPPORTED_TOOLS` first. + +Directories are searched recursively, and for Prowler and Trivy every `*.json` file below the path is a candidate. Point them at the scanner's own output directory, not at a parent that also holds other JSON (a `reports/` folder with cloudg's own `findings.json`, say), or unrelated files are read as scanner output and mostly produce warnings. + +`parse_report` passes the path to the scanner wrappers as a string. A `pathlib.Path` is fine as input to both functions. + +The `resource_id` of a finding is whatever the scanner identified: an ARN for Prowler, a Terraform address such as `aws_s3_bucket.data` for Checkov, an image name for Trivy image scans. Matching those to collected assets is the job of [`AssetIndex`](/api/dependencygraph/) and of the asset map in [`InventoryMapper`](/api/inventorymapper/). + +## Related + +:::links +- [Ingesting existing output](/guides/ingesting/) The guide, with the CLI and the input formats in detail. +- [cloudg ingest](/cli/ingest/) The command built on these functions. +- [FindingsNormaliser](/api/findingsnormaliser/) The step that should follow parsing. +- [CloudGEngine](/api/cloudgengine/) `ingest_reports()` and `run_from_reports()` with hooks. +::: diff --git a/site/content/api/ragexporter.md b/site/content/api/ragexporter.md new file mode 100644 index 0000000..c5b5acb --- /dev/null +++ b/site/content/api/ragexporter.md @@ -0,0 +1,249 @@ +--- +title: RAGExporter +object: cloudg.graph.rag_export.RAGExporter +members: [export_all, export_entity_chunks, export_community_chunks, export_relation_chunks] +lede: Cuts an inventory into text chunks with metadata, ready to embed for retrieval-augmented generation. +since: "0.6.0" +--- + +A language model answering questions about your cloud needs the right slice of the inventory in its context window, not the whole `inventory-map.json`. `RAGExporter` writes three kinds of chunk, each aimed at a different kind of question, as one JSON object per line. Embed the `content` field, keep `metadata` for filtering, and you have a retrieval corpus. Nothing in the exporter calls a model or an embedding API. + +## Chunk kinds + +| Kind | `chunk_id` | One chunk per | Good for | +|---|---|---|---| +| `entity` | `entity::` | asset | "tell me about `orders-db`" | +| `community` | `community::` | cluster of two or more tightly connected nodes (Louvain) | "what sits around this subnet?", blast-radius style questions | +| `relation_group` | `relation_group::` | relation group present (`NETWORK`, `IAM`, `SECURITY` and so on) | "list every rule open to the internet", "show the IAM grants" | + +Every chunk has the same five keys: `chunk_id`, `chunk_type`, `content` (the text to embed), `metadata` (a flat-ish dict for filtering) and `relations` (a list of dicts). + +| Kind | `metadata` keys | `relations` | +|---|---|---| +| `entity` | `asset_type`, `provider`, `region`, `account_id`, `arn`, `is_internet_exposed`, `relation_types`, `severity_max`, `compliance_frameworks`, `neighbour_count`, `finding_count` | every inferred relation of the asset's edges, with `predicate`, `direction` (`outgoing` or `incoming`), `object`, `object_id`, `evidence` | +| `community` | `community_id`, `member_count`, `asset_types`, `internal_edges`, `external_edges`, `internet_exposed_count`, `finding_count`, `severity_max`, `compliance_frameworks`, `risk_score` | empty | +| `relation_group` | `relation_group`, `total_relations`, `relation_type_counts`, `unique_subjects`, `unique_objects` | the first 50 triples, with `predicate`, `object`, `evidence` | + +`severity_max` is the worst severity among the matched findings, or `"NONE"`. A community's `risk_score` is `0.3` per member plus `0.5` per finding plus `2.0` per internet-exposed member, capped at 10. + +The relations are the ones the [ontology](/api/cloudontology/) uses: `infer_relations()` on each edge (so a security group rule from `0.0.0.0/0` on port 443 becomes `INGRESS_ALLOWED`, `INTERNET_REACHABLE` and `ONLY_HTTPS`), plus the asset-level relations from metadata (`ENCRYPTED_BY_KMS`, VPC containment from `vpc_id`, tag ownership, key rotation, EC2 security group membership). Relation-group chunks carry both; entity chunks carry the edge relations only. + +Findings attach to assets through `AssetIndex`, so a finding whose `resource_id` is an ARN or a unique name still counts toward the right entity and community. + +## Output files + +`export_all()` writes two files into `output_dir` (created if missing) and returns their paths as `{"chunks": Path, "index": Path}`. + +| File | Contents | +|---|---| +| `rag_chunks.jsonl` | one chunk per line: entity chunks first, then community, then relation-group | +| `rag_metadata_index.json` | `total_chunks`, `entity_chunks`, `community_chunks`, `relation_group_chunks`, `chunk_ids` (in file order) and `chunk_types` | + +```mermaid caption="What export_all() does" +flowchart LR + A["assets, edges"] --> E["export_entity_chunks"] + G["graph"] --> L["Louvain communities"] --> C["export_community_chunks"] + A --> R["export_relation_chunks"] + F["findings"] --> E + F --> C + E --> J["rag_chunks.jsonl"] + C --> J + R --> J + J --> I["rag_metadata_index.json"] +``` + +`cloudg run` writes these files twice. Phase 2c exports right after the graph is built, when the only findings are the reachability ones. After the scanners finish, a silent second pass rewrites both files with every finding. If that second pass fails, the files from phase 2c stay, with only reachability findings in them. `--no-rag-export` or `rag.enabled: false` turns the export off. + +## Examples + +### Export and reshape for a vector store + +The script needs nothing beyond cloudg. It exports the chunks, then turns every chunk into an `{id, text, metadata}` record with plain-text content and scalar metadata, which is the shape most vector stores accept. + +```python title="rag_demo.py" +"""Export RAG chunks for a small estate and reshape them for a vector store.""" + +import json +from pathlib import Path + +from cloudg.graph.builder import GraphBuilder +from cloudg.graph.rag_export import RAGExporter +from cloudg.schema.models import ( + AssetType, + CloudAsset, + CloudProvider, + EdgeType, + Finding, + NetworkEdge, + Severity, +) + +ACCOUNT = "123456789012" +REGION = "eu-west-1" + + +def asset(asset_id: str, asset_type: AssetType, arn: str, **extra) -> CloudAsset: + return CloudAsset( + id=asset_id, + name=asset_id, + asset_type=asset_type, + provider=CloudProvider.AWS, + region=REGION, + account_id=ACCOUNT, + arn=arn, + **extra, + ) + + +assets = [ + asset("sg-web", AssetType.SECURITY_GROUP, f"arn:aws:ec2:{REGION}:{ACCOUNT}:security-group/sg-web"), + asset("web-1", AssetType.EC2, f"arn:aws:ec2:{REGION}:{ACCOUNT}:instance/i-0web1", + is_internet_exposed=True), + asset("web-2", AssetType.EC2, f"arn:aws:ec2:{REGION}:{ACCOUNT}:instance/i-0web2"), + asset("app-role", AssetType.IAM_ROLE, f"arn:aws:iam::{ACCOUNT}:role/app-role"), + asset("orders-db", AssetType.RDS_INSTANCE, f"arn:aws:rds:{REGION}:{ACCOUNT}:db:orders-db"), +] +edges = [ + NetworkEdge(source_id="0.0.0.0/0", target_id="sg-web", edge_type=EdgeType.SECURITY_GROUP_RULE, + port_range="443", protocol="TCP", cidr="0.0.0.0/0", direction="ingress"), + NetworkEdge(source_id="web-1", target_id="sg-web", edge_type=EdgeType.ATTACHED_TO), + NetworkEdge(source_id="web-2", target_id="sg-web", edge_type=EdgeType.ATTACHED_TO), + NetworkEdge(source_id="web-1", target_id="app-role", edge_type=EdgeType.ASSUMES_ROLE), + NetworkEdge(source_id="app-role", target_id="orders-db", edge_type=EdgeType.GRANTS_ACCESS), +] +findings = [ + Finding( + resource_id="web-1", + severity=Severity.HIGH, + title="Instance metadata service v1 enabled", + description="IMDSv2 is not enforced.", + source_tool="prowler", + compliance_frameworks=["CIS"], + ) +] + +graph = GraphBuilder().build(assets, edges) +paths = RAGExporter().export_all(assets, edges, graph, findings, output_dir="rag-out") +print({kind: path.name for kind, path in paths.items()}) + +index = json.loads(paths["index"].read_text()) +print({k: v for k, v in index.items() if k.endswith("chunks")}) + +chunks = [json.loads(line) for line in paths["chunks"].read_text().splitlines()] + +# The content uses a few symbols; plain words embed and display more reliably +MARKERS = { + "\N{WARNING SIGN}": "WARNING:", + "\N{RIGHTWARDS ARROW}": "->", + "\N{LEFTWARDS ARROW}": "<-", + "\N{BULLET}": "-", +} + + +def clean(text: str) -> str: + for symbol, word in MARKERS.items(): + text = text.replace(symbol, word) + return text + + +def scalar(value): + """Most vector stores accept only str, int, float and bool metadata.""" + if isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, list): + return ",".join(map(str, value)) + return json.dumps(value, sort_keys=True) + + +records = [ + { + "id": c["chunk_id"], + "text": clean(c["content"]), + "metadata": {"chunk_type": c["chunk_type"], + **{k: scalar(v) for k, v in c["metadata"].items()}}, + } + for c in chunks +] +out = Path("rag-out") / "vector-records.jsonl" +out.write_text("".join(json.dumps(r) + "\n" for r in records)) +print(len(records), "records ->", out) + +web_1 = next(r for r in records if r["id"] == "entity::web-1") +print(web_1["text"]) +print(web_1["metadata"]) + +# Pre-filter on metadata before any similarity search +urgent = [r["id"] for r in records if r["metadata"].get("severity_max") in ("CRITICAL", "HIGH")] +print("high or critical:", urgent) +``` + +```console +$ python rag_demo.py +{'chunks': 'rag_chunks.jsonl', 'index': 'rag_metadata_index.json'} +{'total_chunks': 11, 'entity_chunks': 5, 'community_chunks': 2, 'relation_group_chunks': 4} +11 records -> rag-out/vector-records.jsonl +Resource: web-1 +Type: EC2 +Provider: AWS +Region: eu-west-1 +ARN: arn:aws:ec2:eu-west-1:123456789012:instance/i-0web1 +Account: 123456789012 +WARNING: INTERNET EXPOSED + +Relations (2): + -> PROTECTED_BY_SG: sg-web + -> RUNS_ON: app-role + +Findings (1): + [HIGH] Instance metadata service v1 enabled +{'chunk_type': 'entity', 'asset_type': 'EC2', 'provider': 'AWS', 'region': 'eu-west-1', 'account_id': '123456789012', 'is_internet_exposed': True, 'relation_types': 'PROTECTED_BY_SG,RUNS_ON', 'severity_max': 'HIGH', 'compliance_frameworks': 'CIS', 'neighbour_count': 2, 'finding_count': 1, 'arn': 'arn:aws:ec2:eu-west-1:123456789012:instance/i-0web1'} +high or critical: ['entity::web-1', 'community::1'] +``` + +The community number can differ on your run (see the notes). Each line of `vector-records.jsonl` looks like this, wrapped here for reading: + +```json title="rag-out/vector-records.jsonl" +{"id": "community::1", + "text": "Community 1 (3 resources):\n - web-1 (EC2)\n - app-role (IAM_ROLE)\n - orders-db (RDS_INSTANCE)\n\nAsset types: {'EC2': 1, 'IAM_ROLE': 1, 'RDS_INSTANCE': 1}\nInternal edges: 2, External edges: 1\nWARNING: 1 internet-exposed resources\nFindings: 1 (max severity: HIGH)", + "metadata": {"chunk_type": "community", "community_id": 1, "member_count": 3, + "asset_types": "{\"EC2\": 1, \"IAM_ROLE\": 1, \"RDS_INSTANCE\": 1}", + "internal_edges": 2, "external_edges": 1, "internet_exposed_count": 1, + "finding_count": 1, "severity_max": "HIGH", "compliance_frameworks": "CIS", + "risk_score": 3.4}} +``` + +From here, loading into a store is a few lines in that store's own client: pass `id`, `text` (or your embedding of it) and `metadata` to its add or upsert call. Use the `chunk_type` and `severity_max` fields as filters so a question about one asset retrieves entity chunks first. + +### Only one kind of chunk + +The three `export_*` methods return `RAGChunk` objects without writing anything. Call one directly when you only want, say, the relation groups: + +```python title="relation_chunks.py" +from cloudg.graph.rag_export import RAGExporter +from cloudg.inventory import InventoryResult + +result = InventoryResult.load("./reports") +assets_by_id = {a.id: a for a in result.assets} +for chunk in RAGExporter().export_relation_chunks(result.edges, assets_by_id): + data = chunk.to_dict() + print(data["chunk_id"], data["metadata"]["total_relations"]) +``` + +## Notes + +- All three kinds are always written. The `rag.chunk_strategy` key in `config.yaml` (`entity`, `community`, `relation_group` or `hybrid`) is validated but not read by the exporter, and `max_chunk_tokens` (the constructor argument and `rag.max_chunk_tokens`) is stored but not applied. Chunk size is bounded by fixed caps instead: 30 relations and 10 findings in an entity chunk's text, 50 triples in a relation-group chunk. A community chunk lists every member, so a very large community makes a very long chunk; split such chunks yourself before embedding. +- Community numbers come from python-louvain's `best_partition` without a fixed seed, so `community::3` in one run is not the same cluster as `community::3` in the next. Key on members, not on the id. Without python-louvain installed, the exporter logs a warning and uses connected components instead. +- Communities are found on the undirected version of the graph, and nodes left alone in a community (singletons) produce no chunk. Placeholder nodes such as `0.0.0.0/0` take part in clustering like any other node. +- Entity chunks are built from the `NetworkEdge` list, so two rules on the same pair appear separately there. Community chunks read the graph, where `GraphBuilder` has merged them, and take `is_internet_exposed` from the graph's node attributes. +- `content` uses four non-ASCII symbols: a warning sign (U+26A0) before exposure lines, arrows (U+2192, U+2190) before outgoing and incoming relations, and a bullet (U+2022) before community members. `rag_chunks.jsonl` stores them as `\u` escapes, and they come back as the symbols when parsed. The example swaps them for words before embedding. +- `chunk_types` in the index comes from a set, so its order varies between runs. + +## Related + +:::links +- [CloudOntology](/api/cloudontology/) The same relations as RDF. +- [GraphBuilder](/api/graphbuilder/) Builds the graph the community chunks are cut from. +- [Graph, ontology and RAG](/guides/graph-ontology-rag/) The analysis phase end to end. +- [cloudg run](/cli/run/) The `--rag-export` switch. +- [Output files](/reference/output-files/) Where `rag_chunks.jsonl` sits among the other outputs. +::: diff --git a/site/content/api/reachabilityanalyzer.md b/site/content/api/reachabilityanalyzer.md new file mode 100644 index 0000000..354dfae --- /dev/null +++ b/site/content/api/reachabilityanalyzer.md @@ -0,0 +1,255 @@ +--- +title: ReachabilityAnalyzer +object: + - cloudg.graph.reachability.ReachabilityAnalyzer + - cloudg.graph.reachability.finding_id + - cloudg.graph.reachability.network_flow_graph +members: + cloudg.graph.reachability.ReachabilityAnalyzer: [internet_entry_points, find_internet_exposed, flow_hops, flow_successors, generate_findings, compute_blast_radius] +lede: Walks a `GraphBuilder` graph from the internet along the edges traffic can actually take, and reports what it reaches with stable finding ids. +since: "0.6.0" +--- + +`cloudg run` calls `ReachabilityAnalyzer(graph).generate_findings()` right after it builds the graph, and the results show up in `findings.json` with `source_tool` set to `cloudg-reachability`. The class answers two different questions, and they follow different edges: + +- What can the internet reach? `find_internet_exposed()` walks network-flow edges only, with the rules below. +- What could a compromised resource reach? `compute_blast_radius(node)` follows every edge, identity included. + +The analyzer works on the graph you pass in and changes it: `find_internet_exposed()` sets `is_internet_exposed=True` on every node it reaches. Pass `graph.copy()` if you need the original untouched (the MCP layer does). + +## The network-flow walk + +Release 0.6.0 changed the walk. Before it, the analysis followed every edge type out of an exposed node, so a database whose only link to the internet was an IAM role granted access to it came out as a CRITICAL "Internet-exposed RDS_INSTANCE". Now an edge is followed only when traffic can travel along it, and in the direction it travels. + +```mermaid caption="Edges the exposure walk follows, in the direction traffic moves" +flowchart LR + I["0.0.0.0/0, ::/0, Internet tag"] -->|"SG or NACL rule, ingress"| G["Security group, NSG, NACL"] + G -->|"ATTACHED_TO, walked backwards"| R["Resource in the group"] + I -->|"INTERNET_EXPOSED"| P["Public resource"] + R -->|"LOAD_BALANCER_TARGET"| T["Target group"] + T -->|"LOAD_BALANCER_TARGET"| W["Targets"] + E["ENI or Elastic IP"] -->|"ATTACHED_TO"| V["Instance"] + N["VPC, VNet, subnet"] -->|"CONTAINS"| M["Resource placed in it"] + S["Route or peering source"] -->|"ROUTE, PEERING"| D["Destination"] +``` + +| Edge type | Followed | Direction | +|---|---|---| +| `INTERNET_EXPOSED`, `LOAD_BALANCER_TARGET`, `ROUTE`, `PEERING` | always | source to target | +| `SECURITY_GROUP_RULE`, `NACL_RULE` | unless `direction` is `egress` | source to target | +| `ATTACHED_TO` | when the target is a `SECURITY_GROUP`, `NSG` or `NACL`, or the edge's `relationship` is `PROTECTED_BY_SG` or `PROTECTED_BY_NACL` | backwards, from the group to the resource attached to it | +| `ATTACHED_TO` | when the source is a `NETWORK_INTERFACE` or `ELASTIC_IP` | source to target | +| `CONTAINS` | only from a `VPC`, `VNET` or `SUBNET` | parent to child | +| `IAM_TRUST`, `IAM_POLICY_ATTACHMENT`, `ASSUMES_ROLE`, `GRANTS_ACCESS`, `INVOKES`, `REFERENCES`, `USES_IMAGE`, `LOGS_TO`, `PROTECTS`, `MONITORS`, `MANAGES`, `GOVERNS` | never | | + +The backwards `ATTACHED_TO` hop is the one people miss. Collectors and the linker write `web-1 ATTACHED_TO sg-web`, pointing from the resource to its group, while traffic flows the other way: whatever the group admits reaches `web-1`. + +`INVOKES` is deliberately not followed, so a Lambda function behind a public API Gateway is not reported as exposed. The gateway is the exposed resource; the function is only invoked by it. Disk, route-table and gateway attachments are not traffic paths either, and organization, account, cluster and namespace containment are not traversed. + +Where the walk starts: every node named `0.0.0.0/0` or `::/0`, plus the source of every non-egress edge whose `cidr` stands for the whole internet. That test (`cloudg.graph.ports.is_internet_source`) also accepts the Azure NSG service tags `Internet`, `Any` and `*` in any letter case. Private CIDRs such as `10.0.0.0/8`, unresolved references and a group with an egress rule to `0.0.0.0/0` are not entry points. + +The walk over-approximates. It does not intersect ports across hops, and a rule whose source is another security group is followed like any other ingress rule, so a database whose group admits the web tier's group is reported exposed once the web tier is. Treat a finding as "there is a network path", then check the ports. + +## What generate_findings() reports + +| Rule (`finding_id` input) | Raised for | Severity | Frameworks | +|---|---|---|---| +| `internet-exposed-sensitive-asset` | a reached `RDS_INSTANCE`, `AURORA_CLUSTER`, `AZURE_SQL`, `CLOUD_SQL` or `DYNAMODB_TABLE` | CRITICAL | CIS, NIST-800-53 | +| `internet-exposed-unexpected-asset` | any other reached resource, except load balancers, CloudFront, CDNs and internet gateways | HIGH | none | +| `internet-open-sensitive-port:` | a non-egress edge from an internet source whose ports include one of 22, 3389, 3306, 5432, 1433, 27017, 6379, 9200, 5601, 8080 or 8443 | CRITICAL | CIS, NIST-800-53, PCI-DSS | + +Security groups, NSGs, NACLs and target groups are hops: the walk marks them exposed, but they get no exposure finding of their own. A group's open rules show up as open-port findings instead, with the group as `resource_id`, and a target group's exposure shows up on the targets behind it. Placeholder nodes (`is_external`) never get an exposure finding. + +Ports are compared as numbers after `cloudg.graph.ports` parses `port_range`, so `0-65535` contains 22, `2200-2300` does not, and an Azure `*` covers all eleven ports. An edge yields at most one finding per sensitive port, however many of its merged rules overlap it. + +### Deterministic ids + +Each finding id is a UUID5 of the rule name and a stable key, so the same exposure gets the same id on every scan. You can dedupe, track and suppress reachability findings across runs. + +```python +finding_id(rule, asset_key(graph, node_id)) # exposure findings +finding_id(f"{RULE_OPEN_PORT}:{port}", f"{src_key}->{dst_key}") # open-port findings +``` + +The key is the node's ARN, or the node id when there is no ARN (CIDR placeholders). The graph node id itself would not do: collectors give assets a fresh random UUID on every collection. `Finding.resource_id` is still that per-run node id, so match findings to assets of a later run by `resource_arn`. + +## Examples + +### Exposure, findings and blast radius + +Ten assets, fourteen edges, no credentials. The estate has a public load balancer in front of `web-1`, an SSH rule left open on the web tier's group, a database reachable only from a private subnet and through an IAM grant, and a Lambda function behind a public API Gateway. + +```python title="exposure.py" +"""Internet exposure, flow hops, finding ids and blast radius on a small estate.""" + +from cloudg.graph.builder import GraphBuilder +from cloudg.graph.reachability import ( + RULE_UNEXPECTED_EXPOSURE, + ReachabilityAnalyzer, + asset_key, + finding_id, + network_flow_graph, +) +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType, NetworkEdge + +ACCOUNT = "123456789012" +REGION = "eu-west-1" + + +def asset(asset_id: str, asset_type: AssetType, arn: str) -> CloudAsset: + return CloudAsset( + id=asset_id, + name=asset_id, + asset_type=asset_type, + provider=CloudProvider.AWS, + region=REGION, + account_id=ACCOUNT, + arn=arn, + ) + + +assets = [ + asset("sg-alb", AssetType.SECURITY_GROUP, f"arn:aws:ec2:{REGION}:{ACCOUNT}:security-group/sg-alb"), + asset("sg-web", AssetType.SECURITY_GROUP, f"arn:aws:ec2:{REGION}:{ACCOUNT}:security-group/sg-web"), + asset("sg-db", AssetType.SECURITY_GROUP, f"arn:aws:ec2:{REGION}:{ACCOUNT}:security-group/sg-db"), + asset("web-alb", AssetType.LOAD_BALANCER, f"arn:aws:elasticloadbalancing:{REGION}:{ACCOUNT}:loadbalancer/app/web-alb/1"), + asset("web-tg", AssetType.TARGET_GROUP, f"arn:aws:elasticloadbalancing:{REGION}:{ACCOUNT}:targetgroup/web-tg/1"), + asset("web-1", AssetType.EC2, f"arn:aws:ec2:{REGION}:{ACCOUNT}:instance/i-0web1"), + asset("app-role", AssetType.IAM_ROLE, f"arn:aws:iam::{ACCOUNT}:role/app-role"), + asset("orders-db", AssetType.RDS_INSTANCE, f"arn:aws:rds:{REGION}:{ACCOUNT}:db:orders-db"), + asset("orders-api", AssetType.API_GATEWAY, f"arn:aws:apigateway:{REGION}::/restapis/a1b2c3"), + asset("orders-fn", AssetType.LAMBDA_FUNCTION, f"arn:aws:lambda:{REGION}:{ACCOUNT}:function:orders-fn"), +] + + +def ingress(source: str, target: str, ports: str, cidr: str | None = None) -> NetworkEdge: + return NetworkEdge( + source_id=source, + target_id=target, + edge_type=EdgeType.SECURITY_GROUP_RULE, + port_range=ports, + protocol="TCP", + cidr=cidr, + direction="ingress", + ) + + +def edge(source: str, target: str, edge_type: EdgeType, **extra) -> NetworkEdge: + return NetworkEdge(source_id=source, target_id=target, edge_type=edge_type, **extra) + + +edges = [ + ingress("0.0.0.0/0", "sg-alb", "443", cidr="0.0.0.0/0"), + ingress("0.0.0.0/0", "sg-web", "22", cidr="0.0.0.0/0"), # SSH left open + ingress("sg-alb", "sg-web", "8080"), + ingress("10.0.1.0/24", "sg-db", "5432", cidr="10.0.1.0/24"), # private source + # The default egress rule points from the group to the internet + edge("sg-web", "0.0.0.0/0", EdgeType.SECURITY_GROUP_RULE, port_range="0-65535", + protocol="ALL", cidr="0.0.0.0/0", direction="egress"), + edge("web-alb", "sg-alb", EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), + edge("web-1", "sg-web", EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), + edge("orders-db", "sg-db", EdgeType.ATTACHED_TO, relationship="PROTECTED_BY_SG"), + edge("web-alb", "web-tg", EdgeType.LOAD_BALANCER_TARGET), + edge("web-tg", "web-1", EdgeType.LOAD_BALANCER_TARGET), + edge("web-1", "app-role", EdgeType.ASSUMES_ROLE), + edge("app-role", "orders-db", EdgeType.GRANTS_ACCESS), + edge("0.0.0.0/0", "orders-api", EdgeType.INTERNET_EXPOSED, cidr="0.0.0.0/0"), + edge("orders-api", "orders-fn", EdgeType.INVOKES), +] + +graph = GraphBuilder().build(assets, edges) +analyzer = ReachabilityAnalyzer(graph) + +print("entry points:", sorted(analyzer.internet_entry_points())) +print("exposed:", sorted(analyzer.find_internet_exposed())) + +print("hops out of sg-web:") +for nxt, data, walked_back in analyzer.flow_hops("sg-web"): + print(f" -> {nxt} via {data['edge_type']}{' (reversed)' if walked_back else ''}") + +for finding in analyzer.generate_findings(): + print(f"{finding.severity.value:8} {finding.title} id={finding.id[:8]}") + +# The id can be recomputed from the rule name and the asset's ARN +expected = finding_id(RULE_UNEXPECTED_EXPOSURE, asset_key(graph, "web-1")) +print("web-1 exposure id:", expected[:8]) + +flow = network_flow_graph(graph) +print("flow edges:", flow.number_of_edges(), "of", graph.number_of_edges()) +print("flow reversed:", sorted((u, v) for u, v, d in flow.edges(data=True) if d["reversed"])) + +radius = analyzer.compute_blast_radius("web-1") +print("blast radius of web-1:", sorted(radius["reachable_nodes"]), + "depth", radius["depth"], "risk", radius["risk_score"]) +``` + +```console +$ python exposure.py +entry points: ['0.0.0.0/0'] +exposed: ['orders-api', 'sg-alb', 'sg-web', 'web-1', 'web-alb', 'web-tg'] +hops out of sg-web: + -> web-1 via ATTACHED_TO (reversed) +HIGH Unexpected internet-exposed resource: orders-api id=b5fbe235 +HIGH Unexpected internet-exposed resource: web-1 id=4ec3a339 +CRITICAL Security group allows SSH (port 22) from 0.0.0.0/0 id=17cd5168 +web-1 exposure id: 4ec3a339 +flow edges: 10 of 14 +flow reversed: [('sg-alb', 'web-alb'), ('sg-db', 'orders-db'), ('sg-web', 'web-1')] +blast radius of web-1: ['0.0.0.0/0', 'app-role', 'orders-api', 'orders-db', 'orders-fn', 'sg-alb', 'sg-db', 'sg-web'] depth 4 risk 6.0 +``` + +What the output shows: + +- `10.0.1.0/24` and the group with the egress rule are not entry points. `orders-db` is not exposed: its only inbound rule comes from a private CIDR, and the IAM path through `app-role` does not count. +- `orders-fn` is not exposed, because `INVOKES` is not a network hop. The API Gateway is reported instead. +- `web-alb` is reached but gets no finding (load balancers are expected to face the internet), and the groups and the target group are hops only. +- The ids are the same on every run. Run the script twice and compare. +- The only edge leaving `sg-web` in the graph is its egress rule, which the walk skips. Its one flow hop is the attachment from `web-1`, walked backwards. + +The blast radius is a different, much wider question. From `web-1` it follows the IAM edges to `orders-db`, and it also follows the egress rule into the `0.0.0.0/0` placeholder and from there to everything the internet reaches. Read it as an upper bound. + +### Flow-aware paths with NetworkX + +`network_flow_graph()` returns a new `DiGraph` with an edge `u -> v` exactly where the exposure walk would hop, so ordinary NetworkX path functions follow traffic. Save this next to `exposure.py`; the import runs that script first, so its output is printed too. + +```python title="flow_paths.py" +import networkx as nx + +from cloudg.graph.reachability import network_flow_graph +from exposure import graph + +flow = network_flow_graph(graph) +for path in nx.all_simple_paths(flow, "0.0.0.0/0", "web-1", cutoff=6): + hops = [flow.edges[u, v]["edge_type"] + (" (rev)" if flow.edges[u, v]["reversed"] else "") + for u, v in zip(path, path[1:])] + print(" -> ".join(path), "|", ", ".join(hops)) +``` + +```console +$ python flow_paths.py +... +0.0.0.0/0 -> sg-alb -> sg-web -> web-1 | SECURITY_GROUP_RULE, SECURITY_GROUP_RULE, ATTACHED_TO (rev) +0.0.0.0/0 -> sg-alb -> web-alb -> web-tg -> web-1 | SECURITY_GROUP_RULE, ATTACHED_TO (rev), LOAD_BALANCER_TARGET, LOAD_BALANCER_TARGET +0.0.0.0/0 -> sg-web -> web-1 | SECURITY_GROUP_RULE, ATTACHED_TO (rev) +``` + +The first path is the over-approximation at work: `sg-alb` admits 443 from the internet and `sg-web` admits 8080 from `sg-alb`, and the walk does not check that the ports line up. + +## Notes + +- `compute_blast_radius()` returns `reachable_nodes` (every descendant over every edge type), `depth` (the longest shortest-path distance) and `risk_score`: 0.5 per reachable node, plus 2.0 for each reachable `RDS_INSTANCE`, `AURORA_CLUSTER`, `AZURE_SQL`, `CLOUD_SQL` or `DYNAMODB_TABLE`, capped at 10.0. An unknown node gives `{"reachable_nodes": [], "depth": 0, "risk_score": 0.0}`. +- `flow_hops()` can yield the same next node more than once when several edges lead there, and yields nothing for an unknown node. `flow_successors()` is the same walk without the edge data. +- `network_flow_graph()` copies every node with its attributes and leaves the input graph alone. Each edge carries `edge_type` and `reversed`. +- Graphs built by hand can carry a `ports` list on an edge, and the open-port check reads it next to `port_range`. `GraphBuilder` never copies `NetworkEdge.ports` onto the graph. +- The open-port title keeps the historical `0.0.0.0/0` wording for both `0.0.0.0/0` and `::/0`, and names the Azure tag as written (`Internet`, `Any`, `*`) otherwise. +- Stored findings from 0.5.x carry random ids. They will not match the 0.6.0 ids; re-baseline suppressions after upgrading. + +## Related + +:::links +- [GraphBuilder](/api/graphbuilder/) Builds the graph this class walks. +- [Dependencies and blast radius](/guides/dependencies/) The inventory-level blast radius behind `cloudg deps`. +- [DependencyGraph](/api/dependencygraph/) Dependency walks over a saved map. +- [Data models](/api/models/) The `Finding` model these results use. +- [Graph, ontology and RAG](/guides/graph-ontology-rag/) Where reachability sits in the analysis phase. +::: diff --git a/site/content/api/relationshiplinker.md b/site/content/api/relationshiplinker.md new file mode 100644 index 0000000..c228ce7 --- /dev/null +++ b/site/content/api/relationshiplinker.md @@ -0,0 +1,204 @@ +--- +object: cloudg.inventory.linker.RelationshipLinker +members: [link, seed_existing, resolve] +lede: "Turns a flat list of `CloudAsset` objects into a connected map. It reads only asset metadata, so it works on assets from any source: cloudg's collectors, your own plugin, a CMDB export, a previous run." +--- + +## How linking works + +`RelationshipLinker(assets, materialize_external=True)` indexes every asset by the identifiers other resources use to point at it: ARN or resource ID, name, the short ID at the end of the ARN (`i-0b1c...`, `sg-0a1b...`), metadata keys such as `queue_url`, `repository_uri` or `zone_id`, every entry in `metadata["aliases"]`, and a few derived forms (an S3 bucket's `.s3.amazonaws.com`, a load balancer's DNS name). Building the index makes no API calls and keeps no reference to a cloud session. + +`link()` then makes three passes and returns the new edges: + +```mermaid caption="The three passes of RelationshipLinker.link()" +flowchart TD + I["identifier index over all assets"] --> P1["pass 1: declared relations in metadata.relations"] + P1 --> P2["pass 2: provider rules on metadata keys"] + P2 --> S["Identity Center permission sets to SSO roles"] + S --> G{"include_generic?"} + G -->|yes| P3["pass 3: any identifier string in metadata becomes REFERENCES"] + G -->|no| E["edges"] + P3 --> E + P1 -.->|"target in an unmapped account"| X["external CLOUD_ACCOUNT placeholder"] + P1 -.->|"target not found"| U["linker.unresolved"] +``` + +Pass 1 reads `metadata["relations"]`, the list each collector fills with what an asset talks to. Pass 2 applies provider-aware rules to well-known metadata keys: `security_groups` and `nsg_id` (attachment), `subnet_id` and `vpc_config` (containment), `role_arn` (execution roles), `kms_key_id` (encryption), `origins` and `web_acl_id` (CloudFront and WAF), route table routes, ENI and volume attachments, VPC peering, Azure NIC wiring and the GCP `parent_full_resource_name`. Pass 3 walks every other string in the metadata, up to six levels deep, and links anything that looks like an ARN, an Azure resource ID, a GCP full resource name or an AWS short ID and resolves to another asset. Pairs already joined in either direction are skipped, so the generic pass only adds links nothing else found. + +An edge is added once per `(source, target, edge_type)`. Every pass checks that key, and so does anything you register with `seed_existing()`. + +### The relation object + +Each entry in `metadata["relations"]` is a dict: + +| Key | Required | Meaning | +|---|---|---| +| `target` | yes | Any identifier of the other asset: ARN, resource ID, name, short ID, alias. | +| `edge` | no | An [`EdgeType`](/api/models/) value such as `INVOKES` or `ASSUMES_ROLE`. Missing or unknown values become `REFERENCES`. | +| `relationship` | no | A finer label, copied to the edge's `relationship` (`TRIGGERED_BY`, `ENCRYPTED_BY_KMS`, `RUNS_ON`). The [relationship vocabulary](/reference/catalog/relationship-vocabulary/) lists the names cloudg's collectors use. | +| `reverse` | no | When true, the edge runs from the target to the declaring asset. Use it when the declaring side is the passive one: "my queue triggers me", "this account is trusted by me". | +| `description` | no | Copied to the edge's `description`. | +| `properties` | no | Copied to the edge's `properties`. | + +Edges always read "source verb target": a queue `INVOKES` a function, a function `ASSUMES_ROLE` a role, a WAF `PROTECTS` an API. + +### Resolving an identifier + +`resolve(identifier, context=None)` returns the internal asset `id` an identifier points at, or `None`. When several assets share an identifier (two accounts each with a security group called `default`), the referencing asset passed as `context` decides: a candidate in the same account and region wins, then one in the same account, then a single global one. If it is still ambiguous, the answer is `None`; the linker would rather leave a reference unresolved than guess across accounts. Real assets win over placeholder nodes. When the exact string misses, normalised variants are tried (trailing dots and slashes stripped, among others), and a role ARN built from a bare name falls back to the role with that name in that account. + +### External accounts and unresolved references + +When a declared target does not resolve but its identifier names another AWS account, Azure subscription or GCP project, the linker creates a `CLOUD_ACCOUNT` placeholder with `metadata.external` set to `true` and points the edge at it. The original target is kept in the edge's `properties.external_reference`. The placeholders accumulate in `linker.external_assets`; add them to your asset list, or the new edges point at ids your list does not contain. Pass `materialize_external=False` to turn this off, and such references are reported as unresolved instead. + +Declared relations that resolve to nothing at all go to `linker.unresolved`, one dict per reference with `source`, `source_name`, `target` and `edge_type`, capped at 2000 entries. A deleted dead-letter queue that is still configured, or a log group that was never created, typically shows up here. + +## Examples + +### Link a CMDB export + +Five records from a hypothetical CMDB, with one edge the CMDB already had. Runs offline. + +```python title="link_cmdb.py" +from cloudg.graph.builder import GraphBuilder +from cloudg.inventory import RelationshipLinker +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType, NetworkEdge + +ACCOUNT = "123456789012" +REGION = "eu-west-1" +KEY = f"arn:aws:kms:{REGION}:{ACCOUNT}:key/1111aaaa-22bb-33cc-44dd-555555eeeeee" +ROLE = f"arn:aws:iam::{ACCOUNT}:role/orders-worker-role" +QUEUE = f"arn:aws:sqs:{REGION}:{ACCOUNT}:orders-inbound" +TABLE = f"arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/orders" +FUNCTION = f"arn:aws:lambda:{REGION}:{ACCOUNT}:function:orders-worker" + + +def cmdb_row(arn, name, asset_type, region=REGION, **metadata): + """One record from a CMDB export, turned into a CloudAsset.""" + return CloudAsset(arn=arn, name=name, asset_type=asset_type, provider=CloudProvider.AWS, + region=region, account_id=ACCOUNT, metadata=metadata) + + +assets = [ + cmdb_row(KEY, "orders-key", AssetType.KMS_KEY), + cmdb_row(ROLE, "orders-worker-role", AssetType.IAM_ROLE, region="global", relations=[ + # the role trusts an outside account; reverse: the edge runs account -> role + {"target": "arn:aws:iam::999999999999:root", "edge": "IAM_TRUST", + "relationship": "CROSS_ACCOUNT_TRUST", "reverse": True}, + ]), + cmdb_row(QUEUE, "orders-inbound", AssetType.MESSAGE_QUEUE, kms_key_id=KEY), + cmdb_row(TABLE, "orders", AssetType.DYNAMODB_TABLE, kms_key_id=KEY), + cmdb_row(FUNCTION, "orders-worker", AssetType.LAMBDA_FUNCTION, role_arn=ROLE, relations=[ + {"target": QUEUE, "edge": "INVOKES", "relationship": "TRIGGERED_BY", "reverse": True}, + {"target": "orders", "edge": "REFERENCES", "relationship": "WRITES_TO"}, + {"target": f"arn:aws:logs:{REGION}:{ACCOUNT}:log-group:/aws/lambda/orders-worker", + "edge": "LOGS_TO"}, + ]), +] + +# An edge the CMDB already had; seed it so the linker does not add a second one +by_arn = {a.arn: a.id for a in assets} +existing = [NetworkEdge(source_id=by_arn[TABLE], target_id=by_arn[KEY], edge_type=EdgeType.REFERENCES)] + +linker = RelationshipLinker(assets) +linker.seed_existing(existing) +edges = existing + linker.link() +assets += linker.external_assets + +names = {a.id: a.name for a in assets} +for e in edges: + print(f"{names[e.source_id]:28} {e.edge_type.value:13} {e.relationship or '-':20} {names[e.target_id]}") +print("unresolved:", [(u["source_name"], u["target"]) for u in linker.unresolved]) + +graph = GraphBuilder().build(assets, edges) +print(graph.number_of_nodes(), "nodes,", graph.number_of_edges(), "edges") +``` + +```console +$ python link_cmdb.py +orders REFERENCES - orders-key +external account 999999999999 IAM_TRUST CROSS_ACCOUNT_TRUST orders-worker-role +orders-inbound INVOKES TRIGGERED_BY orders-worker +orders-worker REFERENCES WRITES_TO orders +orders-inbound REFERENCES ENCRYPTED_BY_KMS orders-key +orders-worker ASSUMES_ROLE RUNS_ON orders-worker-role +unresolved: [('orders-worker', 'arn:aws:logs:eu-west-1:123456789012:log-group:/aws/lambda/orders-worker')] +6 nodes, 6 edges +``` + +Reading the output line by line: + +- The first edge is the seeded one. The table's `kms_key_id` would have produced the same `(table, key, REFERENCES)` edge, so the linker skipped it; the queue's `kms_key_id` had no seeded twin and became an `ENCRYPTED_BY_KMS` reference. +- The trust relation names account `999999999999`, which is not in the list, so a placeholder account node was created and the edge, reversed, runs from it to the role. +- `"orders"` resolved by name, because exactly one asset has that name. +- `role_arn` on the function produced `ASSUMES_ROLE` / `RUNS_ON` through a pass 2 rule, with no declared relation. +- The log group is not in the export, so that relation is in `unresolved`. + +To feed the result into the rest of cloudg, wrap it in an [`InventoryResult`](/api/inventoryresult/) and call `export()`, or hand `assets` and `edges` to [`CloudGEngine.analyze()`](/api/cloudgengine/). + +### Link only what was declared + +For a strict view without the heuristic passes, switch off the generic scan and the placeholders: + +```python title="declared_only.py" +from cloudg.inventory import RelationshipLinker +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider + +function = CloudAsset( + arn="arn:aws:lambda:eu-west-1:123456789012:function:thumbnailer", + name="thumbnailer", + asset_type=AssetType.LAMBDA_FUNCTION, + provider=CloudProvider.AWS, + region="eu-west-1", + account_id="123456789012", + metadata={ + "relations": [{"target": "uploads", "edge": "INVOKES", "reverse": True}], + "notes": {"previous_bucket": "arn:aws:s3:::uploads-legacy"}, + }, +) +bucket = CloudAsset( + arn="arn:aws:s3:::uploads", + name="uploads", + asset_type=AssetType.S3_BUCKET, + provider=CloudProvider.AWS, + account_id="123456789012", +) +legacy = CloudAsset( + arn="arn:aws:s3:::uploads-legacy", + name="uploads-legacy", + asset_type=AssetType.S3_BUCKET, + provider=CloudProvider.AWS, + account_id="123456789012", +) + +assets = [function, bucket, legacy] +strict = RelationshipLinker(assets, materialize_external=False).link(include_generic=False) +full = RelationshipLinker(assets).link() +print(len(strict), "declared edge(s);", len(full), "with the generic scan") +``` + +```console +$ python declared_only.py +1 declared edge(s); 2 with the generic scan +``` + +The generic scan found the ARN of `uploads-legacy` inside `metadata["notes"]` and added a `REFERENCES` edge for it. That is usually what you want for a map, and usually not what you want when you need every edge to be explainable by a declared relation. + +## Notes + +A linker instance is built for one asset list and one `link()` call. A second call remembers the edges of the first and returns an empty list, while appending the same unresolved references to `linker.unresolved` again. Build a new `RelationshipLinker` when you need to link again. + +The linker never modifies the assets you pass in. It only reads `metadata`, `arn`, `name`, `account_id`, `region` and `asset_type`, plus `raw_data` for a few AWS rules (instance profiles, the raw Lambda `Role` field). Assets loaded from an exported map have empty `raw_data`, so those rules do not fire on them. + +`resolve()` without a `context` returns the first candidate for an ambiguous identifier instead of `None`. Pass the referencing asset when you call it yourself. + +Errors inside a pass are caught per asset and logged at DEBUG on `cloudg.inventory.linker`, so one malformed asset cannot stop the run. If edges you expect are missing, turn on DEBUG for that logger. + +## Related + +:::links +- [InventoryMapper](/api/inventorymapper/) Runs the linker as part of every map. +- [DependencyGraph](/api/dependencygraph/) Dependency questions over the linked edges. +- [Relationship vocabulary](/reference/catalog/relationship-vocabulary/) The relationship names and what they mean. +- [Asset types by category](/reference/catalog/asset-types-by-category/) Which types exist and what they carry. +- [Inventory mapping](/guides/inventory-mapping/) The linker in the context of a full map. +::: diff --git a/site/content/api/resilience.md b/site/content/api/resilience.md new file mode 100644 index 0000000..f04deec --- /dev/null +++ b/site/content/api/resilience.md @@ -0,0 +1,192 @@ +--- +title: cloudg.resilience +object: + - cloudg.resilience.call_with_resilience + - cloudg.resilience.call_with_resilience_sync + - cloudg.resilience.resilient + - cloudg.resilience.Scope + - cloudg.resilience.RetryPolicy + - cloudg.resilience.stats_scope + - cloudg.resilience.configure + - cloudg.resilience.get_governor + - cloudg.resilience.Governor + - cloudg.resilience.classify +members: + cloudg.resilience.Scope: [service_scope, account_scope] + cloudg.resilience.Governor: [summary, budget, try_retry, breaker_scope, provider_enabled, reset] +lede: Put your own cloud API calls through the same rate limits, circuit breakers and retry budget that cloudg's collectors use. +source: docs/RESILIENCE.md +since: "0.6.0" +--- + +Every cloud call cloudg makes goes through one process-wide governor: adaptive token buckets per provider, account, region and service, a concurrency cap per account, a circuit breaker per bucket and a retry budget per provider. If your own code calls the same APIs in the same process (extra enumeration next to an `InventoryMapper` run, an MCP tool of your own, a CI job that maps and then queries), wrap those calls with `call_with_resilience` and they queue behind cloudg's instead of competing with them. A throttle your call hits slows cloudg's calls to that scope too, and the other way round. + +## What happens to one call + +```mermaid caption="call_with_resilience, step by step" +flowchart TD + A["call_with_resilience(fn, scope)"] --> B{"Breaker open?"} + B -->|yes| X["CircuitOpenError, nothing sent"] + B -->|no| C{"Deadline left?"} + C -->|no| Y["DeadlineExceededError"] + C -->|yes| D["Wait for a token, take a bulkhead slot"] + D --> E["fn(*args, **kwargs)"] + E -->|success| S["Return the result"] + E -->|error| K{"classify(exc)"} + K -->|FATAL| R["Re-raise at once"] + K -->|"THROTTLED or TRANSIENT"| F{"Retries, deadline, budget left?"} + F -->|yes| G["Back off, never under Retry-After"] --> B + F -->|no| Z["Re-raise, breaker counts a failure"] +``` + +A throttling error also halves the scope's rate (at most once per second) and pauses the scope for the server's Retry-After, capped at 300 seconds. After five seconds without a throttle each success gives back 5% of the configured rate. The backoff before a retry is decorrelated jitter, `min(cap, uniform(0.5, 3 x previous))`. Retries stop at the first of `max_retries` (8 by default), the per-call deadline (900 seconds) or an empty retry budget (500 per provider; the first two retries of each call are free, and each success refunds 0.1). When retrying stops, your original exception is re-raised, not a wrapper. + +`classify()` decides what is retried. It reads error codes, HTTP statuses, exception class names and `Retry-After` style headers from botocore, azure-core, google-api-core, urllib, aiohttp and httpx errors by duck typing, without importing any SDK. An exception of your own with none of those attributes is FATAL and raised at once, unless its message mentions throttling ("rate exceeded", "throttl", "too many requests"). To make your own errors retryable, give them a `status_code` of 429 or 503 (throttling) or 500, 502, 504 (transient), plus `headers` with a `Retry-After` if you have one. + +## Choosing a scope + +A `Scope` says where the call goes, and its string form is what the telemetry shows: `aws/123456789012/eu-west-1/ec2/DescribeVpcs`, with `*` for a missing part. + +| Part | AWS | Azure | GCP | +|---|---|---|---| +| `provider` | `"aws"` | `"azure"` | `"gcp"` | +| `account` | account id | subscription id, lower-case | `projects/`, `organizations/` or `folders/` | +| `region` | client region | `None` | `None` | +| `service` | botocore service name (`ec2`, `iam`, `sts`) | provider namespace without `Microsoft.` (`network`, `compute`) | `cloudasset` | +| `operation` | API name (`DescribeVpcs`) | `None` | RPC name (`ListAssets`) | + +Use the same values cloudg uses, or your calls land in different buckets and protect nothing. A call takes a token from a leaf bucket per account, region and service; the leaf goes down to the operation only for services marked `per_operation` (AWS `ec2` is, out of the box) or with an override keyed `"."`. Breakers have the same granularity as the leaf. + +Other provider names work too. `Scope("github", "my-org", None, "rest")` is paced at the generic defaults (20 rps per service, 64 calls in flight, 8 retries), but `config.yaml` can only tune `aws`, `azure` and `gcp`. + +## Examples + +### Pace, retry and report your own calls + +The script needs botocore (a cloudg dependency) and no credentials: `describe_vpcs` raises a real botocore `ClientError` twice before answering, and `describe_secret` stands in for an async SDK call. `configure()` caps one service at 2 requests per second, and six concurrent calls show the limiter queueing them. + +```python title="paced_calls.py" +"""Send your own cloud calls through cloudg's shared limiter, breakers and retries.""" + +import asyncio + +from botocore.exceptions import ClientError + +from cloudg.config import RateLimitConfig +from cloudg.resilience import ( + RetryPolicy, + Scope, + call_with_resilience, + configure, + get_governor, + resilient, + stats_scope, +) + +ACCOUNT = "123456789012" + +# 2 rps for this one service; every other AWS service keeps cloudg's defaults +configure(RateLimitConfig.model_validate( + {"aws": {"services": {"secretsmanager": {"max_rps": 2, "burst": 2}}}} +)) + +attempts = {"describe": 0} + + +def describe_vpcs() -> dict: + """Stands in for ec2.describe_vpcs(): throttled twice, then answers.""" + attempts["describe"] += 1 + if attempts["describe"] <= 2: + raise ClientError({"Error": {"Code": "RequestLimitExceeded", "Message": "Rate exceeded"}, + "ResponseMetadata": {"HTTPStatusCode": 503}}, "DescribeVpcs") + return {"Vpcs": [{"VpcId": "vpc-0a1b2c3d"}]} + + +@resilient(lambda name, **_: Scope("aws", ACCOUNT, "eu-west-1", "secretsmanager", "DescribeSecret"), + policy=RetryPolicy(max_retries=2, deadline=30.0)) +async def describe_secret(name: str) -> str: + """Stands in for an async SDK call; the decorator paces every call.""" + return f"arn:aws:secretsmanager:eu-west-1:{ACCOUNT}:secret:{name}" + + +async def main() -> None: + ec2 = Scope("aws", ACCOUNT, "eu-west-1", "ec2", "DescribeVpcs") + loop = asyncio.get_running_loop() + with stats_scope() as stats: + vpcs = await call_with_resilience(describe_vpcs, scope=ec2) + start = loop.time() + arns = await asyncio.gather(*(describe_secret(f"app-{i}") for i in range(6))) + elapsed = loop.time() - start + print(vpcs["Vpcs"], "after", attempts["describe"], "attempts") + print(len(arns), f"secrets in {elapsed:.1f}s at 2 rps") + for line in stats.messages(): + print(" ", line) + summary = get_governor().summary(stats) + print("totals:", summary["totals"]) + print("slowed:", {k: v["rate_rps"] for k, v in summary["slowed_buckets"].items()}) + + +asyncio.run(main()) +``` + +```console +$ python paced_calls.py +[{'VpcId': 'vpc-0a1b2c3d'}] after 3 attempts +6 secrets in 2.0s at 2 rps + aws/123456789012/eu-west-1/ec2: throttled 2x, slowed to 5.0 rps, 2 retries + aws/123456789012/eu-west-1/secretsmanager: waited 5.0s for rate limit +totals: {'calls': 9, 'throttled': 2, 'transient_errors': 0, 'retries': 2, 'gave_up': 0, 'rejected': 0, 'breaker_trips': 0, 'wait_seconds': 4.999} +slowed: {'aws/123456789012/eu-west-1/ec2/DescribeVpcs': 5.0} +``` + +Reading it: + +- `describe_vpcs` is a plain function, so `call_with_resilience` ran it in a worker thread. Two `RequestLimitExceeded` errors classified as THROTTLED, so the call was retried, and the `DescribeVpcs` bucket dropped from its 20 rps ceiling to 10, then to 5. Throttles less than a second apart count as one congestion event, so depending on the backoff your run may stop at 10. Any cloudg collector calling `DescribeVpcs` for that account and region in this process now runs at that rate too, recovering by 1 rps every five seconds of quiet. +- Two of the six secret lookups used the burst; the other four queued at 0.5-second intervals. `wait_seconds` adds up the waits of all six callers, which is why it exceeds the wall-clock 2.0 seconds. +- `stats_scope()` collected this block only. `get_governor().summary()` without an argument reports everything since the process started. +- `summary["slowed_buckets"]` lists buckets below their ceiling or paused. Rates live in the governor, so they outlast the `with` block. + +### A real boto3 client + +The same call against AWS, which needs credentials. A sync boto3 method is passed as is, and its keyword arguments follow `scope=`: + +```python title="describe_vpcs.py" +import asyncio + +import boto3 + +from cloudg.resilience import Scope, call_with_resilience, stats_scope + + +async def main() -> None: + ec2 = boto3.client("ec2", region_name="eu-west-1") + scope = Scope("aws", "123456789012", "eu-west-1", "ec2", "DescribeVpcs") + with stats_scope() as stats: + vpcs = await call_with_resilience(ec2.describe_vpcs, scope=scope, MaxResults=50) + print(len(vpcs["Vpcs"]), stats.messages()) + + +asyncio.run(main()) +``` + +boto3 also retries inside the call, so a throttled request can be retried both by botocore and by `call_with_resilience`. To hand a whole session to the governor instead, so every client created from it is paced with botocore's retries counted against the same budget, call `cloudg.resilience.aws.install_aws_hooks(session, account_id, region)` once and use the clients normally. Azure clients have the equivalent `cloudg.resilience.azure.build_azure_client(ClientClass, credential, subscription_id, ...)`. + +## Notes + +- `call_with_resilience` awaits coroutine functions and runs everything else with `asyncio.to_thread`. In synchronous code that already runs in a worker thread, use `call_with_resilience_sync`, which blocks with `time.sleep`. +- With `ratelimit.enabled: false`, or `ratelimit..enabled: false`, `fn` is called once, directly, with no pacing and no retries. +- `configure()` accepts a `CloudGConfig`, a `RateLimitConfig` or a dict, and ignores anything else. It is idempotent: a provider whose settings did not change keeps its learned rates, breakers, budget and held slots. The settings are process-wide, so the last configuration applied wins for every caller in the process. `InventoryMapper` and `MultiAccountCollector` call it themselves with their config. +- An open breaker raises `CircuitOpenError` without sending the call; it carries `retry_after`. `RetryBudgetExhaustedError` and `DeadlineExceededError` do too. All three subclass `ResilienceError` and have a botocore-shaped `.response`, so error-code helpers that read `response["Error"]["Code"]` keep working. +- A FATAL answer from the provider (`AccessDenied`, a 404) proves the service is up, so it counts like a success for the breaker. An `AccessDenied` storm never opens a circuit. +- `reset_governor()` replaces the governor with a fresh one and forgets every learned rate. It exists for tests. `get_governor().reset()` clears the same learned state on the current governor and keeps its configuration. +- `cloudg.retry.with_retry` is the older decorator. It keeps its signature and also retries provider throttling recognised from codes and statuses, but it does not use the shared buckets or breakers. + +## Related + +:::links +- [Rate limits and throttling](/guides/rate-limits/) What cloudg does under throttling, and how to tune it. +- [ratelimit settings](/reference/config/ratelimit/) Every `ratelimit` key in `config.yaml`. +- [How one call is paced](/reference/resilience/how-one-call-is-paced/) Buckets, AIMD, breakers and the retry budget in depth. +- [Python API](/reference/resilience/python-api/) `LiveOperationGuard`, the bulkhead and other lower-level pieces. +- [CloudGMCPLayer](/api/mcp-layer/) Live MCP tools run under the same governor. +::: diff --git a/site/content/api/results.md b/site/content/api/results.md new file mode 100644 index 0000000..6f09a81 --- /dev/null +++ b/site/content/api/results.md @@ -0,0 +1,203 @@ +--- +object: [cloudg.api.PipelineResult, cloudg.api.CollectionResult, cloudg.api.AnalysisResult] +members: + cloudg.api.PipelineResult: [total_assets, total_findings, severity_breakdown, to_summary] + cloudg.api.CollectionResult: [] + cloudg.api.AnalysisResult: [] +lede: "Three plain dataclasses carry what the engine produces. `PipelineResult` is the one most callers keep; the other two come back from single phases." +--- + +## Which method returns which + +| Returned by | Type | Holds | +|---|---|---| +| `run_pipeline()`, `run_from_reports()` and their `_sync` twins | `PipelineResult` | assets, edges, normalised findings, the `ScanResult`, analysis counts, report paths, run metadata | +| `collect()` | `CollectionResult` | what the collectors found, before any scanning | +| `analyze()` | `AnalysisResult` | graph counts, ontology and RAG output paths, Terraform files, attack paths, reachability findings | + +All three are `@dataclass` classes with defaults for every field, so you can also build them yourself, for instance in a test that feeds a fake `CollectionResult` to code that expects one. They are not pydantic models: there is no `model_dump()`, and `dataclasses.asdict()` will not turn the `CloudAsset` and `Finding` objects inside them into JSON. Use `to_summary()` for the headline numbers and dump the models yourself for the rest (see the examples). + +## How run_pipeline fills a PipelineResult + +`run_pipeline()` gets a `CollectionResult` from `collect()` and an `AnalysisResult` from `analyze()`, normalises the findings, writes the reports, and copies a subset of fields into one `PipelineResult`. + +```mermaid caption="Where each PipelineResult field comes from" +flowchart LR + C["CollectionResult"] -->|"assets, edges, coverage, providers_scanned, regions_scanned"| P["PipelineResult"] + A["AnalysisResult"] -->|"graph_nodes, graph_edges, ontology_triples, rag_chunks_path, terraform_paths, attack_paths"| P + N["FindingsNormaliser"] -->|"findings, scan_result"| P + R["report writers"] -->|"report_paths"| P + E["normalisation and reporting failures"] -->|errors| P +``` + +Some fields do not make the trip. `AnalysisResult.ontology_path`, `rag_chunk_count` and `reachability_findings` have no counterpart on `PipelineResult` (the reachability findings are merged into `findings` instead), and `duration_ms` on the pipeline result is the whole run, not the collection time. If you need those, drive the phases yourself with `collect()` and `analyze()`. + +`run_from_reports()` fills only `findings`, `scan_result`, `report_paths`, `duration_ms` and `errors`. Everything that needs live assets stays at its default: empty lists, empty dicts, zero counts. + +## Field notes + +`findings` holds the normalised list when normalisation succeeded: deduplicated, rescored, sorted from CRITICAL to INFO. If normalisation failed, it holds the raw findings instead, `scan_result` is `None`, and `errors` says why. + +`scan_result` is the [`ScanResult`](/api/models/) the normaliser produced. It is where the compliance mapping lives (`scan_result.compliance`, one `ComplianceResult` per framework control), and its `summary` adds `total_edges` and `compliance_frameworks` to the counts. In `run_pipeline()` its `edges` are set to the collected edges. + +`severity_breakdown` counts `finding.severity.value` over `findings`, recomputed on every access. Only severities that occur appear as keys, so read it with `.get("CRITICAL", 0)`. `total_assets` and `total_findings` are `len()` of the lists. + +`to_summary()` returns `total_assets`, `total_findings`, `severity_breakdown`, `providers_scanned`, `regions_scanned`, `graph_nodes`, `graph_edges`, `ontology_triples`, `attack_paths_count`, `duration_ms` and `errors`. Every value is a plain JSON type, so `json.dumps(result.to_summary())` works as is. `report_paths`, `rag_chunks_path` and `terraform_paths` hold `pathlib.Path` objects and are not part of the summary. + +`regions_scanned` maps a provider to the regions it actually covered after `ALL` was expanded, for example `{"aws": ["eu-west-1", "eu-central-1"]}`. + +`coverage` is a list of `CollectionCoverage` records, one per provider, account and region run, each with a `services` list of `SUCCESS`, `FAILED`, `PARTIAL` or `SKIPPED` entries. This is where a collection failure shows up; `errors` does not record it. `cov.to_summary()` gives the counts and the failed services with their error text. + +`attack_paths` is a `list[list[str]]`. Each inner list is a path of graph node ids, starting at an internet-exposed or external node and ending at the target of an `IAM_TRUST` edge, at most eight hops long. Node ids are asset ids, so map them back with `{a.id: a for a in result.assets}`. The search stops after about a hundred paths. + +`errors` holds strings of the form `": "`, for the `normalisation` and `reporting` phases only. Collection, scanner and analysis failures go to the `on_error` hook and the log; see [Event hooks](/api/event-hooks/). + +`AnalysisResult.ontology_path` is the last file the ontology was saved to. With the default `export_formats: [turtle, json-ld]` that is `ontology.jsonld`; `ontology.ttl` is written next to it. `rag_chunk_count` is never filled in 0.6.0 and stays `0`; count the lines of `rag_chunks_path` if you need the number. + +## Examples + +### Save a run and gate a build on it + +Runs offline against existing Checkov and Trivy output. + +```python title="save_result.py" +import json +import sys +from pathlib import Path + +from cloudg import CloudGConfig, CloudGEngine + +result = CloudGEngine(CloudGConfig()).run_from_reports_sync( + {"checkov": ["./results_json.json"], "trivy": ["./trivy.json"]}, + output_dir="./reports", +) + +# Headline numbers: plain JSON types only, safe to json.dumps +summary = result.to_summary() +print(json.dumps(summary, indent=2)) + +# Everything else: dump the pydantic models and stringify the paths yourself +payload = { + "summary": summary, + "findings": [f.model_dump(mode="json") for f in result.findings], + "compliance": [c.model_dump(mode="json") for c in result.scan_result.compliance] + if result.scan_result + else [], + "report_paths": {kind: str(path) for kind, path in result.report_paths.items()}, +} +Path("./reports/run-summary.json").write_text(json.dumps(payload, indent=2)) + +breakdown = result.severity_breakdown # only severities that occur +blocking = breakdown.get("CRITICAL", 0) + breakdown.get("HIGH", 0) +if blocking: + print(f"{blocking} critical/high findings, failing the build", file=sys.stderr) + sys.exit(1) +``` + +```console +$ python save_result.py +{ + "total_assets": 0, + "total_findings": 2, + "severity_breakdown": { + "CRITICAL": 1, + "HIGH": 1 + }, + "providers_scanned": [], + "regions_scanned": {}, + "graph_nodes": 0, + "graph_edges": 0, + "ontology_triples": 0, + "attack_paths_count": 0, + "duration_ms": 1219, + "errors": [] +} +2 critical/high findings, failing the build +$ echo $? +1 +``` + +### Read attack paths from an AnalysisResult + +Four hand-built assets: a public API that invokes a function, the function's role, and an admin role that trusts it. Ontology and RAG export are switched off to keep the run quick. + +```python title="attack_paths.py" +import asyncio + +from cloudg import CloudGConfig, CloudGEngine +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType, NetworkEdge + +ACCOUNT = "123456789012" + + +def aws(name, asset_type, arn, exposed=False): + return CloudAsset(name=name, asset_type=asset_type, arn=arn, provider=CloudProvider.AWS, + region="eu-west-1", account_id=ACCOUNT, is_internet_exposed=exposed) + + +api = aws("orders-api", AssetType.API_GATEWAY, "arn:aws:apigateway:eu-west-1::/restapis/a1b2c3d4e5", exposed=True) +fn = aws("orders-worker", AssetType.LAMBDA_FUNCTION, f"arn:aws:lambda:eu-west-1:{ACCOUNT}:function:orders-worker") +role = aws("orders-worker", AssetType.IAM_ROLE, f"arn:aws:iam::{ACCOUNT}:role/orders-worker") +admin = aws("ops-admin", AssetType.IAM_ROLE, f"arn:aws:iam::{ACCOUNT}:role/ops-admin") + +assets = [api, fn, role, admin] +edges = [ + NetworkEdge(source_id=api.id, target_id=fn.id, edge_type=EdgeType.INVOKES), + NetworkEdge(source_id=fn.id, target_id=role.id, edge_type=EdgeType.ASSUMES_ROLE), + # ops-admin trusts orders-worker: the worker's role can assume it + NetworkEdge(source_id=role.id, target_id=admin.id, edge_type=EdgeType.IAM_TRUST), +] + + +async def main() -> None: + config = CloudGConfig() + config.ontology.enabled = False + config.rag.enabled = False + analysis = await CloudGEngine(config).analyze(assets, edges, [], output_dir="./reports") + + names = {a.id: f"{a.name} ({a.asset_type.value})" for a in assets} + print(analysis.graph_nodes, "nodes,", analysis.graph_edges, "edges") + for path in analysis.attack_paths: + print(" -> ".join(names.get(node, node) for node in path)) + + +asyncio.run(main()) +``` + +```console +$ python attack_paths.py +4 nodes, 3 edges +orders-api (API_GATEWAY) -> orders-worker (LAMBDA_FUNCTION) -> orders-worker (IAM_ROLE) -> ops-admin (IAM_ROLE) +``` + +### Check collection coverage + +```python title="coverage_check.py" +from cloudg import CloudGConfig, CloudGEngine + +result = CloudGEngine(CloudGConfig(providers=["aws"])).run_pipeline_sync("./reports") + +for cov in result.coverage: + s = cov.to_summary() + print(f"{s['provider']:6} {s['account_id'] or '-':14} {s['region'] or '-':14} " + f"{s['coverage_pct']:5.1f}% ({s['successful']}/{s['total_services']})") + for failure in s["failures"]: + print(" failed:", failure["service"], failure["error"]) +``` + +This one needs AWS credentials. + +## Notes + +The dataclasses are mutable and hold the same objects the engine worked on. Changing `result.findings[0].severity` changes it everywhere that list is referenced, including `result.scan_result.findings`. + +`total_assets` counts what was collected, not what was scanned: a scanner like Checkov reports findings against IaC resources that may not be in `assets` at all. + +## Related + +:::links +- [CloudGEngine](/api/cloudgengine/) The methods that return these objects. +- [Data models](/api/models/) Finding, CloudAsset, ScanResult and the enums. +- [Event hooks](/api/event-hooks/) Where the failures that `errors` leaves out are reported. +- [Output files](/reference/output-files/) What the report paths point at. +::: diff --git a/site/content/cli/auth-flags.md b/site/content/cli/auth-flags.md new file mode 100644 index 0000000..e99d483 --- /dev/null +++ b/site/content/cli/auth-flags.md @@ -0,0 +1,177 @@ +--- +title: Auth flags +lede: "Every `--aws-*`, `--azure-*` and `--gcp-*` flag of `cloudg run`, the config key it fills, the environment variable cloudg falls back to, and which method wins when several are set." +--- + +The credential flags exist on `cloudg run` only. Each one writes a single field of the loaded config and changes nothing else, so you can mix flags with values from `config.yaml`. `cloudg map` takes the same settings from `config.yaml` (it has `--profile` and nothing else). `cloudg collect` uses neither and works from the environment and default chains alone. The [Authentication](/guides/authentication/) guide covers the identities, roles and permissions each method needs; this page is about the flags. + +:::warning Secrets on the command line +Values passed as flags can be seen in your shell history and, while cloudg runs, in the process list of the machine. Prefer the environment variables or a `config.yaml` with tight file permissions for secrets such as `--aws-secret` and `--azure-client-secret`. +::: + +## AWS + +| Flag | Config key | Environment fallback | +|---|---|---| +| `--aws-key` | `aws.access_key_id` | `AWS_ACCESS_KEY_ID`, through the boto3 default chain | +| `--aws-secret` | `aws.secret_access_key` | `AWS_SECRET_ACCESS_KEY`, same | +| `--aws-session-token` | `aws.session_token` | `AWS_SESSION_TOKEN`, same | +| `--aws-role-arn` | `aws.role_arn` | none | +| `--aws-external-id` | `aws.external_id` | none | +| `--aws-web-identity-token-file` | `aws.web_identity_token_file` | `AWS_WEB_IDENTITY_TOKEN_FILE`, read by cloudg | +| `--profile` | `aws.profile` | `AWS_PROFILE`, through the boto3 default chain | + +cloudg builds the base session from the first method that is complete, then optionally assumes a role on top: + +```mermaid caption="How cloudg picks AWS credentials" +flowchart TD + A{"Key and secret both set?"} -->|yes| K["Direct keys, plus session token if set"] + A -->|no| B{"Role ARN and token file set?"} + B -->|yes| W["AssumeRoleWithWebIdentity into the role"] + B -->|no| C{"Profile set?"} + C -->|yes| P["Named profile, SSO included"] + C -->|no| D["boto3 default chain"] + K --> R{"Role ARN set?"} + P --> R + D --> R + R -->|yes| S["sts:AssumeRole, with ExternalId if set"] +``` + +1. Direct keys need both `--aws-key` and `--aws-secret`. One without the other is ignored and the next method is tried. `--aws-session-token` is added when present, for temporary keys. +2. Web identity federation needs a role ARN and a token file. The token file comes from `--aws-web-identity-token-file`, or from `AWS_WEB_IDENTITY_TOKEN_FILE` when the flag is absent. This is the pattern for GitHub Actions, GitLab CI and EKS, with no stored keys. +3. A named profile, from `--profile`. +4. The boto3 default chain: environment keys, the shared credentials file, the SSO cache, container credentials and the instance role. + +Then the role hop. If `--aws-role-arn` is set and step 2 did not already use it, cloudg calls `sts:AssumeRole` into it with the session name `aws.role_session_name` (default `cloudg-scan`) for one hour, passing `--aws-external-id` as `ExternalId` when given. Without a role ARN, `aws.accounts` plus `aws.role_name` in the config build a role ARN per account instead. + +One interaction catches people on EKS. With IRSA, `AWS_WEB_IDENTITY_TOKEN_FILE` is already set in the pod. If you then pass `--aws-role-arn`, cloudg federates straight into that role with the pod's token, instead of assuming it from the pod's own role. The target role's trust policy must accept the cluster's OIDC provider. + +## Azure + +| Flag | Config key | Environment fallback | +|---|---|---| +| `--azure-tenant-id` | `azure.tenant_id` | `AZURE_TENANT_ID` | +| `--azure-client-id` | `azure.client_id` | `AZURE_CLIENT_ID` | +| `--azure-client-secret` | `azure.client_secret` | `AZURE_CLIENT_SECRET` | +| `--azure-cert-path` | `azure.certificate_path` | none read by cloudg | +| `--azure-federated-token-file` | `azure.federated_token_file` | `AZURE_FEDERATED_TOKEN_FILE` | +| `--azure-managed-identity` | `azure.use_managed_identity` | none | +| `--subscription-id` | `azure.subscription_ids` | none on `run`; `AZURE_SUBSCRIPTION_ID` on `collect` | + +cloudg reads each setting from the config first and from the environment variable second, then tries the methods in this order: + +1. Workload identity federation: tenant ID, client ID and a federated token file. +2. Service principal with a client secret: tenant ID, client ID and secret. +3. Service principal with a certificate: tenant ID, client ID and `--azure-cert-path` (PEM or PKCS12). +4. Managed identity, when `--azure-managed-identity` is set. For a user-assigned identity, put its client ID in `azure.managed_identity_client_id`; there is no flag for it. +5. `DefaultAzureCredential`, which covers environment variables, `az login`, Azure PowerShell and managed identity. + +Methods 1 to 3 need the tenant ID and client ID. If either is missing, cloudg skips straight to 4 or 5. The order also means that `AZURE_TENANT_ID`, `AZURE_CLIENT_ID` and `AZURE_CLIENT_SECRET` left in the environment beat `--azure-managed-identity`. Unset them if you want the managed identity. + +## GCP + +| Flag | Config key | Environment fallback | +|---|---|---| +| `--gcp-credentials-file` | `gcp.credentials_file` | `GOOGLE_APPLICATION_CREDENTIALS`, through application default credentials | +| `--gcp-impersonate-sa` | `gcp.impersonate_service_account` | none | +| `--project-id` | `gcp.project_ids` | the credentials' default project | + +`--gcp-credentials-file` takes a service account key JSON or a workload identity federation config (`"type": "external_account"`); both load the same way. Without it, cloudg uses application default credentials: the file in `GOOGLE_APPLICATION_CREDENTIALS`, your `gcloud auth application-default login` session, or the metadata server on GCE, GKE and Cloud Run. + +`--gcp-impersonate-sa` adds a hop on top of either: cloudg impersonates that service account with the `cloud-platform` scope. The base identity needs `roles/iam.serviceAccountTokenCreator` on it. + +## What the scanners get + +The flags above configure cloudg's own collection. The scanner subprocesses that `cloudg run` starts receive much less: + +| Scanner | Credentials passed | +|---|---| +| Prowler (AWS) | `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` from the direct keys, `AWS_DEFAULT_REGION` set to the first entry of `aws.regions`, and `-p` with the `--profile` flag value. Everything else in cloudg's environment is inherited. | +| ScoutSuite (AWS) | `--profile` with the `--profile` flag value | +| Prowler and ScoutSuite (Azure, GCP) | nothing; they use their own defaults and whatever `*_extra_args` you configure | +| Checkov, Trivy | nothing; they scan files and images | + +In particular, `--aws-session-token`, role assumption and web identity federation are not passed on, and `aws.profile` from the config is not either (only the flag is). If temporary keys came in through flags, Prowler gets the key and secret without the token, and its calls fail. When you authenticate any way other than a profile or plain keys, export the credentials in the environment before `cloudg run`, so the scanners inherit them. + +With `--regions all`, the first entry of `aws.regions` is the literal `ALL`, and that is what Prowler receives as `AWS_DEFAULT_REGION`. If Prowler then fails on the region, list the regions explicitly instead of using `all`. + +## Examples + +Keys that your CI secret store already exports as `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN`. The default chain picks them up, so no flag is needed, and Prowler inherits the same environment: + +```bash +cloudg run -p aws --regions us-east-1 +``` + +An auditor assuming a customer's role with an external ID: + +```bash +cloudg run -p aws --profile auditor \ + --aws-role-arn arn:aws:iam::123456789012:role/ThirdPartyAudit \ + --aws-external-id 7f3c2a9e-example +``` + +OIDC federation in CI, with no stored keys: + +```bash +cloudg run -p aws --regions all \ + --aws-role-arn arn:aws:iam::123456789012:role/cloudg-ci \ + --aws-web-identity-token-file "$RUNNER_TEMP/oidc-token" +``` + +An Azure service principal, secret from the environment: + +```bash +export AZURE_CLIENT_SECRET="$(cat ./sp-secret)" +cloudg run -p azure \ + --azure-tenant-id 11111111-1111-1111-1111-111111111111 \ + --azure-client-id 22222222-2222-2222-2222-222222222222 +``` + +Azure workload identity on AKS. The webhook sets `AZURE_TENANT_ID`, `AZURE_CLIENT_ID` and `AZURE_FEDERATED_TOKEN_FILE` in the pod, and cloudg reads all three, so no flag is needed: + +```bash +cloudg run -p azure +``` + +The system-assigned managed identity of an Azure VM: + +```bash +cloudg run -p azure --azure-managed-identity --subscription-id 00000000-0000-0000-0000-000000000000 +``` + +A GCP service account key, impersonating a read-only account: + +```bash +cloudg run -p gcp --project-id my-project \ + --gcp-credentials-file ./sa-key.json \ + --gcp-impersonate-sa cloudg-reader@my-project.iam.gserviceaccount.com +``` + +The same settings kept in `config.yaml`, which also works for `cloudg map`: + +```yaml title="config.yaml" +aws: + role_arn: arn:aws:iam::123456789012:role/cloudg-ci + web_identity_token_file: /var/run/secrets/oidc/token +azure: + tenant_id: 11111111-1111-1111-1111-111111111111 + client_id: 22222222-2222-2222-2222-222222222222 + certificate_path: /etc/cloudg/sp-cert.pem +gcp: + credentials_file: /etc/cloudg/wif-config.json + impersonate_service_account: cloudg-reader@my-project.iam.gserviceaccount.com +``` + +```bash +cloudg -c config.yaml map -p all --regions all +``` + +## Related + +:::links +- [Authentication](/guides/authentication/) Identities, roles and least-privilege permissions per provider. +- [Provider and region flags](/cli/provider-flags/) `--profile`, `--subscription-id` and `--project-id`. +- [cloudg run](/cli/run/) The command these flags belong to. +- [AWS Organizations](/guides/aws-organizations/) Member-account roles for `cloudg map --org`. +::: diff --git a/site/content/cli/collect.md b/site/content/cli/collect.md new file mode 100644 index 0000000..234e817 --- /dev/null +++ b/site/content/cli/collect.md @@ -0,0 +1,93 @@ +--- +command: collect +lede: Collects assets and network edges from one provider and saves them as JSON. No graph analysis, no scanners, no reports. +intro: | + `cloudg collect` is phase 1 of `cloudg run` on its own, for one provider and, on AWS, one region. It is a quick way to test credentials and permissions before a long run. For a full inventory with relationships and dependency analysis, use [cloudg map](/cli/map/). +--- + +## How it works + +`-p/--provider` is required and takes exactly one of `aws`, `azure` or `gcp` (case-insensitive). There is no `all` here. + +The command does not read `config.yaml`. Credentials come from the flags on this page, the environment and each provider's default chain, through `CredentialResolver`. That means the config-only methods (direct keys, role ARNs, service principals in config, credential files, impersonation) are not available. Use [cloudg run](/cli/run/) or `cloudg map` with a config file for those. + +How each provider resolves its identity: + +| Provider | Scope | Credentials | +|---|---|---| +| `aws` | One region: `--region`, default `us-east-1`. The account ID comes from `sts:GetCallerIdentity`. | `--profile`, else `AWS_PROFILE`, else the boto3 default chain (environment keys, SSO cache, instance or task role) | +| `azure` | One subscription: `--subscription-id`, else `AZURE_SUBSCRIPTION_ID`. One of them is required. | A service principal or workload identity when `AZURE_TENANT_ID` and `AZURE_CLIENT_ID` are set together with `AZURE_CLIENT_SECRET` or `AZURE_FEDERATED_TOKEN_FILE`, else `DefaultAzureCredential` (which includes `az login`) | +| `gcp` | One project: `--project-id`, else `GOOGLE_CLOUD_PROJECT`, else the project of the application default credentials | Application default credentials (`GOOGLE_APPLICATION_CREDENTIALS`, `gcloud auth application-default login`, or the metadata server) | + +`--region` only matters for AWS, and because it always has a value, `AWS_DEFAULT_REGION` is never used. `--profile` only matters for AWS. `--subscription-id` and `--project-id` only matter for their provider. There is no `--regions`; to collect several AWS regions, run the command once per region with a different `-o`, or use `cloudg run` or `cloudg map`. + +The AWS collector here is the standard one used by `cloudg run`. It covers the core services (EC2, S3, RDS, VPCs, subnets, security groups, IAM, Lambda, load balancers, ECS, DynamoDB, CloudFront, Secrets Manager, KMS and a few more). It does not run the deep inventory collectors, the Cloud Control sweep or the relationship linker that `cloudg map` uses. + +When collection succeeds, cloudg prints a "Collection Summary" table with the asset and edge counts and the path of the file it wrote. + +## Examples + +Collect the default AWS region with your current credentials: + +```bash +cloudg collect -p aws +``` + +Collect one AWS region with a named profile: + +```bash +cloudg collect -p aws --profile prod --region eu-west-1 -o ./reports/eu-west-1 +``` + +Collect an Azure subscription with your `az login` session: + +```bash +cloudg collect -p azure --subscription-id 00000000-0000-0000-0000-000000000000 +``` + +Collect an Azure subscription as a service principal, configured through the environment: + +```bash +export AZURE_TENANT_ID=11111111-1111-1111-1111-111111111111 +export AZURE_CLIENT_ID=22222222-2222-2222-2222-222222222222 +export AZURE_CLIENT_SECRET="$(cat ./sp-secret)" +cloudg collect -p azure --subscription-id 00000000-0000-0000-0000-000000000000 +``` + +Collect a GCP project with a service account key: + +```bash +export GOOGLE_APPLICATION_CREDENTIALS=./sa-key.json +cloudg collect -p gcp --project-id my-project +``` + +Load the result into the MCP server to explore it from an assistant. The file is loaded as a `generic` dataset named after the file: + +```bash +cloudg mcp serve --dataset ./reports/inventory-aws.json +``` + +## Output files + +| File | Contents | +|---|---| +| `inventory-.json` | `{"assets": [...], "edges": [...]}`, each entry a `CloudAsset` or `NetworkEdge` in Pydantic JSON form | + +The file is written to `-o/--output` (default `./reports`), and a second run for the same provider overwrites it. + +## Exit codes + +| Code | When | +|---|---| +| 0 | The inventory file was written | +| 1 | Collection failed: missing SDK extra, no credentials, no Azure subscription or GCP project. cloudg prints a "Collection failed" panel with the reason. | +| 2 | Click rejected an option, for example `-p all` | + +## Related + +:::links +- [cloudg map](/cli/map/) The full inventory map with relationships and dependencies. +- [cloudg run](/cli/run/) Collection as part of the complete pipeline. +- [Authentication](/guides/authentication/) Every supported identity per provider. +- [Provider and region flags](/cli/provider-flags/) How `--region` and `--regions` differ. +::: diff --git a/site/content/cli/deps.md b/site/content/cli/deps.md new file mode 100644 index 0000000..5af368c --- /dev/null +++ b/site/content/cli/deps.md @@ -0,0 +1,147 @@ +--- +command: deps +lede: Answers dependency questions from a saved inventory map. What does this asset need, what breaks if it goes away, and which assets is everything else leaning on? +intro: | + `cloudg deps` reads the files `cloudg map` wrote. It never calls a cloud API, so it runs offline and needs no credentials. +--- + +## How it works + +The command loads `inventory-map.json` from `-m/--map`. You can pass the file itself or the folder that holds it; the default is `./reports`. If an `inventory-organization.json` sits next to the map, it is loaded too. From the assets and edges it builds a dependency graph in which every edge reads "dependent needs dependency". A Lambda function needs its role, its image, its KMS key and the queue that triggers it. A load balancer needs the WAF in front of it. A resource needs the CloudFormation stack that manages it. + +What happens next depends on whether you name an asset. + +```mermaid caption="The two modes of cloudg deps" +flowchart TD + A["Load inventory-map.json"] --> B{"ASSET given?"} + B -->|yes| C["Resolve the reference"] + C -->|"no unique match"| X["Error, exit 1"] + C -->|found| D{"--direction"} + D -->|up| E["What it depends on"] + D -->|down| F["What depends on it"] + D -->|both| G["Both trees"] + B -->|no| H["Overview: shared deps, blast radius, cross-account edges"] +``` + +### With an asset + +`ASSET` can be an ARN or cloud resource ID, the internal asset ID, a name that is unique in the map, or a unique ARN tail (the part after the last `/`, such as `i-0bast`). If the reference matches nothing, or matches more than one asset, cloudg prints `No unique asset matches ''; use its ARN or resource ID` and exits with 1. Names like `default` are often ambiguous in multi-account maps, so ARNs are the safe choice. + +`-d/--direction` picks the tree. `up` walks what the asset depends on. `down` walks what depends on it, which is its blast radius. `both` (the default) prints the two trees one after the other. `--depth` stops the walk after that many levels (default 3). Every node shows its name, type, account and region, and the edge that led to it in brackets: + +```console +$ cloudg deps bastion --direction up --depth 2 +bastion EC2 arn:aws:ec2:us-east-1:111111111111:instance/i-0bast depends on +├── prod-public-a SUBNET · 111111111111/us-east-1 (CONTAINS) +│ └── prod-vpc VPC · 111111111111/us-east-1 (CONTAINS) +├── sg-admin SECURITY_GROUP · 111111111111/us-east-1 (ATTACHED_TO) +└── bastion-admin IAM_ROLE · 111111111111/global (ASSUMES_ROLE) + └── orders-db-credentials SECRET · 111111111111/us-east-1 (GRANTS_ACCESS) +``` + +```console +$ cloudg deps app-images --direction down --depth 2 +app-images CONTAINER_REGISTRY arn:aws:ecr:eu-west-1:222222222222:repository/app-images needed by +(blast radius) +├── deploy-role IAM_ROLE · 222222222222/global (GRANTS_ACCESS) +│ ├── app-role IAM_ROLE · 111111111111/global (CROSS_ACCOUNT_TRUST) +│ └── vendor-co CLOUD_ACCOUNT · 999999999999/global (CROSS_ACCOUNT_TRUST) +└── api-handler LAMBDA_FUNCTION · 111111111111/us-east-1 (USES_IMAGE) +``` + +Both captures come from the synthetic estate in `tests/mcp/fixtures/sample_estate.py`, with the banner left out. A tree with no entries prints `nothing`. + +### Without an asset + +The overview has three tables, each cut to `--top` rows (default 15): + +| Table | What it ranks | +|---|---| +| Most shared dependencies | Assets with the most direct dependents. Containment and peering edges are not counted, and an asset needs at least two dependents to appear. | +| Largest blast radius | Assets with the most transitive dependents (up to 10 levels), with the number of accounts affected and how many of those dependents are internet-exposed | +| Cross-account edges | Every edge whose two ends sit in different accounts, with `yes` in the External column when one end is an account that was not mapped | + +The blast radius search does not walk every asset. It looks at the `4 x --top` assets with the most direct dependents and ranks those, which keeps the overview fast on big maps. + +### JSON output + +`--json` prints the same data as JSON: the tree object (`asset`, plus `depends_on` and/or `dependents`, each node with `via`, `relationship` and `children`) when you name an asset, or an object with `shared_dependencies`, `largest_blast_radius` and `cross_account_edges` otherwise. In JSON mode the cross-account list is complete; `--top` only limits the other two. + +:::warning The banner is on stdout +The root `cloudg` command prints its banner to stdout before `deps` runs, so `cloudg deps --json | jq` fails to parse. Strip everything before the first `{` line: + +```bash +cloudg deps --map ./reports --json | sed -n '/^{/,$p' | jq '.largest_blast_radius[0]' +``` + +From Python, `InventoryResult.load()` and `dependency_graph()` return the same data with no banner at all (see the Python tab below). +::: + +## Examples + +See the overview for the map in `./reports`: + +```bash +cloudg deps +``` + +Read a map stored somewhere else and show the 30 biggest items in each table: + +```bash +cloudg deps --map ./maps/2026-10-09/inventory-map.json --top 30 +``` + +Check what a role can reach and what uses it, by ARN: + +```bash +cloudg deps arn:aws:iam::123456789012:role/app-role +``` + +Before you delete a queue, see everything that would notice, five levels deep: + +```bash +cloudg deps jobs-queue --direction down --depth 5 +``` + +Feed a function's dependencies into another tool: + +```bash tab="CLI" +cloudg deps api-handler --direction up --json | sed -n '/^{/,$p' > api-handler-deps.json +``` + +```python tab="Python" title="deps_tree.py" +import json + +from cloudg.inventory import InventoryResult + +result = InventoryResult.load("./reports") +graph = result.dependency_graph() + +asset = graph.find("api-handler") +if asset is None: + raise SystemExit("no unique asset matches 'api-handler'") + +tree = graph.tree(asset.id, direction="up", max_depth=3) +print(json.dumps(tree, indent=2, default=str)) + +for row in graph.blast_radius(top=5): + print(row["name"], row["transitive_dependents"]) +``` + +## Exit codes + +| Code | When | +|---|---| +| 0 | The tree or overview was printed | +| 1 | The map could not be loaded ("Could not load the inventory map"), or `ASSET` matched no single asset | +| 2 | Click rejected an option, for example `--direction sideways` | + +## Related + +:::links +- [Dependencies and blast radius](/guides/dependencies/) How to read the trees and what the edge types mean. +- [cloudg map](/cli/map/) Produces the inventory map this command reads. +- [DependencyGraph](/api/dependencygraph/) The Python class behind `deps`. +- [InventoryResult](/api/inventoryresult/) Loading and exporting saved maps. +- [MCP server](/mcp/) Ask the same questions from an AI assistant. +::: diff --git a/site/content/cli/ingest.md b/site/content/cli/ingest.md new file mode 100644 index 0000000..1d8d398 --- /dev/null +++ b/site/content/cli/ingest.md @@ -0,0 +1,150 @@ +--- +command: ingest +lede: Turns scanner output you already have into cloudg findings and reports. No scanner runs and no cloud credentials are needed. +intro: | + Use `cloudg ingest` when Prowler, ScoutSuite, Checkov or Trivy already ran somewhere else: in a CI job, on another host, on a schedule. cloudg does the second half of the pipeline on their files: cross-scanner deduplication, scoring, compliance mapping and report rendering. +--- + +## How it works + +You pass each tool's native output with its own flag. Every flag is repeatable, and any mix of tools works, from one Trivy file to all four scanners at once. A path can be a single file or the directory the tool wrote. + +```mermaid caption="From scanner files to reports" +flowchart LR + A["--prowler / --scoutsuite / --checkov / --trivy"] --> B["Parse each path"] + B -->|"missing or unreadable"| W["Warn and skip"] + B --> C["raw-findings.json"] + B --> D["Normalise: dedupe, score, map compliance"] + D --> E["findings.json"] + D --> F["report.html"] +``` + +1. cloudg groups the paths by tool and prints them in an "Ingest Configuration" panel. With no input flag at all it stops with "No inputs" and exit code 1. +2. Each path goes through that tool's parser. A path that does not exist, or that the parser rejects, is skipped with a warning; the other paths still count. The per-tool totals are printed as they come in. +3. All findings are normalised together. Duplicates within one scanner and across scanners are merged using the check-equivalence rulesets, findings are rescored, and each one is mapped to compliance frameworks. The rulesets come from `rulesets.rules_dir`, so a custom ruleset directory in the config passed with `cloudg -c` takes effect here. +4. The pre-normalisation findings are written to `raw-findings.json`, and the normalised result is rendered in the formats `--format` asks for. + +What each flag accepts: + +| Flag | Accepts | Produce it with | +|---|---|---| +| `--prowler` | An ASFF JSON array or JSONL file, or Prowler's output directory (searched recursively for `*.json`) | `prowler aws -M json-asff -o ./prowler-output` | +| `--scoutsuite` | The `scoutsuite_results_*.js` file, or the report directory | `scout aws --report-dir ./scoutsuite-report --no-browser` | +| `--checkov` | A `checkov --output json` file, or a directory holding `results_json.json` | `checkov -d ./iac --output json > results_json.json` | +| `--trivy` | A `trivy image` or `trivy fs` JSON file, or a directory of them | `trivy image --format json myrepo/app:latest > trivy-image.json` | + +[Ingesting existing output](/guides/ingesting/) lists the exact fields cloudg reads from each format. + +:::warning Prowler needs ASFF +Prowler 4 and later write OCSF JSON by default. The CLI parser only understands ASFF, and it does not reject OCSF: each OCSF record becomes a placeholder finding titled "Unknown Prowler Finding" with no resource and MEDIUM severity. Run Prowler with `-M json-asff`. The MCP server's `load_dataset` and `ingest_reports` tools do detect OCSF and refuse it with a clear error. +::: + +A run on three small test files (the samples in `tests/test_ingest.py`), with one missing path, prints this: + +```console +$ cloudg ingest --prowler prowler.asff.json --checkov results_json.json --trivy trivy-image.json --trivy missing.json -o out +──────────────────────── Ingest Scanner Outputs ──────────────────────── +╭────────── Ingest Configuration ───────────╮ +│ prowler prowler.asff.json │ +│ checkov results_json.json │ +│ trivy trivy-image.json, missing.json │ +╰───────────────────────────────────────────╯ + prowler: 1 findings + checkov: 1 findings + ⚠ trivy: skipping missing.json (Report path does not exist: missing.json) + trivy: 1 findings + ✓ JSON: out/findings.json + ✓ HTML: out/report.html + ✓ Raw findings: out/raw-findings.json + + ✓ Ingested 3 findings from 3 tool(s), 3 after deduplication +``` + +The INFO log lines between those rows are left out. The Prowler sample holds two records, but one of them has `Compliance.Status: PASSED`, and passed checks are dropped, which is why it reports 1. + +## Examples + +Turn a single Prowler run into an HTML report: + +```bash +prowler aws -M json-asff -o ./prowler-output +cloudg ingest --prowler ./prowler-output/ -o ./reports +``` + +Combine image and filesystem scans from Trivy: + +```bash +cloudg ingest --trivy ./trivy-image.json --trivy ./trivy-fs.json +``` + +Merge all four scanners from a CI artifact folder, so that the same issue reported by Prowler and ScoutSuite shows up once: + +```bash +cloudg ingest \ + --prowler ./artifacts/prowler/ \ + --scoutsuite ./artifacts/scoutsuite-report/ \ + --checkov ./artifacts/results_json.json \ + --trivy ./artifacts/trivy/ \ + -o ./reports +``` + +Write only `findings.json` for a downstream job, and use your own rulesets: + +```bash +cloudg -c config.yaml ingest --checkov ./results_json.json --format json -o ./out +``` + +Ingest, then overlay the findings on a fresh inventory map: + +```bash +cloudg ingest --prowler ./prowler-output/ -o ./reports +cloudg map -p aws --regions all --findings ./reports/raw-findings.json -o ./reports +``` + +The same aggregation in Python, with no files written: + +```bash tab="CLI" +cloudg ingest --prowler ./prowler-output/ --trivy ./trivy-image.json --format json +``` + +```python tab="Python" title="ingest_reports.py" +from cloudg.ingest import ingest_reports +from cloudg.normaliser import FindingsNormaliser + +findings = ingest_reports({ + "prowler": ["./prowler-output/"], + "trivy": ["./trivy-image.json"], +}) +result = FindingsNormaliser().normalise(findings) +print(len(findings), "parsed,", len(result.findings), "after deduplication") +``` + +## Output files + +Written to `-o/--output` (default `./reports`). + +| File | Written when | Contents | +|---|---|---| +| `raw-findings.json` | always | Every parsed finding before normalisation, as a JSON list | +| `findings.json` | `--format json` or `all` | The normalised result: metadata, summary, findings, compliance. Its `assets` list and `graph` are empty, since nothing was collected. | +| `report.html` | `--format html` or `all` | The interactive report, without a topology | + +There is no SVG format here, because ingest has no inventory to draw. `raw-findings.json` is the file to pass to `cloudg map --findings` later. + +## Exit codes + +| Code | When | +|---|---| +| 0 | Reports were written. This includes runs where every path was skipped; cloudg then warns "No findings parsed from the given reports" and writes empty reports. | +| 1 | No input flag was given | +| 2 | Click rejected an option, for example `--format svg` | + +## Related + +:::links +- [Ingesting existing output](/guides/ingesting/) Input formats in detail and how deduplication works. +- [cloudg scan](/cli/scan/) Run the scanners from cloudg instead. +- [cloudg report](/cli/report/) Re-render reports from a `findings.json`. +- [parse_report](/api/parse-report/) The parser behind each flag. +- [FindingsNormaliser](/api/findingsnormaliser/) Deduplication, scoring and compliance mapping. +::: diff --git a/site/content/cli/map.md b/site/content/cli/map.md new file mode 100644 index 0000000..07ddbc5 --- /dev/null +++ b/site/content/cli/map.md @@ -0,0 +1,196 @@ +--- +command: map +lede: Builds the scanner-independent inventory map. Everything deployed or default is collected, linked into typed relationships and written to disk for `cloudg deps`, the MCP server or your own code. +intro: | + `cloudg map` needs read access to the cloud APIs and nothing else. No scanner runs and none has to be installed. The [Inventory mapping](/guides/inventory-mapping/) guide explains what the map contains; this page covers the command itself. +--- + +## How it works + +Like `cloudg run`, the command starts from the config loaded by the root command (`cloudg -c config.yaml map ...`), or from built-in defaults when there is no `-c`. It then applies three groups of flags. A flag you leave out never clears a value the config set. + +Scope flags come first. `-p/--provider` replaces `providers` (default `aws`; `all` means all three). `--regions` replaces the region list of every provider, and `all` turns into the `ALL` sentinel that triggers discovery. `--profile`, `--subscription-id` and `--project-id` work as they do on `run`. `--accounts` sets `aws.accounts` and `--role-name` sets `aws.role_name`. + +Organization flags fill `aws.organization`. `--org/--no-org` sets `enabled`, `--org-role` sets `role_name`, `--ou` and `--exclude-account` set `include_ous` and `exclude_accounts`, and `--ct-home-region` sets `home_region`. + +Inventory flags fill `inventory`. `--services` and `--exclude-services` take comma-separated family or collector names. `--kubernetes/--no-kubernetes` and `--cloud-control/--no-cloud-control` only change the config when you pass them, so the config file decides otherwise (both default to on). + +`--sweep/--no-sweep` is different. It defaults to on and is always passed to the mapper, so it overrides `inventory.tagging_sweep` from the config file every time. To turn the tagging sweep off you have to pass `--no-sweep`. + +The command prints the result as a "Map Configuration" panel before it calls any API. For `cloudg map -p aws --org --org-role cloudg-readonly --ou Workloads --regions all --services containers,serverless,security` it shows: + +```console +$ cloudg map -p aws --org --org-role cloudg-readonly --ou Workloads --regions all --services containers,serverless,security +╭─────────────────────────── Map Configuration ───────────────────────────╮ +│ Providers aws │ +│ AWS regions ALL │ +│ Services containers, serverless, security │ +│ Kubernetes workloads enabled │ +│ Catch-all sweep enabled │ +│ Cloud Control sweep enabled │ +│ Scanners none (inventory mapping is scanner-independent) │ +│ Organization discover all accounts under Workloads │ +│ Member role cloudg-readonly │ +╰─────────────────────────────────────────────────────────────────────────╯ +``` + +The mapping run itself goes through these steps: + +```mermaid caption="What cloudg map does" +flowchart TD + A["Config + flags"] --> B{"--org?"} + B -->|yes| C["Discover org, OUs, accounts, Control Tower"] + B -->|no| D["Deep collectors per account and region"] + C --> D + D --> E["Breadth sweeps"] + E --> F["Deduplicate"] + F --> G["Link relationships"] + G --> H["Account hierarchy"] + H --> I["Dependency analysis"] + I --> J["Write inventory files"] + J --> K{"--findings?"} + K -->|yes| L["asset-map.json, compliance-map.json"] +``` + +With `--org`, cloudg first lists the Organization from the account your credentials belong to (the management account, or a delegated administrator for the Organizations APIs). It picks the member accounts under `--ou`, drops `--exclude-account` entries, and, if you also passed `--accounts`, keeps only the accounts in both lists. The role assumed in each member account is `--org-role`, else `--role-name`, else `AWSControlTowerExecution`. If Control Tower is present and you asked for `--regions all`, only its governed regions are mapped (`aws.organization.use_governed_regions`, on by default). + +Without `--org`, `--accounts` plus `--role-name` gives you the same fan-out over an explicit list. In both cases, if your own account is in the list, cloudg collects it with the base credentials instead of assuming the role, since member roles usually do not exist in the management account. + +Deep collection runs per account and region. Account-wide services (IAM, S3, CloudFront, Route 53, Shield and others) run once per account, in the primary region, so `--regions all` does not repeat them. After the deep collectors, the breadth sweeps fill gaps: Cloud Control lists every resource type with a list handler, and the Resource Groups Tagging API finds tagged resources. When the same resource turns up twice, the dedicated collector wins, then Cloud Control, then the tagging API. + +The linker then resolves every identifier the collectors recorded into typed edges (`INVOKES`, `USES_IMAGE`, `ASSUMES_ROLE`, `PROTECTS`, `GOVERNS` and so on), adds an account node above each account's top-level resources, and computes the dependency analysis. The [Inventory mapping](/guides/inventory-mapping/) guide lists the edge types. + +The command prints an organization table (with `--org`), an inventory summary, the 15 services with the most assets, the 10 most shared dependencies, the 10 largest blast radii and up to 15 security service gaps. Here is the summary part for the synthetic estate in the test suite (`tests/mcp/fixtures/sample_estate.py`), so the numbers are small: + +```console + Inventory Summary +╭──────────────────────────────┬───────╮ +│ Metric │ Value │ +├──────────────────────────────┼───────┤ +│ Assets │ 44 │ +│ Interconnections │ 56 │ +│ Accounts │ 5 │ +│ Services │ 23 │ +│ Internet-exposed │ 6 │ +│ Cross-account edges │ 3 │ +│ External accounts referenced │ 1 │ +│ Security service gaps │ 1 │ +│ Unlinked assets │ 2 │ +│ Unresolved references │ 1 │ +╰──────────────────────────────┴───────╯ +``` + +If some collectors failed (an API you lack permission for, a region that is not enabled), the map is still written. cloudg prints one warning with the count and the first eight failures, as `account region service`. Run with `cloudg -v map ...` to see why each one failed, or look at the coverage records. + +### Merging findings + +`--findings` takes a cloudg findings file: a `raw-findings.json` from `cloudg scan` or `cloudg ingest`, or a `findings.json` report. Both shapes work, a bare list or an object with a `findings` key. The option is repeatable. Files that cannot be read or validated are skipped with a warning. Findings attach to assets by ARN, resource ID or name. + +The merge happens after a fresh mapping run, because `--findings` belongs to `map`. To overlay findings on a map you already saved without calling the cloud again, use `InventoryMapper.export_merged()` from Python (see [InventoryMapper](/api/inventorymapper/)) or load both files into the MCP server. + +### Credentials + +`map` has no `--aws-key`, `--azure-*` or `--gcp-*` flags. It takes credentials from the config file, the standard environment variables, `--profile`, or the provider's default chain, in the order described under [Auth flags](/cli/auth-flags/). Put role ARNs, service principals and credential files in `config.yaml` when you map with something other than your default identity. + +## Examples + +Map one AWS account in every enabled region, using your default credentials: + +```bash +cloudg map -p aws --regions all +``` + +Map the regions you care about with a named profile, into a dated folder: + +```bash +cloudg map -p aws --profile audit --regions us-east-1,eu-west-1 -o ./maps/2026-10-09 +``` + +Map every account in the Organization, from the management account, with a dedicated read-only role: + +```bash +cloudg map -p aws --org --org-role cloudg-readonly --regions all +``` + +Limit an Organization run to one OU and skip a sandbox account: + +```bash +cloudg map -p aws --org --ou Workloads --exclude-account 111122223333 --regions all +``` + +Map a fixed set of accounts without Organizations access: + +```bash +cloudg map -p aws --accounts 111111111111,222222222222 --role-name cloudg-readonly --regions all +``` + +Narrow a run to containers and serverless, without reaching into the EKS clusters: + +```bash +cloudg map -p aws --services containers,serverless --no-kubernetes --regions eu-west-1 +``` + +Map Azure and GCP alongside AWS. Azure covers every enabled subscription unless you pass one: + +```bash +cloudg map -p all --regions all --subscription-id 00000000-0000-0000-0000-000000000000 --project-id my-project +``` + +Map and overlay the findings of an earlier scan in one go: + +```bash +cloudg map -p aws --regions all --findings ./reports/raw-findings.json -o ./reports +``` + +The same mapping from Python: + +```bash tab="CLI" +cloudg map -p aws --profile audit --regions all -o ./reports +``` + +```python tab="Python" title="map_inventory.py" +from cloudg import CloudGConfig, CloudGEngine + +config = CloudGConfig(providers=["aws"]) +config.aws.profile = "audit" +config.aws.regions = ["ALL"] + +engine = CloudGEngine(config) +result = engine.map_inventory_sync("./reports") +print(result.summary["total_assets"], "assets") +``` + +## Output files + +Written to `-o/--output` (default `./reports`). The [output files reference](/reference/output-files/) describes each one in more detail. + +| File | Written when | Contents | +|---|---|---| +| `inventory-map.json` | always | Summary, providers, resolved regions, assets, edges, unresolved references, and a `throttling` block when the run was throttled | +| `inventory-map.graphml` | always | The interconnection graph | +| `inventory-graph.json` | always | D3 graph with account and relationship attributes | +| `inventory-dependencies.json` | always | Most shared dependencies, largest blast radius, cross-account edges, security service coverage | +| `inventory-organization.json` | the Organization was mapped | OU tree, accounts, SCPs, Control Tower landing zone and controls | +| `asset-map.json` | `--findings` loaded at least one finding | Per-asset finding counts and severity breakdown, riskiest first | +| `compliance-map.json` | same | Framework to findings and affected assets | + +`cloudg deps` and the MCP server's `load_dataset` read `inventory-map.json` and pick up `inventory-organization.json` from the same folder. + +## Exit codes + +| Code | When | +|---|---| +| 0 | The map was written, even if some collectors failed | +| 1 | Mapping failed as a whole, for example no usable credentials. cloudg prints an "Inventory mapping failed" panel. | +| 2 | Click rejected an option, such as `-p aws2` | + +## Related + +:::links +- [Inventory mapping](/guides/inventory-mapping/) What the map collects and how relationships are built. +- [AWS Organizations](/guides/aws-organizations/) Roles and permissions for `--org`. +- [cloudg deps](/cli/deps/) Query the saved map: what depends on what. +- [Dependencies and blast radius](/guides/dependencies/) How to read dependency trees. +- [Provider and region flags](/cli/provider-flags/) `--regions all` and discovery. +- [InventoryMapper](/api/inventorymapper/) The Python class behind the command. +::: diff --git a/site/content/cli/mcp-call.md b/site/content/cli/mcp-call.md new file mode 100644 index 0000000..7b27f89 --- /dev/null +++ b/site/content/cli/mcp-call.md @@ -0,0 +1,149 @@ +--- +command: mcp call +lede: Calls one MCP tool in-process and prints its result. The policy, privacy transforms and middleware apply exactly as they would for a client. +intro: | + `cloudg mcp call` is the quickest way to try a tool, debug a policy, or use a tool from a shell script. No server runs and no client is needed. +--- + +## How it works + +Each invocation builds a fresh layer from the shared options (see [cloudg mcp serve](/cli/mcp-serve/#building-the-layer)), calls `TOOL` once as the chosen principal, prints the result and exits. Nothing carries over between invocations. A `load_dataset` call in one command is gone by the next, so pass the data you need with `--dataset` every time. + +Arguments go in `--args` as a JSON object. You have three ways to supply it: + +| Form | Example | +|---|---| +| Inline JSON | `--args '{"ref": "bastion"}'` | +| A file, prefixed with `@` | `--args @args.json` (a leading `~` is expanded) | +| stdin | `--args -` | + +Anything that is not a JSON object is refused before the tool runs, with `Invalid value for --args: not valid JSON: ...` or `must be a JSON object` and exit code 2. Without `--args`, the tool is called with no arguments. + +The principal is the local user unless you pass `--role` or `--principal-id`, as on [cloudg mcp tools](/cli/mcp-tools/). A tool the policy hides from that principal behaves as if it did not exist. + +What gets printed depends on the result and on `--raw`: + +| Result | Printed on stdout | +|---|---| +| Has structured content | The structured content, as indented JSON | +| Text only | Each text block as is; any other block as JSON | +| `--raw` | The whole `CallToolResult`: `content`, `isError`, `structuredContent` and `_meta` | + +`--raw` is the way to see what the policy added. Under the default `standard` policy, `_meta` carries a `cloudg/transforms` entry with the result's sensitivity and provenance, and `cloudg/duration_ms`. + +A tool that fails returns `isError: true` rather than raising. `call` prints that result like any other and then exits with 1, which makes it usable in `if` statements. Here a typo in an asset reference returns suggestions in the error data: + +```console +$ cloudg mcp call get_asset --dataset prod=inventory/inventory-map.json --args '{"ref": "bastoin"}' --raw +{ + "content": [ + { + "type": "text", + "text": "No asset matches the reference in this dataset. 2 close matches, see suggestions in the error data. Use find_assets(query=...) to search by name, ARN or tag." + } + ], + "isError": true, + "_meta": { + "cloudg/error_code": -32602, + "cloudg/error_data": { + "value": "bastoin", + "dataset": "prod", + "suggestions": [ + {"id": "bastion", "name": "bastion", "type": "EC2", ...}, + {"id": "admin-role", "name": "bastion-admin", "type": "IAM_ROLE", ...} + ] + } + } +} +$ echo $? +1 +``` + +(Suggestion objects shortened; they also carry `arn`, `account_id` and `dataset`.) This and the next capture use the synthetic estate from `tests/mcp/fixtures/sample_estate.py`. + +stdout carries only the result. The banner and logs go to stderr, so `| jq` works. + +### Live tools + +`map_inventory`, `collect_assets`, `run_scanners` and `run_pipeline` call the cloud. From `call` they use the cloud settings and credentials of the config given with `-c` (on `mcp call` or on the root `cloudg`), and they run their credential preflight before anything else. Their own timeouts (one or two hours) replace the 300-second default of `--timeout`. A policy or `--read-only` that drops them makes them unknown. + +## Examples + +See what is loaded and where tools may read and write: + +```bash +cloudg mcp call workspace_status +``` + +Search a saved map and keep only two fields per asset: + +```console +$ cloudg mcp call find_assets --dataset prod=inventory/inventory-map.json --args '{"query": "bastion", "fields": ["name", "type"]}' +{ + "dataset": "prod", + "total": 2, + "offset": 0, + "returned": 2, + "next_cursor": null, + "truncated": false, + "items": [ + { + "id": "bastion", + "name": "bastion", + "type": "EC2" + }, + { + "id": "admin-role", + "name": "bastion-admin", + "type": "IAM_ROLE" + } + ] +} +``` + +Count assets by type, with the arguments in a file: + +```bash +echo '{"group_by": "type", "top": 10}' > count.json +cloudg mcp call count_assets --dataset prod=./reports/inventory-map.json --args @count.json +``` + +Build the arguments in a script and pipe them in: + +```bash +jq -n --arg ref "arn:aws:iam::123456789012:role/app-role" '{ref: $ref, max_depth: 5}' \ + | cloudg mcp call blast_radius --dataset prod=./reports/inventory-map.json --args - +``` + +Check whether an analyst could call a tool under your policy: + +```bash +cloudg mcp call get_asset_metadata --policy ./mcp-policy.yaml --role analyst --dataset prod=./reports/inventory-map.json --args '{"ref": "bastion"}' +``` + +Stop a script when a tool reports an error: + +```bash +if ! cloudg mcp call get_asset --dataset prod=./reports/inventory-map.json --args '{"ref": "orders-db"}' > orders-db.json; then + echo "lookup failed, see orders-db.json" >&2 + exit 1 +fi +``` + +## Exit codes + +| Code | When | +|---|---| +| 0 | The tool ran and returned `isError: false` | +| 1 | The tool returned `isError: true`; the tool is unknown or hidden (`Error: Unknown tool: NAME`); or the layer could not be built | +| 2 | `--args` is not a JSON object, or another option was rejected | + +## Related + +:::links +- [Tool index](/mcp/tools/tool-index/) Every tool's arguments and results. +- [Conventions shared by every tool](/mcp/tools/conventions-shared-by-every-tool/) References, pagination, fields and errors. +- [Errors](/mcp/server/errors/) Error codes and what the error data holds. +- [cloudg mcp read](/cli/mcp-read/) The same thing for resources. +- [Live tools and the credential preflight](/mcp/server/live-tools-and-the-credential-preflight/) What happens before a live tool calls the cloud. +::: diff --git a/site/content/cli/mcp-config.md b/site/content/cli/mcp-config.md new file mode 100644 index 0000000..427a03d --- /dev/null +++ b/site/content/cli/mcp-config.md @@ -0,0 +1,142 @@ +--- +command: mcp config +lede: Prints a ready-to-paste MCP client configuration for Claude Desktop, Claude Code, Cursor or VS Code. +intro: | + Each client stores MCP servers in a slightly different JSON shape. `cloudg mcp config` writes the right one, with the launch command and arguments filled in, so you don't have to work out paths by hand. +--- + +## How it works + +The command builds a JSON snippet, prints it on stdout, and prints a one-line hint on stderr saying where the snippet goes. Because the hint is on stderr, you can redirect stdout straight into a file. + +`--client` picks the shape. `--transport` picks between two kinds of entry: + +| `--transport` | What the client does | Options that apply | +|---|---|---| +| `stdio` (default) | Launches `cloudg mcp serve` itself as a child process | `--command`, `--policy`, `--dataset`, `--read-only` | +| `http` | Connects to a server you already started with `cloudg mcp serve --transport http` | `--url`, `--token-env` | + +The two groups don't mix. With `--transport http`, `--policy`, `--dataset` and `--read-only` are ignored, because the running server already has its own options. With `stdio`, `--url` and `--token-env` are ignored. + +For stdio, `--command auto` (the default) looks for a launcher in this order: `cloudg-mcp` on `PATH` (giving `cloudg-mcp serve`), then `cloudg` on `PATH` (giving `cloudg mcp serve`), then the current Python interpreter with `-m cloudg.mcp serve`. Found executables are written as absolute paths, which matters for desktop apps that start with a smaller `PATH` than your shell. Any other value is split with `shlex` and used as is. `--policy`, each `--dataset` and `--read-only` are then appended as server arguments, and dataset paths are made absolute. + +The client shapes differ like this: + +| Client | stdio entry | http entry | Where it goes | +|---|---|---|---| +| `claude-desktop` | `mcpServers.` with `command` and `args` | `npx -y mcp-remote ` bridge, since the desktop config only launches local commands | `claude_desktop_config.json` (Settings > Developer > Edit Config), then restart | +| `claude-code` | `mcpServers.` with `type: stdio`, `command`, `args`, `env` | `type: http`, `url`, optional `headers` | `.mcp.json` in the project root, or `claude mcp add-json` | +| `cursor` | `mcpServers.` with `command` and `args` | `url`, optional `headers` | `~/.cursor/mcp.json` or `.cursor/mcp.json` | +| `vscode` | `servers.` with `type: stdio`, `command`, `args` | `type: http`, `url`, optional `headers` | `.vscode/mcp.json`, or under `mcp` in `settings.json` | + +`--token-env VAR` adds `Authorization: Bearer ...` using each client's variable syntax: `${VAR}` for Claude Code and the `mcp-remote` bridge, `${env:VAR}` for Cursor and VS Code. The token itself never appears in the file. + +## Examples + +Print the default Claude Desktop entry. On a machine where neither launcher is on `PATH`, it falls back to the interpreter: + +```console +$ cloudg mcp config +{ + "mcpServers": { + "cloudg": { + "command": "/opt/cloudg/.venv/bin/python3", + "args": [ + "-m", + "cloudg.mcp", + "serve" + ] + } + } +} +Merge into claude_desktop_config.json (Settings > Developer > Edit Config), then restart Claude Desktop. +``` + +Write a project `.mcp.json` for Claude Code with a strict, read-only server and one preloaded map: + +```console +$ cloudg mcp config --client claude-code --command "cloudg mcp serve" --policy strict --dataset prod=./reports/inventory-map.json --read-only > .mcp.json +Save as .mcp.json in your project root, or run: claude mcp add-json ''. +$ cat .mcp.json +{ + "mcpServers": { + "cloudg": { + "type": "stdio", + "command": "cloudg", + "args": [ + "mcp", + "serve", + "--policy", + "strict", + "--dataset", + "prod=/home/me/project/reports/inventory-map.json", + "--read-only" + ], + "env": {} + } + } +} +``` + +Point Claude Code at a shared HTTP server that needs a token: + +```console +$ cloudg mcp config --client claude-code --transport http --token-env CLOUDG_MCP_TOKEN 2>/dev/null +{ + "mcpServers": { + "cloudg": { + "type": "http", + "url": "http://127.0.0.1:8765/mcp", + "headers": { + "Authorization": "Bearer ${CLOUDG_MCP_TOKEN}" + } + } + } +} +``` + +Connect VS Code to a server on another host: + +```bash +cloudg mcp config --client vscode --transport http --url https://mcp.internal.example/mcp --token-env CLOUDG_MCP_TOKEN > .vscode/mcp.json +``` + +Reach an HTTP server from Claude Desktop through the `mcp-remote` bridge (needs Node.js): + +```bash +cloudg mcp config --client claude-desktop --transport http --token-env CLOUDG_MCP_TOKEN +``` + +Give the server a different name, so it can sit next to another cloudg entry: + +```bash +cloudg mcp config --client cursor --name cloudg-prod --dataset prod=./reports/inventory-map.json +``` + +Generate the same snippet from Python, for example in a provisioning script: + +```python title="write_mcp_config.py" +import json + +from cloudg.mcp.cli import client_config + +snippet = client_config( + "claude-code", + command=["cloudg", "mcp", "serve"], + serve_args=["--policy", "strict", "--read-only"], +) +print(json.dumps(snippet, indent=2)) +``` + +## Exit codes + +The command exits 0 after printing. Click returns 2 for an unknown `--client` or `--transport` value. + +## Related + +:::links +- [Connect a client](/mcp/connect/) Where each client keeps its configuration, step by step. +- [cloudg mcp serve](/cli/mcp-serve/) Every option the generated command can take. +- [Quick start with a desktop client](/mcp/server/quick-start-with-a-desktop-client/) From install to the first question. +- [Built-in profiles](/mcp/privacy/built-in-profiles/) Which `--policy` to put in the launch arguments. +::: diff --git a/site/content/cli/mcp-prompts.md b/site/content/cli/mcp-prompts.md new file mode 100644 index 0000000..a9ea061 --- /dev/null +++ b/site/content/cli/mcp-prompts.md @@ -0,0 +1,118 @@ +--- +command: mcp prompts +lede: Lists the MCP prompts a server would expose, with the arguments each one takes. +intro: | + cloudg's prompts are ready-made workflows, such as a security posture review or an incident triage, that a client can offer its user as one-click tasks. `cloudg mcp prompts` shows which of them a principal can see and what arguments they expect. +--- + +## How it works + +The command builds the layer from the shared options (see [cloudg mcp serve](/cli/mcp-serve/#building-the-layer)) and prints the prompts visible to one principal: the local user by default, or the one described by `--role` and `--principal-id` (see [cloudg mcp tools](/cli/mcp-tools/) for how those work). `--prefix` is applied to prompt names as well as tool names. + +The table has four columns. Arguments are listed in order, and a `*` marks the required ones. In 0.6.0 there are 11 prompts, all in the `prompts` category, and every built-in policy lists all of them for the local user: + +```console +$ cloudg mcp prompts + cloudg MCP prompts (11) +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Prompt ┃ Category ┃ Arguments ┃ Description ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ security_posture_review │ prompts │ dataset │ Whole-estate security review: risks, exposure, identity pivots, coverage gaps, compliance. │ +│ executive_summary │ prompts │ dataset │ Non-technical one-page summary of the estate's security state. │ +│ attack_surface_report │ prompts │ dataset │ External attack surface: exposed assets, open admin ports, internet-to-crown-jewel paths. │ +│ investigate_asset │ prompts │ ref*, dataset │ Deep dive on one asset: config, relations, exposure, findings, blast radius. │ +│ blast_radius_assessment │ prompts │ ref*, dataset │ Impact of compromise or loss of one asset. │ +│ change_impact_analysis │ prompts │ ref*, change, dataset │ What a planned change to one asset affects, with rollout and rollback plan. │ +│ cross_account_trust_review │ prompts │ dataset │ IAM trust and grants across account boundaries, external accounts, pivot chains. │ +│ compliance_gap_analysis │ prompts │ framework*, dataset │ Failing controls, responsible assets and a remediation backlog for one framework. │ +│ remediation_plan │ prompts │ severity, dataset │ Phased fix plan for findings at or above a severity. │ +│ incident_triage │ prompts │ finding_id*, dataset │ Exploitability, impact and containment for one finding. │ +│ drift_review │ prompts │ base*, target │ Explain the differences between two datasets (before / after). │ +└────────────────────────────┴──────────┴───────────────────────┴────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +`dataset` is optional everywhere and means the active dataset when left empty. + +`--json` prints the `prompts/list` entries a client receives. Each has `name`, `title`, `description`, `arguments` (with `name`, `required` and `description`) and `_meta`: + +```console +$ cloudg mcp prompts --json | jq '.[3]' +{ + "name": "investigate_asset", + "description": "Deep dive on one asset: config, relations, exposure, findings, blast radius.", + "arguments": [ + { + "name": "ref", + "required": true, + "description": "Asset id, ARN or unique name." + }, + { + "name": "dataset", + "required": false, + "description": "Dataset name; empty = active dataset." + } + ], + "title": "Investigate asset", + "_meta": { + "cloudg/category": "prompts", + "cloudg/sensitivity": "confidential", + "cloudg/tags": [ + "workflow" + ] + } +} +``` + +The CLI lists prompts but has no command to render one. A client does that through `prompts/get`. To see the rendered messages yourself, call the layer from Python (example below). A rendered prompt is a set of user messages: the instructions, a JSON context block, and often an embedded resource with the current data, such as the asset's detail for `investigate_asset`. + +## Examples + +List every prompt: + +```bash +cloudg mcp prompts +``` + +See which prompts a principal with the `analyst` role gets under your own policy: + +```bash +cloudg mcp prompts --policy ./mcp-policy.yaml --role analyst +``` + +List only the names of prompts with a required argument: + +```bash +cloudg mcp prompts --json | jq -r '.[] | select(any(.arguments[]; .required)) | .name' +``` + +Render a prompt in-process to read what the assistant would receive: + +```python title="render_prompt.py" +import asyncio +import json + +from cloudg.mcp.server import create_layer_from_options + +layer = create_layer_from_options(datasets=["prod=./reports/inventory-map.json"]) +result = asyncio.run(layer.get_prompt("investigate_asset", {"ref": "bastion"})) +print(json.dumps(result.to_wire(), indent=2)) +``` + +A missing required argument raises `Missing required prompt arguments: ref`, and an unknown or hidden prompt raises `Unknown prompt: `. + +## Exit codes + +| Code | When | +|---|---| +| 0 | The list was printed | +| 1 | The layer could not be built | +| 2 | Click rejected an option | + +## Related + +:::links +- [Prompts](/mcp/tools/prompts/) Each prompt with its arguments and rendered output. +- [cloudg mcp tools](/cli/mcp-tools/) The tools the prompts tell the assistant to call. +- [In-process use](/mcp/server/in-process-use/) Driving the layer from Python. +- [cloudg mcp serve](/cli/mcp-serve/) The shared layer options in full. +::: diff --git a/site/content/cli/mcp-read.md b/site/content/cli/mcp-read.md new file mode 100644 index 0000000..87fa26d --- /dev/null +++ b/site/content/cli/mcp-read.md @@ -0,0 +1,109 @@ +--- +command: mcp read +lede: Reads one MCP resource in-process and prints its contents, with the policy applied as it would be for a client. +intro: | + Resources are fetched by URI, such as `cloudg://findings/summary` or `cloudg://assets/`. `cloudg mcp read` lets you look at one from the shell, which helps when you are writing a policy or checking what an assistant will be shown. +--- + +## How it works + +Like [cloudg mcp call](/cli/mcp-call/), each invocation builds a fresh layer from the shared options (see [cloudg mcp serve](/cli/mcp-serve/#building-the-layer)), reads `URI` once as the chosen principal, prints the contents and exits. Most resources read the active dataset, so load one with `--dataset` on the same command line. Without a dataset, those resources fail: + +```console +$ cloudg mcp read cloudg://findings/summary +Error: No dataset is loaded. Call load_dataset(path=...) with an inventory-map.json / findings.json / scanner report, or map_inventory to collect live data. +``` + +The URI can be a fixed resource or a filled-in template. [cloudg mcp resources](/cli/mcp-resources/) lists both. For the asset templates, the reference can be the internal ID, a unique name, or a full ARN, and the ARN can be written as is or URL-encoded. Quote the URI in the shell, since ARNs contain `:` and `/` and some templates contain characters the shell treats specially. + +Text contents are printed as they are, which for most resources is indented JSON and for some is another format: `cloudg://ontology/turtle` prints Turtle, `cloudg://graph/graphml` prints GraphML XML, and `cloudg://docs/{topic}` prints Markdown. Binary contents, if a resource returns any, are printed as their wire JSON with the base64 blob. + +The policy decides what the principal may read. A resource it hides gives the same error as one that does not exist. For example, the `strict` profile hides the restricted Turtle export: + +```console +$ cloudg mcp read cloudg://ontology/turtle --policy strict --dataset prod=inventory/inventory-map.json +Error: Unknown resource: cloudg://ontology/turtle +``` + +The principal options (`--role`, `--principal-id`) work as on [cloudg mcp tools](/cli/mcp-tools/). stdout carries only the contents, so the output can be piped or redirected. + +## Examples + +Headline numbers of a dataset, by name. This capture uses the synthetic estate from the test suite and is cut after the first lines: + +```console +$ cloudg mcp read cloudg://datasets/prod/summary --dataset prod=inventory/inventory-map.json +{ + "dataset": "prod", + "kind": "inventory", + "source": "/home/me/estate/inventory/inventory-map.json", + "loaded_at": "2026-10-09T16:24:08+00:00", + "version": 0, + "providers": [ + "aws", + "azure", + "gcp" + ], + "total_assets": 44, + "total_edges": 56, + "total_findings": 12, + "open_findings": 11, + "suppressed_findings": 1, + ... +} +``` + +`total_findings` is not zero although only an inventory map was given: a `findings.json` in the same folder is loaded into the dataset. + +Read one asset by ARN: + +```bash +cloudg mcp read 'cloudg://assets/arn:aws:ec2:us-east-1:111111111111:instance/i-0bast' --dataset prod=./reports/inventory-map.json +``` + +List the open critical findings, highest risk first: + +```bash +cloudg mcp read cloudg://findings/severity/CRITICAL --dataset prod=./reports/inventory-map.json +``` + +Export the ontology as Turtle for a triple store: + +```bash +cloudg mcp read cloudg://ontology/turtle --dataset prod=./reports/inventory-map.json > estate.ttl +``` + +Export the graph for Gephi or yEd: + +```bash +cloudg mcp read cloudg://graph/graphml --dataset prod=./reports/inventory-map.json > estate.graphml +``` + +See which documentation topics the server offers to assistants: + +```bash +cloudg mcp read cloudg://docs +``` + +Check what the active policy does, as seen by a given role: + +```bash +cloudg mcp read cloudg://policy --policy ./mcp-policy.yaml --role analyst +``` + +## Exit codes + +| Code | When | +|---|---| +| 0 | The contents were printed | +| 1 | The URI is unknown or hidden (`Error: Unknown resource: URI`), the resource failed (for example no dataset loaded), or the layer could not be built | +| 2 | Click rejected an option | + +## Related + +:::links +- [Resources and resource templates](/mcp/tools/resources-and-resource-templates/) Every resource with example contents. +- [cloudg mcp resources](/cli/mcp-resources/) List the URIs and templates. +- [cloudg mcp call](/cli/mcp-call/) Call a tool instead of reading a resource. +- [The workspace](/mcp/server/the-workspace/) How datasets are loaded and named. +::: diff --git a/site/content/cli/mcp-resources.md b/site/content/cli/mcp-resources.md new file mode 100644 index 0000000..a4bbd91 --- /dev/null +++ b/site/content/cli/mcp-resources.md @@ -0,0 +1,100 @@ +--- +command: mcp resources +lede: Lists the MCP resources and resource templates a server would expose, with their URIs, sensitivity and MIME types. +intro: | + Resources are the read-only side of the MCP server: summaries, graphs, schemas and single assets that a client can fetch by URI without calling a tool. `cloudg mcp resources` shows which ones the active policy lets a principal see. +--- + +## How it works + +The command builds the layer from the shared options (see [cloudg mcp serve](/cli/mcp-serve/#building-the-layer)), asks it for the resources and resource templates visible to one principal, and prints them. The principal options work as on [cloudg mcp tools](/cli/mcp-tools/): the local user by default, or `--role` and `--principal-id` to see what a token holder gets. + +There are two kinds of entry. A resource has a fixed URI, such as `cloudg://findings/summary`. A resource template has placeholders in braces that the client fills in, such as `cloudg://assets/{+ref}`. The `{+ref}` form means the value may contain `/` and `:`, so a full ARN fits. The table lists resources first, then templates, under one "URI / template" column: + +| Column | Source | +|---|---| +| URI / template | `uri` for a resource, `uriTemplate` for a template | +| Name | The internal name, such as `findings_summary_resource` | +| Category | `_meta["cloudg/category"]` | +| Sensitivity | `_meta["cloudg/sensitivity"]` | +| MIME | `mimeType`, mostly `application/json`; `text/turtle` for the ontology | +| Description | First line of the description | + +In 0.6.0, the default policy shows the local user 14 resources and 11 templates: + +| Fixed URIs | Templates | +|---|---| +| `cloudg://workspace`, `cloudg://datasets` | `cloudg://datasets/{dataset}/summary` | +| `cloudg://findings/summary` | `cloudg://assets/{+ref}`, `cloudg://assets/{+ref}/neighbors`, `cloudg://assets/{+ref}/findings` | +| `cloudg://compliance` | `cloudg://findings/{finding_id}`, `cloudg://findings/severity/{severity}` | +| `cloudg://graph/d3` | `cloudg://compliance/{framework}` | +| `cloudg://ontology/turtle` | `cloudg://graph/{format}`, `cloudg://ontology/{format}` | +| `cloudg://schema/asset-types`, `cloudg://schema/edge-types`, `cloudg://schema/relation-types` | `cloudg://schema/asset-types/{asset_type}` | +| `cloudg://docs` | `cloudg://docs/{topic}` | +| `cloudg://policy`, `cloudg://privacy/detectors`, `cloudg://ratelimit`, `cloudg://metrics` | | + +The list does not depend on loaded datasets. Resources that read data, like `cloudg://findings/summary`, use the active dataset when they are read, so preload one with `--dataset` before you [read](/cli/mcp-read/) them. + +`--include-category` and `--exclude-category` apply to resources and templates as well as tools. `--include-tool` and `--exclude-tool` do not touch them. + +With `--json`, the output is one object with two keys, `resources` and `resourceTemplates`, each holding the wire definitions a client would receive: + +```console +$ cloudg mcp resources --json | jq '.resourceTemplates[0]' +{ + "uriTemplate": "cloudg://datasets/{dataset}/summary", + "name": "dataset_summary_resource", + "mimeType": "application/json", + "description": "Headline numbers of one dataset.", + "title": "Dataset summary", + "_meta": { + "cloudg/category": "workspace", + "cloudg/sensitivity": "internal" + } +} +``` + +stdout carries only the table or the JSON; the banner and logs go to stderr. + +## Examples + +List all resources and templates for the default policy: + +```bash +cloudg mcp resources +``` + +Print only the URIs, one per line: + +```bash +cloudg mcp resources --json | jq -r '.resources[].uri, .resourceTemplates[].uriTemplate' +``` + +Check which resources an analyst sees under your policy: + +```bash +cloudg mcp resources --policy ./mcp-policy.yaml --role analyst +``` + +See what is left when you hide the ontology and privacy categories: + +```bash +cloudg mcp resources --exclude-category ontology --exclude-category privacy +``` + +## Exit codes + +| Code | When | +|---|---| +| 0 | The list was printed | +| 1 | The layer could not be built, for example a missing policy file or dataset | +| 2 | Click rejected an option | + +## Related + +:::links +- [Resources and resource templates](/mcp/tools/resources-and-resource-templates/) Every resource with an example of its content. +- [cloudg mcp read](/cli/mcp-read/) Fetch one of these URIs from the shell. +- [cloudg mcp tools](/cli/mcp-tools/) The same listing for tools. +- [The workspace](/mcp/server/the-workspace/) Datasets and which one is active. +::: diff --git a/site/content/cli/mcp-serve.md b/site/content/cli/mcp-serve.md new file mode 100644 index 0000000..61424b5 --- /dev/null +++ b/site/content/cli/mcp-serve.md @@ -0,0 +1,182 @@ +--- +command: mcp serve +lede: Runs cloudg as a Model Context Protocol server, so an AI assistant can query your inventory, graph, findings and compliance data through tools, resources and prompts. +intro: | + Desktop clients (Claude Desktop, Claude Code, Cursor, VS Code) launch `cloudg mcp serve` themselves over stdio, and you rarely type it by hand. You do type it when you serve over HTTP, test a policy, or preload datasets. [cloudg mcp config](/cli/mcp-config/) writes the client snippet for you. The [MCP server](/mcp/) section covers the server in depth. +--- + +## How it works + +The command builds one MCP layer from its options, picks a server implementation, and serves until the transport closes: end of input on stdio, Ctrl+C on HTTP. + +```mermaid caption="From the command line to a running server" +flowchart TD + A["Options and env vars"] --> B["Validate tokens and option combinations"] + B -->|invalid| X["Error before start"] + B --> C["Load registry, apply --read-only and filters"] + C --> D["Apply policy, add middleware"] + D --> E["Preload --dataset files"] + E --> F{"--transport"} + F -->|stdio| G["JSON-RPC on stdin and stdout"] + F -->|"http, sse"| H["HTTP server on host, port and path"] + G --> I["Stop: save pseudonym vault"] + H --> I +``` + +### Building the layer + +These options are shared with `tools`, `resources`, `prompts`, `call` and `read`, so you can check what a server will expose before you start it. + +The registry is the built-in catalog unless `--registry MODULE:ATTR` names your own. `--read-only` then drops every tool that writes files, calls cloud APIs, runs scanners or is marked destructive. `--include-category`, `--exclude-category`, `--include-tool` and `--exclude-tool` narrow it further, and `--prefix` is put in front of every tool and prompt name (at most 64 characters from `A-Z a-z 0-9 _ . -`). + +The policy comes from `--policy`, which accepts a profile name (`open`, `standard`, `strict`, `read_only`, `airgapped`, `audit`, `soc-analyst`), a YAML or JSON file, or inline JSON. Without the option, cloudg reads `CLOUDG_MCP_POLICY`, and without that it uses `standard`. The policy decides which tools each principal sees and which privacy transforms run on results. See [Built-in profiles](/mcp/privacy/built-in-profiles/). + +Middleware is added from the options. A metrics middleware is always on, which is why `cloudg://metrics` is always listed. `--audit-log` (or `CLOUDG_MCP_AUDIT_LOG`) appends a JSONL record per call with argument values hashed. `--max-concurrency` caps parallel tool calls, and `--cache-ttl` caches results of read-only, idempotent tools for that many seconds. `--timeout` is the default per-call limit, 300 seconds; tools with their own limit, such as the live collection tools, keep theirs. + +Each `--dataset [NAME=]PATH` is loaded before the server accepts a connection. Without a name, an `inventory-map.json` is named after its folder and anything else after its file stem. A `findings.json` next to an inventory map is loaded into the same dataset. Dataset paths you give here are trusted and may sit outside the workspace's allowed roots; those roots only limit the paths tools may open. + +`-c/--config` points the layer at a cloudg `config.yaml`, which the live tools use for providers and credentials. Without it, the config given to the root `cloudg -c` is used. + +### Choosing a flavor + +`--flavor auto` uses the official `mcp` SDK when it is installed and the built-in, dependency-free server otherwise. `native`, `sdk` and `fastmcp` force one. Some options only work with some flavors, and the command refuses the combination instead of ignoring the option: + +```console +$ cloudg mcp serve --transport http --flavor sdk --cors-origin http://x +Error: --cors-origin is only supported by the native flavor (use --flavor native) +$ cloudg mcp serve --transport sse --flavor fastmcp --json-response +Error: --json-response / --stateless apply to Streamable HTTP; the fastmcp flavor cannot combine them with --transport sse +``` + +`--page-size` (items per page in list results) only applies to the native flavor. [Flavors](/mcp/server/flavors/) compares them. + +### stdio + +stdio is the default. The client starts the process and talks newline-delimited JSON-RPC over stdin and stdout. cloudg keeps stdout for protocol messages only: the banner, logs and notices all go to stderr. You can see this by piping requests in by hand. This exchange used the native flavor and a dataset from the test suite's sample estate; output lines are cut: + +```console +$ printf '%s\n' \ + '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"shell","version":"0"}}}' \ + '{"jsonrpc":"2.0","method":"notifications/initialized"}' \ + '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"count_assets","arguments":{"group_by":"provider"}}}' \ + | cloudg mcp serve --flavor native --dataset prod=inventory/inventory-map.json 2>/dev/null +{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-06-18","capabilities":{"tools":{"listChanged":true},...},"serverInfo":{"name":"cloudg","version":"0.6.0","title":"cloudg"},...}} +{"jsonrpc":"2.0","id":2,"result":{"content":[{"type":"text","text":"{\n \"dataset\": \"prod\",\n \"group_by\": \"provider\",\n \"total\": 44, ..."}],"isError":false,"structuredContent":{...}}} +``` + +Auth tokens are ignored on stdio. Whoever launched the process is the local user, and the policy treats them that way. + +### HTTP + +`--transport http` (or its alias `streamable-http`) serves Streamable HTTP at the address built from `--host`, `--port` and `--path`, which is `http://127.0.0.1:8765/mcp` by default. `--transport sse` adds the deprecated `/sse` and `/messages/` endpoints for older clients. When the server is up, it prints one line to stderr: + +```console +$ cloudg mcp serve --transport http --port 8799 +cloudg MCP server (mcp SDK 2.x, http) listening on http://127.0.0.1:8799/mcp +``` + +`--json-response` answers POSTs with plain JSON instead of an SSE stream, and `--stateless` drops sessions for clients that cannot keep one. + +Three checks protect an HTTP server: + +| Check | Default | Change it with | +|---|---|---| +| `Origin` header | Loopback origins only, on every bind and every flavor | `--allowed-origin PATTERN` (repeatable, replaces the default) | +| `Host` header | Loopback names when bound to loopback; not checked on other binds | `--allowed-host PATTERN` (repeatable) | +| Bearer token | None: every HTTP caller is the `anonymous` principal with the `default` role | `--auth-token TOKEN[:ROLES[:ID]]` (repeatable) or `CLOUDG_MCP_AUTH_TOKENS` | + +Each token spec maps a bearer token to a principal. `ROLES` is a comma-separated list, and the principal always gets the `default` role as well. `TOKEN` can be `env:VARNAME` so the secret stays out of your shell history and process list. A token that itself contains `:` needs the full three-part form. `CLOUDG_MCP_AUTH_TOKENS` holds several specs separated by whitespace. + +The tokens are checked before the server starts. An empty token, given directly or through a variable that is not set, stops the command rather than starting a server without the authentication you asked for: + +```console +$ cloudg mcp serve --transport http --auth-token env:CLOUDG_MCP_TOKEN +Error: Invalid value for --auth-token: Environment variable 'CLOUDG_MCP_TOKEN' for an auth token is empty +``` + +:::warning Binding beyond loopback +Keep `--host 127.0.0.1` unless you also set `--auth-token`. On a non-loopback bind the `Host` header is not checked unless you pass `--allowed-host`, so set that too. +::: + +`--cors-origin` grants CORS to a browser origin. Only the native flavor supports it. + +### Stopping + +When the transport closes, cloudg saves the pseudonym vault if the policy configures a `vault.path`. Without one, pseudonyms live in memory and change on the next start unless `CLOUDG_MCP_VAULT_KEY` is set. + +## Examples + +Serve over stdio with the default `standard` policy, the way a desktop client starts it: + +```bash +cloudg mcp serve +``` + +Preload yesterday's map and findings and apply the `strict` policy: + +```bash +cloudg mcp serve --policy strict --dataset prod=/srv/cloudg/reports/inventory-map.json +``` + +Expose only inventory and graph tools, with no way to call the cloud or write files: + +```bash +cloudg mcp serve --read-only --include-category inventory --include-category graph --dataset ./reports/inventory-map.json +``` + +Start without the root banner and config handling, through the standalone `cloudg-mcp` entry point (`python -m cloudg.mcp` does the same): + +```bash +cloudg-mcp serve --dataset ./reports/inventory-map.json +``` + +Serve Streamable HTTP on loopback for a local web client: + +```bash +cloudg mcp serve --transport http +``` + +Serve HTTP with two tokens mapped to roles, and keep an audit trail: + +```bash +export CLOUDG_ANALYST_TOKEN="$(openssl rand -hex 32)" +export CLOUDG_ADMIN_TOKEN="$(openssl rand -hex 32)" +cloudg mcp serve --transport http \ + --auth-token env:CLOUDG_ANALYST_TOKEN:analyst:alice \ + --auth-token env:CLOUDG_ADMIN_TOKEN:admin,analyst:ops \ + --audit-log ./mcp-audit.jsonl +``` + +Make the server reachable from other hosts, with tokens and explicit Host and Origin lists: + +```bash +cloudg mcp serve --transport http --host 0.0.0.0 \ + --auth-token env:CLOUDG_MCP_TOKEN:analyst \ + --allowed-host 'mcp.internal.example:*' \ + --allowed-origin 'https://console.internal.example' +``` + +Debug a client that misbehaves, with full logs on stderr: + +```bash +cloudg mcp serve --log-level DEBUG 2> ./mcp-server.log +``` + +## Exit codes + +| Code | When | +|---|---| +| 0 | The transport closed normally: end of input on stdio, Ctrl+C on HTTP | +| 1 | The layer could not be built: a dataset path that does not exist, an invalid `--prefix`, a policy file that is missing or fails to load (`Error: Policy file not found: nope.yaml`) | +| 2 | An option was rejected before start: an empty or malformed `--auth-token`, a flavor and option combination that does not work, a `--registry` that does not resolve to a `Registry` | + +## Related + +:::links +- [Connect a client](/mcp/connect/) Step-by-step setup for each desktop client. +- [cloudg mcp config](/cli/mcp-config/) Generate the client configuration snippet. +- [Transports and protocol versions](/mcp/server/transports-and-protocol-versions/) stdio, Streamable HTTP and SSE in detail. +- [Security model in brief](/mcp/server/security-model-in-brief/) What the policy, tokens and Origin checks protect. +- [Built-in profiles](/mcp/privacy/built-in-profiles/) What each `--policy` profile shows and hides. +- [cloudg mcp tools](/cli/mcp-tools/) Check what a policy exposes before you serve it. +::: diff --git a/site/content/cli/mcp-tools.md b/site/content/cli/mcp-tools.md new file mode 100644 index 0000000..9479e89 --- /dev/null +++ b/site/content/cli/mcp-tools.md @@ -0,0 +1,108 @@ +--- +command: mcp tools +lede: Lists the MCP tools a server would expose, as a table or as the exact `tools/list` JSON a client receives. +intro: | + Run `cloudg mcp tools` with the same options you plan to give `cloudg mcp serve`. What it prints is what the client will see, after the policy, the filters and `--read-only` have done their work. +--- + +## How it works + +The command builds the same layer `serve` would build from the shared options (`--policy`, `--read-only`, `--include-category` and the rest, listed on [cloudg mcp serve](/cli/mcp-serve/#building-the-layer)), asks it for the tool list of one principal, and prints it. No server starts and no client is involved. + +The principal is the local user unless you pass `--role` or `--principal-id`. The local user has the id `local` and the roles `default` and `local`. With either option, the caller becomes a principal with the id you gave (default `cli`) and the roles you gave (default `default`). This is how you check what a token holder will see before you hand out the token: give the same roles you put in its `--auth-token` spec. + +Without `--json`, the output is a table with five columns: + +| Column | Source | +|---|---| +| Tool | The tool name, with `--prefix` applied | +| Category | `_meta["cloudg/category"]`: `workspace`, `inventory`, `graph`, `findings`, `compliance`, `ontology`, `export`, `live`, `meta` or `privacy` | +| Sensitivity | `_meta["cloudg/sensitivity"]`: `public`, `internal`, `confidential` or `restricted` | +| Hints | The MCP annotations that are true: `read-only`, `destructive`, `idempotent`, `open-world` | +| Description | The first line of the description as written in the source, so it can stop mid-sentence; cut at 90 characters | + +```console +$ cloudg mcp tools --category live + cloudg MCP tools (5) +┏━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Tool ┃ Category ┃ Sensitivity ┃ Hints ┃ Description ┃ +┡━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ map_inventory │ live │ confidential │ open-world │ Map the complete live inventory (no scanners) with read-only cloud │ +│ collect_assets │ live │ confidential │ open-world │ Run the standard multi-provider asset collection (the `cloudg │ +│ run_scanners │ live │ confidential │ open-world │ Run security scanners (external binaries, cloud credentials) │ +│ run_pipeline │ live │ confidential │ open-world │ The complete cloudg pipeline: collect -> scan -> analyse (graph, │ +│ rate_limit_status │ live │ internal │ read-only, idempotent │ Live-operation guard and cloud API throttling state, without │ +└───────────────────┴──────────┴──────────────┴───────────────────────┴────────────────────────────────────────────────────────────────────┘ +``` + +With `--json`, the command prints the list of tool definitions exactly as they go over the wire: `name`, `title`, `description`, `inputSchema`, `annotations`, `_meta`, and `outputSchema` where the policy keeps it. Pipe it to `jq` to inspect argument schemas. + +`--category` filters on the category and is repeatable. It only filters the printout. To change what a server exposes, use `--include-category` or `--exclude-category`, which `serve` also understands. `--include-tool` and `--exclude-tool` take the names without the prefix. + +How many tools you see depends on the policy and the principal. For the local user in 0.6.0: + +| Options | Tools listed | +|---|---| +| `--policy open` | 74 | +| none (`standard`) | 73, without `reveal_token` | +| `--role admin` | 74, `reveal_token` included | +| `--read-only` | 65 | +| `--policy read_only` | 66 | +| `--policy strict` | 61 | + +stdout carries only the table or the JSON. The banner and any log lines go to stderr, so piping is safe. + +## Examples + +List every tool the default policy exposes: + +```bash +cloudg mcp tools +``` + +Show only the graph and inventory tools: + +```bash +cloudg mcp tools --category graph --category inventory +``` + +Check what an analyst token will be able to call under your own policy file: + +```bash +cloudg mcp tools --policy ./mcp-policy.yaml --role analyst +``` + +Confirm that a read-only deployment has no tools that touch the cloud: + +```bash +cloudg mcp tools --read-only --category live +``` + +Print the argument schema of one tool: + +```bash +cloudg mcp tools --json --include-tool find_assets | jq '.[0].inputSchema' +``` + +Save the full definitions to diff them between two releases: + +```bash +cloudg mcp tools --json --policy open > tools-0.6.0.json +``` + +## Exit codes + +| Code | When | +|---|---| +| 0 | The list was printed | +| 1 | The layer could not be built, for example `Error: Policy file not found: ...` or an invalid `--prefix` | +| 2 | Click rejected an option, or `--registry` did not resolve to a `Registry` | + +## Related + +:::links +- [Tool index](/mcp/tools/tool-index/) Every tool with its arguments and an example result. +- [cloudg mcp call](/cli/mcp-call/) Call one of the listed tools from the shell. +- [Built-in profiles](/mcp/privacy/built-in-profiles/) What each policy hides and transforms. +- [cloudg mcp serve](/cli/mcp-serve/) The shared layer options in full. +::: diff --git a/site/content/cli/output-flags.md b/site/content/cli/output-flags.md new file mode 100644 index 0000000..c1279b3 --- /dev/null +++ b/site/content/cli/output-flags.md @@ -0,0 +1,141 @@ +--- +title: Output flags +lede: "Where cloudg writes its files, which formats each command produces, and the flags that switch outputs on and off: `-o`, `--format`, `--ontology`, `--rag-export`, `--terraform` and `--json`." +--- + +## -o, --output + +Every command that writes files takes `-o/--output`, and it defaults to `./reports` everywhere. The directory is created if it doesn't exist, parents included. + +| Command | Writes into `-o` | +|---|---| +| `run` | Reports, graph files, ontology, RAG chunks, native scanner output | +| `map` | Inventory map files, plus asset and compliance maps with `--findings` | +| `collect` | `inventory-.json` | +| `scan` | `raw-findings.json` and native scanner output | +| `ingest` | `raw-findings.json` and the reports picked by `--format` | +| `report` | The reports picked by `--format` | + +`cloudg deps` reads instead of writing. Its `-m/--map` option points at a saved `inventory-map.json` or its folder, with the same `./reports` default, so `map` followed by `deps` works with no options at all. + +File names are fixed, so a second run into the same directory overwrites the first. Keep a history with dated folders: + +```bash +cloudg run -p aws --regions all -o "./reports/$(date +%F)" +``` + +`report.output_dir` in `config.yaml` does not change the default of `-o`. It is the default output directory of the MCP workspace, where MCP tools write exports. + +## The output directory + +After a `cloudg run` with every phase enabled and a `cloudg map` into the same folder, the directory looks like this: + +```text +reports/ +├── findings.json normalised result (run, ingest, report) +├── report.html interactive report (run, ingest, report) +├── topology.svg static topology (run, report) +├── topology.graphml graph for Gephi, yEd, NetworkX (run) +├── topology-cytoscape.json graph for Cytoscape.js (run) +├── ontology.ttl RDF, Turtle (run, --ontology) +├── ontology.jsonld RDF, JSON-LD (run, --ontology) +├── rag_chunks.jsonl retrieval chunks, one per line (run, --rag-export) +├── rag_metadata_index.json chunk index (run, --rag-export) +├── raw-findings.json pre-normalisation findings (scan, ingest) +├── inventory-map.json assets, edges, summary (map) +├── inventory-map.graphml interconnection graph (map) +├── inventory-graph.json D3 graph (map) +├── inventory-dependencies.json shared deps, blast radius, coverage (map) +├── inventory-organization.json org and Control Tower topology (map --org) +├── asset-map.json findings per asset (map --findings) +├── compliance-map.json framework to assets (map --findings) +├── inventory-aws.json assets and edges of one provider (collect) +├── prowler/aws/ Prowler's ASFF output (run; scan writes prowler/) +├── scoutsuite/aws/ ScoutSuite's report (run; scan writes scoutsuite/) +└── terraform/ provider.tf.json, variables.tf.json, main.tf.json, import_commands.sh (run, --terraform) +``` + +The [output files reference](/reference/output-files/) describes each file's structure. + +:::note The Terraform folder doesn't follow -o +The Terraform recreation goes to `terraform.output_dir`, which defaults to `./reports/terraform` whatever `-o` says. cloudg uses `/terraform` only when that key is set to an empty string. If you change `-o`, set `terraform.output_dir` in your config as well. +::: + +## --format + +Two commands let you pick formats: + +| Command | Choices | Default | +|---|---|---| +| `report` | `html`, `json`, `svg`, `all` | `all` | +| `ingest` | `html`, `json`, `all` | `all` | + +`json` writes `findings.json`, `html` writes `report.html`, and `svg` writes `topology.svg`. `ingest` has no SVG choice, since it has no inventory to draw. + +`run` has no `--format` and always writes all three. The `report.formats` key in `config.yaml` is not read by the CLI. `map` writes its fixed set of inventory files. + +## --ontology, --rag-export, --terraform + +These three switches exist on `cloudg run` only. Each pairs with a config key, but they combine differently: + +| Flag | Default | Config key | Phase runs when | +|---|---|---|---| +| `--ontology/--no-ontology` | on | `ontology.enabled` (default true) | the flag and the key are both on | +| `--rag-export/--no-rag-export` | on | `rag.enabled` (default true) | the flag and the key are both on | +| `--terraform/--no-terraform` | off | `terraform.enabled` (default false) | the flag or the key is on | + +So `--ontology` cannot switch the ontology back on when the config turns it off, while `--no-terraform` cannot switch Terraform off when the config turns it on. + +What each one writes is shaped by config: + +| Output | Settings | +|---|---| +| Ontology | One file per entry in `ontology.export_formats`. `turtle` gives `ontology.ttl`, `json-ld` gives `ontology.jsonld`, `xml` gives `ontology.rdf`, `nt` gives `ontology.nt`. The default is Turtle and JSON-LD. | +| RAG export | `rag.max_chunk_tokens` (default 2000) caps the size of each chunk. | +| Terraform | `terraform.output_dir`, see the note above. | + +Turn off the ontology and the RAG export when you only need findings: + +```bash +cloudg run -p aws --no-ontology --no-rag-export +``` + +`--terraform` has a second effect: when no IaC directory is given by `--iac-dir` or `scanners.iac_directories`, Checkov and the Trivy filesystem scan target the generated Terraform. See [cloudg run](/cli/run/#how-it-works). + +## --json + +`--json` switches a command from tables to machine-readable output on stdout. + +| Command | What `--json` prints | +|---|---| +| `deps` | The dependency tree of one asset, or the overview object | +| `mcp tools` | The `tools/list` definitions | +| `mcp resources` | An object with `resources` and `resourceTemplates` | +| `mcp prompts` | The `prompts/list` definitions | + +`cloudg mcp call` prints JSON without a flag, and `--raw` there gives the whole `CallToolResult`. + +The `mcp` commands send the banner and logs to stderr, so their stdout can be piped straight into `jq`. `deps` does not: the root banner is printed to stdout first. Strip it before parsing: + +```bash +cloudg deps --json | sed -n '/^{/,$p' | jq '.shared_dependencies[:3]' +``` + +## Logs + +Two root options control logging for every command. They go before the subcommand: + +```bash +cloudg -v --log-file ./cloudg.log run -p aws +``` + +`-v/--verbose` switches to debug logging, and `--log-file` also writes every log record to a file with timestamps. The config keys `verbose` and `log_file` do the same; the flag wins for the log file, and either one turns on verbose mode. `cloudg mcp serve` has its own `--log-level`, and `--audit-log` for a JSONL audit trail of tool calls. + +## Related + +:::links +- [Output files](/reference/output-files/) The structure of every file in the output directory. +- [Reports](/guides/reports/) Reading the HTML report and `findings.json`. +- [Graph, ontology and RAG](/guides/graph-ontology-rag/) What the ontology and RAG exports are for. +- [Provider and region flags](/cli/provider-flags/) What gets collected in the first place. +::: diff --git a/site/content/cli/provider-flags.md b/site/content/cli/provider-flags.md new file mode 100644 index 0000000..69fbd2a --- /dev/null +++ b/site/content/cli/provider-flags.md @@ -0,0 +1,137 @@ +--- +title: Provider and region flags +lede: "`-p`, `--region`, `--regions`, `--subscription-id`, `--project-id` and `--profile` decide which clouds, accounts and regions a command touches. They look the same on every command but don't behave the same everywhere." +--- + +## Which command takes which flag + +| Flag | `run` | `map` | `collect` | `scan` | +|---|---|---|---|---| +| `-p/--provider` | required, repeatable, `aws`, `azure`, `gcp` or `all` | repeatable, same choices, default `aws` | required, one of `aws`, `azure`, `gcp` | one free-text value, default `aws` | +| `--region` | AWS only, ignored when `--regions` is set | none | AWS only, default `us-east-1` | none | +| `--regions` | yes | yes | none | none | +| `--subscription-id` | yes | yes | yes | none | +| `--project-id` | yes | yes | yes | none | +| `--profile` | yes | yes | yes | yes | + +`deps`, `ingest`, `report` and the `mcp` commands work on files and take none of these. The MCP live tools read providers and regions from the config passed with `-c`, or from their own arguments. + +On `run` and `map` every flag here writes into the loaded config, and a flag you leave out keeps the config value. `collect` and `scan` don't read `config.yaml` at all, so for them the flags and the environment are all there is. + +## -p, --provider + +On `run` and `map`, repeat the flag to cover several clouds in one run (`-p aws -p gcp`), or pass `all` for all three. Matching is case-insensitive. The result replaces `providers` in the config. Collection for the selected providers runs concurrently. + +`collect` takes exactly one provider and has no `all`. `scan` passes its value unchecked to Prowler and ScoutSuite as their provider argument, so it accepts anything those tools accept. + +## --region and --regions + +`--regions` is the flag to reach for. It takes either a comma-separated list or the word `all`: + +```bash +cloudg map -p aws --regions us-east-1,eu-west-1,ap-southeast-2 +cloudg map -p aws --regions all +``` + +A list replaces the region list of all three providers at once (`aws.regions`, `azure.regions` and `gcp.regions`). That does no harm in a mixed run, because Azure Resource Graph and GCP Cloud Asset Inventory return resources from every location in one call and don't iterate regions. It does mean the run configuration panel shows AWS region names next to Azure. To keep separate lists, set them in `config.yaml` and leave `--regions` out. + +`all` (in any case) becomes the sentinel `["ALL"]`, which starts region discovery when collection begins. You can write `regions: [ALL]` in `config.yaml` for the same effect. + +`--region` is older and narrower. On `run` it sets `aws.regions` to one region, and only when `--regions` is absent. On `collect` it is the single AWS region to collect, defaulting to `us-east-1`, and since it always has a value, `AWS_DEFAULT_REGION` is never consulted. + +Without either flag, `run` and `map` use the config: `aws.regions` defaults to `["us-east-1"]`, while `azure.regions` and `gcp.regions` default to `["ALL"]`. + +### How discovery works + +```mermaid caption="Resolving the AWS region list" +flowchart TD + A["aws.regions after flags"] --> B{"Is it ALL?"} + B -->|no| C["Use the list as given"] + B -->|yes| D{"map --org with Control Tower?"} + D -->|yes| E["Control Tower governed regions"] + D -->|no| F["ec2 DescribeRegions"] + F -->|ok| G["Enabled regions, sorted"] + F -->|"fails"| H["Built-in list of 29 regions"] +``` + +| Provider | API | Filter | Fallback | +|---|---|---|---| +| AWS | `ec2:DescribeRegions`, called in `us-east-1` | `opt-in-status` is `opt-in-not-required` or `opted-in` | 29 built-in regions | +| Azure | `SubscriptionClient.subscriptions.list_locations` for the first subscription | none | 47 built-in locations | +| GCP | Compute Engine `regions.list` for the first project | `status == "UP"` | 40 built-in regions | + +Discovery never stops a run. When the SDK is missing or the call fails, cloudg logs a warning and uses the built-in list for that provider. + +:::warning AWS discovery uses the default session +In `cloudg run` and `cloudg map`, the AWS region lookup builds a plain `boto3.Session()` rather than the session made from your `--profile`, `--aws-*` flags or config credentials. If the default chain has no credentials, or different ones, discovery falls back to the built-in 29 regions. Regions in that list that your account hasn't enabled then show up as failed collectors in the coverage table. Set `AWS_PROFILE` to the same profile, or list the regions explicitly, when you rely on non-default credentials. +::: + +On `cloudg map --org`, when Control Tower is found and the region list is `ALL`, the governed regions of the landing zone replace discovery. Turn that off with `aws.organization.use_governed_regions: false`. + +Account-wide AWS services (IAM, S3, CloudFront, Route 53, Shield and others) are collected once per account in the primary region, so a long region list does not repeat them. + +## --subscription-id + +Sets `azure.subscription_ids` to that one subscription. For several, list them in `config.yaml`: + +```yaml title="config.yaml" +azure: + subscription_ids: + - 00000000-0000-0000-0000-000000000000 + - 11111111-1111-1111-1111-111111111111 +``` + +Without it, `run` and `map` collect every subscription the credential can see in the Enabled state, as long as `azure.all_subscriptions` is true (the default). Set that to false to take only the first one. `collect` needs a subscription: the flag, or `AZURE_SUBSCRIPTION_ID` in the environment. + +## --project-id + +Sets `gcp.project_ids` to that one project. Without it, cloudg uses the project of the application default credentials. On `collect`, `GOOGLE_CLOUD_PROJECT` is checked before that. + +To cover a whole GCP organization in one Cloud Asset Inventory listing, set `gcp.organization_id` in the config; `gcp.project_ids` then filters that listing instead of driving it. + +## --profile + +An AWS CLI profile name, SSO profiles included. On `run` and `map` it sets `aws.profile`, which sits third in the AWS credential order: direct keys and OIDC federation win over it. See [Auth flags](/cli/auth-flags/). + +`run` and `scan` also hand the profile to the scanners: Prowler gets `-p ` and ScoutSuite gets `--profile `, for AWS only. On `collect`, `--profile` falls back to `AWS_PROFILE`, and then to the default chain. + +## Examples + +Two AWS regions and one GCP project in one run: + +```bash +cloudg run -p aws -p gcp --regions us-east-1,eu-west-1 --project-id my-project +``` + +Every enabled AWS region, with discovery and collection using the same profile: + +```bash +AWS_PROFILE=audit cloudg map -p aws --profile audit --regions all +``` + +All three clouds with discovery everywhere: + +```bash +cloudg map -p all --regions all +``` + +One Azure subscription, mapped on its own: + +```bash +cloudg map -p azure --subscription-id 00000000-0000-0000-0000-000000000000 +``` + +A quick single-region credential check: + +```bash +cloudg collect -p aws --profile audit --region ap-southeast-2 +``` + +## Related + +:::links +- [Auth flags](/cli/auth-flags/) Credentials for each provider and which method wins. +- [Output flags](/cli/output-flags/) Where results go. +- [cloudg map](/cli/map/) Organization-wide AWS mapping with `--org`. +- [Provider settings](/reference/config/providers/) The matching `config.yaml` keys. +::: diff --git a/site/content/cli/report.md b/site/content/cli/report.md new file mode 100644 index 0000000..6fcde64 --- /dev/null +++ b/site/content/cli/report.md @@ -0,0 +1,126 @@ +--- +command: report +lede: Re-renders the HTML, JSON and SVG reports from a `findings.json` you already have. No cloud access and no scanners. +intro: | + Use `cloudg report` after upgrading cloudg to get the newer HTML template, or to rebuild one format you skipped. Read the notes on what does not survive the round trip before you point it at your only copy of a report. +--- + +## How it works + +`-i/--input` names a `findings.json` written by `cloudg run`, `cloudg ingest` or `CloudGEngine`. cloudg reads three keys from it: + +| Key | Used for | +|---|---| +| `assets` | Validated back into `CloudAsset` objects | +| `findings` | Validated back into `Finding` objects | +| `graph` | The D3 `{nodes, links}` block, passed to the JSON and HTML renderers as is | + +From the assets and findings it builds a fresh `ScanResult` and renders the formats `--format` names: `json`, `svg`, `html`, or `all` (the default). The output goes to `-o/--output`, `./reports` unless you say otherwise. + +Nothing is normalised again and the config file plays no part. + +### What is lost on the way + +The `compliance` list and the edges in `findings.json` are not read back. The re-rendered files therefore differ from the originals in a few ways. This comparison is from a report built on the synthetic estate in the test suite: + +| Field | Original `findings.json` | After `cloudg report` | +|---|---|---| +| `compliance` | 8 entries | empty | +| `summary.compliance_frameworks` | 5 frameworks | empty | +| `summary.total_edges` | 56 | 0 | +| `metadata.scan_id` | original ID | a new ID | +| `assets`, `findings`, `graph` | as written | unchanged | + +In practice the HTML report keeps its topology (it uses the `graph` block) and its findings table, but its compliance matrix is empty. `topology.svg` is drawn from the edges, which are not loaded, so it shows the assets without connections. + +:::warning Don't overwrite the source +`-i ./reports/findings.json` with the default `-o ./reports` replaces the input with the reduced version above. Write to another directory: + +```bash +cloudg report -i ./reports/findings.json -o ./reports/rerendered +``` +::: + +`report` only accepts the report shape, an object with those keys. A `raw-findings.json` is a bare JSON list and makes the command fail with `AttributeError: 'list' object has no attribute 'get'`. To build reports from raw findings, pass the scanner files to [cloudg ingest](/cli/ingest/) instead. + +A successful run prints one line per file: + +```console +$ cloudg report -i ./reports/findings.json -o rep +─────────────────────────── Report Generation ─────────────────────────── + ✓ JSON: rep/findings.json + ✓ SVG: rep/topology.svg + ✓ HTML: rep/report.html +``` + +(Log lines are left out.) + +## Examples + +Rebuild only the HTML report, for example after upgrading cloudg: + +```bash +cloudg report -i ./reports/findings.json --format html -o ./reports/html +``` + +Rebuild every format into a new folder: + +```bash +cloudg report -i ./reports/2026-10-01/findings.json -o ./reports/2026-10-01-rerendered +``` + +Draw a topology picture of an inventory map. `inventory-map.json` has an `assets` key, so `report` accepts it; the picture has no edges for the reason above: + +```bash +cloudg report -i ./reports/inventory-map.json --format svg -o ./reports/svg +``` + +Render from Python instead when you want to keep the compliance results: + +```bash tab="CLI" +cloudg report -i ./reports/findings.json --format html -o ./reports/html +``` + +```python tab="Python" title="rerender.py" +import json +from pathlib import Path + +from cloudg.renderers.html_report import HTMLReportGenerator +from cloudg.schema.models import CloudAsset, ComplianceResult, Finding, ScanResult + +data = json.loads(Path("./reports/findings.json").read_text()) +scan_result = ScanResult( + assets=[CloudAsset.model_validate(a) for a in data.get("assets", [])], + findings=[Finding.model_validate(f) for f in data.get("findings", [])], + compliance=[ComplianceResult.model_validate(c) for c in data.get("compliance", [])], +) +out = HTMLReportGenerator(output_dir="./reports/html").generate( + scan_result, graph_json=data.get("graph") +) +print(out) +``` + +## Output files + +| File | Written when | Contents | +|---|---|---| +| `findings.json` | `--format json` or `all` | The rebuilt result, with the losses listed above | +| `topology.svg` | `--format svg` or `all` | Asset picture without edges | +| `report.html` | `--format html` or `all` | Interactive report; findings and topology intact, compliance matrix empty | + +## Exit codes + +| Code | When | +|---|---| +| 0 | All requested formats were written | +| 1 | The input file does not exist ("Input file not found"), or it is not a report object and the command fails with a traceback | +| 2 | `-i` is missing or `--format` has an unknown value | + +## Related + +:::links +- [Reports](/guides/reports/) What is in the HTML report and how to read it. +- [Output files](/reference/output-files/) The `findings.json` format. +- [cloudg ingest](/cli/ingest/) Build reports from scanner output. +- [Output flags](/cli/output-flags/) `-o` and `--format` across commands. +::: diff --git a/site/content/cli/run.md b/site/content/cli/run.md new file mode 100644 index 0000000..82dbb8a --- /dev/null +++ b/site/content/cli/run.md @@ -0,0 +1,190 @@ +--- +command: run +lede: The whole pipeline in one command. It collects assets, builds the graph, runs the scanners, normalises the findings and writes every report. +intro: | + `cloudg run` is the command most people start with. It is also the only command that takes the full set of credential flags, so it works the same way on a laptop, in CI and on cloud compute. If you only want to know what exists, [cloudg map](/cli/map/) is faster and needs no scanners. +--- + +## How it works + +`cloudg run` builds one `CloudGConfig` and runs a fixed sequence of phases against it. The config comes from the file you pass to the root command (`cloudg -c config.yaml run ...`). Without `-c`, cloudg uses its built-in defaults. It does not look for a `config.yaml` in the current directory on its own. + +On top of that config, the command applies its flags: + +1. `-p/--provider` replaces `providers`. `all` expands to `aws`, `azure` and `gcp`. +2. `--regions` replaces the region list of all three providers at once. `all` becomes the `ALL` sentinel, which triggers region discovery. When `--regions` is absent, `--region` sets `aws.regions` to that single region and leaves Azure and GCP alone. +3. `--subscription-id`, `--project-id` and `--profile` set `azure.subscription_ids`, `gcp.project_ids` and `aws.profile`. +4. Every `--aws-*`, `--azure-*` and `--gcp-*` flag you give overwrites the matching credential field. Flags you leave out keep whatever the config file says. See [Auth flags](/cli/auth-flags/). + +Then the phases run: + +```mermaid caption="Phases of cloudg run" +flowchart TD + A["Config file + flags"] --> B["1: collect assets"] + B --> C["2: graph and reachability"] + C --> D["2c: RAG export"] + C --> E["2d: Terraform recreation"] + C --> F["3: scanners in parallel"] + E -.->|"IaC target if nothing else is set"| F + F --> G["3b: ontology"] + F --> H["4: normalise"] + H --> I["5: JSON, SVG, HTML"] +``` + +Phase 1 is collection. The multi-provider collector walks every configured account, subscription or project, and every region, concurrently. If collection throws, cloudg prints a "Collection failed" panel and carries on with an empty inventory, so the scanners still run. + +In phase 2, assets and edges go into a directed graph. cloudg writes `topology.graphml` and `topology-cytoscape.json`, runs reachability analysis from the internet node, and turns exposed paths into findings. With `graph.compute_attack_paths` on (the default), it also counts lateral movement paths and prints a warning when it finds any. + +Phase 2c, the RAG export, runs when `--rag-export` is on (the default) and `rag.enabled` is true in config. Both have to agree. The chunks are written once here and rewritten silently after the scanners finish, so the final `rag_chunks.jsonl` includes scanner findings. + +Phase 2d, the Terraform recreation, runs when you pass `--terraform` or set `terraform.enabled: true`. Either one is enough. + +Phase 3 runs the scanners. Every enabled scanner gets its own thread. The scanner list comes from `--scanners`, or from `scanners.enabled` in config when the flag is absent. The default config enables all five: `prowler`, `scoutsuite`, `checkov`, `trivy` and `iam`. Prowler and ScoutSuite run once per provider. Checkov runs once per IaC directory. Trivy scans the images from `--images` (or `scanners.trivy_images`), and falls back to a filesystem scan of the IaC directories when there are no images. + +The IaC directory is resolved in this order: `--iac-dir`, then `scanners.iac_directories`, then the Terraform recreation from phase 2d if it produced any `*.tf.json` files. There is no fallback to the current directory. A scan of whatever folder you happened to run cloudg from would report zero findings and look like a clean result, so cloudg warns "nothing to scan" instead. + +The built-in IAM linter runs whenever `iam` is in the scanner list or phase 1 collected any assets. In practice it runs on almost every real run, even with `--scanners prowler`. + +A scanner whose binary is missing logs a warning and returns no findings. A scanner that raises is marked failed in the progress display. Neither stops the run. + +Phase 3b builds the ontology. It runs after the scanners so that findings become part of the RDF graph. It needs `--ontology` (the default) and `ontology.enabled` in config. One file is written per entry in `ontology.export_formats`, which defaults to Turtle and JSON-LD. + +Phase 4 normalises. Reachability, scanner and IAM findings are deduplicated within and across scanners using the rulesets in `rulesets.rules_dir`, rescored, and mapped to compliance frameworks. + +Phase 5 renders. `findings.json`, `topology.svg` and `report.html` are always written, whatever `report.formats` says in config. A summary table and, when collection recorded it, a coverage table close the run. + +The command prints its resolved settings before phase 1. These two panels come from the code that renders them, for the flags shown: + +```console +$ cloudg run -p aws -p azure --regions us-east-1,eu-west-1 --scanners prowler,checkov +╭────────── Run Configuration ──────────╮ +│ Providers aws, azure │ +│ Scanners prowler, checkov │ +│ AWS regions us-east-1, eu-west-1 │ +│ Azure regions us-east-1, eu-west-1 │ +│ Output reports │ +╰───────────────────────────────────────╯ +``` + +```console +$ cloudg run -p aws +╭─────────────────── Run Configuration ───────────────────╮ +│ Providers aws │ +│ Scanners prowler, scoutsuite, checkov, trivy, iam │ +│ AWS regions us-east-1 │ +│ Output reports │ +╰─────────────────────────────────────────────────────────╯ +``` + +The first panel shows a side effect worth knowing: a comma-separated `--regions` list is copied to every provider, so Azure is shown with AWS region names. That is harmless, because Azure Resource Graph and GCP Cloud Asset Inventory return resources from all locations in one call. If you need different lists per provider, set them in `config.yaml` and leave `--regions` off. + +## Examples + +Scan one AWS account in one region with your default credentials, to try cloudg out: + +```bash +cloudg run -p aws --region us-east-1 +``` + +Use a named profile and discover every enabled region: + +```bash +cloudg run -p aws --profile audit --regions all +``` + +Run only Prowler and Checkov, and point Checkov at your Terraform code: + +```bash +cloudg run -p aws --regions us-east-1 --scanners prowler,checkov --iac-dir ./infra/terraform +``` + +Audit the live cloud with Checkov when you have no IaC repository. `--terraform` writes a recreation of what was collected, and the IaC scanners pick it up: + +```bash +cloudg run -p aws --regions us-east-1 --scanners checkov,iam --terraform +``` + +Scan container images with Trivy alongside the cloud scan: + +```bash +cloudg run -p aws --scanners prowler,trivy --images registry.example.com/api:1.4.2,registry.example.com/worker:1.4.2 +``` + +Scan all three clouds at once, with settings for each provider kept in a config file: + +```bash +cloudg -c config.yaml run -p all --regions all -o ./reports/$(date +%F) +``` + +Run in CI with OIDC and no stored keys. Your workflow writes the OIDC token to a file first; pass its path: + +```bash +cloudg run -p aws --regions all \ + --aws-role-arn arn:aws:iam::123456789012:role/cloudg-audit \ + --aws-web-identity-token-file "$RUNNER_TEMP/oidc-token" +``` + +Skip the semantic exports when you only want findings and the HTML report: + +```bash +cloudg run -p aws --no-ontology --no-rag-export +``` + +Drive the pipeline from Python when you want result objects as well as files. `CloudGEngine.run_pipeline_sync()` covers the same ground, though it orders the analysis steps a little differently and writes only the JSON and HTML reports (see [CloudGEngine](/api/cloudgengine/)): + +```bash tab="CLI" +cloudg run -p aws --region us-east-1 --scanners prowler,checkov,iam -o ./reports +``` + +```python tab="Python" title="run_pipeline.py" +from cloudg import CloudGConfig, CloudGEngine + +config = CloudGConfig(providers=["aws"]) +config.aws.regions = ["us-east-1"] +config.scanners.enabled = ["prowler", "checkov", "iam"] + +engine = CloudGEngine(config) +result = engine.run_pipeline_sync(output_dir="./reports") +print(len(result.findings), "findings") +print(result.report_paths) +``` + +## Output files + +All paths are relative to `-o/--output` (default `./reports`), except the Terraform directory. + +| File | Written when | Contents | +|---|---|---| +| `findings.json` | always | Normalised result: metadata, summary, assets, findings, compliance, D3 graph. `cloudg report -i` reads it back. | +| `report.html` | always | Self-contained interactive report | +| `topology.svg` | always | Static topology picture | +| `topology.graphml` | always | The graph for Gephi, yEd or NetworkX | +| `topology-cytoscape.json` | always | The graph in Cytoscape.js format | +| `ontology.ttl`, `ontology.jsonld` | `--ontology` and `ontology.enabled` | RDF graph, one file per `ontology.export_formats` entry (`xml` gives `ontology.rdf`, `nt` gives `ontology.nt`) | +| `rag_chunks.jsonl`, `rag_metadata_index.json` | `--rag-export` and `rag.enabled` | Retrieval chunks, one JSON object per line, and their index | +| `prowler//` | Prowler enabled | Prowler's own ASFF output | +| `scoutsuite//` | ScoutSuite enabled | ScoutSuite's own report | +| `provider.tf.json`, `variables.tf.json`, `main.tf.json`, `import_commands.sh` | `--terraform` or `terraform.enabled` | Terraform recreation, in `terraform.output_dir` | + +:::warning Terraform ignores -o +The Terraform directory is `terraform.output_dir`, which defaults to `./reports/terraform`. It only falls back to `/terraform` when that key is set to an empty string. So `cloudg run -o /tmp/scan --terraform` still writes Terraform files to `./reports/terraform`. Set `terraform.output_dir` in your config if you change `-o`. +::: + +`cloudg run` does not write `raw-findings.json`. Use [cloudg scan](/cli/scan/) or [cloudg ingest](/cli/ingest/) when you need the pre-normalisation findings. + +## Exit codes + +`cloudg run` does not set an exit code of its own. It exits 0 when it reaches the end, even if collection failed or every scanner was skipped. Click returns 2 for a bad flag value (for example `-p aws2`), and an unhandled exception, such as a `config.yaml` that fails validation, exits 1 with a traceback. + +To fail a CI job on findings, read `findings.json` after the run. [CI and automation](/guides/ci-automation/) has a severity gate. + +## Related + +:::links +- [The pipeline](/guides/pipeline/) What each phase does and why it runs in that order. +- [Running scanners](/guides/running-scanners/) Installing Prowler, ScoutSuite, Checkov and Trivy. +- [Auth flags](/cli/auth-flags/) Every credential flag and which one wins. +- [Provider and region flags](/cli/provider-flags/) `-p`, `--region`, `--regions` and discovery. +- [Output flags](/cli/output-flags/) The output directory and what lands in it. +- [cloudg map](/cli/map/) Inventory only, no scanners. +::: diff --git a/site/content/cli/scan.md b/site/content/cli/scan.md new file mode 100644 index 0000000..7cf7e70 --- /dev/null +++ b/site/content/cli/scan.md @@ -0,0 +1,102 @@ +--- +command: scan +lede: Runs the external security scanners on their own and saves what they found, without collecting assets or normalising. +intro: | + `cloudg scan` is the scanner phase of `cloudg run` cut loose from the rest. It is handy for checking that your scanner binaries work, or for producing a `raw-findings.json` to merge into a map later. For a complete assessment with deduplication and reports, use [cloudg run](/cli/run/). +--- + +## How it works + +The command reads its flags and nothing else. Unlike `run` and `map`, it does not use the config loaded with `cloudg -c`, so `scanners.enabled`, the `*_extra_args` lists, `checkov_frameworks`, `iac_directories` and `trivy_images` have no effect here. Pass everything on the command line. + +`--scanners` is a comma-separated list. It defaults to `prowler,checkov`. Each scanner you name becomes one job, and all jobs run in parallel threads with a progress display: + +| Scanner | What it runs | +|---|---| +| `prowler` | `prowler -M json-asff -o /prowler`, with `-p ` when the provider is `aws` | +| `scoutsuite` | `scout --report-dir /scoutsuite --no-browser`, with `--profile` for `aws` | +| `checkov` | Checkov against `--iac-dir` | +| `trivy` | `trivy image` for each entry in `--images`; with no images, a filesystem scan of `--iac-dir` | +| `iam` | Nothing. The IAM linter needs collected assets, which `scan` does not have, so the name is accepted and ignored. | + +`-p/--provider` is passed straight to Prowler and ScoutSuite as their provider argument. It is free text (default `aws`) and takes one value, unlike the repeatable, validated `-p` on `run` and `map`. + +`--iac-dir` defaults to the current directory. That is a difference from `cloudg run`, which refuses to fall back to `.` because a scan of an unrelated folder looks like a clean result. With `scan`, make sure you run it from your IaC repository or pass `--iac-dir`. + +A scanner that is not installed logs a warning such as "Checkov is not installed. Install with: pip install checkov. Skipping Checkov scan." and reports 0 findings. A scanner that raises is shown as failed. In both cases the other scanners carry on, and the command still writes its output file. + +Here is a run on a machine where neither Checkov nor Trivy is installed: + +```console +$ cloudg scan --scanners checkov,trivy,iam --iac-dir ./iac -o out +╭────── Scan Configuration ───────╮ +│ Provider aws │ +│ Scanners checkov, trivy, iam │ +╰─────────────────────────────────╯ + ⚠ Trivy: no images specified, falling back to filesystem scan + ✓ Checkov: 0 findings 0:00:00 + ✓ Trivy (filesystem): 0 findings 0:00:00 + + ✓ Total: 0 findings + ✓ Raw findings: out/raw-findings.json +``` + +The "not installed" warnings are log lines and are left out above. Zero findings next to a green tick does not mean the code is clean; check the log output, or run `checkov --version` and `trivy --version` first. + +## Examples + +Run the defaults, Prowler and Checkov, from the root of your Terraform repository: + +```bash +cloudg scan -p aws --profile audit +``` + +Check Terraform code in another folder with Checkov only: + +```bash +cloudg scan --scanners checkov --iac-dir ./infra/terraform -o ./reports +``` + +Scan two container images with Trivy: + +```bash +cloudg scan --scanners trivy --images registry.example.com/api:1.4.2,registry.example.com/worker:1.4.2 +``` + +Run Prowler against a GCP project with your application default credentials: + +```bash +cloudg scan -p gcp --scanners prowler -o ./reports/gcp +``` + +Prowler and ScoutSuite want an explicit auth argument for Azure (`--az-cli-auth`, `--cli` and similar). `scan` cannot pass extra arguments, so for Azure use `cloudg run` with `scanners.prowler_extra_args` and `scanners.scoutsuite_extra_args` in your config. + +Scan, then overlay the raw findings on an inventory map: + +```bash +cloudg scan -p aws --profile audit --scanners prowler -o ./reports +cloudg map -p aws --profile audit --regions all --findings ./reports/raw-findings.json -o ./reports +``` + +## Output files + +| File | Contents | +|---|---| +| `raw-findings.json` | All findings from all scanners as a JSON list, before deduplication or compliance mapping | +| `prowler/` | Prowler's native ASFF output, when Prowler ran | +| `scoutsuite/` | ScoutSuite's native report, when ScoutSuite ran | + +`scan` writes no `findings.json` and no HTML report. To get them from these results, feed the native files to [cloudg ingest](/cli/ingest/), for example `cloudg ingest --prowler ./reports/prowler/`. + +## Exit codes + +`scan` exits 0 once `raw-findings.json` is written, whether or not any scanner worked. Click returns 2 for a malformed option. If you need a failing exit status when a scanner is missing, check for the binaries in your script before calling cloudg. + +## Related + +:::links +- [Running scanners](/guides/running-scanners/) Install and configure Prowler, ScoutSuite, Checkov and Trivy. +- [cloudg run](/cli/run/) Scanners plus collection, graph, normalisation and reports. +- [cloudg ingest](/cli/ingest/) Normalise and report on scanner output. +- [Provider and region flags](/cli/provider-flags/) How `-p` differs between commands. +::: diff --git a/site/content/guides/authentication.md b/site/content/guides/authentication.md new file mode 100644 index 0000000..382cc87 --- /dev/null +++ b/site/content/guides/authentication.md @@ -0,0 +1,329 @@ +--- +title: Authentication +lede: "Every provider has several ways in, tried in a fixed order, so one `config.yaml` works on a laptop, in CI and on cloud compute." +meta: + - [Code, "`cloudg/credentials.py`"] + - [Providers, "AWS, Azure, GCP"] + - [Flags, "[Auth flags](/cli/auth-flags/)"] +source: cloudg/credentials.py +since: "0.6.0" +--- + +All credential handling lives in three functions in `cloudg/credentials.py`: `build_aws_session`, `build_azure_credential` and `build_gcp_credentials`. `cloudg run`, `cloudg map`, `CloudGEngine`, `InventoryMapper` and the credential preflight of the MCP live tools all call them, so the precedence on this page holds everywhere. Each function walks its list from the top and stops at the first method whose settings are complete. It never tries the next method after a failure, which matters when you debug: if direct keys are set but wrong, cloudg will not fall back to your profile. + +You can supply each setting three ways. CLI flags (on `cloudg run`) are copied onto the loaded config before anything resolves, `config.yaml` holds the same keys under `aws`, `azure` and `gcp`, and a handful of environment variables are read directly. The tabs below show all three where they exist. + +:::note Which commands take which flags +`cloudg run` has the full set of credential flags listed under [auth flags](/cli/auth-flags/). `cloudg map` has only `--profile`, `--accounts`, `--role-name` and `--org-role`; put everything else in `config.yaml` and pass it with `cloudg -c config.yaml map ...`. `cloudg collect` is the exception: it builds credentials from `--profile` (or `AWS_PROFILE`) and the environment only, and ignores the auth keys in `config.yaml`. +::: + +## AWS + +```mermaid caption="How build_aws_session picks the base credentials, then the optional AssumeRole hop" +flowchart TD + A["access_key_id and secret_access_key set?"] -->|yes| K[Direct keys] + A -->|no| B["role_arn and a web identity token file?"] + B -->|yes| W[AssumeRoleWithWebIdentity] + B -->|no| C["profile set?"] + C -->|yes| P["Named profile, SSO included"] + C -->|no| D["boto3 default chain"] + K --> R{"role_arn, or account plus role_name?"} + P --> R + D --> R + W --> R + R -->|yes| S["sts:AssumeRole, optional ExternalId"] + R -->|no| Done[Session ready] + S --> Done +``` + +| Order | Method | Settings that select it | +|---|---|---| +| 1 | Direct access keys | `access_key_id` and `secret_access_key`, plus `session_token` for temporary keys | +| 2 | OIDC web identity | `role_arn` together with `web_identity_token_file` or `$AWS_WEB_IDENTITY_TOKEN_FILE` | +| 3 | Named profile | `profile` (plain or SSO) | +| 4 | Default provider chain | nothing set: env vars, shared config, container credentials, instance role | +| then | STS AssumeRole | `role_arn` (when not already used by step 2), or `accounts` plus `role_name` | + +Whatever the base method, the session is instrumented with cloudg's rate limiter before it is used, so STS, Organizations and every collector call are paced. See [rate limits](/guides/rate-limits/). + +### Direct keys and session tokens + +The third-party auditor case: someone hands you an access key pair, or a temporary key pair with a session token. Keys win over everything else, including a profile set in the same config. + +```bash tab="CLI" +cloudg run -p aws --regions us-east-1 \ + --aws-key AKIAIOSFODNN7EXAMPLE \ + --aws-secret "$AUDIT_SECRET" \ + --aws-session-token "$AUDIT_SESSION_TOKEN" +``` + +```yaml tab="config.yaml" title="config.yaml" +aws: + regions: [us-east-1] + access_key_id: AKIAIOSFODNN7EXAMPLE + secret_access_key: REPLACE_ME # keep this file out of version control + session_token: null # only for temporary credentials +``` + +```bash tab="Environment" +export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE +export AWS_SECRET_ACCESS_KEY="$AUDIT_SECRET" +export AWS_SESSION_TOKEN="$AUDIT_SESSION_TOKEN" +cloudg run -p aws --regions us-east-1 +``` + +The environment tab works through a different door. cloudg does not copy `AWS_ACCESS_KEY_ID` into its config, so resolution falls through to step 4 and boto3's default chain picks the variables up. The effect is the same, and it is the variant to prefer: nothing secret lands in a file or in your shell history. + +### OIDC web identity federation + +For GitHub Actions, GitLab CI and EKS service accounts: no long-lived keys, only a short-lived token file exchanged for role credentials with `AssumeRoleWithWebIdentity`. cloudg reads the token file itself and names the session after `role_session_name` (default `cloudg-scan`). + +```bash tab="CLI" +cloudg run -p aws --regions all \ + --aws-role-arn arn:aws:iam::123456789012:role/cloudg-readonly \ + --aws-web-identity-token-file /var/run/secrets/oidc/token +``` + +```yaml tab="config.yaml" title="config.yaml" +aws: + role_arn: arn:aws:iam::123456789012:role/cloudg-readonly + web_identity_token_file: /var/run/secrets/oidc/token + role_session_name: cloudg-ci +``` + +```bash tab="Environment" +# Read by boto3's default chain (step 4), not by cloudg's step 2 +export AWS_ROLE_ARN=arn:aws:iam::123456789012:role/cloudg-readonly +export AWS_WEB_IDENTITY_TOKEN_FILE=/var/run/secrets/oidc/token +cloudg run -p aws --regions all +``` + +When step 2 succeeds, `role_arn` has been used up as the OIDC target and no second AssumeRole follows. A cross-account hop after federation needs `accounts` plus `role_name` instead. + +:::warning The token file variable on EKS +cloudg also reads `$AWS_WEB_IDENTITY_TOKEN_FILE` from the environment, and EKS (IRSA) sets it in every pod. On such a pod, a `role_arn` meant as a second hop becomes the OIDC target instead, and the exchange fails unless that role trusts the cluster's OIDC provider. Either let the pod's own role do the work (no `role_arn`), or reach other accounts with `accounts` plus `role_name`. +::: + +A failed exchange raises `RuntimeError: AssumeRoleWithWebIdentity failed for : ...`. The usual causes are a trust policy whose `sub` condition does not match the branch or environment, or a token minted for the wrong audience (AWS expects `sts.amazonaws.com`). + +### Profiles and SSO + +A named profile from `~/.aws/config`, plain or SSO. Log in first; cloudg uses the cached SSO token the AWS CLI writes. + +```bash tab="CLI" +aws sso login --profile prod-audit +cloudg map -p aws --profile prod-audit --regions all +``` + +```yaml tab="config.yaml" title="config.yaml" +aws: + profile: prod-audit +``` + +```bash tab="Environment" +export AWS_PROFILE=prod-audit +cloudg map -p aws --regions all +``` + +An expired SSO session shows up as a `TokenRetrievalError` or `UnauthorizedSSOTokenError` from botocore on the first call. Run `aws sso login` again. + +### Instance roles and the default chain + +With nothing configured, boto3 resolves credentials the usual way: environment variables, the shared credentials and config files, ECS and EKS container credentials, then the EC2 instance metadata service. This is the mode for cloudg running on EC2, ECS or Lambda under its own role, and the mode the CI examples in [CI and automation](/guides/ci-automation/) rely on. Nothing to pass: + +```bash +cloudg run -p aws --regions all +``` + +### Role assumption and external IDs + +STS `AssumeRole` sits on top of whichever base method won. It runs in two situations: + +- `role_arn` is set and step 2 did not consume it. This is a single-account hop, for example from your own account into a customer's audit role. +- cloudg is collecting a specific account and `role_name` is set. It builds `arn:aws:iam:::role/` for each account in `accounts`. + +`external_id` is passed as `ExternalId` when set, which is the standard confused-deputy protection for third-party auditors. Sessions last 3600 seconds (fixed in the code) and use `role_session_name`. + +```bash tab="CLI" +cloudg run -p aws --regions eu-west-1 \ + --profile auditor \ + --aws-role-arn arn:aws:iam::210987654321:role/ThirdPartyAudit \ + --aws-external-id 7f2c1a9e-audit-2026 +``` + +```yaml tab="config.yaml" title="config.yaml" +aws: + profile: auditor + role_arn: arn:aws:iam::210987654321:role/ThirdPartyAudit + external_id: 7f2c1a9e-audit-2026 + role_session_name: acme-audit +``` + +For fan-out across accounts, list them and name the role: + +```yaml title="config.yaml" +aws: + regions: [us-east-1, eu-west-1] + accounts: ["111111111111", "222222222222", "333333333333"] + role_name: cloudg-readonly +``` + +When `accounts` and `role_name` are set and `role_arn` is not, cloudg first calls `GetCallerIdentity` to learn its own account. If that account is in the list it is collected with the base credentials, because roles such as `AWSControlTowerExecution` do not exist in the management account. For whole organizations, `cloudg map --org` discovers the account list for you; see [AWS Organizations](/guides/aws-organizations/). + +A failed AssumeRole does not stop the run. The account and region get a coverage record for `sts_assume_role` with status FAILED and the STS error, and the other accounts carry on. Check the coverage table at the end of `cloudg run`, or `coverage` in the result, before trusting a quiet map. + +### What the scanners receive + +The collectors use the resolved session. Prowler and ScoutSuite are separate processes with their own credential handling, and they get less. Prowler runs with a copy of cloudg's environment plus `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_DEFAULT_REGION` from the config, and `-p ` when `--profile` was given. ScoutSuite gets `--profile`. Neither receives `session_token`, the assumed role, or the OIDC exchange. + +:::tip Give the scanners the same identity +When the scanners should run as the same role as the collectors, put the credentials in the environment (the standard `AWS_*` variables, or `AWS_ROLE_ARN` with `AWS_WEB_IDENTITY_TOKEN_FILE`) rather than in cloudg flags. Every process in the job then resolves the same identity. `aws-actions/configure-aws-credentials` does exactly that in GitHub Actions. +::: + +## Azure + +`build_azure_credential` returns one `azure.identity` credential for the whole run. Steps 1 to 3 all need a tenant and a client ID, taken from the config or from `AZURE_TENANT_ID` and `AZURE_CLIENT_ID`. + +| Order | Method | Settings that select it | Credential class | +|---|---|---|---| +| 1 | Workload identity federation | tenant + client + `federated_token_file` or `$AZURE_FEDERATED_TOKEN_FILE` | `WorkloadIdentityCredential` | +| 2 | Service principal secret | tenant + client + `client_secret` or `$AZURE_CLIENT_SECRET` | `ClientSecretCredential` | +| 3 | Service principal certificate | tenant + client + `certificate_path` (config or flag only) | `CertificateCredential` | +| 4 | Managed identity | `use_managed_identity: true`, optional `managed_identity_client_id` | `ManagedIdentityCredential` | +| 5 | Default chain | nothing above | `DefaultAzureCredential` | + +The Azure CLI is not a numbered step. `az login` works through step 5, because `DefaultAzureCredential` tries environment credentials, workload identity, managed identity and then the Azure CLI. On a laptop that is the simplest path: + +```bash tab="CLI" +az login +cloudg map -p azure +``` + +```yaml tab="config.yaml" title="config.yaml" +providers: [azure] +azure: + subscription_ids: [] # empty: every Enabled subscription you can see +``` + +```bash tab="Environment" +az login +export AZURE_SUBSCRIPTION_ID=00000000-0000-0000-0000-000000000000 +cloudg collect -p azure # collect reads the subscription from the env var +``` + +Service principals and federation: + +```bash tab="CLI" +# Secret +cloudg run -p azure \ + --azure-tenant-id "$TENANT" --azure-client-id "$APP_ID" \ + --azure-client-secret "$SP_SECRET" +# Certificate +cloudg run -p azure \ + --azure-tenant-id "$TENANT" --azure-client-id "$APP_ID" \ + --azure-cert-path ./cloudg-sp.pem +# Federated token (AKS workload identity, GitHub OIDC) +cloudg run -p azure \ + --azure-tenant-id "$TENANT" --azure-client-id "$APP_ID" \ + --azure-federated-token-file /var/run/secrets/azure/tokens/azure-identity-token +# Managed identity of the VM, App Service or AKS node +cloudg run -p azure --azure-managed-identity +``` + +```yaml tab="config.yaml" title="config.yaml" +azure: + tenant_id: 11111111-2222-3333-4444-555555555555 + client_id: 66666666-7777-8888-9999-000000000000 + client_secret: null # 2. secret + certificate_path: null # 3. PEM or PKCS12 + federated_token_file: null # 1. wins over 2 and 3 + use_managed_identity: false # 4. + managed_identity_client_id: null # for a user-assigned identity +``` + +```bash tab="Environment" +export AZURE_TENANT_ID=11111111-2222-3333-4444-555555555555 +export AZURE_CLIENT_ID=66666666-7777-8888-9999-000000000000 +export AZURE_CLIENT_SECRET="$SP_SECRET" # or: +export AZURE_FEDERATED_TOKEN_FILE=/path/to/token # takes precedence over the secret +cloudg run -p azure +``` + +Two interactions catch people out. AKS workload identity injects `AZURE_CLIENT_ID`, `AZURE_TENANT_ID` and `AZURE_FEDERATED_TOKEN_FILE` into the pod, so step 1 fires even when you set `use_managed_identity: true`. And a stale `AZURE_CLIENT_SECRET` in a shell silently beats your `az login` session. When the identity is not what you expect, run with `-v`: cloudg logs the method it chose, for example `Azure auth: service principal (client 6666...)`. + +Subscriptions come from `subscription_ids` (or `--subscription-id` on `run` and `map`). When the list is empty and `all_subscriptions` is true, cloudg lists every Enabled subscription the credential can see; with `all_subscriptions: false` it takes only the first. + +## GCP + +| Order | Method | Settings that select it | +|---|---|---| +| 1 | Credentials file | `credentials_file` in config or `--gcp-credentials-file`: a service account key JSON or an `external_account` workload identity federation config | +| 2 | Application Default Credentials | nothing above: `$GOOGLE_APPLICATION_CREDENTIALS`, `gcloud auth application-default login`, or the GCE/GKE metadata server | +| then | Service account impersonation | `impersonate_service_account` or `--gcp-impersonate-sa`, on top of 1 or 2 | + +All three request the `cloud-platform` scope. `GOOGLE_APPLICATION_CREDENTIALS` is handled by step 2 rather than step 1, which makes no practical difference: ADC loads the same file. + +```bash tab="CLI" +gcloud auth application-default login +cloudg map -p gcp --project-id prod-app-123 +# or a key / federation file, with impersonation on top +cloudg run -p gcp --project-id prod-app-123 \ + --gcp-credentials-file ./wif-config.json \ + --gcp-impersonate-sa cloudg-reader@sec-tools.iam.gserviceaccount.com +``` + +```yaml tab="config.yaml" title="config.yaml" +providers: [gcp] +gcp: + project_ids: [prod-app-123, prod-data-456] + organization_id: null # set it for one org-wide listing + credentials_file: null + impersonate_service_account: cloudg-reader@sec-tools.iam.gserviceaccount.com +``` + +```bash tab="Environment" +export GOOGLE_APPLICATION_CREDENTIALS=$HOME/keys/cloudg-reader.json +export GOOGLE_CLOUD_PROJECT=prod-app-123 +cloudg map -p gcp +``` + +Impersonation is the clean way to keep keys off disk: your own ADC identity (or the CI's federated identity) needs only `roles/iam.serviceAccountTokenCreator` on one reader service account, and every API call runs as that account. + +The project list is `project_ids`. When it is empty, cloudg uses the default project the credentials carry (ADC reads it from `GOOGLE_CLOUD_PROJECT` or the gcloud config). With neither, collection fails with `no GCP project to scan: set gcp.project_ids or gcp.organization_id, or use credentials that carry a default project`. With `organization_id` set and `collection_scope: auto`, one Cloud Asset Inventory listing at `organizations/` covers every project, and `project_ids` only filter it. + +## Least-privilege permissions + +cloudg only reads. Grant read-only roles at the widest scope you want mapped, and nothing else. + +| Provider | Grant | Notes | +|---|---|---| +| AWS | `arn:aws:iam::aws:policy/SecurityAudit` and `arn:aws:iam::aws:policy/job-function/ViewOnlyAccess` | Together they cover the describe, list and get-policy calls of the collectors and the Cloud Control sweep | +| AWS, organizations | `organizations:Describe*`, `organizations:List*`, `controltower:List*`, `controltower:Get*` in the management account | Plus a read-only member role in every account, for example deployed with a StackSet. `AWSControlTowerExecution` works but is admin | +| AWS, EKS workloads | An EKS access entry for the role with `AmazonEKSViewPolicy` | Only when `inventory.kubernetes` is on | +| Azure | `Reader` on each subscription, or on a management group to cover everything below it | Management group and Azure Policy mapping needs Reader at the management groups themselves | +| GCP | `roles/cloudasset.viewer` on the organization, folder or project | Covers resource, IAM policy, org policy and access policy listings. The Cloud Asset API must be enabled in the project that makes the calls | +| GCP, impersonation | `roles/iam.serviceAccountTokenCreator` on the target service account | Granted to the caller, not to the target | + +Scanners have their own requirements. Prowler documents extra permissions on top of `SecurityAudit` and `ViewOnlyAccess`, and Prowler and ScoutSuite on Azure also read Microsoft Graph. Check each scanner's documentation before you blame cloudg for an `AccessDenied` in their output. + +:::danger Never point cloudg at admin credentials for convenience +Nothing in cloudg writes to your cloud, but its reports and the RAG and ontology exports describe your estate in detail, and the MCP live tools run with whatever identity the server has. A read-only role keeps a leaked report or a misconfigured agent from becoming a write path. +::: + +## Checking which identity cloudg used + +Run any command with `-v`. Each provider logs the branch it took: + +```console +$ cloudg -v map -p aws --profile prod-audit --regions us-east-1 +INFO AWS auth: profile 'prod-audit' (region us-east-1) +``` + +Other messages you may see are `AWS auth: direct access keys`, `AWS auth: web identity federation into `, `AWS auth: default provider chain`, `AWS auth: assuming role `, `Azure auth: workload identity federation (client ...)`, `Azure auth: DefaultAzureCredential chain`, `GCP auth: file-based identity (...)`, `GCP auth: application default credentials` and `GCP auth: impersonating `. + +:::links +- [Auth flags](/cli/auth-flags/) Every credential flag of `cloudg run`. +- [AWS Organizations](/guides/aws-organizations/) Mapping every account from the management account. +- [CI and automation](/guides/ci-automation/) OIDC from GitHub Actions and GitLab CI. +- [Docker](/guides/docker/) Mounting credentials into the container. +::: diff --git a/site/content/guides/aws-organizations.md b/site/content/guides/aws-organizations.md new file mode 100644 index 0000000..1d9ab98 --- /dev/null +++ b/site/content/guides/aws-organizations.md @@ -0,0 +1,394 @@ +--- +title: AWS Organizations +lede: "With `--org`, `cloudg map` starts at the management account, finds every member account and maps them all, with the OU tree, SCPs and Control Tower around them." +meta: + - [Command, "`cloudg map --org`"] + - [Runs from, "Management account or delegated administrator"] +source: cloudg/inventory/organization.py +--- + +Mapping one account at a time misses what holds an organization together: which OU an account sits in, which SCPs restrict it, which Control Tower controls apply, and which roles in one account trust another. `cloudg map --org` reads the organization first, picks the accounts you asked for, assumes a role in each one, and maps all of them into a single [inventory map](/guides/inventory-mapping/). Accounts hang under their OUs, SCPs and controls point at what they govern, and cross-account trust becomes an edge you can query with [`cloudg deps`](/guides/dependencies/). + +Every call cloudg makes here is read-only: `Describe*`, `List*` and `Get*` against Organizations and Control Tower, then the usual read calls inside each member account. + +## What happens during discovery + +```mermaid caption="Organization discovery, then one role assumption per member account and region" +sequenceDiagram + participant C as cloudg + participant S as STS + participant O as Organizations + participant T as Control Tower + participant M as Member account + C->>S: GetCallerIdentity + C->>O: DescribeOrganization, ListRoots + C->>O: Walk OUs and accounts, list policies and targets + C->>O: Trusted services, delegated administrators + C->>T: ListLandingZones, probing likely home regions + T-->>C: Landing zone, enabled controls and baselines + Note over C: Filter accounts by OU, exclusions, status + loop Each selected account and region + C->>S: AssumeRole into the member role + S-->>C: Temporary credentials + C->>M: Deep collectors (read-only) + end +``` + +In order: + +1. `sts:GetCallerIdentity` records which account is running the discovery, and `organizations:DescribeOrganization` returns the organization ID, the management account and the feature set. If this fails, the caller is not in an organization or can't read it. +2. From each root, cloudg walks down with `ListAccountsForParent` and `ListOrganizationalUnitsForParent`, recording every account's parent and OU path, like `Root/Workloads/Prod`. +3. For each policy type enabled on the root (SCPs, RCPs, tag, backup and AI services opt-out policies), it lists the policies and the roots, OUs and accounts each one is attached to. +4. It lists trusted service access and the delegated administrators of GuardDuty, Security Hub, Inspector, Config, Access Analyzer, Macie, Detective, Firewall Manager and IAM Identity Center. An access denial ends that loop quietly. +5. With Control Tower detection on, it looks for a landing zone (see [Control Tower](#control-tower-and-regions) below). The landing zone manifest is parsed for governed regions and the log archive, audit, Config aggregator and backup accounts, then dropped. +6. The account list is filtered, and collection fans out with one AssumeRole per account and region. + +If discovery fails, the map doesn't stop. cloudg records `organizations: FAILED` in coverage, logs the reason, and maps the caller's account alone. The CLI's failed-collectors warning at the end of the run names it. + +## Permissions for the caller + +Run `cloudg map --org` with credentials for the management account. Organizations discovery also works from a delegated administrator, but the Control Tower APIs only answer in the management account. + +```json title="cloudg-org-discovery-policy.json" +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ReadOrganization", + "Effect": "Allow", + "Action": [ + "organizations:Describe*", + "organizations:List*", + "controltower:List*", + "controltower:Get*" + ], + "Resource": "*" + }, + { + "Sid": "AssumeMemberRole", + "Effect": "Allow", + "Action": "sts:AssumeRole", + "Resource": "arn:aws:iam::*:role/cloudg-readonly" + } + ] +} +``` + +The caller's own account is mapped too, with the caller's credentials and not through a member role (member roles such as `AWSControlTowerExecution` don't exist in the management account). So the caller also needs read access in its own account; attaching `SecurityAudit` and `ViewOnlyAccess` to it is the simplest way. + +## Deploy a read-only member role + +cloudg assumes one role, by name, in every member account. The default is `AWSControlTowerExecution`, because Control Tower creates it in every account it enrolls. + +:::warning AWSControlTowerExecution has admin rights +`AWSControlTowerExecution` carries administrator permissions. cloudg only reads, but anything that can run cloudg with management-account credentials could do more with that role. The read-only role Control Tower creates, `aws-controltower-ReadOnlyExecutionRole`, can't be used instead: it only trusts a role in the audit account. For production, deploy your own read-only role to every account and pass it with `--org-role`. +::: + +A service-managed CloudFormation StackSet deploys the role to every account in the OUs you target, and to accounts that join those OUs later. + +::::steps +### Save the template + +```yaml title="cloudg-readonly-role.yaml" +AWSTemplateFormatVersion: "2010-09-09" +Description: Read-only role that cloudg assumes to map this account + +Parameters: + ManagementAccountId: + Type: String + AllowedPattern: '^[0-9]{12}$' + Description: Account that runs cloudg map --org + +Resources: + CloudgReadOnlyRole: + Type: AWS::IAM::Role + Properties: + RoleName: cloudg-readonly + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + AWS: !Sub "arn:aws:iam::${ManagementAccountId}:root" + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/SecurityAudit + - arn:aws:iam::aws:policy/job-function/ViewOnlyAccess + +Outputs: + RoleArn: + Value: !GetAtt CloudgReadOnlyRole.Arn +``` + +Keep the role at the root path. cloudg builds the ARN as `arn:aws:iam:::role/`, so a role created under a path such as `/audit/` would not be found by its bare name. + +The trust policy above lets any principal in the management account that has `sts:AssumeRole` permission assume the role. To pin it to the one role cloudg runs as, add a `Condition` with `ArnEquals` on `aws:PrincipalArn`. + +### Turn on trusted access for StackSets + +Run this once, in the management account: + +```bash +aws cloudformation activate-organizations-access +``` + +### Create the stack set + +```bash +aws cloudformation create-stack-set \ + --stack-set-name cloudg-readonly \ + --template-body file://cloudg-readonly-role.yaml \ + --parameters ParameterKey=ManagementAccountId,ParameterValue=123456789012 \ + --capabilities CAPABILITY_NAMED_IAM \ + --permission-model SERVICE_MANAGED \ + --auto-deployment Enabled=true,RetainStacksOnAccountRemoval=false +``` + +### Deploy it to your OUs + +IAM roles are global, so deploy to a single region. Targeting the root ID (`r-...`) covers every OU; list OU IDs instead to limit it. + +```bash +aws cloudformation create-stack-instances \ + --stack-set-name cloudg-readonly \ + --deployment-targets OrganizationalUnitIds=r-7h3o \ + --regions us-east-1 +aws cloudformation list-stack-instances --stack-set-name cloudg-readonly +``` + +Service-managed StackSets never deploy to the management account itself. cloudg doesn't need them to, since it maps the management account with the caller's own credentials. +:::: + +To map Kubernetes workloads inside EKS clusters as well, each cluster needs an access entry for `cloudg-readonly` with the `AmazonEKSViewPolicy`. The [inventory mapping guide](/guides/inventory-mapping/#kubernetes-inside-eks) has the two `aws eks` commands. + +## Map the organization + +```bash tab="CLI" +cloudg map -p aws --profile management --org --org-role cloudg-readonly --regions all +``` + +```python tab="Python" title="map_org.py" +from cloudg import CloudGConfig +from cloudg.inventory import InventoryMapper + +config = CloudGConfig(providers=["aws"]) +config.aws.profile = "management" +config.aws.regions = ["ALL"] + +org = config.aws.organization +org.enabled = True +org.role_name = "cloudg-readonly" +org.include_ous = ["Workloads"] +org.exclude_accounts = ["111122223333"] + +mapper = InventoryMapper(config) +inventory = mapper.map_inventory_sync() +paths = inventory.export("./reports") + +topology = mapper.organization # None when discovery failed +if topology is not None: + print(topology.organization_id, len(topology.accounts), "accounts") + print("Control Tower:", topology.control_tower_enabled, topology.governed_regions) +print(paths.get("organization")) +``` + +```bash tab="Docker" +docker run --rm \ + -v ~/.aws:/home/cloudg/.aws:ro \ + -v "$PWD/reports:/app/reports" \ + ghcr.io/morpheuslord/cloudg:latest \ + map -p aws --profile management --org --org-role cloudg-readonly --regions all +``` + +With `--org`, the CLI's configuration panel shows "Organization: discover all accounts" (plus the OU filter, if any) and the member role it will use. After the run an "Organization" table lists the organization ID, the number of accounts and OUs, whether Control Tower was found and the governed regions, ahead of the usual inventory summary. + +## Choose the accounts + +By default every `ACTIVE` account in the organization is mapped, the management account included. These flags narrow it down: + +| Flag | Config key | Effect | +|---|---|---| +| `--ou` | `aws.organization.include_ous` | Only accounts under this OU, nested OUs included. Takes an OU ID, an OU ARN, or a name (case-insensitive). Repeatable. | +| `--exclude-account` | `aws.organization.exclude_accounts` | Skip this account ID. Repeatable. | +| `--accounts` | `aws.accounts` | Comma-separated IDs. With `--org`, the discovered list is intersected with this one. | +| none | `aws.organization.include_management_account` | `true` by default. | +| none | `aws.organization.include_suspended` | `false` by default, so `SUSPENDED` and `PENDING_CLOSURE` accounts are skipped. | + +```bash +# the Workloads OU and everything below it, minus one sandbox +cloudg map -p aws --org --org-role cloudg-readonly --ou Workloads --exclude-account 111122223333 +# two OUs by ID +cloudg map -p aws --org --org-role cloudg-readonly --ou ou-7h3o-ljevwujv --ou ou-7h3o-bqhq9yxz +``` + +OU names are matched against the whole tree and the first match wins. If two OUs share a name (a `Prod` under `Workloads` and another under `Sandbox`), pass the ID. A name that matches no OU is logged as a warning and ignored, and if none of your `--ou` values match, no account is selected and cloudg falls back to mapping the caller's own account. Run with `-v` when the account count looks wrong. + +## Control Tower and regions + +Control Tower detection is on by default (`aws.organization.control_tower`). cloudg calls `controltower:ListLandingZones` region by region until it finds one: first the region given with `--ct-home-region`, then the session's region, then the usual home regions in this order: `us-east-1`, `us-east-2`, `us-west-2`, `eu-west-1`, `eu-central-1`, `eu-west-2`, `ap-southeast-2`, `ap-northeast-1`, `ap-southeast-1`, `ca-central-1`, `ap-south-1`, `eu-north-1`, `sa-east-1`. If your home region isn't on that list, pass it: + +```bash +cloudg map -p aws --org --org-role cloudg-readonly --ct-home-region eu-south-1 --regions all +``` + +`--ct-home-region` also sets the region of the discovery session. Without it, regions where the lookup fails are only logged at debug level. With `--ct-home-region` set, every failed lookup is kept in the topology's `errors` list and coverage shows `controltower: PARTIAL`. + +When a landing zone is found and you passed `--regions all`, cloudg maps only the governed regions instead of every enabled region (`aws.organization.use_governed_regions`, on by default). An explicit region list is always used as given, and so is the default `us-east-1` when you pass no `--regions` at all. + +## What lands on the map + +With `aws.organization.map_structure` on (the default), the topology becomes assets and edges next to the collected resources: + +| Asset | Type | Edges | +|---|---|---| +| The organization | `ORGANIZATION` | the management account `MANAGES` it | +| Each root and OU | `ORG_UNIT` | its parent `CONTAINS` it | +| Each account | `CLOUD_ACCOUNT` | its OU `CONTAINS` it (`ORG_CONTAINS_ACCOUNT`) | +| Each SCP, RCP, tag, backup or AI opt-out policy | `ORG_POLICY` | `GOVERNS` each target; `SCP_RESTRICTS` for SCPs | +| The landing zone | `LANDING_ZONE` | `MANAGES` the organization and each shared account | +| Each enabled control and baseline | `GUARDRAIL` | `GOVERNS` the OU it is enabled on | + +Account nodes use `arn:aws:iam:::root` as their identifier, the same string IAM trust policies and resource policies use for an account principal. So when a role in one account trusts another account of the organization, the `IAM_TRUST` edge starts at that account's node on the map instead of at an external placeholder, and `cloudg deps` lists it among the cross-account edges. Account metadata records whether it is the management account, its Control Tower role (`log_archive`, `audit` and so on) and the services it is delegated administrator for. + +## How member accounts are assumed + +For each selected account and each region, cloudg builds a session from your base credentials (profile, keys, web identity or the default chain) and then calls `sts:AssumeRole` on `arn:aws:iam:::role/`. The role name comes from `--org-role`, else `aws.organization.role_name`, else `--role-name` / `aws.role_name`, else `AWSControlTowerExecution`. The session name is `aws.role_session_name` (`cloudg-scan` by default), the duration one hour, and `aws.external_id` is sent when set. + +An account whose role can't be assumed records `sts_assume_role: FAILED` for that region, and nothing else is collected there. The rest of the organization carries on. + +:::note +`aws.role_arn` takes precedence over the member role when it is set without a web identity token file: every account would then be collected through that single role, which is not what you want for an organization. Leave `aws.role_arn` unset with `--org`, or use it together with `aws.web_identity_token_file` (the GitHub Actions and GitLab CI pattern), where it is the role the pipeline starts from and the member role is assumed on top. +::: + +## inventory-organization.json + +A map made with `--org` writes the whole topology to `inventory-organization.json`, and `InventoryResult.load()` reads it back as `inventory.organization`. Abridged: + +```json title="reports/inventory-organization.json" +{ + "organization_id": "o-1wietecbs4", + "management_account_id": "123456789012", + "caller_account_id": "123456789012", + "feature_set": "ALL", + "ous": { + "ou-7h3o-bqhq9yxz": {"id": "ou-7h3o-bqhq9yxz", "name": "Prod", "parent_id": "ou-7h3o-ljevwujv", + "path": ["Root", "Workloads", "Prod"], "is_root": false} + }, + "accounts": { + "552289857375": {"id": "552289857375", "name": "prod-app", "status": "ACTIVE", + "parent_id": "ou-7h3o-bqhq9yxz", "ou_path": ["Root", "Workloads", "Prod"]} + }, + "policies": [ + {"id": "p-FullAWSAccess", "name": "FullAWSAccess", "type": "SERVICE_CONTROL_POLICY", + "aws_managed": true, "targets": ["r-7h3o"]} + ], + "delegated_administrators": {"guardduty": ["333333333333"]}, + "landing_zone": {"version": "3.3", "status": "ACTIVE", "driftStatus": {"status": "IN_SYNC"}}, + "governed_regions": ["us-east-1", "eu-west-1"], + "shared_accounts": {"log_archive": "222222222222", "audit": "333333333333"}, + "enabled_controls": [], + "control_tower_region": "us-east-1", + "errors": [], + "control_tower_enabled": true +} +``` + +`accounts` holds every account in the organization, before any filter, so the file doubles as an account inventory. A quick listing by OU: + +```python title="accounts_by_ou.py" +import json +from collections import defaultdict + +with open("./reports/inventory-organization.json") as f: + org = json.load(f) + +by_ou = defaultdict(list) +for account in org["accounts"].values(): + by_ou["/".join(account["ou_path"])].append(f'{account["id"]} {account["name"]} {account["status"]}') + +for path in sorted(by_ou): + print(path) + for line in sorted(by_ou[path]): + print(" ", line) +``` + +## Configuration + +The same settings live under `aws.organization` in `config.yaml`, for runs that pass `-c config.yaml`. The flags override them. + +```yaml title="config.yaml" +aws: + profile: management + regions: [ALL] + organization: + enabled: true + role_name: cloudg-readonly # default: aws.role_name, then AWSControlTowerExecution + include_ous: [Workloads] # OU IDs, ARNs or names, nested OUs included + exclude_accounts: ["111122223333"] + include_management_account: true + include_suspended: false + use_governed_regions: true # regions [ALL] -> Control Tower governed regions + control_tower: true # map the landing zone, controls and baselines + home_region: null # Control Tower home region, auto-detected + map_structure: true # org, OU, account, SCP and control nodes on the map +``` + +| Key | Default | Flag | +|---|---|---| +| `enabled` | `false` | `--org / --no-org` | +| `role_name` | `null` | `--org-role` | +| `include_ous` | `[]` | `--ou` | +| `exclude_accounts` | `[]` | `--exclude-account` | +| `include_management_account` | `true` | none | +| `include_suspended` | `false` | none | +| `use_governed_regions` | `true` | none | +| `control_tower` | `true` | none | +| `home_region` | `null` | `--ct-home-region` | +| `map_structure` | `true` | none | + +## Discovery on its own + +`discover_organization()` works without a mapping run. It takes a boto3 session and returns an `OrganizationTopology`: + +```python title="org_topology.py" +import boto3 + +from cloudg.inventory import discover_organization + +session = boto3.Session(profile_name="management", region_name="us-east-1") +topology = discover_organization(session, control_tower=True) + +print(topology.organization_id, "managed by", topology.management_account_id) +print("Control Tower:", topology.control_tower_enabled, topology.governed_regions) + +for account in topology.accounts.values(): + print(account.id, account.name, account.status, "/".join(account.ou_path)) + +for policy in topology.policies: + print(policy.type, policy.name, "->", policy.targets) + +workloads = topology.target_accounts(include_ous=["Workloads"], include_management_account=False) +print("would map:", workloads) + +assets = topology.to_assets() # organization, OUs, accounts, policies, landing zone, controls +``` + +It raises `RuntimeError` when the caller can't describe or list the organization. Other failures, such as a Control Tower call that was denied, end up in `topology.errors`. + +## Troubleshooting + +| What you see | Likely cause | Fix | +|---|---|---| +| Only one account mapped, `organizations` in the failed-collectors line | the credentials aren't from the management account or a delegated administrator, or lack `organizations:List*` | run with management-account credentials | +| `sts_assume_role` failed for some accounts | the member role is missing there, or its trust policy doesn't name the caller's account | check the StackSet instances for those accounts | +| `sts_assume_role` failed for the management account when running from a delegated administrator | the StackSet never deploys to the management account, and `AWSControlTowerExecution` doesn't exist there | `--exclude-account` it, set `include_management_account: false`, or create the role there by hand | +| Control Tower reported as absent | running from a delegated administrator, or the home region isn't in the probe list | run from the management account; pass `--ct-home-region` | +| Far fewer accounts than expected | an `--ou` value matched nothing, or matched a different OU of the same name | use OU IDs; run with `-v` to see the warning | +| Fewer regions than `--regions all` would give | Control Tower governed regions replaced `all` | set `use_governed_regions: false`, or pass the regions explicitly | + +:::links +- [Inventory mapping](/guides/inventory-mapping/) What a mapping run collects in each account. +- [Dependencies and blast radius](/guides/dependencies/) Query cross-account edges on the finished map. +- [Authentication](/guides/authentication/) Profiles, OIDC and role assumption for the base credentials. +- [cloudg map](/cli/map/) Every option of the command. +::: diff --git a/site/content/guides/ci-automation.md b/site/content/guides/ci-automation.md new file mode 100644 index 0000000..9794f47 --- /dev/null +++ b/site/content/guides/ci-automation.md @@ -0,0 +1,400 @@ +--- +title: CI and automation +lede: "Run cloudg on a schedule or on every infrastructure change, with short-lived cloud credentials, a severity gate that fails the job, and the reports kept as build artifacts." +meta: + - [Commands, "`cloudg run`, `cloudg map`, `cloudg ingest`"] + - [CI systems, "GitHub Actions, GitLab CI"] +source: .github/workflows/cloudg.yml +since: "0.6.0" +--- + +There are three useful shapes for cloudg in a pipeline, and most teams end up with two of them: + +:::cells +### Scheduled posture run +`cloudg run` against the live account on a cron, with OIDC credentials, Prowler and the IAM linter. Slow, thorough, read-only. +### Pull request gate +Checkov and Trivy over the IaC in the branch, aggregated with `cloudg ingest`. No cloud access at all, so it is safe for forks. +### Inventory snapshot +`cloudg map` on a cron. No scanners, just the map, kept as an artifact for diffing and for the MCP server. +::: + +Before wiring any of them up, know what cloudg's exit code does and does not tell you. + +## Exit codes + +cloudg exits non-zero when it cannot do its job, never because of what it found. Nothing in the CLI fails on severity. + +| Command | Exits 1 when | Exits 0 even when | +|---|---|---| +| `cloudg run` | an unhandled exception escapes | collection failed for every account, a scanner is missing, the ontology or RAG export failed | +| `cloudg map` | the mapping raised (bad config, no SDK installed) | individual collectors failed or were throttled (recorded in coverage) | +| `cloudg ingest` | no `--prowler`, `--scoutsuite`, `--checkov` or `--trivy` was given | the given paths parsed to zero findings | +| `cloudg report` | the `-i` file does not exist | | +| `cloudg collect` | collection raised | | +| any | click usage errors (unknown flag, bad choice) exit 2 | | + +`cloudg run` is the one to watch. Each phase catches its own errors so the rest of the pipeline still produces something, which is good for a human at a terminal and bad for a CI job: expired credentials give you a green build with `total_assets: 0`. The gate script below checks for that as well as for severity. + +## A severity gate + +Every run writes `findings.json` (see [reports](/guides/reports/)). Its `findings` list holds the normalised, deduplicated findings, each with `severity`, `risk_score`, `source_tool`, `is_suppressed` and the affected resource, and `summary` holds the totals. Save this as `ci/severity_gate.py` in the repository that runs the job: + +```python title="ci/severity_gate.py" +"""Fail a CI job when a cloudg findings.json crosses a severity threshold. + +Usage: + python severity_gate.py reports/findings.json --fail-on HIGH --max 0 --min-assets 1 + +Exit codes: 0 passed, 1 gate failed, 2 the report is missing or unusable. +""" + +import argparse +import json +import sys +from collections import Counter + +ORDER = ["INFO", "LOW", "MEDIUM", "HIGH", "CRITICAL"] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("findings", help="findings.json written by cloudg run or cloudg ingest") + parser.add_argument("--fail-on", default="CRITICAL", choices=ORDER, + help="lowest severity that blocks (default CRITICAL)") + parser.add_argument("--max", type=int, default=0, + help="blocking findings tolerated (default 0)") + parser.add_argument("--min-assets", type=int, default=0, + help="fail when fewer assets were collected (catches silent auth failures)") + args = parser.parse_args() + + try: + with open(args.findings) as fh: + data = json.load(fh) + except (OSError, json.JSONDecodeError) as exc: + print(f"gate: cannot read {args.findings}: {exc}", file=sys.stderr) + return 2 + + assets = data.get("summary", {}).get("total_assets", 0) + if assets < args.min_assets: + print(f"gate: only {assets} asset(s) collected, expected at least {args.min_assets}") + return 2 + + threshold = ORDER.index(args.fail_on) + active = [f for f in data.get("findings", []) if not f.get("is_suppressed")] + blocking = [f for f in active if ORDER.index(f["severity"]) >= threshold] + + print(f"assets: {assets}, findings by severity: {dict(Counter(f['severity'] for f in active))}") + for f in sorted(blocking, key=lambda f: -f.get("risk_score", 0))[:25]: + target = f.get("resource_arn") or f["resource_id"] + print(f" [{f['severity']}] {f['title']} ({f['source_tool']}) {target}") + + if len(blocking) > args.max: + print(f"gate: {len(blocking)} finding(s) at {args.fail_on} or above, {args.max} allowed") + return 1 + print("gate: passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) +``` + +It needs nothing but the standard library, so it runs before or after cloudg is installed. A failing gate prints the worst offenders first: + +```console +$ python ci/severity_gate.py reports/findings.json --fail-on HIGH +assets: 0, findings by severity: {'CRITICAL': 1, 'HIGH': 3} + [CRITICAL] [Trivy] CVE-2024-0001: openssl (alpine) (trivy) myrepo/app:latest + [HIGH] S3 bucket default encryption (prowler) arn:aws:s3:::my-bucket + [HIGH] [Checkov/terraform] Ensure S3 bucket has server-side encryption enabled (checkov) aws_s3_bucket.data + [HIGH] [Trivy/IaC] AVD-AWS-0088: S3 bucket encryption not enabled (trivy) ./iac +gate: 4 finding(s) at HIGH or above, 0 allowed +``` + +Use `--min-assets 1` (or a realistic floor for your account) on live runs, and leave it at 0 for ingest-only jobs, which never collect assets. If you only need the yes or no, `jq` does it in one line: + +```bash +jq -e '[.findings[] | select(.severity == "CRITICAL" and (.is_suppressed | not))] | length == 0' reports/findings.json +``` + +When the gate should run inside Python, for example in a job that already uses the engine, see the gate recipe in [Python recipes](/guides/python-recipes/). + +## GitHub Actions with OIDC to AWS + +No access keys in repository secrets. GitHub mints an OIDC token per job, AWS exchanges it for a role session, and the role only trusts this repository. + +::::steps +### Create the role + +In the target account, add GitHub as an identity provider (`token.actions.githubusercontent.com`, audience `sts.amazonaws.com`) once, then create a role with `SecurityAudit` and `ViewOnlyAccess` attached and this trust policy. Replace the account ID and repository: + +```json title="cloudg-github-trust.json" +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::123456789012:oidc-provider/token.actions.githubusercontent.com" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "token.actions.githubusercontent.com:aud": "sts.amazonaws.com", + "token.actions.githubusercontent.com:sub": "repo:acme/cloud-posture:ref:refs/heads/main" + } + } + } + ] +} +``` + +:::warning Scope the subject +A `sub` of `repo:acme/cloud-posture:*` lets every branch and every pull request from a branch assume the role. Pin it to `main` or to a deployment environment (`repo:acme/cloud-posture:environment:cloud-audit`). +::: +### Add the config and the gate + +Commit `ci/severity_gate.py` from above and a config next to it: + +```yaml title="ci/cloudg.yaml" +providers: [aws] +aws: + regions: [us-east-1, eu-west-1] +scanners: + enabled: [prowler, iam] + timeout_seconds: 3600 # per scanner +``` +### Add the workflow + +```yaml title=".github/workflows/cloud-posture.yml" +name: Cloud posture + +on: + schedule: + - cron: "0 5 * * 1-5" # weekdays 05:00 UTC + workflow_dispatch: + +permissions: + contents: read + id-token: write # lets the job request an OIDC token + +concurrency: + group: cloud-posture + cancel-in-progress: false + +jobs: + posture: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Cache pip and the Trivy database + uses: actions/cache@v6 + with: + path: | + ~/.cache/pip + ~/.cache/trivy + key: cloudg-0.6.0-${{ runner.os }} + + - name: Install cloudg and scanners + run: | + pip install "cloudg[aws]==0.6.0" parliament + pipx install prowler + + - name: Assume the audit role + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: arn:aws:iam::123456789012:role/cloudg-github + role-session-name: cloudg-${{ github.run_id }} + aws-region: us-east-1 + + - name: Run cloudg + run: cloudg -c ci/cloudg.yaml run -p aws -o reports + + - name: Gate on severity + run: python ci/severity_gate.py reports/findings.json --fail-on CRITICAL --min-assets 1 + + - name: Job summary + if: always() + run: | + python - <<'EOF' >> "$GITHUB_STEP_SUMMARY" + import json + s = json.load(open("reports/findings.json"))["summary"] + print(f"### cloudg: {s['total_findings']} findings on {s['total_assets']} assets") + for sev, n in sorted(s["severity_breakdown"].items()): + print(f"- {sev}: {n}") + EOF + + - name: Upload reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: cloudg-${{ github.run_id }} + path: reports/ + retention-days: 30 +``` +:::: + +Why credentials go through `configure-aws-credentials` and not cloudg's own `--aws-role-arn` with `--aws-web-identity-token-file`: the action exports `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` for the whole job. cloudg's collectors pick them up through the default chain, and so does Prowler, which runs as a separate process and does not inherit an exchange cloudg did internally. [Authentication](/guides/authentication/#what-the-scanners-receive) has the details. For a `cloudg map` job with no scanners, cloudg's native OIDC path works equally well. + +The action's session lasts one hour by default. A long organization-wide run can outlive it; raise `role-duration-seconds` (and the role's maximum session duration) or split the run per account. + +`if: always()` on the upload matters: the gate fails the job, and the reports are exactly what you want to look at when it does. + +### Pull requests without cloud access + +Infrastructure changes can be gated before they reach an account. Checkov and Trivy scan the branch, cloudg aggregates and deduplicates, the gate decides. No OIDC permission is needed, so this job is safe on forks. + +```yaml title=".github/workflows/iac-gate.yml" +name: IaC gate + +on: + pull_request: + paths: ["infra/**"] + +permissions: + contents: read + +jobs: + iac: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Install tools + run: | + pip install "cloudg==0.6.0" + pipx install checkov + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin + + - name: Scan + run: | + checkov -d infra --output json > checkov.json || true + trivy fs --format json --scanners vuln,misconfig,secret --output trivy-fs.json infra + + - name: Aggregate + run: cloudg ingest --checkov checkov.json --trivy trivy-fs.json -o reports + + - name: Gate + run: python ci/severity_gate.py reports/findings.json --fail-on HIGH + + - uses: actions/upload-artifact@v7 + if: always() + with: + name: iac-gate-${{ github.event.pull_request.number }} + path: reports/ +``` + +`|| true` after Checkov is deliberate: Checkov exits 1 when any check fails, and the decision belongs to the gate, which sees the deduplicated result. The core `cloudg` package is enough here, since ingest needs no cloud SDK. + +## GitLab CI + +GitLab issues OIDC tokens through `id_tokens`. Write the token to a file and set `AWS_ROLE_ARN` and `AWS_WEB_IDENTITY_TOKEN_FILE`: boto3's default chain then performs the web identity exchange for cloudg and for Prowler alike. The role's trust policy uses your GitLab host as the provider (`gitlab.com` for SaaS), audience `sts.amazonaws.com`, and a `sub` such as `project_path:acme/cloud-posture:ref_type:branch:ref:main`. + +```yaml title=".gitlab-ci.yml" +stages: [posture] + +variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + AWS_ROLE_ARN: arn:aws:iam::123456789012:role/cloudg-gitlab + AWS_DEFAULT_REGION: us-east-1 + +cloudg-posture: + stage: posture + image: python:3.12-slim + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_PIPELINE_SOURCE == "web" + id_tokens: + AWS_ID_TOKEN: + aud: sts.amazonaws.com + cache: + key: cloudg-0.6.0 + paths: [.cache/pip] + before_script: + - pip install "cloudg[aws]==0.6.0" parliament pipx + - pipx install prowler + - export PATH="$HOME/.local/bin:$PATH" + - echo "$AWS_ID_TOKEN" > "$CI_PROJECT_DIR/.web-identity-token" + - export AWS_WEB_IDENTITY_TOKEN_FILE="$CI_PROJECT_DIR/.web-identity-token" + script: + - cloudg -c ci/cloudg.yaml run -p aws -o reports + - python ci/severity_gate.py reports/findings.json --fail-on CRITICAL --min-assets 1 + artifacts: + when: always + paths: [reports/] + expire_in: 30 days +``` + +Do not also pass `--aws-role-arn` here. With `AWS_WEB_IDENTITY_TOKEN_FILE` set in the environment, cloudg would treat that ARN as its own OIDC target and do the exchange itself, which works for the collectors but leaves Prowler on the default chain anyway. Create the schedule under CI/CD, Schedules; the `web` rule lets you start a run by hand. + +## Scheduled inventory snapshots + +A map is cheaper than a full run and changes less often than findings. A nightly `cloudg map` gives you `inventory-map.json`, `inventory-dependencies.json` and, with `--org`, `inventory-organization.json`, ready for `cloudg deps`, for diffing, or for an [MCP server](/cli/mcp-serve/) to load. + +```yaml title=".github/workflows/inventory.yml" +name: Inventory snapshot + +on: + schedule: + - cron: "30 2 * * *" + +permissions: + contents: read + id-token: write + +jobs: + map: + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - run: pip install "cloudg[aws]==0.6.0" + - uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: arn:aws:iam::123456789012:role/cloudg-github + aws-region: us-east-1 + - run: cloudg map -p aws --regions all -o inventory + - uses: actions/upload-artifact@v7 + with: + name: inventory-${{ github.run_id }} + path: inventory/ + retention-days: 90 +``` + +Throttling during a map is recorded in coverage and in the `throttling` block of `inventory-map.json` rather than failing the job, so check that block when a snapshot looks thin. Several pipelines mapping the same account at once share its API quota; stagger their schedules, or lower `ratelimit.aws.max_rps` for the CI config as described in [rate limits](/guides/rate-limits/). + +## Caching and artifacts + +Three things are worth caching. pip's wheel cache saves the install of the cloud SDKs on every run. Trivy's vulnerability database (`~/.cache/trivy`) is a sizeable download; with the cache restored, Trivy only fetches it again when its copy is out of date. Pin the cloudg version in both the install command and the cache key, so an upgrade invalidates the cache instead of mixing versions. + +Do not cache `reports/`. Publish it as an artifact instead. A run's useful files: + +| File | Use in CI | +|---|---| +| `findings.json` | the gate's input; also re-renders later with `cloudg report -i` | +| `report.html` | self-contained, opens from the artifact download | +| `raw-findings.json` | written by `ingest` (and `scan`): findings before deduplication, for debugging the normaliser | +| `inventory-map.json` | the map from `cloudg map`, loadable with `InventoryResult.load` | +| `ontology.ttl`, `rag_chunks.jsonl` | feed downstream analysis or a retrieval index | + +The reports describe your estate in detail: account IDs, ARNs, open ports, IAM grants. Keep artifact retention short and the repository private, and use the Docker image or a private runner when the reports must not leave your network. See [Docker](/guides/docker/) for the image, which bundles all four scanners. + +:::links +- [Authentication](/guides/authentication/) Every credential method and its precedence. +- [Python recipes](/guides/python-recipes/) The gate and the pipeline from Python. +- [Ingesting existing output](/guides/ingesting/) What `cloudg ingest` accepts. +- [cloudg run](/cli/run/) Every flag of the full pipeline. +::: diff --git a/site/content/guides/compliance.md b/site/content/guides/compliance.md new file mode 100644 index 0000000..07a1d94 --- /dev/null +++ b/site/content/guides/compliance.md @@ -0,0 +1,328 @@ +--- +title: Compliance frameworks +lede: "cloudg doesn't run compliance checks of its own. It takes the findings your scanners produced and tags each one with the framework controls it violates, using rulesets that ship in the package." +meta: + - [Rulesets, "`cloudg/rules/`"] + - [Frameworks, "28, with 4,271 controls"] + - [Config, "`rulesets.rules_dir`"] +source: cloudg/normaliser.py +since: "0.6.0" +--- + +Compliance mapping is the last step of normalisation, so it runs wherever findings are normalised: phase 4 of `cloudg run`, `cloudg ingest`, `CloudGEngine.normalise_findings()` and `run_from_reports()`. Its output is a list of `ComplianceResult` records in `findings.json` and the Compliance tab of `report.html`. + +One thing to understand before reading any of that output: a control only appears when at least one finding maps to it. cloudg has no list of the controls you passed, because scanners report failures (Prowler's passing checks are dropped while parsing). Every `ComplianceResult` it writes has status `FAIL`. Read the compliance output as "these controls have open findings", not as a full audit score. + +## The rulesets + +The rulesets are YAML files in `cloudg/rules/`, installed with the package. They come in two kinds. + +The generated rulesets in `cloudg/rules/frameworks/` are converted from Prowler's public compliance data (Apache-2.0) by `scripts/import_prowler_compliance.py`. Each control lists the Prowler check names that assess it, under `checks`. There are 22 of these files, one framework each, with 4,166 controls and 10,236 check mappings over 787 distinct Prowler checks. + +The hand-written rulesets at the top of `cloudg/rules/` hold regex `patterns` instead of check lists. There are 9 files with 105 controls. Three of them (`cis_aws_v3.yaml`, `cis_azure_v2.yaml`, `cis_gcp_v3.yaml`) use the same framework names as the generated CIS 5.0 files and are merged into them on load. The other six define frameworks that only exist in pattern form. + +Loaded together, that is 28 framework names and 4,271 controls. The README's figures are consistent with this: 28 is the total number of frameworks, while 4,166 controls and 10,236 check mappings describe the generated part only. + +| Framework | Controls | Check mappings | Pattern controls | Files | +|---|---|---|---|---| +| `AWS-Foundational-Security-Best-Practices-AWS` | 206 | 226 | 0 | `frameworks/aws_foundational_security_best_practices_aws.yaml` | +| `CIS-AWS` | 81 | 73 | 18 | `frameworks/cis_5.0_aws.yaml`, `cis_aws_v3.yaml` | +| `CIS-AZURE` | 123 | 109 | 16 | `frameworks/cis_5.0_azure.yaml`, `cis_azure_v2.yaml` | +| `CIS-GCP` | 87 | 71 | 16 | `frameworks/cis_5.0_gcp.yaml`, `cis_gcp_v3.yaml` | +| `GDPR` | 5 | 0 | 5 | `gdpr.yaml` | +| `GDPR-AWS` | 3 | 70 | 0 | `frameworks/gdpr_aws.yaml` | +| `HIPAA` | 8 | 0 | 8 | `hipaa_security_rule.yaml` | +| `HIPAA-AWS` | 32 | 307 | 0 | `frameworks/hipaa_aws.yaml` | +| `HIPAA-AZURE` | 34 | 340 | 0 | `frameworks/hipaa_azure.yaml` | +| `HIPAA-GCP` | 23 | 95 | 0 | `frameworks/hipaa_gcp.yaml` | +| `ISO-27001` | 9 | 0 | 9 | `iso_27001_2022.yaml` | +| `ISO27001-AWS` | 37 | 627 | 0 | `frameworks/iso27001_2022_aws.yaml` | +| `ISO27001-AZURE` | 31 | 339 | 0 | `frameworks/iso27001_2022_azure.yaml` | +| `ISO27001-GCP` | 30 | 108 | 0 | `frameworks/iso27001_2022_gcp.yaml` | +| `MITRE-ATTACK-AWS` | 36 | 380 | 0 | `frameworks/mitre_attack_aws.yaml` | +| `MITRE-ATTACK-AZURE` | 31 | 231 | 0 | `frameworks/mitre_attack_azure.yaml` | +| `MITRE-ATTACK-GCP` | 26 | 142 | 0 | `frameworks/mitre_attack_gcp.yaml` | +| `NIST-800-53` | 13 | 0 | 13 | `nist_800_53.yaml` | +| `NIST-800-53-Revision-5-AWS` | 287 | 1944 | 0 | `frameworks/nist_800_53_revision_5_aws.yaml` | +| `NIST-CSF-AWS` | 50 | 645 | 0 | `frameworks/nist_csf_2.0_aws.yaml` | +| `PCI-AWS` | 1507 | 1513 | 0 | `frameworks/pci_4.0_aws.yaml` | +| `PCI-AZURE` | 756 | 756 | 0 | `frameworks/pci_4.0_azure.yaml` | +| `PCI-DSS` | 11 | 0 | 11 | `pci_dss_v4.yaml` | +| `PCI-GCP` | 755 | 1530 | 0 | `frameworks/pci_4.0_gcp.yaml` | +| `SOC2` | 9 | 0 | 9 | `soc2_tsc.yaml` | +| `SOC2-AWS` | 27 | 251 | 0 | `frameworks/soc2_aws.yaml` | +| `SOC2-AZURE` | 26 | 313 | 0 | `frameworks/soc2_azure.yaml` | +| `SOC2-GCP` | 28 | 166 | 0 | `frameworks/soc2_gcp.yaml` | + +The merged CIS frameworks mix two benchmark versions under one name: the generated controls use CIS 5.0 numbering (`1.4`) and the hand-written ones CIS 3.0 or 2.0 numbering with a prefix (`CIS-1.4`). They don't collide, but don't expect one numbering scheme inside `CIS-AWS`. + +`rules/check_equivalence.yaml` sits in the same directory but isn't a ruleset. It tells the deduplicator which checks from different scanners mean the same thing; see [Ingesting existing output](/guides/ingesting/#normalisation-and-deduplication). + +## How a finding gets its controls + +Mapping works in four tiers, most precise first. Each finding carries a list of framework names, `compliance_frameworks`, which the tiers add to. + +```mermaid caption="From a finding to framework controls" +flowchart TD + F["Finding"] --> T1["Tier 1: tags set by the scanner parser"] + T1 --> T2["Tier 2: exact check-ID lookup in ruleset checks"] + T2 --> Q{"Any framework tagged yet?"} + Q -->|yes| G["Group by framework and control"] + Q -->|no| T3["Tier 3: ruleset regex patterns"] + T3 --> Q2{"Matched?"} + Q2 -->|yes| G + Q2 -->|no| T4["Tier 4: built-in fallback patterns"] + T4 --> G + G --> CR["ComplianceResult per control, status FAIL"] +``` + +### Tier 1: what the scanner said + +Each parser sets `compliance_frameworks` while it reads the native output. These tags are coarse framework names, not controls: + +| Source | Tags | +|---|---| +| Prowler | From `Compliance.RelatedRequirements`: `CIS`, `NIST-800-53`, `PCI-DSS`, `GDPR`, `HIPAA`, `SOC2` (substring match) | +| ScoutSuite | The rule's `references` list, copied as is | +| Checkov | `CIS` always; `NIST-800-53` or `PCI-DSS` when the check ID contains those strings | +| Trivy | `CVE` for vulnerabilities, `CIS` for misconfigurations, none for secrets | +| IAM linter | `CIS`, `NIST-800-53` and, for wildcard actions, `SOC2` | +| Reachability | `CIS` and `NIST-800-53` (plus `PCI-DSS` for some), none for unexpected internet exposure | + +### Tier 2: exact check IDs + +This tier always runs. cloudg splits the finding's `source_finding_id` on `-`, `/`, `:` and whitespace, then looks every piece up in the `checks` lists of all loaded rulesets. A Prowler ASFF ID such as `prowler-aws-s3_bucket_default_encryption-123456789012-eu-west-1-abc` yields the piece `s3_bucket_default_encryption`, which appears in 28 controls across eight AWS frameworks. Each hit adds the framework and records the exact control ID and title. + +Prowler check names never contain dashes, so they survive the split intact. A Checkov ID such as `CKV_AWS_19` does too, which means you can list Checkov IDs in your own rulesets. Trivy IDs (`AVD-AWS-0088`, `CVE-2024-0001`) and ScoutSuite rule names (`s3-bucket-no-encryption`) are split into fragments, so they can't be matched by this tier. + +### Tier 3: ruleset patterns + +Only for findings that still have no framework at all after tiers 1 and 2. cloudg joins the title and description, lowercases them, and runs every `patterns` regex of every ruleset against that text (`re.search`, case-insensitive). A match adds the framework name. It does not record which control matched. + +In practice few findings get this far, since Checkov, Trivy vulnerabilities and misconfigurations, and the IAM linter always tag something in tier 1. The usual candidates are ScoutSuite findings with an empty `references` list, Prowler findings whose requirements match none of the six prefixes, Trivy secrets, and reachability findings for unexpected internet exposure. + +### Tier 4: built-in fallback + +When tier 3 matches nothing, a small table in `cloudg/normaliser.py` tries a few broad patterns (`encryption`, `cloudtrail`, `mfa`, `logging` and so on) for `CIS`, `NIST-800-53`, `PCI-DSS`, `GDPR`, `SOC2` and `HIPAA`. A finding that matches nothing here keeps an empty framework list and appears in no `ComplianceResult`. + +## ComplianceResult + +After tagging, cloudg groups findings by framework and control ID and writes one `ComplianceResult` per group. The model lives in `cloudg.schema.models`: + +| Field | Type | Value written by the normaliser | +|---|---|---| +| `id` | `str` | A new UUID | +| `framework` | `str` | The framework name, such as `NIST-CSF-AWS` or `CIS` | +| `control_id` | `str` | See below | +| `control_title` | `str \| None` | The ruleset's control title for exact matches, otherwise `" "` | +| `status` | `ComplianceStatus` | Always `FAIL` | +| `finding_ids` | `list[str]` | IDs of every finding mapped to this control | +| `resource_arn` | `str \| None` | Always `None` | + +`ComplianceStatus` defines four values: `PASS`, `FAIL`, `NOT_APPLICABLE` and `MANUAL`. The normaliser in 0.6.0 only produces `FAIL`. The other values are there for your own code, or for results you build yourself. + +The `control_id` depends on how the framework was attached: + +| How the framework was attached | `control_id` | Example | +|---|---|---| +| Tier 2 exact match | The ruleset control ID | `ds_1` (NIST-CSF-AWS), `3.5.1.30` (PCI-AWS) | +| Tier 1, 3 or 4, Prowler finding | `/` plus the third and fourth tokens of the ASFF ID | `CIS/s3_bucket_default_encryption-123456789012` | +| Tier 1, 3 or 4, Checkov finding | `/` | `CIS/CKV_AWS_19` | +| Tier 1, 3 or 4, Trivy CVE | `/` | `CVE/CVE-2024-0001` | +| Anything else | `-aggregate` | `GDPR-aggregate` | + +Two details follow from that table. For Prowler, the fourth token of the ASFF ID is usually the account ID, so the same check in two accounts gives two control IDs under the coarse `CIS` tag. And the `-aggregate` buckets collect every finding that reached a framework without a specific control: ScoutSuite findings, pattern matches and Trivy misconfigurations. + +## Reading the output + +### findings.json + +`compliance` is a list of the records above, and `summary.compliance_frameworks` lists the framework names that have at least one record. Join `finding_ids` against `findings[].id` to see what is behind each control. This script works on any `findings.json` from `cloudg run`, `cloudg ingest` or `run_from_reports()`: + +```python title="compliance_summary.py" +import json +from collections import defaultdict +from pathlib import Path + +report = json.loads(Path("./reports/findings.json").read_text()) +findings = {f["id"]: f for f in report["findings"]} + +controls = defaultdict(list) +for result in report["compliance"]: + controls[result["framework"]].append(result) + +print(f"{'framework':32} {'controls':>8} {'findings':>8}") +for framework, results in sorted(controls.items()): + finding_ids = {fid for r in results for fid in r["finding_ids"]} + print(f"{framework:32} {len(results):8} {len(finding_ids):8}") + +print() +for result in controls.get("CIS-AWS", []) + controls.get("NIST-CSF-AWS", []): + print(result["framework"], result["control_id"], "-", (result["control_title"] or "")[:60]) + for fid in result["finding_ids"]: + f = findings[fid] + print(" ", f["severity"], f["source_tool"], f["resource_arn"]) +``` + +Against the [sample reports from the ingest guide](/guides/ingesting/#try-it-without-any-scanner), the start of the output is: + +```console +$ python compliance_summary.py +framework controls findings +CIS 3 3 +CIS-AWS 1 1 +CVE 1 1 +GDPR 1 1 +GDPR-AWS 1 1 +HIPAA 1 1 +HIPAA-AWS 6 1 +... +NIST-CSF-AWS ds_1 - Data-at-rest is protected. + HIGH prowler arn:aws:s3:::my-bucket +``` + +The coarse tier 1 tags and the ruleset frameworks appear side by side: `CIS` next to `CIS-AWS`, `HIPAA` next to `HIPAA-AWS`. They are separate keys. Prowler findings usually carry both, with the precise controls under the ruleset name; Checkov and Trivy findings land mostly under the coarse ones. + +### report.html {#report-html} + +The Compliance tab shows one card per framework with Pass and Fail counts and a percentage bar. Fail is the number of distinct controls with findings. Since every result is `FAIL`, Pass is always 0 and the bar reads 0%; treat the card as a count of affected controls. The Analytics tab charts the same counts, and the detail panel of each finding lists its frameworks. [Reports](/guides/reports/) covers the rest of the page. + +### compliance-map.json + +`cloudg map --findings` (and `InventoryMapper.export_merged()`) writes a different, framework-level view: for each framework, how many findings, their severities, and which resources are affected. It is built from each finding's `compliance_frameworks` list, not from `ComplianceResult` records, so it has no control IDs. Pass it the normalised `findings.json`, not `raw-findings.json`, or the tier 2 to 4 frameworks will be missing. + +```python title="compliance_map_demo.py" +import json +from pathlib import Path + +from cloudg import CloudGConfig, InventoryMapper, InventoryResult +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, Finding + +report = json.loads(Path("./reports/findings.json").read_text()) +findings = [Finding.model_validate(f) for f in report["findings"]] + +bucket = CloudAsset( + name="my-bucket", + arn="arn:aws:s3:::my-bucket", + asset_type=AssetType.S3_BUCKET, + provider=CloudProvider.AWS, + region="eu-west-1", + account_id="123456789012", +) +inventory = InventoryResult(assets=[bucket]) + +compliance_map = InventoryMapper(CloudGConfig()).build_compliance_map(inventory, findings) +print(json.dumps(compliance_map["frameworks"]["CIS"], indent=2)) +``` + +```console +$ python compliance_map_demo.py +{ + "findings": 3, + "severity_breakdown": { + "HIGH": 3 + }, + "affected_assets": [ + "./iac", + "arn:aws:s3:::my-bucket", + "aws_s3_bucket.data" + ], + "affected_asset_count": 3 +} +``` + +`affected_assets` holds inventory ARNs where a finding matched an inventory asset, and the finding's own `resource_arn` where it didn't. That is why a Terraform address and a Trivy directory show up next to the bucket. The top level also has `total_frameworks` and `total_inventory_assets`. + +## Adding a custom ruleset {#adding-a-custom-ruleset} + +A ruleset is a YAML file with a `framework` name and a list of `controls`. Each control has an `id`, a `title`, and either `checks` (tier 2), `patterns` (tier 3) or both. `version`, `source` and a per-control `severity` are accepted and ignored by the normaliser. + +```yaml title="my-rules/acme_baseline.yaml" +framework: ACME-BASELINE +version: "2026.1" +controls: + - id: "ACME-S3-01" + title: "Buckets encrypt data at rest" + checks: + - s3_bucket_default_encryption # Prowler check name + - CKV_AWS_19 # Checkov check ID + patterns: + - "bucket.*(encrypt|sse)" + severity: HIGH + - id: "ACME-IMG-01" + title: "Container images carry no secrets" + patterns: + - "secret found in container image" +``` + +cloudg loads every `*.yaml` under `rulesets.rules_dir`, recursively. That directory replaces the packaged one, it doesn't add to it, so start from a copy of the shipped rules or you lose all 28 frameworks and the check-equivalence map: + +```bash +cp -r "$(python -c 'import cloudg, pathlib; print(pathlib.Path(cloudg.__file__).parent / "rules")')" ./my-rules +cp acme_baseline.yaml ./my-rules/ +``` + +Then point cloudg at it. + +```yaml tab="CLI" title="config.yaml" +rulesets: + rules_dir: ./my-rules +``` + +```python tab="Python" title="custom_rules.py" +from cloudg.ingest import ingest_reports +from cloudg.normaliser import FindingsNormaliser + +findings = ingest_reports( + { + "prowler": ["./prowler-output/"], + "checkov": ["./results_json.json"], + "scoutsuite": ["./scoutsuite-report/"], + } +) +scan_result = FindingsNormaliser(rules_dir="./my-rules").normalise(findings) + +for result in scan_result.compliance: + if result.framework == "ACME-BASELINE": + print(result.control_id, result.control_title, len(result.finding_ids)) +``` + +With the CLI, pass the config with `-c`: `cloudg -c config.yaml ingest ...` or `cloudg -c config.yaml run ...`. From `CloudGEngine`, set `config.rulesets.rules_dir`. On the sample reports, the result is: + +```console +$ python custom_rules.py +ACME-BASELINE-aggregate ACME-BASELINE ACME-BASELINE-aggregate 1 +ACME-S3-01 Buckets encrypt data at rest 2 +``` + +The Prowler and Checkov findings matched `ACME-S3-01` exactly through `checks`. The ScoutSuite finding had no frameworks, so tier 3 ran, the pattern matched, and it was filed under `ACME-BASELINE-aggregate`. + +A few rules of thumb for writing them: + +- Prefer `checks` over `patterns`. Exact matches give real control IDs and titles in the output; pattern matches only give the framework. +- Use Prowler check names and Checkov `CKV_*` IDs in `checks`. Trivy and ScoutSuite IDs contain dashes and won't match. +- Several files may use the same `framework` name; their controls are combined. That is how the hand-written and generated CIS files share `CIS-AWS`. +- A file that fails to parse is logged as a warning and skipped, and the rest still load. Run with `cloudg -v` to see `Loaded N rules from ` for each one. + +`rulesets.load_external` is present in the config model but nothing reads it in 0.6.0; rulesets always load. + +### Refreshing the generated rulesets + +From a repository checkout, the import script rebuilds `cloudg/rules/frameworks/` against a newer Prowler release: + +```bash +git clone --depth 1 https://github.com/prowler-cloud/prowler /tmp/prowler +python scripts/import_prowler_compliance.py /tmp/prowler +python scripts/import_prowler_compliance.py /tmp/prowler --out ./my-rules/frameworks +``` + +The list of frameworks it imports is the `FRAMEWORKS` constant at the top of the script. Add a `(provider, file stem)` pair there to import another of Prowler's compliance files. + +## Next + +:::links +- [Ingesting existing output](/guides/ingesting/) Deduplication and the check-equivalence map. +- [Reports](/guides/reports/) The Compliance tab and the rest of `report.html`. +- [Data models](/api/models/) `ComplianceResult`, `ComplianceStatus` and `ScanResult`. +- [FindingsNormaliser](/api/findingsnormaliser/) The class that does the mapping. +::: diff --git a/site/content/guides/dependencies.md b/site/content/guides/dependencies.md new file mode 100644 index 0000000..a65193d --- /dev/null +++ b/site/content/guides/dependencies.md @@ -0,0 +1,321 @@ +--- +title: Dependencies and blast radius +lede: "`cloudg deps` asks two questions of any asset on a saved map: what does it need, and what breaks with it?" +meta: + - [Command, "`cloudg deps`"] + - [Input, "`inventory-map.json` from `cloudg map`"] +source: cloudg/inventory/dependencies.py +--- + +A map from [`cloudg map`](/guides/inventory-mapping/) records edges the way you would describe them out loud: the queue invokes the function, the function assumes its role, the WAF protects the API. That is the right shape for drawing, but not for impact questions. To ask "what breaks if I rotate this KMS key?" you need every edge turned into one arrow that reads "dependent needs dependency". `cloudg deps` and the `DependencyGraph` class behind it do that conversion and then walk the result. + +`cloudg deps` reads a saved `inventory-map.json` and nothing else. It makes no cloud calls, so you can run it on a laptop against a map someone produced in CI last night. + +## How edges become dependencies + +Each edge type has a fixed dependency direction. For "uses" edges the source needs the target. For "acts on" edges the target needs the source. + +| Direction | Edge types | Reads as | +|---|---|---| +| forward: the source depends on the target | `REFERENCES`, `ATTACHED_TO`, `ROUTE`, `PEERING`, `USES_IMAGE`, `ASSUMES_ROLE`, `LOGS_TO`, `LOAD_BALANCER_TARGET`, `GRANTS_ACCESS`, `IAM_POLICY_ATTACHMENT`, `IAM_TRUST` | a function needs its role, image, key and log group; a principal needs what it is granted | +| reverse: the target depends on the source | `CONTAINS`, `INVOKES`, `PROTECTS`, `MONITORS`, `MANAGES`, `GOVERNS` | a function needs the queue that triggers it; an ALB needs the WAF in front of it; a resource needs the stack managing it; an account needs the SCPs governing it | +| ignored | `SECURITY_GROUP_RULE`, `NACL_RULE`, `INTERNET_EXPOSED` | traffic rules carry no dependency meaning | + +Two more rules shape the graph. `CONTAINS` edges whose container is an organization, an OU or an account are dropped, because "this account contains this bucket" says where something lives, not what breaks with it. And an edge only counts when both of its ends are assets on the map, so edges to raw CIDR blocks never take part. + +Here is a small map as `cloudg map` would record it: + +```mermaid caption="A slice of a map, with edges in their natural direction" +flowchart LR + WAF["orders-api-acl (WAF)"] -->|PROTECTS| API["orders-api (API Gateway)"] + API -->|INVOKES| FN["orders-worker (Lambda)"] + Q["orders (SQS)"] -->|INVOKES| FN + Q -->|REFERENCES| DLQ["orders-dlq (SQS)"] + Q -->|REFERENCES| KEY["orders-key (KMS)"] + FN -->|ASSUMES_ROLE| ROLE["orders-worker (IAM role)"] + FN -->|USES_IMAGE| REPO["orders-worker (ECR)"] + FN -->|LOGS_TO| LOGS["/aws/lambda/orders-worker"] + FN -->|REFERENCES| KEY + DLQ -->|REFERENCES| KEY + REPO -->|REFERENCES| KEY + ROLE -->|GRANTS_ACCESS| TABLE["orders (DynamoDB)"] + TABLE -->|REFERENCES| KEY + EXT["account 999999999999"] -->|IAM_TRUST| ROLE +``` + +And the same slice after the conversion. Every arrow now points from the asset that needs something to the thing it needs. Note how the `INVOKES` and `PROTECTS` arrows flipped: + +```mermaid caption="The same slice as dependency arrows. Walking with the arrows is depends_on (up); walking against them is dependents (down)." +flowchart LR + FN["orders-worker (Lambda)"] --> API["orders-api (API Gateway)"] + API --> WAF["orders-api-acl (WAF)"] + FN --> Q["orders (SQS)"] + Q --> DLQ["orders-dlq (SQS)"] + Q --> KEY["orders-key (KMS)"] + DLQ --> KEY + FN --> ROLE["orders-worker (IAM role)"] + FN --> REPO["orders-worker (ECR)"] + FN --> LOGS["/aws/lambda/orders-worker"] + FN --> KEY + REPO --> KEY + ROLE --> TABLE["orders (DynamoDB)"] + TABLE --> KEY + EXT["account 999999999999"] --> ROLE +``` + +Upstream of the function sits everything it needs to run, including the API and queue that feed it work. Downstream of the KMS key sits everything that stops working if the key is disabled. + +This is an availability and change-impact view. Compromise runs the other way along identity edges: a compromised function exposes its role, not the reverse. For that question, walk the `IAM_TRUST`, `ASSUMES_ROLE` and `GRANTS_ACCESS` edges of the map directly, or use the attack path analysis in [GraphBuilder](/api/graphbuilder/). + +## Query one asset + +Give `cloudg deps` an asset and it prints two trees: what the asset depends on, and what depends on it. + +```bash +cloudg deps arn:aws:sqs:us-east-1:123456789012:orders --map ./reports +``` + +The output below was produced by cloudg 0.6.0 against the map drawn above (the banner that cloudg prints first is left out here): + +```console +$ cloudg deps arn:aws:sqs:us-east-1:123456789012:orders --map ./reports +orders MESSAGE_QUEUE arn:aws:sqs:us-east-1:123456789012:orders depends on +├── orders-key KMS_KEY · 123456789012/us-east-1 (ENCRYPTED_BY_KMS) +└── orders-dlq MESSAGE_QUEUE · 123456789012/us-east-1 (WRITES_TO) +orders MESSAGE_QUEUE arn:aws:sqs:us-east-1:123456789012:orders needed by (blast radius) +└── orders-worker LAMBDA_FUNCTION · 123456789012/us-east-1 (TRIGGERED_BY) +``` + +Each line is the asset name, its type, account and region, and in parentheses the edge's `relationship` (or its `edge_type` when no relationship is set). An empty tree prints `nothing`. + +Asking about the function in one direction shows the walk going deeper: + +```console +$ cloudg deps arn:aws:lambda:us-east-1:123456789012:function:orders-worker --direction up +orders-worker LAMBDA_FUNCTION arn:aws:lambda:us-east-1:123456789012:function:orders-worker depends on +├── orders-worker IAM_ROLE · 123456789012/global (RUNS_ON) +│ └── orders DYNAMODB_TABLE · 123456789012/us-east-1 (POLICY_ALLOWS_ACTION) +├── orders-worker CONTAINER_REGISTRY · 123456789012/us-east-1 (USES_IMAGE) +├── /aws/lambda/orders-worker LOG_GROUP · 123456789012/us-east-1 (LOGS_TO) +├── orders-key KMS_KEY · 123456789012/us-east-1 (ENCRYPTED_BY_KMS) +├── orders MESSAGE_QUEUE · 123456789012/us-east-1 (TRIGGERED_BY) +│ └── orders-dlq MESSAGE_QUEUE · 123456789012/us-east-1 (WRITES_TO) +└── orders-api API_GATEWAY · 123456789012/us-east-1 (INVOKES) + └── orders-api-acl WAF_WEB_ACL · 123456789012/us-east-1 (PROTECTED_BY_WAF) +``` + +The walk is breadth-first and every asset appears once, at the shallowest depth where it was first reached. The DynamoDB table also needs `orders-key`, but the key already appeared one level up as a direct dependency of the function, so it isn't repeated under the table. When an asset seems to be missing from a branch, look higher in the tree. + +### Direction, depth and top + +| Option | Default | Effect | +|---|---|---| +| `ASSET` | none | The asset to query. Leave it out for the overview below. | +| `-m, --map` | `./reports` | `inventory-map.json`, or the directory holding it. | +| `-d, --direction` | `both` | `up` is what the asset depends on, `down` is what depends on it, `both` prints the two trees. | +| `--depth` | `3` | Levels to expand in the trees. | +| `--top` | `15` | Rows in the overview tables. | +| `--json` | off | Print JSON instead of tables and trees. | + +`--depth` only limits the trees. The blast radius figures in the overview always walk up to 10 hops. + +### How the asset is matched + +The argument can be any of these, tried in order: + +1. the internal asset ID (the UUID in `inventory-map.json`); +2. the ARN or resource ID; +3. the name, when exactly one asset has it; +4. the part of the ARN after the last `/`, when exactly one asset has it. + +Names repeat more than you would expect. In the map above a role, a function and an ECR repository are all called `orders-worker`, so a bare name matches nothing: + +```console +$ cloudg deps orders-worker + ✗ No unique asset matches 'orders-worker'; use its ARN or resource ID +``` + +The command exits with status 1 in that case, which makes it safe in scripts. Pass the full ARN instead. `orders-key` works as a bare name because only one asset carries it. + +## The overview + +Without an asset, `cloudg deps` prints three tables for the whole map: + +```console +$ cloudg deps --map ./reports --top 5 +──────────────────────────────── Interdependencies ──────────────────────────────── + Most shared dependencies +╭───────────────┬──────────┬──────────────┬───────────────────╮ +│ Asset │ Type │ Account │ Direct dependents │ +├───────────────┼──────────┼──────────────┼───────────────────┤ +│ orders-key │ KMS_KEY │ 123456789012 │ 5 │ +│ orders-worker │ IAM_ROLE │ 123456789012 │ 2 │ +╰───────────────┴──────────┴──────────────┴───────────────────╯ + Largest blast radius +╭────────────────┬────────────────┬────────────┬──────────┬─────────╮ +│ Asset │ Type │ Dependents │ Accounts │ Exposed │ +├────────────────┼────────────────┼────────────┼──────────┼─────────┤ +│ orders-key │ KMS_KEY │ 7 │ 2 │ 0 │ +│ orders │ DYNAMODB_TABLE │ 3 │ 2 │ 0 │ +│ orders-dlq │ MESSAGE_QUEUE │ 2 │ 1 │ 0 │ +│ orders-worker │ IAM_ROLE │ 2 │ 2 │ 0 │ +│ orders-api-acl │ WAF_WEB_ACL │ 2 │ 1 │ 1 │ +╰────────────────┴────────────────┴────────────┴──────────┴─────────╯ + Cross-account edges +╭────────────────────────────────┬─────────────────────┬────────────────────────────────┬──────────╮ +│ Source │ Relationship │ Target │ External │ +├────────────────────────────────┼─────────────────────┼────────────────────────────────┼──────────┤ +│ arn:aws:iam::999999999999:root │ CROSS_ACCOUNT_TRUST │ arn:aws:iam::123456789012:rol… │ yes │ +╰────────────────────────────────┴─────────────────────┴────────────────────────────────┴──────────╯ +``` + +A table with no rows is not printed at all. + +### Shared dependencies + +The assets that the most other assets depend on directly, the classic case being one KMS key behind forty resources. Containment and peering edges don't count, and an asset needs at least two distinct functional dependents to be listed. The count is of distinct assets, so a function with two edges to the same key counts once. + +### Blast radius + +The assets whose loss or change would reach the most others, counted transitively. cloudg takes the `4 × top` assets with the most direct dependents as candidates, walks each one downstream for up to 10 hops, and ranks them by how many assets it reached. Assets with no dependents are left out. + +`Accounts` is how many distinct accounts those dependents live in. In the example the key reaches account `999999999999` because the role trusted by that external account sits downstream of the table the key encrypts. `Exposed` counts internet-exposed assets among the dependents: the WAF scores 1 because the API Gateway it protects faces the internet. + +On a map made with `--org`, SCPs often rank high. Every account an SCP governs depends on it, so an SCP attached to many accounts collects many dependents even though nothing about it is broken. + +### Cross-account edges + +Every edge whose two ends carry different account IDs. `External` is `yes` when either end is a placeholder for an account that was not mapped, such as a vendor account trusted by one of your roles. Edges from an OU to an account are not listed, but on an organization map you will see an `SCP_RESTRICTS` row for every SCP attached directly to a member account, and AWS attaches `FullAWSAccess` to every account by default. Filter those out in JSON when you only care about workload traffic (see the Python example below). + +## Coverage gaps + +`cloudg deps` doesn't print the security coverage section; `cloudg map` writes it to `inventory-dependencies.json` under `security_coverage`: + +| Key | What it holds | +|---|---| +| `services_by_account_region` | account to region to security service to enabled, for GuardDuty, Security Hub, Inspector, Macie, Config, CloudTrail and the others | +| `gaps` | every disabled cell as `"/: "` | +| `workloads_without_vulnerability_scanning` | EC2 instances, container registries and Lambda functions that no enabled vulnerability scanner monitors | +| `internet_facing_without_waf` | internet-exposed load balancers, API Gateways and CloudFront distributions that no WAF protects | + +The same file repeats the shared dependencies, blast radius (top 25), cross-account edges and the map's unresolved references. + +```python title="coverage_gaps.py" +import json + +with open("./reports/inventory-dependencies.json") as f: + analysis = json.load(f) + +coverage = analysis["security_coverage"] +for gap in coverage["gaps"]: + print("disabled:", gap) +for item in coverage["internet_facing_without_waf"]: + print("no WAF:", item["type"], item["name"], item["account_id"]) +for item in coverage["workloads_without_vulnerability_scanning"]: + print("not scanned:", item["type"], item["name"], item["region"]) +``` + +## JSON output + +`--json` prints the tree view (with an asset) or the overview (without one) as JSON. The tree has an `asset` descriptor and, depending on `--direction`, `depends_on` and `dependents` lists. Each node is a descriptor (`id`, `name`, `arn`, `type`, `account_id`, `region`) plus `via` (the edge type), `relationship` and `children`. The overview has `shared_dependencies`, `largest_blast_radius` and `cross_account_edges`; that last list is complete, since `--top` only trims the table view. + +:::warning The banner goes to stdout too +cloudg prints its banner on stdout before every command, `--json` included, so redirecting the output straight to a file does not give you valid JSON. Drop everything before the first line that starts with `{`: +::: + +```bash +cloudg deps arn:aws:iam::123456789012:role/orders-worker --json | sed -n '/^{/,$p' > role-deps.json +cloudg deps --map ./reports --top 50 --json | sed -n '/^{/,$p' > overview.json +``` + +For anything more than a one-off, the Python API below returns the same structures without the parsing step. + +## Working in Python + +`InventoryResult.load()` reads a saved map, and `dependency_graph()` builds the graph `cloudg deps` uses: + +```python title="blast_radius.py" +from cloudg.inventory import InventoryResult + +inventory = InventoryResult.load("./reports") # the directory or inventory-map.json +graph = inventory.dependency_graph() + +key = graph.find("orders-key") # ID, ARN, unique name or unique ARN tail +if key is None: + raise SystemExit("no unique asset matches") + +needs, needed_by = graph.direct_counts(key.id) +print(f"{key.name} ({key.asset_type.value}): needs {needs}, needed by {needed_by} directly") + +for link in graph.dependents(key.id, max_depth=5): + asset = graph.assets[link.asset_id] + parent = graph.assets[link.parent_id] + print(f"{' ' * link.depth}{asset.name} [{asset.asset_type.value}] " + f"via {link.relationship or link.via_edge} from {parent.name}") + +for row in graph.shared_dependencies(top=5): + print(row["name"], row["type"], row["direct_dependents"]) + +for row in graph.blast_radius(top=5): + print(row["name"], row["transitive_dependents"], row["accounts_affected"], + row["internet_exposed_dependents"]) +``` + +`depends_on(asset_id, max_depth=10)` and `dependents(asset_id, max_depth=10)` return `DependencyLink` objects with `asset_id`, `via_edge`, `relationship`, `depth` (starting at 1) and `parent_id`. `tree(asset_id, direction="both", max_depth=3)` returns the nested dict that `--json` prints. `blast_radius()` also takes `candidates`, an iterable of asset IDs, when you want to score a specific set instead of the most-depended-on assets: + +```python title="key_blast_radius.py" +from cloudg.inventory import InventoryResult +from cloudg.inventory.dependencies import cross_account_edges + +inventory = InventoryResult.load("./reports") +graph = inventory.dependency_graph() + +keys = [a.id for a in inventory.assets if a.asset_type.value == "KMS_KEY"] +for row in graph.blast_radius(candidates=keys, top=10): + print(row["arn"], row["transitive_dependents"], row["accounts_affected"]) + +workload_edges = [ + e for e in cross_account_edges(inventory.assets, inventory.edges) + if e["edge_type"] != "GOVERNS" +] +for e in workload_edges: + print(e["source_account"], e["relationship"] or e["edge_type"], e["target_account"], + "(external)" if e["external"] else "") +``` + +Build the graph with `inventory.dependency_graph(include_hierarchy=True)` if you want account and OU containment to count, for example to ask what depends on one OU. + +### Resolving references with AssetIndex + +`graph.find()` uses `AssetIndex`, the matcher cloudg also uses to attach scanner findings to assets. Use it directly when you have many references to resolve, such as every `resource_id` from a scanner export. Its `find()` takes two opt-in tiers for references `cloudg deps` won't accept: the ARN resource part (`function:orders-worker`, `db:orders`, `instance/i-0web1`) and a case-insensitive name. + +```python title="resolve_refs.py" +from cloudg.inventory import InventoryResult +from cloudg.inventory.dependencies import AssetIndex + +inventory = InventoryResult.load("./reports") +index = AssetIndex(inventory.assets) +by_id = {a.id: a for a in inventory.assets} + +refs = [ + "arn:aws:sqs:us-east-1:123456789012:orders", # exact ARN + "orders-key", # unique name + "orders-worker", # three assets share this name + "function:orders-worker", # ARN resource part, opt-in + "ORDERS-KEY", # case-folded name, opt-in +] +for ref in refs: + strict = index.find(ref) + loose = index.find(ref, arn_parts=True, casefold=True) + print(f"{ref!r:48} strict={by_id[strict].asset_type.value if strict else None} " + f"loose={by_id[loose].asset_type.value if loose else None}") +``` + +Against the example map, the ambiguous `orders-worker` stays unresolved in both modes, `function:orders-worker` and `ORDERS-KEY` resolve only with the opt-in tiers, and the other two resolve either way. The opt-in tiers never pick one of several assets that share an exact name. `index.resolve_finding(finding)` tries a finding's `resource_id` and `resource_arn` through the same tiers. + +:::links +- [Inventory mapping](/guides/inventory-mapping/) Produce the map that `cloudg deps` reads. +- [cloudg deps](/cli/deps/) The command reference. +- [DependencyGraph](/api/dependencygraph/) Every method and return type. +- [InventoryResult](/api/inventoryresult/) Loading, analysis and export. +::: diff --git a/site/content/guides/docker.md b/site/content/guides/docker.md new file mode 100644 index 0000000..ca60af2 --- /dev/null +++ b/site/content/guides/docker.md @@ -0,0 +1,359 @@ +--- +title: Docker +lede: "The cloudg image bundles the CLI, the three cloud SDKs, Prowler, Checkov and Trivy. Mount your credentials and a reports directory, and every command runs the same way it does on the host." +meta: + - [Image, "`ghcr.io/morpheuslord/cloudg`"] + - [Base, "`python:3.12-slim`"] + - [Entrypoint, "`cloudg`"] +source: Dockerfile +--- + +The image is the quickest route to a full `cloudg run`, because the scanners are the hard part of a local install and the image has them preinstalled. It is also a clean way to run the MCP server on a host where you would rather not install Python packages. + +## What is in the image + +The Dockerfile has three stages. The first installs the scanners, each Python one in its own virtualenv so their dependencies cannot collide. The second installs cloudg's Python dependencies with uv. The third copies both into a fresh `python:3.12-slim` and installs the cloudg source on top, so a code change rebuilds only the last few layers. + +| Component | Where | Notes | +|---|---|---| +| cloudg | `/usr/local/bin/cloudg`, `/usr/local/bin/cloudg-mcp` | installed with the `aws`, `azure` and `gcp` extras | +| Prowler | `/opt/prowler`, symlinked into `/usr/local/bin` | own venv | +| Checkov | `/opt/checkov`, symlinked into `/usr/local/bin` | own venv | +| Trivy | `/usr/local/bin/trivy` | from Aqua's install script | +| Parliament | system site-packages | used by the IAM linter | +| Graphviz | system package | not needed by cloudg itself | + +Some things are deliberately or incidentally absent, and each one changes how you use the image: + +- ScoutSuite is not installed. The default `scanners.enabled` still lists it, so every run logs that ScoutSuite is not installed and skips it. Pass `--scanners prowler,checkov,trivy,iam` to keep the log clean. +- The `mcp` extra is not installed. `cloudg mcp serve` works anyway, because the default `--flavor auto` falls back to the native server, which has no dependencies. +- The AWS, Azure and gcloud CLIs are not installed. Anything that relies on a CLI login (Azure's `az login` session in particular) does not work inside the container. Use keys, a service principal, a credentials file or an instance identity instead, as shown below. + +At run time the container works like this: + +| Setting | Value | +|---|---| +| `ENTRYPOINT` | `["cloudg"]`, so arguments after the image name are cloudg arguments | +| `CMD` | `["--help"]` | +| `WORKDIR` | `/app`, so the default `-o ./reports` means `/app/reports` | +| `USER` | `cloudg`, a non-root user created with `useradd` (normally UID 1000) | +| `HOME` | `/home/cloudg`, where credential files and scanner caches go | +| `HEALTHCHECK` | `cloudg --version` every 5 minutes | + +## Get the image + +```bash tab="Pull" +docker pull ghcr.io/morpheuslord/cloudg:latest +docker tag ghcr.io/morpheuslord/cloudg:latest cloudg:latest +``` + +```bash tab="Build" +git clone https://github.com/morpheuslord/cloudg.git +cd cloudg +docker build -t cloudg:latest . +``` + +Releases are tagged with their version (`ghcr.io/morpheuslord/cloudg:0.6.0`) as well as `latest`. The examples on this page use the local name `cloudg:latest`, which is what `docker compose` builds too. + +A build needs the full source checkout, not just `cloudg/`: the package build force-includes `docs/DOCUMENTATION.md` and `docs/INVENTORY_REFERENCE.md` (the MCP server serves them as `cloudg://docs/...` resources), and hatchling refuses to build without them. The first build takes a while because of the scanner stage; after that, edits to `cloudg/` rebuild in seconds. + +Check it: + +```console +$ docker run --rm cloudg:latest --version +cloudg, version 0.6.0 +``` + +## The reports volume + +Everything cloudg writes goes to `/app/reports` unless you pass `-o`. Mount a host directory there: + +```bash +mkdir -p reports +docker run --rm -v "$PWD/reports:/app/reports" cloudg:latest report -i /app/reports/findings.json +``` + +Create the directory before the first run. If Docker creates it for you, it belongs to root, and the `cloudg` user inside the container cannot write to it. On Linux the host directory must also be writable by the container's UID. If your own UID is not 1000, either make the directory writable for UID 1000 or run as yourself and point `HOME` somewhere writable: + +```bash +docker run --rm --user "$(id -u):$(id -g)" -e HOME=/tmp \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest ingest --trivy /app/reports/trivy.json +``` + +Docker Desktop on macOS and Windows maps file ownership for you, so this only comes up on Linux. + +## Credentials + +The container sees no credentials until you give it some. Each provider has a few ways in. + +### AWS + +```bash tab="Profile" +docker run --rm \ + -v "$HOME/.aws:/home/cloudg/.aws:ro" \ + -e AWS_PROFILE=audit \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p aws --regions us-east-1 +``` + +```bash tab="Environment" +docker run --rm \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p aws --regions us-east-1 +``` + +```bash tab="Instance role" +# on EC2 or ECS: credentials come from the instance or task role +docker run --rm -v "$PWD/reports:/app/reports" cloudg:latest run -p aws --regions all +``` + +Mount `~/.aws` at `/home/cloudg/.aws`, not `/root/.aws`; the container does not run as root. With SSO profiles, run `aws sso login` on the host first. A read-only mount can use the cached token but cannot store a refreshed one, so log in again on the host when it expires. + +`-e AWS_ACCESS_KEY_ID` with no value copies the variable from your shell, which keeps keys out of your shell history. Prowler inside the container reads the same variables or profile, so the scanners and the collectors use the same identity. + +On EC2 with IMDSv2, a container on Docker's default bridge network is one network hop further from the metadata service than the host. If the instance's metadata hop limit is 1, the container gets no credentials; raise the hop limit to 2 or run with `--network host`. + +### Azure + +The usual container setup is a service principal in environment variables. cloudg reads `AZURE_TENANT_ID`, `AZURE_CLIENT_ID` and `AZURE_CLIENT_SECRET` when the matching flags are not given: + +```bash tab="Client secret" +docker run --rm \ + -e AZURE_TENANT_ID -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p azure --subscription-id 00000000-0000-0000-0000-000000000000 +``` + +```bash tab="Certificate" +docker run --rm \ + -e AZURE_TENANT_ID -e AZURE_CLIENT_ID \ + -v "$PWD/sp-cert.pem:/secrets/sp-cert.pem:ro" \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p azure --azure-cert-path /secrets/sp-cert.pem +``` + +```bash tab="Managed identity" +# on an Azure VM or container host with a managed identity +docker run --rm -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p azure --azure-managed-identity +``` + +Without `--subscription-id`, cloudg collects every enabled subscription the identity can see. + +### GCP + +Mount a key file (or a workload identity federation config) and point `GOOGLE_APPLICATION_CREDENTIALS` at the path inside the container: + +```bash tab="Key file" +docker run --rm \ + -v "$PWD/scanner-sa.json:/secrets/gcp.json:ro" \ + -e GOOGLE_APPLICATION_CREDENTIALS=/secrets/gcp.json \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p gcp --project-id my-project +``` + +```bash tab="gcloud ADC" +# after `gcloud auth application-default login` on the host +docker run --rm \ + -v "$HOME/.config/gcloud/application_default_credentials.json:/home/cloudg/.config/gcloud/application_default_credentials.json:ro" \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p gcp --project-id my-project +``` + +google-auth finds the application default credentials file at its standard path under `HOME` without needing gcloud in the container. + +## Running each command + +The commands below assume the reports mount from above and AWS credentials in environment variables; swap in whichever credential mount you use. Paths in arguments are paths inside the container. + +The full pipeline: + +```bash +docker run --rm \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p aws --regions us-east-1 --scanners prowler,checkov,trivy,iam --terraform +``` + +`--terraform` gives Checkov and Trivy a target (the Terraform recreation of what was collected). To scan your own IaC, mount it and pass `--iac-dir`: + +```bash +docker run --rm \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \ + -v "$PWD/infra:/app/infra:ro" \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest run -p aws --iac-dir /app/infra +``` + +Inventory mapping, and the dependency explorer on its output: + +```bash +docker run --rm \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest map -p aws --regions all +docker run --rm -v "$PWD/reports:/app/reports" \ + cloudg:latest deps arn:aws:iam::123456789012:role/app-role +``` + +Ingest needs no credentials, only the scanner files. Mount them read-only: + +```bash +docker run --rm \ + -v "$PWD/scan-output:/app/input:ro" \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest ingest --prowler /app/input/prowler/ --trivy /app/input/trivy.json +``` + +Re-render a report from an earlier run: + +```bash +docker run --rm -v "$PWD/reports:/app/reports" \ + cloudg:latest report -i /app/reports/findings.json --format all +``` + +### Using a config file + +cloudg reads `config.yaml` only when you pass `-c`, and `-c` is a global option, so it goes before the subcommand: + +```bash +docker run --rm \ + -v "$PWD/config.yaml:/app/config.yaml:ro" \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \ + -v "$PWD/reports:/app/reports" \ + cloudg:latest -c /app/config.yaml run -p aws +``` + +Keep relative paths in the config (`terraform.output_dir: ./reports/terraform`, `scanners.iac_directories`) relative to `/app`, the container's working directory. + +## docker compose + +The repository ships a `docker-compose.yml` with one service, `cloudg`. It builds the image from the checkout, mounts `./reports` and `./config.yaml`, and passes the common credential variables through from your shell: + +```yaml title="docker-compose.yml" +services: + cloudg: + build: . + image: cloudg:latest + container_name: cloudg + volumes: + - ./reports:/app/reports + - ./config.yaml:/app/config.yaml:ro + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-} + - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-us-east-1} + - AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID:-} + - AZURE_TENANT_ID=${AZURE_TENANT_ID:-} + - AZURE_CLIENT_ID=${AZURE_CLIENT_ID:-} + - AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET:-} + - GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS:-} + entrypoint: ["cloudg"] + command: ["--help"] +``` + +Use it with `docker compose run`, which replaces `command` with your arguments: + +```bash +mkdir -p reports +docker compose build +docker compose run --rm cloudg run -p aws --regions us-east-1 +docker compose run --rm cloudg -c /app/config.yaml run -p aws +``` + +Three things to know about this file: + +- `./config.yaml` must exist on the host. If it does not, Docker creates an empty directory with that name and mounts it, and `-c /app/config.yaml` then fails. The repository's own `config.yaml` is a commented template you can edit. +- The mounted config is not used unless you pass `-c /app/config.yaml`, as in the last command above. +- `GOOGLE_APPLICATION_CREDENTIALS` is passed through as a host path, which does not exist inside the container. Mount the key file and set the variable to the container path, for example in an override file. + +An override file is the tidy way to add mounts without editing the shipped one. Compose merges `docker-compose.override.yml` automatically: + +```yaml title="docker-compose.override.yml" +services: + cloudg: + volumes: + - ~/.aws:/home/cloudg/.aws:ro + - ./scanner-sa.json:/secrets/gcp.json:ro + - ./infra:/app/infra:ro + environment: + - AWS_PROFILE=audit + - GOOGLE_APPLICATION_CREDENTIALS=/secrets/gcp.json +``` + +```bash +docker compose run --rm cloudg run -p aws -p gcp --project-id my-project --iac-dir /app/infra +``` + +## The MCP server in Docker + +The MCP server runs from the same image. Over stdio, the AI client starts the container itself; over HTTP, the container runs as a long-lived service. + +```mermaid caption="Two ways a client reaches cloudg in a container" +flowchart LR + D["Desktop client"] -->|"docker run -i (stdio)"| S1["cloudg mcp serve"] + W["HTTP client"] -->|"127.0.0.1:8765/mcp, bearer token"| S2["cloudg mcp serve --transport http"] + S1 --> V[("/app/reports")] + S2 --> V +``` + +### stdio for a desktop client + +Map once on the host or in a container, then serve the map read-only: + +```bash +docker run -i --rm \ + -v "$PWD/reports:/app/reports:ro" \ + cloudg:latest mcp serve --dataset prod=/app/reports/inventory-map.json +``` + +Use `-i` and never `-t`. stdio needs stdin open, and a TTY would mix terminal control codes into the JSON-RPC stream. The banner goes to stderr when the subcommand is `mcp`, so stdout stays clean. + +In the client configuration the command is `docker` and the arguments are the rest of that line. For Claude Desktop, in `claude_desktop_config.json`: + +```json title="claude_desktop_config.json" +{ + "mcpServers": { + "cloudg": { + "command": "docker", + "args": [ + "run", "-i", "--rm", + "-v", "/home/me/cloudg/reports:/app/reports:ro", + "cloudg:latest", + "mcp", "serve", "--policy", "strict", + "--dataset", "prod=/app/reports/inventory-map.json" + ] + } + } +} +``` + +Desktop clients do not run commands through a shell, so use absolute host paths in `-v`. The workspace's allowed roots default to the working directory and the report directory, which inside the container are `/app` and `/app/reports`; tools that load datasets by path can reach files you mount under either. + +### HTTP as a service + +For a shared server, bind to all interfaces inside the container, publish the port on the host's loopback, and require a token: + +```bash +export CLOUDG_MCP_TOKEN="$(openssl rand -hex 32)" +docker run -d --name cloudg-mcp \ + -p 127.0.0.1:8765:8765 \ + -e CLOUDG_MCP_TOKEN \ + -v "$PWD/reports:/app/reports:ro" \ + cloudg:latest mcp serve --transport http --host 0.0.0.0 \ + --auth-token env:CLOUDG_MCP_TOKEN:analyst:shared \ + --allowed-host 'localhost:*' --allowed-host '127.0.0.1:*' \ + --dataset prod=/app/reports/inventory-map.json +curl -s http://127.0.0.1:8765/healthz +``` + +`--host 0.0.0.0` is needed because the default `127.0.0.1` inside the container is not reachable through a published port. Binding to a non-loopback address turns off the default `Host` check, which is why the example sets `--allowed-host` explicitly; if clients reach the server through a reverse proxy, list the proxy's public name instead. The endpoint is `http://127.0.0.1:8765/mcp`, and `/healthz` answers `{"status":"ok"}` without a token, which suits a container health probe. + +### Live tools in a container + +The MCP live tools (`map_inventory`, `collect_assets`, `run_scanners`, `run_pipeline`) call cloud APIs with the server's credentials, whoever is asking. With no credentials mounted, their credential preflight (at most 25 seconds) fails and nothing is collected. If you do mount credentials, restrict who can use them: the `strict`, `read_only`, `airgapped`, `audit` and `soc-analyst` policies hide those tools, and `--read-only` drops them along with anything that writes files. A server that only answers questions about saved maps should run with `--read-only` and a read-only reports mount, as above. + +[Connect a client](/mcp/connect/) has configuration for the other MCP clients, and [cloudg mcp serve](/cli/mcp-serve/) lists every server option. diff --git a/site/content/guides/graph-ontology-rag.md b/site/content/guides/graph-ontology-rag.md new file mode 100644 index 0000000..98f1971 --- /dev/null +++ b/site/content/guides/graph-ontology-rag.md @@ -0,0 +1,455 @@ +--- +title: Graph, ontology and RAG +lede: "Collected assets and edges become a NetworkX graph for reachability and blast radius, an RDF ontology you can query with SPARQL, and retrieval-ready chunks for an LLM." +meta: + - [Modules, "`cloudg.graph`"] + - [Config, "`graph`, `ontology`, `rag`, `terraform`"] + - [Changed in, "0.6.0"] +source: cloudg/graph/reachability.py +since: "0.6.0" +--- + +Every collection ends with two lists: `CloudAsset` objects and directed `NetworkEdge` objects between their ids. Everything on this page is derived from those two lists, plus findings when you have them. Nothing here calls a cloud API, so you can rebuild any of it offline from a saved map or a `findings.json`. + +```mermaid caption="From collected data to the four analysis outputs" +flowchart LR + A["Assets and edges"] --> G["GraphBuilder: nx.DiGraph"] + F[Findings] --> O + G --> R["ReachabilityAnalyzer"] + R --> RF["Exposure and open-port findings"] + RF --> O + A --> O["CloudOntology: RDF triples"] + O --> T["ontology.ttl, ontology.jsonld"] + G --> C["RAGExporter"] + A --> C + F --> C + C --> J["rag_chunks.jsonl"] + A --> TF["TerraformExporter: .tf.json"] +``` + +`cloudg run` does all of it in order: graph and reachability in phase 2, RAG export in 2c, Terraform in 2d, then after the scanners the ontology and a second RAG pass so both include the scanner findings. `CloudGEngine.analyze()` does the graph, ontology, RAG and Terraform steps for assets you collected yourself. The sections below use a small hand-built estate so every output is reproducible; the full script is at the end. + +## The graph + +`GraphBuilder().build(assets, edges)` returns an `nx.DiGraph`. Each asset becomes a node keyed by its `id`, carrying `name`, `asset_type`, `provider`, `region`, `arn`, `account_id`, `tags` (as a JSON string) and `is_internet_exposed`. Each edge becomes a graph edge with `edge_type`, `relationship`, `port_range`, `protocol`, `cidr`, `direction` and `description`. An edge whose endpoint is not a collected asset, such as the CIDR `0.0.0.0/0` or an Azure service tag, gets a placeholder node with `asset_type` and `provider` set to `EXTERNAL` and `is_external` true. + +A `DiGraph` holds one edge per ordered pair, and collectors often emit several rules for one pair: two ingress rules from `0.0.0.0/0` into the same security group, one for 443 and one for 22. Since 0.6.0 `SECURITY_GROUP_RULE`, `NACL_RULE` and `INTERNET_EXPOSED` edges on the same pair are merged into one graph edge when their CIDR and direction match, and the merged `port_range` lists every port: + +```console +$ python graph_tour.py +graph: 9 nodes, 10 edges +merged rule: 443,22 +``` + +Eleven `NetworkEdge` objects became ten graph edges. Any other edge type sharing a pair with an earlier edge replaces it, which is NetworkX's normal behaviour, so a pair with both a `CONTAINS` and a `REFERENCES` edge keeps the second. + +Other methods on the builder: + +| Method | Returns | +|---|---| +| `find_attack_paths(source, target, max_depth=10)` | every simple path between two node ids, over all edge types | +| `find_lateral_movement_paths()` | paths of up to 8 hops from internet-exposed or external nodes to the target of any `IAM_TRUST` edge, capped at about 100 | +| `compute_centrality()` | per node: degree, betweenness, in-degree and out-degree centrality | +| `to_d3_json()`, `to_cytoscape_json()` | the dicts behind the HTML report's topology view and `topology-cytoscape.json` | +| `save_graphml(path)`, `load_graphml(path)` | GraphML round trip | +| `subgraph(node_ids)` | a copy restricted to those nodes | + +:::warning GraphML and assets without an ARN +In 0.6.0 `save_graphml` raises `NetworkXError: GraphML writer does not support as data values` when any asset in the graph has `arn=None`, because the node attribute is stored as `None`. The collectors set ARNs, so this mostly hits hand-built assets. Give each asset an `arn` (any stable identifier will do) before exporting GraphML. +::: + +## Reachability and internet exposure + +`ReachabilityAnalyzer(graph).find_internet_exposed()` is a breadth-first walk from the internet. It starts at nodes named `0.0.0.0/0` or `::/0`, plus the source of every non-egress edge whose CIDR stands for the internet (`0.0.0.0/0`, `::/0`, and the Azure tags `Internet`, `Any` and `*` in any letter case). + +Since 0.6.0 the walk follows network-flow edges only. Earlier releases followed every edge type, so a database whose only link to the internet was an IAM role granted access to it came out as a CRITICAL "Internet-exposed RDS_INSTANCE". The rules now: + +| Edge | Followed | +|---|---| +| `INTERNET_EXPOSED`, `LOAD_BALANCER_TARGET`, `ROUTE`, `PEERING` | source to target | +| `SECURITY_GROUP_RULE`, `NACL_RULE` | source to target for ingress rules; egress rules carry nothing inbound | +| `ATTACHED_TO` | backwards from a security group, NSG or NACL (or an edge declaring `PROTECTED_BY_SG` / `PROTECTED_BY_NACL`) to the resource attached to it; forwards from a network interface or Elastic IP | +| `CONTAINS` | from a VPC, VNet or subnet to what is placed in it; never organization, account, cluster or resource group containment | +| `GRANTS_ACCESS`, `ASSUMES_ROLE`, `IAM_TRUST`, `IAM_POLICY_ATTACHMENT`, `INVOKES`, `REFERENCES`, `LOGS_TO`, ... | never | + +Not following `INVOKES` means a Lambda function behind a public API Gateway is not flagged. The gateway is the exposed resource; the function is only invoked by it. + +Every node reached is marked `is_internet_exposed` in the graph (not on the `CloudAsset`). `generate_findings()` then turns the walk into findings: + +| Rule | Severity | When | +|---|---|---| +| `internet-exposed-sensitive-asset` | CRITICAL | a reached RDS instance, Aurora cluster, Azure SQL, Cloud SQL or DynamoDB table | +| `internet-exposed-unexpected-asset` | HIGH | any other reached resource, except load balancers, CloudFront, CDNs and internet gateways, where exposure is expected | +| `internet-open-sensitive-port` | CRITICAL | an ingress rule from an internet source whose ports include one of 22, 3389, 3306, 5432, 1433, 27017, 6379, 9200, 5601, 8080, 8443 | + +Security groups, NSGs, NACLs and target groups are hops on the walk. They get marked but get no exposure finding of their own: a group's open rules show up as open-port findings, and a target group's exposure is reported on its targets. Ports are parsed as numbers, so `0-65535`, `*` and comma lists such as `443,3380-3390` match the ports inside them and `2200-2300` no longer matches 22. + +Finding ids are deterministic: a UUID5 of the rule and the asset's ARN (or its id when it has no ARN), so the same exposure keeps its id from scan to scan. `cloudg.graph.reachability.finding_id()` and `asset_key()` give you the scheme. + +On the sample estate: + +```console +internet-exposed: ['alb-1', 'db-1', 'i-web', 'sg-db', 'sg-web'] + CRITICAL Internet-exposed RDS_INSTANCE: orders-db id=e0b159a9 + HIGH Unexpected internet-exposed resource: web-1 id=8dcf38d8 + CRITICAL Security group allows SSH (port 22) from 0.0.0.0/0 id=17cd5168 +``` + +Only `sg-web` admits the internet, yet the database is flagged. That is because the walk is an over-approximation: it follows the `sg-web -> sg-db` rule like any other ingress rule and does not intersect ports across hops, so traffic that entered on 443 is treated as able to continue on 5432. The IAM path from `app-role` to the database played no part in it. Treat a CRITICAL exposure finding behind a group-to-group rule as a path worth checking, not as proof. + +`flow_hops(node)`, `flow_successors(node)`, `internet_entry_points()` and the module function `network_flow_graph(graph)` expose the same walk, so your own code can follow the network exactly the way the analysis does. + +## Blast radius + +`compute_blast_radius(node_id)` answers the opposite question: if this node is compromised, what can it reach? Unlike exposure it follows every edge type, identity included, because a stolen role reaches whatever the role is granted. + +```console +blast radius of web-1: {'reachable_nodes': ['sg-web', 'role-app', 'sg-db', 'db-1'], 'depth': 2, 'risk_score': 4.0} +``` + +The score is `0.5` per reachable node plus `2.0` per sensitive data store, capped at 10. It is a quick ranking signal. For dependency questions on a full inventory (what breaks if this KMS key goes, which shared resources have the most dependents) use `DependencyGraph` and `cloudg deps`, described in [dependencies and blast radius](/guides/dependencies/). + +## The ontology + +`CloudOntology().build(assets, edges, findings)` writes an RDF graph with rdflib. Three namespaces: + +| Prefix | IRI | Holds | +|---|---|---| +| `cm:` | `https://cloudg.io/ontology#` | OWL classes (`cm:ComputeInstance`, `cm:RelationalDatabase`, `cm:SecurityFinding`, ...) | +| `cmp:` | `https://cloudg.io/property/` | relations (`cmp:INGRESS_ALLOWED`) and data properties (`cmp:hasName`, `cmp:hasARN`, `cmp:hasSeverity`) | +| `cmr:` | `https://cloudg.io/resource/` | individuals: `cmr:`, `cmr:finding_`, `cmr:compliance_` | + +Each asset becomes an individual of the class for its type, with `hasName`, `hasProvider`, `hasRegion`, `isInternetExposed` (the asset's own flag, not the reachability result), `hasARN` and `hasAccountId` when set, and a `TAGGED_WITH` link per tag. Each finding becomes a `cm:SecurityFinding` with `hasSeverity`, `hasRiskScore` and `FINDING_AFFECTS` pointing at its asset. The asset is resolved by id, then ARN, then unique name, then unique ARN tail, so a Prowler finding keyed by an ARN lands on the right node. Each compliance framework a finding names becomes a `cm:ComplianceControl` that `COMPLIANCE_GOVERNS` the asset. + +### Relation types + +Edges are not copied as-is. `infer_relations()` derives typed relations from the edge type, the endpoint asset types, the CIDR and the ports. There are 64 relation types in 7 groups: + +| Group | Count | Examples | +|---|---|---| +| NETWORK | 16 | `INGRESS_ALLOWED`, `EGRESS_ALLOWED`, `INTERNET_REACHABLE`, `ONLY_SSH`, `ONLY_HTTPS`, `ALL_TRAFFIC`, `PORT_RESTRICTED`, `CIDR_RESTRICTED`, `VPC_PEERED`, `TRANSIT_ROUTED` | +| CONTAINMENT | 8 | `CONTAINS`, `VPC_CONTAINS_SUBNET`, `SUBNET_CONTAINS_INSTANCE`, `CLUSTER_CONTAINS_SERVICE`, `ORG_CONTAINS_ACCOUNT`, `LB_TARGETS_INSTANCE` | +| IAM | 10 | `ROLE_ASSUMES_ROLE`, `CROSS_ACCOUNT_TRUST`, `ROLE_HAS_POLICY`, `POLICY_ALLOWS_ACTION`, `SCP_RESTRICTS` | +| DATA_FLOW | 9 | `READS_FROM`, `WRITES_TO`, `LOGS_TO`, `STREAMS_TO`, `REPLICATES_TO` | +| SECURITY | 9 | `PROTECTED_BY_SG`, `PROTECTED_BY_NACL`, `PROTECTED_BY_WAF`, `ENCRYPTED_BY_KMS`, `FINDING_AFFECTS`, `COMPLIANCE_GOVERNS` | +| COMPUTE | 8 | `RUNS_ON`, `TRIGGERED_BY`, `INVOKES`, `LOAD_BALANCED_BY`, `DEPENDS_ON` | +| GOVERNANCE | 4 | `TAGGED_WITH`, `COST_ALLOCATED_TO`, `OWNED_BY`, `MONITORED_BY` | + +A few inference rules worth knowing, several of them changed in 0.6.0: + +- A security group rule yields `INGRESS_ALLOWED` or `EGRESS_ALLOWED`, `INTERNET_REACHABLE` for an internet source, `CIDR_RESTRICTED` for any other CIDR, and port relations from the parsed ranges: `ALL_TRAFFIC` for every port, else `ONLY_SSH` / `ONLY_HTTP` / `ONLY_HTTPS` / `ONLY_RDP` for 22, 80, 443 and 3389 inside the ranges, else `PORT_RESTRICTED`. `NACL_RULE` edges yield no relation unless they declare one. +- `PROTECTED_BY_SG` and `PROTECTED_BY_NACL` come from `ATTACHED_TO` edges, so the triple reads `web-alb PROTECTED_BY_SG sg-web`. Other attachments are `DEPENDS_ON`. +- `CONTAINS` edges get a specific relation only when both endpoint types fit it; organization to OU, account to VPC or resource group to resource stay plain `CONTAINS`. +- A relationship declared on the edge (`NetworkEdge.relationship`, such as the linker's `TRIGGERED_BY`) replaces the inferred one on containment, attachment and the typed inventory edges, and comes first on the others. +- Asset metadata adds relations of its own: a VPC `CONTAINS` everything sharing its `vpc_id`, an encrypted asset is `ENCRYPTED_BY_KMS` its key (matched against collected KMS keys by id, ARN, key id or alias), and `owner`, `team`, `costcenter` or `monitoring` tags become governance relations. + +Ports, protocol and CIDR of an edge are kept on a blank node of type `cm:EdgeMetadata` with `cm:fromNode`, `cm:toNode`, `cmp:hasPort`, `cmp:hasProtocol` and `cmp:hasCIDR`. + +### Querying with SPARQL + +The exported Turtle file is self-contained. Load it with rdflib in any process, no cloudg import needed: + +```python title="query_ontology.py" +"""Run SPARQL against the ontology.ttl that cloudg wrote.""" + +import sys + +from rdflib import Graph + +PREFIXES = """ +PREFIX cm: +PREFIX cmp: +PREFIX cmr: +PREFIX rdfs: +""" + +FINDINGS_BY_ASSET = PREFIXES + """ +SELECT ?assetName ?type ?severity ?title WHERE { + ?finding a cm:SecurityFinding ; + cmp:hasSeverity ?severity ; + cmp:hasName ?title ; + cmp:FINDING_AFFECTS ?asset . + ?asset cmp:hasName ?assetName ; + a ?type . + FILTER(?type != cm:CloudResource) +} +ORDER BY ?severity ?assetName +""" + +OPEN_FROM_INTERNET = PREFIXES + """ +SELECT ?groupName (GROUP_CONCAT(DISTINCT ?port; separator=", ") AS ?ports) WHERE { + ?src cmp:INTERNET_REACHABLE ?group . + ?group cmp:hasName ?groupName . + ?meta a cm:EdgeMetadata ; + cm:fromNode ?src ; + cm:toNode ?group ; + cmp:hasPort ?port . +} +GROUP BY ?groupName +""" + +WHO_CAN_TOUCH_DATABASES = PREFIXES + """ +SELECT ?principalName ?dbName WHERE { + ?db a cm:RelationalDatabase ; + cmp:hasName ?dbName . + ?principal cmp:POLICY_ALLOWS_ACTION ?db ; + cmp:hasName ?principalName . +} +""" + +path = sys.argv[1] if len(sys.argv) > 1 else "reports/ontology.ttl" +g = Graph() +g.parse(path, format="turtle") +print(f"{len(g)} triples loaded from {path}\n") + +for title, query in [ + ("Findings by asset", FINDINGS_BY_ASSET), + ("Security groups open to the internet", OPEN_FROM_INTERNET), + ("Principals granted access to a database", WHO_CAN_TOUCH_DATABASES), +]: + print(title) + for row in g.query(query): + print(" " + " | ".join(str(v) for v in row)) + print() +``` + +```console +$ python query_ontology.py graph-out/ontology.ttl +757 triples loaded from graph-out/ontology.ttl + +Findings by asset + orders-db | https://cloudg.io/ontology#RelationalDatabase | CRITICAL | Internet-exposed RDS_INSTANCE: orders-db + sg-web | https://cloudg.io/ontology#SecurityGroup | CRITICAL | Security group allows SSH (port 22) from 0.0.0.0/0 + orders-db | https://cloudg.io/ontology#RelationalDatabase | HIGH | RDS instance is not encrypted + web-1 | https://cloudg.io/ontology#ComputeInstance | HIGH | Unexpected internet-exposed resource: web-1 + +Security groups open to the internet + sg-web | 22, 443 + +Principals granted access to a database + app-role | orders-db +``` + +The `FILTER` in the first query drops the generic `cm:CloudResource` type that placeholder nodes carry. On a live `cloudg run` the asset ids in `cmr:` IRIs are random UUIDs that change every collection, so join on `cmp:hasARN` when you compare two runs. + +Without leaving Python, `CloudOntology` has the same entry point plus canned queries: `query(sparql)` returns a list of dicts and accepts the `cm:`, `cmp:` and `cmr:` prefixes without declarations, and `query_internet_exposed()`, `query_by_relation_group(RelationGroup.IAM)`, `query_asset_neighbourhood(asset_id, hops=2)` and `query_compliance_gaps()` cover the common questions. `stats()` returns triple, class and individual counts and per-relation and per-group totals. + +## RAG export + +`RAGExporter().export_all(assets, edges, graph, findings, output_dir)` writes `rag_chunks.jsonl` (one chunk per line, the usual vector-store input) and `rag_metadata_index.json` (counts and every `chunk_id`). Each chunk has `chunk_id`, `chunk_type`, `content` (plain text to embed), `metadata` (to filter on) and `relations`. There are three kinds. + +Entity chunks, `entity::`, one per asset: a header with name, type, provider, region, ARN, account and tags, up to 30 relations from its one-hop neighbourhood, and up to 10 findings, most severe first. The metadata carries `asset_type`, `provider`, `region`, `account_id`, `is_internet_exposed`, `relation_types`, `severity_max`, `compliance_frameworks`, `neighbour_count`, `finding_count` and `arn`. This is the chunk for "tell me about orders-db": + +```json title="rag_chunks.jsonl (one line, formatted)" +{ + "chunk_id": "entity::db-1", + "chunk_type": "entity", + "content": "Resource: orders-db\nType: RDS_INSTANCE\nProvider: AWS\nRegion: eu-west-1\nARN: arn:aws:rds:eu-west-1:123456789012:db:orders-db\nAccount: 123456789012\n\nRelations (2):\n → PROTECTED_BY_SG: sg-db\n ← POLICY_ALLOWS_ACTION: app-role\n\nFindings (2):\n [CRITICAL] Internet-exposed RDS_INSTANCE: orders-db\n [HIGH] RDS instance is not encrypted", + "metadata": { + "asset_type": "RDS_INSTANCE", "provider": "AWS", "region": "eu-west-1", + "account_id": "123456789012", "is_internet_exposed": false, + "relation_types": ["POLICY_ALLOWS_ACTION", "PROTECTED_BY_SG"], + "severity_max": "CRITICAL", "compliance_frameworks": ["CIS", "NIST-800-53"], + "neighbour_count": 2, "finding_count": 2, + "arn": "arn:aws:rds:eu-west-1:123456789012:db:orders-db" + }, + "relations": [ + {"predicate": "PROTECTED_BY_SG", "direction": "outgoing", "object": "sg-db", "object_id": "sg-db", "evidence": ""}, + {"predicate": "POLICY_ALLOWS_ACTION", "direction": "incoming", "object": "app-role", "object_id": "role-app", "evidence": ""} + ] +} +``` + +Community chunks, `community::`: Louvain communities of the undirected graph (python-louvain is a core dependency; without it cloudg falls back to connected components), singletons skipped. They list members, type counts, internal and external edge counts, internet-exposed members and findings, with a `risk_score` of `0.3` per member plus `0.5` per finding plus `2.0` per exposed member, capped at 10. These answer "what is around this subnet" questions. Their `internet_exposed_count` reads the graph's flags, so it reflects the reachability walk when that ran on the same graph. + +Relation-group chunks, `relation_group::`: one per relation group, holding the distribution of relation types and the first 50 triples with port, protocol and CIDR as evidence, for "list every network path" questions. They include the asset-level relations as well as the edge relations, so they match what the ontology holds. + +`finding_count` is the field to filter on when you want only chunks with problems. Since 0.6.0 both exporters resolve findings to assets the same way the ontology does, so findings keyed by ARN or display name count against the right asset; before, they left `finding_count` at 0. The sample exports 8 entity, 3 community and 5 relation-group chunks. + +## Terraform export + +`TerraformExporter(output_dir).export(assets, edges)` writes a Terraform recreation of what was collected in JSON syntax: `provider.tf.json`, `variables.tf.json`, `main.tf.json` and `import_commands.sh`, a script of `terraform import` commands, one per mapped resource. `preview(assets)` reports what maps before you write anything: + +```console +terraform preview: {'aws_vpc': 1, 'aws_subnet': 1, 'aws_security_group': 2, 'aws_lb': 1, 'aws_instance': 1, 'aws_db_instance': 1, 'aws_iam_role': 1} +{'provider': 'provider.tf.json', 'variables': 'variables.tf.json', 'main': 'main.tf.json', 'import_commands': 'import_commands.sh'} +``` + +Asset types without a Terraform mapping are counted under `unmapped_asset_types` and skipped. The output is a starting point for codifying click-ops infrastructure, not a faithful plan: review every resource before `terraform plan`. `cloudg run --terraform` has a second use for it. When no IaC directory is configured, Checkov scans the generated files, so it audits the live estate instead of whatever directory you ran from. + +## Configuration keys + +| Key | Default | Effect in 0.6.0 | +|---|---|---| +| `graph.compute_attack_paths` | `true` | `cloudg run` computes lateral movement paths and prints their count | +| `graph.persist_graphml` | `true` | not read: `cloudg run` always writes `topology.graphml` | +| `graph.export_cytoscape` | `false` | not read: `cloudg run` always writes `topology-cytoscape.json` | +| `graph.max_nodes_warn` | `10000` | not read: the builder warns above a fixed 10,000 assets | +| `ontology.enabled` | `true` | builds the ontology (`run` also needs `--ontology`, the default) | +| `ontology.export_formats` | `[turtle, json-ld]` | one file per format: `turtle` to `ontology.ttl`, `json-ld` to `ontology.jsonld`, `xml` to `ontology.rdf` | +| `ontology.include_raw_metadata` | `false` | not read | +| `rag.enabled` | `true` | writes the chunks (`run` also needs `--rag-export`, the default) | +| `rag.chunk_strategy` | `hybrid` | not read: all three chunk kinds are always written | +| `rag.max_chunk_tokens` | `2000` | passed to `RAGExporter` but not applied; chunk size is bounded by fixed caps (30 relations, 10 findings, 50 triples) | +| `terraform.enabled` | `false` | writes the Terraform files (`run --terraform` does the same) | +| `terraform.output_dir` | `./reports/terraform` | used as given, not relative to `-o`; set it explicitly when you change the output directory | + +```yaml title="config.yaml" +graph: + compute_attack_paths: true +ontology: + enabled: true + export_formats: [turtle, json-ld, xml] +rag: + enabled: true +terraform: + enabled: true + output_dir: ./out/terraform # match your -o directory +``` + +## Output files + +| File | Written by | Contents | +|---|---|---| +| `topology.graphml` | `cloudg run` | the graph, for Gephi, yEd or `GraphBuilder.load_graphml` | +| `topology-cytoscape.json` | `cloudg run` | Cytoscape.js elements | +| `topology.svg` | `cloudg run` | static topology diagram | +| `findings.json` (`graph` key) | `cloudg run`, `run_pipeline` | D3 nodes and links, used by `report.html` | +| `inventory-map.graphml`, `inventory-graph.json` | `cloudg map` | the same graph built from the inventory map | +| `ontology.ttl`, `ontology.jsonld` | `cloudg run`, `analyze()` | RDF, per `ontology.export_formats` | +| `rag_chunks.jsonl`, `rag_metadata_index.json` | `cloudg run`, `analyze()` | the chunks and their index | +| `terraform/*.tf.json`, `terraform/import_commands.sh` | `cloudg run --terraform`, `analyze()` with `terraform.enabled` | Terraform recreation | + +## The whole example + +The script behind every output on this page. It needs only `pip install cloudg`. + +```python title="graph_tour.py" +"""Build a small estate by hand and run the graph, ontology, RAG and Terraform exports.""" + +import json +from pathlib import Path + +from cloudg.graph.builder import GraphBuilder +from cloudg.graph.ontology import CloudOntology +from cloudg.graph.rag_export import RAGExporter +from cloudg.graph.reachability import ReachabilityAnalyzer +from cloudg.renderers.terraform_export import TerraformExporter +from cloudg.schema.models import ( + AssetType, + CloudAsset, + CloudProvider, + EdgeType, + Finding, + NetworkEdge, + Severity, +) + +ACCOUNT, REGION = "123456789012", "eu-west-1" +OUT = Path("./graph-out") + + +def aws(asset_id: str, name: str, kind: AssetType, arn_tail: str, **metadata) -> CloudAsset: + service = arn_tail.split(":", 1)[0] + return CloudAsset( + id=asset_id, + name=name, + asset_type=kind, + provider=CloudProvider.AWS, + region=REGION, + account_id=ACCOUNT, + arn=f"arn:aws:{service}:{REGION}:{ACCOUNT}:{arn_tail.split(':', 1)[1]}", + metadata=metadata, + ) + + +def edge(src: str, dst: str, kind: EdgeType, **attrs) -> NetworkEdge: + return NetworkEdge(source_id=src, target_id=dst, edge_type=kind, **attrs) + + +assets = [ + aws("vpc-1", "prod-vpc", AssetType.VPC, "ec2:vpc/vpc-1", vpc_id="vpc-1"), + aws("subnet-a", "public-a", AssetType.SUBNET, "ec2:subnet/subnet-a", vpc_id="vpc-1"), + aws("sg-web", "sg-web", AssetType.SECURITY_GROUP, "ec2:security-group/sg-web"), + aws("sg-db", "sg-db", AssetType.SECURITY_GROUP, "ec2:security-group/sg-db"), + aws("alb-1", "web-alb", AssetType.LOAD_BALANCER, + "elasticloadbalancing:loadbalancer/app/web-alb/1"), + aws("i-web", "web-1", AssetType.EC2, "ec2:instance/i-0web"), + aws("db-1", "orders-db", AssetType.RDS_INSTANCE, "rds:db:orders-db"), + aws("role-app", "app-role", AssetType.IAM_ROLE, "iam:role/app-role"), +] + +edges = [ + # Internet -> web security group, two rules on the same pair + edge("0.0.0.0/0", "sg-web", EdgeType.SECURITY_GROUP_RULE, + port_range="443", protocol="tcp", cidr="0.0.0.0/0"), + edge("0.0.0.0/0", "sg-web", EdgeType.SECURITY_GROUP_RULE, + port_range="22", protocol="tcp", cidr="0.0.0.0/0"), + # Load balancer and instance sit behind sg-web; the ALB targets the instance + edge("alb-1", "sg-web", EdgeType.ATTACHED_TO), + edge("i-web", "sg-web", EdgeType.ATTACHED_TO), + edge("alb-1", "i-web", EdgeType.LOAD_BALANCER_TARGET), + # The database group admits the web group on 5432 + edge("sg-web", "sg-db", EdgeType.SECURITY_GROUP_RULE, port_range="5432", protocol="tcp"), + edge("db-1", "sg-db", EdgeType.ATTACHED_TO), + # Identity: the instance runs as app-role, which is granted the database + edge("i-web", "role-app", EdgeType.ASSUMES_ROLE), + edge("role-app", "db-1", EdgeType.GRANTS_ACCESS), + # Placement + edge("vpc-1", "subnet-a", EdgeType.CONTAINS), + edge("subnet-a", "i-web", EdgeType.CONTAINS), +] + +scanner_findings = [ + Finding( + resource_id=f"arn:aws:rds:{REGION}:{ACCOUNT}:db:orders-db", # an ARN, not an asset id + severity=Severity.HIGH, + title="RDS instance is not encrypted", + description="Storage encryption is disabled", + source_tool="prowler", + source_finding_id="rds_instance_storage_encrypted", + compliance_frameworks=["CIS"], + ) +] + +# 1. Graph +builder = GraphBuilder() +graph = builder.build(assets, edges) +print(f"graph: {graph.number_of_nodes()} nodes, {graph.number_of_edges()} edges") +print("merged rule:", graph.edges["0.0.0.0/0", "sg-web"]["port_range"]) + +# 2. Reachability and blast radius +analyzer = ReachabilityAnalyzer(graph) +print("internet-exposed:", sorted(analyzer.find_internet_exposed())) +reachability = analyzer.generate_findings() +for f in reachability: + print(f" {f.severity.value:<8} {f.title} id={f.id[:8]}") +print("blast radius of web-1:", analyzer.compute_blast_radius("i-web")) + +# 3. Ontology +findings = scanner_findings + reachability +ontology = CloudOntology() +ontology.build(assets, edges, findings) +stats = ontology.stats() +print(f"ontology: {stats['total_triples']} triples, groups {stats['relation_group_counts']}") +ontology.save(OUT / "ontology.ttl", fmt="turtle") +ontology.save(OUT / "ontology.jsonld", fmt="json-ld") +builder.save_graphml(OUT / "topology.graphml") + +# 4. RAG chunks +paths = RAGExporter().export_all(assets, edges, graph, findings, output_dir=OUT) +index = json.loads(paths["index"].read_text()) +print("rag:", {k: index[k] for k in ("entity_chunks", "community_chunks", "relation_group_chunks")}) + +# 5. Terraform +terraform = TerraformExporter(output_dir=OUT / "terraform") +print("terraform preview:", terraform.preview(assets)["resource_types"]) +print({name: path.name for name, path in terraform.export(assets, edges).items()}) +``` + +:::links +- [Dependencies and blast radius](/guides/dependencies/) `DependencyGraph` and `cloudg deps` on a full inventory. +- [GraphBuilder](/api/graphbuilder/) Every method of the builder. +- [ReachabilityAnalyzer](/api/reachabilityanalyzer/) The walk and its helpers. +- [CloudOntology](/api/cloudontology/) and [RAGExporter](/api/ragexporter/) The exporters' APIs. +::: diff --git a/site/content/guides/ingesting.md b/site/content/guides/ingesting.md new file mode 100644 index 0000000..842fb84 --- /dev/null +++ b/site/content/guides/ingesting.md @@ -0,0 +1,427 @@ +--- +title: Ingesting existing output +lede: "`cloudg ingest` is the second half of the pipeline on its own. Give it files that Prowler, ScoutSuite, Checkov or Trivy already wrote, and it parses, deduplicates, maps compliance and writes the reports." +meta: + - [Command, "`cloudg ingest`"] + - [Python, "`parse_report`, `ingest_reports`, `CloudGEngine.run_from_reports()`"] + - [Needs, "No credentials, no scanner binaries"] +source: cloudg/ingest.py +since: "0.6.0" +--- + +Ingest exists for the common case where the scanners already run somewhere you don't control from cloudg: a CI job that runs Checkov on every pull request, a nightly Prowler container in the security account, a Trivy step in the image build. Those jobs produce files. `cloudg ingest` reads the files and does the rest. Nothing is executed and no cloud API is called, so it runs on a laptop with no AWS profile, in an air-gapped review environment, or in a CI step after the scan step. + +## What ingest does + +```mermaid caption="cloudg ingest, from native files to reports" +flowchart LR + P["--prowler files"] --> PR["parse_report"] + S["--scoutsuite files"] --> PR + C["--checkov files"] --> PR + T["--trivy files"] --> PR + PR --> RAW["raw-findings.json"] + PR --> N["FindingsNormaliser"] + N --> J["findings.json"] + N --> H["report.html"] +``` + +Each path goes through `cloudg.ingest.parse_report(tool, path)`, which hands it to that scanner's own parser (the same code `cloudg run` uses on fresh output). Every parsed `Finding` is collected into one list and written, unchanged, to `raw-findings.json`. The list then goes through the same `FindingsNormaliser` that `cloudg run` uses: two-pass deduplication, CVSS-based rescoring and compliance mapping. The result is written as `findings.json` and `report.html`. + +What ingest doesn't do is anything that needs live assets. There is no collection, so no graph, no reachability findings, no IAM linting, no ontology, no RAG chunks and no `topology.svg`. If you want those as well, see [Combining with a live inventory](#combining-with-a-live-inventory). + +## Running it + +Every input flag is repeatable, and any combination of tools works. + +```bash tab="CLI" +cloudg ingest \ + --prowler ./prowler-output/ \ + --scoutsuite ./scoutsuite-report/ \ + --checkov ./results_json.json \ + --trivy ./trivy-image.json \ + --trivy ./trivy-fs.json \ + -o ./reports +``` + +```python tab="Python" title="ingest_reports_demo.py" +from cloudg import CloudGConfig, CloudGEngine + +engine = CloudGEngine(CloudGConfig()) +result = engine.run_from_reports_sync( + { + "prowler": ["./prowler-output/"], + "scoutsuite": ["./scoutsuite-report/"], + "checkov": ["./results_json.json"], + "trivy": ["./trivy-image.json", "./trivy-fs.json"], + }, + output_dir="./reports", +) + +print(result.total_findings, result.severity_breakdown) +print(result.report_paths) +print(result.errors) +``` + +```bash tab="Docker" +docker run --rm \ + -v "$PWD:/data" \ + ghcr.io/morpheuslord/cloudg:latest ingest \ + --prowler /data/prowler-output/ \ + --checkov /data/results_json.json \ + --trivy /data/trivy-image.json \ + -o /data/reports +``` + +| Flag | Accepts | +|---|---| +| `--prowler` | An ASFF JSON or JSONL file, or a directory searched recursively for `*.json` | +| `--scoutsuite` | A `scoutsuite_results*.js` file, or a report directory searched recursively for one | +| `--checkov` | A Checkov JSON report, or a directory containing `results_json.json` | +| `--trivy` | A `trivy image` or `trivy fs` JSON report, or a directory searched recursively for `*.json` | +| `--format` | `html`, `json` or `all` (default). `raw-findings.json` is written in every case. | +| `-o, --output` | Output directory, `./reports` by default | + +The global `-c` option matters here for one reason: `rulesets.rules_dir` decides which compliance rulesets the normaliser loads. See [Compliance frameworks](/guides/compliance/#adding-a-custom-ruleset). + +```bash +cloudg -c config.yaml ingest --prowler ./prowler-output/ -o ./reports +``` + +### Try it without any scanner {#try-it-without-any-scanner} + +The script below writes one small file per tool in each tool's native shape, the same shapes the test suite uses. Run it in an empty directory, then run the `cloudg ingest` command above. + +```python title="make_sample_reports.py" +"""Write one small native report per scanner, shaped like the real tools' output.""" +import json +from pathlib import Path + +Path("prowler-output").mkdir(exist_ok=True) +Path("prowler-output/prowler-output-123456789012.asff.json").write_text(json.dumps([ + { + "Id": "prowler-aws-s3_bucket_default_encryption-123456789012-eu-west-1-abc", + "Title": "S3 bucket default encryption", + "Description": "Bucket has no default encryption", + "Severity": {"Label": "HIGH"}, + "Resources": [{"Id": "arn:aws:s3:::my-bucket"}], + "Compliance": {"Status": "FAILED", "RelatedRequirements": ["CIS 2.1.1"]}, + "Remediation": {"Recommendation": {"Text": "Enable default encryption"}}, + } +])) + +results = Path("scoutsuite-report/scoutsuite-results") +results.mkdir(parents=True, exist_ok=True) +(results / "scoutsuite_results_aws-123456789012.js").write_text( + "scoutsuite_results =\n" + json.dumps({ + "services": {"s3": {"findings": {"s3-bucket-no-encryption": { + "description": "Bucket without encryption", + "rationale": "Data at rest should be encrypted", + "remediation": "Enable encryption", + "level": "danger", + "flagged_items": 1, + "items": ["s3.buckets.my-bucket"], + "references": [], + }}}} + }) +) + +Path("results_json.json").write_text(json.dumps({ + "check_type": "terraform", + "results": {"failed_checks": [{ + "check_id": "CKV_AWS_19", + "check_name": "Ensure S3 bucket has server-side encryption enabled", + "severity": "HIGH", + "resource": "aws_s3_bucket.data", + "file_path": "/main.tf", + "file_line_range": [1, 10], + "guideline": "https://docs.example/ckv-aws-19", + }]}, +})) + +Path("trivy-image.json").write_text(json.dumps({ + "ArtifactName": "myrepo/app:latest", + "ArtifactType": "container_image", + "Results": [{ + "Target": "myrepo/app:latest (alpine 3.19)", + "Type": "alpine", + "Vulnerabilities": [{ + "VulnerabilityID": "CVE-2024-0001", + "PkgName": "openssl", + "InstalledVersion": "3.1.0", + "FixedVersion": "3.1.1", + "Severity": "CRITICAL", + "Description": "Example vulnerability", + "CVSS": {"nvd": {"V3Score": 9.8}}, + }], + }], +})) + +Path("trivy-fs.json").write_text(json.dumps({ + "ArtifactName": "./iac", + "ArtifactType": "filesystem", + "Results": [{ + "Target": "main.tf", + "Type": "terraform", + "Misconfigurations": [{ + "ID": "AVD-AWS-0088", + "Title": "S3 bucket encryption not enabled", + "Description": "Bucket does not have encryption enabled", + "Severity": "HIGH", + "Resolution": "Enable encryption", + }], + }], +})) +print("sample reports written") +``` + +```console +$ python make_sample_reports.py +sample reports written +$ cloudg ingest --prowler ./prowler-output/ --scoutsuite ./scoutsuite-report/ --checkov ./results_json.json --trivy ./trivy-image.json --trivy ./trivy-fs.json -o ./reports + prowler: 1 findings + scoutsuite: 1 findings + checkov: 1 findings + trivy: 2 findings + ✓ JSON: reports/findings.json + ✓ HTML: reports/report.html + ✓ Raw findings: reports/raw-findings.json + + ✓ Ingested 5 findings from 4 tool(s), 5 after deduplication +``` + +## Input requirements {#input-requirements} + +Each parser expects one specific output format. Feed it anything else and the result ranges from zero findings to findings full of placeholders, so this section is worth reading before you wire up a CI job. + +### Prowler + +Produce ASFF: + +```bash +prowler aws -M json-asff -o ./prowler-output +prowler azure --az-cli-auth -M json-asff -o ./prowler-output-azure +``` + +cloudg accepts a single file or a directory. A directory is searched recursively for `*.json`, and every match is parsed as ASFF. Each file may be a JSON array or JSONL (one ASFF object per line). From each finding cloudg reads: + +| ASFF field | Becomes | +|---|---| +| `Id` | `source_finding_id`; the check name inside it (third dash-separated token) is the dedupe key | +| `Title`, `Description` | `title`, `description` | +| `Severity.Label` (or `ProductFields.Severity`) | `severity`; `informational` maps to INFO, unknown labels to MEDIUM | +| `Resources[0].Id` | `resource_arn` and `resource_id` | +| `Compliance.RelatedRequirements` | Coarse framework tags (`CIS`, `NIST-800-53`, `PCI-DSS`, `GDPR`, `HIPAA`, `SOC2`) | +| `Compliance.Status` | `PASSED` findings are dropped | +| `Remediation.Recommendation.Text` | `remediation` | +| `ProductFields` | `evidence` (first 1000 characters of the JSON) | + +:::warning Only ASFF in the Prowler directory +Prowler's other JSON format, OCSF (`*.ocsf.json`), also ends in `.json`. If the directory holds OCSF files, cloudg parses them as ASFF and produces one `Unknown Prowler Finding` at MEDIUM severity with no resource for every record, passing checks included. Run Prowler with `-M json-asff` only, or pass the `.asff.json` file itself. +::: + +### ScoutSuite + +Produce a report directory: + +```bash +scout aws --profile audit --report-dir ./scoutsuite-report --no-browser +``` + +cloudg accepts the report directory (searched recursively for `scoutsuite_results*.js`, which ScoutSuite puts in `scoutsuite-results/`) or the `.js` file itself. That file is JavaScript, one JSON object assigned to a variable; cloudg strips everything up to the first `=` and parses the rest. + +Findings come from `services..findings.`. A rule with `flagged_items` of 0 is skipped. Otherwise every entry in `items` becomes one finding, with the item string as the resource ID. ScoutSuite items are its own object paths, not ARNs, so ScoutSuite findings rarely merge with other tools' findings for the same resource. + +| ScoutSuite `level` | Severity | +|---|---| +| `danger` | CRITICAL | +| `warning` | HIGH | +| `caution` | MEDIUM | +| `good` | INFO | +| anything else | MEDIUM | + +The rule's `references` list is copied into `compliance_frameworks` as is. If your ScoutSuite rules put documentation URLs there, those URLs appear as framework names in the compliance output. + +### Checkov + +Produce JSON, either redirected from stdout or written by Checkov: + +```bash +checkov -d ./infra --output json > results_json.json +checkov -d ./infra --output json --output-file-path ./checkov-out +``` + +The second form writes `./checkov-out/results_json.json`. cloudg accepts a file or a directory; a directory is searched for `results_json.json` first, then for any `*.json`. Both shapes Checkov emits parse: a single `{check_type, results}` object, or a list of them when several frameworks ran. + +Only `results.failed_checks[]` is read. From each entry: `check_id` (dedupe key), `check_name` (the title, prefixed with `[Checkov/]`), `severity`, `resource` (Checkov's resource address, such as `aws_s3_bucket.data`), `file_path`, `file_line_range` and `guideline`. When `severity` is empty, as it usually is in plain open-source Checkov output, the finding is MEDIUM. Every Checkov finding is tagged `CIS`, plus `NIST-800-53` or `PCI-DSS` when the check ID contains those strings. + +### Trivy + +Produce either or both: + +```bash +trivy image --format json --output trivy-image.json 123456789012.dkr.ecr.us-east-1.amazonaws.com/api:1.4.2 +trivy fs --format json --scanners vuln,misconfig,secret --output trivy-fs.json ./infra +``` + +cloudg accepts a file or a directory of them (searched recursively for `*.json`). The top-level `ArtifactType` decides the parser: + +| `ArtifactType` | Parser | Reads | +|---|---|---| +| `container_image` | Image | `Vulnerabilities`, `Secrets` | +| anything else (`filesystem`, `repository`) | Filesystem | `Vulnerabilities`, `Misconfigurations`, `Secrets` | + +`ArtifactName` becomes the resource ID of every finding in that file. Vulnerabilities use `VulnerabilityID` as the check ID and read `PkgName`, `InstalledVersion`, `FixedVersion`, `Severity`, `Description` and the first `V3Score` found under `CVSS`. Misconfigurations use `ID` (the `AVD-*` identifier), `Title`, `Description`, `Severity` and `Resolution`. Secrets are always HIGH. Trivy's `UNKNOWN` severity maps to INFO. + +A `trivy fs` scan only reports misconfigurations when `misconfig` is in `--scanners`, which is why the command above lists it. Leave out `--include-non-failures`: cloudg doesn't look at a misconfiguration's status, so passing checks would come through as findings. + +## Several files per scanner + +Repeat the flag, point it at a directory, or both. In Python, the value for each tool is a list. + +```bash +cloudg ingest \ + --prowler ./prowler/account-111111111111/ \ + --prowler ./prowler/account-222222222222/ \ + --trivy ./trivy-results/ \ + -o ./reports +``` + +Paths are handled one at a time. A path that doesn't exist is reported (`trivy: skipping ./nope.json (Report path does not exist: ./nope.json)`) and skipped, and a file that isn't valid JSON is logged and skipped by the parser. Neither stops the other inputs. If nothing parses at all, cloudg warns `No findings parsed from the given reports` and still writes empty reports, with exit code 0. The only hard failure is calling `cloudg ingest` with no input flag, which exits 1. + +If you feed the same report twice, or two overlapping directories, the duplicates collapse during normalisation: within one scanner, findings with the same check ID on the same resource are merged into one. + +## Normalisation and deduplication {#normalisation-and-deduplication} + +The normaliser runs two deduplication passes, then rescoring, then compliance mapping. Compliance mapping has [its own page](/guides/compliance/); this is what happens to the findings themselves. + +Pass 1 works inside each scanner. The key is the tool, the check ID (or the normalised title when there is no check ID) and the resource. Two different checks that happen to share a title on the same bucket stay separate. + +Pass 2 works across scanners. Two findings from different tools merge only when all three of these hold: they're on the same resource string, their titles match after normalisation (lowercased, leading `[Tool]` tags removed, punctuation collapsed), and both check IDs map to the same canonical ID in `rules/check_equivalence.yaml`. The shipped file defines 7 canonical checks covering 18 scanner check IDs, such as Prowler `s3_bucket_default_encryption`, Checkov `CKV_AWS_19` and Trivy `AVD-AWS-0088` for S3 default encryption. A merged finding keeps the highest severity and lists every tool in `source_tool`, comma-separated. + +```python title="dedupe_demo.py" +from cloudg.normaliser import FindingsNormaliser +from cloudg.schema.models import Finding, Severity + +bucket = "arn:aws:s3:::my-bucket" +prowler = Finding( + resource_id=bucket, + resource_arn=bucket, + severity=Severity.HIGH, + title="S3 bucket default encryption", + description="Bucket has no default encryption", + source_tool="prowler", + source_finding_id="prowler-aws-s3_bucket_default_encryption-123456789012-eu-west-1-abc", +) +checkov = Finding( + resource_id=bucket, + resource_arn=bucket, + severity=Severity.CRITICAL, + title="[Checkov/terraform] S3 bucket default encryption", + description="S3 bucket default encryption", + source_tool="checkov", + source_finding_id="CKV_AWS_19", +) + +result = FindingsNormaliser().normalise([prowler, checkov]) +for f in result.findings: + print(f.severity.value, "|", f.source_tool, "|", f.title) +``` + +```console +$ python dedupe_demo.py +CRITICAL | checkov, prowler | [Checkov/terraform] S3 bucket default encryption +``` + +Expect that to be the exception with real reports. Prowler reports an ARN, Checkov a Terraform address and Trivy a directory or image name, and their titles are worded differently, so the sample reports above produce three separate S3 encryption findings. cloudg chooses visible duplicates over silently dropping one tool's result. + +Rescoring comes after deduplication. A finding with a CVSS score of 9.0 or more becomes CRITICAL; 7.0 or more raises it to at least HIGH. Every finding also carries `risk_score`, the severity's base score (9.5, 7.5, 5.0, 2.5 or 0.5), averaged with the CVSS score when there is one. The Trivy sample CVE above has CVSS 9.8, so its risk score is 9.7. + +## Parse first, normalise later + +`parse_report` and `ingest_reports` give you findings without running the normaliser or writing anything, which is useful when you want to filter or enrich first. + +```python title="parse_then_normalise.py" +from cloudg.ingest import parse_report, ingest_reports +from cloudg.normaliser import FindingsNormaliser + +# One tool at a time +prowler = parse_report("prowler", "./prowler-output/") +trivy = parse_report("trivy", "./trivy-image.json") +print(len(prowler), "prowler,", len(trivy), "trivy") + +# Or every tool in one call; bad paths are logged and skipped +findings = ingest_reports( + { + "prowler": ["./prowler-output/"], + "checkov": ["./results_json.json"], + "trivy": ["./trivy-image.json", "./trivy-fs.json"], + } +) + +scan_result = FindingsNormaliser().normalise(findings) +print(scan_result.summary["severity_breakdown"]) +for f in scan_result.findings: + print(f"{f.severity.value:8} {f.risk_score:4} {f.source_tool:10} {f.title}") +``` + +```console +$ python parse_then_normalise.py +1 prowler, 1 trivy +{'CRITICAL': 1, 'HIGH': 3} +CRITICAL 9.7 trivy [Trivy] CVE-2024-0001: openssl (alpine) +HIGH 7.5 prowler S3 bucket default encryption +HIGH 7.5 checkov [Checkov/terraform] Ensure S3 bucket has server-side encryption enabled +HIGH 7.5 trivy [Trivy/IaC] AVD-AWS-0088: S3 bucket encryption not enabled +``` + +The functions differ in how they fail. `parse_report` raises `ValueError` for a tool name it doesn't know (only `prowler`, `scoutsuite`, `checkov` and `trivy` are accepted, case-insensitive) and `FileNotFoundError` for a missing path. `ingest_reports` catches both, logs them and moves on. The per-scanner parsers are public too, as class methods: `ProwlerScanner.parse_report(path)`, `ScoutSuiteScanner.parse_report(path)`, `CheckovScanner.parse_report(path)` and `TrivyScanner.parse_report(path)`. + +On the engine, `CloudGEngine.ingest_reports()` wraps `ingest_reports` and fires the `on_phase_start("ingest")`, `on_finding` and `on_scan_complete` [event hooks](/api/event-hooks/). `run_from_reports()` and its sync wrapper go further and write `findings.json` and `report.html` (not `raw-findings.json`), returning a `PipelineResult` whose `report_paths` holds both paths and whose `errors` lists any phase that failed. + +## What comes out + +| File | Contents | +|---|---| +| `raw-findings.json` | A JSON list of every parsed `Finding`, before deduplication and compliance mapping (CLI only) | +| `findings.json` | `metadata`, `summary`, `findings`, `compliance` and an empty `assets` list and `graph` | +| `report.html` | The interactive report: findings table, analytics, compliance cards | + +With no assets, the report's Infrastructure Map tab falls back to a force graph of finding nodes linked to compliance framework nodes. The [Reports](/guides/reports/) guide covers both files in detail, and the field-level layout of `findings.json` is on [Output files](/reference/output-files/). + +## Combining with a live inventory {#combining-with-a-live-inventory} + +Ingest needs no credentials; mapping needs read-only ones. Run them separately, at different times if you like, and merge at the end. [`cloudg map`](/guides/inventory-mapping/) accepts a cloudg findings file and writes `asset-map.json` (each asset with its findings) and `compliance-map.json` (each framework with its affected assets): + +```bash +cloudg ingest --prowler ./ci/prowler-output/ --checkov ./ci/results_json.json -o ./reports +cloudg map -p aws --regions us-east-1 --findings ./reports/findings.json -o ./reports +``` + +From Python, ingest and normalise against collected assets so findings sit next to the inventory in one `ScanResult`: + +```python title="collect_and_ingest.py" +import asyncio + +from cloudg import CloudGConfig, CloudGEngine + + +async def main() -> None: + engine = CloudGEngine(CloudGConfig(providers=["aws"])) + + collection = await engine.collect() # needs AWS credentials + ingested = engine.ingest_reports({"prowler": ["./ci/prowler-output/"]}) + scan_result = engine.normalise_findings(ingested, assets=collection.assets) + print(len(scan_result.assets), "assets,", len(scan_result.findings), "findings") + + +asyncio.run(main()) +``` + +## Next + +:::links +- [Compliance frameworks](/guides/compliance/) How findings are tagged with controls, and how to add your own. +- [Reports](/guides/reports/) Reading `findings.json` and `report.html`, and re-rendering them. +- [cloudg ingest](/cli/ingest/) The command reference. +- [parse_report](/api/parse-report/) The Python parser entry point. +::: diff --git a/site/content/guides/installation.md b/site/content/guides/installation.md new file mode 100644 index 0000000..c0b2880 --- /dev/null +++ b/site/content/guides/installation.md @@ -0,0 +1,245 @@ +--- +title: Installation +lede: "cloudg is a Python package with optional extras per cloud provider. The security scanners it drives are separate programs that you install next to it, or get prebuilt in the Docker image." +meta: + - [Python, "3.11 or newer"] + - [Package, "`cloudg` on PyPI"] + - [Image, "`ghcr.io/morpheuslord/cloudg`"] +source: pyproject.toml +--- + +There are two things to install, and it helps to keep them apart. The first is cloudg itself: the CLI, the graph engine, the normaliser, the report renderers and, through extras, the cloud SDKs. The second is the set of scanners (Prowler, ScoutSuite, Checkov, Trivy) that `cloudg run` and `cloudg scan` call as subprocesses. cloudg works with any subset of them, including none. + +## Requirements + +cloudg needs Python 3.11 or newer (`requires-python = ">=3.11"`). The Docker image is built on `python:3.12-slim`. Nothing else is required for the core package: no database, no daemon, no Node.js. + +Credentials are not needed to install, and not needed at all for `cloudg ingest`, `cloudg report`, `cloudg deps` or the MCP server reading saved files. Only the commands that talk to a cloud (`run`, `collect`, `scan`, `map`) need them. + +## Install cloudg + +```bash tab="pip" +python3 -m venv .venv +source .venv/bin/activate +pip install "cloudg[all]" +``` + +```bash tab="uv" +# as a standalone CLI in its own environment +uv tool install "cloudg[all]" +# or into the current project's virtualenv +uv pip install "cloudg[all]" +``` + +```bash tab="Docker" +docker pull ghcr.io/morpheuslord/cloudg:latest +docker run --rm ghcr.io/morpheuslord/cloudg:latest --version +``` + +Quote the package spec. In zsh, unquoted square brackets are a glob pattern and `pip install cloudg[all]` fails with "no matches found". + +The Docker image already contains cloudg, the AWS, Azure and GCP SDKs, Prowler, Checkov, Trivy and Parliament. It is the shortest way to the full pipeline; [Docker](/guides/docker/) covers running it with credentials and volumes. + +### From a source checkout + +Contributors, and anyone who wants the newest code on `main`, install in editable mode: + +```bash +git clone https://github.com/morpheuslord/cloudg.git +cd cloudg +uv venv +source .venv/bin/activate +uv pip install -e ".[all,dev]" +``` + +Plain pip works the same way: `pip install -e ".[all,dev]"`. + +## Extras + +The cloud SDKs are optional, so a machine that only aggregates scanner output does not pull in boto3 or the Azure management libraries. + +| Extra | Adds | Install it when | +|---|---|---| +| none | pydantic, click, rich, PyYAML, NetworkX, rdflib, python-louvain, Parliament, Jinja2, svgwrite, aiofiles | You only ingest existing scanner output, re-render reports, or explore saved maps | +| `aws` | `boto3>=1.28`, `aioboto3>=12.0` | You collect from or map AWS | +| `azure` | `azure-identity`, the compute, network, storage, resource, SQL and Key Vault management SDKs, `azure-mgmt-resourcegraph` | You collect from or map Azure | +| `gcp` | `google-cloud-asset>=3.0`, `google-auth>=2.22` | You collect from or map GCP | +| `mcp` | the official MCP SDK, `mcp>=1.30,<3` | You want the SDK flavor of the MCP server or to mount cloudg into an SDK server. The native MCP server needs no extra | +| `all` | `aws`, `azure`, `gcp` and `mcp` together | You are not sure yet | +| `dev` | `aws`, `mcp`, pytest, pytest-asyncio, `moto[all]`, ruff | You work on cloudg itself | +| `full` | diagrams, weasyprint, policy-sentry | Rarely; see the note below | + +Extras combine: `pip install "cloudg[aws,gcp]"` gets exactly two providers. + +:::note What the core package can do on its own +Without any extra you still get the graph builder, the ontology, the RAG exporter, the findings normaliser with every compliance ruleset, all four scanner parsers, the HTML, JSON and SVG renderers, and the MCP layer with its native server. That is enough for `cloudg ingest` on an air-gapped analysis host. +::: + +About `full`: in 0.6.0 the IAM linter only checks whether `policy_sentry` can be imported, and nothing in the package imports `diagrams` or `weasyprint`. Installing `full` does no harm, but it changes no output today. + +If the extra for a provider is missing, collection from that provider cannot import its SDK and fails; the error is logged and the other providers carry on. Install the extra and run again. + +## Scanner binaries {#scanner-binaries} + +cloudg looks each scanner up on `PATH` at the moment it runs it. A missing one is logged ("Prowler is not installed... Skipping Prowler scan.") and the pipeline carries on with the others. So you can start with nothing and add scanners as you need them. + +| Scanner | Executable | Install | Used by | +|---|---|---|---| +| Prowler | `prowler` | `pip install prowler` | `run`, `scan`: once per provider | +| ScoutSuite | `scout` | `pip install scoutsuite` | `run`, `scan`: once per provider | +| Checkov | `checkov` | `pip install checkov` | `run`, `scan`: once per IaC directory | +| Trivy | `trivy` | binary, see below | `run`, `scan`: images, or a filesystem fallback | +| IAM linter | none | ships with cloudg (Parliament is a core dependency) | `run`, `scan`: always, when there are IAM assets | + +Prowler, ScoutSuite and Checkov are Python applications with large, pinned dependency trees. Putting them in the same virtualenv as cloudg works until two of them disagree about a version. The Docker image avoids that by giving Prowler and Checkov a venv each and symlinking their executables into `/usr/local/bin`. On a workstation, `pipx` or `uv tool` gives you the same isolation: + +```bash +uv tool install prowler +uv tool install checkov +uv tool install scoutsuite +``` + +Both tools put the executables in `~/.local/bin`, which must be on `PATH` for cloudg to find them. + +Trivy is a Go binary. Use your package manager or Aqua Security's install script: + +```bash tab="macOS" +brew install trivy +``` + +```bash tab="Linux" +curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin +``` + +```bash tab="Windows" +winget install AquaSecurity.Trivy +``` + +The scanners authenticate on their own. cloudg passes Prowler the AWS profile (`-p`) and, when you gave direct keys, the key pair through the environment; ScoutSuite gets the AWS profile. For Azure and GCP they read the same environment and credential files the SDKs do. If a scanner works when you run it by hand, it works under cloudg. + +## Installer scripts + +`install.sh` (Linux and macOS) and `install.bat` (Windows) set up a complete workstation in one go. Run them from the root of a source checkout, because they install cloudg in editable mode from the current directory: + +```bash tab="Linux / macOS" +git clone https://github.com/morpheuslord/cloudg.git +cd cloudg +./install.sh +``` + +```bash tab="Windows" +git clone https://github.com/morpheuslord/cloudg.git +cd cloudg +install.bat +``` + +`install.sh` detects the package manager (apt, dnf, yum, pacman, zypper or Homebrew) and then, in order: + +1. installs curl, git and unzip if they are missing, and finds or installs Python 3.11 or newer; +2. installs Graphviz and Node.js 20; +3. installs uv, creates `.venv` and runs `uv pip install -e ".[all,dev]"`, then tries `.[full]`; +4. installs Prowler, Checkov, ScoutSuite, Parliament and policy-sentry into that venv, Trivy through its install script or the distro repository, CloudSploit through npm, and Cloud Custodian (`c7n`); +5. installs the AWS CLI v2, the Azure CLI and the gcloud CLI where they are missing; +6. runs `cloudg --version` and prints which tools it found. + +`install.bat` does the same on Windows with winget or Chocolatey. It stops if Python is older than 3.11, and if it has to install Python through winget it asks you to open a new terminal and run it again. + +Both scripts use `sudo` (on Linux) or system package managers, and they install more than cloudg strictly needs. CloudSploit and Cloud Custodian are not called by cloudg; Custodian is there so you can run the policy packs in `cloudg/policies/` yourself. Graphviz is not needed either, since the SVG topology is drawn with svgwrite. If you prefer a lean setup, use pip or uv and add only the scanners you want. + +Each step logs `[OK]` or `[WARN]` and keeps going, so read the summary at the end. Afterwards, activate the environment: + +```bash +source .venv/bin/activate +``` + +## Verify the install + +Check the version and that the command group loads: + +```console +$ cloudg --version +cloudg, version 0.6.0 +$ cloudg --help +Usage: cloudg [OPTIONS] COMMAND [ARGS]... +... +Commands: + collect Collect cloud assets from the specified provider. + deps Explore interdependencies in a saved inventory map. + ingest Aggregate existing scanner outputs; no scanners are executed. + map Map the complete infrastructure inventory, with no scanners... + mcp Model Context Protocol server: expose cloudg to AI assistants. + report Generate reports from existing scan data. + run Run the full pipeline: collect → scan → normalise → render. + scan Run security scanners and generate findings. +``` + +Then see which scanners cloudg will find: + +```bash +command -v prowler scout checkov trivy +``` + +Anything missing from that list is skipped at run time. + +Last, an end-to-end smoke test that needs no credentials. Save a minimal Trivy result: + +```json title="trivy-sample.json" +{ + "ArtifactName": "myrepo/app:latest", + "ArtifactType": "container_image", + "Results": [ + { + "Target": "myrepo/app:latest (alpine 3.19)", + "Type": "alpine", + "Vulnerabilities": [ + { + "VulnerabilityID": "CVE-2024-0001", + "PkgName": "openssl", + "InstalledVersion": "3.1.0", + "FixedVersion": "3.1.1", + "Severity": "CRITICAL", + "Description": "Example vulnerability for a smoke test" + } + ] + } + ] +} +``` + +and feed it to `cloudg ingest`: + +```bash +cloudg ingest --trivy trivy-sample.json -o ./smoke-test +``` + +You should see "Ingested 1 findings from 1 tool(s), 1 after deduplication" and three files in `./smoke-test`: `findings.json`, `raw-findings.json` and `report.html`. If that works, the parser, normaliser, compliance rulesets and renderers are all installed correctly. + +## Upgrading + +```bash tab="pip" +pip install --upgrade "cloudg[all]" +``` + +```bash tab="uv" +uv tool upgrade cloudg +``` + +```bash tab="Docker" +docker pull ghcr.io/morpheuslord/cloudg:latest +``` + +For a source checkout, `git pull` and then reinstall: `uv pip install -e ".[all,dev]"`. Reinstalling matters when the entry points change. The `cloudg-mcp` console script arrived in 0.6.0, so an editable install made before that does not have it until you reinstall. + +Image tags follow the release version (`ghcr.io/morpheuslord/cloudg:0.6.0`), and `latest` moves with each release. Pin the version tag in CI so an upgrade happens when you choose. + +Configuration files carry over. A `config.yaml` that still uses the old single `provider:` key is migrated to `providers: [...]` when it loads. Read the [changelog](/changelog/) before upgrading across a minor version; it lists every changed default and output field. + +## Troubleshooting + +`cloudg: command not found` usually means the virtualenv is not active, or `~/.local/bin` is not on `PATH` after `uv tool install`. `python -m cloudg.cli --help` runs the same CLI from whichever interpreter you call it with. + +If pip says no matching distribution was found, or mentions "a different Python", the interpreter is older than 3.11. Create the venv with a newer one, for example `uv venv --python 3.12`. + +A scanner that works in your shell but is "not installed" under cloudg is on a different `PATH`. That happens with scanners installed in a second virtualenv that is not active. Use `uv tool` or `pipx`, or symlink the executable somewhere on `PATH`. + +Next: run your first scan in the [quickstart](/guides/quickstart/). diff --git a/site/content/guides/introduction.md b/site/content/guides/introduction.md new file mode 100644 index 0000000..7cf8ac3 --- /dev/null +++ b/site/content/guides/introduction.md @@ -0,0 +1,128 @@ +--- +title: Introduction +lede: "cloudg (cloud graphing) maps AWS, Azure and GCP infrastructure into a graph, runs security scanners over the same inventory, and merges what they find into one deduplicated, compliance-mapped result." +meta: + - [Version, "0.6.0"] + - [Python, "3.11 or newer"] + - [Providers, "AWS, Azure, GCP"] + - [License, "MIT"] +source: README.md +--- + +cloudg is a Python command-line tool and a library. You point it at one or more cloud accounts and it builds an inventory of what is deployed there: instances, buckets, roles, security groups, load balancers, functions, key vaults, projects. Those assets and the network edges between them go into a directed graph. From the graph cloudg works out what the internet can reach, which paths an attacker could take sideways, and how much of the estate each node could touch if it were compromised. + +The same inventory then goes to the security scanners you already know (Prowler, ScoutSuite, Checkov and Trivy), plus a built-in IAM policy linter. Each scanner reports the same problem in its own words, so cloudg normalises every finding into one model, merges duplicates within and across tools, rescores them against CVSS, and maps them onto controls from 28 compliance frameworks. What you open at the end is a single HTML report, not four. + +None of that requires the whole pipeline. Each half of cloudg works without the other. + +## Three ways in + +Most people arrive with one of three questions, and cloudg has a command for each. + +:::cells +### cloudg run +"What is wrong with my cloud?" Collects every configured provider, builds the graph, runs the scanners in parallel, normalises the findings and renders the reports. Needs credentials and, for full coverage, the scanner binaries. +### cloudg ingest +"I already ran the scanners, can you make sense of the output?" Reads native Prowler, ScoutSuite, Checkov and Trivy files, then deduplicates, maps compliance and renders reports. No cloud credentials and no scanner binaries. +### cloudg map +"What do I have, and how is it wired?" A scanner-free inventory mapper: deep collectors, a sweep across every service, typed relationships and dependency analysis. No scanner runs or needs to be installed. +::: + +They are independent but they compose. You can `cloudg map` an account today, run Prowler in CI next week, and merge that output into the map later with `cloudg map --findings`. You can collect live assets from Python and normalise findings you ingested from someone else's pipeline in the same script. The [quickstart](/guides/quickstart/) walks through all three. + +### Choosing between them + +| You have | You want | Use | +|---|---|---| +| Read-only cloud credentials and the scanners installed | A full security picture with a topology graph | `cloudg run` | +| Scanner output files from CI or another team, no cloud access | One deduplicated report with compliance mapping | `cloudg ingest` | +| Read-only credentials, no interest in scanners right now | A complete asset map with dependencies and blast radius | `cloudg map` | +| A management account in AWS Organizations | Every member account mapped in one run | `cloudg map --org` | +| An AI assistant that speaks MCP | Questions answered from your inventory and findings | `cloudg mcp serve` | + +`cloudg run` and `cloudg map` collect differently. The pipeline uses a fixed set of service collectors per provider (on AWS that is EC2, S3, RDS, VPCs, subnets, security groups, IAM users and roles, Lambda, ELBv2, ECS, DynamoDB, CloudFront, Secrets Manager and KMS), which is what the scanners and the reachability analysis need. The inventory mapper goes much further: 133 AWS collector tasks, a Cloud Control API sweep over every listable resource type, Azure Resource Graph across subscriptions and GCP Cloud Asset Inventory across the organization. If you want to know everything that exists, use `map`. If you want to know what is exposed and misconfigured, use `run`. + +## How the pieces fit + +```mermaid caption="From cloud APIs to the files and tools you use" +flowchart LR + P["AWS, Azure, GCP APIs"] --> C[Collectors] + C --> G["Graph (NetworkX)"] + C --> S[Scanners] + G --> R["Reachability findings"] + S --> N[Normaliser] + R --> N + N --> O["Reports: HTML, JSON, SVG"] + G --> K["Ontology (RDF)"] + G --> A["RAG chunks"] + N --> M["MCP server"] + G --> M +``` + +Collectors talk to the provider APIs through a shared rate limiter, so a big account slows down instead of failing. In the NetworkX graph, edge endpoints that are not assets (the `0.0.0.0/0` on a security group rule, for example) become external nodes, and a breadth-first search from the internet sources `0.0.0.0/0` and `::/0` over network-flow edges is how cloudg decides what is internet-reachable. Scanners run as separate processes, each in its own worker thread, and anything not installed is skipped with a warning. The normaliser is the meeting point: scanner findings, the graph's reachability findings and the IAM linter's results all go through the same deduplication and compliance mapping. + +The MCP server sits beside all of this. It loads the files a run or a map produced (or runs collection itself, when you allow the live tools) and exposes them to an AI client as tools, resources and prompts, behind a privacy policy that redacts secrets by default. + +## Providers + +| Provider | Install extra | How cloudg reads it | Default regions | +|---|---|---|---| +| AWS | `cloudg[aws]` (boto3, aioboto3) | Per account and region, with STS role assumption for multi-account fan-out | `us-east-1` | +| Azure | `cloudg[azure]` (azure-identity and the management SDKs) | Per subscription; every enabled subscription the credential can see when none is set | all locations | +| GCP | `cloudg[gcp]` (google-cloud-asset, google-auth) | Cloud Asset Inventory, per project or once at organization scope | all regions | + +Every provider supports several authentication methods, resolved in a fixed order: direct keys, OIDC federation, profiles and service principals, managed identities, and finally each SDK's default chain. The same configuration works on a laptop, in CI and on cloud compute. [Authentication](/guides/authentication/) has the details. + +## Scanners + +| Scanner | What it checks | Executable cloudg looks for | +|---|---|---| +| Prowler | Live cloud configuration against its check library, once per provider | `prowler` | +| ScoutSuite | Live cloud configuration, once per provider | `scout` | +| Checkov | Infrastructure-as-code in a directory, or the Terraform cloudg generates from the live estate | `checkov` | +| Trivy | Container images, or a filesystem when no images are given | `trivy` | +| IAM linter | IAM policies on collected roles and policies, using Parliament | built in | + +The scanners are executables, not Python dependencies. cloudg checks `PATH` when it runs each one and skips the missing ones. [Installation](/guides/installation/#scanner-binaries) shows how to install each, and [Running scanners](/guides/running-scanners/) covers what cloudg passes to them. + +## What you get + +A `cloudg run` writes everything into `./reports` (change it with `-o`): + +| File | What it is | +|---|---| +| `report.html` | Interactive report in one file: D3 topology, findings table, compliance matrix. The data is embedded; Chart.js and D3 load from a CDN. | +| `findings.json` | Machine-readable result: metadata, summary, assets, findings, compliance and graph data. `cloudg report -i` reads it back. | +| `topology.svg`, `topology.graphml`, `topology-cytoscape.json` | The graph as an image, for Gephi or yEd, and for Cytoscape | +| `ontology.ttl`, `ontology.jsonld` | RDF ontology with 64 inferred relation types, queryable with SPARQL | +| `rag_chunks.jsonl`, `rag_metadata_index.json` | Retrieval-ready chunks of the infrastructure for LLM pipelines | +| `terraform/*.tf.json` | Terraform recreation of the live infrastructure, with `--terraform` | + +`cloudg ingest` writes `findings.json`, `report.html` and `raw-findings.json`. `cloudg map` writes `inventory-map.json`, `inventory-map.graphml`, `inventory-graph.json` and `inventory-dependencies.json`, plus `inventory-organization.json` with `--org` and `asset-map.json` and `compliance-map.json` with `--findings`. [Output files](/reference/output-files/) describes each one field by field. + +## Using it from Python + +The CLI is a thin layer. `CloudGEngine` runs the whole pipeline or any single phase, with hooks that fire as findings arrive: + +```python title="first_run.py" +from cloudg import CloudGConfig, CloudGEngine + +engine = CloudGEngine(CloudGConfig(providers=["aws"])) +engine.on_phase_start = lambda phase: print("phase:", phase) + +result = engine.run_pipeline_sync(output_dir="./reports") +print(result.to_summary()) +``` + +The graph builder, ontology, RAG exporter, scanner parsers, normaliser, inventory mapper and MCP layer are all importable on their own. [The pipeline](/guides/pipeline/) shows the phases from both the CLI and Python, and the [Python API](/api/) tab documents every class. + +## Where to go next + +:::links +- [Installation](/guides/installation/) pip, uv or Docker, the provider extras, and the scanner binaries. +- [Quickstart](/guides/quickstart/) A first report in a few minutes, with or without credentials. +- [The pipeline](/guides/pipeline/) What each phase of cloudg run reads, writes and does when something fails. +- [Inventory mapping](/guides/inventory-mapping/) The scanner-free map of everything deployed. +- [Ingesting existing output](/guides/ingesting/) Reports from scanner files you already have. +- [MCP server](/mcp/) Hand the inventory and findings to an AI assistant. +::: diff --git a/site/content/guides/inventory-mapping.md b/site/content/guides/inventory-mapping.md new file mode 100644 index 0000000..f865a58 --- /dev/null +++ b/site/content/guides/inventory-mapping.md @@ -0,0 +1,343 @@ +--- +title: Inventory mapping +lede: "`cloudg map` asks a different question from `cloudg run`: what exists, and how is it wired together?" +meta: + - [Command, "`cloudg map`"] + - [Providers, "AWS, Azure, GCP"] +source: docs/DOCUMENTATION.md +--- + +`cloudg map` reads your cloud accounts and writes down everything it finds, deployed resources and defaults alike, then links them into one graph of typed relationships. A Lambda function points at the role it runs as, the queue that triggers it, the image it was built from and the KMS key that encrypts its environment. A WAF points at the load balancer it protects. An SCP points at the accounts it restricts. + +No scanner runs during a map, and none needs to be installed. The command only needs read access to the cloud APIs. + +:::note +The map doesn't depend on the scanners. You can map today and pull in next week's scan findings with `--findings ./reports/raw-findings.json`. On the CLI that flag belongs to `cloudg map`, so it maps again; to overlay findings onto a map you already saved without calling the cloud, use the Python call in [Overlay scanner findings](#overlay-scanner-findings). +::: + +## Map an account + +The smallest useful run maps one AWS account in every enabled region: + +```bash tab="CLI" +cloudg map -p aws --profile audit --regions all +``` + +```python tab="Python" title="map_account.py" +from cloudg import CloudGConfig, CloudGEngine + +config = CloudGConfig(providers=["aws"]) +config.aws.profile = "audit" +config.aws.regions = ["ALL"] + +engine = CloudGEngine(config) +inventory = engine.map_inventory_sync(output_dir="./reports") + +summary = inventory.summary +print(summary["total_assets"], "assets,", summary["total_edges"], "edges") +print("internet-exposed:", summary["internet_exposed"]) +print("unlinked:", summary["unlinked_assets"]) +``` + +```bash tab="Docker" +docker run --rm \ + -v ~/.aws:/home/cloudg/.aws:ro \ + -v "$PWD/reports:/app/reports" \ + ghcr.io/morpheuslord/cloudg:latest \ + map -p aws --profile audit --regions all +``` + +The image runs as the user `cloudg` with its home at `/home/cloudg`, which is why the AWS config directory is mounted there. Reports land in `/app/reports`, the container's `./reports`. + +While it runs, the CLI prints a "Map Configuration" panel (providers, regions, selected services, whether the Kubernetes, Cloud Control and catch-all sweeps are on) and then an "Inventory Summary" table: assets, interconnections, accounts, services, internet-exposed assets, cross-account edges, external accounts referenced, security service gaps, unlinked assets and unresolved references. Two ranked tables follow, "Most shared dependencies" and "Largest blast radius", and a warning line for each security service that is not enabled. [Dependencies and blast radius](/guides/dependencies/) explains how to read them. + +Pass a config file with the group option, before the subcommand: `cloudg -c config.yaml map ...`. Without `-c`, cloudg uses its built-in defaults and does not look for a `config.yaml` on its own. + +## Regions + +`--regions` takes `all` or a comma-separated list such as `us-east-1,eu-west-1`. The same value is applied to AWS, Azure and GCP. Leave it out and AWS maps `us-east-1` only, because that is the default of `aws.regions`. + +With `all`, cloudg discovers the enabled regions first and runs one collection unit per account and region. Account-wide services (IAM, Route 53, ECR Public, Direct Connect gateways and the other global tasks) run once per account, in the primary region: `us-east-1` when it is part of the run, otherwise the first region in the list. So `--regions all` doesn't collect IAM once per region. + +With `--org` and Control Tower, `--regions all` is replaced by the landing zone's governed regions. An explicit list is always used as given. The [AWS Organizations](/guides/aws-organizations/) guide covers that case. + +## Many accounts + +`--accounts 111122223333,444455556666 --role-name cloudg-readonly` maps a fixed list of accounts by assuming that role in each one. `--org` discovers the accounts instead, from the management account, and maps the OU tree, SCPs and Control Tower with them: + +```bash +cloudg map -p aws --org --org-role cloudg-readonly --regions all +``` + +:::warning AWSControlTowerExecution has admin rights +Without `--org-role` (or `--role-name`), cloudg assumes `AWSControlTowerExecution` in every member account. That role exists wherever Control Tower enrolled an account, but it carries administrator permissions. Deploy a read-only role to every account and pass it with `--org-role`. [AWS Organizations](/guides/aws-organizations/) has a StackSet template for it. +::: + +Always pair `--accounts` with `--role-name`. Without a role name cloudg doesn't assume anything and maps your own account once per listed ID, labelled with each of those IDs. + +## What gets collected + +Coverage comes in layers. Each one catches what the previous one missed. + +:::cells +### Deep collectors +131 dedicated AWS collector tasks in 19 service families, each reading one service in depth and declaring what every asset talks to. Azure goes through Resource Graph, GCP through Cloud Asset Inventory. +### Breadth sweep +On AWS, a Cloud Control sweep lists every CloudFormation resource type that has a list handler, tagged or not, so services without a dedicated collector still show up. The tagging API sweep adds tagged resources on top. +### Every account +Every account gets a `CLOUD_ACCOUNT` node that contains its top-level resources. With `--org`, accounts hang under the OU tree next to SCPs and Control Tower controls. +::: + +When the same resource turns up twice, the copy from a dedicated collector wins over a Cloud Control hit, which wins over a tagging API hit. Their declared relationships and aliases are merged into the copy that is kept. + +Values that could be secret are never collected: SSM parameter values, environment variable values, passwords, VPN pre-shared keys, Direct Connect auth keys. The Cloud Control sweep strips secret-looking properties at any depth before it stores a resource model. + +Security services that are switched off still appear. GuardDuty, Security Hub, Inspector, Macie, Config, Access Analyzer, Detective, CloudTrail, WAF, Network Firewall and Shield each produce a node per account and region; when one is not enabled (or access to it is denied) the node carries `enabled: false`. That is how detection gaps end up on the map instead of being silently absent. + +## Choose service families + +`--services` and `--exclude-services` take family names or individual task names, comma-separated. The exclusion always wins. + +| Family | What it covers | +|---|---| +| `network` | VPCs, subnets, security groups, route tables, gateways, ENIs, NACLs, peering, transit gateways, endpoints and PrivateLink, load balancers and listeners, CloudFront, Global Accelerator, VPC Lattice | +| `hybrid` | site-to-site VPN, customer and virtual private gateways, Direct Connect, Cloud WAN | +| `compute` | EC2, Auto Scaling, launch templates, AMIs and their sharing, Batch, App Runner, Elastic Beanstalk | +| `containers` | ECR, ECS, Cloud Map, EKS clusters, nodegroups, Fargate profiles, addons, access entries | +| `serverless` | Lambda, API Gateway (REST and HTTP), authorizers, VPC links, custom domains, Step Functions | +| `integration` | SQS, SNS, EventBridge, Scheduler, Pipes, Kinesis, Firehose, MSK, Amazon MQ | +| `storage` | S3, access points, EBS volumes and snapshots, EFS, FSx, Transfer Family, DataSync | +| `data` | RDS, Aurora, proxies, DynamoDB, ElastiCache, MemoryDB, DAX, OpenSearch, Redshift | +| `analytics`, `ml` | Glue, Lake Formation, EMR, Athena; SageMaker and Bedrock | +| `identity` | the IAM graph, IAM Identity Center, Roles Anywhere, Cognito, KMS, Secrets Manager, ACM | +| `governance` | RAM shares, Service Catalog, CloudFormation StackSets | +| `security` | GuardDuty, Security Hub, Inspector, Macie, Config, Access Analyzer, Detective, WAF, Network Firewall, Shield | +| `logging`, `operations`, `backup` | CloudTrail, flow logs, log groups and subscriptions, OAM; CloudWatch alarms, Systems Manager; AWS Backup | +| `dns`, `iac`, `cicd` | Route 53 and Resolver; CloudFormation stacks; CodePipeline, CodeBuild, CodeDeploy, CodeConnections | +| `sweep` | the two breadth sweeps, `cloud_control` and `tagging_sweep` | + +That makes 133 AWS tasks in all. The task-by-task list, with each task's family and whether it runs once per account, is in the [inventory internals](https://github.com/morpheuslord/cloudg/blob/main/docs/INVENTORY_INTERNALS.md#53-service-families-and-tasks) document. + +```bash +# containers and serverless wiring only +cloudg map -p aws --regions all --services containers,serverless,sweep +# everything except the data stores and one noisy task +cloudg map -p aws --regions all --exclude-services data,cloudformation +``` + +:::warning The sweeps are a family too +The two sweeps belong to the family `sweep`. Once you pass `--services`, only what you listed runs, so `--services containers,serverless` also drops the Cloud Control and tagging API sweeps. Add `sweep` to the list if you want them. A family name that matches nothing (a typo, say) is not an error; it selects no tasks. +::: + +Narrowing the run has a side effect on the links. A function whose role lives in the `identity` family can't be linked to that role when `identity` is excluded, so the reference shows up under `unresolved_references` instead of as an edge. + +## Kubernetes inside EKS + +With the `containers` family selected (or `eks`, or `kubernetes`), cloudg also reads the objects inside every `ACTIVE` EKS cluster that has an endpoint: namespaces, Deployments, StatefulSets, DaemonSets, CronJobs, Services, Ingresses and ServiceAccounts. Workloads are linked to the ECR repositories their images come from and to the IAM role their service account maps to (IRSA or Pod Identity). Services are linked to the workloads their selector matches and to the load balancer behind their hostname. + +It authenticates the way `aws eks get-token` does, with a presigned STS request, and only issues `GET`s. Secrets are never read. The identity cloudg runs as needs network reach to the cluster endpoint and read access inside the cluster. An access entry with the `AmazonEKSViewPolicy` is enough: + +```bash +aws eks create-access-entry \ + --cluster-name prod-eks \ + --principal-arn arn:aws:iam::123456789012:role/cloudg-readonly +aws eks associate-access-policy \ + --cluster-name prod-eks \ + --principal-arn arn:aws:iam::123456789012:role/cloudg-readonly \ + --policy-arn arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy \ + --access-scope type=cluster +``` + +A cluster that can't be reached (a private endpoint, a missing access entry) records `kubernetes:` as `FAILED` in coverage. The rest of the EKS collection is unaffected. Turn in-cluster mapping off with `--no-kubernetes`; the default timeout per API call is 10 seconds (`inventory.kubernetes_timeout`). + +## Breadth sweeps + +Two sweeps catch what no deep collector reads. They are separate flags. + +`--cloud-control / --no-cloud-control` (on by default) controls the AWS Cloud Control API sweep. cloudg lists every CloudFormation resource type with a list handler (more than 800), skips the ones a deep collector already maps, runs account-wide types once per account and lists the rest with six types in parallel per region. Each hit becomes an asset with `discovered_via: "cloud-control"` and its redacted resource model under `properties`. The list of listable types is discovered once and cached for a week in `$CLOUDG_CACHE_DIR` (default `~/.cache/cloudg`). Types that need a parent identifier are skipped, and types that fail to list are counted per error code in the `cloud_control_skipped_types` coverage entry. To narrow it from config, set `inventory.cloud_control_types` or `inventory.cloud_control_exclude` to type names or prefixes such as `AWS::Glue::`. + +`--sweep / --no-sweep` (on by default) controls the Resource Groups Tagging API sweep. AWS only returns resources that have been tagged at some point, so this sweep mostly enriches; the Cloud Control sweep is the one that finds untagged resources. Hits carry `discovered_via: "tagging-api"`. + +:::note +`cloudg map` always hands the value of `--sweep` to the mapper, and that value defaults to on, so the CLI overrides `inventory.tagging_sweep: false` from your config file even when you don't type the flag. Use `--no-sweep` on the command line to turn the tagging sweep off. +::: + +## Azure and GCP + +Azure is read through Azure Resource Graph, which returns every resource with its full properties in one query per subscription. That is what lets PaaS services get real relationships: managed identities and their role assignments, private endpoints, AKS node pools and ACR pulls, App Service VNet integration, Application Gateway and Front Door backends, Key Vault access policies and Defender for Cloud plans. Resource Graph needs `azure-mgmt-resourcegraph`, which comes with the `azure` extra (`pip install "cloudg[azure]"`). Without it cloudg falls back to the per-service SDK collectors and the ARM resource sweep. + +Pass one subscription with `--subscription-id`. With none configured and `azure.all_subscriptions: true` (the default), every Enabled subscription the credential can see is mapped. `azure.map_management_groups` adds the management group tree, Azure Landing Zone archetypes and Azure Policy assignments. The identity needs Reader at the tenant root management group. + +GCP is read through Cloud Asset Inventory `list_assets` with the full resource JSON. When `gcp.organization_id` is set, the listing runs once at the organization scope; otherwise once per project (`--project-id` for a single one). Relations cover instances, firewall semantics, the load-balancing chain down to NEGs and Cloud Armor, GKE with Workload Identity, Cloud Run and Functions v2, Pub/Sub, Eventarc, logging sinks, CMEK keys and Shared VPC. With an organization ID, the organization, folder and project tree, organization policies and VPC Service Controls perimeters are mapped too. The identity needs `roles/cloudasset.viewer` on the organization (or each project), and the Cloud Asset API must be enabled on the quota project. + +```bash +cloudg map -p azure --subscription-id 00000000-0000-0000-0000-000000000000 +cloudg map -p gcp --project-id prod-platform-381214 +cloudg map -p all --regions all +``` + +## How a run fits together + +```mermaid caption="One mapping run, from discovery to the files on disk" +flowchart TD + A["Org discovery (--org)"] --> B["Deep collectors, per account and region"] + B --> C["Cloud Control and tagging sweeps"] + C --> D["Deduplicate by ARN"] + D --> E["Relationship linker"] + E --> F["Account hierarchy"] + F --> G["Dependency analysis"] + G --> H["Exports in ./reports"] +``` + +Collectors never create edges across services themselves. They declare what each asset talks to, by any identifier they have (an ARN, a queue URL, an image URI, a bare `sg-` ID), and the relationship linker resolves those identifiers across services, regions and accounts. It runs three passes: declared relations, provider rules for older metadata conventions, and a generic pass that turns any identifier-looking string in an asset's metadata into a `REFERENCES` edge when it resolves to another asset. + +Resolution is scoped. An ambiguous name such as `default` resolves inside the referencing asset's account and region, and is left unresolved rather than guessed when it could belong to several accounts. A reference into an account you didn't map becomes a `CLOUD_ACCOUNT` node with `external: true`, so the vendor account trusted by one of your roles stays visible. A reference that resolves to nothing at all, such as a deleted queue still configured as a DLQ, lands in `unresolved_references` (capped at 2000 per run). Those are worth reading; a missing DLQ or log group is often a real misconfiguration. + +## Typed edges + +Every edge has a coarse `edge_type` and, when known, a finer `relationship` plus `properties`. Direction always reads "source verb target". + +| Edge type | Examples | +|---|---| +| `INVOKES` | S3 bucket to Lambda (notification), SQS to Lambda (`TRIGGERED_BY`), SNS to SQS (`STREAMS_TO`), EventBridge rule to target, API Gateway to Lambda | +| `USES_IMAGE` | task definition, Kubernetes workload or Lambda to an ECR repository | +| `ASSUMES_ROLE` | Lambda, task definition, nodegroup, Kubernetes service account or instance profile to an IAM role | +| `IAM_TRUST` | principal, external account or OIDC provider to a role it may assume (`CROSS_ACCOUNT_TRUST` across accounts) | +| `GRANTS_ACCESS` | principal to a resource its policy names; bucket, queue and repository policy grants | +| `IAM_POLICY_ATTACHMENT` | user, group or role to a managed policy | +| `PROTECTS` | WAF to ALB, API Gateway or CloudFront; Network Firewall to VPC; Shield to resource | +| `MONITORS` | Inspector to each scanned instance, repository or function; GuardDuty, Config and CloudTrail to the account | +| `MANAGES` | CloudFormation stack to resource, ASG to instance, landing zone to OUs | +| `GOVERNS` | SCP, RCP, tag policy or Control Tower control to an OU or account | +| `LOGS_TO` | trail, flow log, load balancer, API stage or function to its log destination | +| `LOAD_BALANCER_TARGET` | load balancer to target group to instance, IP or Lambda; Kubernetes Service to workload | +| `ROUTE` | route table to gateway, DNS record to load balancer, Ingress to Service | +| `CONTAINS`, `ATTACHED_TO`, `PEERING`, `REFERENCES` | containment, attachment, VPC peering, and everything else (KMS keys, secrets, DLQs, layers) | + +## Coverage, gaps and throttling + +Every collection unit (one account and region on AWS, one subscription on Azure, one scope on GCP) produces a coverage record with one entry per task: `SUCCESS` with an asset count, `FAILED` with the error, `PARTIAL` or `SKIPPED`. One failing task never stops the others. The discovery steps get records too: `organizations`, `controltower`, `management_groups`, `organization_hierarchy`. A member account whose role can't be assumed shows as `sts_assume_role: FAILED` and nothing else is collected there. + +After a run the CLI prints one warning line listing the failed entries, for example `3 collectors failed (see inventory coverage / -v): ...`, with the first eight named. Coverage is not written to any of the output files, so read it from Python when you need the details: + +```python title="coverage_report.py" +from cloudg import CloudGConfig +from cloudg.inventory import InventoryMapper + +config = CloudGConfig(providers=["aws"]) +config.aws.regions = ["us-east-1", "eu-west-1"] + +inventory = InventoryMapper(config).map_inventory_sync() + +for record in inventory.coverage: + for entry in record.services: + if entry.status.value != "SUCCESS": + print(record.provider, record.account_id or "-", record.region or "-", + entry.service, entry.status.value, entry.error) + +if inventory.throttling: + for line in inventory.throttling.get("messages", []): + print("throttled:", line) +``` + +Throttling never fails a map. Calls are paced and retried; a service that is still throttled after its retries is recorded as `FAILED` or `PARTIAL` with an error starting `throttled:`. The run's throttling telemetry is attached as `inventory.throttling` and, when present, written under a top-level `throttling` key in `inventory-map.json`. The [rate limits guide](/guides/rate-limits/) covers tuning. + +## Output files + +Everything goes to `-o/--output`, `./reports` by default. + +| File | Written when | Contents | +|---|---|---| +| `inventory-map.json` | always | assets, edges, unresolved references, and a summary: totals, per-service, type, region and account counts, relationship counts, cross-account edges, external accounts, security service gaps, internet exposure, unlinked assets | +| `inventory-map.graphml` | always | the same graph for Gephi, yEd, Neo4j import or `networkx.read_graphml` | +| `inventory-graph.json` | always | D3 force-layout nodes and links with account and relationship attributes | +| `inventory-dependencies.json` | always | most shared dependencies, largest blast radius, cross-account edges, security coverage (including workloads no vulnerability scanner covers and internet-facing endpoints without a WAF), unresolved references | +| `inventory-organization.json` | with `--org` | the Organization and Control Tower topology | +| `asset-map.json` | with `--findings` | per-asset finding counts and severity breakdown, riskiest first | +| `compliance-map.json` | with `--findings` | framework to findings and affected assets | + +`inventory-map.json` is self-contained, and `cloudg deps` reads it back without cloud access. The graph files merge parallel rule edges, so count edges in `inventory-map.json`. The [output files reference](/reference/output-files/) has the field-level detail. + +## Overlay scanner findings + +`--findings` takes a cloudg findings file, either `raw-findings.json` or a report with a top-level `findings` list such as `findings.json`. It is repeatable. A file that can't be read is skipped with a warning. + +```bash +cloudg map -p aws --regions all --findings ./reports/raw-findings.json +``` + +That maps again. When the scan output arrives days after the map, overlay it onto the saved map instead, with no cloud calls: + +```python title="overlay_findings.py" +import json + +from cloudg import CloudGConfig +from cloudg.inventory import InventoryMapper, InventoryResult +from cloudg.schema.models import Finding + +inventory = InventoryResult.load("./reports") + +with open("./reports/raw-findings.json") as f: + data = json.load(f) +raw = data if isinstance(data, list) else data.get("findings", []) +findings = [Finding.model_validate(item) for item in raw] + +mapper = InventoryMapper(CloudGConfig(providers=["aws"])) +paths = mapper.export_merged(inventory, findings, "./reports") +print(paths["asset_map"], paths["compliance_map"]) + +asset_map = mapper.build_asset_map(inventory, findings) +for entry in asset_map["assets"][:10]: + if entry["finding_count"]: + print(entry["name"], entry["type"], entry["severity_breakdown"]) +``` + +A finding is matched to an asset when its `resource_arn` or `resource_id` equals the asset's internal ID, ARN or name. Names are not checked for uniqueness. A finding whose `resource_id` is `orders` lands on every asset named `orders`, a queue and a DynamoDB table alike, even when its `resource_arn` points at only one of them. Read `severity_breakdown` with that in mind when your names repeat across services. + +## Use it as a library + +`cloudg.inventory` works without the engine. `InventoryMapper` takes a `CloudGConfig` and never modifies it; it works on a deep copy, because organization discovery rewrites the account list, role and regions. + +```python title="map_library.py" +from cloudg import CloudGConfig +from cloudg.inventory import InventoryMapper + +config = CloudGConfig(providers=["aws", "azure", "gcp"]) +config.aws.regions = ["ALL"] +config.inventory.services = ["network", "containers", "serverless", "identity", "sweep"] +config.inventory.kubernetes = False + +mapper = InventoryMapper(config, tagging_sweep=False) +inventory = mapper.map_inventory_sync() + +print(inventory.summary["assets_by_service"]) +paths = inventory.export("./reports") +print(sorted(paths)) +``` + +`map_inventory()` is the async form. `export()` returns a dict of paths keyed `map`, `graphml`, `graph`, `dependencies` and, for an organization, `organization`. `InventoryResult.load(path)` reads a saved map back from the directory or the JSON file. + +The relationship linker also works on assets from any source, such as a plugin collector, a CMDB import or a previous run: + +```python title="link_assets.py" +from cloudg.graph.builder import GraphBuilder +from cloudg.inventory import InventoryResult, RelationshipLinker + +assets = InventoryResult.load("./reports").assets + +linker = RelationshipLinker(assets) +edges = linker.link() +print(len(edges), "edges,", len(linker.unresolved), "unresolved") + +graph = GraphBuilder().build(assets + linker.external_assets, edges) +print(graph.number_of_nodes(), "nodes,", graph.number_of_edges(), "edges in the graph") +``` + +Your own collectors can declare relationships the same way the built-in ones do, by putting entries such as `{"target": "", "edge": "INVOKES", "relationship": "TRIGGERED_BY"}` in `metadata["relations"]`. The [RelationshipLinker](/api/relationshiplinker/) page has the details. + +:::links +- [Dependencies and blast radius](/guides/dependencies/) Query the saved map with `cloudg deps`. +- [AWS Organizations](/guides/aws-organizations/) Map every account under the management account. +- [cloudg map](/cli/map/) Every option of the command. +- [InventoryMapper](/api/inventorymapper/) The Python class behind it. +::: diff --git a/site/content/guides/pipeline.md b/site/content/guides/pipeline.md new file mode 100644 index 0000000..40ff78f --- /dev/null +++ b/site/content/guides/pipeline.md @@ -0,0 +1,356 @@ +--- +title: The pipeline +lede: "`cloudg run` is five phases in a fixed order: collect, graph, scan, normalise, render. This page follows the data through each one, says what it reads and writes, and what happens when part of it fails." +meta: + - [Command, "`cloudg run`"] + - [Python, "`CloudGEngine.run_pipeline()`"] + - [Output, "`./reports` by default"] +source: cloudg/cli_run_helpers.py +--- + +Each phase hands its results to the next in memory. Files are written along the way, but no phase reads back what an earlier one wrote, so you can delete `./reports` between runs without breaking anything. The graph phase and the scan phase both start from the collected inventory; the normaliser is where their findings meet. + +```mermaid caption="Phases of cloudg run and what each one writes" +flowchart TD + A["1 Collect: MultiAccountCollector"] --> B["2 Graph: GraphBuilder, reachability"] + B --> B2["2c RAG chunks / 2d Terraform"] + B2 --> C["3 Scan: thread pool"] + C --> C2["3b Ontology, with all findings"] + C2 --> D["4 Normalise: FindingsNormaliser"] + D --> E["5 Render: JSON, SVG, HTML"] + B -. "topology.graphml" .-> F[(reports)] + B2 -. "rag_chunks.jsonl, terraform/" .-> F + C -. "prowler/, scoutsuite/" .-> F + C2 -. "ontology.ttl, ontology.jsonld" .-> F + E -. "findings.json, report.html" .-> F +``` + +The CLI prints each phase as it starts, with the same numbering: + +| Banner | Runs when | Writes | +|---|---|---| +| Phase 1 · Asset Collection | always | nothing | +| Phase 2 · Graph Analysis | always | `topology.graphml`, `topology-cytoscape.json` | +| Phase 2c · RAG Export | `--rag-export` (default) and `rag.enabled` | `rag_chunks.jsonl`, `rag_metadata_index.json` | +| Phase 2d · Terraform Recreation | `--terraform` or `terraform.enabled` | `provider.tf.json`, `variables.tf.json`, `main.tf.json`, `import_commands.sh` | +| Phase 3 · Security Scanning | always | each scanner's own output under `prowler//` and `scoutsuite//` | +| Phase 3b · Semantic Ontology | `--ontology` (default) and `ontology.enabled` | `ontology.ttl`, `ontology.jsonld` | +| Phase 4 · Normalisation | always | nothing | +| Phase 5 · Report Generation | always | `findings.json`, `topology.svg`, `report.html` | + +## Phase 1: collect + +Collection reads the configuration (providers, regions, accounts, credentials) and returns three lists: assets, network edges and coverage records. It writes nothing to disk. If you want the raw inventory as a file, use `cloudg collect`, which writes `inventory-.json`, or `cloudg map` for the full inventory map. + +`cloudg run` always goes through `MultiAccountCollector`, which starts one task per provider and runs them together with `asyncio.gather`. Inside each provider the fan-out differs: + +| Provider | Unit of work | Notes | +|---|---|---| +| AWS | one task per account and region | Each task runs the pipeline's 15 service collectors (EC2, S3, RDS, VPCs, subnets, security groups, IAM users and roles, Lambda, ELBv2, ECS, DynamoDB, CloudFront, Secrets Manager, KMS). With `accounts` and `role_name` in the config, cloudg assumes that role in each account; the caller's own account uses the base credentials. | +| Azure | one task per subscription | List calls return every location at once, so locations are not iterated. With no `subscription_ids`, every enabled subscription the credential can see is collected. | +| GCP | one Cloud Asset Inventory listing per project, or one for the whole organization | Set `gcp.organization_id` for a single organization-scope listing; `project_ids` then filter it. | + +`--regions all` turns into `["ALL"]` for all three providers, and region discovery replaces it with the enabled regions before collection starts. Without `--regions`, AWS uses `us-east-1` (or `--region`), while Azure and GCP default to all. + +Two limits keep collection polite. An `asyncio.Semaphore` sized by `concurrency_limit` (default 5) caps how many AWS account and region tasks run at once. Below that, every API call passes through the shared rate limiter configured under `ratelimit`, which paces calls per service, honours Retry-After, and opens a circuit breaker on an API that stays throttled. [Rate limits](/guides/rate-limits/) explains the knobs. + +### When collection fails + +Failures are contained at the smallest unit that can fail: + +- A single collector, say Lambda in `eu-west-1`, fails: its coverage record says `FAILED` with the error, and the other services in that region carry on. +- An account or region cannot authenticate: that pair contributes no assets and is recorded as `aws_full` `FAILED`. +- A whole provider raises: the other providers still finish, and a `_full` coverage record holds the error. +- Collection as a whole raises: the run continues with zero assets. + +None of these stop the run or change its exit code. You see them in two places: a log line per failed collector during the run ("Collector ec2 failed: ..."), and the Collection Coverage table after the results, with one row per region and account, a coverage percentage and the names of the services that failed. A run with no credentials at all finishes "successfully" with an empty inventory, so read that table before trusting a quiet report. From Python, the same records are in `result.coverage`, with the error text for each service. + +## Phase 2: graph and analysis + +`GraphBuilder.build(assets, edges)` turns the inventory into a NetworkX `DiGraph`. Assets become nodes. Edge endpoints that are not assets, such as the `0.0.0.0/0` on a security group rule, become external nodes. The graph is saved straight away as `topology.graphml` and `topology-cytoscape.json`. + +`ReachabilityAnalyzer` then runs a breadth-first search from the internet sources (`0.0.0.0/0`, `::/0`) over network-flow edges. Every asset it reaches can produce a finding, for example "Security group allows SSH (port 22) from 0.0.0.0/0", and those findings carry deterministic IDs, so the same exposure keeps its ID from one run to the next. With `graph.compute_attack_paths` (on by default) the builder also looks for lateral movement paths and prints a warning with the count when it finds any. + +Two exports hang off the graph: + +- RAG export (2c) chunks the infrastructure per asset, per Louvain community and per relation group, and writes `rag_chunks.jsonl` and its index. At this point it only knows the reachability findings. After the scanners finish, a silent second pass rewrites both files with every finding included. +- Terraform recreation (2d) maps the collected assets to `.tf.json` resources. It writes to `terraform.output_dir`, which defaults to `./reports/terraform` and does not follow `-o`. Set `terraform.output_dir` in `config.yaml` if you write reports somewhere else. + +An exception in the RAG or Terraform export prints a failure line and the run continues without those files. + +## Phase 3: scan + +The scanners run in a `ThreadPoolExecutor`. Each one is a subprocess, so threads are enough. The pool is sized at the number of enabled scanners plus the number of providers plus one, which fits every job of a typical run at once. The jobs submitted depend on what is enabled and what there is to scan: + +| Job | Submitted | Target | +|---|---|---| +| Prowler | once per provider | the live account, output in `reports/prowler//` | +| ScoutSuite | once per provider | the live account, report in `reports/scoutsuite//` | +| Checkov | once per IaC directory | the resolved IaC directories | +| Trivy | once | `--images` if given, otherwise a filesystem scan of the IaC directories | +| IAM linter | whenever any assets were collected | IAM roles and policies from the inventory, via Parliament | + +The scanner list comes from `--scanners`, or from `scanners.enabled` in `config.yaml` (all five by default). + +IaC directories are resolved in this order: `--iac-dir`, then `scanners.iac_directories`, then the Terraform directory from phase 2d if it holds `*.tf.json` files. There is no fallback to the current directory. A clean Checkov scan of whatever folder you happened to run cloudg from would look like a clean bill of health for your cloud, so with nothing to scan, Checkov and Trivy print a warning and are left out. `--terraform` on its own is enough to give them a target: Checkov then audits the Terraform representation of what is really deployed. + +### When a scanner fails + +A missing executable is not an error: the wrapper logs that the tool is not installed and returns no findings. A scanner that exits badly or produces output cloudg cannot parse is marked failed in the progress display with its error, and the other scanners are unaffected. Each wrapper bounds its own subprocess: 3600 seconds for Prowler and ScoutSuite, 1800 seconds for Checkov and each Trivy invocation. + +:::note About scanners.timeout_seconds +The pipeline reads `scanners.timeout_seconds` (default 3600) when it collects each scanner's result, but it only does so after the scanner has finished, so in 0.6.0 it does not cut a slow scanner short. The per-scanner subprocess limits above are what actually apply. +::: + +### Phase 3b: ontology + +The ontology is built after the scanners, not with the other graph exports, so that the RDF can include every finding: scanner results, IAM linter results and reachability findings. `CloudOntology` infers typed relations (`INTERNET_REACHABLE`, `ROLE_ASSUMES_ROLE`, `ENCRYPTED_BY_KMS` and so on) and saves one file per entry in `ontology.export_formats`: `turtle` as `ontology.ttl`, `json-ld` as `ontology.jsonld`, `xml` as `ontology.rdf`, `nt` as `ontology.nt`. A failure here is printed and skipped. + +## Phase 4: normalise + +`FindingsNormaliser` gets three lists (reachability findings, scanner findings, IAM linter findings) plus the assets, and returns one `ScanResult`. In order, it: + +1. deduplicates within each scanner on scanner, check ID and resource; +2. merges across scanners only when both the normalised title and the check semantics in `cloudg/rules/check_equivalence.yaml` match, so a Prowler and a Checkov finding about the same S3 encryption problem become one finding with `source_tool` "prowler, checkov"; +3. rescores severity against CVSS where a score is known; +4. maps every finding to compliance controls, first from what the scanner reported, then by exact check ID from the shipped rulesets, then by regex rules, then from a small built-in table. + +Rulesets come from the package unless `rulesets.rules_dir` points elsewhere. The CLI then attaches the collected edges to the result so the renderers can draw them. + +## Phase 5: render + +Three renderers write the final files: + +- `JSONExporter` writes `findings.json`: metadata, summary, assets, findings, compliance and the D3 graph data. `cloudg report -i findings.json` reads it back, so you can re-render without collecting again. +- `SVGRenderer` writes `topology.svg`. +- `HTMLReportGenerator` writes `report.html` with the data embedded. The page loads Chart.js and D3 from cdn.jsdelivr.net when opened. + +After that the CLI prints the summary table (assets, findings, severity breakdown, frameworks) and the coverage table, and exits with status 0. + +## The whole run, step by step + +```mermaid caption="Who calls whom during cloudg run" +sequenceDiagram + participant CLI as cloudg run + participant Col as MultiAccountCollector + participant G as GraphBuilder + participant Pool as Scanner pool + participant N as FindingsNormaliser + participant R as Renderers + CLI->>Col: collect_all() + Col-->>CLI: assets, edges, coverage + CLI->>G: build(assets, edges) + G-->>CLI: graph, reachability findings + CLI->>Pool: submit Prowler, ScoutSuite, Checkov, Trivy, IAM + Pool-->>CLI: scanner and IAM findings + CLI->>CLI: ontology and RAG update + CLI->>N: normalise(reachability, scanner, iam) + N-->>CLI: ScanResult + CLI->>R: findings.json, topology.svg, report.html +``` + +## Running it from the CLI + +The flags map directly onto the phases above: + +```bash +# one region, scanners from config.yaml +cloudg run -p aws --regions us-east-1 + +# two providers, only Prowler and the IAM linter +cloudg run -p aws -p azure --scanners prowler,iam + +# give Checkov a target by recreating the estate as Terraform +cloudg run -p aws --regions us-east-1 --terraform + +# scan real IaC and two images, skip the ontology +cloudg run -p aws --iac-dir ./infra --images nginx:1.27,myrepo/api:2.3 --no-ontology + +# everything, with settings from a config file, into a dated folder +cloudg -c config.yaml run -p all --regions all -o ./reports/2026-10-09 +``` + +`config.yaml` is only read when you pass `-c`; there is no automatic lookup in the current directory. The [cloudg run reference](/cli/run/) lists every flag. + +## Running it from Python + +`CloudGEngine` implements the same phases. `run_pipeline()` is the async version and `run_pipeline_sync()` wraps it in `asyncio.run` for scripts and notebooks: + +```python tab="Sync" title="run_pipeline.py" +from cloudg import CloudGConfig, CloudGEngine + +config = CloudGConfig(providers=["aws"]) +config.aws.regions = ["us-east-1"] +config.scanners.enabled = ["prowler", "iam"] + +engine = CloudGEngine(config) +engine.on_phase_start = lambda phase: print(f"-> {phase}") +engine.on_error = lambda phase, exc: print(f"!! {phase}: {exc}") + +result = engine.run_pipeline_sync(output_dir="./reports") + +print(result.to_summary()) +print(result.report_paths) # {'json': ..., 'html': ...} +for cov in result.coverage: + failed = [s.service for s in cov.services if s.status.value == "FAILED"] + if failed: + print(cov.provider, cov.region, "failed:", failed) +``` + +```python tab="Async" title="run_pipeline_async.py" +import asyncio + +from cloudg import CloudGConfig, CloudGEngine + + +async def main() -> None: + engine = CloudGEngine(CloudGConfig(providers=["aws", "gcp"])) + result = await engine.run_pipeline(output_dir="./reports") + print(result.total_assets, "assets,", result.total_findings, "findings") + print(result.severity_breakdown) + + +asyncio.run(main()) +``` + +CLI flags become config fields or method arguments: + +| CLI | Python | +|---|---| +| `-p aws -p gcp` | `CloudGConfig(providers=["aws", "gcp"])` | +| `--regions us-east-1,eu-west-1` | `config.aws.regions = ["us-east-1", "eu-west-1"]` (and `azure`, `gcp`) | +| `--profile audit` | `config.aws.profile = "audit"` for collection; `scan(profile="audit")` for the scanners | +| `--scanners prowler,iam` | `config.scanners.enabled = ["prowler", "iam"]` | +| `--iac-dir`, `--images` | `config.scanners.iac_directories`, `config.scanners.trivy_images`, or `scan(iac_dir=..., images=[...])` | +| `--terraform` | `config.terraform.enabled = True` | +| `--no-ontology`, `--no-rag-export` | `config.ontology.enabled = False`, `config.rag.enabled = False` | + +The engine's pipeline is close to the CLI's but not identical, and the differences matter if you compare outputs: + +- The engine order is collect, scan, analyse, normalise, report. Reachability runs inside `scan()`, and `analyze()` builds the graph, ontology, RAG chunks and Terraform together after the scanners. +- `run_pipeline()` writes `findings.json` and `report.html`, plus the ontology and RAG files. It does not write `topology.svg`, `topology.graphml` or `topology-cytoscape.json`; use `GraphBuilder` if you need them. +- `run_pipeline()` calls `scan()` without a profile, so Prowler and ScoutSuite use the default credential chain (for example `AWS_PROFILE`). Collection uses `config.aws.profile`. +- Trivy runs only when there are images. The CLI's filesystem fallback is not in `scan()`. +- When Terraform is enabled and no IaC directory is configured, `scan()` generates the Terraform first so Checkov has a target, the same outcome as `--terraform` on the CLI. +- `PipelineResult.errors` records failures in normalisation and reporting. Collection, scanner, graph, ontology, RAG and Terraform failures go to the `on_error` hook and the log instead, and collection problems also show in `result.coverage`. + +### Event hooks + +Hooks are plain attributes on the engine. Set them before you call a method; an exception raised inside a hook is logged at debug level and ignored. + +| Hook | Called with | When | +|---|---|---| +| `on_phase_start` | phase name | `collection`, `scanning`, `analysis`, `normalisation`, `reporting`; also `ingest` and `inventory_mapping` from those methods | +| `on_collection_complete` | `CollectionResult` | after a successful collection | +| `on_finding` | `Finding` | once per finding when `scan()` or `ingest_reports()` returns | +| `on_scan_complete` | `list[Finding]` | right after the `on_finding` calls | +| `on_analysis_complete` | `AnalysisResult` | at the end of `analyze()` | +| `on_error` | phase name, exception | `collection`, `graph_analysis`, `graph_build`, `ontology`, `rag_export`, `terraform`, `normalisation`, `reporting`, or the scanner job name (`prowler-aws`, `scoutsuite-azure`, `checkov-./infra`, `trivy`, `iam`) | + +`on_finding` fires after all scanners have finished, not as each one completes, and it sees findings before normalisation, so duplicates across scanners are still separate at that point. Use `result.findings` from the pipeline if you want the merged set. + +### Running the phases yourself + +Calling the phases one by one lets you change what flows between them: filter assets before scanning, add findings from somewhere else, skip phases you do not need. + +```python title="phases.py" +import asyncio + +from cloudg import CloudGConfig, CloudGEngine + + +async def main() -> None: + engine = CloudGEngine(CloudGConfig(providers=["aws"])) + + collection = await engine.collect() + print(len(collection.assets), "assets in", collection.duration_ms, "ms") + + # leave out anything in a sandbox account before scanning + assets = [a for a in collection.assets if a.account_id != "999999999999"] + + findings = await engine.scan(assets, collection.edges, iac_dir="./infra") + analysis = await engine.analyze(assets, collection.edges, findings) + scan_result = engine.normalise_findings(findings, assets=assets) + + print(analysis.graph_nodes, "nodes,", len(analysis.attack_paths), "attack paths") + print(scan_result.summary) + + +asyncio.run(main()) +``` + +`normalise_findings()` is the same normaliser the pipeline uses, but it does not write anything. To get the report files, pass its result to the renderers yourself. This example needs no credentials and no scanners: it builds two assets by hand, runs the analysis and normalisation phases, and renders the reports. + +```python title="offline_pipeline.py" +import asyncio + +from cloudg import CloudGConfig, CloudGEngine +from cloudg.graph.builder import GraphBuilder +from cloudg.renderers.html_report import HTMLReportGenerator +from cloudg.renderers.json_export import JSONExporter +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider, EdgeType, NetworkEdge + +# a security group open to the world on port 22, and the instance behind it +sg = CloudAsset( + name="web", + asset_type=AssetType.SECURITY_GROUP, + provider=CloudProvider.AWS, + region="us-east-1", + arn="arn:aws:ec2:us-east-1:111111111111:security-group/sg-0web", +) +vm = CloudAsset( + name="web-1", + asset_type=AssetType.EC2, + provider=CloudProvider.AWS, + region="us-east-1", + arn="arn:aws:ec2:us-east-1:111111111111:instance/i-0web1", +) +assets = [sg, vm] +edges = [ + NetworkEdge( + source_id="0.0.0.0/0", + target_id=sg.id, + edge_type=EdgeType.SECURITY_GROUP_RULE, + cidr="0.0.0.0/0", + port_range="22", + ), + NetworkEdge(source_id=sg.id, target_id=vm.id, edge_type=EdgeType.ATTACHED_TO), +] + + +async def main() -> None: + engine = CloudGEngine(CloudGConfig(providers=["aws"])) + + # graph, reachability, ontology and RAG chunks + analysis = await engine.analyze(assets, edges, findings=[], output_dir="./offline") + + # the same normaliser run_pipeline uses + scan_result = engine.normalise_findings(analysis.reachability_findings, assets=assets) + scan_result.edges = edges + + # findings.json and report.html, with the topology + builder = GraphBuilder() + builder.build(assets, edges) + graph_json = builder.to_d3_json() + JSONExporter(output_dir="./offline").export(scan_result, graph_json=graph_json) + HTMLReportGenerator(output_dir="./offline").generate(scan_result, graph_json=graph_json) + + print(analysis.graph_nodes, "nodes,", analysis.graph_edges, "edges") + for finding in scan_result.findings: + print(finding.severity.value, finding.title) + + +asyncio.run(main()) +``` + +```console +$ python offline_pipeline.py +3 nodes, 2 edges +CRITICAL Security group allows SSH (port 22) from 0.0.0.0/0 +$ ls offline +findings.json ontology.jsonld ontology.ttl rag_chunks.jsonl rag_metadata_index.json report.html +``` + +Three nodes for two assets: the third is the external `0.0.0.0/0` node the security group rule points from. + +For scanner output that already exists, `run_from_reports()` replaces collection and scanning with ingest and runs normalisation and reporting on top; [Ingesting existing output](/guides/ingesting/) covers it. [Python recipes](/guides/python-recipes/) has more combinations, and [CloudGEngine](/api/cloudgengine/) documents every method. diff --git a/site/content/guides/python-recipes.md b/site/content/guides/python-recipes.md new file mode 100644 index 0000000..5d9f089 --- /dev/null +++ b/site/content/guides/python-recipes.md @@ -0,0 +1,565 @@ +--- +title: Python recipes +lede: "Complete scripts for the integrations people build most often: aggregating reports, feeding a SIEM, running phases on your own schedule, overlaying findings on a map, and gating CI." +meta: + - [Entry point, "`cloudg.CloudGEngine`"] + - [Needs credentials, "recipes 3 to 6 only"] +source: docs/DOCUMENTATION.md +since: "0.6.0" +--- + +The CLI is a thin layer over a public API, so anything `cloudg run` does you can do from Python, in pieces, with your own decisions in between. Each recipe below is one file you can save and run. The ones that only read scanner output work on any machine with `pip install cloudg`; the ones that collect need the provider extra (`pip install "cloudg[aws]"`) and credentials resolved as described in [authentication](/guides/authentication/). + +The sample inputs used in the offline recipes are ordinary scanner output: a Prowler `-M json-asff` directory, a `checkov --output json` file and `trivy image` / `trivy fs` JSON. [Ingesting existing output](/guides/ingesting/) lists exactly which fields each parser reads. + +```mermaid caption="Where each recipe enters the pipeline" +flowchart LR + C["collect()"] --> S["scan()"] + I["ingest_reports()"] --> N["normalise_findings()"] + S --> N + C --> A["analyze()"] + S --> A + N --> R["JSON and HTML reports"] + M["map_inventory()"] --> O["export_merged()"] + I --> O +``` + +## 1. Aggregate existing scan results + +The scanners already ran somewhere else and you want one deduplicated, compliance-mapped report. `run_from_reports_sync` chains ingest, normalisation and report rendering and touches no cloud API. + +```python title="aggregate_reports.py" +"""Aggregate scanner output that already exists into one cloudg report.""" + +from cloudg import CloudGConfig, CloudGEngine + +engine = CloudGEngine(CloudGConfig()) +result = engine.run_from_reports_sync( + { + "prowler": ["./prowler-output/"], + "checkov": ["./results_json.json"], + "trivy": ["./trivy-image.json", "./trivy-fs.json"], + }, + output_dir="./reports", +) + +if result.errors: + print("phases that failed:", result.errors) + +print(f"{result.total_findings} findings after deduplication") +print("by severity:", result.severity_breakdown) +print("reports:", {name: str(path) for name, path in result.report_paths.items()}) + +for finding in sorted(result.findings, key=lambda f: f.risk_score, reverse=True): + if finding.severity.value in ("CRITICAL", "HIGH"): + print(f"{finding.risk_score:4.1f} {finding.source_tool:<18} {finding.title}") +``` + +```console +$ python aggregate_reports.py +4 findings after deduplication +by severity: {'CRITICAL': 1, 'HIGH': 3} +reports: {'json': 'reports/findings.json', 'html': 'reports/report.html'} + 9.5 trivy [Trivy] CVE-2024-0001: openssl (alpine) + 7.5 prowler S3 bucket default encryption + 7.5 checkov [Checkov/terraform] Ensure S3 bucket has server-side encryption enabled + 7.5 trivy [Trivy/IaC] AVD-AWS-0088: S3 bucket encryption not enabled +``` + +A path that does not parse is logged and skipped rather than raised, so an empty result usually means a wrong path. Prowler findings with `Compliance.Status: PASSED` are dropped at parse time. `result.errors` lists phases that failed (normalisation or reporting); with no collection, the graph, ontology and Terraform fields stay empty. + +## 2. Parse first, decide later + +When you want to filter, enrich or route findings before they are merged, parse them into `Finding` objects yourself and normalise afterwards. + +```python title="parse_then_decide.py" +"""Parse scanner reports into Finding objects, filter them, then normalise.""" + +from collections import Counter + +from cloudg import CloudGConfig, CloudGEngine +from cloudg.ingest import ingest_reports, parse_report +from cloudg.scanners.trivy import TrivyScanner + +# One tool at a time... +findings = parse_report("prowler", "./prowler-output/") +findings += parse_report("checkov", "./results_json.json") + +# ...or every tool in one call +findings = ingest_reports( + { + "prowler": ["./prowler-output/"], + "checkov": ["./results_json.json"], + "trivy": ["./trivy-image.json", "./trivy-fs.json"], + } +) +print("parsed:", Counter(f.source_tool for f in findings)) + +# Decide before normalising: here, drop INFO and LOW +kept = [f for f in findings if f.severity.value not in ("INFO", "LOW")] + +engine = CloudGEngine(CloudGConfig()) +scan_result = engine.normalise_findings(kept) +print("summary:", scan_result.summary) + +failed = Counter(c.framework for c in scan_result.compliance if c.status.value == "FAIL") +for framework, count in failed.most_common(): + print(f"{framework}: {count} failed control(s)") + +# The per-scanner parsers work without the engine at all +image_findings = TrivyScanner.parse_report("./trivy-image.json") +print("trivy image only:", [f.title for f in image_findings]) +``` + +```console +$ python parse_then_decide.py +parsed: Counter({'trivy': 2, 'prowler': 1, 'checkov': 1}) +summary: {'total_assets': 0, 'total_findings': 4, 'total_edges': 0, 'severity_breakdown': {'CRITICAL': 1, 'HIGH': 3}, 'compliance_frameworks': [...]} +NIST-800-53-Revision-5-AWS: 12 failed control(s) +HIPAA-AWS: 6 failed control(s) +CIS: 3 failed control(s) +... +trivy image only: ['[Trivy] CVE-2024-0001: openssl (alpine)'] +``` + +`normalise_findings` is the same pass `cloudg run` uses: dedupe within each scanner, merge across scanners through `check_equivalence.yaml`, rescore with CVSS, map compliance controls. Filtering before it means a dropped finding never contributes to a merged one. Filtering after it, on `scan_result.findings`, keeps the merge intact and lets you decide on the merged view; pick whichever matches your policy. + +## 3. Stream findings into a SIEM + +`on_finding` is called once per finding after the scan (or ingest) phase. Hooks run inline in the engine and any exception they raise is swallowed, so the shipper below only puts findings on a queue and lets a worker thread batch them to an HTTP collector, keeping a local NDJSON spool either way. + +```python title="stream_to_siem.py" hl="35-36,82" +"""Ship every cloudg finding to a SIEM over HTTP while the pipeline runs.""" + +import asyncio +import json +import logging +import os +import queue +import threading +import urllib.request + +from cloudg import CloudGConfig, CloudGEngine +from cloudg.schema.models import Finding + +log = logging.getLogger("siem-shipper") + +SIEM_URL = os.environ.get("SIEM_URL") # e.g. https://siem.example.com/ingest +SIEM_TOKEN = os.environ.get("SIEM_TOKEN", "") +SPOOL_FILE = "cloudg-findings.ndjson" # local copy, also the fallback + + +class FindingShipper: + """on_finding callback: queue the finding, send batches from a worker thread. + + The engine calls hooks inline, so a slow HTTP call made directly in the + callback would hold up the pipeline. Here the callback only enqueues. + """ + + def __init__(self, url: str | None, token: str, batch_size: int = 200) -> None: + self.url, self.token, self.batch_size = url, token, batch_size + self.sent = 0 + self._queue: queue.Queue[dict | None] = queue.Queue() + self._worker = threading.Thread(target=self._drain, daemon=True) + self._worker.start() + + def __call__(self, finding: Finding) -> None: + self._queue.put({"source": "cloudg", "event": finding.model_dump(mode="json")}) + + def close(self) -> None: + self._queue.put(None) + self._worker.join() + + def _drain(self) -> None: + batch: list[dict] = [] + while True: + item = self._queue.get() + if item is not None: + batch.append(item) + if batch and (item is None or len(batch) >= self.batch_size): + self._send(batch) + batch = [] + if item is None: + return + + def _send(self, batch: list[dict]) -> None: + body = "\n".join(json.dumps(e) for e in batch) + "\n" + with open(SPOOL_FILE, "a") as fh: + fh.write(body) + if not self.url: + self.sent += len(batch) + return + request = urllib.request.Request( + self.url, + data=body.encode(), + headers={"Content-Type": "application/x-ndjson", + "Authorization": f"Bearer {self.token}"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=30) as resp: + resp.read() + self.sent += len(batch) + except OSError as exc: # URLError and timeouts; the spool file keeps the batch + log.warning("SIEM rejected a batch of %d: %s", len(batch), exc) + + +async def main() -> None: + config = CloudGConfig(providers=["aws"]) + config.aws.regions = ["us-east-1", "eu-west-1"] + + shipper = FindingShipper(SIEM_URL, SIEM_TOKEN) + engine = CloudGEngine(config) + engine.on_finding = shipper + engine.on_phase_start = lambda phase: log.info("cloudg phase: %s", phase) + engine.on_error = lambda phase, exc: log.error("cloudg %s failed: %s", phase, exc) + + try: + result = await engine.run_pipeline(output_dir="./reports") + finally: + shipper.close() + + print(f"shipped {shipper.sent} raw findings; {result.total_findings} after normalisation") + print("summary:", json.dumps(result.to_summary(), default=str)) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") + asyncio.run(main()) +``` + +```bash +SIEM_URL=https://siem.example.com/ingest SIEM_TOKEN="$SIEM_TOKEN" python stream_to_siem.py +``` + +Two properties of the hook decide how you model events on the SIEM side. The findings it sees are the raw ones, before normalisation, so two scanners reporting the same unencrypted bucket arrive as two events; the merged view is `result.findings` once the run ends. And they arrive in one burst at the end of the scan phase, not one by one as scanners finish. Reachability findings have deterministic ids (a UUID5 of the rule and the asset's ARN), so the SIEM can deduplicate them across daily runs by `event.id`; scanner findings get a fresh id each run, so key those on `source_tool`, `source_finding_id` and `resource_arn`. + +The same shipper works on `engine.ingest_reports(...)` or `run_from_reports(...)`, which fire `on_finding` too and need no credentials. + +## 4. Run the phases yourself + +`run_pipeline` runs collect, scan, analyse, normalise and report in a fixed order. Calling the phases yourself lets you stop early, pick scanners per run, or do something between phases. This one refuses to scan when collection came back empty, which is what expired credentials look like. + +```python title="run_phases.py" +"""Run collection, scanning, analysis and normalisation as separate steps.""" + +import asyncio +from pathlib import Path + +from cloudg import CloudGConfig, CloudGEngine +from cloudg.coverage import ServiceStatus +from cloudg.renderers.html_report import HTMLReportGenerator +from cloudg.renderers.json_export import JSONExporter + +OUT = Path("./reports") + + +async def main() -> None: + config = CloudGConfig(providers=["aws"]) + config.aws.regions = ["us-east-1"] + config.scanners.enabled = ["prowler", "iam"] # missing binaries are skipped + config.terraform.enabled = False + + engine = CloudGEngine(config) + engine.on_phase_start = lambda phase: print(f"--> {phase}") + + # 1. Collect, then decide whether the rest is worth running + collection = await engine.collect() + failed = [ + (c.account_id, c.region, s.service, s.error) + for c in collection.coverage + for s in c.services + if s.status in (ServiceStatus.FAILED, ServiceStatus.PARTIAL) + ] + print(f"{len(collection.assets)} assets, {len(collection.edges)} edges, " + f"{len(failed)} degraded services in {collection.duration_ms} ms") + if not collection.assets: + raise SystemExit(f"nothing collected; first problems: {failed[:3]}") + + # 2. Scan: reachability analysis, the IAM linter and any scanner on PATH + findings = await engine.scan(collection.assets, collection.edges, output_dir=OUT) + + # 3. Graph, ontology and RAG export, written into OUT + analysis = await engine.analyze(collection.assets, collection.edges, findings, output_dir=OUT) + print(f"graph: {analysis.graph_nodes} nodes, {analysis.graph_edges} edges, " + f"{analysis.ontology_triples} triples, {len(analysis.attack_paths)} attack paths") + + # 4. Normalise: scan() already holds the reachability findings + scan_result = engine.normalise_findings(findings, assets=collection.assets) + scan_result.edges = collection.edges + print("severity:", scan_result.summary["severity_breakdown"]) + + # 5. Reports + print(JSONExporter(output_dir=str(OUT)).export(scan_result)) + print(HTMLReportGenerator(output_dir=str(OUT)).generate(scan_result)) + + +asyncio.run(main()) +``` + +```console +$ python run_phases.py +--> collection +11 assets, 13 edges, 0 degraded services in 5981 ms +--> scanning +--> analysis +graph: 12 nodes, 13 edges, 774 triples, 0 attack paths +severity: {'CRITICAL': 1} +reports/findings.json +reports/report.html +``` + +`scan()` runs the graph reachability analysis itself, and `analyze()` computes the same findings again into `analysis.reachability_findings`. Pass only one of the two lists to the normaliser. Adding both is harmless (the ids are deterministic, so the duplicates merge) but pointless. `analyze()` writes `ontology.ttl`, `ontology.jsonld`, `rag_chunks.jsonl` and `rag_metadata_index.json` into the output directory according to the `ontology` and `rag` config sections; it does not write GraphML, which only `cloudg run` and the inventory export do. + +## 5. Combine live collection with ingested reports + +Collection needs credentials; ingest does not. A common split is Prowler running in a security account's own pipeline while you want its findings next to cloudg's live graph and reachability findings. + +```python title="combine_live_and_ingested.py" +"""Collect live assets, merge in scanner output produced elsewhere, analyse it all.""" + +import asyncio +from pathlib import Path + +from cloudg import CloudGConfig, CloudGEngine +from cloudg.graph.builder import GraphBuilder +from cloudg.renderers.html_report import HTMLReportGenerator +from cloudg.renderers.json_export import JSONExporter + +OUT = Path("./reports") + + +async def main() -> None: + config = CloudGConfig(providers=["aws"]) + config.aws.regions = ["us-east-1"] + engine = CloudGEngine(config) + + collection = await engine.collect() # needs credentials + ingested = engine.ingest_reports({"prowler": ["./prowler-output/"]}) # needs nothing + + # Reachability findings from the live graph, so they sit next to Prowler's + analysis = await engine.analyze(collection.assets, collection.edges, ingested, output_dir=OUT) + scan_result = engine.normalise_findings( + ingested + analysis.reachability_findings, assets=collection.assets + ) + scan_result.edges = collection.edges + + builder = GraphBuilder() + builder.build(collection.assets, collection.edges) + graph_json = builder.to_d3_json() + JSONExporter(output_dir=str(OUT)).export(scan_result, graph_json=graph_json) + HTMLReportGenerator(output_dir=str(OUT)).generate(scan_result, graph_json=graph_json) + + by_tool: dict[str, int] = {} + for finding in scan_result.findings: + by_tool[finding.source_tool] = by_tool.get(finding.source_tool, 0) + 1 + print(f"{len(collection.assets)} assets, {analysis.ontology_triples} ontology triples") + print("findings by tool:", by_tool) + + +asyncio.run(main()) +``` + +```console +$ python combine_live_and_ingested.py +11 assets, 768 ontology triples +findings by tool: {'cloudg-reachability': 1, 'prowler': 1} +``` + +Passing the ingested findings to `analyze()` puts them into the ontology and the RAG chunks, attached to the live assets. The exporters resolve each finding's asset by id, then ARN, then unique name, so Prowler findings keyed by ARN land on the right node. Passing `graph_json` to the renderers gives the HTML report its topology view; without it the report has findings but no graph. + +## 6. Map now, overlay findings later + +Inventory mapping needs read credentials and nothing else. Scanners can run elsewhere and on another schedule, and their output merges in whenever it arrives, even in a different process, because the map round-trips through `inventory-map.json`. + +```python tab="Map" title="map_inventory.py" +"""Map the inventory now; no scanners involved.""" + +from cloudg import CloudGConfig, CloudGEngine + +config = CloudGConfig(providers=["aws"]) +config.aws.regions = ["us-east-1"] # or ["ALL"] + +inventory = CloudGEngine(config).map_inventory_sync(output_dir="./map-reports") + +summary = inventory.summary +print(summary["total_assets"], "assets,", summary["total_edges"], "edges") +print("by service:", summary["assets_by_service"]) +print("internet exposed:", summary["internet_exposed"]) +print("unlinked:", summary["unlinked_assets"]) +if inventory.throttling: + print("throttled:", inventory.throttling["messages"]) +``` + +```python tab="Overlay" title="overlay_findings.py" +"""Overlay scanner findings on an inventory map written earlier.""" + +import json + +from cloudg import CloudGConfig, CloudGEngine +from cloudg.inventory import InventoryMapper, InventoryResult + +inventory = InventoryResult.load("./map-reports") # directory or inventory-map.json +print(len(inventory.assets), "assets in the saved map") + +engine = CloudGEngine(CloudGConfig()) +findings = engine.ingest_reports({"prowler": ["./prowler-output/"]}) + +mapper = InventoryMapper(CloudGConfig()) +paths = mapper.export_merged(inventory, findings, "./map-reports") +print({name: str(path) for name, path in paths.items()}) + +asset_map = json.loads(paths["asset_map"].read_text()) +for entry in asset_map["assets"]: + if entry["finding_count"]: + print(entry["name"], entry["type"], entry["severity_breakdown"]) +``` + +```bash tab="CLI" +cloudg map -p aws --regions all -o ./map-reports +# later, with scanner output that cloudg already normalised: +cloudg map -p aws --regions all -o ./map-reports --findings ./reports/raw-findings.json +``` + +The overlay step needs no credentials. `asset-map.json` lists every asset with `finding_count`, `severity_breakdown` and `finding_ids`, sorted with the worst first; `compliance-map.json` groups affected assets by framework. The asset map matches a finding to an asset by ARN, asset id or exact name (the compliance map by ARN or id only), so scanner output that names resources by ARN (Prowler, ScoutSuite) overlays well, and IaC findings keyed by a Terraform address (Checkov) do not match a live asset at all. The CLI variant re-maps the account before merging; the Python one reuses the saved map. + +## 7. Link relationships into someone else's inventory + +`RelationshipLinker` is pure post-processing over `CloudAsset` objects, whoever produced them: a plugin collector, a CMDB export, a previous run. Declare what each asset points at in `metadata["relations"]` and the linker resolves the references across services, regions and accounts. + +```json title="cmdb_assets.json" +[ + {"id": "cmdb-001", "name": "orders-api", "type": "LAMBDA_FUNCTION", + "arn": "arn:aws:lambda:eu-west-1:123456789012:function:orders-api", + "role_arn": "arn:aws:iam::123456789012:role/orders-api-role", + "queue_arn": "arn:aws:sqs:eu-west-1:123456789012:orders"}, + {"id": "cmdb-002", "name": "orders-api-role", "type": "IAM_ROLE", + "arn": "arn:aws:iam::123456789012:role/orders-api-role"}, + {"id": "cmdb-003", "name": "orders", "type": "MESSAGE_QUEUE", + "arn": "arn:aws:sqs:eu-west-1:123456789012:orders"}, + {"id": "cmdb-004", "name": "audit-bucket", "type": "S3_BUCKET", + "arn": "arn:aws:s3:::audit-bucket", + "replicates_to": "arn:aws:s3:::audit-bucket-dr"} +] +``` + +```python title="link_cmdb.py" +"""Turn a CMDB export into CloudAsset objects, link them, and build a graph.""" + +import json +from collections import Counter + +from cloudg.graph.builder import GraphBuilder +from cloudg.inventory import RelationshipLinker +from cloudg.schema.models import AssetType, CloudAsset, CloudProvider + + +def to_asset(row: dict) -> CloudAsset: + relations = [] + if row.get("role_arn"): + relations.append({"target": row["role_arn"], "edge": "ASSUMES_ROLE"}) + if row.get("queue_arn"): + # The queue triggers the function: the edge points queue -> function + relations.append( + {"target": row["queue_arn"], "edge": "INVOKES", + "relationship": "TRIGGERED_BY", "reverse": True} + ) + if row.get("replicates_to"): + relations.append({"target": row["replicates_to"], "edge": "REFERENCES", + "relationship": "REPLICATES_TO"}) + return CloudAsset( + id=row["id"], + name=row["name"], + arn=row["arn"], + asset_type=AssetType(row["type"]), + provider=CloudProvider.AWS, + region=row["arn"].split(":")[3] or "global", + account_id=row["arn"].split(":")[4] or None, + metadata={"relations": relations}, + ) + + +with open("cmdb_assets.json") as fh: + assets = [to_asset(row) for row in json.load(fh)] + +linker = RelationshipLinker(assets) +edges = linker.link() + +for edge in edges: + print(f"{edge.source_id} -[{edge.edge_type.value}/{edge.relationship}]-> {edge.target_id}") +print("unresolved:", linker.unresolved) +print("external placeholders:", [a.name for a in linker.external_assets]) + +graph = GraphBuilder().build(assets + linker.external_assets, edges) +print(graph.number_of_nodes(), "nodes,", graph.number_of_edges(), "edges") +print(Counter(data["edge_type"] for _, _, data in graph.edges(data=True))) +``` + +```console +$ python link_cmdb.py +cmdb-001 -[ASSUMES_ROLE/None]-> cmdb-002 +cmdb-003 -[INVOKES/TRIGGERED_BY]-> cmdb-001 +unresolved: [{'source': 'arn:aws:s3:::audit-bucket', 'source_name': 'audit-bucket', 'target': 'arn:aws:s3:::audit-bucket-dr', 'edge_type': 'REFERENCES'}] +external placeholders: [] +4 nodes, 2 edges +Counter({'ASSUMES_ROLE': 1, 'INVOKES': 1}) +``` + +Each relation is `{"target", "edge", "relationship", "reverse", "properties", "description"}`; only `target` is required, and `edge` defaults to `REFERENCES`. `reverse: true` flips the direction so the edge still reads "source verb target". A target that matches nothing is reported in `linker.unresolved` instead of producing a dangling edge, as the DR bucket above shows. A reference into an account that was not collected becomes a placeholder account node in `linker.external_assets`. If you already have some edges, call `linker.seed_existing(edges)` before `link()` so they are not duplicated. + +## 8. Gate CI on severity + +For a CI job that already runs Python, the gate can live next to the aggregation. This version fails on HIGH and CRITICAL, honours suppressions and a list of accepted check IDs, and refuses to pass on partial data. For a gate that only reads `findings.json`, with no cloudg import, see [CI and automation](/guides/ci-automation/#a-severity-gate). + +```python title="gate_on_severity.py" +"""Aggregate IaC scanner output and fail the build on blocking findings.""" + +import sys + +from cloudg import CloudGConfig, CloudGEngine + +BLOCKING = {"CRITICAL", "HIGH"} +ALLOWED_CHECKS = {"CKV_AWS_18"} # accepted risks, by scanner check ID + +result = CloudGEngine(CloudGConfig()).run_from_reports_sync( + {"checkov": ["results_json.json"], "trivy": ["trivy-fs.json", "trivy-image.json"]}, + output_dir="./reports", +) + +if result.errors: # a phase failed: do not pass the build on partial data + print("cloudg errors:", result.errors) + sys.exit(2) + +blocking = [ + f for f in result.findings + if f.severity.value in BLOCKING + and not f.is_suppressed + and f.source_finding_id not in ALLOWED_CHECKS +] +print("severity:", result.severity_breakdown, "report:", result.report_paths.get("html")) +for f in sorted(blocking, key=lambda f: f.risk_score, reverse=True): + print(f" [{f.severity.value}] {f.title} ({f.source_tool}: {f.source_finding_id})") + +if blocking: + print(f"{len(blocking)} blocking finding(s), failing the build") + sys.exit(1) +``` + +```console +$ python gate_on_severity.py; echo "exit $?" +severity: {'CRITICAL': 1, 'HIGH': 2} report: reports/report.html + [CRITICAL] [Trivy] CVE-2024-0001: openssl (alpine) (trivy: CVE-2024-0001) + [HIGH] [Checkov/terraform] Ensure S3 bucket has server-side encryption enabled (checkov: CKV_AWS_19) + [HIGH] [Trivy/IaC] AVD-AWS-0088: S3 bucket encryption not enabled (trivy: AVD-AWS-0088) +3 blocking finding(s), failing the build +exit 1 +``` + +After a cross-scanner merge, `source_finding_id` holds one scanner's check ID and `source_tool` becomes a comma list such as `"prowler, checkov"`, so an allow-list keyed on one tool's ID can miss a merged finding. When that matters, filter on the parsed findings before normalising, as in recipe 2. + +:::links +- [CloudGEngine](/api/cloudgengine/) Every method and its signature. +- [Event hooks](/api/event-hooks/) When each hook fires. +- [Result objects](/api/results/) `PipelineResult`, `CollectionResult`, `AnalysisResult`. +- [RelationshipLinker](/api/relationshiplinker/) Resolution rules for declared relations. +::: diff --git a/site/content/guides/quickstart.md b/site/content/guides/quickstart.md new file mode 100644 index 0000000..3d9c405 --- /dev/null +++ b/site/content/guides/quickstart.md @@ -0,0 +1,161 @@ +--- +title: Quickstart +lede: "Scan one AWS region and open the report. Then, if you have no credentials to hand, get the same report from scanner files you already have, or map an account without any scanners." +meta: + - [Time, "about 10 minutes, plus scanner run time"] + - [Needs, "Python 3.11+, read-only AWS credentials"] +source: site/content/guides/quickstart.md +--- + +This page takes the shortest path to each of cloudg's three entry points. It sticks to AWS and one region so the first run is quick; every command takes `-p azure`, `-p gcp` or `-p all` the same way. + +## Scan an AWS account + +::::steps +### Install cloudg and a scanner + +```bash +pip install "cloudg[aws]" +pip install prowler +``` + +Prowler is enough for a first run. cloudg skips scanners it cannot find on `PATH`, so you will see warnings for ScoutSuite, Checkov and Trivy; that is expected. [Installation](/guides/installation/) covers the rest, and the [Docker image](/guides/docker/) has Prowler, Checkov and Trivy built in. + +### Authenticate + +With no credential flags, cloudg uses boto3's default chain: the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` variables, then `AWS_PROFILE` or the default profile in `~/.aws`, cached SSO credentials, and on cloud compute the instance or task role. If the AWS CLI works, cloudg will too: + +```bash +aws sso login --profile audit +export AWS_PROFILE=audit +aws sts get-caller-identity +``` + +:::tip Use read-only credentials +cloudg only reads. The AWS managed policies `SecurityAudit` and `ViewOnlyAccess` are the usual starting point for the role or user you scan with. +::: + +### Run the pipeline + +```bash +cloudg run -p aws --regions us-east-1 +``` + +The run goes through five phases, printed as it goes: asset collection, graph analysis (plus RAG export), security scanning with every available scanner in parallel, normalisation, and report generation. The ontology is built after the scanners so it can include their findings. Prowler usually takes the longest. + +To pick the profile on the command line instead of the environment, add `--profile audit`. To scan every enabled region, use `--regions all`. + +### Open the report + +```bash +open reports/report.html # macOS +xdg-open reports/report.html # Linux +``` + +`report.html` is a single file with the findings, assets and graph data embedded in it: the topology graph, the findings table and the compliance matrix. It loads Chart.js and D3 from cdn.jsdelivr.net, though, so the charts and the topology view need network access when you open it. +:::: + +### What lands in ./reports + +```text +reports/ +├── report.html the report, open this first +├── findings.json assets, normalised findings, compliance, graph data +├── topology.svg static picture of the graph +├── topology.graphml the graph for Gephi, yEd or NetworkX +├── topology-cytoscape.json the graph for Cytoscape +├── ontology.ttl RDF ontology (Turtle) +├── ontology.jsonld RDF ontology (JSON-LD) +├── rag_chunks.jsonl retrieval chunks, one JSON object per line +├── rag_metadata_index.json index over the chunks +└── prowler/aws/ Prowler's own ASFF output +``` + +ScoutSuite, when installed, writes its report under `reports/scoutsuite/aws/`. Add `--terraform` and you also get `reports/terraform/` with `.tf.json` files that recreate what was collected. + +If the summary says 0 assets, collection did not authenticate. The Collection Coverage table at the end shows 0% for the region and lists the failed services, and the log lines above it carry the reason: "could not be found" (a missing profile) or "Unable to locate credentials" means boto3 found nothing to use. The run itself does not fail in that case, so check the table before trusting an empty report. + +To re-render the report later without touching the cloud: + +```bash +cloudg report -i reports/findings.json --format html +``` + +## No credentials: ingest scanner output + +If someone else runs the scanners (in CI, on a schedule, on another network), you only need their output files. `cloudg ingest` needs no cloud access and no scanner binaries, and the core package without extras is enough. + +::::steps +### Install the core package + +```bash +pip install cloudg +``` + +### Point it at the files + +Pass any combination of tools. Each flag takes a file or a directory and can be repeated: + +```bash +cloudg ingest \ + --prowler ./prowler-output/ \ + --checkov ./results_json.json \ + --trivy ./trivy-image.json \ + -o ./reports +``` + +Prowler's input is its ASFF JSON (`prowler aws -M json-asff`), Checkov's is `checkov -o json`, Trivy's is `trivy image -f json` or `trivy fs -f json`. ScoutSuite is read from its `scoutsuite_results_*.js` file or report directory with `--scoutsuite`. + +### Read the result + +```console +$ ls reports +findings.json raw-findings.json report.html +``` + +`raw-findings.json` holds every parsed finding before deduplication. `findings.json` and `report.html` hold the merged, compliance-mapped set. The last line of the output tells you how many duplicates were merged, for example "Ingested 3 findings from 3 tool(s), 3 after deduplication". +:::: + +There is no topology in an ingest report, because nothing was collected. [Ingesting existing output](/guides/ingesting/) covers the exact input formats and how to combine ingested findings with a live collection. + +## No scanners: map the inventory + +`cloudg map` answers "what is deployed, and what depends on what". It needs read-only credentials and the provider extra, but no scanners. + +::::steps +### Map one region + +```bash +cloudg map -p aws --regions us-east-1 +``` + +This runs the deep collectors, a Cloud Control sweep over every listable resource type, and the relationship linker. It reads far more services than `cloudg run` does, so expect it to take longer. + +### Look at the files + +```console +$ ls reports +inventory-dependencies.json inventory-graph.json inventory-map.graphml inventory-map.json +``` + +`inventory-map.json` is the map itself: every asset, every typed relationship, and a summary by service. `inventory-dependencies.json` lists shared dependencies, blast radius, cross-account edges and gaps in security service coverage. + +### Ask about one asset + +```bash +cloudg deps arn:aws:iam::123456789012:role/app-role +``` + +`cloudg deps` reads the saved map from `./reports` and prints what that asset depends on and what would break without it. `-d down` shows only the blast radius. +:::: + +When you do run scanners later, merge their findings into the map with `cloudg map --findings ./reports/findings.json` (or the `raw-findings.json` that `cloudg ingest` and `cloudg scan` write), which adds `asset-map.json` and `compliance-map.json`. [Inventory mapping](/guides/inventory-mapping/) goes through the map in depth. + +## Next steps + +:::links +- [The pipeline](/guides/pipeline/) What each phase of cloudg run does, and how to drive it from Python. +- [Authentication](/guides/authentication/) Profiles, role assumption, OIDC in CI, Azure and GCP. +- [Running scanners](/guides/running-scanners/) IaC directories, container images and scanner arguments. +- [Reports](/guides/reports/) Reading report.html and findings.json. +::: diff --git a/site/content/guides/rate-limits.md b/site/content/guides/rate-limits.md new file mode 100644 index 0000000..caaad8b --- /dev/null +++ b/site/content/guides/rate-limits.md @@ -0,0 +1,299 @@ +--- +title: Rate limits and throttling +lede: "Every cloud API call cloudg makes goes through one process-wide governor that paces it, slows down when a provider pushes back, and records what it lost instead of failing the map." +meta: + - [Package, "`cloudg.resilience`"] + - [Config, "[`ratelimit`](/reference/config/ratelimit/)"] + - [Since, "0.6.0"] +source: docs/RESILIENCE.md +since: "0.6.0" +--- + +A single `cloudg map -p aws --org --regions all` runs 133 deep collector tasks, a Cloud Control sweep and a tagging sweep in every region of every member account, and some collectors fan out per item (one `DescribeImageAttribute` per AMI). Those APIs have quotas, and the quotas are shared with your CI pipelines, Terraform runs and the console. Up to 0.5.x cloudg trusted each SDK's own retries. Forty EC2 clients in one account then backed off independently and kept the account's bucket empty, a service that stayed throttled looked the same in coverage as an access error, and nothing remembered that an API had just said no. + +Since 0.6.0 the `cloudg.resilience` package sits between cloudg and every SDK. You rarely need to touch it. This page explains what it does so you can read its output and tune it when an estate needs it. Every key and default is in the [ratelimit reference](/reference/config/ratelimit/), and the Python side in the [cloudg.resilience API](/api/resilience/). + +## How one call is paced + +Every call is attributed to a scope, `Scope(provider, account, region, service, operation)`, which prints as `aws/123456789012/us-east-1/ec2` (with `/DescribeSubnets` appended where the operation matters). Before the call goes out it passes four gates, and after it comes back the outcome feeds the same state. + +```mermaid caption="call_with_resilience: the full sequence. The SDK integrations reuse the same parts and let the SDK do the retrying." +flowchart TD + A[Call arrives with a scope] --> B{"Breaker open or deadline spent?"} + B -->|yes| X["Fail fast, nothing sent"] + B -->|no| D["Take tokens: provider, account, leaf bucket"] + D --> E["Take a bulkhead slot"] + E --> F[Send the request] + F --> G{Outcome} + G -->|success| H["Recover rate, close breaker"] + G -->|THROTTLED| I["Halve rate, pause for Retry-After"] + I --> J{"Retries, deadline and budget left?"} + G -->|TRANSIENT| J + J -->|yes| B + J -->|no| Z["Breaker counts a failure, record give-up"] + G -->|FATAL| R[Re-raise at once] +``` + +Errors are sorted by `classify(exc)` into three kinds. THROTTLED means the provider asked cloudg to slow down: AWS `Throttling`, `RequestLimitExceeded`, `TooManyRequestsException` and the rest of botocore's list, Azure `SubscriptionRequestsThrottled`, GCP `ResourceExhausted`, or any bare HTTP 429 or 503. TRANSIENT covers 408, 500, 502, 504 and connection resets. FATAL is everything else with a provider error code, `AccessDenied` included. The classifier reads error codes, exception classes and HTTP statuses first and only falls back to message text when none of them says anything. + +### Token buckets per provider, account, region and service + +A call takes a token from up to three buckets, most general first, and waits for the slowest: + +| Level | Key | On by default | +|---|---|---| +| Provider-wide | `global_max_rps` | no, for every provider | +| Account | `account_max_rps` | Azure only: 20 rps, burst 200, per subscription | +| Leaf | account x region x service, or x operation | yes | + +The leaf goes down to the operation for AWS `ec2`, which throttles each API action separately, and for any `"."` override you configure. The built-in leaf rates sit at or just below each provider's documented limits: + +| Provider | Leaf | Rate (rps) | Burst | +|---|---|---|---| +| AWS | default per service | 20 | 40 | +| AWS | `ec2`, per API action | 20 | 100 | +| AWS | `sts` | 50 | 100 | +| AWS | `lambda` | 15 | 15 | +| AWS | `iam` | 10 | 40 | +| AWS | `route53`, `resourcegroupstaggingapi` | 5 | 5, 10 | +| AWS | `organizations`, `cloudformation`, `cloudcontrol` | 5 | 10 | +| AWS | `controltower` | 2 | 5 | +| Azure | default per subscription x resource provider | 15 | 100 | +| Azure | `network` / `storage` | 30 / 0.33 | 200 / 20 | +| Azure | `resourcegraph`, per user | 3 | 15 | +| GCP | default per project or organization x API | 10 | 20 | +| GCP | `cloudasset.ListAssets` | 1.5 | 10 | +| GCP | `cloudasset.SearchAllResources`, `SearchAllIamPolicies` | 6 | 20 | + +Buckets work by reservation. Taking a token never blocks while the limiter's lock is held: the balance can go negative and the caller is told how long to sleep. That queues callers fairly in arrival order and lets asyncio code and worker threads share one limiter. + +### Adaptive halving and additive recovery + +Each leaf bucket adjusts its rate the way TCP congestion control does (AIMD). When a call in the scope is throttled, the bucket's rate is halved, never below the provider's `min_rps` (AWS 0.2, Azure 0.1, GCP 0.05), and its stored burst is drained so queued callers cannot all fire at once. Throttles within one second of the last decrease count as the same event, so forty simultaneous 429s halve the rate once. If the provider sent a Retry-After hint, the bucket is also paused for that long, capped at 300 seconds. + +Recovery is slow on purpose. After five seconds without a throttle, each success adds 5% of the configured ceiling back, at most once per five seconds. An EC2 action at 20 rps that is throttled twice, a second apart, runs at 5 rps, and steady success brings it back to 20 rps in about 75 seconds. botocore's `adaptive` retry mode does something similar per client; the difference is that this state is shared by every client in the process. + +Only leaves adapt. The account and provider-wide buckets stay fixed, and `adaptive: false` freezes a provider's leaves too. + +### Bulkheads + +A bulkhead caps requests in flight per provider and account: 128 for AWS, 16 for Azure and GCP (`max_concurrency`). What holds a slot depends on the integration. On AWS it is one API call, so botocore's retries of that call reuse the slot. Azure holds one per HTTP attempt, and GCP one per whole Cloud Asset listing, all pages included. + +### Circuit breakers + +Each leaf scope has a breaker with the same granularity as its bucket: per EC2 action, per service elsewhere. A throttled `DescribeSubnets` therefore does not block `DescribeVpcs` in the same account and region. + +```mermaid caption="One breaker. Rejected calls never reach the provider and never count as failures." +stateDiagram-v2 + [*] --> Closed + Closed --> Closed: success resets the count + Closed --> Open: breaker_threshold consecutive failures + Open --> HalfOpen: cooldown elapsed + HalfOpen --> Closed: probe succeeds, base cooldown restored + HalfOpen --> Open: probe fails, cooldown doubled up to 600 s + HalfOpen --> HalfOpen: probe lease expires, next call probes +``` + +A failure here means a call that still failed with throttling or a transient error after all its retries. The defaults are 5 consecutive failures (`breaker_threshold`) and a 60-second cooldown (`breaker_cooldown_seconds`). While open, calls fail at once with `circuit open for after repeated throttling; retry in s`. + +Some things deliberately do not count. A call the open circuit rejected was never sent. A call stopped by an empty retry budget was stopped by cloudg, not by the service. And a FATAL answer such as `AccessDenied` or a 404 proves the service is reachable, so it resets the failure count like a success. An `AccessDenied` storm never opens a circuit. + +### Retries and the retry budget + +`call_with_resilience` retries THROTTLED and TRANSIENT errors with decorrelated jitter, `min(cap, uniform(0.5, 3 x previous))`, never sleeping less than the server's Retry-After. It stops at the first of `max_retries` (8), the per-call `deadline_seconds` (900) or the provider's retry budget. + +The budget is the piece that protects a degraded provider. Each provider starts with `retry_budget` tokens (500). Every retry after the first two of a call costs one, and every success refunds 0.1. When a whole provider is struggling, successes stop refilling it, the budget runs dry and cloudg stops multiplying the load. The first two retries of each call stay free, so a long run that drained the budget does not leave every later call without a single retry. + +The SDKs keep their own retry loops, and cloudg plugs into them: + +| Provider | Who retries | Settings that apply | +|---|---|---| +| AWS | botocore, via event hooks on every session cloudg builds | `aws.retry_mode` (adaptive), `aws.max_retries` (10 retries after the first attempt); the budget can end the loop | +| Azure | azure-core, with an explicit `RetryPolicy` and a per-attempt throttle policy | `ratelimit.azure.max_retries`, `max_backoff_seconds`, `deadline_seconds`, budget | +| GCP | google-api-core `Retry` per page | `ratelimit.gcp.max_backoff_seconds`, `deadline_seconds`, at most `max_retries` consecutive retries per page, budget | + +`ratelimit.aws.max_retries`, `max_backoff_seconds` and `deadline_seconds` do not touch botocore. They only bound your own calls through `call_with_resilience`. + +## How coverage records throttled scopes + +Throttling never fails a map. A service that stayed throttled is recorded in coverage and the rest of the run continues. + +Many AWS collectors catch per-item errors so that one bad resource does not stop a sweep. Before 0.6.0 that meant a collector whose calls were all throttled reported SUCCESS with fewer assets. Now a per-task ledger counts calls that gave up after retries and calls an open circuit skipped, and either turns the task's record PARTIAL: + +```json +{"service": "vpc", "status": "SUCCESS", "asset_count": 2, "error": null} +{"service": "subnets", "status": "PARTIAL", "asset_count": 0, "error": "throttled: 1 call(s) gave up after retries (throttled: DescribeSubnets (retries exhausted): RequestLimitExceeded: Request limit exceeded.)"} +``` + +A collector that lets the error escape is recorded FAILED with an error starting `throttled:`. On Azure you get an `azure_` record with FAILED and `azure_full` turns PARTIAL; on GCP the listing record says `listing truncated: throttled: ...`. + +The run's numbers are attached to the result. `InventoryResult.throttling` is `None` unless something eventful happened (a throttle, a transient error, a give-up, a rejection, a breaker trip, or at least a second spent waiting for tokens), `summary["throttling"]` holds the short form, and `inventory-map.json` carries the full block under a top-level `throttling` key: + +```json title="inventory-map.json (excerpt)" +"throttling": { + "totals": {"calls": 2, "throttled": 11, "transient_errors": 0, "retries": 10, + "gave_up": 1, "rejected": 0, "breaker_trips": 0, "wait_seconds": 0.949}, + "messages": [ + "aws/123456789012/us-east-1/ec2: throttled 11x, slowed to 5.0 rps, 10 retries, 1 calls gave up" + ], + "skipped": { + "aws/123456789012/us-east-1/ec2": "throttled: DescribeSubnets (retries exhausted): RequestLimitExceeded: Request limit exceeded." + }, + "scopes": {"...": "per-scope counters"}, + "breakers": {}, + "slowed_buckets": { + "aws/123456789012/us-east-1/ec2/DescribeSubnets": {"rate_rps": 5.0, "ceiling_rps": 20.0, "throttles": 11} + } +} +``` + +`cloudg map` logs each message at WARNING as `Throttling: `. A scope that was never throttled but spent a long time queued for tokens gets a message too, such as `aws/123456789012/us-east-1/ec2: waited 418.6s for rate limit`, which is what an AMI-heavy region looks like at the default EC2 rates. `scopes` is per service; `slowed_buckets` and `breakers` are keyed at bucket granularity (here per EC2 action) and come from the process-wide governor when the run ended, so in a process running several collections they can include other runs' scopes. + +Reading it from Python: + +```python title="throttling_report.py" +from cloudg import CloudGConfig +from cloudg.inventory import InventoryMapper + +config = CloudGConfig(providers=["aws"]) +config.aws.regions = ["us-east-1", "eu-west-1"] + +result = InventoryMapper(config).map_inventory_sync() + +if result.throttling is None: + print("no API pushed back") +else: + for line in result.throttling["messages"]: + print(line) + for scope, reason in result.throttling["skipped"].items(): + print(f"incomplete: {scope}: {reason}") + +for record in result.coverage: + for svc in record.services: + if svc.error and svc.error.startswith("throttled:"): + print(record.account_id, record.region, svc.service, svc.status.value) +``` + +## Guard rails for live collections (MCP) + +People rarely run `cloudg map` twice in a minute. An AI agent might. The MCP live tools (`map_inventory`, `collect_assets`, `run_scanners`, `run_pipeline`) therefore run inside a `LiveOperationGuard`: + +- Identical concurrent calls share one run. The second caller gets the same result with `"joined": true`. +- At most `live_max_concurrent` (1) live operation per scope and `live_max_concurrent_total` (2) overall. +- After an operation touching a scope finishes, successfully or not, the scope cools down for `live_cooldown_seconds` (120). Failures cool down too, since a failure is often throttling. +- `live_caller_max_operations` per caller per `live_caller_window_seconds` is available but off (0). +- An operation still running after `live_operation_timeout_seconds` (3600) is cancelled and frees its scopes. + +Scopes come from the config: `aws/` per configured account, else `aws/profile:`, else `aws`, and the equivalents for Azure subscriptions and GCP projects. A refusal is a tool error with code -31029 and `reason` `cooldown`, `busy` or `quota`, plus `retry_after_seconds` and a `use_dataset` pointer to the newest loaded live dataset covering the same providers. Principals with the `admin` or `operator` role can pass `force=true` to skip the cooldown, though not the concurrency caps. + +The tool `rate_limit_status` and the resource `cloudg://ratelimit` show cooldowns, operations in flight and the governor's throttling state without calling any cloud API. Point agents at them before they reach for a live tool. These guard rails are separate from the MCP policy's per-principal rate limits, which decide how often someone may call a tool at all; see the [MCP server](/mcp/) section. + +## Tuning the ratelimit block + +Start from the defaults and read the run's `throttling` block before changing anything. Unset (`null`) limits use the built-ins, service overrides merge into the built-in table key by key, and a misspelled key anywhere under `ratelimit` is logged and ignored rather than failing the load: + +```text +WARNING cloudg.config: Unknown config key ratelimit.aws.max_rpss is ignored +``` + +### One noisy service + +Override that service, not the provider. For EC2 prefer an operation key, which slows one action instead of all of them: + +```yaml title="config.yaml" +ratelimit: + aws: + services: + resourcegroupstaggingapi: {max_rps: 1, burst: 2} # a cost tool shares this quota + ec2.DescribeImageAttribute: {max_rps: 5, burst: 10} # AMI-heavy region +``` + +:::warning An override replaces the whole built-in entry +`ec2: {max_rps: 10}` drops the built-in burst of 100 to 20 (twice the rate). Repeat the burst if you want to keep it: `ec2: {max_rps: 10, burst: 50}`. EC2 stays per action either way. +::: + +### A shared or large estate + +Throttling spread over many services of one account calls for `account_max_rps`; it is off for AWS, where an organization run across many regions could otherwise send 20 rps per service per region into one account. `global_max_rps` caps everything one process sends to a provider, whatever the account. + +```yaml title="config.yaml" +ratelimit: + aws: + max_rps: 8 # every account x region x service bucket + burst: 16 + account_max_rps: 40 # all services of one account together + global_max_rps: 200 # all of AWS from this process + max_concurrency: 64 # requests in flight per account + min_rps: 0.1 + breaker_cooldown_seconds: 120 + services: + ec2: {max_rps: 10, burst: 50} + iam: {max_rps: 4, burst: 8} + cloudcontrol: {max_rps: 2, burst: 4} + azure: + account_max_rps: 10 # per subscription + services: + resourcegraph: {max_rps: 1, burst: 5} + gcp: + services: + cloudasset.ListAssets: {max_rps: 0.5, burst: 5} +``` + +Lowering `max_rps` leaves headroom for the pipelines and people using the same quota. The cost is a longer run, and the adaptive limiter only reacts after the provider has already throttled someone. + +### Breakers and budgets + +Breakers opening on a flaky service: raise `breaker_threshold` rather than the cooldown. Set `per_operation: true` on a service to give each operation its own bucket and breaker. That only helps where scopes carry an operation, which is AWS and the GCP Cloud Asset RPCs; Azure scopes stop at the resource provider. When a provider is degraded as a whole, a lower `retry_budget` makes cloudg give up sooner. + +```yaml title="config.yaml" +ratelimit: + aws: + breaker_threshold: 10 + retry_budget: 200 + services: + ssm: {max_rps: 10, burst: 20, per_operation: true} +``` + +### Agents that collect too often + +```yaml title="config.yaml" +ratelimit: + live_cooldown_seconds: 600 # ten minutes between live maps of one scope + live_caller_max_operations: 3 # per principal... + live_caller_window_seconds: 3600 # ...per hour + aws: + live_cooldown_seconds: 900 # AWS scopes cool down longer +``` + +### Turning it off + +`ratelimit.enabled: false` removes pacing, breakers, cloudg-level retries and the Azure retry overrides for every provider; botocore keeps its own retries from `aws.max_retries`. `ratelimit..enabled: false` does the same for one provider. That is for benchmarks or a mock cloud, not for production estates. + +The same settings from Python, for embedders: + +```python title="tuned_config.py" +from cloudg import CloudGConfig +from cloudg.resilience import configure, get_governor + +config = CloudGConfig.model_validate( + { + "providers": ["aws"], + "ratelimit": { + "aws": {"account_max_rps": 40, "services": {"iam": {"max_rps": 4, "burst": 8}}}, + "live_cooldown_seconds": 600, + }, + } +) +configure(config) # InventoryMapper and the collectors also do this for you +print(get_governor().budget("aws").remaining) +``` + +## What it does not cover + +Learned rates and breaker state live in memory, so each new `cloudg map` process starts from the configured rates. A few calls are not instrumented yet: the GCP organization-policy and access-context-manager clients outside the asset listing, the urllib fallback that lists Azure subscriptions, and the single region-discovery call. A blocking boto3 or Azure call made on an event-loop thread goes ahead without a bulkhead slot when none is free, so `max_concurrency` is not a hard cap on that path. And per-item fan-out is now held to the documented rates, which makes some maps slower than in 0.5.x: 1,000 `DescribeImageAttribute` calls in one region take about 50 seconds at 20 rps. + +:::links +- [cloudg.resilience](/api/resilience/) `call_with_resilience`, `@resilient`, the governor and the guard from Python. +- [ratelimit reference](/reference/config/ratelimit/) Every key with its default. +- [Inventory mapping](/guides/inventory-mapping/) Where coverage records come from. +::: diff --git a/site/content/guides/reports.md b/site/content/guides/reports.md new file mode 100644 index 0000000..b85ad44 --- /dev/null +++ b/site/content/guides/reports.md @@ -0,0 +1,306 @@ +--- +title: Reports +lede: "Every path through cloudg ends in the same two files: `findings.json` for machines and `report.html` for people. `cloudg report` rebuilds them from an earlier run without touching the cloud." +meta: + - [Command, "`cloudg report`"] + - [Formats, "HTML, JSON, SVG"] + - [Python, "`JSONExporter`, `HTMLReportGenerator`, `SVGRenderer`"] +source: cloudg/renderers/html_report.py +since: "0.6.0" +--- + +The renderers in `cloudg/renderers/` take a normalised `ScanResult` (assets, findings, compliance results) plus, when there is one, the D3 graph of the collected assets. They don't call any cloud API or scanner, which is why `cloudg report` can re-render an old run anywhere. + +## Which command writes what + +```mermaid caption="Report inputs and outputs" +flowchart LR + RUN["cloudg run"] --> SR["ScanResult and graph"] + ING["cloudg ingest"] --> SR + FJ["findings.json from an earlier run"] -->|"cloudg report -i"| SR + SR --> JSON["findings.json"] + SR --> HTML["report.html"] + SR --> SVG["topology.svg"] + SCAN["cloudg scan, cloudg ingest"] --> RAW["raw-findings.json"] +``` + +| Command | `findings.json` | `report.html` | `topology.svg` | `raw-findings.json` | +|---|---|---|---|---| +| `cloudg run` | Yes | Yes | Yes | No | +| `cloudg ingest` | With `--format json` or `all` | With `--format html` or `all` | No | Always | +| `cloudg report` | With `--format json` or `all` | With `--format html` or `all` | With `--format svg` or `all` | No | +| `cloudg scan` | No | No | No | Always | +| `run_pipeline()`, `run_from_reports()` | Yes | Yes | No | No | + +`cloudg run` writes more than these (GraphML, Cytoscape JSON, the ontology, RAG chunks, Terraform); those come from the graph and analysis phases and are listed on [Output files](/reference/output-files/). + +Those three are the only report formats. There is no CSV, SARIF or PDF writer in 0.6.0. `findings.json` is easy to convert, though; there's a [CSV recipe](#exporting-to-csv) further down. + +## findings.json + +`JSONExporter` writes one JSON object with six keys: + +```json title="findings.json" +{ + "metadata": { + "scan_id": "e0d347bc-b1bc-4f3e-8d1a-0cc98878e0d9", + "provider": null, + "account_id": null, + "region": null, + "started_at": "2026-10-09 16:27:37.437338", + "completed_at": "2026-10-09 16:27:37.437314" + }, + "summary": { + "total_assets": 0, + "total_findings": 5, + "total_edges": 0, + "severity_breakdown": {"CRITICAL": 2, "HIGH": 3}, + "compliance_frameworks": ["CIS", "CIS-AWS", "CVE", "NIST-CSF-AWS"] + }, + "assets": [], + "findings": [], + "compliance": [], + "graph": {"nodes": [], "links": []} +} +``` + +`assets`, `findings` and `compliance` are the Pydantic models from `cloudg.schema.models`, serialised in JSON mode. Assets leave out `raw_data`. Findings include the computed `risk_score`. `graph` holds D3 nodes and links for `cloudg run`, and is empty for ingest. In the `metadata` block, the normaliser doesn't fill in `provider`, `account_id` or `region` in 0.6.0, so expect `null` there, and both timestamps are the time of normalisation. The field-by-field reference is on [Output files](/reference/output-files/), and the models are on [Data models](/api/models/). + +`raw-findings.json` is different: a bare JSON list of `Finding` objects as the parsers produced them, before deduplication, rescoring or compliance mapping. It has no `summary`, `compliance` or `graph`. + +## report.html + +The HTML report is one file rendered from the Jinja template `cloudg/templates/report.html.j2`. The data is embedded in the page as JSON, so you can mail it, attach it to a ticket or archive it next to `findings.json`. + +The header shows the scan ID, provider and account, a button that downloads the embedded data, and six counters: total assets, total findings, and the CRITICAL, HIGH, MEDIUM and LOW counts. Below it are four tabs. + +The Infrastructure Map tab draws a D3 tree of account, region, VPC, subnet and resources, built from each asset's `region` and its `vpc_id` / `subnet_id` metadata. Assets with neither land under "Global / Ungrouped". Nodes with findings get a red ring and internet-exposed ones an orange ring. Toggle Layout switches to a force-directed graph of the collected graph plus one triangle node per finding (linked to its resource by a `FINDING_AFFECTS` edge) and one diamond per compliance framework (linked to its findings by `COMPLIANCE_GOVERNS`). With no assets, as after `cloudg ingest`, the tab goes straight to that force view, which then shows findings and frameworks only. + +The Findings tab lists every finding, sorted by severity, with dropdown filters for severity and source tool and a text search over title, resource and description. Clicking a row opens a side panel with the description, evidence, remediation, CVSS score, frameworks and risk score. + +Analytics has four Chart.js charts: severity distribution, asset types, findings by source tool, and compliance per framework. + +Compliance shows one card per framework with Pass and Fail counts. As explained in [Compliance frameworks](/guides/compliance/#report-html), cloudg only records controls that have findings, so Pass is 0 and the percentage bar reads 0%. Read Fail as the number of affected controls. + +The download button saves a file named `findings.json` that holds only `findings`, `graph` and the tree data. It isn't the same file cloudg writes, but `cloudg report -i` and the viewer both accept it. + +:::warning The report needs network access +`report.html` loads Chart.js 4.4.0 and D3 7 from `cdn.jsdelivr.net`. Opened without network access, the findings table and detail panel still work, but the map, the charts and the compliance cards stay empty because the page's script stops at the first D3 call. The `report.inline_js` config key doesn't change this in 0.6.0. For air-gapped use, build an [offline template](#a-report-that-works-offline). +::: + +The page also inserts finding titles, descriptions and evidence into its HTML without escaping them. Scanner output can contain resource names and descriptions that other people control, so treat a report built from someone else's scan files like any other HTML from that source. + +If the template can't be found or fails to render, cloudg logs a warning and writes a plain fallback page instead: the counters and a findings table, with no script at all. + +## topology.svg + +`SVGRenderer` draws a static map of the collected assets: VPCs and VNets as containers, subnets inside them, compute, databases and functions inside the subnets, and IAM principals and security groups in side panels. It needs assets, so it only means something after `cloudg run`, or after `cloudg report` on a `findings.json` that has assets. + +## cloudg report + +`cloudg report` reads a `findings.json` and renders it again. Use it to regenerate the HTML after upgrading cloudg, to produce just the SVG, or to rebuild a report on a machine that never had cloud access. + +```bash tab="CLI" +cloudg report -i ./reports/findings.json -o ./reports-rerendered +cloudg report -i ./reports/findings.json -o ./reports-rerendered --format html +cloudg report -i ./archive/2026-09-30/findings.json -o ./archive/2026-09-30/svg --format svg +``` + +```bash tab="Docker" +docker run --rm \ + -v "$PWD/reports:/data" \ + ghcr.io/morpheuslord/cloudg:latest report -i /data/findings.json -o /data/rerendered +``` + +| Flag | Meaning | +|---|---| +| `-i, --input` | Path to a `findings.json`. Required. | +| `-o, --output` | Output directory, `./reports` by default | +| `--format` | `html`, `json`, `svg` or `all` (default) | + +A missing input file prints `Input file not found` and exits with code 1. + +`cloudg report` rebuilds the `ScanResult` from three keys of the input: `assets`, `findings` and `graph`. The rest is not carried over: + +| In the input | After `cloudg report` | +|---|---| +| `assets`, `findings`, `graph` | Kept | +| `compliance` | Dropped: the new `findings.json` has an empty list, and the Compliance tab and chart are empty | +| Edges between assets | Not stored in `findings.json`, so `topology.svg` has no connections | +| `metadata` | A new scan ID and new timestamps | + +:::warning Don't render into the input's directory +`-o` defaults to `./reports`, which is also where `cloudg run` and `cloudg ingest` write. `cloudg report -i ./reports/findings.json` with no `-o` overwrites the original `findings.json` with a copy that has no compliance results. Always give `-o` a different directory. +::: + +`cloudg report` only reads the `findings.json` object. Given `raw-findings.json`, which is a list, it fails with `AttributeError: 'list' object has no attribute 'get'`. To turn raw findings into reports, use the [Python route below](#from-raw-findings). + +## Rendering from Python + +The three renderers are plain classes. Each takes an output directory and writes one file, returning its path. + +| Class | Call | Default file | +|---|---|---| +| `cloudg.renderers.json_export.JSONExporter(output_dir=".")` | `.export(scan_result, graph_json=None, filename="findings.json")` | `findings.json` | +| `cloudg.renderers.html_report.HTMLReportGenerator(template_dir=None, output_dir=".")` | `.generate(scan_result, graph_json=None, filename="report.html")` | `report.html` | +| `cloudg.renderers.svg.SVGRenderer(output_dir=".")` | `.render(assets, edges, findings_by_resource=None, filename="topology.svg", width=1800, height=1400)` | `topology.svg` | + +### Re-render and keep compliance + +This does what `cloudg report` does, but carries the compliance results over: + +```python title="rerender.py" +import json +from pathlib import Path + +from cloudg.renderers.html_report import HTMLReportGenerator +from cloudg.renderers.json_export import JSONExporter +from cloudg.schema.models import CloudAsset, ComplianceResult, Finding, ScanResult + +data = json.loads(Path("./reports/findings.json").read_text()) + +scan_result = ScanResult( + assets=[CloudAsset.model_validate(a) for a in data.get("assets", [])], + findings=[Finding.model_validate(f) for f in data.get("findings", [])], + compliance=[ComplianceResult.model_validate(c) for c in data.get("compliance", [])], +) +graph = data.get("graph") or {"nodes": [], "links": []} + +out = "./reports-rerendered" +print(HTMLReportGenerator(output_dir=out).generate(scan_result, graph_json=graph)) +print(JSONExporter(output_dir=out).export(scan_result, graph_json=graph)) +print(len(scan_result.compliance), "compliance results kept") +``` + +### From raw findings {#from-raw-findings} + +`raw-findings.json` from `cloudg scan` (or `cloudg ingest`) hasn't been normalised yet. Load it, normalise it, then render: + +```python title="raw_to_reports.py" +import json +from pathlib import Path + +from cloudg import CloudGConfig, CloudGEngine +from cloudg.renderers.html_report import HTMLReportGenerator +from cloudg.renderers.json_export import JSONExporter +from cloudg.schema.models import Finding + +raw = json.loads(Path("./reports/raw-findings.json").read_text()) +findings = [Finding.model_validate(item) for item in raw] + +engine = CloudGEngine(CloudGConfig()) +scan_result = engine.normalise_findings(findings) + +out = "./reports-from-raw" +print(JSONExporter(output_dir=out).export(scan_result)) +print(HTMLReportGenerator(output_dir=out).generate(scan_result)) +print(len(scan_result.compliance), "compliance results") +``` + +`normalise_findings()` uses `config.rulesets.rules_dir`, so a custom ruleset applies here too. + +### A report that works offline {#a-report-that-works-offline} + +`HTMLReportGenerator` accepts a `template_dir` holding a `report.html.j2`. The script below copies cloudg's own template and replaces the two CDN script tags with the libraries' source, wrapped in `{% raw %}` so Jinja leaves the JavaScript alone. Run it once on a machine with network access; the resulting template works anywhere. + +```python title="build_offline_template.py" +"""Make a copy of cloudg's report template with Chart.js and D3 inlined.""" +import urllib.request +from pathlib import Path + +import cloudg + +LIBS = { + "https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js", + "https://cdn.jsdelivr.net/npm/d3@7/dist/d3.min.js", +} + +source = Path(cloudg.__file__).parent / "templates" / "report.html.j2" +template = source.read_text() + +for url in LIBS: + tag = f'' + if tag not in template: + raise SystemExit(f"script tag not found, template changed: {url}") + js = urllib.request.urlopen(url, timeout=30).read().decode("utf-8") + template = template.replace(tag, "") + +out = Path("offline-template") +out.mkdir(exist_ok=True) +(out / "report.html.j2").write_text(template) +print("wrote", out / "report.html.j2", len(template), "bytes") +``` + +Then render with it: + +```python title="offline_report.py" +import json +from pathlib import Path + +from cloudg.renderers.html_report import HTMLReportGenerator +from cloudg.schema.models import CloudAsset, ComplianceResult, Finding, ScanResult + +data = json.loads(Path("./reports/findings.json").read_text()) +scan_result = ScanResult( + assets=[CloudAsset.model_validate(a) for a in data.get("assets", [])], + findings=[Finding.model_validate(f) for f in data.get("findings", [])], + compliance=[ComplianceResult.model_validate(c) for c in data.get("compliance", [])], +) + +generator = HTMLReportGenerator(template_dir="./offline-template", output_dir="./reports-offline") +print(generator.generate(scan_result, graph_json=data.get("graph"))) +``` + +The result is about 530 KB, most of it the two libraries. The same mechanism works for your own design: copy the template, change it, and pass `template_dir`. The variables available to a template are `scan_id`, `provider`, `account_id`, `started_at`, `completed_at`, `total_assets`, `total_findings`, `severity_counts`, `resource_types`, `source_tools`, `compliance_summary`, `findings` and `assets`, plus JSON-string versions (`severity_counts_json`, `resource_types_json`, `source_tools_json`, `compliance_summary_json`, `findings_json`, `assets_json`, `graph_json`, `hierarchy_json`, `findings_per_resource_json`) for embedding in scripts. + +### Exporting to CSV {#exporting-to-csv} + +For a spreadsheet or a ticketing import, flatten the findings: + +```python title="findings_to_csv.py" +import csv +import json +from pathlib import Path + +data = json.loads(Path("./reports/findings.json").read_text()) +columns = ["severity", "risk_score", "source_tool", "title", "resource_arn", + "source_finding_id", "compliance_frameworks", "remediation"] + +with open("findings.csv", "w", newline="") as fh: + writer = csv.DictWriter(fh, fieldnames=columns, extrasaction="ignore") + writer.writeheader() + for finding in data["findings"]: + row = dict(finding) + row["compliance_frameworks"] = ";".join(finding.get("compliance_frameworks", [])) + writer.writerow(row) + +print("wrote", len(data["findings"]), "rows to findings.csv") +``` + +## The report viewer + +`docs/viewer.html` in the repository is a standalone page for browsing results without generating anything. Open it in a browser and drop a file on it, or use Open a report. It reads `findings.json` from `cloudg run`, `cloudg ingest` or `cloudg report`, and the bare list in `raw-findings.json` from `cloudg scan`. The file is parsed in the browser and not uploaded anywhere. The page starts with built-in sample data, marked with a "sample data" badge, until you load your own. + +It shows severity and tool counts, failed compliance controls per framework (top ten), the most affected resources, a sortable and filterable findings table, and the topology when the file has graph data. Raw findings have no compliance or graph, and the page says so. Like the report, it loads D3 from a CDN (`cdnjs.cloudflare.com`). + +## Configuration + +`config.yaml` has a `report:` section: + +```yaml title="config.yaml" +report: + formats: [html, json, svg] + inline_js: true + output_dir: ./reports +``` + +None of the three keys affects report generation in 0.6.0. The commands decide formats with `--format` and the location with `-o`, the HTML always loads its libraries from the CDN, and the only reader of `report.output_dir` is the MCP server, which uses it as the default workspace directory. Set `-o` explicitly. The keys are listed on the [report config page](/reference/config/report/). + +## Next + +:::links +- [cloudg report](/cli/report/) The command reference. +- [Output files](/reference/output-files/) Every file cloudg writes, field by field. +- [Compliance frameworks](/guides/compliance/) What the compliance output means. +- [Ingesting existing output](/guides/ingesting/) Reports from scans that ran elsewhere. +::: diff --git a/site/content/guides/running-scanners.md b/site/content/guides/running-scanners.md new file mode 100644 index 0000000..7543917 --- /dev/null +++ b/site/content/guides/running-scanners.md @@ -0,0 +1,299 @@ +--- +title: Running scanners +lede: "cloudg doesn't ship its own cloud checks. It drives Prowler, ScoutSuite, Checkov and Trivy as subprocesses, adds a built-in IAM linter, and turns everything they report into one list of findings." +meta: + - [Commands, "`cloudg scan`, `cloudg run`"] + - [Python, "`CloudGEngine.scan()`"] + - [Needs, "Scanner binaries on `PATH`"] +source: cloudg/api_scanners.py +since: "0.6.0" +--- + +There are two ways to run the scanners. `cloudg scan` runs them on their own, with no collection, and writes the raw findings to disk. `cloudg run` runs them as phase 3 of the full pipeline, after collection and graph analysis, and passes the findings through the normaliser into the reports. The two commands share the scanner wrappers in `cloudg/scanners/` but differ in defaults and in which config keys they read; [the table near the end](#scan-and-run-compared) lists every difference. + +If the scans already ran somewhere else (a CI job, another host, a nightly cron) you don't need either command. Hand the output files to [`cloudg ingest`](/guides/ingesting/) instead. + +## The scanners + +Each scanner has a name you use in `--scanners` and in `scanners.enabled`. Four of them are external programs that cloudg finds with `shutil.which`; the fifth is built in. + +| Name | Binary | Install | Scans | Needs cloud credentials | +|---|---|---|---|---| +| `prowler` | `prowler` | `pip install prowler` | Live cloud accounts (CSPM checks) | Yes | +| `scoutsuite` | `scout` | `pip install scoutsuite` | Live cloud accounts (configuration audit) | Yes | +| `checkov` | `checkov` | `pip install checkov` | IaC files: Terraform, CloudFormation, ARM, Kubernetes and more | No | +| `trivy` | `trivy` | Binary from [trivy.dev](https://trivy.dev/) | Container images (CVEs, secrets), or a directory (vulnerabilities, misconfigurations, secrets) | Only for private registries | +| `iam` | none | Parliament ships with cloudg | IAM role trust policies and policy documents from the collected inventory | No (reads collected assets) | + +Installing each external tool into its own virtual environment keeps its dependencies away from cloudg's and from the other scanners. The [Docker image](/guides/docker/) does the same: Prowler and Checkov live in separate venvs under `/opt`, with the binaries symlinked onto `PATH`. That image bundles Prowler, Checkov and Trivy. It does not include ScoutSuite. + +```bash +python -m venv ~/.venvs/prowler && ~/.venvs/prowler/bin/pip install prowler +python -m venv ~/.venvs/checkov && ~/.venvs/checkov/bin/pip install checkov +ln -s ~/.venvs/prowler/bin/prowler ~/.local/bin/prowler +ln -s ~/.venvs/checkov/bin/checkov ~/.local/bin/checkov +which prowler checkov trivy scout +``` + +### Prowler + +cloudg runs one Prowler process per provider and asks for ASFF output: + +```text +prowler -M json-asff -o /prowler/ [-p ] +``` + +`-p` is only added for AWS. Under `cloudg run`, the subprocess environment also gets `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` when `aws.access_key_id` and `aws.secret_access_key` are set (from config or `--aws-key` / `--aws-secret`), and `AWS_DEFAULT_REGION` set to the first entry of `aws.regions`. Nothing else is forwarded: a session token, `--aws-role-arn` or an OIDC token file that you gave cloudg for collection does not reach Prowler. Give Prowler its own credentials through a profile, the environment, or its own flags in `scanners.prowler_extra_args`. + +Prowler's stdout is streamed into the cloudg log, with a progress line every 25 checks. A non-zero exit code is logged as a warning and the output is parsed anyway, since Prowler exits non-zero whenever a check fails. Findings with `Compliance.Status` of `PASSED` are dropped during parsing. + +:::warning Regions set to all +With `--regions all`, `aws.regions` holds the literal value `ALL`, and that is what cloudg puts in `AWS_DEFAULT_REGION` for Prowler. If Prowler fails with endpoint errors under `--regions all`, run it with an explicit region list instead. Prowler scans every enabled region by default anyway. +::: + +### ScoutSuite + +Also one process per provider: + +```text +scout --report-dir /scoutsuite/ --no-browser [--profile ] +``` + +`--profile` is only passed for AWS. For Azure and GCP, ScoutSuite needs an authentication mode flag (for example `--cli` for Azure CLI credentials), which you supply through `scanners.scoutsuite_extra_args`. cloudg then searches the report directory for `scoutsuite_results*.js` and parses the JSON inside it. + +### Checkov + +One process per IaC directory: + +```text +checkov -d --output json --quiet [--framework ...] +``` + +With `scanners.checkov_frameworks` empty (the default), cloudg leaves out `--framework` so Checkov auto-detects every framework it finds in the directory. A non-zero exit is normal for Checkov when checks fail and is not treated as an error. cloudg reads the JSON from stdout and does not save it. + +### Trivy + +With images configured, Trivy runs once per image, one after another, inside a single job: + +```text +trivy image --format json --quiet +``` + +With no images, the CLI commands fall back to a filesystem scan of the IaC directories: + +```text +trivy fs --format json --quiet --scanners vuln,misconfig,secret +``` + +Like Checkov, the JSON is read from stdout and not written to disk. Trivy pulls images with its own registry authentication, so log in to private registries (ECR, ACR, Artifact Registry) the way Trivy expects before you start. + +### The IAM linter + +The linter looks at every collected asset of type `IAM_ROLE` or `IAM_POLICY`. It reads `metadata["assume_role_policy"]` (preferred when present, so for roles it is the trust policy) or `metadata["policy_document"]`, and runs two passes over it. The first is Parliament, which reports wildcard abuse and logical errors as `source_tool="parliament"`. The second is cloudg's own check for `Allow` statements with `Action: *`, or with `Resource: *` combined with a wildcard action or an action under `iam:`, `sts:`, `kms:`, `s3:`, `ec2:` or `lambda:`. Those are reported as HIGH with `source_tool="cloudg-iam"`. + +Because it reads collected assets, the linter has nothing to do in `cloudg scan`, which collects nothing. + +### Reachability findings + +The scan phase also includes findings that come from the graph, not from a scanner. `ReachabilityAnalyzer` walks the asset graph from the internet node and reports exposed resources with `source_tool="cloudg-reachability"`. In `cloudg run` this happens in phase 2, and the results join the scanner findings at normalisation. Like the IAM linter, it needs collected assets. + +### Cloud Custodian policies + +`cloudg/policies/` (in the repository and inside the installed package) holds four Cloud Custodian policy packs: `custodian.yml` (AWS cost and governance), `custodian-aws-security.yml`, `custodian-azure.yml` and `custodian-gcp.yml`. cloudg never runs them, and `cloudg ingest` cannot read Custodian output. They are there for you to run with `custodian` directly: + +```bash +pip install c7n c7n-azure c7n-gcp +custodian run --output-dir ./custodian-output cloudg/policies/custodian-aws-security.yml +``` + +## Choosing scanners + +`--scanners` takes a comma-separated list of the names above. Spaces are trimmed and names are lowercased. An unknown name is ignored without a warning. + +```bash tab="CLI" +cloudg run -p aws --regions us-east-1,eu-west-1 --scanners prowler,trivy --images 123456789012.dkr.ecr.us-east-1.amazonaws.com/api:1.4.2 +cloudg scan -p aws --profile audit --scanners prowler,scoutsuite -o ./scan-2026-10-09 +``` + +```python tab="Python" title="scan_only.py" +import asyncio + +from cloudg import CloudGConfig, CloudGEngine +from cloudg.scanners.checkov import CheckovScanner +from cloudg.scanners.prowler import ProwlerScanner +from cloudg.scanners.trivy import TrivyScanner + +for tool in (ProwlerScanner, CheckovScanner, TrivyScanner): + print(tool.__name__, "installed" if tool.is_available() else "missing") + +config = CloudGConfig(providers=["aws"]) +config.scanners.enabled = ["checkov", "trivy"] +config.scanners.checkov_frameworks = ["terraform"] + +engine = CloudGEngine(config) +engine.on_error = lambda name, exc: print("scanner error:", name, exc) + +findings = asyncio.run( + engine.scan( + assets=[], + edges=[], + iac_dir="./terraform", + images=["python:3.12-slim"], + output_dir="./reports", + ) +) +print(len(findings), "findings") +``` + +```bash tab="Docker" +docker run --rm \ + -v ~/.aws:/home/cloudg/.aws:ro \ + -v "$PWD/reports:/app/reports" \ + ghcr.io/morpheuslord/cloudg:latest scan -p aws --scanners prowler -o /app/reports +``` + +`cloudg run` without `--scanners` uses `scanners.enabled`, which defaults to all five names. `cloudg scan` without `--scanners` runs `prowler,checkov` and ignores the config list. + +The IAM linter is the odd one out. In `cloudg run` it runs whenever collection returned any assets, even if `iam` isn't in the list, so leaving it out of `--scanners` does not switch it off. `CloudGEngine.scan()` behaves the same way. + +## IaC targets: --iac-dir and the Terraform fallback + +Checkov, and Trivy when it has no images, need a directory. `cloudg run` and `CloudGEngine.scan()` resolve it in a fixed order and stop at the first hit. + +```mermaid caption="How cloudg run picks the directories for Checkov and Trivy fs" +flowchart TD + A{"--iac-dir given?"} -->|yes| X["Scan that directory"] + A -->|no| B{"scanners.iac_directories set?"} + B -->|yes| Y["Scan each listed directory"] + B -->|no| C{"Terraform recreation on and *.tf.json written?"} + C -->|yes| Z["Scan the recreation of the live cloud"] + C -->|no| S["Skip Checkov, warn"] +``` + +The last step is the interesting one. With `--terraform` (or `terraform.enabled: true`) and nothing else configured, cloudg writes a Terraform recreation of the collected infrastructure and points Checkov at it, so Checkov audits your real cloud through its Terraform representation. There is no fallback to the current directory: scanning whatever folder cloudg was started from would produce a clean-looking result that says nothing about the cloud. + +```bash +cloudg run -p aws --regions us-east-1 --terraform --scanners checkov,trivy +``` + +`cloudg scan` is older and simpler. Its `--iac-dir` defaults to `.`, so Checkov (and the Trivy filesystem fallback) scan the working directory unless you say otherwise. + +## Container images: --images + +`--images` takes a comma-separated list of full image references. Without it, `cloudg run` falls back to `scanners.trivy_images`. If neither is set, the CLI commands run `trivy fs` on the IaC directories instead, and `cloudg run` skips Trivy with a warning when there are no directories either. + +`CloudGEngine.scan()` has no filesystem fallback. From Python, Trivy only runs when there are images, either from the `images` argument or from `config.scanners.trivy_images`. + +## Configuration + +Every key below lives under `scanners:` in `config.yaml`. They apply to `cloudg run` and `CloudGEngine.scan()`; `cloudg scan` ignores all of them. The full reference is on the [scanners config page](/reference/config/scanners/). + +```yaml title="config.yaml" +scanners: + enabled: [prowler, checkov, trivy, iam] + prowler_extra_args: ["--severity", "critical", "high"] + scoutsuite_extra_args: [] + checkov_extra_args: ["--skip-check", "CKV_AWS_18"] + checkov_frameworks: [terraform, cloudformation] # empty = auto-detect + trivy_extra_args: ["--severity", "HIGH,CRITICAL", "--ignore-unfixed"] + trivy_images: + - 123456789012.dkr.ecr.us-east-1.amazonaws.com/api:1.4.2 + - 123456789012.dkr.ecr.us-east-1.amazonaws.com/worker:1.4.2 + iac_directories: [./infra/terraform, ./infra/k8s] + timeout_seconds: 3600 +``` + +The `*_extra_args` lists are appended to the scanner's command line as separate arguments, after cloudg's own flags. They are passed to every run of that scanner. When Prowler or ScoutSuite runs once per provider, the same extra arguments go to all of them, so an Azure-only flag will break the AWS run. If you need different flags per cloud, run the providers as separate `cloudg run` invocations with separate config files. + +```bash +cloudg -c config.yaml run -p aws --regions us-east-1 +``` + +## Parallelism and timeouts + +Every scanner target becomes one job in a thread pool: one Prowler job per provider, one ScoutSuite job per provider, one Checkov job per IaC directory, one Trivy job for all images (or all directories), and one IAM linter job. The pool has `len(scanners) + len(providers) + 1` workers, with a floor of two, so for `cloudg run -p all` with all five scanners that is nine workers. Jobs beyond that wait their turn. Since each job is mostly a subprocess, the threads spend their time waiting and the real CPU load comes from the scanners themselves. + +The time limits that actually stop a scanner are the subprocess timeouts inside each wrapper: + +| Scanner | Subprocess timeout | What happens when it fires | +|---|---|---| +| Prowler | 3600 s | Process killed, no findings from that provider | +| ScoutSuite | 3600 s | No findings from that provider | +| Checkov | 1800 s per directory | No findings from that directory | +| Trivy | 1800 s per image or directory | That image or directory is skipped, the rest continue | + +`scanners.timeout_seconds` (default 3600, minimum 60) is passed to the wait on each job's result. In 0.6.0 that wait only happens after the job has already finished, so the setting does not cut a slow scanner short. Plan around the per-tool limits above. + +## When something goes wrong + +A missing binary does not fail the run. The wrapper logs a warning such as `Prowler is not installed. Install with: pip install prowler. Skipping Prowler scan.` and returns no findings. The progress display then shows a tick and `Prowler: 0 findings`, which looks exactly like a clean scan. Read the warnings, or check the binaries before you trust a zero: + +```console +$ cloudg scan -p aws --scanners prowler,checkov --iac-dir ./infra +[21:50:50] WARNING Checkov is not installed. Install with: pip install checkov. + Skipping Checkov scan. + WARNING Prowler is not installed. Install with: pip install prowler. + Skipping Prowler scan. + ✓ Prowler: 0 findings 0:00:00 + ✓ Checkov: 0 findings 0:00:00 + + ✓ Total: 0 findings + ✓ Raw findings: reports/raw-findings.json +``` + +Other failures follow the same rule: one scanner's problem never stops the others. + +| Symptom | Cause | Fix | +|---|---|---| +| `0 findings` from a scanner straight away | Binary not on `PATH` | Install it, or run inside the Docker image | +| `[Prowler] Exited with code 3 after ...` | Prowler found failing checks | Normal; findings are still parsed | +| `Checkov: nothing to scan; pass --iac-dir, ...` | No `--iac-dir`, no `iac_directories`, no Terraform recreation | Pass `--iac-dir` or add `--terraform` | +| `Trivy: nothing to scan` | No images and no IaC directories | Pass `--images` or set `scanners.trivy_images` | +| `No ScoutSuite results found in ...` | ScoutSuite failed before writing results, often authentication | Run the printed `scout` command by hand to see its error | +| ` failed: ...` in the progress list | The wrapper raised an exception | Run with `cloudg -v` for the traceback | + +From Python, set `engine.on_error` to see failures as they happen. It receives the job name (`prowler-aws`, `checkov-./infra`, `trivy`, `iam`) and the exception. A missing binary is not an error there either; it just yields an empty list. + +## Where raw output lands + +| Scanner | `cloudg run` and `CloudGEngine.scan()` | `cloudg scan` | +|---|---|---| +| Prowler | `/prowler//*.json` (ASFF) | `/prowler/*.json` | +| ScoutSuite | `/scoutsuite//` (HTML report and `scoutsuite-results/scoutsuite_results_*.js`) | `/scoutsuite/` | +| Checkov | Not saved (read from stdout) | Not saved | +| Trivy | Not saved (read from stdout) | Not saved | +| All findings | Normalised into `findings.json` and `report.html` | `raw-findings.json`, before normalisation | + +:::warning Reusing an output directory +Prowler names each output file after the account and a timestamp, and cloudg parses every `*.json` under the Prowler output directory. Point a second run at the same `-o` and the older ASFF files are parsed again. Repeated checks on the same resource are merged during normalisation, but a problem you fixed after the first run still shows up from the old file. Use a fresh output directory per run. +::: + +The Prowler and ScoutSuite directories are in the native formats that `cloudg ingest` reads, so you can re-aggregate them later without scanning again: + +```bash +cloudg ingest --prowler ./reports/prowler/aws/ --scoutsuite ./reports/scoutsuite/aws/ -o ./reports-again +``` + +`raw-findings.json` from `cloudg scan` is a JSON list of cloudg `Finding` objects, not a scanner format. `cloudg report` can't read it. To turn it into reports, normalise it in Python as shown in [Reports](/guides/reports/#from-raw-findings), or merge it into an inventory map with [`cloudg map --findings`](/guides/inventory-mapping/). + +## Scan and run compared {#scan-and-run-compared} + +| | `cloudg scan` | `cloudg run` (phase 3) | +|---|---|---| +| Collects assets first | No | Yes | +| Default scanners | `prowler,checkov` | `scanners.enabled` (all five) | +| Reads `scanners.*` config keys | No | Yes | +| Providers | One, from `-p` (default `aws`) | Every provider from `-p` | +| IaC directory when none is given | `.` | Config, then the Terraform recreation, else skipped | +| Trivy without images | `trivy fs` on the IaC directory | `trivy fs` on the resolved directories, else skipped | +| IAM linter and reachability | Never | Whenever assets were collected | +| Writes | `raw-findings.json` | `findings.json`, `report.html`, `topology.svg` and the graph files | + +## Next + +:::links +- [Ingesting existing output](/guides/ingesting/) Aggregate scans that ran elsewhere, with no binaries and no credentials. +- [Compliance frameworks](/guides/compliance/) How findings are tagged with framework controls after the scan. +- [Reports](/guides/reports/) What `findings.json` and `report.html` contain. +- [cloudg scan](/cli/scan/) Every flag of the standalone scan command. +::: diff --git a/site/content/mcp/connect.md b/site/content/mcp/connect.md new file mode 100644 index 0000000..f0d5e4b --- /dev/null +++ b/site/content/mcp/connect.md @@ -0,0 +1,329 @@ +--- +title: Connect a client +lede: "`cloudg mcp config` writes the JSON your client needs. This page walks through it for Claude Desktop, Claude Code, Cursor and VS Code, then covers datasets, policies, HTTP with a token, and what to check when the client shows no tools." +meta: + - [Command, "`cloudg mcp config`"] + - [Clients, "Claude Desktop, Claude Code, Cursor, VS Code"] + - [Transports, "stdio, streamable HTTP"] +source: docs/MCP.md +since: "0.6.0" +--- + +Most people run cloudg over stdio: the client starts `cloudg mcp serve` as a child process when it opens and stops it when it closes. You don't keep a server running, and nothing listens on a port. HTTP is for the other cases: a server shared by several people, a client on a different machine, or a gateway in front of it. + +```mermaid caption="Which setup to use" +flowchart TD + A{"Does the client run on the machine that holds the reports?"} -->|yes| B{"One user, one client?"} + A -->|no| E["HTTP: cloudg mcp serve --transport http, with --auth-token"] + B -->|yes| C["stdio: the client launches cloudg-mcp serve"] + B -->|"no, shared"| E + E --> F{"Client is Claude Desktop?"} + F -->|yes| G["Bridge with npx mcp-remote"] + F -->|no| H["Client connects to the URL directly"] +``` + +## Before you start + +You need cloudg 0.6.0 or newer on Python 3.11 or newer. The base install includes the MCP layer and a dependency-free server; the `mcp` extra adds the official SDK, which only matters if you want `--flavor sdk`. + +```bash tab="pip" +pip install cloudg +``` + +```bash tab="uv" +uv pip install cloudg +``` + +You also need something to look at: an `inventory-map.json` from [cloudg map](/cli/map/), a `findings.json` from [cloudg run](/cli/run/), or native scanner output that [cloudg ingest](/cli/ingest/) understands. The examples below assume `./reports/inventory-map.json`. + +If you installed cloudg in editable mode before 0.6.0, run `pip install -e .` again so the `cloudg-mcp` console script gets created. + +## Set up a stdio client + +::::steps +### Check that the server loads your data + +Run one tool from a terminal before touching any client config. `cloudg mcp call` goes through the same layer, policy and transforms the server uses, so if this works, the server will too. + +```bash +cloudg mcp call workspace_status --dataset prod=./reports/inventory-map.json +``` + +The result lists the dataset with its asset, edge and finding counts, and the allowed roots. An error here (wrong path, unknown policy) is the same error the client would hide from you later. + +### Generate the snippet + +`cloudg mcp config --client NAME` prints the JSON for that client on stdout and a one-line hint on stderr about where it goes. It finds the launch command itself: a `cloudg-mcp` executable on `PATH` first, then `cloudg` (giving `cloudg mcp serve`), then the current Python interpreter with `-m cloudg.mcp serve`. The command is written as an absolute path, because desktop apps often start with a minimal `PATH`. + +The output below is from an install where `cloudg-mcp` was found on `PATH`; your path will differ. + +```json tab="Claude Desktop" title="claude_desktop_config.json" +{ + "mcpServers": { + "cloudg": { + "command": "/home/you/.venv/bin/cloudg-mcp", + "args": [ + "serve" + ] + } + } +} +``` + +```json tab="Claude Code" title=".mcp.json" +{ + "mcpServers": { + "cloudg": { + "type": "stdio", + "command": "/home/you/.venv/bin/cloudg-mcp", + "args": [ + "serve" + ], + "env": {} + } + } +} +``` + +```json tab="Cursor" title="~/.cursor/mcp.json" +{ + "mcpServers": { + "cloudg": { + "command": "/home/you/.venv/bin/cloudg-mcp", + "args": [ + "serve" + ] + } + } +} +``` + +```json tab="VS Code" title=".vscode/mcp.json" +{ + "servers": { + "cloudg": { + "type": "stdio", + "command": "/home/you/.venv/bin/cloudg-mcp", + "args": [ + "serve" + ] + } + } +} +``` + +The commands that produced them are `cloudg mcp config --client claude-desktop` (the default), `--client claude-code`, `--client cursor` and `--client vscode`. `--name` changes the server key from `cloudg`, and `--command "CMD ARGS"` replaces the detected launch command (it is split with `shlex`). + +:::note Interpreter fallback +In a virtualenv where neither script is on `PATH`, the command is the interpreter and the arguments are `["-m", "cloudg.mcp", "serve"]`. That works just as well; it only looks different. +::: + +### Put it where the client reads it + +Each client reads a different file. cloudg's hint on stderr names it, and the table repeats it. Where a client keeps its global config on macOS, Windows or Linux is the client's business; use the client's own menu to open it when in doubt. + +| Client | File | Notes | +|---|---|---| +| Claude Desktop | `claude_desktop_config.json` | Open it with Settings, Developer, Edit Config. Merge the `cloudg` entry into the existing `mcpServers` object, then restart Claude Desktop | +| Claude Code | `.mcp.json` in the project root | Or register the inner entry with `claude mcp add-json cloudg ''` | +| Cursor | `~/.cursor/mcp.json` (all projects) or `.cursor/mcp.json` (one project) | Same shape as Claude Desktop | +| VS Code | `.vscode/mcp.json` | Or add the same object under `"mcp"` in `settings.json`. Note the top-level key is `servers`, not `mcpServers` | + +For Claude Code, the one-line form with the entry from the snippet above: + +```bash +claude mcp add-json cloudg '{"type": "stdio", "command": "/home/you/.venv/bin/cloudg-mcp", "args": ["serve"], "env": {}}' +``` + +### Restart the client and start a conversation + +The server sends instructions on `initialize` that tell the model to begin with `workspace_status`. If you did not preload a dataset, ask the assistant to load one by absolute path, for example "Load /srv/cloudg/reports/inventory-map.json and show me the internet-exposed assets with HIGH findings." That path has to be inside an allowed root, which the next section explains. +:::: + +## Load your reports with --dataset + +Pass the same options to `config` that you would pass to `serve`, and they are appended to the launch arguments. `--dataset` takes `NAME=PATH` or a bare `PATH`, and `config` resolves the path to an absolute one, since the client starts the server from a working directory you don't control. + +```console +$ cloudg mcp config --client claude-desktop --policy strict \ + --dataset prod=./reports/inventory-map.json --read-only +{ + "mcpServers": { + "cloudg": { + "command": "/home/you/.venv/bin/cloudg-mcp", + "args": [ + "serve", + "--policy", + "strict", + "--dataset", + "prod=/home/you/cloud-audit/reports/inventory-map.json", + "--read-only" + ] + } + } +} +``` + +The path can be an `inventory-map.json`, the directory holding it (a `findings.json` next to it is merged in), a cloudg `findings.json` report, or native output from Prowler (ASFF), ScoutSuite, Checkov or Trivy. Without a name, the dataset is named after the parent directory for `inventory-map.json` and after the file stem otherwise. Repeat `--dataset` to preload several. They load in order and each becomes active in turn, so the last one is active when the conversation starts; the model can switch with `select_dataset`. + +Preloaded paths skip the allowed-roots check, because you chose them. Datasets the model loads later with `load_dataset` don't. By default the allowed roots are the server's working directory and the configured report directory, minus `/` and your home directory, and a desktop client usually starts the server from one of those two. So set the roots explicitly in the entry's `env` block: + +```json title="claude_desktop_config.json" +{ + "mcpServers": { + "cloudg": { + "command": "/home/you/.venv/bin/cloudg-mcp", + "args": ["serve", "--policy", "standard"], + "env": {"CLOUDG_MCP_ALLOWED_ROOTS": "/srv/cloudg/reports:/srv/cloudg/scans"} + } + } +} +``` + +The separator is `os.pathsep`: `:` on macOS and Linux, `;` on Windows. Export tools write under the report directory, which is added to the roots if it is not already inside one. + +## Choose a policy + +`--policy` takes a built-in profile name, a path to a `.yaml`, `.yml` or `.json` policy file, or inline JSON starting with `{`. Without it the server uses `$CLOUDG_MCP_POLICY`, or `standard`. The profiles live in `cloudg/mcp/policies/`: + +| Profile | Use it when | Tools listed | +|---|---|---| +| `standard` | The default. Real identifiers, secrets redacted, cloud text fenced | 73 | +| `read_only` | The model must not call cloud APIs, start scanners or write files | 66 | +| `airgapped` | Same, but local report and Terraform exports are fine | 69 | +| `audit` | `read_only` plus a log of every call and inline provenance labels | 66 | +| `strict` | The model must not see real account ids, ARNs, names, IPs or e-mails | 61 | +| `soc-analyst` | A starting point for a shared server with `analyst`, `lead` and `collector` roles | 62 (local user) | +| `open` | Throwaway local experiments with data you made up. Secrets pass verbatim | 74 | + +The counts are what `cloudg mcp tools --policy NAME` lists for the local user in 0.6.0. Run that command with your policy and, for HTTP, with `--role ROLE`, to see exactly what a client will get before you hand it out. + +Under `strict`, pseudonyms come from a key that is random per process, so they change every time the client restarts the server and old ones stop resolving. Put `CLOUDG_MCP_VAULT_KEY` in the entry's `env` block if pseudonyms must stay stable across conversations, and keep that value away from the model. The [built-in profiles](/mcp/privacy/built-in-profiles/) page shows the same calls under every profile, and [loading and composing policies](/mcp/privacy/loading-and-composing-policies/) covers writing your own with `extends`. + +## Read-only mode + +There are two ways to stop a model from changing anything outside its own memory, and they are not identical. + +The `--read-only` flag removes tools from the registry before the policy sees them: every tool with the `write_fs`, `cloud_access` or `exec` capability, and every tool annotated destructive. That takes out the four live collectors, the three file exports (`export_ontology`, `export_report`, `export_terraform`) and `unload_dataset`. Under `standard`, which already hides `reveal_token`, 65 tools remain. The flag combines with any policy. + +The `read_only` profile hides tools through the policy instead (capabilities `cloud_access`, `exec` and `write_fs` denied) and keeps `unload_dataset`, for 66 tools. Either way the model can still load datasets from the allowed roots, snapshot them and suppress findings in memory, since those don't leave the process. + +## Connect over HTTP with a token + +Start the server first. Give each token an explicit principal id, because the audit log records the id: + +```bash +export CLOUDG_MCP_TOKEN="$(openssl rand -hex 24)" +cloudg mcp serve --transport http --dataset prod=./reports/inventory-map.json \ + --auth-token env:CLOUDG_MCP_TOKEN:analyst:alice +``` + +It listens on `http://127.0.0.1:8765/mcp` (change with `--host`, `--port` and `--path`) and, with the native flavor, answers `GET /healthz` without a token. `--flavor auto` picks the official SDK when `cloudg[mcp]` is installed, and that flavor has no `/healthz`; add `--flavor native` if your load balancer needs one. The spec format is `TOKEN[:ROLES[:ID]]`, read from the right; `env:VARNAME` takes the token from a variable, and the principal always gets the `default` role on top of the ones listed. An empty token, including one from an unset variable, stops the server before it starts. + +Then generate the client side with `--transport http` and `--token-env`, naming the variable that holds the token on the client's machine. `--policy`, `--dataset` and `--read-only` do nothing in this mode, since the server is already running with its own options. Use `--url` for anything other than the default address. + +```json tab="Claude Desktop" title="claude_desktop_config.json" +{ + "mcpServers": { + "cloudg": { + "command": "npx", + "args": [ + "-y", + "mcp-remote", + "http://127.0.0.1:8765/mcp", + "--header", + "Authorization:Bearer ${CLOUDG_MCP_TOKEN}" + ] + } + } +} +``` + +```json tab="Claude Code" title=".mcp.json" +{ + "mcpServers": { + "cloudg": { + "type": "http", + "url": "http://127.0.0.1:8765/mcp", + "headers": { + "Authorization": "Bearer ${CLOUDG_MCP_TOKEN}" + } + } + } +} +``` + +```json tab="Cursor" title="~/.cursor/mcp.json" +{ + "mcpServers": { + "cloudg": { + "url": "http://127.0.0.1:8765/mcp", + "headers": { + "Authorization": "Bearer ${env:CLOUDG_MCP_TOKEN}" + } + } + } +} +``` + +```json tab="VS Code" title=".vscode/mcp.json" +{ + "servers": { + "cloudg": { + "type": "http", + "url": "http://127.0.0.1:8765/mcp", + "headers": { + "Authorization": "Bearer ${env:CLOUDG_MCP_TOKEN}" + } + } + } +} +``` + +Each client expands the variable in its own syntax: `${VAR}` in Claude Code, `${env:VAR}` in Cursor and VS Code. The token itself never appears in the file. Claude Desktop's config only launches local commands, so cloudg bridges to the URL with `npx mcp-remote` and passes the header on its command line. That entry has no `env` block, and whether `${CLOUDG_MCP_TOKEN}` gets a value depends on the environment Claude Desktop hands to `npx`, which the cloudg docs have not tested. If the server answers 401, add `"env": {"CLOUDG_MCP_TOKEN": "..."}` to the entry by hand. + +To reach the server from another host, bind to a public address and keep the token: `--host 0.0.0.0` together with `--allowed-host 'mcp.internal.example:*'` (and `--allowed-origin` for browser clients). The [CLI reference](/mcp/server/cli-reference/) lists every serve option. + +## Troubleshooting + +### The client shows no tools, or disconnects at once + +Copy the `command` and `args` from the client config and run them in a terminal. If the process exits, the reason is on stderr. The usual ones: + +| Message or symptom | Cause | Fix | +|---|---|---| +| `Error: Dataset not found: /path/...` | A `--dataset` path that doesn't exist | Fix the path; regenerate with `cloudg mcp config` so it is absolute | +| `Error: Unknown policy 'NAME': not a profile (...) or an existing file` | Typo in `--policy`, or a relative file path that doesn't resolve from the client's working directory | Use a profile name or an absolute file path | +| `command not found`, or the client log says it can't spawn | The command isn't on the client's `PATH` | Use the absolute path `cloudg mcp config` prints | +| Starts in a terminal, empty tool list in the client | The policy hides everything for this principal | Check `cloudg mcp tools --policy P --role R` with the same options | + +A client also drops the connection when anything other than protocol messages reaches stdout. The server redirects stray `print()` calls to stderr while it runs, but a wrapper script that echoes something before `exec`-ing the server will break the stream. `cloudg -v mcp serve` is safe; verbose logs go to stderr. + +### load_dataset says the path is outside the allowed roots + +The model asked for a file outside `CLOUDG_MCP_ALLOWED_ROOTS`. The error lists the roots it checked. Add the directory to that variable in the entry's `env`, or preload the file with `--dataset`. + +### HTTP answers 401, 403 or 421 + +401 means the `Authorization` header was missing or the token didn't match; check that the variable is set where the client runs. 403 `Forbidden: invalid Origin` means a browser-style `Origin` header that isn't loopback or allowed; add it with `--allowed-origin`. 421 `Invalid Host header` means a loopback-bound server received a `Host` other than `localhost`, `127.0.0.1` or `[::1]`, which is what DNS rebinding looks like and also what a reverse proxy forwarding its public name looks like; add the name with `--allowed-host 'name:*'`. + +### A token with a colon in it is rejected + +Specs are read from the right, so `abc:def` means token `abc` with role `def`. Write such a token in the full form (`abc:def::` or `abc:def:analyst:`) or put it in a variable and use `env:VAR:ROLES:ID`. + +### Pseudonyms from yesterday no longer resolve + +The vault key is random per process unless `CLOUDG_MCP_VAULT_KEY` is set. Set it in the client's `env` block. + +### The client complains about a missing outputSchema + +Under every profile except `open`, tools don't advertise `outputSchema`, because masking can change a field's type and a client must reject content that doesn't match an advertised schema. The structured content is still sent. This is deliberate; see [troubleshooting and FAQ](/mcp/server/troubleshooting-and-faq/) for this and the other cases (timeouts, large results, "cooling down" from live tools). + +:::links +- [Quick start](/mcp/server/quick-start-with-a-desktop-client/) The same setup with captured output from the sample estate. +- [cloudg mcp config](/cli/mcp-config/) Every option of the config command. +- [cloudg mcp serve](/cli/mcp-serve/) Transports, flavors, tokens, Host and Origin checks. +- [Built-in profiles](/mcp/privacy/built-in-profiles/) What each policy changes, call by call. +- [Principals and roles](/mcp/privacy/principals-and-roles/) Map tokens to roles and give each role its own view. +- [Troubleshooting and FAQ](/mcp/server/troubleshooting-and-faq/) The full list of known problems. +::: diff --git a/site/content/mcp/index.md b/site/content/mcp/index.md new file mode 100644 index 0000000..495dbd0 --- /dev/null +++ b/site/content/mcp/index.md @@ -0,0 +1,264 @@ +--- +title: MCP server +lede: "cloudg's inventory, relationship graph, findings, compliance results and RDF ontology, served to AI assistants over the Model Context Protocol, with one access and privacy policy applied to every request." +meta: + - [Command, "`cloudg mcp serve`"] + - [Python, "`cloudg.mcp.CloudGMCPLayer`"] + - [Catalog, "74 tools, 13 resources, 11 templates, 11 prompts"] + - [Default policy, "`standard`"] +source: docs/MCP.md +since: "0.6.0" +--- + +The MCP layer lives in `cloudg/mcp/` and ships with the base `pip install cloudg`. It takes the files cloudg already writes (`inventory-map.json`, `findings.json`, native scanner output) and turns them into tools an assistant can call: "which internet-exposed assets have HIGH findings?", "what breaks if this IAM role goes away?", "which CIS controls fail in the shared-services account?". The model asks, cloudg answers from its own graph and data model, and a policy decides what the model may see and do on the way. + +It is written for three kinds of reader. People who run Claude Desktop, Claude Code, Cursor or VS Code and want to point it at a cloud estate should read [Connect a client](/mcp/connect/) next. Operators who already run an MCP server or gateway can mount cloudg into it. And Python authors (an agent framework, a test suite, a batch job) can call the same tools in-process, with no MCP traffic at all. + +## One layer, three ways in + +The center of the package is one class, `CloudGMCPLayer`. It holds a registry of tools, resources, templates and prompts, a workspace of loaded datasets, a policy, and a middleware chain. Every request ends up in one of four methods on it: `call_tool`, `read_resource`, `get_prompt` or `complete`. Because of that, a policy you write once applies identically to a desktop client on stdio, a web client on HTTP, a server you mounted cloudg into, and a Python script. + +| Mode | Entry point | Use it when | +|---|---|---| +| Standalone server | `cloudg mcp serve` | You want cloudg as its own MCP server for a desktop client or a gateway | +| Mounted | `layer.register_into(server)` | You already run an MCP server (official `mcp` SDK 1.x or 2.x, or `fastmcp` 2, 3 or 4) and want cloudg's tools inside it | +| In-process | `await layer.call_tool(...)` | You are in Python and do not need MCP on the wire | + +```mermaid caption="How the pieces fit together" +flowchart LR + A["Desktop or IDE client"] -->|"stdio or HTTP"| B["cloudg mcp serve (native, sdk or fastmcp)"] + C["Your MCP server"] -->|register_into| L[CloudGMCPLayer] + D["Python script"] -->|"call_tool()"| L + B --> L + L --> R["Registry: tools, resources, prompts"] + L --> P["Policy and token vault"] + L --> M[Middleware] + L --> W["Workspace of datasets"] + W --> F["inventory-map.json, findings.json, scanner output"] + W --> X["Cloud APIs (live tools only)"] +``` + +The [server section](/mcp/server/what-the-layer-is/) covers the layer in depth, [mounting](/mcp/server/mounting-into-an-existing-mcp-server/) covers the second mode and [in-process use](/mcp/server/in-process-use/) the third. + +## What an assistant can do with it + +The default catalog has 74 tools in ten categories. The counts below come from running `cloudg mcp tools --policy open --json` against the 0.6.0 code and grouping by `_meta["cloudg/category"]`. + +| Category | Tools | What the tools do | +|---|---|---| +| `workspace` | 8 | Load, select, snapshot, diff and unload datasets; `workspace_status` is where a model should start | +| `inventory` | 11 | Search assets (`find_assets`), fetch one with its relations (`get_asset`), count and group, list accounts, regions and tags, report collector coverage and the AWS Organization | +| `graph` | 17 | Neighbours, shortest paths, attack paths from the internet, identity-based lateral movement, internet exposure, dependencies, blast radius, centrality, sub-graph export | +| `findings` | 10 | List, summarise and prioritise findings (`top_risks`), suppress and restore them in memory, ingest scanner reports, generate reachability findings | +| `compliance` | 5 | Frameworks, posture per framework, controls, one control's status, assets with the most failures | +| `ontology` | 6 | Ontology statistics, read-only SPARQL, semantic neighbourhoods, relation groups, RAG chunks, ontology export | +| `export` | 3 | Terraform preview and export, report files | +| `live` | 5 | `map_inventory`, `collect_assets`, `run_scanners`, `run_pipeline` against real cloud APIs, plus `rate_limit_status` | +| `meta` | 4 | What this caller may use, and cloudg's vocabulary of asset, edge and relation types | +| `privacy` | 5 | The active policy, a transform preview, the detector list, the decision log, and `reveal_token` | + +The default `standard` policy hides `reveal_token`, so `cloudg mcp tools` lists 73. The stricter profiles hide more (see [policies](#policies-and-privacy) below). Every tool is documented with its arguments, return fields and a captured example in the [tool index](/mcp/tools/tool-index/) and the per-category pages such as [graph](/mcp/tools/graph/) and [findings](/mcp/tools/findings/). + +The four collecting tools in `live` are the only ones that touch a cloud provider. They check credentials before the first API call, refuse to re-collect a scope that was collected a couple of minutes ago, and are hidden by every built-in profile except `open` and `standard`. The details are in [live tools and the credential preflight](/mcp/server/live-tools-and-the-credential-preflight/). + +### Resources and prompts + +Resources are read-only documents a client can attach to a conversation without a tool call. There are 13 fixed resources, such as `cloudg://workspace`, `cloudg://findings/summary`, `cloudg://compliance`, `cloudg://graph/d3`, `cloudg://ontology/turtle` and `cloudg://policy`, and 11 templates that take a parameter: `cloudg://assets/{+ref}`, `cloudg://assets/{+ref}/neighbors`, `cloudg://findings/{finding_id}`, `cloudg://compliance/{framework}` and so on. `cloudg mcp serve` adds a 14th resource, `cloudg://metrics`, with call counts and latencies. Template arguments complete as you type: in the sample estate, asking for `ref` values matching `web` returns asset ids such as `web-1` and `sg-web`. + +The 11 prompts are packaged analysis workflows that a user picks from the client's prompt menu. They tell the model which tools to call and in what order: + +| Prompt | Required arguments | +|---|---| +| `security_posture_review`, `executive_summary`, `attack_surface_report`, `cross_account_trust_review` | none | +| `investigate_asset`, `blast_radius_assessment`, `change_impact_analysis` | `ref` | +| `compliance_gap_analysis` | `framework` | +| `remediation_plan` | none (`severity` optional) | +| `incident_triage` | `finding_id` | +| `drift_review` | `base` | + +The full list with arguments and rendered output is in [resources and resource templates](/mcp/tools/resources-and-resource-templates/) and [prompts](/mcp/tools/prompts/). + +## Workspaces and datasets + +The server keeps one in-memory workspace. It holds named datasets, one of which is active, and every tool takes an optional `dataset` argument that defaults to the active one. A dataset is one loaded view of infrastructure: assets, edges, findings, compliance results, coverage and organization data, plus where it came from. + +Datasets get into the workspace in three ways. The operator can preload them with `--dataset NAME=PATH` when starting the server. The model can call `load_dataset` with a path. Or a live tool collects a fresh one from the cloud. The loader detects what a path holds: a directory with `inventory-map.json` (a `findings.json` next to it is merged in), a cloudg `findings.json`, or Prowler ASFF, ScoutSuite, Checkov or Trivy output, which is normalised and mapped to compliance frameworks on load. + +The graph, dependency graph, centrality scores and RDF ontology are built the first time a tool needs them and cached until the dataset changes. On a large estate the first `attack_paths` or `sparql_query` call is the slow one. The workspace holds 16 datasets; loading a 17th evicts the oldest inactive one. + +Two safety rules matter in practice. First, the model can only read and write inside the allowed roots: by default the directory the server was started from and the configured report directory, with `/` and your home directory left out. Set `CLOUDG_MCP_ALLOWED_ROOTS` to change them. Second, paths given with `--dataset` skip that check, because the operator typed them. `snapshot_dataset` and `diff_datasets` let a model compare two versions of an estate, which is what the `drift_review` prompt does. More in [the workspace](/mcp/server/the-workspace/). + +## The request lifecycle + +Every tool call takes the same route, whatever transport it arrived on. + +```mermaid caption="What happens to one tools/call" +flowchart TD + A[MCP client] --> B["Transport: stdio line or HTTP POST"] + B --> C["Adapter resolves the principal (token, OAuth or local user)"] + C --> D{"Tool visible to this principal?"} + D -->|no| E["Unknown tool (JSON-RPC -32602)"] + D -->|yes| F["Middleware: audit log, metrics, concurrency, cache"] + F --> G["Policy check: access, then rate limit"] + G -->|"denied or limited"| K["isError result with cloudg/error_code"] + G --> H["Input pipeline: refuse secrets, reverse pseudonyms"] + H --> I["Validate arguments, run handler under its timeout"] + I --> J["Output pipeline: sanitize, redact, project, annotate"] + J --> L["Result with cloudg/transforms in _meta"] +``` + +A few details are easy to miss. A tool the policy hides gives exactly the same "Unknown tool" error as a tool that does not exist, so a model cannot probe for what it is missing. Arguments are validated against a pydantic model built from the handler's signature, and unknown keys are rejected. Synchronous handlers run in a worker thread so a slow graph build does not block other requests, and every call has a timeout (300 seconds by default, longer for the live tools). The result is sent twice, as `structuredContent` and as indented JSON text for clients that ignore structured output; the text copy is capped at 200,000 characters. Errors from the handler pass through the same output pipeline as results, because exception messages tend to quote ARNs. + +The step-by-step version, including resources, prompts and completions, is in [the request lifecycle](/mcp/server/the-request-lifecycle/). Error codes are listed in [errors](/mcp/server/errors/) and the middleware hooks in [middleware](/mcp/server/middleware/). + +## Policies and privacy + +A policy is a YAML document that says who may call what, how often, and what happens to the data in each direction. Seven profiles ship in `cloudg/mcp/policies/`, and you choose one with `--policy` or `CLOUDG_MCP_POLICY`: + +| Profile | What it does | Tools listed for the local user | +|---|---|---| +| `open` | Nothing. Secrets reach the model verbatim. Local experiments only | 74 | +| `standard` (default) | Redacts secrets, drops private keys, masks credential ids, fences attacker-controllable text, refuses secrets in arguments | 73 | +| `airgapped` | `standard`, plus no cloud calls and no scanner processes; local exports allowed | 69 | +| `read_only` | `standard`, plus no cloud calls, processes or file writes | 66 | +| `audit` | `read_only` with a decision log of every call and inline provenance labels | 66 | +| `strict` | Pseudonymises accounts, ARNs, names, IPs, e-mails and tag values; hides restricted tools | 61 | +| `soc-analyst` | Example multi-role policy: pseudonymised analysts, revealing leads, a collector role | 62 | + +The tool counts were measured with `cloudg mcp tools --policy NAME` on 0.6.0. `soc-analyst` gives different numbers per role (60 for `analyst`, 70 for `lead`). + +The output pipeline in `standard` runs four transforms in order. `sanitize` strips invisible and bidi characters and wraps text that reads like instructions in `⟦untrusted⟧` fences, because anyone who can tag a resource can plant "ignore previous instructions" in a tag. `redact` replaces secrets found in metadata (EC2 `user_data`, connection strings, SAS signatures) with `[REDACTED:...]`. `project` drops `raw_data` and caps string and result size. `annotate` adds a provenance report. On the way in, `guard_secrets` refuses any argument that contains a secret. + +Under `strict` the redactor also pseudonymises identifiers through the token vault. Pseudonyms keep their format, so a 12-digit account id stays 12 digits and an ARN stays a valid ARN, and they are deterministic for the life of the vault key. The model can pass `res-8de954a6d9` back as a `ref` and the input pipeline resolves the real asset before the handler runs. Set `CLOUDG_MCP_VAULT_KEY` if pseudonyms must survive a server restart; without it a random key is generated per process. + +One side effect surprises people: under every profile except `open`, tools do not advertise an `outputSchema`, since masking can change a field's type and clients reject structured content that does not match a schema. The data is still sent. The [privacy section](/mcp/privacy/threat-model-and-goals/) covers the threat model, [what the policy touches](/mcp/privacy/what-the-policy-touches/), the [built-in profiles](/mcp/privacy/built-in-profiles/) file by file, and [the token vault](/mcp/privacy/the-token-vault/). + +## Transports and flavors + +`cloudg mcp serve --transport stdio` (the default) is for clients that launch the server as a child process and talk newline-delimited JSON-RPC over its stdin and stdout. stdout carries protocol messages only; the server redirects stray `print()` calls and writes to file descriptor 1 to stderr while it runs. + +`--transport http` serves streamable HTTP on `http://127.0.0.1:8765/mcp`, with sessions, server-sent events for progress, and `/healthz`. `--transport sse` adds the deprecated HTTP+SSE endpoints for old clients. Both protocol eras are served: the handshake versions from 2024-11-05 to 2025-11-25, and the stateless 2026-07-28 version. + +The flavor picks which implementation speaks MCP. The layer and results are identical across them. + +| Flavor | Implementation | Needs | +|---|---|---| +| `auto` (default) | `sdk` if the official SDK imports, else `native` | nothing | +| `native` | cloudg's own server, standard library only | nothing | +| `sdk` | the official SDK's low-level `Server`, uvicorn for HTTP | `pip install "cloudg[mcp]"` | +| `fastmcp` | `fastmcp.FastMCP` with cloudg components | `pip install fastmcp` | + +The native flavor supports every serve option, including `--cors-origin`, `--page-size` and `/healthz`. See [transports and protocol versions](/mcp/server/transports-and-protocol-versions/) and [flavors](/mcp/server/flavors/). + +## Security model in brief + +Every transport binds to 127.0.0.1 by default. On HTTP, `--auth-token TOKEN[:ROLES[:ID]]` maps static bearer tokens to principals with roles, which the policy's `roles:` section uses; tokens are kept only as SHA-256 digests and compared in constant time. Every flavor checks `Origin` and `Host` headers against allow-lists to block DNS rebinding, and the native server warns at start-up when bound to a non-loopback address without tokens. On stdio, tokens are ignored and the caller is the local user. + +File access is confined to the allowed roots and writes to the output directory. Error messages never repeat the caller's arguments. The JSONL audit log (`--audit-log`) records every call with hashed argument values, and every pseudonym reversal is logged at WARNING. `--registry` imports and runs Python code, so its value must never come from untrusted input. The longer list is in [security model in brief](/mcp/server/security-model-in-brief/). + +## Examples + +Serve over stdio with the `strict` profile and one preloaded dataset. This is what a desktop client runs: + +```bash +cloudg mcp serve --policy strict --dataset prod=./reports/inventory-map.json +``` + +Serve over streamable HTTP with a bearer token read from the environment. The token principal gets the role `analyst` and the id `alice`, which is what the audit log records: + +```bash +export CLOUDG_MCP_TOKEN="$(openssl rand -hex 24)" +cloudg mcp serve --transport http --dataset prod=./reports/inventory-map.json \ + --auth-token env:CLOUDG_MCP_TOKEN:analyst:alice --audit-log mcp-audit.jsonl +``` + +The server prints `cloudg MCP server (native, http) listening on http://127.0.0.1:8765/mcp` to stderr. A POST without the header gets `401` with `WWW-Authenticate: Bearer realm="cloudg-mcp"`; with `Authorization: Bearer $CLOUDG_MCP_TOKEN` the `initialize` answer carries an `Mcp-Session-Id`. + +`cloudg mcp call` runs one tool in-process through the full layer (policy, transforms, middleware) and prints the structured result. It exits 1 when the tool returns an error, so it works in shell scripts: + +```console +$ cloudg mcp call find_assets --dataset prod=./reports/inventory-map.json \ + --args '{"internet_exposed": true, "limit": 2, "fields": ["name", "type", "account_id"]}' +{ + "dataset": "prod", + "total": 6, + "offset": 0, + "returned": 2, + "next_cursor": "c2.ae39283d581a", + "truncated": true, + "items": [ + {"id": "bastion", "name": "bastion", "type": "EC2", "account_id": "111111111111"}, + {"id": "az-vm", "name": "jump-vm", "type": "VIRTUAL_MACHINE", "account_id": "00000000-aaaa-bbbb-cccc-000000000001"} + ] +} +``` + +That output came from the synthetic estate in `tests/mcp/fixtures/sample_estate.py` (items reformatted onto one line each). Add `--raw` to see the whole `CallToolResult`, including `_meta`. + +The same in Python, under `strict`, showing a pseudonym used as the argument of the next call: + +```python title="ask_estate.py" +import asyncio +from pathlib import Path + +from cloudg.config import CloudGConfig +from cloudg.mcp import CloudGMCPLayer, NotFoundError, Principal, Workspace + + +async def main() -> None: + reports = Path("reports").resolve() + ws = Workspace(CloudGConfig(), allowed_roots=[reports], output_dir=reports / "mcp-out") + layer = CloudGMCPLayer(policy="strict", workspace=ws) + + loaded = await layer.call_tool("load_dataset", {"path": str(reports), "name": "prod"}) + print("loaded", loaded.structured["total_assets"], "assets") + + exposed = await layer.call_tool( + "find_assets", + {"internet_exposed": True, "limit": 3, "fields": ["name", "type", "account_id"]}, + ) + for item in exposed.structured["items"]: + print(item["type"], item["name"], item["account_id"]) + print("transforms:", sorted(exposed.meta["cloudg/transforms"]["pseudonymized"])) + + # A pseudonym goes straight back into the next call. + first = exposed.structured["items"][0]["id"] + detail = await layer.call_tool("get_asset", {"ref": first}) + print(detail.structured["name"], detail.structured["max_severity"]) + + # strict hides the live tools, so to this caller they do not exist. + try: + await layer.call_tool("map_inventory", {}, principal=Principal(id="alice", roles={"analyst"})) + except NotFoundError as exc: + print("refused:", exc) + + +asyncio.run(main()) +``` + +```text +loaded 44 assets +EC2 res-e991bc4472 504423825804 +VIRTUAL_MACHINE res-88ebc3a1d8 5dd921c1-d60a-8312-93ba-971ec3c02712 +S3_BUCKET res-766425e86e 504423825804 +transforms: ['aws_account_id', 'azure_subscription_id', 'dataset_name', 'resource_name'] +res-e991bc4472 HIGH +refused: Unknown tool: map_inventory +``` + +The pseudonyms change between runs unless `CLOUDG_MCP_VAULT_KEY` is set. Note the last line: in Python a hidden tool raises `NotFoundError`, while denials, rate limits and handler errors come back as a `ToolResult` with `is_error` set. + +## Where to go next + +:::links +- [Connect a client](/mcp/connect/) Claude Desktop, Claude Code, Cursor and VS Code, step by step. +- [Quick start](/mcp/server/quick-start-with-a-desktop-client/) Check your data in a terminal before wiring a client. +- [CLI reference](/mcp/server/cli-reference/) Every option of serve, tools, call, read and config. +- [Tool index](/mcp/tools/tool-index/) All 74 tools with category, sensitivity and a one-line summary. +- [Conventions](/mcp/tools/conventions-shared-by-every-tool/) Paging, field selection, asset references and the result envelope. +- [Recipes](/mcp/tools/recipes/) Multi-step investigations as tool-call sequences. +- [Built-in profiles](/mcp/privacy/built-in-profiles/) The seven policies side by side, with the same calls under each. +- [Policy schema](/mcp/privacy/policy-schema-reference/) Write your own policy, roles and rules. +- [In-process use](/mcp/server/in-process-use/) CloudGMCPLayer, Principal and ToolContext from Python. +- [Internals](/mcp/internals/package-map/) How the package is built, for contributors. +::: diff --git a/site/content/reference/output-files.md b/site/content/reference/output-files.md new file mode 100644 index 0000000..f73a9d1 --- /dev/null +++ b/site/content/reference/output-files.md @@ -0,0 +1,280 @@ +--- +title: Output files +lede: Every file cloudg writes, which command writes it, and what is inside. +source: docs/DOCUMENTATION.md +since: "0.6.0" +--- + +Every command writes into one output directory, `-o` / `--output`, which defaults to `./reports` and is created when missing. Files with the same name are overwritten without a prompt, so give each run its own directory when you want to keep history. The exceptions to "everything goes into `-o`" are the Terraform export, the Cloud Control type cache and the MCP server's audit log and vault, listed in their sections below. + +## cloudg run + +```mermaid caption="Where the files of cloudg run come from, phase by phase" +flowchart LR + A["Collect"] --> B["Graph"] + B --> C["RAG export"] + C --> D["Terraform"] + D --> E["Scanners"] + E --> F["Ontology"] + F --> G["Normalise"] + G --> H["Render"] + B -.-> B1["topology.graphml, topology-cytoscape.json"] + C -.-> C1["rag_chunks.jsonl, rag_metadata_index.json"] + E -.-> E1["prowler/, scoutsuite/"] + F -.-> F1["ontology.ttl, ontology.jsonld"] + H -.-> H1["findings.json, topology.svg, report.html"] +``` + +| File | Written by | Contents | +|---|---|---| +| `topology.graphml` | graph phase, always | The asset graph from [GraphBuilder](/api/graphbuilder/), for Gephi, yEd or `networkx.read_graphml`. Saved before the reachability walk, so `is_internet_exposed` is the collector's flag | +| `topology-cytoscape.json` | graph phase, always | The same graph as Cytoscape.js `elements` | +| `rag_chunks.jsonl` | RAG export, unless `--no-rag-export` or `rag.enabled: false` | One retrieval chunk per line: entity, community and relation-group chunks ([RAGExporter](/api/ragexporter/)). Written once after the graph phase and rewritten after the scanners with every finding | +| `rag_metadata_index.json` | RAG export | Chunk counts per kind, every `chunk_id` in file order, the chunk types | +| `provider.tf.json`, `variables.tf.json`, `main.tf.json`, `import_commands.sh` | Terraform phase, with `--terraform` or `terraform.enabled: true` | Terraform JSON that recreates the collected resources, and a script of `terraform import` commands. Written to `terraform.output_dir` (default `./reports/terraform`, relative to the working directory, whatever `-o` says) | +| `prowler//` | Prowler, when it runs | Prowler's own JSON-ASFF output files, which cloudg parses | +| `scoutsuite//` | ScoutSuite, when it runs | ScoutSuite's report directory | +| `ontology.ttl`, `ontology.jsonld` | ontology phase, unless `--no-ontology` or `ontology.enabled: false` | The RDF graph with 64 relation types and the findings ([CloudOntology](/api/cloudontology/)). One file per `ontology.export_formats` entry; `xml` adds `ontology.rdf`, `nt` adds `ontology.nt` | +| `findings.json` | render phase | The normalised result: metadata, summary, assets, findings, compliance mappings, D3 graph. Shape below | +| `topology.svg` | render phase | Static topology diagram | +| `report.html` | render phase | Interactive report: topology, findings table, compliance matrix. The data is embedded, but the page loads Chart.js and D3 from `cdn.jsdelivr.net`, so the charts and the topology need network access when it is opened. `report.inline_js` is not read in 0.6.0 | + +Checkov and Trivy write nothing to disk; cloudg reads their stdout. `cloudg run` writes no `raw-findings.json` and no inventory map. All three render files are written whatever `report.formats` says, and `topology.graphml` and `topology-cytoscape.json` are written whatever `graph.persist_graphml` and `graph.export_cytoscape` say: those keys are not read in 0.6.0. + +## cloudg collect + +| File | Written by | Contents | +|---|---|---| +| `inventory-.json` | `cloudg collect -p ` | `{"assets": [...], "edges": [...]}` straight from the provider's collector, each serialised with `model_dump(mode="json")`. No linking, no summary; use `cloudg map` for the interconnected inventory | + +## cloudg scan + +| File | Written by | Contents | +|---|---|---| +| `raw-findings.json` | every scan | A JSON list of `Finding` objects from all scanners, before normalisation: no deduplication, no compliance mapping | +| `prowler/` | Prowler | Prowler's JSON-ASFF output (not split per provider, unlike `cloudg run`) | +| `scoutsuite/` | ScoutSuite | ScoutSuite's report directory | + +`cloudg map --findings` accepts this file, or a `findings.json`. + +## cloudg map + +| File | Written by | Contents | +|---|---|---| +| `inventory-map.json` | every map | The self-contained map: `summary`, `providers`, `regions`, `assets`, `edges`, `unresolved_references`, and `throttling` when a cloud API pushed back. Read by `cloudg deps`, `InventoryResult.load()` and the MCP workspace. Field reference: [inventory-map.json](/reference/inventory/inventory-map-json/) | +| `inventory-map.graphml` | every map | The map's graph, with parallel rule edges merged ([field list](/reference/inventory/inventory-map-graphml/)) | +| `inventory-graph.json` | every map | D3 `nodes` and `links` for `docs/viewer.html` and other front ends ([field list](/reference/inventory/inventory-graph-json/)) | +| `inventory-dependencies.json` | every map | Shared dependencies and largest blast radius (top 25 each), every cross-account edge, security service coverage, unresolved references | +| `inventory-organization.json` | `--org`, or `aws.organization.enabled` | The AWS Organization and Control Tower topology: accounts, OU tree, SCPs, landing zone | +| `asset-map.json` | `--findings`, when at least one finding loads | Every asset with its matched findings, sorted by finding count | +| `compliance-map.json` | `--findings`, when at least one finding loads | Per framework: finding count, severity breakdown, affected assets | + +The map's coverage records (which service was collected, partly collected or failed) are shown in the terminal but not saved in `inventory-map.json`. + +Outside the output directory, the AWS Cloud Control sweep caches the list of resource types it can enumerate in `~/.cache/cloudg/cloudcontrol-types.json` (or `$CLOUDG_CACHE_DIR/cloudcontrol-types.json`) for seven days. + +## cloudg deps --json + +`cloudg deps` reads a saved map and writes no file. With `--json` it prints JSON to stdout; redirect it to keep it. + +| Invocation | Prints | +|---|---| +| `cloudg deps --json` | The asset's dependency tree: what it depends on (`up`), what depends on it (`down`), or both, to `--depth` levels | +| `cloudg deps --json` | `{"shared_dependencies", "largest_blast_radius", "cross_account_edges"}`; `--top` limits the first two, the third lists every cross-account edge | + +```bash +cloudg deps arn:aws:iam::123456789012:role/orders-worker --json > role-deps.json +cloudg deps --map ./reports --top 50 --json > overview.json +``` + +The key-by-key shapes are in [cloudg deps --json](/reference/inventory/cloudg-deps-json/). + +## cloudg ingest + +| File | Written by | Contents | +|---|---|---| +| `raw-findings.json` | every ingest | The parsed findings before normalisation, as a JSON list | +| `findings.json` | `--format json` or `all` (default) | Normalised findings and compliance mappings. `assets` is empty and `graph` has no nodes, because nothing was collected | +| `report.html` | `--format html` or `all` | The HTML report for those findings | + +## cloudg report + +| File | Written by | Contents | +|---|---|---| +| `findings.json` | `--format json` or `all` (default) | Rewritten from the input's `assets`, `findings` and `graph` | +| `topology.svg` | `--format svg` or `all` | Diagram of the input's assets | +| `report.html` | `--format html` or `all` | The HTML report | + +:::warning cloudg report does not round-trip everything +`cloudg report -i` rebuilds the result from `assets`, `findings` and `graph` only. The `findings.json` it writes has an empty `compliance` list, a new `scan_id`, `total_edges: 0` and `completed_at: "None"`, and `topology.svg` is drawn without edges. With the default `-o ./reports`, running it on `./reports/findings.json` overwrites the original. Point `-o` somewhere else. +::: + +## cloudg mcp + +`cloudg mcp serve` writes nothing unless asked to. These are the files it can produce: + +| File | Written by | Contents | +|---|---|---| +| audit log (`--audit-log PATH` or `$CLOUDG_MCP_AUDIT_LOG`) | `AuditLogMiddleware` | One JSON line per call: time, principal, tool or resource name, argument keys, HMAC hashes of the argument values, duration, outcome, sensitivity, transforms applied. Created with mode `0600` and appended to | +| pseudonym vault (`vault.path` in the policy) | the policy's `TokenVault` | The pseudonym mappings, encrypted by default, written atomically with mode `0600` when serving ends or the process exits | +| `findings.json`, `report.html`, inventory files, `.graphml`, `asset-map.json` and `compliance-map.json` | the `export_report` tool | Depends on its `format` argument; written under the workspace output directory (`report.output_dir`, default `./reports`) or its `subdir` | +| `terraform/*.tf.json`, `terraform/import_commands.sh` | the `export_terraform` tool | As for `cloudg run`, in the workspace output directory | +| `ontology.ttl` (or `.jsonld`, `.rdf`, `.nt`) | the `export_ontology` tool | The dataset's ontology; the file name comes from its `filename` argument | +| `scans/` | the `run_scanners` live tool | Scanner output, as for `cloudg scan` | +| `pipeline/` | the `run_pipeline` live tool | What `CloudGEngine.run_pipeline()` writes: `findings.json`, `report.html`, ontology and RAG files | + +The tools refuse paths that escape the output directory. `cloudg mcp config` prints client configuration to stdout and writes no file. + +The global option `cloudg --log-file PATH ` (or `log_file` in `config.yaml`) also writes the log to that file, for any command. + +## findings.json + +`findings.json` is the main machine-readable result of `cloudg run` and `cloudg ingest`, and the input of `cloudg report`. An excerpt from a run with two assets, an SSH rule open to the internet and nothing else, trimmed to one entry per list (the summary still counts both findings): + +```json title="reports/findings.json" +{ + "metadata": { + "scan_id": "cfc19759-897f-407f-b866-a0b977325467", + "provider": null, + "account_id": null, + "region": null, + "started_at": "2026-10-09 16:47:09.228244", + "completed_at": "2026-10-09 16:47:09.228230" + }, + "summary": {"total_assets": 2, "total_findings": 2, "total_edges": 2, "severity_breakdown": {"CRITICAL": 1, "HIGH": 1}, "compliance_frameworks": ["PCI-DSS", "SOC2", "NIST-800-53", "CIS", "ISO-27001"]}, + "assets": [ + { + "id": "sg-web", + "arn": "arn:aws:ec2:eu-west-1:123456789012:security-group/sg-web", + "name": "sg-web", + "asset_type": "SECURITY_GROUP", + "provider": "AWS", + "region": "eu-west-1", + "account_id": "123456789012", + "tags": {}, + "metadata": {}, + "collected_at": "2026-10-09T16:47:08.038905", + "is_internet_exposed": false, + "display_id": "arn:aws:ec2:eu-west-1:123456789012:security-group/sg-web" + } + ], + "findings": [ + { + "id": "17cd5168-70ea-5cd1-aa9a-10c4dea0a98e", + "resource_id": "sg-web", + "resource_arn": "arn:aws:ec2:eu-west-1:123456789012:security-group/sg-web", + "severity": "CRITICAL", + "title": "Security group allows SSH (port 22) from 0.0.0.0/0", + "description": "A security group rule allows inbound traffic on port 22 (SSH) from 0.0.0.0/0. This is a common attack vector.", + "evidence": "Edge from 0.0.0.0/0 to sg-web, port 22, cidr 0.0.0.0/0", + "remediation": "Restrict port 22 (SSH) access to specific IP ranges. Use a bastion host or VPN for administrative access.", + "source_tool": "cloudg-reachability", + "source_finding_id": null, + "compliance_frameworks": [ + "CIS", + "NIST-800-53", + "PCI-DSS" + ], + "cvss_score": null, + "detected_at": "2026-10-09T16:47:08.039404", + "is_suppressed": false, + "risk_score": 9.5 + } + ], + "compliance": [ + { + "id": "ec4accbd-dd7c-4f79-9561-f903ea125340", + "framework": "CIS", + "control_id": "CIS-aggregate", + "control_title": "CIS CIS-aggregate", + "status": "FAIL", + "finding_ids": [ + "17cd5168-70ea-5cd1-aa9a-10c4dea0a98e" + ], + "resource_arn": null + } + ], + "graph": { + "nodes": [ + {"id": "web-1", "name": "web-1", "type": "EC2", "provider": "AWS", "region": "eu-west-1", "arn": "arn:aws:ec2:eu-west-1:123456789012:instance/i-0web1", "account_id": "123456789012", "is_internet_exposed": false, "is_external": false} + ], + "links": [ + {"source": "web-1", "target": "sg-web", "type": "ATTACHED_TO", "port_range": "", "protocol": "", "cidr": "", "direction": "ingress", "relationship": "PROTECTED_BY_SG", "description": ""} + ] + } +} +``` + +| Key | Notes | +|---|---| +| `metadata` | Run identity. The timestamps are Python `str(datetime)` (a space, not `T`), and a missing value is the string `"None"`, not `null`. The CLI and the engine leave `provider`, `account_id` and `region` at `null` | +| `summary` | Counts. `total_edges` counts the collected `NetworkEdge` objects, which `findings.json` does not otherwise contain | +| `assets` | `CloudAsset` in JSON mode, without `raw_data`, with the computed `display_id` (ARN, else id). Fields: [CloudAsset](/reference/inventory/cloudasset/) | +| `findings` | `Finding` in JSON mode, sorted by severity, with the computed `risk_score` (severity base, averaged with `cvss_score` when present). Reachability findings have deterministic UUID5 ids; most scanner findings get a fresh UUID on every run. The normaliser can add frameworks the scanner did not set | +| `compliance` | One `ComplianceResult` per framework control, with the ids of the findings that failed it. `-aggregate` collects findings mapped to a framework but to no specific control | +| `graph` | D3 force-layout data, the same shape as `inventory-graph.json`. Rule edges on the same pair are merged here, so `links` can be shorter than the edge count | + +Read it back with the models: `CloudAsset.model_validate(...)` and `Finding.model_validate(...)` accept these entries as they are, which is what `cloudg report` does. + +## inventory-map.json + +The map `cloudg map` writes, and `cloudg deps` and the MCP workspace read. The same two assets, as a map: + +```json title="reports/inventory-map.json" +{ + "summary": { + "total_assets": 2, + "total_edges": 2, + "providers": ["aws"], + "assets_by_type": {"EC2": 1, "SECURITY_GROUP": 1}, + "assets_by_service": {"ec2": 2}, + "assets_by_region": {"eu-west-1": 2}, + "assets_by_account": {"123456789012": 2}, + "edges_by_type": {"SECURITY_GROUP_RULE": 1, "ATTACHED_TO": 1}, + "edges_by_relationship": {"PROTECTED_BY_SG": 1}, + "unlinked_assets": 0, + "internet_exposed": 0, + "accounts": 1, + "cross_account_edges": 0, + "external_accounts": 0, + "security_service_gaps": 0, + "unresolved_references": 0 + }, + "providers": ["aws"], + "regions": {"aws": ["eu-west-1"]}, + "assets": [ + {"id": "sg-web", "arn": "arn:aws:ec2:eu-west-1:123456789012:security-group/sg-web", + "name": "sg-web", "asset_type": "SECURITY_GROUP", "provider": "AWS", "region": "eu-west-1", + "account_id": "123456789012", "tags": {}, "metadata": {}, + "collected_at": "2026-10-09T16:29:31.447446", "is_internet_exposed": false, + "display_id": "arn:aws:ec2:eu-west-1:123456789012:security-group/sg-web"} + ], + "edges": [ + {"id": "53172b71-cf60-48f5-8bec-def2fbe139cc", "source_id": "0.0.0.0/0", "target_id": "sg-web", + "edge_type": "SECURITY_GROUP_RULE", "ports": [], "port_range": "22", "protocol": "TCP", + "cidr": "0.0.0.0/0", "direction": "ingress", "description": null, "relationship": null, + "properties": {}} + ], + "unresolved_references": [] +} +``` + +| Key | Notes | +|---|---| +| `summary` | Counts by type, service, region and account, edge counts by type and relationship, and health counters. `organization` and `throttling` blocks appear only when an organization was mapped or an API throttled. See [summary](/reference/inventory/summary/) | +| `assets` | Every asset, linked and deduplicated. Real collector assets carry rich `metadata`, including the `relations` the linker resolved | +| `edges` | Every edge, parallel rules kept. An endpoint can be a CIDR or another non-asset string, so look endpoints up with `.get()` | +| `unresolved_references` | Declared relations the linker could not resolve, with `source`, `source_name`, `target` and `edge_type` | +| `throttling` | Only when a cloud API throttled the run: the full throttling report of the run | + +`edges` here and `links` in `inventory-graph.json` differ in length whenever parallel rules were merged for the graph. Count edges in this file. For every field of every structure, see the [inventory reference](/reference/inventory/inventory-map-json/). + +## Related + +:::links +- [cloudg run](/cli/run/) The flags that switch outputs on and off. +- [cloudg map](/cli/map/) The inventory map and its companions. +- [Reports](/guides/reports/) Reading `report.html` and `findings.json`. +- [Data models](/api/models/) The Pydantic models behind every JSON file. +- [CloudAsset reference](/reference/inventory/cloudasset/) Every asset field and identifier format. +::: diff --git a/site/data/config-notes.yaml b/site/data/config-notes.yaml new file mode 100644 index 0000000..d83757d --- /dev/null +++ b/site/data/config-notes.yaml @@ -0,0 +1,60 @@ +# Descriptions for config.yaml keys that have no comment in the file and no +# Field(description=...) in cloudg/config.py. Keys are "
.". +# A bare section name sets that page's intro. + +aws: Credentials, regions and accounts for the AWS collectors. Any auth method can be combined with role assumption. +azure: Credentials, subscriptions and regions for the Azure collectors. +gcp: Credentials, projects and scope for the GCP collectors. +scanners: Which scanners `cloudg run` and `cloudg scan` start, and the extra arguments each one gets. +graph: Settings for the NetworkX graph built from collected assets and edges. +ontology: The RDF ontology export (Turtle and JSON-LD). +rag: Chunked exports for retrieval-augmented generation pipelines. +terraform: Terraform files that recreate the collected infrastructure. Off by default. +report: Which report files are written and where. +rulesets: Where the compliance rulesets are read from. + +aws.regions: Regions to collect. `ALL` discovers every region enabled for the account. +aws.role_session_name: Session name used when cloudg assumes a role. +aws.organization.exclude_accounts: Account IDs to leave out of an Organizations run. +aws.organization.include_management_account: Map the management account as well as the members. +aws.organization.include_suspended: Include suspended accounts. +azure.tenant_id: Directory (tenant) ID for service principal, certificate or federated auth. +azure.client_id: Application (client) ID of the service principal or workload identity. +azure.client_secret: Client secret for service principal auth. +azure.certificate_path: Path to a PEM certificate for certificate-based service principal auth. +azure.federated_token_file: Path to a federated (OIDC) token file for workload identity federation. +azure.use_managed_identity: Authenticate with the managed identity of the machine cloudg runs on. +gcp.organization_id: Organization to list from. Needed for the folder and project hierarchy. +gcp.credentials_file: Path to a service account key file. Leave null to use Application Default Credentials. +gcp.org_policies: Collect Organization Policy constraints. +gcp.vpc_service_controls: Collect VPC Service Controls perimeters. +scanners.enabled: Scanners to run. Any of prowler, scoutsuite, checkov, trivy and iam (the built-in IAM linter). +scanners.prowler_extra_args: Extra command-line arguments passed to Prowler. +scanners.scoutsuite_extra_args: Extra command-line arguments passed to ScoutSuite. +scanners.checkov_extra_args: Extra command-line arguments passed to Checkov. +scanners.trivy_extra_args: Extra command-line arguments passed to Trivy. +scanners.trivy_images: Container images for Trivy to scan. +scanners.iac_directories: Infrastructure-as-code directories for Checkov to scan. +scanners.timeout_seconds: Time limit for each scanner process, in seconds (minimum 60). +inventory.cloud_control_exclude: CloudFormation types or prefixes the Cloud Control sweep skips. +inventory.cloud_control_concurrency: Parallel Cloud Control list calls (1 to 32). +inventory.exclude_services: Service families or collector names to skip. +inventory.kubernetes_timeout: Seconds to wait on each Kubernetes API call (1 to 120). +inventory.max_images_per_repository: Image digests sampled per ECR repository (0 to 1000). +graph.persist_graphml: Write the graph to GraphML next to the reports. +graph.max_nodes_warn: Log a warning when the graph has more nodes than this. +graph.compute_attack_paths: Search the graph for attack paths during analysis. +graph.export_cytoscape: Also export the graph in Cytoscape JSON format. +ontology.enabled: Build and export the ontology. +ontology.export_formats: Serialisations to write. +ontology.include_raw_metadata: Copy each asset's raw provider metadata into the ontology. +rag.enabled: Write RAG chunks. +rag.max_chunk_tokens: Upper bound on the size of one chunk, in tokens (minimum 100). +terraform.enabled: Write Terraform files. +terraform.output_dir: Where the Terraform files go. +report.formats: Report formats to write. +report.output_dir: Default output directory for reports. +rulesets.load_external: Load the YAML rulesets from rules_dir. +ratelimit.enabled: Turn the shared rate limiter on or off for every provider. +ratelimit.aws.services: Per-service overrides of the built-in limits. +general.verbose: Log at debug level. diff --git a/site/data/nav.yaml b/site/data/nav.yaml new file mode 100644 index 0000000..ce90d23 --- /dev/null +++ b/site/data/nav.yaml @@ -0,0 +1,195 @@ +# Site map. Each tab has a sidebar made of groups; each group lists pages. +# +# page: hand-written page, read from site/content/.md +# split: docs/FILE.md one page per "##" section of a long reference document +# (sections over split_h3_kb are split again at "###") +# generated: built from code: cli-index, api-index, config, mcp-tools-index +# +# Paths are URLs under the site base, without slashes at either end. + +tabs: + - id: guides + label: Guides + root: guides/introduction + groups: + - title: Start + items: + - {title: Introduction, page: guides/introduction} + - {title: Installation, page: guides/installation} + - {title: Quickstart, page: guides/quickstart} + - {title: The pipeline, page: guides/pipeline} + - title: Mapping + items: + - {title: Inventory mapping, page: guides/inventory-mapping} + - {title: Dependencies & blast radius, page: guides/dependencies} + - {title: AWS Organizations, page: guides/aws-organizations} + - title: Scanning + items: + - {title: Running scanners, page: guides/running-scanners} + - {title: Ingesting existing output, page: guides/ingesting} + - {title: Compliance frameworks, page: guides/compliance} + - title: Analyse + items: + - {title: "Graph, ontology & RAG", page: guides/graph-ontology-rag} + - {title: Reports, page: guides/reports} + - title: Operate + items: + - {title: Authentication, page: guides/authentication} + - {title: Rate limits & throttling, page: guides/rate-limits} + - {title: Docker, page: guides/docker} + - {title: CI and automation, page: guides/ci-automation} + - {title: Python recipes, page: guides/python-recipes} + + - id: cli + label: CLI + root: cli + groups: + - title: Overview + items: + - {title: All commands, generated: cli-index, path: cli} + - title: Pipeline + items: + - {title: cloudg run, page: cli/run} + - {title: cloudg collect, page: cli/collect} + - {title: cloudg scan, page: cli/scan} + - {title: cloudg report, page: cli/report} + - title: Inventory + items: + - {title: cloudg map, page: cli/map} + - {title: cloudg deps, page: cli/deps} + - {title: cloudg ingest, page: cli/ingest} + - title: MCP + items: + - {title: cloudg mcp serve, page: cli/mcp-serve} + - {title: cloudg mcp config, page: cli/mcp-config} + - {title: cloudg mcp tools, page: cli/mcp-tools} + - {title: cloudg mcp resources, page: cli/mcp-resources} + - {title: cloudg mcp prompts, page: cli/mcp-prompts} + - {title: cloudg mcp call, page: cli/mcp-call} + - {title: cloudg mcp read, page: cli/mcp-read} + - title: Shared + items: + - {title: Provider & region flags, page: cli/provider-flags} + - {title: Auth flags, page: cli/auth-flags} + - {title: Output flags, page: cli/output-flags} + + - id: api + label: Python API + root: api + groups: + - title: Overview + items: + - {title: Python API, generated: api-index, path: api} + - title: Engine + items: + - {title: CloudGEngine, page: api/cloudgengine} + - {title: CloudGConfig, page: api/cloudgconfig} + - {title: Result objects, page: api/results} + - {title: Event hooks, page: api/event-hooks} + - title: Inventory + items: + - {title: InventoryMapper, page: api/inventorymapper} + - {title: InventoryResult, page: api/inventoryresult} + - {title: RelationshipLinker, page: api/relationshiplinker} + - {title: DependencyGraph, page: api/dependencygraph} + - title: Graph + items: + - {title: GraphBuilder, page: api/graphbuilder} + - {title: ReachabilityAnalyzer, page: api/reachabilityanalyzer} + - {title: CloudOntology, page: api/cloudontology} + - {title: RAGExporter, page: api/ragexporter} + - title: Findings + items: + - {title: FindingsNormaliser, page: api/findingsnormaliser} + - {title: parse_report, page: api/parse-report} + - {title: Data models, page: api/models} + - title: Extend + items: + - {title: Entry points, page: api/entry-points} + - {title: cloudg.resilience, page: api/resilience} + - {title: CloudGMCPLayer, page: api/mcp-layer} + + - id: mcp + label: MCP server + root: mcp + groups: + - title: Start + items: + - {title: Overview, page: mcp/index, path: mcp} + - {title: Connect a client, page: mcp/connect} + - title: Server + split: docs/MCP.md + prefix: mcp/server + skip: [Contents] + - title: Tools + split: docs/MCP_TOOLS.md + prefix: mcp/tools + skip: [Tool reference] + - title: Privacy & policy + split: docs/MCP_PRIVACY.md + prefix: mcp/privacy + skip: [Contents] + - title: Internals + split: docs/MCP_INTERNALS.md + prefix: mcp/internals + skip: [Contents] + + - id: reference + label: Reference + root: reference/config/providers + groups: + - title: config.yaml + generated: config + prefix: reference/config + - title: Outputs + items: + - {title: Output files, page: reference/output-files} + - title: Inventory structures + split: docs/INVENTORY_REFERENCE.md + prefix: reference/inventory + skip: [Contents] + - title: Catalogs + split: docs/INVENTORY_CATALOG.md + prefix: reference/catalog + skip: [Contents] + - title: Inventory internals + split: docs/INVENTORY_INTERNALS.md + prefix: reference/internals + skip: [Contents] + - title: Resilience internals + split: docs/RESILIENCE.md + prefix: reference/resilience + skip: [Contents] + + - id: changelog + label: Changelog + root: changelog + +# Links from the old one-page handbook (index.html#anchor) to their new homes. +redirects: + install: guides/installation + pipeline: guides/pipeline + run: cli/run + collect-scan: cli/collect + map: guides/inventory-mapping + deps: guides/dependencies + ingest: guides/ingesting + report: cli/report + inputs: guides/ingesting#input-requirements + outputs: reference/output-files + models: api/models + inventory-docs: reference/inventory + config: reference/config/providers + ratelimit: guides/rate-limits + api: api + recipes: guides/python-recipes + plugins: api/entry-points + auth: guides/authentication + docker: guides/docker + mcp: mcp + mcp-serve: cli/mcp-serve + mcp-embed: mcp/server/in-process-use + mcp-policies: mcp/privacy/built-in-profiles + mcp-catalog: mcp/tools/tool-index + mcp-cli: cli/mcp-serve + changelog: changelog diff --git a/site/data/stats.yaml b/site/data/stats.yaml new file mode 100644 index 0000000..f76aabc --- /dev/null +++ b/site/data/stats.yaml @@ -0,0 +1,19 @@ +# Figures on the docs home page. Entries with `compute` are counted from the +# code at build time; `value` is the fallback (and the only source for the +# framework count, which follows the README's way of counting). +stats: + - label: compliance frameworks + value: 28 + href: guides/compliance + - label: controls mapped + compute: controls + value: 4166 + href: guides/compliance + - label: asset types catalogued + compute: asset_types + value: 156 + href: reference/catalog + - label: MCP tools + compute: mcp_tools + value: 74 + href: mcp diff --git a/site/docsgen/__init__.py b/site/docsgen/__init__.py new file mode 100644 index 0000000..450dc9f --- /dev/null +++ b/site/docsgen/__init__.py @@ -0,0 +1 @@ +"""Static site generator for the cloudg documentation (see site/AUTHORING.md).""" diff --git a/site/docsgen/gen_api.py b/site/docsgen/gen_api.py new file mode 100644 index 0000000..39f05fd --- /dev/null +++ b/site/docsgen/gen_api.py @@ -0,0 +1,319 @@ +"""Python API reference data, read from the live objects with ``inspect``.""" + +from __future__ import annotations + +import dataclasses +import enum +import importlib +import inspect +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +_SECTION_RE = re.compile( + r"^(Args|Arguments|Parameters|Params|Returns|Return|Yields|Raises|Example|Examples|Note|Notes|Attributes|Usage)\s*:\s*$" +) + + +@dataclass +class Param: + name: str + type: str + default: str + description: str = "" + kind: str = "" + + +@dataclass +class Doc: + summary: str = "" + body: str = "" + params: dict[str, str] = field(default_factory=dict) + returns: str = "" + raises: list[tuple[str, str]] = field(default_factory=list) + examples: str = "" + attributes: dict[str, str] = field(default_factory=dict) + notes: str = "" + + +@dataclass +class Member: + name: str + kind: str # method, async, property, classmethod, staticmethod, attribute + signature: str + returns: str + doc: Doc + params: list[Param] + source_line: int = 0 + + +@dataclass +class ApiObject: + path: str + name: str + module: str + kind: str # class, function, module, enum, model, dataclass + signature: str + doc: Doc + params: list[Param] + returns: str + members: list[Member] = field(default_factory=list) + fields: list[Param] = field(default_factory=list) + enum_values: list[tuple[str, str]] = field(default_factory=list) + bases: list[str] = field(default_factory=list) + source_file: str = "" + source_line: int = 0 + + +def rst_to_md(text: str) -> str: + """Turn the reST bits docstrings use (``:func:`x```, double backticks) into markdown.""" + text = re.sub(r":[\w]+(?::[\w]+)?:`~?\.?([^`]+)`", r"`\1`", text) + return re.sub(r"``([^`]+)``", r"`\1`", text) + + +def parse_docstring(text: str | None) -> Doc: + doc = Doc() + if not text: + return doc + text = rst_to_md(inspect.cleandoc(text)) + sections: dict[str, list[str]] = {"": []} + current = "" + for line in text.split("\n"): + m = _SECTION_RE.match(line.strip()) if not line.startswith(" ") else None + if m: + current = m.group(1).lower() + sections.setdefault(current, []) + continue + sections.setdefault(current, []).append(line) + + main = "\n".join(sections.pop("", [])).strip() + paras = re.split(r"\n\s*\n", main, maxsplit=1) + doc.summary = " ".join(paras[0].split()) + doc.body = paras[1].strip() if len(paras) > 1 else "" + + def items(lines: list[str]) -> list[tuple[str, str]]: + out: list[tuple[str, str]] = [] + lines = inspect.cleandoc("\n".join(lines)).split("\n") if lines else [] + for line in lines: + if not line.strip(): + continue + m = re.match(r"^(\*{0,2}[\w.\[\], |]+?)\s*(\([^)]*\))?\s*:\s*(.*)$", line) if not line.startswith(" ") else None + if m: + out.append((m.group(1).strip("* "), m.group(3).strip())) + elif out: + out[-1] = (out[-1][0], (out[-1][1] + " " + line.strip()).strip()) + return out + + for key in ("args", "arguments", "parameters", "params"): + if key in sections: + doc.params.update(dict(items(sections.pop(key)))) + for key in ("returns", "return", "yields"): + if key in sections: + doc.returns = " ".join(" ".join(sections.pop(key)).split()) + if "raises" in sections: + doc.raises = items(sections.pop("raises")) + if "attributes" in sections: + doc.attributes = dict(items(sections.pop("attributes"))) + for key in ("example", "examples", "usage"): + if key in sections: + doc.examples = inspect.cleandoc("\n".join(sections.pop(key))) + notes = [] + for key in ("note", "notes"): + if key in sections: + notes.append(inspect.cleandoc("\n".join(sections.pop(key)))) + doc.notes = "\n\n".join(notes) + return doc + + +def _ann(a: Any) -> str: + if a is inspect.Parameter.empty or a is inspect.Signature.empty: + return "" + if isinstance(a, str): + return a.strip("'\"") + if isinstance(a, type): + return a.__name__ if a.__module__ == "builtins" else a.__name__ + s = repr(a) + return re.sub(r"\b(?:[a-z_][\w]*\.)+([A-Z]\w*)", r"\1", s).replace("typing.", "") + + +def _default(d: Any) -> str: + if d is inspect.Parameter.empty: + return "" + if isinstance(d, enum.Enum): + return f"{type(d).__name__}.{d.name}" + r = repr(d) + return r if len(r) <= 60 else r[:57] + "..." + + +def _params(sig: inspect.Signature | None, doc: Doc, skip_self: bool) -> list[Param]: + if sig is None: + return [] + out = [] + for i, (name, p) in enumerate(sig.parameters.items()): + if skip_self and i == 0 and name in ("self", "cls"): + continue + prefix = "*" if p.kind is p.VAR_POSITIONAL else "**" if p.kind is p.VAR_KEYWORD else "" + desc = doc.params.get(name) or doc.params.get(prefix + name) or "" + out.append(Param(prefix + name, _ann(p.annotation), _default(p.default), desc, p.kind.name)) + return out + + +def format_signature(name: str, params: list[Param], returns: str, width: int = 88) -> str: + pieces = [] + seen_kwonly = False + for p in params: + if p.kind == "KEYWORD_ONLY" and not seen_kwonly and not any(q.name.startswith("*") and not q.name.startswith("**") for q in params): + pieces.append("*") + if p.kind == "KEYWORD_ONLY": + seen_kwonly = True + s = p.name + if p.type: + s += f": {p.type}" + if p.default: + s += f" = {p.default}" if p.type else f"={p.default}" + pieces.append(s) + ret = f" -> {returns}" if returns else "" + one = f"{name}({', '.join(pieces)}){ret}" + if len(one) <= width: + return one + return f"{name}(\n" + "".join(f" {p},\n" for p in pieces) + f"){ret}" + + +def _signature(obj: Any) -> inspect.Signature | None: + try: + return inspect.signature(obj) + except (TypeError, ValueError): + return None + + +def _source(obj: Any, repo: Path) -> tuple[str, int]: + try: + target = inspect.unwrap(obj) + file = inspect.getsourcefile(target) or "" + line = inspect.getsourcelines(target)[1] + return str(Path(file).resolve().relative_to(repo.resolve())), line + except (TypeError, OSError, ValueError): + return "", 0 + + +def _is_pydantic(cls: type) -> bool: + return hasattr(cls, "model_fields") and hasattr(cls, "model_validate") + + +def _member(cls: type, name: str, repo: Path) -> Member | None: + raw = inspect.getattr_static(cls, name, None) + if raw is None: + return None + kind = "method" + target: Any = raw + if isinstance(raw, property): + kind, target = "property", raw.fget + elif isinstance(raw, classmethod): + kind, target = "classmethod", raw.__func__ + elif isinstance(raw, staticmethod): + kind, target = "staticmethod", raw.__func__ + elif inspect.iscoroutinefunction(raw): + kind = "async" + elif not callable(raw): + return Member(name, "attribute", name, _ann(type(raw)), Doc(), []) + doc = parse_docstring(inspect.getdoc(target)) + sig = _signature(target) + params = _params(sig, doc, skip_self=kind in ("method", "async", "classmethod", "property")) + returns = _ann(sig.return_annotation) if sig else "" + if kind == "property": + signature, params = name, [] + else: + signature = format_signature(name, params, returns) + _, line = _source(target, repo) + return Member(name, kind, signature, returns, doc, params, line) + + +def _public_members(cls: type) -> list[str]: + names = [] + base_names: set[str] = set() + for base in cls.__mro__[1:]: + if base.__module__.startswith(("pydantic", "builtins", "enum", "typing", "abc")): + base_names.update(vars(base)) + for name, value in vars(cls).items(): + if name.startswith("_") or name in base_names: + continue + if callable(value) or isinstance(value, (property, classmethod, staticmethod)): + names.append(name) + return names + + +def load(path: str, repo: Path, members: list[str] | None = None) -> ApiObject: + module_name, _, attr = path.rpartition(".") + try: + module = importlib.import_module(path) + obj, module_name, attr = module, path, path.rsplit(".", 1)[-1] + except ImportError: + module = importlib.import_module(module_name) + obj = getattr(module, attr) + real_module = getattr(obj, "__module__", module_name) or module_name + file, line = _source(obj, repo) + doc = parse_docstring(inspect.getdoc(obj) if not inspect.ismodule(obj) else obj.__doc__) + + if inspect.ismodule(obj): + return ApiObject(path, attr, module_name, "module", path, doc, [], "", source_file=file, source_line=1) + + if inspect.isclass(obj): + if issubclass(obj, enum.Enum): + kind = "enum" + elif _is_pydantic(obj): + kind = "model" + elif dataclasses.is_dataclass(obj): + kind = "dataclass" + else: + kind = "class" + init_sig = None + if kind in ("class", "dataclass"): + init_sig = _signature(obj) + init_doc = parse_docstring(inspect.getdoc(obj.__init__)) if "__init__" in vars(obj) else Doc() + merged = Doc(**{**doc.__dict__, "params": {**init_doc.params, **doc.params}}) + params = _params(init_sig, merged, skip_self=False) if init_sig else [] + signature = format_signature(obj.__name__, params, "") if kind in ("class", "dataclass") else obj.__name__ + bases = [b.__name__ for b in obj.__bases__ if b is not object] + api = ApiObject(path, obj.__name__, real_module, kind, signature, merged, params, "", bases=bases, + source_file=file, source_line=line) + if kind == "enum": + api.enum_values = [(m.name, repr(m.value)) for m in obj] + if kind == "model": + for fname, finfo in obj.model_fields.items(): + ann = _ann(finfo.annotation) + if finfo.is_required(): + default = "required" + elif finfo.default_factory is not None: + name = getattr(finfo.default_factory, "__name__", "") + default = f"{name}()" if name and name != "" else "generated" + else: + default = _default(finfo.default) + desc = finfo.description or doc.attributes.get(fname, "") + api.fields.append(Param(fname, ann, default, desc)) + elif kind == "dataclass": + for f in dataclasses.fields(obj): + if f.default is not dataclasses.MISSING: + default = _default(f.default) + elif f.default_factory is not dataclasses.MISSING: # type: ignore[misc] + name = getattr(f.default_factory, "__name__", "") + default = f"{name}()" if name and name != "" else "generated" + else: + default = "required" + api.fields.append(Param(f.name, _ann(f.type), default, doc.attributes.get(f.name, ""))) + names = members if members is not None else _public_members(obj) + if kind == "enum" and members is None: + names = [] + for name in names: + m = _member(obj, name, repo) + if m is not None: + api.members.append(m) + return api + + # function + sig = _signature(obj) + params = _params(sig, doc, skip_self=False) + returns = _ann(sig.return_annotation) if sig else "" + kind = "async" if inspect.iscoroutinefunction(obj) else "function" + signature = format_signature(attr, params, returns) + return ApiObject(path, attr, real_module, kind, signature, doc, params, returns, source_file=file, source_line=line) diff --git a/site/docsgen/gen_changelog.py b/site/docsgen/gen_changelog.py new file mode 100644 index 0000000..b081571 --- /dev/null +++ b/site/docsgen/gen_changelog.py @@ -0,0 +1,51 @@ +"""CHANGELOG.md split into releases and labelled blocks (Added, Changed, Fixed...).""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path + +_RELEASE_RE = re.compile(r"^##\s+(.+?)\s*$") +_VERSION_RE = re.compile(r"^(\d+\.\d+\.\d+)\s*(?:\((\d{4}-\d{2}-\d{2})\))?") +_LABEL_RE = re.compile(r"^([A-Z][A-Za-z ]{2,30}):\s*$") + + +@dataclass +class Release: + title: str + version: str + date: str + anchor: str + intro: str + blocks: list[tuple[str, str]] = field(default_factory=list) + + +def parse(path: Path) -> tuple[str, list[Release]]: + text = path.read_text() + parts = re.split(r"(?m)^(?=## )", text) + head = parts[0] + head = re.sub(r"(?m)^#\s+.*$", "", head).strip() + releases: list[Release] = [] + for part in parts[1:]: + lines = part.rstrip().split("\n") + title = _RELEASE_RE.match(lines[0]).group(1) + m = _VERSION_RE.match(title) + version = m.group(1) if m else title + date = (m.group(2) or "") if m else "" + anchor = version if m else re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-") + intro: list[str] = [] + blocks: list[tuple[str, list[str]]] = [] + fence = False + for line in lines[1:]: + if line.lstrip().startswith("```"): + fence = not fence + lm = _LABEL_RE.match(line) if not fence else None + if lm: + blocks.append((lm.group(1), [])) + continue + (blocks[-1][1] if blocks else intro).append(line) + releases.append( + Release(title, version, date, anchor, "\n".join(intro).strip(), [(k, "\n".join(v).strip()) for k, v in blocks]) + ) + return head, releases diff --git a/site/docsgen/gen_cli.py b/site/docsgen/gen_cli.py new file mode 100644 index 0000000..73f4bfe --- /dev/null +++ b/site/docsgen/gen_cli.py @@ -0,0 +1,176 @@ +"""CLI reference data, read from the click command tree in ``cloudg.cli``.""" + +from __future__ import annotations + +import inspect +from dataclasses import dataclass, field +from pathlib import Path + +import click + + +@dataclass +class Option: + flags: str + value: str + default: str + help: str + required: bool = False + multiple: bool = False + envvar: str = "" + is_argument: bool = False + anchor: str = "" + + +@dataclass +class Command: + path: str # "map", "mcp serve" + help: str + short_help: str + usage: str + options: list[Option] = field(default_factory=list) + source_file: str = "" + source_line: int = 0 + group: bool = False + + +def _is_unset(value) -> bool: + return value is None or type(value).__name__ in ("Sentinel", "_Missing") or repr(value).startswith("Sentinel.") + + +def _value_of(param: click.Parameter) -> str: + t = param.type + if isinstance(param, click.Option) and param.is_flag: + return "flag" + if isinstance(t, click.Choice): + return "|".join(str(c) for c in t.choices) + if getattr(param, "metavar", None): + return param.metavar + name = getattr(t, "name", "text") + return { + "text": "TEXT", + "integer": "INT", + "float": "FLOAT", + "boolean": "BOOL", + "file": "FILE", + "path": "PATH", + "filename": "FILE", + }.get(name, name.upper()) + + +def _default_of(param: click.Parameter) -> str: + default = param.default + if isinstance(param, click.Option) and param.is_flag and param.secondary_opts: + if _is_unset(default): + return "config" + return param.opts[0] if default else param.secondary_opts[0] + if isinstance(param, click.Option) and param.is_flag: + return "on" if default is True else "off" + if _is_unset(default): + return "none" + if isinstance(default, (tuple, list)): + return ", ".join(str(d) for d in default) if default else "none" + return str(default) + + +def _flags_of(param: click.Parameter) -> str: + if isinstance(param, click.Argument): + return param.human_readable_name + opts = sorted(param.opts, key=lambda o: (o.startswith("--"), o)) + flags = ", ".join(opts) + if getattr(param, "secondary_opts", None): + flags += " / " + ", ".join(param.secondary_opts) + return flags + + +def _usage(path: str, cmd: click.Command) -> str: + parts = [f"cloudg {path}"] + optional = False + for p in cmd.params: + if isinstance(p, click.Argument): + continue + if getattr(p, "hidden", False): + continue + if p.required: + parts.append(f"{min(p.opts, key=len)} {(p.name or 'value').upper()}") + else: + optional = True + if optional: + parts.append("[OPTIONS]") + for p in cmd.params: + if isinstance(p, click.Argument): + name = p.human_readable_name + parts.append(name if p.required else f"[{name}]") + return " ".join(parts) + + +def _source(cmd: click.Command) -> tuple[str, int]: + fn = cmd.callback + while fn is not None and hasattr(fn, "__wrapped__"): + fn = fn.__wrapped__ + try: + file = inspect.getsourcefile(fn) + line = inspect.getsourcelines(fn)[1] + except (TypeError, OSError): + return "", 0 + return file or "", line + + +def collect(repo: Path) -> dict[str, Command]: + from cloudg.cli import cli + + out: dict[str, Command] = {} + + def walk(cmd: click.Command, path: str) -> None: + if path: + options = [] + for p in cmd.params: + if getattr(p, "hidden", False): + continue + help_text = getattr(p, "help", "") or "" + opt = Option( + flags=_flags_of(p), + value=_value_of(p), + default=_default_of(p), + help=" ".join(help_text.split()), + required=bool(p.required), + multiple=bool(getattr(p, "multiple", False)), + envvar=p.envvar if isinstance(getattr(p, "envvar", None), str) else "", + is_argument=isinstance(p, click.Argument), + anchor=(p.opts[0] if isinstance(p, click.Argument) else max(p.opts, key=len)).lstrip("-"), + ) + options.append(opt) + file, line = _source(cmd) if cmd.callback else ("", 0) + try: + file = str(Path(file).resolve().relative_to(repo.resolve())) if file else "" + except ValueError: + pass + help_text = inspect.cleandoc(cmd.help or "") + out[path] = Command( + path=path, + help=help_text, + short_help=cmd.get_short_help_str(limit=200), + usage=_usage(path, cmd), + options=options, + source_file=file, + source_line=line, + group=isinstance(cmd, click.Group), + ) + if isinstance(cmd, click.Group): + for name, sub in cmd.commands.items(): + walk(sub, f"{path} {name}".strip()) + + walk(cli, "") + root_help = inspect.cleandoc(cli.help or "") + out["__root__"] = Command( + path="", + help=root_help, + short_help="", + usage="cloudg [OPTIONS] COMMAND [ARGS]...", + options=[ + Option(flags=_flags_of(p), value=_value_of(p), default=_default_of(p), help=" ".join((p.help or "").split())) + for p in cli.params + if isinstance(p, click.Option) + ], + ) + return out diff --git a/site/docsgen/gen_config.py b/site/docsgen/gen_config.py new file mode 100644 index 0000000..9df0c99 --- /dev/null +++ b/site/docsgen/gen_config.py @@ -0,0 +1,184 @@ +"""config.yaml reference, read from the annotated example file itself. + +Each key's description is its trailing comment, or failing that the comment +lines directly above it. The YAML text of each top-level section is kept +line by line so the page can show it next to the key table. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path + +_KEY_RE = re.compile(r"^(\s*)(-\s+)?([\w.\-]+):(?:\s+(.*?))?\s*$") +_ITEM_RE = re.compile(r"^(\s*)-\s+(.*?)\s*$") +_RULE_RE = re.compile(r"^#\s*[─━═\-]{3,}") +_HEADER_RE = re.compile(r"^#\s*[─━═]{1,}\s*(.+?)\s*[─━═]*\s*$") + + +@dataclass +class Key: + path: str # relative to the section, e.g. "aws.retry_budget"; "" for the section root + type: str + default: str + description: str + line: int # index into Section.lines + + +@dataclass +class Section: + name: str + title: str + intro: str + lines: list[str] = field(default_factory=list) + keys: list[Key] = field(default_factory=list) + + +def _split_comment(value: str) -> tuple[str, str]: + """Split ``value # comment`` respecting quotes.""" + quote = None + for i, ch in enumerate(value): + if ch in "'\"": + quote = None if quote == ch else (ch if quote is None else quote) + elif ch == "#" and quote is None and (i == 0 or value[i - 1] in " \t"): + return value[:i].strip(), value[i + 1 :].strip() + return value.strip(), "" + + +def _is_code_comment(text: str) -> bool: + """Commented-out YAML and linter pragmas are not descriptions.""" + return bool( + text.startswith(("- ", "nosemgrep", "Uncomment", "#")) + or re.match(r"^[\w.\-]+:\s", text + " ") + and not re.match(r"^(e\.g\.|Note|Default|Options|Example)", text) + ) + + +def _clean(comment: str) -> str: + comment = re.sub(r"\s*nosemgrep:.*$", "", comment) + return comment.strip() + + +def _type_of(value: str) -> str: + v = value.strip() + if v in ("null", "~", ""): + return "null" + if v in ("true", "false"): + return "bool" + if re.fullmatch(r"-?\d+", v): + return "int" + if re.fullmatch(r"-?\d+\.\d*", v): + return "float" + if v.startswith("["): + return "list" + if v.startswith("{"): + return "map" + return "str" + + +def parse(path: Path) -> list[Section]: + raw = path.read_text().split("\n") + sections: list[Section] = [] + general = Section("general", "General", "Settings at the top level of the file that apply to the whole run.") + pending_comments: list[str] = [] + header = "" + current: Section | None = None + stack: list[tuple[int, str]] = [] + last_key: Key | None = None + list_items: list[str] = [] + + def flush_list() -> None: + nonlocal list_items, last_key + if last_key is not None and list_items and last_key.type in ("map", "null"): + last_key.type = "list" + last_key.default = "[" + ", ".join(list_items) + "]" + list_items = [] + + for line in raw: + stripped = line.strip() + indent = len(line) - len(line.lstrip(" ")) + if not stripped: + pending_comments = [] + if current is not None and current is not general: + current.lines.append(line) + continue + if stripped.startswith("#"): + hm = _HEADER_RE.match(stripped) + if indent == 0 and (hm or _RULE_RE.match(stripped)): + if hm and not _RULE_RE.match(stripped): + header = hm.group(1).strip("─━═ ") + pending_comments = [] + continue + text = stripped.lstrip("#").strip() + if not _is_code_comment(text): + pending_comments.append(text) + if current is not None and indent > 0: + current.lines.append(line) + continue + + km = _KEY_RE.match(line) + if km and not km.group(2): + key, rest = km.group(3), km.group(4) or "" + value, comment = _split_comment(rest) + comment = _clean(comment) + if _is_code_comment(comment): + comment = "" + if indent == 0: + flush_list() + if value: # top-level scalar + general.lines.append(line) + general.keys.append(Key(key, _type_of(value), value, comment or " ".join(pending_comments), + len(general.lines) - 1)) + current = general + stack = [] + last_key = general.keys[-1] + else: + current = Section(key, header or key, " ".join(pending_comments)) + current.lines.append(line) + sections.append(current) + stack = [(0, key)] + header = "" + last_key = None + pending_comments = [] + continue + if current is None: + continue + flush_list() + while stack and stack[-1][0] >= indent: + stack.pop() + parents = [k for _, k in stack[1:]] if current is not general else [k for _, k in stack] + rel = ".".join(parents + [key]) + current.lines.append(line) + typ = _type_of(value) if value else "map" + k = Key(rel, typ, value if value else "", comment or " ".join(pending_comments), len(current.lines) - 1) + current.keys.append(k) + last_key = k + stack.append((indent, key)) + pending_comments = [] + continue + + im = _ITEM_RE.match(line) + if im and current is not None: + item, comment = _split_comment(im.group(2)) + current.lines.append(line) + if last_key is not None and ":" not in item: + list_items.append(item) + if comment and not last_key.description: + last_key.description = comment + pending_comments = [] + continue + if current is not None: + current.lines.append(line) + flush_list() + + for s in sections: + while s.lines and not s.lines[-1].strip(): + s.lines.pop() + # maps with only children: describe as such + for k in s.keys: + if k.type == "map" and not k.default: + k.default = "" + if general.keys: + sections.append(general) + return sections diff --git a/site/docsgen/highlight.py b/site/docsgen/highlight.py new file mode 100644 index 0000000..ee9a80f --- /dev/null +++ b/site/docsgen/highlight.py @@ -0,0 +1,246 @@ +"""Code blocks rendered as editor or terminal windows. + +Pygments does the tokenising; the HTML is built here, one ```` +per source line, so line numbers, highlighted lines and terminal prompts can be +drawn with CSS and stay out of the copied text. +""" + +from __future__ import annotations + +import re +from html import escape + +from pygments.lexers import get_lexer_by_name +from pygments.token import STANDARD_TYPES, Token +from pygments.util import ClassNotFound + +from docsgen.icons import icon + +TERMINAL_LANGS = {"bash", "sh", "shell", "zsh", "shell-session-commands"} +SESSION_LANGS = {"console", "shell-session"} +LANG_LABELS = { + "python": "Python", + "py": "Python", + "yaml": "YAML", + "yml": "YAML", + "json": "JSON", + "jsonc": "JSON", + "toml": "TOML", + "ini": "INI", + "dockerfile": "Dockerfile", + "docker": "Dockerfile", + "text": "Text", + "txt": "Text", + "sparql": "SPARQL", + "turtle": "Turtle", + "ttl": "Turtle", + "xml": "XML", + "html": "HTML", + "javascript": "JavaScript", + "js": "JavaScript", + "typescript": "TypeScript", + "hcl": "HCL", + "terraform": "HCL", + "sql": "SQL", + "powershell": "PowerShell", + "ps1": "PowerShell", + "diff": "Diff", + "jsonl": "JSON Lines", + "csv": "CSV", +} +_ATTR_RE = re.compile(r'(\w+)=("([^"]*)"|\'([^\']*)\'|(\S+))') + + +def parse_info(info: str) -> tuple[str, dict[str, str]]: + """Split a fence info string into language and attributes. + + ``python title="a.py" hl="2-4"`` gives ``("python", {"title": "a.py", "hl": "2-4"})``. + A bare ``{2-4}`` is accepted as ``hl`` too. + """ + info = info.strip() + if not info: + return "", {} + lang, _, rest = info.partition(" ") + attrs: dict[str, str] = {} + for m in _ATTR_RE.finditer(rest): + attrs[m.group(1)] = next(g for g in (m.group(3), m.group(4), m.group(5)) if g is not None) + brace = re.search(r"\{([\d,\-\s]+)\}", rest) + if brace and "hl" not in attrs: + attrs["hl"] = brace.group(1) + return lang.lower(), attrs + + +def parse_ranges(spec: str) -> set[int]: + out: set[int] = set() + for part in spec.replace(" ", "").split(","): + if not part: + continue + if "-" in part: + a, b = part.split("-", 1) + out.update(range(int(a), int(b) + 1)) + else: + out.add(int(part)) + return out + + +def _css_class(ttype) -> str: + while ttype not in STANDARD_TYPES and ttype is not Token: + ttype = ttype.parent + return STANDARD_TYPES.get(ttype, "") + + +def _lexer(lang: str): + alias = {"jsonl": "json", "yml": "yaml", "docker": "dockerfile", "ttl": "turtle", "terraform": "hcl"} + try: + return get_lexer_by_name(alias.get(lang, lang) or "text", stripnl=False, ensurenl=False) + except ClassNotFound: + return get_lexer_by_name("text", stripnl=False, ensurenl=False) + + +def tokenize_lines(code: str, lang: str) -> list[str]: + """Return one HTML string per source line.""" + lines: list[str] = [""] + for ttype, value in _lexer(lang).get_tokens(code): + cls = _css_class(ttype) + parts = value.split("\n") + for i, part in enumerate(parts): + if i: + lines.append("") + if part: + text = escape(part, quote=False) + lines[-1] += f'{text}' if cls else text + if lines and lines[-1] == "" and code.endswith("\n"): + lines.pop() + return lines + + +def _terminal_roles(raw_lines: list[str]) -> list[str]: + roles = [] + continued = False + in_heredoc: str | None = None + for line in raw_lines: + stripped = line.strip() + if in_heredoc is not None: + roles.append("out") + if stripped == in_heredoc: + in_heredoc = None + continue + if continued: + roles.append("cont") + elif not stripped: + roles.append("blank") + elif stripped.startswith("#"): + roles.append("cmt") + else: + roles.append("cmd") + m = re.search(r"<<-?\s*'?\"?(\w+)'?\"?\s*$", stripped) + if m and roles[-1] in ("cmd", "cont"): + in_heredoc = m.group(1) + continued = stripped.endswith("\\") and roles[-1] != "cmt" + return roles + + +def _session_roles(raw_lines: list[str]) -> list[tuple[str, str]]: + """For console blocks: (role, line without prompt).""" + out = [] + continued = False + for line in raw_lines: + if continued: + out.append(("cont", line)) + elif line.startswith("$ "): + out.append(("cmd", line[2:])) + elif line.startswith("# ") and not out: + out.append(("cmt", line)) + else: + out.append(("out", line)) + continued = out[-1][0] in ("cmd", "cont") and line.rstrip().endswith("\\") + return out + + +def code_bar_label(lang: str, attrs: dict[str, str]) -> tuple[str, str]: + """Icon name and label shown in the editor bar.""" + if lang in TERMINAL_LANGS or lang in SESSION_LANGS: + return "terminal", attrs.get("title", "Terminal") + return "file-code", attrs.get("title", LANG_LABELS.get(lang, lang.upper() if lang else "Text")) + + +def render_code_body(code: str, lang: str, attrs: dict[str, str]) -> tuple[str, str]: + """Render the ``
`` for a block. Returns (variant, html)."""
+    code = code.rstrip("\n") + "\n"
+    hl = parse_ranges(attrs["hl"]) if attrs.get("hl") else set()
+    if lang in SESSION_LANGS:
+        variant = "terminal"
+        rows = _session_roles(code.rstrip("\n").split("\n"))
+        html_lines = []
+        for role, text in rows:
+            inner = tokenize_lines(text + "\n", "bash")[0] if role in ("cmd", "cont") else escape(text, quote=False)
+            html_lines.append((role, inner))
+    elif lang in TERMINAL_LANGS:
+        variant = "terminal"
+        raw = code.rstrip("\n").split("\n")
+        roles = _terminal_roles(raw)
+        toks = tokenize_lines(code, "bash")
+        toks += [""] * (len(raw) - len(toks))
+        html_lines = list(zip(roles, toks))
+    else:
+        variant = "editor"
+        html_lines = [("", t) for t in tokenize_lines(code, lang)]
+    out = []
+    for n, (role, inner) in enumerate(html_lines, start=1):
+        classes = ["line"]
+        if role:
+            classes.append(role)
+        if n in hl:
+            classes.append("hl")
+        out.append(f'{inner or " "}')
+    pre = f'
' + "\n".join(out) + "
"
+    return variant, pre
+
+
+def render_code_block(code: str, info: str) -> str:
+    lang, attrs = parse_info(info)
+    variant, pre = render_code_body(code, lang, attrs)
+    ico, label = code_bar_label(lang, attrs)
+    lang_tag = LANG_LABELS.get(lang, lang) if lang and "title" in attrs and variant == "editor" else ""
+    bar = (
+        f'
{icon(ico, 14)}{escape(label)}'
+        + (f'{escape(lang_tag)}' if lang_tag else "")
+        + copy_button()
+        + "
"
+    )
+    return f'
{bar}{pre}
'
+
+
+def copy_button() -> str:
+    return (
+        f''
+    )
+
+
+def render_tab_group(blocks: list[tuple[str, str]]) -> str:
+    """Blocks are (code, info) pairs that all carry a ``tab`` attribute."""
+    tabs, panes = [], []
+    for i, (code, info) in enumerate(blocks):
+        lang, attrs = parse_info(info)
+        label = attrs.get("tab", f"Tab {i + 1}")
+        variant, pre = render_code_body(code, lang, attrs)
+        selected = "true" if i == 0 else "false"
+        ico = icon("terminal", 14) if variant == "terminal" else ""
+        tabs.append(
+            f''
+        )
+        sub = ""
+        if "title" in attrs:
+            sub_ico = "terminal" if variant == "terminal" else "file-code"
+            sub = f'
{icon(sub_ico, 13)}{escape(attrs["title"])}
'
+        hidden = "" if i == 0 else " hidden"
+        panes.append(
+            f'
{sub}{pre}
'
+        )
+    return (
+        '
'
+        f'
{"".join(tabs)}{copy_button()}
'
+        f'{"".join(panes)}
'
+    )
diff --git a/site/docsgen/icons.py b/site/docsgen/icons.py
new file mode 100644
index 0000000..8466c62
--- /dev/null
+++ b/site/docsgen/icons.py
@@ -0,0 +1,63 @@
+"""Lucide icons used by the site, inlined as SVG (stroke 2, currentColor)."""
+
+_PATHS = {
+    "search": '',
+    "chevron-down": '',
+    "github": (
+        ''
+    ),
+    "moon": '',
+    "sun": (
+        ''
+        ''
+        ''
+    ),
+    "pencil": (
+        ''
+    ),
+    "info": '',
+    "lightbulb": (
+        ''
+    ),
+    "triangle-alert": (
+        ''
+        ''
+    ),
+    "octagon-alert": (
+        '"
+    ),
+    "copy": '',
+    "check": '',
+    "arrow-left": '',
+    "arrow-right": '',
+    "terminal": '',
+    "corner-down-left": '',
+    "file-code": (
+        ''
+    ),
+    "menu": '',
+    "x": '',
+    "external": (
+        ''
+    ),
+    "list": (
+        ''
+        ''
+    ),
+}
+
+
+def icon(name: str, size: int = 16, cls: str = "") -> str:
+    attrs = f' class="{cls}"' if cls else ""
+    return (
+        f'