From 271ffe9bba18c4b718de3150a4e45ffb72bd728c Mon Sep 17 00:00:00 2001 From: Nathan Randall Date: Thu, 15 Oct 2026 14:34:41 -0600 Subject: [PATCH 1/4] fix(release): standardize on ubuntu-26.04 and share Go test job Release run 38748137720 failed because ubuntu-26.04 moves Graphviz's layout engines other than dot into a separate package, and PR checks still ran on ubuntu-latest (24.04). - Standardize Linux workflows on ubuntu-26.04; Linux archives stay on ubuntu-22.04 for the glibc floor. - Add the reusable test-go.yml, shared by PR checks and release-tag. - Add scripts/setup-graphviz.sh to install and check Graphviz engines. - Smoke-test darwin/amd64 under Rosetta on macos-15. - Bump Playwright to 1.61.1 for ubuntu-26.04. --- .github/workflows/actions.lock | 4 +- .github/workflows/build-and-test-go.yml | 91 +++++++---------------- .github/workflows/build-and-test-ui.yml | 16 ++-- .github/workflows/copilot-setup-steps.yml | 8 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/lint-go.yml | 2 +- .github/workflows/query-unit-tests.yml | 4 +- .github/workflows/release-build.yml | 30 +++++--- .github/workflows/release-tag.yml | 36 +++------ .github/workflows/self-audit.yml | 4 +- .github/workflows/test-go.yml | 81 ++++++++++++++++++++ .github/workflows/update-codeql.yml | 10 +-- docs/deployment/source.md | 2 +- plans/tools/deployment.md | 16 ++-- scripts/setup-graphviz.sh | 73 ++++++++++++++++++ ui/tests/e2e/package-lock.json | 27 ++++--- ui/tests/e2e/package.json | 2 +- 17 files changed, 265 insertions(+), 143 deletions(-) create mode 100644 .github/workflows/test-go.yml create mode 100755 scripts/setup-graphviz.sh diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index a9ba6269..506d0dce 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -45,7 +45,6 @@ workflows: - 'anchore/sbom-action@v0.24.2' '.github/workflows/release-tag.yml': - 'actions/checkout@v7.0.1' - - 'actions/setup-go@v7.0.0' '.github/workflows/release.yml': - 'actions/checkout@v7.0.1' '.github/workflows/self-audit.yml': @@ -53,6 +52,9 @@ workflows: - 'actions/checkout@v7.0.1' - 'actions/setup-dotnet@v6.0.0' - 'actions/setup-go@v7.0.0' + '.github/workflows/test-go.yml': + - 'actions/checkout@v7.0.1' + - 'actions/setup-go@v7.0.0' '.github/workflows/update-codeql.yml': - 'actions/cache@v6.1.0' - 'actions/checkout@v7.0.1' diff --git a/.github/workflows/build-and-test-go.yml b/.github/workflows/build-and-test-go.yml index cc082dfa..5d31027c 100644 --- a/.github/workflows/build-and-test-go.yml +++ b/.github/workflows/build-and-test-go.yml @@ -16,6 +16,7 @@ on: - '**/testdata/**' - 'scripts/**' - '.github/workflows/build-and-test-go.yml' + - '.github/workflows/test-go.yml' - 'go.mod' - 'go.sum' - 'Makefile' @@ -26,6 +27,7 @@ on: - '**/testdata/**' - 'scripts/**' - '.github/workflows/build-and-test-go.yml' + - '.github/workflows/test-go.yml' - 'go.mod' - 'go.sum' - 'Makefile' @@ -34,82 +36,43 @@ permissions: contents: read jobs: + # Each of these is test-go.yml, the job a release tests its tree with, so a release runs nothing + # that a pull request has not already run. checks: name: Build and Test - Checks - runs-on: ubuntu-latest - - steps: - - name: Build and Test - Checkout repository - uses: actions/checkout@v7.0.1 - - - name: Build and Test - Setup Go - uses: actions/setup-go@v7.0.0 - with: - go-version-file: go.mod - cache: true - - # First, because a duplicated package clause reads as a syntax error everywhere else and - # sends the reader looking in the wrong place. - - name: Build and Test - Check for corrupted writes - run: make check-corrupt - - - name: Build and Test - Check formatting - run: make fmt - - - name: Build and Test - Vet - run: make vet - - - name: Build and Test - Test the scripts - run: make script-test - - - name: Build and Test - Build the binary - run: make build + permissions: + contents: read + uses: ./.github/workflows/test-go.yml + with: + # check-corrupt first, because a duplicated package clause reads as a syntax error + # everywhere else and sends the reader looking in the wrong place. + targets: check-corrupt fmt vet script-test build + cache: true sqlite: name: Build and Test - Go (SQLite) - runs-on: ubuntu-latest - - steps: - - name: Build and Test - Checkout repository - uses: actions/checkout@v7.0.1 - - - name: Build and Test - Setup Go - uses: actions/setup-go@v7.0.0 - with: - go-version-file: go.mod - cache: true - - # Graphviz is a requirement: serve refuses to start without it, and the layout and figure - # tests fail rather than skip. - - name: Build and Test - Install Graphviz - run: sudo apt-get update && sudo apt-get install --yes --no-install-recommends graphviz - - - name: Build and Test - Run tests - run: make test-sqlite + permissions: + contents: read + uses: ./.github/workflows/test-go.yml + with: + targets: test-sqlite + cache: true postgres: name: Build and Test - Go (PostgreSQL) - runs-on: ubuntu-latest - - steps: - - name: Build and Test - Checkout repository - uses: actions/checkout@v7.0.1 - - - name: Build and Test - Setup Go - uses: actions/setup-go@v7.0.0 - with: - go-version-file: go.mod - cache: true - - # Docker is a requirement: the store suite runs on a PostgreSQL primary and physical standby in - # containers, and fails rather than skips without a daemon. Ubuntu runners have one. - - name: Build and Test - Run tests - run: make test-postgres + permissions: + contents: read + uses: ./.github/workflows/test-go.yml + with: + # Docker is a requirement: the store suite runs on a PostgreSQL primary and physical standby + # in containers, and fails rather than skips without a daemon. Ubuntu runners have one. + targets: test-postgres + cache: true # One name to require in branch protection, whichever jobs the run is split into. build-and-test-go: name: Build and Test - Go - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 needs: [checks, sqlite, postgres] if: always() diff --git a/.github/workflows/build-and-test-ui.yml b/.github/workflows/build-and-test-ui.yml index e514e840..9b5643d3 100644 --- a/.github/workflows/build-and-test-ui.yml +++ b/.github/workflows/build-and-test-ui.yml @@ -6,6 +6,7 @@ on: pull_request: paths: - '.github/workflows/build-and-test-ui.yml' + - 'scripts/setup-graphviz.sh' - '.node-version' - 'Makefile' # The populated run is only as good as what it runs against, and both of these decide that: @@ -28,6 +29,7 @@ on: branches: ['main'] paths: - '.github/workflows/build-and-test-ui.yml' + - 'scripts/setup-graphviz.sh' - '.node-version' - 'Makefile' - 'cmd/abom-corpus/**' @@ -55,7 +57,7 @@ jobs: # seconds rather than behind a toolchain neither of them uses. browser: name: Interface - Browser code - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Browser code - Checkout repository @@ -78,7 +80,7 @@ jobs: components: name: Interface - Components - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Components - Checkout repository @@ -110,7 +112,7 @@ jobs: # `dotnet publish` and a `go build` that together cost more than the tests it then runs. build: name: Interface - Build what the suite drives - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 outputs: specs: ${{ steps.specs.outputs.specs }} @@ -189,7 +191,7 @@ jobs: suite: name: Interface - Browser suite (${{ matrix.spec }}) needs: build - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 strategy: # Every shard reports. "Which spec files are broken" is the question a split suite exists to @@ -240,8 +242,8 @@ jobs: # Graphviz is a requirement: serve refuses to start without it, and the layout and figure # tests fail rather than skip. - - name: Browser suite - Install Graphviz - run: sudo apt-get update && sudo apt-get install --yes --no-install-recommends graphviz + - name: Browser suite - Setup Graphviz + run: scripts/setup-graphviz.sh # `-run` rather than `make e2e`: the binary, the interface and the data set were built once # and unpacked above, and rebuilding them here would be the whole saving spent again. @@ -261,7 +263,7 @@ jobs: name: Build and Test - Interface needs: [browser, build, components, suite] if: always() - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Interface - Summary diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 72ae854c..e23c4945 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -23,6 +23,7 @@ on: - 'go.mod' - 'go.sum' - 'Makefile' + - 'scripts/setup-graphviz.sh' # Pins the PostgreSQL image the setup pulls. - 'scripts/with-postgres.py' - '**/codeql-pack.yml' @@ -42,6 +43,7 @@ on: - 'go.mod' - 'go.sum' - 'Makefile' + - 'scripts/setup-graphviz.sh' - 'scripts/with-postgres.py' - '**/codeql-pack.yml' - '**/codeql-pack.lock.yml' @@ -56,7 +58,7 @@ jobs: copilot-setup-steps: permissions: contents: read - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Copilot Setup - Checkout code @@ -139,8 +141,8 @@ jobs: # Graphviz is a requirement: serve refuses to start without it, and the layout and figure # tests fail rather than skip. - - name: Copilot Setup - Install Graphviz - run: sudo apt-get update && sudo apt-get install --yes --no-install-recommends graphviz + - name: Copilot Setup - Setup Graphviz + run: scripts/setup-graphviz.sh # `make test-postgres` runs the store suite in the PostgreSQL image the harness pins, and pulls it on # first use. The probe starts that image and removes its container; the image stays cached. diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index cfd4736d..d39fb697 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -16,7 +16,7 @@ permissions: jobs: dependency-review: name: Dependency Review - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Dependency Review - Checkout repository diff --git a/.github/workflows/lint-go.yml b/.github/workflows/lint-go.yml index 65f79c64..dcb91f5e 100644 --- a/.github/workflows/lint-go.yml +++ b/.github/workflows/lint-go.yml @@ -13,7 +13,7 @@ permissions: jobs: lint-go: name: Lint - Go - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Lint - Checkout repository diff --git a/.github/workflows/query-unit-tests.yml b/.github/workflows/query-unit-tests.yml index db920614..7ddb3f00 100644 --- a/.github/workflows/query-unit-tests.yml +++ b/.github/workflows/query-unit-tests.yml @@ -36,7 +36,7 @@ jobs: # here rather than at release time. versions: name: Query Unit Tests - Versions - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Versions - Checkout repository @@ -49,7 +49,7 @@ jobs: query-unit-tests: name: Query Unit Tests - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Query Unit Tests - Checkout repository diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 1af5cebc..a199ef17 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -3,8 +3,9 @@ name: Release - Build # Called by release.yml only. Read-only: it tests the tagged commit, builds each platform's -# archive on that platform, and smoke-tests the uploaded bytes on that platform again. Nothing -# here signs or publishes, and nothing restores a cache another branch could have written. +# archive on that platform, and smoke-tests the uploaded bytes on that platform again, darwin/amd64 +# under Rosetta. Nothing here signs or publishes, and nothing restores a cache another branch could +# have written. on: workflow_call: @@ -57,8 +58,8 @@ jobs: # Graphviz is a requirement: serve refuses to start without it, and the layout and figure # tests fail rather than skip. - - name: Verify - Install Graphviz - run: sudo apt-get update && sudo apt-get install --yes --no-install-recommends graphviz + - name: Verify - Setup Graphviz + run: scripts/setup-graphviz.sh # `make verify` less lint, pins and the browser suite, which pull requests already run and # which need tools this job would otherwise install for nothing the archive contains. @@ -161,7 +162,10 @@ jobs: include: - { os: linux, arch: amd64, runner: ubuntu-22.04, codeql: linux64 } - { os: linux, arch: arm64, runner: ubuntu-22.04-arm, codeql: linux-arm64 } - - { os: darwin, arch: amd64, runner: macos-15-intel, codeql: osx64 } + # Under Rosetta on arm64: Homebrew has no Intel bottle for Graphviz or its dependencies, + # so an Intel runner would compile all of them before the smoke could start. The + # manifest check still requires the executable to report darwin/amd64. + - { os: darwin, arch: amd64, runner: macos-15, codeql: osx64 } - { os: darwin, arch: arm64, runner: macos-15, codeql: osx64 } steps: @@ -200,13 +204,17 @@ jobs: # Graphviz is a requirement the release notes name: serve refuses to start without `dot`, so # the archive is proven on a machine that has it, installed the way a user would. - - name: Smoke - Install Graphviz + - name: Smoke - Setup Graphviz + run: scripts/setup-graphviz.sh dot + + # An arm64 Mac runs the amd64 build only through Rosetta, which the image normally ships. + - name: Smoke - Install Rosetta + if: matrix.os == 'darwin' && matrix.arch == 'amd64' run: | - case "$RUNNER_OS" in - Linux) sudo apt-get update && sudo apt-get install --yes --no-install-recommends graphviz ;; - macOS) brew install graphviz ;; - *) echo "::error::no Graphviz install for $RUNNER_OS"; exit 1 ;; - esac + if ! arch -x86_64 /usr/bin/true 2>/dev/null; then + sudo softwareupdate --install-rosetta --agree-to-license + fi + arch -x86_64 /usr/bin/true - name: Smoke - Run run: | diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 08619cd1..bb23b0b2 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -54,37 +54,19 @@ jobs: exit 1 fi + # The job pull requests run, on the tree with every pack set to the release version. Read-only + # whatever this workflow is granted, and uncached: a release restores nothing another branch + # could have written. test: name: Tag - Test the tree to be tagged needs: inspect if: needs.inspect.outputs.exists == 'false' - runs-on: ubuntu-26.04 - - steps: - - name: Test - Checkout repository - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - - name: Test - Setup Go - uses: actions/setup-go@v7.0.0 - with: - go-version-file: go.mod - # A release restores nothing another branch could have written. - cache: false - - - name: Test - Set the release version - env: - VERSION: ${{ inputs.version }} - run: scripts/release-version.sh "$VERSION" - - # Graphviz is a requirement: serve refuses to start without it, and the layout and figure - # tests fail rather than skip. - - name: Test - Install Graphviz - run: sudo apt-get update && sudo apt-get install --yes --no-install-recommends graphviz - - - name: Test - Run the Go checks - run: make check-corrupt check-release-version fmt vet test + permissions: + contents: read + uses: ./.github/workflows/test-go.yml + with: + version: ${{ inputs.version }} + targets: check-corrupt check-release-version fmt vet test create: name: Tag - Create diff --git a/.github/workflows/self-audit.yml b/.github/workflows/self-audit.yml index db60dd36..a70330cc 100644 --- a/.github/workflows/self-audit.yml +++ b/.github/workflows/self-audit.yml @@ -52,7 +52,7 @@ permissions: jobs: self-audit: name: Self Audit - ${{ matrix.name }} - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 strategy: # Every language is reported, not just the first to fail. A cancelled sibling would hide @@ -169,7 +169,7 @@ jobs: # starts, under the repository's `require_lockfile` Actions policy. So this gates. action-pins: name: Self Audit - Action pins - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Action pins - Checkout repository diff --git a/.github/workflows/test-go.yml b/.github/workflows/test-go.yml new file mode 100644 index 00000000..fed7a0ee --- /dev/null +++ b/.github/workflows/test-go.yml @@ -0,0 +1,81 @@ +# This workflow is managed by gh actions-lock. + +name: Test - Go + +# The Go tests, run the same way for a pull request as for a release: build-and-test-go.yml calls +# this for each of its jobs, and release-tag.yml for the tree it is about to tag, so a release fails +# on nothing a pull request has not already run on the same image. +# +# Called, never triggered, and given no secrets. The token can only read, the checkout keeps no +# credentials, and every input reaches the shell through the environment and is checked first. + +on: + workflow_call: + inputs: + targets: + description: 'The make targets to run, separated by spaces' + required: true + type: string + version: + description: 'A release version to set every pack to first, as tagging does' + required: false + type: string + default: '' + cache: + description: 'Restore and save the Go module and build caches' + required: false + type: boolean + default: false + +permissions: + contents: read + +jobs: + test: + name: make ${{ inputs.targets }} + runs-on: ubuntu-26.04 + + steps: + - name: Test - Checkout repository + uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + + - name: Test - Setup Go + uses: actions/setup-go@v7.0.0 + with: + go-version-file: go.mod + # Off unless the caller turns it on: a release restores nothing another branch could + # have written. + cache: ${{ inputs.cache }} + + - name: Test - Set the release version + if: inputs.version != '' + env: + VERSION: ${{ inputs.version }} + run: scripts/release-version.sh "$VERSION" + + # Graphviz is a requirement: serve refuses to start without it, and the layout and figure + # tests fail rather than skip. + - name: Test - Setup Graphviz + run: scripts/setup-graphviz.sh + + # Target names only: an option or a variable assignment would change what make runs. + - name: Test - Run make + env: + TARGETS: ${{ inputs.targets }} + run: | + set -f + # shellcheck disable=SC2086 # one argument per target, with globbing off. + set -- $TARGETS + set +f + [ $# -gt 0 ] || { echo "::error::no make targets given"; exit 1; } + for target in "$@"; do + case "$target" in + -* | *[!A-Za-z0-9._-]*) + echo "::error::'$target' is not a make target" + exit 1 + ;; + esac + done + make "$@" diff --git a/.github/workflows/update-codeql.yml b/.github/workflows/update-codeql.yml index 6d6622ff..d6e92305 100644 --- a/.github/workflows/update-codeql.yml +++ b/.github/workflows/update-codeql.yml @@ -20,7 +20,7 @@ jobs: # here, for the suites that pin upstream, and for the lock files. detect: name: Detect CodeQL CLI Update - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 outputs: current: ${{ steps.check.outputs.current }} @@ -91,7 +91,7 @@ jobs: name: Check for an Existing Upgrade Branch needs: detect if: needs.detect.outputs.needed == 'true' - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 outputs: exists: ${{ steps.look.outputs.exists }} @@ -135,7 +135,7 @@ jobs: name: Upgrade and Open a Pull Request needs: [detect, existing-branch] if: needs.detect.outputs.needed == 'true' && needs.existing-branch.outputs.exists != 'true' - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 permissions: contents: write @@ -207,8 +207,8 @@ jobs: # Graphviz is a requirement: serve refuses to start without it, and the layout and figure # tests fail rather than skip. - - name: Upgrade - Install Graphviz - run: sudo apt-get update && sudo apt-get install --yes --no-install-recommends graphviz + - name: Upgrade - Setup Graphviz + run: scripts/setup-graphviz.sh - name: Upgrade - Verify run: make verify diff --git a/docs/deployment/source.md b/docs/deployment/source.md index 7dcd25f1..4c59e7cf 100644 --- a/docs/deployment/source.md +++ b/docs/deployment/source.md @@ -21,7 +21,7 @@ flowchart LR | .NET SDK | The major version named by `TargetFramework` in [the UI project](../../ui/Abom.WebAssembly.csproj). `dotnet --list-sdks` shows what is installed. No extra workloads are needed. | | GitHub CLI | [gh](https://cli.github.com/), for authentication and ownership verification. | | CodeQL | The version in [.codeql-version](../../.codeql-version), on `PATH`. See [install CodeQL](#install-codeql). | -| Graphviz | [Graphviz](https://graphviz.org/download/), with `dot` on `PATH`: `brew install graphviz` or `apt-get install graphviz`. Every details page draws its figure with it, and the Explorer's stable layout uses it. `serve` refuses to start without it and names the version it found. | +| Graphviz | [Graphviz](https://graphviz.org/download/), with `dot` on `PATH`: `brew install graphviz` or `apt-get install graphviz`. Every details page draws its figure with it, and the Explorer's stable layout uses it. `serve` refuses to start without it and names the version it found. From Ubuntu 26.04 the other layout engines, which `make test` exercises, are a separate package: `apt-get install graphviz libgvplugin-neato-layout8`. [setup-graphviz.sh](../../scripts/setup-graphviz.sh) installs and checks them as CI does. | These are the only requirements for serving and analyzing. Node, Playwright and golangci-lint are development tools. AI authoring and explicit lock verification have diff --git a/plans/tools/deployment.md b/plans/tools/deployment.md index d8eaa3a4..b826ead8 100644 --- a/plans/tools/deployment.md +++ b/plans/tools/deployment.md @@ -249,16 +249,19 @@ run in deployment environments: | Stage | Workflow and environment | What it writes | | --- | --- | --- | | Version | `release.yml` | Nothing; refuses a version that is not a release of the pinned CLI | -| Tag | `release-tag.yml`, `release-tag` | The tag, only for a dispatched version that has none, after the Go tests pass on the tree it tags. An existing tag is checked and never moved. | -| Build | `release-build.yml` | Nothing. Tests the tagged commit, builds each archive on its own platform's runner and smoke-tests the uploaded bytes there. | +| Tag | `release-tag.yml`, `release-tag` | The tag, only for a dispatched version that has none, after the Go tests pass on the tree it tags. They run in [test-go.yml](../../.github/workflows/test-go.yml), the job pull requests run. An existing tag is checked and never moved. | +| Build | `release-build.yml` | Nothing. Tests the tagged commit, builds each archive on its own platform's runner and smoke-tests the uploaded bytes there; darwin/amd64 is smoke-tested under Rosetta on arm64, because Homebrew has no Intel bottle for Graphviz. | | Draft | `release-publish.yml`, `release-draft` | Provenance attestations, then a draft release with the archives, SBOMs, `SHA256SUMS` and the attestation bundle | | Publish | `release-publish.yml`, `release` | Publishes the draft, after checking it holds exactly the staged assets | The [smoke test](../../scripts/release/smoke.sh) verifies the manifest against the extracted bytes and the executable's identity, starts the server from the archive root and reopens its database, then installs the shipped packs' dependencies and requires results from ABOM's own findings and -graph queries against the pinned CLI. Attestations name `release-publish.yml` as the signing -workflow. Artifact attestations on a private repository need GitHub Enterprise Cloud. +graph queries against the pinned CLI. Every job that needs Graphviz installs it with +[setup-graphviz.sh](../../scripts/setup-graphviz.sh), which adds the layout engines that Ubuntu +26.04 splits into a separate package and checks that each one draws. Attestations name +`release-publish.yml` as the signing workflow. Artifact attestations on a private repository need +GitHub Enterprise Cloud. The repository needs, once: the three environments, with `release-tag` limited to `main`, `release-draft` and `release` limited to `main` and `v*` tags, and required reviewers on `release` @@ -422,7 +425,10 @@ organization-wide crawl merely to validate the instructions. recorded floor with the pinned CodeQL, including the Xcode requirements on Apple silicon, before any floor appears in release documentation. Serving and analyzing need no Rosetta there, since ABOM traces no build and the Actions extractor is native; the floor run is what confirms it. The - candidates' archives are enough to start on. + candidates' archives are enough to start on. GitHub retires the `ubuntu-22.04` and + `ubuntu-22.04-arm` runner images in April 2027, after brownouts in which every job on them + fails, so the Linux archive and smoke rows need a new home first: an `ubuntu:22.04` container on + a supported runner keeps the glibc floor, and a newer image raises it. 4. **Exercise the actual download path.** Document download, provenance/checksum verification, extraction into a versioned user-writable prefix, PATH setup, configuration and startup. Use appropriate Linux/macOS verification commands and an expected repository/workflow with diff --git a/scripts/setup-graphviz.sh b/scripts/setup-graphviz.sh new file mode 100755 index 00000000..8e2d6405 --- /dev/null +++ b/scripts/setup-graphviz.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# +# Installs Graphviz with the layout engines given, and proves each one draws. +# +# setup-graphviz.sh [engine ...] default: every engine internal/layout may run +# +# Every workflow installs Graphviz through this, so a pull request installs it exactly as a release +# does. Its packaging differs by runner image: Ubuntu 26.04 splits the layout engines out of +# graphviz, and Homebrew has no Intel macOS bottle. serve itself needs only dot; the layout tests +# run every engine. +set -euo pipefail + +all="dot neato fdp sfdp twopi circo osage" +# shellcheck disable=SC2086 # split into one argument per engine. +[ $# -gt 0 ] || set -- $all + +# Each engine is then run by name, so nothing outside internal/layout's list is. +for engine in "$@"; do + case " $all " in + *" $engine "*) ;; + *) + echo "setup-graphviz: $engine is not one of: $all" >&2 + exit 2 + ;; + esac +done + +as_root() { + if [ "$(id -u)" -eq 0 ]; then + "$@" + else + sudo "$@" + fi +} + +case "$(uname -s)/$(uname -m)" in +Linux/*) + as_root apt-get update + # From Ubuntu 26.04 every engine but dot is a separate package that graphviz does not even + # recommend; before it, they ship inside graphviz's own library. + plugin=$(apt-cache pkgnames libgvplugin-neato-layout | + grep -E '^libgvplugin-neato-layout[0-9]+$' | sort -V | tail -n 1 || true) + # shellcheck disable=SC2086 # $plugin is one package name or nothing. + as_root apt-get install --yes --no-install-recommends graphviz $plugin + ;; +Darwin/arm64) + brew install graphviz + ;; +Darwin/x86_64) + # Homebrew builds no Intel bottle for Graphviz or anything it depends on, so this would compile + # that whole tree from source. An amd64 build runs on arm64 under Rosetta instead. + echo "setup-graphviz: Homebrew has no Intel macOS bottle for Graphviz; use an arm64 machine" >&2 + exit 1 + ;; +*) + echo "setup-graphviz: no Graphviz install for $(uname -s)/$(uname -m)" >&2 + exit 1 + ;; +esac + +# A missing engine is otherwise found minutes later, by a test, as a Graphviz error. +dot -V +missing="" +for engine in "$@"; do + if ! echo 'digraph { subgraph cluster_g { a } a -> b }' | "$engine" -Tplain >/dev/null; then + missing="$missing $engine" + fi +done +if [ -n "$missing" ]; then + echo "setup-graphviz: Graphviz cannot draw with:$missing" >&2 + exit 1 +fi +echo "setup-graphviz: draws with $*" diff --git a/ui/tests/e2e/package-lock.json b/ui/tests/e2e/package-lock.json index 989f5de1..e247424b 100644 --- a/ui/tests/e2e/package-lock.json +++ b/ui/tests/e2e/package-lock.json @@ -8,17 +8,20 @@ "name": "abom-e2e", "version": "0.0.0", "devDependencies": { - "@playwright/test": "1.56.1" + "@playwright/test": "1.61.1" + }, + "engines": { + "node": ">=24.0.0" } }, "node_modules/@playwright/test": { - "version": "1.56.1", - "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@playwright/test/-/test-1.56.1.tgz", - "integrity": "sha1-bjvz0MkMXPlL9kvbVv0VqAXIvT8=", + "version": "1.61.1", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@playwright/test/-/test-1.61.1.tgz", + "integrity": "sha1-SFaNwir3gZ5V+l6OO8ebfmo+ZnU=", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright": "1.56.1" + "playwright": "1.61.1" }, "bin": { "playwright": "cli.js" @@ -42,13 +45,13 @@ } }, "node_modules/playwright": { - "version": "1.56.1", - "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/playwright/-/playwright-1.56.1.tgz", - "integrity": "sha1-YuO5nd6+0NR15ZNqFSyI5ovlX78=", + "version": "1.61.1", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha1-2MDAbrk8KJga/HR7rORTvb1QGLw=", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.56.1" + "playwright-core": "1.61.1" }, "bin": { "playwright": "cli.js" @@ -61,9 +64,9 @@ } }, "node_modules/playwright-core": { - "version": "1.56.1", - "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/playwright-core/-/playwright-core-1.56.1.tgz", - "integrity": "sha1-JKZkgeXNM6BFYyIwqixPDLax2z0=", + "version": "1.61.1", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha1-PJmEEwfvu6vJ1yTEGojJFHBdFfw=", "dev": true, "license": "Apache-2.0", "bin": { diff --git a/ui/tests/e2e/package.json b/ui/tests/e2e/package.json index 16f6af27..68f2ff3c 100644 --- a/ui/tests/e2e/package.json +++ b/ui/tests/e2e/package.json @@ -11,6 +11,6 @@ "test": "playwright test" }, "devDependencies": { - "@playwright/test": "1.56.1" + "@playwright/test": "1.61.1" } } From 2c121aebc3af5964e5203767c73d2d657197ed59 Mon Sep 17 00:00:00 2001 From: Nathan Randall <70299490+data-douser@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:37:00 -0600 Subject: [PATCH 2/4] feat(abom): implement QLaaS session audit log (#58) * feat(abom): implement QLaaS session audit log Implements user interfaces for viewing QLaaS agent-session audit log while implementing safer orphan removal from settings page. Audit log - Record every agent session's hook, permission, tool, policy and measured test events (sanitized, capped at 16 KiB per detail) in SQLite and PostgreSQL. - Add a CodeQL > Audit log tab, plus audit icons on Requests and Runs rows. Search and filters apply as you type or choose. Filters show as named chips. Follow is on by default. Toolbar actions are icons with tooltips. - One card per session: time used against the time limit, Copilot AI credit cap and usage, and events and refusals. Its details are tabs inside the card. - Exports: the filtered log as JSONL, and a session's reproduction guide as Markdown. Both are refused in Demo mode. - Requests and Runs keep the column header and the opened row pinned while you scroll. - Tooltips stay inside the window, and a page left open on an old build offers Reload. Removing what you no longer own - Settings no longer opens a removal dialog by itself; you start it with Review removal. - A removal applies only to the preview that was reviewed. If anything that preview kept would now be removed, nothing changes and the current numbers are shown (409). - Removing half or more of what is in view needs explicit confirmation (`--most` on the command line). Deleting permanently asks first. - The owned-scope walk ignores letter case in repository names. - Remove the unused /api/prune/staleness endpoint and its startup check. Security and quality fixes - Escape all recorded text in the Markdown guide so it cannot form links or images. - Follow shows new events on the last page in oldest-first order. - Prune and compact report server failures as failures, not refusals. - Session lists and cards keep refreshing after a failed read, and until the session ends. - A session, request or run that does not exist reads as not recorded, not as the server being unavailable. - Find versions only the query it returns. - Resolve C# CodeQL findings. The Go, JavaScript and C# suites now report zero findings. * Address PR 58 review findings - Refuse request/run and a conflicting session on the session-scoped audit events and reproduction routes, instead of silently narrowing them - Sanitize a session's ending detail only in the store - Add a test showing a credit-limit request before the session exists does not hold up event delivery * Classify audit errors by kind, not wording Mark refused audit filters (limit, order, cursor) with store.ErrBadAuditFilter and map HTTP status from it in auditError, instead of matching substrings of the error message. --- cmd/codeql-abom/prune.go | 24 +- cmd/codeql-abom/prune_test.go | 54 ++ cmd/codeql-abom/reobserve_test.go | 4 +- cmd/codeql-abom/serve.go | 14 +- docs/architecture.md | 66 ++- docs/tutorials.md | 32 + docs/vocabulary.md | 9 +- internal/audit/sanitize.go | 185 ++++++ internal/audit/sanitize_test.go | 26 + internal/authoring/authoring.go | 41 +- internal/authoring/authoring_test.go | 23 + internal/authoring/record.go | 27 +- internal/authoring/record_test.go | 23 + internal/graph/closure.go | 40 +- internal/graph/closure_test.go | 59 ++ internal/harness/audit.go | 315 ++++++++++ internal/harness/audit_policy_test.go | 110 ++++ internal/harness/audit_test.go | 39 ++ internal/harness/confine.go | 29 +- internal/harness/copilot.go | 37 +- internal/harness/copilot_test.go | 32 + internal/harness/draft.go | 11 +- internal/harness/gate.go | 274 ++++++--- internal/harness/gate_test.go | 27 +- internal/harness/printed.go | 5 + internal/harness/session.go | 60 +- internal/serve/audit.go | 250 ++++++++ internal/serve/audit_markdown.go | 343 +++++++++++ internal/serve/audit_stream_test.go | 40 ++ internal/serve/audit_test.go | 224 +++++++ internal/serve/authoring.go | 33 -- internal/serve/authoring_test.go | 24 +- internal/serve/demo.go | 3 +- internal/serve/prune.go | 260 ++++++-- internal/serve/prune_test.go | 8 +- internal/serve/prunereview_test.go | 322 ++++++++++ internal/serve/serve.go | 16 +- internal/serve/staleness.go | 94 --- internal/serve/staleness_test.go | 172 ------ internal/store/audit.go | 553 ++++++++++++++++++ internal/store/audit_lineage_test.go | 93 +++ internal/store/audit_scale_test.go | 81 +++ internal/store/audit_test.go | 175 ++++++ internal/store/authoringsession.go | 125 +++- internal/store/authoringsession_test.go | 65 ++ internal/store/backend.go | 5 + internal/store/prune.go | 25 + internal/store/prune_test.go | 25 + internal/store/queryrequest.go | 25 +- internal/store/schema.go | 13 + internal/store/store.go | 5 + plans/next.md | 5 +- plans/ui/codeql-requests.md | 17 + ui/Models/AuditModels.cs | 114 ++++ ui/Models/Models.cs | 72 ++- ui/Pages/CodeQL.razor | 9 +- ui/Pages/Settings.razor | 88 --- ui/Program.cs | 1 - ui/Services/AbomApi.cs | 75 ++- ui/Services/AuditFilterException.cs | 4 + ui/Services/AuditSpec.cs | 60 ++ ui/Services/AuditText.cs | 94 +++ ui/Services/PromptMemory.cs | 14 - ui/Shared/AnalysisRuns.razor | 19 +- ui/Shared/AuditEvents.razor | 263 +++++++++ ui/Shared/AuditLog.razor | 252 ++++++++ ui/Shared/AuditSessions.razor | 72 +++ ui/Shared/AuditTestEvidenceView.razor | 25 + ui/Shared/PruneControl.razor | 303 ++++++++-- ui/Shared/QueryRequestForm.razor | 4 + ui/Shared/QueryRequests.razor | 9 +- ui/Shared/RequestSessions.razor | 202 ++----- ui/Shared/SessionCard.razor | 374 ++++++++++++ ui/Shared/SessionPolicy.razor | 103 ++++ ui/Shared/SessionPolicyDetails.razor | 81 +++ .../Abom.WebAssembly.Tests/AbomTestContext.cs | 46 +- .../Abom.WebAssembly.Tests/ApiControlTests.cs | 4 +- ui/tests/Abom.WebAssembly.Tests/AuditTests.cs | 216 +++++++ .../Abom.WebAssembly.Tests/AuditUxTests.cs | 387 ++++++++++++ .../Abom.WebAssembly.Tests/CodeQLTabsTests.cs | 6 +- .../CodeQLTooltipTests.cs | 62 ++ .../DiscoveryFirstRunTests.cs | 49 +- .../PruneReviewTests.cs | 277 +++++++++ .../RequestSessionsTests.cs | 84 ++- .../StalenessPromptTests.cs | 54 -- ui/tests/e2e/specs/audit.spec.mjs | 131 +++++ ui/tests/e2e/specs/authoring.spec.mjs | 8 +- ui/tests/e2e/specs/freshness.spec.mjs | 31 + ui/tests/e2e/specs/geometry.spec.mjs | 173 ++++++ ui/tests/e2e/specs/tips.spec.mjs | 58 +- ui/wwwroot/css/app.css | 294 +++++++++- ui/wwwroot/index.html | 9 + ui/wwwroot/js/build.js | 85 +++ ui/wwwroot/js/build.test.js | 135 +++++ ui/wwwroot/js/tipfit.js | 53 ++ ui/wwwroot/js/tipfit.test.js | 60 ++ 96 files changed, 8040 insertions(+), 987 deletions(-) create mode 100644 internal/audit/sanitize.go create mode 100644 internal/audit/sanitize_test.go create mode 100644 internal/harness/audit.go create mode 100644 internal/harness/audit_policy_test.go create mode 100644 internal/harness/audit_test.go create mode 100644 internal/serve/audit.go create mode 100644 internal/serve/audit_markdown.go create mode 100644 internal/serve/audit_stream_test.go create mode 100644 internal/serve/audit_test.go create mode 100644 internal/serve/prunereview_test.go delete mode 100644 internal/serve/staleness.go delete mode 100644 internal/serve/staleness_test.go create mode 100644 internal/store/audit.go create mode 100644 internal/store/audit_lineage_test.go create mode 100644 internal/store/audit_scale_test.go create mode 100644 internal/store/audit_test.go create mode 100644 ui/Models/AuditModels.cs create mode 100644 ui/Services/AuditFilterException.cs create mode 100644 ui/Services/AuditSpec.cs create mode 100644 ui/Services/AuditText.cs delete mode 100644 ui/Services/PromptMemory.cs create mode 100644 ui/Shared/AuditEvents.razor create mode 100644 ui/Shared/AuditLog.razor create mode 100644 ui/Shared/AuditSessions.razor create mode 100644 ui/Shared/AuditTestEvidenceView.razor create mode 100644 ui/Shared/SessionCard.razor create mode 100644 ui/Shared/SessionPolicy.razor create mode 100644 ui/Shared/SessionPolicyDetails.razor create mode 100644 ui/tests/Abom.WebAssembly.Tests/AuditTests.cs create mode 100644 ui/tests/Abom.WebAssembly.Tests/AuditUxTests.cs create mode 100644 ui/tests/Abom.WebAssembly.Tests/PruneReviewTests.cs delete mode 100644 ui/tests/Abom.WebAssembly.Tests/StalenessPromptTests.cs create mode 100644 ui/tests/e2e/specs/audit.spec.mjs create mode 100644 ui/tests/e2e/specs/freshness.spec.mjs create mode 100644 ui/wwwroot/js/build.js create mode 100644 ui/wwwroot/js/build.test.js create mode 100644 ui/wwwroot/js/tipfit.js create mode 100644 ui/wwwroot/js/tipfit.test.js diff --git a/cmd/codeql-abom/prune.go b/cmd/codeql-abom/prune.go index b126a5b9..26e76ae4 100644 --- a/cmd/codeql-abom/prune.go +++ b/cmd/codeql-abom/prune.go @@ -22,6 +22,8 @@ func runPrune(ctx context.Context, args []string, stdout, stderr io.Writer) erro apply := fs.Bool("apply", false, "mark what is outside the declared scope, rather than explaining it") compact := fs.Bool("compact", false, "delete what has already been marked; every open page reloads afterwards") + most := fs.Bool("most", false, + "with --apply, mark even a removal of at least half of what is in view") if err := fs.Parse(args); err != nil { return err } @@ -55,12 +57,28 @@ func runPrune(ctx context.Context, args []string, stdout, stderr io.Writer) erro } report, err := s.ExplainOutsideClosure(keep) - if *apply { - report, err = s.MarkOutsideClosure(keep) - } if err != nil { return fmt.Errorf("prune: %w", err) } + if *apply { + live, err := s.LiveGraph() + if err != nil { + return fmt.Errorf("prune: %w", err) + } + // A scope that could not be confirmed reads exactly like one that owns little, and both + // put most of the graph outside it. + if report.TakesMostOf(live) && !*most { + fmt.Fprintf(stdout, "%d repositories declared, %d reachable from them\n", + closure.Roots(), closure.Size()) + return fmt.Errorf("prune: this would mark %d of %d nodes and %d of %d edges, most of what "+ + "is in view, and nothing was changed; check what is declared and that it could be "+ + "confirmed, then re-run with --apply --most if this is what you mean", + report.Nodes, live.Nodes, report.Edges, live.Edges) + } + if report, err = s.MarkOutsideClosure(keep); err != nil { + return fmt.Errorf("prune: %w", err) + } + } verb := "would remove" if *apply { diff --git a/cmd/codeql-abom/prune_test.go b/cmd/codeql-abom/prune_test.go index c339b738..3e958ea2 100644 --- a/cmd/codeql-abom/prune_test.go +++ b/cmd/codeql-abom/prune_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "fmt" "path/filepath" "strings" "testing" @@ -58,6 +59,29 @@ func TestPruneRefusesWhenNothingIsDeclared(t *testing.T) { } } +// A removal of most of what is in view is what a scope that could not be confirmed produces — a +// listing that under-reported, an organization a `gh` check could not see — as readily as one that +// owns little. It is applied only when that is said outright. +func TestPruneRefusesToRemoveMostOfTheGraphUnlessToldTo(t *testing.T) { + db := prunableDB(t) + addOutside(t, db, 4) + before := liveNodeCount(t, db) + + var out bytes.Buffer + err := runPrune(context.Background(), []string{"--db", db, "--apply"}, &out, &out) + if err == nil || !strings.Contains(err.Error(), "--most") { + t.Fatalf("a removal of most of the graph was not refused with the way to say it: %v\n%s", err, out.String()) + } + if after := liveNodeCount(t, db); after != before { + t.Fatalf("the refused removal removed rows anyway: %d became %d", before, after) + } + + runPruneOK(t, "--db", db, "--apply", "--most") + if after := liveNodeCount(t, db); after >= before { + t.Errorf("live nodes %d -> %d; --apply --most should have marked what is outside", before, after) + } +} + // Compaction is the irreversible half, so it is its own flag rather than something --apply // escalates into. func TestCompactingIsSeparateFromMarking(t *testing.T) { @@ -111,6 +135,7 @@ func seedPrunable(t *testing.T, s *store.Store) { w := store.NewWriter(s, 0) for _, n := range []store.Node{ {Key: "octo-org/app/.github/workflows/ci.yml", Kind: "workflow", RepositoryID: mine}, + {Key: "octo-org/app/.github/workflows/release.yml", Kind: "workflow", RepositoryID: mine}, {Key: "stranger/repo/.github/workflows/ci.yml", Kind: "workflow", RepositoryID: theirs}, {Key: "actions/checkout", Kind: "action", Owner: "actions", Repo: "checkout", IsExternal: true}, {Key: "stranger/thing", Kind: "action", Owner: "stranger", Repo: "thing", IsExternal: true}, @@ -123,6 +148,9 @@ func seedPrunable(t *testing.T, s *store.Store) { {Key: "in", SrcKey: "octo-org/app/.github/workflows/ci.yml", DstKey: "actions/checkout", Kind: "step-uses-action", RepositoryID: mine, FilePath: ".github/workflows/ci.yml", StartLine: 1}, + {Key: "in-release", SrcKey: "octo-org/app/.github/workflows/release.yml", DstKey: "actions/checkout", + Kind: "step-uses-action", RepositoryID: mine, + FilePath: ".github/workflows/release.yml", StartLine: 1}, {Key: "out", SrcKey: "stranger/repo/.github/workflows/ci.yml", DstKey: "stranger/thing", Kind: "step-uses-action", RepositoryID: theirs, FilePath: ".github/workflows/ci.yml", StartLine: 1}, @@ -136,6 +164,32 @@ func seedPrunable(t *testing.T, s *store.Store) { } } +// addOutside puts n more workflows in the repository nothing owned reaches. +func addOutside(t *testing.T, db string, n int) { + t.Helper() + s, err := store.Open(db, store.Options{}) + if err != nil { + t.Fatalf("store.Open: %v", err) + } + defer s.Close() + theirs, err := s.UpsertRepository("stranger/repo", "") + if err != nil { + t.Fatalf("UpsertRepository: %v", err) + } + w := store.NewWriter(s, 0) + for i := 0; i < n; i++ { + if err := w.AddNode(store.Node{ + Key: fmt.Sprintf("stranger/repo/.github/workflows/extra-%d.yml", i), + Kind: "workflow", RepositoryID: theirs, + }); err != nil { + t.Fatalf("AddNode: %v", err) + } + } + if err := w.Flush(); err != nil { + t.Fatalf("Flush: %v", err) + } +} + func clearScope(t *testing.T, db string) { t.Helper() s, err := store.Open(db, store.Options{}) diff --git a/cmd/codeql-abom/reobserve_test.go b/cmd/codeql-abom/reobserve_test.go index 919dc3d7..ba6d7645 100644 --- a/cmd/codeql-abom/reobserve_test.go +++ b/cmd/codeql-abom/reobserve_test.go @@ -19,7 +19,9 @@ func TestAServerStartPutsNothingBackToReobserve(t *testing.T) { var out, errs bytes.Buffer prepareQueue(s, queue, &out, &errs) - reclaimAbandonedWork(s, queue, &out, &errs) + if err := reclaimAbandonedWork(s, queue, &out, &errs); err != nil { + t.Fatal(err) + } var moved int if err := s.DB().QueryRow(`SELECT COUNT(*) FROM abom_frontier diff --git a/cmd/codeql-abom/serve.go b/cmd/codeql-abom/serve.go index fd2d2683..cdf4791a 100644 --- a/cmd/codeql-abom/serve.go +++ b/cmd/codeql-abom/serve.go @@ -113,7 +113,7 @@ func prepareQueue(s store.Backend, queue *frontier.Queue, stdout, stderr io.Writ } // reclaimAbandonedWork settles claims left by a stopped server before serving. -func reclaimAbandonedWork(s store.Backend, queue *frontier.Queue, stdout, stderr io.Writer) { +func reclaimAbandonedWork(s store.Backend, queue *frontier.Queue, stdout, stderr io.Writer) error { if n, err := queue.ReclaimOrphaned(); err != nil { fmt.Fprintf(stderr, "serve: reclaiming abandoned work: %v\n", err) } else if n > 0 { @@ -131,10 +131,11 @@ func reclaimAbandonedWork(s store.Backend, queue *frontier.Queue, stdout, stderr fmt.Fprintf(stdout, "serve: %d comparisons were interrupted when the server last stopped\n", n) } if n, err := s.InterruptAuthoringSessions(); err != nil { - fmt.Fprintf(stderr, "serve: marking abandoned authoring sessions: %v\n", err) + return fmt.Errorf("serve: cannot record abandoned authoring sessions; refusing to serve: %w", err) } else if n > 0 { fmt.Fprintf(stdout, "serve: %d authoring sessions were interrupted when the server last stopped\n", n) } + return nil } // migratePolicyScopes writes retired policy scopes into predicates for this grouping key. @@ -369,7 +370,9 @@ func runServe(ctx context.Context, args []string, stdout, stderr io.Writer) (res } wireTrees(ctx, loop, cfg.apiURL, tokens) wireCodeScanning(ctx, loop, s, cfg) - primeServedStore(s, loop.Queue, stdout, stderr) + if err := primeServedStore(s, loop.Queue, stdout, stderr); err != nil { + return err + } srv, err := newConfiguredServer(ctx, cfg, s, loop, tokens, stderr) if err != nil { @@ -395,7 +398,6 @@ func runServe(ctx context.Context, args []string, stdout, stderr io.Writer) (res } writeIdentityBanner(stdout, s) - go srv.CheckStaleness() go srv.ReadUnreadDeclarations(ctx) go srv.MeterPools(ctx, poolFlushEvery) go pruneHistory(ctx, s, time.Hour, time.Now, stdout) @@ -575,7 +577,7 @@ func openServeStore(cfg serveConfig, stdout, stderr io.Writer) (store.Backend, e return s, nil } -func primeServedStore(s store.Backend, queue *frontier.Queue, stdout, stderr io.Writer) { +func primeServedStore(s store.Backend, queue *frontier.Queue, stdout, stderr io.Writer) error { if _, err := s.LinkFindingsToNodes(); err != nil { fmt.Fprintf(stderr, "serve: attributing findings: %v\n", err) } @@ -583,7 +585,7 @@ func primeServedStore(s store.Backend, queue *frontier.Queue, stdout, stderr io. fmt.Fprintf(stderr, "serve: classifying edge boundaries: %v\n", err) } prepareQueue(s, queue, stdout, stderr) - reclaimAbandonedWork(s, queue, stdout, stderr) + return reclaimAbandonedWork(s, queue, stdout, stderr) } func newConfiguredServer( diff --git a/docs/architecture.md b/docs/architecture.md index c226b33d..9eba00e0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -342,7 +342,8 @@ for the same query does not repaint it. **An agent can write the query a request asks for, inside bounds ABOM sets.** A person starts a session from the request. ABOM drives the Copilot CLI through the Copilot SDK, with ql-mcp as the -only tool server. Every tool call passes through ABOM's own callbacks before and after it runs. +only tool server. ABOM registers the pinned SDK's lifecycle hooks and separate permission callback; +the audit distinguishes their observations from runtime calls identified by an exact call ID. - Writes are confined to the target's source, help and test fixtures. - There is no shell. @@ -354,6 +355,56 @@ crawls. Its timeline, what it spent and every tree `PrintAST` printed are kept, shown on the request. What the agent says it did is never a mark. Every step is re-derived from the files and from ABOM's own records. +**The CodeQL Audit log is an evidence trail, not a transcript or an authorship verdict.** Requests +and Runs link to the same paged event reader. All ten pinned SDK hooks are registered, and the +SDK observer is installed before session creation. Registered hooks, observed hooks, permissions, +runtime events and ABOM-measured evidence remain distinct. Ordinary pre/post hooks supply no +call ID: matching a tool name and a nearby timestamp never establishes a pair or a duration. +SDK redelivery with the same event identity is idempotent. Agent idle/stop observations are not +the terminal ABOM session state. + +Before model work, the trail records an immutable effective-policy snapshot from the configuration +actually used: default/configured timeout and actual context deadline, the optional runtime AI +credit limit, model/tools, MCP timeout, measured/configured toolchain identities, and the same bounds +definitions that enforce editable files and YAML fixtures. A changed target or TDD hold produces +a new snapshot before the next permitted operation. Sealed drafting has no filesystem or CodeQL +lane. Historical policies are **not recorded**, not today's defaults and not an unlimited budget. +A policy is recorded before session creation; its presence does not prove the runtime accepted +the configuration. An unsupported credit cap can still prevent startup, with the failure recorded. +A recording failure cancels the session and refuses future work; it is a failure with incomplete +evidence, never a silently successful activity. + +`GET /api/authoring-events` combines request/run/session, explicit query/version, purpose, hook, +tool, decision, TDD step, actor, time and literal sanitized-text filters. Pages default to 100 and +are capped at 500. Counts cover all matches; filter-bound keyset cursors preserve the row-ID cutoff +while events append. Structured scopes use indexed SQL. Text, sanitized tool/actor and timestamp +filters scan the safe projection in bounded memory; a small page is not evidence of enterprise +search throughput. Session summaries at `/api/authoring-sessions/{id}` expose policy history, +observed coverage and explicit relations. A version evaluated by a run is not proof that an +agent authored it; broader same-query searches are labeled separately. A filed draft has an +explicit validated draft identity, rather than a time-based guess. + +New event details are sanitized structured JSON, bounded to 16 KiB with omission markers. +Credentials, environment, source/patch/prompt bodies and opaque successful tool bodies are +redacted or omitted; public messages are not hidden reasoning. Historical raw rows remain intact +in storage, but their old detail bodies are not republished. Free-text sanitization is conservative, +not a guarantee that arbitrary prose cannot contain a secret. No new source or patch archive exists. + +The append-only trail keeps every ABOM-measured test iteration, separately from the mutable latest +test result used by existing progress marks. A RESULT failure on marked expectations, compilation +failure, unavailable result and a model's claim remain different evidence. Safe input digests and +existing printed-tree IDs are collected before scratch cleanup; escaping paths are not followed. +Per-file observations after an operation are labeled separately from the measured test's input +digests. Missing inputs and bounded-manifest omissions are explicit. + +`/api/authoring-events/export` streams the **entire** filtered cutoff as JSONL, not just its first +page. `/api/authoring-sessions/{id}/reproduction?download=1` streams a Markdown evidence manifest. +Both have completion markers; an interrupted download is incomplete. Events, policies and coverage +use the same cutoff; current session status/non-event relations are labeled as such. Demo permits +reading but forbids these exports. A digest cannot recover source: obtain matching files and the +recorded toolchain independently before attempting manual reproduction. Nothing replays a session +automatically or changes deterministic analysis runs or security verdicts. + **Nothing makes the query's own output its expected result.** `codeql test accept` is refused at both hooks, and no person accepts a report either. The agent writes, in order, the help, a marked fixture for each example, and the `.expected` those marks call for. The query stays as it is until a @@ -553,6 +604,17 @@ listing can under-report without failing: one read without a token found 91 of 1 and said nothing. A directly recorded repository no declared account covers is treated the same way: it is outside the closure, kept until somebody accepts the prune that says what removing it takes. +The prune is asked for, never offered by itself. Settings reads what it would remove when somebody +opens the review, and applying it names that review: the server reads the scope again and, if +anything the review kept would now go, removes nothing and answers 409 with the removal as it now +stands. The scope can shrink between the two — a listing that under-reports, or a `gh` check that +cannot see an organization's membership, stops a repository counting as owned — and a removal read +against one scope and applied against a smaller one took out what had been shown as kept. A removal +of at least half of what is in view is applied only when said outright, because a scope that could +not be confirmed reads exactly like one that owns little. The closure is walked without regard to +case, as GitHub names it: a `uses:` spelled unlike the repository it resolves to stopped the walk, +and everything past it read as outside. + ## The data model Persistence is a SQLite file for a single-binary installation or a PostgreSQL database for @@ -962,7 +1024,7 @@ flowchart LR | `abom_query_request_example`, `abom_query_request_reference`, `abom_query_request_tag` | A request's examples in order, each pasted or linked — a link keeps the commit, lines and text read at save — its references (CVE, CWE, GHSA, URL), and the `@tags` it asks for | | `abom_query_request_evidence` | The run results a request was asked from: a false alarm names a finding or a result a held run kept, a missed detection a repository | | `abom_authoring_session` | An agent session somebody started on a request, or on a run's picked results to draft one. Whether it finished, was stopped, failed or was interrupted by a restart, and what it spent as the CLI reported it | -| `abom_authoring_event` | Every tool call a session made or was refused, recorded before it ran, with the decision and the reason. Append-only | +| `abom_authoring_event` | Append-only lifecycle, permission, runtime, policy and measured-test events, with safe structured capture, explicit identities and observed coverage. Events are not necessarily correlated tool calls | | `abom_authoring_ast` | What `PrintAST` printed over a query's test database, by a session or by ABOM at somebody's asking, with a digest of the fixtures it was printed from and the CodeQL that printed it. Append-only | | `abom_authoring_review` | An operator's approval or rejection of what a query reported, recorded while a person accepted reports; the server no longer writes it. Append-only | | `abom_query_run_rule_tag` | The tags each rule declared in one evaluation, so a finding matches the tags of the run that reported it | diff --git a/docs/tutorials.md b/docs/tutorials.md index dc714830..abe8a09c 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -63,3 +63,35 @@ ABOM's stricter [fail-first contract](architecture.md#what-produces-the-data) st fixture marks establish expectations before implementation. Accepting current query output as expected results is not the ABOM authoring workflow. Compilation proves syntax and types; tests and comparisons provide separate evidence about behavior. + +## Reading an agent's evidence + +Open **CodeQL → Audit log**, or the Audit log icon (a scroll) on a Request or Run row. These are +the same records; a query-only search is broader than a proven session/run relation. A tab opened +before the server was restarted keeps running the interface it loaded, so a tab added since is +not there yet; the page says so and offers **Reload**. + +1. Each agent session is one card: how long it ran against its time limit, its AI credit cap + (Copilot's, when set), what Copilot reported it used, and its events and refusals. The + refusals open in the Audit log. Its details are tabs inside the card, one at a time: its + events, the agent's last message, **Limits and permissions**, and the trees it printed. + **Not recorded** means unknown, not current defaults. Drafting and query authoring have + different capabilities. Check the session state too: a recorded policy can precede a startup + failure, including a credit cap the installed runtime does not support. +2. Expand an event to see safe arguments, its policy reference, exact correlation where supplied, + measured tests, input digests and existing printed-tree links. Totals count events, not calls. +3. Turn on **TDD and reproduction evidence** to distinguish initial measured red tests, subsequent + iterations and stopping reasons. Compilation failures and agent messages never prove red/green. +4. Search and filters apply as you type or choose; there is nothing to submit. Each shows as a + chip naming what it scopes to (a run by its queries, a request by its title, a session by its + purpose), and closing a chip removes that filter alone. Share the URL. The page arrows keep a + fixed cutoff; **Follow**, on by default, shows newer matching events as they are recorded, or + says they are waiting if you have paged away. Refresh starts a new snapshot; clearing all + filters returns to the global view. +5. Export matching JSONL for the full filtered cutoff, or a session's reproduction Markdown. + Check the final completion marker. Exports are unavailable in Demo. + +The guide is evidence for manual investigation, not automatic replay. Obtain the matching query, +pack dependencies, fixtures, expectations and toolchain independently, and compare their recorded +digests. Source/patch bodies and scratch are not archived; missing or changed inputs are a limit, +not permission to substitute current files and claim the original result. diff --git a/docs/vocabulary.md b/docs/vocabulary.md index ce6e859b..d1b7e263 100644 --- a/docs/vocabulary.md +++ b/docs/vocabulary.md @@ -199,7 +199,14 @@ evidence. Neither a lock context nor a query's source substitutes for this datab | **Selected by you** / **decided by the agent** | Who chose a request's target. Empty means the agent decides | | **False alarm** / **missed detection** | Evidence on a request: a finding that should not have fired, or a repository where nothing fired that should have | | **Session** | An agent writing the query a request asks for, started by a person. Running, finished, stopped, failed or interrupted | -| **What it did** | A session's timeline: every tool call, allowed or refused, and why | +| **Audit log** | The shared, paged record of observed agent lifecycle events, permissions, decisions and measured evidence, opened from the Audit log icon on any request or run | +| **AI credit cap** | The most AI credits Copilot lets one session spend, when one is set. With none set, Copilot's own plan limits still apply | +| **Time limit** | How long a session may run before it ends; shown beside how long it took, or has taken so far | +| **Event / call** | A recorded observation / a runtime invocation with an exact supplied identity. Ordinary SDK pre/post hooks cannot be paired just by name or time; event totals are not call totals | +| **Effective policy** | The immutable limits, tools and file/fixture restrictions captured from the configuration and bounds actually used. Later settings do not repaint an older session | +| **Not recorded** | The historical value or observation was not captured. Distinct from zero, a recorded absence of an AI credit cap, or proof no activity happened | +| **Reproduction evidence** | Ordered policy, input identities, decisions and measured test iterations. Digests and links are not source archives, and missing/changed inputs prevent exact reproduction | +| **What it did** | A request's entry into the same complete audit event reader, including uncorrelated observations and refusals | | **Refused** | A tool call that ABOM's bounds did not let run, or a run that could not start. Either says why | | **Tree** | What `PrintAST` printed over a test database, in a session or by ABOM when somebody asked. It is the evidence for the AST step while the fixtures are unchanged | | **Expected results** | The `.expected` a query's test compares against, written by the agent from the help and the fixtures' marks before the query. Never the query's own output | diff --git a/internal/audit/sanitize.go b/internal/audit/sanitize.go new file mode 100644 index 00000000..b36342bc --- /dev/null +++ b/internal/audit/sanitize.go @@ -0,0 +1,185 @@ +// Package audit defines the safe, bounded evidence shared by capture and readers. +package audit + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "net/url" + "regexp" + "sort" + "strings" + "unicode/utf8" +) + +const Version = 1 +const MaxDetail = 16 * 1024 + +var credentials = regexp.MustCompile(`(?i)(bearer\s+\S+|(?:gh[pousr]_|github_pat_)[a-zA-Z0-9_]+|(?:token|password|secret|authorization|api[_-]?key)\s*[:=]\s*["']?[^"'\s,;]+)`) +var urls = regexp.MustCompile(`https?://[^\s"'<>]+`) +var absolute = regexp.MustCompile(`(^|[\s"'(=])((?:/[A-Za-z0-9_.~-]+){2,}|[A-Za-z]:\\[^\s"'<>]+)`) + +// Text is a conservative public-text projection, not a claim that arbitrary prose is secret-free. +func Text(s string) string { + s = credentials.ReplaceAllString(s, "[redacted credential]") + s = urls.ReplaceAllStringFunc(s, func(raw string) string { + u, err := url.Parse(raw) + if err != nil { + return "[omitted URL]" + } + u.User, u.RawQuery, u.Fragment = nil, "", "" + return u.String() + }) + // URL paths are also reduced: this trail does not retain private checkout locations. + s = absolute.ReplaceAllString(s, "${1}[path]") + return bounded(s, 4096) +} + +func bounded(s string, n int) string { + if len(s) <= n { + return s + } + cut := n - len("[truncated]") + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + return s[:cut] + "[truncated]" +} + +func Digest(body []byte) string { + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]) +} + +func secretKey(k string) bool { + k = strings.ToLower(k) + return strings.Contains(k, "token") || strings.Contains(k, "secret") || + strings.Contains(k, "password") || strings.Contains(k, "authorization") || + strings.Contains(k, "credential") || k == "env" || k == "environment" || k == "api_key" +} + +func sourceKey(k string) bool { + switch strings.ToLower(k) { + case "file_text", "old_str", "new_str", "ql_code", "source", "patch", "prompt", "systemmessage", "compliant", "noncompliant", "non_compliant": + return true + } + return false +} + +func safe(v any, depth int) any { + if depth > 8 { + return "[omitted: nesting limit]" + } + switch x := v.(type) { + case nil, bool, float64, int, int64: + return v + case string: + return Text(x) + case []any: + out := make([]any, 0, min(len(x), 65)) + for i, item := range x { + if i == 64 { + out = append(out, "[truncated: array]") + break + } + out = append(out, safe(item, depth+1)) + } + return out + case map[string]any: + out := map[string]any{} + keys := make([]string, 0, len(x)) + for k := range x { + keys = append(keys, k) + } + sort.Strings(keys) + for i, k := range keys { + if i == 64 { + out["_truncated"] = "object fields" + break + } + name := bounded(Text(k), 128) + switch { + case secretKey(k): + out[name] = "[redacted]" + case sourceKey(k): + body, _ := json.Marshal(x[k]) + out[name] = map[string]any{"omitted": "source or prompt body", "bytes": len(body), "sha256": Digest(body)} + default: + out[name] = safe(x[k], depth+1) + } + } + return out + default: + return "[omitted: unsupported body]" + } +} + +func Detail(v any) string { + b, err := json.Marshal(map[string]any{"schema": Version, "value": safe(v, 0)}) + if err != nil { + return `{"schema":1,"omitted":"unsupported body"}` + } + if len(b) > MaxDetail { + return fmt.Sprintf(`{"schema":1,"truncated":true,"omitted":"detail exceeds 16 KiB","sanitizedBytes":%d}`, len(b)) + } + return string(b) +} + +func ProjectDetail(s string) string { + if s == "" { + return "" + } + var v any + if json.Unmarshal([]byte(s), &v) == nil { + if m, ok := v.(map[string]any); ok && m["schema"] == float64(Version) { + body, err := json.Marshal(safe(v, 0)) + if err == nil && len(body) <= MaxDetail { + return string(body) + } + } + return Detail(v) + } + return Detail(map[string]any{"message": s}) +} + +// Policy is normative: it is never passed through lossy detail truncation. +type Policy struct { + Version int `json:"version"` + Purpose string `json:"purpose"` + TimeoutMs int64 `json:"timeoutMs"` + TimeoutSource string `json:"timeoutSource"` + Deadline string `json:"deadline"` + CreditsKnown bool `json:"creditsKnown"` + MaxAICredits *float64 `json:"maxAiCredits"` + Model string `json:"model"` + Runtime map[string]string `json:"runtime"` + Tools []string `json:"tools"` + MCPTimeoutMs int `json:"mcpTimeoutMs"` + Bounds Bounds `json:"bounds"` + Hooks []string `json:"hooks"` + Enforcement string `json:"enforcement"` +} + +type Bounds struct { + Target string `json:"target"` + Writable map[string]string `json:"writable"` + FixtureRoot string `json:"fixtureRoot"` + FixtureExtensions []string `json:"fixtureExtensions"` + ExcludedNames []string `json:"excludedNames"` + ReadOnly []string `json:"readOnly"` + Denied []string `json:"denied"` + QueryHeld string `json:"queryHeld"` + ToolArguments map[string][]string `json:"toolArguments"` +} + +func (p Policy) JSON() (string, error) { + b, err := json.Marshal(p) + if err != nil { + return "", err + } + if len(b) > MaxDetail { + return "", fmt.Errorf("the complete effective policy exceeds 16 KiB; refusing execution") + } + return string(b), nil +} diff --git a/internal/audit/sanitize_test.go b/internal/audit/sanitize_test.go new file mode 100644 index 00000000..1caed181 --- /dev/null +++ b/internal/audit/sanitize_test.go @@ -0,0 +1,26 @@ +package audit + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestSanitizationKeepsValidBoundedJSONWithoutSourceOrCredentials(t *testing.T) { + body := Detail(map[string]any{ + "args": map[string]any{"token": "credential-canary", "url": "https://user:password@example.com/a?token=url-canary", + "file_text": "source-canary", "path": "/Users/operator/private/query.ql"}, + "result": strings.Repeat("字", 20000), + }) + if !json.Valid([]byte(body)) || len(body) > MaxDetail { + t.Fatalf("invalid or oversized detail: %d", len(body)) + } + for _, secret := range []string{"credential-canary", "url-canary", "source-canary", "password", "/Users/operator"} { + if strings.Contains(body, secret) { + t.Errorf("leaked %s", secret) + } + } + if !strings.Contains(body, "omitted") || !strings.Contains(body, "truncated") { + t.Fatalf("missing coverage markers: %s", body) + } +} diff --git a/internal/authoring/authoring.go b/internal/authoring/authoring.go index 332cb8b9..bf0feb3a 100644 --- a/internal/authoring/authoring.go +++ b/internal/authoring/authoring.go @@ -36,6 +36,18 @@ type Query struct { // Read from the files rather than from `codeql resolve`: the CLI takes the machine-wide lock, and // a listing asked for on every page load must not queue behind a crawl. func Workspace(root string) ([]Query, error) { + out, err := listed(root) + if err != nil { + return nil, err + } + if err := versioned(root, out); err != nil { + return nil, err + } + return out, nil +} + +// listed is every query under root/src with the tests that name it, not yet versioned. +func listed(root string) ([]Query, error) { tests, err := testsByQuery(root) if err != nil { return nil, err @@ -66,24 +78,32 @@ func Workspace(root string) ([]Query, error) { if err != nil && !errors.Is(err, fs.ErrNotExist) { return nil, fmt.Errorf("listing the queries in %s: %w", root, err) } + return out, nil +} - paths := make([]string, 0, len(out)) - for _, q := range out { +// versioned sets each query's version from its file and what it imports. +func versioned(root string, queries []Query) error { + paths := make([]string, 0, len(queries)) + for _, q := range queries { paths = append(paths, q.Path) } - versions, err := runner.QueryVersions(paths) + versions, err := queryVersions(paths) if err != nil { - return nil, fmt.Errorf("versioning the queries in %s: %w", root, err) + return fmt.Errorf("versioning the queries in %s: %w", root, err) } - for i := range out { - out[i].Version = versions[i] + for i := range queries { + queries[i].Version = versions[i] } - return out, nil + return nil } -// Find is the workspace query that declares id. +// queryVersions is runner.QueryVersions, named apart so a test can count what is versioned. +var queryVersions = runner.QueryVersions + +// Find is the workspace query that declares id. Only that query is versioned: Find runs for every +// event an agent session records, and the rest of the pack is not what was asked about. func Find(root, id string) (Query, bool, error) { - queries, err := Workspace(root) + queries, err := listed(root) if err != nil { return Query{}, false, err } @@ -97,6 +117,9 @@ func Find(root, id string) (Query, bool, error) { case 0: return Query{}, false, nil case 1: + if err := versioned(root, found); err != nil { + return Query{}, false, err + } return found[0], true, nil } // Refused rather than picked: nothing would say which of the two a result was about. diff --git a/internal/authoring/authoring_test.go b/internal/authoring/authoring_test.go index e4a06c31..f923c468 100644 --- a/internal/authoring/authoring_test.go +++ b/internal/authoring/authoring_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "reflect" + "slices" "strings" "testing" @@ -69,6 +70,28 @@ func TestAQueryIsFoundByTheIDItDeclares(t *testing.T) { } } +// Find runs for every event an agent session records, under the lock that orders them, so it +// versions the query it finds and nothing else. Versioning the whole pack made each tool call wait +// on every query in it: 211ms a call at 300 queries. +func TestFindingAQueryVersionsThatQueryAlone(t *testing.T) { + root := workspace(t, map[string]string{"A": "acme/a", "B": "acme/b", "C": "acme/c"}) + var versioned [][]string + before := queryVersions + queryVersions = func(paths []string) ([]string, error) { + versioned = append(versioned, slices.Clone(paths)) + return before(paths) + } + t.Cleanup(func() { queryVersions = before }) + + got, found, err := Find(root, "acme/b") + if err != nil || !found { + t.Fatalf("Find: found=%v err=%v", found, err) + } + if len(versioned) != 1 || !reflect.DeepEqual(versioned[0], []string{got.Path}) { + t.Errorf("versioned %v, want only %s", versioned, got.Path) + } +} + // One query's failing test must not fail another: a test belongs to the query its .qlref names // and to no other. func TestATestBelongsOnlyToTheQueryItsQlrefNames(t *testing.T) { diff --git a/internal/authoring/record.go b/internal/authoring/record.go index ba4654c5..1b67be93 100644 --- a/internal/authoring/record.go +++ b/internal/authoring/record.go @@ -20,12 +20,19 @@ var ErrTestsDidNotRun = errors.New("the tests could not be run") // Tests runs a query's own tests. type Tests func(ctx context.Context, tests []string) (runner.TestResult, error) +type testEvidenceKey struct{} + +// WithTestEvidence keeps measured iterations independently of the latest-test projection. +func WithTestEvidence(ctx context.Context, keep func(store.QueryTest) error) context.Context { + return context.WithValue(ctx, testEvidenceKey{}, keep) +} + // RecordTests runs a query version's own tests and records what the CLI said against the version, // its tests, its fixtures and the CodeQL that ran them. // // ABOM never forms the opinion itself: `codeql test run` compiles the query, runs it against a // database built from the fixture tree and compares the output to the checked-in `.expected`. -func RecordTests(ctx context.Context, st store.Backend, run Tests, root string, q Query, cli string) (runner.TestResult, error) { +func RecordTests(ctx context.Context, st store.Backend, run Tests, root string, q Query, cli string) (answer runner.TestResult, answerErr error) { digest, err := DigestTests(root, q.Tests) if err != nil { return runner.TestResult{}, err @@ -39,6 +46,24 @@ func RecordTests(ctx context.Context, st store.Backend, run Tests, root string, result := runner.TestResult{ Detail: fmt.Sprintf("no test names %s, so nothing was shown to work", q.Rel), } + defer func() { + keep, _ := ctx.Value(testEvidenceKey{}).(func(store.QueryTest) error) + if keep == nil { + return + } + test := store.QueryTest{QueryID: q.ID, QueryVersion: q.Version, CLIVersion: cli, + Passed: result.Passed, Total: result.Total, Failed: result.Failed, Detail: result.Detail, + FailureStage: result.FailureStage, TestDigest: digest, FixturesDigest: fixtures, + Missing: result.Missing, Unexpected: result.Unexpected} + if answerErr != nil { + test.Passed = false + test.FailureStage = "UNAVAILABLE" + test.Detail = answerErr.Error() + } + if err := keep(test); err != nil { + answerErr = err + } + }() if len(q.Tests) > 0 { if result, err = run(ctx, q.Tests); err != nil { return runner.TestResult{}, fmt.Errorf("%w: %w", ErrTestsDidNotRun, err) diff --git a/internal/authoring/record_test.go b/internal/authoring/record_test.go index 394773ed..4d6da71c 100644 --- a/internal/authoring/record_test.go +++ b/internal/authoring/record_test.go @@ -16,6 +16,7 @@ func recordingStore(t *testing.T) *store.Store { if err != nil { t.Fatal(err) } + t.Cleanup(func() { st.Close() }) if err := st.Migrate(); err != nil { t.Fatal(err) @@ -23,6 +24,28 @@ func recordingStore(t *testing.T) *store.Store { return st } +func TestEachMeasuredIterationSurvivesTheLatestTestUpsert(t *testing.T) { + root := workspace(t, map[string]string{"A": "acme/a"}) + q, _ := mustVersion(t, root, "acme/a") + st := recordingStore(t) + var evidence []store.QueryTest + ctx := WithTestEvidence(context.Background(), func(test store.QueryTest) error { + evidence = append(evidence, test) + return nil + }) + for _, passed := range []bool{false, true, false} { + run := func(context.Context, []string) (runner.TestResult, error) { + return runner.TestResult{Passed: passed, Total: 1, FailureStage: "RESULT"}, nil + } + if _, err := RecordTests(ctx, st, run, root, q, "2.27.1"); err != nil { + t.Fatal(err) + } + } + if len(evidence) != 3 || evidence[0].Passed || !evidence[1].Passed || evidence[2].Passed { + t.Fatalf("lost measured iterations: %+v", evidence) + } +} + // A recorded run is evidence only with what it ran under: the CodeQL, the tests, the fixtures and // what its expectations said. func TestARecordedRunNamesItsCodeQLFixturesAndExpectations(t *testing.T) { diff --git a/internal/graph/closure.go b/internal/graph/closure.go index 1a0e75ec..700084ff 100644 --- a/internal/graph/closure.go +++ b/internal/graph/closure.go @@ -2,6 +2,7 @@ package graph import ( "sort" + "strings" "github.com/advanced-security/codeql-abom/internal/store" ) @@ -12,18 +13,22 @@ import ( // deep it sits; a dependency of a repository nothing owned reaches is somebody else's, and // following it is how a corpus of 56,830 nodes grows from 204 repositories nobody owns. type Closure struct { - // byRepository holds every repository admitted. Absence means "not reached", which is a real - // answer rather than a missing one: the walk visits the whole graph. - byRepository map[string]bool + // byRepository holds every repository admitted, keyed in lower case with the spelling it was + // first met under. GitHub's names are not case-sensitive and a `uses:` can spell one unlike + // the row it resolves to; keyed by spelling, the walk stopped at the first difference and all + // beyond it read as outside. Absence means "not reached", which is a real answer rather than a + // missing one: the walk visits the whole graph. + byRepository map[string]string roots int } -// Contains reports whether a repository is inside the owned closure. +// Contains reports whether a repository is inside the owned closure, however it is spelled. func (c *Closure) Contains(fullName string) bool { if c == nil { return true } - return c.byRepository[fullName] + _, ok := c.byRepository[strings.ToLower(fullName)] + return ok } // Roots is how many declared repositories the walk started from. Zero means nothing is declared @@ -50,7 +55,7 @@ func (c *Closure) Repositories() []string { return nil } out := make([]string, 0, len(c.byRepository)) - for name := range c.byRepository { + for _, name := range c.byRepository { out = append(out, name) } sort.Strings(out) @@ -100,7 +105,7 @@ func (b *Builder) KeepList() ([]string, *Closure, error) { // ClosureFrom walks outward from an explicit root set, so a declaration that has not been stored // can be measured before anyone commits to it. func (b *Builder) ClosureFrom(roots []string) (*Closure, error) { - out := &Closure{byRepository: map[string]bool{}, roots: len(roots)} + out := &Closure{byRepository: map[string]string{}, roots: len(roots)} // Edges by containing repository to the repository each target belongs to. The target's // owner and repo come from the node rather than its repository_row_id, which a node first @@ -112,25 +117,26 @@ func (b *Builder) ClosureFrom(roots []string) (*Closure, error) { dependsOn := map[string][]string{} for _, d := range rows { - dependsOn[d.From] = append(dependsOn[d.From], d.To) + from := strings.ToLower(d.From) + dependsOn[from] = append(dependsOn[from], d.To) } queue := make([]string, 0, len(roots)) - for _, r := range roots { - if !out.byRepository[r] { - out.byRepository[r] = true - queue = append(queue, r) + admit := func(name string) { + key := strings.ToLower(name) + if _, seen := out.byRepository[key]; !seen { + out.byRepository[key] = name + queue = append(queue, key) } } + for _, r := range roots { + admit(r) + } for len(queue) > 0 { from := queue[0] queue = queue[1:] for _, to := range dependsOn[from] { - if out.byRepository[to] { - continue - } - out.byRepository[to] = true - queue = append(queue, to) + admit(to) } } return out, nil diff --git a/internal/graph/closure_test.go b/internal/graph/closure_test.go index b56e8c02..f44ed3e5 100644 --- a/internal/graph/closure_test.go +++ b/internal/graph/closure_test.go @@ -116,3 +116,62 @@ func declareOwned(t *testing.T, s *store.Store, owners ...string) { t.Fatalf("SetOwnedScope: %v", err) } } + +// GitHub names are case-insensitive, and a `uses:` can spell a repository differently from the +// row it resolves to. The walk keyed both by their exact spelling, so it stopped at the first +// difference, and everything beyond it read as outside the closure — where a prune takes it. +func TestADependencySpelledInAnotherCaseIsStillReachedAndKept(t *testing.T) { + s := closureFixture(t) + declareOwned(t, s, "octo-org") + app, err := s.UpsertRepository("octo-org/app", "") + if err != nil { + t.Fatalf("UpsertRepository: %v", err) + } + upstream, err := s.UpsertRepository("Up-Stream/Lib", "") + if err != nil { + t.Fatalf("UpsertRepository: %v", err) + } + w := store.NewWriter(s, 0) + for _, n := range []store.Node{ + {Key: "octo-org/app/.github/workflows/spelled.yml", Kind: "workflow", RepositoryID: app}, + {Key: "up-stream/lib", Kind: "action", Owner: "up-stream", Repo: "lib", IsExternal: true}, + {Key: "Up-Stream/Lib/.github/workflows/ci.yml", Kind: "workflow", RepositoryID: upstream}, + {Key: "deep/dep", Kind: "action", Owner: "deep", Repo: "dep", IsExternal: true}, + } { + if err := w.AddNode(n); err != nil { + t.Fatalf("AddNode(%s): %v", n.Key, err) + } + } + for _, e := range []store.Edge{ + {Key: "spelled-1", SrcKey: "octo-org/app/.github/workflows/spelled.yml", DstKey: "up-stream/lib", + Kind: "step-uses-action", RepositoryID: app, FilePath: ".github/workflows/spelled.yml", StartLine: 1}, + {Key: "spelled-2", SrcKey: "Up-Stream/Lib/.github/workflows/ci.yml", DstKey: "deep/dep", + Kind: "step-uses-action", RepositoryID: upstream, FilePath: ".github/workflows/ci.yml", StartLine: 1}, + } { + if err := w.AddEdge(e); err != nil { + t.Fatalf("AddEdge(%s): %v", e.Key, err) + } + } + if err := w.Flush(); err != nil { + t.Fatalf("Flush: %v", err) + } + + c := closureOf(t, s) + if !c.Contains("Up-Stream/Lib") { + t.Error("the repository a dependency names in another case is outside the closure") + } + if !c.Contains("deep/dep") { + t.Error("the walk stopped at a repository spelled in another case") + } + keep, _, err := (&Builder{Store: s}).KeepList() + if err != nil { + t.Fatalf("KeepList: %v", err) + } + found := false + for _, name := range keep { + found = found || name == "Up-Stream/Lib" + } + if !found { + t.Errorf("the kept list %v drops Up-Stream/Lib, so a prune would take what it holds", keep) + } +} diff --git a/internal/harness/audit.go b/internal/harness/audit.go new file mode 100644 index 00000000..dc227706 --- /dev/null +++ b/internal/harness/audit.go @@ -0,0 +1,315 @@ +package harness + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "os" + "path/filepath" + "reflect" + "runtime" + "runtime/debug" + "slices" + "strings" + "time" + + "github.com/advanced-security/codeql-abom/internal/audit" + "github.com/advanced-security/codeql-abom/internal/authoring" + "github.com/advanced-security/codeql-abom/internal/store" + copilot "github.com/github/copilot-sdk/go" +) + +var hookNames = []string{HookPreToolUse, HookPermission, HookPostToolUse, HookPostToolUseFailure, + HookSessionStart, HookSessionEnd, HookError, "prompt-submitted", "prompt-transformed", "agent-stop", "pre-mcp-tool-call"} + +func (s *Sessions) timeoutSource() string { + if s.Limit <= 0 { + return "default" + } + return "configured" +} + +func (s *Sessions) finishAudit(l *live, state, detail string) { + e := Event{Source: "abom", Hook: "abom-session-end", Tool: "session", Decision: state, Reason: audit.Text(detail)} + if err := s.finalInputs(l, &e); err != nil { + _ = l.gate.fail(err) + return + } + _ = l.gate.keep(e, Verdict{}) +} + +func (s *Sessions) finalInputs(l *live, e *Event) error { + b := l.gate.Bounds() + if !b.bound() { + return nil + } + meta, err := s.Store.AuditSession(l.session) + if err != nil || meta.RequestID == 0 { + return err + } + id, _, err := s.observedVersion(meta.RequestID) + if err != nil { + return err + } + q, found, _ := authoring.Find(s.Workspace, id) + if found { + e.QueryID, e.QueryVersion = q.ID, q.Version + e.Capture.Inputs = s.inputDigests(b, q) + } + return nil +} + +func (s *Sessions) inputDigests(b Bounds, q authoring.Query) map[string]string { + out := map[string]string{"queryVersion": q.Version, "capturedWhen": "per-file observations after the operation; measured test input digests are recorded separately"} + paths := []string{"qlpack.yml", "codeql-pack.yml", "codeql-pack.lock.yml"} + for path := range b.writableFiles() { + paths = append(paths, path) + } + err := filepath.WalkDir(b.Tests(), func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() && strings.HasSuffix(entry.Name(), ".testproj") { + return filepath.SkipDir + } + if entry.IsDir() || !slices.Contains(fixtureExts, filepath.Ext(path)) { + return nil + } + if len(paths) >= 48 { + out["omission"] = "additional fixture paths omitted: bounded manifest" + return fs.SkipAll + } + paths = append(paths, path) + return nil + }) + if err != nil { + out["fixtures"] = "unavailable: fixture tree could not be enumerated" + } + slices.Sort(paths) + for _, path := range paths { + out[filepath.ToSlash(b.rel(path))] = permittedInputDigest(b, path) + } + out["sourceAvailability"] = "digests only; files are not archived and may have changed" + return out +} + +func permittedInputDigest(b Bounds, path string) string { + real, err := b.resolve(path) + if err != nil || b.readable(real) != nil || !within(b.Workspace(), real) { + return "unavailable: outside permitted roots or unresolved path" + } + f, err := os.Open(real) + if err != nil { + return "unavailable: input not readable" + } + defer f.Close() + info, err := f.Stat() + if err != nil || !info.Mode().IsRegular() { + return "unavailable: not a regular input file" + } + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "unavailable: input not readable" + } + return fmt.Sprintf("sha256:%x", h.Sum(nil)) +} + +func (g *Gate) fail(err error) error { + g.mu.Lock() + if g.fatal == nil { + g.fatal = errors.New("audit recording failed; evidence is incomplete and this session cannot continue") + } + fatal, cancel := g.fatal, g.cancel + g.mu.Unlock() + slog.Error("authoring audit recording failed", "error", audit.Text(err.Error())) + if cancel != nil { + cancel() + } + return fatal +} + +func (g *Gate) Err() error { + g.mu.Lock() + defer g.mu.Unlock() + return g.fatal +} + +func (g *Gate) PromptSubmitted(in copilot.UserPromptSubmittedHookInput, _ copilot.HookInvocation) (*copilot.UserPromptSubmittedHookOutput, error) { + err := g.keep(Event{At: in.Timestamp, Hook: "prompt-submitted", Tool: "session", Decision: DecisionNoted, + Capture: store.AuditCapture{RuntimeSessionID: in.SessionID}, Detail: detail(map[string]any{"prompt": in.Prompt})}, Verdict{}) + return nil, err +} + +func (g *Gate) PromptTransformed(in copilot.UserPromptTransformedHookInput, _ copilot.HookInvocation) (*copilot.UserPromptTransformedHookOutput, error) { + err := g.keep(Event{At: in.Timestamp, Hook: "prompt-transformed", Tool: "session", Decision: DecisionNoted, + Capture: store.AuditCapture{RuntimeSessionID: in.SessionID}, Detail: detail(map[string]any{"prompt": in.TransformedPrompt})}, Verdict{}) + return nil, err +} + +func (g *Gate) AgentStop(in copilot.AgentStopHookInput, _ copilot.HookInvocation) (*copilot.AgentStopHookOutput, error) { + err := g.keep(Event{At: in.Timestamp, Hook: "agent-stop", Tool: "session", Decision: DecisionNoted, + Capture: store.AuditCapture{RuntimeSessionID: in.SessionID}, Reason: in.StopReason, + Detail: detail(map[string]any{"message": "An agent-stop observation is not the end of the ABOM session. Transcript not read."})}, Verdict{}) + return nil, err +} + +func (g *Gate) PreMCP(in copilot.PreMCPToolCallHookInput, _ copilot.HookInvocation) (*copilot.PreMCPToolCallHookOutput, error) { + err := g.keep(Event{At: in.Timestamp, Hook: "pre-mcp-tool-call", Tool: in.ToolName, Decision: DecisionNoted, + Capture: store.AuditCapture{RuntimeSessionID: in.SessionID, CallID: in.ToolCallID}, Detail: detail(in.Arguments)}, Verdict{}) + return nil, err +} + +func (b Bounds) description(sealed bool) audit.Bounds { + d := audit.Bounds{Writable: map[string]string{}, FixtureExtensions: []string{}, ExcludedNames: []string{}, + ReadOnly: []string{"pack (except the explicitly editable files)"}, Denied: []string{}, + ToolArguments: map[string][]string{}} + for i := range b.libraries { + d.ReadOnly = append(d.ReadOnly, fmt.Sprintf("read-root-%d", i+1)) + } + for i := range b.denied { + d.Denied = append(d.Denied, fmt.Sprintf("protected-root-%d", i+1)) + } + if sealed { + d.QueryHeld = "sealed drafting: no filesystem edits, shell, MCP or CodeQL tools" + return d + } + for name, tool := range loopTools { + fields := []string{} + for arg, check := range tool.args { + fn := runtime.FuncForPC(reflect.ValueOf(check).Pointer()).Name() + fn = fn[strings.LastIndex(fn, ".")+1:] + required := "optional" + if slices.Contains(tool.required, arg) { + required = "required" + } + fields = append(fields, arg+": "+fn+" ("+required+")") + } + slices.Sort(fields) + d.ToolArguments[Server+"-"+name] = fields + } + if !b.bound() { + d.QueryHeld = "target not selected; no files are writable" + return d + } + d.Target, d.FixtureRoot = filepath.ToSlash(b.rel(b.query)), filepath.ToSlash(b.rel(b.tests)) + for path, step := range b.writableFiles() { + if !b.isDenied(path) { + d.Writable[filepath.ToSlash(b.rel(path))] = step + } + } + d.FixtureExtensions = slices.Clone(fixtureExts) + d.ExcludedNames = append(slices.Clone(manifests), ".expected (except target)", ".qlref", ".ql", ".qll", ".actual", "test database", "protected roots", "symlinks outside permitted roots") + if b.firstChange != nil { + d.QueryHeld = audit.Text(b.firstChange()) + } + return d +} + +func (g *Gate) startPolicy(ctx context.Context, s Setup, cfg *copilot.SessionConfig, c *Copilot) error { + p := audit.Policy{Version: audit.Version, Purpose: "authoring", TimeoutMs: s.Timeout.Milliseconds(), + TimeoutSource: s.TimeoutSource, Runtime: map[string]string{}, Tools: []string{}, Hooks: slices.Clone(hookNames), + Enforcement: "The session ends at its deadline. Copilot enforces an AI credit cap when one is set. Each CodeQL tool call has its own timeout. Every tool call is still checked against the editable files and its argument rules."} + if s.Sealed { + p.Purpose = "sealed drafting" + } + if deadline, ok := ctx.Deadline(); ok { + p.Deadline = deadline.UTC().Format(time.RFC3339Nano) + } + if info, ok := debug.ReadBuildInfo(); ok { + p.Runtime["abom"] = info.Main.Version + for _, dep := range info.Deps { + if dep.Path == "github.com/github/copilot-sdk/go" { + p.Runtime["sdk"] = dep.Version + } + } + } + if cfg != nil { + p.Tools = slices.Clone(cfg.AvailableTools) + p.Model = cfg.Model + p.CreditsKnown = true + if cfg.SessionLimits != nil { + p.MaxAICredits = cfg.SessionLimits.MaxAiCredits + } + if m, ok := cfg.MCPServers[Server].(copilot.MCPStdioServerConfig); ok { + p.MCPTimeoutMs = m.Timeout + } + } + if c != nil { + p.Runtime["ql-mcp configured command"] = audit.Text(strings.Join(c.QLMCP, " ")) + } + for key, value := range s.Runtime { + p.Runtime[key] = audit.Text(value) + } + g.mu.Lock() + g.policy = &p + g.mu.Unlock() + return g.refreshPolicy() +} + +func (g *Gate) refreshPolicy() error { + g.recordMu.Lock() + defer g.recordMu.Unlock() + return g.refreshPolicyLocked() +} + +func (g *Gate) refreshPolicyLocked() error { + g.mu.Lock() + if g.policy == nil { + g.mu.Unlock() + return nil + } + + p, previous := *g.policy, g.policyJSON + b := g.bounds + g.mu.Unlock() + p.Bounds = b.description(p.Purpose == "sealed drafting") + body, err := p.JSON() + if err != nil { + return g.fail(err) + } + if previous == body { + return nil + } + // Persist the whole normative description before allowing the next operation. + e := Event{At: g.Now(), Source: "abom", Hook: "policy", Tool: "session", Decision: DecisionNoted, + Capture: store.AuditCapture{Version: audit.Version, Policy: &p, Correlation: "ABOM policy snapshot"}} + if err := g.record(e); err != nil { + return g.fail(err) + } + g.mu.Lock() + g.policyJSON = body + g.mu.Unlock() + return nil +} + +func (g *Gate) guardedTools(tools []copilot.Tool) []copilot.Tool { + out := slices.Clone(tools) + for i := range out { + handle, name := out[i].Handler, out[i].Name + out[i].Handler = func(in copilot.ToolInvocation) (copilot.ToolResult, error) { + e := Event{Source: "abom", Hook: "custom-tool-start", Tool: name, Decision: DecisionAllowed, + Detail: detail(in.Arguments), Capture: store.AuditCapture{RuntimeSessionID: in.SessionID, CallID: in.ToolCallID}} + if err := g.keep(e, Verdict{}); err != nil { + return copilot.ToolResult{}, err + } + start := g.Now() + result, err := handle(in) + e.Hook, e.Decision, e.Detail = "custom-tool-complete", DecisionSucceeded, detail(map[string]any{"resultType": result.ResultType, "omitted": "opaque tool output"}) + if err != nil || result.ResultType == "failure" { + e.Decision = DecisionFailed + } + if in.ToolCallID != "" { + e.Took = g.Now().Sub(start) + } + if recordErr := g.keep(e, Verdict{}); recordErr != nil { + return copilot.ToolResult{}, recordErr + } + return result, err + } + } + return out +} diff --git a/internal/harness/audit_policy_test.go b/internal/harness/audit_policy_test.go new file mode 100644 index 00000000..fcec5849 --- /dev/null +++ b/internal/harness/audit_policy_test.go @@ -0,0 +1,110 @@ +package harness + +import ( + "context" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/advanced-security/codeql-abom/internal/authoring" + copilot "github.com/github/copilot-sdk/go" +) + +func TestFinalInputsIdentifyFixturesWithoutFollowingAnEscapingLink(t *testing.T) { + g, _, _ := newGate(t) + b := g.Bounds() + if err := os.MkdirAll(b.Tests(), 0755); err != nil { + t.Fatal(err) + } + fixture := filepath.Join(b.Tests(), "audit-fixture.yml") + if err := os.WriteFile(fixture, []byte("on: push\n"), 0600); err != nil { + t.Fatal(err) + } + outside := filepath.Join(t.TempDir(), "outside.yml") + if err := os.WriteFile(outside, []byte("private canary"), 0600); err != nil { + t.Fatal(err) + } + link := filepath.Join(b.Tests(), "escape.yml") + if err := os.Symlink(outside, link); err != nil { + t.Fatal(err) + } + s := &Sessions{} + digests := s.inputDigests(b, authoring.Query{Version: "v"}) + if !strings.HasPrefix(digests[b.rel(fixture)], "sha256:") { + t.Fatal("fixture digest missing") + } + if !strings.Contains(digests[b.rel(link)], "unavailable") { + t.Fatal("escaping fixture not identified as unavailable") + } +} + +func TestPolicyUsesTheExecutionToolsBoundsCreditsAndActualDeadline(t *testing.T) { + g, j, p := newGate(t) + c := &Copilot{Model: "test-model", Credits: 12, QLMCP: []string{"npx", "codeql-development-mcp-server@2.27.2"}} + ctx, cancel := context.WithTimeout(context.Background(), DefaultLimit) + defer cancel() + setup := Setup{Gate: g, Workspace: p.ws, Timeout: DefaultLimit, TimeoutSource: "default"} + cfg := c.config(setup) + if err := g.startPolicy(ctx, setup, cfg, c); err != nil { + t.Fatal(err) + } + policy := j.all()[0].Capture.Policy + deadline, _ := ctx.Deadline() + if policy == nil || policy.Deadline != deadline.UTC().Format(time.RFC3339Nano) || policy.TimeoutMs != DefaultLimit.Milliseconds() || + policy.MaxAICredits == nil || *policy.MaxAICredits != 12 || !slices.Equal(policy.Tools, cfg.AvailableTools) { + t.Fatalf("policy differs from actual execution: %+v", policy) + } + b := g.Bounds() + for path := range b.writableFiles() { + if _, ok := policy.Bounds.Writable[b.rel(path)]; !ok { + t.Errorf("missing exact file %s", path) + } + } + if !slices.Equal(policy.Bounds.FixtureExtensions, fixtureExts) { + t.Fatal("fixture restrictions drifted") + } + old, _ := policy.JSON() + c.Credits = 99 + after, _ := policy.JSON() + if old != after { + t.Fatal("later settings repainted the policy") + } +} + +func TestSealedPolicyHasNoFilesystemOrCodeQLLane(t *testing.T) { + g, j, p := newGate(t) + c := &Copilot{Model: "draft"} + setup := Setup{Gate: g, Workspace: p.ws, Sealed: true, Timeout: time.Minute, TimeoutSource: "configured"} + if err := g.startPolicy(context.Background(), setup, c.config(setup), c); err != nil { + t.Fatal(err) + } + policy := j.all()[0].Capture.Policy + if policy.Purpose != "sealed drafting" || len(policy.Tools) != 0 || len(policy.Bounds.Writable) != 0 || !policy.CreditsKnown || policy.MaxAICredits != nil { + t.Fatalf("sealed policy widened execution: %+v", policy) + } +} + +func TestSDKIdentityDoesNotPairConcurrentHooksOrIngestReasoning(t *testing.T) { + g, j, _ := newGate(t) + for _, id := range []string{"a", "b"} { + g.Observe(copilot.SessionEvent{ID: "start-" + id, Data: &copilot.ToolExecutionStartData{ToolName: "view", ToolCallID: id}}) + } + _, _ = g.PostToolUse(copilot.PostToolUseHookInput{ToolName: "view", ToolResult: "private-source-canary"}, copilot.HookInvocation{}) + if len(g.InFlight()) != 2 { + t.Fatal("an ambiguous post hook consumed a runtime flight") + } + g.Observe(copilot.SessionEvent{ID: "reasoning", Data: &copilot.AssistantReasoningData{Content: "hidden-reasoning-canary"}}) + g.Observe(copilot.SessionEvent{ID: "done-b", Data: &copilot.ToolExecutionCompleteData{ToolCallID: "b", Success: true}}) + g.Observe(copilot.SessionEvent{ID: "done-b", Data: &copilot.ToolExecutionCompleteData{ToolCallID: "b", Success: true}}) + if len(g.InFlight()) != 1 || g.InFlight()[0].CallID != "a" || len(j.all()) != 4 { + t.Fatalf("wrong pairing or dedup: %+v", j.all()) + } + for _, e := range j.all() { + if strings.Contains(e.Detail, "canary") { + t.Fatal("source or hidden reasoning entered the audit") + } + } +} diff --git a/internal/harness/audit_test.go b/internal/harness/audit_test.go new file mode 100644 index 00000000..abe9812d --- /dev/null +++ b/internal/harness/audit_test.go @@ -0,0 +1,39 @@ +package harness + +import ( + "errors" + "testing" + + copilot "github.com/github/copilot-sdk/go" +) + +func TestAllPinnedHooksAreRegisteredAndPostRecordingFailureIsFatal(t *testing.T) { + g, j, p := newGate(t) + h := g.Hooks() + if h.OnUserPromptSubmitted == nil || h.OnUserPromptTransformed == nil || h.OnAgentStop == nil || h.OnPreMCPToolCall == nil { + t.Fatal("four pinned callbacks are missing") + } + _, _ = h.OnUserPromptSubmitted(copilot.UserPromptSubmittedHookInput{Prompt: "request"}, copilot.HookInvocation{}) + _, _ = h.OnUserPromptTransformed(copilot.UserPromptTransformedHookInput{Prompt: "request"}, copilot.HookInvocation{}) + _, _ = h.OnAgentStop(copilot.AgentStopHookInput{StopReason: "idle"}, copilot.HookInvocation{}) + _, _ = h.OnPreMCPToolCall(copilot.PreMCPToolCallHookInput{ToolName: "codeql_test_run", ToolCallID: "exact"}, copilot.HookInvocation{}) + if len(j.all()) != 4 { + t.Fatalf("lost callback: %+v", j.all()) + } + j.fail = errors.New("disk unavailable") + _, _ = h.OnPostToolUse(copilot.PostToolUseHookInput{ToolName: "view"}, copilot.HookInvocation{}) + j.fail = nil + answer, _ := h.OnPreToolUse(copilot.PreToolUseHookInput{ToolName: "view", ToolArgs: map[string]any{"path": p.ws}}, copilot.HookInvocation{}) + if answer == nil || answer.PermissionDecision != "deny" { + t.Fatal("recording recovered into unrecorded success") + } +} + +func TestSessionConfigObservesCreationBeforeTheSessionExists(t *testing.T) { + g, _, p := newGate(t) + c := &Copilot{Model: "test"} + cfg := c.config(Setup{Gate: g, Workspace: p.root}) + if cfg.OnEvent == nil { + t.Fatal("creation-time SDK observations are lost") + } +} diff --git a/internal/harness/confine.go b/internal/harness/confine.go index 80dda2e0..fa24eac4 100644 --- a/internal/harness/confine.go +++ b/internal/harness/confine.go @@ -266,21 +266,17 @@ func (b Bounds) writable(path string) (string, error) { if err != nil { return "", err } - dir, stem := filepath.Dir(b.query), b.stem() - switch { - case b.isDenied(p): - return "", fmt.Errorf("%s is ABOM's own, which no session writes", path) - case p == b.query: - if b.firstChange != nil { + if step, ok := b.writableFiles()[p]; ok && !b.isDenied(p) { + if p == b.query && b.firstChange != nil { if why := b.firstChange(); why != "" { return "", errors.New(why) } } - return "", nil - case p == filepath.Join(dir, stem+".md") || p == filepath.Join(dir, stem+".qhelp"): - return StepHelp, nil - case p == filepath.Join(b.tests, stem+".expected"): - return StepExpected, nil + return step, nil + } + switch { + case b.isDenied(p): + return "", fmt.Errorf("%s is ABOM's own, which no session writes", path) case !inside(b.tests, p): return "", fmt.Errorf("a session writes %s, its help beside it and fixtures under %s, not %s", b.rel(b.query), b.rel(b.tests), b.rel(p)) @@ -291,7 +287,7 @@ func (b Bounds) writable(path string) (string, error) { switch ext := filepath.Ext(name); { case slices.Contains([]string{".expected", ".qlref", ".ql", ".qll", ".actual"}, ext): return "", fmt.Errorf("%s decides what the test runs, and only ABOM and the CLI write it; the "+ - "target's own is %s", name, b.rel(filepath.Join(b.tests, stem+".expected"))) + "target's own is %s", name, b.rel(filepath.Join(b.tests, b.stem()+".expected"))) case slices.Contains(manifests, name): return "", fmt.Errorf("%s is a pack manifest, and one under a test directory changes how "+ "every test in it resolves", name) @@ -302,6 +298,15 @@ func (b Bounds) writable(path string) (string, error) { return "", nil } +func (b Bounds) writableFiles() map[string]string { + if !b.bound() { + return map[string]string{} + } + dir, stem := filepath.Dir(b.query), b.stem() + return map[string]string{b.query: "", filepath.Join(dir, stem+".md"): StepHelp, + filepath.Join(dir, stem+".qhelp"): StepHelp, filepath.Join(b.tests, stem+".expected"): StepExpected} +} + // Call decides a tool call as the agent makes it, before anything runs. func (b Bounds) Call(tool string, args any) Verdict { fields, _ := args.(map[string]any) diff --git a/internal/harness/copilot.go b/internal/harness/copilot.go index 5cf1c78c..889db053 100644 --- a/internal/harness/copilot.go +++ b/internal/harness/copilot.go @@ -20,6 +20,9 @@ import ( // Setup is everything one session starts with. type Setup struct { + Timeout time.Duration + TimeoutSource string + Runtime map[string]string // Home is where the CLI keeps the session's state; Scratch is where ql-mcp writes what CodeQL // printed. Both are scrubbed when the session ends. Home string @@ -97,19 +100,34 @@ func (c *Copilot) Run(ctx context.Context, s Setup) (string, error) { } defer func() { _ = client.Stop() }() - session, err := client.CreateSession(ctx, c.config(s)) - if err != nil { - return "", fmt.Errorf("creating the session: %w", err) + if status, err := client.GetStatus(ctx); err == nil { + if s.Runtime == nil { + s.Runtime = map[string]string{} + } + s.Runtime["copilot-cli"] = status.Version } - defer func() { _ = session.Disconnect() }() - defer session.On(s.Gate.Observe)() + cfg := c.config(s) + if err := s.Gate.startPolicy(ctx, s, cfg, c); err != nil { + return "", err + } + var session *copilot.Session + ready := make(chan struct{}) limit := &creditLimit{answer: func(r rpc.UIHandlePendingSessionLimitsExhaustedRequest) { + <-ready + if session == nil { + return + } answer, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _, _ = session.RPC.UI.HandlePendingSessionLimitsExhausted(answer, &r) }} - defer session.On(limit.observe)() - + cfg.OnEvent = func(e copilot.SessionEvent) { s.Gate.Observe(e); limit.observe(e) } + session, err = client.CreateSession(ctx, cfg) + close(ready) + if err != nil { + return "", fmt.Errorf("creating the session: %w", err) + } + defer func() { _ = session.Disconnect() }() var last string for send := s.Prompt; send != ""; { reply, err := session.SendAndWait(ctx, copilot.MessageOptions{Prompt: send}) @@ -118,6 +136,9 @@ func (c *Copilot) Run(ctx context.Context, s Setup) (string, error) { abort, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _ = session.Abort(abort) + if err := s.Gate.Err(); err != nil { + return "", err + } return "", ctx.Err() } // Anything more sent past the limit is refused in turn, so the session ends here. @@ -190,6 +211,7 @@ func (c *Copilot) config(s Setup) *copilot.SessionConfig { }, OnPermissionRequest: s.Gate.Permission, Hooks: s.Gate.Hooks(), + OnEvent: s.Gate.Observe, EnableConfigDiscovery: copilot.Bool(false), SkipCustomInstructions: copilot.Bool(true), EnableSkills: copilot.Bool(false), @@ -207,6 +229,7 @@ func (c *Copilot) config(s Setup) *copilot.SessionConfig { if s.Sealed { cfg.MCPServers = nil } + cfg.Tools = s.Gate.guardedTools(cfg.Tools) return cfg } diff --git a/internal/harness/copilot_test.go b/internal/harness/copilot_test.go index 7da12141..1e106436 100644 --- a/internal/harness/copilot_test.go +++ b/internal/harness/copilot_test.go @@ -190,6 +190,38 @@ func TestASessionAtItsCreditLimitDeclinesToSpendMoreAndRemembersWhatItSpent(t *t } } +// Copilot CLI may deliver the limit request while the session is still being created. Answering it +// must wait for the session without holding up event delivery, or creating the session could wait +// on an event handler that is itself waiting on the session. +func TestALimitRequestBeforeTheSessionExistsDoesNotHoldUpEventDelivery(t *testing.T) { + ready := make(chan struct{}) + answered := make(chan struct{}) + limit := &creditLimit{answer: func(rpc.UIHandlePendingSessionLimitsExhaustedRequest) { + <-ready + close(answered) + }} + delivered := make(chan struct{}) + go func() { + limit.observe(copilot.SessionEvent{Data: &copilot.SessionLimitsExhaustedRequestedData{ + RequestID: "r1", MaxAiCredits: 30, UsedAiCredits: 31}}) + close(delivered) + }() + select { + case <-delivered: + case <-time.After(5 * time.Second): + t.Fatal("event delivery waited on a session that did not yet exist") + } + if limit.reached() == nil { + t.Error("the limit was not remembered before the session existed") + } + close(ready) + select { + case <-answered: + case <-time.After(5 * time.Second): + t.Fatal("the request was never answered once the session existed") + } +} + // Copilot CLI 1.0.88 asks nothing of a session like this one: it blocks the model call, says so in // a limits status the SDK does not type, and fails the turn. The payload is the one it emitted. func TestASessionTheRuntimeBlockedAtItsCreditLimitRemembersWhatItSpent(t *testing.T) { diff --git a/internal/harness/draft.go b/internal/harness/draft.go index cda3dd5f..02e81622 100644 --- a/internal/harness/draft.go +++ b/internal/harness/draft.go @@ -91,12 +91,13 @@ func (s *Sessions) draft(run int64, queryID string, picked []Picked, by string) }) limit := s.limit() ctx, cancel := context.WithTimeout(context.Background(), limit) + gate.cancel = cancel l := &live{session: id, gate: gate, cancel: cancel} s.recordDraft(run, l) d := drafter{store: s.Store, workspace: s.Workspace, session: id, run: run, query: queryID, picked: picked, databases: databases} - setup := Setup{Home: home, Workspace: bounds.Workspace(), Prompt: d.prompt(), Rules: d.rules(), + setup := Setup{Timeout: limit, TimeoutSource: s.timeoutSource(), Home: home, Workspace: bounds.Workspace(), Prompt: d.prompt(), Rules: d.rules(), Gate: gate, Tools: d.tools(), Sealed: true} done := s.launchDraft(draftRun{ run: run, id: id, dir: dir, limit: limit, ctx: ctx, cancel: cancel, live: l, setup: setup, @@ -159,12 +160,18 @@ func (s *Sessions) launchDraft(run draftRun) <-chan struct{} { defer close(done) final, err := s.run(run.ctx, run.setup) state, detail := s.outcome(run.ctx, run.live, run.limit, final, err) + s.finishAudit(run.live, state, detail) + if err := run.live.gate.Err(); err != nil { + state, detail = store.SessionFailed, err.Error() + } run.cancel() s.mu.Lock() delete(s.drafting, run.run) s.mu.Unlock() _ = os.RemoveAll(run.dir) - _ = s.Store.EndAuthoringSession(run.id, state, detail) + if err := s.Store.EndAuthoringSession(run.id, state, detail); err != nil { + _ = run.live.gate.fail(err) + } }() return done } diff --git a/internal/harness/gate.go b/internal/harness/gate.go index 9e13b376..0b4ba89a 100644 --- a/internal/harness/gate.go +++ b/internal/harness/gate.go @@ -2,6 +2,7 @@ package harness import ( "encoding/json" + "errors" "fmt" "math" "slices" @@ -10,6 +11,8 @@ import ( "time" "unicode/utf8" + "github.com/advanced-security/codeql-abom/internal/audit" + "github.com/advanced-security/codeql-abom/internal/store" copilot "github.com/github/copilot-sdk/go" "github.com/github/copilot-sdk/go/rpc" ) @@ -56,7 +59,12 @@ type Event struct { Reason string `json:"reason"` Detail string `json:"detail"` // Took is how long a call ran, from its approval to its result; zero when not measured. - Took time.Duration `json:"-"` + Took time.Duration `json:"-"` + Source string `json:"source"` + SourceID string `json:"sourceId"` + QueryID string `json:"queryId"` + QueryVersion string `json:"queryVersion"` + Capture store.AuditCapture `json:"capture"` } // MarshalJSON writes Took in milliseconds, the unit every other duration in the API uses. @@ -75,6 +83,7 @@ type Flight struct { Subject string `json:"subject"` Step string `json:"step"` Since time.Time `json:"since"` + CallID string `json:"callId"` } // Gate turns Bounds into a session's callbacks and records every invocation of them. No callback @@ -84,10 +93,12 @@ type Gate struct { // Now stamps events. Now func() time.Time - mu sync.Mutex - bounds Bounds - record func(Event) error - flights []Flight + mu sync.Mutex + recordMu sync.Mutex + decisionMu sync.Mutex + bounds Bounds + record func(Event) error + flights []Flight // scratch is where ql-mcp prints, and keepPrinted keeps what it printed there. scratch string @@ -97,8 +108,13 @@ type Gate struct { keepSpend func(Spend) error // call and request decide, as fields so a test can break them. - call func(Bounds, string, any) Verdict - request func(Bounds, rpc.PermissionRequest) Verdict + call func(Bounds, string, any) Verdict + request func(Bounds, rpc.PermissionRequest) Verdict + fatal error + cancel func() + policy *audit.Policy + policyJSON string + seen map[string]bool } // NewGate decides by b, and keeps each event with record before the call it describes may run. @@ -108,17 +124,22 @@ func NewGate(b Bounds, record func(Event) error) *Gate { // Bind narrows the gate to one target. A session is about one query, so a second is refused. func (g *Gate) Bind(query, tests string) error { + g.decisionMu.Lock() + defer g.decisionMu.Unlock() g.mu.Lock() - defer g.mu.Unlock() if g.bounds.bound() { - return fmt.Errorf("this session is already about %s", g.bounds.rel(g.bounds.query)) + target := g.bounds.rel(g.bounds.query) + g.mu.Unlock() + return fmt.Errorf("this session is already about %s", target) } b, err := g.bounds.Bind(query, tests) if err != nil { + g.mu.Unlock() return err } g.bounds = b - return nil + g.mu.Unlock() + return g.refreshPolicy() } // Bounds is what the gate decides by now. @@ -138,12 +159,16 @@ func (g *Gate) InFlight() []Flight { // Hooks wires the gate into a session. func (g *Gate) Hooks() *copilot.SessionHooks { return &copilot.SessionHooks{ - OnPreToolUse: g.PreToolUse, - OnPostToolUse: g.PostToolUse, - OnPostToolUseFailure: g.PostToolUseFailure, - OnSessionStart: g.SessionStart, - OnSessionEnd: g.SessionEnd, - OnErrorOccurred: g.ErrorOccurred, + OnPreToolUse: g.PreToolUse, + OnPostToolUse: g.PostToolUse, + OnPostToolUseFailure: g.PostToolUseFailure, + OnSessionStart: g.SessionStart, + OnSessionEnd: g.SessionEnd, + OnErrorOccurred: g.ErrorOccurred, + OnUserPromptSubmitted: g.PromptSubmitted, + OnUserPromptTransformed: g.PromptTransformed, + OnAgentStop: g.AgentStop, + OnPreMCPToolCall: g.PreMCP, } } @@ -151,13 +176,16 @@ func (g *Gate) Hooks() *copilot.SessionHooks { // typed permission request that follows is judged again with what the CLI says it will touch. func (g *Gate) PreToolUse(in copilot.PreToolUseHookInput, _ copilot.HookInvocation) ( out *copilot.PreToolUseHookOutput, err error) { + g.decisionMu.Lock() + defer g.decisionMu.Unlock() defer func() { if r := recover(); r != nil { + _ = g.fail(fmt.Errorf("pre-action callback panicked")) out, err = deny(fmt.Sprintf("the gate failed deciding this call, so it is refused: %v", r)), nil } }() b := g.Bounds() - e := Event{Hook: HookPreToolUse, Tool: in.ToolName, Subject: b.subject(in.ToolName, in.ToolArgs), + e := Event{At: in.Timestamp, Capture: store.AuditCapture{RuntimeSessionID: in.SessionID}, Hook: HookPreToolUse, Tool: in.ToolName, Subject: b.subject(in.ToolName, in.ToolArgs), Detail: detail(in.ToolArgs)} v := decide(func() Verdict { return g.call(b, in.ToolName, in.ToolArgs) }) if err := g.keep(e, v); err != nil { @@ -170,16 +198,25 @@ func (g *Gate) PreToolUse(in copilot.PreToolUseHookInput, _ copilot.HookInvocati } // Permission decides a typed permission request, and marks an approved ql-mcp call as under way. -func (g *Gate) Permission(req copilot.PermissionRequest, _ copilot.PermissionInvocation) ( +func (g *Gate) Permission(req copilot.PermissionRequest, invocation copilot.PermissionInvocation) ( d rpc.PermissionDecision, err error) { + g.decisionMu.Lock() + defer g.decisionMu.Unlock() defer func() { if r := recover(); r != nil { + _ = g.fail(fmt.Errorf("permission callback panicked")) d, err = reject(fmt.Sprintf("the gate failed deciding this request, so it is refused: %v", r)), nil } }() b := g.Bounds() tool, subject, args := b.describe(req) - e := Event{Hook: HookPermission, Tool: tool, Subject: subject, Detail: detail(args)} + var ids struct { + ToolCallID string `json:"toolCallId"` + } + raw, _ := json.Marshal(req) + _ = json.Unmarshal(raw, &ids) + e := Event{Hook: HookPermission, Tool: tool, Subject: subject, Detail: detail(args), + Capture: store.AuditCapture{RuntimeSessionID: invocation.SessionID, CallID: ids.ToolCallID}} v := decide(func() Verdict { return g.request(b, req) }) if err := g.keep(e, v); err != nil { return reject(unrecorded(err)), nil @@ -187,11 +224,6 @@ func (g *Gate) Permission(req copilot.PermissionRequest, _ copilot.PermissionInv if !v.Allowed { return reject(v.Reason), nil } - if _, ok := req.(*rpc.PermissionRequestMCP); ok { - g.mu.Lock() - g.flights = append(g.flights, Flight{Tool: tool, Subject: subject, Step: v.Step, Since: g.Now()}) - g.mu.Unlock() - } return &rpc.PermissionDecisionApproveOnce{}, nil } @@ -200,10 +232,15 @@ func (g *Gate) PostToolUse(in copilot.PostToolUseHookInput, _ copilot.HookInvoca out *copilot.PostToolUseHookOutput, err error) { defer func() { if r := recover(); r != nil { + _ = g.fail(fmt.Errorf("post-action capture panicked")) out, err = nil, nil } }() - g.land(HookPostToolUse, in.ToolName, in.ToolArgs, DecisionSucceeded, resultText(in.ToolResult)) + g.land(Event{At: in.Timestamp, Hook: HookPostToolUse, Tool: in.ToolName, Decision: DecisionSucceeded, + Capture: store.AuditCapture{RuntimeSessionID: in.SessionID}}, in.ToolArgs, resultText(in.ToolResult)) + if err := g.Err(); err != nil { + return nil, err + } if context := g.print(in.ToolName, in.ToolArgs, resultText(in.ToolResult)); context != "" { return &copilot.PostToolUseHookOutput{AdditionalContext: context}, nil } @@ -215,10 +252,12 @@ func (g *Gate) PostToolUseFailure(in copilot.PostToolUseFailureHookInput, _ copi out *copilot.PostToolUseFailureHookOutput, err error) { defer func() { if r := recover(); r != nil { + _ = g.fail(fmt.Errorf("post-failure capture panicked")) out, err = nil, nil } }() - g.land(HookPostToolUseFailure, in.ToolName, in.ToolArgs, DecisionFailed, in.Error) + g.land(Event{At: in.Timestamp, Hook: HookPostToolUseFailure, Tool: in.ToolName, Decision: DecisionFailed, + Capture: store.AuditCapture{RuntimeSessionID: in.SessionID}}, in.ToolArgs, in.Error) return nil, nil } @@ -227,10 +266,12 @@ func (g *Gate) SessionStart(in copilot.SessionStartHookInput, _ copilot.HookInvo out *copilot.SessionStartHookOutput, err error) { defer func() { if r := recover(); r != nil { + _ = g.fail(fmt.Errorf("session-start capture panicked")) out, err = nil, nil } }() - _ = g.keep(Event{Hook: HookSessionStart, Tool: "session", Subject: in.Source}, Verdict{Allowed: true}) + _ = g.keep(Event{At: in.Timestamp, Capture: store.AuditCapture{RuntimeSessionID: in.SessionID}, + Hook: HookSessionStart, Tool: "session", Subject: in.Source}, Verdict{Allowed: true}) return nil, nil } @@ -239,6 +280,7 @@ func (g *Gate) SessionEnd(in copilot.SessionEndHookInput, _ copilot.HookInvocati out *copilot.SessionEndHookOutput, err error) { defer func() { if r := recover(); r != nil { + _ = g.fail(fmt.Errorf("session-end capture panicked")) out, err = nil, nil } }() @@ -249,7 +291,8 @@ func (g *Gate) SessionEnd(in copilot.SessionEndHookInput, _ copilot.HookInvocati if in.Error != "" { text = in.Error } - _ = g.keep(Event{Hook: HookSessionEnd, Tool: "session", Subject: in.Reason, Detail: clip(text)}, + _ = g.keep(Event{At: in.Timestamp, Capture: store.AuditCapture{RuntimeSessionID: in.SessionID}, + Hook: HookSessionEnd, Tool: "session", Subject: in.Reason, Detail: detail(map[string]any{"message": text})}, Verdict{Allowed: true, Reason: in.Reason}) return nil, nil } @@ -259,32 +302,108 @@ func (g *Gate) ErrorOccurred(in copilot.ErrorOccurredHookInput, _ copilot.HookIn out *copilot.ErrorOccurredHookOutput, err error) { defer func() { if r := recover(); r != nil { + _ = g.fail(fmt.Errorf("error capture panicked")) out, err = &copilot.ErrorOccurredHookOutput{ErrorHandling: "abort"}, nil } }() - _ = g.keep(Event{Hook: HookError, Tool: in.ErrorContext, Detail: clip(in.Error)}, + _ = g.keep(Event{At: in.Timestamp, Capture: store.AuditCapture{RuntimeSessionID: in.SessionID}, + Hook: HookError, Tool: in.ErrorContext, Detail: detail(map[string]any{"message": in.Error})}, Verdict{Allowed: true, Reason: in.Error}) return &copilot.ErrorOccurredHookOutput{ErrorHandling: "abort"}, nil } // Observe records the session events no hook sees: an MCP server's state, and what the agent said. func (g *Gate) Observe(e copilot.SessionEvent) { - defer func() { _ = recover() }() + defer func() { + if recover() != nil { + _ = g.fail(fmt.Errorf("SDK observation capture panicked")) + } + }() + if e.ID != "" { + g.mu.Lock() + if g.seen == nil { + g.seen = map[string]bool{} + } + seen := g.seen[e.ID] + g.seen[e.ID] = true + g.mu.Unlock() + if seen { + return + } + } + event := Event{At: e.Timestamp, Source: "sdk", SourceID: e.ID, Decision: DecisionNoted} switch d := e.Data.(type) { case *copilot.SessionMCPServerStatusChangedData: var why string if d.Error != nil { why = *d.Error } - _ = g.keep(Event{Hook: HookServer, Tool: d.ServerName, Subject: string(d.Status), - Decision: DecisionNoted, Detail: clip(why)}, Verdict{}) + event.Hook, event.Tool, event.Subject, event.Detail = HookServer, d.ServerName, string(d.Status), detail(map[string]any{"message": why}) case *copilot.AssistantMessageData: if strings.TrimSpace(d.Content) != "" { - _ = g.keep(Event{Hook: HookMessage, Tool: "agent", Decision: DecisionNoted, - Detail: clip(d.Content)}, Verdict{}) + event.Hook, event.Tool, event.Detail = HookMessage, "agent", detail(map[string]any{"message": d.Content}) } case *copilot.AssistantUsageData: g.spent(d) + return + case *copilot.ToolExecutionStartData: + event.Hook, event.Tool, event.Detail = "tool-start", d.ToolName, detail(d.Arguments) + event.Subject = g.Bounds().subject(d.ToolName, d.Arguments) + event.Capture.CallID = d.ToolCallID + if d.ToolCallID != "" { + g.mu.Lock() + g.flights = append(g.flights, Flight{Tool: d.ToolName, Subject: event.Subject, CallID: d.ToolCallID, Since: g.Now()}) + g.mu.Unlock() + } + case *copilot.ToolExecutionCompleteData: + event.Hook, event.Tool = "tool-complete", "runtime" + event.Capture.CallID = d.ToolCallID + event.Decision = DecisionFailed + if d.Success { + event.Decision = DecisionSucceeded + } + event.Detail = detail(map[string]any{"success": d.Success, "result": "[omitted: opaque tool result]"}) + g.mu.Lock() + i := slices.IndexFunc(g.flights, func(f Flight) bool { return d.ToolCallID != "" && f.CallID == d.ToolCallID }) + if i >= 0 { + f := g.flights[i] + event.Tool, event.Subject, event.Took = f.Tool, f.Subject, g.Now().Sub(f.Since) + g.flights = slices.Delete(g.flights, i, i+1) + } + g.mu.Unlock() + case *copilot.SessionLimitsExhaustedRequestedData: + event.Hook, event.Tool = "credit-limit", "session" + event.Detail = detail(map[string]any{"maxAiCredits": d.MaxAiCredits, "usedAiCredits": d.UsedAiCredits, "outcome": "limit reported by runtime"}) + case *copilot.RawSessionEventData: + if d.EventType != "model.response_limits_status" { + return + } + var status struct { + Status map[string]any `json:"status"` + } + if json.Unmarshal(d.Raw, &status) != nil { + return + } + event.Hook, event.Tool = "credit-status", "session" + fields := map[string]any{} + for _, key := range []string{"maxAiCredits", "aiCreditsUsed", "isLimitsExhausted"} { + if value, ok := status.Status[key]; ok { + fields[key] = value + } + } + event.Detail = detail(fields) + default: + if e.Data == nil { + return + } + kind := string(e.Data.Type()) + if !slices.Contains([]string{"session.start", "session.resume", "session.shutdown", "session.error", "session.idle", "session.limits", "abort", "agent.interrupted"}, kind) { + return + } + event.Hook, event.Tool = kind, "session" + } + if event.Hook != "" { + _ = g.keep(event, Verdict{}) } } @@ -320,34 +439,38 @@ func (g *Gate) spent(d *copilot.AssistantUsageData) { s.NanoAIU = &nano } if err := keep(s); err != nil { - _ = g.keep(Event{Hook: HookSpend, Tool: d.Model, Decision: DecisionNoted, - Detail: clip("what this model call cost could not be kept: " + err.Error())}, Verdict{}) + _ = g.fail(fmt.Errorf("recording session spend failed")) } } // land records a call's outcome, with how long it ran if the gate saw it approved. -func (g *Gate) land(hook, tool string, args any, outcome, text string) { +func (g *Gate) land(e Event, args any, text string) { b := g.Bounds() - e := Event{Hook: hook, Tool: tool, Subject: b.subject(tool, args), Decision: outcome, Detail: clip(text)} - g.mu.Lock() - i := slices.IndexFunc(g.flights, func(f Flight) bool { return f.Tool == tool && f.Subject == e.Subject }) - if i < 0 { - i = slices.IndexFunc(g.flights, func(f Flight) bool { return f.Tool == tool }) - } - if i >= 0 { - f := g.flights[i] - g.flights = slices.Delete(g.flights, i, i+1) - e.Took, e.Step = g.Now().Sub(f.Since), f.Step - } - g.mu.Unlock() - if i < 0 { - e.Step = decide(func() Verdict { return g.call(b, tool, args) }).Step + e.Subject, e.Detail = b.subject(e.Tool, args), detail(map[string]any{"message": text}) + if e.Decision == DecisionSucceeded { + e.Detail = detail(map[string]any{"result": map[string]any{"omitted": "opaque tool result; measured tests are separate evidence", "bytes": len(text), "sha256": audit.Digest([]byte(text))}}) } + e.Step = decide(func() Verdict { return g.call(b, e.Tool, args) }).Step _ = g.keep(e, Verdict{}) } // keep stamps and records an event, taking its decision from v unless it already has an outcome. -func (g *Gate) keep(e Event, v Verdict) error { +func (g *Gate) keep(e Event, v Verdict) (err error) { + defer func() { + if recover() != nil { + err = g.fail(errors.New("audit callback panicked")) + } + }() + g.recordMu.Lock() + defer g.recordMu.Unlock() + if err := g.Err(); err != nil { + return err + } + if e.Hook != "policy" { + if err := g.refreshPolicyLocked(); err != nil { + return err + } + } if e.At.IsZero() { e.At = g.Now() } @@ -357,7 +480,26 @@ func (g *Gate) keep(e Event, v Verdict) error { e.Decision = DecisionNoted } } - return g.record(e) + if e.Source == "" { + e.Source = "hook" + if e.Hook == HookPermission { + e.Source = "permission" + } + if e.Hook == HookArtifact || e.Hook == HookMessage && e.Tool == "abom" { + e.Source = "abom" + } + } + e.Capture.Version = audit.Version + e.Capture.Correlation = "uncorrelated hook observation" + if e.Capture.CallID != "" { + e.Capture.Correlation = "reported runtime call ID" + } + e.Tool, e.Subject, e.Reason = audit.Text(e.Tool), audit.Text(e.Subject), audit.Text(e.Reason) + e.Detail = audit.ProjectDetail(e.Detail) + if err := g.record(e); err != nil { + return g.fail(err) + } + return nil } func decisionOf(v Verdict) string { @@ -456,30 +598,8 @@ func (b Bounds) name(p string) string { return b.rel(real) } -// textArgs carry the agent's working text, which the timeline replaces with its size. -var textArgs = map[string]bool{"file_text": true, "old_str": true, "new_str": true, "ql_code": true} - func detail(args any) string { - if args == nil { - return "" - } - fields, ok := args.(map[string]any) - if !ok { - return clip(fmt.Sprint(args)) - } - shown := make(map[string]any, len(fields)) - for k, v := range fields { - if s, ok := v.(string); ok && textArgs[k] { - shown[k] = fmt.Sprintf("<%d characters>", len(s)) - continue - } - shown[k] = v - } - body, err := json.Marshal(shown) - if err != nil { - return "" - } - return clip(string(body)) + return audit.Detail(args) } // resultText is what a tool reported back to the agent. @@ -502,7 +622,7 @@ func clip(s string) string { if len(s) <= maxDetail { return s } - const mark = "…" + const mark = "[truncated]" cut := maxDetail - len(mark) for cut > 0 && !utf8.RuneStart(s[cut]) { cut-- diff --git a/internal/harness/gate_test.go b/internal/harness/gate_test.go index 1586dfbc..d30a83aa 100644 --- a/internal/harness/gate_test.go +++ b/internal/harness/gate_test.go @@ -112,7 +112,7 @@ func TestTheTimelineKeepsWhatWasTouchedNotTheText(t *testing.T) { } } -func TestAnApprovedCodeQLCallIsUnderWayUntilItsResultArrives(t *testing.T) { +func TestOnlyReportedCallIdentitiesMeasureAnExecution(t *testing.T) { g, j, p := newGate(t) clock := time.Date(2026, 9, 23, 12, 0, 0, 0, time.UTC) g.Now = func() time.Time { return clock } @@ -123,6 +123,11 @@ func TestAnApprovedCodeQLCallIsUnderWayUntilItsResultArrives(t *testing.T) { if _, ok := d.(*rpc.PermissionDecisionApproveOnce); !ok || err != nil { t.Fatalf("test run: %T %v, want approved", d, err) } + if len(g.InFlight()) != 0 { + t.Fatal("permission without an ID became an exact call") + } + g.Observe(copilot.SessionEvent{ID: "start", Data: &copilot.ToolExecutionStartData{ + ToolName: Server + "-codeql_test_run", ToolCallID: "call-1", Arguments: args}}) flights := g.InFlight() if len(flights) != 1 || flights[0].Tool != Server+"-codeql_test_run" || flights[0].Subject != "test/CheckoutTagPin" || !flights[0].Since.Equal(clock) { @@ -135,11 +140,16 @@ func TestAnApprovedCodeQLCallIsUnderWayUntilItsResultArrives(t *testing.T) { copilot.HookInvocation{}); err != nil { t.Fatal(err) } + hook := j.all()[len(j.all())-1] + if hook.Took != 0 { + t.Fatal("ordinary post hook was paired by guessing") + } + g.Observe(copilot.SessionEvent{ID: "complete", Data: &copilot.ToolExecutionCompleteData{ToolCallID: "call-1", Success: true}}) if left := g.InFlight(); len(left) != 0 { t.Errorf("still under way after its result: %+v", left) } last := j.all()[len(j.all())-1] - if last.Decision != DecisionSucceeded || last.Took != 8*time.Second || last.Step != StepTests { + if last.Decision != DecisionSucceeded || last.Took != 8*time.Second || last.Capture.CallID != "call-1" { t.Errorf("result recorded as %+v", last) } } @@ -198,7 +208,7 @@ func TestACallTheTimelineCannotRecordIsRefused(t *testing.T) { out, err := g.PreToolUse(copilot.PreToolUseHookInput{ToolName: "view", ToolArgs: map[string]any{"path": p.ws}}, copilot.HookInvocation{}) if err != nil || out == nil || out.PermissionDecision != "deny" || - !strings.Contains(out.PermissionDecisionReason, "disk full") { + !strings.Contains(out.PermissionDecisionReason, "evidence is incomplete") { t.Errorf("unrecordable read: %+v %v, want a denial naming why", out, err) } d, err := g.Permission(&rpc.PermissionRequestRead{Path: p.ws}, copilot.PermissionInvocation{}) @@ -299,14 +309,14 @@ func TestTheAgentsWordsAndItsServersReachTheTimeline(t *testing.T) { } events := j.all() - if len(events) != 2 { + if len(events) != 3 { t.Fatalf("recorded %d events, want the server's failure and the agent's words: %+v", len(events), events) } if e := events[0]; e.Hook != HookServer || e.Tool != Server || e.Subject != "failed" || - e.Detail != failure || e.Decision != DecisionNoted { + !strings.Contains(e.Detail, failure) || e.Decision != DecisionNoted { t.Errorf("server event %+v", e) } - if e := events[1]; e.Hook != HookMessage || e.Detail != "Extracting the test database first." || + if e := events[1]; e.Hook != HookMessage || !strings.Contains(e.Detail, "Extracting the test database first.") || e.Decision != DecisionNoted { t.Errorf("message event %+v", e) } @@ -343,9 +353,8 @@ func TestWhatEachModelCallCostIsKeptAsItArrives(t *testing.T) { g.KeepSpend(func(Spend) error { return errors.New("the store is closed") }) g.Observe(copilot.SessionEvent{Data: &copilot.AssistantUsageData{Model: "claude-opus-5", InputTokens: &in}}) - if events := j.all(); len(events) != 1 || events[0].Hook != HookSpend || - !strings.Contains(events[0].Detail, "the store is closed") { - t.Errorf("a cost that could not be kept left %+v, want it said on the timeline", events) + if g.Err() == nil { + t.Error("a cost that could not be kept did not fail the session") } } diff --git a/internal/harness/printed.go b/internal/harness/printed.go index 4a972dee..fc8658b2 100644 --- a/internal/harness/printed.go +++ b/internal/harness/printed.go @@ -69,7 +69,12 @@ func (g *Gate) print(tool string, args any, result string) string { var id int64 id, err = keep(Printed{At: g.Now(), Query: name, Database: b.rel(b.database()), SourceFiles: sourceFiles, Body: body, Truncated: truncated}) + if err != nil { + _ = g.fail(err) + return "ABOM could not retain the printed evidence; the session is stopping." + } if err == nil { + e.Capture.ArtifactID = id e.Detail = fmt.Sprintf("kept %d bytes of what %s printed, as printout %d", len(body), name, id) context = fmt.Sprintf("ABOM kept what %s printed. Read the tree in %s.", name, strings.Join(files, " and ")) diff --git a/internal/harness/session.go b/internal/harness/session.go index ab2ba7c6..405250d1 100644 --- a/internal/harness/session.go +++ b/internal/harness/session.go @@ -146,10 +146,12 @@ func (s *Sessions) start(request int64, by string) (store.AuthoringSession, <-ch }) limit := s.limit() ctx, cancel := context.WithTimeout(context.Background(), limit) + gate.cancel = cancel l := &live{session: id, gate: gate, cancel: cancel} s.recordLive(request, l) setup := Setup{ + Timeout: limit, TimeoutSource: s.timeoutSource(), Home: home, Scratch: scratch, Workspace: bounds.Workspace(), @@ -221,12 +223,18 @@ func (s *Sessions) launchAuthoring(run startedAuthoring) <-chan struct{} { final, err = s.run(run.ctx, run.setup) } state, detail := s.outcome(run.ctx, run.live, run.limit, final, err) + s.finishAudit(run.live, state, detail) + if err := run.live.gate.Err(); err != nil { + state, detail = store.SessionFailed, err.Error() + } run.cancel() s.mu.Lock() delete(s.live, run.request) s.mu.Unlock() _ = os.RemoveAll(run.dir) - _ = s.Store.EndAuthoringSession(run.id, state, detail) + if err := s.Store.EndAuthoringSession(run.id, state, detail); err != nil { + _ = run.live.gate.fail(err) + } }() return done } @@ -254,6 +262,17 @@ func (s *Sessions) run(ctx context.Context, setup Setup) (final string, err erro if agent == nil { return "", ErrNoAgent } + if s.Version != nil { + if cli, err := s.codeQL(ctx); err == nil { + setup.Runtime = map[string]string{"codeql": cli} + } + } + if _, ok := agent.(*Copilot); !ok { + setup.Tools = setup.Gate.guardedTools(setup.Tools) + if err := setup.Gate.startPolicy(ctx, setup, nil, nil); err != nil { + return "", err + } + } return agent.Run(ctx, setup) } @@ -263,6 +282,8 @@ func (s *Sessions) outcome(ctx context.Context, l *live, limit time.Duration, fi s.mu.Unlock() var spent *CreditLimitError switch { + case l.gate.Err() != nil: + return store.SessionFailed, l.gate.Err().Error() case errors.Is(ctx.Err(), context.DeadlineExceeded): return store.SessionStopped, fmt.Sprintf("stopped at the session's time limit of %s", limit) case stopped: @@ -322,7 +343,18 @@ func (s *Sessions) acquire() (func(), error) { } func (s *Sessions) recorder(session int64) func(Event) error { + owner, ownerErr := s.Store.AuditSession(session) return func(e Event) error { + if ownerErr != nil { + return ownerErr + } + if owner.RequestID > 0 && e.QueryVersion == "" { + var err error + e.QueryID, e.QueryVersion, err = s.observedVersion(owner.RequestID) + if err != nil { + return err + } + } var took *int64 if e.Took > 0 { ms := e.Took.Milliseconds() @@ -332,11 +364,25 @@ func (s *Sessions) recorder(session int64) func(Event) error { SessionID: session, At: e.At.UTC().Format(time.RFC3339Nano), Hook: e.Hook, Tool: e.Tool, Subject: e.Subject, Decision: e.Decision, Step: e.Step, Reason: e.Reason, Detail: e.Detail, TookMs: took, + Source: e.Source, SourceID: e.SourceID, QueryID: e.QueryID, QueryVersion: e.QueryVersion, Capture: e.Capture, }) return err } } +func (s *Sessions) observedVersion(request int64) (string, string, error) { + req, err := s.requestNamed(request) + if err != nil || req.QueryID == "" { + return "", "", err + } + // A partial edit can temporarily remove query metadata; that makes the version unknown. + q, found, _ := authoring.Find(s.Workspace, req.QueryID) + if !found { + return req.QueryID, "", nil + } + return q.ID, q.Version, nil +} + // keeper keeps a printout against the request's query as it stands, the fixtures it was printed // from and the CodeQL that printed it, so a tree printed before either changed reads as old. func (s *Sessions) keeper(session, request int64) func(Printed) (int64, error) { @@ -727,6 +773,9 @@ func (s *Sessions) tools(ctx context.Context, request int64, gate *Gate) []copil // testTarget runs the bound query's own tests through ABOM's runner and records them, saying // whether the failure is one on a marked result. func (s *Sessions) testTarget(ctx context.Context, request int64, gate *Gate) (string, error) { + if err := gate.Err(); err != nil { + return "", err + } if !gate.Bounds().bound() { return "", fmt.Errorf("decide the query this request is about with %s before testing it", ToolTarget) } @@ -748,6 +797,15 @@ func (s *Sessions) testTarget(ctx context.Context, request int64, gate *Gate) (s if err != nil { return "", err } + ctx = authoring.WithTestEvidence(ctx, func(test store.QueryTest) error { + decision := DecisionFailed + if test.Passed { + decision = DecisionSucceeded + } + return gate.keep(Event{Source: "abom", Hook: "test-evidence", Tool: ToolTest, Decision: decision, + Step: StepTests, QueryID: test.QueryID, QueryVersion: test.QueryVersion, + Capture: store.AuditCapture{Test: &test, Inputs: s.inputDigests(gate.Bounds(), q)}}, Verdict{}) + }) result, err := authoring.RecordTests(ctx, s.Store, s.RunTests, s.Workspace, q, cli) if err != nil { return "", err diff --git a/internal/serve/audit.go b/internal/serve/audit.go new file mode 100644 index 00000000..9f529a32 --- /dev/null +++ b/internal/serve/audit.go @@ -0,0 +1,250 @@ +package serve + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/advanced-security/codeql-abom/internal/audit" + "github.com/advanced-security/codeql-abom/internal/store" +) + +var auditTerms = []string{"event", "request", "run", "session", "query", "version", "purpose", "hook", "tool", "decision", "step", "actor", "from", "to", "text", "order", "limit", "cursor", "through"} + +func auditFilter(r *http.Request) (store.AuditFilter, error) { + q := r.URL.Query() + for key, values := range q { + if len(values) != 1 { + return store.AuditFilter{}, fmt.Errorf("%s must appear once", key) + } + } + f := store.AuditFilter{Query: q.Get("query"), Version: q.Get("version"), Purpose: q.Get("purpose"), Hook: q.Get("hook"), Tool: q.Get("tool"), + Decision: q.Get("decision"), Step: q.Get("step"), Actor: q.Get("actor"), From: q.Get("from"), To: q.Get("to"), Text: q.Get("text"), Order: q.Get("order"), Cursor: q.Get("cursor")} + for key, p := range map[string]*int64{"event": &f.Event, "request": &f.Request, "run": &f.Run, "session": &f.Session, "through": &f.Through} { + if raw := q.Get(key); raw != "" { + n, err := strconv.ParseInt(raw, 10, 64) + if err != nil || n < 0 || n == 0 && key != "through" { + return f, fmt.Errorf("%s must be a positive integer", key) + } + *p = n + if key == "through" { + f.FixedCutoff = true + } + } + } + if raw := q.Get("limit"); raw != "" { + n, err := strconv.Atoi(raw) + if err != nil || n < 1 || n > 500 { + return f, errors.New("limit must be between 1 and 500") + } + f.Limit = n + } + if f.Purpose != "" && f.Purpose != "authoring" && f.Purpose != "sealed drafting" { + return f, errors.New("purpose is authoring or sealed drafting") + } + if f.Order != "" && f.Order != "asc" && f.Order != "desc" { + return f, errors.New("order is asc or desc") + } + for _, key := range []string{"from", "to"} { + if raw := q.Get(key); raw != "" { + if _, err := time.Parse(time.RFC3339Nano, raw); err != nil { + return f, fmt.Errorf("%s must be an RFC3339 timestamp", key) + } + } + } + if f.From != "" && f.To != "" { + from, _ := time.Parse(time.RFC3339Nano, f.From) + to, _ := time.Parse(time.RFC3339Nano, f.To) + if from.After(to) { + return f, errors.New("from must precede to") + } + } + if len(f.Text) > 4096 || len(f.Cursor) > 4096 { + return f, errors.New("audit search or cursor is too long") + } + return f, nil +} + +// scopeToSession narrows a filter to the session its route names. A request, run or other session +// in the query could only narrow that session further, to nothing, so it is refused. +func scopeToSession(f *store.AuditFilter, id int64) error { + switch { + case f.Session != 0 && f.Session != id: + return errors.New("session conflicts with route") + case f.Request != 0 || f.Run != 0: + return errors.New("the route names the session; request and run do not apply") + } + f.Session = id + return nil +} + +func auditError(w http.ResponseWriter, err error) { + code := http.StatusInternalServerError + switch { + case errors.Is(err, store.ErrNoAuditSession): + code = http.StatusNotFound + case errors.Is(err, store.ErrBadAuditFilter): + code = http.StatusBadRequest + } + writeError(w, code, err.Error()) +} + +func (s *Server) auditEvents(w http.ResponseWriter, r *http.Request) { + f, err := auditFilter(r) + if err != nil { + writeError(w, 400, err.Error()) + return + } + if raw := r.PathValue("id"); raw != "" { + id, err := strconv.ParseInt(raw, 10, 64) + if err != nil || id <= 0 { + writeError(w, 400, "a session is a positive integer") + return + } + if err := scopeToSession(&f, id); err != nil { + writeError(w, 400, err.Error()) + return + } + } + page, err := s.Store.AuditEvents(f) + if err != nil { + auditError(w, err) + return + } + writeJSON(w, page, nil) +} + +func (s *Server) auditSession(w http.ResponseWriter, r *http.Request) { + id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) + if err != nil || id <= 0 { + writeError(w, 400, "a session is a positive integer") + return + } + meta, err := s.Store.AuditSession(id) + if err != nil { + auditError(w, err) + return + } + writeJSON(w, meta, nil) +} + +func (s *Server) auditSessions(w http.ResponseWriter, r *http.Request) { + f, err := auditFilter(r) + if err != nil || f.Request == 0 && f.Run == 0 { + writeError(w, 400, "session summaries require a request or run") + return + } + items, err := s.Store.AuditSessions(f.Request, f.Run) + if err != nil { + auditError(w, err) + return + } + writeJSON(w, map[string]any{"items": items, "total": len(items)}, nil) +} + +func (s *Server) auditExport(w http.ResponseWriter, r *http.Request) { + f, err := auditFilter(r) + if err != nil { + writeError(w, 400, err.Error()) + return + } + if f.Cursor != "" { + writeError(w, 400, "export takes filters and a cutoff, not a page cursor") + return + } + page, err := s.Store.AuditEvents(f) + if err != nil { + auditError(w, err) + return + } + f.Through, f.FixedCutoff = page.Through, true + w.Header().Set("Content-Type", "application/x-ndjson; charset=utf-8") + w.Header().Set("Content-Disposition", `attachment; filename="codeql-audit.jsonl"`) + enc := json.NewEncoder(w) + if err := enc.Encode(map[string]any{"type": "metadata", "schema": audit.Version, "filter": f, "through": page.Through, "total": page.Total, "coverage": page.Coverage}); err != nil { + return + } + count := 0 + // Cutoff zero denotes an empty snapshot, not a later reading of newly appended events. + if page.Through > 0 { + err = s.Store.WalkAudit(f, func(e store.AuthoringEvent) error { + if err := r.Context().Err(); err != nil { + return err + } + if err := enc.Encode(map[string]any{"type": "event", "event": e}); err != nil { + return err + } + count++ + return nil + }) + } + if err != nil || r.Context().Err() != nil || count != page.Total { + return + } + _ = enc.Encode(map[string]any{"type": "completion", "complete": true, "events": count, "through": page.Through}) +} + +const reproductionLimits = "This is recorded evidence, not a replay or an authorship claim. Digests cannot recover files. Source, patches and scratch are not archived. Obtain matching query, pack dependencies, fixtures, expected results and toolchain independently; unavailable or changed inputs prevent a claim of exact reproduction. Model messages and successful tool returns do not prove a CodeQL test passed. Events, policies and observed coverage use the stated cutoff; session status and non-event relations describe the current record." + +func (s *Server) auditReproduction(w http.ResponseWriter, r *http.Request) { + id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) + if err != nil || id <= 0 { + writeError(w, 400, "a session is a positive integer") + return + } + f, err := auditFilter(r) + if err != nil { + writeError(w, 400, err.Error()) + return + } + if err := scopeToSession(&f, id); err != nil { + writeError(w, 400, err.Error()) + return + } + page, err := s.Store.AuditEvents(f) + if err != nil { + auditError(w, err) + return + } + meta, err := s.Store.AuditSession(id, page.Through) + if err != nil { + auditError(w, err) + return + } + if !r.URL.Query().Has("download") { + writeJSON(w, map[string]any{"session": meta, "evidence": page, "limitations": reproductionLimits}, nil) + return + } + f.Cursor, f.Through, f.FixedCutoff = "", page.Through, true + w.Header().Set("Content-Type", "text/markdown; charset=utf-8") + w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="session-%d-reproduction.md"`, id)) + s.writeAuditGuide(w, r, meta, f, page) +} + +// md makes recorded text inert in the guide. CommonMark lets any ASCII punctuation character be +// backslash-escaped, so escaping every one leaves no link, image, emphasis, code span, HTML, +// entity, math or bare-address autolink to form from text an agent controls, and no `|` to split +// a table cell, while a viewer still shows exactly what was recorded. A line break would end a +// table row, so it is a space. +func md(s string) string { + var b strings.Builder + b.Grow(len(s) + len(s)/4) + for _, r := range s { + switch { + case r == '\r' || r == '\n': + b.WriteByte(' ') + case strings.ContainsRune(markdownPunctuation, r): + b.WriteByte('\\') + b.WriteRune(r) + default: + b.WriteRune(r) + } + } + return b.String() +} + +const markdownPunctuation = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" diff --git a/internal/serve/audit_markdown.go b/internal/serve/audit_markdown.go new file mode 100644 index 00000000..133d1c57 --- /dev/null +++ b/internal/serve/audit_markdown.go @@ -0,0 +1,343 @@ +package serve + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "slices" + "strconv" + "strings" + + "github.com/advanced-security/codeql-abom/internal/audit" + "github.com/advanced-security/codeql-abom/internal/store" +) + +// The guide is a presentation of the same fixed-cutoff evidence as JSONL. It reads +// no referenced file, follows no artifact URL and derives no progress verdict. +type auditGuide struct { + w io.Writer + err error +} + +func (g *auditGuide) printf(format string, args ...any) { + if g.err == nil { + _, g.err = fmt.Fprintf(g.w, format, args...) + } +} + +func guideCode(value string) string { + if value == "" { + return "Not recorded" + } + // Markdown inside an HTML element is still Markdown, so the value is escaped as any recorded + // text is; the element only sets it in monospace. + return "" + md(value) + "" +} + +func guideCredits(p *audit.Policy) string { + if !p.CreditsKnown { + return "AI credit cap not recorded" + } + if p.MaxAICredits == nil { + return "No AI credit cap (Copilot plan limits still apply)" + } + return fmt.Sprintf("Capped at %g AI credits, enforced by Copilot", *p.MaxAICredits) +} + +func guideCondition(p *audit.Policy) string { + if p.Bounds.QueryHeld == "" { + return "No query hold recorded in this policy." + } + return p.Bounds.QueryHeld +} + +func guideBoundaryList(values []string) string { + if len(values) == 0 { + return "No entries recorded" + } + return strings.Join(values, "; ") +} + +func (g *auditGuide) policy(p *audit.Policy) { + g.printf("| Limit / boundary | Recorded value |\n| --- | --- |\n") + timeout := "Not recorded" + if p.TimeoutMs > 0 { + timeout = fmt.Sprintf("%g min (%s)", float64(p.TimeoutMs)/60000, p.TimeoutSource) + } + mcpTimeout := "Not configured in this policy" + if p.MCPTimeoutMs > 0 { + mcpTimeout = fmt.Sprintf("%g min", float64(p.MCPTimeoutMs)/60000) + } + for _, row := range [][2]string{ + {"Session timeout", timeout}, {"AI credits", guideCredits(p)}, {"Deadline", p.Deadline}, + {"MCP call timeout", mcpTimeout}, {"Target", p.Bounds.Target}, + {"Conditions", guideCondition(p)}, {"Read-only", guideBoundaryList(p.Bounds.ReadOnly)}, + {"Denied scope", guideBoundaryList(p.Bounds.Denied)}, {"Excluded from edits", guideBoundaryList(p.Bounds.ExcludedNames)}, + } { + g.printf("| %s | %s |\n", row[0], guideCode(row[1])) + } + g.printf("\n**Available tools** (still subject to bounds): ") + for i, tool := range p.Tools { + if i > 0 { + g.printf(", ") + } + g.printf("%s", guideCode(tool)) + } + g.printf("\n\n**Editable paths:**\n\n") + if len(p.Bounds.Writable) == 0 { + g.printf("No editable files in this policy.\n\n") + } + for _, path := range sortedAuditKeys(p.Bounds.Writable) { + role := p.Bounds.Writable[path] + if role == "" { + role = "query, subject to the recorded conditions" + } + g.printf("- %s — %s\n", guideCode(path), guideCode(role)) + } + g.printf("\nFixture root: %s. Extensions: %s.\n\n", guideCode(p.Bounds.FixtureRoot), guideCode(strings.Join(p.Bounds.FixtureExtensions, ", "))) + g.printf("**Runtime identities:** model %s", guideCode(p.Model)) + for _, key := range sortedAuditKeys(p.Runtime) { + g.printf("; %s: %s", md(key), guideCode(p.Runtime[key])) + } + g.printf(". Other versions were not recorded.\n\n") +} + +type guideInput struct{ name, value string } + +func guideInputs(e store.AuthoringEvent) []guideInput { + var values []guideInput + for _, key := range sortedAuditKeys(e.Capture.Inputs) { + values = append(values, guideInput{key, e.Capture.Inputs[key]}) + } + if test := e.Capture.Test; test != nil { + values = append(values, guideInput{"Query version", test.QueryVersion}, guideInput{"Test input digest", test.TestDigest}, guideInput{"Fixture digest", test.FixturesDigest}) + } + return values +} + +func sortedAuditKeys[V any](values map[string]V) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} + +type auditGuideIndex struct { + seen map[string]bool + additional bool + iterations []store.AuthoringEvent + moreIterations bool +} + +func (index *auditGuideIndex) observe(g *auditGuide, e store.AuthoringEvent) error { + if e.Capture.Test != nil { + if len(index.iterations) < 48 { + index.iterations = append(index.iterations, e) + } else { + index.moreIterations = true + } + } + for _, input := range guideInputs(e) { + key := input.name + "\x00" + input.value + if index.seen[key] { + continue + } + if len(index.seen) >= 48 { + index.additional = true + continue + } + index.seen[key] = true + g.printf("| %s | %s | [Event #%d](/codeql?tab=audit&session=%d&event=%d) |\n", md(input.name), guideCode(input.value), e.RowID, e.SessionID, e.RowID) + } + return g.err +} + +func (s *Server) writeAuditGuide(w http.ResponseWriter, r *http.Request, meta store.AuditSession, f store.AuditFilter, page store.AuditPage) { + g := &auditGuide{w: w} + g.printf("# Session %d — recorded reproduction evidence\n\n", meta.RowID) + g.printf("Fixed cutoff: event #%d; %d matching events. %s\n\n", page.Through, page.Total, reproductionLimits) + g.printf("## Manual reproduction checklist\n\n" + + "1. Read the originating request and the policy applicable to each recorded event.\n" + + "2. Obtain the recorded query, pack dependencies, help, marked fixtures and expected results independently. Compare the identities below; source and scratch are not archived.\n" + + "3. Use the recorded CodeQL CLI and tool arguments to prepare an equivalent manual operation. Inspect refusals first; this guide neither authorizes nor executes those operations.\n" + + "4. Compare each measured test result, failure stage and missing/unexpected count. A tool return or model message is not proof of a passing test.\n\n") + g.printf("## Session and limits\n\nPurpose: %s. Current recorded state: %s. Started by: %s.\n\n", md(meta.Purpose), md(meta.State), md(meta.StartedBy)) + var latest *store.AuthoringEvent + for i := range meta.Policies { + if meta.Policies[i].Capture.Policy != nil && (latest == nil || meta.Policies[i].RowID > latest.RowID) { + latest = &meta.Policies[i] + } + } + if latest == nil { + g.printf("Limits and permissions were not recorded. Current settings cannot describe this historical session.\n\n") + } else { + g.printf("Latest policy at this cutoff: [event #%d](/codeql?tab=audit&session=%d&event=%d). Earlier steps retain their own policy references.\n\n", latest.RowID, meta.RowID, latest.RowID) + g.policy(latest.Capture.Policy) + } + for _, relation := range meta.Relations { + g.printf("- %s:", md(relation.Kind)) + if relation.RequestID > 0 { + g.printf(" [request #%d](/codeql?tab=requests&request=%d)", relation.RequestID, relation.RequestID) + } + if relation.RunID > 0 { + g.printf(" [run #%d](/codeql?tab=runs&run=%d)", relation.RunID, relation.RunID) + } + if relation.QueryID != "" { + g.printf(" %s; version %s", guideCode(relation.QueryID), guideCode(relation.Version)) + } + g.printf("\n") + } + g.printf("\n## Required inputs\n\nThese are identities or references, not recovered source. Availability and missing-input notes are retained verbatim. Pack dependencies, help and expected results without recorded identities must be obtained and verified independently.\n\n" + + "| Input / reference | Recorded digest or value | Evidence |\n| --- | --- | --- |\n") + // A compact index is bounded independently of the number of events. Every + // step below still carries its own full input observations. + index := auditGuideIndex{seen: map[string]bool{}} + if page.Through > 0 { + err := s.Store.WalkAudit(f, func(e store.AuthoringEvent) error { + if err := r.Context().Err(); err != nil { + return err + } + return index.observe(g, e) + }) + if err != nil { + return + } + } + if len(index.seen) == 0 { + g.printf("| Required query, fixtures, expectations and dependencies | No input identities recorded | See the original request |\n") + } + if index.additional { + g.printf("\nThe compact index shows 48 distinct references. Additional input observations remain with their steps below; they are not omitted from this guide.\n") + } + g.printf("\n## Measured test iterations\n\n") + if len(index.iterations) == 0 { + g.printf("No measured test iteration matches these filters. Tool-return events and model messages are not substitutes.\n") + } else { + g.printf("| Evidence | Outcome | Compared tests | Query version | CodeQL CLI |\n| --- | --- | --- | --- | --- |\n") + for _, e := range index.iterations { + test := e.Capture.Test + g.printf("| [Event #%d](/codeql?tab=audit&session=%d&event=%d) | %s | %d of %d passed | %s | %s |\n", + e.RowID, e.SessionID, e.RowID, guideTestOutcome(test), test.Total-test.Failed, test.Total, guideCode(test.QueryVersion), guideCode(test.CLIVersion)) + } + } + if index.moreIterations { + g.printf("\nThis compact index shows the first 48 matching iterations. Every iteration, including the remainder, is retained with its step below.\n") + } + g.printf("\n## Recorded steps and test iterations\n\nAll matching events follow the selected order. Append IDs establish sequence; ordinary hook observations are not guessed tool-call pairs. Refused operations are evidence to inspect, not instructions to repeat.\n\n") + count := 0 + if page.Through > 0 { + err := s.Store.WalkAudit(f, func(e store.AuthoringEvent) error { + if err := r.Context().Err(); err != nil { + return err + } + count++ + g.event(e, count) + return g.err + }) + if err != nil { + return + } + } + if g.err != nil || r.Context().Err() != nil || count != page.Total { + return + } + terms := r.URL.Query() + terms.Del("download") + terms.Del("cursor") + terms.Set("session", strconv.FormatInt(meta.RowID, 10)) + terms.Set("through", strconv.FormatInt(page.Through, 10)) + g.printf("## Technical evidence appendix\n\n[Download the full filtered JSONL snapshot](/api/authoring-events/export?%s). Each event link above retains access to the complete sanitized envelope and its policy. No new source archive is included.\n\n", terms.Encode()) + g.printf("\n", page.Through, count) +} + +func (g *auditGuide) event(e store.AuthoringEvent, number int) { + g.printf("### %d. %s — %s\n\n[Event #%d](/codeql?tab=audit&session=%d&event=%d) · %s · policy %s\n\n", number, guideCode(e.Tool), md(e.Decision), e.RowID, e.SessionID, e.RowID, md(e.At), guidePolicyID(e.PolicyID)) + g.printf("Recorded observation: %s. Step: %s. Subject: %s.\n\n", guideCode(e.Hook), guideCode(e.Step), guideCode(e.Subject)) + if e.Reason != "" { + g.printf("Recorded reason: %s\n\n", md(e.Reason)) + } + if p := e.Capture.Policy; p != nil { + g.printf("Policy snapshot: %g min timeout; %s. Conditions: %s\n\nRead the linked event for its complete exact tool and file restrictions.\n\n", float64(p.TimeoutMs)/60000, md(guideCredits(p)), md(guideCondition(p))) + } + if test := e.Capture.Test; test != nil { + g.printf("**Measured test: %s — %d of %d passed.**\n\n", guideTestOutcome(test), test.Total-test.Failed, test.Total) + g.printf("| Measurement | Recorded value |\n| --- | --- |\n") + for _, row := range [][2]string{{"Query", test.QueryID}, {"Query version", test.QueryVersion}, {"CodeQL CLI", test.CLIVersion}, {"Failure stage", test.FailureStage}, {"Missing results", guideCount(test.Missing)}, {"Unexpected results", guideCount(test.Unexpected)}} { + g.printf("| %s | %s |\n", row[0], guideCode(row[1])) + } + g.printf("\nRecorded diagnostic: %s\n\n", md(test.Detail)) + } + inputs := guideInputs(e) + if len(inputs) > 0 { + g.printf("| Required input / reference | Recorded value |\n| --- | --- |\n") + for _, input := range inputs { + g.printf("| %s | %s |\n", md(input.name), guideCode(input.value)) + } + g.printf("\n") + } + if e.Detail != "" { + var detail any + if json.Unmarshal([]byte(e.Detail), &detail) == nil { + g.printf("Safe recorded arguments and details (not an executable command):\n\n| Field | Value |\n| --- | --- |\n") + g.fields("", detail) + g.printf("\n") + } else { + g.printf("Safe recorded detail: %s\n\n", md(e.Detail)) + } + } + if e.Capture.ArtifactID > 0 { + g.printf("[Existing printed tree #%d](/api/authoring-asts/%d)\n\n", e.Capture.ArtifactID, e.Capture.ArtifactID) + } +} + +func guideTestOutcome(test *store.QueryTest) string { + if test.Passed { + return "Test passed" + } + switch strings.ToUpper(test.FailureStage) { + case "RESULT": + return "Result comparison failed" + case "COMPILE", "COMPILATION": + return "Compilation failed" + case "EXTRACT", "EXTRACTION": + return "Extraction failed" + default: + return "Test failed" + } +} + +func (g *auditGuide) fields(prefix string, value any) { + switch typed := value.(type) { + case map[string]any: + for _, key := range sortedAuditKeys(typed) { + name := key + if prefix != "" { + name = prefix + "." + key + } + g.fields(name, typed[key]) + } + case []any: + for i, element := range typed { + g.fields(fmt.Sprintf("%s[%d]", prefix, i), element) + } + default: + g.printf("| %s | %s |\n", md(prefix), guideCode(fmt.Sprint(value))) + } +} + +func guideCount(value *int64) string { + if value == nil { + return "Not recorded" + } + return strconv.FormatInt(*value, 10) +} + +func guidePolicyID(id int64) string { + if id == 0 { + return "not recorded" + } + return fmt.Sprintf("event #%d", id) +} diff --git a/internal/serve/audit_stream_test.go b/internal/serve/audit_stream_test.go new file mode 100644 index 00000000..9fbf307e --- /dev/null +++ b/internal/serve/audit_stream_test.go @@ -0,0 +1,40 @@ +package serve + +import ( + "context" + "net/http/httptest" + "strings" + "testing" + + "github.com/advanced-security/codeql-abom/internal/store" +) + +func TestCancelledAuditDownloadNeverClaimsCompletion(t *testing.T) { + s, st, _, request := authoringServer(t) + session, err := st.StartAuthoringSession(request, "operator", "test") + if err != nil { + t.Fatal(err) + } + if _, err := st.RecordAuthoringEvent(store.AuthoringEvent{SessionID: session, At: "2026-10-01T00:00:00Z", Hook: "pre-tool-use", Tool: "view", Decision: "allowed"}); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + r := localRequest("GET", "/api/authoring-events/export", nil).WithContext(ctx) + w := httptest.NewRecorder() + s.Handler().ServeHTTP(w, r) + if strings.Contains(w.Body.String(), `"complete":true`) { + t.Fatal("canceled export claimed completeness") + } +} + +func TestUnknownAuditOwnersAreNotSuccessfulEmptyLogs(t *testing.T) { + s, _, _, _ := authoringServer(t) + for _, path := range []string{"/api/authoring-events?request=9999", "/api/authoring-events?run=9999"} { + w := httptest.NewRecorder() + s.Handler().ServeHTTP(w, localRequest("GET", path, nil)) + if w.Code != 404 { + t.Errorf("%s answered %d, want missing owner", path, w.Code) + } + } +} diff --git a/internal/serve/audit_test.go b/internal/serve/audit_test.go new file mode 100644 index 00000000..fafab732 --- /dev/null +++ b/internal/serve/audit_test.go @@ -0,0 +1,224 @@ +package serve + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "regexp" + "strings" + "testing" + + "github.com/advanced-security/codeql-abom/internal/audit" + "github.com/advanced-security/codeql-abom/internal/store" +) + +func TestReproductionLabelsRecordedEmptyPolicyBoundaries(t *testing.T) { + var body bytes.Buffer + g := &auditGuide{w: &body} + g.policy(&audit.Policy{TimeoutMs: 600000, CreditsKnown: true}) + for _, wanted := range []string{ + "No AI credit cap (Copilot plan limits still apply)", "No editable files in this policy.", + "No query hold recorded in this policy.", "| Denied scope | No entries recorded |", + } { + if !strings.Contains(rendered(body.String()), wanted) { + t.Errorf("recorded empty policy boundary missing %q", wanted) + } + } + // The cap is Copilot's on one session; the product's own name explains nothing about it. + if strings.Contains(body.String(), "ABOM") { + t.Errorf("the guide names ABOM in a policy it did not set:\n%s", body.String()) + } +} + +// rendered is the guide as a Markdown viewer shows its text: every backslash escape resolved. +func rendered(markdown string) string { + return regexp.MustCompile("\\\\([!-/:-@\\[-`{-~])").ReplaceAllString(markdown, "$1") +} + +// What an agent writes reaches the downloaded guide verbatim. Opened in a Markdown viewer it drew +// the agent's links as recorded evidence and loaded its images, whose address carries out whatever +// the agent put in it; the guide's own links are the only ones it may hold. +func TestRecordedTextCannotFormLinksOrImagesInTheGuide(t *testing.T) { + s, st, _, request := authoringServer(t) + session, err := st.StartAuthoringSession(request, "operator", "test") + if err != nil { + t.Fatal(err) + } + said := strings.Join([]string{ + "![status](https://attacker.example/x/leak.png)", + "[Open the passing test log](https://attacker.example/login)", + `\[Approve\](https://attacker.example/p)`, + "", + "", + "www.attacker.example/www", + "*bold* _under_ `tick` $math$ | pipe", + }, " ") + detail, err := json.Marshal(map[string]any{"message": said, "[key](https://attacker.example/k)": said}) + if err != nil { + t.Fatal(err) + } + if _, err := st.RecordAuthoringEvent(store.AuthoringEvent{SessionID: session, At: "2026-10-01T00:00:00Z", + Hook: "message", Tool: said, Subject: said, Reason: said, Decision: "refused", Detail: string(detail), + Capture: store.AuditCapture{Version: 1, Inputs: map[string]string{said: said}, + Test: &store.QueryTest{QueryID: said, QueryVersion: said, Total: 1, Failed: 1, FailureStage: "RESULT", Detail: said}}, + }); err != nil { + t.Fatal(err) + } + + w := httptest.NewRecorder() + s.Handler().ServeHTTP(w, localRequest("GET", fmt.Sprintf("/api/authoring-sessions/%d/reproduction?download=1", session), nil)) + body := w.Body.String() + if w.Code != http.StatusOK || !strings.Contains(body, " + + + diff --git a/ui/wwwroot/js/build.js b/ui/wwwroot/js/build.js new file mode 100644 index 00000000..b53d5009 --- /dev/null +++ b/ui/wwwroot/js/build.js @@ -0,0 +1,85 @@ +// A server restarted onto a new build keeps answering pages loaded from the old one, and those pages +// keep working, so nothing on them says what they are missing: a tab the server now offers looks +// absent rather than unloaded. This compares what the page booted from with what the server serves +// now, and says so when they differ. Only a positive difference counts; a server that cannot be +// reached, or an answer that is not the interface's own page, is not evidence of a newer build. +window.abomBuild = (function () { + var notice = document.getElementById('abom-stale-build'); + var map = document.querySelector('script[type="importmap"]'); + var booted = map ? map.textContent.trim() : ''; + // First-party files loaded by a fixed name. The framework's are fingerprinted in the import + // map, and the vendored libraries change only as a deliberate upgrade. + var assets = Array.prototype.map.call( + document.querySelectorAll('link[rel="stylesheet"][href], script[src]'), + function (el) { return el.getAttribute('href') || el.getAttribute('src'); }) + .filter(function (path) { return /^(css|js)\//.test(path) && !/\.min\.js$/.test(path); }); + var dates = {}; + var stale = false; + var busy = false; + var last = 0; + + function served(html) { + var found = /`; + +// What index.html loads by a fixed name, beside what it does not need to watch. +const LOADED = [ + ['href', '_content/MudBlazor/MudBlazor.min.css'], + ['href', 'css/app.css'], + ['href', 'css/tutorial.css'], + ['src', 'js/three.min.js'], + ['src', 'js/theme.js'], + ['src', 'js/tutorial.js'], + ['src', 'js/tipfit.js'], + ['src', 'js/build.js'], + ['src', '_framework/blazor.webassembly.66stpp682q.js'], +]; + +// build.js is a plain script, like theme.js: index.html loads it before Blazor exists, so it +// publishes onto window, and evaluating the source is the only way to reach it from here. +function load({ served = () => page('aaa'), dates = () => ({}), unreachable = false } = {}) { + const asked = []; + const reloads = []; + let pressed; + const notice = { + hidden: true, + querySelector: () => ({ addEventListener: (_, fn) => { pressed = fn; } }), + }; + const document = { + hidden: false, + getElementById: (id) => (id === 'abom-stale-build' ? notice : null), + querySelector: (selector) => (selector === 'script[type="importmap"]' ? { textContent: `\n${importMap('aaa')}\n` } : null), + querySelectorAll: () => LOADED.map(([attribute, value]) => ({ + getAttribute: (name) => (name === attribute ? value : null), + })), + addEventListener: () => {}, + }; + const window = { addEventListener: () => {} }; + const fetch = async (url, init = {}) => { + asked.push(`${init.method ?? 'GET'} ${url}`); + if (unreachable) throw new TypeError('Failed to fetch'); + if (init.method === 'HEAD') { + const date = dates()[url]; + return { ok: true, headers: { get: (name) => (name.toLowerCase() === 'last-modified' ? date ?? null : null) } }; + } + return { ok: true, text: async () => served() }; + }; + const location = { reload: () => reloads.push(true) }; + const source = readFileSync(fileURLToPath(new URL('./build.js', import.meta.url)), 'utf8'); + new Function('window', 'document', 'fetch', 'location', 'setInterval', source)( + window, document, fetch, location, () => 0); + return { build: window.abomBuild, notice, asked, reloads, press: () => pressed() }; +} + +const MONDAY = 'Mon, 12 Oct 2026 07:41:20 GMT'; +const TUESDAY = 'Tue, 13 Oct 2026 07:41:20 GMT'; + +test('a page the server still serves says nothing', async () => { + const { build, notice } = load({ dates: () => ({ 'css/app.css': MONDAY }) }); + await build.ready; + + assert.equal(await build.check(), false); + assert.equal(notice.hidden, true); +}); + +// The reported failure: `make restart` replaced the server, and a page loaded the night before +// kept running the build that had no Audit log tab, so the tab looked missing rather than unloaded. +test('a page running an earlier build than the server says so', async () => { + const { build, notice } = load({ served: () => page('bbb') }); + await build.ready; + + assert.equal(await build.check(), true); + assert.equal(notice.hidden, false); +}); + +// A stylesheet is not fingerprinted, so a change to it alone never reaches the import map. +test('a restyled interface is a newer build too', async () => { + let date = MONDAY; + const { build, notice } = load({ dates: () => ({ 'css/app.css': date }) }); + await build.ready; + date = TUESDAY; + + assert.equal(await build.check(), true); + assert.equal(notice.hidden, false); +}); + +test('only first-party files loaded by a fixed name are watched', async () => { + const { build, asked } = load(); + await build.ready; + + assert.deepEqual(asked.filter((a) => a.startsWith('HEAD')).sort(), + ['HEAD css/app.css', 'HEAD css/tutorial.css', 'HEAD js/build.js', 'HEAD js/theme.js', 'HEAD js/tipfit.js', 'HEAD js/tutorial.js']); +}); + +// A server restarting refuses the connection for a moment; that is not news about the build. +test('a server that cannot be reached is not a newer build', async () => { + const { build, notice } = load({ unreachable: true }); + await build.ready; + + assert.equal(await build.check(), false); + assert.equal(notice.hidden, true); +}); + +// A proxy's error page, or anything that is not the interface's own entry point, has no import +// map, and the absence of one says nothing about which build the server holds. +test('an answer that is not the interface page proves nothing', async () => { + const { build, notice } = load({ served: () => 'Bad gateway' }); + await build.ready; + + assert.equal(await build.check(), false); + assert.equal(notice.hidden, true); +}); + +test('a stylesheet answered without a date proves nothing', async () => { + let date = MONDAY; + const { build } = load({ dates: () => ({ 'css/app.css': date }) }); + await build.ready; + date = undefined; + + assert.equal(await build.check(), false); +}); + +test('the notice reloads the page into the build the server serves', async () => { + const { build, reloads, press } = load({ served: () => page('bbb') }); + await build.ready; + await build.check(); + + press(); + assert.equal(reloads.length, 1); +}); diff --git a/ui/wwwroot/js/tipfit.js b/ui/wwwroot/js/tipfit.js new file mode 100644 index 00000000..5e2ceab7 --- /dev/null +++ b/ui/wwwroot/js/tipfit.js @@ -0,0 +1,53 @@ +// A MudBlazor tip is placed on the side of its control it was asked for, and flipped to the other +// side when that one has no room; it is never moved along the side it is on. Beside a control near +// the edge of the window, one ran past it and widened the page. This slides an open tip back inside +// the window along its own side, and changes nothing else about it. +(function () { + var MARGIN = 8; + + // How far to move a box so it lies inside the window less a margin. One wider than that starts + // at the left margin, so its first words are the ones read. + function shift(rect, width) { + if (rect.width + 2 * MARGIN >= width || rect.left < MARGIN) return MARGIN - rect.left; + if (rect.right > width - MARGIN) return width - MARGIN - rect.right; + return 0; + } + + // Measured from where MudBlazor placed it, which is where it is now less the move already on + // it: fitting again changes nothing, and a tip placed again is fitted from its new place. + // Compared as numbers, because a browser reads a translate back in its own spelling. + function fit(tip) { + var current = parseFloat(tip.style.translate) || 0; + var rect = tip.getBoundingClientRect(); + var placed = { left: rect.left - current, right: rect.right - current, width: rect.width }; + var dx = Math.round(shift(placed, document.documentElement.clientWidth)); + if (dx !== current) tip.style.translate = dx ? dx + 'px' : ''; + } + + window.abomTipFit = { shift: shift, fit: fit }; + + if (typeof MutationObserver === 'undefined' || !document.body) return; + + var pending = false; + function fitOpen() { + pending = false; + var open = document.querySelectorAll('.mud-tooltip.mud-popover-open'); + for (var i = 0; i < open.length; i++) fit(open[i]); + } + function soon() { + if (pending) return; + pending = true; + requestAnimationFrame(fitOpen); + } + // A tip opens by gaining a class and is placed by its style, both on the tip itself. + new MutationObserver(function (records) { + for (var i = 0; i < records.length; i++) { + var target = records[i].target; + if (target.classList && target.classList.contains('mud-tooltip')) { + soon(); + return; + } + } + }).observe(document.body, { subtree: true, attributes: true, attributeFilter: ['class', 'style'] }); + window.addEventListener('resize', soon); +})(); diff --git a/ui/wwwroot/js/tipfit.test.js b/ui/wwwroot/js/tipfit.test.js new file mode 100644 index 00000000..b293f447 --- /dev/null +++ b/ui/wwwroot/js/tipfit.test.js @@ -0,0 +1,60 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +// tipfit.js is a plain script, like theme.js: index.html loads it before Blazor exists and it +// publishes onto window. Evaluating the source is the only way to reach it from here. +function load(width) { + const window = {}; + const document = { documentElement: { clientWidth: width } }; + const source = readFileSync(fileURLToPath(new URL('./tipfit.js', import.meta.url)), 'utf8'); + new Function('window', 'document', source)(window, document); + return window.abomTipFit; +} + +// What MudBlazor leaves: a box at `left`, moved only by whatever translate is on it now. +function tip(left, width) { + return { + left, + style: { translate: '' }, + getBoundingClientRect() { + const dx = parseFloat(this.style.translate) || 0; + return { left: this.left + dx, right: this.left + width + dx, width }; + }, + }; +} + +test('a tip inside the window stays where it was placed', () => { + const { shift } = load(390); + assert.equal(shift({ left: 30, right: 350, width: 320 }, 390), 0); +}); + +test('a tip past either edge moves back by its overflow and a margin', () => { + const { shift } = load(390); + assert.equal(shift({ left: 200, right: 520, width: 320 }, 390), -138); + assert.equal(shift({ left: -40, right: 280, width: 320 }, 390), 48); +}); + +test('a tip wider than the window starts at its left margin', () => { + const { shift } = load(300); + assert.equal(shift({ left: 40, right: 360, width: 320 }, 300), -32); +}); + +// The fit is read back off the element it moved, so running it again, as every mutation of the +// tip does, changes nothing; and a tip MudBlazor has since placed somewhere else is fitted from +// where it now is, not from where it was. +test('fitting a tip again changes nothing, and a tip placed again is fitted afresh', () => { + const { fit } = load(390); + const el = tip(200, 320); + fit(el); + assert.equal(el.style.translate, '-138px'); + assert.equal(el.getBoundingClientRect().right, 382); + fit(el); + assert.equal(el.style.translate, '-138px'); + + el.left = 10; + fit(el); + assert.equal(el.style.translate, ''); + assert.equal(el.getBoundingClientRect().left, 10); +}); From bca2d077866f60f44c5d9679f0849785c42cdd7f Mon Sep 17 00:00:00 2001 From: Nathan Randall <70299490+data-douser@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:01:34 -0600 Subject: [PATCH 3/4] Update Graphviz installation instructions in source.md Clarify Graphviz installation requirements for Ubuntu 26.04 and later. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: Nathan Randall <70299490+data-douser@users.noreply.github.com> --- docs/deployment/source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deployment/source.md b/docs/deployment/source.md index 4c59e7cf..1fac9f19 100644 --- a/docs/deployment/source.md +++ b/docs/deployment/source.md @@ -21,7 +21,7 @@ flowchart LR | .NET SDK | The major version named by `TargetFramework` in [the UI project](../../ui/Abom.WebAssembly.csproj). `dotnet --list-sdks` shows what is installed. No extra workloads are needed. | | GitHub CLI | [gh](https://cli.github.com/), for authentication and ownership verification. | | CodeQL | The version in [.codeql-version](../../.codeql-version), on `PATH`. See [install CodeQL](#install-codeql). | -| Graphviz | [Graphviz](https://graphviz.org/download/), with `dot` on `PATH`: `brew install graphviz` or `apt-get install graphviz`. Every details page draws its figure with it, and the Explorer's stable layout uses it. `serve` refuses to start without it and names the version it found. From Ubuntu 26.04 the other layout engines, which `make test` exercises, are a separate package: `apt-get install graphviz libgvplugin-neato-layout8`. [setup-graphviz.sh](../../scripts/setup-graphviz.sh) installs and checks them as CI does. | +| Graphviz | [Graphviz](https://graphviz.org/download/), with `dot` on `PATH`: `brew install graphviz` or `apt-get install graphviz`. Every details page draws its figure with it, and the Explorer's stable layout uses it. `serve` refuses to start without it and names the version it found. On Ubuntu 26.04 and later, `make test` also needs the separately packaged neato layout plugin; run [setup-graphviz.sh](../../scripts/setup-graphviz.sh) as CI does, or install the available `libgvplugin-neato-layout*` package for your release. | These are the only requirements for serving and analyzing. Node, Playwright and golangci-lint are development tools. AI authoring and explicit lock verification have From 1b85a828ac70167ae2d33d3b815da0f48d288b1a Mon Sep 17 00:00:00 2001 From: Nathan Randall <70299490+data-douser@users.noreply.github.com> Date: Thu, 15 Oct 2026 20:20:02 -0600 Subject: [PATCH 4/4] Fix script execution command for Graphviz setup Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: Nathan Randall <70299490+data-douser@users.noreply.github.com> --- .github/workflows/update-codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-codeql.yml b/.github/workflows/update-codeql.yml index d6e92305..33e93c4c 100644 --- a/.github/workflows/update-codeql.yml +++ b/.github/workflows/update-codeql.yml @@ -208,7 +208,7 @@ jobs: # Graphviz is a requirement: serve refuses to start without it, and the layout and figure # tests fail rather than skip. - name: Upgrade - Setup Graphviz - run: scripts/setup-graphviz.sh + run: bash scripts/setup-graphviz.sh - name: Upgrade - Verify run: make verify