From a9494568eae0e210136337e97b3eaf76e498e727 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 02:13:50 +0200 Subject: [PATCH 01/56] feat: run firstmate as a Kubernetes agent OS --- .agents/skills/kubernetes-fleet/SKILL.md | 26 ++ .dockerignore | 16 ++ AGENTS.md | 1 + Dockerfile | 54 ++++ README.md | 6 + THIRD_PARTY_NOTICES.md | 9 + bin/agent-os-container-entrypoint.sh | 15 + bin/agent-os-crewmate.sh | 98 +++++++ bin/agent-os-local.sh | 50 ++++ deploy/orbstack/kustomization.yaml | 7 + deploy/orbstack/namespace.yaml | 8 + deploy/orbstack/primary.yaml | 74 +++++ deploy/orbstack/rbac.yaml | 23 ++ docs/kubernetes.md | 83 ++++++ .../2026-07-12-orbstack-agent-os-demo.md | 258 ++++++++++++++++++ tests/agent-os-container.test.sh | 22 ++ tests/agent-os-kubernetes.test.sh | 75 +++++ tests/agent-os-local.test.sh | 93 +++++++ 18 files changed, 918 insertions(+) create mode 100644 .agents/skills/kubernetes-fleet/SKILL.md create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 THIRD_PARTY_NOTICES.md create mode 100755 bin/agent-os-container-entrypoint.sh create mode 100755 bin/agent-os-crewmate.sh create mode 100755 bin/agent-os-local.sh create mode 100644 deploy/orbstack/kustomization.yaml create mode 100644 deploy/orbstack/namespace.yaml create mode 100644 deploy/orbstack/primary.yaml create mode 100644 deploy/orbstack/rbac.yaml create mode 100644 docs/kubernetes.md create mode 100644 docs/superpowers/plans/2026-07-12-orbstack-agent-os-demo.md create mode 100755 tests/agent-os-container.test.sh create mode 100755 tests/agent-os-kubernetes.test.sh create mode 100755 tests/agent-os-local.test.sh diff --git a/.agents/skills/kubernetes-fleet/SKILL.md b/.agents/skills/kubernetes-fleet/SKILL.md new file mode 100644 index 000000000..a43913e54 --- /dev/null +++ b/.agents/skills/kubernetes-fleet/SKILL.md @@ -0,0 +1,26 @@ +--- +name: kubernetes-fleet +description: "Operate Agent OS crewmates as Kubernetes Pods with persistent homes and explicit authority boundaries." +user-invocable: false +metadata: + internal: true +--- + +# Kubernetes fleet + +Load this skill only when the current firstmate is running in Kubernetes or is explicitly managing Kubernetes-backed crewmates. + +## Operating contract + +- Keep every crewmate general-purpose; its brief, tools, and authority specialize it for the current task. +- A crewmate normally communicates only with its parent through its terminal, status files, reports, and delivered Git state. +- Use `bin/agent-os-crewmate.sh create ` to create a separate Pod and persistent home. +- Use `status` to inspect it and `delete` only after unique work is checkpointed or delivered. +- Never mount the primary home into a child Pod. +- The demo child receives no Kubernetes ServiceAccount token by default. +- The OrbStack primary's cluster-admin binding is a local-demo trust decision, not a production-safe default. +- Pin an explicit host context with `AGENT_OS_CONTEXT`; the launcher refuses ambient host contexts. + +Use ordinary `kubectl exec`, Herdr, files, and Git to supervise the child. +Do not add a custom inter-agent chat protocol or Task/Run service. + diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..e1767260e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +.git +.github +.pi +.codex +.claude +.grok +.opencode +.no-mistakes +.env +config +data +projects +state +worktrees +.worktrees + diff --git a/AGENTS.md b/AGENTS.md index 16aa0b034..831acb54f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -770,6 +770,7 @@ These skills are not captain-invocable; they are conditional operating reference - `fmx-respond` - load on an `x-mention ` `check:` wake to handle the mention, on an `x-mode-error ...` `check:` wake to report the X-mode configuration blocker, and on any milestone or terminal wake for an X-mode-linked task before posting its completion follow-up; relevant only when X mode is on. - `firstmate-codexapp` - load before coordinating a visible Codex Desktop thread, evaluating a Codex App backend request, or reconciling Codex Desktop host-tool smoke evidence for Firstmate work. - `firstmate-coding-guidelines` - load before changing firstmate's shared, tracked material, as defined by section 1's list, whether editing directly or briefing a crewmate for a firstmate-repo task. +- `kubernetes-fleet` - load before creating, supervising, recovering, or deleting Kubernetes-backed crewmates. ## 14. X mode diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..82cbec6f2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,54 @@ +FROM node:24-bookworm-slim + +ARG TARGETARCH +ARG HERDR_VERSION=0.7.3 +ARG KUBECTL_VERSION=1.34.8 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + ca-certificates \ + curl \ + git \ + jq \ + openssh-client \ + procps \ + tmux \ + && rm -rf /var/lib/apt/lists/* + +RUN set -eu; \ + case "$TARGETARCH" in \ + amd64) asset=herdr-linux-x86_64; sha=043ef43ecbabda28465dcff1eec3184518150d567b8b8f20cda9c6c88770641d ;; \ + arm64) asset=herdr-linux-aarch64; sha=ea490094f2c7c39099870857d00c64c628ef7b5eba1967df4258033455ee2cb1 ;; \ + *) echo "unsupported TARGETARCH: $TARGETARCH" >&2; exit 1 ;; \ + esac; \ + curl -fsSL "https://github.com/ogulcancelik/herdr/releases/download/v${HERDR_VERSION}/${asset}" -o /usr/local/bin/herdr; \ + echo "$sha /usr/local/bin/herdr" | sha256sum -c -; \ + chmod 0755 /usr/local/bin/herdr; \ + mkdir -p /usr/share/licenses/herdr; \ + curl -fsSL "https://raw.githubusercontent.com/ogulcancelik/herdr/v${HERDR_VERSION}/LICENSE" -o /usr/share/licenses/herdr/LICENSE + +RUN set -eu; \ + case "$TARGETARCH" in \ + amd64) sha=f6249132865c13abe3c9dd5038f5da65849cb86eee1608c001831504e481aa8c ;; \ + arm64) sha=4c9fe1f717738950c638c38056130a8db5075e6413ae36d8687221a240cdf88b ;; \ + *) echo "unsupported TARGETARCH: $TARGETARCH" >&2; exit 1 ;; \ + esac; \ + curl -fsSL "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl" -o /usr/local/bin/kubectl; \ + echo "$sha /usr/local/bin/kubectl" | sha256sum -c -; \ + chmod 0755 /usr/local/bin/kubectl + +RUN npm install --global @earendil-works/pi-coding-agent@0.80.6 + +ENV FM_HOME=/home/agent \ + HOME=/home/agent \ + HERDR_SESSION=default + +RUN mkdir -p /home/agent /opt/agent-os \ + && chown -R node:node /home/agent /opt/agent-os + +COPY --chown=node:node . /opt/agent-os + +USER node +WORKDIR /opt/agent-os +ENTRYPOINT ["/opt/agent-os/bin/agent-os-container-entrypoint.sh"] diff --git a/README.md b/README.md index 6ec155d69..432984eb5 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,11 @@ An agent distro is a portable directory of instructions, skills, tooling, polici There is no app to install: the cloned repo is the distro - `AGENTS.md`, bundled firstmate skills, and helper scripts that any terminal coding agent can follow. Launching a supported harness inside it instantiates your first mate - and makes you the captain. +This Agent OS fork also packages the distro for Kubernetes: one persistent +first mate can allocate isolated, persistent crewmate containers using ordinary +`kubectl`, while Herdr keeps every agent terminal visible. Start locally with +the [OrbStack Kubernetes demo](docs/kubernetes.md). + ## Features - **One liaison** - you talk only to the first mate; it dispatches, supervises, escalates only real decisions, and reports plain outcomes. @@ -188,6 +193,7 @@ Firstmate's skills live in two separate places with different audiences: ## Documentation +- [docs/kubernetes.md](docs/kubernetes.md) - run the Agent OS controller and isolated crewmates on local OrbStack Kubernetes. - [docs/architecture.md](docs/architecture.md) - how the crew, supervision, worktrees, secondmates, and project modes work. - [docs/configuration.md](docs/configuration.md) - environment variables, `FM_HOME`, runtime backend selection, optional X mode, the files you set, and harness support. - [docs/wedge-alarm.md](docs/wedge-alarm.md) - configure the active alert for a wedged away-mode escalation delivery. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 000000000..86fa4f8dc --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,9 @@ +# Third-party notices + +## Herdr + +The Agent OS demo image includes an unmodified Herdr 0.7.3 executable as a separate program. +Herdr is available under AGPL-3.0-or-later or a commercial license. +The image includes Herdr's license at `/usr/share/licenses/herdr/LICENSE`. +The exact corresponding source is available at . +Agent OS invokes Herdr through its documented CLI and socket interfaces. diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh new file mode 100755 index 000000000..57eaf2507 --- /dev/null +++ b/bin/agent-os-container-entrypoint.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# agent-os-container-entrypoint.sh - seed a persistent home and run Herdr. +set -eu + +FM_HOME=${FM_HOME:-/home/agent} +export FM_HOME HOME="$FM_HOME" + +mkdir -p "$FM_HOME/config" "$FM_HOME/data" "$FM_HOME/projects" "$FM_HOME/state" +if [ ! -e "$FM_HOME/config/backend" ]; then + printf 'herdr\n' > "$FM_HOME/config/backend" +fi + +cd /opt/agent-os +exec herdr server + diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh new file mode 100755 index 000000000..32218f87c --- /dev/null +++ b/bin/agent-os-crewmate.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# agent-os-crewmate.sh - create, inspect, or delete one isolated crewmate Pod. +# Usage: bin/agent-os-crewmate.sh create|status|delete +set -eu + +COMMAND=${1:-} +ID=${2:-} +NAMESPACE=${AGENT_OS_NAMESPACE:-agent-os-demo} +IMAGE=${AGENT_OS_IMAGE:-agent-os:dev} +KUBECTL=${AGENT_OS_KUBECTL:-kubectl} + +case "$ID" in + ''|*[!a-z0-9-]*|-*|*-) echo "error: invalid crewmate id '$ID'" >&2; exit 2 ;; +esac + +KUBECTL_ARGS=() +if [ -n "${AGENT_OS_CONTEXT:-}" ]; then + KUBECTL_ARGS=(--context "$AGENT_OS_CONTEXT") +elif [ "${AGENT_OS_IN_CLUSTER:-0}" != 1 ] && [ ! -f /var/run/secrets/kubernetes.io/serviceaccount/token ]; then + echo "error: set AGENT_OS_CONTEXT outside Kubernetes; ambient contexts are refused" >&2 + exit 2 +fi + +POD="agent-os-crewmate-$ID" +PVC="$POD-home" + +case "$COMMAND" in + create) + "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" apply -f - <" >&2 + exit 2 + ;; +esac diff --git a/bin/agent-os-local.sh b/bin/agent-os-local.sh new file mode 100755 index 000000000..03c1830dd --- /dev/null +++ b/bin/agent-os-local.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# agent-os-local.sh - build and operate the local OrbStack Agent OS demo. +# Usage: bin/agent-os-local.sh build|deploy|status|shell|attach|destroy [--yes] +set -eu + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +CONTEXT=${AGENT_OS_CONTEXT:-orbstack} +NAMESPACE=${AGENT_OS_NAMESPACE:-agent-os-demo} +IMAGE=${AGENT_OS_IMAGE:-agent-os:dev} +COMMAND=${1:-} + +if [ "$CONTEXT" != orbstack ] && [ "${AGENT_OS_ALLOW_NON_ORBSTACK:-0}" != 1 ]; then + echo "error: refusing Kubernetes context '$CONTEXT'; set AGENT_OS_ALLOW_NON_ORBSTACK=1 to opt in" >&2 + exit 2 +fi + +cd "$ROOT" + +case "$COMMAND" in + build) + docker build -t "$IMAGE" . + ;; + deploy) + if [ "$CONTEXT" = orbstack ]; then + orbctl start k8s + kubectl --context "$CONTEXT" wait --for=condition=Ready node/orbstack --timeout=120s + fi + kubectl --context "$CONTEXT" apply -k deploy/orbstack + ;; + status) + kubectl --context "$CONTEXT" -n "$NAMESPACE" get statefulset agent-os-firstmate + ;; + shell) + kubectl --context "$CONTEXT" -n "$NAMESPACE" exec -it statefulset/agent-os-firstmate -- bash + ;; + attach) + kubectl --context "$CONTEXT" -n "$NAMESPACE" exec -it statefulset/agent-os-firstmate -- herdr + ;; + destroy) + if [ "${2:-}" != --yes ]; then + echo "error: destroy requires --yes and deletes only namespace '$NAMESPACE'" >&2 + exit 2 + fi + kubectl --context "$CONTEXT" delete namespace "$NAMESPACE" + ;; + *) + echo "usage: $0 build|deploy|status|shell|attach|destroy [--yes]" >&2 + exit 2 + ;; +esac diff --git a/deploy/orbstack/kustomization.yaml b/deploy/orbstack/kustomization.yaml new file mode 100644 index 000000000..57f9eaf5f --- /dev/null +++ b/deploy/orbstack/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - rbac.yaml + - primary.yaml + diff --git a/deploy/orbstack/namespace.yaml b/deploy/orbstack/namespace.yaml new file mode 100644 index 000000000..8e919fbef --- /dev/null +++ b/deploy/orbstack/namespace.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: agent-os-demo + labels: + app.kubernetes.io/name: agent-os + app.kubernetes.io/part-of: agent-os-demo + diff --git a/deploy/orbstack/primary.yaml b/deploy/orbstack/primary.yaml new file mode 100644 index 000000000..7c67cbd6e --- /dev/null +++ b/deploy/orbstack/primary.yaml @@ -0,0 +1,74 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: agent-os-firstmate-home + namespace: agent-os-demo + labels: + app.kubernetes.io/name: agent-os + app.kubernetes.io/component: firstmate +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: agent-os-firstmate + namespace: agent-os-demo + labels: + app.kubernetes.io/name: agent-os + app.kubernetes.io/component: firstmate +spec: + serviceName: agent-os-firstmate + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: agent-os + app.kubernetes.io/component: firstmate + template: + metadata: + labels: + app.kubernetes.io/name: agent-os + app.kubernetes.io/component: firstmate + spec: + serviceAccountName: agent-os-firstmate + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: firstmate + image: agent-os:dev + imagePullPolicy: Never + env: + - name: FM_HOME + value: /home/agent + - name: HOME + value: /home/agent + readinessProbe: + exec: + command: + - herdr + - status + - --json + initialDelaySeconds: 2 + periodSeconds: 5 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "4" + memory: 8Gi + volumeMounts: + - name: home + mountPath: /home/agent + volumes: + - name: home + persistentVolumeClaim: + claimName: agent-os-firstmate-home + diff --git a/deploy/orbstack/rbac.yaml b/deploy/orbstack/rbac.yaml new file mode 100644 index 000000000..db3773b90 --- /dev/null +++ b/deploy/orbstack/rbac.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: agent-os-firstmate + namespace: agent-os-demo +--- +# Local demo only. Production installations require a reviewed narrower role. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: agent-os-firstmate-local-demo + labels: + app.kubernetes.io/name: agent-os + app.kubernetes.io/part-of: agent-os-demo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: agent-os-firstmate + namespace: agent-os-demo + diff --git a/docs/kubernetes.md b/docs/kubernetes.md new file mode 100644 index 000000000..45b0f088b --- /dev/null +++ b/docs/kubernetes.md @@ -0,0 +1,83 @@ +# Kubernetes Agent OS demo + +This demo runs Firstmate as the persistent controller of an isolated agent +cluster. Kubernetes distributes and isolates the crew; it does not replace +Firstmate's supervision model or Herdr's terminal/session interface. + +The local topology is deliberately small: + +- `agent-os-firstmate-0` is a StatefulSet with a 20 Gi persistent home. +- Herdr 0.7.3 is PID 1 and remains the visible session backend. +- the primary's local-demo ServiceAccount has `cluster-admin`, so it can shape + its own crew and workspace; +- every crewmate is an adaptable Agent OS container with its own 10 Gi PVC; +- crewmates do not receive Kubernetes service-account credentials by default. + +The broad primary grant is only for the isolated local agent cluster. It is not +a production-cluster access pattern. + +## Requirements + +- OrbStack with Kubernetes enabled +- Docker +- `kubectl` + +The helper refuses ambient Kubernetes contexts. It uses `orbstack` unless a +different context is deliberately enabled with `AGENT_OS_ALLOW_NON_ORBSTACK=1`. + +## Run the demo + +```sh +bin/agent-os-local.sh build +bin/agent-os-local.sh deploy +bin/agent-os-local.sh status +bin/agent-os-local.sh shell +``` + +Inside the primary container, authenticate the Pi harness using the provider +flow you choose. Host model credentials are intentionally excluded from the +image and are never copied automatically. The authenticated home persists on +the primary PVC. + +Start Pi from the tracked distro and attach to Herdr from another terminal: + +```sh +# inside the primary shell +cd /opt/agent-os +pi + +# on the host +bin/agent-os-local.sh attach +``` + +The primary can create and manage isolated crewmates directly: + +```sh +bin/agent-os-crewmate.sh create scout-1 +bin/agent-os-crewmate.sh status scout-1 +bin/agent-os-crewmate.sh delete scout-1 +``` + +When run on a host, the crewmate helper requires an explicit context: + +```sh +AGENT_OS_CONTEXT=orbstack bin/agent-os-crewmate.sh status scout-1 +``` + +Destroying the demo requires confirmation and deletes only the demo namespace: + +```sh +bin/agent-os-local.sh destroy --yes +``` + +## What the demo proves + +The primary and child homes survive Pod replacement. A child cannot use the +Kubernetes API through an automatically mounted token. The primary can create, +inspect, and delete child Pods and PVCs using ordinary `kubectl`; no custom +agent communication protocol or GitOps controller is required. + +This is the local substrate, not the final access model. Remote Agent OS +clusters should keep the intelligence cluster isolated and grant product or +production access deliberately per task, with Akua providing stable +infrastructure primitives and guardrails. diff --git a/docs/superpowers/plans/2026-07-12-orbstack-agent-os-demo.md b/docs/superpowers/plans/2026-07-12-orbstack-agent-os-demo.md new file mode 100644 index 000000000..75da12bad --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-orbstack-agent-os-demo.md @@ -0,0 +1,258 @@ +# OrbStack Agent OS Demo Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Package this Firstmate fork as a persistent Kubernetes-native Agent OS demo that runs on OrbStack and can launch one isolated crewmate Pod. + +**Architecture:** Keep Firstmate's supervision and Herdr backend unchanged. Add one OCI image, one namespace-scoped manifest set, and thin Bash CLIs that always target an explicit Kubernetes context. The trusted primary Pod gets cluster-admin only for the local demo; crewmate Pods get no service-account token and keep work on their own PVC. + +**Tech Stack:** Bash, Docker, Kubernetes YAML, Kustomize, OrbStack Kubernetes, Herdr 0.7.3, Node.js 24, Pi 0.80.6. + +## Global Constraints + +- Every host-side `kubectl` command must pass `--context`, defaulting to `orbstack`. +- The demo namespace is exactly `agent-os-demo`; scripts must not enumerate, mutate, or delete unrelated namespaces. +- No host model credentials, kubeconfigs, Git credentials, or home directories may enter the image build context. +- The primary home is a PVC mounted at `/home/agent`; each crewmate gets a different PVC. +- Herdr remains an unmodified separate AGPL executable and must ship with its license and exact source link. +- Firstmate remains usable outside Kubernetes; no existing backend contract is replaced. +- Tests use fake `docker`, `kubectl`, and `orbctl` commands and run before any live mutation. + +--- + +### Task 1: Guarded local bootstrap CLI + +**Files:** +- Create: `tests/agent-os-local.test.sh` +- Create: `bin/agent-os-local.sh` + +**Interfaces:** +- Consumes: `docker`, `kubectl`, and optional `orbctl` executables from `PATH`. +- Produces: `bin/agent-os-local.sh build|deploy|status|shell|attach|destroy` and environment variables `AGENT_OS_CONTEXT`, `AGENT_OS_NAMESPACE`, and `AGENT_OS_IMAGE`. + +- [ ] **Step 1: Write the failing CLI test** + +Create a fake-tool directory that appends every invocation to a log, run `status`, `deploy`, and `destroy`, and assert the exact safety contract: + +```bash +PATH="$fakebin:$PATH" AGENT_OS_TEST_LOG="$log" bin/agent-os-local.sh status +grep -F 'kubectl --context orbstack -n agent-os-demo get statefulset agent-os-firstmate' "$log" + +PATH="$fakebin:$PATH" AGENT_OS_TEST_LOG="$log" bin/agent-os-local.sh deploy +grep -F 'kubectl --context orbstack apply -k deploy/orbstack' "$log" + +if PATH="$fakebin:$PATH" bin/agent-os-local.sh destroy 2>/dev/null; then + fail "destroy must require --yes" +fi +``` + +Also assert that `AGENT_OS_CONTEXT=minekube-prod` is rejected unless `AGENT_OS_ALLOW_NON_ORBSTACK=1`, and that no command omits `--context`. + +- [ ] **Step 2: Run the test to verify RED** + +Run: `bash tests/agent-os-local.test.sh` + +Expected: FAIL because `bin/agent-os-local.sh` does not exist. + +- [ ] **Step 3: Implement the minimal guarded CLI** + +Implement strict Bash with these command mappings: + +```bash +build) docker build -t "$IMAGE" . ;; +deploy) kubectl --context "$CONTEXT" apply -k deploy/orbstack ;; +status) kubectl --context "$CONTEXT" -n "$NAMESPACE" get statefulset agent-os-firstmate ;; +shell) kubectl --context "$CONTEXT" -n "$NAMESPACE" exec -it statefulset/agent-os-firstmate -- bash ;; +attach) kubectl --context "$CONTEXT" -n "$NAMESPACE" exec -it statefulset/agent-os-firstmate -- herdr ;; +destroy) [ "${2:-}" = --yes ] || exit 2; kubectl --context "$CONTEXT" delete namespace "$NAMESPACE" ;; +``` + +`deploy` may call `orbctl start k8s` only when the context is `orbstack`; it must then wait with `kubectl --context orbstack wait --for=condition=Ready node/orbstack --timeout=120s`. + +- [ ] **Step 4: Run the test to verify GREEN** + +Run: `bash tests/agent-os-local.test.sh` + +Expected: all assertions print `ok` and exit 0. + +### Task 2: Reproducible container and persistent primary + +**Files:** +- Create: `tests/agent-os-container.test.sh` +- Create: `.dockerignore` +- Create: `Dockerfile` +- Create: `bin/agent-os-container-entrypoint.sh` +- Create: `THIRD_PARTY_NOTICES.md` + +**Interfaces:** +- Consumes: repository source as Docker build context. +- Produces: image `agent-os:dev`, `/opt/agent-os`, persistent `FM_HOME=/home/agent`, Herdr server as PID 1, Pi on `PATH`. + +- [ ] **Step 1: Write the failing static container test** + +Assert: + +```bash +grep -F 'FROM node:24-bookworm-slim' Dockerfile +grep -F 'HERDR_VERSION=0.7.3' Dockerfile +grep -F '@earendil-works/pi-coding-agent@0.80.6' Dockerfile +grep -F 'FM_HOME=/home/agent' Dockerfile +grep -F 'exec herdr server' bin/agent-os-container-entrypoint.sh +grep -F '.git' .dockerignore +grep -F '.pi' .dockerignore +grep -F 'https://github.com/ogulcancelik/herdr/tree/v0.7.3' THIRD_PARTY_NOTICES.md +``` + +Also assert `bash -n bin/agent-os-container-entrypoint.sh` succeeds. + +- [ ] **Step 2: Run the test to verify RED** + +Run: `bash tests/agent-os-container.test.sh` + +Expected: FAIL on the first missing file. + +- [ ] **Step 3: Implement the image and entrypoint** + +Use `node:24-bookworm-slim`, install only `bash`, `ca-certificates`, `curl`, `git`, `jq`, `openssh-client`, `procps`, and `tmux`, and download the architecture-specific Herdr 0.7.3 release binary. +Install Pi at the exact package version. +Copy the repository to `/opt/agent-os`. +The entrypoint must create `$FM_HOME/config`, write `herdr` to `config/backend` only when absent, set `HOME=$FM_HOME`, and execute `herdr server` without copying any credentials. + +- [ ] **Step 4: Run static tests and build the image** + +Run: + +```bash +bash tests/agent-os-container.test.sh +docker build -t agent-os:dev . +docker run --rm --entrypoint bash agent-os:dev -lc 'herdr version && pi --version && test -x /opt/agent-os/bin/fm-spawn.sh' +``` + +Expected: static test exits 0, image builds, Herdr reports 0.7.3, Pi reports 0.80.6, and the Firstmate toolbelt exists. + +### Task 3: Kubernetes manifests and isolated crewmate launcher + +**Files:** +- Create: `tests/agent-os-kubernetes.test.sh` +- Create: `deploy/orbstack/kustomization.yaml` +- Create: `deploy/orbstack/namespace.yaml` +- Create: `deploy/orbstack/rbac.yaml` +- Create: `deploy/orbstack/primary.yaml` +- Create: `bin/agent-os-crewmate.sh` +- Create: `.agents/skills/kubernetes-fleet/SKILL.md` +- Modify: `AGENTS.md` + +**Interfaces:** +- Consumes: image `agent-os:dev`, explicit `kubectl` context on a host or in-cluster service-account configuration in the primary Pod. +- Produces: namespace `agent-os-demo`, StatefulSet `agent-os-firstmate`, PVC `agent-os-firstmate-home`, and `bin/agent-os-crewmate.sh create|status|delete `. + +- [ ] **Step 1: Write failing manifest and launcher tests** + +Assert rendered Kustomize output contains: + +```text +Namespace/agent-os-demo +ServiceAccount/agent-os-firstmate +ClusterRoleBinding/agent-os-firstmate-local-demo +PersistentVolumeClaim/agent-os-firstmate-home +StatefulSet/agent-os-firstmate +imagePullPolicy: Never +``` + +Use a fake `kubectl` to assert `agent-os-crewmate.sh create scout-1` applies exactly one PVC and one Pod in `agent-os-demo`, labels both with `agent-os.akua.dev/crewmate=scout-1`, sets `automountServiceAccountToken: false`, and refuses IDs outside `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`. + +- [ ] **Step 2: Run tests to verify RED** + +Run: `bash tests/agent-os-kubernetes.test.sh` + +Expected: FAIL because the manifests and launcher do not exist. + +- [ ] **Step 3: Implement manifests and launcher** + +The primary StatefulSet must mount the PVC at `/home/agent`, run as UID/GID 1000, request `500m` CPU and `1Gi` memory, limit at `4` CPU and `8Gi`, and use the dedicated ServiceAccount. +The local-demo ClusterRoleBinding may grant `cluster-admin` only to `system:serviceaccount:agent-os-demo:agent-os-firstmate` and must include a comment that production installations need a reviewed narrower role. +The crewmate launcher creates a separate PVC mounted at `/home/agent`, reuses `agent-os:dev`, and disables service-account token automount. + +- [ ] **Step 4: Add the conditional operating skill** + +Add a focused internal skill that tells Firstmate to use the launcher only when it is running in Kubernetes, keep children general-purpose, communicate parent-to-child through terminals/files, preserve unique work on the child PVC, and never claim the local-demo RBAC is production-safe. +Add exactly one trigger line to `AGENTS.md` section 13: load the skill before creating, supervising, recovering, or deleting Kubernetes crewmates. + +- [ ] **Step 5: Run tests to verify GREEN** + +Run: + +```bash +bash tests/agent-os-kubernetes.test.sh +for script in bin/agent-os-*.sh; do bash -n "$script"; done +kubectl kustomize deploy/orbstack >/tmp/agent-os-rendered.yaml +``` + +Expected: all tests exit 0 and rendering succeeds. + +### Task 4: Live OrbStack proof and concise documentation + +**Files:** +- Create: `docs/kubernetes.md` +- Modify: `README.md` + +**Interfaces:** +- Consumes: Tasks 1-3 and an active OrbStack Kubernetes context. +- Produces: reproducible local-demo commands and empirical proof of primary/child persistence. + +- [ ] **Step 1: Document only the verified local path** + +Document `build`, `deploy`, `status`, `shell`, `attach`, crewmate create/status/delete, and `destroy --yes`. +State that no credentials are copied automatically and that the user launches Pi from `/opt/agent-os` after authenticating inside the persistent home. +Add one concise README link; keep mechanism detail in `docs/kubernetes.md`. + +- [ ] **Step 2: Deploy to OrbStack** + +Run: + +```bash +bin/agent-os-local.sh build +bin/agent-os-local.sh deploy +kubectl --context orbstack -n agent-os-demo rollout status statefulset/agent-os-firstmate --timeout=180s +``` + +Expected: the StatefulSet has one Ready replica. + +- [ ] **Step 3: Prove Herdr, RBAC, and persistence** + +Run: + +```bash +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- herdr status --json +kubectl --context orbstack -n agent-os-demo auth can-i create pods --as system:serviceaccount:agent-os-demo:agent-os-firstmate +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- sh -lc 'printf persisted > /home/agent/persistence-proof' +kubectl --context orbstack -n agent-os-demo delete pod agent-os-firstmate-0 +kubectl --context orbstack -n agent-os-demo wait --for=condition=Ready pod/agent-os-firstmate-0 --timeout=180s +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- grep -F persisted /home/agent/persistence-proof +``` + +Expected: Herdr server is compatible, RBAC says `yes`, and the proof survives Pod replacement. + +- [ ] **Step 4: Prove isolated crewmate lifecycle** + +Run the launcher inside the primary Pod to create `scout-1`, wait for readiness, confirm it has a different PVC and no automounted token, write a proof file, restart it, and confirm the file remains. +Delete only `scout-1` through the launcher and confirm the primary remains Ready. + +- [ ] **Step 5: Run final repository verification** + +Run: + +```bash +for script in bin/*.sh bin/backends/*.sh; do bash -n "$script"; done +bin/fm-lint.sh +bash tests/agent-os-local.test.sh +bash tests/agent-os-container.test.sh +bash tests/agent-os-kubernetes.test.sh +git diff --check +``` + +Expected: every command exits 0 with no warnings attributable to the new files. + +- [ ] **Step 6: Commit and ship through the repository gate** + +Commit the reviewed files on `feat/orbstack-demo`, initialize no-mistakes for the `akua-dev/agent-os` push target, run the configured pipeline without `--yes`, and stop at a CI-green PR for the captain's merge decision. diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh new file mode 100755 index 000000000..6aaf81499 --- /dev/null +++ b/tests/agent-os-container.test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Static reproducibility and credential-boundary tests for the Agent OS image. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +assert_grep 'FROM node:24-bookworm-slim' "$ROOT/Dockerfile" "image must pin Node 24 Bookworm" +assert_grep 'ARG HERDR_VERSION=0.7.3' "$ROOT/Dockerfile" "image must pin Herdr 0.7.3" +assert_grep 'ARG KUBECTL_VERSION=1.34.8' "$ROOT/Dockerfile" "image must pin kubectl 1.34.8" +assert_grep 'sha256sum -c -' "$ROOT/Dockerfile" "downloaded runtime binaries must be checksum verified" +assert_grep '@earendil-works/pi-coding-agent@0.80.6' "$ROOT/Dockerfile" "image must pin Pi 0.80.6" +assert_grep 'FM_HOME=/home/agent' "$ROOT/Dockerfile" "image must declare the persistent firstmate home" +assert_grep 'exec herdr server' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must keep Herdr as PID 1" +assert_grep '.git' "$ROOT/.dockerignore" "git metadata must stay out of the build context" +assert_grep '.pi' "$ROOT/.dockerignore" "Pi credentials must stay out of the build context" +assert_grep '.codex' "$ROOT/.dockerignore" "Codex credentials must stay out of the build context" +assert_grep 'https://github.com/ogulcancelik/herdr/tree/v0.7.3' "$ROOT/THIRD_PARTY_NOTICES.md" \ + "Herdr's exact corresponding source must be named" + +bash -n "$ROOT/bin/agent-os-container-entrypoint.sh" +pass "container files pin dependencies and exclude host credentials" diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh new file mode 100755 index 000000000..dfaac9040 --- /dev/null +++ b/tests/agent-os-kubernetes.test.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Kubernetes manifest and isolated crewmate launcher tests. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +LAUNCHER="$ROOT/bin/agent-os-crewmate.sh" +TMP=$(fm_test_tmproot agent-os-kubernetes) +FAKEBIN=$(fm_fakebin "$TMP") +CALLS="$TMP/calls.log" +STDIN_LOG="$TMP/stdin.yaml" + +rendered=$(kubectl kustomize "$ROOT/deploy/orbstack") || fail "OrbStack kustomization did not render" +assert_contains "$rendered" 'kind: Namespace' "render must include the namespace" +assert_contains "$rendered" 'name: agent-os-demo' "render must use the isolated namespace" +assert_contains "$rendered" 'kind: ServiceAccount' "render must include the primary ServiceAccount" +assert_contains "$rendered" 'kind: ClusterRoleBinding' "render must include the explicit local-demo grant" +assert_contains "$rendered" 'name: agent-os-firstmate-home' "render must include the persistent primary home" +assert_contains "$rendered" 'kind: StatefulSet' "render must include the primary StatefulSet" +assert_contains "$rendered" 'imagePullPolicy: Never' "OrbStack must use the locally built image" +assert_contains "$rendered" 'mountPath: /home/agent' "the primary home must mount at FM_HOME" +pass "OrbStack manifests render the isolated persistent primary" + +cat > "$FAKEBIN/kubectl" <<'SH' +#!/usr/bin/env bash +printf 'kubectl' >> "$AGENT_OS_TEST_LOG" +printf ' %s' "$@" >> "$AGENT_OS_TEST_LOG" +printf '\n' >> "$AGENT_OS_TEST_LOG" +if [ "${*: -2}" = "-f -" ]; then + cat > "$AGENT_OS_STDIN_LOG" +fi +SH +chmod +x "$FAKEBIN/kubectl" + +run_launcher() { + PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_STDIN_LOG="$STDIN_LOG" \ + AGENT_OS_IN_CLUSTER=1 "$LAUNCHER" "$@" +} + +: > "$CALLS" +run_launcher create scout-1 +grep -Fqx 'kubectl -n agent-os-demo apply -f -' "$CALLS" || fail "create must apply only to agent-os-demo" +[ "$(grep -Fc 'kind: PersistentVolumeClaim' "$STDIN_LOG")" -eq 1 ] || fail "create must emit one PVC" +[ "$(grep -Fc 'kind: Pod' "$STDIN_LOG")" -eq 1 ] || fail "create must emit one Pod" +assert_grep 'agent-os.akua.dev/crewmate: scout-1' "$STDIN_LOG" "child resources need the stable crewmate label" +assert_grep 'automountServiceAccountToken: false' "$STDIN_LOG" "children must not receive Kubernetes credentials" +assert_grep 'claimName: agent-os-crewmate-scout-1-home' "$STDIN_LOG" "child work must use its own PVC" +pass "crewmate create emits one isolated Pod and PVC" + +: > "$CALLS" +run_launcher delete scout-1 +grep -Fqx 'kubectl -n agent-os-demo delete pod agent-os-crewmate-scout-1 --ignore-not-found' "$CALLS" || \ + fail "delete must target the crewmate Pod explicitly" +grep -Fqx 'kubectl -n agent-os-demo delete pvc agent-os-crewmate-scout-1-home --ignore-not-found' "$CALLS" || \ + fail "delete must target the crewmate PVC explicitly" +pass "crewmate delete removes the Pod and PVC explicitly" + +if run_launcher create 'Bad_ID' >/dev/null 2>&1; then + fail "invalid Kubernetes crewmate IDs must be rejected" +fi +pass "crewmate IDs are validated before kubectl" + +if PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_STDIN_LOG="$STDIN_LOG" \ + "$LAUNCHER" status scout-1 >/dev/null 2>&1; then + fail "host execution without an explicit context must be rejected" +fi +pass "launcher refuses an ambient host Kubernetes context" + +: > "$CALLS" +PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_STDIN_LOG="$STDIN_LOG" \ + AGENT_OS_CONTEXT=orbstack "$LAUNCHER" status scout-1 +grep -Fqx 'kubectl --context orbstack -n agent-os-demo get pod agent-os-crewmate-scout-1' "$CALLS" || \ + fail "host status must pin the selected context" +pass "host launcher calls require and pin an explicit context" diff --git a/tests/agent-os-local.test.sh b/tests/agent-os-local.test.sh new file mode 100755 index 000000000..8c9ce018d --- /dev/null +++ b/tests/agent-os-local.test.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Safety and command-contract tests for the local OrbStack Agent OS CLI. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +CLI="$ROOT/bin/agent-os-local.sh" +TMP=$(fm_test_tmproot agent-os-local) +FAKEBIN=$(fm_fakebin "$TMP") +LOG="$TMP/calls.log" + +make_fake() { + local name=$1 + cat > "$FAKEBIN/$name" <<'SH' +#!/usr/bin/env bash +printf '%s' "$(basename "$0")" >> "$AGENT_OS_TEST_LOG" +printf ' %s' "$@" >> "$AGENT_OS_TEST_LOG" +printf '\n' >> "$AGENT_OS_TEST_LOG" +SH + chmod +x "$FAKEBIN/$name" +} + +make_fake docker +make_fake kubectl +make_fake orbctl + +run_cli() { + PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$LOG" "$CLI" "$@" +} + +assert_call() { + grep -Fqx -- "$1" "$LOG" || fail "$2 (missing exact call: $1)" +} + +test_status_pins_context_and_namespace() { + : > "$LOG" + run_cli status + assert_call 'kubectl --context orbstack -n agent-os-demo get statefulset agent-os-firstmate' \ + "status must target only the OrbStack Agent OS StatefulSet" + pass "status pins the OrbStack context and Agent OS namespace" +} + +test_deploy_starts_local_kubernetes_and_applies_kustomize() { + : > "$LOG" + run_cli deploy + assert_call 'orbctl start k8s' "deploy must start OrbStack Kubernetes" + assert_call 'kubectl --context orbstack wait --for=condition=Ready node/orbstack --timeout=120s' \ + "deploy must wait for the explicit OrbStack node" + assert_call 'kubectl --context orbstack apply -k deploy/orbstack' \ + "deploy must apply only the checked-in OrbStack kustomization" + pass "deploy starts and targets only OrbStack" +} + +test_build_uses_local_image_name() { + : > "$LOG" + run_cli build + assert_call 'docker build -t agent-os:dev .' "build must use the local demo image tag" + pass "build uses the deterministic local image tag" +} + +test_destroy_requires_exact_confirmation() { + local out rc=0 + : > "$LOG" + out=$(run_cli destroy 2>&1) || rc=$? + [ "$rc" -eq 2 ] || fail "destroy without --yes must exit 2, got $rc: $out" + [ ! -s "$LOG" ] || fail "destroy without --yes invoked an external command" + + run_cli destroy --yes + assert_call 'kubectl --context orbstack delete namespace agent-os-demo' \ + "confirmed destroy must delete only agent-os-demo" + pass "destroy requires confirmation and remains namespace-scoped" +} + +test_non_orbstack_context_is_fail_closed() { + local out rc=0 + : > "$LOG" + out=$(PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$LOG" AGENT_OS_CONTEXT=minekube-prod "$CLI" status 2>&1) || rc=$? + [ "$rc" -eq 2 ] || fail "non-OrbStack context must exit 2, got $rc: $out" + [ ! -s "$LOG" ] || fail "rejected non-OrbStack context invoked an external command" + + PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$LOG" AGENT_OS_CONTEXT=kind-agent-os \ + AGENT_OS_ALLOW_NON_ORBSTACK=1 "$CLI" status + assert_call 'kubectl --context kind-agent-os -n agent-os-demo get statefulset agent-os-firstmate' \ + "explicit non-OrbStack opt-in must still pin the chosen context" + pass "non-OrbStack contexts require an explicit opt-in" +} + +test_status_pins_context_and_namespace +test_deploy_starts_local_kubernetes_and_applies_kustomize +test_build_uses_local_image_name +test_destroy_requires_exact_confirmation +test_non_orbstack_context_is_fail_closed From ec8e5e3d8ec6875e1e36242102cd748c79f659e2 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 08:13:57 +0200 Subject: [PATCH 02/56] docs: design persistent root agent toolchains --- ...root-userns-persistent-toolchain-design.md | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-12-root-userns-persistent-toolchain-design.md diff --git a/docs/superpowers/specs/2026-07-12-root-userns-persistent-toolchain-design.md b/docs/superpowers/specs/2026-07-12-root-userns-persistent-toolchain-design.md new file mode 100644 index 000000000..f0a682e77 --- /dev/null +++ b/docs/superpowers/specs/2026-07-12-root-userns-persistent-toolchain-design.md @@ -0,0 +1,109 @@ +# Root user-namespace and persistent toolchain design + +**Date:** 2026-07-12 + +## Purpose + +Agent OS Pods must be able to administer their own container environment without becoming root on the Kubernetes node. +The image must already contain the complete Firstmate toolchain so a fresh controller can begin useful work without an installation round. +Tools and authentication added by an agent at runtime must survive Pod replacement. + +## Decisions + +Every Agent OS Pod runs as UID 0 inside a Kubernetes Pod user namespace. +Every Pod sets `hostUsers: false`, so Kubernetes maps container UID 0 to a distinct unprivileged host UID range. +The deployment fails closed when the node, filesystem, container runtime, or cluster configuration cannot provide Pod user namespaces. +No Agent OS Pod uses privileged mode, a host user namespace, a host PID namespace, a host IPC namespace, a host network namespace, a raw block device, or a host-path mount. + +The image remains the reproducible baseline. +Each agent receives its own PVC-backed home and persistent `/usr/local` tree for runtime adaptation. +No agent mounts another agent's persistent state. + +## Image baseline + +The image includes the universal Firstmate toolchain and the Kubernetes demo runtime: + +- Bash, CA certificates, curl, Git, OpenSSH client, jq, tmux, procps, and rsync. +- GitHub CLI, ripgrep, and fd. +- Node.js, npm, and Pi. +- Herdr and kubectl. +- treehouse and no-mistakes. +- gh-axi, chrome-devtools-axi, lavish-axi, tasks-axi, and quota-axi. + +The Herdr backend does not require Orca. +Agent harnesses other than Pi remain optional runtime additions rather than baseline requirements. +Downloaded release binaries use exact versions and checksums where upstream publishes them. +Global npm packages use exact versions. +The build fails for an unsupported architecture or failed integrity check. + +## Persistent filesystem model + +Every agent PVC contains two independently used trees: + +- `/home/agent` stores Agent OS state, GitHub and model authentication, configuration, projects, package-manager state, and user-installed binaries. +- a PVC subdirectory mounted at `/usr/local` stores global npm packages and tools that installers place in `/usr/local`. + +An init container runs from the same image before the agent starts. +It copies the image's `/usr/local` baseline into the PVC-backed `/usr/local` without deleting agent-installed files. +Image-owned files overwrite older image-owned copies so a new image can update the baseline. +Agent-added paths that do not collide with the baseline remain intact. + +The runtime environment puts `/home/agent/.local/bin`, `/home/agent/.bun/bin`, `/home/agent/.cargo/bin`, and `/usr/local/bin` before system paths. +It sets persistent XDG directories below `/home/agent` and configures npm's global prefix to `/usr/local`. +Installers that honor HOME, XDG paths, npm prefix, Bun home, Cargo home, or `/usr/local` therefore survive Pod replacement. + +`apt` remains available because the process is root inside the container. +Packages installed into `/usr`, `/etc`, or `/var` with `apt` are intentionally ephemeral and disappear on Pod replacement. +Firstmate's required tools do not depend on runtime `apt` because they are part of the image baseline. +An agent that needs an additional durable CLI should prefer `/usr/local` or a persistent home prefix. + +## Pod security and authority + +The primary and crewmate containers set `runAsUser: 0` and `runAsGroup: 0` inside their Pod user namespace. +The init container uses the same user namespace and container-root identity to seed the persistent tool tree. +The Pods use the runtime's normal namespaced capabilities and do not request privileged mode. + +The Firstmate ServiceAccount retains cluster-admin only for the isolated local Agent OS cluster. +Crewmates continue to receive no Kubernetes ServiceAccount token by default. +Container root and Kubernetes API authority remain separate controls. + +Kubernetes v1.34 marks Pod user namespaces beta. +The target node must use a compatible Linux kernel, idmapped-mount filesystem, containerd 2.0 or later or another supported CRI runtime, and a compatible OCI runtime. +The local demo must verify the actual UID map rather than infer support from the Kubernetes version. + +## Startup flow + +1. Kubernetes creates or reattaches the agent's PVC. +2. The init container ensures the persistent home and `/usr/local` directories exist. +3. The init container refreshes image-owned `/usr/local` files while preserving runtime additions. +4. The main container starts as container root with the persistent environment configured. +5. Herdr starts as PID 1. +6. Firstmate bootstrap finds its required tools and reports only missing authentication or genuinely optional additions. + +## Failure behavior + +The demo must refuse to claim readiness when Pod user namespaces are unavailable. +The verification path checks `/proc/self/uid_map` and proves that container UID 0 maps to a nonzero host UID range. +The image build stops on missing releases, unsupported architectures, checksum failures, or missing baseline commands. +The init container stops startup if it cannot seed the persistent tool tree. +No fallback silently removes `hostUsers: false` or grants privileged mode. + +## Verification + +Static tests assert that both primary and generated crewmate Pod specs use `hostUsers: false` and container UID 0. +Container tests assert every baseline command is present and Firstmate bootstrap emits no `MISSING:` diagnostics. +Runtime tests prove the Pod UID map is remapped, root can write to normal container paths, and durable installs can write to both persistent prefixes. +Persistence tests write one tool below `/home/agent/.local/bin` and one below `/usr/local/bin`, replace the Pod, and execute both tools afterward. +Isolation tests prove crewmates have distinct PVCs and no ServiceAccount token. +Authentication verification checks that GitHub CLI configuration survives Pod replacement without embedding credentials in the image. + +## Out of scope + +This change does not persist the entire mutable container root filesystem. +It does not add privileged Pods, host mounts, a custom package service, GitOps, or a new inter-agent protocol. +It does not define production-cluster RBAC beyond retaining the existing local-demo boundary. + +## Sources + +- Kubernetes v1.34 user-namespace documentation: . +- Firstmate's universal toolchain contract: `docs/configuration.md` under `Toolchain` and `bin/fm-bootstrap.sh`. From 3a7d3f4885d253ea0e02730a398e341504ec66b2 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 08:18:00 +0200 Subject: [PATCH 03/56] docs: plan persistent root agent toolchains --- ...-07-12-root-userns-persistent-toolchain.md | 476 ++++++++++++++++++ 1 file changed, 476 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-12-root-userns-persistent-toolchain.md diff --git a/docs/superpowers/plans/2026-07-12-root-userns-persistent-toolchain.md b/docs/superpowers/plans/2026-07-12-root-userns-persistent-toolchain.md new file mode 100644 index 000000000..b665ae958 --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-root-userns-persistent-toolchain.md @@ -0,0 +1,476 @@ +# Root User Namespace and Persistent Toolchain Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Run every Agent OS Pod as container root mapped to an unprivileged host UID while providing Firstmate's complete baseline toolchain and preserving runtime-installed tools across Pod replacement. + +**Architecture:** Extend the reproducible image with exact binary and npm package versions, then seed the image's `/usr/local` tree into each agent PVC from an init container. The main container mounts that tree at `/usr/local`, uses the same PVC for `/home/agent`, and runs as UID 0 inside a Pod user namespace selected by `hostUsers: false`. + +**Tech Stack:** Docker, Bash, Kubernetes v1.34, Kustomize, StatefulSet, Pods, PVCs, Linux user namespaces, Herdr, Pi, Node.js, npm. + +## Global Constraints + +- Every primary and crewmate Pod sets `hostUsers: false`, `runAsUser: 0`, and `runAsGroup: 0`. +- The primary retains the existing local-demo `cluster-admin` ServiceAccount binding, while crewmates receive no Kubernetes credentials by default. +- No Pod uses privileged mode, host PID, host IPC, host networking, raw block devices, or host-path mounts. +- The deployment fails closed when the Pod UID map does not map container UID 0 to a nonzero host UID. +- Each agent has its own PVC-backed `/home/agent` and `/usr/local`; no persistent state is shared between agents. +- Runtime additions in `/usr/local` and persistent home prefixes survive Pod replacement. +- Runtime `apt` changes outside the persistent prefixes remain intentionally ephemeral. +- Host model and GitHub credentials never enter the image or Docker build context. +- Host Kubernetes commands always pin `--context orbstack`. +- Use TDD for every behavior change and run `bin/fm-lint.sh` before shipping. +- Do not run no-mistakes unless the captain asks for it again. + +--- + +### Task 1: Complete the reproducible image toolchain + +**Files:** +- Modify: `Dockerfile` +- Modify: `tests/agent-os-container.test.sh` + +**Interfaces:** +- Consumes: Docker BuildKit `TARGETARCH` values `amd64` and `arm64`. +- Produces: image commands `gh`, `rg`, `fd`, `treehouse`, `no-mistakes`, `gh-axi`, `chrome-devtools-axi`, `lavish-axi`, `tasks-axi`, and `quota-axi`. + +- [ ] **Step 1: Add failing static pin assertions** + +Add these assertions to `tests/agent-os-container.test.sh`: + +```bash +assert_grep 'ARG GH_VERSION=2.96.0' "$ROOT/Dockerfile" "image must pin GitHub CLI 2.96.0" +assert_grep 'ARG TREEHOUSE_VERSION=2.0.0' "$ROOT/Dockerfile" "image must pin treehouse 2.0.0" +assert_grep 'ARG NO_MISTAKES_VERSION=1.34.0' "$ROOT/Dockerfile" "image must pin no-mistakes 1.34.0" +assert_grep 'gh-axi@0.1.27' "$ROOT/Dockerfile" "image must pin gh-axi 0.1.27" +assert_grep 'chrome-devtools-axi@0.1.26' "$ROOT/Dockerfile" "image must pin chrome-devtools-axi 0.1.26" +assert_grep 'lavish-axi@0.1.40' "$ROOT/Dockerfile" "image must pin lavish-axi 0.1.40" +assert_grep 'tasks-axi@0.2.2' "$ROOT/Dockerfile" "image must pin tasks-axi 0.2.2" +assert_grep 'quota-axi@0.1.5' "$ROOT/Dockerfile" "image must pin quota-axi 0.1.5" +assert_grep 'ripgrep' "$ROOT/Dockerfile" "image must install ripgrep" +assert_grep 'fd-find' "$ROOT/Dockerfile" "image must install fd" +``` + +- [ ] **Step 2: Run the test and observe the missing pins** + +Run: `bash tests/agent-os-container.test.sh` + +Expected: FAIL at `image must pin GitHub CLI 2.96.0`. + +- [ ] **Step 3: Install the Debian and exact npm baseline** + +Add `fd-find`, `ripgrep`, and `rsync` to the existing apt package list. +Create `/usr/local/bin/fd` as a symlink to `/usr/bin/fdfind`. +Replace the existing Pi-only npm install with this exact install: + +```dockerfile +RUN npm install --global \ + @earendil-works/pi-coding-agent@0.80.6 \ + gh-axi@0.1.27 \ + chrome-devtools-axi@0.1.26 \ + lavish-axi@0.1.40 \ + tasks-axi@0.2.2 \ + quota-axi@0.1.5 +``` + +- [ ] **Step 4: Install checksum-verified GitHub CLI** + +Add `ARG GH_VERSION=2.96.0` and a `RUN` block that downloads `gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz`, verifies the architecture checksum, and copies `bin/gh` into `/usr/local/bin`. +Use these checksums: + +```text +amd64 83d5c2ccad5498f58bf6368acb1ab32588cf43ab3a4b1c301bf36328b1c8bd60 +arm64 06f86ec7103d41993b76cd78072f43595c34aaa56506d971d9860e67140bf909 +``` + +- [ ] **Step 5: Install checksum-verified treehouse and no-mistakes** + +Add `ARG TREEHOUSE_VERSION=2.0.0` and `ARG NO_MISTAKES_VERSION=1.34.0`. +Download the matching Linux tarball for `TARGETARCH`, verify it, and install its binary into `/usr/local/bin`. +Use these checksums: + +```text +treehouse amd64 b7926c19633ee94582b7f1b58369f22b304ae7228a47253c2148e3a8176f03b0 +treehouse arm64 91bca451bab84df685ee17975c8a9d8cf671b3e95c96b7fc6ff0121ea0aae991 +no-mistakes amd64 449d0276e1b35369ea332dae0eddb5be326c2d4fc9643270af98858cf3906536 +no-mistakes arm64 f157df3e18350edea8abdaa065681bd115a9d321fca86f51e9a0184b3a9d8756 +``` + +- [ ] **Step 6: Run static tests** + +Run: `bash tests/agent-os-container.test.sh` + +Expected: PASS with `container files pin dependencies and exclude host credentials`. + +- [ ] **Step 7: Build and verify every baseline command** + +Run: + +```bash +bin/agent-os-local.sh build +docker run --rm --entrypoint bash agent-os:dev -lc ' + for tool in bash curl git ssh jq tmux ps rsync gh rg fd node npm pi herdr kubectl treehouse no-mistakes gh-axi chrome-devtools-axi lavish-axi tasks-axi quota-axi; do + command -v "$tool" || exit 1 + done + FM_HOME=/tmp/fm HOME=/tmp/fm mkdir -p /tmp/fm/config /tmp/fm/data /tmp/fm/state + FM_HOME=/tmp/fm HOME=/tmp/fm FM_BOOTSTRAP_DETECT_ONLY=1 /opt/agent-os/bin/fm-bootstrap.sh | grep "^MISSING:" && exit 1 || true +' +``` + +Expected: every command resolves and bootstrap emits no `MISSING:` line. + +- [ ] **Step 8: Commit the image baseline** + +```bash +git add Dockerfile tests/agent-os-container.test.sh +git commit -m "feat: bundle the Firstmate toolchain" +``` + +--- + +### Task 2: Add the user-namespace and persistent-prefix initializer + +**Files:** +- Create: `bin/agent-os-init.sh` +- Create: `tests/agent-os-init.test.sh` +- Modify: `Dockerfile` +- Modify: `bin/agent-os-container-entrypoint.sh` + +**Interfaces:** +- Consumes: `AGENT_OS_UID_MAP_PATH`, defaulting to `/proc/self/uid_map`; `AGENT_OS_PERSISTENT_ROOT`, defaulting to `/persistent-agent`. +- Produces: a PVC root used as `/home/agent` and a seeded `$AGENT_OS_PERSISTENT_ROOT/usr-local`; exit zero only for a remapped UID 0 range. + +- [ ] **Step 1: Write failing initializer tests** + +Create `tests/agent-os-init.test.sh` with fixtures for a mapped and host UID map: + +```bash +#!/usr/bin/env bash +set -u +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +TMP=$(fm_test_tmproot agent-os-init) +mkdir -p "$TMP/source/bin" "$TMP/persistent/.local/bin" "$TMP/persistent/usr-local/bin" +printf '0 100000 65536\n' > "$TMP/mapped" +printf '0 0 4294967295\n' > "$TMP/host" +printf 'baseline\n' > "$TMP/source/bin/baseline" +printf 'runtime\n' > "$TMP/persistent/usr-local/bin/runtime-added" + +AGENT_OS_UID_MAP_PATH="$TMP/mapped" \ +AGENT_OS_IMAGE_USR_LOCAL="$TMP/source" \ +AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ + "$ROOT/bin/agent-os-init.sh" + +assert_grep baseline "$TMP/persistent/usr-local/bin/baseline" "initializer must seed image tools" +assert_grep runtime "$TMP/persistent/usr-local/bin/runtime-added" "initializer must preserve runtime tools" + +if AGENT_OS_UID_MAP_PATH="$TMP/host" \ + AGENT_OS_IMAGE_USR_LOCAL="$TMP/source" \ + AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ + "$ROOT/bin/agent-os-init.sh" >/dev/null 2>&1; then + fail "initializer must reject the host user namespace" +fi +pass "initializer verifies remapping and preserves persistent tools" +``` + +- [ ] **Step 2: Run the test and observe the missing initializer** + +Run: `bash tests/agent-os-init.test.sh` + +Expected: FAIL because `bin/agent-os-init.sh` does not exist. + +- [ ] **Step 3: Implement the initializer** + +Create `bin/agent-os-init.sh` as a strict Bash script. +Read three integers from the first UID-map row. +Require inside UID `0`, outside UID greater than `0`, and range at least `65536`. +Create persistent `.config`, `.cache`, `.local/bin`, `.local/share`, `.bun`, `.cargo`, and `usr-local` directories below the PVC root. +Copy the image baseline with `rsync -a "$IMAGE_USR_LOCAL/" "$PERSISTENT_ROOT/usr-local/"` and do not pass `--delete`. + +- [ ] **Step 4: Configure persistent package-manager paths** + +In `Dockerfile`, set: + +```dockerfile +ENV FM_HOME=/home/agent \ + HOME=/home/agent \ + XDG_CONFIG_HOME=/home/agent/.config \ + XDG_DATA_HOME=/home/agent/.local/share \ + XDG_CACHE_HOME=/home/agent/.cache \ + NPM_CONFIG_PREFIX=/usr/local \ + BUN_INSTALL=/home/agent/.bun \ + CARGO_HOME=/home/agent/.cargo \ + PATH=/home/agent/.local/bin:/home/agent/.bun/bin:/home/agent/.cargo/bin:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin \ + HERDR_SESSION=default +``` + +Remove `USER node` and create `/opt/image-usr-local` as a snapshot of the built `/usr/local` tree before runtime mounts hide it. + +- [ ] **Step 5: Make runtime setup persistent and idempotent** + +Update `bin/agent-os-container-entrypoint.sh` to create persistent XDG, local-bin, Bun, and Cargo directories. +After the Herdr backend configuration, install the three AXI SessionStart hooks into persistent HOME only when their marker is absent: + +```bash +for tool in gh-axi chrome-devtools-axi lavish-axi; do + marker="$HOME/.config/agent-os/setup-$tool" + if [ ! -e "$marker" ]; then + "$tool" setup hooks + mkdir -p "$(dirname "$marker")" + : > "$marker" + fi +done +``` + +- [ ] **Step 6: Run initializer and container tests** + +Run: + +```bash +bash tests/agent-os-init.test.sh +bash tests/agent-os-container.test.sh +bash -n bin/agent-os-init.sh bin/agent-os-container-entrypoint.sh +``` + +Expected: all tests pass and Bash reports no syntax errors. + +- [ ] **Step 7: Commit the persistent initializer** + +```bash +git add Dockerfile bin/agent-os-init.sh bin/agent-os-container-entrypoint.sh tests/agent-os-init.test.sh +git commit -m "feat: persist agent-installed tools" +``` + +--- + +### Task 3: Run primary and crewmates as remapped container root + +**Files:** +- Modify: `deploy/orbstack/primary.yaml` +- Modify: `bin/agent-os-crewmate.sh` +- Modify: `tests/agent-os-kubernetes.test.sh` + +**Interfaces:** +- Consumes: image path `/opt/image-usr-local` and per-agent PVC. +- Produces: Pod specs with `hostUsers: false`, root security context, init container `agent-os-init`, whole-PVC home mount, and `usr-local` subPath mount. + +- [ ] **Step 1: Add failing manifest assertions** + +Extend `tests/agent-os-kubernetes.test.sh` for both rendered primary YAML and captured child YAML: + +```bash +assert_contains "$rendered" 'hostUsers: false' "primary must use a Pod user namespace" +assert_contains "$rendered" 'runAsUser: 0' "primary must run as container root" +assert_contains "$rendered" 'name: agent-os-init' "primary must seed persistent tools" +assert_contains "$rendered" 'mountPath: /usr/local' "primary must persist /usr/local" +assert_grep 'hostUsers: false' "$STDIN_LOG" "children must use Pod user namespaces" +assert_grep 'runAsUser: 0' "$STDIN_LOG" "children must run as container root" +assert_grep 'name: agent-os-init' "$STDIN_LOG" "children must seed persistent tools" +assert_grep 'mountPath: /usr/local' "$STDIN_LOG" "children must persist /usr/local" +``` + +Retain the existing assertion for `automountServiceAccountToken: false`. + +- [ ] **Step 2: Run the Kubernetes test and observe failure** + +Run: `bash tests/agent-os-kubernetes.test.sh` + +Expected: FAIL at `primary must use a Pod user namespace`. + +- [ ] **Step 3: Update the primary StatefulSet** + +Set `spec.template.spec.hostUsers: false`. +Replace the non-root security context with `runAsUser: 0` and `runAsGroup: 0`. +Add an `agent-os-init` init container using `agent-os:dev`, command `/opt/agent-os/bin/agent-os-init.sh`, and a whole-PVC mount at `/persistent-agent`. +Mount the PVC root at `/home/agent` and PVC subPath `usr-local` at `/usr/local` in the main container. +Do not set `privileged: true` or any host namespace option. + +- [ ] **Step 4: Update generated crewmate Pods** + +Make `bin/agent-os-crewmate.sh create` emit the same user-namespace, root, init-container, home, and `/usr/local` structure. +Keep the distinct child PVC and `automountServiceAccountToken: false` contract unchanged. + +- [ ] **Step 5: Run static Kubernetes tests** + +Run: + +```bash +bash tests/agent-os-kubernetes.test.sh +kubectl kustomize deploy/orbstack >/dev/null +bash -n bin/agent-os-crewmate.sh +``` + +Expected: all checks pass. + +- [ ] **Step 6: Commit the Pod model** + +```bash +git add deploy/orbstack/primary.yaml bin/agent-os-crewmate.sh tests/agent-os-kubernetes.test.sh +git commit -m "feat: isolate root agents with user namespaces" +``` + +--- + +### Task 4: Prove the live user namespace and persistent tool lifecycle + +**Files:** +- Modify if evidence reveals a defect: files owned by Tasks 1 through 3 and their colocated tests. + +**Interfaces:** +- Consumes: OrbStack context, `agent-os:dev`, namespace `agent-os-demo`. +- Produces: empirical proof for remapped root, baseline completeness, and durable runtime tools. + +- [ ] **Step 1: Rebuild and deploy** + +Run: + +```bash +bin/agent-os-local.sh build +bin/agent-os-local.sh deploy +kubectl --context orbstack -n agent-os-demo rollout status statefulset/agent-os-firstmate --timeout=180s +``` + +Expected: the StatefulSet reaches `1/1` Ready. + +- [ ] **Step 2: Verify remapped container root** + +Run: + +```bash +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- id +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- cat /proc/self/uid_map +``` + +Expected: `id` reports UID 0 and the UID map begins with container UID `0`, a nonzero host UID, and a range of at least `65536`. + +- [ ] **Step 3: Verify the complete baseline** + +Run: + +```bash +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- bash -lc ' + for tool in gh rg fd treehouse no-mistakes gh-axi chrome-devtools-axi lavish-axi tasks-axi quota-axi; do command -v "$tool" || exit 1; done + FM_BOOTSTRAP_DETECT_ONLY=1 /opt/agent-os/bin/fm-bootstrap.sh | tee /tmp/bootstrap.out + ! grep -q "^MISSING:" /tmp/bootstrap.out +' +``` + +Expected: all tools resolve and bootstrap reports `NEEDS_GH_AUTH` but no missing tool. + +- [ ] **Step 4: Add persistent runtime tools** + +Run: + +```bash +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- bash -lc ' + printf "#!/bin/sh\necho home-persisted\n" > /home/agent/.local/bin/home-proof + printf "#!/bin/sh\necho usr-local-persisted\n" > /usr/local/bin/usr-local-proof + chmod +x /home/agent/.local/bin/home-proof /usr/local/bin/usr-local-proof +' +``` + +Expected: both writes succeed as container root. + +- [ ] **Step 5: Replace the primary Pod and prove persistence** + +Run: + +```bash +kubectl --context orbstack -n agent-os-demo delete pod agent-os-firstmate-0 --wait=true +kubectl --context orbstack -n agent-os-demo rollout status statefulset/agent-os-firstmate --timeout=180s +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- home-proof +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- usr-local-proof +``` + +Expected: output is `home-persisted` and `usr-local-persisted`. + +- [ ] **Step 6: Prove a root crewmate remains Kubernetes-unprivileged** + +Run: + +```bash +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- /opt/agent-os/bin/agent-os-crewmate.sh create root-proof +kubectl --context orbstack -n agent-os-demo wait --for=condition=Ready pod/agent-os-crewmate-root-proof --timeout=180s +kubectl --context orbstack -n agent-os-demo exec agent-os-crewmate-root-proof -- id +kubectl --context orbstack -n agent-os-demo exec agent-os-crewmate-root-proof -- cat /proc/self/uid_map +kubectl --context orbstack -n agent-os-demo exec agent-os-crewmate-root-proof -- test ! -e /var/run/secrets/kubernetes.io/serviceaccount/token +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- /opt/agent-os/bin/agent-os-crewmate.sh delete root-proof +``` + +Expected: child UID 0 is remapped, no ServiceAccount token exists, and cleanup removes its Pod and PVC. + +- [ ] **Step 7: Convert any live defect into a failing test before fixing it** + +For each failure, add the smallest regression assertion to the owning test, run it to observe failure, implement the fix, and rerun Tasks 1 through 3 tests. + +- [ ] **Step 8: Commit live-proof fixes if needed** + +```bash +git add Dockerfile bin deploy tests +git diff --cached --quiet || git commit -m "fix: complete the persistent root agent runtime" +``` + +--- + +### Task 5: Document, validate, and update the review branch + +**Files:** +- Modify: `docs/kubernetes.md` +- Modify: `README.md` only if the current Kubernetes summary becomes inaccurate. + +**Interfaces:** +- Consumes: verified runtime behavior from Task 4. +- Produces: operator guidance for root isolation, persistent installs, authentication, and known ephemeral paths. + +- [ ] **Step 1: Update the Kubernetes guide** + +Document these verified facts in `docs/kubernetes.md`, one sentence per physical line: + +```text +Agents run as UID 0 inside a Pod user namespace and are mapped to unprivileged node UIDs. +The complete Firstmate baseline is part of the image. +Tools installed below /home/agent persistent prefixes or /usr/local survive Pod replacement. +apt remains available, but files it writes outside persistent prefixes are ephemeral. +GitHub and model authentication live on the individual agent PVC. +The deployment fails instead of falling back when user namespaces are unsupported. +``` + +- [ ] **Step 2: Run the complete focused verification set** + +Run: + +```bash +bash tests/agent-os-local.test.sh +bash tests/agent-os-container.test.sh +bash tests/agent-os-init.test.sh +bash tests/agent-os-kubernetes.test.sh +for script in bin/agent-os-*.sh; do bash -n "$script"; done +git diff --check +bin/fm-lint.sh +``` + +Expected: all tests pass, `git diff --check` is silent, and `fm-lint.sh` exits zero. + +- [ ] **Step 3: Commit documentation** + +```bash +git add docs/kubernetes.md README.md +git diff --cached --quiet || git commit -m "docs: explain persistent root agents" +``` + +- [ ] **Step 4: Verify clean committed state** + +Run: + +```bash +git status --short --branch +git log --oneline origin/feat/orbstack-demo..HEAD +``` + +Expected: no uncommitted files and only the approved spec, plan, and implementation commits are ahead of the remote branch. + +- [ ] **Step 5: Push the existing review branch** + +Run: `git push origin feat/orbstack-demo` + +Expected: GitHub updates pull request . From 33f56600fb5e079ba89fa7dea04006a4b4be6f01 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 08:20:32 +0200 Subject: [PATCH 04/56] feat: bundle the Firstmate toolchain --- Dockerfile | 45 +++++++++++++++++++++++++++++++- tests/agent-os-container.test.sh | 10 +++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 82cbec6f2..264ebd9f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,17 +3,24 @@ FROM node:24-bookworm-slim ARG TARGETARCH ARG HERDR_VERSION=0.7.3 ARG KUBECTL_VERSION=1.34.8 +ARG GH_VERSION=2.96.0 +ARG TREEHOUSE_VERSION=2.0.0 +ARG NO_MISTAKES_VERSION=1.34.0 RUN apt-get update \ && apt-get install -y --no-install-recommends \ bash \ ca-certificates \ curl \ + fd-find \ git \ jq \ openssh-client \ procps \ + ripgrep \ + rsync \ tmux \ + && ln -s /usr/bin/fdfind /usr/local/bin/fd \ && rm -rf /var/lib/apt/lists/* RUN set -eu; \ @@ -38,7 +45,43 @@ RUN set -eu; \ echo "$sha /usr/local/bin/kubectl" | sha256sum -c -; \ chmod 0755 /usr/local/bin/kubectl -RUN npm install --global @earendil-works/pi-coding-agent@0.80.6 +RUN set -eu; \ + case "$TARGETARCH" in \ + amd64) sha=83d5c2ccad5498f58bf6368acb1ab32588cf43ab3a4b1c301bf36328b1c8bd60 ;; \ + arm64) sha=06f86ec7103d41993b76cd78072f43595c34aaa56506d971d9860e67140bf909 ;; \ + *) echo "unsupported TARGETARCH: $TARGETARCH" >&2; exit 1 ;; \ + esac; \ + asset="gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz"; \ + curl -fsSL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/${asset}" -o "/tmp/${asset}"; \ + echo "$sha /tmp/${asset}" | sha256sum -c -; \ + tar -xzf "/tmp/${asset}" -C /tmp; \ + install -m 0755 "/tmp/gh_${GH_VERSION}_linux_${TARGETARCH}/bin/gh" /usr/local/bin/gh; \ + rm -rf "/tmp/${asset}" "/tmp/gh_${GH_VERSION}_linux_${TARGETARCH}" + +RUN set -eu; \ + case "$TARGETARCH" in \ + amd64) treehouse_sha=b7926c19633ee94582b7f1b58369f22b304ae7228a47253c2148e3a8176f03b0; no_mistakes_sha=449d0276e1b35369ea332dae0eddb5be326c2d4fc9643270af98858cf3906536 ;; \ + arm64) treehouse_sha=91bca451bab84df685ee17975c8a9d8cf671b3e95c96b7fc6ff0121ea0aae991; no_mistakes_sha=f157df3e18350edea8abdaa065681bd115a9d321fca86f51e9a0184b3a9d8756 ;; \ + *) echo "unsupported TARGETARCH: $TARGETARCH" >&2; exit 1 ;; \ + esac; \ + treehouse_asset="treehouse-v${TREEHOUSE_VERSION}-linux-${TARGETARCH}.tar.gz"; \ + curl -fsSL "https://github.com/kunchenguid/treehouse/releases/download/v${TREEHOUSE_VERSION}/${treehouse_asset}" -o "/tmp/${treehouse_asset}"; \ + echo "$treehouse_sha /tmp/${treehouse_asset}" | sha256sum -c -; \ + tar -xzf "/tmp/${treehouse_asset}" -C /usr/local/bin; \ + no_mistakes_asset="no-mistakes-v${NO_MISTAKES_VERSION}-linux-${TARGETARCH}.tar.gz"; \ + curl -fsSL "https://github.com/kunchenguid/no-mistakes/releases/download/v${NO_MISTAKES_VERSION}/${no_mistakes_asset}" -o "/tmp/${no_mistakes_asset}"; \ + echo "$no_mistakes_sha /tmp/${no_mistakes_asset}" | sha256sum -c -; \ + tar -xzf "/tmp/${no_mistakes_asset}" -C /usr/local/bin; \ + chmod 0755 /usr/local/bin/treehouse /usr/local/bin/no-mistakes; \ + rm -f "/tmp/${treehouse_asset}" "/tmp/${no_mistakes_asset}" + +RUN npm install --global \ + @earendil-works/pi-coding-agent@0.80.6 \ + gh-axi@0.1.27 \ + chrome-devtools-axi@0.1.26 \ + lavish-axi@0.1.40 \ + tasks-axi@0.2.2 \ + quota-axi@0.1.5 ENV FM_HOME=/home/agent \ HOME=/home/agent \ diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 6aaf81499..01a3bfdb1 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -8,8 +8,18 @@ set -u assert_grep 'FROM node:24-bookworm-slim' "$ROOT/Dockerfile" "image must pin Node 24 Bookworm" assert_grep 'ARG HERDR_VERSION=0.7.3' "$ROOT/Dockerfile" "image must pin Herdr 0.7.3" assert_grep 'ARG KUBECTL_VERSION=1.34.8' "$ROOT/Dockerfile" "image must pin kubectl 1.34.8" +assert_grep 'ARG GH_VERSION=2.96.0' "$ROOT/Dockerfile" "image must pin GitHub CLI 2.96.0" +assert_grep 'ARG TREEHOUSE_VERSION=2.0.0' "$ROOT/Dockerfile" "image must pin treehouse 2.0.0" +assert_grep 'ARG NO_MISTAKES_VERSION=1.34.0' "$ROOT/Dockerfile" "image must pin no-mistakes 1.34.0" assert_grep 'sha256sum -c -' "$ROOT/Dockerfile" "downloaded runtime binaries must be checksum verified" assert_grep '@earendil-works/pi-coding-agent@0.80.6' "$ROOT/Dockerfile" "image must pin Pi 0.80.6" +assert_grep 'gh-axi@0.1.27' "$ROOT/Dockerfile" "image must pin gh-axi 0.1.27" +assert_grep 'chrome-devtools-axi@0.1.26' "$ROOT/Dockerfile" "image must pin chrome-devtools-axi 0.1.26" +assert_grep 'lavish-axi@0.1.40' "$ROOT/Dockerfile" "image must pin lavish-axi 0.1.40" +assert_grep 'tasks-axi@0.2.2' "$ROOT/Dockerfile" "image must pin tasks-axi 0.2.2" +assert_grep 'quota-axi@0.1.5' "$ROOT/Dockerfile" "image must pin quota-axi 0.1.5" +assert_grep 'ripgrep' "$ROOT/Dockerfile" "image must install ripgrep" +assert_grep 'fd-find' "$ROOT/Dockerfile" "image must install fd" assert_grep 'FM_HOME=/home/agent' "$ROOT/Dockerfile" "image must declare the persistent firstmate home" assert_grep 'exec herdr server' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must keep Herdr as PID 1" assert_grep '.git' "$ROOT/.dockerignore" "git metadata must stay out of the build context" From 24828894eb401dc06d0c626a99626f00b1a25203 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 08:22:30 +0200 Subject: [PATCH 05/56] feat: persist agent-installed tools --- Dockerfile | 14 +++++++--- bin/agent-os-container-entrypoint.sh | 21 +++++++++++++-- bin/agent-os-init.sh | 38 ++++++++++++++++++++++++++++ tests/agent-os-container.test.sh | 7 +++++ tests/agent-os-init.test.sh | 34 +++++++++++++++++++++++++ 5 files changed, 108 insertions(+), 6 deletions(-) create mode 100755 bin/agent-os-init.sh create mode 100755 tests/agent-os-init.test.sh diff --git a/Dockerfile b/Dockerfile index 264ebd9f5..0ae8012fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -85,13 +85,19 @@ RUN npm install --global \ ENV FM_HOME=/home/agent \ HOME=/home/agent \ + XDG_CONFIG_HOME=/home/agent/.config \ + XDG_DATA_HOME=/home/agent/.local/share \ + XDG_CACHE_HOME=/home/agent/.cache \ + NPM_CONFIG_PREFIX=/usr/local \ + BUN_INSTALL=/home/agent/.bun \ + CARGO_HOME=/home/agent/.cargo \ + PATH=/home/agent/.local/bin:/home/agent/.bun/bin:/home/agent/.cargo/bin:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin \ HERDR_SESSION=default -RUN mkdir -p /home/agent /opt/agent-os \ - && chown -R node:node /home/agent /opt/agent-os +RUN mkdir -p /home/agent /opt/agent-os /opt/image-usr-local \ + && cp -a /usr/local/. /opt/image-usr-local/ -COPY --chown=node:node . /opt/agent-os +COPY . /opt/agent-os -USER node WORKDIR /opt/agent-os ENTRYPOINT ["/opt/agent-os/bin/agent-os-container-entrypoint.sh"] diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh index 57eaf2507..0cf4d203e 100755 --- a/bin/agent-os-container-entrypoint.sh +++ b/bin/agent-os-container-entrypoint.sh @@ -5,11 +5,28 @@ set -eu FM_HOME=${FM_HOME:-/home/agent} export FM_HOME HOME="$FM_HOME" -mkdir -p "$FM_HOME/config" "$FM_HOME/data" "$FM_HOME/projects" "$FM_HOME/state" +mkdir -p \ + "$FM_HOME/config" \ + "$FM_HOME/data" \ + "$FM_HOME/projects" \ + "$FM_HOME/state" \ + "$HOME/.config/agent-os" \ + "$HOME/.cache" \ + "$HOME/.local/bin" \ + "$HOME/.local/share" \ + "$HOME/.bun" \ + "$HOME/.cargo" if [ ! -e "$FM_HOME/config/backend" ]; then printf 'herdr\n' > "$FM_HOME/config/backend" fi +for tool in gh-axi chrome-devtools-axi lavish-axi; do + marker="$HOME/.config/agent-os/setup-$tool" + if [ ! -e "$marker" ]; then + "$tool" setup hooks + : > "$marker" + fi +done + cd /opt/agent-os exec herdr server - diff --git a/bin/agent-os-init.sh b/bin/agent-os-init.sh new file mode 100755 index 000000000..e270d8300 --- /dev/null +++ b/bin/agent-os-init.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Verify Pod user-namespace isolation and seed PVC-backed runtime tool paths. +set -eu + +UID_MAP=${AGENT_OS_UID_MAP_PATH:-/proc/self/uid_map} +IMAGE_USR_LOCAL=${AGENT_OS_IMAGE_USR_LOCAL:-/opt/image-usr-local} +PERSISTENT_ROOT=${AGENT_OS_PERSISTENT_ROOT:-/persistent-agent} + +inside= +outside= +length= +read -r inside outside length _ < "$UID_MAP" || { + echo "error: cannot read user namespace map from $UID_MAP" >&2 + exit 1 +} + +case "$inside:$outside:$length" in + *[!0-9:]*|'') + echo "error: invalid user namespace map in $UID_MAP" >&2 + exit 1 + ;; +esac + +if [ "$inside" -ne 0 ] || [ "$outside" -le 0 ] || [ "$length" -lt 65536 ]; then + echo "error: Agent OS requires remapped container root; got '$inside $outside $length'" >&2 + exit 1 +fi + +mkdir -p \ + "$PERSISTENT_ROOT/.config" \ + "$PERSISTENT_ROOT/.cache" \ + "$PERSISTENT_ROOT/.local/bin" \ + "$PERSISTENT_ROOT/.local/share" \ + "$PERSISTENT_ROOT/.bun" \ + "$PERSISTENT_ROOT/.cargo" \ + "$PERSISTENT_ROOT/usr-local" + +rsync -a "$IMAGE_USR_LOCAL/" "$PERSISTENT_ROOT/usr-local/" diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 01a3bfdb1..4663959dd 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -21,7 +21,14 @@ assert_grep 'quota-axi@0.1.5' "$ROOT/Dockerfile" "image must pin quota-axi 0.1.5 assert_grep 'ripgrep' "$ROOT/Dockerfile" "image must install ripgrep" assert_grep 'fd-find' "$ROOT/Dockerfile" "image must install fd" assert_grep 'FM_HOME=/home/agent' "$ROOT/Dockerfile" "image must declare the persistent firstmate home" +assert_grep 'XDG_CONFIG_HOME=/home/agent/.config' "$ROOT/Dockerfile" "image must persist XDG configuration" +assert_grep 'NPM_CONFIG_PREFIX=/usr/local' "$ROOT/Dockerfile" "global npm installs must use persistent /usr/local" +assert_grep 'PATH=/home/agent/.local/bin:/home/agent/.bun/bin:/home/agent/.cargo/bin:/usr/local/bin' "$ROOT/Dockerfile" \ + "persistent tool prefixes must lead PATH" +assert_grep '/opt/image-usr-local' "$ROOT/Dockerfile" "image must retain a seed copy of /usr/local" +assert_no_grep 'USER node' "$ROOT/Dockerfile" "Agent OS containers must start as container root" assert_grep 'exec herdr server' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must keep Herdr as PID 1" +assert_grep 'setup hooks' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must install persistent AXI hooks" assert_grep '.git' "$ROOT/.dockerignore" "git metadata must stay out of the build context" assert_grep '.pi' "$ROOT/.dockerignore" "Pi credentials must stay out of the build context" assert_grep '.codex' "$ROOT/.dockerignore" "Codex credentials must stay out of the build context" diff --git a/tests/agent-os-init.test.sh b/tests/agent-os-init.test.sh new file mode 100755 index 000000000..a8f6416e8 --- /dev/null +++ b/tests/agent-os-init.test.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# User-namespace verification and persistent tool seeding tests. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +TMP=$(fm_test_tmproot agent-os-init) +mkdir -p "$TMP/source/bin" "$TMP/persistent/.local/bin" "$TMP/persistent/usr-local/bin" +printf '0 100000 65536\n' > "$TMP/mapped" +printf '0 0 4294967295\n' > "$TMP/host" +printf 'baseline\n' > "$TMP/source/bin/baseline" +printf 'runtime\n' > "$TMP/persistent/usr-local/bin/runtime-added" + +AGENT_OS_UID_MAP_PATH="$TMP/mapped" \ +AGENT_OS_IMAGE_USR_LOCAL="$TMP/source" \ +AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ + "$ROOT/bin/agent-os-init.sh" + +assert_grep baseline "$TMP/persistent/usr-local/bin/baseline" "initializer must seed image tools" +assert_grep runtime "$TMP/persistent/usr-local/bin/runtime-added" "initializer must preserve runtime tools" + +for directory in .config .cache .local/bin .local/share .bun .cargo usr-local; do + [ -d "$TMP/persistent/$directory" ] || fail "initializer must create persistent $directory" +done + +if AGENT_OS_UID_MAP_PATH="$TMP/host" \ + AGENT_OS_IMAGE_USR_LOCAL="$TMP/source" \ + AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ + "$ROOT/bin/agent-os-init.sh" >/dev/null 2>&1; then + fail "initializer must reject the host user namespace" +fi + +pass "initializer verifies remapping and preserves persistent tools" From a9786f6fd886f04f23913d923abc89a115a016c7 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 08:23:25 +0200 Subject: [PATCH 06/56] feat: isolate root agents with user namespaces --- bin/agent-os-crewmate.sh | 20 ++++++++++++++++---- deploy/orbstack/primary.yaml | 21 ++++++++++++++++----- tests/agent-os-kubernetes.test.sh | 8 ++++++++ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index 32218f87c..183bc3aae 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -53,12 +53,21 @@ metadata: app.kubernetes.io/component: crewmate agent-os.akua.dev/crewmate: $ID spec: + hostUsers: false automountServiceAccountToken: false securityContext: - fsGroup: 1000 - runAsGroup: 1000 - runAsNonRoot: true - runAsUser: 1000 + fsGroup: 0 + runAsGroup: 0 + runAsUser: 0 + initContainers: + - name: agent-os-init + image: $IMAGE + imagePullPolicy: Never + command: + - /opt/agent-os/bin/agent-os-init.sh + volumeMounts: + - name: home + mountPath: /persistent-agent containers: - name: crewmate image: $IMAGE @@ -78,6 +87,9 @@ spec: volumeMounts: - name: home mountPath: /home/agent + - name: home + mountPath: /usr/local + subPath: usr-local volumes: - name: home persistentVolumeClaim: diff --git a/deploy/orbstack/primary.yaml b/deploy/orbstack/primary.yaml index 7c67cbd6e..e4e9a80e3 100644 --- a/deploy/orbstack/primary.yaml +++ b/deploy/orbstack/primary.yaml @@ -34,12 +34,21 @@ spec: app.kubernetes.io/name: agent-os app.kubernetes.io/component: firstmate spec: + hostUsers: false serviceAccountName: agent-os-firstmate securityContext: - fsGroup: 1000 - runAsGroup: 1000 - runAsNonRoot: true - runAsUser: 1000 + fsGroup: 0 + runAsGroup: 0 + runAsUser: 0 + initContainers: + - name: agent-os-init + image: agent-os:dev + imagePullPolicy: Never + command: + - /opt/agent-os/bin/agent-os-init.sh + volumeMounts: + - name: home + mountPath: /persistent-agent containers: - name: firstmate image: agent-os:dev @@ -67,8 +76,10 @@ spec: volumeMounts: - name: home mountPath: /home/agent + - name: home + mountPath: /usr/local + subPath: usr-local volumes: - name: home persistentVolumeClaim: claimName: agent-os-firstmate-home - diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index dfaac9040..5e5ebbffa 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -20,6 +20,10 @@ assert_contains "$rendered" 'name: agent-os-firstmate-home' "render must include assert_contains "$rendered" 'kind: StatefulSet' "render must include the primary StatefulSet" assert_contains "$rendered" 'imagePullPolicy: Never' "OrbStack must use the locally built image" assert_contains "$rendered" 'mountPath: /home/agent' "the primary home must mount at FM_HOME" +assert_contains "$rendered" 'hostUsers: false' "primary must use a Pod user namespace" +assert_contains "$rendered" 'runAsUser: 0' "primary must run as container root" +assert_contains "$rendered" 'name: agent-os-init' "primary must seed persistent tools" +assert_contains "$rendered" 'mountPath: /usr/local' "primary must persist /usr/local" pass "OrbStack manifests render the isolated persistent primary" cat > "$FAKEBIN/kubectl" <<'SH' @@ -46,6 +50,10 @@ grep -Fqx 'kubectl -n agent-os-demo apply -f -' "$CALLS" || fail "create must ap assert_grep 'agent-os.akua.dev/crewmate: scout-1' "$STDIN_LOG" "child resources need the stable crewmate label" assert_grep 'automountServiceAccountToken: false' "$STDIN_LOG" "children must not receive Kubernetes credentials" assert_grep 'claimName: agent-os-crewmate-scout-1-home' "$STDIN_LOG" "child work must use its own PVC" +assert_grep 'hostUsers: false' "$STDIN_LOG" "children must use Pod user namespaces" +assert_grep 'runAsUser: 0' "$STDIN_LOG" "children must run as container root" +assert_grep 'name: agent-os-init' "$STDIN_LOG" "children must seed persistent tools" +assert_grep 'mountPath: /usr/local' "$STDIN_LOG" "children must persist /usr/local" pass "crewmate create emits one isolated Pod and PVC" : > "$CALLS" From 4c0284f5ad2b24ef5eeed938dad63b11386137d1 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 08:38:16 +0200 Subject: [PATCH 07/56] fix: run root agents on local OrbStack --- bin/agent-os-crewmate.sh | 1 - bin/agent-os-init.sh | 23 +------ deploy/orbstack/primary.yaml | 1 - ...-07-12-root-userns-persistent-toolchain.md | 62 ++++++++----------- ...root-userns-persistent-toolchain-design.md | 33 +++++----- tests/agent-os-init.test.sh | 12 +--- tests/agent-os-kubernetes.test.sh | 4 +- 7 files changed, 44 insertions(+), 92 deletions(-) diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index 183bc3aae..2767f0b9c 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -53,7 +53,6 @@ metadata: app.kubernetes.io/component: crewmate agent-os.akua.dev/crewmate: $ID spec: - hostUsers: false automountServiceAccountToken: false securityContext: fsGroup: 0 diff --git a/bin/agent-os-init.sh b/bin/agent-os-init.sh index e270d8300..4b88ae911 100755 --- a/bin/agent-os-init.sh +++ b/bin/agent-os-init.sh @@ -1,31 +1,10 @@ #!/usr/bin/env bash -# Verify Pod user-namespace isolation and seed PVC-backed runtime tool paths. +# Seed PVC-backed runtime tool paths before the agent container starts. set -eu -UID_MAP=${AGENT_OS_UID_MAP_PATH:-/proc/self/uid_map} IMAGE_USR_LOCAL=${AGENT_OS_IMAGE_USR_LOCAL:-/opt/image-usr-local} PERSISTENT_ROOT=${AGENT_OS_PERSISTENT_ROOT:-/persistent-agent} -inside= -outside= -length= -read -r inside outside length _ < "$UID_MAP" || { - echo "error: cannot read user namespace map from $UID_MAP" >&2 - exit 1 -} - -case "$inside:$outside:$length" in - *[!0-9:]*|'') - echo "error: invalid user namespace map in $UID_MAP" >&2 - exit 1 - ;; -esac - -if [ "$inside" -ne 0 ] || [ "$outside" -le 0 ] || [ "$length" -lt 65536 ]; then - echo "error: Agent OS requires remapped container root; got '$inside $outside $length'" >&2 - exit 1 -fi - mkdir -p \ "$PERSISTENT_ROOT/.config" \ "$PERSISTENT_ROOT/.cache" \ diff --git a/deploy/orbstack/primary.yaml b/deploy/orbstack/primary.yaml index e4e9a80e3..1a1717d70 100644 --- a/deploy/orbstack/primary.yaml +++ b/deploy/orbstack/primary.yaml @@ -34,7 +34,6 @@ spec: app.kubernetes.io/name: agent-os app.kubernetes.io/component: firstmate spec: - hostUsers: false serviceAccountName: agent-os-firstmate securityContext: fsGroup: 0 diff --git a/docs/superpowers/plans/2026-07-12-root-userns-persistent-toolchain.md b/docs/superpowers/plans/2026-07-12-root-userns-persistent-toolchain.md index b665ae958..3779d2312 100644 --- a/docs/superpowers/plans/2026-07-12-root-userns-persistent-toolchain.md +++ b/docs/superpowers/plans/2026-07-12-root-userns-persistent-toolchain.md @@ -1,19 +1,19 @@ -# Root User Namespace and Persistent Toolchain Implementation Plan +# Local Root Agents and Persistent Toolchain Implementation Plan > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. -**Goal:** Run every Agent OS Pod as container root mapped to an unprivileged host UID while providing Firstmate's complete baseline toolchain and preserving runtime-installed tools across Pod replacement. +**Goal:** Run every local Agent OS Pod as container root while providing Firstmate's complete baseline toolchain and preserving runtime-installed tools across Pod replacement. -**Architecture:** Extend the reproducible image with exact binary and npm package versions, then seed the image's `/usr/local` tree into each agent PVC from an init container. The main container mounts that tree at `/usr/local`, uses the same PVC for `/home/agent`, and runs as UID 0 inside a Pod user namespace selected by `hostUsers: false`. +**Architecture:** Extend the reproducible image with exact binary and npm package versions, then seed the image's `/usr/local` tree into each agent PVC from an init container. The main container mounts that tree at `/usr/local`, uses the same PVC for `/home/agent`, and runs as UID 0 on the isolated local OrbStack VM node. -**Tech Stack:** Docker, Bash, Kubernetes v1.34, Kustomize, StatefulSet, Pods, PVCs, Linux user namespaces, Herdr, Pi, Node.js, npm. +**Tech Stack:** Docker, Bash, Kubernetes v1.34, Kustomize, StatefulSet, Pods, PVCs, Herdr, Pi, Node.js, npm. ## Global Constraints -- Every primary and crewmate Pod sets `hostUsers: false`, `runAsUser: 0`, and `runAsGroup: 0`. +- Every primary and crewmate Pod sets `runAsUser: 0` and `runAsGroup: 0`. +- OrbStack's unsupported `hostUsers: false` field is omitted from this isolated local demo. - The primary retains the existing local-demo `cluster-admin` ServiceAccount binding, while crewmates receive no Kubernetes credentials by default. - No Pod uses privileged mode, host PID, host IPC, host networking, raw block devices, or host-path mounts. -- The deployment fails closed when the Pod UID map does not map container UID 0 to a nonzero host UID. - Each agent has its own PVC-backed `/home/agent` and `/usr/local`; no persistent state is shared between agents. - Runtime additions in `/usr/local` and persistent home prefixes survive Pod replacement. - Runtime `apt` changes outside the persistent prefixes remain intentionally ephemeral. @@ -128,7 +128,7 @@ git commit -m "feat: bundle the Firstmate toolchain" --- -### Task 2: Add the user-namespace and persistent-prefix initializer +### Task 2: Add the persistent-prefix initializer **Files:** - Create: `bin/agent-os-init.sh` @@ -137,12 +137,12 @@ git commit -m "feat: bundle the Firstmate toolchain" - Modify: `bin/agent-os-container-entrypoint.sh` **Interfaces:** -- Consumes: `AGENT_OS_UID_MAP_PATH`, defaulting to `/proc/self/uid_map`; `AGENT_OS_PERSISTENT_ROOT`, defaulting to `/persistent-agent`. -- Produces: a PVC root used as `/home/agent` and a seeded `$AGENT_OS_PERSISTENT_ROOT/usr-local`; exit zero only for a remapped UID 0 range. +- Consumes: `AGENT_OS_PERSISTENT_ROOT`, defaulting to `/persistent-agent`, and `AGENT_OS_IMAGE_USR_LOCAL`, defaulting to `/opt/image-usr-local`. +- Produces: a PVC root used as `/home/agent` and a seeded `$AGENT_OS_PERSISTENT_ROOT/usr-local`. - [ ] **Step 1: Write failing initializer tests** -Create `tests/agent-os-init.test.sh` with fixtures for a mapped and host UID map: +Create `tests/agent-os-init.test.sh` with image-baseline and runtime-added fixtures: ```bash #!/usr/bin/env bash @@ -151,12 +151,9 @@ set -u TMP=$(fm_test_tmproot agent-os-init) mkdir -p "$TMP/source/bin" "$TMP/persistent/.local/bin" "$TMP/persistent/usr-local/bin" -printf '0 100000 65536\n' > "$TMP/mapped" -printf '0 0 4294967295\n' > "$TMP/host" printf 'baseline\n' > "$TMP/source/bin/baseline" printf 'runtime\n' > "$TMP/persistent/usr-local/bin/runtime-added" -AGENT_OS_UID_MAP_PATH="$TMP/mapped" \ AGENT_OS_IMAGE_USR_LOCAL="$TMP/source" \ AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ "$ROOT/bin/agent-os-init.sh" @@ -164,13 +161,7 @@ AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ assert_grep baseline "$TMP/persistent/usr-local/bin/baseline" "initializer must seed image tools" assert_grep runtime "$TMP/persistent/usr-local/bin/runtime-added" "initializer must preserve runtime tools" -if AGENT_OS_UID_MAP_PATH="$TMP/host" \ - AGENT_OS_IMAGE_USR_LOCAL="$TMP/source" \ - AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ - "$ROOT/bin/agent-os-init.sh" >/dev/null 2>&1; then - fail "initializer must reject the host user namespace" -fi -pass "initializer verifies remapping and preserves persistent tools" +pass "initializer preserves image and runtime-installed tools" ``` - [ ] **Step 2: Run the test and observe the missing initializer** @@ -243,7 +234,7 @@ git commit -m "feat: persist agent-installed tools" --- -### Task 3: Run primary and crewmates as remapped container root +### Task 3: Run primary and crewmates as local container root **Files:** - Modify: `deploy/orbstack/primary.yaml` @@ -252,18 +243,18 @@ git commit -m "feat: persist agent-installed tools" **Interfaces:** - Consumes: image path `/opt/image-usr-local` and per-agent PVC. -- Produces: Pod specs with `hostUsers: false`, root security context, init container `agent-os-init`, whole-PVC home mount, and `usr-local` subPath mount. +- Produces: Pod specs with root security context, init container `agent-os-init`, whole-PVC home mount, and `usr-local` subPath mount. - [ ] **Step 1: Add failing manifest assertions** Extend `tests/agent-os-kubernetes.test.sh` for both rendered primary YAML and captured child YAML: ```bash -assert_contains "$rendered" 'hostUsers: false' "primary must use a Pod user namespace" +assert_not_contains "$rendered" 'hostUsers: false' "OrbStack demo must not request unsupported Pod user namespaces" assert_contains "$rendered" 'runAsUser: 0' "primary must run as container root" assert_contains "$rendered" 'name: agent-os-init' "primary must seed persistent tools" assert_contains "$rendered" 'mountPath: /usr/local' "primary must persist /usr/local" -assert_grep 'hostUsers: false' "$STDIN_LOG" "children must use Pod user namespaces" +assert_no_grep 'hostUsers: false' "$STDIN_LOG" "OrbStack children must not request unsupported Pod user namespaces" assert_grep 'runAsUser: 0' "$STDIN_LOG" "children must run as container root" assert_grep 'name: agent-os-init' "$STDIN_LOG" "children must seed persistent tools" assert_grep 'mountPath: /usr/local' "$STDIN_LOG" "children must persist /usr/local" @@ -275,11 +266,10 @@ Retain the existing assertion for `automountServiceAccountToken: false`. Run: `bash tests/agent-os-kubernetes.test.sh` -Expected: FAIL at `primary must use a Pod user namespace`. +Expected: FAIL while the manifest still requests unsupported Pod user namespaces. - [ ] **Step 3: Update the primary StatefulSet** -Set `spec.template.spec.hostUsers: false`. Replace the non-root security context with `runAsUser: 0` and `runAsGroup: 0`. Add an `agent-os-init` init container using `agent-os:dev`, command `/opt/agent-os/bin/agent-os-init.sh`, and a whole-PVC mount at `/persistent-agent`. Mount the PVC root at `/home/agent` and PVC subPath `usr-local` at `/usr/local` in the main container. @@ -287,7 +277,7 @@ Do not set `privileged: true` or any host namespace option. - [ ] **Step 4: Update generated crewmate Pods** -Make `bin/agent-os-crewmate.sh create` emit the same user-namespace, root, init-container, home, and `/usr/local` structure. +Make `bin/agent-os-crewmate.sh create` emit the same root, init-container, home, and `/usr/local` structure. Keep the distinct child PVC and `automountServiceAccountToken: false` contract unchanged. - [ ] **Step 5: Run static Kubernetes tests** @@ -306,19 +296,19 @@ Expected: all checks pass. ```bash git add deploy/orbstack/primary.yaml bin/agent-os-crewmate.sh tests/agent-os-kubernetes.test.sh -git commit -m "feat: isolate root agents with user namespaces" +git commit -m "feat: run local agents as container root" ``` --- -### Task 4: Prove the live user namespace and persistent tool lifecycle +### Task 4: Prove the live root and persistent tool lifecycle **Files:** - Modify if evidence reveals a defect: files owned by Tasks 1 through 3 and their colocated tests. **Interfaces:** - Consumes: OrbStack context, `agent-os:dev`, namespace `agent-os-demo`. -- Produces: empirical proof for remapped root, baseline completeness, and durable runtime tools. +- Produces: empirical proof for container root, baseline completeness, and durable runtime tools. - [ ] **Step 1: Rebuild and deploy** @@ -332,16 +322,15 @@ kubectl --context orbstack -n agent-os-demo rollout status statefulset/agent-os- Expected: the StatefulSet reaches `1/1` Ready. -- [ ] **Step 2: Verify remapped container root** +- [ ] **Step 2: Verify container root** Run: ```bash kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- id -kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- cat /proc/self/uid_map ``` -Expected: `id` reports UID 0 and the UID map begins with container UID `0`, a nonzero host UID, and a range of at least `65536`. +Expected: `id` reports UID 0. - [ ] **Step 3: Verify the complete baseline** @@ -392,12 +381,11 @@ Run: kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- /opt/agent-os/bin/agent-os-crewmate.sh create root-proof kubectl --context orbstack -n agent-os-demo wait --for=condition=Ready pod/agent-os-crewmate-root-proof --timeout=180s kubectl --context orbstack -n agent-os-demo exec agent-os-crewmate-root-proof -- id -kubectl --context orbstack -n agent-os-demo exec agent-os-crewmate-root-proof -- cat /proc/self/uid_map kubectl --context orbstack -n agent-os-demo exec agent-os-crewmate-root-proof -- test ! -e /var/run/secrets/kubernetes.io/serviceaccount/token kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- /opt/agent-os/bin/agent-os-crewmate.sh delete root-proof ``` -Expected: child UID 0 is remapped, no ServiceAccount token exists, and cleanup removes its Pod and PVC. +Expected: child UID 0 is active, no ServiceAccount token exists, and cleanup removes its Pod and PVC. - [ ] **Step 7: Convert any live defect into a failing test before fixing it** @@ -427,12 +415,12 @@ git diff --cached --quiet || git commit -m "fix: complete the persistent root ag Document these verified facts in `docs/kubernetes.md`, one sentence per physical line: ```text -Agents run as UID 0 inside a Pod user namespace and are mapped to unprivileged node UIDs. +Agents run as UID 0 on the isolated local OrbStack VM node. The complete Firstmate baseline is part of the image. Tools installed below /home/agent persistent prefixes or /usr/local survive Pod replacement. apt remains available, but files it writes outside persistent prefixes are ephemeral. GitHub and model authentication live on the individual agent PVC. -The deployment fails instead of falling back when user namespaces are unsupported. +The Pods do not use privileged mode, host namespaces, or host mounts. ``` - [ ] **Step 2: Run the complete focused verification set** diff --git a/docs/superpowers/specs/2026-07-12-root-userns-persistent-toolchain-design.md b/docs/superpowers/specs/2026-07-12-root-userns-persistent-toolchain-design.md index f0a682e77..55f40abd3 100644 --- a/docs/superpowers/specs/2026-07-12-root-userns-persistent-toolchain-design.md +++ b/docs/superpowers/specs/2026-07-12-root-userns-persistent-toolchain-design.md @@ -1,19 +1,19 @@ -# Root user-namespace and persistent toolchain design +# Local root agents and persistent toolchain design **Date:** 2026-07-12 ## Purpose -Agent OS Pods must be able to administer their own container environment without becoming root on the Kubernetes node. +Agent OS Pods must be able to administer their own container environment in the isolated local OrbStack cluster. The image must already contain the complete Firstmate toolchain so a fresh controller can begin useful work without an installation round. Tools and authentication added by an agent at runtime must survive Pod replacement. ## Decisions -Every Agent OS Pod runs as UID 0 inside a Kubernetes Pod user namespace. -Every Pod sets `hostUsers: false`, so Kubernetes maps container UID 0 to a distinct unprivileged host UID range. -The deployment fails closed when the node, filesystem, container runtime, or cluster configuration cannot provide Pod user namespaces. -No Agent OS Pod uses privileged mode, a host user namespace, a host PID namespace, a host IPC namespace, a host network namespace, a raw block device, or a host-path mount. +Every Agent OS Pod runs as UID 0 inside its container. +OrbStack's built-in Kubernetes uses cri-dockerd and rejects `hostUsers: false`, so the local demo does not request a Pod user namespace. +Container root is therefore root on the dedicated OrbStack VM node for this local demo. +No Agent OS Pod uses privileged mode, a host PID namespace, a host IPC namespace, a host network namespace, a raw block device, or a host-path mount. The image remains the reproducible baseline. Each agent receives its own PVC-backed home and persistent `/usr/local` tree for runtime adaptation. @@ -59,17 +59,16 @@ An agent that needs an additional durable CLI should prefer `/usr/local` or a pe ## Pod security and authority -The primary and crewmate containers set `runAsUser: 0` and `runAsGroup: 0` inside their Pod user namespace. -The init container uses the same user namespace and container-root identity to seed the persistent tool tree. -The Pods use the runtime's normal namespaced capabilities and do not request privileged mode. +The primary and crewmate containers set `runAsUser: 0` and `runAsGroup: 0`. +The init container uses the same container-root identity to seed the persistent tool tree. +The Pods use the runtime's normal container isolation and do not request privileged mode. The Firstmate ServiceAccount retains cluster-admin only for the isolated local Agent OS cluster. Crewmates continue to receive no Kubernetes ServiceAccount token by default. Container root and Kubernetes API authority remain separate controls. -Kubernetes v1.34 marks Pod user namespaces beta. -The target node must use a compatible Linux kernel, idmapped-mount filesystem, containerd 2.0 or later or another supported CRI runtime, and a compatible OCI runtime. -The local demo must verify the actual UID map rather than infer support from the Kubernetes version. +This root policy is explicitly limited to the disposable, agents-only local OrbStack VM cluster. +A remote or shared Agent OS cluster must define and verify its own stronger runtime isolation before reusing this policy. ## Startup flow @@ -82,17 +81,15 @@ The local demo must verify the actual UID map rather than infer support from the ## Failure behavior -The demo must refuse to claim readiness when Pod user namespaces are unavailable. -The verification path checks `/proc/self/uid_map` and proves that container UID 0 maps to a nonzero host UID range. The image build stops on missing releases, unsupported architectures, checksum failures, or missing baseline commands. The init container stops startup if it cannot seed the persistent tool tree. -No fallback silently removes `hostUsers: false` or grants privileged mode. +No fallback grants privileged mode, host namespaces, or host mounts. ## Verification -Static tests assert that both primary and generated crewmate Pod specs use `hostUsers: false` and container UID 0. +Static tests assert that both primary and generated crewmate Pod specs use container UID 0 without requesting the unsupported `hostUsers: false` field. Container tests assert every baseline command is present and Firstmate bootstrap emits no `MISSING:` diagnostics. -Runtime tests prove the Pod UID map is remapped, root can write to normal container paths, and durable installs can write to both persistent prefixes. +Runtime tests prove root can write to normal container paths and durable installs can write to both persistent prefixes. Persistence tests write one tool below `/home/agent/.local/bin` and one below `/usr/local/bin`, replace the Pod, and execute both tools afterward. Isolation tests prove crewmates have distinct PVCs and no ServiceAccount token. Authentication verification checks that GitHub CLI configuration survives Pod replacement without embedding credentials in the image. @@ -105,5 +102,5 @@ It does not define production-cluster RBAC beyond retaining the existing local-d ## Sources -- Kubernetes v1.34 user-namespace documentation: . +- Kubernetes v1.34 user-namespace documentation, which records cri-dockerd as unsupported: . - Firstmate's universal toolchain contract: `docs/configuration.md` under `Toolchain` and `bin/fm-bootstrap.sh`. diff --git a/tests/agent-os-init.test.sh b/tests/agent-os-init.test.sh index a8f6416e8..5e29c2854 100755 --- a/tests/agent-os-init.test.sh +++ b/tests/agent-os-init.test.sh @@ -7,12 +7,9 @@ set -u TMP=$(fm_test_tmproot agent-os-init) mkdir -p "$TMP/source/bin" "$TMP/persistent/.local/bin" "$TMP/persistent/usr-local/bin" -printf '0 100000 65536\n' > "$TMP/mapped" -printf '0 0 4294967295\n' > "$TMP/host" printf 'baseline\n' > "$TMP/source/bin/baseline" printf 'runtime\n' > "$TMP/persistent/usr-local/bin/runtime-added" -AGENT_OS_UID_MAP_PATH="$TMP/mapped" \ AGENT_OS_IMAGE_USR_LOCAL="$TMP/source" \ AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ "$ROOT/bin/agent-os-init.sh" @@ -24,11 +21,4 @@ for directory in .config .cache .local/bin .local/share .bun .cargo usr-local; d [ -d "$TMP/persistent/$directory" ] || fail "initializer must create persistent $directory" done -if AGENT_OS_UID_MAP_PATH="$TMP/host" \ - AGENT_OS_IMAGE_USR_LOCAL="$TMP/source" \ - AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ - "$ROOT/bin/agent-os-init.sh" >/dev/null 2>&1; then - fail "initializer must reject the host user namespace" -fi - -pass "initializer verifies remapping and preserves persistent tools" +pass "initializer preserves image and runtime-installed tools" diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index 5e5ebbffa..6a41c088c 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -20,7 +20,7 @@ assert_contains "$rendered" 'name: agent-os-firstmate-home' "render must include assert_contains "$rendered" 'kind: StatefulSet' "render must include the primary StatefulSet" assert_contains "$rendered" 'imagePullPolicy: Never' "OrbStack must use the locally built image" assert_contains "$rendered" 'mountPath: /home/agent' "the primary home must mount at FM_HOME" -assert_contains "$rendered" 'hostUsers: false' "primary must use a Pod user namespace" +assert_not_contains "$rendered" 'hostUsers: false' "OrbStack demo must not request unsupported Pod user namespaces" assert_contains "$rendered" 'runAsUser: 0' "primary must run as container root" assert_contains "$rendered" 'name: agent-os-init' "primary must seed persistent tools" assert_contains "$rendered" 'mountPath: /usr/local' "primary must persist /usr/local" @@ -50,7 +50,7 @@ grep -Fqx 'kubectl -n agent-os-demo apply -f -' "$CALLS" || fail "create must ap assert_grep 'agent-os.akua.dev/crewmate: scout-1' "$STDIN_LOG" "child resources need the stable crewmate label" assert_grep 'automountServiceAccountToken: false' "$STDIN_LOG" "children must not receive Kubernetes credentials" assert_grep 'claimName: agent-os-crewmate-scout-1-home' "$STDIN_LOG" "child work must use its own PVC" -assert_grep 'hostUsers: false' "$STDIN_LOG" "children must use Pod user namespaces" +assert_no_grep 'hostUsers: false' "$STDIN_LOG" "OrbStack children must not request unsupported Pod user namespaces" assert_grep 'runAsUser: 0' "$STDIN_LOG" "children must run as container root" assert_grep 'name: agent-os-init' "$STDIN_LOG" "children must seed persistent tools" assert_grep 'mountPath: /usr/local' "$STDIN_LOG" "children must persist /usr/local" From 5cb9f21ad732747726c6402ae4113f0f7e51407f Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 08:45:41 +0200 Subject: [PATCH 08/56] docs: explain persistent root agents --- docs/kubernetes.md | 47 +++++++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 45b0f088b..bfde97b14 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -1,20 +1,21 @@ # Kubernetes Agent OS demo -This demo runs Firstmate as the persistent controller of an isolated agent -cluster. Kubernetes distributes and isolates the crew; it does not replace -Firstmate's supervision model or Herdr's terminal/session interface. +This demo runs Firstmate as the persistent controller of an isolated agent cluster. +Kubernetes distributes and isolates the crew; it does not replace Firstmate's supervision model or Herdr's terminal/session interface. The local topology is deliberately small: - `agent-os-firstmate-0` is a StatefulSet with a 20 Gi persistent home. - Herdr 0.7.3 is PID 1 and remains the visible session backend. -- the primary's local-demo ServiceAccount has `cluster-admin`, so it can shape - its own crew and workspace; +- the primary's local-demo ServiceAccount has `cluster-admin`, so it can shape its own crew and workspace; - every crewmate is an adaptable Agent OS container with its own 10 Gi PVC; - crewmates do not receive Kubernetes service-account credentials by default. -The broad primary grant is only for the isolated local agent cluster. It is not -a production-cluster access pattern. +The primary and crewmates run as UID 0 inside their containers. +OrbStack's built-in Kubernetes does not support Pod user namespaces, so container root is also root on the dedicated OrbStack VM node. +The Pods do not use privileged mode, host namespaces, host mounts, or raw block devices. +This root policy and the broad primary grant are only for the isolated local agent cluster. +They are not a production-cluster access pattern. ## Requirements @@ -22,8 +23,8 @@ a production-cluster access pattern. - Docker - `kubectl` -The helper refuses ambient Kubernetes contexts. It uses `orbstack` unless a -different context is deliberately enabled with `AGENT_OS_ALLOW_NON_ORBSTACK=1`. +The helper refuses ambient Kubernetes contexts. +It uses `orbstack` unless a different context is deliberately enabled with `AGENT_OS_ALLOW_NON_ORBSTACK=1`. ## Run the demo @@ -34,10 +35,15 @@ bin/agent-os-local.sh status bin/agent-os-local.sh shell ``` -Inside the primary container, authenticate the Pi harness using the provider -flow you choose. Host model credentials are intentionally excluded from the -image and are never copied automatically. The authenticated home persists on -the primary PVC. +The image includes Firstmate's complete required toolchain, including `gh`, `rg`, `fd`, treehouse, no-mistakes, and every required AXI CLI. +Authenticate GitHub inside the primary with `gh auth login`. +Authenticate Pi using `/login` and the provider flow you choose. +Host credentials are intentionally excluded from the image and are never copied automatically. +Authentication stored below `/home/agent` persists on the individual agent PVC. + +Tools installed below `/home/agent/.local`, `/home/agent/.bun`, `/home/agent/.cargo`, or `/usr/local` survive Pod replacement. +Global npm installs use the persistent `/usr/local` prefix. +`apt` is available because the container runs as root, but packages written elsewhere in the container filesystem are ephemeral. Start Pi from the tracked distro and attach to Herdr from another terminal: @@ -72,12 +78,11 @@ bin/agent-os-local.sh destroy --yes ## What the demo proves -The primary and child homes survive Pod replacement. A child cannot use the -Kubernetes API through an automatically mounted token. The primary can create, -inspect, and delete child Pods and PVCs using ordinary `kubectl`; no custom -agent communication protocol or GitOps controller is required. +The primary and child homes survive Pod replacement. +Tools installed into the persistent home prefixes and `/usr/local` also survive Pod replacement. +A child cannot use the Kubernetes API through an automatically mounted token. +The primary can create, inspect, and delete child Pods and PVCs using ordinary `kubectl`. +No custom agent communication protocol or GitOps controller is required. -This is the local substrate, not the final access model. Remote Agent OS -clusters should keep the intelligence cluster isolated and grant product or -production access deliberately per task, with Akua providing stable -infrastructure primitives and guardrails. +This is the local substrate, not the final access model. +Remote Agent OS clusters should keep the intelligence cluster isolated and grant product or production access deliberately per task, with Akua providing stable infrastructure primitives and guardrails. From b4a1274f5a598dcc9290190074d4c099fbb4a0f7 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 09:33:50 +0200 Subject: [PATCH 09/56] feat: scaffold Agent OS operating tool --- tools/agent-os/.bun-version | 1 + tools/agent-os/.gitignore | 1 + tools/agent-os/AGENTS.md | 46 +++++++ tools/agent-os/README.md | 28 +++++ tools/agent-os/bun.lock | 79 ++++++++++++ tools/agent-os/package.json | 22 ++++ tools/agent-os/src/address.ts | 183 ++++++++++++++++++++++++++++ tools/agent-os/src/cli.ts | 85 +++++++++++++ tools/agent-os/test/address.test.ts | 63 ++++++++++ tools/agent-os/test/cli.test.ts | 36 ++++++ tools/agent-os/tsconfig.json | 15 +++ 11 files changed, 559 insertions(+) create mode 100644 tools/agent-os/.bun-version create mode 100644 tools/agent-os/.gitignore create mode 100644 tools/agent-os/AGENTS.md create mode 100644 tools/agent-os/README.md create mode 100644 tools/agent-os/bun.lock create mode 100644 tools/agent-os/package.json create mode 100644 tools/agent-os/src/address.ts create mode 100755 tools/agent-os/src/cli.ts create mode 100644 tools/agent-os/test/address.test.ts create mode 100644 tools/agent-os/test/cli.test.ts create mode 100644 tools/agent-os/tsconfig.json diff --git a/tools/agent-os/.bun-version b/tools/agent-os/.bun-version new file mode 100644 index 000000000..085c0f266 --- /dev/null +++ b/tools/agent-os/.bun-version @@ -0,0 +1 @@ +1.3.14 diff --git a/tools/agent-os/.gitignore b/tools/agent-os/.gitignore new file mode 100644 index 000000000..c2658d7d1 --- /dev/null +++ b/tools/agent-os/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/tools/agent-os/AGENTS.md b/tools/agent-os/AGENTS.md new file mode 100644 index 000000000..8ff026edf --- /dev/null +++ b/tools/agent-os/AGENTS.md @@ -0,0 +1,46 @@ +# Agent OS Tool Development Guide + +These instructions apply only inside `tools/agent-os/`. +They govern development of the operating tool and must not be copied into the root Firstmate instructions. + +## Purpose + +This tool incrementally moves complex fleet mechanics from Bash into a small Bun and TypeScript CLI. +It extends Firstmate's existing files and lifecycle rather than introducing a parallel state store, controller, or agent model. +Keep compatibility with the root distro's `data/`, `state/`, briefs, status events, and guarded teardown behavior. +Use the Bun channel declared in `.bun-version`. +Keep `.bun-version`, `packageManager`, and the matching Bun type definitions aligned on the latest stable release. + +## Addresses + +Use Multiaddr strings as the single locator for supervised mates. +Persist the canonical string form only. +Do not persist Multiaddr bytes while Agent OS protocols use application-private numeric codes. +Keep credentials, resource requests, authority grants, Pod names, UIDs, and IP addresses out of addresses. +Use stable mate IDs and let each resolver discover the current runtime resource. +Treat `in-cluster` as the reserved alias for the cluster containing the caller. +The address grammar and validation contract are owned by `src/address.ts` and its tests. + +## Command Execution + +Use Bun Shell's `$` for finite calls to trusted CLIs such as `kubectl`, `herdr`, `treehouse`, and Git. +Interpolate dynamic values directly so Bun treats each value as a literal argument. +Never use `{ raw: value }` for dynamic data. +Avoid `bash -c` and validate values that an external command could interpret as flags. +Keep non-zero exits throwing by default and use `.nothrow()` only for an expected negative probe. +Set environment and working directory per command instead of mutating the global `$` instance. +Use `Bun.spawn()` for interactive agents, servers, watches, live logs, cancellation, and other long-running processes. +Existing Bash scripts may be invoked explicitly during migration, but new orchestration and state transitions belong in TypeScript. + +## Scope Discipline + +Prefer a thin adapter over a new framework. +Do not add a database, daemon, custom inter-agent protocol, Kubernetes controller, or duplicate task specification. +Keep mates general-purpose and keep parent-only communication unchanged across local and remote addresses. +Use Kubernetes APIs and later Agent Sandbox only as runtime resolvers behind the same logical address. + +## Verification + +Run `bun run check` from this directory after changes. +Add focused Bun tests beside every new address, resolver, or command contract. +Test failure, ambiguity, and unsafe input paths as well as successful execution. diff --git a/tools/agent-os/README.md b/tools/agent-os/README.md new file mode 100644 index 000000000..f37699a54 --- /dev/null +++ b/tools/agent-os/README.md @@ -0,0 +1,28 @@ +# Agent OS operating tool + +This directory contains the Bun and TypeScript operating tool that will gradually absorb Firstmate's complex fleet mechanics. +The root agent distro remains instructions, skills, trusted CLIs, and durable local files. + +The first implemented contract is a self-describing Agent OS profile over [Multiaddr](https://github.com/multiformats/multiaddr). + +```text +/local/mate/task-x/herdr +/k8s/in-cluster/namespace/agent-os/mate/task-x/herdr +``` + +The Kubernetes form names the cluster alias, namespace, stable mate ID, and terminal protocol without encoding an ephemeral Pod or IP. +The resolver may later find a raw Pod or an Agent Sandbox without changing the stored address. + +## Development + +The tool pins the latest stable Bun release in `.bun-version` and `package.json`. + +```sh +bun install +bun run check +bun run src/cli.ts address local task-x +bun run src/cli.ts address k8s task-x +bun run src/cli.ts address inspect /k8s/in-cluster/namespace/agent-os/mate/task-x/herdr +``` + +See `AGENTS.md` in this directory for the tool-specific development rules. diff --git a/tools/agent-os/bun.lock b/tools/agent-os/bun.lock new file mode 100644 index 000000000..717d11312 --- /dev/null +++ b/tools/agent-os/bun.lock @@ -0,0 +1,79 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "@akua-dev/agent-os-tool", + "dependencies": { + "@multiformats/multiaddr": "13.0.3", + }, + "devDependencies": { + "@types/bun": "1.3.14", + "typescript": "7.0.2", + }, + }, + }, + "packages": { + "@chainsafe/is-ip": ["@chainsafe/is-ip@2.1.0", "", {}, "sha512-KIjt+6IfysQ4GCv66xihEitBjvhU/bixbbbFxdJ1sqCp4uJ0wuZiYBPhksZoy4lfaF0k9cwNzY5upEW/VWdw3w=="], + + "@multiformats/multiaddr": ["@multiformats/multiaddr@13.0.3", "", { "dependencies": { "@chainsafe/is-ip": "^2.0.1", "multiformats": "^14.0.0", "uint8-varint": "^3.0.0", "uint8arrays": "^6.1.1" } }, "sha512-mEqqJ4r3a/uuFMTpRkU316wGNIDQNhuVWpm+ebKTQeYsfv9jXbPONWM6VVnj3KGUrwfsX7GZOyp4TFqEA2SPCw=="], + + "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], + + "@types/node": ["@types/node@26.1.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw=="], + + "@typescript/typescript-aix-ppc64": ["@typescript/typescript-aix-ppc64@7.0.2", "", { "os": "aix", "cpu": "ppc64" }, "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ=="], + + "@typescript/typescript-darwin-arm64": ["@typescript/typescript-darwin-arm64@7.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA=="], + + "@typescript/typescript-darwin-x64": ["@typescript/typescript-darwin-x64@7.0.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA=="], + + "@typescript/typescript-freebsd-arm64": ["@typescript/typescript-freebsd-arm64@7.0.2", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ=="], + + "@typescript/typescript-freebsd-x64": ["@typescript/typescript-freebsd-x64@7.0.2", "", { "os": "freebsd", "cpu": "x64" }, "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw=="], + + "@typescript/typescript-linux-arm": ["@typescript/typescript-linux-arm@7.0.2", "", { "os": "linux", "cpu": "arm" }, "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ=="], + + "@typescript/typescript-linux-arm64": ["@typescript/typescript-linux-arm64@7.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ=="], + + "@typescript/typescript-linux-loong64": ["@typescript/typescript-linux-loong64@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ=="], + + "@typescript/typescript-linux-mips64el": ["@typescript/typescript-linux-mips64el@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA=="], + + "@typescript/typescript-linux-ppc64": ["@typescript/typescript-linux-ppc64@7.0.2", "", { "os": "linux", "cpu": "ppc64" }, "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA=="], + + "@typescript/typescript-linux-riscv64": ["@typescript/typescript-linux-riscv64@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ=="], + + "@typescript/typescript-linux-s390x": ["@typescript/typescript-linux-s390x@7.0.2", "", { "os": "linux", "cpu": "s390x" }, "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw=="], + + "@typescript/typescript-linux-x64": ["@typescript/typescript-linux-x64@7.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A=="], + + "@typescript/typescript-netbsd-arm64": ["@typescript/typescript-netbsd-arm64@7.0.2", "", { "os": "none", "cpu": "arm64" }, "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA=="], + + "@typescript/typescript-netbsd-x64": ["@typescript/typescript-netbsd-x64@7.0.2", "", { "os": "none", "cpu": "x64" }, "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA=="], + + "@typescript/typescript-openbsd-arm64": ["@typescript/typescript-openbsd-arm64@7.0.2", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ=="], + + "@typescript/typescript-openbsd-x64": ["@typescript/typescript-openbsd-x64@7.0.2", "", { "os": "openbsd", "cpu": "x64" }, "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg=="], + + "@typescript/typescript-sunos-x64": ["@typescript/typescript-sunos-x64@7.0.2", "", { "os": "sunos", "cpu": "x64" }, "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g=="], + + "@typescript/typescript-win32-arm64": ["@typescript/typescript-win32-arm64@7.0.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ=="], + + "@typescript/typescript-win32-x64": ["@typescript/typescript-win32-x64@7.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g=="], + + "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + + "multiformats": ["multiformats@14.0.4", "", {}, "sha512-+SXItmOUWzT0kjlIfkZ4NawJaC9jW/mPlyn902V9Qgpc7YDwdEiwqbiZxTyvC0XWh1jZguXca3A/WQ+UOPRLUA=="], + + "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], + + "uint8-varint": ["uint8-varint@3.0.0", "", { "dependencies": { "uint8arraylist": "^3.0.1", "uint8arrays": "^6.1.0" } }, "sha512-S4DdpXBaLwKcFo7f0bWzWfHjbZ/i3QhM842qn+ZvHjxqFCfUcEB9SQNcmI69S+zMlcmIcKxsk9Iyw77S2Kxv6Q=="], + + "uint8arraylist": ["uint8arraylist@3.0.2", "", { "dependencies": { "uint8arrays": "^6.0.0" } }, "sha512-LDVoq9BQaGJzGDUovEnoX6rpKCvnY/Jbtws4ikwnBzjRbq5qBAFpBZevUEbSmMM87aO0Sp+wOZy2ZXf5yODmXQ=="], + + "uint8arrays": ["uint8arrays@6.1.1", "", { "dependencies": { "multiformats": "^14.0.0" } }, "sha512-iz7JN0XCSZYA111lhFG2Ui9EhFvTNekqSRHw3lvMHq+dzwWy1OQftxFQREEh4rffU0oSoXdQHsk2TiHKVm4fsA=="], + + "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], + } +} diff --git a/tools/agent-os/package.json b/tools/agent-os/package.json new file mode 100644 index 000000000..1dd58ead0 --- /dev/null +++ b/tools/agent-os/package.json @@ -0,0 +1,22 @@ +{ + "name": "@akua-dev/agent-os-tool", + "version": "0.0.0", + "private": true, + "type": "module", + "bin": { + "agent-os": "./src/cli.ts" + }, + "scripts": { + "check": "bun run typecheck && bun test", + "test": "bun test", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@multiformats/multiaddr": "13.0.3" + }, + "devDependencies": { + "@types/bun": "1.3.14", + "typescript": "7.0.2" + }, + "packageManager": "bun@1.3.14" +} diff --git a/tools/agent-os/src/address.ts b/tools/agent-os/src/address.ts new file mode 100644 index 000000000..9aee6f568 --- /dev/null +++ b/tools/agent-os/src/address.ts @@ -0,0 +1,183 @@ +import { + V, + multiaddr, + registry, + type Component, + type ProtocolCodec, +} from "@multiformats/multiaddr"; + +const APPLICATION_PROTOCOL_BASE = 0x300000; +const DNS_LABEL = /^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/; + +export const AGENT_OS_PROTOCOL_CODES = Object.freeze({ + local: APPLICATION_PROTOCOL_BASE, + k8s: APPLICATION_PROTOCOL_BASE + 1, + namespace: APPLICATION_PROTOCOL_BASE + 2, + mate: APPLICATION_PROTOCOL_BASE + 3, + herdr: APPLICATION_PROTOCOL_BASE + 4, +}); + +export type AgentAddress = + | { + address: string; + location: "local"; + mateId: string; + protocol: "herdr"; + } + | { + address: string; + cluster: string; + location: "kubernetes"; + mateId: string; + namespace: string; + protocol: "herdr"; + }; + +export interface KubernetesAddressOptions { + cluster?: string; + namespace?: string; +} + +export class AgentAddressError extends Error { + override name = "AgentAddressError"; +} + +function validateLabel(value: string, component: string): void { + if (!DNS_LABEL.test(value)) { + throw new AgentAddressError( + `Invalid ${component} value ${JSON.stringify(value)}: expected a lowercase DNS label`, + ); + } +} + +const agentOsProtocols: ProtocolCodec[] = [ + { + code: AGENT_OS_PROTOCOL_CODES.local, + name: "local", + size: 0, + }, + { + code: AGENT_OS_PROTOCOL_CODES.k8s, + name: "k8s", + size: V, + validate: (value) => validateLabel(value, "k8s cluster alias"), + }, + { + code: AGENT_OS_PROTOCOL_CODES.namespace, + name: "namespace", + size: V, + validate: (value) => validateLabel(value, "namespace"), + }, + { + code: AGENT_OS_PROTOCOL_CODES.mate, + name: "mate", + size: V, + validate: (value) => validateLabel(value, "mate ID"), + }, + { + code: AGENT_OS_PROTOCOL_CODES.herdr, + name: "herdr", + size: 0, + }, +]; + +function findProtocol(key: string | number): ProtocolCodec | undefined { + try { + return registry.getProtocol(key); + } catch { + return undefined; + } +} + +export function registerAgentOsProtocols(): void { + for (const protocol of agentOsProtocols) { + const byCode = findProtocol(protocol.code); + const byName = findProtocol(protocol.name); + + if (byCode == null && byName == null) { + registry.addProtocol(protocol); + continue; + } + + if (byCode?.name !== protocol.name || byName?.code !== protocol.code) { + throw new AgentAddressError( + `Multiaddr protocol collision for ${protocol.name}/${protocol.code}`, + ); + } + } +} + +registerAgentOsProtocols(); + +function componentValue(component: Component, name: string): string { + if (component.value == null || component.value === "") { + throw new AgentAddressError(`Agent OS address component ${name} needs a value`); + } + + return component.value; +} + +function hasNames(components: Component[], names: string[]): boolean { + return ( + components.length === names.length && + components.every((component, index) => component.name === names[index]) + ); +} + +export function parseAgentAddress(input: string): AgentAddress { + let parsed; + + try { + parsed = multiaddr(input); + } catch (error) { + throw new AgentAddressError(`Invalid Agent OS address: ${String(error)}`); + } + + const address = parsed.toString(); + const components = parsed.getComponents(); + + if (hasNames(components, ["local", "mate", "herdr"])) { + return { + address, + location: "local", + mateId: componentValue(components[1]!, "mate"), + protocol: "herdr", + }; + } + + if ( + hasNames(components, ["k8s", "namespace", "mate", "herdr"]) + ) { + return { + address, + cluster: componentValue(components[0]!, "k8s"), + location: "kubernetes", + mateId: componentValue(components[2]!, "mate"), + namespace: componentValue(components[1]!, "namespace"), + protocol: "herdr", + }; + } + + throw new AgentAddressError(`Unsupported Agent OS address: ${address}`); +} + +export function formatLocalAgentAddress(mateId: string): string { + validateLabel(mateId, "mate ID"); + return parseAgentAddress(`/local/mate/${mateId}/herdr`).address; +} + +export function formatKubernetesAgentAddress( + mateId: string, + options: KubernetesAddressOptions = {}, +): string { + const cluster = options.cluster ?? "in-cluster"; + const namespace = options.namespace ?? "agent-os"; + + validateLabel(cluster, "k8s cluster alias"); + validateLabel(namespace, "namespace"); + validateLabel(mateId, "mate ID"); + + return parseAgentAddress( + `/k8s/${cluster}/namespace/${namespace}/mate/${mateId}/herdr`, + ).address; +} diff --git a/tools/agent-os/src/cli.ts b/tools/agent-os/src/cli.ts new file mode 100755 index 000000000..5cd221ed4 --- /dev/null +++ b/tools/agent-os/src/cli.ts @@ -0,0 +1,85 @@ +#!/usr/bin/env bun + +import { parseArgs } from "node:util"; + +import { + formatKubernetesAgentAddress, + formatLocalAgentAddress, + parseAgentAddress, +} from "./address.ts"; + +const usage = `Usage: + agent-os address local + agent-os address k8s [--cluster ] [--namespace ] + agent-os address inspect `; + +function requireSingleArgument(args: string[]): string { + if (args.length !== 1 || args[0] == null) { + throw new Error(usage); + } + + return args[0]; +} + +function runAddressCommand(args: string[]): void { + const [command, ...rest] = args; + + switch (command) { + case "local": + console.log(formatLocalAgentAddress(requireSingleArgument(rest))); + return; + + case "k8s": { + const { positionals, values } = parseArgs({ + allowPositionals: true, + args: rest, + options: { + cluster: { + default: "in-cluster", + type: "string", + }, + namespace: { + default: "agent-os", + type: "string", + }, + }, + strict: true, + }); + const mateId = requireSingleArgument(positionals); + + console.log( + formatKubernetesAgentAddress(mateId, { + cluster: values.cluster, + namespace: values.namespace, + }), + ); + return; + } + + case "inspect": + console.log( + JSON.stringify(parseAgentAddress(requireSingleArgument(rest)), null, 2), + ); + return; + + default: + throw new Error(usage); + } +} + +function main(): void { + const [command, ...rest] = Bun.argv.slice(2); + + if (command !== "address") { + throw new Error(usage); + } + + runAddressCommand(rest); +} + +try { + main(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +} diff --git a/tools/agent-os/test/address.test.ts b/tools/agent-os/test/address.test.ts new file mode 100644 index 000000000..4f5a7b4d7 --- /dev/null +++ b/tools/agent-os/test/address.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, test } from "bun:test"; + +import { + formatKubernetesAgentAddress, + formatLocalAgentAddress, + parseAgentAddress, +} from "../src/address.ts"; + +describe("Agent OS Multiaddrs", () => { + test("formats and parses a co-located Herdr mate", () => { + const address = formatLocalAgentAddress("task-x"); + + expect(address).toBe("/local/mate/task-x/herdr"); + expect(parseAgentAddress(address)).toEqual({ + address, + location: "local", + mateId: "task-x", + protocol: "herdr", + }); + }); + + test("defaults Kubernetes addresses to the caller's cluster", () => { + const address = formatKubernetesAgentAddress("task-x"); + + expect(address).toBe( + "/k8s/in-cluster/namespace/agent-os/mate/task-x/herdr", + ); + expect(parseAgentAddress(address)).toEqual({ + address, + cluster: "in-cluster", + location: "kubernetes", + mateId: "task-x", + namespace: "agent-os", + protocol: "herdr", + }); + }); + + test("accepts explicit registered cluster and namespace aliases", () => { + const address = formatKubernetesAgentAddress("review-7", { + cluster: "agents-eu", + namespace: "company-factory", + }); + + expect(address).toBe( + "/k8s/agents-eu/namespace/company-factory/mate/review-7/herdr", + ); + }); + + test.each([ + "/k8s/in-cluster/mate/task-x/herdr", + "/k8s/in-cluster/namespace/agent-os/herdr/mate/task-x", + "/local/namespace/agent-os/mate/task-x/herdr", + ])("rejects an unsupported component sequence: %s", (address) => { + expect(() => parseAgentAddress(address)).toThrow("Unsupported Agent OS address"); + }); + + test.each(["Task-X", "-task", "task/other", "--namespace"])( + "rejects an unsafe mate ID: %s", + (mateId) => { + expect(() => formatLocalAgentAddress(mateId)).toThrow("mate"); + }, + ); +}); diff --git a/tools/agent-os/test/cli.test.ts b/tools/agent-os/test/cli.test.ts new file mode 100644 index 000000000..1d6baf2f3 --- /dev/null +++ b/tools/agent-os/test/cli.test.ts @@ -0,0 +1,36 @@ +import { $ } from "bun"; +import { describe, expect, test } from "bun:test"; +import { fileURLToPath } from "node:url"; + +const cli = fileURLToPath(new URL("../src/cli.ts", import.meta.url)); + +describe("agent-os address CLI", () => { + test("prints a Kubernetes address with in-cluster defaults", async () => { + const output = await $`${process.execPath} ${cli} address k8s task-x`.text(); + + expect(output.trim()).toBe( + "/k8s/in-cluster/namespace/agent-os/mate/task-x/herdr", + ); + }); + + test("inspects an address as structured JSON", async () => { + const address = "/local/mate/task-x/herdr"; + const output = await $`${process.execPath} ${cli} address inspect ${address}`.json(); + + expect(output).toEqual({ + address, + location: "local", + mateId: "task-x", + protocol: "herdr", + }); + }); + + test("fails closed for an invalid address", async () => { + const result = await $`${process.execPath} ${cli} address inspect /mate/task-x` + .quiet() + .nothrow(); + + expect(result.exitCode).toBe(1); + expect(result.stderr.toString()).toContain("Unsupported Agent OS address"); + }); +}); diff --git a/tools/agent-os/tsconfig.json b/tools/agent-os/tsconfig.json new file mode 100644 index 000000000..ae04ceb65 --- /dev/null +++ b/tools/agent-os/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "allowImportingTsExtensions": true, + "forceConsistentCasingInFileNames": true, + "module": "Preserve", + "moduleDetection": "force", + "moduleResolution": "Bundler", + "noEmit": true, + "noUncheckedIndexedAccess": true, + "strict": true, + "target": "ESNext", + "types": ["bun"] + }, + "include": ["src/**/*.ts", "test/**/*.ts"] +} From c9423d48a33df3c73989340ebc62e7e27423a037 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 10:02:33 +0200 Subject: [PATCH 10/56] feat: add Akua mate packaging --- .agents/skills/kubernetes-fleet/SKILL.md | 4 +- .dockerignore | 3 +- Dockerfile | 41 +- THIRD_PARTY_NOTICES.md | 8 + bin/fm-bootstrap.sh | 10 +- docs/kubernetes.md | 12 +- tests/agent-os-container.test.sh | 14 +- tests/fm-bootstrap.test.sh | 18 +- tests/fm-secondmate-harness.test.sh | 2 +- tests/fm-secondmate-liveness.test.sh | 2 +- tests/fm-secondmate-sync.test.sh | 2 +- tests/fm-session-start.test.sh | 2 +- tests/fm-x-mode.test.sh | 2 +- .../.agents/skills/effect-ts/SKILL.md | 191 ++++ .../skills/effect-ts/references/features.md | 504 ++++++++ .../effect-ts/references/guide-effect.md | 447 ++++++++ .../references/guide-error-handling.md | 540 +++++++++ .../effect-ts/references/guide-layers.md | 1018 +++++++++++++++++ .../references/guide-observability.md | 724 ++++++++++++ .../effect-ts/references/guide-retries.md | 433 +++++++ .../effect-ts/references/guide-schedule.md | 379 ++++++ .../effect-ts/references/guide-schema.md | 624 ++++++++++ .../skills/effect-ts/references/guide-sql.md | 536 +++++++++ .../effect-ts/references/guide-testing.md | 504 ++++++++ .../skills/effect-ts/references/setup.md | 89 ++ tools/agent-os/.gitignore | 1 + tools/agent-os/AGENTS.md | 18 +- tools/agent-os/README.md | 5 + tools/agent-os/bun.lock | 42 + tools/agent-os/package.json | 5 +- tools/agent-os/packages/mate/README.md | 25 + tools/agent-os/packages/mate/akua.toml | 5 + .../packages/mate/inputs.example.yaml | 5 + tools/agent-os/packages/mate/package.k | 101 ++ tools/agent-os/scripts/prepare-effect.sh | 13 + tools/agent-os/skills-lock.json | 11 + tools/agent-os/src/cli.ts | 65 +- tools/agent-os/src/json-modules.d.ts | 4 + tools/agent-os/src/mate-package.ts | 151 +++ tools/agent-os/test/mate-package.test.ts | 81 ++ 40 files changed, 6617 insertions(+), 24 deletions(-) create mode 100644 tools/agent-os/.agents/skills/effect-ts/SKILL.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/features.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-effect.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-error-handling.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-layers.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-observability.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-retries.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-schedule.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-schema.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-sql.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-testing.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/setup.md create mode 100644 tools/agent-os/packages/mate/README.md create mode 100644 tools/agent-os/packages/mate/akua.toml create mode 100644 tools/agent-os/packages/mate/inputs.example.yaml create mode 100644 tools/agent-os/packages/mate/package.k create mode 100755 tools/agent-os/scripts/prepare-effect.sh create mode 100644 tools/agent-os/skills-lock.json create mode 100644 tools/agent-os/src/json-modules.d.ts create mode 100644 tools/agent-os/src/mate-package.ts create mode 100644 tools/agent-os/test/mate-package.test.ts diff --git a/.agents/skills/kubernetes-fleet/SKILL.md b/.agents/skills/kubernetes-fleet/SKILL.md index a43913e54..d39f7c1b1 100644 --- a/.agents/skills/kubernetes-fleet/SKILL.md +++ b/.agents/skills/kubernetes-fleet/SKILL.md @@ -14,6 +14,9 @@ Load this skill only when the current firstmate is running in Kubernetes or is e - Keep every crewmate general-purpose; its brief, tools, and authority specialize it for the current task. - A crewmate normally communicates only with its parent through its terminal, status files, reports, and delivered Git state. +- The image includes Akua's CLI for direct use and the Agent OS tool uses `@akua-dev/sdk` in process. +- Use `agent-os mate render ` when the prepared package is useful, then inspect or edit its ordinary YAML before applying it. +- The package is optional; direct `akua render`, raw YAML, and `kubectl` remain supported. - Use `bin/agent-os-crewmate.sh create ` to create a separate Pod and persistent home. - Use `status` to inspect it and `delete` only after unique work is checkpointed or delivered. - Never mount the primary home into a child Pod. @@ -23,4 +26,3 @@ Load this skill only when the current firstmate is running in Kubernetes or is e Use ordinary `kubectl exec`, Herdr, files, and Git to supervise the child. Do not add a custom inter-agent chat protocol or Task/Run service. - diff --git a/.dockerignore b/.dockerignore index e1767260e..90b53692f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,4 +13,5 @@ projects state worktrees .worktrees - +node_modules +.repos diff --git a/Dockerfile b/Dockerfile index 0ae8012fd..2a1979ece 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:24-bookworm-slim +FROM node:24-trixie-slim ARG TARGETARCH ARG HERDR_VERSION=0.7.3 @@ -6,6 +6,8 @@ ARG KUBECTL_VERSION=1.34.8 ARG GH_VERSION=2.96.0 ARG TREEHOUSE_VERSION=2.0.0 ARG NO_MISTAKES_VERSION=1.34.0 +ARG BUN_VERSION=1.3.14 +ARG AKUA_VERSION=0.8.25 RUN apt-get update \ && apt-get install -y --no-install-recommends \ @@ -20,6 +22,7 @@ RUN apt-get update \ ripgrep \ rsync \ tmux \ + unzip \ && ln -s /usr/bin/fdfind /usr/local/bin/fd \ && rm -rf /var/lib/apt/lists/* @@ -75,6 +78,34 @@ RUN set -eu; \ chmod 0755 /usr/local/bin/treehouse /usr/local/bin/no-mistakes; \ rm -f "/tmp/${treehouse_asset}" "/tmp/${no_mistakes_asset}" +RUN set -eu; \ + case "$TARGETARCH" in \ + amd64) bun_arch=x64; sha=951ee2aee855f08595aeec6225226a298d3fea83a3dcd6465c09cbccdf7e848f ;; \ + arm64) bun_arch=aarch64; sha=a27ffb63a8310375836e0d6f668ae17fa8d8d18b88c37c821c65331973a19a3b ;; \ + *) echo "unsupported TARGETARCH: $TARGETARCH" >&2; exit 1 ;; \ + esac; \ + asset="bun-linux-${bun_arch}.zip"; \ + curl -fsSL "https://github.com/oven-sh/bun/releases/download/bun-v${BUN_VERSION}/${asset}" -o "/tmp/${asset}"; \ + echo "$sha /tmp/${asset}" | sha256sum -c -; \ + unzip -q "/tmp/${asset}" -d /tmp/bun; \ + install -m 0755 "/tmp/bun/bun-linux-${bun_arch}/bun" /usr/local/bin/bun; \ + rm -rf "/tmp/${asset}" /tmp/bun + +RUN set -eu; \ + case "$TARGETARCH" in \ + amd64) triple=x86_64-unknown-linux-gnu; sha=bc57afbffe7e18aacd2146e2cd67151c56e7a3c279fe659312ff7ffb359cd03a ;; \ + arm64) triple=aarch64-unknown-linux-gnu; sha=3a3c6bae72764cbd85a6e4e0877a05e5def8f7aeee8563b7918099214a1a313a ;; \ + *) echo "unsupported TARGETARCH: $TARGETARCH" >&2; exit 1 ;; \ + esac; \ + asset="akua-v${AKUA_VERSION}-${triple}.tar.gz"; \ + curl -fsSL "https://github.com/akua-dev/akua/releases/download/v${AKUA_VERSION}/${asset}" -o "/tmp/${asset}"; \ + echo "$sha /tmp/${asset}" | sha256sum -c -; \ + tar -xzf "/tmp/${asset}" -C /tmp; \ + install -m 0755 /tmp/akua /usr/local/bin/akua; \ + rm -f "/tmp/${asset}" /tmp/akua; \ + mkdir -p /usr/share/licenses/akua; \ + curl -fsSL "https://raw.githubusercontent.com/akua-dev/akua/v${AKUA_VERSION}/LICENSE" -o /usr/share/licenses/akua/LICENSE + RUN npm install --global \ @earendil-works/pi-coding-agent@0.80.6 \ gh-axi@0.1.27 \ @@ -94,10 +125,14 @@ ENV FM_HOME=/home/agent \ PATH=/home/agent/.local/bin:/home/agent/.bun/bin:/home/agent/.cargo/bin:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin \ HERDR_SESSION=default -RUN mkdir -p /home/agent /opt/agent-os /opt/image-usr-local \ - && cp -a /usr/local/. /opt/image-usr-local/ +RUN mkdir -p /home/agent /opt/agent-os /opt/image-usr-local COPY . /opt/agent-os +RUN cd /opt/agent-os/tools/agent-os \ + && bun install --frozen-lockfile --production --ignore-scripts \ + && ln -s /opt/agent-os/tools/agent-os/src/cli.ts /usr/local/bin/agent-os \ + && cp -a /usr/local/. /opt/image-usr-local/ + WORKDIR /opt/agent-os ENTRYPOINT ["/opt/agent-os/bin/agent-os-container-entrypoint.sh"] diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 86fa4f8dc..0049de365 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -7,3 +7,11 @@ Herdr is available under AGPL-3.0-or-later or a commercial license. The image includes Herdr's license at `/usr/share/licenses/herdr/LICENSE`. The exact corresponding source is available at . Agent OS invokes Herdr through its documented CLI and socket interfaces. + +## Akua + +The Agent OS demo image includes an unmodified Akua 0.8.25 executable as a separate program. +Akua is available under the Apache License 2.0. +The image includes Akua's license at `/usr/share/licenses/akua/LICENSE`. +The exact source is available at . +Agent OS invokes Akua through its documented CLI. diff --git a/bin/fm-bootstrap.sh b/bin/fm-bootstrap.sh index 7a2b2a9f9..d46478b6b 100755 --- a/bin/fm-bootstrap.sh +++ b/bin/fm-bootstrap.sh @@ -43,7 +43,10 @@ # "treehouse get --lease" support. # no-mistakes is also MISSING when its installed version is older than # 1.31.2. -# tasks-axi and quota-axi are required bootstrap tools (same class as +# Akua, tasks-axi, and quota-axi are required bootstrap tools. +# Akua renders the optional Agent OS mate package into ordinary YAML; +# firstmate may also author and apply Kubernetes YAML directly. +# tasks-axi and quota-axi are in the same class as # lavish-axi). tasks-axi is also version and feature gated (0.1.1+ # with update --archive-body and mv [...]); an installed but incompatible build # reports MISSING like no-mistakes. When @@ -314,6 +317,7 @@ install_cmd() { tmux|node|git|gh|curl|jq|orca) echo "brew install $1 # or the platform's package manager" ;; treehouse) echo "curl -fsSL https://kunchenguid.github.io/treehouse/install.sh | sh" ;; no-mistakes) echo "curl -fsSL https://raw.githubusercontent.com/kunchenguid/no-mistakes/main/docs/install.sh | sh" ;; + akua) echo "curl -fsSL https://cli.akua.dev/install | sh" ;; gh-axi|chrome-devtools-axi|lavish-axi) echo "npm install -g $1 && $1 setup hooks" ;; tasks-axi|quota-axi) echo "npm install -g $1" ;; *) return 1 ;; @@ -322,8 +326,8 @@ install_cmd() { BACKEND=$(fm_backend_name) case "$BACKEND" in - orca) TOOLS="orca node git gh no-mistakes gh-axi chrome-devtools-axi lavish-axi tasks-axi quota-axi" ;; - *) TOOLS="tmux node git gh treehouse no-mistakes gh-axi chrome-devtools-axi lavish-axi tasks-axi quota-axi" ;; + orca) TOOLS="orca node git gh akua no-mistakes gh-axi chrome-devtools-axi lavish-axi tasks-axi quota-axi" ;; + *) TOOLS="tmux node git gh akua treehouse no-mistakes gh-axi chrome-devtools-axi lavish-axi tasks-axi quota-axi" ;; esac NO_MISTAKES_MIN_MAJOR=1 NO_MISTAKES_MIN_MINOR=31 diff --git a/docs/kubernetes.md b/docs/kubernetes.md index bfde97b14..553bba78e 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -35,7 +35,7 @@ bin/agent-os-local.sh status bin/agent-os-local.sh shell ``` -The image includes Firstmate's complete required toolchain, including `gh`, `rg`, `fd`, treehouse, no-mistakes, and every required AXI CLI. +The image includes Firstmate's complete required toolchain, including `gh`, `rg`, `fd`, Akua, treehouse, no-mistakes, and every required AXI CLI. Authenticate GitHub inside the primary with `gh auth login`. Authenticate Pi using `/login` and the provider flow you choose. Host credentials are intentionally excluded from the image and are never copied automatically. @@ -64,6 +64,16 @@ bin/agent-os-crewmate.sh status scout-1 bin/agent-os-crewmate.sh delete scout-1 ``` +It can also render the prepared Akua package without spawning a subprocess from the TypeScript tool: + +```sh +agent-os mate render scout-1 --namespace agent-os-demo --image agent-os:dev --out /tmp/scout-1 +kubectl apply -f /tmp/scout-1 +``` + +This command uses `@akua-dev/sdk` and only writes ordinary manifests. +Firstmate can inspect or edit those files, invoke the bundled `akua` CLI directly, modify the package, or author equivalent YAML without the helper. + When run on a host, the crewmate helper requires an explicit context: ```sh diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 4663959dd..1ceefb506 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -5,12 +5,15 @@ set -u # shellcheck source=tests/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -assert_grep 'FROM node:24-bookworm-slim' "$ROOT/Dockerfile" "image must pin Node 24 Bookworm" +assert_grep 'FROM node:24-trixie-slim' "$ROOT/Dockerfile" "image must pin Node 24 Trixie" assert_grep 'ARG HERDR_VERSION=0.7.3' "$ROOT/Dockerfile" "image must pin Herdr 0.7.3" assert_grep 'ARG KUBECTL_VERSION=1.34.8' "$ROOT/Dockerfile" "image must pin kubectl 1.34.8" assert_grep 'ARG GH_VERSION=2.96.0' "$ROOT/Dockerfile" "image must pin GitHub CLI 2.96.0" assert_grep 'ARG TREEHOUSE_VERSION=2.0.0' "$ROOT/Dockerfile" "image must pin treehouse 2.0.0" assert_grep 'ARG NO_MISTAKES_VERSION=1.34.0' "$ROOT/Dockerfile" "image must pin no-mistakes 1.34.0" +assert_grep 'ARG BUN_VERSION=1.3.14' "$ROOT/Dockerfile" "image must pin stable Bun 1.3.14" +assert_grep 'ARG AKUA_VERSION=0.8.25' "$ROOT/Dockerfile" "image must pin Akua 0.8.25" +assert_grep '"@akua-dev/sdk": "0.8.24"' "$ROOT/tools/agent-os/package.json" "tool must pin the Akua SDK" assert_grep 'sha256sum -c -' "$ROOT/Dockerfile" "downloaded runtime binaries must be checksum verified" assert_grep '@earendil-works/pi-coding-agent@0.80.6' "$ROOT/Dockerfile" "image must pin Pi 0.80.6" assert_grep 'gh-axi@0.1.27' "$ROOT/Dockerfile" "image must pin gh-axi 0.1.27" @@ -26,14 +29,23 @@ assert_grep 'NPM_CONFIG_PREFIX=/usr/local' "$ROOT/Dockerfile" "global npm instal assert_grep 'PATH=/home/agent/.local/bin:/home/agent/.bun/bin:/home/agent/.cargo/bin:/usr/local/bin' "$ROOT/Dockerfile" \ "persistent tool prefixes must lead PATH" assert_grep '/opt/image-usr-local' "$ROOT/Dockerfile" "image must retain a seed copy of /usr/local" +assert_grep 'akua-dev/akua/releases/download/v${AKUA_VERSION}' "$ROOT/Dockerfile" "image must install Akua from its release" +assert_grep 'ln -s /opt/agent-os/tools/agent-os/src/cli.ts /usr/local/bin/agent-os' "$ROOT/Dockerfile" \ + "image must expose the Agent OS tool" +assert_grep 'bun install --frozen-lockfile --production --ignore-scripts' "$ROOT/Dockerfile" \ + "image install must not clone development source checkouts" assert_no_grep 'USER node' "$ROOT/Dockerfile" "Agent OS containers must start as container root" assert_grep 'exec herdr server' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must keep Herdr as PID 1" assert_grep 'setup hooks' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must install persistent AXI hooks" assert_grep '.git' "$ROOT/.dockerignore" "git metadata must stay out of the build context" assert_grep '.pi' "$ROOT/.dockerignore" "Pi credentials must stay out of the build context" assert_grep '.codex' "$ROOT/.dockerignore" "Codex credentials must stay out of the build context" +assert_grep 'node_modules' "$ROOT/.dockerignore" "host dependencies must stay out of the build context" +assert_grep '.repos' "$ROOT/.dockerignore" "development source checkouts must stay out of the build context" assert_grep 'https://github.com/ogulcancelik/herdr/tree/v0.7.3' "$ROOT/THIRD_PARTY_NOTICES.md" \ "Herdr's exact corresponding source must be named" +assert_grep 'https://github.com/akua-dev/akua/tree/v0.8.25' "$ROOT/THIRD_PARTY_NOTICES.md" \ + "Akua's exact source must be named" bash -n "$ROOT/bin/agent-os-container-entrypoint.sh" pass "container files pin dependencies and exclude host credentials" diff --git a/tests/fm-bootstrap.test.sh b/tests/fm-bootstrap.test.sh index f16314e95..83204e381 100755 --- a/tests/fm-bootstrap.test.sh +++ b/tests/fm-bootstrap.test.sh @@ -27,7 +27,7 @@ TMP_ROOT=$(fm_test_tmproot fm-bootstrap-tests) make_fake_toolchain() { local dir=$1 fakebin fakebin=$(fm_fakebin "$dir") - fm_fake_exit0 "$fakebin" tmux node gh-axi chrome-devtools-axi lavish-axi + fm_fake_exit0 "$fakebin" tmux node akua gh-axi chrome-devtools-axi lavish-axi cat > "$fakebin/gh" <<'SH' #!/usr/bin/env bash if [ "${1:-}" = auth ] && [ "${2:-}" = status ]; then @@ -327,6 +327,21 @@ SH pass "bootstrap requires git with an install instruction" } +test_akua_is_required_with_supported_install_instruction() { + local case_dir fakebin out expected + case_dir="$TMP_ROOT/akua-required" + mkdir -p "$case_dir/home/config" + printf '%s\n' manual > "$case_dir/home/config/backlog-backend" + fakebin=$(make_fake_toolchain "$case_dir") + rm -f "$fakebin/akua" + + out=$(PATH="$fakebin:$BASE_PATH" FM_HOME="$case_dir/home" FM_ROOT_OVERRIDE="$case_dir/home" \ + FM_FAKE_TREEHOUSE_LEASE_HELP=1 "$ROOT/bin/fm-bootstrap.sh") + expected="MISSING: akua (install: curl -fsSL https://cli.akua.dev/install | sh)" + [ "$out" = "$expected" ] || fail "missing Akua should report the supported install instruction, got: $out" + pass "bootstrap requires Akua with an install instruction" +} + test_orca_backend_gates_orca_tool_only_when_selected() { local case_dir fakebin out missing_orca missing_orca="MISSING: orca (install: brew install orca # or the platform's package manager)" @@ -495,6 +510,7 @@ ROWS test_bootstrap_reporting test_no_mistakes_min_version test_git_is_required_with_supported_install_instruction +test_akua_is_required_with_supported_install_instruction test_orca_backend_gates_orca_tool_only_when_selected test_fleet_sync_timeout_scales_with_origin_backed_project_count test_fleet_sync_timeout_floor_preserves_small_fleets diff --git a/tests/fm-secondmate-harness.test.sh b/tests/fm-secondmate-harness.test.sh index 1a314c050..71b22ae11 100755 --- a/tests/fm-secondmate-harness.test.sh +++ b/tests/fm-secondmate-harness.test.sh @@ -720,7 +720,7 @@ make_fake_toolchain() { local dir=$1 fakebin fakebin="$dir/fakebin" mkdir -p "$fakebin" - fm_fake_exit0 "$fakebin" tmux node gh-axi chrome-devtools-axi lavish-axi + fm_fake_exit0 "$fakebin" tmux node akua gh-axi chrome-devtools-axi lavish-axi cat > "$fakebin/gh" <<'SH' #!/usr/bin/env bash exit 0 diff --git a/tests/fm-secondmate-liveness.test.sh b/tests/fm-secondmate-liveness.test.sh index f639e836e..b7308e28c 100755 --- a/tests/fm-secondmate-liveness.test.sh +++ b/tests/fm-secondmate-liveness.test.sh @@ -169,7 +169,7 @@ test_agent_alive_dispatcher_routes_and_falls_back() { make_toolchain() { local dir=$1 fakebin fakebin=$(fm_fakebin "$dir") - fm_fake_exit0 "$fakebin" node gh-axi chrome-devtools-axi lavish-axi + fm_fake_exit0 "$fakebin" node akua gh-axi chrome-devtools-axi lavish-axi cat > "$fakebin/gh" <<'SH' #!/usr/bin/env bash exit 0 diff --git a/tests/fm-secondmate-sync.test.sh b/tests/fm-secondmate-sync.test.sh index bc806523d..80c30c4da 100755 --- a/tests/fm-secondmate-sync.test.sh +++ b/tests/fm-secondmate-sync.test.sh @@ -289,7 +289,7 @@ make_fake_toolchain() { local dir=$1 fakebin fakebin="$dir/fakebin" mkdir -p "$fakebin" - fm_fake_exit0 "$fakebin" tmux node gh-axi chrome-devtools-axi lavish-axi + fm_fake_exit0 "$fakebin" tmux node akua gh-axi chrome-devtools-axi lavish-axi cat > "$fakebin/gh" <<'SH' #!/usr/bin/env bash exit 0 diff --git a/tests/fm-session-start.test.sh b/tests/fm-session-start.test.sh index 4e1d68eff..c7b3f953a 100755 --- a/tests/fm-session-start.test.sh +++ b/tests/fm-session-start.test.sh @@ -53,7 +53,7 @@ new_world() { # test deliberately breaks one. Mirrors fm-bootstrap.test.sh's fixture. make_fake_toolchain() { local fakebin=$1 - fm_fake_exit0 "$fakebin" tmux node gh-axi chrome-devtools-axi lavish-axi + fm_fake_exit0 "$fakebin" tmux node akua gh-axi chrome-devtools-axi lavish-axi cat > "$fakebin/gh" <<'SH' #!/usr/bin/env bash exit 0 diff --git a/tests/fm-x-mode.test.sh b/tests/fm-x-mode.test.sh index 89b71187c..9b45d9870 100755 --- a/tests/fm-x-mode.test.sh +++ b/tests/fm-x-mode.test.sh @@ -442,7 +442,7 @@ test_bootstrap_reports_missing_x_dependency() { local home fakebin out tool tool_path home="$TMP_ROOT/boot-missing-x"; mkdir -p "$home" fakebin=$(fm_fakebin "$home") - fm_fake_exit0 "$fakebin" tmux node no-mistakes gh-axi chrome-devtools-axi lavish-axi curl + fm_fake_exit0 "$fakebin" tmux node akua no-mistakes gh-axi chrome-devtools-axi lavish-axi curl for tool in dirname grep tail; do tool_path=$(command -v "$tool") || fail "test host must provide $tool" ln -s "$tool_path" "$fakebin/$tool" diff --git a/tools/agent-os/.agents/skills/effect-ts/SKILL.md b/tools/agent-os/.agents/skills/effect-ts/SKILL.md new file mode 100644 index 000000000..62a95f239 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/SKILL.md @@ -0,0 +1,191 @@ +--- +name: effect-ts +description: Use this skill whenever working in a repository that uses Effect, even if the current task is in a new file or the user does not explicitly ask for Effect help. Apply it to any work that should follow the repository's Effect patterns, conventions, architecture, or supporting tooling. Also use it for questions about Effect patterns, services, layers, schemas, streams, runtimes, or typed error handling. +--- + +# Effect Expert + +Expert guidance for programming with the Effect library, covering error handling, dependency injection, composability, and testing patterns. + +## Prerequisite + +Before doing any other Effect-related work, check that `./.repos/effect` exists at the root of the repository where the skill is being used. + +If it does not exist, stop and prompt the user with the setup task documented in `./references/setup.md`. + +## Research Strategy + +Effect has many ways to accomplish the same task. Proactively research best practices when working with Effect patterns, especially for moderate to high complexity tasks. + +Use the local guides in `./references/` first. They are the preferred source for best practices, conventions, and common implementation patterns. + +Only go directly to the vendored Effect repo when: + +- the guides do not cover the question +- you need exact API details or signatures +- you need deeper implementation details +- you need to verify a behavior against the source + +### Research Sources + +1. Local skill guides first. Start with the relevant files in `./references/` before doing deeper research. +2. Codebase patterns second. Examine similar patterns in the current project before implementing. If Effect patterns already exist, follow them for consistency. If no patterns exist, skip this step. +3. Effect source code last. For gaps in the guides, complex type errors, unclear behavior, or implementation details, examine the vendored Effect source at `./.repos/effect/packages/effect/src/`. + +### When To Research + +- Always research for services, layers, or complex dependency injection. +- Always research for error handling with multiple error types or complex error hierarchies. +- Always research for stream-based operations and reactive patterns. +- Always research for resource management with scoped effects and cleanup. +- Always research for concurrent or performance-critical code. +- Always research for unfamiliar testing patterns. +- Research when needed for complex refactors from promises or try/catch into Effect. +- Research when needed for new service dependencies or layer restructuring. +- Research when needed for custom error types or extensions of existing error hierarchies. +- Research when needed for integrations with external systems such as databases, APIs, or third-party services. + +### Research Approach + +- Focus on canonical, readable, and maintainable solutions rather than clever optimizations. +- Verify suggested approaches against existing codebase patterns when those patterns exist. +- When multiple approaches are possible, prefer the most idiomatic Effect solution supported by the codebase and the vendored source. + +### Codebase Pattern Discovery + +When working in a project that uses Effect, check for existing patterns before implementing new code: + +1. Search for Effect imports and existing module usage to understand current conventions. +2. Identify how services and layers are structured in the project. +3. Note how errors are defined and propagated. +4. Examine how Effect code is tested in the project. + +If no Effect patterns exist in the codebase, proceed using canonical patterns from the vendored Effect source and examples. Do not block on missing codebase patterns. + +### Feature Discovery + +When you need to discover available Effect modules, packages, or capabilities, search `./references/features.md` first. + +- Use it to identify the right package or module for a task. +- Use the listed repo paths to jump directly into the vendored source under `./.repos/effect`. +- Use it before inventing custom abstractions when Effect may already provide the functionality. + +### Guide Discovery + +When the task touches one of these areas, consult the matching guide before implementing: + +- `./references/guide-effect.md` for core `Effect` usage patterns, common constructors, composition, provisioning, and runtime boundaries +- `./references/guide-error-handling.md` for defining errors, schema-based errors, failure handling, defects, and interrupts +- `./references/guide-layers.md` for services, layer construction, composition, and provisioning patterns +- `./references/guide-observability.md` for `Effect.fn`, spans, logging, metrics, and telemetry wiring +- `./references/guide-retries.md` for retry policies, retry conditions, fallback strategies, and `ExecutionPlan` +- `./references/guide-schedule.md` for retries, repeats, backoff, polling, cron, and schedule composition +- `./references/guide-schema.md` for schema design, transformations, unions, recursion, opaque/branded types, and schema best practices +- `./references/guide-sql.md` for Effect SQL usage, transactions, resolvers, schema-aware SQL, and migrations +- `./references/guide-testing.md` for detailed `@effect/vitest` usage, layered test setup, property tests, and test services + +These guides should be treated as the default implementation guidance. Do not skip them and jump straight to `./.repos/effect` unless you need source-level confirmation or the guides do not answer the question. + +## Effect Principles + +Apply these core principles when writing Effect code. + +## Installation + +When installing Effect packages in a user repository: + +- use `effect@beta` +- keep all `@effect/*` packages on aligned versions +- install only the packages needed for the user's runtime and actual task + +### Version Rules + +- `effect` should be installed as `effect@beta` +- if you install any `@effect/*` package, make sure all `@effect/*` packages use matching versions +- do not mix unrelated `@effect/*` versions in the same project + +### Package Selection + +Choose packages based on the runtime and the work being done. + +- core library: `effect@beta` +- Node.js runtime needs: install the matching `@effect/platform-node` +- browser runtime needs: install the matching `@effect/platform-browser` +- Bun runtime needs: install the matching `@effect/platform-bun` +- Vitest integration needs: install the matching `@effect/vitest` +- OpenTelemetry integration needs: install the matching `@effect/opentelemetry` + +Install additional `@effect/*` packages only when the user task actually needs them. + +### Practical Rule + +- start with `effect@beta` +- add `@effect/*` packages as needed by runtime and features +- keep the full installed Effect package set version-aligned + +### Error Handling + +- Use Effect's typed error system instead of throwing exceptions. +- Define descriptive error types with proper error propagation. +- Prefer `Schema.TaggedErrorClass` when the error can be schema-defined. +- Use `Effect.fail`, `Effect.catchTag`, and `Effect.catch` for error control flow. + +### Dependency Injection + +- Implement dependency injection using services and layers. +- Define services with `Context.Tag`. +- Compose layers with `Layer.merge` and `Layer.provide`. +- Use `Effect.provide` to inject dependencies at the edge, avoid providing locally. +- Keep services encapsulated; avoid exporting trivial accessor wrappers that only forward to one service method. + +### Composability + +- Leverage Effect composability for complex operations. +- Use appropriate constructors such as `Effect.succeed`, `Effect.fail`, `Effect.tryPromise`, `Effect.try`, and `Effect.sync`. +- Apply proper resource management with scoped effects. +- Chain operations with `Effect.flatMap`, `Effect.map`, and `Effect.tap`. + +### Business Logic Functions + +- Prefer `Effect.fn` for reusable business-logic functions that return `Effect`. +- Prefer `Effect.fn` over raw `Effect.gen` definitions even when the function takes no arguments. +- If you do not want an explicit named span, use `Effect.fn` without a span name. +- Do not use `Effect.fnUntraced` as the default. +- Use `Effect.fnUntraced` only for edge cases with a concrete low-level reason, such as measured hot-path overhead. + +### TypeScript Preferences + +- Never use `any`. +- Never use `as` casts. +- Never use unsafe type assertions or escape hatches. +- Never use `namespace`. +- Prefer correct typing, schema-driven decoding, narrowing, and proper generic constraints instead of forcing types. +- If a value comes from an external boundary, validate or decode it instead of asserting its type. +- If a type is hard to express, simplify the design or introduce a properly typed helper instead of using unsafe TypeScript. +- For layers, do not hide them inside `namespace` blocks. Prefer either `static` members on the service class or plain exported layer constants. + +### Code Quality + +- Write type-safe code that leverages Effect's type system. +- Use `Effect.gen` for readable sequential code. +- Implement proper testing patterns using Effect testing utilities. +- Prefer existing Effect primitives before introducing custom helpers. +- Prefer `Schema.Class` / `Schema.TaggedClass` variants over plain `Schema.Struct` for named reusable schemas when possible. + +### Explaining Solutions + +When providing solutions, explain the Effect concepts being used and why they fit the specific use case. If you encounter patterns not covered in local references, prefer consistency with the codebase when possible and otherwise rely on the vendored Effect source. + +## References + +- `./references/features.md` +- `./references/guide-effect.md` +- `./references/guide-error-handling.md` +- `./references/guide-layers.md` +- `./references/guide-observability.md` +- `./references/guide-retries.md` +- `./references/guide-schedule.md` +- `./references/guide-schema.md` +- `./references/guide-sql.md` +- `./references/guide-testing.md` +- `./references/setup.md` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/features.md b/tools/agent-os/.agents/skills/effect-ts/references/features.md new file mode 100644 index 000000000..a605d3eab --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/features.md @@ -0,0 +1,504 @@ +# Features + +Public package and module surface area to be aware of when researching or implementing a solution. Each module entry includes its vendored repo path so it can be located quickly. + +## `effect` Package + +Package path: `packages/effect` + +- `Array` - `packages/effect/src/Array.ts` - Immutable array utilities. Use: transform arrays immutably. +- `BigDecimal` - `packages/effect/src/BigDecimal.ts` - Arbitrary-precision decimal arithmetic. Use: avoid floating-point errors. +- `BigInt` - `packages/effect/src/BigInt.ts` - `bigint` utilities and instances. Use: work with large integers. +- `Boolean` - `packages/effect/src/Boolean.ts` - Boolean utilities and instances. Use: compose boolean logic. +- `Brand` - `packages/effect/src/Brand.ts` - Branded type helpers. Use: prevent accidental type mixing. +- `Cache` - `packages/effect/src/Cache.ts` - Effect cache utilities. Use: memoize effectful lookups. +- `Cause` - `packages/effect/src/Cause.ts` - Structured effect failure representation. Use: inspect failures and defects. +- `Channel` - `packages/effect/src/Channel.ts` - Bidirectional streaming I/O abstraction. Use: build stream pipelines. +- `ChannelSchema` - `packages/effect/src/ChannelSchema.ts` - Schema helpers for channels. Use: type channel protocols. +- `Chunk` - `packages/effect/src/Chunk.ts` - Immutable high-performance sequence. Use: efficient functional sequences. +- `Clock` - `packages/effect/src/Clock.ts` - Time service and sleep utilities. Use: schedule and measure time. +- `Combiner` - `packages/effect/src/Combiner.ts` - Combine two values of one type. Use: merge values consistently. +- `Config` - `packages/effect/src/Config.ts` - Schema-driven configuration loading. Use: read typed app config. +- `ConfigProvider` - `packages/effect/src/ConfigProvider.ts` - Configuration data source abstraction. Use: load config from env/files. +- `Console` - `packages/effect/src/Console.ts` - Effectful console operations. Use: log and debug safely. +- `Context` - `packages/effect/src/Context.ts` - Dependency injection context table. Use: provide and access services. +- `Cron` - `packages/effect/src/Cron.ts` - Cron scheduling utilities. Use: express recurring schedules. +- `Data` - `packages/effect/src/Data.ts` - Immutable data and tagged unions. Use: model domain data and errors. +- `DateTime` - `packages/effect/src/DateTime.ts` - Date-time utilities. Use: handle timestamps and calendars. +- `Deferred` - `packages/effect/src/Deferred.ts` - Single-assignment async variable. Use: coordinate concurrent fibers. +- `Differ` - `packages/effect/src/Differ.ts` - Value diffing utilities. Use: compute incremental updates. +- `Duration` - `packages/effect/src/Duration.ts` - Precise time span type. Use: represent delays and intervals. +- `Effect` - `packages/effect/src/Effect.ts` - Core effect type and operators. Use: model async and concurrent work. +- `Effectable` - `packages/effect/src/Effectable.ts` - Protocols for effect-like values. Use: integrate custom yieldables. +- `Encoding` - `packages/effect/src/Encoding.ts` - Base64, Base64Url, and Hex codecs. Use: encode binary/text values. +- `Equal` - `packages/effect/src/Equal.ts` - Structural and custom equality. Use: compare immutable values deeply. +- `Equivalence` - `packages/effect/src/Equivalence.ts` - Equivalence relation helpers. Use: dedupe or compare with custom rules. +- `ErrorReporter` - `packages/effect/src/ErrorReporter.ts` - Pluggable error reporting. Use: forward failures to observability tools. +- `ExecutionPlan` - `packages/effect/src/ExecutionPlan.ts` - Execution plan utilities. Use: inspect planned work. +- `Exit` - `packages/effect/src/Exit.ts` - Synchronous effect outcome value. Use: inspect success or failure. +- `Fiber` - `packages/effect/src/Fiber.ts` - Lightweight concurrency primitive. Use: fork and join work. +- `FiberHandle` - `packages/effect/src/FiberHandle.ts` - Managed fiber handle helpers. Use: control child fibers. +- `FiberMap` - `packages/effect/src/FiberMap.ts` - Fiber map utilities. Use: track fibers by key. +- `FiberSet` - `packages/effect/src/FiberSet.ts` - Fiber set utilities. Use: manage groups of fibers. +- `FileSystem` - `packages/effect/src/FileSystem.ts` - Effectful file system abstraction. Use: read and write files safely. +- `Filter` - `packages/effect/src/Filter.ts` - Filter result utilities. Use: compose filtering operations. +- `Formatter` - `packages/effect/src/Formatter.ts` - Human-readable value formatting. Use: pretty-print data for logs. +- `Function` - `packages/effect/src/Function.ts` - Core function helpers. Use: pipe and compose functions. +- `Graph` - `packages/effect/src/Graph.ts` - Graph utilities. Use: model dependency graphs. +- `Hash` - `packages/effect/src/Hash.ts` - Hashing utilities. Use: hash values for collections. +- `HashMap` - `packages/effect/src/HashMap.ts` - Immutable hash map. Use: keyed immutable storage. +- `HashRing` - `packages/effect/src/HashRing.ts` - Consistent hashing utilities. Use: distribute keys across nodes. +- `HashSet` - `packages/effect/src/HashSet.ts` - Immutable hash set. Use: store unique immutable values. +- `HKT` - `packages/effect/src/HKT.ts` - Higher-kinded type utilities. Use: define generic type classes. +- `Inspectable` - `packages/effect/src/Inspectable.ts` - Custom inspection helpers. Use: improve debugging output. +- `Iterable` - `packages/effect/src/Iterable.ts` - Functional iterable utilities. Use: process lazy iterables. +- `JsonPatch` - `packages/effect/src/JsonPatch.ts` - JSON Patch operations. Use: diff and patch JSON documents. +- `JsonPointer` - `packages/effect/src/JsonPointer.ts` - JSON Pointer token helpers. Use: escape pointer path segments. +- `JsonSchema` - `packages/effect/src/JsonSchema.ts` - JSON Schema dialect conversion. Use: convert schemas between formats. +- `Latch` - `packages/effect/src/Latch.ts` - Latch synchronization primitive. Use: gate concurrent progress. +- `Layer` - `packages/effect/src/Layer.ts` - Service construction recipes. Use: build and wire dependencies. +- `LayerMap` - `packages/effect/src/LayerMap.ts` - Layer registry helpers. Use: share and look up layers. +- `Logger` - `packages/effect/src/Logger.ts` - Structured logging system. Use: emit runtime logs. +- `LogLevel` - `packages/effect/src/LogLevel.ts` - Log level utilities. Use: filter logs by severity. +- `ManagedRuntime` - `packages/effect/src/ManagedRuntime.ts` - Managed runtime helpers. Use: own runtime lifecycle. +- `Match` - `packages/effect/src/Match.ts` - Type-safe pattern matching. Use: replace switch-heavy branching. +- `Metric` - `packages/effect/src/Metric.ts` - Application metrics system. Use: count, gauge, and histogram events. +- `MutableHashMap` - `packages/effect/src/MutableHashMap.ts` - Mutable hash map. Use: high-performance mutable key-value storage. +- `MutableHashSet` - `packages/effect/src/MutableHashSet.ts` - Mutable hash set. Use: high-performance mutable uniqueness checks. +- `MutableList` - `packages/effect/src/MutableList.ts` - Mutable linked-list-like buffer. Use: efficient append/prepend workloads. +- `MutableRef` - `packages/effect/src/MutableRef.ts` - Mutable reference container. Use: hold local mutable state. +- `Newtype` - `packages/effect/src/Newtype.ts` - Zero-cost wrapper types. Use: distinguish same-shaped values. +- `NonEmptyIterable` - `packages/effect/src/NonEmptyIterable.ts` - Non-empty iterable helpers. Use: require at least one element. +- `Number` - `packages/effect/src/Number.ts` - Number utilities and instances. Use: do typed numeric operations. +- `Optic` - `packages/effect/src/Optic.ts` - Immutable data accessors and updaters. Use: read or update nested state. +- `Option` - `packages/effect/src/Option.ts` - Optional value type. Use: model absence safely. +- `Order` - `packages/effect/src/Order.ts` - Total ordering type class. Use: sort and compare values. +- `Ordering` - `packages/effect/src/Ordering.ts` - Comparison result helpers. Use: combine sort outcomes. +- `PartitionedSemaphore` - `packages/effect/src/PartitionedSemaphore.ts` - Partition-aware semaphore. Use: limit concurrency by key. +- `Path` - `packages/effect/src/Path.ts` - Path utilities. Use: work with filesystem-style paths. +- `Pipeable` - `packages/effect/src/Pipeable.ts` - Pipeable protocol helpers. Use: support fluent pipelines. +- `PlatformError` - `packages/effect/src/PlatformError.ts` - Platform error types. Use: normalize platform-specific failures. +- `Pool` - `packages/effect/src/Pool.ts` - Resource pool utilities. Use: reuse scarce resources. +- `Predicate` - `packages/effect/src/Predicate.ts` - Predicate and refinement helpers. Use: filter and narrow values. +- `PrimaryKey` - `packages/effect/src/PrimaryKey.ts` - Primary key helpers. Use: define stable string identifiers. +- `PubSub` - `packages/effect/src/PubSub.ts` - Publish-subscribe hub. Use: broadcast messages to subscribers. +- `Pull` - `packages/effect/src/Pull.ts` - Pull protocol helpers. Use: model pull-based streaming. +- `Queue` - `packages/effect/src/Queue.ts` - Effect queue utilities. Use: buffer producer-consumer work. +- `Random` - `packages/effect/src/Random.ts` - Randomness service. Use: generate testable random values. +- `RcMap` - `packages/effect/src/RcMap.ts` - Reference-counted map. Use: share keyed resources. +- `RcRef` - `packages/effect/src/RcRef.ts` - Reference-counted ref. Use: share scoped resources. +- `Record` - `packages/effect/src/Record.ts` - Record utilities. Use: transform string-keyed objects. +- `Redactable` - `packages/effect/src/Redactable.ts` - Context-aware redaction protocol. Use: mask sensitive values. +- `Redacted` - `packages/effect/src/Redacted.ts` - Sensitive value wrapper. Use: keep secrets out of logs. +- `Reducer` - `packages/effect/src/Reducer.ts` - Fold values into one result. Use: aggregate collections. +- `Ref` - `packages/effect/src/Ref.ts` - Atomic mutable reference. Use: manage concurrent state. +- `References` - `packages/effect/src/References.ts` - Runtime reference services. Use: tune runtime configuration. +- `RegExp` - `packages/effect/src/RegExp.ts` - RegExp utilities. Use: work with regular expressions. +- `Request` - `packages/effect/src/Request.ts` - External request description. Use: batch and cache data fetching. +- `RequestResolver` - `packages/effect/src/RequestResolver.ts` - Request execution abstraction. Use: resolve batched requests. +- `Resource` - `packages/effect/src/Resource.ts` - Resource utilities. Use: acquire and release shared resources. +- `Result` - `packages/effect/src/Result.ts` - Synchronous success/failure type. Use: validate without effects. +- `Runtime` - `packages/effect/src/Runtime.ts` - Effect runtime helpers. Use: run main programs. +- `Schedule` - `packages/effect/src/Schedule.ts` - Retry and repetition schedules. Use: control retry timing. +- `Scheduler` - `packages/effect/src/Scheduler.ts` - Scheduler utilities. Use: control task execution. +- `Schema` - `packages/effect/src/Schema.ts` - Data schema, validation, and codecs. Use: decode and encode typed data. +- `SchemaAST` - `packages/effect/src/SchemaAST.ts` - Runtime schema AST. Use: inspect or transform schemas. +- `SchemaGetter` - `packages/effect/src/SchemaGetter.ts` - One-way schema transformations. Use: decode individual fields. +- `SchemaIssue` - `packages/effect/src/SchemaIssue.ts` - Structured schema validation errors. Use: inspect parse failures. +- `SchemaParser` - `packages/effect/src/SchemaParser.ts` - Schema parsing helpers. Use: build parsing workflows. +- `SchemaRepresentation` - `packages/effect/src/SchemaRepresentation.ts` - Serializable schema IR. Use: round-trip schemas through JSON/codegen. +- `SchemaTransformation` - `packages/effect/src/SchemaTransformation.ts` - Bidirectional schema transformations. Use: map encoded and decoded forms. +- `SchemaUtils` - `packages/effect/src/SchemaUtils.ts` - Schema utility helpers. Use: support schema internals. +- `Scope` - `packages/effect/src/Scope.ts` - Resource lifetime scope. Use: ensure cleanup of acquired resources. +- `ScopedCache` - `packages/effect/src/ScopedCache.ts` - Scoped cache utilities. Use: cache scoped resources. +- `ScopedRef` - `packages/effect/src/ScopedRef.ts` - Scoped mutable reference. Use: swap scoped resources safely. +- `Semaphore` - `packages/effect/src/Semaphore.ts` - Concurrency semaphore. Use: limit parallel access. +- `Sink` - `packages/effect/src/Sink.ts` - Stream consumer abstraction. Use: fold or write stream inputs. +- `Stdio` - `packages/effect/src/Stdio.ts` - Standard I/O utilities. Use: access stdin/stdout/stderr. +- `Stream` - `packages/effect/src/Stream.ts` - Functional stream abstraction. Use: process streaming data. +- `String` - `packages/effect/src/String.ts` - String utilities and instances. Use: manipulate text functionally. +- `Struct` - `packages/effect/src/Struct.ts` - Immutable object utilities. Use: transform plain objects. +- `SubscriptionRef` - `packages/effect/src/SubscriptionRef.ts` - Subscribable reference. Use: observe state changes. +- `Symbol` - `packages/effect/src/Symbol.ts` - Symbol utilities. Use: work with JavaScript symbols. +- `SynchronizedRef` - `packages/effect/src/SynchronizedRef.ts` - Effect-synchronized ref. Use: update state with effects. +- `Take` - `packages/effect/src/Take.ts` - Stream take representation. Use: carry chunk, end, or error. +- `Terminal` - `packages/effect/src/Terminal.ts` - Terminal interaction utilities. Use: build CLI prompts/output. +- `Tracer` - `packages/effect/src/Tracer.ts` - Tracing abstractions. Use: create spans and traces. +- `Trie` - `packages/effect/src/Trie.ts` - Prefix tree for strings. Use: do prefix lookups. +- `Tuple` - `packages/effect/src/Tuple.ts` - Immutable tuple utilities. Use: transform fixed-length arrays. +- `TxChunk` - `packages/effect/src/TxChunk.ts` - Transactional chunk. Use: mutate chunk state in STM. +- `TxDeferred` - `packages/effect/src/TxDeferred.ts` - Transactional deferred cell. Use: coordinate STM completion. +- `TxHashMap` - `packages/effect/src/TxHashMap.ts` - Transactional hash map. Use: keyed STM state. +- `TxHashSet` - `packages/effect/src/TxHashSet.ts` - Transactional hash set. Use: unique STM state. +- `TxPriorityQueue` - `packages/effect/src/TxPriorityQueue.ts` - Transactional priority queue. Use: ordered STM queueing. +- `TxPubSub` - `packages/effect/src/TxPubSub.ts` - Transactional pub-sub hub. Use: broadcast within STM. +- `TxQueue` - `packages/effect/src/TxQueue.ts` - Transactional queue. Use: queue data within STM. +- `TxReentrantLock` - `packages/effect/src/TxReentrantLock.ts` - Transactional reentrant RW lock. Use: coordinate STM access. +- `TxRef` - `packages/effect/src/TxRef.ts` - Transactional reference. Use: read and write STM state. +- `TxSemaphore` - `packages/effect/src/TxSemaphore.ts` - Transactional semaphore. Use: limit STM concurrency. +- `TxSubscriptionRef` - `packages/effect/src/TxSubscriptionRef.ts` - Transactional subscribable ref. Use: observe committed STM changes. +- `Types` - `packages/effect/src/Types.ts` - Type-level utility aliases. Use: shape complex TypeScript types. +- `UndefinedOr` - `packages/effect/src/UndefinedOr.ts` - Helpers for `A | undefined`. Use: model lightweight optional values. +- `Unify` - `packages/effect/src/Unify.ts` - Type unification helpers. Use: simplify inferred types. +- `Utils` - `packages/effect/src/Utils.ts` - Generator and HKT internals. Use: support yieldable APIs. + +### `effect/testing` + +- `FastCheck` - `packages/effect/src/testing/FastCheck.ts` - Re-export of fast-check for property testing. Use: generate random test cases. +- `TestClock` - `packages/effect/src/testing/TestClock.ts` - Controllable clock for tests. Use: advance time deterministically. +- `TestConsole` - `packages/effect/src/testing/TestConsole.ts` - Test console implementation. Use: assert console output. +- `TestSchema` - `packages/effect/src/testing/TestSchema.ts` - Schema assertion helpers. Use: verify decode/encode behavior. + +### `effect/unstable/ai` + +- `AiError` - `packages/effect/src/unstable/ai/AiError.ts` - AI-related error types. Use: handle model failures. +- `AnthropicStructuredOutput` - `packages/effect/src/unstable/ai/AnthropicStructuredOutput.ts` - Anthropic structured-output codec helpers. Use: parse structured model responses. +- `Chat` - `packages/effect/src/unstable/ai/Chat.ts` - Stateful AI conversation API. Use: manage chat sessions. +- `EmbeddingModel` - `packages/effect/src/unstable/ai/EmbeddingModel.ts` - Provider-agnostic embedding API. Use: generate text vectors. +- `IdGenerator` - `packages/effect/src/unstable/ai/IdGenerator.ts` - Pluggable ID generation. Use: issue stable request IDs. +- `LanguageModel` - `packages/effect/src/unstable/ai/LanguageModel.ts` - AI text generation with tools. Use: call LLMs. +- `McpSchema` - `packages/effect/src/unstable/ai/McpSchema.ts` - MCP schema helpers. Use: type MCP messages. +- `McpServer` - `packages/effect/src/unstable/ai/McpServer.ts` - MCP server support. Use: expose model context tools. +- `Model` - `packages/effect/src/unstable/ai/Model.ts` - Unified AI provider interface. Use: swap AI backends. +- `OpenAiStructuredOutput` - `packages/effect/src/unstable/ai/OpenAiStructuredOutput.ts` - OpenAI structured-output codecs. Use: decode typed model output. +- `Prompt` - `packages/effect/src/unstable/ai/Prompt.ts` - Prompt-building data structures. Use: compose prompts. +- `Response` - `packages/effect/src/unstable/ai/Response.ts` - AI response data structures. Use: inspect completions. +- `ResponseIdTracker` - `packages/effect/src/unstable/ai/ResponseIdTracker.ts` - Response ID tracking utilities. Use: correlate model replies. +- `Telemetry` - `packages/effect/src/unstable/ai/Telemetry.ts` - OpenTelemetry integration for AI operations. Use: trace model calls. +- `Tokenizer` - `packages/effect/src/unstable/ai/Tokenizer.ts` - Tokenization and truncation utilities. Use: enforce token budgets. +- `Tool` - `packages/effect/src/unstable/ai/Tool.ts` - Tool definition and management. Use: expose callable tools. +- `Toolkit` - `packages/effect/src/unstable/ai/Toolkit.ts` - Collection of tools. Use: bundle tool implementations. + +### `effect/unstable/cli` + +- `Argument` - `packages/effect/src/unstable/cli/Argument.ts` - CLI argument definitions. Use: positional arguments. +- `CliError` - `packages/effect/src/unstable/cli/CliError.ts` - CLI error types. Use: report parse failures. +- `CliOutput` - `packages/effect/src/unstable/cli/CliOutput.ts` - CLI rendering/output helpers. Use: format command output. +- `Command` - `packages/effect/src/unstable/cli/Command.ts` - CLI command definitions. Use: build subcommands. +- `Completions` - `packages/effect/src/unstable/cli/Completions.ts` - Shell completion generation. Use: generate completion scripts. +- `Flag` - `packages/effect/src/unstable/cli/Flag.ts` - CLI flag definitions. Use: parse options. +- `GlobalFlag` - `packages/effect/src/unstable/cli/GlobalFlag.ts` - Global CLI flags. Use: shared top-level options. +- `HelpDoc` - `packages/effect/src/unstable/cli/HelpDoc.ts` - CLI help document model. Use: render usage text. +- `Param` - `packages/effect/src/unstable/cli/Param.ts` - CLI parameter helpers. Use: define typed params. +- `Primitive` - `packages/effect/src/unstable/cli/Primitive.ts` - Primitive CLI parsers. Use: parse numbers or strings. +- `Prompt` - `packages/effect/src/unstable/cli/Prompt.ts` - Interactive CLI prompts. Use: ask users for input. + +### `effect/unstable/cluster` + +- `ClusterCron` - `packages/effect/src/unstable/cluster/ClusterCron.ts` - Cluster cron scheduling. Use: distributed recurring jobs. +- `ClusterError` - `packages/effect/src/unstable/cluster/ClusterError.ts` - Cluster error types. Use: classify cluster failures. +- `ClusterMetrics` - `packages/effect/src/unstable/cluster/ClusterMetrics.ts` - Cluster metrics helpers. Use: observe cluster health. +- `ClusterSchema` - `packages/effect/src/unstable/cluster/ClusterSchema.ts` - Cluster schema types. Use: encode cluster messages. +- `ClusterWorkflowEngine` - `packages/effect/src/unstable/cluster/ClusterWorkflowEngine.ts` - Workflow engine for clusters. Use: run distributed workflows. +- `DeliverAt` - `packages/effect/src/unstable/cluster/DeliverAt.ts` - Deferred delivery scheduling. Use: schedule message delivery. +- `Entity` - `packages/effect/src/unstable/cluster/Entity.ts` - Cluster entity abstraction. Use: model sharded actors. +- `EntityAddress` - `packages/effect/src/unstable/cluster/EntityAddress.ts` - Entity addressing types. Use: route entity messages. +- `EntityId` - `packages/effect/src/unstable/cluster/EntityId.ts` - Typed entity identifiers. Use: identify actor instances. +- `EntityProxy` - `packages/effect/src/unstable/cluster/EntityProxy.ts` - Client proxy for entities. Use: call remote entities. +- `EntityProxyServer` - `packages/effect/src/unstable/cluster/EntityProxyServer.ts` - Server for entity proxies. Use: serve entity calls. +- `EntityResource` - `packages/effect/src/unstable/cluster/EntityResource.ts` - Entity-backed resource helpers. Use: manage entity state. +- `EntityType` - `packages/effect/src/unstable/cluster/EntityType.ts` - Entity type descriptors. Use: register entity kinds. +- `Envelope` - `packages/effect/src/unstable/cluster/Envelope.ts` - Message envelope types. Use: wrap cluster messages. +- `HttpRunner` - `packages/effect/src/unstable/cluster/HttpRunner.ts` - HTTP-based runner. Use: transport cluster traffic over HTTP. +- `K8sHttpClient` - `packages/effect/src/unstable/cluster/K8sHttpClient.ts` - Kubernetes HTTP client helpers. Use: discover cluster peers. +- `MachineId` - `packages/effect/src/unstable/cluster/MachineId.ts` - Machine identifier utilities. Use: label cluster nodes. +- `Message` - `packages/effect/src/unstable/cluster/Message.ts` - Cluster message model. Use: send typed payloads. +- `MessageStorage` - `packages/effect/src/unstable/cluster/MessageStorage.ts` - Message persistence abstraction. Use: durable message storage. +- `Reply` - `packages/effect/src/unstable/cluster/Reply.ts` - Reply message helpers. Use: respond to cluster requests. +- `Runner` - `packages/effect/src/unstable/cluster/Runner.ts` - Cluster runner abstraction. Use: host cluster workloads. +- `RunnerAddress` - `packages/effect/src/unstable/cluster/RunnerAddress.ts` - Runner addressing types. Use: locate a runner. +- `RunnerHealth` - `packages/effect/src/unstable/cluster/RunnerHealth.ts` - Runner health reporting. Use: track node readiness. +- `Runners` - `packages/effect/src/unstable/cluster/Runners.ts` - Runner collection utilities. Use: manage active runners. +- `RunnerServer` - `packages/effect/src/unstable/cluster/RunnerServer.ts` - Runner server implementation. Use: expose runner endpoints. +- `RunnerStorage` - `packages/effect/src/unstable/cluster/RunnerStorage.ts` - Runner persistence abstraction. Use: store runner metadata. +- `ShardId` - `packages/effect/src/unstable/cluster/ShardId.ts` - Shard identifier type. Use: map keys to shards. +- `Sharding` - `packages/effect/src/unstable/cluster/Sharding.ts` - Sharding utilities. Use: distribute entities. +- `ShardingConfig` - `packages/effect/src/unstable/cluster/ShardingConfig.ts` - Sharding configuration. Use: tune partitioning behavior. +- `ShardingRegistrationEvent` - `packages/effect/src/unstable/cluster/ShardingRegistrationEvent.ts` - Sharding registration events. Use: observe topology changes. +- `SingleRunner` - `packages/effect/src/unstable/cluster/SingleRunner.ts` - Single-process runner. Use: local cluster execution. +- `Singleton` - `packages/effect/src/unstable/cluster/Singleton.ts` - Cluster singleton abstraction. Use: one active service instance. +- `SingletonAddress` - `packages/effect/src/unstable/cluster/SingletonAddress.ts` - Singleton addressing types. Use: route singleton calls. +- `Snowflake` - `packages/effect/src/unstable/cluster/Snowflake.ts` - Snowflake-style ID generation. Use: unique distributed IDs. +- `SocketRunner` - `packages/effect/src/unstable/cluster/SocketRunner.ts` - Socket-based runner. Use: cluster transport over sockets. +- `SqlMessageStorage` - `packages/effect/src/unstable/cluster/SqlMessageStorage.ts` - SQL-backed message storage. Use: persist cluster messages. +- `SqlRunnerStorage` - `packages/effect/src/unstable/cluster/SqlRunnerStorage.ts` - SQL-backed runner storage. Use: persist runner state. +- `TestRunner` - `packages/effect/src/unstable/cluster/TestRunner.ts` - Test runner utilities. Use: simulate cluster execution. + +### `effect/unstable/devtools` + +- `DevTools` - `packages/effect/src/unstable/devtools/DevTools.ts` - Devtools integration. Use: inspect runtime behavior. +- `DevToolsClient` - `packages/effect/src/unstable/devtools/DevToolsClient.ts` - Devtools client API. Use: connect to devtools server. +- `DevToolsSchema` - `packages/effect/src/unstable/devtools/DevToolsSchema.ts` - Devtools message schemas. Use: type devtools payloads. +- `DevToolsServer` - `packages/effect/src/unstable/devtools/DevToolsServer.ts` - Devtools server API. Use: expose inspection endpoints. + +### `effect/unstable/encoding` + +- `Msgpack` - `packages/effect/src/unstable/encoding/Msgpack.ts` - MessagePack encoding helpers. Use: compact binary serialization. +- `Ndjson` - `packages/effect/src/unstable/encoding/Ndjson.ts` - NDJSON encoding helpers. Use: stream JSON lines. +- `Sse` - `packages/effect/src/unstable/encoding/Sse.ts` - Server-sent event encoding helpers. Use: emit SSE streams. + +### `effect/unstable/eventlog` + +- `Event` - `packages/effect/src/unstable/eventlog/Event.ts` - Event model types. Use: represent domain events. +- `EventGroup` - `packages/effect/src/unstable/eventlog/EventGroup.ts` - Event grouping helpers. Use: batch related events. +- `EventJournal` - `packages/effect/src/unstable/eventlog/EventJournal.ts` - Event journal abstraction. Use: append and read events. +- `EventLog` - `packages/effect/src/unstable/eventlog/EventLog.ts` - Event log API. Use: stream recorded events. +- `EventLogEncryption` - `packages/effect/src/unstable/eventlog/EventLogEncryption.ts` - Event log encryption helpers. Use: protect event payloads. +- `EventLogMessage` - `packages/effect/src/unstable/eventlog/EventLogMessage.ts` - Event log message types. Use: transport log entries. +- `EventLogRemote` - `packages/effect/src/unstable/eventlog/EventLogRemote.ts` - Remote event log client/server helpers. Use: access remote logs. +- `EventLogServer` - `packages/effect/src/unstable/eventlog/EventLogServer.ts` - Event log server support. Use: host event streams. +- `EventLogServerEncrypted` - `packages/effect/src/unstable/eventlog/EventLogServerEncrypted.ts` - Encrypted event log server. Use: secure event serving. +- `EventLogServerUnencrypted` - `packages/effect/src/unstable/eventlog/EventLogServerUnencrypted.ts` - Unencrypted event log server. Use: plain local serving. +- `EventLogSessionAuth` - `packages/effect/src/unstable/eventlog/EventLogSessionAuth.ts` - Event log session auth helpers. Use: authorize event sessions. +- `SqlEventJournal` - `packages/effect/src/unstable/eventlog/SqlEventJournal.ts` - SQL-backed event journal. Use: persist events in SQL. +- `SqlEventLogServerEncrypted` - `packages/effect/src/unstable/eventlog/SqlEventLogServerEncrypted.ts` - SQL-backed encrypted event server. Use: secure persisted logs. +- `SqlEventLogServerUnencrypted` - `packages/effect/src/unstable/eventlog/SqlEventLogServerUnencrypted.ts` - SQL-backed plain event server. Use: simple persisted logs. + +### `effect/unstable/http` + +- `Cookies` - `packages/effect/src/unstable/http/Cookies.ts` - HTTP cookie helpers. Use: parse and set cookies. +- `Etag` - `packages/effect/src/unstable/http/Etag.ts` - ETag utilities. Use: cache validation headers. +- `FetchHttpClient` - `packages/effect/src/unstable/http/FetchHttpClient.ts` - Fetch-based HTTP client. Use: run requests in browsers. +- `FindMyWay` - `packages/effect/src/unstable/http/FindMyWay.ts` - `find-my-way` router integration. Use: path routing. +- `Headers` - `packages/effect/src/unstable/http/Headers.ts` - HTTP header utilities. Use: manage request headers. +- `HttpBody` - `packages/effect/src/unstable/http/HttpBody.ts` - HTTP body representations. Use: send JSON or bytes. +- `HttpClient` - `packages/effect/src/unstable/http/HttpClient.ts` - Effect HTTP client API. Use: perform outbound requests. +- `HttpClientError` - `packages/effect/src/unstable/http/HttpClientError.ts` - HTTP client error types. Use: handle request failures. +- `HttpClientRequest` - `packages/effect/src/unstable/http/HttpClientRequest.ts` - HTTP client request builders. Use: construct requests. +- `HttpClientResponse` - `packages/effect/src/unstable/http/HttpClientResponse.ts` - HTTP client response utilities. Use: read status and body. +- `HttpEffect` - `packages/effect/src/unstable/http/HttpEffect.ts` - HTTP-specific effect helpers. Use: compose HTTP workflows. +- `HttpIncomingMessage` - `packages/effect/src/unstable/http/HttpIncomingMessage.ts` - Incoming message abstraction. Use: read request bodies. +- `HttpMethod` - `packages/effect/src/unstable/http/HttpMethod.ts` - HTTP method helpers. Use: branch on verbs. +- `HttpMiddleware` - `packages/effect/src/unstable/http/HttpMiddleware.ts` - HTTP middleware utilities. Use: add auth or logging. +- `HttpPlatform` - `packages/effect/src/unstable/http/HttpPlatform.ts` - Platform HTTP integration. Use: supply runtime adapters. +- `HttpRouter` - `packages/effect/src/unstable/http/HttpRouter.ts` - HTTP routing abstraction. Use: define endpoints. +- `HttpServer` - `packages/effect/src/unstable/http/HttpServer.ts` - HTTP server API. Use: serve requests. +- `HttpServerError` - `packages/effect/src/unstable/http/HttpServerError.ts` - HTTP server error types. Use: render server failures. +- `HttpServerRequest` - `packages/effect/src/unstable/http/HttpServerRequest.ts` - Server request utilities. Use: inspect inbound requests. +- `HttpServerRespondable` - `packages/effect/src/unstable/http/HttpServerRespondable.ts` - Response conversion protocol. Use: return custom response types. +- `HttpServerResponse` - `packages/effect/src/unstable/http/HttpServerResponse.ts` - Server response builders. Use: send JSON or text. +- `HttpStaticServer` - `packages/effect/src/unstable/http/HttpStaticServer.ts` - Static file serving helpers. Use: serve assets. +- `HttpTraceContext` - `packages/effect/src/unstable/http/HttpTraceContext.ts` - HTTP trace-context propagation. Use: carry tracing headers. +- `Multipart` - `packages/effect/src/unstable/http/Multipart.ts` - Multipart form-data helpers. Use: file uploads. +- `Multipasta` - `packages/effect/src/unstable/http/Multipasta.ts` - Multipasta integration. Use: parse multipart streams. +- `Template` - `packages/effect/src/unstable/http/Template.ts` - HTTP templating helpers. Use: generate responses from templates. +- `Url` - `packages/effect/src/unstable/http/Url.ts` - URL utilities. Use: parse and build URLs. +- `UrlParams` - `packages/effect/src/unstable/http/UrlParams.ts` - URL parameter helpers. Use: encode query strings. + +### `effect/unstable/httpapi` + +- `HttpApi` - `packages/effect/src/unstable/httpapi/HttpApi.ts` - Typed HTTP API description. Use: define an API contract. +- `HttpApiBuilder` - `packages/effect/src/unstable/httpapi/HttpApiBuilder.ts` - Build servers from `HttpApi`. Use: implement typed endpoints. +- `HttpApiClient` - `packages/effect/src/unstable/httpapi/HttpApiClient.ts` - Client generation for `HttpApi`. Use: call typed APIs. +- `HttpApiEndpoint` - `packages/effect/src/unstable/httpapi/HttpApiEndpoint.ts` - Endpoint descriptors. Use: define route inputs/outputs. +- `HttpApiError` - `packages/effect/src/unstable/httpapi/HttpApiError.ts` - HTTP API error types. Use: model typed API failures. +- `HttpApiGroup` - `packages/effect/src/unstable/httpapi/HttpApiGroup.ts` - Group API endpoints. Use: organize related routes. +- `HttpApiMiddleware` - `packages/effect/src/unstable/httpapi/HttpApiMiddleware.ts` - API middleware helpers. Use: add auth policies. +- `HttpApiScalar` - `packages/effect/src/unstable/httpapi/HttpApiScalar.ts` - Scalar UI/integration helpers. Use: expose API docs tooling. +- `HttpApiSchema` - `packages/effect/src/unstable/httpapi/HttpApiSchema.ts` - Schema annotations for HTTP metadata. Use: tag schema fields for APIs. +- `HttpApiSecurity` - `packages/effect/src/unstable/httpapi/HttpApiSecurity.ts` - Security scheme helpers. Use: define auth requirements. +- `HttpApiSwagger` - `packages/effect/src/unstable/httpapi/HttpApiSwagger.ts` - Swagger/OpenAPI integration. Use: serve interactive docs. +- `HttpApiTest` - `packages/effect/src/unstable/httpapi/HttpApiTest.ts` - HttpApi test helpers. Use: test typed endpoints. +- `OpenApi` - `packages/effect/src/unstable/httpapi/OpenApi.ts` - OpenAPI generation helpers. Use: export API specs. + +### `effect/unstable/observability` + +- `Otlp` - `packages/effect/src/unstable/observability/Otlp.ts` - OTLP integration entrypoint. Use: configure OTLP telemetry. +- `OtlpExporter` - `packages/effect/src/unstable/observability/OtlpExporter.ts` - OTLP exporter utilities. Use: send telemetry externally. +- `OtlpLogger` - `packages/effect/src/unstable/observability/OtlpLogger.ts` - OTLP log export support. Use: ship structured logs. +- `OtlpMetrics` - `packages/effect/src/unstable/observability/OtlpMetrics.ts` - OTLP metrics export support. Use: export counters and histograms. +- `OtlpResource` - `packages/effect/src/unstable/observability/OtlpResource.ts` - OTLP resource metadata helpers. Use: tag service identity. +- `OtlpSerialization` - `packages/effect/src/unstable/observability/OtlpSerialization.ts` - OTLP serialization helpers. Use: encode telemetry payloads. +- `OtlpTracer` - `packages/effect/src/unstable/observability/OtlpTracer.ts` - OTLP tracing export support. Use: export spans. +- `PrometheusMetrics` - `packages/effect/src/unstable/observability/PrometheusMetrics.ts` - Prometheus metrics exporter. Use: expose scrape endpoint. + +### `effect/unstable/persistence` + +- `KeyValueStore` - `packages/effect/src/unstable/persistence/KeyValueStore.ts` - Key-value storage abstraction. Use: persist simple state. +- `Persistable` - `packages/effect/src/unstable/persistence/Persistable.ts` - Persistable type helpers. Use: encode stored values. +- `PersistedCache` - `packages/effect/src/unstable/persistence/PersistedCache.ts` - Durable cache. Use: cache across restarts. +- `PersistedQueue` - `packages/effect/src/unstable/persistence/PersistedQueue.ts` - Durable queue. Use: persist work items. +- `Persistence` - `packages/effect/src/unstable/persistence/Persistence.ts` - Persistence service APIs. Use: back durable components. +- `RateLimiter` - `packages/effect/src/unstable/persistence/RateLimiter.ts` - Persistent rate limiter. Use: enforce cross-process limits. +- `Redis` - `packages/effect/src/unstable/persistence/Redis.ts` - Redis-backed persistence helpers. Use: store state in Redis. + +### `effect/unstable/process` + +- `ChildProcess` - `packages/effect/src/unstable/process/ChildProcess.ts` - Child process abstractions. Use: manage spawned commands. +- `ChildProcessSpawner` - `packages/effect/src/unstable/process/ChildProcessSpawner.ts` - Generic child-process spawning service. Use: launch subprocesses. + +### `effect/unstable/reactivity` + +- `AsyncResult` - `packages/effect/src/unstable/reactivity/AsyncResult.ts` - Reactive async result type. Use: represent loading/error/data. +- `Atom` - `packages/effect/src/unstable/reactivity/Atom.ts` - Reactive atom state primitive. Use: local reactive state. +- `AtomHttpApi` - `packages/effect/src/unstable/reactivity/AtomHttpApi.ts` - HttpApi integration for atoms. Use: sync state over HTTP. +- `AtomRef` - `packages/effect/src/unstable/reactivity/AtomRef.ts` - Ref-backed atom helpers. Use: bridge refs to reactivity. +- `AtomRegistry` - `packages/effect/src/unstable/reactivity/AtomRegistry.ts` - Atom registry utilities. Use: track application atoms. +- `AtomRpc` - `packages/effect/src/unstable/reactivity/AtomRpc.ts` - RPC integration for atoms. Use: sync state over RPC. +- `Hydration` - `packages/effect/src/unstable/reactivity/Hydration.ts` - Hydration helpers. Use: restore reactive state. +- `Reactivity` - `packages/effect/src/unstable/reactivity/Reactivity.ts` - Core reactivity utilities. Use: compose reactive computations. + +### `effect/unstable/rpc` + +- `Rpc` - `packages/effect/src/unstable/rpc/Rpc.ts` - Typed RPC description. Use: define RPC contracts. +- `RpcClient` - `packages/effect/src/unstable/rpc/RpcClient.ts` - RPC client API. Use: call remote procedures. +- `RpcClientError` - `packages/effect/src/unstable/rpc/RpcClientError.ts` - RPC client error types. Use: handle transport failures. +- `RpcGroup` - `packages/effect/src/unstable/rpc/RpcGroup.ts` - Group RPC endpoints. Use: organize procedures. +- `RpcMessage` - `packages/effect/src/unstable/rpc/RpcMessage.ts` - RPC message model. Use: encode request/response payloads. +- `RpcMiddleware` - `packages/effect/src/unstable/rpc/RpcMiddleware.ts` - RPC middleware helpers. Use: add auth or tracing. +- `RpcSchema` - `packages/effect/src/unstable/rpc/RpcSchema.ts` - Schema helpers for RPC. Use: type RPC payloads. +- `RpcSerialization` - `packages/effect/src/unstable/rpc/RpcSerialization.ts` - RPC serialization helpers. Use: encode wire formats. +- `RpcServer` - `packages/effect/src/unstable/rpc/RpcServer.ts` - RPC server API. Use: expose procedures. +- `RpcTest` - `packages/effect/src/unstable/rpc/RpcTest.ts` - RPC testing helpers. Use: test RPC handlers. +- `RpcWorker` - `packages/effect/src/unstable/rpc/RpcWorker.ts` - Worker integration for RPC. Use: run RPC over workers. +- `Utils` - `packages/effect/src/unstable/rpc/Utils.ts` - Shared RPC utilities. Use: support custom RPC plumbing. + +### `effect/unstable/schema` + +- `Model` - `packages/effect/src/unstable/schema/Model.ts` - Unstable schema model helpers. Use: define schema-backed models. +- `VariantSchema` - `packages/effect/src/unstable/schema/VariantSchema.ts` - Variant/discriminated schema helpers. Use: model tagged unions. + +### `effect/unstable/socket` + +- `Socket` - `packages/effect/src/unstable/socket/Socket.ts` - Socket abstractions. Use: connect stream transports. +- `SocketServer` - `packages/effect/src/unstable/socket/SocketServer.ts` - Socket server helpers. Use: accept socket clients. + +### `effect/unstable/sql` + +- `Migrator` - `packages/effect/src/unstable/sql/Migrator.ts` - SQL migration helpers. Use: run schema migrations. +- `SqlClient` - `packages/effect/src/unstable/sql/SqlClient.ts` - SQL client API. Use: execute queries. +- `SqlConnection` - `packages/effect/src/unstable/sql/SqlConnection.ts` - SQL connection abstraction. Use: manage DB sessions. +- `SqlError` - `packages/effect/src/unstable/sql/SqlError.ts` - SQL error types. Use: classify database failures. +- `SqlModel` - `packages/effect/src/unstable/sql/SqlModel.ts` - SQL-backed model helpers. Use: map tables to models. +- `SqlResolver` - `packages/effect/src/unstable/sql/SqlResolver.ts` - SQL request resolution helpers. Use: batch DB-backed requests. +- `SqlSchema` - `packages/effect/src/unstable/sql/SqlSchema.ts` - Schema helpers for SQL. Use: type query values. +- `SqlStream` - `packages/effect/src/unstable/sql/SqlStream.ts` - Streaming SQL query helpers. Use: consume large result sets. +- `Statement` - `packages/effect/src/unstable/sql/Statement.ts` - SQL statement builders/types. Use: prepare typed statements. + +### `effect/unstable/workers` + +- `Transferable` - `packages/effect/src/unstable/workers/Transferable.ts` - Transferable value helpers. Use: send data between workers. +- `Worker` - `packages/effect/src/unstable/workers/Worker.ts` - Worker abstractions. Use: run isolated tasks. +- `WorkerError` - `packages/effect/src/unstable/workers/WorkerError.ts` - Worker error types. Use: handle worker failures. +- `WorkerRunner` - `packages/effect/src/unstable/workers/WorkerRunner.ts` - Worker runner utilities. Use: host jobs in workers. + +### `effect/unstable/workflow` + +- `Activity` - `packages/effect/src/unstable/workflow/Activity.ts` - Workflow activity helpers. Use: define durable steps. +- `DurableClock` - `packages/effect/src/unstable/workflow/DurableClock.ts` - Durable clock API. Use: schedule workflow time. +- `DurableDeferred` - `packages/effect/src/unstable/workflow/DurableDeferred.ts` - Durable deferred primitive. Use: await external completion. +- `DurableQueue` - `packages/effect/src/unstable/workflow/DurableQueue.ts` - Durable queue primitive. Use: persist workflow worklists. +- `Workflow` - `packages/effect/src/unstable/workflow/Workflow.ts` - Workflow definition API. Use: declare durable workflows. +- `WorkflowEngine` - `packages/effect/src/unstable/workflow/WorkflowEngine.ts` - Workflow engine runtime. Use: execute workflows. +- `WorkflowProxy` - `packages/effect/src/unstable/workflow/WorkflowProxy.ts` - Workflow client proxy. Use: call workflow instances. +- `WorkflowProxyServer` - `packages/effect/src/unstable/workflow/WorkflowProxyServer.ts` - Workflow proxy server. Use: expose workflow endpoints. + +## `@effect/opentelemetry` Package + +Package path: `packages/opentelemetry` + +- `Logger` - `packages/opentelemetry/src/Logger.ts` - OpenTelemetry logging integration. Use: export structured logs. +- `Metrics` - `packages/opentelemetry/src/Metrics.ts` - OpenTelemetry metrics integration. Use: publish application metrics. +- `NodeSdk` - `packages/opentelemetry/src/NodeSdk.ts` - Node OpenTelemetry SDK wiring. Use: bootstrap telemetry in Node. +- `Resource` - `packages/opentelemetry/src/Resource.ts` - OpenTelemetry resource helpers. Use: describe service metadata. +- `Tracer` - `packages/opentelemetry/src/Tracer.ts` - OpenTelemetry tracing integration. Use: create and export spans. +- `WebSdk` - `packages/opentelemetry/src/WebSdk.ts` - Web OpenTelemetry SDK wiring. Use: bootstrap telemetry in browsers. + +## `@effect/platform-browser` Package + +Package path: `packages/platform-browser` + +- `BrowserHttpClient` - `packages/platform-browser/src/BrowserHttpClient.ts` - Browser HTTP client implementation. Use: make fetch-based requests. +- `BrowserKeyValueStore` - `packages/platform-browser/src/BrowserKeyValueStore.ts` - Browser key-value storage adapter. Use: persist small client values. +- `BrowserPersistence` - `packages/platform-browser/src/BrowserPersistence.ts` - Browser persistence services. Use: store app state locally. +- `BrowserRuntime` - `packages/platform-browser/src/BrowserRuntime.ts` - Browser runtime entrypoints. Use: run Effect apps in browsers. +- `BrowserSocket` - `packages/platform-browser/src/BrowserSocket.ts` - Browser socket implementation. Use: open WebSocket-style connections. +- `BrowserStream` - `packages/platform-browser/src/BrowserStream.ts` - Browser stream adapters. Use: bridge web streams. +- `BrowserWorker` - `packages/platform-browser/src/BrowserWorker.ts` - Browser worker integration. Use: communicate with web workers. +- `BrowserWorkerRunner` - `packages/platform-browser/src/BrowserWorkerRunner.ts` - Worker-side runtime helpers. Use: run Effect code inside workers. +- `Clipboard` - `packages/platform-browser/src/Clipboard.ts` - Clipboard API wrappers. Use: read or write clipboard data. +- `Geolocation` - `packages/platform-browser/src/Geolocation.ts` - Geolocation API wrappers. Use: access device location. +- `IndexedDb` - `packages/platform-browser/src/IndexedDb.ts` - IndexedDB integration. Use: build browser databases. +- `IndexedDbDatabase` - `packages/platform-browser/src/IndexedDbDatabase.ts` - IndexedDB database helpers. Use: define database handles. +- `IndexedDbQueryBuilder` - `packages/platform-browser/src/IndexedDbQueryBuilder.ts` - IndexedDB query builder. Use: compose indexed queries. +- `IndexedDbTable` - `packages/platform-browser/src/IndexedDbTable.ts` - IndexedDB table helpers. Use: work with object stores. +- `IndexedDbVersion` - `packages/platform-browser/src/IndexedDbVersion.ts` - IndexedDB versioning helpers. Use: manage schema upgrades. +- `Permissions` - `packages/platform-browser/src/Permissions.ts` - Permissions API wrappers. Use: query browser permissions. + +## `@effect/platform-bun` Package + +Package path: `packages/platform-bun` + +- `BunChildProcessSpawner` - `packages/platform-bun/src/BunChildProcessSpawner.ts` - Bun child-process spawner. Use: launch subprocesses. +- `BunClusterHttp` - `packages/platform-bun/src/BunClusterHttp.ts` - Bun clustered HTTP helpers. Use: scale HTTP servers. +- `BunClusterSocket` - `packages/platform-bun/src/BunClusterSocket.ts` - Bun clustered socket helpers. Use: scale socket servers. +- `BunFileSystem` - `packages/platform-bun/src/BunFileSystem.ts` - Bun file system implementation. Use: do Bun-based file I/O. +- `BunHttpClient` - `packages/platform-bun/src/BunHttpClient.ts` - Bun HTTP client implementation. Use: make outbound HTTP requests. +- `BunHttpPlatform` - `packages/platform-bun/src/BunHttpPlatform.ts` - Bun HTTP platform services. Use: provide HTTP runtime pieces. +- `BunHttpServer` - `packages/platform-bun/src/BunHttpServer.ts` - Bun HTTP server implementation. Use: serve HTTP endpoints. +- `BunHttpServerRequest` - `packages/platform-bun/src/BunHttpServerRequest.ts` - Bun request adapters. Use: read incoming HTTP requests. +- `BunMultipart` - `packages/platform-bun/src/BunMultipart.ts` - Bun multipart parsing. Use: handle form uploads. +- `BunPath` - `packages/platform-bun/src/BunPath.ts` - Bun path service. Use: resolve filesystem paths. +- `BunRedis` - `packages/platform-bun/src/BunRedis.ts` - Bun Redis integration. Use: talk to Redis. +- `BunRuntime` - `packages/platform-bun/src/BunRuntime.ts` - Bun runtime entrypoints. Use: run Effect apps on Bun. +- `BunServices` - `packages/platform-bun/src/BunServices.ts` - Bun service bundle. Use: provide common Bun services. +- `BunSink` - `packages/platform-bun/src/BunSink.ts` - Bun sink adapters. Use: write streamed output. +- `BunSocket` - `packages/platform-bun/src/BunSocket.ts` - Bun socket implementation. Use: manage socket connections. +- `BunSocketServer` - `packages/platform-bun/src/BunSocketServer.ts` - Bun socket server implementation. Use: accept socket clients. +- `BunStdio` - `packages/platform-bun/src/BunStdio.ts` - Bun stdio integration. Use: access stdin/stdout/stderr. +- `BunStream` - `packages/platform-bun/src/BunStream.ts` - Bun stream adapters. Use: bridge Bun streams. +- `BunTerminal` - `packages/platform-bun/src/BunTerminal.ts` - Bun terminal integration. Use: build CLI terminal interactions. +- `BunWorker` - `packages/platform-bun/src/BunWorker.ts` - Bun worker integration. Use: communicate with workers. +- `BunWorkerRunner` - `packages/platform-bun/src/BunWorkerRunner.ts` - Bun worker runtime helpers. Use: run Effect code in workers. + +## `@effect/platform-node` Package + +Package path: `packages/platform-node` + +- `Mime` - `packages/platform-node/src/Mime.ts` - MIME type helpers. Use: detect or assign content types. +- `NodeChildProcessSpawner` - `packages/platform-node/src/NodeChildProcessSpawner.ts` - Node child-process spawner. Use: launch subprocesses. +- `NodeClusterHttp` - `packages/platform-node/src/NodeClusterHttp.ts` - Node clustered HTTP helpers. Use: scale HTTP servers. +- `NodeClusterSocket` - `packages/platform-node/src/NodeClusterSocket.ts` - Node clustered socket helpers. Use: scale socket servers. +- `NodeFileSystem` - `packages/platform-node/src/NodeFileSystem.ts` - Node file system implementation. Use: do Node-based file I/O. +- `NodeHttpClient` - `packages/platform-node/src/NodeHttpClient.ts` - Node HTTP client implementation. Use: make outbound HTTP requests. +- `NodeHttpIncomingMessage` - `packages/platform-node/src/NodeHttpIncomingMessage.ts` - Node incoming message adapters. Use: read raw Node HTTP messages. +- `NodeHttpPlatform` - `packages/platform-node/src/NodeHttpPlatform.ts` - Node HTTP platform services. Use: provide HTTP runtime pieces. +- `NodeHttpServer` - `packages/platform-node/src/NodeHttpServer.ts` - Node HTTP server implementation. Use: serve HTTP endpoints. +- `NodeHttpServerRequest` - `packages/platform-node/src/NodeHttpServerRequest.ts` - Node request adapters. Use: read incoming HTTP requests. +- `NodeMultipart` - `packages/platform-node/src/NodeMultipart.ts` - Node multipart parsing. Use: handle form uploads. +- `NodePath` - `packages/platform-node/src/NodePath.ts` - Node path service. Use: resolve filesystem paths. +- `NodeRedis` - `packages/platform-node/src/NodeRedis.ts` - Node Redis integration. Use: talk to Redis. +- `NodeRuntime` - `packages/platform-node/src/NodeRuntime.ts` - Node runtime entrypoints. Use: run Effect apps on Node. +- `NodeServices` - `packages/platform-node/src/NodeServices.ts` - Node service bundle. Use: provide common Node services. +- `NodeSink` - `packages/platform-node/src/NodeSink.ts` - Node sink adapters. Use: write streamed output. +- `NodeSocket` - `packages/platform-node/src/NodeSocket.ts` - Node socket implementation. Use: manage socket connections. +- `NodeSocketServer` - `packages/platform-node/src/NodeSocketServer.ts` - Node socket server implementation. Use: accept socket clients. +- `NodeStdio` - `packages/platform-node/src/NodeStdio.ts` - Node stdio integration. Use: access stdin/stdout/stderr. +- `NodeStream` - `packages/platform-node/src/NodeStream.ts` - Node stream adapters. Use: bridge Node streams. +- `NodeTerminal` - `packages/platform-node/src/NodeTerminal.ts` - Node terminal integration. Use: build CLI terminal interactions. +- `NodeWorker` - `packages/platform-node/src/NodeWorker.ts` - Node worker integration. Use: communicate with worker threads. +- `NodeWorkerRunner` - `packages/platform-node/src/NodeWorkerRunner.ts` - Node worker runtime helpers. Use: run Effect code in workers. +- `Undici` - `packages/platform-node/src/Undici.ts` - Undici integration helpers. Use: use Undici-based HTTP features. + +## `@effect/platform-node-shared` Package + +Package path: `packages/platform-node-shared` + +- No public `src/index.ts` barrel is present in this vendored repo, so there are no barrel exports to inventory here. + +## `@effect/vitest` Package + +Package path: `packages/vitest` + +- `vitest` - `packages/vitest/src/index.ts` - Re-export of `vitest` APIs. Use: use standard Vitest APIs. +- `API` - `packages/vitest/src/index.ts` - Test API type alias. Use: type custom test helpers. +- `Vitest` - `packages/vitest/src/index.ts` - Effect-aware Vitest namespace types. Use: type Effect-based tests. +- `addEqualityTesters` - `packages/vitest/src/index.ts` - Installs equality testers. Use: compare Effect values in assertions. +- `effect` - `packages/vitest/src/index.ts` - Effect-aware `it` variant. Use: write scoped Effect tests. +- `live` - `packages/vitest/src/index.ts` - Live-service test variant. Use: run tests with live services. +- `layer` - `packages/vitest/src/index.ts` - Share a `Layer` across tests. Use: provide services to test groups. +- `flakyTest` - `packages/vitest/src/index.ts` - Flaky-test wrapper. Use: stabilize eventually consistent checks. +- `prop` - `packages/vitest/src/index.ts` - Property-test helper. Use: generate property-based cases. +- `it` - `packages/vitest/src/index.ts` - Extended Vitest `it`. Use: mix regular and Effect tests. +- `makeMethods` - `packages/vitest/src/index.ts` - Build extended test methods. Use: wrap a custom test API. +- `describeWrapped` - `packages/vitest/src/index.ts` - `describe` helper with Effect methods. Use: define grouped Effect test suites. \ No newline at end of file diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-effect.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-effect.md new file mode 100644 index 000000000..086b1260a --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-effect.md @@ -0,0 +1,447 @@ +# Effect Guide + +This guide is based on common usage patterns in the vendored repo at `./.repos/effect`. + +Key source areas: + +- `./.repos/effect/packages/effect/src/Effect.ts` +- `./.repos/effect/packages/tools/` +- `./.repos/effect/packages/platform-*` +- `./.repos/effect/packages/opentelemetry/` +- `./.repos/effect/packages/vitest/` + +## Mental Model + +`Effect` is the default way to represent application work. + +It describes a computation that: + +- succeeds with `A` +- fails with `E` +- requires services `R` + +The repo consistently uses `Effect` as the main abstraction for: + +- business workflows +- service methods +- platform integrations +- resource lifecycles +- tests + +## Most Common Patterns In The Repo + +The dominant usage pattern is: + +1. use `Effect.gen` for workflows and orchestration +2. use `Effect.fn` for reusable effectful functions +3. use precise constructors such as `succeed`, `fail`, `sync`, `try`, and `tryPromise` +4. use `map`, `flatMap`, and `tap` for local transformations +5. access services in implementations and `provide*` only at edges +6. use `acquireRelease` and `scoped` for owned resources +7. use `catchTag` and `match` for typed recovery +8. use `run*` only at runtime boundaries + +## Prefer `Effect.fn` For Reusable Operations + +For reusable effectful operations, prefer `Effect.fn`. + +```ts +import { Effect } from "effect" + +const loadUser = Effect.fn("loadUser")(function*(userId: string) { + return { id: userId, name: "Ada" } +}) +``` + +Use `Effect.fn` when: + +- the operation is reusable +- the operation takes parameters +- the operation is part of business logic or a module API +- you want consistent tracing and stack frames + +Do not treat `Effect.fnUntraced` as the default. If you do not want an explicit named span, use `Effect.fn` without a span name. + +Repo examples: + +- `./.repos/effect/packages/tools/utils/src/Codegen.ts` +- `./.repos/effect/packages/tools/openapi-generator/src/OpenApiPatch.ts` + +## Use `Effect.gen` For Workflows + +Use `Effect.gen` for orchestration and sequential workflows, especially when there are multiple `yield*` steps. + +```ts +const program = Effect.gen(function*() { + const config = yield* Config + const repo = yield* UserRepo + const user = yield* repo.getById("u_123") + return { config, user } +}) +``` + +Use `Effect.gen` when: + +- the body is a workflow +- you are reading multiple services +- you have branching or multiple sequential steps +- you are implementing a layer, handler, or orchestration + +Repo examples: + +- `./.repos/effect/packages/opentelemetry/src/NodeSdk.ts` +- `./.repos/effect/packages/tools/openapi-generator/src/OpenApiGenerator.ts` + +## `Effect.fn` vs `Effect.gen` + +Use this rule: + +- reusable operation: `Effect.fn` +- inline workflow block: `Effect.gen` + +Good split: + +```ts +const loadUser = Effect.fn("loadUser")(function*(userId: string) { + const repo = yield* UserRepo + return yield* repo.getById(userId) +}) + +const program = Effect.gen(function*() { + const user = yield* loadUser("u_123") + yield* Effect.logInfo("loaded user", user) +}) +``` + +## `Effect.fnUntraced` Is An Escape Hatch + +For application and business code, `Effect.fnUntraced` is not the default. + +Use it only when: + +- the function is an internal low-level helper +- observability is intentionally being traded away +- there is a concrete performance or tracing reason + +If the only goal is to avoid an explicit named span, prefer: + +```ts +const normalizeUser = Effect.fn(function*(input: string) { + return input.trim().toLowerCase() +}) +``` + +Instead of: + +```ts +const normalizeUser = Effect.fnUntraced(function*(input: string) { + return input.trim().toLowerCase() +}) +``` + +## Constructor Functions + +The repo uses constructor functions very deliberately. + +### `Effect.succeed` + +Use for pure successful values. + +```ts +const ok = Effect.succeed(42) +``` + +### `Effect.fail` + +Use for expected typed failures. + +```ts +const notFound = Effect.fail(UserNotFound.make({ userId: "u_123" })) +``` + +### `Effect.sync` + +Use for synchronous side effects or pure synchronous construction that should live inside `Effect`. + +```ts +const buildConfig = Effect.sync(() => ({ retries: 3 })) +``` + +### `Effect.try` + +Use for synchronous code that may throw. + +```ts +import { Effect, Schema } from "effect" + +class ParseError extends Schema.TaggedErrorClass()("ParseError", { + cause: Schema.Defect +}) {} + +const parseJson = (input: string) => + Effect.try({ + try: () => JSON.parse(input), + catch: (cause) => ParseError.make({ cause }) + }) +``` + +### `Effect.tryPromise` + +Use for Promise-returning APIs. + +```ts +import { Effect, Schema } from "effect" + +class FetchError extends Schema.TaggedErrorClass()("FetchError", { + cause: Schema.Defect +}) {} + +const fetchText = (url: string) => + Effect.tryPromise({ + try: () => fetch(url).then((response) => response.text()), + catch: (cause) => FetchError.make({ cause }) + }) +``` + +Preferred rule: + +- pure value: `succeed` +- expected failure: `fail` +- synchronous non-throwing effect: `sync` +- synchronous throwing boundary: `try` +- Promise boundary: `tryPromise` + +## Local Composition + +The repo uses `map`, `flatMap`, and `tap` constantly for small local transformations. + +### `Effect.map` + +Use to transform successful values. + +```ts +const userName = loadUser("u_123").pipe( + Effect.map((user) => user.name) +) +``` + +### `Effect.flatMap` + +Use when the next step returns another `Effect`. + +```ts +const result = loadUser("u_123").pipe( + Effect.flatMap((user) => saveAudit(user.id)) +) +``` + +### `Effect.tap` + +Use for side effects that should preserve the main value. + +```ts +const result = loadUser("u_123").pipe( + Effect.tap((user) => Effect.logDebug("loaded user", { userId: user.id })) +) +``` + +Preferred rule: + +- outer workflow: `Effect.gen` +- local transformation: `map`, `flatMap`, `tap` + +## Services And Provisioning + +Repo style is: + +- access services in implementation code +- provide them at boundaries + +### Access services in implementations + +```ts +const loadUser = Effect.fn("loadUser")(function*(userId: string) { + const repo = yield* UserRepo + return yield* repo.getById(userId) +}) +``` + +or: + +```ts +const loadUser = (userId: string) => + Effect.service(UserRepo).pipe( + Effect.flatMap((repo) => repo.getById(userId)) + ) +``` + +### Provide at the edge + +```ts +const program = loadUser("u_123").pipe( + Effect.provide(AppLayer) +) +``` + +Use `provideService` and `provideServiceEffect` for targeted overrides, especially in tests or framework boundaries. + +Do not default to exporting thin accessor functions that just fetch a service and forward to one service method. Prefer real business operations or direct service usage within the owning workflow. + +Repo examples: + +- `./.repos/effect/packages/tools/utils/src/bin.ts` +- `./.repos/effect/packages/tools/openapi-generator/test/` + +## Error Handling + +Common repo patterns: + +- `catchTag` for expected tagged errors +- `match` for totalizing an effect into a value +- `catchCause` for full-cause infra handling + +### `Effect.catchTag` + +Use for targeted typed recovery. + +```ts +const safe = loadUser("u_123").pipe( + Effect.catchTag("UserNotFound", () => Effect.succeed(null)) +) +``` + +### `Effect.match` + +Use when the caller wants a value either way. + +```ts +const result = loadUser("u_123").pipe( + Effect.match({ + onFailure: () => null, + onSuccess: (user) => user + }) +) +``` + +For deeper guidance, see `./references/guide-error-handling.md`. + +## Resource Management + +One of the strongest repo patterns is explicit resource ownership. + +### `Effect.acquireRelease` + +Use for resources that must be cleaned up. + +```ts +const connection = Effect.acquireRelease( + openConnection, + (conn) => closeConnection(conn) +) +``` + +### `Effect.scoped` + +Use when a workflow consumes scoped resources and should tie cleanup to scope lifetime. + +```ts +const program = Effect.scoped( + Effect.gen(function*() { + const conn = yield* connection + return yield* conn.query("select 1") + }) +) +``` + +Repo examples: + +- `./.repos/effect/packages/platform-node/` +- `./.repos/effect/packages/opentelemetry/src/NodeSdk.ts` + +## SQL And Runtime Integrations + +When Effect already provides a domain module for a capability, prefer that module over direct raw runtime client usage in business code. + +Important example: + +- prefer Effect SQL modules from `effect/unstable/sql/*` over embedding a native SQL driver directly in domain services + +Why: + +- transactions, spans, and typed errors stay inside the Effect model +- layering stays cleaner +- migrations and query conventions stay consistent + +For SQL-specific guidance, see `./references/guide-sql.md`. + +## Observability + +The repo uses observability around meaningful boundaries, not every tiny helper. + +Common patterns: + +- `Effect.fn` for named operations +- `Effect.withSpan` for nested span boundaries +- `Effect.log*` for operational events +- `Effect.track` for metrics + +For detailed guidance, see `./references/guide-observability.md`. + +## Runtime Boundaries + +The repo keeps `run*` APIs at true runtime boundaries. + +### `Effect.runPromise` + +Use when leaving Effect world into Promise-based hosts. + +### `Effect.runFork` + +Use for background fibers or long-running integration hooks. + +### `Effect.runSync` + +Use sparingly, mostly in specialized internals where synchrony is guaranteed. + +Preferred rule: + +- library/business code should return `Effect` +- entrypoints and integration boundaries should run `Effect` + +If you have multiple external entrypoints, prefer `ManagedRuntime`. + +## Commonly Used Effect APIs In This Repo + +These are the most practically important `Effect` functions to know first: + +- `Effect.fn` +- `Effect.gen` +- `Effect.succeed` +- `Effect.fail` +- `Effect.sync` +- `Effect.try` +- `Effect.tryPromise` +- `Effect.map` +- `Effect.flatMap` +- `Effect.tap` +- `Effect.service` +- `Effect.provide` +- `Effect.provideService` +- `Effect.catchTag` +- `Effect.match` +- `Effect.acquireRelease` +- `Effect.scoped` +- `Effect.withSpan` +- `Effect.logInfo` +- `Effect.logDebug` +- `Effect.runPromise` + +## Good Repo Examples To Study + +- `./.repos/effect/packages/tools/utils/src/Codegen.ts` +- `./.repos/effect/packages/tools/openapi-generator/src/OpenApiPatch.ts` +- `./.repos/effect/packages/tools/openapi-generator/src/OpenApiGenerator.ts` +- `./.repos/effect/packages/opentelemetry/src/NodeSdk.ts` +- `./.repos/effect/packages/opentelemetry/src/Tracer.ts` +- `./.repos/effect/packages/platform-node/` +- `./.repos/effect/packages/vitest/src/index.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-error-handling.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-error-handling.md new file mode 100644 index 000000000..b842bca1d --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-error-handling.md @@ -0,0 +1,540 @@ +# Error Handling Guide + +This guide is based on the vendored Effect source in `./.repos/effect`. + +Key source files: + +- `./.repos/effect/packages/effect/src/Data.ts` +- `./.repos/effect/packages/effect/src/Schema.ts` +- `./.repos/effect/packages/effect/src/Cause.ts` +- `./.repos/effect/packages/effect/src/Effect.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlError.ts` + +## Mental Model + +Effect distinguishes three failure modes: + +- failure: expected, typed errors in the `E` channel of `Effect` +- defect: unexpected unchecked failures, represented as `Cause.Die` +- interrupt: cooperative cancellation, represented as `Cause.Interrupt` + +This distinction is explicit in `Cause`. + +Repo references: + +- `./.repos/effect/packages/effect/src/Cause.ts` +- `./.repos/effect/packages/effect/src/Effect.ts` + +## Preferred Error Definition Styles + +Preference order: + +1. use schema-based errors when possible +2. fall back to `Data.TaggedError` only when the error payload is not meaningfully serializable or schema-shaped + +Schema-based errors are strictly more powerful because they give you: + +- typed yieldable errors +- schema-defined fields +- encode/decode support +- better protocol and boundary interoperability +- stronger documentation and tooling hooks + +### 1. `Schema.TaggedErrorClass` for schema-backed tagged errors + +Use `Schema.TaggedErrorClass` by default when the error can be described with schemas. + +Why: + +- it creates a yieldable tagged error +- fields are defined with `Schema` +- the error shape can participate in schema-based tooling and encode/decode flows + +Repo references: + +- `./.repos/effect/packages/effect/src/Schema.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlError.ts` + +Example: + +```ts +import { Effect, Schema } from "effect" + +class InvalidPayload extends Schema.TaggedErrorClass()( + "InvalidPayload", + { + field: Schema.String, + reason: Schema.String + } +) {} + +const validate = Effect.fail( + InvalidPayload.make({ + field: "email", + reason: "missing" + }) +) +``` + +Use this when: + +- the error is part of a protocol or transport boundary +- the error needs a precise schema representation +- the error should be serializable or documented structurally + +This is also a good default for domain errors when their payload is schema-friendly. + +### 2. `Schema.ErrorClass` for schema-backed errors without `_tag` routing + +Use `Schema.ErrorClass` when you want schema-defined error objects but do not specifically need tag-based pattern matching. + +Repo references: + +- `./.repos/effect/packages/effect/src/Schema.ts` +- examples across `./.repos/effect/packages/effect/src/unstable/*` + +Example shape from the vendored repo: + +- `./.repos/effect/packages/effect/src/unstable/httpapi/HttpApiError.ts` +- `./.repos/effect/packages/effect/src/unstable/workers/WorkerError.ts` + +### 3. `Data.TaggedError` for non-serializable or lightweight domain errors + +Use `Data.TaggedError` when schema-based errors are not a good fit. + +This is mainly the fallback for: + +- non-serializable payloads +- ad hoc in-memory-only errors +- cases where schema shape would be artificial or misleading + +Repo reference: + +- `./.repos/effect/packages/effect/src/Data.ts` + +Example: + +```ts +import { Data, Effect } from "effect" + +class UserNotFound extends Data.TaggedError("UserNotFound")<{ + readonly userId: string +}> {} + +const loadUser = (userId: string) => + Effect.fail(new UserNotFound({ userId })) + +const program = Effect.gen(function*() { + yield* loadUser("u_123") +}) +``` + +## When To Prefer `Data.TaggedError` vs `Schema.TaggedErrorClass` + +Prefer `Schema.TaggedErrorClass` when: + +- the error can be expressed as a schema +- the error payload must be described by schemas +- the error crosses process, protocol, persistence, or serialization boundaries +- you want the error type to participate in schema tooling + +Prefer `Data.TaggedError` when: + +- the payload is not meaningfully serializable +- the payload cannot reasonably be modeled as a schema +- the error is intentionally local and in-memory only + +## Schema-Based Error Workflows + +### Boundary validation should fail with `SchemaError` + +When validating external input, Effect's schema APIs return `SchemaError` in the error channel. + +Repo references: + +- `./.repos/effect/packages/effect/src/Schema.ts` +- `Schema.decodeUnknownEffect` +- `Schema.decodeUnknownExit` +- `Schema.encodeUnknownEffect` + +Example: + +```ts +import { Effect, Schema } from "effect" + +const UserPayload = Schema.Struct({ + id: Schema.String, + email: Schema.String +}) + +const decodeUser = Schema.decodeUnknownEffect(UserPayload) +``` + +This gives you: + +- success: validated typed data +- failure: `Schema.SchemaError` + +### Normalize `SchemaError` at the boundary + +For application code, it is often better to convert `SchemaError` into a domain error near the boundary. + +Example: + +```ts +import { Data, Effect, Schema } from "effect" + +class InvalidRequestBody extends Data.TaggedError("InvalidRequestBody")<{ + readonly message: string +}> {} + +const UserPayload = Schema.Struct({ + id: Schema.String, + email: Schema.String +}) + +const decodeUser = (input: unknown) => + Schema.decodeUnknownEffect(UserPayload)(input).pipe( + Effect.catchTag("SchemaError", (error) => + Effect.fail(new InvalidRequestBody({ message: error.message })) + ) + ) +``` + +Why: + +- transport validation stays close to the transport layer +- the rest of the application can work with domain-specific errors + +### Use schema-backed errors for protocol errors + +The vendored repo uses schema-backed errors in places like SQL, RPC, sockets, and HTTP APIs. + +Strong examples: + +- `./.repos/effect/packages/effect/src/unstable/sql/SqlError.ts` +- `./.repos/effect/packages/effect/src/unstable/socket/Socket.ts` +- `./.repos/effect/packages/effect/src/unstable/eventlog/EventLogMessage.ts` + +These are good reference points when the error contract matters externally. + +## Wrapping Foreign Or Generic Errors + +When an error comes from a library, runtime API, or generic `Error`, prefer wrapping it in a typed error instead of leaking the foreign error directly through your domain or protocol boundary. + +This is a very common pattern in the vendored repo. + +Good examples: + +- `./.repos/effect/packages/effect/src/unstable/sql/SqlError.ts` +- `./.repos/effect/packages/effect/src/unstable/rpc/RpcClientError.ts` +- `./.repos/effect/packages/effect/src/unstable/socket/Socket.ts` +- `./.repos/effect/packages/effect/src/unstable/workers/WorkerError.ts` +- `./.repos/effect/packages/effect/src/unstable/persistence/Redis.ts` + +### Preferred Pattern + +Wrap the foreign error in a schema-backed typed error and preserve the original error in a `cause` field. + +Prefer using: + +- `Schema.Defect` when you want to preserve a generic encoded defect +- `Schema.DefectWithStack` when the stack should be preserved in the schema contract + +Example: + +```ts +import { Effect, Schema } from "effect" + +class TodoStorageError extends Schema.TaggedErrorClass()( + "TodoStorageError", + { + operation: Schema.String, + cause: Schema.Defect + } +) {} + +const makeStorageError = (operation: string) => (cause: unknown) => + TodoStorageError.make({ + operation, + cause + }) + +const loadTodo = (id: number) => + Effect.try({ + try: () => someLibraryCall(id), + catch: makeStorageError("loadTodo") + }) +``` + +When stack preservation matters in the encoded schema, prefer: + +```ts +class WorkerFailure extends Schema.TaggedErrorClass()( + "WorkerFailure", + { + cause: Schema.DefectWithStack + } +) {} +``` + +### Why This Is Preferred + +- the application still exposes a typed error contract +- the original foreign failure is preserved for diagnostics +- schema-aware transports can encode and decode the failure shape +- business code does not become coupled to a raw library error type + +### When To Use This Pattern + +Use it when: + +- a third-party library throws or rejects with `Error` +- a runtime API returns generic failures +- a lower-level subsystem failure should be surfaced through a typed domain or protocol error +- you need to preserve the underlying failure for debugging without leaking the foreign type as the public error contract + +### `Schema.Defect` vs `Schema.DefectWithStack` + +Prefer `Schema.Defect` by default. + +Use `Schema.DefectWithStack` when: + +- stack information is part of the intended encoded error contract +- the error is primarily infrastructural or diagnostic +- the downstream consumer benefits from the preserved stack + +### Avoid This Anti-Pattern + +Avoid exposing raw generic errors directly as the application error contract. + +Bad: + +```ts +const loadTodo = (id: number) => + Effect.try({ + try: () => someLibraryCall(id), + catch: (cause) => cause as Error + }) +``` + +Why this is bad: + +- the error channel loses a stable typed contract +- the code depends on unsafe assertions +- transport and schema integration become weaker +- callers must understand foreign error shapes instead of your own typed error model + +## Handling Failures + +### Handle specific tagged errors with `Effect.catchTag` + +Use `catchTag` when your error type has `_tag` and you want focused recovery. + +Repo reference: + +- `./.repos/effect/packages/effect/src/Effect.ts` + +Example: + +```ts +const recovered = program.pipe( + Effect.catchTag("UserNotFound", (error) => + Effect.succeed({ id: error.userId, guest: true }) + ) +) +``` + +### Handle several tagged errors with `Effect.catchTags` + +Use `catchTags` when multiple domain errors should be handled together. + +```ts +const recovered = program.pipe( + Effect.catchTags({ + UserNotFound: () => Effect.succeed(null), + InvalidPayload: (error) => Effect.succeed({ error: error.reason }) + }) +) +``` + +### Handle predicate-based subsets with `Effect.catchIf` + +Use `catchIf` when matching on a predicate or refinement, not just `_tag`. + +### Turn failure into a value with `Effect.match` + +Use `match` when you want to fully fold the typed error channel into a success value. + +```ts +const outcome = program.pipe( + Effect.match({ + onFailure: (error) => ({ ok: false as const, error }), + onSuccess: (value) => ({ ok: true as const, value }) + }) +) +``` + +## Handling Defects + +Defects are not normal domain failures. + +They come from: + +- `Effect.die` +- unchecked exceptions in effectful code +- invariants that were broken + +Repo references: + +- `./.repos/effect/packages/effect/src/Cause.ts` +- `./.repos/effect/packages/effect/src/Effect.ts` + +### Preferred rule + +Do not model expected business failures as defects. + +Use defects for: + +- impossible states +- programmer errors +- unrecoverable infrastructure corruption + +### Inspect defects with `sandbox`, `catchCause`, or `matchCause` + +Use `sandbox` to expose `Cause` in the error channel. + +```ts +import { Cause, Effect } from "effect" + +const diagnosed = program.pipe( + Effect.sandbox, + Effect.catchCause((cause) => { + if (Cause.hasDies(cause)) { + return Effect.succeed("defect") + } + return Effect.failCause(cause) + }) +) +``` + +Use `matchCause` or `matchCauseEffect` when you need to distinguish: + +- typed failures +- defects +- interrupts + +### Boundary-only recovery for defects + +If you recover from defects at all, do it only at clear boundaries. + +Examples: + +- worker or RPC boundary +- CLI top-level runner +- HTTP server adapter + +Typical pattern: + +- log or report defect details +- translate to a safe external error +- avoid continuing as if it were a normal domain failure + +### `Effect.orDie` + +Use `orDie` when an error channel should be treated as unrecoverable from this point onward. + +That is appropriate when: + +- a failure has already been validated elsewhere as impossible +- continuing with typed recovery would only obscure a broken invariant + +Do not use `orDie` just to silence a type you do not want to handle. + +## Handling Interrupts + +Interrupts are cancellation, not business failure. + +Repo references: + +- `./.repos/effect/packages/effect/src/Cause.ts` +- `./.repos/effect/packages/effect/src/Effect.ts` + +### Use `Effect.interrupt` to stop work cooperatively + +Interrupts signal that the fiber should stop. They should not usually be translated into a domain error. + +### Use `Effect.onInterrupt` for cleanup + +If interrupted work needs special cleanup, use `onInterrupt`. + +```ts +import { Console, Effect } from "effect" + +const program = longRunningTask.pipe( + Effect.onInterrupt(() => Console.log("cleaning up after interrupt")) +) +``` + +### Use `Cause` inspection when interrupts must be distinguished + +When handling full causes, use `Cause.isInterruptReason`, `Cause.hasInterrupts`, or filtering over `cause.reasons`. + +This is useful for: + +- deciding whether to suppress logs for normal cancellation +- keeping retries for failure but not for cancellation +- distinguishing timeout/cancel flows from real errors + +### Do not treat interrupts as ordinary failures + +Avoid patterns that collapse all causes into a single error value too early. Interrupts often need different operational behavior. + +## Recommended Patterns + +### Pattern: domain errors inside the app, schema errors at the edge + +- decode external input with `Schema.decodeUnknownEffect` +- convert `SchemaError` into a domain or transport error near the boundary +- keep the rest of the application on domain errors + +### Pattern: tagged errors for recovery + +- define domain failures with `Data.TaggedError` +- recover with `catchTag` or `catchTags` +- keep `_tag` names stable and descriptive + +### Pattern: schema-backed errors for protocols + +- use `Schema.TaggedErrorClass` or `Schema.ErrorClass` when the error contract itself matters +- follow examples in SQL, socket, RPC, and HTTP modules + +### Pattern: only inspect `Cause` when you really need the full failure structure + +Use `catchCause`, `matchCause`, or `sandbox` when you must distinguish: + +- expected failures +- defects +- interrupts + +Otherwise prefer the simpler typed error operators. + +## Anti-Patterns + +- using defects for expected validation or business-rule failures +- converting every error immediately to `unknown` or `string` +- using `orDie` to avoid proper handling of expected errors +- treating interrupts as ordinary business failures +- leaking `SchemaError` deep into domain code when it should be normalized at the boundary + +## Good Repo Examples To Study + +- `./.repos/effect/packages/effect/src/Data.ts` +- `./.repos/effect/packages/effect/src/Cause.ts` +- `./.repos/effect/packages/effect/src/Effect.ts` +- `./.repos/effect/packages/effect/src/Schema.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlError.ts` +- `./.repos/effect/packages/effect/src/unstable/http/HttpClientError.ts` +- `./.repos/effect/packages/effect/src/unstable/http/HttpServerError.ts` +- `./.repos/effect/packages/effect/src/unstable/socket/Socket.ts` +- `./.repos/effect/packages/effect/src/unstable/httpapi/HttpApiError.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-layers.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-layers.md new file mode 100644 index 000000000..0419b5d45 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-layers.md @@ -0,0 +1,1018 @@ +# Layers Guide + +This guide is based on the vendored Effect source in `./.repos/effect`. + +Key source files: + +- `./.repos/effect/packages/effect/src/Context.ts` +- `./.repos/effect/packages/effect/src/Layer.ts` +- `./.repos/effect/packages/effect/src/Effect.ts` +- `./.repos/effect/packages/effect/src/ManagedRuntime.ts` + +## Mental Model + +A service is a typed dependency. + +A layer is a recipe for building one or more services, possibly using other services as dependencies. + +Effect's model is: + +- define service identifiers with `Context.Service` or `Context.Service(...)` +- require services from effects with `Effect.service` or by yielding the service key directly +- build implementations with `Layer` +- provide layers at the program boundary or subsystem boundary + +`Layer` means: + +- `ROut`: services produced by the layer +- `E`: possible failures while constructing the layer +- `RIn`: dependencies required to build it + +Repo references: + +- `./.repos/effect/packages/effect/src/Context.ts` +- `./.repos/effect/packages/effect/src/Layer.ts` + +## Services + +## What A Service Is + +A service is a typed key plus its implementation shape. + +In Effect, services are values in `Context`, not global singletons. + +This gives you: + +- explicit dependencies +- easy substitution in tests +- layer-based composition +- multiple implementations for the same interface + +## Preferred Service Definition Style + +Prefer the class syntax with `Context.Service`. + +This matches the vendored repo's current style. + +There are two good definition styles: + +- explicit service shape in the `Context.Service<...>` generic +- inferred service shape from the `make` argument + +Example: + +```ts +import { Context, Effect, Schema } from "effect" + +class UserRepoError extends Schema.TaggedErrorClass()( + "UserRepoError", + { + message: Schema.String + } +) {} + +class UserRepo extends Context.Service Effect.Effect<{ id: string; name: string }, UserRepoError> +}>()("UserRepo") {} +``` + +Why this style is preferred: + +- the service identifier and shape live in one place +- it works naturally with `yield* UserRepo` +- it matches the patterns used across `.repos/effect` + +Repo reference: + +- `./.repos/effect/packages/effect/src/Context.ts` + +## Service Shape Inference With `make` + +When the implementation shape is clearer than the interface declaration, prefer using the `make` argument so the service shape is inferred from the implementation. + +```ts +import { Context, Effect, Schema } from "effect" + +class UserRepoError extends Schema.TaggedErrorClass()( + "UserRepoError", + { + message: Schema.String + } +) {} + +class UserRepo extends Context.Service()("UserRepo", { + make: Effect.succeed({ + getById: Effect.fn("UserRepo.getById")(function*(id: string) { + return yield* Effect.fail( + UserRepoError.make({ message: `User ${id} not found` }) + ) + }) + }) +}) {} +``` + +Why this style is useful: + +- the implementation and inferred API stay together +- TypeScript derives the service shape automatically +- it avoids repeating the same method signatures twice + +Prefer this style when: + +- the implementation is small and obvious +- the explicit service shape would only duplicate the implementation + +Prefer the explicit generic shape when: + +- you want the contract stated before the implementation +- the API surface should be emphasized separately from the implementation + +## Service Example + +```ts +import { Context, Effect, Schema } from "effect" + +class UserNotFound extends Schema.TaggedErrorClass()( + "UserNotFound", + { + userId: Schema.String + } +) {} + +class UserRepo extends Context.Service Effect.Effect<{ id: string; name: string }, UserNotFound> +}>()("UserRepo") {} + +const loadUser = (userId: string) => + Effect.gen(function*() { + const repo = yield* UserRepo + return yield* repo.getById(userId) + }) +``` + +Key points: + +- `UserRepo` is both the identifier and a value you can yield from `Effect.gen` +- the implementation shape is explicit in the service definition +- the effect that uses it stays abstract over the implementation + +## When To Use `Context.Reference` + +Use `Context.Reference` for contextual values with defaults, not for full service APIs. + +Good use cases: + +- current configuration knobs +- current request metadata +- feature flags or tracing flags with defaults + +Repo reference: + +- `./.repos/effect/packages/effect/src/Context.ts` + +Use a full service instead when: + +- behavior matters more than data +- you need multiple methods +- you want a concrete test double or alternate implementation + +## Accessing Services + +Common patterns: + +```ts +const program = Effect.gen(function*() { + const repo = yield* UserRepo + return yield* repo.getById("u_123") +}) +``` + +or: + +```ts +const program = Effect.service(UserRepo).pipe( + Effect.flatMap((repo) => repo.getById("u_123")) +) +``` + +Best practice: + +- use `yield* Service` or `Effect.service(Service)` inside business logic +- do not manually thread service implementations through function arguments when they are real application dependencies + +## Service Encapsulation + +Prefer keeping service access inside the business operation that needs it rather than exporting thin accessor wrappers for every method. + +Avoid this pattern: + +```ts +export const createTodo = Effect.fn(function*(title: string) { + const todos = yield* TodoService + return yield* todos.create(title) +}) +``` + +Why this is usually a bad pattern: + +- it leaks the service dependency into a second public API layer +- it encourages a redundant accessor function per service method +- it spreads dependency access patterns across the codebase +- it weakens service encapsulation instead of improving it + +Prefer one of these patterns instead: + +1. Put the real business logic in a function that uses the service internally because it adds behavior beyond simple forwarding. +2. Expose the service itself and call its methods from the module that owns the workflow. +3. If you need a public operation, make it a real business operation, not a trivial alias of one service method. + +Good: + +```ts +export const completeTodo = Effect.fn("completeTodo")(function*(id: number) { + const todos = yield* TodoService + const todo = yield* todos.getById(id) + if (todo.completed) { + return todo + } + return yield* todos.setCompleted(id, true) +}) +``` + +This is good because: + +- the exported function represents a business operation +- the service remains an internal dependency of that operation +- the function adds behavior rather than just forwarding one method call + +## Layers + +## What A Layer Is + +A layer constructs services from dependencies. + +Use a layer when: + +- service construction is effectful +- the service depends on other services +- the service owns resources that must be acquired and released safely +- you want composition and reuse across modules + +## Preferred Layer Constructors + +### `Layer.succeed` + +Use for pure, already-constructed implementations. + +```ts +const UserRepoTest = Layer.succeed(UserRepo)({ + getById: (id) => Effect.succeed({ id, name: "Test User" }) +}) +``` + +Use this when: + +- construction is pure +- no dependencies are needed +- no scoped resources are involved + +### `Layer.effect` + +Use when constructing a service requires effects, other services, or scoped resource acquisition. + +```ts +class Config extends Context.Service()("Config") {} + +const UserRepoLayer = Layer.effect(UserRepo)( + Effect.gen(function*() { + const config = yield* Config + + return { + getById: (id) => + Effect.succeed({ + id, + name: `Fetched from ${config.apiBaseUrl}` + }) + } + }) +) +``` + +Use this when: + +- construction is effectful +- construction depends on other services +- construction needs `Scope` and finalization +- you want typed construction failure + +This is also the correct constructor for services that own resources with acquisition and release semantics. In this repo, `Layer.effect` replaces the old `Layer.scoped` API. + +Typical examples: + +- database pools +- sockets +- background worker processes +- long-lived subscriptions + +### `Layer.effectDiscard` + +Use `Layer.effectDiscard` for scoped startup effects that do not provide a service. + +Good use cases: + +- starting background fibers in a layer +- one-time scoped initialization side effects +- subsystem startup hooks + +### `Layer.effectContext` + +Use when one effect constructs a full `Context` containing multiple services. + +This is useful for subsystem builders that provide several related services together. + +## Layer Composition + +These operators do different things. Do not treat them as interchangeable. + +### Composition Cheat Sheet + +- `Layer.mergeAll(a, b, ...)`: combine outputs of multiple layers +- `Layer.provide(target, dependencies)`: feed dependency outputs into `target` and keep only `target` outputs +- `Layer.provideMerge(target, dependencies)`: feed dependency outputs into `target` and keep both dependency outputs and target outputs +- `Layer.flatMap(layer, f)`: choose the next layer based on the built service value + +### Example Services + +```ts +import { Context, Effect, Layer } from "effect" + +class Config extends Context.Service()("Config") {} + +class Logger extends Context.Service Effect.Effect +}>()("Logger") {} + +class UserRepo extends Context.Service Effect.Effect<{ id: string; name: string }> +}>()("UserRepo") {} + +const ConfigLayer = Layer.succeed(Config)({ + apiBaseUrl: "https://api.example.com" +}) + +const LoggerLayer = Layer.succeed(Logger)({ + log: (message) => Effect.sync(() => console.log(message)) +}) + +const UserRepoLayer = Layer.effect(UserRepo)( + Effect.gen(function*() { + const config = yield* Config + const logger = yield* Logger + + return { + getById: (id) => + Effect.gen(function*() { + yield* logger.log(`loading ${id} from ${config.apiBaseUrl}`) + return { id, name: "Ada" } + }) + } + }) +) +``` + +### `Layer.mergeAll` + +Use `Layer.mergeAll` to combine outputs of independent layers. + +```ts +const Dependencies = Layer.mergeAll( + ConfigLayer, + LoggerLayer +) +``` + +Use this when: + +- the layers provide different services +- neither needs to transform the other directly + +Semantics: + +- inputs are combined +- outputs are combined +- no dependency feeding happens automatically + +Important: + +- `Layer.mergeAll(ConfigLayer, UserRepoLayer)` is wrong if `UserRepoLayer` requires `Config` and `Logger` +- `mergeAll` does not satisfy `UserRepoLayer`'s requirements +- it only places both layers side by side in the output graph + +Correct pattern: + +```ts +const Dependencies = Layer.mergeAll( + ConfigLayer, + LoggerLayer +) +``` + +### `Layer.provide` + +Use `Layer.provide` to satisfy a target layer's dependencies with another layer, while keeping only the target layer's outputs. + +```ts +const Dependencies = Layer.mergeAll( + ConfigLayer, + LoggerLayer +) + +const UserRepoLayerReady = Layer.provide(UserRepoLayer, Dependencies) +``` + +Interpretation: + +- `UserRepoLayer` requires `Config` and `Logger` +- `Dependencies` provides those dependencies +- the resulting layer provides only `UserRepo` +- `Config` and `Logger` are used for construction but are not kept in the final output + +This is the operator to use when you want to hide construction dependencies behind a narrower public layer. + +Example program: + +```ts +const program = Effect.gen(function*() { + const repo = yield* UserRepo + return yield* repo.getById("u_123") +}).pipe( + Effect.provide(UserRepoLayerReady) +) +``` + +### `Layer.provideMerge` + +Use `provideMerge` when you want to satisfy dependencies and retain both the dependency outputs and the target outputs. + +```ts +const Dependencies = Layer.mergeAll( + ConfigLayer, + LoggerLayer +) + +const AppLayer = Layer.provideMerge(UserRepoLayer, Dependencies) +``` + +Interpretation: + +- `UserRepoLayer` still gets `Config` and `Logger` +- the resulting layer provides `UserRepo`, `Config`, and `Logger` + +This is useful for assembling larger application layers incrementally, especially when downstream code still needs access to the dependencies. + +Example program: + +```ts +const program = Effect.gen(function*() { + const repo = yield* UserRepo + const logger = yield* Logger + + const user = yield* repo.getById("u_123") + yield* logger.log(user.name) + return user +}).pipe( + Effect.provide(AppLayer) +) +``` + +Preferred rule: + +- use `provide` when you want to hide dependency details +- use `provideMerge` when you want to keep dependency services available downstream + +### `Layer.mergeAll` vs `Layer.provide` vs `Layer.provideMerge` + +Think of them like this: + +- `mergeAll`: put layers next to each other +- `provide`: plug one layer into another, expose only the target +- `provideMerge`: plug one layer into another, expose both sides + +### Composition Style Best Practice + +Layers should almost always be fully composed locally before they are assembled into the final application layer. + +Preferred style: + +- define each service layer separately +- define local subsystem dependency bundles separately +- fully compose each subsystem locally with `Layer.provide` or `Layer.provideMerge` +- assemble the final application layer with `Layer.mergeAll(...)` +- apply top-level cross-cutting provisioning in a small number of explicit trailing steps + +Good pattern: + +```ts +const UserDependencies = Layer.mergeAll( + ConfigLayer, + LoggerLayer +) + +const UserLayer = Layer.provide(UserRepoLayer, UserDependencies) + +const BillingDependencies = Layer.mergeAll( + ConfigLayer, + LoggerLayer, + DatabaseLayer +) + +const BillingLayer = Layer.provide(BillingServiceLayer, BillingDependencies) + +const AppLayer = Layer.mergeAll( + UserLayer, + BillingLayer, + HttpLayer +).pipe( + Layer.provide(Telemetry), + Layer.provide(NodeSdk) +) +``` + +Why this style is preferred: + +- subsystem wiring stays local to the subsystem +- the final application layer reads as a high-level composition map +- cross-cutting concerns such as telemetry stay visible at the top level +- it avoids deeply nested inline layer expressions + +Avoid this style when a clearer local name would help: + +```ts +const AppLayer = Layer.provide( + Layer.mergeAll( + Layer.provide(UserRepoLayer, Layer.mergeAll(ConfigLayer, LoggerLayer)), + Layer.provide(BillingServiceLayer, Layer.mergeAll(ConfigLayer, LoggerLayer, DatabaseLayer)), + HttpLayer + ), + Telemetry +).pipe(Layer.provide(NodeSdk)) +``` + +That style is harder to read because: + +- subsystem composition is hidden inside the final assembly +- shared dependencies are harder to spot +- it is harder to refactor or reuse subsystem layers + +### `Layer.flatMap` + +Use `flatMap` when the next layer depends on the actual constructed service value, not just its type-level requirement. + +Example: + +```ts +const UserRepoLayerFromConfig = Layer.flatMap(ConfigLayer, (config) => + Layer.succeed(UserRepo)({ + getById: (id) => Effect.succeed({ id, name: config.apiBaseUrl }) + }) +) +``` + +This is more specialized than `merge` or `provide`. + +Prefer simpler composition first: + +- `merge` for combining +- `provide` for dependency satisfaction +- `flatMap` only when construction logic truly depends on the built value + +## Providing Layers To Effects + +## Preferred Rule + +Provide layers at boundaries. + +Usually that means: + +- the application entrypoint +- a subsystem entrypoint +- a test boundary + +Avoid repeatedly providing layers deep inside business logic unless you are deliberately isolating a subsystem. + +### Anti-Pattern: Local `Effect.provide` + +`Effect.provide` should be used only once at the entry of your program in normal application code. + +Bad pattern: + +```ts +const loadUser = (userId: string) => + Effect.gen(function*() { + const repo = yield* UserRepo + return yield* repo.getById(userId) + }).pipe( + Effect.provide(UserRepoLayer) + ) +``` + +Why this is an anti-pattern: + +- it hides dependency wiring inside business logic +- it makes implementations harder to swap in tests +- it prevents clean top-level composition +- it encourages many small local runtimes instead of one coherent application graph +- it makes shared cross-cutting services harder to reason about + +Preferred pattern: + +```ts +const loadUser = (userId: string) => + Effect.gen(function*() { + const repo = yield* UserRepo + return yield* repo.getById(userId) + }) + +const program = loadUser("u_123").pipe( + Effect.provide(AppLayer) +) +``` + +Rule of thumb: + +- business logic should require services +- composition should happen in layers +- `Effect.provide` should happen at the outermost entry boundary + +### Multiple Entry Points + +If your code integrates with a framework and has multiple entry points, prefer `ManagedRuntime` instead of repeatedly calling `Effect.provide` at many call sites. + +Typical examples: + +- HTTP handlers registered separately +- queue consumers +- cron jobs +- framework lifecycle hooks +- RPC handlers or worker callbacks + +Preferred pattern: + +```ts +const runtime = ManagedRuntime.make(AppLayer) + +const handleRequest = (id: string) => + runtime.runPromise(loadUser(id)) +``` + +Why: + +- the layer graph is still composed once +- services remain shared according to layer semantics +- the framework integration gets a stable runtime boundary +- resource lifecycle is explicit through `ManagedRuntime` + +Repo reference: + +- `./.repos/effect/packages/effect/src/ManagedRuntime.ts` + +## `Effect.provide` + +Use `Effect.provide` to satisfy an effect's dependencies with a layer or context. + +```ts +const program = loadUser("u_123").pipe( + Effect.provide(UserRepoLayerReady) +) +``` + +This is the main boundary provisioning operator. + +## `Effect.provideService` + +Use `provideService` for a single ad hoc implementation. + +```ts +const program = loadUser("u_123").pipe( + Effect.provideService(UserRepo, { + getById: (id) => Effect.succeed({ id, name: "Inline User" }) + }) +) +``` + +Good use cases: + +- small tests +- one-off overrides +- local customization + +Do not use this as the default replacement for real application layers. + +## `Effect.provideServiceEffect` + +Use `provideServiceEffect` when one service instance must be built effectfully without creating a reusable layer. + +This is useful for targeted overrides, but if the construction is reusable or part of application wiring, prefer a named `Layer.effect`. + +## Best Practices + +## 1. Keep service interfaces small and focused + +Prefer cohesive services over giant "everything" services. + +Good: + +- `UserRepo` +- `Mailer` +- `Clock`-like configuration or time abstractions + +Avoid: + +- large service shapes that mix unrelated responsibilities + +## 2. Prefer layers over manual wiring + +If construction has dependencies or effects, represent it as a layer. + +Avoid manually grabbing dependencies and assembling concrete objects all over the codebase. + +## 2.5 Prefer Effect-native integrations over raw runtime clients + +When Effect already provides a module for a capability, prefer the Effect-native integration over directly embedding a raw runtime client in service code. + +Examples: + +- prefer `effect/unstable/sql` modules over directly coupling business services to native SQL driver APIs +- prefer Effect HTTP modules over direct ad hoc request clients when the project is already using Effect HTTP abstractions + +Why: + +- resource handling, tracing, and errors stay inside the Effect model +- integrations compose better with layers and services +- observability and transactions are easier to keep consistent + +## 3. Keep business logic abstract over implementations + +Business functions should require services, not construct them. + +Good: + +```ts +const sendWelcomeEmail = (userId: string) => + Effect.gen(function*() { + const repo = yield* UserRepo + const user = yield* repo.getById(userId) + return user + }) +``` + +Avoid constructing `UserRepo` inside `sendWelcomeEmail`. + +## 4. Use `Layer.succeed` only for pure values + +Do not hide effectful initialization inside supposedly pure service objects. + +If initialization can fail, depends on effects, or needs scoped acquisition, use `Layer.effect`. + +## 5. Use `Layer.effect` for owned resources + +If the service opens something that must later close, model that lifecycle explicitly. + +This is one of the main reasons layers exist. + +## 6. Prefer top-level composition + +Compose major application layers once near the boundary. + +Good pattern: + +- define `ConfigLayer` +- define `UserRepoLayer` +- define `Dependencies = Layer.mergeAll(...)` +- define `AppLayer` separately with `Layer.provide(...)` or `Layer.provideMerge(...)` +- provide `AppLayer` to the top-level program + +## 7. Use `Layer.fresh` only when you really need a new instance + +Layers are shared by default. + +That is usually what you want. + +Use `Layer.fresh` only when you intentionally need to bypass sharing and rebuild the layer. + +## 7.5 Understand Layer Memoization + +Layers are memoized by reference. + +That means: + +- reusing the same layer value preserves memoization and sharing +- creating a new layer value creates a new memoization identity + +Because of this, functions that return layers should be avoided unless they are absolutely necessary. + +Prefer plain named layer constants over layer factory functions. + +Only use a function returning a layer when: + +- the layer genuinely depends on runtime parameters +- the caller truly needs distinct configurations or instances +- a constant layer value cannot express the construction cleanly + +Even in those cases, call the function once during construction and reuse the resulting layer value. + +### Function Dependencies Should Stay Unprovided + +If a dependency of a layer is itself represented by a function that returns a layer, do not call that function locally just to satisfy the dependency. + +Instead, leave that dependency unprovided and let it be supplied at the edge. + +Bad pattern: + +```ts +const UserLayer = Layer.provide(UserRepoLayer, makeDatabaseLayer(config)) +``` + +Why this is bad: + +- it creates a fresh layer reference locally +- it breaks or weakens memoization and sharing assumptions +- it hides an important construction dependency inside subsystem wiring +- it makes the final application graph harder to understand + +Preferred pattern: + +```ts +const UserLayer = UserRepoLayer + +const DatabaseLayer = makeDatabaseLayer(config) + +const AppLayer = Layer.provideMerge(UserLayer, DatabaseLayer) +``` + +More generally: + +- if a layer depends on a parameterized layer factory, keep that dependency in the required environment when possible +- construct the concrete parameterized layer once at the outer boundary +- provide it only at the edge where the full application graph is assembled + +Rule: + +- do not call layer-producing dependency functions deep inside subsystem composition +- keep those dependencies unprovided until the edge +- provide the concrete layer once in the final top-level assembly + +Bad pattern: + +```ts +const makeDatabaseLayer = () => Layer.effect(DatabaseService)(/* ... */) + +const AppLayer = Layer.mergeAll( + makeDatabaseLayer(), + makeDatabaseLayer() +) +``` + +Why this is bad: + +- each call creates a distinct layer reference +- memoization does not apply across those distinct references +- the underlying resource or service may be constructed more than once +- sharing assumptions become incorrect + +Preferred pattern: + +```ts +const DatabaseLayer = makeDatabaseLayer() + +const AppLayer = Layer.mergeAll( + DatabaseLayer, + OtherLayer +) +``` + +Rule: + +- avoid layer-producing functions unless they are truly necessary +- if a function returns a layer, call it once during construction and bind the result to a named constant +- reuse that layer value everywhere else + +This is especially important for: + +- database layers +- HTTP client layers +- telemetry layers +- queues, workers, and other resource-owning services + +If you intentionally need a distinct instance, make that explicit with a new layer value or `Layer.fresh`, rather than accidentally creating multiple instances by repeatedly calling a layer factory. + +## 8. Treat `Layer.orDie` carefully + +`Layer.orDie` converts layer construction failures into defects. + +Only use it when failure is truly unrecoverable at that boundary. + +Do not use it to hide legitimate configuration or infrastructure failures. + +## 9. Use `ManagedRuntime.make` at true runtime boundaries + +If you need a reusable runtime built from a layer, `ManagedRuntime.make` is the edge tool for that. + +Good use cases: + +- embedding Effect into external frameworks +- scripts or hosts that repeatedly run Effect programs + +Repo reference: + +- `./.repos/effect/packages/effect/src/ManagedRuntime.ts` + +## 10. Prefer explicit test layers + +For tests, prefer: + +- `Layer.succeed` for simple fakes +- `Layer.mock` for partial mocks when appropriate + +This keeps test wiring explicit and close to production composition style. + +## Recommended Patterns + +## Pattern: service definition plus live layer + +```ts +import { Context, Effect, Layer } from "effect" + +class Config extends Context.Service()("Config") {} + +class UserRepo extends Context.Service Effect.Effect<{ id: string; name: string }> +}>()("UserRepo") {} + +const ConfigLayer = Layer.succeed(Config)({ + apiBaseUrl: "https://api.example.com" +}) + +const UserRepoLayer = Layer.effect(UserRepo)( + Effect.gen(function*() { + const config = yield* Config + + return { + getById: (id) => + Effect.succeed({ + id, + name: `Loaded via ${config.apiBaseUrl}` + }) + } + }) +) + +const Dependencies = Layer.mergeAll(ConfigLayer) + +const AppLayer = Layer.provide(UserRepoLayer, Dependencies) +``` + +## Pattern: provide at the top level + +```ts +const program = Effect.gen(function*() { + const repo = yield* UserRepo + return yield* repo.getById("u_123") +}).pipe( + Effect.provide(AppLayer) +) +``` + +## Pattern: single-service override in tests + +```ts +const TestRepo = Layer.succeed(UserRepo)({ + getById: (id) => Effect.succeed({ id, name: "Test" }) +}) +``` + +## Anti-Patterns + +- constructing live services directly inside business logic +- using `Layer.succeed` for values that actually require effectful initialization +- providing the same large layer repeatedly throughout the call graph +- collapsing unrelated responsibilities into one service +- using `Layer.orDie` to hide normal initialization failures +- bypassing layers entirely for resource-owning services + +## Good Repo Examples To Study + +- `./.repos/effect/packages/effect/src/Context.ts` +- `./.repos/effect/packages/effect/src/Layer.ts` +- `./.repos/effect/packages/effect/src/ManagedRuntime.ts` +- `./.repos/effect/packages/effect/src/Stream.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlClient.ts` +- `./.repos/effect/packages/effect/src/unstable/persistence/Persistence.ts` +- `./.repos/effect/packages/effect/src/unstable/rpc/RpcSerialization.ts` +- `./.repos/effect/packages/effect/src/unstable/reactivity/Reactivity.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-observability.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-observability.md new file mode 100644 index 000000000..0483b7fd5 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-observability.md @@ -0,0 +1,724 @@ +# Observability Guide + +This guide is based on the vendored Effect source in `./.repos/effect`. + +Key source files: + +- `./.repos/effect/packages/effect/src/Effect.ts` +- `./.repos/effect/packages/effect/src/Tracer.ts` +- `./.repos/effect/packages/effect/src/Logger.ts` +- `./.repos/effect/packages/effect/src/Metric.ts` +- `./.repos/effect/packages/opentelemetry/src/NodeSdk.ts` +- `./.repos/effect/packages/opentelemetry/src/Tracer.ts` + +## Mental Model + +Observable Effect code should make business operations visible by default. + +That means: + +- business logic should show up clearly in stack traces +- important operations should produce spans +- logs should inherit execution context +- metrics should be attached at meaningful boundaries + +The most important best practice is: + +- prefer `Effect.fn(...)` whenever possible for business logic + +Why: + +- it adds stack frames +- it creates spans automatically +- it gives you better tracing and debugging for free +- it keeps business logic observable without extra boilerplate + +Repo reference: + +- `./.repos/effect/packages/effect/src/Effect.ts` + +## Preferred Rule + +Use `Effect.fn` as the default constructor for business-logic functions that return `Effect`. + +Prefer this: + +```ts +import { Effect } from "effect" + +const loadUser = Effect.fn("loadUser")(function*(userId: string) { + return { id: userId, name: "Ada" } +}) +``` + +Over this: + +```ts +import { Effect } from "effect" + +const loadUser = (userId: string) => + Effect.gen(function*() { + return { id: userId, name: "Ada" } + }) +``` + +The second version works, but it throws away useful observability structure that `Effect.fn` gives you automatically. + +## Prefer `Effect.fn` Over Raw `Effect.gen` + +For business-logic definitions, prefer `Effect.fn` over writing raw `Effect.gen` directly, even when the operation takes no arguments. + +Prefer this: + +```ts +const refreshCache = Effect.fn("refreshCache")(function*() { + yield* Effect.logInfo("refreshing cache") +}) +``` + +Over this: + +```ts +const refreshCache = Effect.gen(function*() { + yield* Effect.logInfo("refreshing cache") +}) +``` + +Why: + +- `Effect.fn` gives the operation a clear observable identity +- stack traces are better +- tracing is more consistent +- the codebase gets a uniform shape for business operations + +Use raw `Effect.gen` when necessary, for example: + +- inline effect blocks inside another `Effect.fn` +- small one-off composition at call sites +- top-level assembly code where you are not defining a reusable business operation + +Rule of thumb: + +- reusable business operation: `Effect.fn` +- inline composition block: `Effect.gen` + +## `Effect.fn` vs `Effect.fnUntraced` + +### `Effect.fn` + +Use `Effect.fn` for almost all business logic. + +It is the preferred default because it adds: + +- stack frames +- tracing spans +- optional post-processing of the produced effect + +Example: + +```ts +import { Effect } from "effect" + +const createUser = Effect.fn("createUser")(function*(name: string) { + return { id: "u_123", name } +}) +``` + +Use this for: + +- domain operations +- application services +- handlers +- workflows +- orchestrations +- repository calls + +### `Effect.fnUntraced` + +Use `Effect.fnUntraced` only for edge cases. + +The vendored Effect repo itself uses `fnUntraced` in a number of low-level internals and integration helpers. That does not make it the default recommendation for downstream application or business code. + +If you do not want an explicit named span, prefer `Effect.fn` without a span name so you still keep stack traces and the normal traced-function behavior. + +Prefer this: + +```ts +const normalizeUser = Effect.fn(function*(input: string) { + return input.trim().toLowerCase() +}) +``` + +Over this: + +```ts +const normalizeUser = Effect.fnUntraced(function*(input: string) { + return input.trim().toLowerCase() +}) +``` + +Typical cases: + +- extremely hot low-level internal helpers +- very small internal combinators +- tight loops where you have measured overhead and need to reduce it + +Preferred rule: + +- `Effect.fn` by default +- `Effect.fn` without a span name when you want to avoid an explicit named span +- `Effect.fnUntraced` only with a concrete measured reason + +## Business Logic Patterns + +### Pattern: one named `Effect.fn` per meaningful operation + +Good: + +```ts +const parseCommand = Effect.fn("parseCommand")(function*(input: string) { + return input.trim() +}) + +const loadUser = Effect.fn("loadUser")(function*(userId: string) { + return { id: userId, name: "Ada" } +}) + +const sendWelcomeEmail = Effect.fn("sendWelcomeEmail")(function*(userId: string) { + const user = yield* loadUser(userId) + return user.email +}) +``` + +Why this is good: + +- each operation has a clear span name +- traces reflect business concepts +- stack traces reflect the actual workflow + +### Pattern: use meaningful names + +Span names created through `Effect.fn` should represent business operations, not generic implementation detail. + +Prefer: + +- `loadUser` +- `chargeInvoice` +- `syncGithubInstallation` + +Avoid: + +- `helper` +- `run` +- `process` +- `step1` + +## Explicit Spans + +### `Effect.withSpan` + +Use `withSpan` when you need an explicit span around an effect that is not already naturally represented by a named `Effect.fn`, or when you want a nested sub-operation span. + +Example: + +```ts +import { Effect } from "effect" + +const syncUser = Effect.fn("syncUser")(function*(userId: string) { + const profile = yield* fetchProfile(userId).pipe( + Effect.withSpan("fetchProfile") + ) + + return yield* persistProfile(profile).pipe( + Effect.withSpan("persistProfile") + ) +}) +``` + +Use this when: + +- you want a nested span inside a larger operation +- you are instrumenting an existing effect pipeline +- you need more detailed trace structure than `Effect.fn` alone provides + +### `Effect.withSpanScoped` + +Use `withSpanScoped` when the span should remain open for the lifetime of a scope. + +This is less common in business logic and more common in long-lived resource or streaming workflows. + +### `Effect.withParentSpan` + +Use `withParentSpan` when integrating with an externally created span or continuing a parent span manually. + +This is useful in framework or interoperability boundaries. + +## Span Enrichment + +### `Effect.annotateCurrentSpan` + +Use `annotateCurrentSpan` to attach important structured fields to the current span. + +Example: + +```ts +import { Effect } from "effect" + +const loadUser = Effect.fn("loadUser")(function*(userId: string) { + yield* Effect.annotateCurrentSpan({ userId }) + return { id: userId, name: "Ada" } +}) +``` + +Good span annotations: + +- stable identifiers +- domain-relevant keys +- request or resource identifiers +- small structured values + +Avoid: + +- giant payloads +- secrets +- noisy transient data with little diagnostic value + +## Logging Patterns + +### Use Effect logging inside effects + +Prefer: + +- `Effect.log` +- `Effect.logInfo` +- `Effect.logDebug` +- `Effect.logWarning` +- `Effect.logError` + +These integrate with the current Effect execution context. + +Example: + +```ts +const loadUser = Effect.fn("loadUser")(function*(userId: string) { + yield* Effect.logDebug("loading user", { userId }) + return { id: userId, name: "Ada" } +}) +``` + +### `Effect.withLogSpan` + +Use `withLogSpan` when you want log messages to carry a local logical span label even when you are not creating a full tracing span. + +Example: + +```ts +const program = Effect.logInfo("starting sync").pipe( + Effect.withLogSpan("user-sync") +) +``` + +This is useful for: + +- log grouping +- quick local context +- correlation in plain log output + +### Logging Best Practices + +- log at business boundaries, not every tiny helper +- prefer structured values over concatenated strings +- keep logs high-signal +- avoid duplicate logs at every layer of the stack +- rely on spans plus a few well-placed logs, not log spam + +## Metrics Patterns + +### Track effects at meaningful boundaries + +Use metric tracking on meaningful operations such as: + +- requests +- jobs +- retries +- external calls +- queue handlers + +Repo reference: + +- `./.repos/effect/packages/effect/src/Effect.ts` +- `Effect.track` + +Example: + +```ts +import { Effect, Metric } from "effect" + +const requests = Metric.counter("user_load_requests").pipe( + Metric.withConstantInput(1) +) + +const loadUser = Effect.fn("loadUser")( + function*(userId: string) { + return { id: userId, name: "Ada" } + }, + Effect.track(requests) +) +``` + +### Prefer boundary metrics over micro-metrics + +Good metrics are usually attached to: + +- endpoint handlers +- queue/job handlers +- repository operations +- external API boundaries + +Avoid putting a metric on every tiny internal helper. + +## OpenTelemetry Integration + +For real application observability, compose telemetry at the layer level. + +The vendored repo provides `@effect/opentelemetry` layers such as: + +- `NodeSdk.layer` +- `Tracer.layer` +- `Logger.layer` +- `Metrics.layer` + +Repo references: + +- `./.repos/effect/packages/opentelemetry/src/NodeSdk.ts` +- `./.repos/effect/packages/opentelemetry/src/Tracer.ts` + +Preferred composition style: + +```ts +const AppLayer = Layer.mergeAll( + UserLayer, + BillingLayer, + HttpLayer +).pipe( + Layer.provide(Telemetry), + Layer.provide(NodeSdk) +) +``` + +Why: + +- business code stays observability-agnostic +- observability is configured once at the boundary +- spans, logs, and metrics remain consistent across the app + +## OpenTelemetry JS Framework Integration + +The vendored repo includes a real integration layer for the OpenTelemetry JavaScript ecosystem in `@effect/opentelemetry`. + +This is the preferred integration path when the application needs to participate in the standard OpenTelemetry JS framework, exporters, and SDKs. + +Relevant modules: + +- `./.repos/effect/packages/opentelemetry/src/NodeSdk.ts` +- `./.repos/effect/packages/opentelemetry/src/Tracer.ts` +- `./.repos/effect/packages/opentelemetry/src/Metrics.ts` +- `./.repos/effect/packages/opentelemetry/src/Logger.ts` +- `./.repos/effect/packages/opentelemetry/src/Resource.ts` +- `./.repos/effect/packages/opentelemetry/src/WebSdk.ts` + +### Preferred Integration Model + +Use `@effect/opentelemetry` layers to bridge Effect observability into OpenTelemetry JS. + +Do not manually wire OpenTelemetry SDK objects inside business code. + +Prefer: + +- configuring tracer, metrics, logger, and resource layers once +- composing them into the application layer graph +- keeping business code written against Effect tracing, logging, and metrics APIs + +This means: + +- application code should keep using `Effect.fn`, `Effect.withSpan`, `Effect.log*`, and Effect metrics +- OpenTelemetry JS should be introduced at the infrastructure layer, not inside domain operations + +### `NodeSdk.layer` + +`NodeSdk.layer(...)` is the main Node.js integration entrypoint. + +From the vendored source, it accepts a configuration that can include: + +- span processors +- tracer config +- metric readers +- temporality preference +- log record processors +- logger provider config +- resource information such as service name and version +- shutdown timeout + +It then builds and merges: + +- resource layer +- tracer layer +- metrics layer +- logger layer + +This makes it the preferred high-level integration for Node applications. + +### Resource Configuration + +OpenTelemetry JS integration should define resource metadata explicitly. + +From `NodeSdk.layer`, the supported resource configuration includes: + +- `serviceName` +- `serviceVersion` +- additional attributes + +This is important because tracer and logger setup depend on the configured resource. + +Best practice: + +- always provide a meaningful service name +- provide service version when available +- use resource attributes for stable deployment or environment metadata + +### Tracing Integration + +The `Tracer` module bridges Effect spans into OpenTelemetry spans. + +Important integration points from the vendored source: + +- `Tracer.layer` +- `Tracer.layerGlobal` +- `Tracer.layerGlobalProvider` +- `Tracer.currentOtelSpan` +- `Tracer.makeExternalSpan` + +Use these when: + +- you need Effect tracing to export through OpenTelemetry JS +- you need to continue or bridge external trace context +- you need access to the current OpenTelemetry span object + +Best practice: + +- keep creating spans with Effect APIs in application code +- use the OpenTelemetry tracer layer to export and bridge them +- use `makeExternalSpan` or parent-span wiring only at integration boundaries + +### Metrics Integration + +The `Metrics` module connects Effect metrics to OpenTelemetry JS metric readers. + +Important details from the vendored implementation: + +- `Metrics.layer(...)` registers a producer against one or more metric readers +- it supports temporality preferences: + - `cumulative` + - `delta` +- it handles shutdown through scoped layer cleanup + +Best practice: + +- choose temporality based on the backend +- configure metric readers in the telemetry layer +- keep application code focused on recording Effect metrics, not exporter mechanics + +### Logger Integration + +The `Logger` module connects Effect logging to OpenTelemetry JS logs. + +Important details from the vendored implementation: + +- it maps Effect log levels to OpenTelemetry severity numbers +- it includes fiber ID, span context, log annotations, and log span timing in emitted attributes +- `Logger.layer({ mergeWithExisting })` can merge with or replace existing application loggers + +Best practice: + +- prefer merging with existing loggers unless there is a strong reason to replace them +- use Effect log annotations and log spans so the OpenTelemetry logger receives structured context automatically + +### Shutdown And Lifecycle + +The vendored layers use scoped acquisition and release for tracer providers, metric readers, and logger providers. + +This is the correct lifecycle model. + +Do not manually call provider shutdown methods from arbitrary business logic. + +Instead: + +- let the OpenTelemetry layers own provider lifecycle +- compose them into the application layer graph +- let the runtime or outer layer scope manage shutdown + +### External Trace Context + +When integrating with frameworks or inbound protocols that already carry trace context, prefer using the OpenTelemetry integration helpers rather than hand-rolling context propagation. + +The vendored tracer module provides: + +- `makeExternalSpan` +- `currentOtelSpan` + +Use these only at integration boundaries such as: + +- HTTP adapters +- RPC adapters +- worker or queue adapters + +Keep business operations oblivious to propagation mechanics. + +### Recommended Pattern + +Preferred architecture: + +1. business code uses Effect observability APIs +2. infrastructure composes `@effect/opentelemetry` layers +3. the final app layer provides telemetry once at the top level + +Example shape: + +```ts +const TelemetryLayer = NodeSdk.layer(() => ({ + resource: { + serviceName: "todo-service", + serviceVersion: "1.0.0" + }, + spanProcessor: mySpanProcessor, + metricReader: myMetricReader, + logRecordProcessor: myLogProcessor +})) + +const AppLayer = Layer.mergeAll( + DomainLayer, + HttpLayer +).pipe( + Layer.provide(TelemetryLayer) +) +``` + +This keeps: + +- app code portable +- OTel JS setup centralized +- shutdown semantics correct +- exported spans, logs, and metrics aligned + +### Anti-Patterns + +- constructing OpenTelemetry SDK clients directly inside business services +- mixing manual exporter setup into domain code +- bypassing Effect logging and tracing APIs in normal business operations +- scattering provider shutdown logic across the application +- configuring telemetry separately in many subsystems instead of one top-level layer + +## Anti-Patterns + +### Anti-Pattern: business logic built from anonymous `Effect.gen` functions everywhere + +Bad: + +```ts +const loadUser = (userId: string) => + Effect.gen(function*() { + return { id: userId, name: "Ada" } + }) +``` + +Why this is bad: + +- weaker tracing structure +- poorer stack traces +- less consistent naming in debugging output + +Preferred: + +```ts +const loadUser = Effect.fn("loadUser")(function*(userId: string) { + return { id: userId, name: "Ada" } +}) +``` + +### Anti-Pattern: using `Effect.fnUntraced` by default + +This throws away free observability. + +If you just do not want an explicit span name, use `Effect.fn` without a name instead. + +Only use `fnUntraced` when you have a specific low-level reason. + +### Anti-Pattern: logging without structure or context + +Bad: + +- giant interpolated strings +- duplicate logs at every layer +- logs with no business identifiers + +Prefer: + +- named operations via `Effect.fn` +- structured logs with IDs and context +- a few high-value logs at operation boundaries + +### Anti-Pattern: hand-instrumenting every helper with spans + +Do not create explicit spans everywhere just because you can. + +Preferred order: + +1. start with `Effect.fn` +2. add `Effect.withSpan` only where extra detail is actually useful +3. add metrics at meaningful boundaries + +## Recommended Patterns + +### Pattern: observable business operation + +```ts +import { Effect } from "effect" + +const fetchUser = Effect.fn("fetchUser")(function*(userId: string) { + yield* Effect.annotateCurrentSpan({ userId }) + yield* Effect.logDebug("fetching user", { userId }) + return { id: userId, name: "Ada" } +}) +``` + +### Pattern: orchestration with nested spans + +```ts +const syncUser = Effect.fn("syncUser")(function*(userId: string) { + const profile = yield* fetchRemoteProfile(userId).pipe( + Effect.withSpan("fetchRemoteProfile") + ) + + return yield* persistProfile(profile).pipe( + Effect.withSpan("persistProfile") + ) +}) +``` + +### Pattern: framework boundary with runtime + +```ts +const runtime = ManagedRuntime.make(AppLayer) + +const handleRequest = (userId: string) => + runtime.runPromise(fetchUser(userId)) +``` + +## Good Repo Examples To Study + +- `./.repos/effect/packages/effect/src/Effect.ts` +- `./.repos/effect/packages/effect/src/Tracer.ts` +- `./.repos/effect/packages/effect/src/Logger.ts` +- `./.repos/effect/packages/effect/src/Metric.ts` +- `./.repos/effect/packages/opentelemetry/src/NodeSdk.ts` +- `./.repos/effect/packages/opentelemetry/src/Tracer.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-retries.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-retries.md new file mode 100644 index 000000000..3ef116fa6 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-retries.md @@ -0,0 +1,433 @@ +# Retries Guide + +This guide is based on retry patterns and `ExecutionPlan` usage in the vendored Effect repo. + +Key source files: + +- `./.repos/effect/packages/effect/src/Effect.ts` +- `./.repos/effect/packages/effect/src/Schedule.ts` +- `./.repos/effect/packages/effect/src/ExecutionPlan.ts` +- `./.repos/effect/packages/effect/test/Effect.test.ts` +- `./.repos/effect/packages/effect/test/ExecutionPlan.test.ts` + +Representative repo usage: + +- `./.repos/effect/packages/effect/src/unstable/workflow/Activity.ts` +- `./.repos/effect/packages/effect/src/unstable/workflow/WorkflowEngine.ts` +- `./.repos/effect/packages/effect/src/unstable/rpc/RpcClient.ts` +- `./.repos/effect/packages/effect/src/unstable/observability/OtlpExporter.ts` +- `./.repos/effect/packages/vitest/src/internal/internal.ts` + +## Mental Model + +Retries in Effect are not just loops. + +The repo uses three increasingly powerful levels: + +1. simple `Effect.retry` options for bounded or condition-based retries +2. `Schedule` for timing-aware retry policies +3. `ExecutionPlan` for fallback across different provided resources or layers + +Choose the smallest model that correctly expresses the retry policy. + +## Preferred Rule + +Prefer structured retry policies over ad hoc retry loops. + +Use: + +- simple `Effect.retry({ ... })` for straightforward conditions +- `Effect.retry(schedule)` when timing matters +- `ExecutionPlan` when retries should escalate across different layers or resources + +Avoid: + +- hand-written loops with mutable counters +- inline `catch` plus recursive retry logic +- resource fallback logic encoded as nested `catch` chains when `ExecutionPlan` is a better fit + +## `Effect.retry` + +`Effect.retry` is the main retry operator. + +The vendored tests show several important supported forms. + +## Retry On Success vs Failure + +`Effect.retry` only retries failures. + +If the effect succeeds, nothing is retried. + +This is explicitly covered in the module tests. + +## Simple Retry Options + +### `{ times: n }` + +Use this for the simplest bounded retry case. + +```ts +const retried = effect.pipe( + Effect.retry({ times: 3 }) +) +``` + +Use this when: + +- timing does not matter +- you only need a fixed retry count + +### `{ until: predicate }` + +Use `until` when retries should stop once the failure value satisfies a condition. + +The module tests show: + +- pure `until` +- effectful `until` +- that `until` is still evaluated at least once + +Example: + +```ts +const retried = effect.pipe( + Effect.retry({ until: (error) => error._tag === "Done" }) +) +``` + +### `{ while: predicate }` + +Use `while` when retries should continue only while the failure value satisfies a condition. + +The tests also show pure and effectful `while` variants. + +Example: + +```ts +const retried = effect.pipe( + Effect.retry({ while: (error) => error._tag === "Retryable" }) +) +``` + +## Retry With Schedule + +Use a `Schedule` whenever timing matters. + +```ts +const retried = effect.pipe( + Effect.retry(Schedule.recurs(3)) +) +``` + +Or with the richer object form: + +```ts +const retried = effect.pipe( + Effect.retry({ + schedule: Schedule.recurs(3), + while: (error) => error._tag === "Retryable" + }) +) +``` + +This is a very important repo pattern because it lets you combine: + +- retry timing +- retry limits +- retry predicates + +## Current Schedule Metadata During Retries + +The module tests show that retry execution updates `Schedule.CurrentMetadata`. + +This means retry policies and retry-aware effects can inspect: + +- attempt number +- elapsed time +- previous delay timing +- schedule output + +Use this when: + +- logging retry behavior +- building retry-aware diagnostics +- implementing advanced adaptive retry behavior + +## When To Use Simple Options Vs Schedule + +Prefer simple options when: + +- only the retry count matters +- retry timing does not matter +- the retry rule is just a condition on the error + +Prefer a schedule when: + +- retry timing matters +- backoff matters +- jitter matters +- the policy should evolve over time + +## Common Retry Schedules In The Repo + +The vendored repo repeatedly uses these patterns: + +### Fixed retry count + +```ts +Schedule.recurs(3) +``` + +### Exponential backoff + +```ts +Schedule.exponential(500, 1.5) +``` + +### Exponential plus steady fallback spacing + +```ts +Schedule.exponential(500, 1.5).pipe( + Schedule.either(Schedule.spaced(5000)) +) +``` + +This appears in production modules such as RPC and workflow code. + +### Error-sensitive delay policy + +```ts +Schedule.forever.pipe( + Schedule.addDelay((error) => Effect.succeed("1 second")) +) +``` + +The OTLP exporter uses this shape to derive delays from actual HTTP failure details such as rate limits. + +## Retry Only For Specific Failures + +The workflow `Activity` module shows an important advanced pattern: + +- sandbox the effect +- retry only when the `Cause` matches a specific retryable category +- fail or die differently once retries are exhausted + +Example shape from the repo: + +```ts +effect.pipe( + Effect.sandbox, + Effect.retry(policy), + Effect.catch((cause) => { + if (!Cause.hasInterrupts(cause)) { + return Effect.failCause(cause) + } + return Effect.die("interrupted and retries exhausted") + }) +) +``` + +Use this when: + +- retryability depends on the full cause, not just typed failures +- interrupt-specific retry behavior is required +- infrastructure policy is more nuanced than a simple tagged error rule + +## Retry Observability + +Retry logic should be observable. + +Good patterns: + +- keep retries inside named `Effect.fn` operations +- use `Schedule.CurrentMetadata` for diagnostics when needed +- log or annotate retry attempts at meaningful boundaries +- prefer central retry policies over duplicating timing logic everywhere + +Do not spread retry behavior across many small helpers where it becomes hard to see the operational policy. + +## `ExecutionPlan` + +Use `ExecutionPlan` when retries should escalate across different provided resources or layers. + +This is not just about retry timing. It is about retrying the same operation under different provided environments. + +The core use case from `ExecutionPlan.ts` is: + +- try one layer some number of times +- possibly with a schedule and conditions +- then fall back to another layer +- then possibly fall back again + +### What `ExecutionPlan` Solves + +`ExecutionPlan` is the right tool when: + +- the same effect should be retried against multiple alternative providers +- fallback should move across tiers, regions, models, or implementations +- retry policy includes both attempt counts and provider changes + +Examples: + +- fail over between multiple language model providers +- try one upstream cluster, then another +- fall back from a fast but unreliable service to a slower but more reliable one + +## `ExecutionPlan.make` + +Use `ExecutionPlan.make(...)` to define ordered retry/fallback steps. + +Each step can include: + +- `provide` +- `attempts` +- `while` +- `schedule` + +Example shape: + +```ts +const Plan = ExecutionPlan.make( + { + provide: FastLayer, + attempts: 2, + schedule: Schedule.spaced("3 seconds") + }, + { + provide: SafeLayer, + attempts: 3, + schedule: Schedule.spaced("1 second") + }, + { + provide: FinalFallbackLayer + } +) +``` + +### Step Semantics + +For each step: + +- `provide` is the context or layer to use +- `attempts` bounds how many times that step is tried +- `while` can stop retries for that step based on the input +- `schedule` defines the timing policy for retries within that step + +If `attempts` is omitted, the step attempts once unless a schedule is involved in a way that causes further retries. + +## `Effect.withExecutionPlan` And `Stream.withExecutionPlan` + +Use: + +- `Effect.withExecutionPlan` for effects +- `Stream.withExecutionPlan` for streams + +The vendored tests focus on `Stream.withExecutionPlan` and demonstrate: + +- fallback from one provider to another +- fallback after partial stream failure +- the ability to prevent fallback on partial streams + +This is a strong signal that `ExecutionPlan` is particularly useful for long-running or streaming integrations where failure can happen after partial success. + +## `ExecutionPlan.CurrentMetadata` + +`ExecutionPlan` exposes metadata with: + +- `attempt` +- `stepIndex` + +This is useful for: + +- diagnostics +- logging which fallback tier is being used +- understanding which plan step ultimately succeeded + +## `captureRequirements` + +`ExecutionPlan.captureRequirements` converts a plan with requirements into one whose requirements are satisfied from the current context. + +Use this when the plan should be frozen with the current environment before being applied later. + +## `ExecutionPlan.merge` + +Use `ExecutionPlan.merge(...)` when you need to concatenate multiple plans into one ordered plan. + +This is useful for assembling more complex fallback policies out of smaller ones. + +## When To Use `ExecutionPlan` Instead Of `Schedule` + +Use `Schedule` when: + +- only timing and retry conditions change +- the same environment/provider is used for every retry + +Use `ExecutionPlan` when: + +- the provider or layer should change across retry phases +- retries are tied to alternative resources, not just delays +- fallback is part of dependency provisioning strategy + +## Recommended Patterns + +### Pattern: simple bounded retry + +```ts +const retried = effect.pipe( + Effect.retry({ times: 3 }) +) +``` + +### Pattern: retryable-error backoff + +```ts +const retryPolicy = Schedule.exponential(500, 1.5).pipe( + Schedule.either(Schedule.spaced(5000)) +) + +const retried = effect.pipe( + Effect.retry({ + schedule: retryPolicy, + while: (error) => error._tag === "Retryable" + }) +) +``` + +### Pattern: fallback across providers + +```ts +const Plan = ExecutionPlan.make( + { + provide: PrimaryLayer, + attempts: 2, + schedule: Schedule.spaced("1 second") + }, + { + provide: SecondaryLayer, + attempts: 3, + schedule: Schedule.exponential(500, 1.5) + }, + { + provide: FinalFallbackLayer + } +) +``` + +## Anti-Patterns + +- hand-writing retry recursion instead of using `Effect.retry` +- embedding sleep and counters directly in business logic +- using `ExecutionPlan` when a simple `Schedule` is enough +- encoding provider fallback as a maze of nested `catch` branches +- retrying indiscriminately without checking whether the failure is actually retryable + +## Good Repo Examples To Study + +- `./.repos/effect/packages/effect/test/Effect.test.ts` +- `./.repos/effect/packages/effect/src/Schedule.ts` +- `./.repos/effect/packages/effect/src/ExecutionPlan.ts` +- `./.repos/effect/packages/effect/test/ExecutionPlan.test.ts` +- `./.repos/effect/packages/effect/src/unstable/workflow/Activity.ts` +- `./.repos/effect/packages/effect/src/unstable/workflow/WorkflowEngine.ts` +- `./.repos/effect/packages/effect/src/unstable/rpc/RpcClient.ts` +- `./.repos/effect/packages/effect/src/unstable/observability/OtlpExporter.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-schedule.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-schedule.md new file mode 100644 index 000000000..2f44ca0d4 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-schedule.md @@ -0,0 +1,379 @@ +# Schedule Guide + +This guide is based on the vendored `Schedule` module and its usage across `./.repos/effect`. + +Key source files: + +- `./.repos/effect/packages/effect/src/Schedule.ts` +- `./.repos/effect/packages/effect/test/Schedule.test.ts` + +Representative repo usage: + +- `./.repos/effect/packages/effect/src/unstable/workflow/WorkflowEngine.ts` +- `./.repos/effect/packages/effect/src/unstable/rpc/RpcClient.ts` +- `./.repos/effect/packages/effect/src/unstable/observability/OtlpExporter.ts` +- `./.repos/effect/packages/vitest/src/internal/internal.ts` + +## Mental Model + +`Schedule` is the standard Effect abstraction for: + +- retries +- repeats +- polling +- backoff +- cadence and timing policies + +A schedule describes when the next step should happen and when execution should stop. + +The repo uses schedules heavily for: + +- retry policies +- recurring work +- time-window alignment +- cron-based triggering +- bounded flaky test retries + +## Preferred Rule + +When the timing behavior of an effect matters, prefer expressing it with `Schedule` rather than ad hoc loops, counters, sleeps, or manual retry recursion. + +Prefer: + +- `Effect.retry(schedule)` +- `Effect.repeat(schedule)` +- `Effect.schedule(schedule)` +- `Stream.fromSchedule(schedule)` + +Over: + +- custom retry loops with mutable counters +- `Effect.forever` plus hand-written `Effect.sleep` +- manual backoff code scattered across business logic + +## Common Repo Patterns + +The most common patterns in the vendored repo are: + +1. `Schedule.recurs(n)` for bounded retries or repeats +2. `Schedule.spaced(...)` for simple fixed spacing +3. `Schedule.fixed(...)` or `Schedule.windowed(...)` for interval-aligned work +4. `Schedule.exponential(...)` for backoff +5. `Schedule.either(...)` or sequencing combinators to combine retry policies +6. `Schedule.while(...)` to stop based on metadata or input +7. `Schedule.addDelay(...)` for custom backoff logic +8. `Schedule.jittered(...)` to avoid retry stampedes + +## Retry Vs Repeat + +Use schedules with the right operator: + +- `Effect.retry(schedule)` for failures +- `Effect.repeat(schedule)` for successes +- `Effect.schedule(schedule)` when you want to delay/reschedule an effect directly + +Good rule of thumb: + +- failing workflow: `retry` +- recurring successful workflow: `repeat` +- one effect that should run on a cadence: `schedule` + +## Core Constructors + +### `Schedule.recurs` + +Use `recurs(n)` for a bounded number of additional runs. + +```ts +const retryPolicy = Schedule.recurs(3) +``` + +This is one of the most common repo retry policies. + +### `Schedule.forever` + +Use `forever` when the schedule should never terminate on its own. + +```ts +const retryForever = Schedule.forever +``` + +This is common in infrastructure code and long-running retry loops. + +### `Schedule.spaced` + +Use `spaced(duration)` for simple constant spacing. + +```ts +const pollEverySecond = Schedule.spaced("1 second") +``` + +This is the most straightforward schedule for polling or retry spacing. + +### `Schedule.fixed` + +Use `fixed(duration)` when work should align to fixed interval boundaries. + +The tests show that this differs from simple spacing when the action itself takes time. + +Use it when interval alignment matters more than naïve spacing. + +### `Schedule.windowed` + +Use `windowed(duration)` when you want delays to align to the nearest window boundary. + +This is useful for periodic flush or batching behavior. + +### `Schedule.duration` + +Use `duration(duration)` for a one-shot delay schedule. + +```ts +const onceAfterOneSecond = Schedule.duration("1 second") +``` + +### `Schedule.cron` + +Use `cron(...)` for calendar-based scheduling. + +The module tests cover: + +- minute-level cron +- second-level cron +- calendar matching for specific weekdays and month days + +Use this for: + +- jobs that should follow wall-clock time +- operational schedules +- calendar-driven execution + +## Backoff Patterns + +### `Schedule.exponential` + +Use `exponential(base, factor)` for retry backoff. + +This is a dominant repo pattern. + +Examples from production code: + +- `WorkflowEngine` +- `RpcClient` +- persistence and eventlog modules + +Typical pattern: + +```ts +const retryPolicy = Schedule.exponential(500, 1.5) +``` + +Use this for: + +- network retries +- external system retries +- contention or lock retries + +### Combine exponential with a cap or fallback spacing + +The repo often combines exponential backoff with a more stable spaced fallback using `Schedule.either(...)`. + +Example pattern from production modules: + +```ts +const retryPolicy = Schedule.exponential(500, 1.5).pipe( + Schedule.either(Schedule.spaced(5000)) +) +``` + +This keeps early retries responsive without letting delays grow without bound. + +### `Schedule.jittered` + +Use `jittered(...)` to randomize timing within safe bounds. + +The module tests verify jittered delays remain within a bounded percentage of the original schedule. + +Use it when: + +- many workers or clients may retry at once +- you want to avoid synchronized retry storms +- the system would suffer from coordinated polling spikes + +## Combinators + +### `Schedule.while` + +Use `while(...)` to continue only while a predicate on schedule metadata holds. + +The repo uses this for: + +- stopping after a number of attempts +- filtering retries based on the input error or cause +- bounding retry windows by elapsed time + +Example pattern: + +```ts +const bounded = Schedule.spaced("1 second").pipe( + Schedule.while(({ attempt }) => Effect.succeed(attempt <= 5)) +) +``` + +### `Schedule.andThenResult` + +Use `andThenResult(left, right)` when one schedule should run to completion and then another should take over. + +The module tests show this clearly. + +Use this when: + +- you want an initial aggressive policy followed by a slower steady-state policy +- you want phase-based retry or repeat timing + +### `Schedule.either` + +Use `either` to combine two schedules so both policies influence the resulting timing. + +This appears frequently in repo retry policies that combine exponential growth with a stable fallback cadence. + +### `Schedule.addDelay` + +Use `addDelay` when the delay should depend on the schedule input or output. + +This is a strong fit for custom retry behavior based on the actual error. + +The OTLP exporter uses this style to honor `retry-after` behavior and otherwise fall back to a default delay. + +Example shape: + +```ts +const policy = Schedule.forever.pipe( + Schedule.addDelay((error) => + Effect.succeed("1 second") + ) +) +``` + +Use this when: + +- the delay should depend on the error +- upstream metadata such as rate-limit headers matters +- you need custom backoff without leaving the Schedule model + +## Metadata + +Schedules expose rich metadata including: + +- input +- attempt count +- start time +- current time +- elapsed time +- elapsed since previous run +- output +- duration + +This is one of the reasons schedules are better than ad hoc retry loops. + +Use metadata when: + +- retry behavior depends on the input error +- stop conditions depend on elapsed time +- you want to log or collect retry state + +## Collection And Inspection Helpers + +The module tests highlight several useful helpers: + +- `Schedule.collectInputs(...)` +- `Schedule.collectOutputs(...)` +- `Schedule.collectWhile(...)` +- `Schedule.delays(...)` +- `Schedule.reduce(...)` + +Use these when: + +- you need to inspect or test a schedule +- you want to accumulate state across schedule steps +- you are building a more specialized scheduling policy + +These are especially useful in tests and low-level policy construction. + +## Typical Policies + +### Simple bounded retry + +```ts +const retryPolicy = Schedule.recurs(3) +``` + +### Spaced polling + +```ts +const pollPolicy = Schedule.spaced("5 seconds") +``` + +### Exponential retry with a stable fallback cadence + +```ts +const retryPolicy = Schedule.exponential(500, 1.5).pipe( + Schedule.either(Schedule.spaced("5 seconds")) +) +``` + +### Retry forever with custom delay logic + +```ts +const retryPolicy = Schedule.forever.pipe( + Schedule.addDelay((error) => Effect.succeed("1 second")) +) +``` + +### Cron-driven recurring job + +```ts +const nightly = Schedule.cron("0 0 * * *") +``` + +## Testing Schedules + +The repo tests schedules with `TestClock` and controlled stepping. + +Preferred pattern: + +- use `TestClock` +- advance time explicitly +- inspect emitted delays or outputs + +This keeps schedule tests deterministic. + +The module tests also use helpers built on `Schedule.toStepWithSleep(...)` to inspect schedule behavior precisely. + +## Best Practices + +1. Prefer `Schedule` over ad hoc retry loops. +2. Prefer `Schedule.recurs(...)` for simple bounded retries. +3. Prefer `Schedule.exponential(...)` for backoff. +4. Prefer `Schedule.either(...)` or sequencing combinators to compose retry phases. +5. Prefer `Schedule.addDelay(...)` when delay depends on the actual error. +6. Prefer `Schedule.jittered(...)` for distributed retry behavior. +7. Prefer metadata-driven stop conditions over mutable counters. +8. Prefer testing schedules with `TestClock`. + +## Anti-Patterns + +- hand-writing retry loops with mutable counters and sleeps +- putting backoff logic directly in business code instead of in a schedule +- using `Effect.forever` with embedded `sleep` as a substitute for a schedule +- scattering retry timing logic across many call sites +- ignoring jitter when many clients or workers retry concurrently + +## Good Repo Examples To Study + +- `./.repos/effect/packages/effect/src/Schedule.ts` +- `./.repos/effect/packages/effect/test/Schedule.test.ts` +- `./.repos/effect/packages/effect/src/unstable/workflow/WorkflowEngine.ts` +- `./.repos/effect/packages/effect/src/unstable/rpc/RpcClient.ts` +- `./.repos/effect/packages/effect/src/unstable/observability/OtlpExporter.ts` +- `./.repos/effect/packages/vitest/src/internal/internal.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-schema.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-schema.md new file mode 100644 index 000000000..3d45cfa84 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-schema.md @@ -0,0 +1,624 @@ +# Schema Guide + +This guide is based on the vendored Schema module and common repo usage in `./.repos/effect`. + +Key source files: + +- `./.repos/effect/packages/effect/src/Schema.ts` +- `./.repos/effect/packages/effect/src/SchemaTransformation.ts` +- `./.repos/effect/packages/effect/src/SchemaGetter.ts` +- `./.repos/effect/packages/effect/src/SchemaIssue.ts` +- `./.repos/effect/packages/effect/src/JsonSchema.ts` + +Representative repo usage: + +- `./.repos/effect/packages/tools/ai-codegen/src/Config.ts` +- `./.repos/effect/packages/platform-node/test/fixtures/rpc-schemas.ts` +- `./.repos/effect/packages/platform-browser/test/IndexedDbQueryBuilder.test.ts` +- `./.repos/effect/packages/tools/openapi-generator/` + +## Mental Model + +Schema is the standard way to: + +- define data shapes +- validate unknown input +- encode typed values back to serialized form +- transform between encoded and decoded representations +- attach metadata and constraints + +The repo uses Schema pervasively for: + +- protocol payloads +- configuration +- HTTP and RPC contracts +- database row decoding +- error types +- derived tooling such as JSON Schema and arbitrary generation + +## Preferred Rule + +Prefer Schema-based types whenever data crosses a boundary or should be validated, transformed, documented, or encoded. + +Typical boundaries: + +- HTTP requests and responses +- RPC payloads +- database rows +- config files +- worker messages +- persisted data +- domain errors + +## What A Schema Actually Is + +A schema is not just a static shape. + +It is a contract between: + +- the decoded in-memory value you want to work with +- the encoded representation that comes from or goes to some boundary + +This is the most important thing many implementations get wrong. + +Do not think of Schema as “a typed struct definition.” +Think of it as: + +- validation +- decoding +- encoding +- transformation +- metadata +- reuse across boundaries + +Because of that, schemas should not be duplicated unless there is a real semantic difference. + +If two schemas describe the same logical model but differ only because one boundary encodes a field differently, prefer one schema with transformations instead of two parallel schemas. + +## Avoid Duplicating Schemas + +Do not create multiple parallel schemas for the same logical entity unless they truly represent different models. + +Bad pattern: + +```ts +const Todo = Schema.Struct({ + id: Schema.Number, + title: Schema.String, + completed: Schema.Boolean +}) + +const TodoSql = Schema.Struct({ + id: Schema.Number, + title: Schema.String, + completed: Schema.BooleanFromBit +}) +``` + +This is usually a sign that transformations are not being used properly. + +If the model is still “Todo”, do not define a second schema just because one boundary stores `completed` as a bit. + +Prefer deriving or transforming the representation instead. + +Why duplication is bad: + +- the same model is now maintained in multiple places +- fields drift over time +- boundary logic gets copied instead of centralized +- refactors become error-prone + +Only duplicate schemas when there is a real semantic difference, for example: + +- a creation payload really is a different model from a persisted entity +- a public API contract intentionally differs from an internal domain model +- a projection or partial view is intentionally a different type + +If the difference is only encoding, use a transformation. + +## Prefer `Class` Variants Over `Struct` Variants When Possible + +When a schema represents a named domain model, reusable payload, or long-lived API shape, prefer `Schema.Class`, `Schema.TaggedClass`, or `Schema.TaggedErrorClass` over a bare `Schema.Struct`. + +Prefer: + +```ts +import { Schema } from "effect" + +export class User extends Schema.Class("User")({ + id: Schema.String, + name: Schema.String +}) {} +``` + +Over: + +```ts +import { Schema } from "effect" + +export const User = Schema.Struct({ + id: Schema.String, + name: Schema.String +}) +``` + +Why `Class` variants are usually better: + +- the schema has a stable, named identity +- reusable models are easier to recognize in code and traces +- constructors and validation are packaged together +- extension patterns are clearer +- named schemas read better in contracts and tooling output + +Use `Struct` when: + +- the shape is local and anonymous +- it is a small inline request or response shape +- introducing a class would add unnecessary ceremony +- the schema is primarily a one-off composition fragment + +Good rule of thumb: + +- reusable named model: `Class` +- reusable tagged union member: `TaggedClass` +- reusable error payload: `TaggedErrorClass` +- small inline object shape: `Struct` + +## One Logical Model, Multiple Representations + +The right Schema mindset is: + +- one logical model +- multiple encoded forms when needed +- transformations connecting them + +For example, a `Todo` may be: + +- a boolean in memory +- a bit in SQL +- a string in some external API + +That does not automatically mean you need three separate top-level schemas. + +Prefer: + +- one main schema for the logical model +- transformed field schemas or transformed object schemas for boundary-specific encoding +- derived request/result schemas when the shape is actually different + +## Common Schema Building Blocks + +Common primitives and collections used throughout the repo: + +- `Schema.String` +- `Schema.Number` +- `Schema.Boolean` +- `Schema.BigInt` +- `Schema.Array(...)` +- `Schema.Record(key, value)` +- `Schema.Tuple([...])` +- `Schema.Struct({...})` +- `Schema.Union([...])` + +Example: + +```ts +const Todo = Schema.Struct({ + id: Schema.Number, + title: Schema.String, + completed: Schema.Boolean +}) +``` + +## `Class`, `TaggedClass`, and `TaggedErrorClass` + +### `Schema.Class` + +Use for named reusable schema-backed models. + +```ts +class Product extends Schema.Class("Product")({ + id: Schema.String, + price: Schema.Number +}) {} +``` + +### Constructor Rule + +When constructing schema classes, prefer `X.make(...)` over `new X(...)`. + +Prefer: + +```ts +const todo = Todo.make({ + id: 1, + title: "write docs", + completed: false +}) +``` + +Over: + +```ts +const todo = new Todo({ + id: 1, + title: "write docs", + completed: false +}) +``` + +Why: + +- it is the intended schema-class construction style +- it makes schema-backed construction explicit +- it keeps the codebase consistent +- it reads better across `Class`, `TaggedClass`, and `TaggedErrorClass` + +Use this rule consistently for: + +- `Schema.Class` +- `Schema.TaggedClass` +- `Schema.TaggedErrorClass` + +### `Schema.TaggedClass` + +Use for members of tagged unions. + +```ts +class Circle extends Schema.TaggedClass()("Circle", { + radius: Schema.Number +}) {} + +class Rectangle extends Schema.TaggedClass()("Rectangle", { + width: Schema.Number, + height: Schema.Number +}) {} +``` + +### `Schema.TaggedErrorClass` + +Use for schema-backed typed errors. + +```ts +class NotFound extends Schema.TaggedErrorClass()("NotFound", { + id: Schema.String +}) {} +``` + +## Optional Fields + +Be precise about optionality. + +Important rule from the vendored docs: + +- `Schema.optional(schema)` means `T | undefined` +- `Schema.optionalKey(schema)` means an exact optional property in a struct + +Prefer `optionalKey` for object fields. + +Prefer: + +```ts +const Query = Schema.Struct({ + search: Schema.optionalKey(Schema.String) +}) +``` + +Use `optional` when the value itself should be `A | undefined`, not just an omitted field. + +## Unions + +Use `Schema.Union([...])` for ordinary unions. + +```ts +const Id = Schema.Union([ + Schema.String, + Schema.Number +]) +``` + +Prefer tagged unions for domain variants. + +```ts +class Created extends Schema.TaggedClass()("Created", { + id: Schema.String +}) {} + +class Deleted extends Schema.TaggedClass()("Deleted", { + id: Schema.String +}) {} + +const TodoEvent = Schema.Union([Created, Deleted]) +``` + +Why: + +- decoding and branching are clearer +- `_tag`-based matching aligns with Effect code style + +## Recursive Schemas + +Use `Schema.suspend` for recursive schemas. + +```ts +type Tree = { + readonly name: string + readonly children: ReadonlyArray +} + +const Tree: Schema.Schema = Schema.Struct({ + name: Schema.String, + children: Schema.Array(Schema.suspend((): Schema.Schema => Tree)) +}) +``` + +Use it whenever a schema refers to itself, directly or indirectly. + +Without `suspend`, recursive definitions will not work correctly. + +## Transformations + +Transformations are one of the most important Schema features. + +Use them when decoded and encoded shapes differ. + +This is the main tool that avoids needless schema duplication. + +If your instinct is “I need another schema because this boundary encodes the same value differently”, stop and first ask whether this should be one schema with a transformation instead. + +### `Schema.decodeTo` + +Use `decodeTo` when you want one schema to decode into another schema's type. + +```ts +const TrimmedString = Schema.String.pipe( + Schema.decodeTo(Schema.String, { + decode: (value) => value.trim(), + encode: (value) => value + }) +) +``` + +The vendored docs explicitly note that `decodeTo` is curried and should be used with `pipe`. + +### `Schema.encodeTo` + +Use `encodeTo` when the reverse direction reads more clearly. + +### `SchemaTransformation.transformOrFail` + +Use `transformOrFail` when the transformation itself is effectful or may fail. + +```ts +import * as Effect from "effect/Effect" +import * as SchemaTransformation from "effect/SchemaTransformation" + +const VerifiedString = Schema.String.pipe( + Schema.decodeTo( + Schema.String, + SchemaTransformation.transformOrFail({ + decode: (value) => Effect.succeed(value.trim()), + encode: (value) => Effect.succeed(value) + }) + ) +) +``` + +Use this when: + +- validation depends on services or effects +- decoding can fail with structured issues +- encoding also needs logic beyond identity + +## Field-Level Transformations + +Very often, the right answer is not a second object schema but a transformed field schema. + +Example shape: + +```ts +const Completed = Schema.BooleanFromBit + +const Todo = Schema.Struct({ + id: Schema.Number, + title: Schema.String, + completed: Completed +}) +``` + +In this pattern: + +- the logical model still has `completed: boolean` +- the encoded SQL-facing representation can still be a bit +- the transformation lives at the field where it belongs + +This is usually better than defining `Todo` and `TodoSql` as separate object schemas. + +## Object-Level Transformations + +Use object-level transformations when the whole object encoding differs, not just one field. + +Good use cases: + +- external keys differ from internal keys +- several fields need coordinated transformation +- the encoded shape is a structurally different representation of the same model + +Still prefer a single logical schema plus a transformation pipeline over maintaining multiple duplicated top-level schemas. + +## Rename Keys + +Schema supports key renaming through struct transformations. + +The vendored `Schema.ts` implements key renaming by mapping fields and using decode/encode transformations with renamed key maps. + +Use key renaming when: + +- external payload keys differ from internal keys +- you want stable internal names while honoring external contract names + +Preferred pattern: + +- keep the internal decoded shape idiomatic +- use schema-level transformation or field-mapping to adapt external keys + +This is another example of avoiding duplication. If the only difference is key naming, do not define a second schema just to rename fields manually later. + +In practice, use struct field mapping helpers and transformation composition rather than manual post-parse object rewriting. + +## Opaque And Branded Types + +Use opaque or branded schemas when a value should stay distinct from its structural base type. + +### `Schema.brand` + +Use `brand` for refined nominal distinctions. + +```ts +const UserId = Schema.String.pipe( + Schema.brand("UserId") +) +``` + +This is useful for: + +- IDs +- validated domain scalars +- preventing accidental interchange of same-shaped values + +### `Schema.Opaque` + +Use `Opaque` when you want an opaque schema-backed type with the same structure as its underlying schema. + +This is especially useful when the type should remain distinct at the type level without changing its runtime shape. + +## Picking, Omitting, Partial Shapes, And Mutability + +Common struct operations include: + +- `pick` +- `omit` +- `partial` +- `mutable` + +Use them to derive variations instead of redefining near-identical schemas manually. + +Good examples: + +- request subset from a domain model +- patch/update payloads +- mutable representations for specific adapters + +Prefer deriving from one source schema rather than maintaining parallel copies. + +This is the second major tool for avoiding duplication: + +- use transformations when encoded and decoded representations differ +- use derivation when one schema is a subset, superset, or variation of another + +## Constraints And Validation + +Use schema checks and filters for validation. + +Examples from the module docs include: + +- `isMinLength` +- `isGreaterThan` +- `isPattern` +- `isUUID` + +Attach them with `.check(...)`. + +Use this when: + +- the validation is intrinsic to the schema +- the rule belongs to the data contract + +For business-rule validation that depends on services or current state, prefer effectful logic outside the schema or use effectful transformations. + +## Decoding And Encoding + +Common operations: + +- `Schema.decodeUnknownSync` +- `Schema.decodeUnknownEffect` +- `Schema.decodeUnknownExit` +- `Schema.encodeUnknownSync` +- `Schema.encodeUnknownEffect` + +Preferred rule: + +- use `decodeUnknownEffect` and `encodeUnknownEffect` in Effect code +- avoid throwing sync decode APIs in application flows unless you are intentionally at a sync boundary + +Good pattern: + +```ts +const decodeUser = Schema.decodeUnknownEffect(User) +``` + +## Schema Metadata And Derived Tooling + +Schema is also used for: + +- annotations and documentation metadata +- JSON Schema generation +- arbitrary generation for tests +- derived equivalence + +Useful operations from the module docs: + +- `.annotate(...)` +- `Schema.toJsonSchemaDocument(...)` +- `Schema.toArbitrary(...)` +- `Schema.toEquivalence(...)` + +Use annotations when the schema participates in: + +- API docs +- codegen +- contract generation + +## Common Repo Patterns + +Patterns visible in the vendored repo: + +- `Schema.Class` for named reusable contract types +- `Schema.Struct` for inline shapes and anonymous fragments +- `Schema.Union` for alternative payloads +- `Schema.optionalKey` for request/query/body optional fields +- `Schema.suspend` for recursive generated schemas +- `Schema.decodeTo` and `transformOrFail` for non-trivial decode/encode logic +- `Schema.TaggedErrorClass` for typed error payloads + +## Best Practices + +1. Prefer `Class` variants over plain `Struct` for named reusable schemas. +2. Prefer tagged variants for unions and errors. +3. Prefer `optionalKey` for optional object properties. +4. Do not duplicate schemas unless there is a real semantic difference. +5. Prefer schema-level transformations over ad hoc post-parse object rewriting. +6. Prefer deriving schema variants with `pick`, `omit`, `partial`, and `mutable` instead of duplicating definitions. +7. Prefer field-level transformations when only a field encoding differs. +8. Prefer branded or opaque types for important domain identifiers. +9. Prefer `decodeUnknownEffect` in application code. +10. Keep internal decoded shapes idiomatic and use schema transforms for external representation differences. + +## Anti-Patterns + +- using plain `Struct` for every reusable domain model even when `Class` would give a clearer named type +- duplicating whole schemas when only one field encoding differs +- creating `Foo` and `FooSql` schemas for the same logical model when a transformation would do +- using `optional` when you actually want an optional key +- duplicating near-identical schemas instead of deriving variants +- rewriting keys manually after decode instead of using schema transformations +- hand-validating external data after decode when the constraint belongs in the schema +- exposing unvalidated external payloads deep into business logic + +## Good Repo Examples To Study + +- `./.repos/effect/packages/tools/ai-codegen/src/Config.ts` +- `./.repos/effect/packages/platform-node/test/fixtures/rpc-schemas.ts` +- `./.repos/effect/packages/platform-browser/test/IndexedDbQueryBuilder.test.ts` +- `./.repos/effect/packages/tools/openapi-generator/src/JsonSchemaGenerator.ts` +- `./.repos/effect/packages/effect/src/Schema.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-sql.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-sql.md new file mode 100644 index 000000000..ccaa8d0a8 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-sql.md @@ -0,0 +1,536 @@ +# SQL Guide + +This guide is based on the vendored Effect SQL modules in `./.repos/effect`. + +Key source files: + +- `./.repos/effect/packages/effect/src/unstable/sql/SqlClient.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/Migrator.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlResolver.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlSchema.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlModel.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlError.ts` + +## Preferred Rule + +When a project uses Effect, prefer the Effect SQL modules over directly coupling business code to a native SQL driver API. + +Prefer: + +- `effect/unstable/sql/SqlClient` +- `effect/unstable/sql/Migrator` +- `effect/unstable/sql/SqlResolver` +- `effect/unstable/sql/SqlSchema` +- `effect/unstable/sql/SqlModel` + +Over: + +- embedding raw driver calls directly in business services +- hand-rolling transactions in service methods +- ad hoc migration scripts disconnected from the Effect runtime + +Why: + +- transactions are integrated into the Effect model +- spans and SQL observability are built in +- SQL errors stay typed and consistent +- schema decoding and request resolution compose better +- layers and services stay portable across runtimes and tests + +## Mental Model + +The Effect SQL stack is organized around: + +- `SqlClient` as the main database capability +- `withTransaction` for transaction boundaries +- `SqlResolver` for request-style batched and validated access +- `SqlSchema` and `SqlModel` for schema-aware query/model patterns +- `Migrator` for managed migrations + +The business layer should depend on Effect SQL abstractions, not on a raw driver object. + +## `SqlClient` + +`SqlClient` is the primary service for executing SQL. + +Repo reference: + +- `./.repos/effect/packages/effect/src/unstable/sql/SqlClient.ts` + +Use it when: + +- you need to execute queries +- you need transactions +- you want SQL operations to participate in Effect spans and context + +Important capabilities from the repo: + +- transaction support with `withTransaction` +- connection reservation with `reserve` +- reactive queries +- integration with transaction-scoped context + +## Typed Queries + +Prefer typed SQL queries instead of leaving row shapes implicit. + +The repo uses typed query literals like: + +```ts +const rows = yield* sql<{ id: number; name: string }>`SELECT * FROM test` +``` + +This is the first level of typed SQL usage and is already better than untyped row access. + +Use typed query literals when: + +- the row shape is small and obvious +- the query is local and does not justify a reusable schema +- you want immediate row typing without introducing extra helpers + +Avoid: + +- leaving query results untyped and then recovering shape with unsafe assertions +- using `as` on rows after query execution + +## Schema Integration + +Prefer integrating SQL with Schema whenever the row shape matters or the query result crosses a meaningful boundary. + +Why: + +- Schema validates the shape rather than trusting the database blindly +- row decoding stays explicit and typed +- the same schema can often be reused for transport, domain, or contract layers +- this avoids unsafe row assertions such as `as TodoRow` + +### Prefer schema-decoded results over `as`-based rows + +Avoid this pattern: + +```ts +const row = yield* sql`SELECT id, title FROM todos WHERE id = ${id}` +const todo = row[0] as TodoRow +``` + +Prefer: + +- a typed SQL query plus Schema decoding +- or a SQL helper such as `SqlResolver`, `SqlSchema`, or `SqlModel` when appropriate + +Example boundary decode: + +```ts +const TodoRow = Schema.Struct({ + id: Schema.Number, + title: Schema.String, + completed: Schema.Boolean +}) + +const decodeTodoRows = Schema.decodeUnknownEffect(Schema.Array(TodoRow)) +``` + +Then keep the query and decoding together in one SQL-aware operation. + +### `SqlResolver` + Schema + +`SqlResolver` is one of the clearest examples of SQL and Schema integration in the vendored repo. + +It uses: + +- a `Request` schema for validating resolver input +- a `Result` schema for validating query output + +Example shape from the repo tests: + +```ts +const resolver = SqlResolver.findById({ + Id: Schema.Number, + Result: Schema.Struct({ + id: Schema.Number, + name: Schema.String + }), + ResultId: (row) => row.id, + execute: (ids) => sql`SELECT * FROM test WHERE id IN ${sql.in(ids)}` +}) +``` + +This is a preferred pattern when: + +- multiple IDs are fetched together +- request batching is useful +- the query contract should be schema-validated + +### `SqlSchema` and `SqlModel` + +Use `SqlSchema` and `SqlModel` when you want tighter schema integration with SQL itself. + +They are preferred over hand-written row types when: + +- the row shape is central to the module +- you want reusable schema-aware model logic +- manual row mapping is becoming repetitive + +## Prefer SQL Services Over Native Driver Services + +Avoid this pattern: + +```ts +class TodoService extends Context.Service()("TodoService") { + static readonly layer = Layer.effect(this)( + Effect.acquireRelease( + Effect.try({ try: () => new Database("todos.sqlite") }) + ) + ) +} +``` + +Why this is usually a bad pattern in an Effect codebase: + +- the service is now tightly coupled to one runtime-specific database client +- you lose the shared SQL abstraction that the Effect repo already provides +- transaction and query conventions become ad hoc +- it is easier to drift away from typed SQL errors, SQL tracing, and reusable query helpers + +Prefer a layer that provides `SqlClient`, and let domain services depend on that. + +## Domain Services Should Depend On `SqlClient` + +Good pattern: + +```ts +import * as Context from "effect/Context" +import * as Effect from "effect/Effect" +import * as SqlClient from "effect/unstable/sql/SqlClient" + +class TodoRepo extends Context.Service()("TodoRepo", { + make: Effect.succeed({ + getById: Effect.fn("TodoRepo.getById")(function*(id: number) { + const sql = yield* SqlClient.SqlClient + return yield* sql`SELECT id, title, completed FROM todos WHERE id = ${id}` + }) + }) +}) {} +``` + +Why this is better: + +- the domain service depends on an Effect SQL capability +- SQL stays observable and transactional +- the SQL client implementation can be provided separately from the business service + +## Transactions + +Use `SqlClient.withTransaction` for transaction boundaries. + +Repo reference: + +- `./.repos/effect/packages/effect/src/unstable/sql/SqlClient.ts` + +Prefer: + +```ts +const createAndAudit = Effect.fn("TodoRepo.createAndAudit")(function*(title: string) { + const sql = yield* SqlClient.SqlClient + + return yield* sql.withTransaction( + Effect.gen(function*() { + yield* sql`INSERT INTO todos ${sql.insert({ title, completed: false })}` + yield* sql`INSERT INTO audit_log ${sql.insert({ event: "todo_created" })}` + }) + ) +}) +``` + +Avoid: + +- manual `BEGIN` / `COMMIT` / `ROLLBACK` in application service code +- driver-specific transaction logic spread across multiple service methods + +## Query Composition + +Prefer keeping query logic inside SQL-aware services or repositories. + +Good patterns: + +- keep queries close to the service that owns the behavior +- use named business operations for multi-step workflows +- use small local helpers only when they improve clarity + +Avoid exposing one exported accessor function per SQL service method if it only forwards to the service. + +Bad: + +```ts +export const createTodo = Effect.fn(function*(title: string) { + const todos = yield* TodoRepo + return yield* todos.create(title) +}) +``` + +Prefer: + +- use the service method directly within the owning workflow +- or export a real business operation that adds behavior beyond simple forwarding + +## SQL Resolvers + +Use `SqlResolver` when request-style batching or schema-validated request/response handling is useful. + +Repo reference: + +- `./.repos/effect/packages/effect/src/unstable/sql/SqlResolver.ts` + +It is especially good for: + +- batched lookup patterns +- `findById`-style resolvers +- grouped query resolution +- integrating SQL with request batching patterns + +Important repo pattern: + +- request schema validates inputs +- result schema validates outputs +- execution remains effectful and transactional + +This is one of the strongest typed-query patterns in the repo and should be preferred over ad hoc batched row mapping when the query fits the resolver model. + +## SQL Schemas And Models + +When schema-aware SQL helpers fit the task, prefer them over hand-mapped rows. + +Look at: + +- `SqlSchema` +- `SqlModel` + +These are good fits when: + +- the row shape matters strongly +- schema-based decode/encode should stay aligned with database access +- you want to reduce ad hoc row mapping logic + +Avoid overusing manual `type Row = { ... }` plus custom conversion if the schema-aware modules already express the shape clearly. + +Preferred order for query typing: + +1. schema-aware SQL module such as `SqlResolver`, `SqlSchema`, or `SqlModel` when it fits +2. typed query literals plus Schema decoding when the query is local +3. only use manual row mapping when the first two options are clearly heavier than the problem + +## Migrations + +Use `Migrator` for migrations. + +Repo reference: + +- `./.repos/effect/packages/effect/src/unstable/sql/Migrator.ts` + +The repo shows these best practices: + +- maintain a dedicated migrations table +- load migrations through a managed loader +- run migrations through the Effect runtime +- keep migration execution observable with logs and spans +- use SQL client transaction and locking semantics handled by the migrator + +Important behavior in the vendored repo: + +- migrations table creation is dialect-aware +- duplicate migration IDs are detected +- concurrent migration runs are guarded +- each migration is logged and wrapped in a span + +### Concrete Migration Loader Example + +The runtime-specific SQL migrator packages expose `fromRecord(...)` to define migrations from an ordered record. + +Example shape: + +```ts +import * as SqliteMigrator from "@effect/sql-sqlite-bun/SqliteMigrator" +import * as Effect from "effect/Effect" + +const migrations = SqliteMigrator.fromRecord({ + "1_create_todos": Effect.gen(function*() { + yield* sql` + CREATE TABLE todos ( + id INTEGER PRIMARY KEY NOT NULL, + title TEXT NOT NULL, + completed INTEGER NOT NULL DEFAULT 0 + ) + `.withoutTransform + }), + "2_add_todo_index": Effect.gen(function*() { + yield* sql` + CREATE INDEX todos_completed_idx ON todos (completed) + `.withoutTransform + }) +}) +``` + +This matches the model used by the vendored migrator implementation: + +- each migration has a numeric prefix and descriptive name +- each migration resolves to an `Effect` +- migrations are ordered by ID + +### Running Migrations Directly + +Use the runtime-specific `run(...)` helper when you want a startup effect that runs migrations explicitly. + +Example shape: + +```ts +import * as SqliteMigrator from "@effect/sql-sqlite-bun/SqliteMigrator" + +const runMigrations = SqliteMigrator.run({ + loader: migrations +}) +``` + +This is a good fit when: + +- startup explicitly runs migrations before launching the main app +- deployment tooling runs migrations as a separate command +- you want migration results as an ordinary `Effect` + +The return value includes the applied migration IDs and names. + +### Running Migrations As A Layer + +Use the runtime-specific `layer(...)` helper when migrations should run as part of top-level infrastructure setup. + +Example shape: + +```ts +import * as SqliteMigrator from "@effect/sql-sqlite-bun/SqliteMigrator" + +const MigrationLayer = SqliteMigrator.layer({ + loader: migrations +}) +``` + +From the vendored packages, this is implemented as `Layer.effectDiscard(run(options))`. + +That means: + +- the layer performs the migration effect +- it does not provide a new service of its own +- it is intended to be composed into startup infrastructure + +### Concrete Startup Composition Example + +Preferred shape: + +```ts +const SqlLayer = SqliteClient.layer({ filename: "todos.sqlite" }) + +const MigrationLayer = SqliteMigrator.layer({ + loader: migrations +}) + +const MigratedSqlLayer = Layer.merge( + SqlLayer, + MigrationLayer.pipe(Layer.provide(SqlLayer)) +) +``` + +This keeps the structure explicit: + +- one layer provides the SQL client +- one layer runs migrations +- the merged layer represents a migrated SQL environment + +If the same migrated SQL environment is reused in tests, create it once and reuse the layer value. + +### Migration Best Practices + +- use stable numeric migration IDs +- keep migration files ordered and unique +- do not hand-roll a separate migrations subsystem if `Migrator` already fits the project +- keep migration execution at startup or a dedicated operational boundary +- do not bury migration execution inside arbitrary service construction unless startup is explicitly the right place + +### Preferred Migration Boundary + +Good pattern: + +- construct the SQL layer +- run migrations once at startup or deployment entry +- then run the main application + +Concrete shapes: + +- separate startup effect: `SqliteMigrator.run({ loader })` +- startup layer: `SqliteMigrator.layer({ loader })` +- migrated environment layer: merge the SQL client layer with the migration layer provided by that client layer + +Avoid: + +- opportunistic migrations inside ordinary request handlers +- schema creation hidden inside unrelated business service constructors + +## Errors + +Prefer SQL errors that stay inside the Effect SQL model as long as possible. + +Use domain-level translation only where it helps the business boundary. + +Good: + +- SQL layer or repository works with `SqlError` and schema decode failures where appropriate +- higher-level service translates expected cases into domain errors when needed + +Avoid: + +- converting every SQL error immediately into a string +- hiding SQL failure details too early + +## Observability + +The vendored SQL modules already integrate with spans and transaction context. + +This is another reason to prefer them over raw driver usage. + +Good pattern: + +- use `SqlClient` and `withTransaction` +- keep business operations wrapped with `Effect.fn` +- add explicit spans only where business-level detail matters beyond the built-in SQL spans + +## Layering Pattern + +Preferred layering shape: + +1. runtime-specific database layer provides `SqlClient` +2. migrations run at startup boundary +3. domain repository/service depends on `SqlClient` +4. top-level application layer composes the database layer with the business layers + +This keeps: + +- driver choice at the edge +- SQL capability in the middle +- business logic above it + +## Anti-Patterns + +- embedding a native driver directly in business services when Effect SQL modules are available +- hand-rolling transactions with raw SQL statements in service methods +- hiding schema creation inside unrelated service constructors +- exporting one trivial accessor function per repository/service method +- converting SQL errors to strings too early +- bypassing `Migrator` when the project already uses Effect SQL +- creating ad hoc migration effects without a stable loader shape when `fromRecord(...)` already fits the project +- scattering migration execution across multiple subsystems instead of one explicit startup boundary + +## Good Repo Examples To Study + +- `./.repos/effect/packages/effect/src/unstable/sql/SqlClient.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/Migrator.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlResolver.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlSchema.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlModel.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlError.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-testing.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-testing.md new file mode 100644 index 000000000..2f777cbc4 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-testing.md @@ -0,0 +1,504 @@ +# Testing Guide + +This guide is based on the vendored `@effect/vitest` package in `./.repos/effect`. + +Key source files: + +- `./.repos/effect/packages/vitest/src/index.ts` +- `./.repos/effect/packages/vitest/src/internal/internal.ts` +- `./.repos/effect/packages/vitest/test/index.test.ts` +- `./.repos/effect/packages/vitest/typetest/index.tst.ts` + +## Preferred Rule + +When testing Effect code with Vitest, prefer `@effect/vitest` over manually calling `Effect.runPromise`, `Effect.runSync`, or ad hoc runtime setup inside ordinary Vitest tests. + +Layer provisioning in tests should follow these rules: + +1. If multiple tests should share the same layered setup, use `layer(...)`. +2. If a nested group needs extra dependencies, use `it.layer(...)`. +3. If tests need isolated layer instances per test, use multiple separate `it.layer(...)` calls. +4. Do not default to local `.pipe(Effect.provide(...))` inside test bodies. + +Use: + +- `it.effect` for Effect-based tests with test services +- `it.live` for Effect-based tests that should use live services +- `layer(...)` and `it.layer(...)` for shared layered test setup +- `it.effect.prop` for Effect-based property tests + +Do not default to `.pipe(Effect.provide(SomeLayer))` inside test bodies when `layer(...)` or `it.layer(...)` expresses the setup more clearly. + +## Imports + +Preferred imports for Effect tests: + +```ts +import { assert, describe, it, layer } from "@effect/vitest" +import { Effect } from "effect" +``` + +`@effect/vitest` re-exports Vitest, so it is the normal entrypoint for test APIs in an Effect codebase. + +## Core Test Modes + +## `it.effect` + +Use `it.effect` for most Effect tests. + +It automatically: + +- runs the effect +- scopes it correctly +- provides the default test environment + +The default test environment includes: + +- `TestConsole` +- `TestClock` + +This comes directly from the internal implementation. + +Example: + +```ts +import { assert, it } from "@effect/vitest" +import { Effect } from "effect" + +it.effect("loads a user", () => + Effect.gen(function*() { + yield* Effect.void + assert.isTrue(true) + }) +) +``` + +Use `it.effect` when: + +- the test uses ordinary Effect code +- the test benefits from `TestClock` +- the test benefits from `TestConsole` +- the test should run in a scoped Effect runtime + +## `it.live` + +Use `it.live` when the test should use live services instead of the default test environment. + +Example: + +```ts +it.live("uses live services", () => + Effect.gen(function*() { + yield* Effect.void + }) +) +``` + +Use `it.live` when: + +- you need the real `Clock` +- the test should interact with live runtime services +- you deliberately do not want the test environment overrides + +Rule of thumb: + +- default: `it.effect` +- opt into `it.live` only when you actually need live behavior + +## Effect Test Structure + +Preferred pattern: + +```ts +it.effect("does something", () => + Effect.gen(function*() { + const value = yield* someEffect + assert.strictEqual(value, 1) + }) +) +``` + +Prefer `Effect.gen` inside `it.effect` and `it.live` for readability. + +Avoid: + +- `it("...", async () => ...)` for Effect programs +- `Effect.runPromise(...)` inside plain Vitest tests +- manual runtime setup for routine Effect tests + +## Assertions + +Use `assert` for Effect tests. + +The vendored repo also uses `expect` in some tests, but for skill guidance prefer `assert` in Effect-based tests because it keeps tests more uniform and explicit inside Effect programs. + +Examples: + +```ts +assert.isTrue(value === 1) +assert.strictEqual(a + b, b + a) +assert.include(text, substring) +``` + +## Test Context + +`it.effect` and `it.live` test functions can receive a Vitest `TestContext`. + +Example: + +```ts +it.effect("uses context", (ctx) => + Effect.gen(function*() { + ctx.onTestFailed(() => { + // cleanup or diagnostics + }) + }) +) +``` + +Use this when you need: + +- failure hooks +- abort signals +- ordinary Vitest test context integration + +## `each`, `skip`, `skipIf`, `runIf`, `only`, `fails` + +The Effect testers support the standard test variants: + +- `it.effect.each(...)` +- `it.live.each(...)` +- `it.effect.skip(...)` +- `it.effect.skipIf(...)` +- `it.effect.runIf(...)` +- `it.effect.only(...)` +- `it.live.fails(...)` + +Examples from the vendored tests show these are first-class parts of the API. + +Use them exactly as you would with Vitest, but return `Effect` from the test body. + +## Property Testing + +There are two different property-test entrypoints and they do not have identical behavior. + +## Top-level `it.prop` + +Use `it.prop` for non-Effect property tests. + +Example: + +```ts +import { it } from "@effect/vitest" +import { FastCheck } from "effect/testing" + +const realNumber = FastCheck.float({ noNaN: true, noDefaultInfinity: true }) + +it.prop("symmetry", [realNumber, FastCheck.integer()], ([a, b]) => a + b === b + a) +``` + +Important limitation from the internal implementation: + +- top-level `it.prop` does not support `Schema` arbitraries yet +- if you pass a `Schema`, it throws + +So use top-level `it.prop` only with explicit `FastCheck` arbitraries. + +## `it.effect.prop` + +Use `it.effect.prop` for property tests that return `Effect`. + +This is the more powerful property-testing mode for Effect code. + +Example: + +```ts +import { assert, it } from "@effect/vitest" +import { Effect } from "effect" +import { FastCheck } from "effect/testing" + +const realNumber = FastCheck.float({ noNaN: true, noDefaultInfinity: true }) + +it.effect.prop("symmetry", [realNumber, FastCheck.integer()], ([a, b]) => + Effect.gen(function*() { + assert.strictEqual(a + b, b + a) + }) +) +``` + +Unlike top-level `it.prop`, `it.effect.prop` does support `Schema` inputs by converting them with `Schema.toArbitrary`. + +So prefer `it.effect.prop` when: + +- the property test needs `Effect` +- you want to use `Schema` values as arbitraries +- the test needs Effect services or scope + +## `layer(...)` + +Use top-level `layer(...)` to share a `Layer` across a group of tests. + +Hard rule: + +- use `layer(...)` when tests should share one layered setup +- do not use it when each test needs its own isolated layer instance + +This is one of the most important `@effect/vitest` features. + +Example: + +```ts +import { describe, it, layer } from "@effect/vitest" +import { Context, Effect, Layer } from "effect" + +class Foo extends Context.Service()("Foo") { + static readonly layer = Layer.succeed(Foo)("foo") +} + +describe("foo", () => { + layer(Foo.layer)((it) => { + it.effect("gets foo", () => + Effect.gen(function*() { + const foo = yield* Foo + return foo + }) + ) + }) +}) +``` + +What it does internally: + +- builds the layer once for the test group +- memoizes the built layer with a `MemoMap` +- keeps the scope open for the group +- closes the scope in `afterAll` + +This makes it the preferred way to share layer setup across related tests. + +This is also the preferred alternative to calling `Effect.provide(...)` inside each individual test body. + +## Anti-Pattern: Local `Effect.provide(...)` In Tests + +If multiple tests use the same layer, do not write tests like this: + +```ts +it.effect("creates and lists todos", () => + Effect.gen(function*() { + const service = yield* TodoService + yield* service.create("write tests") + yield* service.create("ship feature") + }).pipe( + Effect.provide(TodoService.inMemoryLayer) + ) +) +``` + +Why this is the wrong pattern: + +- it repeats provisioning in every test +- it hides the layered test setup inside each test body +- it fights the `@effect/vitest` layer helpers +- it makes shared setup and teardown less explicit +- it bypasses the clearer grouped-layer style that the library is designed for + +Prefer: + +```ts +describe("TodoService", () => { + layer(TodoService.inMemoryLayer)((it) => { + it.effect("creates and lists todos", () => + Effect.gen(function*() { + const service = yield* TodoService + yield* service.create("write tests") + yield* service.create("ship feature") + }) + ) + + it.effect("updates completion and deletes todos", () => + Effect.gen(function*() { + const service = yield* TodoService + const todo = yield* service.create("close issue") + yield* service.setCompleted(todo.id, true) + yield* service.remove(todo.id) + }) + ) + }) +}) +``` + +Rule: + +- if a test or group of tests depends on a layer, prefer `layer(...)` +- if a nested group needs extra dependencies, prefer `it.layer(...)` +- if tests need isolated layer instances per test, use multiple separate `it.layer(...)` calls +- use local `Effect.provide(...)` in tests only for true one-off edge cases, not as the normal pattern + +This matches the general layer guidance: provisioning belongs at the boundary, and in `@effect/vitest` the test boundary should usually be expressed with `layer(...)` rather than ad hoc local provisioning. + +## `it.layer(...)` + +Use `it.layer(...)` inside an existing layered test group to add nested layer context. + +Hard rule: + +- use `it.layer(...)` when a specific test or nested test block should get its own isolated layered setup +- if isolation matters, prefer multiple separate `it.layer(...)` calls over one shared `layer(...)` group + +Example: + +```ts +layer(Foo.layer)((it) => { + it.layer(Bar.layer)("nested", (it) => { + it.effect("gets both", () => + Effect.gen(function*() { + const foo = yield* Foo + const bar = yield* Bar + return [foo, bar] + }) + ) + }) +}) +``` + +Important behavior from the implementation: + +- nested `it.layer(...)` reuses the parent memo map +- nested `it.layer(...)` does not accept `memoMap` or `excludeTestServices` +- nested layering is meant to extend the current layered test environment, not redefine its runtime policy + +Use multiple `it.layer(...)` blocks when each test should get its own isolated layered setup instead of sharing one top-level `layer(...)` group. + +## `layer` Options + +The top-level `layer(...)` helper accepts: + +- `timeout` +- `memoMap` +- `excludeTestServices` + +### `excludeTestServices` + +By default, `layer(...)` merges your layer with the test environment (`TestClock` and `TestConsole`). + +Use `excludeTestServices: true` when you want your layer group to run without those test-service overrides. + +This is useful for tests that should keep live runtime behavior. + +### `memoMap` + +Use `memoMap` only when you have a specific reason to coordinate layer memoization manually across test setups. + +Most tests should let `@effect/vitest` manage it. + +## Scoped Resources In Tests + +`@effect/vitest` is designed to work correctly with scoped effects and layered resources. + +The vendored tests explicitly verify resource release through `afterAll`. + +Use this normally: + +- define scoped services in layers +- use `layer(...)` to share them +- let the helper own scope setup and teardown + +Avoid manually managing large scopes in test code unless the test specifically needs that control. + +## `flakyTest` + +Use `flakyTest` for tests that need bounded retrying. + +Repo behavior: + +- wraps the test in `Effect.scoped` +- retries using a schedule +- retries for up to a timeout window +- converts final failure to defect with `Effect.orDie` + +Use this only for truly flaky integration-style conditions, not as a substitute for deterministic tests. + +## `makeMethods` And `describeWrapped` + +These exist for custom integration and wrapper scenarios. + +### `makeMethods` + +Use `makeMethods` when you need to extend or wrap a custom Vitest `it` instance while preserving Effect-aware helpers. + +### `describeWrapped` + +Use `describeWrapped` when you want a `describe` wrapper that hands you the augmented Effect-aware `it` methods directly. + +Most test code can just use imported `describe` and `it` from `@effect/vitest`. + +## Equality Testers + +`addEqualityTesters()` exists as part of the public API. + +Use it only when you have a concrete need to install equality testers for your Vitest environment. + +It is not needed for ordinary test structure. + +## Recommended Patterns + +### Pattern: normal Effect test + +```ts +it.effect("does work", () => + Effect.gen(function*() { + const value = yield* Effect.succeed(1) + assert.strictEqual(value, 1) + }) +) +``` + +### Pattern: shared layer for a test group + +```ts +layer(AppLayer)("app", (it) => { + it.effect("uses app services", () => + Effect.gen(function*() { + yield* Effect.void + }) + ) +}) +``` + +### Pattern: property test with Effect + +```ts +it.effect.prop("law", [FastCheck.integer()], ([n]) => + Effect.gen(function*() { + assert.strictEqual(n + 0, n) + }) +) +``` + +### Pattern: use `TestClock` + +```ts +it.effect("uses TestClock", () => + Effect.gen(function*() { + const fiber = yield* Effect.forkChild(Effect.sleep("1 second")) + yield* TestClock.adjust("1 second") + yield* Fiber.join(fiber) + }) +) +``` + +## Anti-Patterns + +- using plain `it(...)` with `Effect.runPromise(...)` for normal Effect tests +- using `it.live` by default when `it.effect` is sufficient +- manually building and tearing down large layer graphs instead of using `layer(...)` +- using top-level `it.prop` with `Schema` inputs +- using `flakyTest` to hide deterministic failures +- duplicating runtime setup instead of sharing a layer + +## Good Repo Examples To Study + +- `./.repos/effect/packages/vitest/test/index.test.ts` +- `./.repos/effect/packages/vitest/src/index.ts` +- `./.repos/effect/packages/vitest/src/internal/internal.ts` +- `./.repos/effect/packages/vitest/typetest/index.tst.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/setup.md b/tools/agent-os/.agents/skills/effect-ts/references/setup.md new file mode 100644 index 000000000..be471a678 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/setup.md @@ -0,0 +1,89 @@ +# Effect Source Setup + +This setup task is required when `./.repos/effect` is missing from the root of the repository where this skill is used. + +## Prompt + +The local Effect source checkout was not found at `./.repos/effect`. + +Choose one of these setup options before continuing: + +1. Add `https://github.com/Effect-TS/effect-smol` as a git subtree with squashed history at `./.repos/effect` +2. Add `https://github.com/Effect-TS/effect-smol` as a git submodule at `./.repos/effect` +3. Use `git clone` into `./.repos/effect`, ignore it via `.gitignore`, and add a prepare script that bootstraps it when missing + +## Supported Options + +### 1. Git Subtree + +Use this when the repository should vendor the Effect source directly while keeping history compact. + +- Repo path: `./.repos/effect` +- Source: `https://github.com/Effect-TS/effect-smol` +- Preferred shape: subtree with squashed history + +### 2. Git Submodule + +Use this when the repository should track the Effect source explicitly as a separate Git dependency. + +- Repo path: `./.repos/effect` +- Source: `https://github.com/Effect-TS/effect-smol` +- Preferred shape: standard Git submodule + +### 3. Local Clone + Gitignore + Prepare Task + +Use this when the repository should avoid vendoring or submodule management, but still provide a reproducible local setup. + +- Repo path: `./.repos/effect` +- Source: `https://github.com/Effect-TS/effect-smol` +- Add `.repos/effect` to the repository `.gitignore` +- Add a `prepare` task that clones the repo automatically when the directory is missing + +#### Concrete Shape + +Use this exact shape for the setup. Do not invent a different script. + +`package.json`: + +```json +{ + "scripts": { + "prepare": "./scripts/prepare-effect.sh" + } +} +``` + +`.gitignore`: + +```gitignore +.repos/effect +``` + +`scripts/prepare-effect.sh`: + +```sh +#!/usr/bin/env sh + +set -eu + +repo_dir=".repos/effect" +repo_url="https://github.com/Effect-TS/effect-smol" + +if [ -d "$repo_dir/.git" ]; then + exit 0 +fi + +mkdir -p ".repos" +git clone "$repo_url" "$repo_dir" +``` + +#### Notes + +- This keeps `./.repos/effect` available for local research without forcing it into version control +- The script is only responsible for ensuring the checkout exists; it does not update or reset an existing clone +- If you choose this option, the setup task should add this exact script, wire it via `prepare`, and add `.repos/effect` to `.gitignore` + +## Guidance + +- Do not continue with Effect-specific work until one of the setup options is chosen. +- Prefer the option that matches the host repository's dependency management style. diff --git a/tools/agent-os/.gitignore b/tools/agent-os/.gitignore index c2658d7d1..4304d7d0d 100644 --- a/tools/agent-os/.gitignore +++ b/tools/agent-os/.gitignore @@ -1 +1,2 @@ node_modules/ +.repos/effect diff --git a/tools/agent-os/AGENTS.md b/tools/agent-os/AGENTS.md index 8ff026edf..872166ee7 100644 --- a/tools/agent-os/AGENTS.md +++ b/tools/agent-os/AGENTS.md @@ -11,6 +11,14 @@ Keep compatibility with the root distro's `data/`, `state/`, briefs, status even Use the Bun channel declared in `.bun-version`. Keep `.bun-version`, `packageManager`, and the matching Bun type definitions aligned on the latest stable release. +## Effect + +Use Effect for effectful workflows whose failure modes, dependencies, cleanup, concurrency, retries, or observability benefit from typed composition. +Keep small total functions and straightforward value transformations in plain TypeScript. +Load the scoped `effect-ts` skill installed from `Effect-TS/skills` before changing Effect code. +Use the ignored `.repos/effect` checkout for source verification when the skill's focused guides do not answer the question. +Do not copy that checkout into the container image or commit it as a source snapshot. + ## Addresses Use Multiaddr strings as the single locator for supervised mates. @@ -23,7 +31,9 @@ The address grammar and validation contract are owned by `src/address.ts` and it ## Command Execution -Use Bun Shell's `$` for finite calls to trusted CLIs such as `kubectl`, `herdr`, `treehouse`, and Git. +Use the official `@akua-dev/sdk` for Akua operations implemented inside this TypeScript tool. +Keep the bundled `akua` CLI available for Firstmate's direct and interactive use. +Use Bun Shell's `$` for other finite calls to trusted CLIs such as `kubectl`, `herdr`, `treehouse`, and Git. Interpolate dynamic values directly so Bun treats each value as a literal argument. Never use `{ raw: value }` for dynamic data. Avoid `bash -c` and validate values that an external command could interpret as flags. @@ -32,6 +42,12 @@ Set environment and working directory per command instead of mutating the global Use `Bun.spawn()` for interactive agents, servers, watches, live logs, cancellation, and other long-running processes. Existing Bash scripts may be invoked explicitly during migration, but new orchestration and state transitions belong in TypeScript. +## Akua packages + +The prepared mate package lives at `packages/mate/` and renders ordinary Kubernetes YAML. +Treat it as an editable convenience, not a required abstraction or controller. +Firstmate may use `agent-os mate render`, invoke `akua render` itself, edit the result, or author equivalent YAML directly. + ## Scope Discipline Prefer a thin adapter over a new framework. diff --git a/tools/agent-os/README.md b/tools/agent-os/README.md index f37699a54..7776509b3 100644 --- a/tools/agent-os/README.md +++ b/tools/agent-os/README.md @@ -23,6 +23,11 @@ bun run check bun run src/cli.ts address local task-x bun run src/cli.ts address k8s task-x bun run src/cli.ts address inspect /k8s/in-cluster/namespace/agent-os/mate/task-x/herdr +bun run src/cli.ts mate render task-x --out /tmp/task-x ``` +Akua package operations inside this tool use the official `@akua-dev/sdk` in process. +The prepared package under `packages/mate/` remains optional and renders editable Kubernetes YAML. +The scoped official `Effect-TS/skills` installation guides typed Effect workflows; its Effect source checkout is local and ignored rather than committed as a snapshot. + See `AGENTS.md` in this directory for the tool-specific development rules. diff --git a/tools/agent-os/bun.lock b/tools/agent-os/bun.lock index 717d11312..c0ea2c18d 100644 --- a/tools/agent-os/bun.lock +++ b/tools/agent-os/bun.lock @@ -5,7 +5,9 @@ "": { "name": "@akua-dev/agent-os-tool", "dependencies": { + "@akua-dev/sdk": "0.8.24", "@multiformats/multiaddr": "13.0.3", + "effect": "3.21.4", }, "devDependencies": { "@types/bun": "1.3.14", @@ -14,10 +16,32 @@ }, }, "packages": { + "@akua-dev/native": ["@akua-dev/native@0.8.24", "", { "dependencies": { "@akua-dev/native-engines": "0.8.24" }, "optionalDependencies": { "@akua-dev/native-darwin-arm64": "0.8.24", "@akua-dev/native-darwin-x64": "0.8.24", "@akua-dev/native-linux-arm64-gnu": "0.8.24", "@akua-dev/native-linux-arm64-musl": "0.8.24", "@akua-dev/native-linux-x64-gnu": "0.8.24", "@akua-dev/native-linux-x64-musl": "0.8.24", "@akua-dev/native-win32-x64-msvc": "0.8.24" } }, "sha512-DZsUSd0SaLrqBvbmsEmPD9rUwxrUZSC/QTNfclzRgmKxnK2y5x90pT+J4L/Aoa3lCFWpCp9MOM9pZ7ztjHL1Nw=="], + + "@akua-dev/native-darwin-arm64": ["@akua-dev/native-darwin-arm64@0.8.24", "", { "os": "darwin", "cpu": "arm64" }, "sha512-6Wa2zDcrRWQUPouHKmZUom4kNmffvWYxcrvnaeoVwbQFj3TXWKAaCzcVYjZGeLcV8HxMkTrYx4rt/NH5GRLkWA=="], + + "@akua-dev/native-darwin-x64": ["@akua-dev/native-darwin-x64@0.8.24", "", { "os": "darwin", "cpu": "x64" }, "sha512-hhXiqfpw9itn6cdNhC0IVW1WUaPBmpYmBPW7hfkBuaTlP/2wPyAUx+Wu0cJWpna1YCUg3WzcUOj5rm/lyciP9w=="], + + "@akua-dev/native-engines": ["@akua-dev/native-engines@0.8.24", "", {}, "sha512-3rKiWQn1pIURERUmxNiqPw7SxtMrNwX5tDxsCRxsAZ7Ovzfzcwx/VcycfuY+z1DNhEJrkKChkYiMT3N/EZanmg=="], + + "@akua-dev/native-linux-arm64-gnu": ["@akua-dev/native-linux-arm64-gnu@0.8.24", "", { "os": "linux", "cpu": "arm64" }, "sha512-NTeuJRCjy68JJT5LuHwPHf11mnZ1a5hNeKNfoBYMzUeViN3BdnBuszjNxwYHAgQYEN8+Rf5EGtTMdOTm0xDy/Q=="], + + "@akua-dev/native-linux-arm64-musl": ["@akua-dev/native-linux-arm64-musl@0.8.24", "", { "os": "linux", "cpu": "arm64" }, "sha512-QbGJfTY2Ilv+FSVaIU5MnlHjprbKVM41Ya+YfanHJsTBwO0dUw6J9C/fNVKTHeQM98vLVBIWdLiAPl73jSI/Qw=="], + + "@akua-dev/native-linux-x64-gnu": ["@akua-dev/native-linux-x64-gnu@0.8.24", "", { "os": "linux", "cpu": "x64" }, "sha512-jJDnr4MGYxB/vflhGklC1dUxHsO7HNuWexDFGB/vPhzo6RGsAWsApnrqPYYfBDmsk0nvvZKO9OWTB+e837gCzA=="], + + "@akua-dev/native-linux-x64-musl": ["@akua-dev/native-linux-x64-musl@0.8.24", "", { "os": "linux", "cpu": "x64" }, "sha512-21EsbM2fArJjX4RKXeVXDPxSBHkeMtYVCvT9vDj4S1lo5Ld0xRM/utm6EQkfbShdpwWZRhkv/t7evgdVM8Zb2A=="], + + "@akua-dev/native-win32-x64-msvc": ["@akua-dev/native-win32-x64-msvc@0.8.24", "", { "os": "win32", "cpu": "x64" }, "sha512-boG/GW6KTucoq7PUlU92tvB91DGxl4hNdQICcsXwgO7npzLqNEhCzKYnpuwKgqhK6vSpoPd0/jZdmu6EwUyzKQ=="], + + "@akua-dev/sdk": ["@akua-dev/sdk@0.8.24", "", { "dependencies": { "@akua-dev/native": "0.8.24", "@standard-schema/spec": "^1.1.0", "ajv": "^8.18.0", "yaml": "^2.8.3" } }, "sha512-8OM8luAqmxnA4kRJUGbf4Jhr4YuSf/lXKAOUD4kZTUepoRvgYP3xHlXJRCVd1Qv00L1NmeluXG8sr5rh/soGIg=="], + "@chainsafe/is-ip": ["@chainsafe/is-ip@2.1.0", "", {}, "sha512-KIjt+6IfysQ4GCv66xihEitBjvhU/bixbbbFxdJ1sqCp4uJ0wuZiYBPhksZoy4lfaF0k9cwNzY5upEW/VWdw3w=="], "@multiformats/multiaddr": ["@multiformats/multiaddr@13.0.3", "", { "dependencies": { "@chainsafe/is-ip": "^2.0.1", "multiformats": "^14.0.0", "uint8-varint": "^3.0.0", "uint8arrays": "^6.1.1" } }, "sha512-mEqqJ4r3a/uuFMTpRkU316wGNIDQNhuVWpm+ebKTQeYsfv9jXbPONWM6VVnj3KGUrwfsX7GZOyp4TFqEA2SPCw=="], + "@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], + "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], "@types/node": ["@types/node@26.1.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw=="], @@ -62,10 +86,26 @@ "@typescript/typescript-win32-x64": ["@typescript/typescript-win32-x64@7.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g=="], + "ajv": ["ajv@8.20.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA=="], + "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + "effect": ["effect@3.21.4", "", { "dependencies": { "@standard-schema/spec": "^1.0.0", "fast-check": "^3.23.1" } }, "sha512-B89v/xSgPbl1J2Ai2u18jxq3odpFauU1rC6/eSs4FeNHi72kwKdJp12VGigvRV2lK+kRnx+OOz41XV8guZd4gQ=="], + + "fast-check": ["fast-check@3.23.2", "", { "dependencies": { "pure-rand": "^6.1.0" } }, "sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A=="], + + "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], + + "fast-uri": ["fast-uri@3.1.3", "", {}, "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg=="], + + "json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="], + "multiformats": ["multiformats@14.0.4", "", {}, "sha512-+SXItmOUWzT0kjlIfkZ4NawJaC9jW/mPlyn902V9Qgpc7YDwdEiwqbiZxTyvC0XWh1jZguXca3A/WQ+UOPRLUA=="], + "pure-rand": ["pure-rand@6.1.0", "", {}, "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA=="], + + "require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="], + "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], "uint8-varint": ["uint8-varint@3.0.0", "", { "dependencies": { "uint8arraylist": "^3.0.1", "uint8arrays": "^6.1.0" } }, "sha512-S4DdpXBaLwKcFo7f0bWzWfHjbZ/i3QhM842qn+ZvHjxqFCfUcEB9SQNcmI69S+zMlcmIcKxsk9Iyw77S2Kxv6Q=="], @@ -75,5 +115,7 @@ "uint8arrays": ["uint8arrays@6.1.1", "", { "dependencies": { "multiformats": "^14.0.0" } }, "sha512-iz7JN0XCSZYA111lhFG2Ui9EhFvTNekqSRHw3lvMHq+dzwWy1OQftxFQREEh4rffU0oSoXdQHsk2TiHKVm4fsA=="], "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], + + "yaml": ["yaml@2.9.0", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA=="], } } diff --git a/tools/agent-os/package.json b/tools/agent-os/package.json index 1dd58ead0..ba519baaa 100644 --- a/tools/agent-os/package.json +++ b/tools/agent-os/package.json @@ -8,11 +8,14 @@ }, "scripts": { "check": "bun run typecheck && bun test", + "prepare": "./scripts/prepare-effect.sh", "test": "bun test", "typecheck": "tsc --noEmit" }, "dependencies": { - "@multiformats/multiaddr": "13.0.3" + "@akua-dev/sdk": "0.8.24", + "@multiformats/multiaddr": "13.0.3", + "effect": "3.21.4" }, "devDependencies": { "@types/bun": "1.3.14", diff --git a/tools/agent-os/packages/mate/README.md b/tools/agent-os/packages/mate/README.md new file mode 100644 index 000000000..5576ebec4 --- /dev/null +++ b/tools/agent-os/packages/mate/README.md @@ -0,0 +1,25 @@ +# Agent OS mate package + +This Akua package is a prepared convenience for rendering one general-purpose mate Pod and its persistent home. +It is not a controller, required operating model, or replacement for Kubernetes YAML. + +Render the example directly: + +```sh +akua render --package package.k --inputs inputs.example.yaml --out /tmp/scout-1 +``` + +Or use the Agent OS convenience command from any directory: + +```sh +agent-os mate render scout-1 --namespace agent-os-demo --image agent-os:dev --out /tmp/scout-1 +``` + +Inspect and edit the rendered YAML before applying it: + +```sh +kubectl apply -f /tmp/scout-1 +``` + +Firstmate may change this package, supply additional inputs, edit the rendered files, or author equivalent YAML directly. +The package intentionally grants no Kubernetes token and no privileged, host-namespace, or host-mount access. diff --git a/tools/agent-os/packages/mate/akua.toml b/tools/agent-os/packages/mate/akua.toml new file mode 100644 index 000000000..06303c226 --- /dev/null +++ b/tools/agent-os/packages/mate/akua.toml @@ -0,0 +1,5 @@ +[package] +name = "agent-os-mate" +version = "0.1.0" +edition = "akua.dev/v1alpha1" +strict_signing = true diff --git a/tools/agent-os/packages/mate/inputs.example.yaml b/tools/agent-os/packages/mate/inputs.example.yaml new file mode 100644 index 000000000..89d21d97f --- /dev/null +++ b/tools/agent-os/packages/mate/inputs.example.yaml @@ -0,0 +1,5 @@ +mateId: scout-1 +namespace: agent-os-demo +address: /k8s/in-cluster/namespace/agent-os-demo/mate/scout-1/herdr +image: agent-os:dev +imagePullPolicy: Never diff --git a/tools/agent-os/packages/mate/package.k b/tools/agent-os/packages/mate/package.k new file mode 100644 index 000000000..f12829369 --- /dev/null +++ b/tools/agent-os/packages/mate/package.k @@ -0,0 +1,101 @@ +import akua.ctx + +schema Input: + """Inputs for one general-purpose Kubernetes-backed Agent OS mate.""" + + mateId: str + namespace: str = "agent-os" + address: str + image: str = "agent-os:dev" + imagePullPolicy: "Always" | "IfNotPresent" | "Never" = "IfNotPresent" + storage: str = "10Gi" + cpuRequest: str = "250m" + memoryRequest: str = "512Mi" + cpuLimit: str = "2" + memoryLimit: str = "4Gi" + + check: + len(mateId) <= 43, "mateId must leave room for the agent-os-mate- resource prefix" + +input: Input = ctx.input() + +labels = { + "app.kubernetes.io/name" = "agent-os" + "app.kubernetes.io/component" = "mate" + "agent-os.akua.dev/mate" = input.mateId +} + +resources = [ + { + apiVersion = "v1" + kind = "PersistentVolumeClaim" + metadata = { + name = "agent-os-mate-${input.mateId}-home" + namespace = input.namespace + labels = labels + } + spec = { + accessModes = ["ReadWriteOnce"] + resources = { + requests = { storage = input.storage } + } + } + }, + { + apiVersion = "v1" + kind = "Pod" + metadata = { + name = "agent-os-mate-${input.mateId}" + namespace = input.namespace + labels = labels + annotations = { + "agent-os.akua.dev/address" = input.address + } + } + spec = { + automountServiceAccountToken = False + securityContext = { + fsGroup = 0 + runAsGroup = 0 + runAsUser = 0 + } + initContainers = [{ + name = "agent-os-init" + image = input.image + imagePullPolicy = input.imagePullPolicy + command = ["/opt/agent-os/bin/agent-os-init.sh"] + volumeMounts = [{ + name = "home" + mountPath = "/persistent-agent" + }] + }] + containers = [{ + name = "mate" + image = input.image + imagePullPolicy = input.imagePullPolicy + env = [ + { name = "FM_HOME", value = "/home/agent" }, + { name = "HOME", value = "/home/agent" }, + ] + resources = { + requests = { + cpu = input.cpuRequest + memory = input.memoryRequest + } + limits = { + cpu = input.cpuLimit + memory = input.memoryLimit + } + } + volumeMounts = [ + { name = "home", mountPath = "/home/agent" }, + { name = "home", mountPath = "/usr/local", subPath = "usr-local" }, + ] + }] + volumes = [{ + name = "home" + persistentVolumeClaim = { claimName = "agent-os-mate-${input.mateId}-home" } + }] + } + }, +] diff --git a/tools/agent-os/scripts/prepare-effect.sh b/tools/agent-os/scripts/prepare-effect.sh new file mode 100755 index 000000000..24a9d3145 --- /dev/null +++ b/tools/agent-os/scripts/prepare-effect.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env sh + +set -eu + +repo_dir=".repos/effect" +repo_url="https://github.com/Effect-TS/effect-smol" + +if [ -d "$repo_dir/.git" ]; then + exit 0 +fi + +mkdir -p ".repos" +git clone "$repo_url" "$repo_dir" diff --git a/tools/agent-os/skills-lock.json b/tools/agent-os/skills-lock.json new file mode 100644 index 000000000..192e1e1d3 --- /dev/null +++ b/tools/agent-os/skills-lock.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "skills": { + "effect-ts": { + "source": "Effect-TS/skills", + "sourceType": "github", + "skillPath": "skills/effect-ts/SKILL.md", + "computedHash": "301521e167410dd42d834dd0e7cedbe9cfffa519f630a12241dbea7b1d932324" + } + } +} diff --git a/tools/agent-os/src/cli.ts b/tools/agent-os/src/cli.ts index 5cd221ed4..48830c7d0 100755 --- a/tools/agent-os/src/cli.ts +++ b/tools/agent-os/src/cli.ts @@ -1,17 +1,20 @@ #!/usr/bin/env bun import { parseArgs } from "node:util"; +import { Effect } from "effect"; import { formatKubernetesAgentAddress, formatLocalAgentAddress, parseAgentAddress, } from "./address.ts"; +import { renderMatePackage } from "./mate-package.ts"; const usage = `Usage: agent-os address local agent-os address k8s [--cluster ] [--namespace ] - agent-os address inspect `; + agent-os address inspect + agent-os mate render [--cluster ] [--namespace ] [--image ] [--out ]`; function requireSingleArgument(args: string[]): string { if (args.length !== 1 || args[0] == null) { @@ -67,18 +70,68 @@ function runAddressCommand(args: string[]): void { } } -function main(): void { - const [command, ...rest] = Bun.argv.slice(2); +async function runMateCommand(args: string[]): Promise { + const [command, ...rest] = args; - if (command !== "address") { + if (command !== "render") { throw new Error(usage); } - runAddressCommand(rest); + const { positionals, values } = parseArgs({ + allowPositionals: true, + args: rest, + options: { + cluster: { + default: "in-cluster", + type: "string", + }, + image: { + default: "agent-os:dev", + type: "string", + }, + namespace: { + default: "agent-os", + type: "string", + }, + out: { + type: "string", + }, + }, + strict: true, + }); + const mateId = requireSingleArgument(positionals); + const result = await Effect.runPromise( + renderMatePackage({ + cluster: values.cluster, + image: values.image, + mateId, + namespace: values.namespace, + outDir: values.out, + }).pipe( + Effect.catchAll((error) => Effect.fail(new Error(error.message))), + ), + ); + + console.log(JSON.stringify(result, null, 2)); +} + +async function main(): Promise { + const [command, ...rest] = Bun.argv.slice(2); + + switch (command) { + case "address": + runAddressCommand(rest); + return; + case "mate": + await runMateCommand(rest); + return; + default: + throw new Error(usage); + } } try { - main(); + await main(); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; diff --git a/tools/agent-os/src/json-modules.d.ts b/tools/agent-os/src/json-modules.d.ts new file mode 100644 index 000000000..b14866845 --- /dev/null +++ b/tools/agent-os/src/json-modules.d.ts @@ -0,0 +1,4 @@ +declare module "*.json" { + const value: { readonly $defs: Readonly> }; + export default value; +} diff --git a/tools/agent-os/src/mate-package.ts b/tools/agent-os/src/mate-package.ts new file mode 100644 index 000000000..9d9e0306d --- /dev/null +++ b/tools/agent-os/src/mate-package.ts @@ -0,0 +1,151 @@ +import { + Akua, + AkuaRateLimitedError, + AkuaUserError, + type RenderOptions, + type RenderSummary, +} from "@akua-dev/sdk"; +import { Data, Effect } from "effect"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { formatKubernetesAgentAddress } from "./address.ts"; + +export interface RenderMatePackageOptions { + cluster?: string; + image?: string; + mateId: string; + namespace?: string; + outDir?: string; +} + +export interface RenderedMatePackage { + address: string; + outDir: string; +} + +export class MatePackageInputError extends Data.TaggedError( + "MatePackageInputError", +)<{ + readonly cause: unknown; + readonly message: string; +}> {} + +export class AkuaRenderError extends Data.TaggedError("AkuaRenderError")<{ + readonly cause: unknown; + readonly message: string; +}> {} + +export class AkuaPackageUserError extends Data.TaggedError( + "AkuaPackageUserError", +)<{ + readonly cause: Error; + readonly message: string; +}> {} + +export class AkuaPackageRateLimitedError extends Data.TaggedError( + "AkuaPackageRateLimitedError", +)<{ + readonly cause: Error; + readonly message: string; +}> {} + +export interface AkuaRenderer { + render(options: RenderOptions): Promise; +} + +const packageDirectory = resolve( + dirname(fileURLToPath(import.meta.url)), + "../packages/mate", +); + +const defaultAkua = new Akua(); + +function mapAkuaError(cause: unknown) { + if (cause instanceof AkuaRateLimitedError) { + return new AkuaPackageRateLimitedError({ + cause, + message: "Akua rate-limited the mate package render", + }); + } + if (cause instanceof AkuaUserError) { + return new AkuaPackageUserError({ + cause, + message: cause.message, + }); + } + return new AkuaRenderError({ + cause, + message: "Akua could not render the mate package", + }); +} + +export const renderMatePackage = Effect.fn("agent-os.renderMatePackage")( + function* ( + options: RenderMatePackageOptions, + akua: AkuaRenderer = defaultAkua, + ) { + const cluster = options.cluster ?? "in-cluster"; + const namespace = options.namespace ?? "agent-os"; + const address = formatKubernetesAgentAddress(options.mateId, { + cluster, + namespace, + }); + const outDir = resolve( + options.outDir ?? resolve(process.cwd(), "deploy", options.mateId), + ); + const temporaryDirectory = yield* Effect.tryPromise({ + try: () => mkdtemp(resolve(tmpdir(), "agent-os-mate-")), + catch: (cause) => + new MatePackageInputError({ + cause, + message: "Could not create temporary Akua inputs", + }), + }); + + const render = Effect.gen(function* () { + const inputsPath = resolve(temporaryDirectory, "inputs.json"); + yield* Effect.tryPromise({ + try: () => + writeFile( + inputsPath, + JSON.stringify( + { + address, + image: options.image ?? "agent-os:dev", + mateId: options.mateId, + namespace, + }, + null, + 2, + ), + ), + catch: (cause) => + new MatePackageInputError({ + cause, + message: "Could not write temporary Akua inputs", + }), + }); + yield* Effect.tryPromise({ + try: () => + akua.render({ + inputs: inputsPath, + out: outDir, + package: resolve(packageDirectory, "package.k"), + timeout: "30s", + }), + catch: mapAkuaError, + }); + + return { address, outDir } satisfies RenderedMatePackage; + }); + + return yield* render.pipe( + Effect.ensuring( + Effect.promise(() => rm(temporaryDirectory, { force: true, recursive: true })), + ), + ); + }, +); diff --git a/tools/agent-os/test/mate-package.test.ts b/tools/agent-os/test/mate-package.test.ts new file mode 100644 index 000000000..97b72dfee --- /dev/null +++ b/tools/agent-os/test/mate-package.test.ts @@ -0,0 +1,81 @@ +import { Effect } from "effect"; +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; + +import { + type AkuaRenderer, + renderMatePackage, +} from "../src/mate-package.ts"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map((directory) => + rm(directory, { force: true, recursive: true }), + ), + ); +}); + +describe("agent-os mate render", () => { + test("passes typed inputs to the Akua SDK and reports the address", async () => { + const root = await mkdtemp(resolve(tmpdir(), "agent-os-mate-test-")); + temporaryDirectories.push(root); + const outDir = resolve(root, "rendered"); + let capturedInputs: unknown; + const akua: AkuaRenderer = { + async render(options) { + capturedInputs = JSON.parse( + await readFile(options.inputs ?? "", "utf8"), + ); + return { + files: [], + format: "raw-manifests", + hash: "sha256:test", + manifests: 2, + target: options.out ?? outDir, + }; + }, + }; + + const output = await Effect.runPromise( + renderMatePackage( + { + image: "ghcr.io/akua-dev/agent-os:test", + mateId: "scout-1", + namespace: "agent-os-demo", + outDir, + }, + akua, + ), + ); + + expect(output).toEqual({ + address: + "/k8s/in-cluster/namespace/agent-os-demo/mate/scout-1/herdr", + outDir, + }); + expect(capturedInputs).toEqual({ + address: + "/k8s/in-cluster/namespace/agent-os-demo/mate/scout-1/herdr", + image: "ghcr.io/akua-dev/agent-os:test", + mateId: "scout-1", + namespace: "agent-os-demo", + }); + }); + + test("keeps unexpected SDK failures in the typed error channel", async () => { + const akua: AkuaRenderer = { + render: () => Promise.reject(new Error("native addon failed")), + }; + const exit = await Effect.runPromiseExit( + renderMatePackage({ mateId: "scout-1" }, akua), + ); + + expect(exit._tag).toBe("Failure"); + expect(String(exit)).toContain("AkuaRenderError"); + expect(String(exit)).toContain("Akua could not render the mate package"); + }); +}); From c542f0cdaad97d7fb7685dd2231029a4e407b38e Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 10:21:11 +0200 Subject: [PATCH 11/56] feat: bundle k9s in agent image --- .agents/skills/kubernetes-fleet/SKILL.md | 1 + Dockerfile | 16 ++++++++++++++++ THIRD_PARTY_NOTICES.md | 7 +++++++ docs/kubernetes.md | 2 +- tests/agent-os-container.test.sh | 4 ++++ 5 files changed, 29 insertions(+), 1 deletion(-) diff --git a/.agents/skills/kubernetes-fleet/SKILL.md b/.agents/skills/kubernetes-fleet/SKILL.md index d39f7c1b1..4947dd9d0 100644 --- a/.agents/skills/kubernetes-fleet/SKILL.md +++ b/.agents/skills/kubernetes-fleet/SKILL.md @@ -15,6 +15,7 @@ Load this skill only when the current firstmate is running in Kubernetes or is e - Keep every crewmate general-purpose; its brief, tools, and authority specialize it for the current task. - A crewmate normally communicates only with its parent through its terminal, status files, reports, and delivered Git state. - The image includes Akua's CLI for direct use and the Agent OS tool uses `@akua-dev/sdk` in process. +- Use the bundled K9s terminal UI when it makes live Kubernetes inspection faster than individual `kubectl` reads. - Use `agent-os mate render ` when the prepared package is useful, then inspect or edit its ordinary YAML before applying it. - The package is optional; direct `akua render`, raw YAML, and `kubectl` remain supported. - Use `bin/agent-os-crewmate.sh create ` to create a separate Pod and persistent home. diff --git a/Dockerfile b/Dockerfile index 2a1979ece..23f4753ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ ARG TREEHOUSE_VERSION=2.0.0 ARG NO_MISTAKES_VERSION=1.34.0 ARG BUN_VERSION=1.3.14 ARG AKUA_VERSION=0.8.25 +ARG K9S_VERSION=0.51.0 RUN apt-get update \ && apt-get install -y --no-install-recommends \ @@ -106,6 +107,21 @@ RUN set -eu; \ mkdir -p /usr/share/licenses/akua; \ curl -fsSL "https://raw.githubusercontent.com/akua-dev/akua/v${AKUA_VERSION}/LICENSE" -o /usr/share/licenses/akua/LICENSE +RUN set -eu; \ + case "$TARGETARCH" in \ + amd64) sha=c3752ad51a5a4015a113819c4eeb6e55a4d0e4b8e652494797532f6fc8161dd7 ;; \ + arm64) sha=3ee05c82e5f9198928a4e86133608ba6a2c10a2244d6a7789e820f78319d640c ;; \ + *) echo "unsupported TARGETARCH: $TARGETARCH" >&2; exit 1 ;; \ + esac; \ + asset="k9s_Linux_${TARGETARCH}.tar.gz"; \ + curl -fsSL "https://github.com/derailed/k9s/releases/download/v${K9S_VERSION}/${asset}" -o "/tmp/${asset}"; \ + echo "$sha /tmp/${asset}" | sha256sum -c -; \ + tar -xzf "/tmp/${asset}" -C /tmp k9s; \ + install -m 0755 /tmp/k9s /usr/local/bin/k9s; \ + rm -f "/tmp/${asset}" /tmp/k9s; \ + mkdir -p /usr/share/licenses/k9s; \ + curl -fsSL "https://raw.githubusercontent.com/derailed/k9s/v${K9S_VERSION}/LICENSE" -o /usr/share/licenses/k9s/LICENSE + RUN npm install --global \ @earendil-works/pi-coding-agent@0.80.6 \ gh-axi@0.1.27 \ diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 0049de365..7e07a1233 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -15,3 +15,10 @@ Akua is available under the Apache License 2.0. The image includes Akua's license at `/usr/share/licenses/akua/LICENSE`. The exact source is available at . Agent OS invokes Akua through its documented CLI. + +## K9s + +The Agent OS demo image includes an unmodified K9s 0.51.0 executable as a separate program. +K9s is available under the Apache License 2.0. +The image includes K9s's license at `/usr/share/licenses/k9s/LICENSE`. +The exact source is available at . diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 553bba78e..afad836fa 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -35,7 +35,7 @@ bin/agent-os-local.sh status bin/agent-os-local.sh shell ``` -The image includes Firstmate's complete required toolchain, including `gh`, `rg`, `fd`, Akua, treehouse, no-mistakes, and every required AXI CLI. +The image includes Firstmate's complete required toolchain, including `gh`, `rg`, `fd`, Akua, `kubectl`, K9s, treehouse, no-mistakes, and every required AXI CLI. Authenticate GitHub inside the primary with `gh auth login`. Authenticate Pi using `/login` and the provider flow you choose. Host credentials are intentionally excluded from the image and are never copied automatically. diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 1ceefb506..c37ea3b2b 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -13,6 +13,7 @@ assert_grep 'ARG TREEHOUSE_VERSION=2.0.0' "$ROOT/Dockerfile" "image must pin tre assert_grep 'ARG NO_MISTAKES_VERSION=1.34.0' "$ROOT/Dockerfile" "image must pin no-mistakes 1.34.0" assert_grep 'ARG BUN_VERSION=1.3.14' "$ROOT/Dockerfile" "image must pin stable Bun 1.3.14" assert_grep 'ARG AKUA_VERSION=0.8.25' "$ROOT/Dockerfile" "image must pin Akua 0.8.25" +assert_grep 'ARG K9S_VERSION=0.51.0' "$ROOT/Dockerfile" "image must pin K9s 0.51.0" assert_grep '"@akua-dev/sdk": "0.8.24"' "$ROOT/tools/agent-os/package.json" "tool must pin the Akua SDK" assert_grep 'sha256sum -c -' "$ROOT/Dockerfile" "downloaded runtime binaries must be checksum verified" assert_grep '@earendil-works/pi-coding-agent@0.80.6' "$ROOT/Dockerfile" "image must pin Pi 0.80.6" @@ -30,6 +31,7 @@ assert_grep 'PATH=/home/agent/.local/bin:/home/agent/.bun/bin:/home/agent/.cargo "persistent tool prefixes must lead PATH" assert_grep '/opt/image-usr-local' "$ROOT/Dockerfile" "image must retain a seed copy of /usr/local" assert_grep 'akua-dev/akua/releases/download/v${AKUA_VERSION}' "$ROOT/Dockerfile" "image must install Akua from its release" +assert_grep 'derailed/k9s/releases/download/v${K9S_VERSION}' "$ROOT/Dockerfile" "image must install K9s from its release" assert_grep 'ln -s /opt/agent-os/tools/agent-os/src/cli.ts /usr/local/bin/agent-os' "$ROOT/Dockerfile" \ "image must expose the Agent OS tool" assert_grep 'bun install --frozen-lockfile --production --ignore-scripts' "$ROOT/Dockerfile" \ @@ -46,6 +48,8 @@ assert_grep 'https://github.com/ogulcancelik/herdr/tree/v0.7.3' "$ROOT/THIRD_PAR "Herdr's exact corresponding source must be named" assert_grep 'https://github.com/akua-dev/akua/tree/v0.8.25' "$ROOT/THIRD_PARTY_NOTICES.md" \ "Akua's exact source must be named" +assert_grep 'https://github.com/derailed/k9s/tree/v0.51.0' "$ROOT/THIRD_PARTY_NOTICES.md" \ + "K9s's exact source must be named" bash -n "$ROOT/bin/agent-os-container-entrypoint.sh" pass "container files pin dependencies and exclude host credentials" From 282d05e17a76051565350ba9e8eafeb71449cbb3 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 10:50:23 +0200 Subject: [PATCH 12/56] refactor: keep mate workflows tool-native --- .agents/skills/kubernetes-fleet/SKILL.md | 11 +- bin/agent-os-container-entrypoint.sh | 2 + bin/agent-os-init.sh | 1 + bin/agent-os-kubeconfig.sh | 35 ++++ docs/agent-evals.md | 48 ++++++ docs/kubernetes.md | 42 ++++- tests/agent-os-container.test.sh | 15 +- tests/agent-os-kubeconfig.test.sh | 36 +++++ tools/agent-os/AGENTS.md | 18 ++- tools/agent-os/README.md | 4 +- tools/agent-os/bun.lock | 42 ----- tools/agent-os/package.json | 4 +- tools/agent-os/packages/mate/README.md | 10 +- .../packages/mate/inputs.example.yaml | 1 + tools/agent-os/packages/mate/package.k | 20 ++- tools/agent-os/src/cli.ts | 65 +------- tools/agent-os/src/json-modules.d.ts | 4 - tools/agent-os/src/mate-package.ts | 151 ------------------ tools/agent-os/test/mate-package.test.ts | 81 ---------- 19 files changed, 223 insertions(+), 367 deletions(-) create mode 100755 bin/agent-os-kubeconfig.sh create mode 100644 docs/agent-evals.md create mode 100755 tests/agent-os-kubeconfig.test.sh delete mode 100644 tools/agent-os/src/json-modules.d.ts delete mode 100644 tools/agent-os/src/mate-package.ts delete mode 100644 tools/agent-os/test/mate-package.test.ts diff --git a/.agents/skills/kubernetes-fleet/SKILL.md b/.agents/skills/kubernetes-fleet/SKILL.md index 4947dd9d0..6ecbd247c 100644 --- a/.agents/skills/kubernetes-fleet/SKILL.md +++ b/.agents/skills/kubernetes-fleet/SKILL.md @@ -14,16 +14,21 @@ Load this skill only when the current firstmate is running in Kubernetes or is e - Keep every crewmate general-purpose; its brief, tools, and authority specialize it for the current task. - A crewmate normally communicates only with its parent through its terminal, status files, reports, and delivered Git state. -- The image includes Akua's CLI for direct use and the Agent OS tool uses `@akua-dev/sdk` in process. +- The image includes Akua's CLI and the prepared mate package lives at `tools/agent-os/packages/mate/`. - Use the bundled K9s terminal UI when it makes live Kubernetes inspection faster than individual `kubectl` reads. -- Use `agent-os mate render ` when the prepared package is useful, then inspect or edit its ordinary YAML before applying it. +- Render the package with `akua render`, inspect or edit its ordinary YAML when useful, and apply it with `kubectl`. +- Treat AI credentials as explicit per-mate grants, never as ambient inheritance merely because Firstmate can read them. +- Create or select a Kubernetes Secret only after the grant is authorized, then pass its name as the package's `piAuthSecret` input. +- Give every launched Herdr agent a task-unique name, close only a confirmed dead restored pane before reuse, and never replace a live agent. +- Grade completion by the promised artifact or delivered Git state; Herdr `idle` alone is not a completion signal. - The package is optional; direct `akua render`, raw YAML, and `kubectl` remain supported. - Use `bin/agent-os-crewmate.sh create ` to create a separate Pod and persistent home. - Use `status` to inspect it and `delete` only after unique work is checkpointed or delivered. - Never mount the primary home into a child Pod. - The demo child receives no Kubernetes ServiceAccount token by default. +- A Pod with an authorized ServiceAccount gets a token-file-backed `in-cluster` kubeconfig automatically, so Firstmate does not need to copy a bearer token into a temporary kubeconfig. - The OrbStack primary's cluster-admin binding is a local-demo trust decision, not a production-safe default. -- Pin an explicit host context with `AGENT_OS_CONTEXT`; the launcher refuses ambient host contexts. +- Pin an explicit host context for host-side `kubectl`; inside an authorized Pod use the generated `in-cluster` context. Use ordinary `kubectl exec`, Herdr, files, and Git to supervise the child. Do not add a custom inter-agent chat protocol or Task/Run service. diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh index 0cf4d203e..a94ddaaf6 100755 --- a/bin/agent-os-container-entrypoint.sh +++ b/bin/agent-os-container-entrypoint.sh @@ -16,6 +16,8 @@ mkdir -p \ "$HOME/.local/share" \ "$HOME/.bun" \ "$HOME/.cargo" + +"$(dirname "$0")/agent-os-kubeconfig.sh" if [ ! -e "$FM_HOME/config/backend" ]; then printf 'herdr\n' > "$FM_HOME/config/backend" fi diff --git a/bin/agent-os-init.sh b/bin/agent-os-init.sh index 4b88ae911..385f5bd7d 100755 --- a/bin/agent-os-init.sh +++ b/bin/agent-os-init.sh @@ -10,6 +10,7 @@ mkdir -p \ "$PERSISTENT_ROOT/.cache" \ "$PERSISTENT_ROOT/.local/bin" \ "$PERSISTENT_ROOT/.local/share" \ + "$PERSISTENT_ROOT/.pi/agent" \ "$PERSISTENT_ROOT/.bun" \ "$PERSISTENT_ROOT/.cargo" \ "$PERSISTENT_ROOT/usr-local" diff --git a/bin/agent-os-kubeconfig.sh b/bin/agent-os-kubeconfig.sh new file mode 100755 index 000000000..aeef3c2c6 --- /dev/null +++ b/bin/agent-os-kubeconfig.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Create a token-file-backed kubeconfig when this container has a ServiceAccount. +set -eu + +HOME=${HOME:-/home/agent} +service_account_dir=${AGENT_OS_SERVICE_ACCOUNT_DIR:-/var/run/secrets/kubernetes.io/serviceaccount} +kubeconfig=${AGENT_OS_KUBECONFIG_PATH:-$HOME/.kube/config} + +[ -r "$service_account_dir/token" ] || exit 0 +[ -r "$service_account_dir/ca.crt" ] || exit 0 +[ -e "$kubeconfig" ] && exit 0 + +: "${KUBERNETES_SERVICE_HOST:?KUBERNETES_SERVICE_HOST is required with a ServiceAccount token}" +mkdir -p "$(dirname "$kubeconfig")" +umask 077 +cat > "$kubeconfig" < "$tmp/serviceaccount/token" +printf '%s\n' demo-ca > "$tmp/serviceaccount/ca.crt" + +HOME="$tmp/home" \ +AGENT_OS_SERVICE_ACCOUNT_DIR="$tmp/serviceaccount" \ +KUBERNETES_SERVICE_HOST=kubernetes.default.svc \ +KUBERNETES_SERVICE_PORT_HTTPS=443 \ +AGENT_OS_NAMESPACE=agent-os-eval \ + "$ROOT/bin/agent-os-kubeconfig.sh" + +config="$tmp/home/.kube/config" +[ -f "$config" ] || fail "in-cluster kubeconfig was not created" +assert_grep 'current-context: in-cluster' "$config" "kubeconfig must select the local cluster alias" +assert_grep 'namespace: agent-os-eval' "$config" "kubeconfig must select the Agent OS namespace" +assert_grep "tokenFile: $tmp/serviceaccount/token" "$config" "kubeconfig must follow the rotating token file" +assert_no_grep 'demo-token' "$config" "kubeconfig must not copy token contents" +[ "$(stat -f '%Lp' "$config" 2>/dev/null || stat -c '%a' "$config")" = 600 ] || fail "kubeconfig must be mode 600" + +printf '%s\n' sentinel > "$config" +HOME="$tmp/home" \ +AGENT_OS_SERVICE_ACCOUNT_DIR="$tmp/serviceaccount" \ +KUBERNETES_SERVICE_HOST=kubernetes.default.svc \ + "$ROOT/bin/agent-os-kubeconfig.sh" +[ "$(cat "$config")" = sentinel ] || fail "existing kubeconfig must not be overwritten" + +pass "Agent OS creates a rotation-safe in-cluster kubeconfig without exposing token contents" diff --git a/tools/agent-os/AGENTS.md b/tools/agent-os/AGENTS.md index 872166ee7..ed564429e 100644 --- a/tools/agent-os/AGENTS.md +++ b/tools/agent-os/AGENTS.md @@ -5,7 +5,7 @@ They govern development of the operating tool and must not be copied into the ro ## Purpose -This tool incrementally moves complex fleet mechanics from Bash into a small Bun and TypeScript CLI. +This tool owns only stable Agent OS primitives that the underlying general-purpose tools do not already provide. It extends Firstmate's existing files and lifecycle rather than introducing a parallel state store, controller, or agent model. Keep compatibility with the root distro's `data/`, `state/`, briefs, status events, and guarded teardown behavior. Use the Bun channel declared in `.bun-version`. @@ -31,26 +31,30 @@ The address grammar and validation contract are owned by `src/address.ts` and it ## Command Execution -Use the official `@akua-dev/sdk` for Akua operations implemented inside this TypeScript tool. -Keep the bundled `akua` CLI available for Firstmate's direct and interactive use. -Use Bun Shell's `$` for other finite calls to trusted CLIs such as `kubectl`, `herdr`, `treehouse`, and Git. +Keep the bundled `akua` CLI available for Firstmate's direct use instead of proxying it through this tool. +Use Bun Shell's `$` for finite calls to trusted CLIs only when implementing a genuinely missing Agent OS primitive. Interpolate dynamic values directly so Bun treats each value as a literal argument. Never use `{ raw: value }` for dynamic data. Avoid `bash -c` and validate values that an external command could interpret as flags. Keep non-zero exits throwing by default and use `.nothrow()` only for an expected negative probe. Set environment and working directory per command instead of mutating the global `$` instance. Use `Bun.spawn()` for interactive agents, servers, watches, live logs, cancellation, and other long-running processes. -Existing Bash scripts may be invoked explicitly during migration, but new orchestration and state transitions belong in TypeScript. +Existing Bash scripts may be invoked explicitly during migration, but do not move adaptable orchestration into TypeScript merely because it is currently written in Bash or instructions. ## Akua packages The prepared mate package lives at `packages/mate/` and renders ordinary Kubernetes YAML. Treat it as an editable convenience, not a required abstraction or controller. -Firstmate may use `agent-os mate render`, invoke `akua render` itself, edit the result, or author equivalent YAML directly. +Firstmate may invoke `akua render`, edit the result, change the package, or author equivalent YAML directly. ## Scope Discipline -Prefer a thin adapter over a new framework. +Prefer no adapter when an agent can reliably compose the underlying tools itself. +When a stable safety boundary truly needs code, prefer the thinnest adapter that owns only that boundary. +Do not wrap a capable general-purpose CLI merely to encode one Agent OS workflow. +Prefer instructions, skills, and direct composition of Akua, Kubernetes, Herdr, Git, and other trusted tools. +Add TypeScript only for a stable primitive the underlying tools do not already provide, such as the Agent OS address grammar, or when repeated failures prove a reusable safety boundary is missing. +Keep policy and desired outcomes in instructions instead of hardcoding how an adaptable agent must sequence tools. Do not add a database, daemon, custom inter-agent protocol, Kubernetes controller, or duplicate task specification. Keep mates general-purpose and keep parent-only communication unchanged across local and remote addresses. Use Kubernetes APIs and later Agent Sandbox only as runtime resolvers behind the same logical address. diff --git a/tools/agent-os/README.md b/tools/agent-os/README.md index 7776509b3..f0362dc01 100644 --- a/tools/agent-os/README.md +++ b/tools/agent-os/README.md @@ -23,11 +23,9 @@ bun run check bun run src/cli.ts address local task-x bun run src/cli.ts address k8s task-x bun run src/cli.ts address inspect /k8s/in-cluster/namespace/agent-os/mate/task-x/herdr -bun run src/cli.ts mate render task-x --out /tmp/task-x ``` -Akua package operations inside this tool use the official `@akua-dev/sdk` in process. -The prepared package under `packages/mate/` remains optional and renders editable Kubernetes YAML. +The prepared Akua package under `packages/mate/` remains optional and renders editable Kubernetes YAML through the bundled `akua` CLI. The scoped official `Effect-TS/skills` installation guides typed Effect workflows; its Effect source checkout is local and ignored rather than committed as a snapshot. See `AGENTS.md` in this directory for the tool-specific development rules. diff --git a/tools/agent-os/bun.lock b/tools/agent-os/bun.lock index c0ea2c18d..717d11312 100644 --- a/tools/agent-os/bun.lock +++ b/tools/agent-os/bun.lock @@ -5,9 +5,7 @@ "": { "name": "@akua-dev/agent-os-tool", "dependencies": { - "@akua-dev/sdk": "0.8.24", "@multiformats/multiaddr": "13.0.3", - "effect": "3.21.4", }, "devDependencies": { "@types/bun": "1.3.14", @@ -16,32 +14,10 @@ }, }, "packages": { - "@akua-dev/native": ["@akua-dev/native@0.8.24", "", { "dependencies": { "@akua-dev/native-engines": "0.8.24" }, "optionalDependencies": { "@akua-dev/native-darwin-arm64": "0.8.24", "@akua-dev/native-darwin-x64": "0.8.24", "@akua-dev/native-linux-arm64-gnu": "0.8.24", "@akua-dev/native-linux-arm64-musl": "0.8.24", "@akua-dev/native-linux-x64-gnu": "0.8.24", "@akua-dev/native-linux-x64-musl": "0.8.24", "@akua-dev/native-win32-x64-msvc": "0.8.24" } }, "sha512-DZsUSd0SaLrqBvbmsEmPD9rUwxrUZSC/QTNfclzRgmKxnK2y5x90pT+J4L/Aoa3lCFWpCp9MOM9pZ7ztjHL1Nw=="], - - "@akua-dev/native-darwin-arm64": ["@akua-dev/native-darwin-arm64@0.8.24", "", { "os": "darwin", "cpu": "arm64" }, "sha512-6Wa2zDcrRWQUPouHKmZUom4kNmffvWYxcrvnaeoVwbQFj3TXWKAaCzcVYjZGeLcV8HxMkTrYx4rt/NH5GRLkWA=="], - - "@akua-dev/native-darwin-x64": ["@akua-dev/native-darwin-x64@0.8.24", "", { "os": "darwin", "cpu": "x64" }, "sha512-hhXiqfpw9itn6cdNhC0IVW1WUaPBmpYmBPW7hfkBuaTlP/2wPyAUx+Wu0cJWpna1YCUg3WzcUOj5rm/lyciP9w=="], - - "@akua-dev/native-engines": ["@akua-dev/native-engines@0.8.24", "", {}, "sha512-3rKiWQn1pIURERUmxNiqPw7SxtMrNwX5tDxsCRxsAZ7Ovzfzcwx/VcycfuY+z1DNhEJrkKChkYiMT3N/EZanmg=="], - - "@akua-dev/native-linux-arm64-gnu": ["@akua-dev/native-linux-arm64-gnu@0.8.24", "", { "os": "linux", "cpu": "arm64" }, "sha512-NTeuJRCjy68JJT5LuHwPHf11mnZ1a5hNeKNfoBYMzUeViN3BdnBuszjNxwYHAgQYEN8+Rf5EGtTMdOTm0xDy/Q=="], - - "@akua-dev/native-linux-arm64-musl": ["@akua-dev/native-linux-arm64-musl@0.8.24", "", { "os": "linux", "cpu": "arm64" }, "sha512-QbGJfTY2Ilv+FSVaIU5MnlHjprbKVM41Ya+YfanHJsTBwO0dUw6J9C/fNVKTHeQM98vLVBIWdLiAPl73jSI/Qw=="], - - "@akua-dev/native-linux-x64-gnu": ["@akua-dev/native-linux-x64-gnu@0.8.24", "", { "os": "linux", "cpu": "x64" }, "sha512-jJDnr4MGYxB/vflhGklC1dUxHsO7HNuWexDFGB/vPhzo6RGsAWsApnrqPYYfBDmsk0nvvZKO9OWTB+e837gCzA=="], - - "@akua-dev/native-linux-x64-musl": ["@akua-dev/native-linux-x64-musl@0.8.24", "", { "os": "linux", "cpu": "x64" }, "sha512-21EsbM2fArJjX4RKXeVXDPxSBHkeMtYVCvT9vDj4S1lo5Ld0xRM/utm6EQkfbShdpwWZRhkv/t7evgdVM8Zb2A=="], - - "@akua-dev/native-win32-x64-msvc": ["@akua-dev/native-win32-x64-msvc@0.8.24", "", { "os": "win32", "cpu": "x64" }, "sha512-boG/GW6KTucoq7PUlU92tvB91DGxl4hNdQICcsXwgO7npzLqNEhCzKYnpuwKgqhK6vSpoPd0/jZdmu6EwUyzKQ=="], - - "@akua-dev/sdk": ["@akua-dev/sdk@0.8.24", "", { "dependencies": { "@akua-dev/native": "0.8.24", "@standard-schema/spec": "^1.1.0", "ajv": "^8.18.0", "yaml": "^2.8.3" } }, "sha512-8OM8luAqmxnA4kRJUGbf4Jhr4YuSf/lXKAOUD4kZTUepoRvgYP3xHlXJRCVd1Qv00L1NmeluXG8sr5rh/soGIg=="], - "@chainsafe/is-ip": ["@chainsafe/is-ip@2.1.0", "", {}, "sha512-KIjt+6IfysQ4GCv66xihEitBjvhU/bixbbbFxdJ1sqCp4uJ0wuZiYBPhksZoy4lfaF0k9cwNzY5upEW/VWdw3w=="], "@multiformats/multiaddr": ["@multiformats/multiaddr@13.0.3", "", { "dependencies": { "@chainsafe/is-ip": "^2.0.1", "multiformats": "^14.0.0", "uint8-varint": "^3.0.0", "uint8arrays": "^6.1.1" } }, "sha512-mEqqJ4r3a/uuFMTpRkU316wGNIDQNhuVWpm+ebKTQeYsfv9jXbPONWM6VVnj3KGUrwfsX7GZOyp4TFqEA2SPCw=="], - "@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], - "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], "@types/node": ["@types/node@26.1.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw=="], @@ -86,26 +62,10 @@ "@typescript/typescript-win32-x64": ["@typescript/typescript-win32-x64@7.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g=="], - "ajv": ["ajv@8.20.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA=="], - "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], - "effect": ["effect@3.21.4", "", { "dependencies": { "@standard-schema/spec": "^1.0.0", "fast-check": "^3.23.1" } }, "sha512-B89v/xSgPbl1J2Ai2u18jxq3odpFauU1rC6/eSs4FeNHi72kwKdJp12VGigvRV2lK+kRnx+OOz41XV8guZd4gQ=="], - - "fast-check": ["fast-check@3.23.2", "", { "dependencies": { "pure-rand": "^6.1.0" } }, "sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A=="], - - "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], - - "fast-uri": ["fast-uri@3.1.3", "", {}, "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg=="], - - "json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="], - "multiformats": ["multiformats@14.0.4", "", {}, "sha512-+SXItmOUWzT0kjlIfkZ4NawJaC9jW/mPlyn902V9Qgpc7YDwdEiwqbiZxTyvC0XWh1jZguXca3A/WQ+UOPRLUA=="], - "pure-rand": ["pure-rand@6.1.0", "", {}, "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA=="], - - "require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="], - "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], "uint8-varint": ["uint8-varint@3.0.0", "", { "dependencies": { "uint8arraylist": "^3.0.1", "uint8arrays": "^6.1.0" } }, "sha512-S4DdpXBaLwKcFo7f0bWzWfHjbZ/i3QhM842qn+ZvHjxqFCfUcEB9SQNcmI69S+zMlcmIcKxsk9Iyw77S2Kxv6Q=="], @@ -115,7 +75,5 @@ "uint8arrays": ["uint8arrays@6.1.1", "", { "dependencies": { "multiformats": "^14.0.0" } }, "sha512-iz7JN0XCSZYA111lhFG2Ui9EhFvTNekqSRHw3lvMHq+dzwWy1OQftxFQREEh4rffU0oSoXdQHsk2TiHKVm4fsA=="], "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], - - "yaml": ["yaml@2.9.0", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA=="], } } diff --git a/tools/agent-os/package.json b/tools/agent-os/package.json index ba519baaa..cf3f52ee6 100644 --- a/tools/agent-os/package.json +++ b/tools/agent-os/package.json @@ -13,9 +13,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@akua-dev/sdk": "0.8.24", - "@multiformats/multiaddr": "13.0.3", - "effect": "3.21.4" + "@multiformats/multiaddr": "13.0.3" }, "devDependencies": { "@types/bun": "1.3.14", diff --git a/tools/agent-os/packages/mate/README.md b/tools/agent-os/packages/mate/README.md index 5576ebec4..71c5fe0b0 100644 --- a/tools/agent-os/packages/mate/README.md +++ b/tools/agent-os/packages/mate/README.md @@ -9,12 +9,6 @@ Render the example directly: akua render --package package.k --inputs inputs.example.yaml --out /tmp/scout-1 ``` -Or use the Agent OS convenience command from any directory: - -```sh -agent-os mate render scout-1 --namespace agent-os-demo --image agent-os:dev --out /tmp/scout-1 -``` - Inspect and edit the rendered YAML before applying it: ```sh @@ -23,3 +17,7 @@ kubectl apply -f /tmp/scout-1 Firstmate may change this package, supply additional inputs, edit the rendered files, or author equivalent YAML directly. The package intentionally grants no Kubernetes token and no privileged, host-namespace, or host-mount access. + +AI credentials are an explicit grant. +Set `piAuthSecret` to the name of an existing Kubernetes Secret whose `auth.json` key contains the selected Pi credential set. +The package mounts only that Secret reference read-only; it never discovers or copies credentials by itself. diff --git a/tools/agent-os/packages/mate/inputs.example.yaml b/tools/agent-os/packages/mate/inputs.example.yaml index 89d21d97f..3f0136f73 100644 --- a/tools/agent-os/packages/mate/inputs.example.yaml +++ b/tools/agent-os/packages/mate/inputs.example.yaml @@ -3,3 +3,4 @@ namespace: agent-os-demo address: /k8s/in-cluster/namespace/agent-os-demo/mate/scout-1/herdr image: agent-os:dev imagePullPolicy: Never +piAuthSecret: agent-os-mate-scout-1-pi-auth diff --git a/tools/agent-os/packages/mate/package.k b/tools/agent-os/packages/mate/package.k index f12829369..2ed2259c6 100644 --- a/tools/agent-os/packages/mate/package.k +++ b/tools/agent-os/packages/mate/package.k @@ -8,6 +8,7 @@ schema Input: address: str image: str = "agent-os:dev" imagePullPolicy: "Always" | "IfNotPresent" | "Never" = "IfNotPresent" + piAuthSecret: str = "" storage: str = "10Gi" cpuRequest: str = "250m" memoryRequest: str = "512Mi" @@ -25,6 +26,21 @@ labels = { "agent-os.akua.dev/mate" = input.mateId } +piAuthMounts = [{ + name = "pi-auth" + mountPath = "/home/agent/.pi/agent/auth.json" + subPath = "auth.json" + readOnly = True +}] if input.piAuthSecret else [] + +piAuthVolumes = [{ + name = "pi-auth" + secret = { + secretName = input.piAuthSecret + defaultMode = 256 + } +}] if input.piAuthSecret else [] + resources = [ { apiVersion = "v1" @@ -90,12 +106,12 @@ resources = [ volumeMounts = [ { name = "home", mountPath = "/home/agent" }, { name = "home", mountPath = "/usr/local", subPath = "usr-local" }, - ] + ] + piAuthMounts }] volumes = [{ name = "home" persistentVolumeClaim = { claimName = "agent-os-mate-${input.mateId}-home" } - }] + }] + piAuthVolumes } }, ] diff --git a/tools/agent-os/src/cli.ts b/tools/agent-os/src/cli.ts index 48830c7d0..5cd221ed4 100755 --- a/tools/agent-os/src/cli.ts +++ b/tools/agent-os/src/cli.ts @@ -1,20 +1,17 @@ #!/usr/bin/env bun import { parseArgs } from "node:util"; -import { Effect } from "effect"; import { formatKubernetesAgentAddress, formatLocalAgentAddress, parseAgentAddress, } from "./address.ts"; -import { renderMatePackage } from "./mate-package.ts"; const usage = `Usage: agent-os address local agent-os address k8s [--cluster ] [--namespace ] - agent-os address inspect - agent-os mate render [--cluster ] [--namespace ] [--image ] [--out ]`; + agent-os address inspect `; function requireSingleArgument(args: string[]): string { if (args.length !== 1 || args[0] == null) { @@ -70,68 +67,18 @@ function runAddressCommand(args: string[]): void { } } -async function runMateCommand(args: string[]): Promise { - const [command, ...rest] = args; +function main(): void { + const [command, ...rest] = Bun.argv.slice(2); - if (command !== "render") { + if (command !== "address") { throw new Error(usage); } - const { positionals, values } = parseArgs({ - allowPositionals: true, - args: rest, - options: { - cluster: { - default: "in-cluster", - type: "string", - }, - image: { - default: "agent-os:dev", - type: "string", - }, - namespace: { - default: "agent-os", - type: "string", - }, - out: { - type: "string", - }, - }, - strict: true, - }); - const mateId = requireSingleArgument(positionals); - const result = await Effect.runPromise( - renderMatePackage({ - cluster: values.cluster, - image: values.image, - mateId, - namespace: values.namespace, - outDir: values.out, - }).pipe( - Effect.catchAll((error) => Effect.fail(new Error(error.message))), - ), - ); - - console.log(JSON.stringify(result, null, 2)); -} - -async function main(): Promise { - const [command, ...rest] = Bun.argv.slice(2); - - switch (command) { - case "address": - runAddressCommand(rest); - return; - case "mate": - await runMateCommand(rest); - return; - default: - throw new Error(usage); - } + runAddressCommand(rest); } try { - await main(); + main(); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; diff --git a/tools/agent-os/src/json-modules.d.ts b/tools/agent-os/src/json-modules.d.ts deleted file mode 100644 index b14866845..000000000 --- a/tools/agent-os/src/json-modules.d.ts +++ /dev/null @@ -1,4 +0,0 @@ -declare module "*.json" { - const value: { readonly $defs: Readonly> }; - export default value; -} diff --git a/tools/agent-os/src/mate-package.ts b/tools/agent-os/src/mate-package.ts deleted file mode 100644 index 9d9e0306d..000000000 --- a/tools/agent-os/src/mate-package.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { - Akua, - AkuaRateLimitedError, - AkuaUserError, - type RenderOptions, - type RenderSummary, -} from "@akua-dev/sdk"; -import { Data, Effect } from "effect"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { dirname, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; - -import { formatKubernetesAgentAddress } from "./address.ts"; - -export interface RenderMatePackageOptions { - cluster?: string; - image?: string; - mateId: string; - namespace?: string; - outDir?: string; -} - -export interface RenderedMatePackage { - address: string; - outDir: string; -} - -export class MatePackageInputError extends Data.TaggedError( - "MatePackageInputError", -)<{ - readonly cause: unknown; - readonly message: string; -}> {} - -export class AkuaRenderError extends Data.TaggedError("AkuaRenderError")<{ - readonly cause: unknown; - readonly message: string; -}> {} - -export class AkuaPackageUserError extends Data.TaggedError( - "AkuaPackageUserError", -)<{ - readonly cause: Error; - readonly message: string; -}> {} - -export class AkuaPackageRateLimitedError extends Data.TaggedError( - "AkuaPackageRateLimitedError", -)<{ - readonly cause: Error; - readonly message: string; -}> {} - -export interface AkuaRenderer { - render(options: RenderOptions): Promise; -} - -const packageDirectory = resolve( - dirname(fileURLToPath(import.meta.url)), - "../packages/mate", -); - -const defaultAkua = new Akua(); - -function mapAkuaError(cause: unknown) { - if (cause instanceof AkuaRateLimitedError) { - return new AkuaPackageRateLimitedError({ - cause, - message: "Akua rate-limited the mate package render", - }); - } - if (cause instanceof AkuaUserError) { - return new AkuaPackageUserError({ - cause, - message: cause.message, - }); - } - return new AkuaRenderError({ - cause, - message: "Akua could not render the mate package", - }); -} - -export const renderMatePackage = Effect.fn("agent-os.renderMatePackage")( - function* ( - options: RenderMatePackageOptions, - akua: AkuaRenderer = defaultAkua, - ) { - const cluster = options.cluster ?? "in-cluster"; - const namespace = options.namespace ?? "agent-os"; - const address = formatKubernetesAgentAddress(options.mateId, { - cluster, - namespace, - }); - const outDir = resolve( - options.outDir ?? resolve(process.cwd(), "deploy", options.mateId), - ); - const temporaryDirectory = yield* Effect.tryPromise({ - try: () => mkdtemp(resolve(tmpdir(), "agent-os-mate-")), - catch: (cause) => - new MatePackageInputError({ - cause, - message: "Could not create temporary Akua inputs", - }), - }); - - const render = Effect.gen(function* () { - const inputsPath = resolve(temporaryDirectory, "inputs.json"); - yield* Effect.tryPromise({ - try: () => - writeFile( - inputsPath, - JSON.stringify( - { - address, - image: options.image ?? "agent-os:dev", - mateId: options.mateId, - namespace, - }, - null, - 2, - ), - ), - catch: (cause) => - new MatePackageInputError({ - cause, - message: "Could not write temporary Akua inputs", - }), - }); - yield* Effect.tryPromise({ - try: () => - akua.render({ - inputs: inputsPath, - out: outDir, - package: resolve(packageDirectory, "package.k"), - timeout: "30s", - }), - catch: mapAkuaError, - }); - - return { address, outDir } satisfies RenderedMatePackage; - }); - - return yield* render.pipe( - Effect.ensuring( - Effect.promise(() => rm(temporaryDirectory, { force: true, recursive: true })), - ), - ); - }, -); diff --git a/tools/agent-os/test/mate-package.test.ts b/tools/agent-os/test/mate-package.test.ts deleted file mode 100644 index 97b72dfee..000000000 --- a/tools/agent-os/test/mate-package.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { Effect } from "effect"; -import { afterEach, describe, expect, test } from "bun:test"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { resolve } from "node:path"; - -import { - type AkuaRenderer, - renderMatePackage, -} from "../src/mate-package.ts"; - -const temporaryDirectories: string[] = []; - -afterEach(async () => { - await Promise.all( - temporaryDirectories.splice(0).map((directory) => - rm(directory, { force: true, recursive: true }), - ), - ); -}); - -describe("agent-os mate render", () => { - test("passes typed inputs to the Akua SDK and reports the address", async () => { - const root = await mkdtemp(resolve(tmpdir(), "agent-os-mate-test-")); - temporaryDirectories.push(root); - const outDir = resolve(root, "rendered"); - let capturedInputs: unknown; - const akua: AkuaRenderer = { - async render(options) { - capturedInputs = JSON.parse( - await readFile(options.inputs ?? "", "utf8"), - ); - return { - files: [], - format: "raw-manifests", - hash: "sha256:test", - manifests: 2, - target: options.out ?? outDir, - }; - }, - }; - - const output = await Effect.runPromise( - renderMatePackage( - { - image: "ghcr.io/akua-dev/agent-os:test", - mateId: "scout-1", - namespace: "agent-os-demo", - outDir, - }, - akua, - ), - ); - - expect(output).toEqual({ - address: - "/k8s/in-cluster/namespace/agent-os-demo/mate/scout-1/herdr", - outDir, - }); - expect(capturedInputs).toEqual({ - address: - "/k8s/in-cluster/namespace/agent-os-demo/mate/scout-1/herdr", - image: "ghcr.io/akua-dev/agent-os:test", - mateId: "scout-1", - namespace: "agent-os-demo", - }); - }); - - test("keeps unexpected SDK failures in the typed error channel", async () => { - const akua: AkuaRenderer = { - render: () => Promise.reject(new Error("native addon failed")), - }; - const exit = await Effect.runPromiseExit( - renderMatePackage({ mateId: "scout-1" }, akua), - ); - - expect(exit._tag).toBe("Failure"); - expect(String(exit)).toContain("AkuaRenderError"); - expect(String(exit)).toContain("Akua could not render the mate package"); - }); -}); From e1e3b108365f835dc86098293aa4d5d0d7bc9f15 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 12:01:10 +0200 Subject: [PATCH 13/56] feat: add Akua Firstmate bootstrap package --- .../akua-intelligence-bootstrap/SKILL.md | 84 ++++++++ .agents/skills/kubernetes-fleet/SKILL.md | 1 + .github/workflows/agent-os-image.yml | 61 ++++++ AGENTS.md | 1 + README.md | 6 +- docs/acceptance.md | 26 +++ docs/evidence/2026-07-12-firstmate-package.md | 70 +++++++ tests/agent-os-container.test.sh | 6 + tests/agent-os-packages.test.sh | 36 ++++ tools/agent-os/packages/firstmate/README.md | 21 ++ tools/agent-os/packages/firstmate/akua.toml | 5 + .../packages/firstmate/inputs.example.yaml | 5 + tools/agent-os/packages/firstmate/package.k | 182 ++++++++++++++++++ 13 files changed, 503 insertions(+), 1 deletion(-) create mode 100644 .agents/skills/akua-intelligence-bootstrap/SKILL.md create mode 100644 .github/workflows/agent-os-image.yml create mode 100644 docs/acceptance.md create mode 100644 docs/evidence/2026-07-12-firstmate-package.md create mode 100755 tests/agent-os-packages.test.sh create mode 100644 tools/agent-os/packages/firstmate/README.md create mode 100644 tools/agent-os/packages/firstmate/akua.toml create mode 100644 tools/agent-os/packages/firstmate/inputs.example.yaml create mode 100644 tools/agent-os/packages/firstmate/package.k diff --git a/.agents/skills/akua-intelligence-bootstrap/SKILL.md b/.agents/skills/akua-intelligence-bootstrap/SKILL.md new file mode 100644 index 000000000..d32a28491 --- /dev/null +++ b/.agents/skills/akua-intelligence-bootstrap/SKILL.md @@ -0,0 +1,84 @@ +--- +name: akua-intelligence-bootstrap +description: "Bootstrap or recover a dedicated Agent OS intelligence cluster through Akua's native API, MCP, CLI, Akua packages, and Kubernetes without adding an Agent OS provisioning service." +user-invocable: false +metadata: + internal: true +--- + +# Akua intelligence bootstrap + +Load this skill before provisioning, recovering, or handing off a Firstmate in an Akua-managed intelligence cluster. + +## Boundary + +- The Akua workspace and Hetzner project must be dedicated to Agent OS infrastructure and contain no product, production, or customer resources. +- Provisioning machines, creating or revoking credentials, retrieving kubeconfig, and granting cluster-admin require the captain's explicit approval for the exact run. +- An available credential proves capability, not approval. +- Use Akua's native Platform MCP, REST API, CLI, and packages directly. +- Do not build or invoke an Agent OS provisioning wrapper, controller, Install, or GitOps workflow. +- Use a fresh idempotency key for each intended create and reuse that key only when retrying the same intended operation. +- Never put an Akua token, Hetzner token, kubeconfig, or authorization header in Git, a prompt, a status file, command arguments, or retained evidence. + +## Native API surface + +The current public API base is `https://api.akua.dev/v1`. +When using REST, keep a complete HTTP authorization header in a mode-`0400` file and pass it with curl's native `-H @file` support. +Write secret-bearing JSON through protected files or stdin, never shell arguments. + +Use the Platform MCP `search` tool again before a mutation so current schemas outrank this routing table. + +| Outcome | Native endpoint | +| --- | --- | +| Store the approved Hetzner token | `POST /secrets`, kind `cloud_provider/hcloud` | +| Validate the provider token | `POST /secrets/validate_token` | +| Create managed KaaS | `POST /clusters` | +| Wait for cluster creation | `POST /operations/{id}:wait?timeout=60` | +| Create Hetzner compute configuration | `POST /compute/configs` | +| Inspect available capacity and price | `GET /compute/instance_types?config={config}` | +| Create a worker | `POST /compute/machines` | +| Inspect worker state | `GET /compute/machines/{providerId}` | +| Retrieve the approved owner kubeconfig | `GET /clusters/{id}/kubeconfig` | +| Create the clustered-Firstmate token | `POST /api_tokens` | +| Inventory tokens | `GET /api_tokens` | +| Revoke the local bootstrap token | `DELETE /api_tokens/{id}` | + +## Bootstrap procedure + +1. Record the approved workspace ID, intended region, machine constraints, expiration or cleanup condition, and evidence directory without recording secrets. +2. Read the live workspace, Secret, cluster, compute configuration, machine, and token inventories before creating anything. +3. Stop if the workspace or Hetzner project contains production resources or if an existing matching resource makes intent ambiguous. +4. Validate the provider token without storing it, then create or select the `cloud_provider/hcloud` Secret and require server validation state `valid` before creating compute. +5. Create the managed cluster with an idempotency key, preserve its operation ID, and wait for terminal `SUCCEEDED` state. +6. Create the compute configuration from the validated Secret, inspect available instance types, and select the smallest type satisfying the approved CPU, RAM, disk, architecture, price, and region constraints. +7. Create the worker, then verify both Akua machine state and Kubernetes Node readiness. +8. Retrieve kubeconfig only into a protected temporary file and verify the cluster identity before applying anything. +9. Build or select a published Agent OS image by immutable digest. +10. Create a distinct clustered-Firstmate API token and immediately convert it into a Kubernetes Secret named by `akuaAuthSecret`. + The Secret's `authorization` key contains the complete header consumed through `AKUA_AUTH_HEADER_FILE`. +11. Render `tools/agent-os/packages/firstmate/package.k` with `akua render`, inspect the ordinary YAML, and apply it with `kubectl`. +12. Verify the StatefulSet is ready, Herdr responds, the persistent home is writable, the `in-cluster` context is cluster-admin only in this intelligence cluster, and the Pod can list its Akua workspace through `curl -H @"$AKUA_AUTH_HEADER_FILE"`. +13. From the clustered Firstmate, inventory all workspace token IDs and prove the new token can perform the required Akua and Kubernetes reads. +14. Only after that handoff succeeds, revoke the known local bootstrap token and prove it now receives an authentication failure. +15. Destroy every protected temporary file and record only non-secret resource IDs, operation IDs, timestamps, states, image digest, test results, costs, and interventions. + +## Recovery and cleanup + +- A Pod restart reuses the Firstmate PVC and mounted Kubernetes Secret. +- A worker replacement must preserve or reattach every PVC holding unique work before deleting the old worker. +- Do not claim cluster-loss recovery until an external encrypted backup has been restored in a clean cluster. +- Machine, cluster, Secret, token, and workspace deletion are separate destructive actions. +- Execute only the approved cleanup scope and verify retained resources afterward. + +## Completion evidence + +The bootstrap is complete only when current external state proves all of these: + +- Akua operation IDs reached `SUCCEEDED`; +- the worker is ready in Akua and Kubernetes; +- the Firstmate StatefulSet and Herdr server are ready; +- the home and installed-tool paths survive a Pod replacement; +- the clustered token works from the Pod; +- the bootstrap token is revoked and fails authentication; +- no product or production resource exists in the intelligence workspace; and +- the evidence record contains no secret values. diff --git a/.agents/skills/kubernetes-fleet/SKILL.md b/.agents/skills/kubernetes-fleet/SKILL.md index 6ecbd247c..37782cf7e 100644 --- a/.agents/skills/kubernetes-fleet/SKILL.md +++ b/.agents/skills/kubernetes-fleet/SKILL.md @@ -15,6 +15,7 @@ Load this skill only when the current firstmate is running in Kubernetes or is e - Keep every crewmate general-purpose; its brief, tools, and authority specialize it for the current task. - A crewmate normally communicates only with its parent through its terminal, status files, reports, and delivered Git state. - The image includes Akua's CLI and the prepared mate package lives at `tools/agent-os/packages/mate/`. +- The optional persistent controller package lives at `tools/agent-os/packages/firstmate/`; load `akua-intelligence-bootstrap` before using it against Akua-managed infrastructure. - Use the bundled K9s terminal UI when it makes live Kubernetes inspection faster than individual `kubectl` reads. - Render the package with `akua render`, inspect or edit its ordinary YAML when useful, and apply it with `kubectl`. - Treat AI credentials as explicit per-mate grants, never as ambient inheritance merely because Firstmate can read them. diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml new file mode 100644 index 000000000..a529a5864 --- /dev/null +++ b/.github/workflows/agent-os-image.yml @@ -0,0 +1,61 @@ +name: Agent OS image + +on: + push: + branches: [main] + tags: ["v*"] + pull_request: + branches: [main] + +permissions: + contents: read + packages: write + +concurrency: + group: agent-os-image-${{ github.ref }} + cancel-in-progress: true + +env: + IMAGE: ghcr.io/akua-dev/agent-os + +jobs: + build: + name: Build linux/amd64 and linux/arm64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: docker/setup-qemu-action@v3 + + - uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Compute image tags + id: metadata + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE }} + tags: | + type=sha,format=long + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=tag + + - name: Build and optionally publish + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: mode=max + sbom: true diff --git a/AGENTS.md b/AGENTS.md index 831acb54f..8518c841f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -771,6 +771,7 @@ These skills are not captain-invocable; they are conditional operating reference - `firstmate-codexapp` - load before coordinating a visible Codex Desktop thread, evaluating a Codex App backend request, or reconciling Codex Desktop host-tool smoke evidence for Firstmate work. - `firstmate-coding-guidelines` - load before changing firstmate's shared, tracked material, as defined by section 1's list, whether editing directly or briefing a crewmate for a firstmate-repo task. - `kubernetes-fleet` - load before creating, supervising, recovering, or deleting Kubernetes-backed crewmates. +- `akua-intelligence-bootstrap` - load before provisioning, recovering, or handing off a Firstmate in an Akua-managed intelligence cluster. ## 14. X mode diff --git a/README.md b/README.md index 432984eb5..ec05344ef 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,10 @@ Launching a supported harness inside it instantiates your first mate - and makes This Agent OS fork also packages the distro for Kubernetes: one persistent first mate can allocate isolated, persistent crewmate containers using ordinary `kubectl`, while Herdr keeps every agent terminal visible. Start locally with -the [OrbStack Kubernetes demo](docs/kubernetes.md). +the [OrbStack Kubernetes demo](docs/kubernetes.md), then use the optional +[Firstmate Akua package](tools/agent-os/packages/firstmate/README.md) and the +[focused bootstrap guidance](.agents/skills/akua-intelligence-bootstrap/SKILL.md) +for a dedicated Akua-managed intelligence cluster. ## Features @@ -194,6 +197,7 @@ Firstmate's skills live in two separate places with different audiences: ## Documentation - [docs/kubernetes.md](docs/kubernetes.md) - run the Agent OS controller and isolated crewmates on local OrbStack Kubernetes. +- [docs/acceptance.md](docs/acceptance.md) - requirement-to-evidence ledger for the full distributed Agent OS proof. - [docs/architecture.md](docs/architecture.md) - how the crew, supervision, worktrees, secondmates, and project modes work. - [docs/configuration.md](docs/configuration.md) - environment variables, `FM_HOME`, runtime backend selection, optional X mode, the files you set, and harness support. - [docs/wedge-alarm.md](docs/wedge-alarm.md) - configure the active alert for a wedged away-mode escalation delivery. diff --git a/docs/acceptance.md b/docs/acceptance.md new file mode 100644 index 000000000..7d7463890 --- /dev/null +++ b/docs/acceptance.md @@ -0,0 +1,26 @@ +# Agent OS acceptance ledger + +This ledger maps the Cortex distributed-Firstmate acceptance contract to current claim-matched evidence. +Update it from real runs; planned behavior and model narration never count as proof. + +| Requirement | Current evidence | State | +| --- | --- | --- | +| Persistent Firstmate and Herdr in Kubernetes | OrbStack demo plus generic package render and disposable-cluster run in `docs/evidence/2026-07-12-firstmate-package.md` | Proven locally | +| Firstmate cluster-admin limited to the intelligence cluster | Dedicated local namespace and explicit demo ClusterRoleBinding | Proven locally only | +| Direct Akua-managed KaaS and Hetzner bootstrap | Public endpoint study and `akua-intelligence-bootstrap` skill | Not yet run | +| Distinct clustered token and bootstrap-token revocation | Explicit Secret mount in the Firstmate package and handoff procedure | Not yet run | +| Firstmate-native Akua worker lifecycle | Native endpoint routing in the bootstrap skill | Not yet run | +| Same-Pod general-purpose crewmate | Existing Firstmate Herdr backend | Not yet captured as acceptance evidence | +| Separate persistent crewmate Pod with explicit authority | Mate package, no ServiceAccount token by default, explicit Pi Secret grant | Package proven; full task run incomplete | +| Parent-only supervision and human attach | Firstmate contract, Herdr CLI, live workspaces | Mechanism proven; distributed task trace incomplete | +| Real issue to tested review-ready PR | Existing Firstmate delivery machinery | Not yet proven through the Kubernetes fleet | +| Parent and child restart without unique-work loss | Parent PVC restart proven | Child unique-work recovery not yet proven | +| Separate production read-only and scoped write identity | Cortex contract only | Not yet implemented or Red-approved | +| Optional GitHub Issue/Project client | Cortex contract only | Deferred until terminal-native path passes | +| Repeatable end-to-end eval and recordable demo | Deterministic unit checks and local smoke evidence | Full critical-path run not yet captured | +| Public installable image and release | Multi-architecture GHCR workflow added; local image only | Publication not yet proven | + +## Definition of done + +Agent OS is finished only when every row is backed by current external evidence and no row remains partial or unproven. +The complete run must record time-to-cluster, time-to-Firstmate, time-to-crewmate, time-to-PR, recovery time, model and infrastructure cost, human interventions, immutable image digest, resource and operation IDs, and sanitized failure evidence. diff --git a/docs/evidence/2026-07-12-firstmate-package.md b/docs/evidence/2026-07-12-firstmate-package.md new file mode 100644 index 000000000..aa49fc105 --- /dev/null +++ b/docs/evidence/2026-07-12-firstmate-package.md @@ -0,0 +1,70 @@ +# Firstmate package verification + +Date: 2026-07-12 +Akua CLI: 0.8.20 +Kubernetes: OrbStack 1.34.8 +Agent OS image: `sha256:f87ae0f5f93d2781700a7c3a0828833433565322b7bb3e8f1d49244b40b8580e` + +## Claim + +The optional Firstmate Akua package renders ordinary resources and can start a persistent, cluster-admin Firstmate with a read-only Akua authorization mount in a dedicated Kubernetes namespace. + +## Render evidence + +Command: + +```sh +akua render --no-json --no-interactive \ + --package tools/agent-os/packages/firstmate/package.k \ + --inputs tools/agent-os/packages/firstmate/inputs.example.yaml \ + --out /tmp/agent-os-firstmate-render +``` + +Result: + +```text +rendered: 6 manifest(s) (sha256:4965345d355a00530a887f8f24454d5e858664c7fa4328ba584c15f96703ca0d) +Namespace +ServiceAccount +PersistentVolumeClaim +Service +ClusterRoleBinding +StatefulSet +``` + +## Disposable-cluster evidence + +The live eval used namespace `agent-os-package-eval`, the local immutable image above, `imagePullPolicy: Never`, and a synthetic authorization Secret. +It rendered with `createNamespace: false`, applied five manifests, waited for the StatefulSet, and ran the checks inside the Pod. + +Commands: + +```sh +kubectl --context orbstack apply -f /tmp/agent-os-firstmate-eval +kubectl --context orbstack -n agent-os-package-eval \ + rollout status statefulset/agent-os-firstmate --timeout=120s +kubectl --context orbstack -n agent-os-package-eval \ + exec statefulset/agent-os-firstmate -- herdr status --json +kubectl --context orbstack -n agent-os-package-eval \ + exec statefulset/agent-os-firstmate -- kubectl auth can-i '*' '*' --all-namespaces +``` + +Result: + +```text +rendered: 5 manifest(s) (sha256:ecf9cfb0052ca90677f14987ff823637aa27d3e914be9b7ab66906b55e99acc4) +partitioned roll out complete: 1 new pods have been updated +herdr=ready +home=writable +auth_mount=readonly +context=in-cluster +cluster_admin=yes +``` + +The write probe against `/var/run/secrets/agent-os/akua/authorization` failed with `Read-only file system`. +The eval namespace and its cluster-scoped binding were deleted after the checks. + +## Scope + +This proves the Kubernetes package contract locally. +It does not prove Akua workspace authentication, managed KaaS creation, Hetzner worker lifecycle, successor-token handoff, public-image availability, or bootstrap-token revocation. diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index be15dd2d1..53c260ac7 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -62,6 +62,12 @@ assert_grep 'https://github.com/akua-dev/akua/tree/v0.8.25' "$ROOT/THIRD_PARTY_N "Akua's exact source must be named" assert_grep 'https://github.com/derailed/k9s/tree/v0.51.0' "$ROOT/THIRD_PARTY_NOTICES.md" \ "K9s's exact source must be named" +assert_grep 'ghcr.io/akua-dev/agent-os' "$ROOT/.github/workflows/agent-os-image.yml" \ + "release workflow must publish the image expected by the Akua packages" +assert_grep 'linux/amd64,linux/arm64' "$ROOT/.github/workflows/agent-os-image.yml" \ + "release workflow must build the two supported container architectures" +assert_grep "push: \${{ github.event_name != 'pull_request' }}" "$ROOT/.github/workflows/agent-os-image.yml" \ + "pull requests must build but never publish images" bash -n "$ROOT/bin/agent-os-container-entrypoint.sh" bash -n "$ROOT/bin/agent-os-kubeconfig.sh" diff --git a/tests/agent-os-packages.test.sh b/tests/agent-os-packages.test.sh new file mode 100755 index 000000000..0b82902bc --- /dev/null +++ b/tests/agent-os-packages.test.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Static contracts for the optional Akua packages used by Agent OS. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +FIRSTMATE="$ROOT/tools/agent-os/packages/firstmate" + +[ -f "$FIRSTMATE/package.k" ] || fail "firstmate package must exist" +[ -f "$FIRSTMATE/inputs.example.yaml" ] || fail "firstmate example inputs must exist" + +assert_grep 'ghcr.io/akua-dev/agent-os:latest' "$FIRSTMATE/package.k" \ + "firstmate package must default to the public Agent OS image" +assert_grep 'kind = "StatefulSet"' "$FIRSTMATE/package.k" \ + "firstmate package must create a restartable controller" +assert_grep 'kind = "PersistentVolumeClaim"' "$FIRSTMATE/package.k" \ + "firstmate package must persist its home" +assert_grep 'kind = "ServiceAccount"' "$FIRSTMATE/package.k" \ + "firstmate package must create its cluster identity" +assert_grep 'name = "cluster-admin"' "$FIRSTMATE/package.k" \ + "dedicated intelligence clusters must support the explicit Firstmate admin grant" +assert_grep 'akuaAuthSecret: str = ""' "$FIRSTMATE/package.k" \ + "Akua workspace access must be an explicit Secret reference" +assert_grep 'mountPath = "/var/run/secrets/agent-os/akua"' "$FIRSTMATE/package.k" \ + "Akua authorization material must use a runtime Secret mount" +assert_grep 'readOnly = True' "$FIRSTMATE/package.k" \ + "Akua authorization material must be mounted read-only" +assert_grep 'herdr", "status", "--json"' "$FIRSTMATE/package.k" \ + "Firstmate readiness must prove the Herdr server is responsive" +assert_grep 'mountPath = "/usr/local"' "$FIRSTMATE/package.k" \ + "Firstmate-installed tools must persist" +assert_no_grep 'token:' "$FIRSTMATE/inputs.example.yaml" \ + "example inputs must never contain an Akua token value" + +pass "Akua packages keep Firstmate persistent and authority explicit" diff --git a/tools/agent-os/packages/firstmate/README.md b/tools/agent-os/packages/firstmate/README.md new file mode 100644 index 000000000..59245da08 --- /dev/null +++ b/tools/agent-os/packages/firstmate/README.md @@ -0,0 +1,21 @@ +# Firstmate package + +This optional Akua package renders ordinary Kubernetes resources for one persistent Firstmate in a dedicated intelligence cluster. +It creates a namespace, ServiceAccount, persistent home, headless Service, StatefulSet, and - when explicitly enabled - the intelligence-cluster `cluster-admin` binding. + +Render it directly with Akua: + +```sh +akua render \ + --package package.k \ + --inputs inputs.example.yaml \ + --out /tmp/agent-os-firstmate +``` + +Inspect the YAML before applying it with `kubectl`. +The package is an editable convenience, not a controller or required workflow. +Use an immutable image digest for a real intelligence cluster; the `latest` default is only a discoverable starting point. + +`akuaAuthSecret` names an existing Kubernetes Secret with an `authorization` key containing a complete HTTP authorization header for the dedicated Agent OS workspace token. +The value is mounted read-only at runtime and never belongs in inputs, rendered YAML, Git, logs, or command arguments. +Leave the input empty when Firstmate should not receive Akua workspace access. diff --git a/tools/agent-os/packages/firstmate/akua.toml b/tools/agent-os/packages/firstmate/akua.toml new file mode 100644 index 000000000..245f6a593 --- /dev/null +++ b/tools/agent-os/packages/firstmate/akua.toml @@ -0,0 +1,5 @@ +[package] +name = "agent-os-firstmate" +version = "0.1.0" +edition = "akua.dev/v1alpha1" +strict_signing = true diff --git a/tools/agent-os/packages/firstmate/inputs.example.yaml b/tools/agent-os/packages/firstmate/inputs.example.yaml new file mode 100644 index 000000000..30b57eb03 --- /dev/null +++ b/tools/agent-os/packages/firstmate/inputs.example.yaml @@ -0,0 +1,5 @@ +namespace: agent-os +image: ghcr.io/akua-dev/agent-os:latest +imagePullPolicy: IfNotPresent +clusterAdmin: true +akuaAuthSecret: agent-os-firstmate-akua-auth diff --git a/tools/agent-os/packages/firstmate/package.k b/tools/agent-os/packages/firstmate/package.k new file mode 100644 index 000000000..ff2f6f233 --- /dev/null +++ b/tools/agent-os/packages/firstmate/package.k @@ -0,0 +1,182 @@ +import akua.ctx + +schema Input: + """Inputs for one persistent Firstmate in a dedicated intelligence cluster.""" + + namespace: str = "agent-os" + createNamespace: bool = True + image: str = "ghcr.io/akua-dev/agent-os:latest" + imagePullPolicy: "Always" | "IfNotPresent" | "Never" = "IfNotPresent" + clusterAdmin: bool = True + akuaAuthSecret: str = "" + storage: str = "20Gi" + cpuRequest: str = "500m" + memoryRequest: str = "1Gi" + cpuLimit: str = "4" + memoryLimit: str = "8Gi" + +input: Input = ctx.input() + +labels = { + "app.kubernetes.io/name" = "agent-os" + "app.kubernetes.io/component" = "firstmate" +} + +akuaAuthMounts = [{ + name = "akua-auth" + mountPath = "/var/run/secrets/agent-os/akua" + readOnly = True +}] if input.akuaAuthSecret else [] + +akuaAuthVolumes = [{ + name = "akua-auth" + secret = { + secretName = input.akuaAuthSecret + defaultMode = 256 + } +}] if input.akuaAuthSecret else [] + +akuaAuthEnv = [{ + name = "AKUA_AUTH_HEADER_FILE" + value = "/var/run/secrets/agent-os/akua/authorization" +}] if input.akuaAuthSecret else [] + +namespaceResources = [{ + apiVersion = "v1" + kind = "Namespace" + metadata = { + name = input.namespace + labels = { "app.kubernetes.io/part-of" = "agent-os" } + } +}] if input.createNamespace else [] + +adminResources = [{ + apiVersion = "rbac.authorization.k8s.io/v1" + kind = "ClusterRoleBinding" + metadata = { + name = "agent-os-firstmate-${input.namespace}" + labels = labels + } + roleRef = { + apiGroup = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "cluster-admin" + } + subjects = [{ + kind = "ServiceAccount" + name = "agent-os-firstmate" + namespace = input.namespace + }] +}] if input.clusterAdmin else [] + +resources = namespaceResources + [ + { + apiVersion = "v1" + kind = "ServiceAccount" + metadata = { + name = "agent-os-firstmate" + namespace = input.namespace + labels = labels + } + }, + { + apiVersion = "v1" + kind = "PersistentVolumeClaim" + metadata = { + name = "agent-os-firstmate-home" + namespace = input.namespace + labels = labels + } + spec = { + accessModes = ["ReadWriteOnce"] + resources = { requests = { storage = input.storage } } + } + }, + { + apiVersion = "v1" + kind = "Service" + metadata = { + name = "agent-os-firstmate" + namespace = input.namespace + labels = labels + } + spec = { + clusterIP = "None" + selector = labels + } + }, +] + adminResources + [ + { + apiVersion = "apps/v1" + kind = "StatefulSet" + metadata = { + name = "agent-os-firstmate" + namespace = input.namespace + labels = labels + } + spec = { + serviceName = "agent-os-firstmate" + replicas = 1 + selector = { matchLabels = labels } + template = { + metadata = { + labels = labels + annotations = { + "agent-os.akua.dev/address" = "/k8s/in-cluster/namespace/${input.namespace}/mate/firstmate/herdr" + } + } + spec = { + serviceAccountName = "agent-os-firstmate" + securityContext = { + fsGroup = 0 + runAsGroup = 0 + runAsUser = 0 + } + initContainers = [{ + name = "agent-os-init" + image = input.image + imagePullPolicy = input.imagePullPolicy + command = ["/opt/agent-os/bin/agent-os-init.sh"] + volumeMounts = [{ + name = "home" + mountPath = "/persistent-agent" + }] + }] + containers = [{ + name = "firstmate" + image = input.image + imagePullPolicy = input.imagePullPolicy + env = [ + { name = "FM_HOME", value = "/home/agent" }, + { name = "HOME", value = "/home/agent" }, + { name = "AKUA_API_URL", value = "https://api.akua.dev/v1" }, + ] + akuaAuthEnv + readinessProbe = { + exec = { command = ["herdr", "status", "--json"] } + initialDelaySeconds = 2 + periodSeconds = 5 + } + resources = { + requests = { + cpu = input.cpuRequest + memory = input.memoryRequest + } + limits = { + cpu = input.cpuLimit + memory = input.memoryLimit + } + } + volumeMounts = [ + { name = "home", mountPath = "/home/agent" }, + { name = "home", mountPath = "/usr/local", subPath = "usr-local" }, + ] + akuaAuthMounts + }] + volumes = [{ + name = "home" + persistentVolumeClaim = { claimName = "agent-os-firstmate-home" } + }] + akuaAuthVolumes + } + } + } + }, +] From b40df4bee687a0d0fb257b7ac01a80c7b0edd1bc Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 12:06:29 +0200 Subject: [PATCH 14/56] fix: probe model capacity and portable evidence --- .agents/skills/akua-intelligence-bootstrap/SKILL.md | 9 ++++++--- .agents/skills/kubernetes-fleet/SKILL.md | 1 + docs/acceptance.md | 1 + tests/agent-os-kubeconfig.test.sh | 6 +++++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.agents/skills/akua-intelligence-bootstrap/SKILL.md b/.agents/skills/akua-intelligence-bootstrap/SKILL.md index d32a28491..8da717917 100644 --- a/.agents/skills/akua-intelligence-bootstrap/SKILL.md +++ b/.agents/skills/akua-intelligence-bootstrap/SKILL.md @@ -58,9 +58,11 @@ Use the Platform MCP `search` tool again before a mutation so current schemas ou The Secret's `authorization` key contains the complete header consumed through `AKUA_AUTH_HEADER_FILE`. 11. Render `tools/agent-os/packages/firstmate/package.k` with `akua render`, inspect the ordinary YAML, and apply it with `kubectl`. 12. Verify the StatefulSet is ready, Herdr responds, the persistent home is writable, the `in-cluster` context is cluster-admin only in this intelligence cluster, and the Pod can list its Akua workspace through `curl -H @"$AKUA_AUTH_HEADER_FILE"`. -13. From the clustered Firstmate, inventory all workspace token IDs and prove the new token can perform the required Akua and Kubernetes reads. -14. Only after that handoff succeeds, revoke the known local bootstrap token and prove it now receives an authentication failure. -15. Destroy every protected temporary file and record only non-secret resource IDs, operation IDs, timestamps, states, image digest, test results, costs, and interventions. +13. Probe the approved primary model route with a bounded request and record only provider, model, result, timing, and cost. + If there is no independently approved fallback provider, record the single-provider availability risk instead of presenting the fleet as quota-resilient. +14. From the clustered Firstmate, inventory all workspace token IDs and prove the new token can perform the required Akua and Kubernetes reads. +15. Only after that handoff succeeds, revoke the known local bootstrap token and prove it now receives an authentication failure. +16. Destroy every protected temporary file and record only non-secret resource IDs, operation IDs, timestamps, states, image digest, test results, costs, and interventions. ## Recovery and cleanup @@ -77,6 +79,7 @@ The bootstrap is complete only when current external state proves all of these: - Akua operation IDs reached `SUCCEEDED`; - the worker is ready in Akua and Kubernetes; - the Firstmate StatefulSet and Herdr server are ready; +- at least one approved model route completes a bounded request; - the home and installed-tool paths survive a Pod replacement; - the clustered token works from the Pod; - the bootstrap token is revoked and fails authentication; diff --git a/.agents/skills/kubernetes-fleet/SKILL.md b/.agents/skills/kubernetes-fleet/SKILL.md index 37782cf7e..cd4529f21 100644 --- a/.agents/skills/kubernetes-fleet/SKILL.md +++ b/.agents/skills/kubernetes-fleet/SKILL.md @@ -20,6 +20,7 @@ Load this skill only when the current firstmate is running in Kubernetes or is e - Render the package with `akua render`, inspect or edit its ordinary YAML when useful, and apply it with `kubectl`. - Treat AI credentials as explicit per-mate grants, never as ambient inheritance merely because Firstmate can read them. - Create or select a Kubernetes Secret only after the grant is authorized, then pass its name as the package's `piAuthSecret` input. +- Probe the selected model route before launching work; when quota is unavailable, use another explicitly granted provider or report the capacity blocker instead of repeatedly spawning agents. - Give every launched Herdr agent a task-unique name, close only a confirmed dead restored pane before reuse, and never replace a live agent. - Grade completion by the promised artifact or delivered Git state; Herdr `idle` alone is not a completion signal. - The package is optional; direct `akua render`, raw YAML, and `kubectl` remain supported. diff --git a/docs/acceptance.md b/docs/acceptance.md index 7d7463890..90a7ab1ce 100644 --- a/docs/acceptance.md +++ b/docs/acceptance.md @@ -10,6 +10,7 @@ Update it from real runs; planned behavior and model narration never count as pr | Direct Akua-managed KaaS and Hetzner bootstrap | Public endpoint study and `akua-intelligence-bootstrap` skill | Not yet run | | Distinct clustered token and bootstrap-token revocation | Explicit Secret mount in the Firstmate package and handoff procedure | Not yet run | | Firstmate-native Akua worker lifecycle | Native endpoint routing in the bootstrap skill | Not yet run | +| Replaceable usable model supply | Pi auth is explicit, but the live demo has one Codex provider and current quota exhaustion | Blocked until an approved route has capacity | | Same-Pod general-purpose crewmate | Existing Firstmate Herdr backend | Not yet captured as acceptance evidence | | Separate persistent crewmate Pod with explicit authority | Mate package, no ServiceAccount token by default, explicit Pi Secret grant | Package proven; full task run incomplete | | Parent-only supervision and human attach | Firstmate contract, Herdr CLI, live workspaces | Mechanism proven; distributed task trace incomplete | diff --git a/tests/agent-os-kubeconfig.test.sh b/tests/agent-os-kubeconfig.test.sh index 80aa70097..a06d76f07 100755 --- a/tests/agent-os-kubeconfig.test.sh +++ b/tests/agent-os-kubeconfig.test.sh @@ -24,7 +24,11 @@ assert_grep 'current-context: in-cluster' "$config" "kubeconfig must select the assert_grep 'namespace: agent-os-eval' "$config" "kubeconfig must select the Agent OS namespace" assert_grep "tokenFile: $tmp/serviceaccount/token" "$config" "kubeconfig must follow the rotating token file" assert_no_grep 'demo-token' "$config" "kubeconfig must not copy token contents" -[ "$(stat -f '%Lp' "$config" 2>/dev/null || stat -c '%a' "$config")" = 600 ] || fail "kubeconfig must be mode 600" +case $(uname -s) in + Darwin) mode=$(stat -f '%Lp' "$config") ;; + *) mode=$(stat -c '%a' "$config") ;; +esac +[ "$mode" = 600 ] || fail "kubeconfig must be mode 600" printf '%s\n' sentinel > "$config" HOME="$tmp/home" \ From 3ffbec9aabe723927a33e39cd708eda700656b85 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 10:48:55 +0200 Subject: [PATCH 15/56] Prove persistent Kubernetes mate recovery --- .agents/skills/harness-adapters/SKILL.md | 2 + AGENTS.md | 1 + bin/agent-os-container-entrypoint.sh | 64 ++++++++++ deploy/orbstack/primary.yaml | 4 + docs/acceptance.md | 14 +-- .../evidence/2026-07-13-same-pod-firstmate.md | 97 ++++++++++++++ .../2026-07-13-separate-pod-recovery.md | 119 ++++++++++++++++++ docs/kubernetes.md | 11 +- tests/agent-os-container.test.sh | 12 ++ 9 files changed, 316 insertions(+), 8 deletions(-) create mode 100644 docs/evidence/2026-07-13-same-pod-firstmate.md create mode 100644 docs/evidence/2026-07-13-separate-pod-recovery.md diff --git a/.agents/skills/harness-adapters/SKILL.md b/.agents/skills/harness-adapters/SKILL.md index 67bffd886..704f457d3 100644 --- a/.agents/skills/harness-adapters/SKILL.md +++ b/.agents/skills/harness-adapters/SKILL.md @@ -87,6 +87,8 @@ The supported launch-profile flags below were verified locally on 2026-06-30 wit | pi | `--model ` | `--thinking ` | Verified on pi 0.80.2. `max` prints an invalid-thinking warning, so firstmate omits Pi effort when the requested effort is `max`. | | opencode | `--model ` | none for firstmate's interactive launch | Verified on opencode 1.17.6. `opencode run` has `--variant`, but firstmate launches the interactive `opencode --prompt` path, which has no verified effort flag. | +When a Pi provider route is known, pass a provider-qualified model id such as `openai-codex/gpt-5.6-terra`; unqualified names may resolve to an unauthenticated provider. + When a requested effort value is outside the harness-specific accepted set, `fm-spawn` records the requested `effort=` in meta but emits no effort flag for that harness. This preserves launch success instead of passing a known-bad value. diff --git a/AGENTS.md b/AGENTS.md index 8518c841f..dfe6c186d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -65,6 +65,7 @@ Never add an agent name as co-author. `FM_HOME` selects the operational home for a firstmate instance. When it is unset, most scripts use this repo root as the home, which is today's behavior. When it is set, scripts still use their own `bin/` from the repo they live in, but operational dirs come from `$FM_HOME`: `state/`, `data/`, `config/`, and `projects/`. +When `FM_HOME` is set, never create or read operational state through repo-relative `state/`, `data/`, `config/`, or `projects/` paths; use `$FM_HOME/...` or firstmate scripts that resolve it. Existing overrides remain compatible: `FM_STATE_OVERRIDE` can still point at a custom state dir, and `FM_ROOT_OVERRIDE` still behaves like the old whole-root override when `FM_HOME` is unset. `bin/fm-send.sh` is the fail-closed exception: it requires `FM_HOME` to be set so target resolution is always scoped to an explicit firstmate home. Each secondmate gets its own persistent `FM_HOME`, so its local state, backlog, projects, and session lock are isolated from the main firstmate. diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh index a94ddaaf6..a8958c471 100755 --- a/bin/agent-os-container-entrypoint.sh +++ b/bin/agent-os-container-entrypoint.sh @@ -22,6 +22,70 @@ if [ ! -e "$FM_HOME/config/backend" ]; then printf 'herdr\n' > "$FM_HOME/config/backend" fi +# The local evaluation overlay may pin every spawned Pi agent while a model is +# under test. This is deliberately opt-in so the reusable image and Akua +# packages remain model-agnostic. Converge both dispatch files on every Pod +# start: crew-dispatch governs crewmates/scouts, while secondmate-harness is the +# separate profile used to launch Secondmates. +if [ -n "${AGENT_OS_TEST_PI_MODEL:-}" ]; then + case "$AGENT_OS_TEST_PI_MODEL" in + */*) ;; + *) + echo "error: AGENT_OS_TEST_PI_MODEL must include its provider" >&2 + exit 2 + ;; + esac + case "$AGENT_OS_TEST_PI_MODEL" in + *[!A-Za-z0-9._:/-]*) + echo "error: AGENT_OS_TEST_PI_MODEL must be one provider-qualified token" >&2 + exit 2 + ;; + esac + case "${AGENT_OS_TEST_PI_EFFORT:-}" in + low|medium|high|xhigh) ;; + *) + echo "error: AGENT_OS_TEST_PI_EFFORT must be low, medium, high, or xhigh" >&2 + exit 2 + ;; + esac + cat > "$FM_HOME/config/crew-dispatch.json" < "$FM_HOME/config/secondmate-harness" + + pi_provider=${AGENT_OS_TEST_PI_MODEL%%/*} + pi_model=${AGENT_OS_TEST_PI_MODEL#*/} + pi_settings_dir="$HOME/.pi/agent" + pi_settings="$pi_settings_dir/settings.json" + pi_settings_tmp="$pi_settings.tmp.$$" + mkdir -p "$pi_settings_dir" + umask 077 + if [ -s "$pi_settings" ] && jq -e 'type == "object"' "$pi_settings" >/dev/null 2>&1; then + jq \ + --arg provider "$pi_provider" \ + --arg model "$pi_model" \ + --arg thinking "$AGENT_OS_TEST_PI_EFFORT" \ + '. + {defaultProvider: $provider, defaultModel: $model, defaultThinkingLevel: $thinking}' \ + "$pi_settings" > "$pi_settings_tmp" + else + jq -n \ + --arg provider "$pi_provider" \ + --arg model "$pi_model" \ + --arg thinking "$AGENT_OS_TEST_PI_EFFORT" \ + '{defaultProvider: $provider, defaultModel: $model, defaultThinkingLevel: $thinking}' \ + > "$pi_settings_tmp" + fi + mv "$pi_settings_tmp" "$pi_settings" +fi + for tool in gh-axi chrome-devtools-axi lavish-axi; do marker="$HOME/.config/agent-os/setup-$tool" if [ ! -e "$marker" ]; then diff --git a/deploy/orbstack/primary.yaml b/deploy/orbstack/primary.yaml index 1a1717d70..4debe6814 100644 --- a/deploy/orbstack/primary.yaml +++ b/deploy/orbstack/primary.yaml @@ -57,6 +57,10 @@ spec: value: /home/agent - name: HOME value: /home/agent + - name: AGENT_OS_TEST_PI_MODEL + value: openai-codex/gpt-5.6-terra + - name: AGENT_OS_TEST_PI_EFFORT + value: low readinessProbe: exec: command: diff --git a/docs/acceptance.md b/docs/acceptance.md index 90a7ab1ce..b69ba6a5f 100644 --- a/docs/acceptance.md +++ b/docs/acceptance.md @@ -10,16 +10,16 @@ Update it from real runs; planned behavior and model narration never count as pr | Direct Akua-managed KaaS and Hetzner bootstrap | Public endpoint study and `akua-intelligence-bootstrap` skill | Not yet run | | Distinct clustered token and bootstrap-token revocation | Explicit Secret mount in the Firstmate package and handoff procedure | Not yet run | | Firstmate-native Akua worker lifecycle | Native endpoint routing in the bootstrap skill | Not yet run | -| Replaceable usable model supply | Pi auth is explicit, but the live demo has one Codex provider and current quota exhaustion | Blocked until an approved route has capacity | -| Same-Pod general-purpose crewmate | Existing Firstmate Herdr backend | Not yet captured as acceptance evidence | -| Separate persistent crewmate Pod with explicit authority | Mate package, no ServiceAccount token by default, explicit Pi Secret grant | Package proven; full task run incomplete | -| Parent-only supervision and human attach | Firstmate contract, Herdr CLI, live workspaces | Mechanism proven; distributed task trace incomplete | +| Replaceable usable model supply | `openai-codex/gpt-5.4-mini` completed the same-Pod task; Terra-low completed the separate-Pod and recovery tasks; the current local overlay converges direct Pi, crewmate, and Secondmate defaults to `openai-codex/gpt-5.6-terra` with low thinking | One provider with multiple models proven locally; replacement provider still unproven | +| Same-Pod general-purpose crewmate | Firstmate-supervised Pi scout, real Kubernetes test, in-cluster context check, and report in `docs/evidence/2026-07-13-same-pod-firstmate.md` | Proven locally | +| Separate persistent crewmate Pod with explicit authority | Terra-low task, explicit resources, read-only Pi Secret, no ServiceAccount token, and retained PVC in `docs/evidence/2026-07-13-separate-pod-recovery.md` | Proven locally | +| Parent-only supervision and human attach | Same-Pod and separate-Pod tasks accepted only Firstmate briefs and steers; a connected Herdr client observed the live workspaces | Proven locally | | Real issue to tested review-ready PR | Existing Firstmate delivery machinery | Not yet proven through the Kubernetes fleet | -| Parent and child restart without unique-work loss | Parent PVC restart proven | Child unique-work recovery not yet proven | +| Parent and child restart without unique-work loss | Primary session recovery plus separate-Pod replacement with unique artifact, report, tools, and exact Pi session resumed | Proven locally | | Separate production read-only and scoped write identity | Cortex contract only | Not yet implemented or Red-approved | | Optional GitHub Issue/Project client | Cortex contract only | Deferred until terminal-native path passes | -| Repeatable end-to-end eval and recordable demo | Deterministic unit checks and local smoke evidence | Full critical-path run not yet captured | -| Public installable image and release | Multi-architecture GHCR workflow added; local image only | Publication not yet proven | +| Repeatable end-to-end eval and recordable demo | Same-Pod and separate-Pod run records now capture model, time, cost estimate, interventions, resource IDs, and failures | Local agent lifecycle captured; Akua/GitHub/product path incomplete | +| Public installable image and release | Multi-architecture workflow run `29188585384` succeeded; local image only because pull-request builds do not publish | Publication not yet proven | ## Definition of done diff --git a/docs/evidence/2026-07-13-same-pod-firstmate.md b/docs/evidence/2026-07-13-same-pod-firstmate.md new file mode 100644 index 000000000..45a063e80 --- /dev/null +++ b/docs/evidence/2026-07-13-same-pod-firstmate.md @@ -0,0 +1,97 @@ +# Same-Pod Firstmate verification + +Date: 2026-07-13 +Kubernetes context: `orbstack` +Namespace: `agent-os-demo` +Pod: `agent-os-firstmate-0` +Image ID: `docker-pullable://agent-os@sha256:1e6696938aafcec21748b83e3d68773d8b3cf1ccb967974f2cfbeda0c4fbed65` +Herdr: 0.7.3, protocol 16 +Pi: 0.80.6 + +## Claim + +A persistent Firstmate running in Herdr inside the Kubernetes Pod can create, supervise, recover, and finish a general-purpose Pi crewmate in another Herdr task pane in the same Pod. +The crewmate receives work only from Firstmate, runs real verification commands, writes its report to the persistent Firstmate home, and remains observable through Herdr. + +## Topology + +- Primary Firstmate: Herdr workspace `w3`, pane `w3:p1`, persistent `FM_HOME=/home/agent`. +- Crewmate task: `aeval-p4`, Herdr endpoint `default:w2:p6`. +- Model: `openai-codex/gpt-5.4-mini`, medium thinking. +- Worktree: `/home/agent/.treehouse/agent-os-eval-f1de56/2/agent-os-eval`. +- Report: `/home/agent/data/aeval-p4/report.md`. +- Both agents ran in `agent-os-firstmate-0`; no separate task service or communication protocol was used. + +## Task evidence + +Firstmate created a minimal local-only evaluation repository, registered it, scaffolded a scout brief, and dispatched the crewmate through `fm-spawn.sh` with the Herdr backend. +The successful spawn recorded: + +```text +spawned aeval-p4 harness=pi kind=scout mode=local-only yolo=off window=default:w2:p6 worktree=/home/agent/.treehouse/agent-os-eval-f1de56/2/agent-os-eval +``` + +The crewmate ran the real kubeconfig regression test: + +```sh +bash /opt/agent-os/tests/agent-os-kubeconfig.test.sh +``` + +```text +ok - Agent OS creates a rotation-safe in-cluster kubeconfig without exposing token contents +``` + +It also checked the active Kubernetes context: + +```text +CURRENT NAME CLUSTER AUTHINFO NAMESPACE +* in-cluster in-cluster in-cluster agent-os-demo +``` + +The task status reached: + +```text +working: inspecting project and kube integration +resolved: recovered with provider-qualified openai-codex/gpt-5.4-mini +done: report written with verification and evidence +``` + +The recorded task interval was `2026-07-13T08:18:30Z` to `2026-07-13T08:20:47Z`, or 2 minutes 17 seconds. +Pi displayed an approximate crewmate-session cost of `$0.195`; this is a model UI estimate, not a provider invoice. + +## Parent-only supervision and human observation + +The child received its initial brief from `fm-spawn.sh` and later steers through Firstmate's `fm-send.sh` path. +The external evaluator made only read-only Herdr observations of the child pane and never sent a prompt or command directly to the child. +Herdr server logs recorded a connected client while the fleet was live, proving that a human could attach and observe the same workspaces. + +## Failure and recovery evidence + +The first spawn used the unqualified model name `gpt-5.4-mini`. +Pi resolved it to the unauthenticated `azure-openai-responses` provider and failed with: + +```text +Error: No API key found for azure-openai-responses. +``` + +Firstmate preserved the task brief, state, report path, and worktree allocation, then respawned the same task id with `openai-codex/gpt-5.4-mini`. +The qualified route completed successfully. + +A connected Herdr client also closed the original primary workspace during the run. +The primary Pi session, persistent Firstmate home, task metadata, brief, and status survived. +A resumed primary reacquired the fleet through `bin/fm-session-start.sh` and supervised the task to completion. +The resumed primary had to remain in its own Herdr workspace; placing it inside the task tab made ordinary dead-task replacement close the supervisor with that tab. + +Three operator interventions were required: qualify the provider route, restart the primary after its workspace closed, and move the resumed primary into a dedicated workspace. +Firstmate then handled child steering, report acceptance, and agent exit itself. + +## Lessons promoted into the system + +- Operational state must always resolve through `$FM_HOME`; repo-relative `data/` and `projects/` created non-persistent duplicates during the first attempt. +- Pi dispatch must use a provider-qualified model id when authentication depends on the provider route. +- A primary Firstmate Herdr pane must not share a task tab that the backend may replace during recovery. + +## Scope + +This proves the same-Pod task path, parent-only task communication, human observation, provider-qualified model recovery, and primary-session recovery locally. +It does not prove a separately resourced crewmate Pod, unique child work surviving a Pod restart, Akua-managed infrastructure, GitHub delivery, or production access boundaries. diff --git a/docs/evidence/2026-07-13-separate-pod-recovery.md b/docs/evidence/2026-07-13-separate-pod-recovery.md new file mode 100644 index 000000000..972d38614 --- /dev/null +++ b/docs/evidence/2026-07-13-separate-pod-recovery.md @@ -0,0 +1,119 @@ +# Separate-Pod mate and recovery verification + +Date: 2026-07-13 +Kubernetes context: `orbstack` +Namespace: `agent-os-demo` +Package: `tools/agent-os/packages/mate/package.k` +Model policy: `openai-codex/gpt-5.6-terra`, low thinking + +## Claim + +Firstmate can use ordinary Akua package rendering, Kubernetes, and Herdr to create and supervise a separately resourced persistent mate Pod with explicit AI credentials and no ambient Kubernetes authority. +The mate can complete real work, survive Pod replacement without losing its unique artifact or Pi session, resume the exact persisted session, append recovery evidence, and return control to Firstmate without a custom controller or communication protocol. + +## Resource and authority boundary + +- Pod: `agent-os-mate-separate-eval`. +- Address: `/k8s/in-cluster/ns/agent-os-demo/pod/agent-os-mate-separate-eval`. +- Requests: `500m` CPU and `1Gi` memory. +- Limits: `2` CPU and `4Gi` memory. +- PVC: `agent-os-mate-separate-eval-home`, UID `2c141e3b-0585-4648-96e8-2bc496450a91`, retained and Bound. +- Pi Secret: `agent-os-mate-separate-eval-pi-auth`, UID `c4698208-bb08-437c-87e9-9346e8b14ed2`, retained. +- `automountServiceAccountToken`: `false`. +- No ServiceAccount token existed in the container. +- `/home/agent/.pi/agent/auth.json` was an explicit read-only Secret mount. +- The package added no privileged setting, host namespace, or host mount. + +Parent-side live verification returned: + +```text +NO_SA_TOKEN +/home/agent/.pi/agent/auth.json ro,relatime +{"server":"running","running":true,"socket":"/home/agent/.config/herdr/herdr.sock","compatible":true} +``` + +The child initially misread `/proc/mounts` and claimed that no auth material was mounted. +Firstmate caught the discrepancy and retained the authoritative parent-side `/proc/self/mountinfo` evidence above. + +## Initial task + +Firstmate launched `separate-eval-terra` through the mate's Herdr server with `openai-codex/gpt-5.6-terra` and low thinking. +The child created `/home/agent/unique-work.txt`: + +```text +separate-eval-20260713T083121Z-b8a6ef90edac4d55 +Short note: child evaluator artifact; parent-only communication. +``` + +Its immutable artifact hash was: + +```text +2aeb922cd51230ed7b62bc40cc7ce30d41e44b4df3963911fe860030c2f7765a +``` + +The task ran from `2026-07-13T08:31:12Z` through `2026-07-13T08:31:35Z` and wrote `/home/agent/separate-eval-report.md`. +The child communicated only through its initial Firstmate brief and returned its result through the persistent report and Herdr pane. + +## Pod and session recovery + +Firstmate exited the child, preserved the PVC and Secret, deleted only the Pod, and recreated it from the same Akua-rendered manifests. + +```text +Initial Pod UID: 960cc460-cd06-44d1-a8fa-da200d4384b9 +Replacement Pod UID: 1cb7ff92-5e0d-4ccd-9324-38515ae9823b +``` + +After replacement, Firstmate verified: + +```text +UNIQUE_SURVIVED +REPORT_SURVIVED +PI_SESSION_SURVIVED +/usr/local/bin/herdr +/usr/local/bin/pi +{"running":true,"socket":"/home/agent/.config/herdr/herdr.sock"} +``` + +The persisted Pi session was `019f5a99-f031-7f51-8cc0-156777204dc5`. +Firstmate relaunched `pi --session 019f5a99-f031-7f51-8cc0-156777204dc5` on Terra-low through the replacement Pod's Herdr server. +The resumed agent rehashed the unique artifact, confirmed Herdr health, and appended a recovery record at `2026-07-13T08:33:26Z`. +It then exited, leaving no live Terra child agent. + +The approximate model cost displayed by the final recovery pane was `$0.127`; this is a Pi subscription estimate, not provider billing evidence. + +## Model-policy intervention + +An earlier `openai-codex/gpt-5.4-mini` child was stopped immediately when the captain changed the testing policy. +The primary was restarted on Terra-low, and `/home/agent/config/crew-dispatch.json` now pins all subsequent testing crewmates to: + +```json +{ + "rules": [], + "default": { + "harness": "pi", + "model": "openai-codex/gpt-5.6-terra", + "effort": "low" + } +} +``` + +The aborted pane restored as a stale Herdr entry after Pod replacement but had no live Pi process or accepted result. + +The policy was then closed over all three local launch paths: + +- Pi's persisted defaults are `openai-codex`, `gpt-5.6-terra`, and `low`, so a + direct primary `pi` session uses Terra-low; +- `crew-dispatch.json` pins ordinary crewmates and scouts; +- `secondmate-harness` contains + `pi openai-codex/gpt-5.6-terra low` for Secondmate launches. + +After rebuilding and replacing the Firstmate Pod, image +`docker-pullable://agent-os@sha256:94b7eb6c435f1a226e7279c40491e49c505b30bf54c65de1cd3d5b8e0a102611` +converged all three settings and Herdr returned healthy. The retained separate +mate PVC was also updated to the same direct-Pi defaults. No child agent was +left running after verification. + +## Scope + +This proves the separate-Pod authority boundary, parent-only supervision, persistent home, Pod replacement, unique-work survival, persistent toolchain, and exact Pi session recovery in the local intelligence cluster. +It does not prove Akua-managed KaaS or worker lifecycle, GitHub issue-to-PR delivery, product-cluster read/write boundaries, or a second model provider. diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 4a16458f3..271bc8f3c 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -50,12 +50,21 @@ Start Pi from the tracked distro and attach to Herdr from another terminal: ```sh # inside the primary shell cd /opt/agent-os -pi +pi --model openai-codex/gpt-5.6-terra --thinking low # on the host bin/agent-os-local.sh attach ``` +During the current local evaluation, the OrbStack manifest also converges +`config/crew-dispatch.json` and `config/secondmate-harness` on every primary Pod +start. Crewmates and Secondmates therefore launch through Pi with +`openai-codex/gpt-5.6-terra` and low thinking. The reusable image and Akua +packages remain model-agnostic when those local test environment variables are +absent. The same opt-in policy updates Pi's own `defaultProvider`, +`defaultModel`, and `defaultThinkingLevel`, so a plain `pi` primary session uses +Terra-low too. + The primary can create and manage isolated crewmates directly: ```sh diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 53c260ac7..b82738e62 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -24,6 +24,10 @@ assert_grep 'quota-axi@0.1.5' "$ROOT/Dockerfile" "image must pin quota-axi 0.1.5 assert_grep 'ripgrep' "$ROOT/Dockerfile" "image must install ripgrep" assert_grep 'fd-find' "$ROOT/Dockerfile" "image must install fd" assert_grep 'FM_HOME=/home/agent' "$ROOT/Dockerfile" "image must declare the persistent firstmate home" +assert_grep 'never create or read operational state through repo-relative' "$ROOT/AGENTS.md" \ + "Firstmate must anchor operational state to FM_HOME" +assert_grep 'pass a provider-qualified model id' "$ROOT/.agents/skills/harness-adapters/SKILL.md" \ + "Pi dispatch must preserve the selected provider route" assert_grep 'XDG_CONFIG_HOME=/home/agent/.config' "$ROOT/Dockerfile" "image must persist XDG configuration" assert_grep 'NPM_CONFIG_PREFIX=/usr/local' "$ROOT/Dockerfile" "global npm installs must use persistent /usr/local" assert_grep 'PATH=/home/agent/.local/bin:/home/agent/.bun/bin:/home/agent/.cargo/bin:/usr/local/bin' "$ROOT/Dockerfile" \ @@ -41,6 +45,14 @@ assert_no_grep 'USER node' "$ROOT/Dockerfile" "Agent OS containers must start as assert_grep 'exec herdr server' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must keep Herdr as PID 1" assert_grep 'setup hooks' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must install persistent AXI hooks" assert_grep 'agent-os-kubeconfig.sh' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must prepare in-cluster kubectl access" +assert_grep 'AGENT_OS_TEST_PI_MODEL' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "test-mode Pods must converge the Pi model policy" +assert_grep 'defaultThinkingLevel' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "test-mode Pods must also pin direct Pi sessions" +assert_grep 'openai-codex/gpt-5.6-terra' "$ROOT/deploy/orbstack/primary.yaml" \ + "the local test fleet must pin Terra" +assert_grep 'AGENT_OS_TEST_PI_EFFORT' "$ROOT/deploy/orbstack/primary.yaml" \ + "the local test fleet must pin Pi reasoning effort" assert_grep 'tokenFile:' "$ROOT/bin/agent-os-kubeconfig.sh" "kubeconfig must follow the projected token file" assert_no_grep 'set-credentials.*--token' "$ROOT/bin/agent-os-kubeconfig.sh" "kubeconfig must not copy a bearer token" assert_grep 'automountServiceAccountToken = False' "$ROOT/tools/agent-os/packages/mate/package.k" \ From 4580220fa111c001ee3b31b26afca9af718896e7 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 10:52:24 +0200 Subject: [PATCH 16/56] Pin Terra in remote mate demo --- docs/kubernetes.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 271bc8f3c..e45df8603 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -103,7 +103,8 @@ Use ordinary Herdr and Kubernetes commands to launch, inspect, steer, and retrie ```sh kubectl -n agent-os-demo exec agent-os-mate-scout-1 -- \ - herdr agent start scout-1-research --cwd /home/agent --no-focus -- pi "$(cat /tmp/scout-1.prompt)" + herdr agent start scout-1-research --cwd /home/agent --no-focus -- \ + pi --model openai-codex/gpt-5.6-terra --thinking low "$(cat /tmp/scout-1.prompt)" kubectl -n agent-os-demo exec agent-os-mate-scout-1 -- \ test -s /home/agent/data/scout-1.md ``` From 6e636298104272fa9d09802d87f465ab4bee1dfb Mon Sep 17 00:00:00 2001 From: root Date: Mon, 13 Jul 2026 09:00:53 +0000 Subject: [PATCH 17/56] fix: select rebuilt local demo image --- bin/agent-os-local.sh | 15 ++++++++++++ docs/kubernetes.md | 2 ++ tests/agent-os-local.test.sh | 46 +++++++++++++++++++++++++++++++----- 3 files changed, 57 insertions(+), 6 deletions(-) diff --git a/bin/agent-os-local.sh b/bin/agent-os-local.sh index 03c1830dd..d0c4e7ad8 100755 --- a/bin/agent-os-local.sh +++ b/bin/agent-os-local.sh @@ -7,6 +7,7 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) CONTEXT=${AGENT_OS_CONTEXT:-orbstack} NAMESPACE=${AGENT_OS_NAMESPACE:-agent-os-demo} IMAGE=${AGENT_OS_IMAGE:-agent-os:dev} +IMAGE_IS_OVERRIDE=${AGENT_OS_IMAGE+x} COMMAND=${1:-} if [ "$CONTEXT" != orbstack ] && [ "${AGENT_OS_ALLOW_NON_ORBSTACK:-0}" != 1 ]; then @@ -14,11 +15,20 @@ if [ "$CONTEXT" != orbstack ] && [ "${AGENT_OS_ALLOW_NON_ORBSTACK:-0}" != 1 ]; t exit 2 fi +local_image_tag() { + local image_id + image_id=$(docker image inspect --format '{{.Id}}' "$IMAGE") + printf 'agent-os:local-%s\n' "${image_id#sha256:}" +} + cd "$ROOT" case "$COMMAND" in build) docker build -t "$IMAGE" . + if [ -z "$IMAGE_IS_OVERRIDE" ]; then + docker tag "$IMAGE" "$(local_image_tag)" + fi ;; deploy) if [ "$CONTEXT" = orbstack ]; then @@ -26,6 +36,11 @@ case "$COMMAND" in kubectl --context "$CONTEXT" wait --for=condition=Ready node/orbstack --timeout=120s fi kubectl --context "$CONTEXT" apply -k deploy/orbstack + if [ -z "$IMAGE_IS_OVERRIDE" ]; then + IMAGE=$(local_image_tag) + fi + kubectl --context "$CONTEXT" -n "$NAMESPACE" set image statefulset/agent-os-firstmate \ + "agent-os-init=$IMAGE" "firstmate=$IMAGE" ;; status) kubectl --context "$CONTEXT" -n "$NAMESPACE" get statefulset agent-os-firstmate diff --git a/docs/kubernetes.md b/docs/kubernetes.md index e45df8603..12aa5ff82 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -35,6 +35,8 @@ bin/agent-os-local.sh status bin/agent-os-local.sh shell ``` +With the default local image, each `build` also creates a content-addressed local tag. The following `deploy` updates both StatefulSet containers to that tag, so a rebuilt `agent-os:dev` cannot reuse a stale OrbStack runtime digest. Set `AGENT_OS_IMAGE` to build and deploy an explicit image name without replacing or retagging that override. + The image includes Firstmate's complete required toolchain, including `gh`, `rg`, `fd`, Akua, `kubectl`, K9s, treehouse, no-mistakes, and every required AXI CLI. Authenticate GitHub inside the primary with `gh auth login`. Authenticate Pi using `/login` and the provider flow you choose. diff --git a/tests/agent-os-local.test.sh b/tests/agent-os-local.test.sh index 8c9ce018d..a95379b1c 100755 --- a/tests/agent-os-local.test.sh +++ b/tests/agent-os-local.test.sh @@ -22,11 +22,22 @@ SH } make_fake docker +cat > "$FAKEBIN/docker" <<'SH' +#!/usr/bin/env bash +printf '%s' "$(basename "$0")" >> "$AGENT_OS_TEST_LOG" +printf ' %s' "$@" >> "$AGENT_OS_TEST_LOG" +printf '\n' >> "$AGENT_OS_TEST_LOG" +if [ "${1:-}" = image ] && [ "${2:-}" = inspect ]; then + printf '%s\n' "${AGENT_OS_TEST_IMAGE_ID:?AGENT_OS_TEST_IMAGE_ID is required for image inspection}" +fi +SH +chmod +x "$FAKEBIN/docker" make_fake kubectl make_fake orbctl run_cli() { - PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$LOG" "$CLI" "$@" + PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$LOG" \ + AGENT_OS_TEST_IMAGE_ID="${AGENT_OS_TEST_IMAGE_ID:-sha256:default}" "$CLI" "$@" } assert_call() { @@ -52,11 +63,33 @@ test_deploy_starts_local_kubernetes_and_applies_kustomize() { pass "deploy starts and targets only OrbStack" } -test_build_uses_local_image_name() { +test_rebuild_deploy_uses_a_new_immutable_local_tag() { : > "$LOG" - run_cli build - assert_call 'docker build -t agent-os:dev .' "build must use the local demo image tag" - pass "build uses the deterministic local image tag" + AGENT_OS_TEST_IMAGE_ID=sha256:stale run_cli build + AGENT_OS_TEST_IMAGE_ID=sha256:rebuilt run_cli build + AGENT_OS_TEST_IMAGE_ID=sha256:rebuilt run_cli deploy + + assert_call 'docker build -t agent-os:dev .' "build must retain the local demo image tag" + assert_call 'docker tag agent-os:dev agent-os:local-rebuilt' \ + "build must assign the rebuilt image a unique local tag" + assert_call 'kubectl --context orbstack -n agent-os-demo set image statefulset/agent-os-firstmate agent-os-init=agent-os:local-rebuilt firstmate=agent-os:local-rebuilt' \ + "deploy must replace the stale mutable tag with the rebuilt local tag" + pass "rebuild deploy selects the rebuilt local image instead of a stale mutable tag" +} + +test_explicit_image_override_is_used_without_retagging() { + : > "$LOG" + AGENT_OS_IMAGE=example.test/agent-os:custom run_cli build + AGENT_OS_IMAGE=example.test/agent-os:custom run_cli deploy + + assert_call 'docker build -t example.test/agent-os:custom .' \ + "build must preserve an explicit image override" + assert_call 'kubectl --context orbstack -n agent-os-demo set image statefulset/agent-os-firstmate agent-os-init=example.test/agent-os:custom firstmate=example.test/agent-os:custom' \ + "deploy must preserve an explicit image override" + if grep -F 'docker tag example.test/agent-os:custom' "$LOG" >/dev/null; then + fail "explicit image overrides must not be retagged" + fi + pass "explicit image override remains intact" } test_destroy_requires_exact_confirmation() { @@ -88,6 +121,7 @@ test_non_orbstack_context_is_fail_closed() { test_status_pins_context_and_namespace test_deploy_starts_local_kubernetes_and_applies_kustomize -test_build_uses_local_image_name +test_rebuild_deploy_uses_a_new_immutable_local_tag +test_explicit_image_override_is_used_without_retagging test_destroy_requires_exact_confirmation test_non_orbstack_context_is_fail_closed From 7765088c7d6d219a0551eaed04f7ce3ee5db51bb Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 11:07:05 +0200 Subject: [PATCH 18/56] Record GitHub issue to PR fleet proof --- docs/acceptance.md | 4 +- docs/evidence/2026-07-13-github-issue-pr.md | 112 ++++++++++++++++++++ 2 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 docs/evidence/2026-07-13-github-issue-pr.md diff --git a/docs/acceptance.md b/docs/acceptance.md index b69ba6a5f..328b4f363 100644 --- a/docs/acceptance.md +++ b/docs/acceptance.md @@ -14,10 +14,10 @@ Update it from real runs; planned behavior and model narration never count as pr | Same-Pod general-purpose crewmate | Firstmate-supervised Pi scout, real Kubernetes test, in-cluster context check, and report in `docs/evidence/2026-07-13-same-pod-firstmate.md` | Proven locally | | Separate persistent crewmate Pod with explicit authority | Terra-low task, explicit resources, read-only Pi Secret, no ServiceAccount token, and retained PVC in `docs/evidence/2026-07-13-separate-pod-recovery.md` | Proven locally | | Parent-only supervision and human attach | Same-Pod and separate-Pod tasks accepted only Firstmate briefs and steers; a connected Herdr client observed the live workspaces | Proven locally | -| Real issue to tested review-ready PR | Existing Firstmate delivery machinery | Not yet proven through the Kubernetes fleet | +| Real issue to tested review-ready PR | Issue [#2](https://github.com/akua-dev/agent-os/issues/2) became Terra-low task `fix-local-rebuild-image-r2`, tested commit `761223c`, and PR [#3](https://github.com/akua-dev/agent-os/pull/3), then merged into review branch PR #1; see `docs/evidence/2026-07-13-github-issue-pr.md` | Proven locally | | Parent and child restart without unique-work loss | Primary session recovery plus separate-Pod replacement with unique artifact, report, tools, and exact Pi session resumed | Proven locally | | Separate production read-only and scoped write identity | Cortex contract only | Not yet implemented or Red-approved | -| Optional GitHub Issue/Project client | Cortex contract only | Deferred until terminal-native path passes | +| Optional GitHub Issue/Project client | Repository Issues enabled only when the native intake proof needed it; Firstmate read Issue #2 through `gh-axi` and delivered PR #3 with no custom intake service | Issue client proven locally; Project client remains optional and unproven | | Repeatable end-to-end eval and recordable demo | Same-Pod and separate-Pod run records now capture model, time, cost estimate, interventions, resource IDs, and failures | Local agent lifecycle captured; Akua/GitHub/product path incomplete | | Public installable image and release | Multi-architecture workflow run `29188585384` succeeded; local image only because pull-request builds do not publish | Publication not yet proven | diff --git a/docs/evidence/2026-07-13-github-issue-pr.md b/docs/evidence/2026-07-13-github-issue-pr.md new file mode 100644 index 000000000..f9ba605a2 --- /dev/null +++ b/docs/evidence/2026-07-13-github-issue-pr.md @@ -0,0 +1,112 @@ +# GitHub Issue to Kubernetes crewmate PR verification + +Date: 2026-07-13 +Kubernetes context: `orbstack` +Namespace: `agent-os-demo` +Issue: [#2 Make local rebuilds select the new Agent OS image](https://github.com/akua-dev/agent-os/issues/2) +Delivery PR: [#3 fix: select rebuilt local demo image](https://github.com/akua-dev/agent-os/pull/3) +Final review surface: [#1 feat: run Firstmate as a Kubernetes agent OS](https://github.com/akua-dev/agent-os/pull/1) + +## Claim + +A GitHub Issue can act as optional Agent OS intake without becoming the runtime +or communication layer. A Kubernetes-resident Firstmate can read that issue, +turn it into a precise brief, dispatch and supervise a general-purpose +crewmate, and return a tested review-ready PR. Ordinary Git, GitHub, Firstmate +files, treehouse worktrees, and Herdr terminals remain the operating substrate. + +## Topology and authority + +- Primary Firstmate: Pi in dedicated Herdr workspace `w6`, persistent + `FM_HOME=/home/agent`. +- Crewmate task: `fix-local-rebuild-image-r2`, Herdr pane `w2:p9`. +- Model policy: `openai-codex/gpt-5.6-terra`, low thinking, recorded in the + task metadata. +- Worktree: + `/home/agent/.treehouse/agent-os-eaf3be/1/agent-os`. +- Delivery mode: `direct-PR`; no custom task service, workflow engine, or + inter-agent protocol. +- GitHub authority came from the selected credential already persisted in the + Firstmate home. No credential value was copied into the brief or evidence. + +The evaluator initially discovered that the public Agent OS repository had +Issues disabled. It enabled only that native repository surface, then created +Issue #2 from the observed stale-image failure. No GitHub Project or additional +intake service was required. + +## Parent-only supervision + +The evaluator prompted only the primary Firstmate. Firstmate read Issue #2 with +`gh-axi`, registered the existing Agent OS clone, wrote the child brief, and +spawned the crewmate through: + +```text +spawned fix-local-rebuild-image-r2 harness=pi kind=ship mode=direct-PR +model=openai-codex/gpt-5.6-terra effort=low +window=default:w2:p9 +``` + +Firstmate handled the Pi trust prompt through `fm-send.sh`, polled the child's +status and pane, armed `fm-pr-check.sh`, and inspected the delivered diff. The +evaluator made read-only Herdr observations and never prompted or steered the +child directly. The primary pane was moved into its own workspace early in the +run so task-tab cleanup could not remove the supervisor. + +## RED and GREEN evidence + +The child first extended `tests/agent-os-local.test.sh`. Against the old helper, +the focused test failed for the missing immutable tag operation: + +```text +not ok - build must assign the rebuilt image a unique local tag +(missing exact call: docker tag agent-os:dev agent-os:local-rebuilt) +RED_EXIT=1 +``` + +Commit `761223c533de5cba5a96681cf769ec12164a267e` then: + +- tags the default local image with its Docker image ID; +- updates both StatefulSet containers to that content-specific tag on deploy; +- preserves an explicit `AGENT_OS_IMAGE` without retagging it; +- documents the behavior; and +- adds hermetic stale-tag and override tests. + +The child passed `tests/agent-os-local.test.sh` and shell syntax checks. Its +attempt to run `tests/agent-os-kubernetes.test.sh` correctly exposed an +environment boundary: an in-cluster Firstmate Pod has a ServiceAccount token, +so it cannot represent the test's token-free host case. + +The evaluator closed that evidence gap from an isolated host worktree at the +exact child commit. All `tests/agent-os-*.test.sh` passed, including the +host-only refusal case. Focused ShellCheck, `bash -n`, and `git diff --check` +also passed. This independent result was recorded as a PR #3 comment before +integration. + +## GitHub result and timings + +- Issue #2 created: `2026-07-13T08:55:04Z`. +- PR #3 created: `2026-07-13T09:01:43Z`. +- Issue-to-review-ready-PR: 6 minutes 39 seconds. +- PR #3 merged into `feat/orbstack-demo`: `2026-07-13T09:05:28Z`. +- Issue-to-verified-integration: 10 minutes 24 seconds. +- Merge commit on the review branch: + `7679bda383d044aaec6d6fae7231bb7c0fb80576`. +- Files changed by the child: `AGENTS.md`, `bin/agent-os-local.sh`, + `docs/kubernetes.md`, and `tests/agent-os-local.test.sh`. + +Pi displayed approximate subscription cost estimates of `$0.966` for the child +and `$0.815` for the primary by the end of their panes. These are UI estimates, +not provider invoices. + +Three evaluator interventions occurred: move the primary into a dedicated +workspace, run the host-only verification, and merge the verified stacked PR +into the final review branch. No evaluator intervention changed the child's +implementation. + +## Scope + +This proves native GitHub Issue intake, parent-only same-Pod delegation, real +RED-to-GREEN implementation, a tested review-ready PR, and integration into the +single Agent OS review branch. It does not prove GitHub Projects, an +Akua-managed intelligence cluster, Akua worker lifecycle, or product-cluster +access boundaries. From 0da294a1e84f03e41fdf440e710a6bfed49e8f67 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 11:10:04 +0200 Subject: [PATCH 19/56] Record fleet task reconciliation --- docs/evidence/2026-07-13-github-issue-pr.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/evidence/2026-07-13-github-issue-pr.md b/docs/evidence/2026-07-13-github-issue-pr.md index f9ba605a2..efe7bd4f3 100644 --- a/docs/evidence/2026-07-13-github-issue-pr.md +++ b/docs/evidence/2026-07-13-github-issue-pr.md @@ -95,13 +95,18 @@ integration. `docs/kubernetes.md`, and `tests/agent-os-local.test.sh`. Pi displayed approximate subscription cost estimates of `$0.966` for the child -and `$0.815` for the primary by the end of their panes. These are UI estimates, +and `$0.917` for the primary after final reconciliation. These are UI estimates, not provider invoices. -Three evaluator interventions occurred: move the primary into a dedicated -workspace, run the host-only verification, and merge the verified stacked PR -into the final review branch. No evaluator intervention changed the child's -implementation. +Four evaluator interventions occurred: move the primary into a dedicated +workspace, run the host-only verification, merge the verified stacked PR into +the final review branch, and ask Firstmate to reconcile and tear down the +completed child after that external merge. No evaluator intervention changed +the child's implementation or communicated with it directly. + +Firstmate then ran its ordinary teardown path, marked the task done in the +backlog, removed the child pane and task metadata, and left the retained +separate-Pod recovery fixture, primary workspace, and committed evidence intact. ## Scope From 0b36ea7cb6ab771a69317998fcaa92d4c62b7ccf Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 12:01:51 +0200 Subject: [PATCH 20/56] docs: keep Agent OS work tracking private --- docs/acceptance.md | 4 ++-- docs/evidence/2026-07-13-github-issue-pr.md | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/acceptance.md b/docs/acceptance.md index 328b4f363..55ea1956e 100644 --- a/docs/acceptance.md +++ b/docs/acceptance.md @@ -14,10 +14,10 @@ Update it from real runs; planned behavior and model narration never count as pr | Same-Pod general-purpose crewmate | Firstmate-supervised Pi scout, real Kubernetes test, in-cluster context check, and report in `docs/evidence/2026-07-13-same-pod-firstmate.md` | Proven locally | | Separate persistent crewmate Pod with explicit authority | Terra-low task, explicit resources, read-only Pi Secret, no ServiceAccount token, and retained PVC in `docs/evidence/2026-07-13-separate-pod-recovery.md` | Proven locally | | Parent-only supervision and human attach | Same-Pod and separate-Pod tasks accepted only Firstmate briefs and steers; a connected Herdr client observed the live workspaces | Proven locally | -| Real issue to tested review-ready PR | Issue [#2](https://github.com/akua-dev/agent-os/issues/2) became Terra-low task `fix-local-rebuild-image-r2`, tested commit `761223c`, and PR [#3](https://github.com/akua-dev/agent-os/pull/3), then merged into review branch PR #1; see `docs/evidence/2026-07-13-github-issue-pr.md` | Proven locally | +| Real issue to tested review-ready PR | A private Cortex tracker issue became Terra-low task `fix-local-rebuild-image-r2`, tested commit `761223c`, and public PR [#3](https://github.com/akua-dev/agent-os/pull/3), then merged into review branch PR #1; see `docs/evidence/2026-07-13-github-issue-pr.md` | Proven locally | | Parent and child restart without unique-work loss | Primary session recovery plus separate-Pod replacement with unique artifact, report, tools, and exact Pi session resumed | Proven locally | | Separate production read-only and scoped write identity | Cortex contract only | Not yet implemented or Red-approved | -| Optional GitHub Issue/Project client | Repository Issues enabled only when the native intake proof needed it; Firstmate read Issue #2 through `gh-axi` and delivered PR #3 with no custom intake service | Issue client proven locally; Project client remains optional and unproven | +| Optional GitHub Issue/Project client | Firstmate read a private Cortex GitHub issue through `gh-axi` and delivered public PR #3 with no custom intake service; public Agent OS Issues are disabled because work tracking is private | Issue client proven locally; Project client remains optional and unproven | | Repeatable end-to-end eval and recordable demo | Same-Pod and separate-Pod run records now capture model, time, cost estimate, interventions, resource IDs, and failures | Local agent lifecycle captured; Akua/GitHub/product path incomplete | | Public installable image and release | Multi-architecture workflow run `29188585384` succeeded; local image only because pull-request builds do not publish | Publication not yet proven | diff --git a/docs/evidence/2026-07-13-github-issue-pr.md b/docs/evidence/2026-07-13-github-issue-pr.md index efe7bd4f3..b32b28858 100644 --- a/docs/evidence/2026-07-13-github-issue-pr.md +++ b/docs/evidence/2026-07-13-github-issue-pr.md @@ -3,7 +3,7 @@ Date: 2026-07-13 Kubernetes context: `orbstack` Namespace: `agent-os-demo` -Issue: [#2 Make local rebuilds select the new Agent OS image](https://github.com/akua-dev/agent-os/issues/2) +Issue: private Cortex tracker item; public implementation PR below Delivery PR: [#3 fix: select rebuilt local demo image](https://github.com/akua-dev/agent-os/pull/3) Final review surface: [#1 feat: run Firstmate as a Kubernetes agent OS](https://github.com/akua-dev/agent-os/pull/1) @@ -30,15 +30,16 @@ files, treehouse worktrees, and Herdr terminals remain the operating substrate. Firstmate home. No credential value was copied into the brief or evidence. The evaluator initially discovered that the public Agent OS repository had -Issues disabled. It enabled only that native repository surface, then created -Issue #2 from the observed stale-image failure. No GitHub Project or additional -intake service was required. +Issues disabled. It enabled only that native repository surface and created an +implementation issue from the observed stale-image failure. The tracker item +was later transferred into private Cortex and public Agent OS Issues were +disabled again. No GitHub Project or additional intake service was required. ## Parent-only supervision -The evaluator prompted only the primary Firstmate. Firstmate read Issue #2 with -`gh-axi`, registered the existing Agent OS clone, wrote the child brief, and -spawned the crewmate through: +The evaluator prompted only the primary Firstmate. Firstmate read the issue +with `gh-axi`, registered the existing Agent OS clone, wrote the child brief, +and spawned the crewmate through: ```text spawned fix-local-rebuild-image-r2 harness=pi kind=ship mode=direct-PR @@ -84,7 +85,7 @@ integration. ## GitHub result and timings -- Issue #2 created: `2026-07-13T08:55:04Z`. +- Private issue created: `2026-07-13T08:55:04Z`. - PR #3 created: `2026-07-13T09:01:43Z`. - Issue-to-review-ready-PR: 6 minutes 39 seconds. - PR #3 merged into `feat/orbstack-demo`: `2026-07-13T09:05:28Z`. From 25f62328a28631edd71f3eeb37cc60cb5e3c5016 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 12:33:13 +0200 Subject: [PATCH 21/56] docs: separate Kubernetes core from Akua integration --- README.md | 4 ++++ docs/acceptance.md | 2 ++ docs/kubernetes.md | 8 ++++++++ 3 files changed, 14 insertions(+) diff --git a/README.md b/README.md index ec05344ef..fbafb8d48 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,10 @@ the [OrbStack Kubernetes demo](docs/kubernetes.md), then use the optional [focused bootstrap guidance](.agents/skills/akua-intelligence-bootstrap/SKILL.md) for a dedicated Akua-managed intelligence cluster. +Agent OS works on Kubernetes and is better with Akua. +The portable core can be built and tested on local OrbStack without an Akua account, API key, managed control plane, or Akua-hosted worker, and is intended for any conformant Kubernetes cluster. +Akua adds the preferred credential-to-cluster bootstrap, managed capacity, identity and secret primitives, and guarded product delivery; it is an integration advantage, not a dependency of the core runtime. + ## Features - **One liaison** - you talk only to the first mate; it dispatches, supervises, escalates only real decisions, and reports plain outcomes. diff --git a/docs/acceptance.md b/docs/acceptance.md index 55ea1956e..601ab5ab1 100644 --- a/docs/acceptance.md +++ b/docs/acceptance.md @@ -5,6 +5,7 @@ Update it from real runs; planned behavior and model narration never count as pr | Requirement | Current evidence | State | | --- | --- | --- | +| Portable Kubernetes core without Akua | `docs/kubernetes.md` and `bin/agent-os-local.sh` provide the OrbStack build, deploy, status, shell, and attach path without an Akua account, credential, managed control plane, or hosted worker | Documented locally; clean published-source run remains required | | Persistent Firstmate and Herdr in Kubernetes | OrbStack demo plus generic package render and disposable-cluster run in `docs/evidence/2026-07-12-firstmate-package.md` | Proven locally | | Firstmate cluster-admin limited to the intelligence cluster | Dedicated local namespace and explicit demo ClusterRoleBinding | Proven locally only | | Direct Akua-managed KaaS and Hetzner bootstrap | Public endpoint study and `akua-intelligence-bootstrap` skill | Not yet run | @@ -24,4 +25,5 @@ Update it from real runs; planned behavior and model narration never count as pr ## Definition of done Agent OS is finished only when every row is backed by current external evidence and no row remains partial or unproven. +The portable Kubernetes gate and Akua integration gate are separate: the core must pass from published sources on local OrbStack without Akua, while the enhanced path separately proves Akua-managed bootstrap and guarded delivery. The complete run must record time-to-cluster, time-to-Firstmate, time-to-crewmate, time-to-PR, recovery time, model and infrastructure cost, human interventions, immutable image digest, resource and operation IDs, and sanitized failure evidence. diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 12aa5ff82..2e349f233 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -3,6 +3,10 @@ This demo runs Firstmate as the persistent controller of an isolated agent cluster. Kubernetes distributes and isolates the crew; it does not replace Firstmate's supervision model or Herdr's terminal/session interface. +This is the provider-independent Agent OS acceptance environment. +Running it requires no Akua account, API key, managed control plane, or Akua-hosted worker. +The same core is intended to run on any conformant Kubernetes cluster; Akua is an optional enhanced integration for faster bootstrap, managed capacity, identity, secrets, and guarded product delivery. + The local topology is deliberately small: - `agent-os-firstmate-0` is a StatefulSet with a 20 Gi persistent home. @@ -35,6 +39,10 @@ bin/agent-os-local.sh status bin/agent-os-local.sh shell ``` +Those four commands are the clean local smoke path. +They build the image into OrbStack, start the local Kubernetes node, deploy the isolated persistent fleet, show its status, and open the Firstmate shell. +No Akua authentication step is part of this path. + With the default local image, each `build` also creates a content-addressed local tag. The following `deploy` updates both StatefulSet containers to that tag, so a rebuilt `agent-os:dev` cannot reuse a stale OrbStack runtime digest. Set `AGENT_OS_IMAGE` to build and deploy an explicit image name without replacing or retagging that override. The image includes Firstmate's complete required toolchain, including `gh`, `rg`, `fd`, Akua, `kubectl`, K9s, treehouse, no-mistakes, and every required AXI CLI. From bdbcbce8e7a9b133ce7779ea804abbe714ad39f3 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 13:37:25 +0200 Subject: [PATCH 22/56] feat: make Agent OS package portable --- .github/workflows/agent-os-image.yml | 10 + .github/workflows/ci.yml | 7 + README.md | 9 +- THIRD_PARTY_NOTICES.md | 1 + bin/agent-os-crewmate.sh | 88 ++------- bin/agent-os-kubernetes.sh | 56 ++++++ bin/agent-os-local.sh | 17 +- deploy/orbstack/inputs.yaml | 7 + deploy/orbstack/kustomization.yaml | 7 - deploy/orbstack/namespace.yaml | 8 - deploy/orbstack/primary.yaml | 88 --------- deploy/orbstack/rbac.yaml | 23 --- docs/acceptance.md | 6 +- docs/kubernetes.md | 173 +++++++----------- tests/agent-os-container.test.sh | 34 ++-- tests/agent-os-kubernetes.test.sh | 86 ++++++++- tests/agent-os-local.test.sh | 52 ++++-- tests/agent-os-packages.test.sh | 97 ++++++++-- tools/agent-os/AGENTS.md | 8 +- tools/agent-os/README.md | 3 +- tools/agent-os/packages/firstmate/README.md | 24 ++- .../agent-os/packages/firstmate/crewmate.yaml | 66 +++++++ .../packages/firstmate/inputs.example.yaml | 7 +- tools/agent-os/packages/firstmate/package.k | 92 ++++++---- tools/agent-os/packages/mate/README.md | 23 --- tools/agent-os/packages/mate/akua.toml | 5 - .../packages/mate/inputs.example.yaml | 6 - tools/agent-os/packages/mate/package.k | 117 ------------ 28 files changed, 555 insertions(+), 565 deletions(-) create mode 100755 bin/agent-os-kubernetes.sh create mode 100644 deploy/orbstack/inputs.yaml delete mode 100644 deploy/orbstack/kustomization.yaml delete mode 100644 deploy/orbstack/namespace.yaml delete mode 100644 deploy/orbstack/primary.yaml delete mode 100644 deploy/orbstack/rbac.yaml create mode 100644 tools/agent-os/packages/firstmate/crewmate.yaml delete mode 100644 tools/agent-os/packages/mate/README.md delete mode 100644 tools/agent-os/packages/mate/akua.toml delete mode 100644 tools/agent-os/packages/mate/inputs.example.yaml delete mode 100644 tools/agent-os/packages/mate/package.k diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index a529a5864..32f776403 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -48,6 +48,7 @@ jobs: type=ref,event=tag - name: Build and optionally publish + id: build uses: docker/build-push-action@v6 with: context: . @@ -59,3 +60,12 @@ jobs: cache-to: type=gha,mode=max provenance: mode=max sbom: true + + - name: Record published image digest + if: github.event_name != 'pull_request' + run: | + { + echo '### Agent OS image' + echo + echo "Digest: \`${{ steps.build.outputs.digest }}\`" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a75d82d3b..b2ea28704 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,13 @@ jobs: set -eu npm install -g tasks-axi tasks-axi --version + - name: Install Akua renderer + run: | + set -eu + export AKUA_INSTALL="$RUNNER_TEMP/akua" + curl -fsSL https://cli.akua.dev/install | sh -s -- 0.8.25 + echo "$AKUA_INSTALL/bin" >> "$GITHUB_PATH" + "$AKUA_INSTALL/bin/akua" --version - run: | set -eu for test_script in tests/*.test.sh; do diff --git a/README.md b/README.md index fbafb8d48..dfb14d5ce 100644 --- a/README.md +++ b/README.md @@ -47,11 +47,10 @@ Launching a supported harness inside it instantiates your first mate - and makes This Agent OS fork also packages the distro for Kubernetes: one persistent first mate can allocate isolated, persistent crewmate containers using ordinary -`kubectl`, while Herdr keeps every agent terminal visible. Start locally with -the [OrbStack Kubernetes demo](docs/kubernetes.md), then use the optional -[Firstmate Akua package](tools/agent-os/packages/firstmate/README.md) and the -[focused bootstrap guidance](.agents/skills/akua-intelligence-bootstrap/SKILL.md) -for a dedicated Akua-managed intelligence cluster. +`kubectl`, while Herdr keeps every agent terminal visible. Follow the +[Agent OS Kubernetes quickstart](docs/kubernetes.md) to render the one public +package for any conformant cluster. OrbStack is a local test profile of that +same package, not a separate installer. Agent OS works on Kubernetes and is better with Akua. The portable core can be built and tested on local OrbStack without an Akua account, API key, managed control plane, or Akua-hosted worker, and is intended for any conformant Kubernetes cluster. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 7e07a1233..2fca029bb 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -4,6 +4,7 @@ The Agent OS demo image includes an unmodified Herdr 0.7.3 executable as a separate program. Herdr is available under AGPL-3.0-or-later or a commercial license. +Public image publication is gated on a compliant license path being selected for this distribution. The image includes Herdr's license at `/usr/share/licenses/herdr/LICENSE`. The exact corresponding source is available at . Agent OS invokes Herdr through its documented CLI and socket interfaces. diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index 2767f0b9c..79510d347 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -5,9 +5,15 @@ set -eu COMMAND=${1:-} ID=${2:-} -NAMESPACE=${AGENT_OS_NAMESPACE:-agent-os-demo} -IMAGE=${AGENT_OS_IMAGE:-agent-os:dev} +NAMESPACE=${AGENT_OS_NAMESPACE:-agent-os} +IMAGE=${AGENT_OS_IMAGE:-} +IMAGE_PULL_POLICY=${AGENT_OS_IMAGE_PULL_POLICY:-IfNotPresent} KUBECTL=${AGENT_OS_KUBECTL:-kubectl} +TEMPLATE=${AGENT_OS_CREWMATE_TEMPLATE:-/opt/agent-os/tools/agent-os/packages/firstmate/crewmate.yaml} + +if [ ! -f "$TEMPLATE" ]; then + TEMPLATE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/tools/agent-os/packages/firstmate/crewmate.yaml" +fi case "$ID" in ''|*[!a-z0-9-]*|-*|*-) echo "error: invalid crewmate id '$ID'" >&2; exit 2 ;; @@ -26,74 +32,16 @@ PVC="$POD-home" case "$COMMAND" in create) - "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" apply -f - <&2 + exit 2 + fi + sed \ + -e "s|__AGENT_OS_CREWMATE_ID__|$ID|g" \ + -e "s|__AGENT_OS_NAMESPACE__|$NAMESPACE|g" \ + -e "s|__AGENT_OS_IMAGE__|$IMAGE|g" \ + -e "s|__AGENT_OS_IMAGE_PULL_POLICY__|$IMAGE_PULL_POLICY|g" \ + "$TEMPLATE" | "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" apply -f - ;; status) "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" get pod "$POD" diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh new file mode 100755 index 000000000..9cf83f8d8 --- /dev/null +++ b/bin/agent-os-kubernetes.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# agent-os-kubernetes.sh - render and operate the portable Agent OS package. +# Usage: bin/agent-os-kubernetes.sh install|upgrade|rollback|status|uninstall [--yes] +set -eu + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +COMMAND=${1:-} +CONFIRM=${2:-} +PACKAGE=${AGENT_OS_PACKAGE:-"$ROOT/tools/agent-os/packages/firstmate/package.k"} +INPUTS=${AGENT_OS_INPUTS:-"$ROOT/tools/agent-os/packages/firstmate/inputs.example.yaml"} +CONTEXT=${AGENT_OS_CONTEXT:-} +NAMESPACE=${AGENT_OS_NAMESPACE:-agent-os} +AKUA=${AGENT_OS_AKUA:-akua} +KUBECTL=${AGENT_OS_KUBECTL:-kubectl} +OUT=$(mktemp -d) + +cleanup() { + rm -rf "$OUT" +} +trap cleanup EXIT + +if [ -z "$CONTEXT" ]; then + echo "error: set AGENT_OS_CONTEXT to the Kubernetes context to operate" >&2 + exit 2 +fi + +render() { + "$AKUA" render --no-agent-mode --package "$PACKAGE" --inputs "$INPUTS" --out "$OUT" +} + +case "$COMMAND" in + install|upgrade) + render + "$KUBECTL" --context "$CONTEXT" apply -f "$OUT" + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s + ;; + rollback) + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout undo statefulset/agent-os-firstmate + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s + ;; + status) + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get statefulset agent-os-firstmate + ;; + uninstall) + if [ "$CONFIRM" != --yes ]; then + echo "error: uninstall requires --yes" >&2 + exit 2 + fi + render + "$KUBECTL" --context "$CONTEXT" delete --ignore-not-found -f "$OUT" + ;; + *) + echo "usage: $0 install|upgrade|rollback|status|uninstall [--yes]" >&2 + exit 2 + ;; +esac diff --git a/bin/agent-os-local.sh b/bin/agent-os-local.sh index d0c4e7ad8..e77176d78 100755 --- a/bin/agent-os-local.sh +++ b/bin/agent-os-local.sh @@ -9,6 +9,7 @@ NAMESPACE=${AGENT_OS_NAMESPACE:-agent-os-demo} IMAGE=${AGENT_OS_IMAGE:-agent-os:dev} IMAGE_IS_OVERRIDE=${AGENT_OS_IMAGE+x} COMMAND=${1:-} +PROFILE="$ROOT/deploy/orbstack/inputs.yaml" if [ "$CONTEXT" != orbstack ] && [ "${AGENT_OS_ALLOW_NON_ORBSTACK:-0}" != 1 ]; then echo "error: refusing Kubernetes context '$CONTEXT'; set AGENT_OS_ALLOW_NON_ORBSTACK=1 to opt in" >&2 @@ -21,6 +22,15 @@ local_image_tag() { printf 'agent-os:local-%s\n' "${image_id#sha256:}" } +render_profile() { + local image=$1 inputs + inputs=$(mktemp) + trap 'rm -f "$inputs"' RETURN + sed "s|^image: .*|image: $image|" "$PROFILE" > "$inputs" + AGENT_OS_CONTEXT="$CONTEXT" AGENT_OS_NAMESPACE="$NAMESPACE" AGENT_OS_INPUTS="$inputs" \ + "$ROOT/bin/agent-os-kubernetes.sh" install +} + cd "$ROOT" case "$COMMAND" in @@ -35,12 +45,10 @@ case "$COMMAND" in orbctl start k8s kubectl --context "$CONTEXT" wait --for=condition=Ready node/orbstack --timeout=120s fi - kubectl --context "$CONTEXT" apply -k deploy/orbstack if [ -z "$IMAGE_IS_OVERRIDE" ]; then IMAGE=$(local_image_tag) fi - kubectl --context "$CONTEXT" -n "$NAMESPACE" set image statefulset/agent-os-firstmate \ - "agent-os-init=$IMAGE" "firstmate=$IMAGE" + render_profile "$IMAGE" ;; status) kubectl --context "$CONTEXT" -n "$NAMESPACE" get statefulset agent-os-firstmate @@ -56,7 +64,8 @@ case "$COMMAND" in echo "error: destroy requires --yes and deletes only namespace '$NAMESPACE'" >&2 exit 2 fi - kubectl --context "$CONTEXT" delete namespace "$NAMESPACE" + AGENT_OS_CONTEXT="$CONTEXT" AGENT_OS_NAMESPACE="$NAMESPACE" AGENT_OS_INPUTS="$PROFILE" \ + "$ROOT/bin/agent-os-kubernetes.sh" uninstall --yes ;; *) echo "usage: $0 build|deploy|status|shell|attach|destroy [--yes]" >&2 diff --git a/deploy/orbstack/inputs.yaml b/deploy/orbstack/inputs.yaml new file mode 100644 index 000000000..31c932636 --- /dev/null +++ b/deploy/orbstack/inputs.yaml @@ -0,0 +1,7 @@ +# Local profile for OrbStack's image store and isolated demo cluster. +namespace: agent-os-demo +image: agent-os:dev +imagePullPolicy: Never +allowMutableImage: true +rbac: cluster-admin +storage: 20Gi diff --git a/deploy/orbstack/kustomization.yaml b/deploy/orbstack/kustomization.yaml deleted file mode 100644 index 57f9eaf5f..000000000 --- a/deploy/orbstack/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - namespace.yaml - - rbac.yaml - - primary.yaml - diff --git a/deploy/orbstack/namespace.yaml b/deploy/orbstack/namespace.yaml deleted file mode 100644 index 8e919fbef..000000000 --- a/deploy/orbstack/namespace.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: agent-os-demo - labels: - app.kubernetes.io/name: agent-os - app.kubernetes.io/part-of: agent-os-demo - diff --git a/deploy/orbstack/primary.yaml b/deploy/orbstack/primary.yaml deleted file mode 100644 index 4debe6814..000000000 --- a/deploy/orbstack/primary.yaml +++ /dev/null @@ -1,88 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: agent-os-firstmate-home - namespace: agent-os-demo - labels: - app.kubernetes.io/name: agent-os - app.kubernetes.io/component: firstmate -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 20Gi ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: agent-os-firstmate - namespace: agent-os-demo - labels: - app.kubernetes.io/name: agent-os - app.kubernetes.io/component: firstmate -spec: - serviceName: agent-os-firstmate - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: agent-os - app.kubernetes.io/component: firstmate - template: - metadata: - labels: - app.kubernetes.io/name: agent-os - app.kubernetes.io/component: firstmate - spec: - serviceAccountName: agent-os-firstmate - securityContext: - fsGroup: 0 - runAsGroup: 0 - runAsUser: 0 - initContainers: - - name: agent-os-init - image: agent-os:dev - imagePullPolicy: Never - command: - - /opt/agent-os/bin/agent-os-init.sh - volumeMounts: - - name: home - mountPath: /persistent-agent - containers: - - name: firstmate - image: agent-os:dev - imagePullPolicy: Never - env: - - name: FM_HOME - value: /home/agent - - name: HOME - value: /home/agent - - name: AGENT_OS_TEST_PI_MODEL - value: openai-codex/gpt-5.6-terra - - name: AGENT_OS_TEST_PI_EFFORT - value: low - readinessProbe: - exec: - command: - - herdr - - status - - --json - initialDelaySeconds: 2 - periodSeconds: 5 - resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: "4" - memory: 8Gi - volumeMounts: - - name: home - mountPath: /home/agent - - name: home - mountPath: /usr/local - subPath: usr-local - volumes: - - name: home - persistentVolumeClaim: - claimName: agent-os-firstmate-home diff --git a/deploy/orbstack/rbac.yaml b/deploy/orbstack/rbac.yaml deleted file mode 100644 index db3773b90..000000000 --- a/deploy/orbstack/rbac.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: agent-os-firstmate - namespace: agent-os-demo ---- -# Local demo only. Production installations require a reviewed narrower role. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: agent-os-firstmate-local-demo - labels: - app.kubernetes.io/name: agent-os - app.kubernetes.io/part-of: agent-os-demo -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: - - kind: ServiceAccount - name: agent-os-firstmate - namespace: agent-os-demo - diff --git a/docs/acceptance.md b/docs/acceptance.md index 601ab5ab1..bb3969598 100644 --- a/docs/acceptance.md +++ b/docs/acceptance.md @@ -5,8 +5,8 @@ Update it from real runs; planned behavior and model narration never count as pr | Requirement | Current evidence | State | | --- | --- | --- | -| Portable Kubernetes core without Akua | `docs/kubernetes.md` and `bin/agent-os-local.sh` provide the OrbStack build, deploy, status, shell, and attach path without an Akua account, credential, managed control plane, or hosted worker | Documented locally; clean published-source run remains required | -| Persistent Firstmate and Herdr in Kubernetes | OrbStack demo plus generic package render and disposable-cluster run in `docs/evidence/2026-07-12-firstmate-package.md` | Proven locally | +| Portable Kubernetes core without Akua | `tools/agent-os/packages/firstmate/` and focused render/behavior tests prove a digest-required package, namespace-scoped default RBAC, persistent home, explicit lifecycle commands, and no package credential input | Render and command contracts proven; clean published-image run remains required | +| Persistent Firstmate and Herdr in Kubernetes | Canonical package render plus local lifecycle records in `docs/evidence/2026-07-12-firstmate-package.md` | Package render and local evidence proven; published-image run remains required | | Firstmate cluster-admin limited to the intelligence cluster | Dedicated local namespace and explicit demo ClusterRoleBinding | Proven locally only | | Direct Akua-managed KaaS and Hetzner bootstrap | Public endpoint study and `akua-intelligence-bootstrap` skill | Not yet run | | Distinct clustered token and bootstrap-token revocation | Explicit Secret mount in the Firstmate package and handoff procedure | Not yet run | @@ -20,7 +20,7 @@ Update it from real runs; planned behavior and model narration never count as pr | Separate production read-only and scoped write identity | Cortex contract only | Not yet implemented or Red-approved | | Optional GitHub Issue/Project client | Firstmate read a private Cortex GitHub issue through `gh-axi` and delivered public PR #3 with no custom intake service; public Agent OS Issues are disabled because work tracking is private | Issue client proven locally; Project client remains optional and unproven | | Repeatable end-to-end eval and recordable demo | Same-Pod and separate-Pod run records now capture model, time, cost estimate, interventions, resource IDs, and failures | Local agent lifecycle captured; Akua/GitHub/product path incomplete | -| Public installable image and release | Multi-architecture workflow run `29188585384` succeeded; local image only because pull-request builds do not publish | Publication not yet proven | +| Public installable image and release | Multi-architecture release workflow builds `linux/amd64` and `linux/arm64`, publishes only outside pull requests, and records the resulting digest in its job summary | Publication not yet proven | ## Definition of done diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 2e349f233..1af81f6aa 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -1,150 +1,105 @@ -# Kubernetes Agent OS demo +# Kubernetes Agent OS -This demo runs Firstmate as the persistent controller of an isolated agent cluster. +Agent OS installs one persistent Firstmate from the public, versioned package at `tools/agent-os/packages/firstmate/`. +The package renders ordinary Kubernetes resources. +It is free to render and apply on any conformant Kubernetes cluster without an Akua account, API key, managed control plane, or Akua-hosted worker. Kubernetes distributes and isolates the crew; it does not replace Firstmate's supervision model or Herdr's terminal/session interface. -This is the provider-independent Agent OS acceptance environment. -Running it requires no Akua account, API key, managed control plane, or Akua-hosted worker. -The same core is intended to run on any conformant Kubernetes cluster; Akua is an optional enhanced integration for faster bootstrap, managed capacity, identity, secrets, and guarded product delivery. - -The local topology is deliberately small: - -- `agent-os-firstmate-0` is a StatefulSet with a 20 Gi persistent home. -- Herdr 0.7.3 is PID 1 and remains the visible session backend. -- the primary's local-demo ServiceAccount has `cluster-admin`, so it can shape its own crew and workspace; -- every crewmate is an adaptable Agent OS container with its own 10 Gi PVC; -- crewmates do not receive Kubernetes service-account credentials by default. - -The primary and crewmates run as UID 0 inside their containers. -OrbStack's built-in Kubernetes does not support Pod user namespaces, so container root is also root on the dedicated OrbStack VM node. -The Pods do not use privileged mode, host namespaces, host mounts, or raw block devices. -This root policy and the broad primary grant are only for the isolated local agent cluster. -They are not a production-cluster access pattern. - ## Requirements -- OrbStack with Kubernetes enabled -- Docker -- `kubectl` +- A Kubernetes context that you are authorized to use explicitly. +- `kubectl` and the public `akua` renderer CLI on your PATH. +- Permission to create the selected namespace when `createNamespace: true`, plus the rendered ServiceAccount, Role, RoleBinding, Service, PVC, and StatefulSet. +- A default or selected StorageClass that satisfies the `ReadWriteOnce` PVC request. +- An immutable image digest published by an Agent OS release. -The helper refuses ambient Kubernetes contexts. -It uses `orbstack` unless a different context is deliberately enabled with `AGENT_OS_ALLOW_NON_ORBSTACK=1`. +No Kubernetes Secret is required for the portable install. +The package has no credential field or Secret reference. +Runtime AI, GitHub, or other authority is created separately by its owner after installation and is never supplied through package inputs, command arguments, or rendered YAML. -## Run the demo +The package requires an image digest because a mutable tag cannot identify an upgrade or recovery input. +The source tree contains an intentionally non-installable placeholder until the first public image release is published. +Replace it only with the digest recorded by that release workflow. -```sh -bin/agent-os-local.sh build -bin/agent-os-local.sh deploy -bin/agent-os-local.sh status -bin/agent-os-local.sh shell -``` +## Generic quickstart -Those four commands are the clean local smoke path. -They build the image into OrbStack, start the local Kubernetes node, deploy the isolated persistent fleet, show its status, and open the Firstmate shell. -No Akua authentication step is part of this path. +Start from a tagged Agent OS source checkout that matches the image release. +Copy the stable input schema and replace its image value with that release's immutable digest. -With the default local image, each `build` also creates a content-addressed local tag. The following `deploy` updates both StatefulSet containers to that tag, so a rebuilt `agent-os:dev` cannot reuse a stale OrbStack runtime digest. Set `AGENT_OS_IMAGE` to build and deploy an explicit image name without replacing or retagging that override. +```sh +cp tools/agent-os/packages/firstmate/inputs.example.yaml /tmp/agent-os-inputs.yaml +$EDITOR /tmp/agent-os-inputs.yaml -The image includes Firstmate's complete required toolchain, including `gh`, `rg`, `fd`, Akua, `kubectl`, K9s, treehouse, no-mistakes, and every required AXI CLI. -Authenticate GitHub inside the primary with `gh auth login`. -Authenticate Pi using `/login` and the provider flow you choose. -Host credentials are intentionally excluded from the image and are never copied automatically. -Authentication stored below `/home/agent` persists on the individual agent PVC. +export AGENT_OS_CONTEXT=your-kubernetes-context +export AGENT_OS_NAMESPACE=agent-os +export AGENT_OS_INPUTS=/tmp/agent-os-inputs.yaml -Tools installed below `/home/agent/.local`, `/home/agent/.bun`, `/home/agent/.cargo`, or `/usr/local` survive Pod replacement. -Global npm installs use the persistent `/usr/local` prefix. -`apt` is available because the container runs as root, but packages written elsewhere in the container filesystem are ephemeral. +bin/agent-os-kubernetes.sh install +``` -Start Pi from the tracked distro and attach to Herdr from another terminal: +The installer renders only `tools/agent-os/packages/firstmate/package.k`, applies that fresh output to the named context, and waits for `agent-os-firstmate` to roll out. +It never reads an ambient Kubernetes context. -```sh -# inside the primary shell -cd /opt/agent-os -pi --model openai-codex/gpt-5.6-terra --thinking low +The default `rbac: namespace` creates a ServiceAccount plus a Role and RoleBinding scoped to the selected namespace. +That Role allows Firstmate to manage runtime crewmate Pods and PVCs and inspect its StatefulSet. +Set `rbac: none` only when another reviewed authority handles those runtime operations. +Set `rbac: cluster-admin` only for an isolated intelligence cluster after reviewing the broader ClusterRoleBinding. -# on the host -bin/agent-os-local.sh attach -``` +The persistent home PVC defaults to `20Gi` and is mounted at `/home/agent`. +`/usr/local` is a subpath of that same PVC so user-installed global tools survive Pod replacement. -During the current local evaluation, the OrbStack manifest also converges -`config/crew-dispatch.json` and `config/secondmate-harness` on every primary Pod -start. Crewmates and Secondmates therefore launch through Pi with -`openai-codex/gpt-5.6-terra` and low thinking. The reusable image and Akua -packages remain model-agnostic when those local test environment variables are -absent. The same opt-in policy updates Pi's own `defaultProvider`, -`defaultModel`, and `defaultThinkingLevel`, so a plain `pi` primary session uses -Terra-low too. +## Operations -The primary can create and manage isolated crewmates directly: +To upgrade, use a new released digest in the same input file and apply it through the same package. ```sh -bin/agent-os-crewmate.sh create scout-1 -bin/agent-os-crewmate.sh status scout-1 -bin/agent-os-crewmate.sh delete scout-1 +bin/agent-os-kubernetes.sh upgrade ``` -The prepared package is an optional alternative when typed, editable manifests are useful: +To roll back only the Firstmate workload revision, use Kubernetes StatefulSet history. +This does not roll back package inputs, RBAC, or persistent data. ```sh -akua render \ - --package tools/agent-os/packages/mate/package.k \ - --inputs tools/agent-os/packages/mate/inputs.example.yaml \ - --out /tmp/scout-1 -kubectl apply -f /tmp/scout-1 +bin/agent-os-kubernetes.sh rollback ``` -Firstmate may inspect or edit the result, change the package, compose raw YAML, or use the compatibility helper. -Agent OS does not wrap these capable tools into a second workflow CLI. - -AI credentials are an explicit per-mate grant. -Create or select a Kubernetes Secret containing an `auth.json` key, then pass only its name as the package's `piAuthSecret` input. -The package mounts that file read-only and never discovers credentials by itself. +To inspect the workload, use the same explicit context and namespace. ```sh -kubectl -n agent-os-demo create secret generic agent-os-mate-scout-1-pi-auth \ - --from-file=auth.json=/home/agent/.pi/agent/auth.json +bin/agent-os-kubernetes.sh status ``` -Do this only when copying that selected credential set was authorized. -Prefer an already curated Secret when one exists. - -Give each launched Herdr agent a task-unique name and an explicit completion artifact in its brief. -Use ordinary Herdr and Kubernetes commands to launch, inspect, steer, and retrieve it. +Uninstall is deliberately confirmed and bounded to a fresh render of the selected package inputs. +When those inputs create the namespace, deleting that rendered Namespace also removes everything in that namespace. +When `createNamespace: false`, the command deletes only the rendered Agent OS resources in the existing namespace. ```sh -kubectl -n agent-os-demo exec agent-os-mate-scout-1 -- \ - herdr agent start scout-1-research --cwd /home/agent --no-focus -- \ - pi --model openai-codex/gpt-5.6-terra --thinking low "$(cat /tmp/scout-1.prompt)" -kubectl -n agent-os-demo exec agent-os-mate-scout-1 -- \ - test -s /home/agent/data/scout-1.md +bin/agent-os-kubernetes.sh uninstall --yes ``` -Do not treat Herdr `idle` alone as completion because a newly launched agent may briefly be idle before it starts work. -Check the declared artifact or delivered Git state and read the pane when it is absent. -Before reusing a name restored from a persistent Herdr session, verify that no agent process is live and close only the confirmed stale pane. +## Runtime mates -When run on a host, the compatibility helper requires an explicit context: +Firstmate creates a separate-Pod crewmate at runtime with the internal `crewmate.yaml` template in the canonical package. +It is not a second public package and is not a Marketplace product. +Each runtime mate has its own PVC and no ambient Kubernetes ServiceAccount token. -```sh -AGENT_OS_CONTEXT=orbstack bin/agent-os-crewmate.sh status scout-1 -``` +The existing [same-Pod](evidence/2026-07-13-same-pod-firstmate.md) and [separate-Pod recovery](evidence/2026-07-13-separate-pod-recovery.md) records remain local lifecycle evidence. +They do not substitute for a clean published-image installation. -Inside an authorized Firstmate Pod, Agent OS creates a rotation-safe kubeconfig that references the projected ServiceAccount token file. -It never copies the token value into the kubeconfig. +## OrbStack profile -Destroying the demo requires confirmation and deletes only the demo namespace: +OrbStack is a test profile of the canonical package, not the default product contract. +Its `deploy/orbstack/inputs.yaml` changes only the local environment inputs: the isolated `agent-os-demo` namespace, a local image source with `imagePullPolicy: Never` and `allowMutableImage: true`, and its explicit local-demo `cluster-admin` grant. +The profile uses the same package, ServiceAccount, persistent home, runtime mate template, and lifecycle commands. ```sh -bin/agent-os-local.sh destroy --yes +bin/agent-os-local.sh build +bin/agent-os-local.sh deploy +bin/agent-os-local.sh status +bin/agent-os-local.sh shell ``` -## What the demo proves - -The primary and child homes survive Pod replacement. -Tools installed into the persistent home prefixes and `/usr/local` also survive Pod replacement. -A child cannot use the Kubernetes API through an automatically mounted token. -The primary can create, inspect, and delete child Pods and PVCs using ordinary `kubectl`. -No custom agent communication protocol or GitOps controller is required. - -This is the local substrate, not the final access model. -Remote Agent OS clusters should keep the intelligence cluster isolated and grant product or production access deliberately per task, with Akua providing stable infrastructure primitives and guardrails. +The local helper refuses a non-OrbStack context unless `AGENT_OS_ALLOW_NON_ORBSTACK=1` is set deliberately. +It tags each rebuilt local image by content before rendering the profile so a deployment cannot reuse a stale mutable image. +The local profile is an isolated test environment only. +Its broad RoleBinding and container-root policy are not a production-cluster access pattern. diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index b82738e62..10e6f6361 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -49,20 +49,22 @@ assert_grep 'AGENT_OS_TEST_PI_MODEL' "$ROOT/bin/agent-os-container-entrypoint.sh "test-mode Pods must converge the Pi model policy" assert_grep 'defaultThinkingLevel' "$ROOT/bin/agent-os-container-entrypoint.sh" \ "test-mode Pods must also pin direct Pi sessions" -assert_grep 'openai-codex/gpt-5.6-terra' "$ROOT/deploy/orbstack/primary.yaml" \ - "the local test fleet must pin Terra" -assert_grep 'AGENT_OS_TEST_PI_EFFORT' "$ROOT/deploy/orbstack/primary.yaml" \ - "the local test fleet must pin Pi reasoning effort" +assert_grep 'image: agent-os:dev' "$ROOT/deploy/orbstack/inputs.yaml" \ + "the OrbStack profile must define its local image source" +assert_grep 'imagePullPolicy: Never' "$ROOT/deploy/orbstack/inputs.yaml" \ + "the OrbStack profile must use its local image store" +assert_grep 'rbac: cluster-admin' "$ROOT/deploy/orbstack/inputs.yaml" \ + "the OrbStack profile must make its local-demo grant explicit" assert_grep 'tokenFile:' "$ROOT/bin/agent-os-kubeconfig.sh" "kubeconfig must follow the projected token file" assert_no_grep 'set-credentials.*--token' "$ROOT/bin/agent-os-kubeconfig.sh" "kubeconfig must not copy a bearer token" -assert_grep 'automountServiceAccountToken = False' "$ROOT/tools/agent-os/packages/mate/package.k" \ - "mate package must deny ambient Kubernetes credentials" -assert_grep 'piAuthSecret: str = ""' "$ROOT/tools/agent-os/packages/mate/package.k" \ - "mate package must default to no AI credential grant" -assert_grep 'readOnly = True' "$ROOT/tools/agent-os/packages/mate/package.k" \ - "granted AI credentials must be mounted read-only" -assert_grep 'secretName = input.piAuthSecret' "$ROOT/tools/agent-os/packages/mate/package.k" \ - "mate package must use only the explicitly selected Secret" +assert_grep 'automountServiceAccountToken: false' "$ROOT/tools/agent-os/packages/firstmate/crewmate.yaml" \ + "runtime mate template must deny ambient Kubernetes credentials" +assert_grep 'agent-os.dev/crewmate' "$ROOT/tools/agent-os/packages/firstmate/crewmate.yaml" \ + "runtime mate template must use the portable Agent OS label" +assert_grep 'AGENT_OS_CREWMATE_TEMPLATE' "$ROOT/bin/agent-os-crewmate.sh" \ + "mate runtime must render the canonical package template" +assert_absent "$ROOT/tools/agent-os/packages/mate/package.k" \ + "mate creation must not remain a separately installable package" assert_grep '.git' "$ROOT/.dockerignore" "git metadata must stay out of the build context" assert_grep '.pi' "$ROOT/.dockerignore" "Pi credentials must stay out of the build context" assert_grep '.codex' "$ROOT/.dockerignore" "Codex credentials must stay out of the build context" @@ -75,11 +77,17 @@ assert_grep 'https://github.com/akua-dev/akua/tree/v0.8.25' "$ROOT/THIRD_PARTY_N assert_grep 'https://github.com/derailed/k9s/tree/v0.51.0' "$ROOT/THIRD_PARTY_NOTICES.md" \ "K9s's exact source must be named" assert_grep 'ghcr.io/akua-dev/agent-os' "$ROOT/.github/workflows/agent-os-image.yml" \ - "release workflow must publish the image expected by the Akua packages" + "release workflow must publish the image expected by the portable package" assert_grep 'linux/amd64,linux/arm64' "$ROOT/.github/workflows/agent-os-image.yml" \ "release workflow must build the two supported container architectures" assert_grep "push: \${{ github.event_name != 'pull_request' }}" "$ROOT/.github/workflows/agent-os-image.yml" \ "pull requests must build but never publish images" +assert_grep 'id: build' "$ROOT/.github/workflows/agent-os-image.yml" \ + "release workflow must expose its build result" +assert_grep 'steps.build.outputs.digest' "$ROOT/.github/workflows/agent-os-image.yml" \ + "release workflow must record the immutable published digest" +assert_grep 'publication is gated on a compliant license path' "$ROOT/THIRD_PARTY_NOTICES.md" \ + "Herdr notice must not claim publication approval before the license decision" bash -n "$ROOT/bin/agent-os-container-entrypoint.sh" bash -n "$ROOT/bin/agent-os-kubeconfig.sh" diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index 6a41c088c..6f35aaf6a 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -11,9 +11,19 @@ FAKEBIN=$(fm_fakebin "$TMP") CALLS="$TMP/calls.log" STDIN_LOG="$TMP/stdin.yaml" -rendered=$(kubectl kustomize "$ROOT/deploy/orbstack") || fail "OrbStack kustomization did not render" -assert_contains "$rendered" 'kind: Namespace' "render must include the namespace" -assert_contains "$rendered" 'name: agent-os-demo' "render must use the isolated namespace" +PROFILE="$ROOT/deploy/orbstack/inputs.yaml" +PROFILE_OUT="$TMP/orbstack-rendered" + +[ -f "$PROFILE" ] || fail "OrbStack profile inputs must exist" +[ ! -e "$ROOT/deploy/orbstack/kustomization.yaml" ] || fail "OrbStack must not keep a second static installer" +assert_grep 'namespace: agent-os-demo' "$PROFILE" "OrbStack profile must preserve its isolated namespace" +assert_grep 'imagePullPolicy: Never' "$PROFILE" "OrbStack profile must use its local image store" +assert_grep 'allowMutableImage: true' "$PROFILE" "OrbStack profile must explicitly allow its local image tag" +assert_grep 'rbac: cluster-admin' "$PROFILE" "OrbStack profile must make its local-demo grant explicit" + +akua render --no-agent-mode --package "$ROOT/tools/agent-os/packages/firstmate/package.k" \ + --inputs "$PROFILE" --out "$PROFILE_OUT" >/dev/null || fail "OrbStack profile did not render the canonical package" +rendered=$(cat "$PROFILE_OUT"/*.yaml) assert_contains "$rendered" 'kind: ServiceAccount' "render must include the primary ServiceAccount" assert_contains "$rendered" 'kind: ClusterRoleBinding' "render must include the explicit local-demo grant" assert_contains "$rendered" 'name: agent-os-firstmate-home' "render must include the persistent primary home" @@ -24,7 +34,7 @@ assert_not_contains "$rendered" 'hostUsers: false' "OrbStack demo must not reque assert_contains "$rendered" 'runAsUser: 0' "primary must run as container root" assert_contains "$rendered" 'name: agent-os-init' "primary must seed persistent tools" assert_contains "$rendered" 'mountPath: /usr/local' "primary must persist /usr/local" -pass "OrbStack manifests render the isolated persistent primary" +pass "OrbStack profile renders the canonical persistent primary" cat > "$FAKEBIN/kubectl" <<'SH' #!/usr/bin/env bash @@ -39,7 +49,8 @@ chmod +x "$FAKEBIN/kubectl" run_launcher() { PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_STDIN_LOG="$STDIN_LOG" \ - AGENT_OS_IN_CLUSTER=1 "$LAUNCHER" "$@" + AGENT_OS_IN_CLUSTER=1 AGENT_OS_NAMESPACE=agent-os-demo AGENT_OS_IMAGE=agent-os:local-test \ + AGENT_OS_IMAGE_PULL_POLICY=Never "$LAUNCHER" "$@" } : > "$CALLS" @@ -47,7 +58,7 @@ run_launcher create scout-1 grep -Fqx 'kubectl -n agent-os-demo apply -f -' "$CALLS" || fail "create must apply only to agent-os-demo" [ "$(grep -Fc 'kind: PersistentVolumeClaim' "$STDIN_LOG")" -eq 1 ] || fail "create must emit one PVC" [ "$(grep -Fc 'kind: Pod' "$STDIN_LOG")" -eq 1 ] || fail "create must emit one Pod" -assert_grep 'agent-os.akua.dev/crewmate: scout-1' "$STDIN_LOG" "child resources need the stable crewmate label" +assert_grep 'agent-os.dev/crewmate: scout-1' "$STDIN_LOG" "child resources need the stable crewmate label" assert_grep 'automountServiceAccountToken: false' "$STDIN_LOG" "children must not receive Kubernetes credentials" assert_grep 'claimName: agent-os-crewmate-scout-1-home' "$STDIN_LOG" "child work must use its own PVC" assert_no_grep 'hostUsers: false' "$STDIN_LOG" "OrbStack children must not request unsupported Pod user namespaces" @@ -70,14 +81,73 @@ fi pass "crewmate IDs are validated before kubectl" if PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_STDIN_LOG="$STDIN_LOG" \ - "$LAUNCHER" status scout-1 >/dev/null 2>&1; then + AGENT_OS_NAMESPACE=agent-os-demo AGENT_OS_IMAGE=agent-os:local-test "$LAUNCHER" status scout-1 >/dev/null 2>&1; then fail "host execution without an explicit context must be rejected" fi pass "launcher refuses an ambient host Kubernetes context" : > "$CALLS" PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_STDIN_LOG="$STDIN_LOG" \ - AGENT_OS_CONTEXT=orbstack "$LAUNCHER" status scout-1 + AGENT_OS_CONTEXT=orbstack AGENT_OS_NAMESPACE=agent-os-demo AGENT_OS_IMAGE=agent-os:local-test "$LAUNCHER" status scout-1 grep -Fqx 'kubectl --context orbstack -n agent-os-demo get pod agent-os-crewmate-scout-1' "$CALLS" || \ fail "host status must pin the selected context" pass "host launcher calls require and pin an explicit context" + +GENERIC="$ROOT/bin/agent-os-kubernetes.sh" +GENERIC_INPUTS="$TMP/portable-inputs.yaml" + +cat > "$GENERIC_INPUTS" <<'YAML' +namespace: portable-agent-os +image: ghcr.io/akua-dev/agent-os@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +imagePullPolicy: IfNotPresent +rbac: namespace +storage: 20Gi +YAML + +cat > "$FAKEBIN/akua" <<'SH' +#!/usr/bin/env bash +printf 'akua' >> "$AGENT_OS_TEST_LOG" +printf ' %s' "$@" >> "$AGENT_OS_TEST_LOG" +printf '\n' >> "$AGENT_OS_TEST_LOG" +out='' +while [ "$#" -gt 0 ]; do + if [ "$1" = '--out' ]; then + out=$2 + break + fi + shift +done +mkdir -p "$out" +printf 'apiVersion: v1\nkind: ConfigMap\n' > "$out/rendered.yaml" +SH +chmod +x "$FAKEBIN/akua" + +run_generic() { + PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS="$GENERIC_INPUTS" \ + AGENT_OS_CONTEXT=kind-agent-os AGENT_OS_NAMESPACE=portable-agent-os "$GENERIC" "$@" +} + +: > "$CALLS" +run_generic install +grep -Fq -- "akua render --no-agent-mode --package $ROOT/tools/agent-os/packages/firstmate/package.k --inputs $GENERIC_INPUTS --out " "$CALLS" || \ + fail "generic install must render the canonical package before applying it" +grep -Fq 'kubectl --context kind-agent-os apply -f ' "$CALLS" || \ + fail "generic install must apply only its freshly rendered package output" +grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os rollout status statefulset/agent-os-firstmate --timeout=180s' "$CALLS" || \ + fail "generic install must wait for the rendered Firstmate StatefulSet" +pass "generic install renders and applies the canonical package on an explicit context" + +: > "$CALLS" +run_generic rollback +grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os rollout undo statefulset/agent-os-firstmate' "$CALLS" || \ + fail "generic rollback must target only the Firstmate StatefulSet" +pass "generic rollback remains StatefulSet-scoped" + +: > "$CALLS" +if run_generic uninstall >/dev/null 2>&1; then + fail "generic uninstall must require --yes" +fi +run_generic uninstall --yes +grep -Fq 'kubectl --context kind-agent-os delete --ignore-not-found -f ' "$CALLS" || \ + fail "confirmed uninstall must delete only resources from the fresh package render" +pass "generic uninstall requires confirmation and remains package-scoped" diff --git a/tests/agent-os-local.test.sh b/tests/agent-os-local.test.sh index a95379b1c..6257a6905 100755 --- a/tests/agent-os-local.test.sh +++ b/tests/agent-os-local.test.sh @@ -34,6 +34,25 @@ SH chmod +x "$FAKEBIN/docker" make_fake kubectl make_fake orbctl +cat > "$FAKEBIN/akua" <<'SH' +#!/usr/bin/env bash +printf '%s' "$(basename "$0")" >> "$AGENT_OS_TEST_LOG" +printf ' %s' "$@" >> "$AGENT_OS_TEST_LOG" +printf '\n' >> "$AGENT_OS_TEST_LOG" +inputs='' +out='' +while [ "$#" -gt 0 ]; do + case "$1" in + --inputs) inputs=$2; shift 2 ;; + --out) out=$2; shift 2 ;; + *) shift ;; + esac +done +printf 'akua-input-image %s\n' "$(awk '/^image:/{print $2}' "$inputs")" >> "$AGENT_OS_TEST_LOG" +mkdir -p "$out" +printf 'apiVersion: v1\nkind: ConfigMap\n' > "$out/rendered.yaml" +SH +chmod +x "$FAKEBIN/akua" run_cli() { PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$LOG" \ @@ -52,15 +71,22 @@ test_status_pins_context_and_namespace() { pass "status pins the OrbStack context and Agent OS namespace" } -test_deploy_starts_local_kubernetes_and_applies_kustomize() { +test_deploy_starts_local_kubernetes_and_renders_the_orbstack_profile() { : > "$LOG" run_cli deploy assert_call 'orbctl start k8s' "deploy must start OrbStack Kubernetes" assert_call 'kubectl --context orbstack wait --for=condition=Ready node/orbstack --timeout=120s' \ "deploy must wait for the explicit OrbStack node" - assert_call 'kubectl --context orbstack apply -k deploy/orbstack' \ - "deploy must apply only the checked-in OrbStack kustomization" - pass "deploy starts and targets only OrbStack" + grep -Fq -- "akua render --no-agent-mode --package $ROOT/tools/agent-os/packages/firstmate/package.k --inputs " "$LOG" || \ + fail "deploy must render the canonical portable package" + assert_call 'akua-input-image agent-os:local-default' \ + "OrbStack must derive its manifest from the rebuilt local image" + grep -Fq 'kubectl --context orbstack apply -f ' "$LOG" || \ + fail "deploy must apply only the freshly rendered OrbStack profile" + if grep -F 'kubectl --context orbstack apply -k' "$LOG" >/dev/null; then + fail "OrbStack must not maintain a second static installer" + fi + pass "deploy starts OrbStack and applies the canonical package profile" } test_rebuild_deploy_uses_a_new_immutable_local_tag() { @@ -72,9 +98,9 @@ test_rebuild_deploy_uses_a_new_immutable_local_tag() { assert_call 'docker build -t agent-os:dev .' "build must retain the local demo image tag" assert_call 'docker tag agent-os:dev agent-os:local-rebuilt' \ "build must assign the rebuilt image a unique local tag" - assert_call 'kubectl --context orbstack -n agent-os-demo set image statefulset/agent-os-firstmate agent-os-init=agent-os:local-rebuilt firstmate=agent-os:local-rebuilt' \ - "deploy must replace the stale mutable tag with the rebuilt local tag" - pass "rebuild deploy selects the rebuilt local image instead of a stale mutable tag" + assert_call 'akua-input-image agent-os:local-rebuilt' \ + "the rendered OrbStack profile must select the rebuilt local image" + pass "rebuild deploy renders the rebuilt local image instead of a stale mutable tag" } test_explicit_image_override_is_used_without_retagging() { @@ -84,8 +110,8 @@ test_explicit_image_override_is_used_without_retagging() { assert_call 'docker build -t example.test/agent-os:custom .' \ "build must preserve an explicit image override" - assert_call 'kubectl --context orbstack -n agent-os-demo set image statefulset/agent-os-firstmate agent-os-init=example.test/agent-os:custom firstmate=example.test/agent-os:custom' \ - "deploy must preserve an explicit image override" + assert_call 'akua-input-image example.test/agent-os:custom' \ + "the rendered OrbStack profile must preserve an explicit image override" if grep -F 'docker tag example.test/agent-os:custom' "$LOG" >/dev/null; then fail "explicit image overrides must not be retagged" fi @@ -100,9 +126,9 @@ test_destroy_requires_exact_confirmation() { [ ! -s "$LOG" ] || fail "destroy without --yes invoked an external command" run_cli destroy --yes - assert_call 'kubectl --context orbstack delete namespace agent-os-demo' \ - "confirmed destroy must delete only agent-os-demo" - pass "destroy requires confirmation and remains namespace-scoped" + grep -Fq 'kubectl --context orbstack delete --ignore-not-found -f ' "$LOG" || \ + fail "confirmed destroy must delete only resources from the rendered OrbStack profile" + pass "destroy requires confirmation and remains package-scoped" } test_non_orbstack_context_is_fail_closed() { @@ -120,7 +146,7 @@ test_non_orbstack_context_is_fail_closed() { } test_status_pins_context_and_namespace -test_deploy_starts_local_kubernetes_and_applies_kustomize +test_deploy_starts_local_kubernetes_and_renders_the_orbstack_profile test_rebuild_deploy_uses_a_new_immutable_local_tag test_explicit_image_override_is_used_without_retagging test_destroy_requires_exact_confirmation diff --git a/tests/agent-os-packages.test.sh b/tests/agent-os-packages.test.sh index 0b82902bc..cf929db4d 100755 --- a/tests/agent-os-packages.test.sh +++ b/tests/agent-os-packages.test.sh @@ -1,36 +1,105 @@ #!/usr/bin/env bash -# Static contracts for the optional Akua packages used by Agent OS. +# Render contracts for the one portable Agent OS package. set -u # shellcheck source=tests/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" FIRSTMATE="$ROOT/tools/agent-os/packages/firstmate" +MATE="$ROOT/tools/agent-os/packages/mate" +TMP=$(fm_test_tmproot agent-os-packages) +INPUTS="$TMP/inputs.yaml" +OUT="$TMP/rendered" +MUTABLE_INPUTS="$TMP/mutable-image.yaml" + +mkdir -p "$TMP" [ -f "$FIRSTMATE/package.k" ] || fail "firstmate package must exist" [ -f "$FIRSTMATE/inputs.example.yaml" ] || fail "firstmate example inputs must exist" -assert_grep 'ghcr.io/akua-dev/agent-os:latest' "$FIRSTMATE/package.k" \ - "firstmate package must default to the public Agent OS image" assert_grep 'kind = "StatefulSet"' "$FIRSTMATE/package.k" \ "firstmate package must create a restartable controller" assert_grep 'kind = "PersistentVolumeClaim"' "$FIRSTMATE/package.k" \ "firstmate package must persist its home" assert_grep 'kind = "ServiceAccount"' "$FIRSTMATE/package.k" \ "firstmate package must create its cluster identity" -assert_grep 'name = "cluster-admin"' "$FIRSTMATE/package.k" \ - "dedicated intelligence clusters must support the explicit Firstmate admin grant" -assert_grep 'akuaAuthSecret: str = ""' "$FIRSTMATE/package.k" \ - "Akua workspace access must be an explicit Secret reference" -assert_grep 'mountPath = "/var/run/secrets/agent-os/akua"' "$FIRSTMATE/package.k" \ - "Akua authorization material must use a runtime Secret mount" -assert_grep 'readOnly = True' "$FIRSTMATE/package.k" \ - "Akua authorization material must be mounted read-only" assert_grep 'herdr", "status", "--json"' "$FIRSTMATE/package.k" \ "Firstmate readiness must prove the Herdr server is responsive" assert_grep 'mountPath = "/usr/local"' "$FIRSTMATE/package.k" \ "Firstmate-installed tools must persist" -assert_no_grep 'token:' "$FIRSTMATE/inputs.example.yaml" \ - "example inputs must never contain an Akua token value" -pass "Akua packages keep Firstmate persistent and authority explicit" +assert_no_grep 'image: str = ' "$FIRSTMATE/package.k" \ + "the portable package must require an explicit immutable image digest" +assert_grep 'allowMutableImage: bool = False' "$FIRSTMATE/package.k" \ + "only an explicit local profile may allow a mutable image" +assert_grep 'rbac: "namespace" | "cluster-admin" | "none" = "namespace"' "$FIRSTMATE/package.k" \ + "the portable package must default to namespace-scoped runtime RBAC" +assert_grep 'kind = "Role"' "$FIRSTMATE/package.k" \ + "the portable package must render the namespace runtime Role" +assert_grep 'kind = "RoleBinding"' "$FIRSTMATE/package.k" \ + "the portable package must bind its explicit ServiceAccount to that Role" +assert_no_grep 'akuaAuthSecret' "$FIRSTMATE/package.k" \ + "the portable package must not require Akua authorization" +assert_no_grep 'agent-os.akua.dev' "$FIRSTMATE/package.k" \ + "the portable package must not depend on Akua-owned resource annotations" +assert_grep '@sha256:' "$FIRSTMATE/inputs.example.yaml" \ + "the example must use an immutable image digest" +[ ! -f "$MATE/package.k" ] || fail "mate creation must not remain a separately installable package" + +cat > "$INPUTS" <<'YAML' +namespace: portable-agent-os +image: ghcr.io/akua-dev/agent-os@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +imagePullPolicy: IfNotPresent +rbac: namespace +storage: 20Gi +YAML + +akua render --no-agent-mode --package "$FIRSTMATE/package.k" --inputs "$INPUTS" --out "$OUT" >/dev/null || \ + fail "the portable package must render without an Akua account or credential" + +rendered=$(cat "$OUT"/*.yaml) +assert_contains "$rendered" 'kind: ServiceAccount' "rendered topology must create a ServiceAccount" +assert_contains "$rendered" 'kind: Role' "rendered topology must create namespace-scoped runtime RBAC" +assert_contains "$rendered" 'kind: RoleBinding' "rendered topology must bind namespace runtime RBAC" +assert_contains "$rendered" 'kind: PersistentVolumeClaim' "rendered topology must persist Firstmate home" +assert_contains "$rendered" 'kind: StatefulSet' "rendered topology must run Firstmate as a StatefulSet" +assert_contains "$rendered" 'ghcr.io/akua-dev/agent-os@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ + "rendered topology must retain the immutable image digest" +assert_not_contains "$rendered" 'kind: ClusterRoleBinding' \ + "the default portable topology must not grant cluster-admin" +assert_not_contains "$rendered" 'AKUA_' \ + "the portable rendered topology must not require Akua configuration" +assert_not_contains "$rendered" 'agent-os.akua.dev' \ + "the portable rendered topology must not use Akua-owned annotations" + +cat > "$MUTABLE_INPUTS" <<'YAML' +namespace: portable-agent-os +image: ghcr.io/akua-dev/agent-os:latest +imagePullPolicy: IfNotPresent +rbac: namespace +storage: 20Gi +YAML + +if akua render --no-agent-mode --package "$FIRSTMATE/package.k" --inputs "$MUTABLE_INPUTS" --out "$TMP/mutable-rendered" >/dev/null 2>&1; then + fail "the portable package must reject a mutable image tag" +fi +pass "the portable package rejects mutable image tags" + +assert_grep 'bin/agent-os-kubernetes.sh install' "$ROOT/docs/kubernetes.md" \ + "Kubernetes docs must make the generic installer the default quickstart" +assert_grep 'bin/agent-os-kubernetes.sh rollback' "$ROOT/docs/kubernetes.md" \ + "Kubernetes docs must document bounded rollback" +assert_grep 'bin/agent-os-kubernetes.sh uninstall --yes' "$ROOT/docs/kubernetes.md" \ + "Kubernetes docs must document confirmed bounded uninstall" +assert_grep 'No Kubernetes Secret is required' "$ROOT/docs/kubernetes.md" \ + "Kubernetes docs must state the portable secret requirement" +assert_grep 'OrbStack profile' "$ROOT/docs/kubernetes.md" \ + "Kubernetes docs must describe OrbStack as a profile of the canonical package" +assert_grep '[Agent OS Kubernetes quickstart](docs/kubernetes.md)' "$ROOT/README.md" \ + "README must point to the portable Kubernetes quickstart" +assert_no_grep 'Firstmate Akua package' "$ROOT/README.md" \ + "README must not present a separate Akua package as the portable install path" +assert_grep 'Install Akua renderer' "$ROOT/.github/workflows/ci.yml" \ + "CI behavior tests must install the renderer used by package contracts" + +pass "the portable Agent OS package renders a digest-pinned topology without Akua" diff --git a/tools/agent-os/AGENTS.md b/tools/agent-os/AGENTS.md index ed564429e..8efa46445 100644 --- a/tools/agent-os/AGENTS.md +++ b/tools/agent-os/AGENTS.md @@ -41,11 +41,11 @@ Set environment and working directory per command instead of mutating the global Use `Bun.spawn()` for interactive agents, servers, watches, live logs, cancellation, and other long-running processes. Existing Bash scripts may be invoked explicitly during migration, but do not move adaptable orchestration into TypeScript merely because it is currently written in Bash or instructions. -## Akua packages +## Kubernetes package -The prepared mate package lives at `packages/mate/` and renders ordinary Kubernetes YAML. -Treat it as an editable convenience, not a required abstraction or controller. -Firstmate may invoke `akua render`, edit the result, change the package, or author equivalent YAML directly. +The canonical portable Firstmate package lives at `packages/firstmate/` and renders ordinary Kubernetes YAML. +Mate creation uses that package's internal `crewmate.yaml` template at runtime instead of a separately installable package. +Keep the portable package free of Akua accounts, credentials, services, and marketplace-specific behavior. ## Scope Discipline diff --git a/tools/agent-os/README.md b/tools/agent-os/README.md index f0362dc01..f424a8065 100644 --- a/tools/agent-os/README.md +++ b/tools/agent-os/README.md @@ -25,7 +25,8 @@ bun run src/cli.ts address k8s task-x bun run src/cli.ts address inspect /k8s/in-cluster/namespace/agent-os/mate/task-x/herdr ``` -The prepared Akua package under `packages/mate/` remains optional and renders editable Kubernetes YAML through the bundled `akua` CLI. +The portable Agent OS package under `packages/firstmate/` renders the persistent Firstmate topology without an Akua account or credential. +Mate creation uses its internal runtime template instead of a separately installable package. The scoped official `Effect-TS/skills` installation guides typed Effect workflows; its Effect source checkout is local and ignored rather than committed as a snapshot. See `AGENTS.md` in this directory for the tool-specific development rules. diff --git a/tools/agent-os/packages/firstmate/README.md b/tools/agent-os/packages/firstmate/README.md index 59245da08..c48f2d80d 100644 --- a/tools/agent-os/packages/firstmate/README.md +++ b/tools/agent-os/packages/firstmate/README.md @@ -1,21 +1,25 @@ -# Firstmate package +# Agent OS package -This optional Akua package renders ordinary Kubernetes resources for one persistent Firstmate in a dedicated intelligence cluster. -It creates a namespace, ServiceAccount, persistent home, headless Service, StatefulSet, and - when explicitly enabled - the intelligence-cluster `cluster-admin` binding. +This is the one public, versioned Agent OS package. +It renders ordinary Kubernetes resources for one persistent Firstmate: an optional namespace, ServiceAccount, persistent home, headless Service, StatefulSet, and explicit RBAC. +It requires an immutable Agent OS image digest and no Akua account, API key, service, or credential. -Render it directly with Akua: +Render it directly: ```sh akua render \ --package package.k \ --inputs inputs.example.yaml \ - --out /tmp/agent-os-firstmate + --out /tmp/agent-os ``` Inspect the YAML before applying it with `kubectl`. -The package is an editable convenience, not a controller or required workflow. -Use an immutable image digest for a real intelligence cluster; the `latest` default is only a discoverable starting point. +The example digest is deliberately non-installable until the first public Agent OS image release exists. +Replace it only with that release's published digest. -`akuaAuthSecret` names an existing Kubernetes Secret with an `authorization` key containing a complete HTTP authorization header for the dedicated Agent OS workspace token. -The value is mounted read-only at runtime and never belongs in inputs, rendered YAML, Git, logs, or command arguments. -Leave the input empty when Firstmate should not receive Akua workspace access. +The default `rbac: namespace` grants the Firstmate ServiceAccount only the namespace-scoped Pod, Pod exec/log, PVC, and StatefulSet reads needed for runtime mate management. +Set `rbac: none` when a separate authority mechanism manages mates. +Set `rbac: cluster-admin` only for an isolated intelligence cluster after a reviewed grant. + +The package carries no credential value or Secret reference. +Any runtime credential is a separately created Kubernetes Secret, mounted only after its owner has approved that authority. diff --git a/tools/agent-os/packages/firstmate/crewmate.yaml b/tools/agent-os/packages/firstmate/crewmate.yaml new file mode 100644 index 000000000..c8f7d4273 --- /dev/null +++ b/tools/agent-os/packages/firstmate/crewmate.yaml @@ -0,0 +1,66 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: agent-os-crewmate-__AGENT_OS_CREWMATE_ID__-home + namespace: __AGENT_OS_NAMESPACE__ + labels: + app.kubernetes.io/name: agent-os + app.kubernetes.io/component: crewmate + agent-os.dev/crewmate: __AGENT_OS_CREWMATE_ID__ +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: agent-os-crewmate-__AGENT_OS_CREWMATE_ID__ + namespace: __AGENT_OS_NAMESPACE__ + labels: + app.kubernetes.io/name: agent-os + app.kubernetes.io/component: crewmate + agent-os.dev/crewmate: __AGENT_OS_CREWMATE_ID__ +spec: + automountServiceAccountToken: false + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsUser: 0 + initContainers: + - name: agent-os-init + image: __AGENT_OS_IMAGE__ + imagePullPolicy: __AGENT_OS_IMAGE_PULL_POLICY__ + command: + - /opt/agent-os/bin/agent-os-init.sh + volumeMounts: + - name: home + mountPath: /persistent-agent + containers: + - name: crewmate + image: __AGENT_OS_IMAGE__ + imagePullPolicy: __AGENT_OS_IMAGE_PULL_POLICY__ + env: + - name: FM_HOME + value: /home/agent + - name: HOME + value: /home/agent + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 4Gi + volumeMounts: + - name: home + mountPath: /home/agent + - name: home + mountPath: /usr/local + subPath: usr-local + volumes: + - name: home + persistentVolumeClaim: + claimName: agent-os-crewmate-__AGENT_OS_CREWMATE_ID__-home diff --git a/tools/agent-os/packages/firstmate/inputs.example.yaml b/tools/agent-os/packages/firstmate/inputs.example.yaml index 30b57eb03..22090ce94 100644 --- a/tools/agent-os/packages/firstmate/inputs.example.yaml +++ b/tools/agent-os/packages/firstmate/inputs.example.yaml @@ -1,5 +1,6 @@ namespace: agent-os -image: ghcr.io/akua-dev/agent-os:latest +# Replace only with the immutable digest emitted by an Agent OS release. +image: ghcr.io/akua-dev/agent-os@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa imagePullPolicy: IfNotPresent -clusterAdmin: true -akuaAuthSecret: agent-os-firstmate-akua-auth +rbac: namespace +storage: 20Gi diff --git a/tools/agent-os/packages/firstmate/package.k b/tools/agent-os/packages/firstmate/package.k index ff2f6f233..7bbcfdb99 100644 --- a/tools/agent-os/packages/firstmate/package.k +++ b/tools/agent-os/packages/firstmate/package.k @@ -1,20 +1,23 @@ import akua.ctx schema Input: - """Inputs for one persistent Firstmate in a dedicated intelligence cluster.""" + """Inputs for one persistent Firstmate on a conformant Kubernetes cluster.""" namespace: str = "agent-os" createNamespace: bool = True - image: str = "ghcr.io/akua-dev/agent-os:latest" + image: str imagePullPolicy: "Always" | "IfNotPresent" | "Never" = "IfNotPresent" - clusterAdmin: bool = True - akuaAuthSecret: str = "" + allowMutableImage: bool = False + rbac: "namespace" | "cluster-admin" | "none" = "namespace" storage: str = "20Gi" cpuRequest: str = "500m" memoryRequest: str = "1Gi" cpuLimit: str = "4" memoryLimit: str = "8Gi" + check: + allowMutableImage or "@sha256:" in image, "image must use an immutable @sha256 digest unless the local profile explicitly allows a mutable image" + input: Input = ctx.input() labels = { @@ -22,25 +25,6 @@ labels = { "app.kubernetes.io/component" = "firstmate" } -akuaAuthMounts = [{ - name = "akua-auth" - mountPath = "/var/run/secrets/agent-os/akua" - readOnly = True -}] if input.akuaAuthSecret else [] - -akuaAuthVolumes = [{ - name = "akua-auth" - secret = { - secretName = input.akuaAuthSecret - defaultMode = 256 - } -}] if input.akuaAuthSecret else [] - -akuaAuthEnv = [{ - name = "AKUA_AUTH_HEADER_FILE" - value = "/var/run/secrets/agent-os/akua/authorization" -}] if input.akuaAuthSecret else [] - namespaceResources = [{ apiVersion = "v1" kind = "Namespace" @@ -50,7 +34,50 @@ namespaceResources = [{ } }] if input.createNamespace else [] -adminResources = [{ +namespaceRbacResources = [ + { + apiVersion = "rbac.authorization.k8s.io/v1" + kind = "Role" + metadata = { + name = "agent-os-firstmate-runtime" + namespace = input.namespace + labels = labels + } + rules = [ + { + apiGroups = [""] + resources = ["pods", "pods/log", "pods/exec", "persistentvolumeclaims"] + verbs = ["get", "list", "watch", "create", "delete"] + }, + { + apiGroups = ["apps"] + resources = ["statefulsets"] + verbs = ["get", "list", "watch"] + }, + ] + }, + { + apiVersion = "rbac.authorization.k8s.io/v1" + kind = "RoleBinding" + metadata = { + name = "agent-os-firstmate-runtime" + namespace = input.namespace + labels = labels + } + roleRef = { + apiGroup = "rbac.authorization.k8s.io" + kind = "Role" + name = "agent-os-firstmate-runtime" + } + subjects = [{ + kind = "ServiceAccount" + name = "agent-os-firstmate" + namespace = input.namespace + }] + }, +] if input.rbac == "namespace" else [] + +clusterAdminResources = [{ apiVersion = "rbac.authorization.k8s.io/v1" kind = "ClusterRoleBinding" metadata = { @@ -67,7 +94,7 @@ adminResources = [{ name = "agent-os-firstmate" namespace = input.namespace }] -}] if input.clusterAdmin else [] +}] if input.rbac == "cluster-admin" else [] resources = namespaceResources + [ { @@ -78,6 +105,7 @@ resources = namespaceResources + [ namespace = input.namespace labels = labels } + automountServiceAccountToken = True }, { apiVersion = "v1" @@ -105,7 +133,7 @@ resources = namespaceResources + [ selector = labels } }, -] + adminResources + [ +] + namespaceRbacResources + clusterAdminResources + [ { apiVersion = "apps/v1" kind = "StatefulSet" @@ -122,7 +150,7 @@ resources = namespaceResources + [ metadata = { labels = labels annotations = { - "agent-os.akua.dev/address" = "/k8s/in-cluster/namespace/${input.namespace}/mate/firstmate/herdr" + "agent-os.dev/address" = "/k8s/in-cluster/namespace/${input.namespace}/mate/firstmate/herdr" } } spec = { @@ -149,8 +177,10 @@ resources = namespaceResources + [ env = [ { name = "FM_HOME", value = "/home/agent" }, { name = "HOME", value = "/home/agent" }, - { name = "AKUA_API_URL", value = "https://api.akua.dev/v1" }, - ] + akuaAuthEnv + { name = "AGENT_OS_NAMESPACE", value = input.namespace }, + { name = "AGENT_OS_IMAGE", value = input.image }, + { name = "AGENT_OS_IMAGE_PULL_POLICY", value = input.imagePullPolicy }, + ] readinessProbe = { exec = { command = ["herdr", "status", "--json"] } initialDelaySeconds = 2 @@ -169,12 +199,12 @@ resources = namespaceResources + [ volumeMounts = [ { name = "home", mountPath = "/home/agent" }, { name = "home", mountPath = "/usr/local", subPath = "usr-local" }, - ] + akuaAuthMounts + ] }] volumes = [{ name = "home" persistentVolumeClaim = { claimName = "agent-os-firstmate-home" } - }] + akuaAuthVolumes + }] } } } diff --git a/tools/agent-os/packages/mate/README.md b/tools/agent-os/packages/mate/README.md deleted file mode 100644 index 71c5fe0b0..000000000 --- a/tools/agent-os/packages/mate/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Agent OS mate package - -This Akua package is a prepared convenience for rendering one general-purpose mate Pod and its persistent home. -It is not a controller, required operating model, or replacement for Kubernetes YAML. - -Render the example directly: - -```sh -akua render --package package.k --inputs inputs.example.yaml --out /tmp/scout-1 -``` - -Inspect and edit the rendered YAML before applying it: - -```sh -kubectl apply -f /tmp/scout-1 -``` - -Firstmate may change this package, supply additional inputs, edit the rendered files, or author equivalent YAML directly. -The package intentionally grants no Kubernetes token and no privileged, host-namespace, or host-mount access. - -AI credentials are an explicit grant. -Set `piAuthSecret` to the name of an existing Kubernetes Secret whose `auth.json` key contains the selected Pi credential set. -The package mounts only that Secret reference read-only; it never discovers or copies credentials by itself. diff --git a/tools/agent-os/packages/mate/akua.toml b/tools/agent-os/packages/mate/akua.toml deleted file mode 100644 index 06303c226..000000000 --- a/tools/agent-os/packages/mate/akua.toml +++ /dev/null @@ -1,5 +0,0 @@ -[package] -name = "agent-os-mate" -version = "0.1.0" -edition = "akua.dev/v1alpha1" -strict_signing = true diff --git a/tools/agent-os/packages/mate/inputs.example.yaml b/tools/agent-os/packages/mate/inputs.example.yaml deleted file mode 100644 index 3f0136f73..000000000 --- a/tools/agent-os/packages/mate/inputs.example.yaml +++ /dev/null @@ -1,6 +0,0 @@ -mateId: scout-1 -namespace: agent-os-demo -address: /k8s/in-cluster/namespace/agent-os-demo/mate/scout-1/herdr -image: agent-os:dev -imagePullPolicy: Never -piAuthSecret: agent-os-mate-scout-1-pi-auth diff --git a/tools/agent-os/packages/mate/package.k b/tools/agent-os/packages/mate/package.k deleted file mode 100644 index 2ed2259c6..000000000 --- a/tools/agent-os/packages/mate/package.k +++ /dev/null @@ -1,117 +0,0 @@ -import akua.ctx - -schema Input: - """Inputs for one general-purpose Kubernetes-backed Agent OS mate.""" - - mateId: str - namespace: str = "agent-os" - address: str - image: str = "agent-os:dev" - imagePullPolicy: "Always" | "IfNotPresent" | "Never" = "IfNotPresent" - piAuthSecret: str = "" - storage: str = "10Gi" - cpuRequest: str = "250m" - memoryRequest: str = "512Mi" - cpuLimit: str = "2" - memoryLimit: str = "4Gi" - - check: - len(mateId) <= 43, "mateId must leave room for the agent-os-mate- resource prefix" - -input: Input = ctx.input() - -labels = { - "app.kubernetes.io/name" = "agent-os" - "app.kubernetes.io/component" = "mate" - "agent-os.akua.dev/mate" = input.mateId -} - -piAuthMounts = [{ - name = "pi-auth" - mountPath = "/home/agent/.pi/agent/auth.json" - subPath = "auth.json" - readOnly = True -}] if input.piAuthSecret else [] - -piAuthVolumes = [{ - name = "pi-auth" - secret = { - secretName = input.piAuthSecret - defaultMode = 256 - } -}] if input.piAuthSecret else [] - -resources = [ - { - apiVersion = "v1" - kind = "PersistentVolumeClaim" - metadata = { - name = "agent-os-mate-${input.mateId}-home" - namespace = input.namespace - labels = labels - } - spec = { - accessModes = ["ReadWriteOnce"] - resources = { - requests = { storage = input.storage } - } - } - }, - { - apiVersion = "v1" - kind = "Pod" - metadata = { - name = "agent-os-mate-${input.mateId}" - namespace = input.namespace - labels = labels - annotations = { - "agent-os.akua.dev/address" = input.address - } - } - spec = { - automountServiceAccountToken = False - securityContext = { - fsGroup = 0 - runAsGroup = 0 - runAsUser = 0 - } - initContainers = [{ - name = "agent-os-init" - image = input.image - imagePullPolicy = input.imagePullPolicy - command = ["/opt/agent-os/bin/agent-os-init.sh"] - volumeMounts = [{ - name = "home" - mountPath = "/persistent-agent" - }] - }] - containers = [{ - name = "mate" - image = input.image - imagePullPolicy = input.imagePullPolicy - env = [ - { name = "FM_HOME", value = "/home/agent" }, - { name = "HOME", value = "/home/agent" }, - ] - resources = { - requests = { - cpu = input.cpuRequest - memory = input.memoryRequest - } - limits = { - cpu = input.cpuLimit - memory = input.memoryLimit - } - } - volumeMounts = [ - { name = "home", mountPath = "/home/agent" }, - { name = "home", mountPath = "/usr/local", subPath = "usr-local" }, - ] + piAuthMounts - }] - volumes = [{ - name = "home" - persistentVolumeClaim = { claimName = "agent-os-mate-${input.mateId}-home" } - }] + piAuthVolumes - } - }, -] From 25873dee3a87590f3e218849c4f282a23fad4e6d Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 14:42:40 +0200 Subject: [PATCH 23/56] docs: add Herdr source compliance bundle --- Dockerfile | 3 +++ THIRD_PARTY_NOTICES.md | 7 +++---- THIRD_PARTY_SOURCES.md | 31 +++++++++++++++++++++++++++++ docs/acceptance.md | 1 + docs/herdr-compliance.md | 34 ++++++++++++++++++++++++++++++++ tests/agent-os-container.test.sh | 25 ++++++++++++++++++++--- 6 files changed, 94 insertions(+), 7 deletions(-) create mode 100644 THIRD_PARTY_SOURCES.md create mode 100644 docs/herdr-compliance.md diff --git a/Dockerfile b/Dockerfile index 23f4753ba..7c18f04dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -145,6 +145,9 @@ RUN mkdir -p /home/agent /opt/agent-os /opt/image-usr-local COPY . /opt/agent-os +RUN install -D -m 0644 /opt/agent-os/THIRD_PARTY_NOTICES.md /usr/share/doc/agent-os/THIRD_PARTY_NOTICES.md \ + && install -D -m 0644 /opt/agent-os/THIRD_PARTY_SOURCES.md /usr/share/doc/agent-os/THIRD_PARTY_SOURCES.md + RUN cd /opt/agent-os/tools/agent-os \ && bun install --frozen-lockfile --production --ignore-scripts \ && ln -s /opt/agent-os/tools/agent-os/src/cli.ts /usr/local/bin/agent-os \ diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 2fca029bb..7cda95879 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -2,11 +2,10 @@ ## Herdr -The Agent OS demo image includes an unmodified Herdr 0.7.3 executable as a separate program. -Herdr is available under AGPL-3.0-or-later or a commercial license. -Public image publication is gated on a compliant license path being selected for this distribution. +The Agent OS image includes an unmodified Herdr 0.7.3 executable as a separate program under AGPL-3.0-or-later. The image includes Herdr's license at `/usr/share/licenses/herdr/LICENSE`. -The exact corresponding source is available at . +The source offer is available in the image at `/usr/share/doc/agent-os/THIRD_PARTY_SOURCES.md` and in this repository at [THIRD_PARTY_SOURCES.md](THIRD_PARTY_SOURCES.md). +The exact upstream source is available at . Agent OS invokes Herdr through its documented CLI and socket interfaces. ## Akua diff --git a/THIRD_PARTY_SOURCES.md b/THIRD_PARTY_SOURCES.md new file mode 100644 index 000000000..898e17554 --- /dev/null +++ b/THIRD_PARTY_SOURCES.md @@ -0,0 +1,31 @@ +# Third-party source offers + +This file is copied into every Agent OS image at `/usr/share/doc/agent-os/THIRD_PARTY_SOURCES.md`. +It gives image recipients a no-account network path to the corresponding source for included copyleft software. + +## Herdr 0.7.3 + +Agent OS conveys the upstream `herdr-linux-x86_64` or `herdr-linux-aarch64` executable from Herdr v0.7.3 as an unmodified executable. +Agent OS does not patch, link, embed, or copy Herdr source into Agent OS. +The Agent OS entrypoint executes the executable as the separate process `herdr server`. + +The immutable upstream source commit is `299dd4163a96381ec2d8e5bde13d7ba6d6432373`. +The complete source archive is . +The SHA-256 of that archive is `4e4a536fff8cd74019a1f8b4f1eef7fce556042f2b3e389eb6f9a155c1a7c6d5`. +The archive contains Herdr's source, `Cargo.lock`, `Cargo.toml`, `build.rs`, and its vendored `portable-pty` patch source. + +To retrieve and verify the source, run: + +```bash +curl --fail --location --output herdr-v0.7.3.tar.gz \ + https://github.com/ogulcancelik/herdr/archive/299dd4163a96381ec2d8e5bde13d7ba6d6432373.tar.gz +printf '%s %s\n' \ + 4e4a536fff8cd74019a1f8b4f1eef7fce556042f2b3e389eb6f9a155c1a7c6d5 \ + herdr-v0.7.3.tar.gz | sha256sum --check +tar -xzf herdr-v0.7.3.tar.gz +cd herdr-299dd4163a96381ec2d8e5bde13d7ba6d6432373 +cargo build --release +``` + +The source is offered under the upstream GNU Affero General Public License v3.0 or later at . +The complete distribution record and the rule for any future Herdr modification are in `docs/herdr-compliance.md`. diff --git a/docs/acceptance.md b/docs/acceptance.md index bb3969598..14ed57397 100644 --- a/docs/acceptance.md +++ b/docs/acceptance.md @@ -21,6 +21,7 @@ Update it from real runs; planned behavior and model narration never count as pr | Optional GitHub Issue/Project client | Firstmate read a private Cortex GitHub issue through `gh-axi` and delivered public PR #3 with no custom intake service; public Agent OS Issues are disabled because work tracking is private | Issue client proven locally; Project client remains optional and unproven | | Repeatable end-to-end eval and recordable demo | Same-Pod and separate-Pod run records now capture model, time, cost estimate, interventions, resource IDs, and failures | Local agent lifecycle captured; Akua/GitHub/product path incomplete | | Public installable image and release | Multi-architecture release workflow builds `linux/amd64` and `linux/arm64`, publishes only outside pull requests, and records the resulting digest in its job summary | Publication not yet proven | +| Herdr 0.7.3 source and notice bundle | `THIRD_PARTY_NOTICES.md`, `THIRD_PARTY_SOURCES.md`, the image documentation paths, and `tests/agent-os-container.test.sh` prove the unmodified-binary source offer and notice contract | Source and render contract proven; publication availability remains required | ## Definition of done diff --git a/docs/herdr-compliance.md b/docs/herdr-compliance.md new file mode 100644 index 000000000..e8637a901 --- /dev/null +++ b/docs/herdr-compliance.md @@ -0,0 +1,34 @@ +# Herdr 0.7.3 distribution record + +This is a factual distribution record for the Agent OS image and is not legal advice. + +## License and source record + +Herdr v0.7.3 declares AGPL-3.0-or-later or a commercial license in its upstream `LICENSE` and README. +Agent OS uses the AGPL-3.0-or-later grant for this distribution. +The fixed upstream v0.7.3 tag resolves to commit `299dd4163a96381ec2d8e5bde13d7ba6d6432373`. +The matching source archive and its SHA-256 are recorded in [`THIRD_PARTY_SOURCES.md`](../THIRD_PARTY_SOURCES.md). +The image includes the upstream license at `/usr/share/licenses/herdr/LICENSE` and its source offer at `/usr/share/doc/agent-os/THIRD_PARTY_SOURCES.md`. + +## Technical boundary + +The Dockerfile downloads only Herdr's published Linux executable, verifies its release SHA-256, and marks it executable. +The Dockerfile does not patch the executable or copy Herdr source into Agent OS. +The image starts Herdr as the separate process `herdr server`. +Agent OS shell adapters invoke Herdr through its CLI and socket interfaces rather than importing, linking, or embedding Herdr code. + +These facts support the AGPL section 5 aggregate condition only while Herdr remains a separate and independent work that is not an extension of, or combined into a larger program with, Agent OS. +The AGPL states that inclusion of a covered work in such an aggregate does not apply the license to the other parts of the aggregate. +Any change that patches Herdr, copies its source into Agent OS, links it with Agent OS code, or creates another derivative coupling must stop this release path and receive a fresh license review before publication. + +## Conveyance obligations + +AGPL section 4 requires the applicable license and notices to remain with verbatim copies. +The image preserves Herdr's upstream license and puts the recipient-facing notice and source offer in standard documentation paths. +AGPL section 6 permits network conveyance when equivalent corresponding source access is clearly directed from the object-code distribution. +The checked source URL, immutable commit, archive checksum, and build command are present both in this repository and inside the image without an Agent OS, Akua, or Herdr account requirement. +Release maintainers must keep that source path available for every published image carrying this executable. + +Section 13 requires a no-charge network source offer when an operator modifies Herdr and users interact with that modified version remotely. +Agent OS conveys Herdr unmodified, so this image distribution does not rely on section 13 for the source offer. +If an operator or future release modifies Herdr, that version must prominently offer its corresponding source to remote users as section 13 requires. diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 10e6f6361..478b8af40 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -72,6 +72,28 @@ assert_grep 'node_modules' "$ROOT/.dockerignore" "host dependencies must stay ou assert_grep '.repos' "$ROOT/.dockerignore" "development source checkouts must stay out of the build context" assert_grep 'https://github.com/ogulcancelik/herdr/tree/v0.7.3' "$ROOT/THIRD_PARTY_NOTICES.md" \ "Herdr's exact corresponding source must be named" +assert_present "$ROOT/THIRD_PARTY_SOURCES.md" \ + "the image must ship an operator-visible Herdr source offer" +assert_present "$ROOT/docs/herdr-compliance.md" \ + "the repository must record the Herdr distribution boundary" +assert_grep '299dd4163a96381ec2d8e5bde13d7ba6d6432373' "$ROOT/THIRD_PARTY_SOURCES.md" \ + "the Herdr source offer must pin the v0.7.3 commit" +assert_grep '4e4a536fff8cd74019a1f8b4f1eef7fce556042f2b3e389eb6f9a155c1a7c6d5' "$ROOT/THIRD_PARTY_SOURCES.md" \ + "the Herdr source offer must checksum the source archive" +assert_grep 'cargo build --release' "$ROOT/THIRD_PARTY_SOURCES.md" \ + "the Herdr source offer must name its build command" +assert_grep 'unmodified executable' "$ROOT/THIRD_PARTY_SOURCES.md" \ + "the Herdr source offer must state the modification boundary" +assert_grep 'install -D -m 0644 /opt/agent-os/THIRD_PARTY_NOTICES.md /usr/share/doc/agent-os/THIRD_PARTY_NOTICES.md' "$ROOT/Dockerfile" \ + "the image must expose third-party notices to operators" +assert_grep 'install -D -m 0644 /opt/agent-os/THIRD_PARTY_SOURCES.md /usr/share/doc/agent-os/THIRD_PARTY_SOURCES.md' "$ROOT/Dockerfile" \ + "the image must expose Herdr's source offer to operators" +assert_grep '/usr/share/doc/agent-os/THIRD_PARTY_SOURCES.md' "$ROOT/THIRD_PARTY_NOTICES.md" \ + "the Herdr notice must direct image recipients to the bundled source offer" +assert_no_grep 'publication is gated on a compliant license path' "$ROOT/THIRD_PARTY_NOTICES.md" \ + "the Herdr notice must state the selected compliance path" +assert_grep 'Section 13' "$ROOT/docs/herdr-compliance.md" \ + "the Herdr audit must account for the network-interaction clause" assert_grep 'https://github.com/akua-dev/akua/tree/v0.8.25' "$ROOT/THIRD_PARTY_NOTICES.md" \ "Akua's exact source must be named" assert_grep 'https://github.com/derailed/k9s/tree/v0.51.0' "$ROOT/THIRD_PARTY_NOTICES.md" \ @@ -86,9 +108,6 @@ assert_grep 'id: build' "$ROOT/.github/workflows/agent-os-image.yml" \ "release workflow must expose its build result" assert_grep 'steps.build.outputs.digest' "$ROOT/.github/workflows/agent-os-image.yml" \ "release workflow must record the immutable published digest" -assert_grep 'publication is gated on a compliant license path' "$ROOT/THIRD_PARTY_NOTICES.md" \ - "Herdr notice must not claim publication approval before the license decision" - bash -n "$ROOT/bin/agent-os-container-entrypoint.sh" bash -n "$ROOT/bin/agent-os-kubeconfig.sh" pass "container files pin dependencies and exclude host credentials" From c94408e7ffd3361d9c423c8e54c265a1b8301c82 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 15:35:00 +0200 Subject: [PATCH 24/56] no-mistakes(review): Captain, harden Kubernetes credentials and RBAC lifecycle --- .../akua-intelligence-bootstrap/SKILL.md | 23 ++++- .agents/skills/kubernetes-fleet/SKILL.md | 11 ++- bin/agent-os-crewmate.sh | 17 ++++ bin/agent-os-kubernetes.sh | 39 +++++++- bin/agent-os-local.sh | 3 +- bin/fm-bootstrap.sh | 13 ++- deploy/akua/firstmate-auth-grant.yaml | 17 ++++ deploy/akua/firstmate-auth-revoke.yaml | 14 +++ docs/acceptance.md | 2 +- docs/kubernetes.md | 16 +++- tests/agent-os-kubernetes.test.sh | 94 +++++++++++++++++-- tests/agent-os-local.test.sh | 15 +++ tests/agent-os-packages.test.sh | 18 ++++ tests/fm-bootstrap.test.sh | 13 ++- tools/agent-os/packages/firstmate/README.md | 4 +- .../agent-os/packages/firstmate/crewmate.yaml | 8 ++ tools/agent-os/packages/firstmate/package.k | 7 +- 17 files changed, 273 insertions(+), 41 deletions(-) create mode 100644 deploy/akua/firstmate-auth-grant.yaml create mode 100644 deploy/akua/firstmate-auth-revoke.yaml diff --git a/.agents/skills/akua-intelligence-bootstrap/SKILL.md b/.agents/skills/akua-intelligence-bootstrap/SKILL.md index 8da717917..728434482 100644 --- a/.agents/skills/akua-intelligence-bootstrap/SKILL.md +++ b/.agents/skills/akua-intelligence-bootstrap/SKILL.md @@ -43,6 +43,20 @@ Use the Platform MCP `search` tool again before a mutation so current schemas ou | Inventory tokens | `GET /api_tokens` | | Revoke the local bootstrap token | `DELETE /api_tokens/{id}` | +## Authorization overlay contract + +The public Firstmate package never accepts an Akua credential input or renders an Akua Secret reference. +Akua authorization belongs to the separate namespace-local integration overlay under `deploy/akua/`. +The grant patch mounts only the selected Secret at `/var/run/secrets/agent-os/akua` and sets `AKUA_AUTH_HEADER_FILE=/var/run/secrets/agent-os/akua/authorization`. +The Secret must contain one `authorization` key with the complete header and must live in the same namespace as the Firstmate StatefulSet. +Render `__AKUA_AUTH_SECRET__` into a mode-`0600` temporary copy of `firstmate-auth-grant.yaml`, apply it with `kubectl patch statefulset agent-os-firstmate --type=strategic --patch-file`, wait for rollout, and destroy the temporary copy. +Reject a Secret name that is not a Kubernetes DNS subdomain before substituting it. +Never put the Secret value into the overlay, package inputs, or command arguments. + +Revocation is owned by the same integration boundary. +Revoke the Akua API token and prove it fails first, apply `firstmate-auth-revoke.yaml`, wait for rollout, then delete only the named namespace-local Secret after explicit cleanup approval. +The revoke patch removes the environment entry, volume mount, and volume without changing the public package or any other Firstmate setting. + ## Bootstrap procedure 1. Record the approved workspace ID, intended region, machine constraints, expiration or cleanup condition, and evidence directory without recording secrets. @@ -54,9 +68,8 @@ Use the Platform MCP `search` tool again before a mutation so current schemas ou 7. Create the worker, then verify both Akua machine state and Kubernetes Node readiness. 8. Retrieve kubeconfig only into a protected temporary file and verify the cluster identity before applying anything. 9. Build or select a published Agent OS image by immutable digest. -10. Create a distinct clustered-Firstmate API token and immediately convert it into a Kubernetes Secret named by `akuaAuthSecret`. - The Secret's `authorization` key contains the complete header consumed through `AKUA_AUTH_HEADER_FILE`. -11. Render `tools/agent-os/packages/firstmate/package.k` with `akua render`, inspect the ordinary YAML, and apply it with `kubectl`. +10. Render `tools/agent-os/packages/firstmate/package.k` with `akua render`, inspect the ordinary credential-free YAML, and apply it with `kubectl`. +11. Create a distinct clustered-Firstmate API token, create the namespace-local authorization Secret from a protected header file, and apply the grant overlay exactly as defined above. 12. Verify the StatefulSet is ready, Herdr responds, the persistent home is writable, the `in-cluster` context is cluster-admin only in this intelligence cluster, and the Pod can list its Akua workspace through `curl -H @"$AKUA_AUTH_HEADER_FILE"`. 13. Probe the approved primary model route with a bounded request and record only provider, model, result, timing, and cost. If there is no independently approved fallback provider, record the single-provider availability risk instead of presenting the fleet as quota-resilient. @@ -66,10 +79,10 @@ Use the Platform MCP `search` tool again before a mutation so current schemas ou ## Recovery and cleanup -- A Pod restart reuses the Firstmate PVC and mounted Kubernetes Secret. +- A Pod restart reuses the Firstmate PVC and the authorization mount declared by the separate overlay. - A worker replacement must preserve or reattach every PVC holding unique work before deleting the old worker. - Do not claim cluster-loss recovery until an external encrypted backup has been restored in a clean cluster. -- Machine, cluster, Secret, token, and workspace deletion are separate destructive actions. +- Machine, cluster, authorization overlay, Secret, token, and workspace deletion are separate destructive actions. - Execute only the approved cleanup scope and verify retained resources afterward. ## Completion evidence diff --git a/.agents/skills/kubernetes-fleet/SKILL.md b/.agents/skills/kubernetes-fleet/SKILL.md index cd4529f21..4a2d07309 100644 --- a/.agents/skills/kubernetes-fleet/SKILL.md +++ b/.agents/skills/kubernetes-fleet/SKILL.md @@ -14,16 +14,19 @@ Load this skill only when the current firstmate is running in Kubernetes or is e - Keep every crewmate general-purpose; its brief, tools, and authority specialize it for the current task. - A crewmate normally communicates only with its parent through its terminal, status files, reports, and delivered Git state. -- The image includes Akua's CLI and the prepared mate package lives at `tools/agent-os/packages/mate/`. -- The optional persistent controller package lives at `tools/agent-os/packages/firstmate/`; load `akua-intelligence-bootstrap` before using it against Akua-managed infrastructure. +- The image includes Akua's CLI and the canonical runtime template lives at `tools/agent-os/packages/firstmate/crewmate.yaml` inside the one public Firstmate package. +- The optional persistent controller package lives at `tools/agent-os/packages/firstmate/`; load `akua-intelligence-bootstrap` before using the separate Akua authorization overlay against Akua-managed infrastructure. - Use the bundled K9s terminal UI when it makes live Kubernetes inspection faster than individual `kubectl` reads. - Render the package with `akua render`, inspect or edit its ordinary YAML when useful, and apply it with `kubectl`. - Treat AI credentials as explicit per-mate grants, never as ambient inheritance merely because Firstmate can read them. -- Create or select a Kubernetes Secret only after the grant is authorized, then pass its name as the package's `piAuthSecret` input. +- Create or select a namespace-local Kubernetes Secret only after its AI authority is explicitly authorized. +- The Secret must contain the selected provider's `auth.json` key and must be provisioned independently instead of cloning or sharing the primary credential. +- Pass only its name through `AGENT_OS_AI_SECRET` when invoking `bin/agent-os-crewmate.sh create `. +- The helper never reads or discovers the Secret, and the default Role intentionally has no Secret-read permission. +- Kubernetes mounts the selected `auth.json` read-only into the child; a missing Secret or key keeps the Pod unready, makes create fail, removes the non-running Pod, and retains the PVC for an authorized retry. - Probe the selected model route before launching work; when quota is unavailable, use another explicitly granted provider or report the capacity blocker instead of repeatedly spawning agents. - Give every launched Herdr agent a task-unique name, close only a confirmed dead restored pane before reuse, and never replace a live agent. - Grade completion by the promised artifact or delivered Git state; Herdr `idle` alone is not a completion signal. -- The package is optional; direct `akua render`, raw YAML, and `kubectl` remain supported. - Use `bin/agent-os-crewmate.sh create ` to create a separate Pod and persistent home. - Use `status` to inspect it and `delete` only after unique work is checkpointed or delivered. - Never mount the primary home into a child Pod. diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index 79510d347..08597608e 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -8,6 +8,7 @@ ID=${2:-} NAMESPACE=${AGENT_OS_NAMESPACE:-agent-os} IMAGE=${AGENT_OS_IMAGE:-} IMAGE_PULL_POLICY=${AGENT_OS_IMAGE_PULL_POLICY:-IfNotPresent} +AI_SECRET=${AGENT_OS_AI_SECRET:-} KUBECTL=${AGENT_OS_KUBECTL:-kubectl} TEMPLATE=${AGENT_OS_CREWMATE_TEMPLATE:-/opt/agent-os/tools/agent-os/packages/firstmate/crewmate.yaml} @@ -36,12 +37,28 @@ case "$COMMAND" in echo "error: AGENT_OS_IMAGE must name the immutable image selected for this cluster" >&2 exit 2 fi + case "$AI_SECRET" in + ''|*[!a-z0-9.-]*|[.-]*|*[-.]) + echo "error: AGENT_OS_AI_SECRET must name an explicitly authorized namespace-local Secret" >&2 + exit 2 + ;; + esac + if [ "${#AI_SECRET}" -gt 253 ]; then + echo "error: AGENT_OS_AI_SECRET must be a valid Kubernetes Secret name" >&2 + exit 2 + fi sed \ -e "s|__AGENT_OS_CREWMATE_ID__|$ID|g" \ -e "s|__AGENT_OS_NAMESPACE__|$NAMESPACE|g" \ -e "s|__AGENT_OS_IMAGE__|$IMAGE|g" \ -e "s|__AGENT_OS_IMAGE_PULL_POLICY__|$IMAGE_PULL_POLICY|g" \ + -e "s|__AGENT_OS_AI_SECRET__|$AI_SECRET|g" \ "$TEMPLATE" | "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" apply -f - + if ! "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" wait --for=condition=Ready "pod/$POD" --timeout=180s; then + "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" delete pod "$POD" --ignore-not-found + echo "error: crewmate Pod did not become ready with the authorized AI Secret" >&2 + exit 1 + fi ;; status) "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" get pod "$POD" diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh index 9cf83f8d8..7ec9bcc6f 100755 --- a/bin/agent-os-kubernetes.sh +++ b/bin/agent-os-kubernetes.sh @@ -25,14 +25,45 @@ if [ -z "$CONTEXT" ]; then fi render() { + if ! command -v "$AKUA" >/dev/null 2>&1; then + echo "error: Akua renderer '$AKUA' is required for Kubernetes package operations" >&2 + exit 2 + fi "$AKUA" render --no-agent-mode --package "$PACKAGE" --inputs "$INPUTS" --out "$OUT" } +render_has_kind() { + grep -R -Eq "^kind:[[:space:]]*$1$" "$OUT" +} + +delete_namespace_rbac() { + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" delete rolebinding agent-os-firstmate-runtime --ignore-not-found + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" delete role agent-os-firstmate-runtime --ignore-not-found +} + +delete_cluster_admin_rbac() { + "$KUBECTL" --context "$CONTEXT" delete clusterrolebinding "agent-os-firstmate-$NAMESPACE" --ignore-not-found +} + +reconcile_rbac() { + render_has_kind Role || delete_namespace_rbac + render_has_kind ClusterRoleBinding || delete_cluster_admin_rbac +} + +apply_rendered() { + "$KUBECTL" --context "$CONTEXT" apply -f "$OUT" + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s +} + case "$COMMAND" in - install|upgrade) + install) render - "$KUBECTL" --context "$CONTEXT" apply -f "$OUT" - "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s + apply_rendered + ;; + upgrade) + render + reconcile_rbac + apply_rendered ;; rollback) "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout undo statefulset/agent-os-firstmate @@ -47,6 +78,8 @@ case "$COMMAND" in exit 2 fi render + delete_cluster_admin_rbac + delete_namespace_rbac "$KUBECTL" --context "$CONTEXT" delete --ignore-not-found -f "$OUT" ;; *) diff --git a/bin/agent-os-local.sh b/bin/agent-os-local.sh index e77176d78..242fc5014 100755 --- a/bin/agent-os-local.sh +++ b/bin/agent-os-local.sh @@ -7,7 +7,8 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) CONTEXT=${AGENT_OS_CONTEXT:-orbstack} NAMESPACE=${AGENT_OS_NAMESPACE:-agent-os-demo} IMAGE=${AGENT_OS_IMAGE:-agent-os:dev} -IMAGE_IS_OVERRIDE=${AGENT_OS_IMAGE+x} +IMAGE_IS_OVERRIDE= +[ -z "${AGENT_OS_IMAGE:-}" ] || IMAGE_IS_OVERRIDE=1 COMMAND=${1:-} PROFILE="$ROOT/deploy/orbstack/inputs.yaml" diff --git a/bin/fm-bootstrap.sh b/bin/fm-bootstrap.sh index d46478b6b..1daf373d8 100755 --- a/bin/fm-bootstrap.sh +++ b/bin/fm-bootstrap.sh @@ -43,11 +43,10 @@ # "treehouse get --lease" support. # no-mistakes is also MISSING when its installed version is older than # 1.31.2. -# Akua, tasks-axi, and quota-axi are required bootstrap tools. -# Akua renders the optional Agent OS mate package into ordinary YAML; -# firstmate may also author and apply Kubernetes YAML directly. -# tasks-axi and quota-axi are in the same class as -# lavish-axi). tasks-axi is also version and feature gated (0.1.1+ +# Akua is optional and required only when an Agent OS Kubernetes or +# Akua integration path invokes it. +# tasks-axi and quota-axi are required bootstrap tools in the same class as +# lavish-axi. tasks-axi is also version and feature gated (0.1.1+ # with update --archive-body and mv [...]); an installed but incompatible build # reports MISSING like no-mistakes. When # config/backlog-backend is not manual and tasks-axi is compatible, @@ -326,8 +325,8 @@ install_cmd() { BACKEND=$(fm_backend_name) case "$BACKEND" in - orca) TOOLS="orca node git gh akua no-mistakes gh-axi chrome-devtools-axi lavish-axi tasks-axi quota-axi" ;; - *) TOOLS="tmux node git gh akua treehouse no-mistakes gh-axi chrome-devtools-axi lavish-axi tasks-axi quota-axi" ;; + orca) TOOLS="orca node git gh no-mistakes gh-axi chrome-devtools-axi lavish-axi tasks-axi quota-axi" ;; + *) TOOLS="tmux node git gh treehouse no-mistakes gh-axi chrome-devtools-axi lavish-axi tasks-axi quota-axi" ;; esac NO_MISTAKES_MIN_MAJOR=1 NO_MISTAKES_MIN_MINOR=31 diff --git a/deploy/akua/firstmate-auth-grant.yaml b/deploy/akua/firstmate-auth-grant.yaml new file mode 100644 index 000000000..20a68d2c7 --- /dev/null +++ b/deploy/akua/firstmate-auth-grant.yaml @@ -0,0 +1,17 @@ +spec: + template: + spec: + containers: + - name: firstmate + env: + - name: AKUA_AUTH_HEADER_FILE + value: /var/run/secrets/agent-os/akua/authorization + volumeMounts: + - name: akua-auth + mountPath: /var/run/secrets/agent-os/akua + readOnly: true + volumes: + - name: akua-auth + secret: + secretName: __AKUA_AUTH_SECRET__ + defaultMode: 256 diff --git a/deploy/akua/firstmate-auth-revoke.yaml b/deploy/akua/firstmate-auth-revoke.yaml new file mode 100644 index 000000000..3ea842388 --- /dev/null +++ b/deploy/akua/firstmate-auth-revoke.yaml @@ -0,0 +1,14 @@ +spec: + template: + spec: + containers: + - name: firstmate + env: + - name: AKUA_AUTH_HEADER_FILE + $patch: delete + volumeMounts: + - mountPath: /var/run/secrets/agent-os/akua + $patch: delete + volumes: + - name: akua-auth + $patch: delete diff --git a/docs/acceptance.md b/docs/acceptance.md index 14ed57397..1851c6b64 100644 --- a/docs/acceptance.md +++ b/docs/acceptance.md @@ -9,7 +9,7 @@ Update it from real runs; planned behavior and model narration never count as pr | Persistent Firstmate and Herdr in Kubernetes | Canonical package render plus local lifecycle records in `docs/evidence/2026-07-12-firstmate-package.md` | Package render and local evidence proven; published-image run remains required | | Firstmate cluster-admin limited to the intelligence cluster | Dedicated local namespace and explicit demo ClusterRoleBinding | Proven locally only | | Direct Akua-managed KaaS and Hetzner bootstrap | Public endpoint study and `akua-intelligence-bootstrap` skill | Not yet run | -| Distinct clustered token and bootstrap-token revocation | Explicit Secret mount in the Firstmate package and handoff procedure | Not yet run | +| Distinct clustered token and bootstrap-token revocation | Separate namespace-local Akua grant/revoke overlay and handoff procedure; the public package remains credential-free | Overlay contract implemented; live handoff and revocation not yet run | | Firstmate-native Akua worker lifecycle | Native endpoint routing in the bootstrap skill | Not yet run | | Replaceable usable model supply | `openai-codex/gpt-5.4-mini` completed the same-Pod task; Terra-low completed the separate-Pod and recovery tasks; the current local overlay converges direct Pi, crewmate, and Secondmate defaults to `openai-codex/gpt-5.6-terra` with low thinking | One provider with multiple models proven locally; replacement provider still unproven | | Same-Pod general-purpose crewmate | Firstmate-supervised Pi scout, real Kubernetes test, in-cluster context check, and report in `docs/evidence/2026-07-13-same-pod-firstmate.md` | Proven locally | diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 1af81f6aa..d5f8704b7 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -56,6 +56,9 @@ To upgrade, use a new released digest in the same input file and apply it throug bin/agent-os-kubernetes.sh upgrade ``` +Upgrade reconciles the mutually exclusive namespace and cluster-admin RBAC resources before applying the new render. +Downgrading from `cluster-admin` therefore requires permission to delete the prior ClusterRoleBinding and fails instead of silently retaining authority. + To roll back only the Firstmate workload revision, use Kubernetes StatefulSet history. This does not roll back package inputs, RBAC, or persistent data. @@ -69,9 +72,10 @@ To inspect the workload, use the same explicit context and namespace. bin/agent-os-kubernetes.sh status ``` -Uninstall is deliberately confirmed and bounded to a fresh render of the selected package inputs. +Uninstall is deliberately confirmed and bounded to a fresh render plus the package's deterministic RBAC resource names. +It attempts all package-owned RBAC modes so a binding omitted by newer inputs cannot survive. When those inputs create the namespace, deleting that rendered Namespace also removes everything in that namespace. -When `createNamespace: false`, the command deletes only the rendered Agent OS resources in the existing namespace. +When `createNamespace: false`, the command leaves unrelated resources in the existing namespace untouched. ```sh bin/agent-os-kubernetes.sh uninstall --yes @@ -82,6 +86,14 @@ bin/agent-os-kubernetes.sh uninstall --yes Firstmate creates a separate-Pod crewmate at runtime with the internal `crewmate.yaml` template in the canonical package. It is not a second public package and is not a Marketplace product. Each runtime mate has its own PVC and no ambient Kubernetes ServiceAccount token. +Creating one requires an explicitly authorized, pre-created Secret in the same namespace with an `auth.json` key. +Pass only that Secret's name through `AGENT_OS_AI_SECRET`; the helper never discovers or copies the primary credential. +The authorization file is mounted read-only at `/home/agent/.pi/agent/auth.json`. +A missing Secret or key keeps the Pod unready, so creation fails closed and removes the non-running Pod while retaining its PVC for an authorized retry. + +```sh +AGENT_OS_AI_SECRET=scout-1-ai-auth bin/agent-os-crewmate.sh create scout-1 +``` The existing [same-Pod](evidence/2026-07-13-same-pod-firstmate.md) and [separate-Pod recovery](evidence/2026-07-13-separate-pod-recovery.md) records remain local lifecycle evidence. They do not substitute for a clean published-image installation. diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index 6f35aaf6a..c9face05f 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -44,13 +44,16 @@ printf '\n' >> "$AGENT_OS_TEST_LOG" if [ "${*: -2}" = "-f -" ]; then cat > "$AGENT_OS_STDIN_LOG" fi +if [ "${AGENT_OS_TEST_FAIL_WAIT:-0}" = 1 ] && [ "${1:-}" = -n ] && [ "${3:-}" = wait ]; then + exit 1 +fi SH chmod +x "$FAKEBIN/kubectl" run_launcher() { PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_STDIN_LOG="$STDIN_LOG" \ AGENT_OS_IN_CLUSTER=1 AGENT_OS_NAMESPACE=agent-os-demo AGENT_OS_IMAGE=agent-os:local-test \ - AGENT_OS_IMAGE_PULL_POLICY=Never "$LAUNCHER" "$@" + AGENT_OS_IMAGE_PULL_POLICY=Never AGENT_OS_AI_SECRET=scout-1-ai-auth "$LAUNCHER" "$@" } : > "$CALLS" @@ -65,8 +68,35 @@ assert_no_grep 'hostUsers: false' "$STDIN_LOG" "OrbStack children must not reque assert_grep 'runAsUser: 0' "$STDIN_LOG" "children must run as container root" assert_grep 'name: agent-os-init' "$STDIN_LOG" "children must seed persistent tools" assert_grep 'mountPath: /usr/local' "$STDIN_LOG" "children must persist /usr/local" +assert_grep 'mountPath: /home/agent/.pi/agent/auth.json' "$STDIN_LOG" \ + "children must mount only the explicitly granted AI authorization file" +assert_grep 'secretName: scout-1-ai-auth' "$STDIN_LOG" \ + "children must reference the explicitly selected namespace-local Secret" +assert_grep 'readOnly: true' "$STDIN_LOG" "child AI authorization must be read-only" +grep -Fqx 'kubectl -n agent-os-demo wait --for=condition=Ready pod/agent-os-crewmate-scout-1 --timeout=180s' "$CALLS" || \ + fail "create must fail when the authorized Secret cannot produce a ready Pod" pass "crewmate create emits one isolated Pod and PVC" +: > "$CALLS" +if PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_STDIN_LOG="$STDIN_LOG" \ + AGENT_OS_IN_CLUSTER=1 AGENT_OS_NAMESPACE=agent-os-demo AGENT_OS_IMAGE=agent-os:local-test \ + "$LAUNCHER" create scout-1 >/dev/null 2>&1; then + fail "crewmate create must require an explicit AI Secret reference" +fi +[ ! -s "$CALLS" ] || fail "missing AI Secret reference must fail before kubectl" +pass "crewmate create requires an explicit AI Secret grant" + +: > "$CALLS" +if AGENT_OS_TEST_FAIL_WAIT=1 run_launcher create scout-1 >/dev/null 2>&1; then + fail "crewmate create must fail when its authorized Secret cannot produce a ready Pod" +fi +grep -Fqx 'kubectl -n agent-os-demo delete pod agent-os-crewmate-scout-1 --ignore-not-found' "$CALLS" || \ + fail "failed create must remove the non-running Pod" +if grep -F 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" >/dev/null; then + fail "failed create must retain the crewmate PVC for an authorized retry" +fi +pass "crewmate create fails closed while retaining its persistent home" + : > "$CALLS" run_launcher delete scout-1 grep -Fqx 'kubectl -n agent-os-demo delete pod agent-os-crewmate-scout-1 --ignore-not-found' "$CALLS" || \ @@ -109,16 +139,22 @@ cat > "$FAKEBIN/akua" <<'SH' printf 'akua' >> "$AGENT_OS_TEST_LOG" printf ' %s' "$@" >> "$AGENT_OS_TEST_LOG" printf '\n' >> "$AGENT_OS_TEST_LOG" +inputs='' out='' while [ "$#" -gt 0 ]; do - if [ "$1" = '--out' ]; then - out=$2 - break - fi - shift + case "$1" in + --inputs) inputs=$2; shift 2 ;; + --out) out=$2; shift 2 ;; + *) shift ;; + esac done mkdir -p "$out" -printf 'apiVersion: v1\nkind: ConfigMap\n' > "$out/rendered.yaml" +rbac=$(awk '/^rbac:/{print $2}' "$inputs") +case "$rbac" in + namespace) printf 'apiVersion: rbac.authorization.k8s.io/v1\nkind: Role\n' > "$out/rendered.yaml" ;; + cluster-admin) printf 'apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\n' > "$out/rendered.yaml" ;; + *) printf 'apiVersion: v1\nkind: ConfigMap\n' > "$out/rendered.yaml" ;; +esac SH chmod +x "$FAKEBIN/akua" @@ -135,8 +171,44 @@ grep -Fq 'kubectl --context kind-agent-os apply -f ' "$CALLS" || \ fail "generic install must apply only its freshly rendered package output" grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os rollout status statefulset/agent-os-firstmate --timeout=180s' "$CALLS" || \ fail "generic install must wait for the rendered Firstmate StatefulSet" +if grep -F 'delete clusterrolebinding' "$CALLS" >/dev/null; then + fail "fresh namespace-scoped install must not require cluster RBAC deletion authority" +fi pass "generic install renders and applies the canonical package on an explicit context" +missing_akua_out='' +missing_akua_rc=0 +missing_akua_out=$(PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS="$GENERIC_INPUTS" \ + AGENT_OS_AKUA=missing-agent-os-akua AGENT_OS_CONTEXT=kind-agent-os AGENT_OS_NAMESPACE=portable-agent-os \ + "$GENERIC" install 2>&1) || missing_akua_rc=$? +[ "$missing_akua_rc" -eq 2 ] || fail "Kubernetes package operations must reject a missing renderer with exit 2" +assert_contains "$missing_akua_out" "error: Akua renderer 'missing-agent-os-akua' is required for Kubernetes package operations" \ + "Kubernetes package operations must explain their optional renderer dependency" +pass "Kubernetes package operations require the optional Akua renderer explicitly" + +: > "$CALLS" +run_generic upgrade +grep -Fqx 'kubectl --context kind-agent-os delete clusterrolebinding agent-os-firstmate-portable-agent-os --ignore-not-found' "$CALLS" || \ + fail "namespace RBAC upgrade must revoke any stale cluster-admin binding" +if grep -F 'delete rolebinding agent-os-firstmate-runtime' "$CALLS" >/dev/null; then + fail "namespace RBAC upgrade must retain its rendered RoleBinding" +fi +pass "namespace RBAC upgrade revokes stale cluster-admin authority" + +CLUSTER_ADMIN_INPUTS="$TMP/cluster-admin-inputs.yaml" +sed 's/rbac: namespace/rbac: cluster-admin/' "$GENERIC_INPUTS" > "$CLUSTER_ADMIN_INPUTS" +: > "$CALLS" +PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS="$CLUSTER_ADMIN_INPUTS" \ + AGENT_OS_CONTEXT=kind-agent-os AGENT_OS_NAMESPACE=portable-agent-os "$GENERIC" upgrade +grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete rolebinding agent-os-firstmate-runtime --ignore-not-found' "$CALLS" || \ + fail "cluster-admin upgrade must delete the stale namespace RoleBinding" +grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete role agent-os-firstmate-runtime --ignore-not-found' "$CALLS" || \ + fail "cluster-admin upgrade must delete the stale namespace Role" +if grep -F 'delete clusterrolebinding agent-os-firstmate-portable-agent-os' "$CALLS" >/dev/null; then + fail "cluster-admin upgrade must retain its rendered ClusterRoleBinding" +fi +pass "cluster-admin RBAC upgrade removes stale namespace authority" + : > "$CALLS" run_generic rollback grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os rollout undo statefulset/agent-os-firstmate' "$CALLS" || \ @@ -150,4 +222,10 @@ fi run_generic uninstall --yes grep -Fq 'kubectl --context kind-agent-os delete --ignore-not-found -f ' "$CALLS" || \ fail "confirmed uninstall must delete only resources from the fresh package render" -pass "generic uninstall requires confirmation and remains package-scoped" +grep -Fqx 'kubectl --context kind-agent-os delete clusterrolebinding agent-os-firstmate-portable-agent-os --ignore-not-found' "$CALLS" || \ + fail "uninstall must revoke a stale cluster-admin binding even when current inputs omit it" +grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete rolebinding agent-os-firstmate-runtime --ignore-not-found' "$CALLS" || \ + fail "uninstall must remove namespace runtime binding regardless of current inputs" +grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete role agent-os-firstmate-runtime --ignore-not-found' "$CALLS" || \ + fail "uninstall must remove namespace runtime Role regardless of current inputs" +pass "generic uninstall requires confirmation and revokes every package RBAC mode" diff --git a/tests/agent-os-local.test.sh b/tests/agent-os-local.test.sh index 6257a6905..b7845eb26 100755 --- a/tests/agent-os-local.test.sh +++ b/tests/agent-os-local.test.sh @@ -118,6 +118,20 @@ test_explicit_image_override_is_used_without_retagging() { pass "explicit image override remains intact" } +test_empty_image_override_uses_content_addressed_default() { + : > "$LOG" + AGENT_OS_IMAGE='' AGENT_OS_TEST_IMAGE_ID=sha256:empty-default run_cli build + AGENT_OS_IMAGE='' AGENT_OS_TEST_IMAGE_ID=sha256:empty-default run_cli deploy + + assert_call 'docker build -t agent-os:dev .' \ + "an empty image override must use the local demo default" + assert_call 'docker tag agent-os:dev agent-os:local-empty-default' \ + "an empty image override must still receive a content-addressed tag" + assert_call 'akua-input-image agent-os:local-empty-default' \ + "an empty image override must deploy the content-addressed local image" + pass "empty image override uses the content-addressed local default" +} + test_destroy_requires_exact_confirmation() { local out rc=0 : > "$LOG" @@ -149,5 +163,6 @@ test_status_pins_context_and_namespace test_deploy_starts_local_kubernetes_and_renders_the_orbstack_profile test_rebuild_deploy_uses_a_new_immutable_local_tag test_explicit_image_override_is_used_without_retagging +test_empty_image_override_uses_content_addressed_default test_destroy_requires_exact_confirmation test_non_orbstack_context_is_fail_closed diff --git a/tests/agent-os-packages.test.sh b/tests/agent-os-packages.test.sh index cf929db4d..3319dbaa2 100755 --- a/tests/agent-os-packages.test.sh +++ b/tests/agent-os-packages.test.sh @@ -38,10 +38,28 @@ assert_grep 'kind = "Role"' "$FIRSTMATE/package.k" \ "the portable package must render the namespace runtime Role" assert_grep 'kind = "RoleBinding"' "$FIRSTMATE/package.k" \ "the portable package must bind its explicit ServiceAccount to that Role" +assert_grep 'resources = ["pods", "persistentvolumeclaims"]' "$FIRSTMATE/package.k" \ + "runtime apply authority must exclude Pod subresources from patch access" +assert_grep 'verbs = ["get", "list", "watch", "create", "delete", "patch"]' "$FIRSTMATE/package.k" \ + "runtime RBAC must allow kubectl apply to patch retained crewmate PVCs" assert_no_grep 'akuaAuthSecret' "$FIRSTMATE/package.k" \ "the portable package must not require Akua authorization" assert_no_grep 'agent-os.akua.dev' "$FIRSTMATE/package.k" \ "the portable package must not depend on Akua-owned resource annotations" +[ -f "$ROOT/deploy/akua/firstmate-auth-grant.yaml" ] || \ + fail "Akua authorization must use a separate integration grant overlay" +[ -f "$ROOT/deploy/akua/firstmate-auth-revoke.yaml" ] || \ + fail "Akua authorization overlay must own explicit mount cleanup" +assert_grep 'name: AKUA_AUTH_HEADER_FILE' "$ROOT/deploy/akua/firstmate-auth-grant.yaml" \ + "Akua overlay must set the authorization header file path" +assert_grep 'mountPath: /var/run/secrets/agent-os/akua' "$ROOT/deploy/akua/firstmate-auth-grant.yaml" \ + "Akua overlay must mount only the Akua authorization path" +assert_grep 'secretName: __AKUA_AUTH_SECRET__' "$ROOT/deploy/akua/firstmate-auth-grant.yaml" \ + "Akua overlay must reference a namespace-local Secret by name" +assert_grep "\$patch: delete" "$ROOT/deploy/akua/firstmate-auth-revoke.yaml" \ + "Akua overlay must define explicit authorization mount cleanup" +assert_grep 'mountPath: /var/run/secrets/agent-os/akua' "$ROOT/deploy/akua/firstmate-auth-revoke.yaml" \ + "Akua overlay cleanup must use the strategic merge key for volume mounts" assert_grep '@sha256:' "$FIRSTMATE/inputs.example.yaml" \ "the example must use an immutable image digest" [ ! -f "$MATE/package.k" ] || fail "mate creation must not remain a separately installable package" diff --git a/tests/fm-bootstrap.test.sh b/tests/fm-bootstrap.test.sh index 83204e381..b541950f4 100755 --- a/tests/fm-bootstrap.test.sh +++ b/tests/fm-bootstrap.test.sh @@ -327,9 +327,9 @@ SH pass "bootstrap requires git with an install instruction" } -test_akua_is_required_with_supported_install_instruction() { - local case_dir fakebin out expected - case_dir="$TMP_ROOT/akua-required" +test_akua_is_optional_for_routine_bootstrap() { + local case_dir fakebin out + case_dir="$TMP_ROOT/akua-optional" mkdir -p "$case_dir/home/config" printf '%s\n' manual > "$case_dir/home/config/backlog-backend" fakebin=$(make_fake_toolchain "$case_dir") @@ -337,9 +337,8 @@ test_akua_is_required_with_supported_install_instruction() { out=$(PATH="$fakebin:$BASE_PATH" FM_HOME="$case_dir/home" FM_ROOT_OVERRIDE="$case_dir/home" \ FM_FAKE_TREEHOUSE_LEASE_HELP=1 "$ROOT/bin/fm-bootstrap.sh") - expected="MISSING: akua (install: curl -fsSL https://cli.akua.dev/install | sh)" - [ "$out" = "$expected" ] || fail "missing Akua should report the supported install instruction, got: $out" - pass "bootstrap requires Akua with an install instruction" + [ -z "$out" ] || fail "routine bootstrap must not require optional Akua tooling, got: $out" + pass "routine bootstrap leaves Akua tooling optional" } test_orca_backend_gates_orca_tool_only_when_selected() { @@ -510,7 +509,7 @@ ROWS test_bootstrap_reporting test_no_mistakes_min_version test_git_is_required_with_supported_install_instruction -test_akua_is_required_with_supported_install_instruction +test_akua_is_optional_for_routine_bootstrap test_orca_backend_gates_orca_tool_only_when_selected test_fleet_sync_timeout_scales_with_origin_backed_project_count test_fleet_sync_timeout_floor_preserves_small_fleets diff --git a/tools/agent-os/packages/firstmate/README.md b/tools/agent-os/packages/firstmate/README.md index c48f2d80d..d116d2797 100644 --- a/tools/agent-os/packages/firstmate/README.md +++ b/tools/agent-os/packages/firstmate/README.md @@ -17,9 +17,9 @@ Inspect the YAML before applying it with `kubectl`. The example digest is deliberately non-installable until the first public Agent OS image release exists. Replace it only with that release's published digest. -The default `rbac: namespace` grants the Firstmate ServiceAccount only the namespace-scoped Pod, Pod exec/log, PVC, and StatefulSet reads needed for runtime mate management. +The default `rbac: namespace` grants the Firstmate ServiceAccount only the namespace-scoped Pod, Pod exec/log, PVC, and StatefulSet operations needed for runtime mate management, including patch for idempotent apply. Set `rbac: none` when a separate authority mechanism manages mates. Set `rbac: cluster-admin` only for an isolated intelligence cluster after a reviewed grant. The package carries no credential value or Secret reference. -Any runtime credential is a separately created Kubernetes Secret, mounted only after its owner has approved that authority. +Any runtime credential is a separately created namespace-local Kubernetes Secret, referenced by the runtime helper only after its owner has approved that authority. diff --git a/tools/agent-os/packages/firstmate/crewmate.yaml b/tools/agent-os/packages/firstmate/crewmate.yaml index c8f7d4273..eef72650f 100644 --- a/tools/agent-os/packages/firstmate/crewmate.yaml +++ b/tools/agent-os/packages/firstmate/crewmate.yaml @@ -60,7 +60,15 @@ spec: - name: home mountPath: /usr/local subPath: usr-local + - name: ai-auth + mountPath: /home/agent/.pi/agent/auth.json + subPath: auth.json + readOnly: true volumes: - name: home persistentVolumeClaim: claimName: agent-os-crewmate-__AGENT_OS_CREWMATE_ID__-home + - name: ai-auth + secret: + secretName: __AGENT_OS_AI_SECRET__ + defaultMode: 256 diff --git a/tools/agent-os/packages/firstmate/package.k b/tools/agent-os/packages/firstmate/package.k index 7bbcfdb99..6f23368eb 100644 --- a/tools/agent-os/packages/firstmate/package.k +++ b/tools/agent-os/packages/firstmate/package.k @@ -46,7 +46,12 @@ namespaceRbacResources = [ rules = [ { apiGroups = [""] - resources = ["pods", "pods/log", "pods/exec", "persistentvolumeclaims"] + resources = ["pods", "persistentvolumeclaims"] + verbs = ["get", "list", "watch", "create", "delete", "patch"] + }, + { + apiGroups = [""] + resources = ["pods/log", "pods/exec"] verbs = ["get", "list", "watch", "create", "delete"] }, { From 82419a3c7fad093bb72028e231d41c66a89b6aba Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 16:23:09 +0200 Subject: [PATCH 25/56] no-mistakes(review): Captain, harden Kubernetes lifecycle and authorization boundaries --- .../akua-intelligence-bootstrap/SKILL.md | 21 +- .github/workflows/agent-os-image.yml | 12 +- bin/agent-os-kubernetes.sh | 283 +++++++++++++++-- bin/agent-os-local.sh | 2 +- docs/agent-evals.md | 4 +- docs/kubernetes.md | 28 +- tests/agent-os-container.test.sh | 15 + tests/agent-os-kubernetes.test.sh | 288 ++++++++++++++++-- tests/agent-os-local.test.sh | 35 ++- tests/agent-os-packages.test.sh | 47 +++ tools/agent-os/packages/firstmate/README.md | 5 +- .../agent-os/packages/firstmate/crewmate.yaml | 8 + tools/agent-os/packages/firstmate/package.k | 16 +- 13 files changed, 701 insertions(+), 63 deletions(-) diff --git a/.agents/skills/akua-intelligence-bootstrap/SKILL.md b/.agents/skills/akua-intelligence-bootstrap/SKILL.md index 728434482..4adc24ffd 100644 --- a/.agents/skills/akua-intelligence-bootstrap/SKILL.md +++ b/.agents/skills/akua-intelligence-bootstrap/SKILL.md @@ -49,12 +49,29 @@ The public Firstmate package never accepts an Akua credential input or renders a Akua authorization belongs to the separate namespace-local integration overlay under `deploy/akua/`. The grant patch mounts only the selected Secret at `/var/run/secrets/agent-os/akua` and sets `AKUA_AUTH_HEADER_FILE=/var/run/secrets/agent-os/akua/authorization`. The Secret must contain one `authorization` key with the complete header and must live in the same namespace as the Firstmate StatefulSet. -Render `__AKUA_AUTH_SECRET__` into a mode-`0600` temporary copy of `firstmate-auth-grant.yaml`, apply it with `kubectl patch statefulset agent-os-firstmate --type=strategic --patch-file`, wait for rollout, and destroy the temporary copy. +Render `__AKUA_AUTH_SECRET__` into a mode-`0600` temporary copy of `firstmate-auth-grant.yaml` and keep its path in `grant_patch`. Reject a Secret name that is not a Kubernetes DNS subdomain before substituting it. Never put the Secret value into the overlay, package inputs, or command arguments. +Set `context` and `namespace` from the separately approved target and grant the overlay with this exact context-pinned strategic patch command: + +```sh +kubectl --context "$context" -n "$namespace" patch statefulset agent-os-firstmate --type=strategic --patch-file "$grant_patch" +kubectl --context "$context" -n "$namespace" rollout status statefulset/agent-os-firstmate --timeout=180s +``` + +Destroy the temporary grant patch after rollout. + Revocation is owned by the same integration boundary. -Revoke the Akua API token and prove it fails first, apply `firstmate-auth-revoke.yaml`, wait for rollout, then delete only the named namespace-local Secret after explicit cleanup approval. +Keep the path to `firstmate-auth-revoke.yaml` in `revoke_patch`. +Revoke the Akua API token and prove it fails first, then revoke the overlay with the same exact target and patch type: + +```sh +kubectl --context "$context" -n "$namespace" patch statefulset agent-os-firstmate --type=strategic --patch-file "$revoke_patch" +kubectl --context "$context" -n "$namespace" rollout status statefulset/agent-os-firstmate --timeout=180s +``` + +Delete only the named namespace-local Secret after explicit cleanup approval. The revoke patch removes the environment entry, volume mount, and volume without changing the public package or any other Firstmate setting. ## Bootstrap procedure diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index 32f776403..bedcb3c3a 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -23,15 +23,15 @@ jobs: name: Build linux/amd64 and linux/arm64 runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 - - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 - - uses: docker/setup-buildx-action@v3 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f - name: Log in to GHCR if: github.event_name != 'pull_request' - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 with: registry: ghcr.io username: ${{ github.actor }} @@ -39,7 +39,7 @@ jobs: - name: Compute image tags id: metadata - uses: docker/metadata-action@v5 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 with: images: ${{ env.IMAGE }} tags: | @@ -49,7 +49,7 @@ jobs: - name: Build and optionally publish id: build - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 with: context: . platforms: linux/amd64,linux/arm64 diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh index 7ec9bcc6f..c106fad79 100755 --- a/bin/agent-os-kubernetes.sh +++ b/bin/agent-os-kubernetes.sh @@ -1,53 +1,134 @@ #!/usr/bin/env bash # agent-os-kubernetes.sh - render and operate the portable Agent OS package. -# Usage: bin/agent-os-kubernetes.sh install|upgrade|rollback|status|uninstall [--yes] +# Usage: bin/agent-os-kubernetes.sh install|upgrade|rollback|status|uninstall|cleanup-cluster-rbac [--yes] [--delete-namespace] set -eu ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) COMMAND=${1:-} -CONFIRM=${2:-} +shift || true PACKAGE=${AGENT_OS_PACKAGE:-"$ROOT/tools/agent-os/packages/firstmate/package.k"} INPUTS=${AGENT_OS_INPUTS:-"$ROOT/tools/agent-os/packages/firstmate/inputs.example.yaml"} CONTEXT=${AGENT_OS_CONTEXT:-} -NAMESPACE=${AGENT_OS_NAMESPACE:-agent-os} +REQUESTED_NAMESPACE=${AGENT_OS_NAMESPACE:-} +NAMESPACE=${REQUESTED_NAMESPACE:-agent-os} AKUA=${AGENT_OS_AKUA:-akua} KUBECTL=${AGENT_OS_KUBECTL:-kubectl} OUT=$(mktemp -d) +CONFIRMED=0 +DELETE_NAMESPACE=0 +MANAGES_NAMESPACE=0 +DESIRED_RBAC=none +INSTALLATION_ID= cleanup() { rm -rf "$OUT" } trap cleanup EXIT +usage() { + echo "usage: $0 install|upgrade|rollback|status|uninstall|cleanup-cluster-rbac [--yes] [--delete-namespace]" >&2 + exit 2 +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --yes) CONFIRMED=1 ;; + --delete-namespace) DELETE_NAMESPACE=1 ;; + *) usage ;; + esac + shift +done + if [ -z "$CONTEXT" ]; then echo "error: set AGENT_OS_CONTEXT to the Kubernetes context to operate" >&2 exit 2 fi +render_has_kind() { + grep -R -Eq "^kind:[[:space:]]*$1$" "$OUT" +} + +rendered_statefulset() { + grep -Rl '^kind:[[:space:]]*StatefulSet$' "$OUT" || true +} + render() { + local statefulset statefulset_count rendered_namespace if ! command -v "$AKUA" >/dev/null 2>&1; then echo "error: Akua renderer '$AKUA' is required for Kubernetes package operations" >&2 exit 2 fi "$AKUA" render --no-agent-mode --package "$PACKAGE" --inputs "$INPUTS" --out "$OUT" + statefulset=$(rendered_statefulset) + statefulset_count=$(printf '%s\n' "$statefulset" | sed '/^$/d' | wc -l | tr -d ' ') + if [ "$statefulset_count" -ne 1 ]; then + echo "error: package must render exactly one StatefulSet" >&2 + exit 2 + fi + rendered_namespace=$(awk '$1 == "namespace:" { print $2; exit }' "$statefulset") + if [ -z "$rendered_namespace" ]; then + echo "error: rendered StatefulSet has no namespace" >&2 + exit 2 + fi + if [ -n "$REQUESTED_NAMESPACE" ] && [ "$REQUESTED_NAMESPACE" != "$rendered_namespace" ]; then + echo "error: AGENT_OS_NAMESPACE '$REQUESTED_NAMESPACE' does not match rendered namespace '$rendered_namespace'" >&2 + exit 2 + fi + NAMESPACE=$rendered_namespace + INSTALLATION_ID="agent-os-firstmate:$NAMESPACE" + if render_has_kind Namespace; then + MANAGES_NAMESPACE=1 + fi + if render_has_kind ClusterRoleBinding; then + DESIRED_RBAC=cluster-admin + elif render_has_kind Role; then + DESIRED_RBAC=namespace + fi } -render_has_kind() { - grep -R -Eq "^kind:[[:space:]]*$1$" "$OUT" +namespace_name() { + "$KUBECTL" --context "$CONTEXT" get namespace "$NAMESPACE" --ignore-not-found -o name } -delete_namespace_rbac() { - "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" delete rolebinding agent-os-firstmate-runtime --ignore-not-found - "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" delete role agent-os-firstmate-runtime --ignore-not-found +namespace_identity() { + "$KUBECTL" --context "$CONTEXT" get namespace "$NAMESPACE" \ + -o 'jsonpath={.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}' +} + +require_namespace_contract() { + local name identity + name=$(namespace_name) + if [ "$MANAGES_NAMESPACE" -eq 1 ]; then + if [ -z "$name" ]; then + return + fi + identity=$(namespace_identity) + if [ "$identity" != "agent-os"$'\t'"$INSTALLATION_ID" ]; then + echo "error: namespace '$NAMESPACE' exists without the exact Agent OS installation identity" >&2 + exit 2 + fi + return + fi + if [ -z "$name" ]; then + echo "error: createNamespace=false requires the pre-existing namespace '$NAMESPACE'" >&2 + exit 2 + fi + identity=$(namespace_identity) + if [ "$identity" != $'\t' ]; then + echo "error: createNamespace=false requires an unowned namespace; '$NAMESPACE' carries ownership metadata" >&2 + exit 2 + fi } -delete_cluster_admin_rbac() { - "$KUBECTL" --context "$CONTEXT" delete clusterrolebinding "agent-os-firstmate-$NAMESPACE" --ignore-not-found +workload_state() { + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get statefulset agent-os-firstmate \ + --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.annotations.agent-os\.dev/rbac-mode}{"\t"}{.metadata.annotations.agent-os\.dev/cluster-rbac-cleanup}' } -reconcile_rbac() { - render_has_kind Role || delete_namespace_rbac - render_has_kind ClusterRoleBinding || delete_cluster_admin_rbac +delete_namespace_rbac() { + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" delete rolebinding agent-os-firstmate-runtime --ignore-not-found + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" delete role agent-os-firstmate-runtime --ignore-not-found } apply_rendered() { @@ -55,35 +136,197 @@ apply_rendered() { "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s } +verify_desired_rbac() { + local role_name binding_identity expected_identity + if [ "$DESIRED_RBAC" = namespace ]; then + role_name=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get role agent-os-firstmate-runtime \ + -o 'jsonpath={.metadata.name}') + binding_identity=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get rolebinding agent-os-firstmate-runtime \ + -o 'jsonpath={.roleRef.kind}{"\t"}{.roleRef.name}{"\t"}{.subjects[0].kind}{"\t"}{.subjects[0].name}{"\t"}{.subjects[0].namespace}') + expected_identity="Role"$'\t'"agent-os-firstmate-runtime"$'\t'"ServiceAccount"$'\t'"agent-os-firstmate"$'\t'"$NAMESPACE" + if [ "$role_name" != agent-os-firstmate-runtime ] || [ "$binding_identity" != "$expected_identity" ]; then + echo "error: desired namespace RBAC did not verify after apply" >&2 + exit 2 + fi + fi +} + +cleanup_command() { + printf 'AGENT_OS_CONTEXT=%q AGENT_OS_NAMESPACE=%q AGENT_OS_PACKAGE=%q AGENT_OS_INPUTS=%q AGENT_OS_AKUA=%q AGENT_OS_KUBECTL=%q %q cleanup-cluster-rbac --yes' \ + "$CONTEXT" "$NAMESPACE" "$PACKAGE" "$INPUTS" "$AKUA" "$KUBECTL" "$0" +} + +report_cluster_cleanup() { + echo "incomplete: privileged cleanup is required: $(cleanup_command)" >&2 + echo "required evidence: clusterrolebinding/agent-os-firstmate-$NAMESPACE absent" >&2 +} + +delete_rendered_namespaced_resources() { + local files file kind + files=$(find "$OUT" -type f -name '*.yaml' -print) + while IFS= read -r file; do + [ -n "$file" ] || continue + kind=$(awk '$1 == "kind:" { print $2; exit }' "$file") + case "$kind" in + Namespace|ClusterRoleBinding) ;; + *) "$KUBECTL" --context "$CONTEXT" delete --ignore-not-found -f "$file" ;; + esac + done <<< "$files" +} + +namespace_is_empty() { + local resources resource objects object + if ! resources=$("$KUBECTL" --context "$CONTEXT" api-resources --verbs=list --namespaced -o name); then + echo "error: could not inventory namespaced Kubernetes resource types" >&2 + return 1 + fi + while IFS= read -r resource; do + [ -n "$resource" ] || continue + case "$resource" in + events|events.events.k8s.io|leases.coordination.k8s.io) continue ;; + esac + if ! objects=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get "$resource" -o name); then + echo "error: could not inventory '$resource' in namespace '$NAMESPACE'" >&2 + return 1 + fi + while IFS= read -r object; do + [ -n "$object" ] || continue + case "$object" in + serviceaccount/default|configmap/kube-root-ca.crt) ;; + *) + echo "error: namespace '$NAMESPACE' still contains foreign resource '$object'" >&2 + return 1 + ;; + esac + done <<< "$objects" + done <<< "$resources" +} + +delete_owned_empty_namespace() { + local identity + if [ "$MANAGES_NAMESPACE" -ne 1 ]; then + echo "error: --delete-namespace requires createNamespace=true in the current inputs" >&2 + exit 2 + fi + identity=$(namespace_identity) + if [ "$identity" != "agent-os"$'\t'"$INSTALLATION_ID" ]; then + echo "error: namespace '$NAMESPACE' no longer has the exact Agent OS installation identity" >&2 + exit 2 + fi + if ! namespace_is_empty; then + exit 2 + fi + identity=$(namespace_identity) + if [ "$identity" != "agent-os"$'\t'"$INSTALLATION_ID" ]; then + echo "error: namespace '$NAMESPACE' ownership changed during deletion checks" >&2 + exit 2 + fi + "$KUBECTL" --context "$CONTEXT" delete namespace "$NAMESPACE" +} + +cleanup_cluster_rbac() { + local binding identity workload + binding="agent-os-firstmate-$NAMESPACE" + identity=$("$KUBECTL" --context "$CONTEXT" get clusterrolebinding "$binding" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}') + if [ -n "$identity" ] && [ "$identity" != "$binding"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" ]; then + echo "error: clusterrolebinding '$binding' does not have the exact Agent OS installation identity" >&2 + exit 2 + fi + if [ -n "$identity" ]; then + "$KUBECTL" --context "$CONTEXT" delete clusterrolebinding "$binding" + "$KUBECTL" --context "$CONTEXT" wait --for=delete "clusterrolebinding/$binding" --timeout=60s + fi + if [ -n "$(namespace_name)" ]; then + workload=$(workload_state) + if [ -n "$workload" ]; then + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" annotate statefulset agent-os-firstmate \ + agent-os.dev/cluster-rbac-cleanup- >/dev/null + fi + fi + echo "evidence: clusterrolebinding/$binding absent" +} + case "$COMMAND" in install) + [ "$CONFIRMED" -eq 0 ] && [ "$DELETE_NAMESPACE" -eq 0 ] || usage render + require_namespace_contract + if [ -n "$(workload_state)" ]; then + echo "error: agent-os-firstmate already exists; use upgrade" >&2 + exit 2 + fi apply_rendered ;; upgrade) + [ "$CONFIRMED" -eq 0 ] && [ "$DELETE_NAMESPACE" -eq 0 ] || usage render - reconcile_rbac + require_namespace_contract + previous=$(workload_state) + if [ -z "$previous" ]; then + echo "error: agent-os-firstmate does not exist; use install" >&2 + exit 2 + fi + previous_mode=$(printf '%s' "$previous" | cut -f2) + previous_cleanup=$(printf '%s' "$previous" | cut -f3) + cleanup_required=0 + if [ "$DESIRED_RBAC" != cluster-admin ] && \ + { [ "$previous_mode" = cluster-admin ] || [ -z "$previous_mode" ] || [ "$previous_cleanup" = required ]; }; then + cleanup_required=1 + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" annotate statefulset agent-os-firstmate \ + agent-os.dev/cluster-rbac-cleanup=required --overwrite >/dev/null + fi apply_rendered + verify_desired_rbac + if [ "$DESIRED_RBAC" != namespace ]; then + delete_namespace_rbac + fi + if [ "$DESIRED_RBAC" = cluster-admin ] && [ "$previous_cleanup" = required ]; then + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" annotate statefulset agent-os-firstmate \ + agent-os.dev/cluster-rbac-cleanup- >/dev/null + fi + if [ "$cleanup_required" -eq 1 ]; then + report_cluster_cleanup + exit 3 + fi ;; rollback) + [ "$CONFIRMED" -eq 0 ] && [ "$DELETE_NAMESPACE" -eq 0 ] || usage "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout undo statefulset/agent-os-firstmate "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s ;; status) + [ "$CONFIRMED" -eq 0 ] && [ "$DELETE_NAMESPACE" -eq 0 ] || usage "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get statefulset agent-os-firstmate ;; uninstall) - if [ "$CONFIRM" != --yes ]; then + if [ "$CONFIRMED" -ne 1 ]; then echo "error: uninstall requires --yes" >&2 exit 2 fi render - delete_cluster_admin_rbac + require_namespace_contract + previous=$(workload_state) + previous_mode=$(printf '%s' "$previous" | cut -f2) + previous_cleanup=$(printf '%s' "$previous" | cut -f3) + delete_rendered_namespaced_resources delete_namespace_rbac - "$KUBECTL" --context "$CONTEXT" delete --ignore-not-found -f "$OUT" + if [ "$DELETE_NAMESPACE" -eq 1 ]; then + delete_owned_empty_namespace + fi + if [ "$DESIRED_RBAC" = cluster-admin ] || [ "$previous_mode" = cluster-admin ] || \ + [ "$previous_cleanup" = required ] || { [ -n "$previous" ] && [ -z "$previous_mode" ]; }; then + report_cluster_cleanup + exit 3 + fi ;; - *) - echo "usage: $0 install|upgrade|rollback|status|uninstall [--yes]" >&2 - exit 2 + cleanup-cluster-rbac) + if [ "$CONFIRMED" -ne 1 ] || [ "$DELETE_NAMESPACE" -eq 1 ]; then + echo "error: cleanup-cluster-rbac requires --yes" >&2 + exit 2 + fi + render + cleanup_cluster_rbac ;; + *) usage ;; esac diff --git a/bin/agent-os-local.sh b/bin/agent-os-local.sh index 242fc5014..eb4ba359d 100755 --- a/bin/agent-os-local.sh +++ b/bin/agent-os-local.sh @@ -62,7 +62,7 @@ case "$COMMAND" in ;; destroy) if [ "${2:-}" != --yes ]; then - echo "error: destroy requires --yes and deletes only namespace '$NAMESPACE'" >&2 + echo "error: destroy requires --yes and removes only namespaced Agent OS resources from '$NAMESPACE'" >&2 exit 2 fi AGENT_OS_CONTEXT="$CONTEXT" AGENT_OS_NAMESPACE="$NAMESPACE" AGENT_OS_INPUTS="$PROFILE" \ diff --git a/docs/agent-evals.md b/docs/agent-evals.md index 4bf41ad5f..3696982c7 100644 --- a/docs/agent-evals.md +++ b/docs/agent-evals.md @@ -10,8 +10,8 @@ Transcripts explain failures but do not override an incorrect outcome. The first regression set covers these observed contracts: -- a mate receives no AI credential unless a Secret reference or explicit grant is selected; -- an empty credential grant produces a mate with no AI credential mount; +- a mate receives no AI credential unless an explicit namespace-local Secret reference is selected; +- a missing credential grant is rejected before any mate resource is created; - a granted Pi auth file is mounted read-only from a Kubernetes Secret; - an in-cluster kubeconfig follows the projected token file and never embeds its contents; - a restored dead Herdr terminal may be replaced, while an idle or working agent may not; diff --git a/docs/kubernetes.md b/docs/kubernetes.md index d5f8704b7..4cfd266ca 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -17,7 +17,7 @@ No Kubernetes Secret is required for the portable install. The package has no credential field or Secret reference. Runtime AI, GitHub, or other authority is created separately by its owner after installation and is never supplied through package inputs, command arguments, or rendered YAML. -The package requires an image digest because a mutable tag cannot identify an upgrade or recovery input. +The package requires one complete 64-hex image digest because a mutable or malformed reference cannot identify an upgrade or recovery input. The source tree contains an intentionally non-installable placeholder until the first public image release is published. Replace it only with the digest recorded by that release workflow. @@ -39,6 +39,10 @@ bin/agent-os-kubernetes.sh install The installer renders only `tools/agent-os/packages/firstmate/package.k`, applies that fresh output to the named context, and waits for `agent-os-firstmate` to roll out. It never reads an ambient Kubernetes context. +The namespace in the rendered StatefulSet is authoritative, and an inconsistent `AGENT_OS_NAMESPACE` stops the operation before any Kubernetes request. +With `createNamespace: true`, install creates only an absent namespace or reuses one carrying the exact package installation identity. +It refuses to adopt a pre-existing unowned or foreign namespace. +With `createNamespace: false`, the selected namespace must already exist without Agent OS ownership metadata and remains outside package ownership. The default `rbac: namespace` creates a ServiceAccount plus a Role and RoleBinding scoped to the selected namespace. That Role allows Firstmate to manage runtime crewmate Pods and PVCs and inspect its StatefulSet. @@ -56,8 +60,11 @@ To upgrade, use a new released digest in the same input file and apply it throug bin/agent-os-kubernetes.sh upgrade ``` -Upgrade reconciles the mutually exclusive namespace and cluster-admin RBAC resources before applying the new render. -Downgrading from `cluster-admin` therefore requires permission to delete the prior ClusterRoleBinding and fails instead of silently retaining authority. +Upgrade applies and verifies the desired workload and RBAC before removing obsolete namespace Role and RoleBinding resources. +Routine `namespace` and `none` operations never inspect or delete cluster-scoped RBAC. +When a downgrade may leave the exact package-owned ClusterRoleBinding, upgrade exits incomplete and prints a separately confirmed `cleanup-cluster-rbac --yes` command plus the required absence evidence. +Run that command only through an explicitly approved cluster-admin identity. +It refuses to delete a same-name binding unless its ownership label and installation annotation both match. To roll back only the Firstmate workload revision, use Kubernetes StatefulSet history. This does not roll back package inputs, RBAC, or persistent data. @@ -72,15 +79,22 @@ To inspect the workload, use the same explicit context and namespace. bin/agent-os-kubernetes.sh status ``` -Uninstall is deliberately confirmed and bounded to a fresh render plus the package's deterministic RBAC resource names. -It attempts all package-owned RBAC modes so a binding omitted by newer inputs cannot survive. -When those inputs create the namespace, deleting that rendered Namespace also removes everything in that namespace. -When `createNamespace: false`, the command leaves unrelated resources in the existing namespace untouched. +Uninstall is deliberately confirmed and bounded to namespaced resources from a fresh render plus the deterministic namespace Role and RoleBinding names. +It retains the namespace by default and reports possible cluster-scoped residue without requesting cluster-wide authority. +Use the separately printed privileged cleanup command to remove an exactly owned ClusterRoleBinding. +The optional `--delete-namespace` flag works only for `createNamespace: true`, rechecks the exact installation identity, inventories every listable namespaced resource type, and refuses deletion while any foreign resource remains. +With `createNamespace: false`, the namespace is never deleted. ```sh bin/agent-os-kubernetes.sh uninstall --yes ``` +To delete an exactly owned and otherwise empty namespace as part of the confirmed uninstall, use: + +```sh +bin/agent-os-kubernetes.sh uninstall --yes --delete-namespace +``` + ## Runtime mates Firstmate creates a separate-Pod crewmate at runtime with the internal `crewmate.yaml` template in the canonical package. diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 478b8af40..6e8607ac3 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -108,6 +108,21 @@ assert_grep 'id: build' "$ROOT/.github/workflows/agent-os-image.yml" \ "release workflow must expose its build result" assert_grep 'steps.build.outputs.digest' "$ROOT/.github/workflows/agent-os-image.yml" \ "release workflow must record the immutable published digest" +assert_grep 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' "$ROOT/.github/workflows/agent-os-image.yml" \ + "release checkout action must be pinned to the reviewed full SHA" +assert_grep 'docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130' "$ROOT/.github/workflows/agent-os-image.yml" \ + "release QEMU action must be pinned to the reviewed full SHA" +assert_grep 'docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f' "$ROOT/.github/workflows/agent-os-image.yml" \ + "release Buildx action must be pinned to the reviewed full SHA" +assert_grep 'docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9' "$ROOT/.github/workflows/agent-os-image.yml" \ + "release login action must be pinned to the reviewed full SHA" +assert_grep 'docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051' "$ROOT/.github/workflows/agent-os-image.yml" \ + "release metadata action must be pinned to the reviewed full SHA" +assert_grep 'docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8' "$ROOT/.github/workflows/agent-os-image.yml" \ + "release build action must be pinned to the reviewed full SHA" +if grep -E 'uses: (actions/checkout|docker/[^@]+)@v[0-9]+' "$ROOT/.github/workflows/agent-os-image.yml" >/dev/null; then + fail "release workflow must not use mutable major action tags" +fi bash -n "$ROOT/bin/agent-os-container-entrypoint.sh" bash -n "$ROOT/bin/agent-os-kubeconfig.sh" pass "container files pin dependencies and exclude host credentials" diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index c9face05f..5afeae2b5 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -47,6 +47,59 @@ fi if [ "${AGENT_OS_TEST_FAIL_WAIT:-0}" = 1 ] && [ "${1:-}" = -n ] && [ "${3:-}" = wait ]; then exit 1 fi +if [ "${AGENT_OS_TEST_FAIL_ANNOTATE:-0}" = 1 ] && [[ " $* " = *" annotate statefulset agent-os-firstmate "* ]]; then + exit 1 +fi +case " $* " in + *" get namespace "*" --ignore-not-found -o name "*) + case "${AGENT_OS_TEST_NAMESPACE_STATE:-absent}" in + absent) ;; + *) printf 'namespace/%s\n' "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" ;; + esac + ;; + *" get namespace "*" -o jsonpath="*) + case "${AGENT_OS_TEST_NAMESPACE_STATE:-absent}" in + owned) printf 'agent-os\tagent-os-firstmate:%s' "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" ;; + foreign) printf 'other\tother-installation' ;; + *) printf '\t' ;; + esac + ;; + *" get statefulset agent-os-firstmate --ignore-not-found -o jsonpath="*) + case "${AGENT_OS_TEST_WORKLOAD_STATE:-absent}" in + absent) ;; + namespace) printf 'agent-os-firstmate\tnamespace\t' ;; + cluster-admin) printf 'agent-os-firstmate\tcluster-admin\t' ;; + none) printf 'agent-os-firstmate\tnone\t' ;; + pending) printf 'agent-os-firstmate\tnamespace\trequired' ;; + unknown) printf 'agent-os-firstmate\t\t' ;; + esac + ;; + *" get role agent-os-firstmate-runtime -o jsonpath="*) + printf 'agent-os-firstmate-runtime' + ;; + *" get rolebinding agent-os-firstmate-runtime -o jsonpath="*) + printf 'Role\tagent-os-firstmate-runtime\tServiceAccount\tagent-os-firstmate\t%s' \ + "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" + ;; + *" get clusterrolebinding agent-os-firstmate-"*" --ignore-not-found -o jsonpath="*) + case "${AGENT_OS_TEST_CLUSTER_RBAC_STATE:-absent}" in + absent) ;; + owned) + printf 'agent-os-firstmate-%s\tagent-os\tagent-os-firstmate:%s' \ + "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" + ;; + foreign) printf 'agent-os-firstmate-portable-agent-os\tother\tother-installation' ;; + esac + ;; + *" api-resources --verbs=list --namespaced -o name "*) + printf '%s\n' pods serviceaccounts configmaps + ;; + *" get pods -o name "*) + [ -z "${AGENT_OS_TEST_FOREIGN_RESOURCE:-}" ] || printf 'pod/%s\n' "$AGENT_OS_TEST_FOREIGN_RESOURCE" + ;; + *" get serviceaccounts -o name "*) printf '%s\n' serviceaccount/default ;; + *" get configmaps -o name "*) printf '%s\n' configmap/kube-root-ca.crt ;; +esac SH chmod +x "$FAKEBIN/kubectl" @@ -73,6 +126,9 @@ assert_grep 'mountPath: /home/agent/.pi/agent/auth.json' "$STDIN_LOG" \ assert_grep 'secretName: scout-1-ai-auth' "$STDIN_LOG" \ "children must reference the explicitly selected namespace-local Secret" assert_grep 'readOnly: true' "$STDIN_LOG" "child AI authorization must be read-only" +assert_grep 'readinessProbe:' "$STDIN_LOG" "child readiness must wait for Herdr health" +assert_grep 'herdr' "$STDIN_LOG" "child readiness must invoke Herdr" +assert_grep 'status' "$STDIN_LOG" "child readiness must inspect Herdr status" grep -Fqx 'kubectl -n agent-os-demo wait --for=condition=Ready pod/agent-os-crewmate-scout-1 --timeout=180s' "$CALLS" || \ fail "create must fail when the authorized Secret cannot produce a ready Pod" pass "crewmate create emits one isolated Pod and PVC" @@ -150,16 +206,59 @@ while [ "$#" -gt 0 ]; do done mkdir -p "$out" rbac=$(awk '/^rbac:/{print $2}' "$inputs") +namespace=$(awk '/^namespace:/{print $2}' "$inputs") +create_namespace=$(awk '/^createNamespace:/{print $2}' "$inputs") +[ -n "$create_namespace" ] || create_namespace=true +cat > "$out/statefulset.yaml" < "$out/namespace.yaml" < "$out/rendered.yaml" ;; - cluster-admin) printf 'apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\n' > "$out/rendered.yaml" ;; - *) printf 'apiVersion: v1\nkind: ConfigMap\n' > "$out/rendered.yaml" ;; + namespace) + printf 'apiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n namespace: %s\n' "$namespace" \ + > "$out/role.yaml" + printf 'apiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n namespace: %s\n' "$namespace" \ + > "$out/rolebinding.yaml" + ;; + cluster-admin) + cat > "$out/clusterrolebinding.yaml" </dev/null; then fi pass "generic install renders and applies the canonical package on an explicit context" +: > "$CALLS" +owned_namespace_out='' +owned_namespace_rc=0 +owned_namespace_out=$(AGENT_OS_TEST_NAMESPACE_STATE=foreign run_generic install 2>&1) || owned_namespace_rc=$? +[ "$owned_namespace_rc" -eq 2 ] || \ + fail "install into an existing unowned namespace must exit 2, got $owned_namespace_rc: $owned_namespace_out" +if grep -F 'kubectl --context kind-agent-os apply' "$CALLS" >/dev/null; then + fail "install must reject an unowned existing namespace before apply" +fi +pass "createNamespace install refuses implicit namespace adoption" + +UNOWNED_INPUTS="$TMP/unowned-namespace-inputs.yaml" +awk '{ print; if ($1 == "namespace:") print "createNamespace: false" }' "$GENERIC_INPUTS" > "$UNOWNED_INPUTS" +: > "$CALLS" +PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS="$UNOWNED_INPUTS" \ + AGENT_OS_TEST_NAMESPACE=portable-agent-os AGENT_OS_TEST_NAMESPACE_STATE=unowned \ + AGENT_OS_TEST_WORKLOAD_STATE=absent AGENT_OS_CONTEXT=kind-agent-os \ + AGENT_OS_NAMESPACE=portable-agent-os "$GENERIC" install +grep -Fq 'kubectl --context kind-agent-os apply -f ' "$CALLS" || \ + fail "createNamespace=false must install into a pre-existing unowned namespace" +pass "createNamespace=false requires and preserves an unowned namespace" + +: > "$CALLS" +owned_unmanaged_out='' +owned_unmanaged_rc=0 +owned_unmanaged_out=$(PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS="$UNOWNED_INPUTS" \ + AGENT_OS_TEST_NAMESPACE=portable-agent-os AGENT_OS_TEST_NAMESPACE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=absent AGENT_OS_CONTEXT=kind-agent-os \ + AGENT_OS_NAMESPACE=portable-agent-os "$GENERIC" install 2>&1) || owned_unmanaged_rc=$? +[ "$owned_unmanaged_rc" -eq 2 ] || \ + fail "createNamespace=false must reject an Agent-OS-owned namespace: $owned_unmanaged_out" +pass "createNamespace=false refuses an owned namespace" + +: > "$CALLS" +mismatch_out='' +mismatch_rc=0 +mismatch_out=$(PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS="$GENERIC_INPUTS" \ + AGENT_OS_TEST_NAMESPACE=portable-agent-os AGENT_OS_CONTEXT=kind-agent-os \ + AGENT_OS_NAMESPACE=wrong-namespace "$GENERIC" install 2>&1) || mismatch_rc=$? +[ "$mismatch_rc" -eq 2 ] || fail "namespace mismatch must exit 2, got $mismatch_rc: $mismatch_out" +if grep -F '^kubectl ' "$CALLS" >/dev/null; then + fail "namespace mismatch must fail before any kubectl mutation" +fi +pass "rendered namespace is authoritative over an inconsistent environment" + missing_akua_out='' missing_akua_rc=0 missing_akua_out=$(PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS="$GENERIC_INPUTS" \ @@ -187,27 +331,96 @@ assert_contains "$missing_akua_out" "error: Akua renderer 'missing-agent-os-akua pass "Kubernetes package operations require the optional Akua renderer explicitly" : > "$CALLS" -run_generic upgrade -grep -Fqx 'kubectl --context kind-agent-os delete clusterrolebinding agent-os-firstmate-portable-agent-os --ignore-not-found' "$CALLS" || \ - fail "namespace RBAC upgrade must revoke any stale cluster-admin binding" -if grep -F 'delete rolebinding agent-os-firstmate-runtime' "$CALLS" >/dev/null; then - fail "namespace RBAC upgrade must retain its rendered RoleBinding" +cleanup_out='' +cleanup_rc=0 +cleanup_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=cluster-admin \ + run_generic upgrade 2>&1) || cleanup_rc=$? +[ "$cleanup_rc" -eq 3 ] || \ + fail "cluster-admin downgrade must stop for privileged cleanup with exit 3, got $cleanup_rc: $cleanup_out" +apply_line=$(grep -Fn 'kubectl --context kind-agent-os apply -f ' "$CALLS" | head -n 1 | cut -d: -f1) +marker_line=$(grep -Fn 'annotate statefulset agent-os-firstmate agent-os.dev/cluster-rbac-cleanup=required' "$CALLS" | head -n 1 | cut -d: -f1) +rollout_line=$(grep -Fn 'kubectl --context kind-agent-os -n portable-agent-os rollout status' "$CALLS" | head -n 1 | cut -d: -f1) +verify_line=$(grep -Fn 'kubectl --context kind-agent-os -n portable-agent-os get role agent-os-firstmate-runtime' "$CALLS" | head -n 1 | cut -d: -f1) +[ -n "$marker_line" ] && [ -n "$apply_line" ] && [ -n "$rollout_line" ] && [ -n "$verify_line" ] && \ + [ "$marker_line" -lt "$apply_line" ] && [ "$apply_line" -lt "$rollout_line" ] && \ + [ "$rollout_line" -lt "$verify_line" ] || \ + fail "downgrade must apply and roll out desired namespaced RBAC before privileged cleanup is requested" +if grep -E 'kubectl .* (get|delete) clusterrolebinding' "$CALLS" >/dev/null; then + fail "routine namespace upgrade must never request cluster-wide RBAC authority" fi -pass "namespace RBAC upgrade revokes stale cluster-admin authority" +assert_contains "$cleanup_out" 'cleanup-cluster-rbac --yes' \ + "downgrade must print the exact separately confirmed privileged cleanup command" +assert_contains "$cleanup_out" 'clusterrolebinding/agent-os-firstmate-portable-agent-os absent' \ + "downgrade must print the required cleanup evidence" +pass "cluster-admin downgrade applies desired RBAC before requiring privileged cleanup" + +: > "$CALLS" +marker_failure_rc=0 +AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=cluster-admin \ + AGENT_OS_TEST_FAIL_ANNOTATE=1 run_generic upgrade >/dev/null 2>&1 || marker_failure_rc=$? +[ "$marker_failure_rc" -ne 0 ] || fail "downgrade must fail if its durable cleanup marker cannot be recorded" +if grep -F 'kubectl --context kind-agent-os apply -f ' "$CALLS" >/dev/null; then + fail "downgrade must record its durable cleanup marker before changing the workload RBAC mode" +fi +pass "cluster-admin downgrade records cleanup state before mutation" + +: > "$CALLS" +if AGENT_OS_TEST_CLUSTER_RBAC_STATE=owned run_generic cleanup-cluster-rbac >/dev/null 2>&1; then + fail "privileged cluster RBAC cleanup must require --yes" +fi +[ ! -s "$CALLS" ] || fail "unconfirmed cluster RBAC cleanup invoked an external command" + +: > "$CALLS" +AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=pending \ + AGENT_OS_TEST_CLUSTER_RBAC_STATE=owned run_generic cleanup-cluster-rbac --yes +grep -Fq 'kubectl --context kind-agent-os get clusterrolebinding agent-os-firstmate-portable-agent-os --ignore-not-found' "$CALLS" || \ + fail "privileged cleanup must inspect only the exact stale ClusterRoleBinding" +grep -Fqx 'kubectl --context kind-agent-os delete clusterrolebinding agent-os-firstmate-portable-agent-os' "$CALLS" || \ + fail "privileged cleanup must delete only the exact owned ClusterRoleBinding" +grep -Fqx 'kubectl --context kind-agent-os wait --for=delete clusterrolebinding/agent-os-firstmate-portable-agent-os --timeout=60s' "$CALLS" || \ + fail "privileged cleanup must produce deletion evidence for the exact binding" +pass "privileged cleanup verifies ownership and deletes one exact binding" + +: > "$CALLS" +absent_binding_out=$(AGENT_OS_TEST_NAMESPACE_STATE=absent AGENT_OS_TEST_WORKLOAD_STATE=absent \ + AGENT_OS_TEST_CLUSTER_RBAC_STATE=absent run_generic cleanup-cluster-rbac --yes) +assert_contains "$absent_binding_out" 'clusterrolebinding/agent-os-firstmate-portable-agent-os absent' \ + "privileged cleanup must accept exact absence as completion evidence" +if grep -F 'delete clusterrolebinding' "$CALLS" >/dev/null; then + fail "privileged cleanup must not delete an already absent ClusterRoleBinding" +fi +pass "privileged cleanup records absence after namespace deletion" + +: > "$CALLS" +foreign_binding_out='' +foreign_binding_rc=0 +foreign_binding_out=$(AGENT_OS_TEST_CLUSTER_RBAC_STATE=foreign run_generic cleanup-cluster-rbac --yes 2>&1) || \ + foreign_binding_rc=$? +[ "$foreign_binding_rc" -eq 2 ] || fail "foreign ClusterRoleBinding cleanup must exit 2: $foreign_binding_out" +if grep -F 'delete clusterrolebinding' "$CALLS" >/dev/null; then + fail "privileged cleanup must never delete a binding with mismatched ownership" +fi +pass "privileged cleanup refuses a foreign exact-name binding" CLUSTER_ADMIN_INPUTS="$TMP/cluster-admin-inputs.yaml" sed 's/rbac: namespace/rbac: cluster-admin/' "$GENERIC_INPUTS" > "$CLUSTER_ADMIN_INPUTS" : > "$CALLS" PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS="$CLUSTER_ADMIN_INPUTS" \ - AGENT_OS_CONTEXT=kind-agent-os AGENT_OS_NAMESPACE=portable-agent-os "$GENERIC" upgrade + AGENT_OS_TEST_NAMESPACE=portable-agent-os AGENT_OS_TEST_NAMESPACE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_CONTEXT=kind-agent-os \ + AGENT_OS_NAMESPACE=portable-agent-os "$GENERIC" upgrade grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete rolebinding agent-os-firstmate-runtime --ignore-not-found' "$CALLS" || \ fail "cluster-admin upgrade must delete the stale namespace RoleBinding" grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete role agent-os-firstmate-runtime --ignore-not-found' "$CALLS" || \ fail "cluster-admin upgrade must delete the stale namespace Role" +apply_line=$(grep -Fn 'kubectl --context kind-agent-os apply -f ' "$CALLS" | head -n 1 | cut -d: -f1) +delete_line=$(grep -Fn 'delete rolebinding agent-os-firstmate-runtime' "$CALLS" | head -n 1 | cut -d: -f1) +[ -n "$apply_line" ] && [ -n "$delete_line" ] && [ "$apply_line" -lt "$delete_line" ] || \ + fail "cluster-admin upgrade must apply replacement authority before removing namespace RBAC" if grep -F 'delete clusterrolebinding agent-os-firstmate-portable-agent-os' "$CALLS" >/dev/null; then - fail "cluster-admin upgrade must retain its rendered ClusterRoleBinding" + fail "routine cluster-admin upgrade must retain its rendered ClusterRoleBinding" fi -pass "cluster-admin RBAC upgrade removes stale namespace authority" +pass "cluster-admin upgrade reconciles namespaced authority after apply" : > "$CALLS" run_generic rollback @@ -219,13 +432,52 @@ pass "generic rollback remains StatefulSet-scoped" if run_generic uninstall >/dev/null 2>&1; then fail "generic uninstall must require --yes" fi -run_generic uninstall --yes -grep -Fq 'kubectl --context kind-agent-os delete --ignore-not-found -f ' "$CALLS" || \ - fail "confirmed uninstall must delete only resources from the fresh package render" -grep -Fqx 'kubectl --context kind-agent-os delete clusterrolebinding agent-os-firstmate-portable-agent-os --ignore-not-found' "$CALLS" || \ - fail "uninstall must revoke a stale cluster-admin binding even when current inputs omit it" +: > "$CALLS" +AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=namespace run_generic uninstall --yes +if grep -E 'kubectl .* (get|delete) clusterrolebinding' "$CALLS" >/dev/null; then + fail "routine namespace uninstall must never request cluster-wide RBAC authority" +fi +if grep -F 'delete namespace portable-agent-os' "$CALLS" >/dev/null; then + fail "bounded uninstall must retain its namespace by default" +fi grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete rolebinding agent-os-firstmate-runtime --ignore-not-found' "$CALLS" || \ fail "uninstall must remove namespace runtime binding regardless of current inputs" grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete role agent-os-firstmate-runtime --ignore-not-found' "$CALLS" || \ fail "uninstall must remove namespace runtime Role regardless of current inputs" -pass "generic uninstall requires confirmation and revokes every package RBAC mode" +pass "routine uninstall removes namespaced resources without cluster-wide authority" + +: > "$CALLS" +uninstall_residue_out='' +uninstall_residue_rc=0 +uninstall_residue_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=cluster-admin \ + run_generic uninstall --yes 2>&1) || uninstall_residue_rc=$? +[ "$uninstall_residue_rc" -eq 3 ] || \ + fail "cluster-admin uninstall must report residue with exit 3: $uninstall_residue_out" +if grep -E 'kubectl .* (get|delete) clusterrolebinding' "$CALLS" >/dev/null; then + fail "routine uninstall must report cluster residue without inspecting or deleting it" +fi +assert_contains "$uninstall_residue_out" 'cleanup-cluster-rbac --yes' \ + "cluster-admin uninstall must print the exact privileged cleanup command" +pass "routine uninstall reports cluster-scoped residue for separate cleanup" + +: > "$CALLS" +AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=namespace \ + run_generic uninstall --yes --delete-namespace +grep -Fqx 'kubectl --context kind-agent-os delete namespace portable-agent-os' "$CALLS" || \ + fail "optional namespace deletion must target only the exactly owned namespace" +grep -Fq 'kubectl --context kind-agent-os api-resources --verbs=list --namespaced -o name' "$CALLS" || \ + fail "optional namespace deletion must inventory every listable namespaced resource type" +pass "optional namespace deletion proves ownership and no foreign resources" + +: > "$CALLS" +foreign_resource_out='' +foreign_resource_rc=0 +foreign_resource_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=namespace \ + AGENT_OS_TEST_FOREIGN_RESOURCE=foreign-workload run_generic uninstall --yes --delete-namespace 2>&1) || \ + foreign_resource_rc=$? +[ "$foreign_resource_rc" -eq 2 ] || \ + fail "foreign namespace resources must block namespace deletion: $foreign_resource_out" +if grep -F 'delete namespace portable-agent-os' "$CALLS" >/dev/null; then + fail "namespace deletion must fail closed when foreign resources remain" +fi +pass "optional namespace deletion refuses foreign resources" diff --git a/tests/agent-os-local.test.sh b/tests/agent-os-local.test.sh index b7845eb26..3d66f8066 100755 --- a/tests/agent-os-local.test.sh +++ b/tests/agent-os-local.test.sh @@ -50,7 +50,27 @@ while [ "$#" -gt 0 ]; do done printf 'akua-input-image %s\n' "$(awk '/^image:/{print $2}' "$inputs")" >> "$AGENT_OS_TEST_LOG" mkdir -p "$out" -printf 'apiVersion: v1\nkind: ConfigMap\n' > "$out/rendered.yaml" +cat > "$out/statefulset.yaml" < "$out/rendered.yaml" +cat > "$out/namespace.yaml" <<'YAML' +apiVersion: v1 +kind: Namespace +metadata: + name: agent-os-demo + labels: + app.kubernetes.io/managed-by: agent-os + annotations: + agent-os.dev/installation-id: agent-os-firstmate:agent-os-demo +YAML +printf 'apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\n' > "$out/clusterrolebinding.yaml" SH chmod +x "$FAKEBIN/akua" @@ -133,16 +153,23 @@ test_empty_image_override_uses_content_addressed_default() { } test_destroy_requires_exact_confirmation() { - local out rc=0 + local out rc=0 cleanup_out cleanup_rc=0 : > "$LOG" out=$(run_cli destroy 2>&1) || rc=$? [ "$rc" -eq 2 ] || fail "destroy without --yes must exit 2, got $rc: $out" [ ! -s "$LOG" ] || fail "destroy without --yes invoked an external command" - run_cli destroy --yes + cleanup_out=$(run_cli destroy --yes 2>&1) || cleanup_rc=$? + [ "$cleanup_rc" -eq 3 ] || \ + fail "cluster-admin demo destroy must stop for separate privileged cleanup, got $cleanup_rc: $cleanup_out" grep -Fq 'kubectl --context orbstack delete --ignore-not-found -f ' "$LOG" || \ fail "confirmed destroy must delete only resources from the rendered OrbStack profile" - pass "destroy requires confirmation and remains package-scoped" + if grep -E 'kubectl .* (get|delete) clusterrolebinding' "$LOG" >/dev/null; then + fail "routine demo destroy must not inspect or delete cluster-scoped RBAC" + fi + assert_contains "$cleanup_out" 'cleanup-cluster-rbac --yes' \ + "demo destroy must print the separately confirmed privileged cleanup command" + pass "destroy requires confirmation and reports privileged RBAC cleanup" } test_non_orbstack_context_is_fail_closed() { diff --git a/tests/agent-os-packages.test.sh b/tests/agent-os-packages.test.sh index 3319dbaa2..9c380c5cc 100755 --- a/tests/agent-os-packages.test.sh +++ b/tests/agent-os-packages.test.sh @@ -11,6 +11,7 @@ TMP=$(fm_test_tmproot agent-os-packages) INPUTS="$TMP/inputs.yaml" OUT="$TMP/rendered" MUTABLE_INPUTS="$TMP/mutable-image.yaml" +CLUSTER_ADMIN_INPUTS="$TMP/cluster-admin.yaml" mkdir -p "$TMP" @@ -38,6 +39,12 @@ assert_grep 'kind = "Role"' "$FIRSTMATE/package.k" \ "the portable package must render the namespace runtime Role" assert_grep 'kind = "RoleBinding"' "$FIRSTMATE/package.k" \ "the portable package must bind its explicit ServiceAccount to that Role" +assert_grep '"app.kubernetes.io/managed-by" = "agent-os"' "$FIRSTMATE/package.k" \ + "package resources must carry the Agent OS ownership label" +assert_grep '"agent-os.dev/installation-id"' "$FIRSTMATE/package.k" \ + "cluster and namespace resources must carry the installation identity" +assert_grep '"agent-os.dev/rbac-mode" = input.rbac' "$FIRSTMATE/package.k" \ + "the workload must record the applied RBAC mode for safe reconciliation" assert_grep 'resources = ["pods", "persistentvolumeclaims"]' "$FIRSTMATE/package.k" \ "runtime apply authority must exclude Pod subresources from patch access" assert_grep 'verbs = ["get", "list", "watch", "create", "delete", "patch"]' "$FIRSTMATE/package.k" \ @@ -60,6 +67,14 @@ assert_grep "\$patch: delete" "$ROOT/deploy/akua/firstmate-auth-revoke.yaml" \ "Akua overlay must define explicit authorization mount cleanup" assert_grep 'mountPath: /var/run/secrets/agent-os/akua' "$ROOT/deploy/akua/firstmate-auth-revoke.yaml" \ "Akua overlay cleanup must use the strategic merge key for volume mounts" +assert_grep "kubectl --context \"\$context\" -n \"\$namespace\" patch statefulset agent-os-firstmate --type=strategic --patch-file \"\$grant_patch\"" \ + "$ROOT/.agents/skills/akua-intelligence-bootstrap/SKILL.md" \ + "Akua authorization grant must pin context, namespace, target, patch type, and patch file" +assert_grep "kubectl --context \"\$context\" -n \"\$namespace\" patch statefulset agent-os-firstmate --type=strategic --patch-file \"\$revoke_patch\"" \ + "$ROOT/.agents/skills/akua-intelligence-bootstrap/SKILL.md" \ + "Akua authorization revocation must use the same explicit targeting contract" +assert_grep 'a missing credential grant is rejected before any mate resource is created' "$ROOT/docs/agent-evals.md" \ + "the eval contract must match fail-closed runtime authorization" assert_grep '@sha256:' "$FIRSTMATE/inputs.example.yaml" \ "the example must use an immutable image digest" [ ! -f "$MATE/package.k" ] || fail "mate creation must not remain a separately installable package" @@ -76,6 +91,7 @@ akua render --no-agent-mode --package "$FIRSTMATE/package.k" --inputs "$INPUTS" fail "the portable package must render without an Akua account or credential" rendered=$(cat "$OUT"/*.yaml) +statefulset_rendered=$(cat "$(grep -Rl '^kind: StatefulSet$' "$OUT")") assert_contains "$rendered" 'kind: ServiceAccount' "rendered topology must create a ServiceAccount" assert_contains "$rendered" 'kind: Role' "rendered topology must create namespace-scoped runtime RBAC" assert_contains "$rendered" 'kind: RoleBinding' "rendered topology must bind namespace runtime RBAC" @@ -83,6 +99,12 @@ assert_contains "$rendered" 'kind: PersistentVolumeClaim' "rendered topology mus assert_contains "$rendered" 'kind: StatefulSet' "rendered topology must run Firstmate as a StatefulSet" assert_contains "$rendered" 'ghcr.io/akua-dev/agent-os@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ "rendered topology must retain the immutable image digest" +assert_contains "$rendered" 'app.kubernetes.io/managed-by: agent-os' \ + "rendered resources must carry the Agent OS ownership label" +assert_contains "$rendered" 'agent-os.dev/installation-id: agent-os-firstmate:portable-agent-os' \ + "rendered resources must carry the exact installation identity" +assert_contains "$statefulset_rendered" 'agent-os.dev/rbac-mode: namespace' \ + "the rendered StatefulSet must record its RBAC mode" assert_not_contains "$rendered" 'kind: ClusterRoleBinding' \ "the default portable topology must not grant cluster-admin" assert_not_contains "$rendered" 'AKUA_' \ @@ -90,6 +112,18 @@ assert_not_contains "$rendered" 'AKUA_' \ assert_not_contains "$rendered" 'agent-os.akua.dev' \ "the portable rendered topology must not use Akua-owned annotations" +sed 's/^rbac: namespace/rbac: cluster-admin/' "$INPUTS" > "$CLUSTER_ADMIN_INPUTS" +akua render --no-agent-mode --package "$FIRSTMATE/package.k" --inputs "$CLUSTER_ADMIN_INPUTS" \ + --out "$TMP/cluster-admin-rendered" >/dev/null || \ + fail "the reviewed cluster-admin package profile must render" +cluster_binding=$(cat "$(grep -Rl '^kind: ClusterRoleBinding$' "$TMP/cluster-admin-rendered")") +assert_contains "$cluster_binding" 'name: agent-os-firstmate-portable-agent-os' \ + "cluster-admin RBAC must use the deterministic installation binding name" +assert_contains "$cluster_binding" 'app.kubernetes.io/managed-by: agent-os' \ + "cluster-admin RBAC must carry the exact ownership label" +assert_contains "$cluster_binding" 'agent-os.dev/installation-id: agent-os-firstmate:portable-agent-os' \ + "cluster-admin RBAC must carry the exact installation annotation" + cat > "$MUTABLE_INPUTS" <<'YAML' namespace: portable-agent-os image: ghcr.io/akua-dev/agent-os:latest @@ -103,6 +137,19 @@ if akua render --no-agent-mode --package "$FIRSTMATE/package.k" --inputs "$MUTAB fi pass "the portable package rejects mutable image tags" +for invalid_image in \ + 'ghcr.io/akua-dev/agent-os@sha256:abc' \ + 'ghcr.io/akua-dev/agent-os@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaextra' \ + 'prefix@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:suffix'; do + invalid_inputs="$TMP/invalid-$(printf '%s' "$invalid_image" | tr '/:@' '---').yaml" + sed "s|^image: .*|image: $invalid_image|" "$INPUTS" > "$invalid_inputs" + if akua render --no-agent-mode --package "$FIRSTMATE/package.k" --inputs "$invalid_inputs" \ + --out "$TMP/invalid-rendered" >/dev/null 2>&1; then + fail "the portable package accepted malformed digest reference '$invalid_image'" + fi +done +pass "the portable package requires one complete 64-hex SHA-256 digest" + assert_grep 'bin/agent-os-kubernetes.sh install' "$ROOT/docs/kubernetes.md" \ "Kubernetes docs must make the generic installer the default quickstart" assert_grep 'bin/agent-os-kubernetes.sh rollback' "$ROOT/docs/kubernetes.md" \ diff --git a/tools/agent-os/packages/firstmate/README.md b/tools/agent-os/packages/firstmate/README.md index d116d2797..15ed71f14 100644 --- a/tools/agent-os/packages/firstmate/README.md +++ b/tools/agent-os/packages/firstmate/README.md @@ -2,7 +2,7 @@ This is the one public, versioned Agent OS package. It renders ordinary Kubernetes resources for one persistent Firstmate: an optional namespace, ServiceAccount, persistent home, headless Service, StatefulSet, and explicit RBAC. -It requires an immutable Agent OS image digest and no Akua account, API key, service, or credential. +It requires one complete 64-hex Agent OS image digest and no Akua account, API key, service, or credential. Render it directly: @@ -21,5 +21,8 @@ The default `rbac: namespace` grants the Firstmate ServiceAccount only the names Set `rbac: none` when a separate authority mechanism manages mates. Set `rbac: cluster-admin` only for an isolated intelligence cluster after a reviewed grant. +With `createNamespace: true`, the lifecycle helper creates an absent namespace with the package's exact installation identity and refuses to adopt an existing unowned namespace. +With `createNamespace: false`, the namespace must already exist without Agent OS ownership metadata, and the lifecycle helper never deletes it. + The package carries no credential value or Secret reference. Any runtime credential is a separately created namespace-local Kubernetes Secret, referenced by the runtime helper only after its owner has approved that authority. diff --git a/tools/agent-os/packages/firstmate/crewmate.yaml b/tools/agent-os/packages/firstmate/crewmate.yaml index eef72650f..ee153d228 100644 --- a/tools/agent-os/packages/firstmate/crewmate.yaml +++ b/tools/agent-os/packages/firstmate/crewmate.yaml @@ -47,6 +47,14 @@ spec: value: /home/agent - name: HOME value: /home/agent + readinessProbe: + exec: + command: + - herdr + - status + - --json + initialDelaySeconds: 2 + periodSeconds: 5 resources: requests: cpu: 250m diff --git a/tools/agent-os/packages/firstmate/package.k b/tools/agent-os/packages/firstmate/package.k index 6f23368eb..4e587f9ee 100644 --- a/tools/agent-os/packages/firstmate/package.k +++ b/tools/agent-os/packages/firstmate/package.k @@ -1,4 +1,5 @@ import akua.ctx +import regex schema Input: """Inputs for one persistent Firstmate on a conformant Kubernetes cluster.""" @@ -16,21 +17,26 @@ schema Input: memoryLimit: str = "8Gi" check: - allowMutableImage or "@sha256:" in image, "image must use an immutable @sha256 digest unless the local profile explicitly allows a mutable image" + allowMutableImage or regex.match(image, r"^.+@sha256:[0-9a-fA-F]{64}$"), "image must use one complete immutable @sha256 digest unless the local profile explicitly allows a mutable image" input: Input = ctx.input() labels = { "app.kubernetes.io/name" = "agent-os" "app.kubernetes.io/component" = "firstmate" + "app.kubernetes.io/managed-by" = "agent-os" } +installationId = "agent-os-firstmate:${input.namespace}" +ownershipAnnotations = { "agent-os.dev/installation-id" = installationId } + namespaceResources = [{ apiVersion = "v1" kind = "Namespace" metadata = { name = input.namespace - labels = { "app.kubernetes.io/part-of" = "agent-os" } + labels = labels | { "app.kubernetes.io/part-of" = "agent-os" } + annotations = ownershipAnnotations } }] if input.createNamespace else [] @@ -88,6 +94,7 @@ clusterAdminResources = [{ metadata = { name = "agent-os-firstmate-${input.namespace}" labels = labels + annotations = ownershipAnnotations } roleRef = { apiGroup = "rbac.authorization.k8s.io" @@ -109,6 +116,7 @@ resources = namespaceResources + [ name = "agent-os-firstmate" namespace = input.namespace labels = labels + annotations = ownershipAnnotations } automountServiceAccountToken = True }, @@ -132,6 +140,7 @@ resources = namespaceResources + [ name = "agent-os-firstmate" namespace = input.namespace labels = labels + annotations = ownershipAnnotations } spec = { clusterIP = "None" @@ -146,6 +155,9 @@ resources = namespaceResources + [ name = "agent-os-firstmate" namespace = input.namespace labels = labels + annotations = ownershipAnnotations | { + "agent-os.dev/rbac-mode" = input.rbac + } } spec = { serviceName = "agent-os-firstmate" From 69b1d2b9010c948a660c55844e75fa0def15ed51 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 17:36:19 +0200 Subject: [PATCH 26/56] no-mistakes(review): Captain, harden Kubernetes lifecycle and image supply chain --- .agents/skills/kubernetes-fleet/SKILL.md | 15 +- .github/workflows/ci.yml | 42 ++- Dockerfile | 16 +- THIRD_PARTY_NOTICES.md | 35 ++ bin/agent-os-crewmate.sh | 189 +++++++++-- bin/agent-os-kubernetes.sh | 150 +++++++-- bin/agent-os-local.sh | 21 +- docs/kubernetes.md | 7 +- tests/agent-os-container.test.sh | 41 +++ tests/agent-os-kubernetes.test.sh | 306 ++++++++++++++++-- tests/agent-os-local.test.sh | 77 ++++- tests/agent-os-packages.test.sh | 7 + tools/agent-os/packages/firstmate/README.md | 1 + .../agent-os/packages/firstmate/crewmate.yaml | 20 +- tools/agent-os/packages/firstmate/package.k | 5 +- 15 files changed, 831 insertions(+), 101 deletions(-) diff --git a/.agents/skills/kubernetes-fleet/SKILL.md b/.agents/skills/kubernetes-fleet/SKILL.md index 4a2d07309..af345f49d 100644 --- a/.agents/skills/kubernetes-fleet/SKILL.md +++ b/.agents/skills/kubernetes-fleet/SKILL.md @@ -22,18 +22,27 @@ Load this skill only when the current firstmate is running in Kubernetes or is e - Create or select a namespace-local Kubernetes Secret only after its AI authority is explicitly authorized. - The Secret must contain the selected provider's `auth.json` key and must be provisioned independently instead of cloning or sharing the primary credential. - Pass only its name through `AGENT_OS_AI_SECRET` when invoking `bin/agent-os-crewmate.sh create `. -- The helper never reads or discovers the Secret, and the default Role intentionally has no Secret-read permission. -- Kubernetes mounts the selected `auth.json` read-only into the child; a missing Secret or key keeps the Pod unready, makes create fail, removes the non-running Pod, and retains the PVC for an authorized retry. +- The helper never reads or discovers the Secret value, and the default Role intentionally has no Secret-read permission. +- Kubernetes projects only the selected `auth.json` key into a read-only authorization directory; a missing Secret or key keeps the Pod unready, makes create fail, removes the non-running Pod, and retains the PVC for an authorized retry. - Probe the selected model route before launching work; when quota is unavailable, use another explicitly granted provider or report the capacity blocker instead of repeatedly spawning agents. - Give every launched Herdr agent a task-unique name, close only a confirmed dead restored pane before reuse, and never replace a live agent. - Grade completion by the promised artifact or delivered Git state; Herdr `idle` alone is not a completion signal. - Use `bin/agent-os-crewmate.sh create ` to create a separate Pod and persistent home. -- Use `status` to inspect it and `delete` only after unique work is checkpointed or delivered. +- Use `status` to inspect it, `stop` to remove only the Pod, and `restart` to replace only the Pod on its retained PVC. +- The ambiguous `delete` command is rejected. +- `purge --yes` is the only operation that destroys a persistent home. +- Before purge, independently checkpoint or deliver unique work, then annotate the owned PVC with `agent-os.dev/checkpoint-state=clean` and a non-secret RFC3339 `agent-os.dev/checkpoint-at` value. +- Purge verifies exact installation and crewmate ownership, displays the target, requires its own confirmation, and records requested and completed phases in `AGENT_OS_PURGE_EVIDENCE_FILE` or `$FM_HOME/data/crewmate-purge-evidence.log`. +- Purge evidence contains only time, namespace, crewmate ID, resource names, phase, and checkpoint time. - Never mount the primary home into a child Pod. - The demo child receives no Kubernetes ServiceAccount token by default. - A Pod with an authorized ServiceAccount gets a token-file-backed `in-cluster` kubeconfig automatically, so Firstmate does not need to copy a bearer token into a temporary kubeconfig. - The OrbStack primary's cluster-admin binding is a local-demo trust decision, not a production-safe default. - Pin an explicit host context for host-side `kubectl`; inside an authorized Pod use the generated `in-cluster` context. +For normal credential rotation, update the explicitly authorized namespace-local Secret, restart the owned Pod with the same PVC, prove a bounded request uses the replacement credential, and revoke the old credential only after that proof succeeds. +For urgent revocation, stop the owned Pod first, revoke the old credential, update or select the approved replacement Secret, and restart only when that replacement is ready. +Never print, copy, persist, or place credential values in command arguments or evidence during either flow. + Use ordinary `kubectl exec`, Herdr, files, and Git to supervise the child. Do not add a custom inter-agent chat protocol or Task/Run service. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2ea28704..33b3e5fe4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: name: Lint shell scripts runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 - name: Install pinned ShellCheck run: | set -eu @@ -31,7 +31,7 @@ jobs: # hung watcher or tmux test instead of riding GitHub's 360-minute default. timeout-minutes: 15 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 with: fetch-depth: 0 - name: Install pinned ShellCheck @@ -55,10 +55,38 @@ jobs: - name: Install Akua renderer run: | set -eu - export AKUA_INSTALL="$RUNNER_TEMP/akua" - curl -fsSL https://cli.akua.dev/install | sh -s -- 0.8.25 - echo "$AKUA_INSTALL/bin" >> "$GITHUB_PATH" - "$AKUA_INSTALL/bin/akua" --version + version=0.8.25 + case "$(uname -m)" in + x86_64) + triple=x86_64-unknown-linux-gnu + sha=bc57afbffe7e18aacd2146e2cd67151c56e7a3c279fe659312ff7ffb359cd03a + ;; + aarch64) + triple=aarch64-unknown-linux-gnu + sha=3a3c6bae72764cbd85a6e4e0877a05e5def8f7aeee8563b7918099214a1a313a + ;; + *) + echo "::error::unsupported Akua runner architecture: $(uname -m)" + exit 1 + ;; + esac + asset="akua-v${version}-${triple}.tar.gz" + mkdir -p "$RUNNER_TEMP/bin" + curl -fsSL "https://github.com/akua-dev/akua/releases/download/v${version}/${asset}" -o "$RUNNER_TEMP/$asset" + printf '%s %s\n' "$sha" "$RUNNER_TEMP/$asset" | sha256sum -c - + tar -xzf "$RUNNER_TEMP/$asset" -C "$RUNNER_TEMP" + install -m 0755 "$RUNNER_TEMP/akua" "$RUNNER_TEMP/bin/akua" + echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + "$RUNNER_TEMP/bin/akua" --version + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 + with: + bun-version-file: tools/agent-os/.bun-version + - name: Check Agent OS tool + working-directory: tools/agent-os + run: | + set -eu + bun install --frozen-lockfile --ignore-scripts + bun run check - run: | set -eu for test_script in tests/*.test.sh; do @@ -69,7 +97,7 @@ jobs: name: Repo invariants runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 - name: Symlinks must stay intact run: | set -eu diff --git a/Dockerfile b/Dockerfile index 7c18f04dd..c27179c2f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,7 +47,10 @@ RUN set -eu; \ esac; \ curl -fsSL "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl" -o /usr/local/bin/kubectl; \ echo "$sha /usr/local/bin/kubectl" | sha256sum -c -; \ - chmod 0755 /usr/local/bin/kubectl + chmod 0755 /usr/local/bin/kubectl; \ + mkdir -p /usr/share/licenses/kubectl; \ + curl -fsSL "https://raw.githubusercontent.com/kubernetes/kubernetes/1f328c5e9dd683d0c5e69f3d7d58f8371278dec2/LICENSE" -o /usr/share/licenses/kubectl/LICENSE; \ + echo "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30 /usr/share/licenses/kubectl/LICENSE" | sha256sum -c - RUN set -eu; \ case "$TARGETARCH" in \ @@ -60,6 +63,9 @@ RUN set -eu; \ echo "$sha /tmp/${asset}" | sha256sum -c -; \ tar -xzf "/tmp/${asset}" -C /tmp; \ install -m 0755 "/tmp/gh_${GH_VERSION}_linux_${TARGETARCH}/bin/gh" /usr/local/bin/gh; \ + mkdir -p /usr/share/licenses/gh; \ + curl -fsSL "https://raw.githubusercontent.com/cli/cli/b300f2ec7ec9dc9addc39b2ad88c54097ded7ca0/LICENSE" -o /usr/share/licenses/gh/LICENSE; \ + echo "6da4adc42392c8485e40b4251c7e332fc3352df1947c9ffade71dd60b14a7a4f /usr/share/licenses/gh/LICENSE" | sha256sum -c -; \ rm -rf "/tmp/${asset}" "/tmp/gh_${GH_VERSION}_linux_${TARGETARCH}" RUN set -eu; \ @@ -77,6 +83,11 @@ RUN set -eu; \ echo "$no_mistakes_sha /tmp/${no_mistakes_asset}" | sha256sum -c -; \ tar -xzf "/tmp/${no_mistakes_asset}" -C /usr/local/bin; \ chmod 0755 /usr/local/bin/treehouse /usr/local/bin/no-mistakes; \ + mkdir -p /usr/share/licenses/treehouse /usr/share/licenses/no-mistakes; \ + curl -fsSL "https://raw.githubusercontent.com/kunchenguid/treehouse/68fa3d2556542add76bf80255787b8625a5041a6/LICENSE" -o /usr/share/licenses/treehouse/LICENSE; \ + echo "1b962d20f826f6a758c737f8aa4e8e76dc719b8aa78fcfacdfb46681bb36c2f4 /usr/share/licenses/treehouse/LICENSE" | sha256sum -c -; \ + curl -fsSL "https://raw.githubusercontent.com/kunchenguid/no-mistakes/dc5a80059d3c0f1abbf28f20f43d994b8399bee6/LICENSE" -o /usr/share/licenses/no-mistakes/LICENSE; \ + echo "945016bd37e1ba7211622ef60ee1d23ab727896ba7710edd21e8fbe983863969 /usr/share/licenses/no-mistakes/LICENSE" | sha256sum -c -; \ rm -f "/tmp/${treehouse_asset}" "/tmp/${no_mistakes_asset}" RUN set -eu; \ @@ -90,6 +101,9 @@ RUN set -eu; \ echo "$sha /tmp/${asset}" | sha256sum -c -; \ unzip -q "/tmp/${asset}" -d /tmp/bun; \ install -m 0755 "/tmp/bun/bun-linux-${bun_arch}/bun" /usr/local/bin/bun; \ + mkdir -p /usr/share/licenses/bun; \ + curl -fsSL "https://raw.githubusercontent.com/oven-sh/bun/0d9b296af33f2b851fcbf4df3e9ec89751734ba4/LICENSE.md" -o /usr/share/licenses/bun/LICENSE.md; \ + echo "2c6160ec8fb853f7e8f97d9b249e756c9b0ac44860a68b6bf4f1b0bcbc5c3741 /usr/share/licenses/bun/LICENSE.md" | sha256sum -c -; \ rm -rf "/tmp/${asset}" /tmp/bun RUN set -eu; \ diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 7cda95879..232278847 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -22,3 +22,38 @@ The Agent OS demo image includes an unmodified K9s 0.51.0 executable as a separa K9s is available under the Apache License 2.0. The image includes K9s's license at `/usr/share/licenses/k9s/LICENSE`. The exact source is available at . + +## kubectl + +The Agent OS image includes the unmodified kubectl 1.34.8 executable from Kubernetes. +Kubernetes is available under the Apache License 2.0. +The image includes the license at `/usr/share/licenses/kubectl/LICENSE`. +The exact source commit is . + +## GitHub CLI + +The Agent OS image includes the unmodified GitHub CLI 2.96.0 executable. +GitHub CLI is available under the MIT License. +The image includes the license at `/usr/share/licenses/gh/LICENSE`. +The exact source commit is . + +## Bun + +The Agent OS image includes the unmodified Bun 1.3.14 executable. +Bun and its bundled components use the licenses recorded in the upstream `LICENSE.md` file. +The image includes that complete license record at `/usr/share/licenses/bun/LICENSE.md`. +The exact source commit is . + +## Treehouse + +The Agent OS image includes the unmodified Treehouse 2.0.0 executable. +Treehouse is available under the MIT License. +The image includes the license at `/usr/share/licenses/treehouse/LICENSE`. +The exact source commit is . + +## no-mistakes + +The Agent OS image includes the unmodified no-mistakes 1.34.0 executable. +no-mistakes is available under the MIT License. +The image includes the license at `/usr/share/licenses/no-mistakes/LICENSE`. +The exact source commit is . diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index 08597608e..05bed64aa 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -1,19 +1,22 @@ #!/usr/bin/env bash -# agent-os-crewmate.sh - create, inspect, or delete one isolated crewmate Pod. -# Usage: bin/agent-os-crewmate.sh create|status|delete +# agent-os-crewmate.sh - operate one isolated crewmate Pod and persistent home. +# Usage: bin/agent-os-crewmate.sh create|status|stop|restart|purge|delete [--yes] set -eu +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) COMMAND=${1:-} ID=${2:-} +CONFIRM=${3:-} NAMESPACE=${AGENT_OS_NAMESPACE:-agent-os} IMAGE=${AGENT_OS_IMAGE:-} IMAGE_PULL_POLICY=${AGENT_OS_IMAGE_PULL_POLICY:-IfNotPresent} AI_SECRET=${AGENT_OS_AI_SECRET:-} KUBECTL=${AGENT_OS_KUBECTL:-kubectl} TEMPLATE=${AGENT_OS_CREWMATE_TEMPLATE:-/opt/agent-os/tools/agent-os/packages/firstmate/crewmate.yaml} +INSTALLATION_ID="agent-os-firstmate:$NAMESPACE" if [ ! -f "$TEMPLATE" ]; then - TEMPLATE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/tools/agent-os/packages/firstmate/crewmate.yaml" + TEMPLATE="$ROOT/tools/agent-os/packages/firstmate/crewmate.yaml" fi case "$ID" in @@ -30,45 +33,171 @@ fi POD="agent-os-crewmate-$ID" PVC="$POD-home" +EXPECTED_POD="$POD"$'\t'"agent-os"$'\t'"$ID"$'\t'"$INSTALLATION_ID" +EXPECTED_PVC="$PVC"$'\t'"agent-os"$'\t'"$ID"$'\t'"$INSTALLATION_ID" + +kube() { + "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" "$@" +} + +resource_identity() { + local kind=$1 name=$2 + kube get "$kind" "$name" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}' +} + +pvc_record() { + kube get pvc "$PVC" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-state}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-at}' +} + +require_owned_or_absent() { + local kind=$1 name=$2 expected=$3 identity + identity=$(resource_identity "$kind" "$name") + if [ -n "$identity" ] && [ "$identity" != "$expected" ]; then + echo "error: $kind '$name' does not have the exact crewmate installation identity" >&2 + exit 2 + fi + printf '%s' "$identity" +} + +require_owned_pvc_or_absent() { + local record identity + record=$(pvc_record) + identity=$(printf '%s' "$record" | cut -f1-4) + if [ -n "$record" ] && [ "$identity" != "$EXPECTED_PVC" ]; then + echo "error: pvc '$PVC' does not have the exact crewmate installation identity" >&2 + exit 2 + fi + printf '%s' "$record" +} + +validate_ai_grant() { + if [ -z "$IMAGE" ]; then + echo "error: AGENT_OS_IMAGE must name the immutable image selected for this cluster" >&2 + exit 2 + fi + case "$AI_SECRET" in + ''|*[!a-z0-9.-]*|[.-]*|*[-.]) + echo "error: AGENT_OS_AI_SECRET must name an explicitly authorized namespace-local Secret" >&2 + exit 2 + ;; + esac + if [ "${#AI_SECRET}" -gt 253 ]; then + echo "error: AGENT_OS_AI_SECRET must be a valid Kubernetes Secret name" >&2 + exit 2 + fi +} + +render_resources() { + sed \ + -e "s|__AGENT_OS_CREWMATE_ID__|$ID|g" \ + -e "s|__AGENT_OS_NAMESPACE__|$NAMESPACE|g" \ + -e "s|__AGENT_OS_IMAGE__|$IMAGE|g" \ + -e "s|__AGENT_OS_IMAGE_PULL_POLICY__|$IMAGE_PULL_POLICY|g" \ + -e "s|__AGENT_OS_AI_SECRET__|$AI_SECRET|g" \ + "$TEMPLATE" +} + +apply_and_wait() { + render_resources | kube apply -f - + if ! kube wait --for=condition=Ready "pod/$POD" --timeout=180s; then + kube delete pod "$POD" --ignore-not-found + echo "error: crewmate Pod did not become ready with the authorized AI Secret" >&2 + exit 1 + fi +} + +preflight_create() { + require_owned_or_absent pod "$POD" "$EXPECTED_POD" >/dev/null + require_owned_pvc_or_absent >/dev/null +} + +preflight_existing_home() { + local pvc + require_owned_or_absent pod "$POD" "$EXPECTED_POD" >/dev/null + pvc=$(require_owned_pvc_or_absent) + if [ -z "$pvc" ]; then + echo "error: crewmate persistent home '$PVC' does not exist" >&2 + exit 2 + fi + printf '%s' "$pvc" +} + +record_purge() { + local phase=$1 checkpoint_at=$2 timestamp + timestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ') + printf '%s\t%s\tnamespace=%s\tcrewmate=%s\tpod=%s\tpvc=%s\tcheckpoint-at=%s\n' \ + "$timestamp" "$phase" "$NAMESPACE" "$ID" "$POD" "$PVC" "$checkpoint_at" >&3 +} case "$COMMAND" in create) - if [ -z "$IMAGE" ]; then - echo "error: AGENT_OS_IMAGE must name the immutable image selected for this cluster" >&2 + [ -z "$CONFIRM" ] || { echo "usage: $0 create " >&2; exit 2; } + validate_ai_grant + preflight_create + apply_and_wait + ;; + status) + [ -z "$CONFIRM" ] || { echo "usage: $0 status " >&2; exit 2; } + if [ -z "$(require_owned_or_absent pod "$POD" "$EXPECTED_POD")" ]; then + echo "error: crewmate Pod '$POD' does not exist" >&2 exit 2 fi - case "$AI_SECRET" in - ''|*[!a-z0-9.-]*|[.-]*|*[-.]) - echo "error: AGENT_OS_AI_SECRET must name an explicitly authorized namespace-local Secret" >&2 - exit 2 - ;; - esac - if [ "${#AI_SECRET}" -gt 253 ]; then - echo "error: AGENT_OS_AI_SECRET must be a valid Kubernetes Secret name" >&2 + require_owned_pvc_or_absent >/dev/null + kube get pod "$POD" + ;; + stop) + [ -z "$CONFIRM" ] || { echo "usage: $0 stop " >&2; exit 2; } + require_owned_or_absent pod "$POD" "$EXPECTED_POD" >/dev/null + require_owned_pvc_or_absent >/dev/null + kube delete pod "$POD" --ignore-not-found + ;; + restart) + [ -z "$CONFIRM" ] || { echo "usage: $0 restart " >&2; exit 2; } + validate_ai_grant + preflight_existing_home >/dev/null + kube delete pod "$POD" --ignore-not-found + apply_and_wait + ;; + purge) + echo "purge target: namespace/$NAMESPACE pod/$POD pvc/$PVC" >&2 + if [ "$CONFIRM" != --yes ]; then + echo "error: purge requires the purge-specific --yes confirmation" >&2 exit 2 fi - sed \ - -e "s|__AGENT_OS_CREWMATE_ID__|$ID|g" \ - -e "s|__AGENT_OS_NAMESPACE__|$NAMESPACE|g" \ - -e "s|__AGENT_OS_IMAGE__|$IMAGE|g" \ - -e "s|__AGENT_OS_IMAGE_PULL_POLICY__|$IMAGE_PULL_POLICY|g" \ - -e "s|__AGENT_OS_AI_SECRET__|$AI_SECRET|g" \ - "$TEMPLATE" | "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" apply -f - - if ! "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" wait --for=condition=Ready "pod/$POD" --timeout=180s; then - "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" delete pod "$POD" --ignore-not-found - echo "error: crewmate Pod did not become ready with the authorized AI Secret" >&2 - exit 1 + pvc=$(preflight_existing_home) + checkpoint_state=$(printf '%s' "$pvc" | cut -f5) + checkpoint_at=$(printf '%s' "$pvc" | cut -f6) + if [[ ! "$checkpoint_at" =~ ^[0-9]{4}-(0[1-9]|1[0-2])-([0-2][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]Z$ ]]; then + echo "error: purge requires a valid non-secret checkpoint timestamp" >&2 + exit 2 fi - ;; - status) - "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" get pod "$POD" + if [ "$checkpoint_state" != clean ]; then + echo "error: purge requires agent-os.dev/checkpoint-state=clean on '$PVC'" >&2 + exit 2 + fi + evidence_file=${AGENT_OS_PURGE_EVIDENCE_FILE:-} + if [ -z "$evidence_file" ] && [ -n "${FM_HOME:-}" ]; then + evidence_file="$FM_HOME/data/crewmate-purge-evidence.log" + fi + if [ -z "$evidence_file" ]; then + echo "error: set AGENT_OS_PURGE_EVIDENCE_FILE or FM_HOME before purge" >&2 + exit 2 + fi + mkdir -p "$(dirname "$evidence_file")" + exec 3>>"$evidence_file" + record_purge purge-requested "$checkpoint_at" + kube delete pod "$POD" --ignore-not-found + kube delete pvc "$PVC" --ignore-not-found + record_purge purge-complete "$checkpoint_at" ;; delete) - "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" delete pod "$POD" --ignore-not-found - "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" delete pvc "$PVC" --ignore-not-found + echo "error: delete is ambiguous; use stop to preserve the home or purge --yes" >&2 + exit 2 ;; *) - echo "usage: $0 create|status|delete " >&2 + echo "usage: $0 create|status|stop|restart|purge|delete [--yes]" >&2 exit 2 ;; esac diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh index c106fad79..1299d2375 100755 --- a/bin/agent-os-kubernetes.sh +++ b/bin/agent-os-kubernetes.sh @@ -123,10 +123,72 @@ require_namespace_contract() { workload_state() { "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get statefulset agent-os-firstmate \ --ignore-not-found \ - -o 'jsonpath={.metadata.name}{"\t"}{.metadata.annotations.agent-os\.dev/rbac-mode}{"\t"}{.metadata.annotations.agent-os\.dev/cluster-rbac-cleanup}' + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.annotations.agent-os\.dev/rbac-mode}{"\t"}{.metadata.annotations.agent-os\.dev/cluster-rbac-cleanup}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}' +} + +require_workload_owned() { + local state=$1 required=${2:-required} identity + if [ -z "$state" ]; then + if [ "$required" = required ]; then + echo "error: agent-os-firstmate does not exist" >&2 + exit 2 + fi + return + fi + identity=$(printf '%s' "$state" | cut -f1,4,5) + if [ "$identity" != "agent-os-firstmate"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" ]; then + echo "error: StatefulSet 'agent-os-firstmate' does not have the exact installation identity" >&2 + exit 2 + fi +} + +rendered_resource_field() { + local file=$1 field=$2 + awk -v field="$field" '$1 == field ":" { print $2; exit }' "$file" +} + +live_resource_identity() { + local kind=$1 name=$2 + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get "$kind" "$name" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}' +} + +require_namespaced_resource_owned_or_absent() { + local kind=$1 name=$2 identity expected + identity=$(live_resource_identity "$kind" "$name") + expected="$name"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" + if [ -n "$identity" ] && [ "$identity" != "$expected" ]; then + echo "error: $kind '$name' does not have the exact Agent OS installation identity" >&2 + exit 2 + fi +} + +preflight_rendered_resources() { + local include_cluster=${1:-0} file kind name identity binding + while IFS= read -r file; do + [ -n "$file" ] || continue + kind=$(rendered_resource_field "$file" kind) + name=$(rendered_resource_field "$file" name) + case "$kind" in + Namespace) ;; + ClusterRoleBinding) + [ "$include_cluster" -eq 1 ] || continue + identity=$("$KUBECTL" --context "$CONTEXT" get clusterrolebinding "$name" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}') + binding="$name"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" + if [ -n "$identity" ] && [ "$identity" != "$binding" ]; then + echo "error: ClusterRoleBinding '$name' does not have the exact Agent OS installation identity" >&2 + exit 2 + fi + ;; + *) require_namespaced_resource_owned_or_absent "$kind" "$name" ;; + esac + done < <(find "$OUT" -type f -name '*.yaml' -print) } delete_namespace_rbac() { + require_namespaced_resource_owned_or_absent RoleBinding agent-os-firstmate-runtime + require_namespaced_resource_owned_or_absent Role agent-os-firstmate-runtime "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" delete rolebinding agent-os-firstmate-runtime --ignore-not-found "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" delete role agent-os-firstmate-runtime --ignore-not-found } @@ -137,14 +199,24 @@ apply_rendered() { } verify_desired_rbac() { - local role_name binding_identity expected_identity + local role_json binding_json if [ "$DESIRED_RBAC" = namespace ]; then - role_name=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get role agent-os-firstmate-runtime \ - -o 'jsonpath={.metadata.name}') - binding_identity=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get rolebinding agent-os-firstmate-runtime \ - -o 'jsonpath={.roleRef.kind}{"\t"}{.roleRef.name}{"\t"}{.subjects[0].kind}{"\t"}{.subjects[0].name}{"\t"}{.subjects[0].namespace}') - expected_identity="Role"$'\t'"agent-os-firstmate-runtime"$'\t'"ServiceAccount"$'\t'"agent-os-firstmate"$'\t'"$NAMESPACE" - if [ "$role_name" != agent-os-firstmate-runtime ] || [ "$binding_identity" != "$expected_identity" ]; then + if ! command -v jq >/dev/null 2>&1; then + echo "error: jq is required to verify namespace RBAC exactly" >&2 + exit 2 + fi + role_json=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get role agent-os-firstmate-runtime -o json) + binding_json=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get rolebinding agent-os-firstmate-runtime -o json) + if ! printf '%s' "$role_json" | jq -e ' + .metadata.name == "agent-os-firstmate-runtime" and + .rules == [ + {"apiGroups":[""],"resources":["pods","persistentvolumeclaims"],"verbs":["get","list","watch","create","delete","patch"]}, + {"apiGroups":[""],"resources":["pods/log","pods/exec"],"verbs":["get","list","watch","create","delete"]}, + {"apiGroups":["apps"],"resources":["statefulsets"],"verbs":["get","list","watch"]} + ]' >/dev/null || + ! printf '%s' "$binding_json" | jq -e --arg namespace "$NAMESPACE" ' + .roleRef == {"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"agent-os-firstmate-runtime"} and + .subjects == [{"kind":"ServiceAccount","name":"agent-os-firstmate","namespace":$namespace}]' >/dev/null; then echo "error: desired namespace RBAC did not verify after apply" >&2 exit 2 fi @@ -161,17 +233,25 @@ report_cluster_cleanup() { echo "required evidence: clusterrolebinding/agent-os-firstmate-$NAMESPACE absent" >&2 } -delete_rendered_namespaced_resources() { - local files file kind - files=$(find "$OUT" -type f -name '*.yaml' -print) +delete_rendered_kind() { + local desired_kind=$1 file kind while IFS= read -r file; do [ -n "$file" ] || continue - kind=$(awk '$1 == "kind:" { print $2; exit }' "$file") - case "$kind" in - Namespace|ClusterRoleBinding) ;; - *) "$KUBECTL" --context "$CONTEXT" delete --ignore-not-found -f "$file" ;; - esac - done <<< "$files" + kind=$(rendered_resource_field "$file" kind) + if [ "$kind" = "$desired_kind" ]; then + "$KUBECTL" --context "$CONTEXT" delete --ignore-not-found --wait=true -f "$file" + fi + done < <(find "$OUT" -type f -name '*.yaml' -print) +} + +delete_rendered_namespaced_resources() { + local kind + delete_rendered_kind StatefulSet + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" wait --for=delete pod/agent-os-firstmate-0 --timeout=180s + for kind in Service RoleBinding Role ServiceAccount; do + delete_rendered_kind "$kind" + done + delete_rendered_kind PersistentVolumeClaim } namespace_is_empty() { @@ -183,7 +263,7 @@ namespace_is_empty() { while IFS= read -r resource; do [ -n "$resource" ] || continue case "$resource" in - events|events.events.k8s.io|leases.coordination.k8s.io) continue ;; + events|events.events.k8s.io) continue ;; esac if ! objects=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get "$resource" -o name); then echo "error: could not inventory '$resource' in namespace '$NAMESPACE'" >&2 @@ -225,7 +305,7 @@ delete_owned_empty_namespace() { } cleanup_cluster_rbac() { - local binding identity workload + local binding identity workload mode cleanup_marker binding="agent-os-firstmate-$NAMESPACE" identity=$("$KUBECTL" --context "$CONTEXT" get clusterrolebinding "$binding" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}') @@ -234,12 +314,26 @@ cleanup_cluster_rbac() { exit 2 fi if [ -n "$identity" ]; then + workload= + if [ -n "$(namespace_name)" ]; then + workload=$(workload_state) + require_workload_owned "$workload" optional + fi + if [ -n "$workload" ]; then + mode=$(printf '%s' "$workload" | cut -f2) + cleanup_marker=$(printf '%s' "$workload" | cut -f3) + if [ "$mode" = cluster-admin ] || [ "$cleanup_marker" != required ]; then + echo "error: ClusterRoleBinding '$binding' is not proven stale by the workload cleanup marker" >&2 + exit 2 + fi + fi "$KUBECTL" --context "$CONTEXT" delete clusterrolebinding "$binding" "$KUBECTL" --context "$CONTEXT" wait --for=delete "clusterrolebinding/$binding" --timeout=60s fi if [ -n "$(namespace_name)" ]; then workload=$(workload_state) - if [ -n "$workload" ]; then + require_workload_owned "$workload" optional + if [ -n "$workload" ] && [ "$(printf '%s' "$workload" | cut -f3)" = required ]; then "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" annotate statefulset agent-os-firstmate \ agent-os.dev/cluster-rbac-cleanup- >/dev/null fi @@ -252,7 +346,10 @@ case "$COMMAND" in [ "$CONFIRMED" -eq 0 ] && [ "$DELETE_NAMESPACE" -eq 0 ] || usage render require_namespace_contract - if [ -n "$(workload_state)" ]; then + preflight_rendered_resources 1 + previous=$(workload_state) + require_workload_owned "$previous" optional + if [ -n "$previous" ]; then echo "error: agent-os-firstmate already exists; use upgrade" >&2 exit 2 fi @@ -262,11 +359,13 @@ case "$COMMAND" in [ "$CONFIRMED" -eq 0 ] && [ "$DELETE_NAMESPACE" -eq 0 ] || usage render require_namespace_contract + preflight_rendered_resources 1 previous=$(workload_state) if [ -z "$previous" ]; then echo "error: agent-os-firstmate does not exist; use install" >&2 exit 2 fi + require_workload_owned "$previous" previous_mode=$(printf '%s' "$previous" | cut -f2) previous_cleanup=$(printf '%s' "$previous" | cut -f3) cleanup_required=0 @@ -292,6 +391,10 @@ case "$COMMAND" in ;; rollback) [ "$CONFIRMED" -eq 0 ] && [ "$DELETE_NAMESPACE" -eq 0 ] || usage + render + require_namespace_contract + previous=$(workload_state) + require_workload_owned "$previous" "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout undo statefulset/agent-os-firstmate "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s ;; @@ -306,7 +409,9 @@ case "$COMMAND" in fi render require_namespace_contract + preflight_rendered_resources previous=$(workload_state) + require_workload_owned "$previous" optional previous_mode=$(printf '%s' "$previous" | cut -f2) previous_cleanup=$(printf '%s' "$previous" | cut -f3) delete_rendered_namespaced_resources @@ -315,7 +420,8 @@ case "$COMMAND" in delete_owned_empty_namespace fi if [ "$DESIRED_RBAC" = cluster-admin ] || [ "$previous_mode" = cluster-admin ] || \ - [ "$previous_cleanup" = required ] || { [ -n "$previous" ] && [ -z "$previous_mode" ]; }; then + [ "$previous_cleanup" = required ] || [ -z "$previous" ] || \ + { [ -n "$previous" ] && [ -z "$previous_mode" ]; }; then report_cluster_cleanup exit 3 fi diff --git a/bin/agent-os-local.sh b/bin/agent-os-local.sh index eb4ba359d..f1bf9d6f0 100755 --- a/bin/agent-os-local.sh +++ b/bin/agent-os-local.sh @@ -12,6 +12,13 @@ IMAGE_IS_OVERRIDE= COMMAND=${1:-} PROFILE="$ROOT/deploy/orbstack/inputs.yaml" +case "$NAMESPACE" in + ''|*[!a-z0-9.-]*|[.-]*|*[-.]) + echo "error: AGENT_OS_NAMESPACE must be a valid Kubernetes namespace" >&2 + exit 2 + ;; +esac + if [ "$CONTEXT" != orbstack ] && [ "${AGENT_OS_ALLOW_NON_ORBSTACK:-0}" != 1 ]; then echo "error: refusing Kubernetes context '$CONTEXT'; set AGENT_OS_ALLOW_NON_ORBSTACK=1 to opt in" >&2 exit 2 @@ -24,12 +31,20 @@ local_image_tag() { } render_profile() { - local image=$1 inputs + local image=$1 inputs lifecycle inputs=$(mktemp) trap 'rm -f "$inputs"' RETURN - sed "s|^image: .*|image: $image|" "$PROFILE" > "$inputs" + awk -v image="$image" -v namespace="$NAMESPACE" ' + $1 == "image:" { print "image: " image; next } + $1 == "namespace:" { print "namespace: " namespace; next } + { print } + ' "$PROFILE" > "$inputs" + lifecycle=install + if [ -n "$(kubectl --context "$CONTEXT" -n "$NAMESPACE" get statefulset agent-os-firstmate --ignore-not-found -o name)" ]; then + lifecycle=upgrade + fi AGENT_OS_CONTEXT="$CONTEXT" AGENT_OS_NAMESPACE="$NAMESPACE" AGENT_OS_INPUTS="$inputs" \ - "$ROOT/bin/agent-os-kubernetes.sh" install + "$ROOT/bin/agent-os-kubernetes.sh" "$lifecycle" } cd "$ROOT" diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 4cfd266ca..10ac7100d 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -102,13 +102,18 @@ It is not a second public package and is not a Marketplace product. Each runtime mate has its own PVC and no ambient Kubernetes ServiceAccount token. Creating one requires an explicitly authorized, pre-created Secret in the same namespace with an `auth.json` key. Pass only that Secret's name through `AGENT_OS_AI_SECRET`; the helper never discovers or copies the primary credential. -The authorization file is mounted read-only at `/home/agent/.pi/agent/auth.json`. +The Secret projects only `auth.json` into the read-only `/home/agent/.pi/agent` directory without a `subPath` mount. A missing Secret or key keeps the Pod unready, so creation fails closed and removes the non-running Pod while retaining its PVC for an authorized retry. ```sh AGENT_OS_AI_SECRET=scout-1-ai-auth bin/agent-os-crewmate.sh create scout-1 ``` +Use `stop` for a Pod-only shutdown and `restart` for a Pod-only replacement after an approved Secret rotation. +The ambiguous `delete` operation is rejected. +Only `purge --yes` removes the PVC, and it requires exact ownership, a clean checkpoint annotation, and a non-secret evidence file. +The full rotation, urgent-revocation, checkpoint, and purge procedure is owned by the `kubernetes-fleet` operating skill. + The existing [same-Pod](evidence/2026-07-13-same-pod-firstmate.md) and [separate-Pod recovery](evidence/2026-07-13-separate-pod-recovery.md) records remain local lifecycle evidence. They do not substitute for a clean published-image installation. diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 6e8607ac3..1387002aa 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -98,6 +98,47 @@ assert_grep 'https://github.com/akua-dev/akua/tree/v0.8.25' "$ROOT/THIRD_PARTY_N "Akua's exact source must be named" assert_grep 'https://github.com/derailed/k9s/tree/v0.51.0' "$ROOT/THIRD_PARTY_NOTICES.md" \ "K9s's exact source must be named" +assert_grep '1f328c5e9dd683d0c5e69f3d7d58f8371278dec2/LICENSE' "$ROOT/Dockerfile" \ + "the image must bundle kubectl's license from the immutable Kubernetes source commit" +assert_grep 'b300f2ec7ec9dc9addc39b2ad88c54097ded7ca0/LICENSE' "$ROOT/Dockerfile" \ + "the image must bundle GitHub CLI's license from its immutable source commit" +assert_grep '0d9b296af33f2b851fcbf4df3e9ec89751734ba4/LICENSE.md' "$ROOT/Dockerfile" \ + "the image must bundle Bun's complete license from its immutable source commit" +assert_grep '68fa3d2556542add76bf80255787b8625a5041a6/LICENSE' "$ROOT/Dockerfile" \ + "the image must bundle Treehouse's license from its immutable source commit" +assert_grep 'dc5a80059d3c0f1abbf28f20f43d994b8399bee6/LICENSE' "$ROOT/Dockerfile" \ + "the image must bundle no-mistakes' license from its immutable source commit" +assert_grep '/usr/share/licenses/kubectl/LICENSE' "$ROOT/Dockerfile" \ + "kubectl's license must be installed in the image license bundle" +assert_grep '/usr/share/licenses/gh/LICENSE' "$ROOT/Dockerfile" \ + "GitHub CLI's license must be installed in the image license bundle" +assert_grep '/usr/share/licenses/bun/LICENSE.md' "$ROOT/Dockerfile" \ + "Bun's complete license must be installed in the image license bundle" +assert_grep '/usr/share/licenses/treehouse/LICENSE' "$ROOT/Dockerfile" \ + "Treehouse's license must be installed in the image license bundle" +assert_grep '/usr/share/licenses/no-mistakes/LICENSE' "$ROOT/Dockerfile" \ + "no-mistakes' license must be installed in the image license bundle" +assert_grep '## kubectl' "$ROOT/THIRD_PARTY_NOTICES.md" "third-party notices must cover kubectl" +assert_grep '## GitHub CLI' "$ROOT/THIRD_PARTY_NOTICES.md" "third-party notices must cover GitHub CLI" +assert_grep '## Bun' "$ROOT/THIRD_PARTY_NOTICES.md" "third-party notices must cover Bun" +assert_grep '## Treehouse' "$ROOT/THIRD_PARTY_NOTICES.md" "third-party notices must cover Treehouse" +assert_grep '## no-mistakes' "$ROOT/THIRD_PARTY_NOTICES.md" "third-party notices must cover no-mistakes" +assert_grep 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' "$ROOT/.github/workflows/ci.yml" \ + "CI checkout actions must be pinned to the reviewed full SHA" +assert_grep 'oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6' "$ROOT/.github/workflows/ci.yml" \ + "CI must pin Bun setup to a reviewed full SHA" +assert_grep 'bun-version-file: tools/agent-os/.bun-version' "$ROOT/.github/workflows/ci.yml" \ + "CI must use the repository-pinned Bun version" +assert_grep 'bun run check' "$ROOT/.github/workflows/ci.yml" \ + "CI behavior tests must execute the Agent OS tool check" +assert_grep "mkdir -p \"\$RUNNER_TEMP/bin\"" "$ROOT/.github/workflows/ci.yml" \ + "CI must create its local verified-tool destination before install" +assert_grep 'bc57afbffe7e18aacd2146e2cd67151c56e7a3c279fe659312ff7ffb359cd03a' "$ROOT/.github/workflows/ci.yml" \ + "CI must authenticate the x86_64 Akua release artifact" +assert_grep '3a3c6bae72764cbd85a6e4e0877a05e5def8f7aeee8563b7918099214a1a313a' "$ROOT/.github/workflows/ci.yml" \ + "CI must authenticate the aarch64 Akua release artifact" +assert_no_grep 'https://cli.akua.dev/install' "$ROOT/.github/workflows/ci.yml" \ + "CI must not execute the mutable remote Akua installer" assert_grep 'ghcr.io/akua-dev/agent-os' "$ROOT/.github/workflows/agent-os-image.yml" \ "release workflow must publish the image expected by the portable package" assert_grep 'linux/amd64,linux/arm64' "$ROOT/.github/workflows/agent-os-image.yml" \ diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index 5afeae2b5..c4110cb0b 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -10,6 +10,7 @@ TMP=$(fm_test_tmproot agent-os-kubernetes) FAKEBIN=$(fm_fakebin "$TMP") CALLS="$TMP/calls.log" STDIN_LOG="$TMP/stdin.yaml" +PURGE_EVIDENCE="$TMP/purge-evidence.log" PROFILE="$ROOT/deploy/orbstack/inputs.yaml" PROFILE_OUT="$TMP/orbstack-rendered" @@ -51,6 +52,24 @@ if [ "${AGENT_OS_TEST_FAIL_ANNOTATE:-0}" = 1 ] && [[ " $* " = *" annotate statef exit 1 fi case " $* " in + *" get pod agent-os-crewmate-"*" --ignore-not-found -o jsonpath="*) + id=${AGENT_OS_TEST_CREWMATE_ID:-scout-1} + case "${AGENT_OS_TEST_POD_STATE:-absent}" in + absent) ;; + owned) printf 'agent-os-crewmate-%s\tagent-os\t%s\tagent-os-firstmate:agent-os-demo' "$id" "$id" ;; + foreign) printf 'agent-os-crewmate-%s\tother\tother\tother-installation' "$id" ;; + esac + ;; + *" get pvc agent-os-crewmate-"*" --ignore-not-found -o jsonpath="*) + id=${AGENT_OS_TEST_CREWMATE_ID:-scout-1} + case "${AGENT_OS_TEST_PVC_STATE:-absent}" in + absent) ;; + owned) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tpending\t' "$id" "$id" ;; + clean) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tclean\t2026-07-13T12:00:00Z' "$id" "$id" ;; + invalid-checkpoint) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tclean\t2026' "$id" "$id" ;; + foreign) printf 'agent-os-crewmate-%s-home\tother\tother\tother-installation\tclean\t2026-07-13T12:00:00Z' "$id" ;; + esac + ;; *" get namespace "*" --ignore-not-found -o name "*) case "${AGENT_OS_TEST_NAMESPACE_STATE:-absent}" in absent) ;; @@ -67,13 +86,49 @@ case " $* " in *" get statefulset agent-os-firstmate --ignore-not-found -o jsonpath="*) case "${AGENT_OS_TEST_WORKLOAD_STATE:-absent}" in absent) ;; - namespace) printf 'agent-os-firstmate\tnamespace\t' ;; - cluster-admin) printf 'agent-os-firstmate\tcluster-admin\t' ;; - none) printf 'agent-os-firstmate\tnone\t' ;; - pending) printf 'agent-os-firstmate\tnamespace\trequired' ;; - unknown) printf 'agent-os-firstmate\t\t' ;; + namespace) printf 'agent-os-firstmate\tnamespace\t\tagent-os\tagent-os-firstmate:portable-agent-os' ;; + cluster-admin) printf 'agent-os-firstmate\tcluster-admin\t\tagent-os\tagent-os-firstmate:portable-agent-os' ;; + none) printf 'agent-os-firstmate\tnone\t\tagent-os\tagent-os-firstmate:portable-agent-os' ;; + pending) printf 'agent-os-firstmate\tnamespace\trequired\tagent-os\tagent-os-firstmate:portable-agent-os' ;; + unknown) printf 'agent-os-firstmate\t\t\tagent-os\tagent-os-firstmate:portable-agent-os' ;; + foreign) printf 'agent-os-firstmate\tnamespace\t\tother\tother-installation' ;; esac ;; + *" get StatefulSet agent-os-firstmate --ignore-not-found -o jsonpath="*|\ + *" get ServiceAccount agent-os-firstmate --ignore-not-found -o jsonpath="*|\ + *" get PersistentVolumeClaim agent-os-firstmate-home --ignore-not-found -o jsonpath="*|\ + *" get Service agent-os-firstmate --ignore-not-found -o jsonpath="*|\ + *" get Role agent-os-firstmate-runtime --ignore-not-found -o jsonpath="*|\ + *" get RoleBinding agent-os-firstmate-runtime --ignore-not-found -o jsonpath="*) + name='' + case " $* " in + *" StatefulSet "*) name=agent-os-firstmate ;; + *" ServiceAccount "*) name=agent-os-firstmate ;; + *" PersistentVolumeClaim "*) name=agent-os-firstmate-home ;; + *" Service "*) name=agent-os-firstmate ;; + *" RoleBinding "*) name=agent-os-firstmate-runtime ;; + *" Role "*) name=agent-os-firstmate-runtime ;; + esac + case "${AGENT_OS_TEST_RESOURCE_STATE:-absent}" in + absent) ;; + owned) printf '%s\tagent-os\tagent-os-firstmate:portable-agent-os' "$name" ;; + foreign) printf '%s\tother\tother-installation' "$name" ;; + esac + ;; + *" get role agent-os-firstmate-runtime -o json "*) + if [ "${AGENT_OS_TEST_RBAC_STATE:-exact}" = bad-rules ]; then + printf '%s\n' '{"metadata":{"name":"agent-os-firstmate-runtime"},"rules":[]}' + else + printf '%s\n' '{"metadata":{"name":"agent-os-firstmate-runtime"},"rules":[{"apiGroups":[""],"resources":["pods","persistentvolumeclaims"],"verbs":["get","list","watch","create","delete","patch"]},{"apiGroups":[""],"resources":["pods/log","pods/exec"],"verbs":["get","list","watch","create","delete"]},{"apiGroups":["apps"],"resources":["statefulsets"],"verbs":["get","list","watch"]}]}' + fi + ;; + *" get rolebinding agent-os-firstmate-runtime -o json "*) + if [ "${AGENT_OS_TEST_RBAC_STATE:-exact}" = extra-subject ]; then + printf '%s\n' '{"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"agent-os-firstmate-runtime"},"subjects":[{"kind":"ServiceAccount","name":"agent-os-firstmate","namespace":"portable-agent-os"},{"kind":"ServiceAccount","name":"foreign","namespace":"portable-agent-os"}]}' + else + printf '%s\n' '{"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"agent-os-firstmate-runtime"},"subjects":[{"kind":"ServiceAccount","name":"agent-os-firstmate","namespace":"portable-agent-os"}]}' + fi + ;; *" get role agent-os-firstmate-runtime -o jsonpath="*) printf 'agent-os-firstmate-runtime' ;; @@ -92,13 +147,16 @@ case " $* " in esac ;; *" api-resources --verbs=list --namespaced -o name "*) - printf '%s\n' pods serviceaccounts configmaps + printf '%s\n' pods serviceaccounts configmaps leases.coordination.k8s.io ;; *" get pods -o name "*) [ -z "${AGENT_OS_TEST_FOREIGN_RESOURCE:-}" ] || printf 'pod/%s\n' "$AGENT_OS_TEST_FOREIGN_RESOURCE" ;; *" get serviceaccounts -o name "*) printf '%s\n' serviceaccount/default ;; *" get configmaps -o name "*) printf '%s\n' configmap/kube-root-ca.crt ;; + *" get leases.coordination.k8s.io -o name "*) + [ -z "${AGENT_OS_TEST_FOREIGN_LEASE:-}" ] || printf 'lease/%s\n' "$AGENT_OS_TEST_FOREIGN_LEASE" + ;; esac SH chmod +x "$FAKEBIN/kubectl" @@ -106,7 +164,8 @@ chmod +x "$FAKEBIN/kubectl" run_launcher() { PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_STDIN_LOG="$STDIN_LOG" \ AGENT_OS_IN_CLUSTER=1 AGENT_OS_NAMESPACE=agent-os-demo AGENT_OS_IMAGE=agent-os:local-test \ - AGENT_OS_IMAGE_PULL_POLICY=Never AGENT_OS_AI_SECRET=scout-1-ai-auth "$LAUNCHER" "$@" + AGENT_OS_IMAGE_PULL_POLICY=Never AGENT_OS_AI_SECRET=scout-1-ai-auth \ + AGENT_OS_PURGE_EVIDENCE_FILE="$PURGE_EVIDENCE" "$LAUNCHER" "$@" } : > "$CALLS" @@ -115,15 +174,25 @@ grep -Fqx 'kubectl -n agent-os-demo apply -f -' "$CALLS" || fail "create must ap [ "$(grep -Fc 'kind: PersistentVolumeClaim' "$STDIN_LOG")" -eq 1 ] || fail "create must emit one PVC" [ "$(grep -Fc 'kind: Pod' "$STDIN_LOG")" -eq 1 ] || fail "create must emit one Pod" assert_grep 'agent-os.dev/crewmate: scout-1' "$STDIN_LOG" "child resources need the stable crewmate label" +assert_grep 'app.kubernetes.io/managed-by: agent-os' "$STDIN_LOG" \ + "child resources need the exact Agent OS ownership label" +assert_grep 'agent-os.dev/installation-id: agent-os-firstmate:agent-os-demo' "$STDIN_LOG" \ + "child resources need the exact installation identity" assert_grep 'automountServiceAccountToken: false' "$STDIN_LOG" "children must not receive Kubernetes credentials" assert_grep 'claimName: agent-os-crewmate-scout-1-home' "$STDIN_LOG" "child work must use its own PVC" assert_no_grep 'hostUsers: false' "$STDIN_LOG" "OrbStack children must not request unsupported Pod user namespaces" assert_grep 'runAsUser: 0' "$STDIN_LOG" "children must run as container root" assert_grep 'name: agent-os-init' "$STDIN_LOG" "children must seed persistent tools" assert_grep 'mountPath: /usr/local' "$STDIN_LOG" "children must persist /usr/local" -assert_grep 'mountPath: /home/agent/.pi/agent/auth.json' "$STDIN_LOG" \ - "children must mount only the explicitly granted AI authorization file" -assert_grep 'secretName: scout-1-ai-auth' "$STDIN_LOG" \ +assert_grep 'mountPath: /home/agent/.pi/agent' "$STDIN_LOG" \ + "children must mount the explicitly granted AI authorization directory" +assert_no_grep 'subPath: auth.json' "$STDIN_LOG" \ + "projected AI authorization must support Secret rotation without a subPath mount" +assert_grep 'path: auth.json' "$STDIN_LOG" \ + "the projected authorization directory must expose only the approved auth.json key" +assert_grep 'optional: false' "$STDIN_LOG" \ + "a missing Secret or auth.json key must keep the crewmate Pod unready" +assert_grep 'name: scout-1-ai-auth' "$STDIN_LOG" \ "children must reference the explicitly selected namespace-local Secret" assert_grep 'readOnly: true' "$STDIN_LOG" "child AI authorization must be read-only" assert_grep 'readinessProbe:' "$STDIN_LOG" "child readiness must wait for Herdr health" @@ -133,6 +202,15 @@ grep -Fqx 'kubectl -n agent-os-demo wait --for=condition=Ready pod/agent-os-crew fail "create must fail when the authorized Secret cannot produce a ready Pod" pass "crewmate create emits one isolated Pod and PVC" +: > "$CALLS" +if AGENT_OS_TEST_POD_STATE=foreign run_launcher create scout-1 >/dev/null 2>&1; then + fail "crewmate create must reject a same-name foreign Pod" +fi +if grep -F ' apply -f -' "$CALLS" >/dev/null; then + fail "crewmate create must reject foreign ownership before apply" +fi +pass "crewmate create refuses foreign deterministic-name resources" + : > "$CALLS" if PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_STDIN_LOG="$STDIN_LOG" \ AGENT_OS_IN_CLUSTER=1 AGENT_OS_NAMESPACE=agent-os-demo AGENT_OS_IMAGE=agent-os:local-test \ @@ -154,12 +232,67 @@ fi pass "crewmate create fails closed while retaining its persistent home" : > "$CALLS" -run_launcher delete scout-1 +AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=owned run_launcher stop scout-1 grep -Fqx 'kubectl -n agent-os-demo delete pod agent-os-crewmate-scout-1 --ignore-not-found' "$CALLS" || \ - fail "delete must target the crewmate Pod explicitly" + fail "stop must target the exactly owned crewmate Pod" +if grep -F 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" >/dev/null; then + fail "stop must preserve the crewmate persistent home" +fi +pass "crewmate stop preserves its persistent home" + +: > "$CALLS" +AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=owned run_launcher restart scout-1 +delete_line=$(grep -Fn 'delete pod agent-os-crewmate-scout-1' "$CALLS" | head -n 1 | cut -d: -f1) +apply_line=$(grep -Fn 'apply -f -' "$CALLS" | head -n 1 | cut -d: -f1) +[ -n "$delete_line" ] && [ -n "$apply_line" ] && [ "$delete_line" -lt "$apply_line" ] || \ + fail "restart must replace the owned Pod on its retained PVC" +if grep -F 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" >/dev/null; then + fail "restart must preserve the crewmate persistent home" +fi +pass "crewmate restart replaces only its Pod" + +: > "$CALLS" +delete_out='' +delete_rc=0 +delete_out=$(run_launcher delete scout-1 2>&1) || delete_rc=$? +[ "$delete_rc" -eq 2 ] || fail "legacy delete must refuse destructive ambiguity: $delete_out" +[ ! -s "$CALLS" ] || fail "legacy delete must not mutate Pod or PVC state" +pass "legacy delete never silently destroys persistent work" + +: > "$CALLS" +purge_out='' +purge_rc=0 +purge_out=$(run_launcher purge scout-1 2>&1) || purge_rc=$? +[ "$purge_rc" -eq 2 ] || fail "purge without --yes must exit 2: $purge_out" +assert_contains "$purge_out" 'agent-os-crewmate-scout-1-home' \ + "unconfirmed purge must display the exact persistent target" +[ ! -s "$CALLS" ] || fail "unconfirmed purge must not query or mutate cluster state" + +: > "$CALLS" +if AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=owned run_launcher purge scout-1 --yes >/dev/null 2>&1; then + fail "purge must reject a persistent home without a clean checkpoint" +fi +if grep -F 'delete pvc' "$CALLS" >/dev/null; then + fail "purge must not delete a home without a clean checkpoint" +fi + +: > "$CALLS" +if AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=invalid-checkpoint run_launcher purge scout-1 --yes >/dev/null 2>&1; then + fail "purge must reject a malformed checkpoint timestamp" +fi +assert_no_grep 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" \ + "purge must not delete a home with malformed checkpoint evidence" + +: > "$CALLS" +: > "$PURGE_EVIDENCE" +AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=clean run_launcher purge scout-1 --yes +grep -Fqx 'kubectl -n agent-os-demo delete pod agent-os-crewmate-scout-1 --ignore-not-found' "$CALLS" || \ + fail "purge must delete the exactly owned Pod" grep -Fqx 'kubectl -n agent-os-demo delete pvc agent-os-crewmate-scout-1-home --ignore-not-found' "$CALLS" || \ - fail "delete must target the crewmate PVC explicitly" -pass "crewmate delete removes the Pod and PVC explicitly" + fail "purge must delete the exactly owned persistent home" +assert_grep 'purge-complete' "$PURGE_EVIDENCE" "purge must record non-secret completion evidence" +assert_no_grep 'scout-1-ai-auth' "$PURGE_EVIDENCE" "purge evidence must never contain credential references" +pass "crewmate purge requires confirmation and a clean checkpoint" if run_launcher create 'Bad_ID' >/dev/null 2>&1; then fail "invalid Kubernetes crewmate IDs must be rejected" @@ -174,7 +307,8 @@ pass "launcher refuses an ambient host Kubernetes context" : > "$CALLS" PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_STDIN_LOG="$STDIN_LOG" \ - AGENT_OS_CONTEXT=orbstack AGENT_OS_NAMESPACE=agent-os-demo AGENT_OS_IMAGE=agent-os:local-test "$LAUNCHER" status scout-1 + AGENT_OS_CONTEXT=orbstack AGENT_OS_NAMESPACE=agent-os-demo AGENT_OS_IMAGE=agent-os:local-test \ + AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=owned "$LAUNCHER" status scout-1 grep -Fqx 'kubectl --context orbstack -n agent-os-demo get pod agent-os-crewmate-scout-1' "$CALLS" || \ fail "host status must pin the selected context" pass "host launcher calls require and pin an explicit context" @@ -209,15 +343,43 @@ rbac=$(awk '/^rbac:/{print $2}' "$inputs") namespace=$(awk '/^namespace:/{print $2}' "$inputs") create_namespace=$(awk '/^createNamespace:/{print $2}' "$inputs") [ -n "$create_namespace" ] || create_namespace=true +cat > "$out/00-pvc.yaml" < "$out/statefulset.yaml" < "$out/$file.yaml" < "$out/namespace.yaml" < "$out/role.yaml" - printf 'apiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n namespace: %s\n' "$namespace" \ - > "$out/rolebinding.yaml" + for resource in Role RoleBinding; do + file=$(printf '%s' "$resource" | tr '[:upper:]' '[:lower:]') + cat > "$out/$file.yaml" < "$out/clusterrolebinding.yaml" < "$CALLS" +foreign_resource_install_rc=0 +AGENT_OS_TEST_NAMESPACE_STATE=unowned AGENT_OS_TEST_RESOURCE_STATE=foreign \ + PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS="$UNOWNED_INPUTS" \ + AGENT_OS_TEST_NAMESPACE=portable-agent-os AGENT_OS_TEST_WORKLOAD_STATE=absent \ + AGENT_OS_CONTEXT=kind-agent-os AGENT_OS_NAMESPACE=portable-agent-os \ + "$GENERIC" install >/dev/null 2>&1 || foreign_resource_install_rc=$? +[ "$foreign_resource_install_rc" -eq 2 ] || \ + fail "install must reject same-name foreign namespaced resources" +if grep -F ' apply -f ' "$CALLS" >/dev/null; then + fail "foreign namespaced ownership must fail before package apply" +fi +pass "install refuses foreign deterministic-name resources" + : > "$CALLS" owned_unmanaged_out='' owned_unmanaged_rc=0 @@ -364,6 +550,22 @@ if grep -F 'kubectl --context kind-agent-os apply -f ' "$CALLS" >/dev/null; then fi pass "cluster-admin downgrade records cleanup state before mutation" +: > "$CALLS" +tainted_rbac_rc=0 +AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=cluster-admin \ + AGENT_OS_TEST_RBAC_STATE=extra-subject run_generic upgrade >/dev/null 2>&1 || tainted_rbac_rc=$? +[ "$tainted_rbac_rc" -eq 2 ] || \ + fail "downgrade must reject a RoleBinding with extra subjects, got $tainted_rbac_rc" +pass "replacement RBAC verification requires exact subject cardinality" + +: > "$CALLS" +bad_rules_rc=0 +AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=cluster-admin \ + AGENT_OS_TEST_RBAC_STATE=bad-rules run_generic upgrade >/dev/null 2>&1 || bad_rules_rc=$? +[ "$bad_rules_rc" -eq 2 ] || \ + fail "downgrade must reject a Role without the exact runtime rules, got $bad_rules_rc" +pass "replacement RBAC verification requires exact rules" + : > "$CALLS" if AGENT_OS_TEST_CLUSTER_RBAC_STATE=owned run_generic cleanup-cluster-rbac >/dev/null 2>&1; then fail "privileged cluster RBAC cleanup must require --yes" @@ -381,6 +583,18 @@ grep -Fqx 'kubectl --context kind-agent-os wait --for=delete clusterrolebinding/ fail "privileged cleanup must produce deletion evidence for the exact binding" pass "privileged cleanup verifies ownership and deletes one exact binding" +: > "$CALLS" +active_cleanup_rc=0 +AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=cluster-admin \ + AGENT_OS_TEST_CLUSTER_RBAC_STATE=owned run_generic cleanup-cluster-rbac --yes >/dev/null 2>&1 || \ + active_cleanup_rc=$? +[ "$active_cleanup_rc" -eq 2 ] || \ + fail "privileged cleanup must refuse an active cluster-admin grant, got $active_cleanup_rc" +if grep -F 'delete clusterrolebinding' "$CALLS" >/dev/null; then + fail "cleanup must not revoke an active cluster-admin installation" +fi +pass "privileged cleanup refuses active cluster-admin authority" + : > "$CALLS" absent_binding_out=$(AGENT_OS_TEST_NAMESPACE_STATE=absent AGENT_OS_TEST_WORKLOAD_STATE=absent \ AGENT_OS_TEST_CLUSTER_RBAC_STATE=absent run_generic cleanup-cluster-rbac --yes) @@ -423,10 +637,23 @@ fi pass "cluster-admin upgrade reconciles namespaced authority after apply" : > "$CALLS" -run_generic rollback +AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace run_generic rollback grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os rollout undo statefulset/agent-os-firstmate' "$CALLS" || \ fail "generic rollback must target only the Firstmate StatefulSet" -pass "generic rollback remains StatefulSet-scoped" +grep -Fq 'akua render --no-agent-mode' "$CALLS" || \ + fail "rollback must derive its namespace and identity from the current package render" +pass "generic rollback verifies its rendered installation identity" + +: > "$CALLS" +foreign_rollback_rc=0 +AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=foreign AGENT_OS_TEST_WORKLOAD_STATE=foreign \ + run_generic rollback >/dev/null 2>&1 || foreign_rollback_rc=$? +[ "$foreign_rollback_rc" -eq 2 ] || fail "rollback must refuse a foreign same-name StatefulSet" +if grep -F 'rollout undo' "$CALLS" >/dev/null; then + fail "rollback must verify ownership before mutation" +fi +pass "rollback refuses foreign workload ownership" : > "$CALLS" if run_generic uninstall >/dev/null 2>&1; then @@ -444,8 +671,23 @@ grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete rolebindi fail "uninstall must remove namespace runtime binding regardless of current inputs" grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete role agent-os-firstmate-runtime --ignore-not-found' "$CALLS" || \ fail "uninstall must remove namespace runtime Role regardless of current inputs" +stateful_delete_line=$(grep -Fn '/statefulset.yaml' "$CALLS" | grep ' delete ' | head -n 1 | cut -d: -f1) +pvc_delete_line=$(grep -Fn '/00-pvc.yaml' "$CALLS" | grep ' delete ' | head -n 1 | cut -d: -f1) +[ -n "$stateful_delete_line" ] && [ -n "$pvc_delete_line" ] && \ + [ "$stateful_delete_line" -lt "$pvc_delete_line" ] || \ + fail "uninstall must delete the StatefulSet before waiting on PVC deletion" pass "routine uninstall removes namespaced resources without cluster-wide authority" +: > "$CALLS" +foreign_uninstall_rc=0 +AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=namespace \ + AGENT_OS_TEST_RESOURCE_STATE=foreign run_generic uninstall --yes >/dev/null 2>&1 || foreign_uninstall_rc=$? +[ "$foreign_uninstall_rc" -eq 2 ] || fail "uninstall must reject same-name foreign namespaced resources" +if grep -F ' delete --ignore-not-found -f ' "$CALLS" >/dev/null; then + fail "uninstall must preflight all namespaced ownership before deletion" +fi +pass "uninstall refuses foreign deterministic-name resources" + : > "$CALLS" uninstall_residue_out='' uninstall_residue_rc=0 @@ -460,6 +702,17 @@ assert_contains "$uninstall_residue_out" 'cleanup-cluster-rbac --yes' \ "cluster-admin uninstall must print the exact privileged cleanup command" pass "routine uninstall reports cluster-scoped residue for separate cleanup" +: > "$CALLS" +retry_residue_out='' +retry_residue_rc=0 +retry_residue_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=absent \ + run_generic uninstall --yes 2>&1) || retry_residue_rc=$? +[ "$retry_residue_rc" -eq 3 ] || \ + fail "uninstall retry without workload history must require cluster absence evidence: $retry_residue_out" +assert_contains "$retry_residue_out" 'cleanup-cluster-rbac --yes' \ + "history-free uninstall retry must print the privileged absence-evidence command" +pass "uninstall retry cannot lose cluster-RBAC residue state" + : > "$CALLS" AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=namespace \ run_generic uninstall --yes --delete-namespace @@ -481,3 +734,14 @@ if grep -F 'delete namespace portable-agent-os' "$CALLS" >/dev/null; then fail "namespace deletion must fail closed when foreign resources remain" fi pass "optional namespace deletion refuses foreign resources" + +: > "$CALLS" +foreign_lease_rc=0 +AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=namespace \ + AGENT_OS_TEST_FOREIGN_LEASE=foreign-leader run_generic uninstall --yes --delete-namespace >/dev/null 2>&1 || \ + foreign_lease_rc=$? +[ "$foreign_lease_rc" -eq 2 ] || fail "foreign Lease must block namespace deletion" +if grep -F 'delete namespace portable-agent-os' "$CALLS" >/dev/null; then + fail "namespace deletion must include Leases in its foreign-resource proof" +fi +pass "optional namespace deletion refuses foreign Leases" diff --git a/tests/agent-os-local.test.sh b/tests/agent-os-local.test.sh index 3d66f8066..1ab9a33be 100755 --- a/tests/agent-os-local.test.sh +++ b/tests/agent-os-local.test.sh @@ -32,7 +32,34 @@ if [ "${1:-}" = image ] && [ "${2:-}" = inspect ]; then fi SH chmod +x "$FAKEBIN/docker" -make_fake kubectl +cat > "$FAKEBIN/kubectl" <<'SH' +#!/usr/bin/env bash +printf 'kubectl' >> "$AGENT_OS_TEST_LOG" +printf ' %s' "$@" >> "$AGENT_OS_TEST_LOG" +printf '\n' >> "$AGENT_OS_TEST_LOG" +namespace=${AGENT_OS_NAMESPACE:-agent-os-demo} +if [[ " $* " = *" get statefulset agent-os-firstmate --ignore-not-found -o name "* ]] && \ + [ "${AGENT_OS_TEST_LOCAL_WORKLOAD:-absent}" = present ]; then + printf 'statefulset.apps/agent-os-firstmate\n' +fi +if [[ " $* " = *" get namespace "*" --ignore-not-found -o name "* ]] && \ + [ "${AGENT_OS_TEST_LOCAL_WORKLOAD:-absent}" = present ]; then + printf 'namespace/%s\n' "$namespace" +fi +if [[ " $* " = *" get namespace "*" -o jsonpath="* ]] && \ + [ "${AGENT_OS_TEST_LOCAL_WORKLOAD:-absent}" = present ]; then + printf 'agent-os\tagent-os-firstmate:%s' "$namespace" +fi +if [[ " $* " = *" get statefulset agent-os-firstmate --ignore-not-found -o jsonpath="* ]] && \ + [ "${AGENT_OS_TEST_LOCAL_WORKLOAD:-absent}" = present ]; then + printf 'agent-os-firstmate\tcluster-admin\t\tagent-os\tagent-os-firstmate:%s' "$namespace" +fi +if [[ " $* " = *" get clusterrolebinding agent-os-firstmate-"*" --ignore-not-found -o jsonpath="* ]] && \ + [ "${AGENT_OS_TEST_LOCAL_WORKLOAD:-absent}" = present ]; then + printf 'agent-os-firstmate-%s\tagent-os\tagent-os-firstmate:%s' "$namespace" "$namespace" +fi +SH +chmod +x "$FAKEBIN/kubectl" make_fake orbctl cat > "$FAKEBIN/akua" <<'SH' #!/usr/bin/env bash @@ -49,28 +76,38 @@ while [ "$#" -gt 0 ]; do esac done printf 'akua-input-image %s\n' "$(awk '/^image:/{print $2}' "$inputs")" >> "$AGENT_OS_TEST_LOG" +namespace=$(awk '/^namespace:/{print $2}' "$inputs") +printf 'akua-input-namespace %s\n' "$namespace" >> "$AGENT_OS_TEST_LOG" mkdir -p "$out" cat > "$out/statefulset.yaml" < "$out/rendered.yaml" -cat > "$out/namespace.yaml" <<'YAML' +printf 'apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: agent-os-firstmate-state\n namespace: %s\n' "$namespace" > "$out/rendered.yaml" +cat > "$out/namespace.yaml" < "$out/clusterrolebinding.yaml" < "$out/clusterrolebinding.yaml" SH chmod +x "$FAKEBIN/akua" @@ -123,6 +160,28 @@ test_rebuild_deploy_uses_a_new_immutable_local_tag() { pass "rebuild deploy renders the rebuilt local image instead of a stale mutable tag" } +test_redeploy_upgrades_an_existing_local_installation() { + : > "$LOG" + AGENT_OS_TEST_LOCAL_WORKLOAD=present AGENT_OS_TEST_IMAGE_ID=sha256:redeploy run_cli deploy + + assert_call 'kubectl --context orbstack -n agent-os-demo get statefulset agent-os-firstmate --ignore-not-found -o name' \ + "redeploy must detect the existing local installation" + grep -Fq 'kubectl --context orbstack apply -f ' "$LOG" || \ + fail "redeploy must upgrade the existing local installation" + pass "redeploy upgrades the content-addressed local installation" +} + +test_namespace_override_updates_the_rendered_profile() { + : > "$LOG" + AGENT_OS_NAMESPACE=agent-os-custom run_cli deploy + + assert_call 'akua-input-namespace agent-os-custom' \ + "namespace override must update the canonical profile inputs" + grep -Fq 'kubectl --context orbstack -n agent-os-custom' "$LOG" || \ + fail "namespace override must target the same rendered namespace" + pass "namespace override stays consistent with package rendering" +} + test_explicit_image_override_is_used_without_retagging() { : > "$LOG" AGENT_OS_IMAGE=example.test/agent-os:custom run_cli build @@ -162,7 +221,7 @@ test_destroy_requires_exact_confirmation() { cleanup_out=$(run_cli destroy --yes 2>&1) || cleanup_rc=$? [ "$cleanup_rc" -eq 3 ] || \ fail "cluster-admin demo destroy must stop for separate privileged cleanup, got $cleanup_rc: $cleanup_out" - grep -Fq 'kubectl --context orbstack delete --ignore-not-found -f ' "$LOG" || \ + grep -Fq 'kubectl --context orbstack delete --ignore-not-found --wait=true -f ' "$LOG" || \ fail "confirmed destroy must delete only resources from the rendered OrbStack profile" if grep -E 'kubectl .* (get|delete) clusterrolebinding' "$LOG" >/dev/null; then fail "routine demo destroy must not inspect or delete cluster-scoped RBAC" @@ -189,6 +248,8 @@ test_non_orbstack_context_is_fail_closed() { test_status_pins_context_and_namespace test_deploy_starts_local_kubernetes_and_renders_the_orbstack_profile test_rebuild_deploy_uses_a_new_immutable_local_tag +test_redeploy_upgrades_an_existing_local_installation +test_namespace_override_updates_the_rendered_profile test_explicit_image_override_is_used_without_retagging test_empty_image_override_uses_content_addressed_default test_destroy_requires_exact_confirmation diff --git a/tests/agent-os-packages.test.sh b/tests/agent-os-packages.test.sh index 9c380c5cc..54f047ae0 100755 --- a/tests/agent-os-packages.test.sh +++ b/tests/agent-os-packages.test.sh @@ -103,6 +103,12 @@ assert_contains "$rendered" 'app.kubernetes.io/managed-by: agent-os' \ "rendered resources must carry the Agent OS ownership label" assert_contains "$rendered" 'agent-os.dev/installation-id: agent-os-firstmate:portable-agent-os' \ "rendered resources must carry the exact installation identity" +for resource_file in "$OUT"/*.yaml; do + assert_grep 'app.kubernetes.io/managed-by: agent-os' "$resource_file" \ + "every rendered resource must carry the Agent OS ownership label" + assert_grep 'agent-os.dev/installation-id: agent-os-firstmate:portable-agent-os' "$resource_file" \ + "every rendered resource must carry the exact installation identity" +done assert_contains "$statefulset_rendered" 'agent-os.dev/rbac-mode: namespace' \ "the rendered StatefulSet must record its RBAC mode" assert_not_contains "$rendered" 'kind: ClusterRoleBinding' \ @@ -140,6 +146,7 @@ pass "the portable package rejects mutable image tags" for invalid_image in \ 'ghcr.io/akua-dev/agent-os@sha256:abc' \ 'ghcr.io/akua-dev/agent-os@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaextra' \ + 'invalid@@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ 'prefix@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:suffix'; do invalid_inputs="$TMP/invalid-$(printf '%s' "$invalid_image" | tr '/:@' '---').yaml" sed "s|^image: .*|image: $invalid_image|" "$INPUTS" > "$invalid_inputs" diff --git a/tools/agent-os/packages/firstmate/README.md b/tools/agent-os/packages/firstmate/README.md index 15ed71f14..f8d96f6ce 100644 --- a/tools/agent-os/packages/firstmate/README.md +++ b/tools/agent-os/packages/firstmate/README.md @@ -23,6 +23,7 @@ Set `rbac: cluster-admin` only for an isolated intelligence cluster after a revi With `createNamespace: true`, the lifecycle helper creates an absent namespace with the package's exact installation identity and refuses to adopt an existing unowned namespace. With `createNamespace: false`, the namespace must already exist without Agent OS ownership metadata, and the lifecycle helper never deletes it. +Every rendered namespaced resource carries the exact installation identity so lifecycle operations refuse foreign same-name resources before apply or deletion. The package carries no credential value or Secret reference. Any runtime credential is a separately created namespace-local Kubernetes Secret, referenced by the runtime helper only after its owner has approved that authority. diff --git a/tools/agent-os/packages/firstmate/crewmate.yaml b/tools/agent-os/packages/firstmate/crewmate.yaml index ee153d228..487da9145 100644 --- a/tools/agent-os/packages/firstmate/crewmate.yaml +++ b/tools/agent-os/packages/firstmate/crewmate.yaml @@ -6,7 +6,11 @@ metadata: labels: app.kubernetes.io/name: agent-os app.kubernetes.io/component: crewmate + app.kubernetes.io/managed-by: agent-os agent-os.dev/crewmate: __AGENT_OS_CREWMATE_ID__ + annotations: + agent-os.dev/installation-id: agent-os-firstmate:__AGENT_OS_NAMESPACE__ + agent-os.dev/checkpoint-state: pending spec: accessModes: - ReadWriteOnce @@ -22,7 +26,10 @@ metadata: labels: app.kubernetes.io/name: agent-os app.kubernetes.io/component: crewmate + app.kubernetes.io/managed-by: agent-os agent-os.dev/crewmate: __AGENT_OS_CREWMATE_ID__ + annotations: + agent-os.dev/installation-id: agent-os-firstmate:__AGENT_OS_NAMESPACE__ spec: automountServiceAccountToken: false securityContext: @@ -69,14 +76,19 @@ spec: mountPath: /usr/local subPath: usr-local - name: ai-auth - mountPath: /home/agent/.pi/agent/auth.json - subPath: auth.json + mountPath: /home/agent/.pi/agent readOnly: true volumes: - name: home persistentVolumeClaim: claimName: agent-os-crewmate-__AGENT_OS_CREWMATE_ID__-home - name: ai-auth - secret: - secretName: __AGENT_OS_AI_SECRET__ + projected: defaultMode: 256 + sources: + - secret: + name: __AGENT_OS_AI_SECRET__ + optional: false + items: + - key: auth.json + path: auth.json diff --git a/tools/agent-os/packages/firstmate/package.k b/tools/agent-os/packages/firstmate/package.k index 4e587f9ee..be8159633 100644 --- a/tools/agent-os/packages/firstmate/package.k +++ b/tools/agent-os/packages/firstmate/package.k @@ -17,7 +17,7 @@ schema Input: memoryLimit: str = "8Gi" check: - allowMutableImage or regex.match(image, r"^.+@sha256:[0-9a-fA-F]{64}$"), "image must use one complete immutable @sha256 digest unless the local profile explicitly allows a mutable image" + allowMutableImage or regex.match(image, r"^([a-z0-9]+(([._]|__|-+)[a-z0-9]+)*(:[0-9]+)?/)*[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*(:[A-Za-z0-9_][A-Za-z0-9_.-]{0,127})?@sha256:[0-9a-f]{64}$"), "image must use one complete OCI reference with an immutable @sha256 digest unless the local profile explicitly allows a mutable image" input: Input = ctx.input() @@ -48,6 +48,7 @@ namespaceRbacResources = [ name = "agent-os-firstmate-runtime" namespace = input.namespace labels = labels + annotations = ownershipAnnotations } rules = [ { @@ -74,6 +75,7 @@ namespaceRbacResources = [ name = "agent-os-firstmate-runtime" namespace = input.namespace labels = labels + annotations = ownershipAnnotations } roleRef = { apiGroup = "rbac.authorization.k8s.io" @@ -127,6 +129,7 @@ resources = namespaceResources + [ name = "agent-os-firstmate-home" namespace = input.namespace labels = labels + annotations = ownershipAnnotations } spec = { accessModes = ["ReadWriteOnce"] From a8e56ee1339b42acb2af63fcae2ca859a4112a76 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 18:47:55 +0200 Subject: [PATCH 27/56] no-mistakes(review): Captain, harden Kubernetes lifecycle and image provenance --- .agents/skills/kubernetes-fleet/SKILL.md | 5 +- Dockerfile | 2 +- THIRD_PARTY_NOTICES.md | 5 + bin/agent-os-container-entrypoint.sh | 9 ++ bin/agent-os-crewmate.sh | 61 ++++++++-- bin/agent-os-kubernetes.sh | 85 ++++++++++++-- bin/agent-os-local.sh | 18 ++- docs/kubernetes.md | 4 +- tests/agent-os-container.test.sh | 11 +- tests/agent-os-kubernetes.test.sh | 106 ++++++++++++++---- tests/agent-os-local.test.sh | 16 ++- tests/agent-os-packages.test.sh | 9 ++ .../agent-os/packages/firstmate/crewmate.yaml | 4 +- tools/agent-os/packages/firstmate/package.k | 2 +- 14 files changed, 290 insertions(+), 47 deletions(-) diff --git a/.agents/skills/kubernetes-fleet/SKILL.md b/.agents/skills/kubernetes-fleet/SKILL.md index af345f49d..f3150514e 100644 --- a/.agents/skills/kubernetes-fleet/SKILL.md +++ b/.agents/skills/kubernetes-fleet/SKILL.md @@ -23,7 +23,8 @@ Load this skill only when the current firstmate is running in Kubernetes or is e - The Secret must contain the selected provider's `auth.json` key and must be provisioned independently instead of cloning or sharing the primary credential. - Pass only its name through `AGENT_OS_AI_SECRET` when invoking `bin/agent-os-crewmate.sh create `. - The helper never reads or discovers the Secret value, and the default Role intentionally has no Secret-read permission. -- Kubernetes projects only the selected `auth.json` key into a read-only authorization directory; a missing Secret or key keeps the Pod unready, makes create fail, removes the non-running Pod, and retains the PVC for an authorized retry. +- Kubernetes projects only the selected `auth.json` key into a dedicated read-only runtime directory, and the entrypoint links that file into the writable PVC-backed Pi state without copying credential bytes. +- A missing Secret or key keeps the Pod unready, makes create fail, removes the non-running Pod, and retains the PVC for an authorized retry. - Probe the selected model route before launching work; when quota is unavailable, use another explicitly granted provider or report the capacity blocker instead of repeatedly spawning agents. - Give every launched Herdr agent a task-unique name, close only a confirmed dead restored pane before reuse, and never replace a live agent. - Grade completion by the promised artifact or delivered Git state; Herdr `idle` alone is not a completion signal. @@ -31,7 +32,7 @@ Load this skill only when the current firstmate is running in Kubernetes or is e - Use `status` to inspect it, `stop` to remove only the Pod, and `restart` to replace only the Pod on its retained PVC. - The ambiguous `delete` command is rejected. - `purge --yes` is the only operation that destroys a persistent home. -- Before purge, independently checkpoint or deliver unique work, then annotate the owned PVC with `agent-os.dev/checkpoint-state=clean` and a non-secret RFC3339 `agent-os.dev/checkpoint-at` value. +- Before purge, stop the owned Pod and wait for its absence, independently checkpoint or deliver unique work from the stopped home, then annotate the owned PVC with `agent-os.dev/checkpoint-state=clean` and a non-secret RFC3339 `agent-os.dev/checkpoint-at` value. - Purge verifies exact installation and crewmate ownership, displays the target, requires its own confirmation, and records requested and completed phases in `AGENT_OS_PURGE_EVIDENCE_FILE` or `$FM_HOME/data/crewmate-purge-evidence.log`. - Purge evidence contains only time, namespace, crewmate ID, resource names, phase, and checkpoint time. - Never mount the primary home into a child Pod. diff --git a/Dockerfile b/Dockerfile index c27179c2f..153386b7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:24-trixie-slim +FROM node:24-trixie-slim@sha256:366fdef91728b1b7fa18c84fba63b6e79ed77b7e10cc206878e9705da4d7b169 ARG TARGETARCH ARG HERDR_VERSION=0.7.3 diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 232278847..45bbf9986 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -1,5 +1,10 @@ # Third-party notices +## Node.js base image + +The Agent OS image uses the official multi-architecture `node:24-trixie-slim` base image pinned to index digest `sha256:366fdef91728b1b7fa18c84fba63b6e79ed77b7e10cc206878e9705da4d7b169`. +The reviewed Docker Node source revision is . + ## Herdr The Agent OS image includes an unmodified Herdr 0.7.3 executable as a separate program under AGPL-3.0-or-later. diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh index a8958c471..40e3a5e3b 100755 --- a/bin/agent-os-container-entrypoint.sh +++ b/bin/agent-os-container-entrypoint.sh @@ -17,6 +17,15 @@ mkdir -p \ "$HOME/.bun" \ "$HOME/.cargo" +if [ -n "${AGENT_OS_PI_AUTH_FILE:-}" ]; then + if [ ! -f "$AGENT_OS_PI_AUTH_FILE" ]; then + echo "error: projected Pi authorization is unavailable" >&2 + exit 2 + fi + mkdir -p "$HOME/.pi/agent" + ln -sfn -- "$AGENT_OS_PI_AUTH_FILE" "$HOME/.pi/agent/auth.json" +fi + "$(dirname "$0")/agent-os-kubeconfig.sh" if [ ! -e "$FM_HOME/config/backend" ]; then printf 'herdr\n' > "$FM_HOME/config/backend" diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index 05bed64aa..a4368e327 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -46,6 +46,11 @@ resource_identity() { -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}' } +pod_record() { + kube get pod "$POD" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.uid}' +} + pvc_record() { kube get pvc "$PVC" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-state}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-at}' @@ -99,10 +104,40 @@ render_resources() { "$TEMPLATE" } +cleanup_new_owned_pod() { + local before_uid=$1 after current identity after_uid + after=$(pod_record) + if [ -z "$after" ]; then + echo "partial state: crewmate apply left no Pod; persistent home retained" >&2 + return + fi + identity=$(printf '%s' "$after" | cut -f1-4) + after_uid=$(printf '%s' "$after" | cut -f5) + if [ "$identity" != "$EXPECTED_POD" ] || [ -z "$after_uid" ] || [ -n "$before_uid" ]; then + echo "partial state: Pod was not proven newly created and owned; persistent home retained" >&2 + return + fi + current=$(pod_record) + if [ "$current" != "$after" ]; then + echo "partial state: Pod identity changed during cleanup; no Pod deleted and persistent home retained" >&2 + return + fi + echo "partial state: removing newly created owned Pod '$POD' uid=$after_uid; persistent home retained" >&2 + kube delete pod "$POD" --ignore-not-found --wait=false + kube wait --for=delete "pod/$POD" --timeout=180s +} + apply_and_wait() { - render_resources | kube apply -f - + local before before_uid + before=$(pod_record) + before_uid=$(printf '%s' "$before" | cut -f5) + if ! render_resources | kube apply -f -; then + cleanup_new_owned_pod "$before_uid" + echo "error: crewmate resources were only partially applied" >&2 + exit 1 + fi if ! kube wait --for=condition=Ready "pod/$POD" --timeout=180s; then - kube delete pod "$POD" --ignore-not-found + cleanup_new_owned_pod "$before_uid" echo "error: crewmate Pod did not become ready with the authorized AI Secret" >&2 exit 1 fi @@ -124,6 +159,16 @@ preflight_existing_home() { printf '%s' "$pvc" } +stop_owned_pod() { + local pod + pod=$(require_owned_or_absent pod "$POD" "$EXPECTED_POD") + if [ -z "$pod" ]; then + return + fi + kube delete pod "$POD" --ignore-not-found --wait=false + kube wait --for=delete "pod/$POD" --timeout=180s +} + record_purge() { local phase=$1 checkpoint_at=$2 timestamp timestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ') @@ -149,15 +194,14 @@ case "$COMMAND" in ;; stop) [ -z "$CONFIRM" ] || { echo "usage: $0 stop " >&2; exit 2; } - require_owned_or_absent pod "$POD" "$EXPECTED_POD" >/dev/null require_owned_pvc_or_absent >/dev/null - kube delete pod "$POD" --ignore-not-found + stop_owned_pod ;; restart) [ -z "$CONFIRM" ] || { echo "usage: $0 restart " >&2; exit 2; } validate_ai_grant preflight_existing_home >/dev/null - kube delete pod "$POD" --ignore-not-found + stop_owned_pod apply_and_wait ;; purge) @@ -166,6 +210,10 @@ case "$COMMAND" in echo "error: purge requires the purge-specific --yes confirmation" >&2 exit 2 fi + if [ -n "$(require_owned_or_absent pod "$POD" "$EXPECTED_POD")" ]; then + echo "error: stop the owned crewmate Pod and prove its absence before checkpointing for purge" >&2 + exit 2 + fi pvc=$(preflight_existing_home) checkpoint_state=$(printf '%s' "$pvc" | cut -f5) checkpoint_at=$(printf '%s' "$pvc" | cut -f6) @@ -188,8 +236,7 @@ case "$COMMAND" in mkdir -p "$(dirname "$evidence_file")" exec 3>>"$evidence_file" record_purge purge-requested "$checkpoint_at" - kube delete pod "$POD" --ignore-not-found - kube delete pvc "$PVC" --ignore-not-found + kube delete pvc "$PVC" --ignore-not-found --wait=true --timeout=180s record_purge purge-complete "$checkpoint_at" ;; delete) diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh index 1299d2375..3bf64f43c 100755 --- a/bin/agent-os-kubernetes.sh +++ b/bin/agent-os-kubernetes.sh @@ -189,8 +189,14 @@ preflight_rendered_resources() { delete_namespace_rbac() { require_namespaced_resource_owned_or_absent RoleBinding agent-os-firstmate-runtime require_namespaced_resource_owned_or_absent Role agent-os-firstmate-runtime - "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" delete rolebinding agent-os-firstmate-runtime --ignore-not-found - "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" delete role agent-os-firstmate-runtime --ignore-not-found + if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" delete rolebinding \ + agent-os-firstmate-runtime --ignore-not-found --wait=true --timeout=180s; then + bounded_delete_failure "RoleBinding/agent-os-firstmate-runtime" + fi + if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" delete role \ + agent-os-firstmate-runtime --ignore-not-found --wait=true --timeout=180s; then + bounded_delete_failure "Role/agent-os-firstmate-runtime" + fi } apply_rendered() { @@ -233,13 +239,72 @@ report_cluster_cleanup() { echo "required evidence: clusterrolebinding/agent-os-firstmate-$NAMESPACE absent" >&2 } +uninstall_command() { + printf 'AGENT_OS_CONTEXT=%q AGENT_OS_NAMESPACE=%q AGENT_OS_PACKAGE=%q AGENT_OS_INPUTS=%q AGENT_OS_AKUA=%q AGENT_OS_KUBECTL=%q %q uninstall --yes' \ + "$CONTEXT" "$NAMESPACE" "$PACKAGE" "$INPUTS" "$AKUA" "$KUBECTL" "$0" + [ "$DELETE_NAMESPACE" -eq 0 ] || printf ' --delete-namespace' +} + +report_retained_resources() { + local file kind name record identity expected finalizers + while IFS= read -r file; do + [ -n "$file" ] || continue + kind=$(rendered_resource_field "$file" kind) + name=$(rendered_resource_field "$file" name) + case "$kind" in + ClusterRoleBinding) continue ;; + Namespace) + [ "$DELETE_NAMESPACE" -eq 1 ] || continue + if ! record=$("$KUBECTL" --context "$CONTEXT" get namespace "$name" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.finalizers}'); then + echo "retained-unverified: Namespace/$name could not be inspected; no further deletion attempted" >&2 + continue + fi + ;; + *) + if ! record=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get "$kind" "$name" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.finalizers}'); then + echo "retained-unverified: $kind/$name could not be inspected; no further deletion attempted" >&2 + continue + fi + ;; + esac + [ -n "$record" ] || continue + identity=$(printf '%s' "$record" | cut -f1-3) + expected="$name"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" + if [ "$identity" = "$expected" ]; then + finalizers=$(printf '%s' "$record" | cut -f4-) + [ -n "$finalizers" ] || finalizers='[]' + echo "retained: $kind/$name finalizers=$finalizers" >&2 + else + echo "retained-unverified: $kind/$name ownership changed; no further deletion attempted" >&2 + fi + done < <(find "$OUT" -type f -name '*.yaml' -print) + echo "safe retry: $(uninstall_command)" >&2 +} + +bounded_delete_failure() { + local target=$1 + if [ "$COMMAND" = uninstall ]; then + echo "incomplete: timed out deleting $target after 180s" >&2 + report_retained_resources + return 3 + fi + echo "error: timed out deleting $target after 180s" >&2 + return 1 +} + delete_rendered_kind() { - local desired_kind=$1 file kind + local desired_kind=$1 file kind name while IFS= read -r file; do [ -n "$file" ] || continue kind=$(rendered_resource_field "$file" kind) if [ "$kind" = "$desired_kind" ]; then - "$KUBECTL" --context "$CONTEXT" delete --ignore-not-found --wait=true -f "$file" + name=$(rendered_resource_field "$file" name) + if ! "$KUBECTL" --context "$CONTEXT" delete --ignore-not-found \ + --wait=true --timeout=180s -f "$file"; then + bounded_delete_failure "$kind/$name" + fi fi done < <(find "$OUT" -type f -name '*.yaml' -print) } @@ -247,7 +312,10 @@ delete_rendered_kind() { delete_rendered_namespaced_resources() { local kind delete_rendered_kind StatefulSet - "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" wait --for=delete pod/agent-os-firstmate-0 --timeout=180s + if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" wait \ + --for=delete pod/agent-os-firstmate-0 --timeout=180s; then + bounded_delete_failure Pod/agent-os-firstmate-0 + fi for kind in Service RoleBinding Role ServiceAccount; do delete_rendered_kind "$kind" done @@ -301,7 +369,10 @@ delete_owned_empty_namespace() { echo "error: namespace '$NAMESPACE' ownership changed during deletion checks" >&2 exit 2 fi - "$KUBECTL" --context "$CONTEXT" delete namespace "$NAMESPACE" + if ! "$KUBECTL" --context "$CONTEXT" delete namespace "$NAMESPACE" \ + --wait=true --timeout=180s; then + bounded_delete_failure "Namespace/$NAMESPACE" + fi } cleanup_cluster_rbac() { @@ -327,7 +398,7 @@ cleanup_cluster_rbac() { exit 2 fi fi - "$KUBECTL" --context "$CONTEXT" delete clusterrolebinding "$binding" + "$KUBECTL" --context "$CONTEXT" delete clusterrolebinding "$binding" --wait=false "$KUBECTL" --context "$CONTEXT" wait --for=delete "clusterrolebinding/$binding" --timeout=60s fi if [ -n "$(namespace_name)" ]; then diff --git a/bin/agent-os-local.sh b/bin/agent-os-local.sh index f1bf9d6f0..adb7997f0 100755 --- a/bin/agent-os-local.sh +++ b/bin/agent-os-local.sh @@ -30,15 +30,20 @@ local_image_tag() { printf 'agent-os:local-%s\n' "${image_id#sha256:}" } -render_profile() { - local image=$1 inputs lifecycle - inputs=$(mktemp) - trap 'rm -f "$inputs"' RETURN +render_profile_inputs() { + local image=$1 inputs=$2 awk -v image="$image" -v namespace="$NAMESPACE" ' $1 == "image:" { print "image: " image; next } $1 == "namespace:" { print "namespace: " namespace; next } { print } ' "$PROFILE" > "$inputs" +} + +render_profile() { + local image=$1 inputs lifecycle + inputs=$(mktemp) + trap 'rm -f "$inputs"' RETURN + render_profile_inputs "$image" "$inputs" lifecycle=install if [ -n "$(kubectl --context "$CONTEXT" -n "$NAMESPACE" get statefulset agent-os-firstmate --ignore-not-found -o name)" ]; then lifecycle=upgrade @@ -80,7 +85,10 @@ case "$COMMAND" in echo "error: destroy requires --yes and removes only namespaced Agent OS resources from '$NAMESPACE'" >&2 exit 2 fi - AGENT_OS_CONTEXT="$CONTEXT" AGENT_OS_NAMESPACE="$NAMESPACE" AGENT_OS_INPUTS="$PROFILE" \ + inputs=$(mktemp) + trap 'rm -f "$inputs"' EXIT + render_profile_inputs "$IMAGE" "$inputs" + AGENT_OS_CONTEXT="$CONTEXT" AGENT_OS_NAMESPACE="$NAMESPACE" AGENT_OS_INPUTS="$inputs" \ "$ROOT/bin/agent-os-kubernetes.sh" uninstall --yes ;; *) diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 10ac7100d..8c3cca9f4 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -102,7 +102,7 @@ It is not a second public package and is not a Marketplace product. Each runtime mate has its own PVC and no ambient Kubernetes ServiceAccount token. Creating one requires an explicitly authorized, pre-created Secret in the same namespace with an `auth.json` key. Pass only that Secret's name through `AGENT_OS_AI_SECRET`; the helper never discovers or copies the primary credential. -The Secret projects only `auth.json` into the read-only `/home/agent/.pi/agent` directory without a `subPath` mount. +The Secret projects only `auth.json` into a dedicated read-only runtime directory, and the entrypoint links that file into the writable PVC-backed Pi state without copying credential bytes. A missing Secret or key keeps the Pod unready, so creation fails closed and removes the non-running Pod while retaining its PVC for an authorized retry. ```sh @@ -111,7 +111,7 @@ AGENT_OS_AI_SECRET=scout-1-ai-auth bin/agent-os-crewmate.sh create scout-1 Use `stop` for a Pod-only shutdown and `restart` for a Pod-only replacement after an approved Secret rotation. The ambiguous `delete` operation is rejected. -Only `purge --yes` removes the PVC, and it requires exact ownership, a clean checkpoint annotation, and a non-secret evidence file. +Only `purge --yes` removes the PVC, and it requires the owned Pod to be absent, exact ownership, a fresh clean checkpoint annotation from the stopped home, and a non-secret evidence file. The full rotation, urgent-revocation, checkpoint, and purge procedure is owned by the `kubernetes-fleet` operating skill. The existing [same-Pod](evidence/2026-07-13-same-pod-firstmate.md) and [separate-Pod recovery](evidence/2026-07-13-separate-pod-recovery.md) records remain local lifecycle evidence. diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 1387002aa..1eca072bb 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -5,7 +5,10 @@ set -u # shellcheck source=tests/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -assert_grep 'FROM node:24-trixie-slim' "$ROOT/Dockerfile" "image must pin Node 24 Trixie" +assert_grep 'FROM node:24-trixie-slim@sha256:366fdef91728b1b7fa18c84fba63b6e79ed77b7e10cc206878e9705da4d7b169' \ + "$ROOT/Dockerfile" "image must pin the multi-architecture Node 24 Trixie base" +assert_grep 'sha256:366fdef91728b1b7fa18c84fba63b6e79ed77b7e10cc206878e9705da4d7b169' \ + "$ROOT/THIRD_PARTY_NOTICES.md" "base image provenance must record the reviewed index digest" assert_grep 'ARG HERDR_VERSION=0.7.3' "$ROOT/Dockerfile" "image must pin Herdr 0.7.3" assert_grep 'ARG KUBECTL_VERSION=1.34.8' "$ROOT/Dockerfile" "image must pin kubectl 1.34.8" assert_grep 'ARG GH_VERSION=2.96.0' "$ROOT/Dockerfile" "image must pin GitHub CLI 2.96.0" @@ -49,6 +52,12 @@ assert_grep 'AGENT_OS_TEST_PI_MODEL' "$ROOT/bin/agent-os-container-entrypoint.sh "test-mode Pods must converge the Pi model policy" assert_grep 'defaultThinkingLevel' "$ROOT/bin/agent-os-container-entrypoint.sh" \ "test-mode Pods must also pin direct Pi sessions" +# shellcheck disable=SC2016 # Match literal entrypoint variables. +assert_grep 'ln -sfn -- "$AGENT_OS_PI_AUTH_FILE" "$HOME/.pi/agent/auth.json"' \ + "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "entrypoint must link projected authorization without copying Secret bytes" +assert_no_grep 'cp .*AGENT_OS_PI_AUTH_FILE' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "entrypoint must never persist projected Secret bytes" assert_grep 'image: agent-os:dev' "$ROOT/deploy/orbstack/inputs.yaml" \ "the OrbStack profile must define its local image source" assert_grep 'imagePullPolicy: Never' "$ROOT/deploy/orbstack/inputs.yaml" \ diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index c4110cb0b..8b175c41b 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -45,19 +45,36 @@ printf '\n' >> "$AGENT_OS_TEST_LOG" if [ "${*: -2}" = "-f -" ]; then cat > "$AGENT_OS_STDIN_LOG" fi +if [ "${AGENT_OS_TEST_FAIL_APPLY:-0}" = 1 ] && [[ " $* " = *" apply -f - "* ]]; then + exit 1 +fi if [ "${AGENT_OS_TEST_FAIL_WAIT:-0}" = 1 ] && [ "${1:-}" = -n ] && [ "${3:-}" = wait ]; then exit 1 fi if [ "${AGENT_OS_TEST_FAIL_ANNOTATE:-0}" = 1 ] && [[ " $* " = *" annotate statefulset agent-os-firstmate "* ]]; then exit 1 fi +if [ -n "${AGENT_OS_TEST_FAIL_DELETE_FILE:-}" ] && [[ " $* " = *" delete "* ]] && \ + [[ " $* " = *"$AGENT_OS_TEST_FAIL_DELETE_FILE"* ]]; then + exit 1 +fi case " $* " in *" get pod agent-os-crewmate-"*" --ignore-not-found -o jsonpath="*) id=${AGENT_OS_TEST_CREWMATE_ID:-scout-1} - case "${AGENT_OS_TEST_POD_STATE:-absent}" in + pod_state=${AGENT_OS_TEST_POD_STATE:-absent} + if [[ " $* " = *'.metadata.uid'* ]] && grep -F ' apply -f -' "$AGENT_OS_TEST_LOG" >/dev/null; then + pod_state=${AGENT_OS_TEST_POD_AFTER_APPLY:-$pod_state} + fi + case "$pod_state" in absent) ;; - owned) printf 'agent-os-crewmate-%s\tagent-os\t%s\tagent-os-firstmate:agent-os-demo' "$id" "$id" ;; - foreign) printf 'agent-os-crewmate-%s\tother\tother\tother-installation' "$id" ;; + owned) + printf 'agent-os-crewmate-%s\tagent-os\t%s\tagent-os-firstmate:agent-os-demo' "$id" "$id" + [[ " $* " != *'.metadata.uid'* ]] || printf '\tuid-owned' + ;; + foreign) + printf 'agent-os-crewmate-%s\tother\tother\tother-installation' "$id" + [[ " $* " != *'.metadata.uid'* ]] || printf '\tuid-foreign' + ;; esac ;; *" get pvc agent-os-crewmate-"*" --ignore-not-found -o jsonpath="*) @@ -111,7 +128,10 @@ case " $* " in esac case "${AGENT_OS_TEST_RESOURCE_STATE:-absent}" in absent) ;; - owned) printf '%s\tagent-os\tagent-os-firstmate:portable-agent-os' "$name" ;; + owned) + printf '%s\tagent-os\tagent-os-firstmate:portable-agent-os' "$name" + [[ " $* " != *'.metadata.finalizers'* ]] || printf '\t[kubernetes.io/pvc-protection]' + ;; foreign) printf '%s\tother\tother-installation' "$name" ;; esac ;; @@ -184,8 +204,14 @@ assert_no_grep 'hostUsers: false' "$STDIN_LOG" "OrbStack children must not reque assert_grep 'runAsUser: 0' "$STDIN_LOG" "children must run as container root" assert_grep 'name: agent-os-init' "$STDIN_LOG" "children must seed persistent tools" assert_grep 'mountPath: /usr/local' "$STDIN_LOG" "children must persist /usr/local" -assert_grep 'mountPath: /home/agent/.pi/agent' "$STDIN_LOG" \ - "children must mount the explicitly granted AI authorization directory" +assert_grep 'mountPath: /var/run/secrets/agent-os/pi' "$STDIN_LOG" \ + "children must mount AI authorization outside writable Pi state" +assert_grep 'name: AGENT_OS_PI_AUTH_FILE' "$STDIN_LOG" \ + "children must expose the projected authorization path to the entrypoint" +assert_grep 'value: /var/run/secrets/agent-os/pi/auth.json' "$STDIN_LOG" \ + "the entrypoint must receive only the projected auth.json path" +assert_no_grep 'mountPath: /home/agent/.pi/agent' "$STDIN_LOG" \ + "the read-only Secret projection must not shadow writable Pi state" assert_no_grep 'subPath: auth.json' "$STDIN_LOG" \ "projected AI authorization must support Secret rotation without a subPath mount" assert_grep 'path: auth.json' "$STDIN_LOG" \ @@ -221,20 +247,38 @@ fi pass "crewmate create requires an explicit AI Secret grant" : > "$CALLS" -if AGENT_OS_TEST_FAIL_WAIT=1 run_launcher create scout-1 >/dev/null 2>&1; then +if AGENT_OS_TEST_FAIL_WAIT=1 AGENT_OS_TEST_POD_AFTER_APPLY=owned run_launcher create scout-1 >/dev/null 2>&1; then fail "crewmate create must fail when its authorized Secret cannot produce a ready Pod" fi -grep -Fqx 'kubectl -n agent-os-demo delete pod agent-os-crewmate-scout-1 --ignore-not-found' "$CALLS" || \ +grep -Fqx 'kubectl -n agent-os-demo delete pod agent-os-crewmate-scout-1 --ignore-not-found --wait=false' "$CALLS" || \ fail "failed create must remove the non-running Pod" if grep -F 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" >/dev/null; then fail "failed create must retain the crewmate PVC for an authorized retry" fi pass "crewmate create fails closed while retaining its persistent home" +: > "$CALLS" +partial_out='' +partial_rc=0 +partial_out=$(AGENT_OS_TEST_FAIL_APPLY=1 AGENT_OS_TEST_POD_AFTER_APPLY=owned \ + run_launcher create scout-1 2>&1) || partial_rc=$? +[ "$partial_rc" -eq 1 ] || fail "partial apply must fail after cleanup: $partial_out" +assert_contains "$partial_out" 'uid-owned' \ + "partial apply cleanup must report the exact newly created Pod UID" +assert_grep 'metadata.uid' "$CALLS" \ + "partial apply cleanup must collect exact Pod UID evidence" +grep -Fqx 'kubectl -n agent-os-demo delete pod agent-os-crewmate-scout-1 --ignore-not-found --wait=false' "$CALLS" || \ + fail "partial apply cleanup must remove only the newly created owned Pod" +assert_no_grep 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" \ + "partial apply cleanup must retain the persistent home" +pass "crewmate partial apply cleans only a newly created owned Pod" + : > "$CALLS" AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=owned run_launcher stop scout-1 -grep -Fqx 'kubectl -n agent-os-demo delete pod agent-os-crewmate-scout-1 --ignore-not-found' "$CALLS" || \ +grep -Fqx 'kubectl -n agent-os-demo delete pod agent-os-crewmate-scout-1 --ignore-not-found --wait=false' "$CALLS" || \ fail "stop must target the exactly owned crewmate Pod" +grep -Fqx 'kubectl -n agent-os-demo wait --for=delete pod/agent-os-crewmate-scout-1 --timeout=180s' "$CALLS" || \ + fail "stop must prove Pod absence before checkpointing can begin" if grep -F 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" >/dev/null; then fail "stop must preserve the crewmate persistent home" fi @@ -268,6 +312,13 @@ assert_contains "$purge_out" 'agent-os-crewmate-scout-1-home' \ "unconfirmed purge must display the exact persistent target" [ ! -s "$CALLS" ] || fail "unconfirmed purge must not query or mutate cluster state" +: > "$CALLS" +if AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=clean run_launcher purge scout-1 --yes >/dev/null 2>&1; then + fail "purge must refuse a clean checkpoint while the owned Pod can still write" +fi +assert_no_grep 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" \ + "purge must not delete a home while its Pod still exists" + : > "$CALLS" if AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=owned run_launcher purge scout-1 --yes >/dev/null 2>&1; then fail "purge must reject a persistent home without a clean checkpoint" @@ -285,10 +336,10 @@ assert_no_grep 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" \ : > "$CALLS" : > "$PURGE_EVIDENCE" -AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=clean run_launcher purge scout-1 --yes -grep -Fqx 'kubectl -n agent-os-demo delete pod agent-os-crewmate-scout-1 --ignore-not-found' "$CALLS" || \ - fail "purge must delete the exactly owned Pod" -grep -Fqx 'kubectl -n agent-os-demo delete pvc agent-os-crewmate-scout-1-home --ignore-not-found' "$CALLS" || \ +AGENT_OS_TEST_POD_STATE=absent AGENT_OS_TEST_PVC_STATE=clean run_launcher purge scout-1 --yes +assert_no_grep 'delete pod agent-os-crewmate-scout-1' "$CALLS" \ + "purge must accept checkpoint evidence only after the Pod is absent" +grep -Fqx 'kubectl -n agent-os-demo delete pvc agent-os-crewmate-scout-1-home --ignore-not-found --wait=true --timeout=180s' "$CALLS" || \ fail "purge must delete the exactly owned persistent home" assert_grep 'purge-complete' "$PURGE_EVIDENCE" "purge must record non-secret completion evidence" assert_no_grep 'scout-1-ai-auth' "$PURGE_EVIDENCE" "purge evidence must never contain credential references" @@ -577,7 +628,7 @@ AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=pending \ AGENT_OS_TEST_CLUSTER_RBAC_STATE=owned run_generic cleanup-cluster-rbac --yes grep -Fq 'kubectl --context kind-agent-os get clusterrolebinding agent-os-firstmate-portable-agent-os --ignore-not-found' "$CALLS" || \ fail "privileged cleanup must inspect only the exact stale ClusterRoleBinding" -grep -Fqx 'kubectl --context kind-agent-os delete clusterrolebinding agent-os-firstmate-portable-agent-os' "$CALLS" || \ +grep -Fqx 'kubectl --context kind-agent-os delete clusterrolebinding agent-os-firstmate-portable-agent-os --wait=false' "$CALLS" || \ fail "privileged cleanup must delete only the exact owned ClusterRoleBinding" grep -Fqx 'kubectl --context kind-agent-os wait --for=delete clusterrolebinding/agent-os-firstmate-portable-agent-os --timeout=60s' "$CALLS" || \ fail "privileged cleanup must produce deletion evidence for the exact binding" @@ -623,9 +674,9 @@ PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS="$CLUSTER_ADMIN AGENT_OS_TEST_NAMESPACE=portable-agent-os AGENT_OS_TEST_NAMESPACE_STATE=owned \ AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_CONTEXT=kind-agent-os \ AGENT_OS_NAMESPACE=portable-agent-os "$GENERIC" upgrade -grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete rolebinding agent-os-firstmate-runtime --ignore-not-found' "$CALLS" || \ +grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete rolebinding agent-os-firstmate-runtime --ignore-not-found --wait=true --timeout=180s' "$CALLS" || \ fail "cluster-admin upgrade must delete the stale namespace RoleBinding" -grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete role agent-os-firstmate-runtime --ignore-not-found' "$CALLS" || \ +grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete role agent-os-firstmate-runtime --ignore-not-found --wait=true --timeout=180s' "$CALLS" || \ fail "cluster-admin upgrade must delete the stale namespace Role" apply_line=$(grep -Fn 'kubectl --context kind-agent-os apply -f ' "$CALLS" | head -n 1 | cut -d: -f1) delete_line=$(grep -Fn 'delete rolebinding agent-os-firstmate-runtime' "$CALLS" | head -n 1 | cut -d: -f1) @@ -667,9 +718,9 @@ fi if grep -F 'delete namespace portable-agent-os' "$CALLS" >/dev/null; then fail "bounded uninstall must retain its namespace by default" fi -grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete rolebinding agent-os-firstmate-runtime --ignore-not-found' "$CALLS" || \ +grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete rolebinding agent-os-firstmate-runtime --ignore-not-found --wait=true --timeout=180s' "$CALLS" || \ fail "uninstall must remove namespace runtime binding regardless of current inputs" -grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete role agent-os-firstmate-runtime --ignore-not-found' "$CALLS" || \ +grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete role agent-os-firstmate-runtime --ignore-not-found --wait=true --timeout=180s' "$CALLS" || \ fail "uninstall must remove namespace runtime Role regardless of current inputs" stateful_delete_line=$(grep -Fn '/statefulset.yaml' "$CALLS" | grep ' delete ' | head -n 1 | cut -d: -f1) pvc_delete_line=$(grep -Fn '/00-pvc.yaml' "$CALLS" | grep ' delete ' | head -n 1 | cut -d: -f1) @@ -678,6 +729,23 @@ pvc_delete_line=$(grep -Fn '/00-pvc.yaml' "$CALLS" | grep ' delete ' | head -n 1 fail "uninstall must delete the StatefulSet before waiting on PVC deletion" pass "routine uninstall removes namespaced resources without cluster-wide authority" +: > "$CALLS" +bounded_out='' +bounded_rc=0 +bounded_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=namespace \ + AGENT_OS_TEST_RESOURCE_STATE=owned AGENT_OS_TEST_FAIL_DELETE_FILE=00-pvc.yaml \ + run_generic uninstall --yes 2>&1) || bounded_rc=$? +[ "$bounded_rc" -eq 3 ] || fail "timed-out uninstall must exit incomplete: $bounded_out" +assert_contains "$bounded_out" 'retained: PersistentVolumeClaim/agent-os-firstmate-home' \ + "timed-out uninstall must report the retained owned resource" +assert_contains "$bounded_out" 'finalizers=[kubernetes.io/pvc-protection]' \ + "timed-out uninstall must report retained finalizers" +assert_contains "$bounded_out" 'safe retry:' \ + "timed-out uninstall must print exact safe retry evidence" +grep -F '/00-pvc.yaml' "$CALLS" | grep -F -- '--wait=true --timeout=180s' >/dev/null || \ + fail "uninstall deletion must have an explicit timeout" +pass "uninstall timeouts report retained owned resources and safe retry evidence" + : > "$CALLS" foreign_uninstall_rc=0 AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=namespace \ @@ -716,7 +784,7 @@ pass "uninstall retry cannot lose cluster-RBAC residue state" : > "$CALLS" AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=namespace \ run_generic uninstall --yes --delete-namespace -grep -Fqx 'kubectl --context kind-agent-os delete namespace portable-agent-os' "$CALLS" || \ +grep -Fqx 'kubectl --context kind-agent-os delete namespace portable-agent-os --wait=true --timeout=180s' "$CALLS" || \ fail "optional namespace deletion must target only the exactly owned namespace" grep -Fq 'kubectl --context kind-agent-os api-resources --verbs=list --namespaced -o name' "$CALLS" || \ fail "optional namespace deletion must inventory every listable namespaced resource type" diff --git a/tests/agent-os-local.test.sh b/tests/agent-os-local.test.sh index 1ab9a33be..747d8caa1 100755 --- a/tests/agent-os-local.test.sh +++ b/tests/agent-os-local.test.sh @@ -221,7 +221,7 @@ test_destroy_requires_exact_confirmation() { cleanup_out=$(run_cli destroy --yes 2>&1) || cleanup_rc=$? [ "$cleanup_rc" -eq 3 ] || \ fail "cluster-admin demo destroy must stop for separate privileged cleanup, got $cleanup_rc: $cleanup_out" - grep -Fq 'kubectl --context orbstack delete --ignore-not-found --wait=true -f ' "$LOG" || \ + grep -Fq 'kubectl --context orbstack delete --ignore-not-found --wait=true --timeout=180s -f ' "$LOG" || \ fail "confirmed destroy must delete only resources from the rendered OrbStack profile" if grep -E 'kubectl .* (get|delete) clusterrolebinding' "$LOG" >/dev/null; then fail "routine demo destroy must not inspect or delete cluster-scoped RBAC" @@ -231,6 +231,19 @@ test_destroy_requires_exact_confirmation() { pass "destroy requires confirmation and reports privileged RBAC cleanup" } +test_destroy_uses_the_namespace_adjusted_profile() { + local out rc=0 + : > "$LOG" + out=$(AGENT_OS_NAMESPACE=agent-os-custom AGENT_OS_TEST_LOCAL_WORKLOAD=present \ + run_cli destroy --yes 2>&1) || rc=$? + [ "$rc" -eq 3 ] || fail "custom namespace destroy must reach privileged cleanup, got $rc: $out" + assert_call 'akua-input-namespace agent-os-custom' \ + "destroy must render the same namespace-adjusted profile as deploy" + grep -Fq 'kubectl --context orbstack -n agent-os-custom' "$LOG" || \ + fail "destroy must target the custom rendered namespace" + pass "destroy stays consistent with namespace-adjusted rendering" +} + test_non_orbstack_context_is_fail_closed() { local out rc=0 : > "$LOG" @@ -253,4 +266,5 @@ test_namespace_override_updates_the_rendered_profile test_explicit_image_override_is_used_without_retagging test_empty_image_override_uses_content_addressed_default test_destroy_requires_exact_confirmation +test_destroy_uses_the_namespace_adjusted_profile test_non_orbstack_context_is_fail_closed diff --git a/tests/agent-os-packages.test.sh b/tests/agent-os-packages.test.sh index 54f047ae0..f34c38e53 100755 --- a/tests/agent-os-packages.test.sh +++ b/tests/agent-os-packages.test.sh @@ -147,6 +147,7 @@ for invalid_image in \ 'ghcr.io/akua-dev/agent-os@sha256:abc' \ 'ghcr.io/akua-dev/agent-os@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaextra' \ 'invalid@@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ + 'registry.example/org:123/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ 'prefix@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:suffix'; do invalid_inputs="$TMP/invalid-$(printf '%s' "$invalid_image" | tr '/:@' '---').yaml" sed "s|^image: .*|image: $invalid_image|" "$INPUTS" > "$invalid_inputs" @@ -157,6 +158,14 @@ for invalid_image in \ done pass "the portable package requires one complete 64-hex SHA-256 digest" +PORT_INPUTS="$TMP/registry-port.yaml" +sed 's|^image: .*|image: registry.example:5000/org/agent-os@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa|' \ + "$INPUTS" > "$PORT_INPUTS" +akua render --no-agent-mode --package "$FIRSTMATE/package.k" --inputs "$PORT_INPUTS" \ + --out "$TMP/registry-port-rendered" >/dev/null || \ + fail "the portable package must allow a registry authority port" +pass "OCI reference ports are confined to registry authority" + assert_grep 'bin/agent-os-kubernetes.sh install' "$ROOT/docs/kubernetes.md" \ "Kubernetes docs must make the generic installer the default quickstart" assert_grep 'bin/agent-os-kubernetes.sh rollback' "$ROOT/docs/kubernetes.md" \ diff --git a/tools/agent-os/packages/firstmate/crewmate.yaml b/tools/agent-os/packages/firstmate/crewmate.yaml index 487da9145..d44ef00dc 100644 --- a/tools/agent-os/packages/firstmate/crewmate.yaml +++ b/tools/agent-os/packages/firstmate/crewmate.yaml @@ -54,6 +54,8 @@ spec: value: /home/agent - name: HOME value: /home/agent + - name: AGENT_OS_PI_AUTH_FILE + value: /var/run/secrets/agent-os/pi/auth.json readinessProbe: exec: command: @@ -76,7 +78,7 @@ spec: mountPath: /usr/local subPath: usr-local - name: ai-auth - mountPath: /home/agent/.pi/agent + mountPath: /var/run/secrets/agent-os/pi readOnly: true volumes: - name: home diff --git a/tools/agent-os/packages/firstmate/package.k b/tools/agent-os/packages/firstmate/package.k index be8159633..a770ecdc4 100644 --- a/tools/agent-os/packages/firstmate/package.k +++ b/tools/agent-os/packages/firstmate/package.k @@ -17,7 +17,7 @@ schema Input: memoryLimit: str = "8Gi" check: - allowMutableImage or regex.match(image, r"^([a-z0-9]+(([._]|__|-+)[a-z0-9]+)*(:[0-9]+)?/)*[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*(:[A-Za-z0-9_][A-Za-z0-9_.-]{0,127})?@sha256:[0-9a-f]{64}$"), "image must use one complete OCI reference with an immutable @sha256 digest unless the local profile explicitly allows a mutable image" + allowMutableImage or regex.match(image, r"^([a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*(:[0-9]+)?/)?[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*(\/[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*)*(:[A-Za-z0-9_][A-Za-z0-9_.-]{0,127})?@sha256:[0-9a-f]{64}$"), "image must use one complete OCI reference with an immutable @sha256 digest unless the local profile explicitly allows a mutable image" input: Input = ctx.input() From f8bb5cec4bf01e10f8ed3bf39db67f32a0b1c604 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 19:32:50 +0200 Subject: [PATCH 28/56] no-mistakes(review): Captain, harden Kubernetes recovery and reproducible image builds --- .agents/skills/kubernetes-fleet/SKILL.md | 3 +- .github/workflows/agent-os-image.yml | 37 +- Dockerfile | 22 +- THIRD_PARTY_NOTICES.md | 2 + bin/agent-os-crewmate.sh | 44 +- bin/agent-os-kubernetes.sh | 148 +- image/debian.sources | 13 + image/npm/package-lock.json | 3335 +++++++++++++++++ image/npm/package.json | 13 + tests/agent-os-container.test.sh | 48 +- tests/agent-os-kubernetes.test.sh | 190 +- tests/agent-os-packages.test.sh | 10 + .../agent-os/packages/firstmate/crewmate.yaml | 2 + tools/agent-os/packages/firstmate/package.k | 11 +- 14 files changed, 3787 insertions(+), 91 deletions(-) create mode 100644 image/debian.sources create mode 100644 image/npm/package-lock.json create mode 100644 image/npm/package.json diff --git a/.agents/skills/kubernetes-fleet/SKILL.md b/.agents/skills/kubernetes-fleet/SKILL.md index f3150514e..395c278aa 100644 --- a/.agents/skills/kubernetes-fleet/SKILL.md +++ b/.agents/skills/kubernetes-fleet/SKILL.md @@ -32,7 +32,8 @@ Load this skill only when the current firstmate is running in Kubernetes or is e - Use `status` to inspect it, `stop` to remove only the Pod, and `restart` to replace only the Pod on its retained PVC. - The ambiguous `delete` command is rejected. - `purge --yes` is the only operation that destroys a persistent home. -- Before purge, stop the owned Pod and wait for its absence, independently checkpoint or deliver unique work from the stopped home, then annotate the owned PVC with `agent-os.dev/checkpoint-state=clean` and a non-secret RFC3339 `agent-os.dev/checkpoint-at` value. +- Stopping the owned Pod invalidates earlier checkpoint evidence and records a non-secret `agent-os.dev/quiesced-operation` generation only after proving Pod absence. +- Before purge, independently checkpoint or deliver unique work from the stopped home, then annotate the owned PVC with `agent-os.dev/checkpoint-state=clean`, a non-secret RFC3339 `agent-os.dev/checkpoint-at` value, and `agent-os.dev/checkpoint-operation` equal to the current `agent-os.dev/quiesced-operation` value. - Purge verifies exact installation and crewmate ownership, displays the target, requires its own confirmation, and records requested and completed phases in `AGENT_OS_PURGE_EVIDENCE_FILE` or `$FM_HOME/data/crewmate-purge-evidence.log`. - Purge evidence contains only time, namespace, crewmate ID, resource names, phase, and checkpoint time. - Never mount the primary home into a child Pod. diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index bedcb3c3a..c3a1d168a 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -9,7 +9,6 @@ on: permissions: contents: read - packages: write concurrency: group: agent-os-image-${{ github.ref }} @@ -19,8 +18,9 @@ env: IMAGE: ghcr.io/akua-dev/agent-os jobs: - build: - name: Build linux/amd64 and linux/arm64 + validate: + if: github.event_name == 'pull_request' + name: Validate linux/amd64 and linux/arm64 runs-on: ubuntu-latest steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 @@ -29,8 +29,32 @@ jobs: - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f + - name: Build without publication credentials + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: false + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: mode=max + sbom: true + + publish: + if: github.event_name == 'push' + name: Publish linux/amd64 and linux/arm64 + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 + + - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 + + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f + - name: Log in to GHCR - if: github.event_name != 'pull_request' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 with: registry: ghcr.io @@ -47,13 +71,13 @@ jobs: type=raw,value=latest,enable={{is_default_branch}} type=ref,event=tag - - name: Build and optionally publish + - name: Build and publish id: build uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 with: context: . platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} + push: true tags: ${{ steps.metadata.outputs.tags }} labels: ${{ steps.metadata.outputs.labels }} cache-from: type=gha @@ -62,7 +86,6 @@ jobs: sbom: true - name: Record published image digest - if: github.event_name != 'pull_request' run: | { echo '### Agent OS image' diff --git a/Dockerfile b/Dockerfile index 153386b7c..77718e545 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,10 @@ ARG BUN_VERSION=1.3.14 ARG AKUA_VERSION=0.8.25 ARG K9S_VERSION=0.51.0 +COPY image/debian.sources /etc/apt/sources.list.d/debian.sources + +RUN echo "9767ac71230276e282fdb39a087c889a277835b47751a0c0e5a9da0e8352e289 /etc/apt/sources.list.d/debian.sources" | sha256sum -c - + RUN apt-get update \ && apt-get install -y --no-install-recommends \ bash \ @@ -136,13 +140,17 @@ RUN set -eu; \ mkdir -p /usr/share/licenses/k9s; \ curl -fsSL "https://raw.githubusercontent.com/derailed/k9s/v${K9S_VERSION}/LICENSE" -o /usr/share/licenses/k9s/LICENSE -RUN npm install --global \ - @earendil-works/pi-coding-agent@0.80.6 \ - gh-axi@0.1.27 \ - chrome-devtools-axi@0.1.26 \ - lavish-axi@0.1.40 \ - tasks-axi@0.2.2 \ - quota-axi@0.1.5 +COPY image/npm/package.json image/npm/package-lock.json /opt/agent-os-npm/ + +RUN echo "3646e31389155fbce155c828d8db46bc60ff2976c2d8d29e6633f260f56fd06d /opt/agent-os-npm/package.json" | sha256sum -c - \ + && echo "d815b10a3ea0bf79d55a9e2245b422cbb401bc9b38f27a4575a639ce2caabe1f /opt/agent-os-npm/package-lock.json" | sha256sum -c - \ + && npm ci --omit=dev --ignore-scripts --no-audit --no-fund --prefix /opt/agent-os-npm \ + && mkdir -p /usr/local/lib/node_modules \ + && cp -a /opt/agent-os-npm/node_modules/. /usr/local/lib/node_modules/ \ + && for command in pi gh-axi chrome-devtools-axi lavish-axi tasks-axi quota-axi; do \ + ln -s "/usr/local/lib/node_modules/.bin/$command" "/usr/local/bin/$command"; \ + done \ + && rm -rf /opt/agent-os-npm ENV FM_HOME=/home/agent \ HOME=/home/agent \ diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 45bbf9986..01b6ef8b5 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -4,6 +4,8 @@ The Agent OS image uses the official multi-architecture `node:24-trixie-slim` base image pinned to index digest `sha256:366fdef91728b1b7fa18c84fba63b6e79ed77b7e10cc206878e9705da4d7b169`. The reviewed Docker Node source revision is . +System packages resolve only from the Debian and Debian security snapshots dated `20260624T235959Z` in `image/debian.sources`. +Runtime npm packages resolve from the committed `image/npm/package-lock.json`, including registry integrity checks for every artifact. ## Herdr diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index a4368e327..e5cf36242 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -14,6 +14,7 @@ AI_SECRET=${AGENT_OS_AI_SECRET:-} KUBECTL=${AGENT_OS_KUBECTL:-kubectl} TEMPLATE=${AGENT_OS_CREWMATE_TEMPLATE:-/opt/agent-os/tools/agent-os/packages/firstmate/crewmate.yaml} INSTALLATION_ID="agent-os-firstmate:$NAMESPACE" +OPERATION_ID=${AGENT_OS_OPERATION_ID:-"$(date -u '+%Y%m%d%H%M%S')-$$-$RANDOM"} if [ ! -f "$TEMPLATE" ]; then TEMPLATE="$ROOT/tools/agent-os/packages/firstmate/crewmate.yaml" @@ -22,6 +23,10 @@ fi case "$ID" in ''|*[!a-z0-9-]*|-*|*-) echo "error: invalid crewmate id '$ID'" >&2; exit 2 ;; esac +case "$OPERATION_ID" in + ''|*[!a-z0-9.-]*|[.-]*|*[-.]) echo "error: invalid operation id" >&2; exit 2 ;; +esac +[ "${#OPERATION_ID}" -le 63 ] || { echo "error: invalid operation id" >&2; exit 2; } KUBECTL_ARGS=() if [ -n "${AGENT_OS_CONTEXT:-}" ]; then @@ -48,12 +53,12 @@ resource_identity() { pod_record() { kube get pod "$POD" --ignore-not-found \ - -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.uid}' + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}{"\t"}{.metadata.uid}' } pvc_record() { kube get pvc "$PVC" --ignore-not-found \ - -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-state}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-at}' + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-state}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-at}{"\t"}{.metadata.annotations.agent-os\.dev/quiesced-operation}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-operation}' } require_owned_or_absent() { @@ -101,20 +106,23 @@ render_resources() { -e "s|__AGENT_OS_IMAGE__|$IMAGE|g" \ -e "s|__AGENT_OS_IMAGE_PULL_POLICY__|$IMAGE_PULL_POLICY|g" \ -e "s|__AGENT_OS_AI_SECRET__|$AI_SECRET|g" \ + -e "s|__AGENT_OS_OPERATION_ID__|$OPERATION_ID|g" \ "$TEMPLATE" } cleanup_new_owned_pod() { - local before_uid=$1 after current identity after_uid + local before_uid=$1 after current identity after_operation after_uid after=$(pod_record) if [ -z "$after" ]; then echo "partial state: crewmate apply left no Pod; persistent home retained" >&2 return fi identity=$(printf '%s' "$after" | cut -f1-4) - after_uid=$(printf '%s' "$after" | cut -f5) - if [ "$identity" != "$EXPECTED_POD" ] || [ -z "$after_uid" ] || [ -n "$before_uid" ]; then - echo "partial state: Pod was not proven newly created and owned; persistent home retained" >&2 + after_operation=$(printf '%s' "$after" | cut -f5) + after_uid=$(printf '%s' "$after" | cut -f6) + if [ "$identity" != "$EXPECTED_POD" ] || [ "$after_operation" != "$OPERATION_ID" ] || \ + [ -z "$after_uid" ] || [ -n "$before_uid" ]; then + echo "partial state: replacement or ownership mismatch retained; persistent home retained" >&2 return fi current=$(pod_record) @@ -123,14 +131,18 @@ cleanup_new_owned_pod() { return fi echo "partial state: removing newly created owned Pod '$POD' uid=$after_uid; persistent home retained" >&2 - kube delete pod "$POD" --ignore-not-found --wait=false + if ! printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s"}}\n' "$after_uid" | \ + kube delete --raw "/api/v1/namespaces/$NAMESPACE/pods/$POD" -f -; then + echo "partial state: UID-precondition rejected; replacement or ownership mismatch retained" >&2 + return + fi kube wait --for=delete "pod/$POD" --timeout=180s } apply_and_wait() { local before before_uid before=$(pod_record) - before_uid=$(printf '%s' "$before" | cut -f5) + before_uid=$(printf '%s' "$before" | cut -f6) if ! render_resources | kube apply -f -; then cleanup_new_owned_pod "$before_uid" echo "error: crewmate resources were only partially applied" >&2 @@ -169,6 +181,14 @@ stop_owned_pod() { kube wait --for=delete "pod/$POD" --timeout=180s } +invalidate_checkpoint_evidence() { + local pvc + pvc=$(require_owned_pvc_or_absent) + [ -n "$pvc" ] || return + kube annotate pvc "$PVC" agent-os.dev/checkpoint-state=pending agent-os.dev/checkpoint-at- \ + agent-os.dev/quiesced-operation="$OPERATION_ID" agent-os.dev/checkpoint-operation- --overwrite +} + record_purge() { local phase=$1 checkpoint_at=$2 timestamp timestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ') @@ -196,12 +216,14 @@ case "$COMMAND" in [ -z "$CONFIRM" ] || { echo "usage: $0 stop " >&2; exit 2; } require_owned_pvc_or_absent >/dev/null stop_owned_pod + invalidate_checkpoint_evidence ;; restart) [ -z "$CONFIRM" ] || { echo "usage: $0 restart " >&2; exit 2; } validate_ai_grant preflight_existing_home >/dev/null stop_owned_pod + invalidate_checkpoint_evidence apply_and_wait ;; purge) @@ -217,6 +239,8 @@ case "$COMMAND" in pvc=$(preflight_existing_home) checkpoint_state=$(printf '%s' "$pvc" | cut -f5) checkpoint_at=$(printf '%s' "$pvc" | cut -f6) + quiesced_operation=$(printf '%s' "$pvc" | cut -f7) + checkpoint_operation=$(printf '%s' "$pvc" | cut -f8) if [[ ! "$checkpoint_at" =~ ^[0-9]{4}-(0[1-9]|1[0-2])-([0-2][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]Z$ ]]; then echo "error: purge requires a valid non-secret checkpoint timestamp" >&2 exit 2 @@ -225,6 +249,10 @@ case "$COMMAND" in echo "error: purge requires agent-os.dev/checkpoint-state=clean on '$PVC'" >&2 exit 2 fi + if [ -z "$quiesced_operation" ] || [ "$checkpoint_operation" != "$quiesced_operation" ]; then + echo "error: purge requires checkpoint evidence created for the current quiesced PVC generation" >&2 + exit 2 + fi evidence_file=${AGENT_OS_PURGE_EVIDENCE_FILE:-} if [ -z "$evidence_file" ] && [ -n "${FM_HOME:-}" ]; then evidence_file="$FM_HOME/data/crewmate-purge-evidence.log" diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh index 3bf64f43c..921b7ac1e 100755 --- a/bin/agent-os-kubernetes.sh +++ b/bin/agent-os-kubernetes.sh @@ -14,14 +14,16 @@ NAMESPACE=${REQUESTED_NAMESPACE:-agent-os} AKUA=${AGENT_OS_AKUA:-akua} KUBECTL=${AGENT_OS_KUBECTL:-kubectl} OUT=$(mktemp -d) +RENDER_INPUTS=$(mktemp) CONFIRMED=0 DELETE_NAMESPACE=0 MANAGES_NAMESPACE=0 DESIRED_RBAC=none INSTALLATION_ID= +OPERATION_ID=${AGENT_OS_OPERATION_ID:-"$(date -u '+%Y%m%d%H%M%S')-$$-$RANDOM"} cleanup() { - rm -rf "$OUT" + rm -rf "$OUT" "$RENDER_INPUTS" } trap cleanup EXIT @@ -43,6 +45,10 @@ if [ -z "$CONTEXT" ]; then echo "error: set AGENT_OS_CONTEXT to the Kubernetes context to operate" >&2 exit 2 fi +case "$OPERATION_ID" in + ''|*[!a-z0-9.-]*|[.-]*|*[-.]) echo "error: invalid operation id" >&2; exit 2 ;; +esac +[ "${#OPERATION_ID}" -le 63 ] || { echo "error: invalid operation id" >&2; exit 2; } render_has_kind() { grep -R -Eq "^kind:[[:space:]]*$1$" "$OUT" @@ -58,7 +64,13 @@ render() { echo "error: Akua renderer '$AKUA' is required for Kubernetes package operations" >&2 exit 2 fi - "$AKUA" render --no-agent-mode --package "$PACKAGE" --inputs "$INPUTS" --out "$OUT" + if grep -Eq '^[[:space:]]*operationId:' "$INPUTS"; then + echo "error: operationId is reserved for the lifecycle helper" >&2 + exit 2 + fi + cp "$INPUTS" "$RENDER_INPUTS" + printf '\noperationId: %s\n' "$OPERATION_ID" >> "$RENDER_INPUTS" + "$AKUA" render --no-agent-mode --package "$PACKAGE" --inputs "$RENDER_INPUTS" --out "$OUT" statefulset=$(rendered_statefulset) statefulset_count=$(printf '%s\n' "$statefulset" | sed '/^$/d' | wc -l | tr -d ' ') if [ "$statefulset_count" -ne 1 ]; then @@ -199,9 +211,74 @@ delete_namespace_rbac() { fi } +resource_observation() { + local scope=$1 kind=$2 name=$3 + if [ "$scope" = cluster ]; then + "$KUBECTL" --context "$CONTEXT" get "$kind" "$name" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.uid}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}{"\t"}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\t"}{.metadata.finalizers}' + else + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get "$kind" "$name" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.uid}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}{"\t"}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\t"}{.metadata.finalizers}' + fi +} + +lifecycle_command() { + printf 'AGENT_OS_CONTEXT=%q AGENT_OS_NAMESPACE=%q AGENT_OS_PACKAGE=%q AGENT_OS_INPUTS=%q AGENT_OS_AKUA=%q AGENT_OS_KUBECTL=%q %q %q' \ + "$CONTEXT" "$NAMESPACE" "$PACKAGE" "$INPUTS" "$AKUA" "$KUBECTL" "$0" "$COMMAND" +} + +report_partial_observation() { + local kind=$1 name=$2 scope=$3 record managed installation uid operation ready finalizers prefix + if ! record=$(resource_observation "$scope" "$kind" "$name"); then + echo "partial apply: $kind/$name observation=unavailable expected-operation=$OPERATION_ID" >&2 + return + fi + if [ -z "$record" ]; then + echo "partial apply: $kind/$name observed=absent expected-operation=$OPERATION_ID" >&2 + return + fi + managed=$(printf '%s' "$record" | cut -f2) + installation=$(printf '%s' "$record" | cut -f3) + uid=$(printf '%s' "$record" | cut -f4) + operation=$(printf '%s' "$record" | cut -f5) + ready=$(printf '%s' "$record" | cut -f6) + finalizers=$(printf '%s' "$record" | cut -f7-) + [ -n "$ready" ] || ready=unknown + [ -n "$finalizers" ] || finalizers='[]' + prefix='partial apply' + [ "$kind" != ClusterRoleBinding ] || prefix='residual-authority' + echo "$prefix: $kind/$name uid=$uid operation=$operation ready=$ready ownership=$managed installation=$installation finalizers=$finalizers" >&2 +} + +report_partial_apply() { + local phase=$1 file kind name scope + echo "incomplete: primary $phase failed; automatic cleanup withheld pending exact recovery" >&2 + while IFS= read -r file; do + [ -n "$file" ] || continue + kind=$(rendered_resource_field "$file" kind) + name=$(rendered_resource_field "$file" name) + scope=namespaced + case "$kind" in + ClusterRoleBinding) continue ;; + Namespace) scope=cluster ;; + esac + report_partial_observation "$kind" "$name" "$scope" + done < <(find "$OUT" -type f -name '*.yaml' -print) + report_partial_observation Pod agent-os-firstmate-0 namespaced + report_partial_observation ClusterRoleBinding "agent-os-firstmate-$NAMESPACE" cluster + echo "safe recovery: $(lifecycle_command)" >&2 + echo "privileged cleanup if the reported grant is stale: $(cleanup_command)" >&2 + return 3 +} + apply_rendered() { - "$KUBECTL" --context "$CONTEXT" apply -f "$OUT" - "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s + if ! "$KUBECTL" --context "$CONTEXT" apply -f "$OUT"; then + report_partial_apply apply + fi + if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status \ + statefulset/agent-os-firstmate --timeout=180s; then + report_partial_apply rollout + fi } verify_desired_rbac() { @@ -245,8 +322,41 @@ uninstall_command() { [ "$DELETE_NAMESPACE" -eq 0 ] || printf ' --delete-namespace' } +report_retained_observation() { + local kind=$1 name=$2 scope=$3 record managed installation uid operation ready finalizers identity expected + if ! record=$(resource_observation "$scope" "$kind" "$name"); then + echo "retained-unverified: $kind/$name could not be inspected; no further deletion attempted" >&2 + return + fi + [ -n "$record" ] || return + managed=$(printf '%s' "$record" | cut -f2) + installation=$(printf '%s' "$record" | cut -f3) + uid=$(printf '%s' "$record" | cut -f4) + operation=$(printf '%s' "$record" | cut -f5) + ready=$(printf '%s' "$record" | cut -f6) + finalizers=$(printf '%s' "$record" | cut -f7-) + identity="$managed"$'\t'"$installation" + expected="agent-os"$'\t'"$INSTALLATION_ID" + [ -n "$ready" ] || ready=unknown + [ -n "$finalizers" ] || finalizers='[]' + if [ "$identity" = "$expected" ]; then + echo "retained: $kind/$name uid=$uid operation=$operation ready=$ready ownership=$managed installation=$installation finalizers=$finalizers" >&2 + else + echo "retained-unverified: $kind/$name uid=$uid operation=$operation ownership=$managed installation=$installation; no further deletion attempted" >&2 + fi +} + report_retained_resources() { - local file kind name record identity expected finalizers + local failed_target=$1 file kind name scope + echo "failed-target: $failed_target timeout=180s" >&2 + kind=${failed_target%%/*} + name=${failed_target#*/} + scope=namespaced + [ "$kind" != Namespace ] || scope=cluster + report_retained_observation "$kind" "$name" "$scope" + report_retained_observation Pod agent-os-firstmate-0 namespaced + report_retained_observation Role agent-os-firstmate-runtime namespaced + report_retained_observation RoleBinding agent-os-firstmate-runtime namespaced while IFS= read -r file; do [ -n "$file" ] || continue kind=$(rendered_resource_field "$file" kind) @@ -255,39 +365,21 @@ report_retained_resources() { ClusterRoleBinding) continue ;; Namespace) [ "$DELETE_NAMESPACE" -eq 1 ] || continue - if ! record=$("$KUBECTL" --context "$CONTEXT" get namespace "$name" --ignore-not-found \ - -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.finalizers}'); then - echo "retained-unverified: Namespace/$name could not be inspected; no further deletion attempted" >&2 - continue - fi - ;; - *) - if ! record=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get "$kind" "$name" --ignore-not-found \ - -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.finalizers}'); then - echo "retained-unverified: $kind/$name could not be inspected; no further deletion attempted" >&2 - continue - fi + scope=cluster ;; + *) scope=namespaced ;; esac - [ -n "$record" ] || continue - identity=$(printf '%s' "$record" | cut -f1-3) - expected="$name"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" - if [ "$identity" = "$expected" ]; then - finalizers=$(printf '%s' "$record" | cut -f4-) - [ -n "$finalizers" ] || finalizers='[]' - echo "retained: $kind/$name finalizers=$finalizers" >&2 - else - echo "retained-unverified: $kind/$name ownership changed; no further deletion attempted" >&2 - fi + report_retained_observation "$kind" "$name" "$scope" done < <(find "$OUT" -type f -name '*.yaml' -print) echo "safe retry: $(uninstall_command)" >&2 + echo "cluster cleanup if required: $(cleanup_command)" >&2 } bounded_delete_failure() { local target=$1 if [ "$COMMAND" = uninstall ]; then echo "incomplete: timed out deleting $target after 180s" >&2 - report_retained_resources + report_retained_resources "$target" return 3 fi echo "error: timed out deleting $target after 180s" >&2 diff --git a/image/debian.sources b/image/debian.sources new file mode 100644 index 000000000..f12f97c47 --- /dev/null +++ b/image/debian.sources @@ -0,0 +1,13 @@ +Types: deb +URIs: http://snapshot.debian.org/archive/debian/20260624T235959Z +Suites: trixie trixie-updates +Components: main +Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg +Check-Valid-Until: no + +Types: deb +URIs: http://snapshot.debian.org/archive/debian-security/20260624T235959Z +Suites: trixie-security +Components: main +Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg +Check-Valid-Until: no diff --git a/image/npm/package-lock.json b/image/npm/package-lock.json new file mode 100644 index 000000000..270b77504 --- /dev/null +++ b/image/npm/package-lock.json @@ -0,0 +1,3335 @@ +{ + "name": "agent-os-image-runtime", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "agent-os-image-runtime", + "version": "1.0.0", + "dependencies": { + "@earendil-works/pi-coding-agent": "0.80.6", + "chrome-devtools-axi": "0.1.26", + "gh-axi": "0.1.27", + "lavish-axi": "0.1.40", + "quota-axi": "0.1.5", + "tasks-axi": "0.2.2" + } + }, + "node_modules/@earendil-works/pi-coding-agent": { + "version": "0.80.6", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-coding-agent/-/pi-coding-agent-0.80.6.tgz", + "integrity": "sha512-vcfD6tOk402isLl3Cm/qbn2O10TvgroMp1+/fEGM24ZdvETFCdOYv5VZ7m59EI5fPsjfSJh+CpQ5bhBrhfOg7g==", + "hasShrinkwrap": true, + "license": "MIT", + "dependencies": { + "@earendil-works/pi-agent-core": "^0.80.6", + "@earendil-works/pi-ai": "^0.80.6", + "@earendil-works/pi-tui": "^0.80.6", + "@silvia-odwyer/photon-node": "0.3.4", + "chalk": "5.6.2", + "cross-spawn": "7.0.6", + "diff": "8.0.4", + "glob": "13.0.6", + "highlight.js": "10.7.3", + "hosted-git-info": "9.0.3", + "ignore": "7.0.5", + "jiti": "2.7.0", + "minimatch": "10.2.5", + "proper-lockfile": "4.1.2", + "semver": "7.8.0", + "typebox": "1.1.38", + "undici": "8.5.0", + "yaml": "2.9.0" + }, + "bin": { + "pi": "dist/cli.js" + }, + "engines": { + "node": ">=22.19.0" + }, + "optionalDependencies": { + "@mariozechner/clipboard": "0.3.9" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@anthropic-ai/sdk": { + "version": "0.91.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.91.1.tgz", + "integrity": "sha512-LAmu761tSN9r66ixvmciswUj/ZC+1Q4iAfpedTfSVLeswRwnY3n2Nb6Tsk+cLPP28aLOPWeMgIuTuCcMC6W/iw==", + "license": "MIT", + "dependencies": { + "json-schema-to-ts": "^3.1.1" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-crypto/crc32": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-5.2.0.tgz", + "integrity": "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/client-bedrock-runtime": { + "version": "3.1048.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-runtime/-/client-bedrock-runtime-3.1048.0.tgz", + "integrity": "sha512-u+NT61JZEkRFtpL0CAw1N1dwxnaLgwVXQl/zjJxTGgLyS/jTIdg2SdoEoCTHxgDyCnqa1HEi9QOoE9/pYRNpOQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/credential-provider-node": "^3.972.42", + "@aws-sdk/eventstream-handler-node": "^3.972.16", + "@aws-sdk/middleware-eventstream": "^3.972.12", + "@aws-sdk/middleware-websocket": "^3.972.19", + "@aws-sdk/token-providers": "3.1048.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/fetch-http-handler": "^5.4.2", + "@smithy/node-http-handler": "^4.7.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/core": { + "version": "3.974.11", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.974.11.tgz", + "integrity": "sha512-QpnINq5FZH6EOaDEkmHdT7eUunbvD27pDNQypaWjFyYz7Zl1q3UCMQErBZxpmfGfI7MvI2TlK8KTkgNpv8b1ug==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.8", + "@aws-sdk/xml-builder": "^3.972.24", + "@aws/lambda-invoke-store": "^0.2.2", + "@smithy/core": "^3.24.2", + "@smithy/signature-v4": "^5.4.2", + "@smithy/types": "^4.14.1", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.37", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.37.tgz", + "integrity": "sha512-/jpPvEh6f7ntmIzf7dNxoNX6Q8vt8UpesCjbW6mFfk4V1NW6bIy9qxcQ6WbA8As5yQhsZOe+xeNd4xHX8kdY2Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.39", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.39.tgz", + "integrity": "sha512-pIgTpisWyWg7X1bUbzSjuUYosYTD0Ghz2M0hkSTmb3a6i3qV3uU+NYJPI/E2XSC0HcsZh5rsLPzeXrkb2DS0Cg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/fetch-http-handler": "^5.4.2", + "@smithy/node-http-handler": "^4.7.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.41.tgz", + "integrity": "sha512-u2tyjaxJJzW8UtW4SM1ZcPMDwO6y+kV+llvou+Adts0FAKyzes5jG4izQN+KX3yE8ZROpS5y1LJ//xL2iSf76w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/credential-provider-env": "^3.972.37", + "@aws-sdk/credential-provider-http": "^3.972.39", + "@aws-sdk/credential-provider-login": "^3.972.41", + "@aws-sdk/credential-provider-process": "^3.972.37", + "@aws-sdk/credential-provider-sso": "^3.972.41", + "@aws-sdk/credential-provider-web-identity": "^3.972.41", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/credential-provider-imds": "^4.3.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.41.tgz", + "integrity": "sha512-0LBitxXiAiaE5nlFPfpNIww/8FRY/I7WIndWsc9GmNFOM7cE1wNpVNQEGEk9Outg5l8xl+3vybxFyUy4l9q/LQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.42", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.42.tgz", + "integrity": "sha512-D4oon2zbqqsWOJUM99Gm3/ZyJ0IJvTXVN3PyloGb3kQEyI36fjCZheZj422lAgTWWd6TSHgiImLt3RIaLdv3dQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.37", + "@aws-sdk/credential-provider-http": "^3.972.39", + "@aws-sdk/credential-provider-ini": "^3.972.41", + "@aws-sdk/credential-provider-process": "^3.972.37", + "@aws-sdk/credential-provider-sso": "^3.972.41", + "@aws-sdk/credential-provider-web-identity": "^3.972.41", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/credential-provider-imds": "^4.3.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.37", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.37.tgz", + "integrity": "sha512-7nVaHBUaWIddASYfVaA9O4D5ZVjewU3sCol9WqZPGfW0nR+0WqE0xHZnD/U2L33PlOB8KNXGKZ6wOES/QijKzg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.41.tgz", + "integrity": "sha512-IOWAWEHe5LkjSKkkUUX9ciV6Y1scHTsnfEkdt5yyC4Slrc7AGbkLPrpntjqh18ksJAMOaVhoBsO8p2WyTcY2wQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/token-providers": "3.1048.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.41.tgz", + "integrity": "sha512-mbACk9Yypa8nm4iGZLs0PofOXEcTDOUw6wDnsPXNDNSd2WNXs1tSo+6nc/fh0jLYdfVZThhBL98PHW4aXFsG5A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/eventstream-handler-node": { + "version": "3.972.16", + "resolved": "https://registry.npmjs.org/@aws-sdk/eventstream-handler-node/-/eventstream-handler-node-3.972.16.tgz", + "integrity": "sha512-yedpPgKftqjU5SlPFHfqWpOw6xSCRieWRG1euWOlXn4WJxt2VX92VprCa2PpSOXjVCAeK6dTjW9eJRXVig9yGA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/middleware-eventstream": { + "version": "3.972.12", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-eventstream/-/middleware-eventstream-3.972.12.tgz", + "integrity": "sha512-tHTHHCHNrq6XklQvlzHBDJG4Iuhh7NVPRdtmvP+nHFA+5sxPlIDzlAHHgfoYHGvT3NXP1yVP/L5c3opUn6T3Qg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/middleware-websocket": { + "version": "3.972.19", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-websocket/-/middleware-websocket-3.972.19.tgz", + "integrity": "sha512-mkEhOGYozqKQkbFaVrjwr0faiwwZza1v5/jSY6Tucm3bD+uKTazIUH/4Yo6aMnQD2ua2W9cMP6s8mvwTcjtqHw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/fetch-http-handler": "^5.4.2", + "@smithy/signature-v4": "^5.4.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/nested-clients": { + "version": "3.997.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.9.tgz", + "integrity": "sha512-jPR3rnmRI4hWYyzfmTGBr7NblMp8QYYeflHXba1H6+7CGrWVqWKQzaXFQ4qbExqPRsXN3T3L3JxFhr6aouXUGQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/signature-v4-multi-region": "^3.996.27", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/fetch-http-handler": "^5.4.2", + "@smithy/node-http-handler": "^4.7.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.27", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.27.tgz", + "integrity": "sha512-0Phbz4t6HI3D3skxvG2uI+VWU034/nSIw1T8d+FPzzQG9EQTrw94o9mOKO2Gv3n3Oc8P7JD7RAUxkoneLWv5Eg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/signature-v4": "^5.4.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/token-providers": { + "version": "3.1048.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1048.0.tgz", + "integrity": "sha512-k0y/GcuesuSfWyUM0WamrGyeZmltRYaPbHO82UDA6mZ/doB+FOHKutikPAtSXMn/hDz970cF+iRuuiYO9VEbAA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/types": { + "version": "3.973.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.973.8.tgz", + "integrity": "sha512-gjlAdtHMbtR9X5iIhVUvbVcy55KnznpC6bkDUWW9z915bi0ckdUr5cjf16Kp6xq0bP5HBD2xzgbL9F9Quv5vUw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/util-locate-window": { + "version": "3.965.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.965.5.tgz", + "integrity": "sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws-sdk/xml-builder": { + "version": "3.972.24", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.24.tgz", + "integrity": "sha512-V8z5YcDPfsvzrBlj0xR1vhRtocblhYbqdreCJB/voGd4Sr5zjNAeWxexbnqVtskTJe0vFb5KMqbSL++ePl+zRw==", + "license": "Apache-2.0", + "dependencies": { + "@nodable/entities": "2.1.0", + "@smithy/types": "^4.14.1", + "fast-xml-parser": "5.7.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@aws/lambda-invoke-store": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.4.tgz", + "integrity": "sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@babel/runtime": { + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", + "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-agent-core": { + "version": "0.80.6", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.80.6.tgz", + "license": "MIT", + "dependencies": { + "@earendil-works/pi-ai": "^0.80.6", + "ignore": "7.0.5", + "typebox": "1.1.38", + "yaml": "2.9.0" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-ai": { + "version": "0.80.6", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.80.6.tgz", + "license": "MIT", + "dependencies": { + "@anthropic-ai/sdk": "0.91.1", + "@aws-sdk/client-bedrock-runtime": "3.1048.0", + "@google/genai": "1.52.0", + "@mistralai/mistralai": "2.2.6", + "@opentelemetry/api": "1.9.0", + "@smithy/node-http-handler": "4.7.3", + "http-proxy-agent": "7.0.2", + "https-proxy-agent": "7.0.6", + "openai": "6.26.0", + "partial-json": "0.1.7", + "typebox": "1.1.38" + }, + "bin": { + "pi-ai": "./dist/cli.js" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-tui": { + "version": "0.80.6", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.80.6.tgz", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "1.6.0", + "marked": "18.0.5" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@google/genai": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.52.0.tgz", + "integrity": "sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "google-auth-library": "^10.3.0", + "p-retry": "^4.6.2", + "protobufjs": "^7.5.4", + "ws": "^8.18.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@modelcontextprotocol/sdk": "^1.25.2" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/sdk": { + "optional": true + } + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard/-/clipboard-0.3.9.tgz", + "integrity": "sha512-ABnA53mdfkGZwOFUdZNv2S0CWGO/EIuPj8Vv9xmBFmSYg/qFc7ihO6q5FcQjvoE67kZpWkEc4AhD6B/os04yuA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@mariozechner/clipboard-darwin-arm64": "0.3.9", + "@mariozechner/clipboard-darwin-universal": "0.3.9", + "@mariozechner/clipboard-darwin-x64": "0.3.9", + "@mariozechner/clipboard-linux-arm64-gnu": "0.3.9", + "@mariozechner/clipboard-linux-arm64-musl": "0.3.9", + "@mariozechner/clipboard-linux-riscv64-gnu": "0.3.9", + "@mariozechner/clipboard-linux-x64-gnu": "0.3.9", + "@mariozechner/clipboard-linux-x64-musl": "0.3.9", + "@mariozechner/clipboard-win32-arm64-msvc": "0.3.9", + "@mariozechner/clipboard-win32-x64-msvc": "0.3.9" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-darwin-arm64": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-arm64/-/clipboard-darwin-arm64-0.3.9.tgz", + "integrity": "sha512-BfgV7vCEWZwJwZJw03r6bP5+tf0iI/ANuQYCxi9RNn7FrWB3yzGuMKCrNLRl6V761vXRdL8+OqZ0wd4TqlsNOQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-darwin-universal": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-universal/-/clipboard-darwin-universal-0.3.9.tgz", + "integrity": "sha512-BGGR4iA9Z2shAjI65eI5xtyb3LYNlDW9X3gxKxDbqtbnREohsrqznov6zpKoIrsRWpzlYVEdKphS7ksJ0/ndSQ==", + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-darwin-x64": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-darwin-x64/-/clipboard-darwin-x64-0.3.9.tgz", + "integrity": "sha512-4kURmCbS6nt8uYhtmWpUcJWyPHfmAr5dTpXD1nO3pIfa+TSQ9DbrGOYCKH+aEFW47XhQ4Vp8ZTszie+wfFvDKg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-linux-arm64-gnu": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-arm64-gnu/-/clipboard-linux-arm64-gnu-0.3.9.tgz", + "integrity": "sha512-g59OkUGP2DDfCOIKypHeYgv2M55u/cKvXa5dSxFbEJ34XvIQMdcVmpKCkGUro3ZgefXiGVdwguvTMQGpHWzIXw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-linux-arm64-musl": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-arm64-musl/-/clipboard-linux-arm64-musl-0.3.9.tgz", + "integrity": "sha512-AGuJdgKsmJdm4Pych7kv3sqe591ERRaAHW3xjLooiFzn8J+PxUyof++7YZrB5Y5tpnTO+K18Og3taj2NpluCRQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-linux-riscv64-gnu": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-riscv64-gnu/-/clipboard-linux-riscv64-gnu-0.3.9.tgz", + "integrity": "sha512-DXBEAiuMpk7dhS1a9NzNxVAFi1vaKoPu7rQNgY8LIDLGrK3lnIp3nT10DUum+PKVJoJppIP+NAA8IZe4DMNDPw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-linux-x64-gnu": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-x64-gnu/-/clipboard-linux-x64-gnu-0.3.9.tgz", + "integrity": "sha512-WORrMLd6EpElEME7JRKfSaY34nW1P5LbdgK5YNCS1ncG2LqmITsSMEJ8nh2mpvxb3TxqbOOKgY7k9eMJYlW9Mw==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-linux-x64-musl": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-linux-x64-musl/-/clipboard-linux-x64-musl-0.3.9.tgz", + "integrity": "sha512-/DHn+1DrfL6oRaPPWXaOKvonFFrni666fxd+zFqiQEfvBH0tsHVWjq9iqBk0oDp0qaPA72lIMy5BptxISBEhZQ==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-win32-arm64-msvc": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-win32-arm64-msvc/-/clipboard-win32-arm64-msvc-0.3.9.tgz", + "integrity": "sha512-O5FHD3ErkMwMhNzAfu3ggy0ug4z7btZuoQgwwxlzPrwV2bxlD6WDpqBY4NCgICAgZdDKdp+loUEKVAVt8aYnhQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mariozechner/clipboard-win32-x64-msvc": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@mariozechner/clipboard-win32-x64-msvc/-/clipboard-win32-x64-msvc-0.3.9.tgz", + "integrity": "sha512-ihQC3EufqEY81vhXBgVBtK4prL+wc62zJsSvxrgz7K1hsdt6OObz6v9p3Rn1OG3GJksTTKMJF0u/guMISHPhSA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@mistralai/mistralai": { + "version": "2.2.6", + "resolved": "https://registry.npmjs.org/@mistralai/mistralai/-/mistralai-2.2.6.tgz", + "integrity": "sha512-W8pX7zHxjJvMIpw8JMxeJEleapXX0Q9NPszdNzqkM3MIEoIGPObdodujj+WHteXEvGfaP/AMwlNyRfEzSY6dQQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.40.0", + "ws": "^8.18.0", + "zod": "^3.25.0 || ^4.0.0", + "zod-to-json-schema": "^3.25.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + } + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@nodable/entities": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-2.1.0.tgz", + "integrity": "sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.41.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.41.1.tgz", + "integrity": "sha512-/UhIkaZgPutTFmQ7RnIJGgDXZmtEJ7Dvi86xNTFWcnRxVRNk/aotsqDJYeEvDP+FSMB2SdW+pQzNMcWP0rwuNA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@protobufjs/utf8": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", + "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@silvia-odwyer/photon-node": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@silvia-odwyer/photon-node/-/photon-node-0.3.4.tgz", + "integrity": "sha512-bnly4BKB3KDTFxrUIcgCLbaeVVS8lrAkri1pEzskpmxu9MdfGQTy8b8EgcD83ywD3RPMsIulY8xJH5Awa+t9fA==", + "license": "Apache-2.0" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/core": { + "version": "3.24.3", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.24.3.tgz", + "integrity": "sha512-Ep/7tPamGY8mgESE3LyLKtxJyy6U52WWAqr/3wial47Sj4u3PiIF73AOGI27UyLy9duTkhZbgzodOfLV4TduZg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/crc32": "5.2.0", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/credential-provider-imds": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.3.3.tgz", + "integrity": "sha512-I2Bti0DKFo2IJyN28ijCsx51BAumEYR4/1yZ1FXyBygy9MqbnMqCev4JPth/MbpRfBSRAX35hITSnAdJRo1u5w==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.24.3", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/fetch-http-handler": { + "version": "5.4.3", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.4.3.tgz", + "integrity": "sha512-F+DRf8IJazRJgYog2A/yJK7eYVc0rqTlRzO+5ZxjJd4WkZoKz0IJRncf7G6t1pdVT3kryJcwuTFhN1c5m6N47A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.24.3", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/node-http-handler": { + "version": "4.7.3", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.7.3.tgz", + "integrity": "sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.24.3", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/signature-v4": { + "version": "5.4.3", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.4.3.tgz", + "integrity": "sha512-53+75QuPl6DL+ct6vVEB51FDO5oulXr20TPV46VvJZg76lIlXNWfxi8j+G2V/t0I2qxCBOa3vX/8bmjrpFVo9g==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.24.3", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/types": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.14.2.tgz", + "integrity": "sha512-P+otAxbV4CqBybp7EkcJCrig63yE2E7PuNVOmilVMRcx/O+QDzGULTrKsq4DV13gSfak9ObPrWaHl/9bL5YcWw==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/@types/node": { + "version": "22.19.19", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.19.tgz", + "integrity": "sha512-dyh/xO2Fh5bYrfWaaqGrRQQGkNdmYw6AmaAUvYeUMNTWQtvb796ikLdmTchRmOlOiIJ1TDXfWgVx1QkUlQ6Hew==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/brace-expansion": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", + "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/chalk": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/diff": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.4.tgz", + "integrity": "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/fast-xml-builder": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.2.0.tgz", + "integrity": "sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.5.0", + "xml-naming": "^0.1.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/fast-xml-parser": { + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.7.3.tgz", + "integrity": "sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^2.1.0", + "fast-xml-builder": "^1.1.7", + "path-expression-matcher": "^1.5.0", + "strnum": "^2.2.3" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/gaxios": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.4.tgz", + "integrity": "sha512-bTIgTsM2bWn3XklZISBTQX7ZSddGW+IO3bMdGaemHZ3tbqExMENHLx6kKZ/KlejgrMtj8q7wBItt51yegqalrA==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/gcp-metadata": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^7.0.0", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/get-east-asian-width": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz", + "integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/google-auth-library": { + "version": "10.6.2", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.6.2.tgz", + "integrity": "sha512-e27Z6EThmVNNvtYASwQxose/G57rkRuaRbQyxM2bvYLLX/GqWZ5chWq2EBoUchJbCc57eC9ArzO5wMsEmWftCw==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "8.1.2", + "google-logging-utils": "1.1.3", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/highlight.js": { + "version": "10.7.3", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", + "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==", + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/hosted-git-info": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-9.0.3.tgz", + "integrity": "sha512-Hc+ghLoSt6QaYZUv0WBiIvmMDZuZZ7oaDvdH8MbfOO4lOsxdXLEvuC6ePoGs9H1X9oCLyq6+NVN0MKqD+ydxyg==", + "license": "ISC", + "dependencies": { + "lru-cache": "^11.1.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/jiti": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.7.0.tgz", + "integrity": "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==", + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/lru-cache": { + "version": "11.4.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.4.0.tgz", + "integrity": "sha512-W+R+kFL4HgVxONq2bhXPi3bGpzGe/yEhVOp233qw9wCRtgncJ15P3bC+e4zZMu4Cq7d+WAJjXGW0uUkifhcatA==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/marked": { + "version": "18.0.5", + "resolved": "https://registry.npmjs.org/marked/-/marked-18.0.5.tgz", + "integrity": "sha512-S6GcvALHg6K4ohtu4E7x0a1AqhAjp6cV8KhLSyN9qVapnzJkusVBxZRcIU9AeYsbe6P1hKDusSbEOzGyyuce6w==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/openai": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.26.0.tgz", + "integrity": "sha512-zd23dbWTjiJ6sSAX6s0HrCZi41JwTA1bQVs0wLQPZ2/5o2gxOJA5wh7yOAUgwYybfhDXyhwlpeQf7Mlgx8EOCA==", + "license": "Apache-2.0", + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/p-retry/node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/partial-json": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/partial-json/-/partial-json-0.1.7.tgz", + "integrity": "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/path-expression-matcher": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.5.0.tgz", + "integrity": "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/proper-lockfile/node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/protobufjs": { + "version": "7.6.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.4.tgz", + "integrity": "sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/semver": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.0.tgz", + "integrity": "sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/strnum": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.3.0.tgz", + "integrity": "sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/typebox": { + "version": "1.1.38", + "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.1.38.tgz", + "integrity": "sha512-pZ0aQPmMmXoUvSbeuWf/Hzsc+avNw/Zd6VeE8CFgkVGWyuHPJvqeJJDeJqLve+K70LvjYIoleGcoJHPT17cWoA==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/undici": { + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.5.0.tgz", + "integrity": "sha512-xamtWoB1EshgjpmlXd7GGm2VfdDtw1+rD8uhry8pSNW3If6S8E0m2T2+orSKeZXEn/aPJMviCpDBA65WJt8zhg==", + "license": "MIT", + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/xml-naming": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.1.0.tgz", + "integrity": "sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/@earendil-works/pi-coding-agent/node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + }, + "node_modules/@hono/node-server": { + "version": "1.19.14", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz", + "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==", + "license": "MIT", + "engines": { + "node": ">=18.14.1" + }, + "peerDependencies": { + "hono": "^4" + } + }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.29.0.tgz", + "integrity": "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==", + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/@tailwindcss/browser": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@tailwindcss/browser/-/browser-4.2.4.tgz", + "integrity": "sha512-yd+3CxxuF1KDt9Q+405JR3/vyotvS5eMIsZPEynEak/JybvFVn8mVmLjVUxgNmrFB6EGCc89lXBECzIZA+YeXQ==", + "license": "MIT" + }, + "node_modules/@toon-format/toon": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@toon-format/toon/-/toon-2.3.0.tgz", + "integrity": "sha512-/Ew9etdRQKVMnm9fDaCG0JjyAOK/O7T0M97oum1aW4W+UR8ZhVVPBanIV7oWgHBiGlnVxV9M55PWQCHofDV07w==", + "license": "MIT" + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/axi-sdk-js": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/axi-sdk-js/-/axi-sdk-js-0.1.8.tgz", + "integrity": "sha512-N8Qd/9sVBpG8QRVSw0lLevdFuaBfjHHPdcbcj8DCuxr68sA1/c5KCnGEDqO7lI3kt52/bgymCZajmpTqz/rLOw==", + "license": "MIT", + "dependencies": { + "@toon-format/toon": "^2.1.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bundle-name": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", + "integrity": "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==", + "license": "MIT", + "dependencies": { + "run-applescript": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/chokidar": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/chrome-devtools-axi": { + "version": "0.1.26", + "resolved": "https://registry.npmjs.org/chrome-devtools-axi/-/chrome-devtools-axi-0.1.26.tgz", + "integrity": "sha512-ZAquXPHjvQQL+r6rb0rrr0sHvIf6uPPzBCm7NgVpcPYDjBvdkeYfuX0WINLDdIFDga1PTu2jTC8CdcXLLKusFw==", + "license": "MIT", + "dependencies": { + "@modelcontextprotocol/sdk": "^1.12.1", + "@toon-format/toon": "^2.1.0", + "axi-sdk-js": "^0.1.8" + }, + "bin": { + "chrome-devtools-axi": "dist/bin/chrome-devtools-axi.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/cors": { + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", + "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/daisyui": { + "version": "5.6.18", + "resolved": "https://registry.npmjs.org/daisyui/-/daisyui-5.6.18.tgz", + "integrity": "sha512-6y9rboRQl8fosnusy/5pg11yp+H0LtK3YdtfS4Vf2QVnaYOKasNdYm6hDxeiT6bwZPdl4WRPzVIlIApdo2ssBg==", + "license": "MIT", + "funding": { + "url": "https://github.com/saadeghi/daisyui?sponsor=1" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/default-browser": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.5.0.tgz", + "integrity": "sha512-H9LMLr5zwIbSxrmvikGuI/5KGhZ8E2zH3stkMgM5LpOWDutGM2JZaj460Udnf1a+946zc7YBgrqEWwbk7zHvGw==", + "license": "MIT", + "dependencies": { + "bundle-name": "^4.1.0", + "default-browser-id": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/default-browser-id": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.1.tgz", + "integrity": "sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/define-lazy-prop": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", + "integrity": "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/entities": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-8.0.0.tgz", + "integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.0.tgz", + "integrity": "sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "8.5.2", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.5.2.tgz", + "integrity": "sha512-5Kb34ipNX694DH48vN9irak1Qx30nb0PLYHXfJgw4YEjiC3ZEmZJhwOp+VfiCYwFzvFTdB9QkArYS5kXa2cx2A==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.3.tgz", + "integrity": "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gh-axi": { + "version": "0.1.27", + "resolved": "https://registry.npmjs.org/gh-axi/-/gh-axi-0.1.27.tgz", + "integrity": "sha512-SoA5VgSa/hil7/TAQaaAeU4guOCCXKoJLdpOnAO8Bzt5jnZohqUHYJQ1P1C85jwKDN1urQn2RezE2wz5t6va9Q==", + "license": "MIT", + "dependencies": { + "@toon-format/toon": "^2.1.0", + "axi-sdk-js": "^0.1.8" + }, + "bin": { + "gh-axi": "dist/bin/gh-axi.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hono": { + "version": "4.12.30", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.30.tgz", + "integrity": "sha512-emn+JoJjrN9YTpRDS5it/UI2SO9BAE37T6I3d963RxcZ81G9A4pr2SZTEiiaiKbzx+NKRg5BZ89fCL7gCJCUog==", + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ip-address": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", + "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-docker": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", + "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==", + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-inside-container": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", + "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==", + "license": "MIT", + "dependencies": { + "is-docker": "^3.0.0" + }, + "bin": { + "is-inside-container": "cli.js" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/is-wsl": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.1.tgz", + "integrity": "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==", + "license": "MIT", + "dependencies": { + "is-inside-container": "^1.0.0" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jose": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.3.tgz", + "integrity": "sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "license": "BSD-2-Clause" + }, + "node_modules/lavish-axi": { + "version": "0.1.40", + "resolved": "https://registry.npmjs.org/lavish-axi/-/lavish-axi-0.1.40.tgz", + "integrity": "sha512-vISqcfR3KcdInhzJHE8vUM2yaoZbGHY+uxrAUjMC5psyInfJwDvbgiRwK9GZFcCT2Rvs652wZqXtMa8dLCYoCA==", + "license": "MIT", + "dependencies": { + "@tailwindcss/browser": "4.2.4", + "axi-sdk-js": "^0.1.8", + "chokidar": "^4.0.3", + "daisyui": "^5.5.19", + "express": "^5.2.1", + "open": "^10.2.0", + "parse5": "^8.0.1" + }, + "bin": { + "lavish-axi": "dist/cli.mjs" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/open": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/open/-/open-10.2.0.tgz", + "integrity": "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==", + "license": "MIT", + "dependencies": { + "default-browser": "^5.2.1", + "define-lazy-prop": "^3.0.0", + "is-inside-container": "^1.0.0", + "wsl-utils": "^0.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parse5": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz", + "integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==", + "license": "MIT", + "dependencies": { + "entities": "^8.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/quota-axi": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/quota-axi/-/quota-axi-0.1.5.tgz", + "integrity": "sha512-EGaA78dBNnnIu5i2t7JSjhhwZogsAsHKpdG67wrxsVEy/CTw9uuYCBi+DJ3bO6+lrFnzXpDXs4h1NZ6J8DOLfA==", + "license": "MIT", + "dependencies": { + "@toon-format/toon": "2.1.0" + }, + "bin": { + "quota-axi": "dist/bin/quota-axi.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/quota-axi/node_modules/@toon-format/toon": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@toon-format/toon/-/toon-2.1.0.tgz", + "integrity": "sha512-JwWptdF5eOA0HaQxbKAzkpQtR4wSWTEfDlEy/y3/4okmOAX1qwnpLZMmtEWr+ncAhTTY1raCKH0kteHhSXnQqg==", + "license": "MIT" + }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/readdirp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/run-applescript": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.1.0.tgz", + "integrity": "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/tasks-axi": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/tasks-axi/-/tasks-axi-0.2.2.tgz", + "integrity": "sha512-k5kN2RYFAaVkslN5nWKndf6Wmv99EFl0GuQTbDvVNfu0reoDZmBnZV1a+ZYHqb/owQhFcHiUykMbn8mO/I0Utw==", + "license": "MIT", + "dependencies": { + "@toon-format/toon": "^2.1.0", + "axi-sdk-js": "^0.1.7" + }, + "bin": { + "tasks-axi": "dist/bin/tasks-axi.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/wsl-utils": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/wsl-utils/-/wsl-utils-0.1.0.tgz", + "integrity": "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==", + "license": "MIT", + "dependencies": { + "is-wsl": "^3.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + } + } +} diff --git a/image/npm/package.json b/image/npm/package.json new file mode 100644 index 000000000..712959dbe --- /dev/null +++ b/image/npm/package.json @@ -0,0 +1,13 @@ +{ + "name": "agent-os-image-runtime", + "private": true, + "version": "1.0.0", + "dependencies": { + "@earendil-works/pi-coding-agent": "0.80.6", + "chrome-devtools-axi": "0.1.26", + "gh-axi": "0.1.27", + "lavish-axi": "0.1.40", + "quota-axi": "0.1.5", + "tasks-axi": "0.2.2" + } +} diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 1eca072bb..2aaa39a19 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -5,6 +5,8 @@ set -u # shellcheck source=tests/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +IMAGE_WORKFLOW="$ROOT/.github/workflows/agent-os-image.yml" + assert_grep 'FROM node:24-trixie-slim@sha256:366fdef91728b1b7fa18c84fba63b6e79ed77b7e10cc206878e9705da4d7b169' \ "$ROOT/Dockerfile" "image must pin the multi-architecture Node 24 Trixie base" assert_grep 'sha256:366fdef91728b1b7fa18c84fba63b6e79ed77b7e10cc206878e9705da4d7b169' \ @@ -17,13 +19,30 @@ assert_grep 'ARG NO_MISTAKES_VERSION=1.34.0' "$ROOT/Dockerfile" "image must pin assert_grep 'ARG BUN_VERSION=1.3.14' "$ROOT/Dockerfile" "image must pin stable Bun 1.3.14" assert_grep 'ARG AKUA_VERSION=0.8.25' "$ROOT/Dockerfile" "image must pin Akua 0.8.25" assert_grep 'ARG K9S_VERSION=0.51.0' "$ROOT/Dockerfile" "image must pin K9s 0.51.0" +assert_present "$ROOT/image/debian.sources" "image must commit immutable Debian snapshot inputs" +assert_grep 'snapshot.debian.org/archive/debian/20260624T235959Z' "$ROOT/image/debian.sources" \ + "image must use the reviewed immutable Debian snapshot" +assert_grep 'snapshot.debian.org/archive/debian-security/20260624T235959Z' "$ROOT/image/debian.sources" \ + "image must pin the matching Debian security snapshot" +assert_grep 'debian.sources' "$ROOT/Dockerfile" \ + "image build must install and checksum the committed Debian source input" +assert_present "$ROOT/image/npm/package.json" "image runtime npm dependencies must have a committed manifest" +assert_present "$ROOT/image/npm/package-lock.json" "image runtime npm dependencies must have a committed lockfile" +assert_grep '"lockfileVersion": 3' "$ROOT/image/npm/package-lock.json" \ + "image runtime npm lock must use the reproducible current lock format" +assert_grep '"integrity": "sha512-' "$ROOT/image/npm/package-lock.json" \ + "image runtime npm lock must checksum resolved artifacts" +assert_grep 'npm ci --omit=dev --ignore-scripts' "$ROOT/Dockerfile" \ + "image runtime npm installation must consume only the committed lock" +assert_no_grep 'npm install --global' "$ROOT/Dockerfile" \ + "image build must not resolve mutable global npm dependency graphs" assert_grep 'sha256sum -c -' "$ROOT/Dockerfile" "downloaded runtime binaries must be checksum verified" -assert_grep '@earendil-works/pi-coding-agent@0.80.6' "$ROOT/Dockerfile" "image must pin Pi 0.80.6" -assert_grep 'gh-axi@0.1.27' "$ROOT/Dockerfile" "image must pin gh-axi 0.1.27" -assert_grep 'chrome-devtools-axi@0.1.26' "$ROOT/Dockerfile" "image must pin chrome-devtools-axi 0.1.26" -assert_grep 'lavish-axi@0.1.40' "$ROOT/Dockerfile" "image must pin lavish-axi 0.1.40" -assert_grep 'tasks-axi@0.2.2' "$ROOT/Dockerfile" "image must pin tasks-axi 0.2.2" -assert_grep 'quota-axi@0.1.5' "$ROOT/Dockerfile" "image must pin quota-axi 0.1.5" +assert_grep '"@earendil-works/pi-coding-agent": "0.80.6"' "$ROOT/image/npm/package.json" "image must pin Pi 0.80.6" +assert_grep '"gh-axi": "0.1.27"' "$ROOT/image/npm/package.json" "image must pin gh-axi 0.1.27" +assert_grep '"chrome-devtools-axi": "0.1.26"' "$ROOT/image/npm/package.json" "image must pin chrome-devtools-axi 0.1.26" +assert_grep '"lavish-axi": "0.1.40"' "$ROOT/image/npm/package.json" "image must pin lavish-axi 0.1.40" +assert_grep '"tasks-axi": "0.2.2"' "$ROOT/image/npm/package.json" "image must pin tasks-axi 0.2.2" +assert_grep '"quota-axi": "0.1.5"' "$ROOT/image/npm/package.json" "image must pin quota-axi 0.1.5" assert_grep 'ripgrep' "$ROOT/Dockerfile" "image must install ripgrep" assert_grep 'fd-find' "$ROOT/Dockerfile" "image must install fd" assert_grep 'FM_HOME=/home/agent' "$ROOT/Dockerfile" "image must declare the persistent firstmate home" @@ -101,6 +120,17 @@ assert_grep '/usr/share/doc/agent-os/THIRD_PARTY_SOURCES.md' "$ROOT/THIRD_PARTY_ "the Herdr notice must direct image recipients to the bundled source offer" assert_no_grep 'publication is gated on a compliant license path' "$ROOT/THIRD_PARTY_NOTICES.md" \ "the Herdr notice must state the selected compliance path" +if grep -Eq '^ packages: write$' "$IMAGE_WORKFLOW"; then + fail "workflow defaults must keep pull-request validation read-only" +fi +[ "$(grep -c '^ packages: write$' "$IMAGE_WORKFLOW")" -eq 1 ] || \ + fail "only the protected publication job may receive packages write" +assert_grep ' validate:' "$IMAGE_WORKFLOW" \ + "pull requests must use a distinct read-only validation job" +assert_grep ' publish:' "$IMAGE_WORKFLOW" \ + "push and tag publication must use a distinct privileged job" +assert_grep "if: github.event_name == 'push'" "$IMAGE_WORKFLOW" \ + "the packages-write job must be restricted to protected push and tag events" assert_grep 'Section 13' "$ROOT/docs/herdr-compliance.md" \ "the Herdr audit must account for the network-interaction clause" assert_grep 'https://github.com/akua-dev/akua/tree/v0.8.25' "$ROOT/THIRD_PARTY_NOTICES.md" \ @@ -152,8 +182,10 @@ assert_grep 'ghcr.io/akua-dev/agent-os' "$ROOT/.github/workflows/agent-os-image. "release workflow must publish the image expected by the portable package" assert_grep 'linux/amd64,linux/arm64' "$ROOT/.github/workflows/agent-os-image.yml" \ "release workflow must build the two supported container architectures" -assert_grep "push: \${{ github.event_name != 'pull_request' }}" "$ROOT/.github/workflows/agent-os-image.yml" \ - "pull requests must build but never publish images" +[ "$(grep -c '^ push: false$' "$IMAGE_WORKFLOW")" -eq 1 ] || \ + fail "the read-only pull-request job must build without publishing" +[ "$(grep -c '^ push: true$' "$IMAGE_WORKFLOW")" -eq 1 ] || \ + fail "only the protected publication job may push images" assert_grep 'id: build' "$ROOT/.github/workflows/agent-os-image.yml" \ "release workflow must expose its build result" assert_grep 'steps.build.outputs.digest' "$ROOT/.github/workflows/agent-os-image.yml" \ diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index 8b175c41b..250c1d0a4 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -48,6 +48,10 @@ fi if [ "${AGENT_OS_TEST_FAIL_APPLY:-0}" = 1 ] && [[ " $* " = *" apply -f - "* ]]; then exit 1 fi +if [ "${AGENT_OS_TEST_FAIL_GENERIC_APPLY:-0}" = 1 ] && [[ " $* " = *" apply -f "* ]] && \ + [[ " $* " != *" apply -f - "* ]]; then + exit 1 +fi if [ "${AGENT_OS_TEST_FAIL_WAIT:-0}" = 1 ] && [ "${1:-}" = -n ] && [ "${3:-}" = wait ]; then exit 1 fi @@ -69,11 +73,15 @@ case " $* " in absent) ;; owned) printf 'agent-os-crewmate-%s\tagent-os\t%s\tagent-os-firstmate:agent-os-demo' "$id" "$id" - [[ " $* " != *'.metadata.uid'* ]] || printf '\tuid-owned' + [[ " $* " != *'.metadata.uid'* ]] || printf '\toperation-test\tuid-owned' + ;; + replacement) + printf 'agent-os-crewmate-%s\tagent-os\t%s\tagent-os-firstmate:agent-os-demo' "$id" "$id" + [[ " $* " != *'.metadata.uid'* ]] || printf '\tother-operation\tuid-replacement' ;; foreign) printf 'agent-os-crewmate-%s\tother\tother\tother-installation' "$id" - [[ " $* " != *'.metadata.uid'* ]] || printf '\tuid-foreign' + [[ " $* " != *'.metadata.uid'* ]] || printf '\tother-operation\tuid-foreign' ;; esac ;; @@ -81,10 +89,11 @@ case " $* " in id=${AGENT_OS_TEST_CREWMATE_ID:-scout-1} case "${AGENT_OS_TEST_PVC_STATE:-absent}" in absent) ;; - owned) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tpending\t' "$id" "$id" ;; - clean) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tclean\t2026-07-13T12:00:00Z' "$id" "$id" ;; - invalid-checkpoint) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tclean\t2026' "$id" "$id" ;; - foreign) printf 'agent-os-crewmate-%s-home\tother\tother\tother-installation\tclean\t2026-07-13T12:00:00Z' "$id" ;; + owned) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tpending\t\toperation-test\t' "$id" "$id" ;; + clean) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tclean\t2026-07-13T12:00:00Z\toperation-test\toperation-test' "$id" "$id" ;; + stale-clean) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tclean\t2026-07-13T12:00:00Z\toperation-test\told-operation' "$id" "$id" ;; + invalid-checkpoint) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tclean\t2026\toperation-test\toperation-test' "$id" "$id" ;; + foreign) printf 'agent-os-crewmate-%s-home\tother\tother\tother-installation\tclean\t2026-07-13T12:00:00Z\toperation-test\toperation-test' "$id" ;; esac ;; *" get namespace "*" --ignore-not-found -o name "*) @@ -93,9 +102,16 @@ case " $* " in *) printf 'namespace/%s\n' "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" ;; esac ;; - *" get namespace "*" -o jsonpath="*) + *" get namespace "*" -o jsonpath="*|*" get Namespace "*" -o jsonpath="*) case "${AGENT_OS_TEST_NAMESPACE_STATE:-absent}" in - owned) printf 'agent-os\tagent-os-firstmate:%s' "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" ;; + owned) + if [[ " $* " = *'.metadata.uid'* ]]; then + printf '%s\tagent-os\tagent-os-firstmate:%s\tuid-namespace\toperation-test\tTrue\t[]' \ + "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" + else + printf 'agent-os\tagent-os-firstmate:%s' "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" + fi + ;; foreign) printf 'other\tother-installation' ;; *) printf '\t' ;; esac @@ -117,20 +133,26 @@ case " $* " in *" get Service agent-os-firstmate --ignore-not-found -o jsonpath="*|\ *" get Role agent-os-firstmate-runtime --ignore-not-found -o jsonpath="*|\ *" get RoleBinding agent-os-firstmate-runtime --ignore-not-found -o jsonpath="*) + kind='' name='' case " $* " in - *" StatefulSet "*) name=agent-os-firstmate ;; - *" ServiceAccount "*) name=agent-os-firstmate ;; - *" PersistentVolumeClaim "*) name=agent-os-firstmate-home ;; - *" Service "*) name=agent-os-firstmate ;; - *" RoleBinding "*) name=agent-os-firstmate-runtime ;; - *" Role "*) name=agent-os-firstmate-runtime ;; + *" StatefulSet "*) kind=StatefulSet; name=agent-os-firstmate ;; + *" ServiceAccount "*) kind=ServiceAccount; name=agent-os-firstmate ;; + *" PersistentVolumeClaim "*) kind=PersistentVolumeClaim; name=agent-os-firstmate-home ;; + *" Service "*) kind=Service; name=agent-os-firstmate ;; + *" RoleBinding "*) kind=RoleBinding; name=agent-os-firstmate-runtime ;; + *" Role "*) kind=Role; name=agent-os-firstmate-runtime ;; esac - case "${AGENT_OS_TEST_RESOURCE_STATE:-absent}" in + resource_state=${AGENT_OS_TEST_RESOURCE_STATE:-absent} + if grep -F ' apply -f ' "$AGENT_OS_TEST_LOG" >/dev/null; then + resource_state=${AGENT_OS_TEST_RESOURCE_AFTER_APPLY:-$resource_state} + fi + case "$resource_state" in absent) ;; owned) printf '%s\tagent-os\tagent-os-firstmate:portable-agent-os' "$name" - [[ " $* " != *'.metadata.finalizers'* ]] || printf '\t[kubernetes.io/pvc-protection]' + [[ " $* " != *'.metadata.uid'* ]] || \ + printf '\tuid-%s\toperation-test\tTrue\t[kubernetes.io/pvc-protection]' "$(printf '%s' "$kind" | tr '[:upper:]' '[:lower:]')" ;; foreign) printf '%s\tother\tother-installation' "$name" ;; esac @@ -156,16 +178,31 @@ case " $* " in printf 'Role\tagent-os-firstmate-runtime\tServiceAccount\tagent-os-firstmate\t%s' \ "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" ;; - *" get clusterrolebinding agent-os-firstmate-"*" --ignore-not-found -o jsonpath="*) - case "${AGENT_OS_TEST_CLUSTER_RBAC_STATE:-absent}" in + *" get clusterrolebinding agent-os-firstmate-"*" --ignore-not-found -o jsonpath="*|\ + *" get ClusterRoleBinding agent-os-firstmate-"*" --ignore-not-found -o jsonpath="*) + cluster_state=${AGENT_OS_TEST_CLUSTER_RBAC_STATE:-absent} + if grep -F ' apply -f ' "$AGENT_OS_TEST_LOG" >/dev/null; then + cluster_state=${AGENT_OS_TEST_CLUSTER_RBAC_AFTER_APPLY:-$cluster_state} + fi + case "$cluster_state" in absent) ;; owned) printf 'agent-os-firstmate-%s\tagent-os\tagent-os-firstmate:%s' \ "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" + [[ " $* " != *'.metadata.uid'* ]] || printf '\tuid-clusterrolebinding\toperation-test\tTrue\t[]' ;; foreign) printf 'agent-os-firstmate-portable-agent-os\tother\tother-installation' ;; esac ;; + *" get Pod agent-os-firstmate-0 --ignore-not-found -o jsonpath="*) + case "${AGENT_OS_TEST_PRIMARY_POD_STATE:-absent}" in + absent) ;; + owned) + printf 'agent-os-firstmate-0\tagent-os\tagent-os-firstmate:portable-agent-os\tuid-pod\toperation-test\tFalse\t[kubernetes.io/pvc-protection]' + ;; + foreign) printf 'agent-os-firstmate-0\tother\tother-installation\tuid-foreign\tother-operation\tFalse\t[]' ;; + esac + ;; *" api-resources --verbs=list --namespaced -o name "*) printf '%s\n' pods serviceaccounts configmaps leases.coordination.k8s.io ;; @@ -185,7 +222,8 @@ run_launcher() { PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_STDIN_LOG="$STDIN_LOG" \ AGENT_OS_IN_CLUSTER=1 AGENT_OS_NAMESPACE=agent-os-demo AGENT_OS_IMAGE=agent-os:local-test \ AGENT_OS_IMAGE_PULL_POLICY=Never AGENT_OS_AI_SECRET=scout-1-ai-auth \ - AGENT_OS_PURGE_EVIDENCE_FILE="$PURGE_EVIDENCE" "$LAUNCHER" "$@" + AGENT_OS_OPERATION_ID=operation-test AGENT_OS_PURGE_EVIDENCE_FILE="$PURGE_EVIDENCE" \ + "$LAUNCHER" "$@" } : > "$CALLS" @@ -198,6 +236,8 @@ assert_grep 'app.kubernetes.io/managed-by: agent-os' "$STDIN_LOG" \ "child resources need the exact Agent OS ownership label" assert_grep 'agent-os.dev/installation-id: agent-os-firstmate:agent-os-demo' "$STDIN_LOG" \ "child resources need the exact installation identity" +assert_grep 'agent-os.dev/operation-id: operation-test' "$STDIN_LOG" \ + "each crewmate apply must carry its unique operation identity" assert_grep 'automountServiceAccountToken: false' "$STDIN_LOG" "children must not receive Kubernetes credentials" assert_grep 'claimName: agent-os-crewmate-scout-1-home' "$STDIN_LOG" "child work must use its own PVC" assert_no_grep 'hostUsers: false' "$STDIN_LOG" "OrbStack children must not request unsupported Pod user namespaces" @@ -250,8 +290,10 @@ pass "crewmate create requires an explicit AI Secret grant" if AGENT_OS_TEST_FAIL_WAIT=1 AGENT_OS_TEST_POD_AFTER_APPLY=owned run_launcher create scout-1 >/dev/null 2>&1; then fail "crewmate create must fail when its authorized Secret cannot produce a ready Pod" fi -grep -Fqx 'kubectl -n agent-os-demo delete pod agent-os-crewmate-scout-1 --ignore-not-found --wait=false' "$CALLS" || \ - fail "failed create must remove the non-running Pod" +grep -Fqx 'kubectl -n agent-os-demo delete --raw /api/v1/namespaces/agent-os-demo/pods/agent-os-crewmate-scout-1 -f -' "$CALLS" || \ + fail "failed create must use an atomic UID-preconditioned delete" +assert_grep '"uid":"uid-owned"' "$STDIN_LOG" \ + "failed create must precondition deletion on the observed Pod UID" if grep -F 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" >/dev/null; then fail "failed create must retain the crewmate PVC for an authorized retry" fi @@ -267,18 +309,35 @@ assert_contains "$partial_out" 'uid-owned' \ "partial apply cleanup must report the exact newly created Pod UID" assert_grep 'metadata.uid' "$CALLS" \ "partial apply cleanup must collect exact Pod UID evidence" -grep -Fqx 'kubectl -n agent-os-demo delete pod agent-os-crewmate-scout-1 --ignore-not-found --wait=false' "$CALLS" || \ - fail "partial apply cleanup must remove only the newly created owned Pod" +grep -Fqx 'kubectl -n agent-os-demo delete --raw /api/v1/namespaces/agent-os-demo/pods/agent-os-crewmate-scout-1 -f -' "$CALLS" || \ + fail "partial apply cleanup must use an atomic UID-preconditioned delete" +assert_grep '"uid":"uid-owned"' "$STDIN_LOG" \ + "partial apply cleanup must bind deletion to the observed Pod UID" assert_no_grep 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" \ "partial apply cleanup must retain the persistent home" pass "crewmate partial apply cleans only a newly created owned Pod" +: > "$CALLS" +replacement_out='' +replacement_rc=0 +replacement_out=$(AGENT_OS_TEST_FAIL_APPLY=1 AGENT_OS_TEST_POD_AFTER_APPLY=replacement \ + run_launcher create scout-1 2>&1) || replacement_rc=$? +[ "$replacement_rc" -eq 1 ] || fail "replacement partial apply must remain failed: $replacement_out" +assert_contains "$replacement_out" 'replacement or ownership mismatch retained' \ + "partial apply must report a replacement instead of deleting it" +assert_no_grep 'delete --raw' "$CALLS" \ + "partial apply must retain a same-name Pod from another operation" + : > "$CALLS" AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=owned run_launcher stop scout-1 grep -Fqx 'kubectl -n agent-os-demo delete pod agent-os-crewmate-scout-1 --ignore-not-found --wait=false' "$CALLS" || \ fail "stop must target the exactly owned crewmate Pod" grep -Fqx 'kubectl -n agent-os-demo wait --for=delete pod/agent-os-crewmate-scout-1 --timeout=180s' "$CALLS" || \ fail "stop must prove Pod absence before checkpointing can begin" +wait_line=$(grep -Fn 'wait --for=delete pod/agent-os-crewmate-scout-1' "$CALLS" | head -n 1 | cut -d: -f1) +invalidate_line=$(grep -Fn 'annotate pvc agent-os-crewmate-scout-1-home agent-os.dev/checkpoint-state=pending agent-os.dev/checkpoint-at- agent-os.dev/quiesced-operation=operation-test agent-os.dev/checkpoint-operation- --overwrite' "$CALLS" | head -n 1 | cut -d: -f1) +[ -n "$wait_line" ] && [ -n "$invalidate_line" ] && [ "$wait_line" -lt "$invalidate_line" ] || \ + fail "stop must invalidate every pre-stop checkpoint only after Pod absence" if grep -F 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" >/dev/null; then fail "stop must preserve the crewmate persistent home" fi @@ -334,6 +393,13 @@ fi assert_no_grep 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" \ "purge must not delete a home with malformed checkpoint evidence" +: > "$CALLS" +if AGENT_OS_TEST_POD_STATE=absent AGENT_OS_TEST_PVC_STATE=stale-clean run_launcher purge scout-1 --yes >/dev/null 2>&1; then + fail "purge must reject checkpoint evidence from before the latest stop" +fi +assert_no_grep 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" \ + "purge must not delete a home using stale clean checkpoint evidence" + : > "$CALLS" : > "$PURGE_EVIDENCE" AGENT_OS_TEST_POD_STATE=absent AGENT_OS_TEST_PVC_STATE=clean run_launcher purge scout-1 --yes @@ -392,6 +458,8 @@ done mkdir -p "$out" rbac=$(awk '/^rbac:/{print $2}' "$inputs") namespace=$(awk '/^namespace:/{print $2}' "$inputs") +operation=$(awk '/^operationId:/{print $2}' "$inputs") +printf 'akua-input-operation %s\n' "$operation" >> "$AGENT_OS_TEST_LOG" create_namespace=$(awk '/^createNamespace:/{print $2}' "$inputs") [ -n "$create_namespace" ] || create_namespace=true cat > "$out/00-pvc.yaml" < "$CALLS" run_generic install -grep -Fq -- "akua render --no-agent-mode --package $ROOT/tools/agent-os/packages/firstmate/package.k --inputs $GENERIC_INPUTS --out " "$CALLS" || \ +grep -Fq -- "akua render --no-agent-mode --package $ROOT/tools/agent-os/packages/firstmate/package.k --inputs " "$CALLS" || \ fail "generic install must render the canonical package before applying it" grep -Fq 'kubectl --context kind-agent-os apply -f ' "$CALLS" || \ fail "generic install must apply only its freshly rendered package output" +grep -Fqx 'akua-input-operation operation-test' "$CALLS" || \ + fail "generic install must label every resource with its unique operation identity" grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os rollout status statefulset/agent-os-firstmate --timeout=180s' "$CALLS" || \ fail "generic install must wait for the rendered Firstmate StatefulSet" if grep -F 'delete clusterrolebinding' "$CALLS" >/dev/null; then @@ -498,6 +575,48 @@ if grep -F 'delete clusterrolebinding' "$CALLS" >/dev/null; then fi pass "generic install renders and applies the canonical package on an explicit context" +PARTIAL_CLUSTER_INPUTS="$TMP/partial-cluster-inputs.yaml" +sed 's/rbac: namespace/rbac: cluster-admin/' "$GENERIC_INPUTS" > "$PARTIAL_CLUSTER_INPUTS" +: > "$CALLS" +partial_primary_out='' +partial_primary_rc=0 +partial_primary_out=$(PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" \ + AGENT_OS_INPUTS="$PARTIAL_CLUSTER_INPUTS" AGENT_OS_TEST_NAMESPACE=portable-agent-os \ + AGENT_OS_TEST_NAMESPACE_STATE=absent AGENT_OS_TEST_WORKLOAD_STATE=absent \ + AGENT_OS_TEST_RESOURCE_AFTER_APPLY=owned AGENT_OS_TEST_CLUSTER_RBAC_AFTER_APPLY=owned \ + AGENT_OS_TEST_FAIL_GENERIC_APPLY=1 AGENT_OS_OPERATION_ID=operation-test \ + AGENT_OS_CONTEXT=kind-agent-os AGENT_OS_NAMESPACE=portable-agent-os \ + "$GENERIC" install 2>&1) || partial_primary_rc=$? +[ "$partial_primary_rc" -eq 3 ] || \ + fail "partial primary apply must exit incomplete with 3: $partial_primary_out" +assert_contains "$partial_primary_out" 'partial apply: StatefulSet/agent-os-firstmate' \ + "failed primary apply must inventory expected namespaced resources" +assert_contains "$partial_primary_out" 'uid=uid-statefulset operation=operation-test ready=True' \ + "failed primary apply must report UID, operation identity, and readiness" +assert_contains "$partial_primary_out" 'residual-authority: ClusterRoleBinding/agent-os-firstmate-portable-agent-os' \ + "failed cluster-admin apply must report the exact residual grant" +assert_contains "$partial_primary_out" 'cleanup-cluster-rbac --yes' \ + "failed cluster-admin apply must print the privileged cleanup command" +assert_contains "$partial_primary_out" 'safe recovery:' \ + "failed primary apply must print a bounded exact recovery command" +pass "primary partial apply reports residual resources and authority" + +: > "$CALLS" +namespace_partial_out='' +namespace_partial_rc=0 +namespace_partial_out=$(AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_NAMESPACE_STATE=owned \ + AGENT_OS_TEST_RESOURCE_AFTER_APPLY=owned \ + AGENT_OS_TEST_CLUSTER_RBAC_AFTER_APPLY=owned AGENT_OS_TEST_FAIL_GENERIC_APPLY=1 \ + run_generic upgrade 2>&1) || namespace_partial_rc=$? +[ "$namespace_partial_rc" -eq 3 ] || \ + fail "namespace partial upgrade must exit incomplete with 3: $namespace_partial_out" +assert_contains "$namespace_partial_out" \ + 'residual-authority: ClusterRoleBinding/agent-os-firstmate-portable-agent-os' \ + "every failed primary mutation must inspect the deterministic cluster grant" +assert_contains "$namespace_partial_out" 'cleanup-cluster-rbac --yes' \ + "a failed namespaced mutation with residual authority must print exact cleanup" +pass "namespace partial apply reports stale cluster authority" + : > "$CALLS" owned_namespace_out='' owned_namespace_rc=0 @@ -730,18 +849,33 @@ pvc_delete_line=$(grep -Fn '/00-pvc.yaml' "$CALLS" | grep ' delete ' | head -n 1 pass "routine uninstall removes namespaced resources without cluster-wide authority" : > "$CALLS" +NONE_INPUTS="$TMP/none-rbac-inputs.yaml" +sed 's/rbac: namespace/rbac: none/' "$GENERIC_INPUTS" > "$NONE_INPUTS" bounded_out='' bounded_rc=0 -bounded_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=namespace \ - AGENT_OS_TEST_RESOURCE_STATE=owned AGENT_OS_TEST_FAIL_DELETE_FILE=00-pvc.yaml \ - run_generic uninstall --yes 2>&1) || bounded_rc=$? +bounded_out=$(PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS="$NONE_INPUTS" \ + AGENT_OS_TEST_NAMESPACE=portable-agent-os AGENT_OS_TEST_NAMESPACE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=none AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_PRIMARY_POD_STATE=owned AGENT_OS_TEST_FAIL_DELETE_FILE=00-pvc.yaml \ + AGENT_OS_OPERATION_ID=operation-test AGENT_OS_CONTEXT=kind-agent-os \ + AGENT_OS_NAMESPACE=portable-agent-os "$GENERIC" uninstall --yes 2>&1) || bounded_rc=$? [ "$bounded_rc" -eq 3 ] || fail "timed-out uninstall must exit incomplete: $bounded_out" -assert_contains "$bounded_out" 'retained: PersistentVolumeClaim/agent-os-firstmate-home' \ +assert_contains "$bounded_out" 'failed-target: PersistentVolumeClaim/agent-os-firstmate-home' \ + "timed-out uninstall must report the actual failed target" +assert_contains "$bounded_out" 'retained: PersistentVolumeClaim/agent-os-firstmate-home uid=uid-persistentvolumeclaim' \ "timed-out uninstall must report the retained owned resource" assert_contains "$bounded_out" 'finalizers=[kubernetes.io/pvc-protection]' \ "timed-out uninstall must report retained finalizers" +assert_contains "$bounded_out" 'retained: Pod/agent-os-firstmate-0 uid=uid-pod' \ + "timed-out uninstall must report the StatefulSet Pod outside the fresh render" +assert_contains "$bounded_out" 'retained: Role/agent-os-firstmate-runtime uid=uid-role' \ + "timed-out rbac:none uninstall must report stale deterministic Role residue" +assert_contains "$bounded_out" 'retained: RoleBinding/agent-os-firstmate-runtime uid=uid-rolebinding' \ + "timed-out rbac:none uninstall must report stale deterministic RoleBinding residue" assert_contains "$bounded_out" 'safe retry:' \ "timed-out uninstall must print exact safe retry evidence" +assert_contains "$bounded_out" 'cluster cleanup if required:' \ + "timed-out uninstall must print exact privileged cleanup evidence" grep -F '/00-pvc.yaml' "$CALLS" | grep -F -- '--wait=true --timeout=180s' >/dev/null || \ fail "uninstall deletion must have an explicit timeout" pass "uninstall timeouts report retained owned resources and safe retry evidence" diff --git a/tests/agent-os-packages.test.sh b/tests/agent-os-packages.test.sh index f34c38e53..b6a97c940 100755 --- a/tests/agent-os-packages.test.sh +++ b/tests/agent-os-packages.test.sh @@ -148,6 +148,8 @@ for invalid_image in \ 'ghcr.io/akua-dev/agent-os@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaextra' \ 'invalid@@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ 'registry.example/org:123/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ + '[2001:db8::1]:5000/org:123/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ + '[2001:db8::g]:5000/org/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ 'prefix@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:suffix'; do invalid_inputs="$TMP/invalid-$(printf '%s' "$invalid_image" | tr '/:@' '---').yaml" sed "s|^image: .*|image: $invalid_image|" "$INPUTS" > "$invalid_inputs" @@ -166,6 +168,14 @@ akua render --no-agent-mode --package "$FIRSTMATE/package.k" --inputs "$PORT_INP fail "the portable package must allow a registry authority port" pass "OCI reference ports are confined to registry authority" +IPV6_INPUTS="$TMP/registry-ipv6.yaml" +sed 's|^image: .*|image: "[2001:db8::1]:5000/org/agent-os@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"|' \ + "$INPUTS" > "$IPV6_INPUTS" +akua render --no-agent-mode --package "$FIRSTMATE/package.k" --inputs "$IPV6_INPUTS" \ + --out "$TMP/registry-ipv6-rendered" >/dev/null || \ + fail "the portable package must allow a bracketed IPv6 registry authority" +pass "OCI references accept bracketed IPv6 registry authorities" + assert_grep 'bin/agent-os-kubernetes.sh install' "$ROOT/docs/kubernetes.md" \ "Kubernetes docs must make the generic installer the default quickstart" assert_grep 'bin/agent-os-kubernetes.sh rollback' "$ROOT/docs/kubernetes.md" \ diff --git a/tools/agent-os/packages/firstmate/crewmate.yaml b/tools/agent-os/packages/firstmate/crewmate.yaml index d44ef00dc..f529a2870 100644 --- a/tools/agent-os/packages/firstmate/crewmate.yaml +++ b/tools/agent-os/packages/firstmate/crewmate.yaml @@ -8,6 +8,7 @@ metadata: app.kubernetes.io/component: crewmate app.kubernetes.io/managed-by: agent-os agent-os.dev/crewmate: __AGENT_OS_CREWMATE_ID__ + agent-os.dev/operation-id: __AGENT_OS_OPERATION_ID__ annotations: agent-os.dev/installation-id: agent-os-firstmate:__AGENT_OS_NAMESPACE__ agent-os.dev/checkpoint-state: pending @@ -28,6 +29,7 @@ metadata: app.kubernetes.io/component: crewmate app.kubernetes.io/managed-by: agent-os agent-os.dev/crewmate: __AGENT_OS_CREWMATE_ID__ + agent-os.dev/operation-id: __AGENT_OS_OPERATION_ID__ annotations: agent-os.dev/installation-id: agent-os-firstmate:__AGENT_OS_NAMESPACE__ spec: diff --git a/tools/agent-os/packages/firstmate/package.k b/tools/agent-os/packages/firstmate/package.k index a770ecdc4..05d744db7 100644 --- a/tools/agent-os/packages/firstmate/package.k +++ b/tools/agent-os/packages/firstmate/package.k @@ -15,17 +15,20 @@ schema Input: memoryRequest: str = "1Gi" cpuLimit: str = "4" memoryLimit: str = "8Gi" + operationId: str = "" check: - allowMutableImage or regex.match(image, r"^([a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*(:[0-9]+)?/)?[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*(\/[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*)*(:[A-Za-z0-9_][A-Za-z0-9_.-]{0,127})?@sha256:[0-9a-f]{64}$"), "image must use one complete OCI reference with an immutable @sha256 digest unless the local profile explicitly allows a mutable image" + allowMutableImage or regex.match(image, r"^((([a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*|(\[[0-9A-Fa-f:]+\]))(:[0-9]+)?)/)?[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*(\/[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*)*(:[A-Za-z0-9_][A-Za-z0-9_.-]{0,127})?@sha256:[0-9a-f]{64}$"), "image must use one complete OCI reference with an immutable @sha256 digest unless the local profile explicitly allows a mutable image" + operationId == "" or regex.match(operationId, r"^[a-z0-9]([a-z0-9.-]{0,61}[a-z0-9])?$"), "operationId must be empty or a valid Kubernetes label value" input: Input = ctx.input() -labels = { +selectorLabels = { "app.kubernetes.io/name" = "agent-os" "app.kubernetes.io/component" = "firstmate" "app.kubernetes.io/managed-by" = "agent-os" } +labels = selectorLabels | ({ "agent-os.dev/operation-id" = input.operationId } if input.operationId != "" else {}) installationId = "agent-os-firstmate:${input.namespace}" ownershipAnnotations = { "agent-os.dev/installation-id" = installationId } @@ -147,7 +150,7 @@ resources = namespaceResources + [ } spec = { clusterIP = "None" - selector = labels + selector = selectorLabels } }, ] + namespaceRbacResources + clusterAdminResources + [ @@ -165,7 +168,7 @@ resources = namespaceResources + [ spec = { serviceName = "agent-os-firstmate" replicas = 1 - selector = { matchLabels = labels } + selector = { matchLabels = selectorLabels } template = { metadata = { labels = labels From d9fa6519af838d3588346cc00ddc88b2ab2ba55d Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 20:13:57 +0200 Subject: [PATCH 29/56] no-mistakes(review): Captain, harden Kubernetes lifecycle and dependency integrity --- Dockerfile | 2 +- bin/agent-os-crewmate.sh | 273 +++++++++++++++++--- bin/agent-os-kubernetes.sh | 143 +++++++++- image/npm/package-lock.json | 3 + tests/agent-os-container.test.sh | 11 + tests/agent-os-kubernetes.test.sh | 198 +++++++++++--- tests/agent-os-packages.test.sh | 18 +- tools/agent-os/packages/firstmate/package.k | 7 +- 8 files changed, 571 insertions(+), 84 deletions(-) diff --git a/Dockerfile b/Dockerfile index 77718e545..711bb8c4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -143,7 +143,7 @@ RUN set -eu; \ COPY image/npm/package.json image/npm/package-lock.json /opt/agent-os-npm/ RUN echo "3646e31389155fbce155c828d8db46bc60ff2976c2d8d29e6633f260f56fd06d /opt/agent-os-npm/package.json" | sha256sum -c - \ - && echo "d815b10a3ea0bf79d55a9e2245b422cbb401bc9b38f27a4575a639ce2caabe1f /opt/agent-os-npm/package-lock.json" | sha256sum -c - \ + && echo "f77f31c67455d6f72e6411d5fa82669b9cc95306d518ff655b7c7795cfd41ca2 /opt/agent-os-npm/package-lock.json" | sha256sum -c - \ && npm ci --omit=dev --ignore-scripts --no-audit --no-fund --prefix /opt/agent-os-npm \ && mkdir -p /usr/local/lib/node_modules \ && cp -a /opt/agent-os-npm/node_modules/. /usr/local/lib/node_modules/ \ diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index e5cf36242..938f5979e 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -38,8 +38,11 @@ fi POD="agent-os-crewmate-$ID" PVC="$POD-home" +LOCK="$POD-lifecycle" EXPECTED_POD="$POD"$'\t'"agent-os"$'\t'"$ID"$'\t'"$INSTALLATION_ID" EXPECTED_PVC="$PVC"$'\t'"agent-os"$'\t'"$ID"$'\t'"$INSTALLATION_ID" +EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"$ID"$'\t'"$INSTALLATION_ID" +LOCK_UID= kube() { "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" "$@" @@ -53,12 +56,17 @@ resource_identity() { pod_record() { kube get pod "$POD" --ignore-not-found \ - -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}{"\t"}{.metadata.uid}' + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}' } pvc_record() { kube get pvc "$PVC" --ignore-not-found \ - -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-state}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-at}{"\t"}{.metadata.annotations.agent-os\.dev/quiesced-operation}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-operation}' + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-state}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-at}{"\t"}{.metadata.annotations.agent-os\.dev/quiesced-operation}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-operation}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}' +} + +lock_record() { + kube get lease "$LOCK" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.spec.holderIdentity}{"\t"}{.metadata.uid}' } require_owned_or_absent() { @@ -110,18 +118,83 @@ render_resources() { "$TEMPLATE" } +render_pvc() { + render_resources | awk '/^---$/ { exit } { print }' +} + +render_pod() { + render_resources | awk 'found { print } /^---$/ { found=1 }' +} + +render_lock() { + cat </dev/null || true + LOCK_UID= +} + +acquire_lock() { + local record identity holder + if ! render_lock | kube create -f - >/dev/null; then + record=$(lock_record) + identity=$(printf '%s' "$record" | cut -f1-4) + holder=$(printf '%s' "$record" | cut -f5) + if [ -z "$record" ] || [ "$identity" != "$EXPECTED_LOCK" ]; then + echo "error: lifecycle Lease '$LOCK' is absent or has foreign ownership after create conflict" >&2 + exit 2 + fi + if ! kube wait --for=delete "lease/$LOCK" --timeout=30s >/dev/null; then + echo "error: lifecycle operation '$holder' still holds Lease '$LOCK' after 30s" >&2 + exit 3 + fi + if ! render_lock | kube create -f - >/dev/null; then + echo "error: lifecycle Lease '$LOCK' changed during bounded acquisition" >&2 + exit 3 + fi + fi + record=$(lock_record) + identity=$(printf '%s' "$record" | cut -f1-4) + holder=$(printf '%s' "$record" | cut -f5) + LOCK_UID=$(printf '%s' "$record" | cut -f6) + if [ "$identity" != "$EXPECTED_LOCK" ] || [ "$holder" != "$OPERATION_ID" ] || [ -z "$LOCK_UID" ]; then + LOCK_UID= + echo "error: lifecycle Lease '$LOCK' did not verify after acquisition" >&2 + exit 2 + fi +} + +trap release_lock EXIT + cleanup_new_owned_pod() { - local before_uid=$1 after current identity after_operation after_uid + local expected_uid=${1:-} after current identity after_operation after_uid after=$(pod_record) if [ -z "$after" ]; then - echo "partial state: crewmate apply left no Pod; persistent home retained" >&2 + echo "partial state: crewmate create left no Pod; persistent home retained" >&2 return fi identity=$(printf '%s' "$after" | cut -f1-4) after_operation=$(printf '%s' "$after" | cut -f5) after_uid=$(printf '%s' "$after" | cut -f6) if [ "$identity" != "$EXPECTED_POD" ] || [ "$after_operation" != "$OPERATION_ID" ] || \ - [ -z "$after_uid" ] || [ -n "$before_uid" ]; then + [ -z "$after_uid" ] || { [ -n "$expected_uid" ] && [ "$after_uid" != "$expected_uid" ]; }; then echo "partial state: replacement or ownership mismatch retained; persistent home retained" >&2 return fi @@ -139,24 +212,62 @@ cleanup_new_owned_pod() { kube wait --for=delete "pod/$POD" --timeout=180s } -apply_and_wait() { - local before before_uid - before=$(pod_record) - before_uid=$(printf '%s' "$before" | cut -f6) - if ! render_resources | kube apply -f -; then - cleanup_new_owned_pod "$before_uid" - echo "error: crewmate resources were only partially applied" >&2 +create_and_wait() { + local pvc_before pvc_current pvc_uid pvc_rv pod pod_uid pod_rv + pvc_before=$(require_owned_pvc_or_absent) + if [ -z "$pvc_before" ]; then + if ! render_pvc | kube create -f - >/dev/null; then + echo "error: create-only PVC operation conflicted; no resource was adopted" >&2 + exit 2 + fi + pvc_before=$(require_owned_pvc_or_absent) + fi + pvc_current=$(require_owned_pvc_or_absent) + if [ -z "$pvc_before" ] || [ "$pvc_current" != "$pvc_before" ]; then + echo "error: PVC identity or resourceVersion changed before Pod creation" >&2 + exit 2 + fi + pvc_uid=$(printf '%s' "$pvc_before" | cut -f9) + pvc_rv=$(printf '%s' "$pvc_before" | cut -f10) + if [ -z "$pvc_uid" ] || [ -z "$pvc_rv" ]; then + echo "error: Pod creation requires exact PVC UID and resourceVersion evidence" >&2 + exit 2 + fi + if ! render_pod | kube create -f - >/dev/null; then + cleanup_new_owned_pod + echo "error: create-only Pod operation conflicted or was only partially acknowledged" >&2 + exit 1 + fi + pod=$(pod_record) + if [ "$(printf '%s' "$pod" | cut -f1-4)" != "$EXPECTED_POD" ] || \ + [ "$(printf '%s' "$pod" | cut -f5)" != "$OPERATION_ID" ]; then + echo "error: created Pod did not retain the exact operation identity; no cleanup attempted" >&2 + exit 1 + fi + pod_uid=$(printf '%s' "$pod" | cut -f6) + pod_rv=$(printf '%s' "$pod" | cut -f7) + if [ -z "$pod_uid" ] || [ -z "$pod_rv" ]; then + echo "error: created Pod lacks exact UID or resourceVersion evidence" >&2 exit 1 fi if ! kube wait --for=condition=Ready "pod/$POD" --timeout=180s; then - cleanup_new_owned_pod "$before_uid" + cleanup_new_owned_pod "$pod_uid" echo "error: crewmate Pod did not become ready with the authorized AI Secret" >&2 exit 1 fi } preflight_create() { - require_owned_or_absent pod "$POD" "$EXPECTED_POD" >/dev/null + local identity + identity=$(resource_identity pod "$POD") + if [ -n "$identity" ] && [ "$identity" != "$EXPECTED_POD" ]; then + echo "error: pod '$POD' does not have the exact crewmate installation identity" >&2 + exit 2 + fi + if [ -n "$identity" ]; then + echo "error: crewmate Pod '$POD' already exists; use restart" >&2 + exit 2 + fi require_owned_pvc_or_absent >/dev/null } @@ -172,21 +283,71 @@ preflight_existing_home() { } stop_owned_pod() { - local pod - pod=$(require_owned_or_absent pod "$POD" "$EXPECTED_POD") + local pod uid rv + pod=$(pod_record) if [ -z "$pod" ]; then return fi - kube delete pod "$POD" --ignore-not-found --wait=false + if [ "$(printf '%s' "$pod" | cut -f1-4)" != "$EXPECTED_POD" ]; then + echo "error: pod '$POD' does not have the exact crewmate installation identity" >&2 + exit 2 + fi + uid=$(printf '%s' "$pod" | cut -f6) + rv=$(printf '%s' "$pod" | cut -f7) + if [ -z "$uid" ] || [ -z "$rv" ]; then + echo "error: Pod deletion requires exact UID and resourceVersion evidence" >&2 + exit 2 + fi + printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s"}}\n' "$uid" | \ + kube delete --raw "/api/v1/namespaces/$NAMESPACE/pods/$POD" -f - kube wait --for=delete "pod/$POD" --timeout=180s } invalidate_checkpoint_evidence() { - local pvc + local pvc uid rv patch + pvc=$(require_owned_pvc_or_absent) + [ -n "$pvc" ] || return + uid=$(printf '%s' "$pvc" | cut -f9) + rv=$(printf '%s' "$pvc" | cut -f10) + if [ -z "$uid" ] || [ -z "$rv" ]; then + echo "error: checkpoint invalidation requires exact PVC UID and resourceVersion evidence" >&2 + exit 2 + fi + patch=$(printf '{"metadata":{"uid":"%s","resourceVersion":"%s","annotations":{"agent-os.dev/checkpoint-state":"pending","agent-os.dev/checkpoint-at":null,"agent-os.dev/quiesced-operation":null,"agent-os.dev/checkpoint-operation":null}}}' "$uid" "$rv") + kube patch pvc "$PVC" --type=merge -p "$patch" >/dev/null +} + +record_quiesced_generation() { + local pvc uid rv patch pvc=$(require_owned_pvc_or_absent) [ -n "$pvc" ] || return - kube annotate pvc "$PVC" agent-os.dev/checkpoint-state=pending agent-os.dev/checkpoint-at- \ - agent-os.dev/quiesced-operation="$OPERATION_ID" agent-os.dev/checkpoint-operation- --overwrite + if [ "$(printf '%s' "$pvc" | cut -f5)" != pending ]; then + echo "error: checkpoint evidence changed before quiesced generation could be recorded" >&2 + exit 2 + fi + uid=$(printf '%s' "$pvc" | cut -f9) + rv=$(printf '%s' "$pvc" | cut -f10) + if [ -z "$uid" ] || [ -z "$rv" ]; then + echo "error: quiesced generation requires exact PVC UID and resourceVersion evidence" >&2 + exit 2 + fi + patch=$(printf '{"metadata":{"uid":"%s","resourceVersion":"%s","annotations":{"agent-os.dev/quiesced-operation":"%s"}}}' "$uid" "$rv" "$OPERATION_ID") + kube patch pvc "$PVC" --type=merge -p "$patch" >/dev/null +} + +quiesce_owned_home() { + local pod + invalidate_checkpoint_evidence + stop_owned_pod + if ! pod=$(pod_record); then + echo "error: could not prove Pod absence after deletion" >&2 + exit 3 + fi + if [ -n "$pod" ]; then + echo "error: Pod '$POD' reappeared before quiesced checkpoint generation" >&2 + exit 3 + fi + record_quiesced_generation } record_purge() { @@ -200,31 +361,37 @@ case "$COMMAND" in create) [ -z "$CONFIRM" ] || { echo "usage: $0 create " >&2; exit 2; } validate_ai_grant + acquire_lock preflight_create - apply_and_wait + create_and_wait ;; status) [ -z "$CONFIRM" ] || { echo "usage: $0 status " >&2; exit 2; } - if [ -z "$(require_owned_or_absent pod "$POD" "$EXPECTED_POD")" ]; then + pod=$(pod_record) + if [ -z "$pod" ]; then echo "error: crewmate Pod '$POD' does not exist" >&2 exit 2 fi + if [ "$(printf '%s' "$pod" | cut -f1-4)" != "$EXPECTED_POD" ]; then + echo "error: pod '$POD' does not have the exact crewmate installation identity" >&2 + exit 2 + fi require_owned_pvc_or_absent >/dev/null kube get pod "$POD" ;; stop) [ -z "$CONFIRM" ] || { echo "usage: $0 stop " >&2; exit 2; } + acquire_lock require_owned_pvc_or_absent >/dev/null - stop_owned_pod - invalidate_checkpoint_evidence + quiesce_owned_home ;; restart) [ -z "$CONFIRM" ] || { echo "usage: $0 restart " >&2; exit 2; } validate_ai_grant + acquire_lock preflight_existing_home >/dev/null - stop_owned_pod - invalidate_checkpoint_evidence - apply_and_wait + quiesce_owned_home + create_and_wait ;; purge) echo "purge target: namespace/$NAMESPACE pod/$POD pvc/$PVC" >&2 @@ -232,7 +399,16 @@ case "$COMMAND" in echo "error: purge requires the purge-specific --yes confirmation" >&2 exit 2 fi - if [ -n "$(require_owned_or_absent pod "$POD" "$EXPECTED_POD")" ]; then + acquire_lock + if ! pod=$(pod_record); then + echo "error: could not prove crewmate Pod absence before purge" >&2 + exit 3 + fi + if [ -n "$pod" ]; then + if [ "$(printf '%s' "$pod" | cut -f1-4)" != "$EXPECTED_POD" ]; then + echo "error: pod '$POD' does not have the exact crewmate installation identity" >&2 + exit 2 + fi echo "error: stop the owned crewmate Pod and prove its absence before checkpointing for purge" >&2 exit 2 fi @@ -253,6 +429,42 @@ case "$COMMAND" in echo "error: purge requires checkpoint evidence created for the current quiesced PVC generation" >&2 exit 2 fi + if ! command -v jq >/dev/null 2>&1; then + echo "error: jq is required to prove that the persistent home is detached" >&2 + exit 2 + fi + if ! pods_json=$(kube get pods -o json); then + echo "error: purge could not inventory Pods that may reference PVC '$PVC'" >&2 + exit 3 + fi + if ! attached=$(printf '%s' "$pods_json" | jq -r --arg claim "$PVC" \ + '[.items[]? | any(.spec.volumes[]?; .persistentVolumeClaim.claimName == $claim)] | any'); then + echo "error: purge could not validate the Pod attachment inventory" >&2 + exit 3 + fi + if [ "$attached" = true ]; then + echo "error: purge refuses PVC '$PVC' while any Pod still references it" >&2 + exit 2 + fi + if ! pod=$(pod_record); then + echo "error: could not re-prove crewmate Pod absence during purge" >&2 + exit 3 + fi + if [ -n "$pod" ]; then + echo "error: crewmate Pod '$POD' reappeared during purge validation" >&2 + exit 3 + fi + current_pvc=$(require_owned_pvc_or_absent) + if [ "$current_pvc" != "$pvc" ]; then + echo "error: PVC identity, checkpoint, UID, or resourceVersion changed during purge validation" >&2 + exit 3 + fi + pvc_uid=$(printf '%s' "$pvc" | cut -f9) + pvc_rv=$(printf '%s' "$pvc" | cut -f10) + if [ -z "$pvc_uid" ] || [ -z "$pvc_rv" ]; then + echo "error: purge requires exact PVC UID and resourceVersion evidence" >&2 + exit 2 + fi evidence_file=${AGENT_OS_PURGE_EVIDENCE_FILE:-} if [ -z "$evidence_file" ] && [ -n "${FM_HOME:-}" ]; then evidence_file="$FM_HOME/data/crewmate-purge-evidence.log" @@ -264,7 +476,10 @@ case "$COMMAND" in mkdir -p "$(dirname "$evidence_file")" exec 3>>"$evidence_file" record_purge purge-requested "$checkpoint_at" - kube delete pvc "$PVC" --ignore-not-found --wait=true --timeout=180s + printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s","resourceVersion":"%s"}}\n' \ + "$pvc_uid" "$pvc_rv" | \ + kube delete --raw "/api/v1/namespaces/$NAMESPACE/persistentvolumeclaims/$PVC" -f - + kube wait --for=delete "pvc/$PVC" --timeout=180s record_purge purge-complete "$checkpoint_at" ;; delete) diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh index 921b7ac1e..273b95179 100755 --- a/bin/agent-os-kubernetes.sh +++ b/bin/agent-os-kubernetes.sh @@ -213,6 +213,16 @@ delete_namespace_rbac() { resource_observation() { local scope=$1 kind=$2 name=$3 + if [ "$kind" = StatefulSet ]; then + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get "$kind" "$name" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.uid}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}{"\t"}{"\t"}{.metadata.finalizers}{"\t"}{.spec.replicas}{"\t"}{.status.currentReplicas}{"\t"}{.status.readyReplicas}{"\t"}{.status.updatedReplicas}{"\t"}{.status.availableReplicas}{"\t"}{.status.currentRevision}{"\t"}{.status.updateRevision}{"\t"}{.metadata.generation}{"\t"}{.status.observedGeneration}' + return + fi + if [ "$kind" = Pod ]; then + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get "$kind" "$name" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.uid}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}{"\t"}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\t"}{.metadata.finalizers}{"\t"}{range .status.containerStatuses[*]}{.name}{"="}{.state.waiting.reason}{.state.terminated.reason}{","}{end}' + return + fi if [ "$scope" = cluster ]; then "$KUBECTL" --context "$CONTEXT" get "$kind" "$name" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.uid}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}{"\t"}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\t"}{.metadata.finalizers}' @@ -223,12 +233,13 @@ resource_observation() { } lifecycle_command() { + local action=${1:-$COMMAND} printf 'AGENT_OS_CONTEXT=%q AGENT_OS_NAMESPACE=%q AGENT_OS_PACKAGE=%q AGENT_OS_INPUTS=%q AGENT_OS_AKUA=%q AGENT_OS_KUBECTL=%q %q %q' \ - "$CONTEXT" "$NAMESPACE" "$PACKAGE" "$INPUTS" "$AKUA" "$KUBECTL" "$0" "$COMMAND" + "$CONTEXT" "$NAMESPACE" "$PACKAGE" "$INPUTS" "$AKUA" "$KUBECTL" "$0" "$action" } report_partial_observation() { - local kind=$1 name=$2 scope=$3 record managed installation uid operation ready finalizers prefix + local kind=$1 name=$2 scope=$3 record managed installation uid operation ready finalizers prefix details if ! record=$(resource_observation "$scope" "$kind" "$name"); then echo "partial apply: $kind/$name observation=unavailable expected-operation=$OPERATION_ID" >&2 return @@ -242,16 +253,111 @@ report_partial_observation() { uid=$(printf '%s' "$record" | cut -f4) operation=$(printf '%s' "$record" | cut -f5) ready=$(printf '%s' "$record" | cut -f6) - finalizers=$(printf '%s' "$record" | cut -f7-) + finalizers=$(printf '%s' "$record" | cut -f7) [ -n "$ready" ] || ready=unknown [ -n "$finalizers" ] || finalizers='[]' prefix='partial apply' [ "$kind" != ClusterRoleBinding ] || prefix='residual-authority' - echo "$prefix: $kind/$name uid=$uid operation=$operation ready=$ready ownership=$managed installation=$installation finalizers=$finalizers" >&2 + details="ready=$ready" + if [ "$kind" = StatefulSet ]; then + details="desired=$(printf '%s' "$record" | cut -f8) current=$(printf '%s' "$record" | cut -f9) ready=$(printf '%s' "$record" | cut -f10) updated=$(printf '%s' "$record" | cut -f11) available=$(printf '%s' "$record" | cut -f12) current-revision=$(printf '%s' "$record" | cut -f13) update-revision=$(printf '%s' "$record" | cut -f14) generation=$(printf '%s' "$record" | cut -f15) observed-generation=$(printf '%s' "$record" | cut -f16)" + elif [ "$kind" = Pod ]; then + details="ready=$ready reasons=$(printf '%s' "$record" | cut -f8-)" + fi + echo "$prefix: $kind/$name uid=$uid operation=$operation $details ownership=$managed installation=$installation finalizers=$finalizers" >&2 +} + +partial_install_is_applicable() { + local file kind name identity expected namespace + if ! namespace=$(namespace_name 2>/dev/null); then + return 1 + fi + if [ "$MANAGES_NAMESPACE" -eq 1 ]; then + if [ -n "$namespace" ]; then + if ! identity=$(namespace_identity 2>/dev/null); then + return 1 + fi + [ "$identity" = "agent-os"$'\t'"$INSTALLATION_ID" ] || return 1 + fi + else + [ -n "$namespace" ] || return 1 + if ! identity=$(namespace_identity 2>/dev/null); then + return 1 + fi + [ "$identity" = $'\t' ] || return 1 + fi + while IFS= read -r file; do + [ -n "$file" ] || continue + kind=$(rendered_resource_field "$file" kind) + name=$(rendered_resource_field "$file" name) + case "$kind" in + Namespace) continue ;; + ClusterRoleBinding) + if ! identity=$("$KUBECTL" --context "$CONTEXT" get clusterrolebinding "$name" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}' 2>/dev/null); then + return 1 + fi + ;; + *) + if ! identity=$(live_resource_identity "$kind" "$name" 2>/dev/null); then + return 1 + fi + ;; + esac + expected="$name"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" + [ -z "$identity" ] || [ "$identity" = "$expected" ] || return 1 + done < <(find "$OUT" -type f -name '*.yaml' -print) +} + +partial_recovery_action() { + local state identity namespace + if ! namespace=$(namespace_name 2>/dev/null); then + return 1 + fi + state= + if [ -n "$namespace" ] && ! state=$(workload_state 2>/dev/null); then + return 1 + fi + if [ -n "$state" ]; then + identity=$(printf '%s' "$state" | cut -f1,4,5) + if [ "$identity" = "agent-os-firstmate"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" ]; then + printf upgrade + return + fi + return 1 + fi + if partial_install_is_applicable; then + printf install + return + fi + return 1 +} + +partial_cleanup_is_applicable() { + local binding workload mode marker identity namespace + if ! binding=$(resource_observation cluster ClusterRoleBinding "agent-os-firstmate-$NAMESPACE" 2>/dev/null); then + return 1 + fi + [ -n "$binding" ] || return 1 + identity=$(printf '%s' "$binding" | cut -f2,3) + [ "$identity" = "agent-os"$'\t'"$INSTALLATION_ID" ] || return 1 + if ! namespace=$(namespace_name 2>/dev/null); then + return 1 + fi + [ -n "$namespace" ] || return 0 + if ! workload=$(workload_state 2>/dev/null); then + return 1 + fi + [ -n "$workload" ] || return 0 + identity=$(printf '%s' "$workload" | cut -f1,4,5) + [ "$identity" = "agent-os-firstmate"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" ] || return 1 + mode=$(printf '%s' "$workload" | cut -f2) + marker=$(printf '%s' "$workload" | cut -f3) + [ "$mode" != cluster-admin ] && [ "$marker" = required ] } report_partial_apply() { - local phase=$1 file kind name scope + local phase=$1 file kind name scope recovery echo "incomplete: primary $phase failed; automatic cleanup withheld pending exact recovery" >&2 while IFS= read -r file; do [ -n "$file" ] || continue @@ -266,8 +372,16 @@ report_partial_apply() { done < <(find "$OUT" -type f -name '*.yaml' -print) report_partial_observation Pod agent-os-firstmate-0 namespaced report_partial_observation ClusterRoleBinding "agent-os-firstmate-$NAMESPACE" cluster - echo "safe recovery: $(lifecycle_command)" >&2 - echo "privileged cleanup if the reported grant is stale: $(cleanup_command)" >&2 + if recovery=$(partial_recovery_action); then + echo "safe recovery: $(lifecycle_command "$recovery")" >&2 + else + echo "safe recovery unavailable: workload ownership changed; inspect retained resources" >&2 + fi + if partial_cleanup_is_applicable; then + echo "privileged cleanup with proven predicates: $(cleanup_command)" >&2 + else + echo "privileged cleanup unavailable: stale-grant predicates are not proven" >&2 + fi return 3 } @@ -295,7 +409,8 @@ verify_desired_rbac() { .rules == [ {"apiGroups":[""],"resources":["pods","persistentvolumeclaims"],"verbs":["get","list","watch","create","delete","patch"]}, {"apiGroups":[""],"resources":["pods/log","pods/exec"],"verbs":["get","list","watch","create","delete"]}, - {"apiGroups":["apps"],"resources":["statefulsets"],"verbs":["get","list","watch"]} + {"apiGroups":["apps"],"resources":["statefulsets"],"verbs":["get","list","watch"]}, + {"apiGroups":["coordination.k8s.io"],"resources":["leases"],"verbs":["get","list","watch","create","delete"]} ]' >/dev/null || ! printf '%s' "$binding_json" | jq -e --arg namespace "$NAMESPACE" ' .roleRef == {"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"agent-os-firstmate-runtime"} and @@ -323,7 +438,7 @@ uninstall_command() { } report_retained_observation() { - local kind=$1 name=$2 scope=$3 record managed installation uid operation ready finalizers identity expected + local kind=$1 name=$2 scope=$3 record managed installation uid operation ready finalizers identity expected details if ! record=$(resource_observation "$scope" "$kind" "$name"); then echo "retained-unverified: $kind/$name could not be inspected; no further deletion attempted" >&2 return @@ -334,13 +449,19 @@ report_retained_observation() { uid=$(printf '%s' "$record" | cut -f4) operation=$(printf '%s' "$record" | cut -f5) ready=$(printf '%s' "$record" | cut -f6) - finalizers=$(printf '%s' "$record" | cut -f7-) + finalizers=$(printf '%s' "$record" | cut -f7) identity="$managed"$'\t'"$installation" expected="agent-os"$'\t'"$INSTALLATION_ID" [ -n "$ready" ] || ready=unknown [ -n "$finalizers" ] || finalizers='[]' + details="ready=$ready" + if [ "$kind" = StatefulSet ]; then + details="desired=$(printf '%s' "$record" | cut -f8) current=$(printf '%s' "$record" | cut -f9) ready=$(printf '%s' "$record" | cut -f10) updated=$(printf '%s' "$record" | cut -f11) available=$(printf '%s' "$record" | cut -f12) current-revision=$(printf '%s' "$record" | cut -f13) update-revision=$(printf '%s' "$record" | cut -f14) generation=$(printf '%s' "$record" | cut -f15) observed-generation=$(printf '%s' "$record" | cut -f16)" + elif [ "$kind" = Pod ]; then + details="ready=$ready reasons=$(printf '%s' "$record" | cut -f8-)" + fi if [ "$identity" = "$expected" ]; then - echo "retained: $kind/$name uid=$uid operation=$operation ready=$ready ownership=$managed installation=$installation finalizers=$finalizers" >&2 + echo "retained: $kind/$name uid=$uid operation=$operation $details ownership=$managed installation=$installation finalizers=$finalizers" >&2 else echo "retained-unverified: $kind/$name uid=$uid operation=$operation ownership=$managed installation=$installation; no further deletion attempted" >&2 fi diff --git a/image/npm/package-lock.json b/image/npm/package-lock.json index 270b77504..bf2aca81d 100644 --- a/image/npm/package-lock.json +++ b/image/npm/package-lock.json @@ -490,6 +490,7 @@ "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-agent-core": { "version": "0.80.6", "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.80.6.tgz", + "integrity": "sha512-Lvn89ko42h5ETUb6Z0Ku6ldskEqXaTdQBYvSa0+7bdG9V6rUEpXptv5e0OVZ1HDcvi8s6/2lGCQWsxKX+DFHNw==", "license": "MIT", "dependencies": { "@earendil-works/pi-ai": "^0.80.6", @@ -504,6 +505,7 @@ "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-ai": { "version": "0.80.6", "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.80.6.tgz", + "integrity": "sha512-7xfLk8sANBp+bpPEbjoOZTbPxsa+++b1JXAoSJsNa3vbs9AHHEclmvg54XLQcxH+fuwaeti/g2jeIfJ+mVYLpA==", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "0.91.1", @@ -528,6 +530,7 @@ "node_modules/@earendil-works/pi-coding-agent/node_modules/@earendil-works/pi-tui": { "version": "0.80.6", "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.80.6.tgz", + "integrity": "sha512-bSuzS4EVSqEPj/Qr/p9eqCESfKsGuDNbl77EGci8Iaqqt/C/XCBZL1MjXaxSWW1NsT5afjp/Cb0NTPzOLv/aPA==", "license": "MIT", "dependencies": { "get-east-asian-width": "1.6.0", diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 2aaa39a19..50d7cba78 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -32,6 +32,17 @@ assert_grep '"lockfileVersion": 3' "$ROOT/image/npm/package-lock.json" \ "image runtime npm lock must use the reproducible current lock format" assert_grep '"integrity": "sha512-' "$ROOT/image/npm/package-lock.json" \ "image runtime npm lock must checksum resolved artifacts" +node - "$ROOT/image/npm/package-lock.json" <<'NODE' || fail "every fetched npm artifact must have committed integrity" +const fs = require("fs"); +const lock = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); +const missing = Object.entries(lock.packages) + .filter(([, value]) => value.resolved && !value.integrity) + .map(([name]) => name); +if (missing.length) { + console.error(`missing npm integrity: ${missing.join(", ")}`); + process.exit(1); +} +NODE assert_grep 'npm ci --omit=dev --ignore-scripts' "$ROOT/Dockerfile" \ "image runtime npm installation must consume only the committed lock" assert_no_grep 'npm install --global' "$ROOT/Dockerfile" \ diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index 250c1d0a4..7b5ae1e8d 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -42,10 +42,25 @@ cat > "$FAKEBIN/kubectl" <<'SH' printf 'kubectl' >> "$AGENT_OS_TEST_LOG" printf ' %s' "$@" >> "$AGENT_OS_TEST_LOG" printf '\n' >> "$AGENT_OS_TEST_LOG" +stdin_data='' if [ "${*: -2}" = "-f -" ]; then - cat > "$AGENT_OS_STDIN_LOG" + stdin_data=$(cat) + printf '%s\n' "$stdin_data" >> "$AGENT_OS_STDIN_LOG" + stdin_kind=$(printf '%s\n' "$stdin_data" | awk '$1 == "kind:" { print $2; exit }') + printf 'stdin-kind %s\n' "$stdin_kind" >> "$AGENT_OS_TEST_LOG" fi -if [ "${AGENT_OS_TEST_FAIL_APPLY:-0}" = 1 ] && [[ " $* " = *" apply -f - "* ]]; then +if [ "${AGENT_OS_TEST_FAIL_APPLY:-0}" = 1 ] && [[ " $* " = *" create -f - "* ]] && \ + [ "$stdin_kind" = Pod ]; then + exit 1 +fi +if [ "${AGENT_OS_TEST_LOCK_STATE:-free}" != free ] && [[ " $* " = *" create -f - "* ]] && \ + [ "$stdin_kind" = Lease ]; then + exit 1 +fi +if [ "${AGENT_OS_TEST_FAIL_PVC_PATCH:-0}" = 1 ] && [[ " $* " = *" patch pvc "* ]]; then + exit 1 +fi +if [ "${AGENT_OS_TEST_FAIL_POD_LIST:-0}" = 1 ] && [[ " $* " = *" get pods -o json "* ]]; then exit 1 fi if [ "${AGENT_OS_TEST_FAIL_GENERIC_APPLY:-0}" = 1 ] && [[ " $* " = *" apply -f "* ]] && \ @@ -55,6 +70,9 @@ fi if [ "${AGENT_OS_TEST_FAIL_WAIT:-0}" = 1 ] && [ "${1:-}" = -n ] && [ "${3:-}" = wait ]; then exit 1 fi +if [ "${AGENT_OS_TEST_LOCK_STATE:-free}" = held ] && [[ " $* " = *" wait --for=delete lease/"* ]]; then + exit 1 +fi if [ "${AGENT_OS_TEST_FAIL_ANNOTATE:-0}" = 1 ] && [[ " $* " = *" annotate statefulset agent-os-firstmate "* ]]; then exit 1 fi @@ -66,44 +84,73 @@ case " $* " in *" get pod agent-os-crewmate-"*" --ignore-not-found -o jsonpath="*) id=${AGENT_OS_TEST_CREWMATE_ID:-scout-1} pod_state=${AGENT_OS_TEST_POD_STATE:-absent} - if [[ " $* " = *'.metadata.uid'* ]] && grep -F ' apply -f -' "$AGENT_OS_TEST_LOG" >/dev/null; then - pod_state=${AGENT_OS_TEST_POD_AFTER_APPLY:-$pod_state} + if grep -F 'stdin-kind Pod' "$AGENT_OS_TEST_LOG" >/dev/null; then + pod_state=${AGENT_OS_TEST_POD_AFTER_APPLY:-owned} + fi + last_create=$(grep -Fn 'stdin-kind Pod' "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d: -f1) + last_delete=$(grep -Fn '/pods/agent-os-crewmate-' "$AGENT_OS_TEST_LOG" | grep 'delete --raw' | tail -n 1 | cut -d: -f1) + if [ -n "$last_delete" ] && { [ -z "$last_create" ] || [ "$last_delete" -gt "$last_create" ]; }; then + pod_state=absent fi case "$pod_state" in absent) ;; owned) printf 'agent-os-crewmate-%s\tagent-os\t%s\tagent-os-firstmate:agent-os-demo' "$id" "$id" - [[ " $* " != *'.metadata.uid'* ]] || printf '\toperation-test\tuid-owned' + [[ " $* " != *'.metadata.uid'* ]] || printf '\toperation-test\tuid-owned\trv-pod-owned' ;; replacement) printf 'agent-os-crewmate-%s\tagent-os\t%s\tagent-os-firstmate:agent-os-demo' "$id" "$id" - [[ " $* " != *'.metadata.uid'* ]] || printf '\tother-operation\tuid-replacement' + [[ " $* " != *'.metadata.uid'* ]] || printf '\tother-operation\tuid-replacement\trv-pod-replacement' ;; foreign) printf 'agent-os-crewmate-%s\tother\tother\tother-installation' "$id" - [[ " $* " != *'.metadata.uid'* ]] || printf '\tother-operation\tuid-foreign' + [[ " $* " != *'.metadata.uid'* ]] || printf '\tother-operation\tuid-foreign\trv-pod-foreign' ;; esac ;; *" get pvc agent-os-crewmate-"*" --ignore-not-found -o jsonpath="*) id=${AGENT_OS_TEST_CREWMATE_ID:-scout-1} - case "${AGENT_OS_TEST_PVC_STATE:-absent}" in + pvc_state=${AGENT_OS_TEST_PVC_STATE:-absent} + if grep -F 'stdin-kind PersistentVolumeClaim' "$AGENT_OS_TEST_LOG" >/dev/null; then + [ "$pvc_state" != absent ] || pvc_state=owned + fi + case "$pvc_state" in absent) ;; - owned) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tpending\t\toperation-test\t' "$id" "$id" ;; - clean) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tclean\t2026-07-13T12:00:00Z\toperation-test\toperation-test' "$id" "$id" ;; - stale-clean) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tclean\t2026-07-13T12:00:00Z\toperation-test\told-operation' "$id" "$id" ;; - invalid-checkpoint) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tclean\t2026\toperation-test\toperation-test' "$id" "$id" ;; - foreign) printf 'agent-os-crewmate-%s-home\tother\tother\tother-installation\tclean\t2026-07-13T12:00:00Z\toperation-test\toperation-test' "$id" ;; + owned) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tpending\t\toperation-test\t\tuid-pvc-owned\trv-pvc-owned' "$id" "$id" ;; + clean) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tclean\t2026-07-13T12:00:00Z\toperation-test\toperation-test\tuid-pvc-owned\trv-pvc-owned' "$id" "$id" ;; + stale-clean) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tclean\t2026-07-13T12:00:00Z\toperation-test\told-operation\tuid-pvc-owned\trv-pvc-owned' "$id" "$id" ;; + invalid-checkpoint) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tclean\t2026\toperation-test\toperation-test\tuid-pvc-owned\trv-pvc-owned' "$id" "$id" ;; + foreign) printf 'agent-os-crewmate-%s-home\tother\tother\tother-installation\tclean\t2026-07-13T12:00:00Z\toperation-test\toperation-test\tuid-pvc-foreign\trv-pvc-foreign' "$id" ;; + esac + ;; + *" get lease agent-os-crewmate-"*" --ignore-not-found -o jsonpath="*) + id=${AGENT_OS_TEST_CREWMATE_ID:-scout-1} + case "${AGENT_OS_TEST_LOCK_STATE:-free}" in + held) printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tother-operation\tuid-lock-other' "$id" "$id" ;; + foreign) printf 'agent-os-crewmate-%s-lifecycle\tother\tother\tother-installation\tother-operation\tuid-lock-foreign' "$id" ;; + *) + if grep -F 'stdin-kind Lease' "$AGENT_OS_TEST_LOG" >/dev/null; then + printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\toperation-test\tuid-lock' "$id" "$id" + fi + ;; esac ;; *" get namespace "*" --ignore-not-found -o name "*) - case "${AGENT_OS_TEST_NAMESPACE_STATE:-absent}" in + namespace_state=${AGENT_OS_TEST_NAMESPACE_STATE:-absent} + if grep -F ' apply -f ' "$AGENT_OS_TEST_LOG" >/dev/null; then + namespace_state=${AGENT_OS_TEST_NAMESPACE_AFTER_APPLY:-$namespace_state} + fi + case "$namespace_state" in absent) ;; *) printf 'namespace/%s\n' "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" ;; esac ;; *" get namespace "*" -o jsonpath="*|*" get Namespace "*" -o jsonpath="*) - case "${AGENT_OS_TEST_NAMESPACE_STATE:-absent}" in + namespace_state=${AGENT_OS_TEST_NAMESPACE_STATE:-absent} + if grep -F ' apply -f ' "$AGENT_OS_TEST_LOG" >/dev/null; then + namespace_state=${AGENT_OS_TEST_NAMESPACE_AFTER_APPLY:-$namespace_state} + fi + case "$namespace_state" in owned) if [[ " $* " = *'.metadata.uid'* ]]; then printf '%s\tagent-os\tagent-os-firstmate:%s\tuid-namespace\toperation-test\tTrue\t[]' \ @@ -117,7 +164,11 @@ case " $* " in esac ;; *" get statefulset agent-os-firstmate --ignore-not-found -o jsonpath="*) - case "${AGENT_OS_TEST_WORKLOAD_STATE:-absent}" in + workload_state=${AGENT_OS_TEST_WORKLOAD_STATE:-absent} + if grep -F ' apply -f ' "$AGENT_OS_TEST_LOG" >/dev/null; then + workload_state=${AGENT_OS_TEST_WORKLOAD_AFTER_APPLY:-$workload_state} + fi + case "$workload_state" in absent) ;; namespace) printf 'agent-os-firstmate\tnamespace\t\tagent-os\tagent-os-firstmate:portable-agent-os' ;; cluster-admin) printf 'agent-os-firstmate\tcluster-admin\t\tagent-os\tagent-os-firstmate:portable-agent-os' ;; @@ -151,8 +202,11 @@ case " $* " in absent) ;; owned) printf '%s\tagent-os\tagent-os-firstmate:portable-agent-os' "$name" - [[ " $* " != *'.metadata.uid'* ]] || \ + if [[ " $* " = *'.spec.replicas'* ]]; then + printf '\tuid-statefulset\toperation-test\t\t[kubernetes.io/pvc-protection]\t1\t1\t0\t0\t0\trev-old\trev-new\t2\t1' + elif [[ " $* " = *'.metadata.uid'* ]]; then printf '\tuid-%s\toperation-test\tTrue\t[kubernetes.io/pvc-protection]' "$(printf '%s' "$kind" | tr '[:upper:]' '[:lower:]')" + fi ;; foreign) printf '%s\tother\tother-installation' "$name" ;; esac @@ -161,7 +215,7 @@ case " $* " in if [ "${AGENT_OS_TEST_RBAC_STATE:-exact}" = bad-rules ]; then printf '%s\n' '{"metadata":{"name":"agent-os-firstmate-runtime"},"rules":[]}' else - printf '%s\n' '{"metadata":{"name":"agent-os-firstmate-runtime"},"rules":[{"apiGroups":[""],"resources":["pods","persistentvolumeclaims"],"verbs":["get","list","watch","create","delete","patch"]},{"apiGroups":[""],"resources":["pods/log","pods/exec"],"verbs":["get","list","watch","create","delete"]},{"apiGroups":["apps"],"resources":["statefulsets"],"verbs":["get","list","watch"]}]}' + printf '%s\n' '{"metadata":{"name":"agent-os-firstmate-runtime"},"rules":[{"apiGroups":[""],"resources":["pods","persistentvolumeclaims"],"verbs":["get","list","watch","create","delete","patch"]},{"apiGroups":[""],"resources":["pods/log","pods/exec"],"verbs":["get","list","watch","create","delete"]},{"apiGroups":["apps"],"resources":["statefulsets"],"verbs":["get","list","watch"]},{"apiGroups":["coordination.k8s.io"],"resources":["leases"],"verbs":["get","list","watch","create","delete"]}]}' fi ;; *" get rolebinding agent-os-firstmate-runtime -o json "*) @@ -209,6 +263,13 @@ case " $* " in *" get pods -o name "*) [ -z "${AGENT_OS_TEST_FOREIGN_RESOURCE:-}" ] || printf 'pod/%s\n' "$AGENT_OS_TEST_FOREIGN_RESOURCE" ;; + *" get pods -o json "*) + if [ "${AGENT_OS_TEST_PVC_ATTACHED:-0}" = 1 ]; then + printf '%s\n' '{"items":[{"spec":{"volumes":[{"persistentVolumeClaim":{"claimName":"agent-os-crewmate-scout-1-home"}}]}}]}' + else + printf '%s\n' '{"items":[]}' + fi + ;; *" get serviceaccounts -o name "*) printf '%s\n' serviceaccount/default ;; *" get configmaps -o name "*) printf '%s\n' configmap/kube-root-ca.crt ;; *" get leases.coordination.k8s.io -o name "*) @@ -228,7 +289,11 @@ run_launcher() { : > "$CALLS" run_launcher create scout-1 -grep -Fqx 'kubectl -n agent-os-demo apply -f -' "$CALLS" || fail "create must apply only to agent-os-demo" +if grep -F ' apply -f -' "$CALLS" >/dev/null; then + fail "crewmate create must never adopt a resource through apply" +fi +grep -Fqx 'kubectl -n agent-os-demo create -f -' "$CALLS" || \ + fail "crewmate resources must use create-only semantics when absent" [ "$(grep -Fc 'kind: PersistentVolumeClaim' "$STDIN_LOG")" -eq 1 ] || fail "create must emit one PVC" [ "$(grep -Fc 'kind: Pod' "$STDIN_LOG")" -eq 1 ] || fail "create must emit one Pod" assert_grep 'agent-os.dev/crewmate: scout-1' "$STDIN_LOG" "child resources need the stable crewmate label" @@ -237,7 +302,7 @@ assert_grep 'app.kubernetes.io/managed-by: agent-os' "$STDIN_LOG" \ assert_grep 'agent-os.dev/installation-id: agent-os-firstmate:agent-os-demo' "$STDIN_LOG" \ "child resources need the exact installation identity" assert_grep 'agent-os.dev/operation-id: operation-test' "$STDIN_LOG" \ - "each crewmate apply must carry its unique operation identity" + "each crewmate creation attempt must carry its unique operation identity" assert_grep 'automountServiceAccountToken: false' "$STDIN_LOG" "children must not receive Kubernetes credentials" assert_grep 'claimName: agent-os-crewmate-scout-1-home' "$STDIN_LOG" "child work must use its own PVC" assert_no_grep 'hostUsers: false' "$STDIN_LOG" "OrbStack children must not request unsupported Pod user namespaces" @@ -272,8 +337,8 @@ pass "crewmate create emits one isolated Pod and PVC" if AGENT_OS_TEST_POD_STATE=foreign run_launcher create scout-1 >/dev/null 2>&1; then fail "crewmate create must reject a same-name foreign Pod" fi -if grep -F ' apply -f -' "$CALLS" >/dev/null; then - fail "crewmate create must reject foreign ownership before apply" +if grep -F 'stdin-kind Pod' "$CALLS" >/dev/null; then + fail "crewmate create must reject foreign ownership before Pod creation" fi pass "crewmate create refuses foreign deterministic-name resources" @@ -315,7 +380,7 @@ assert_grep '"uid":"uid-owned"' "$STDIN_LOG" \ "partial apply cleanup must bind deletion to the observed Pod UID" assert_no_grep 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" \ "partial apply cleanup must retain the persistent home" -pass "crewmate partial apply cleans only a newly created owned Pod" +pass "crewmate partial create cleans only a newly created owned Pod" : > "$CALLS" replacement_out='' @@ -325,29 +390,42 @@ replacement_out=$(AGENT_OS_TEST_FAIL_APPLY=1 AGENT_OS_TEST_POD_AFTER_APPLY=repla [ "$replacement_rc" -eq 1 ] || fail "replacement partial apply must remain failed: $replacement_out" assert_contains "$replacement_out" 'replacement or ownership mismatch retained' \ "partial apply must report a replacement instead of deleting it" -assert_no_grep 'delete --raw' "$CALLS" \ - "partial apply must retain a same-name Pod from another operation" +if grep -F '/pods/agent-os-crewmate-scout-1' "$CALLS" | grep -F 'delete --raw' >/dev/null; then + fail "partial apply must retain a same-name Pod from another operation" +fi : > "$CALLS" AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=owned run_launcher stop scout-1 -grep -Fqx 'kubectl -n agent-os-demo delete pod agent-os-crewmate-scout-1 --ignore-not-found --wait=false' "$CALLS" || \ - fail "stop must target the exactly owned crewmate Pod" +grep -Fqx 'kubectl -n agent-os-demo delete --raw /api/v1/namespaces/agent-os-demo/pods/agent-os-crewmate-scout-1 -f -' "$CALLS" || \ + fail "stop must UID-precondition deletion of the exactly owned crewmate Pod" grep -Fqx 'kubectl -n agent-os-demo wait --for=delete pod/agent-os-crewmate-scout-1 --timeout=180s' "$CALLS" || \ fail "stop must prove Pod absence before checkpointing can begin" -wait_line=$(grep -Fn 'wait --for=delete pod/agent-os-crewmate-scout-1' "$CALLS" | head -n 1 | cut -d: -f1) -invalidate_line=$(grep -Fn 'annotate pvc agent-os-crewmate-scout-1-home agent-os.dev/checkpoint-state=pending agent-os.dev/checkpoint-at- agent-os.dev/quiesced-operation=operation-test agent-os.dev/checkpoint-operation- --overwrite' "$CALLS" | head -n 1 | cut -d: -f1) -[ -n "$wait_line" ] && [ -n "$invalidate_line" ] && [ "$wait_line" -lt "$invalidate_line" ] || \ - fail "stop must invalidate every pre-stop checkpoint only after Pod absence" +delete_line=$(grep -Fn '/pods/agent-os-crewmate-scout-1' "$CALLS" | grep 'delete --raw' | head -n 1 | cut -d: -f1) +invalidate_line=$(grep -Fn 'patch pvc agent-os-crewmate-scout-1-home --type=merge' "$CALLS" | head -n 1 | cut -d: -f1) +quiesced_line=$(grep -Fn 'patch pvc agent-os-crewmate-scout-1-home --type=merge' "$CALLS" | tail -n 1 | cut -d: -f1) +[ -n "$delete_line" ] && [ -n "$invalidate_line" ] && [ -n "$quiesced_line" ] && \ + [ "$invalidate_line" -lt "$delete_line" ] && [ "$delete_line" -lt "$quiesced_line" ] || \ + fail "stop must invalidate before deletion and record quiescence only after Pod absence" if grep -F 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" >/dev/null; then fail "stop must preserve the crewmate persistent home" fi pass "crewmate stop preserves its persistent home" +: > "$CALLS" +if AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=clean AGENT_OS_TEST_FAIL_PVC_PATCH=1 \ + run_launcher stop scout-1 >/dev/null 2>&1; then + fail "stop must fail when checkpoint invalidation cannot be recorded" +fi +if grep -F '/pods/agent-os-crewmate-scout-1' "$CALLS" | grep -F 'delete --raw' >/dev/null; then + fail "stop must leave the Pod running when checkpoint invalidation fails" +fi +pass "crewmate stop invalidates checkpoint evidence before Pod deletion" + : > "$CALLS" AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=owned run_launcher restart scout-1 -delete_line=$(grep -Fn 'delete pod agent-os-crewmate-scout-1' "$CALLS" | head -n 1 | cut -d: -f1) -apply_line=$(grep -Fn 'apply -f -' "$CALLS" | head -n 1 | cut -d: -f1) -[ -n "$delete_line" ] && [ -n "$apply_line" ] && [ "$delete_line" -lt "$apply_line" ] || \ +delete_line=$(grep -Fn '/pods/agent-os-crewmate-scout-1' "$CALLS" | grep 'delete --raw' | head -n 1 | cut -d: -f1) +create_line=$(grep -Fn 'stdin-kind Pod' "$CALLS" | tail -n 1 | cut -d: -f1) +[ -n "$delete_line" ] && [ -n "$create_line" ] && [ "$delete_line" -lt "$create_line" ] || \ fail "restart must replace the owned Pod on its retained PVC" if grep -F 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" >/dev/null; then fail "restart must preserve the crewmate persistent home" @@ -405,12 +483,43 @@ assert_no_grep 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" \ AGENT_OS_TEST_POD_STATE=absent AGENT_OS_TEST_PVC_STATE=clean run_launcher purge scout-1 --yes assert_no_grep 'delete pod agent-os-crewmate-scout-1' "$CALLS" \ "purge must accept checkpoint evidence only after the Pod is absent" -grep -Fqx 'kubectl -n agent-os-demo delete pvc agent-os-crewmate-scout-1-home --ignore-not-found --wait=true --timeout=180s' "$CALLS" || \ - fail "purge must delete the exactly owned persistent home" +grep -Fqx 'kubectl -n agent-os-demo delete --raw /api/v1/namespaces/agent-os-demo/persistentvolumeclaims/agent-os-crewmate-scout-1-home -f -' "$CALLS" || \ + fail "purge must atomically delete the exactly owned persistent home" +assert_grep '"uid":"uid-pvc-owned","resourceVersion":"rv-pvc-owned"' "$STDIN_LOG" \ + "purge must precondition deletion on the captured PVC UID and resourceVersion" assert_grep 'purge-complete' "$PURGE_EVIDENCE" "purge must record non-secret completion evidence" assert_no_grep 'scout-1-ai-auth' "$PURGE_EVIDENCE" "purge evidence must never contain credential references" pass "crewmate purge requires confirmation and a clean checkpoint" +: > "$CALLS" +if AGENT_OS_TEST_POD_STATE=absent AGENT_OS_TEST_PVC_STATE=clean AGENT_OS_TEST_PVC_ATTACHED=1 \ + run_launcher purge scout-1 --yes >/dev/null 2>&1; then + fail "purge must refuse a PVC still referenced by any Pod" +fi +if grep -F '/persistentvolumeclaims/agent-os-crewmate-scout-1-home' "$CALLS" | grep -F 'delete --raw' >/dev/null; then + fail "purge must retain an attached persistent home" +fi +pass "crewmate purge rejects attached persistent homes" + +: > "$CALLS" +if AGENT_OS_TEST_POD_STATE=absent AGENT_OS_TEST_PVC_STATE=clean AGENT_OS_TEST_FAIL_POD_LIST=1 \ + run_launcher purge scout-1 --yes >/dev/null 2>&1; then + fail "purge must fail closed when Pod attachments cannot be inventoried" +fi +if grep -F '/persistentvolumeclaims/agent-os-crewmate-scout-1-home' "$CALLS" | grep -F 'delete --raw' >/dev/null; then + fail "purge must retain the PVC after an attachment inventory failure" +fi +pass "crewmate purge fails closed on attachment inventory errors" + +: > "$CALLS" +lock_out='' +lock_rc=0 +lock_out=$(AGENT_OS_TEST_LOCK_STATE=held run_launcher stop scout-1 2>&1) || lock_rc=$? +[ "$lock_rc" -eq 3 ] || fail "bounded lifecycle lock contention must exit incomplete: $lock_out" +assert_contains "$lock_out" "still holds Lease 'agent-os-crewmate-scout-1-lifecycle' after 30s" \ + "lifecycle contention must report the exact holder and bounded timeout" +pass "crewmate lifecycle operations use a bounded coordination lock" + if run_launcher create 'Bad_ID' >/dev/null 2>&1; then fail "invalid Kubernetes crewmate IDs must be rejected" fi @@ -583,6 +692,8 @@ partial_primary_rc=0 partial_primary_out=$(PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" \ AGENT_OS_INPUTS="$PARTIAL_CLUSTER_INPUTS" AGENT_OS_TEST_NAMESPACE=portable-agent-os \ AGENT_OS_TEST_NAMESPACE_STATE=absent AGENT_OS_TEST_WORKLOAD_STATE=absent \ + AGENT_OS_TEST_NAMESPACE_AFTER_APPLY=owned \ + AGENT_OS_TEST_WORKLOAD_AFTER_APPLY=cluster-admin \ AGENT_OS_TEST_RESOURCE_AFTER_APPLY=owned AGENT_OS_TEST_CLUSTER_RBAC_AFTER_APPLY=owned \ AGENT_OS_TEST_FAIL_GENERIC_APPLY=1 AGENT_OS_OPERATION_ID=operation-test \ AGENT_OS_CONTEXT=kind-agent-os AGENT_OS_NAMESPACE=portable-agent-os \ @@ -591,20 +702,27 @@ partial_primary_out=$(PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" \ fail "partial primary apply must exit incomplete with 3: $partial_primary_out" assert_contains "$partial_primary_out" 'partial apply: StatefulSet/agent-os-firstmate' \ "failed primary apply must inventory expected namespaced resources" -assert_contains "$partial_primary_out" 'uid=uid-statefulset operation=operation-test ready=True' \ - "failed primary apply must report UID, operation identity, and readiness" +assert_contains "$partial_primary_out" 'uid=uid-statefulset operation=operation-test desired=1 current=1 ready=0' \ + "failed primary apply must report UID, operation identity, and StatefulSet readiness" assert_contains "$partial_primary_out" 'residual-authority: ClusterRoleBinding/agent-os-firstmate-portable-agent-os' \ "failed cluster-admin apply must report the exact residual grant" -assert_contains "$partial_primary_out" 'cleanup-cluster-rbac --yes' \ - "failed cluster-admin apply must print the privileged cleanup command" assert_contains "$partial_primary_out" 'safe recovery:' \ "failed primary apply must print a bounded exact recovery command" +assert_contains "$partial_primary_out" 'agent-os-kubernetes.sh upgrade' \ + "failed install with an exact-owned StatefulSet must recover through upgrade" +assert_not_contains "$partial_primary_out" 'cleanup-cluster-rbac --yes' \ + "active cluster-admin authority must not advertise an inapplicable cleanup command" +assert_contains "$partial_primary_out" 'desired=1 current=1 ready=0 updated=0 available=0' \ + "failed StatefulSet rollout must report replica readiness" +assert_contains "$partial_primary_out" 'current-revision=rev-old update-revision=rev-new generation=2 observed-generation=1' \ + "failed StatefulSet rollout must report revision and generation progress" pass "primary partial apply reports residual resources and authority" : > "$CALLS" namespace_partial_out='' namespace_partial_rc=0 namespace_partial_out=$(AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_NAMESPACE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_AFTER_APPLY=pending \ AGENT_OS_TEST_RESOURCE_AFTER_APPLY=owned \ AGENT_OS_TEST_CLUSTER_RBAC_AFTER_APPLY=owned AGENT_OS_TEST_FAIL_GENERIC_APPLY=1 \ run_generic upgrade 2>&1) || namespace_partial_rc=$? diff --git a/tests/agent-os-packages.test.sh b/tests/agent-os-packages.test.sh index b6a97c940..22c8fb9e5 100755 --- a/tests/agent-os-packages.test.sh +++ b/tests/agent-os-packages.test.sh @@ -48,7 +48,9 @@ assert_grep '"agent-os.dev/rbac-mode" = input.rbac' "$FIRSTMATE/package.k" \ assert_grep 'resources = ["pods", "persistentvolumeclaims"]' "$FIRSTMATE/package.k" \ "runtime apply authority must exclude Pod subresources from patch access" assert_grep 'verbs = ["get", "list", "watch", "create", "delete", "patch"]' "$FIRSTMATE/package.k" \ - "runtime RBAC must allow kubectl apply to patch retained crewmate PVCs" + "runtime RBAC must allow checkpoint updates on retained crewmate PVCs" +assert_grep 'resources = ["leases"]' "$FIRSTMATE/package.k" \ + "runtime RBAC must permit serialized crewmate lifecycle operations" assert_no_grep 'akuaAuthSecret' "$FIRSTMATE/package.k" \ "the portable package must not require Akua authorization" assert_no_grep 'agent-os.akua.dev' "$FIRSTMATE/package.k" \ @@ -150,9 +152,13 @@ for invalid_image in \ 'registry.example/org:123/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ '[2001:db8::1]:5000/org:123/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ '[2001:db8::g]:5000/org/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ + '[1]:5000/org/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ + '[::::]:5000/org/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ + '[1::2::3]:5000/org/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ + '[::ffff:999.0.2.1]:5000/org/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ 'prefix@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:suffix'; do invalid_inputs="$TMP/invalid-$(printf '%s' "$invalid_image" | tr '/:@' '---').yaml" - sed "s|^image: .*|image: $invalid_image|" "$INPUTS" > "$invalid_inputs" + sed "s|^image: .*|image: \"$invalid_image\"|" "$INPUTS" > "$invalid_inputs" if akua render --no-agent-mode --package "$FIRSTMATE/package.k" --inputs "$invalid_inputs" \ --out "$TMP/invalid-rendered" >/dev/null 2>&1; then fail "the portable package accepted malformed digest reference '$invalid_image'" @@ -174,6 +180,14 @@ sed 's|^image: .*|image: "[2001:db8::1]:5000/org/agent-os@sha256:aaaaaaaaaaaaaaa akua render --no-agent-mode --package "$FIRSTMATE/package.k" --inputs "$IPV6_INPUTS" \ --out "$TMP/registry-ipv6-rendered" >/dev/null || \ fail "the portable package must allow a bracketed IPv6 registry authority" +for ipv6_registry in '::' '::1' '1::' '1:2:3:4:5:6:7:8' '::ffff:192.0.2.1'; do + ipv6_inputs="$TMP/registry-ipv6-$(printf '%s' "$ipv6_registry" | tr ':' '-').yaml" + sed "s|^image: .*|image: \"[$ipv6_registry]:5000/org/agent-os@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"|" \ + "$INPUTS" > "$ipv6_inputs" + akua render --no-agent-mode --package "$FIRSTMATE/package.k" --inputs "$ipv6_inputs" \ + --out "$TMP/registry-ipv6-$(printf '%s' "$ipv6_registry" | tr ':' '-')-rendered" >/dev/null || \ + fail "the portable package rejected valid IPv6 registry '$ipv6_registry'" +done pass "OCI references accept bracketed IPv6 registry authorities" assert_grep 'bin/agent-os-kubernetes.sh install' "$ROOT/docs/kubernetes.md" \ diff --git a/tools/agent-os/packages/firstmate/package.k b/tools/agent-os/packages/firstmate/package.k index 05d744db7..901c20992 100644 --- a/tools/agent-os/packages/firstmate/package.k +++ b/tools/agent-os/packages/firstmate/package.k @@ -18,7 +18,7 @@ schema Input: operationId: str = "" check: - allowMutableImage or regex.match(image, r"^((([a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*|(\[[0-9A-Fa-f:]+\]))(:[0-9]+)?)/)?[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*(\/[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*)*(:[A-Za-z0-9_][A-Za-z0-9_.-]{0,127})?@sha256:[0-9a-f]{64}$"), "image must use one complete OCI reference with an immutable @sha256 digest unless the local profile explicitly allows a mutable image" + allowMutableImage or regex.match(image, r"^((([a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*|(\[(?:(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}|(?:[0-9A-Fa-f]{1,4}:){1,7}:|(?:[0-9A-Fa-f]{1,4}:){1,6}:[0-9A-Fa-f]{1,4}|(?:[0-9A-Fa-f]{1,4}:){1,5}(?::[0-9A-Fa-f]{1,4}){1,2}|(?:[0-9A-Fa-f]{1,4}:){1,4}(?::[0-9A-Fa-f]{1,4}){1,3}|(?:[0-9A-Fa-f]{1,4}:){1,3}(?::[0-9A-Fa-f]{1,4}){1,4}|(?:[0-9A-Fa-f]{1,4}:){1,2}(?::[0-9A-Fa-f]{1,4}){1,5}|[0-9A-Fa-f]{1,4}:(?:(?::[0-9A-Fa-f]{1,4}){1,6})|:(?:(?::[0-9A-Fa-f]{1,4}){1,7}|:))\]))(:[0-9]+)?)/)?[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*(\/[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*)*(:[A-Za-z0-9_][A-Za-z0-9_.-]{0,127})?@sha256:[0-9a-f]{64}$") or regex.match(image, r"^\[(?:(?:[0-9A-Fa-f]{1,4}:){6}|::(?:[0-9A-Fa-f]{1,4}:){0,5}|[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}:){0,4}|(?:[0-9A-Fa-f]{1,4}:){1}[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}:){0,3}|(?:[0-9A-Fa-f]{1,4}:){2}[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}:){0,2}|(?:[0-9A-Fa-f]{1,4}:){3}[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}:){0,1}|(?:[0-9A-Fa-f]{1,4}:){4}[0-9A-Fa-f]{1,4}::)(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}\](:[0-9]+)?\/[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*(\/[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*)*(:[A-Za-z0-9_][A-Za-z0-9_.-]{0,127})?@sha256:[0-9a-f]{64}$"), "image must use one complete OCI reference with an immutable @sha256 digest unless the local profile explicitly allows a mutable image" operationId == "" or regex.match(operationId, r"^[a-z0-9]([a-z0-9.-]{0,61}[a-z0-9])?$"), "operationId must be empty or a valid Kubernetes label value" input: Input = ctx.input() @@ -69,6 +69,11 @@ namespaceRbacResources = [ resources = ["statefulsets"] verbs = ["get", "list", "watch"] }, + { + apiGroups = ["coordination.k8s.io"] + resources = ["leases"] + verbs = ["get", "list", "watch", "create", "delete"] + }, ] }, { From d38fb6dd19b5136feb1bcf7b4518cb82a7ae5e58 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 21:31:10 +0200 Subject: [PATCH 30/56] no-mistakes(review): Captain, harden Kubernetes lifecycle locking and recovery --- bin/agent-os-crewmate.sh | 91 +++--- bin/agent-os-kubernetes-lease.sh | 157 ++++++++++ bin/agent-os-kubernetes.sh | 324 ++++++++++++++++++-- tests/agent-os-kubernetes.test.sh | 261 ++++++++++++---- tests/agent-os-local.test.sh | 83 +++-- tests/agent-os-packages.test.sh | 2 + tools/agent-os/packages/firstmate/package.k | 2 +- 7 files changed, 769 insertions(+), 151 deletions(-) create mode 100644 bin/agent-os-kubernetes-lease.sh diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index 938f5979e..440021d7b 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -39,10 +39,21 @@ fi POD="agent-os-crewmate-$ID" PVC="$POD-home" LOCK="$POD-lifecycle" +LOCK_NAMESPACE=$NAMESPACE EXPECTED_POD="$POD"$'\t'"agent-os"$'\t'"$ID"$'\t'"$INSTALLATION_ID" EXPECTED_PVC="$PVC"$'\t'"agent-os"$'\t'"$ID"$'\t'"$INSTALLATION_ID" EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"$ID"$'\t'"$INSTALLATION_ID" LOCK_UID= +LOCK_RV= +LOCK_RENEW_PID= +LOCK_DURATION_SECONDS=${AGENT_OS_LOCK_DURATION_SECONDS:-300} +LOCK_CLOCK_SKEW_SECONDS=${AGENT_OS_LOCK_CLOCK_SKEW_SECONDS:-5} +LOCK_ACQUIRE_SECONDS=${AGENT_OS_LOCK_ACQUIRE_SECONDS:-30} + +for seconds in "$LOCK_DURATION_SECONDS" "$LOCK_CLOCK_SKEW_SECONDS" "$LOCK_ACQUIRE_SECONDS"; do + case "$seconds" in ''|*[!0-9]*) echo "error: lifecycle Lease timing must use whole seconds" >&2; exit 2 ;; esac +done +[ "$LOCK_DURATION_SECONDS" -ge 3 ] || { echo "error: lifecycle Lease duration must be at least 3 seconds" >&2; exit 2; } kube() { "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" "$@" @@ -66,7 +77,7 @@ pvc_record() { lock_record() { kube get lease "$LOCK" --ignore-not-found \ - -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.spec.holderIdentity}{"\t"}{.metadata.uid}' + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.spec.holderIdentity}{"\t"}{.spec.acquireTime}{"\t"}{.spec.renewTime}{"\t"}{.spec.leaseDurationSeconds}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}' } require_owned_or_absent() { @@ -127,12 +138,15 @@ render_pod() { } render_lock() { + local acquired_at=$1 renewed_at=$2 uid=${3:-} rv=${4:-} cat </dev/null || true - LOCK_UID= -} +. "$ROOT/bin/agent-os-kubernetes-lease.sh" -acquire_lock() { - local record identity holder - if ! render_lock | kube create -f - >/dev/null; then - record=$(lock_record) - identity=$(printf '%s' "$record" | cut -f1-4) - holder=$(printf '%s' "$record" | cut -f5) - if [ -z "$record" ] || [ "$identity" != "$EXPECTED_LOCK" ]; then - echo "error: lifecycle Lease '$LOCK' is absent or has foreign ownership after create conflict" >&2 - exit 2 - fi - if ! kube wait --for=delete "lease/$LOCK" --timeout=30s >/dev/null; then - echo "error: lifecycle operation '$holder' still holds Lease '$LOCK' after 30s" >&2 - exit 3 - fi - if ! render_lock | kube create -f - >/dev/null; then - echo "error: lifecycle Lease '$LOCK' changed during bounded acquisition" >&2 - exit 3 - fi - fi - record=$(lock_record) - identity=$(printf '%s' "$record" | cut -f1-4) - holder=$(printf '%s' "$record" | cut -f5) - LOCK_UID=$(printf '%s' "$record" | cut -f6) - if [ "$identity" != "$EXPECTED_LOCK" ] || [ "$holder" != "$OPERATION_ID" ] || [ -z "$LOCK_UID" ]; then - LOCK_UID= - echo "error: lifecycle Lease '$LOCK' did not verify after acquisition" >&2 - exit 2 +finish_lifecycle() { + local status=$? + trap - EXIT + if ! release_lock && [ "$status" -eq 0 ]; then + status=3 fi + exit "$status" } -trap release_lock EXIT +trap finish_lifecycle EXIT +trap lock_renewal_failed TERM cleanup_new_owned_pod() { local expected_uid=${1:-} after current identity after_operation after_uid @@ -223,8 +214,8 @@ create_and_wait() { pvc_before=$(require_owned_pvc_or_absent) fi pvc_current=$(require_owned_pvc_or_absent) - if [ -z "$pvc_before" ] || [ "$pvc_current" != "$pvc_before" ]; then - echo "error: PVC identity or resourceVersion changed before Pod creation" >&2 + if [ -z "$pvc_before" ] || [ -z "$pvc_current" ]; then + echo "error: PVC identity disappeared before Pod creation" >&2 exit 2 fi pvc_uid=$(printf '%s' "$pvc_before" | cut -f9) @@ -233,6 +224,16 @@ create_and_wait() { echo "error: Pod creation requires exact PVC UID and resourceVersion evidence" >&2 exit 2 fi + if [ "$(printf '%s' "$pvc_current" | cut -f9)" != "$pvc_uid" ]; then + echo "error: PVC UID changed before Pod creation" >&2 + exit 2 + fi + invalidate_checkpoint_evidence + pvc_current=$(require_owned_pvc_or_absent) + if [ -z "$pvc_current" ] || [ "$(printf '%s' "$pvc_current" | cut -f9)" != "$pvc_uid" ]; then + echo "error: PVC UID changed while activating the writer" >&2 + exit 2 + fi if ! render_pod | kube create -f - >/dev/null; then cleanup_new_owned_pod echo "error: create-only Pod operation conflicted or was only partially acknowledged" >&2 @@ -250,6 +251,12 @@ create_and_wait() { echo "error: created Pod lacks exact UID or resourceVersion evidence" >&2 exit 1 fi + pvc_current=$(require_owned_pvc_or_absent) + if [ -z "$pvc_current" ] || [ "$(printf '%s' "$pvc_current" | cut -f9)" != "$pvc_uid" ]; then + cleanup_new_owned_pod "$pod_uid" + echo "error: PVC UID changed after Pod creation; replacement claim retained" >&2 + exit 3 + fi if ! kube wait --for=condition=Ready "pod/$POD" --timeout=180s; then cleanup_new_owned_pod "$pod_uid" echo "error: crewmate Pod did not become ready with the authorized AI Secret" >&2 @@ -306,21 +313,21 @@ stop_owned_pod() { invalidate_checkpoint_evidence() { local pvc uid rv patch pvc=$(require_owned_pvc_or_absent) - [ -n "$pvc" ] || return + [ -n "$pvc" ] || return 0 uid=$(printf '%s' "$pvc" | cut -f9) rv=$(printf '%s' "$pvc" | cut -f10) if [ -z "$uid" ] || [ -z "$rv" ]; then echo "error: checkpoint invalidation requires exact PVC UID and resourceVersion evidence" >&2 exit 2 fi - patch=$(printf '{"metadata":{"uid":"%s","resourceVersion":"%s","annotations":{"agent-os.dev/checkpoint-state":"pending","agent-os.dev/checkpoint-at":null,"agent-os.dev/quiesced-operation":null,"agent-os.dev/checkpoint-operation":null}}}' "$uid" "$rv") + patch=$(printf '{"metadata":{"uid":"%s","resourceVersion":"%s","annotations":{"agent-os.dev/checkpoint-state":"pending","agent-os.dev/writer-state":"active","agent-os.dev/checkpoint-at":null,"agent-os.dev/quiesced-operation":null,"agent-os.dev/checkpoint-operation":null}}}' "$uid" "$rv") kube patch pvc "$PVC" --type=merge -p "$patch" >/dev/null } record_quiesced_generation() { local pvc uid rv patch pvc=$(require_owned_pvc_or_absent) - [ -n "$pvc" ] || return + [ -n "$pvc" ] || return 0 if [ "$(printf '%s' "$pvc" | cut -f5)" != pending ]; then echo "error: checkpoint evidence changed before quiesced generation could be recorded" >&2 exit 2 @@ -331,7 +338,7 @@ record_quiesced_generation() { echo "error: quiesced generation requires exact PVC UID and resourceVersion evidence" >&2 exit 2 fi - patch=$(printf '{"metadata":{"uid":"%s","resourceVersion":"%s","annotations":{"agent-os.dev/quiesced-operation":"%s"}}}' "$uid" "$rv" "$OPERATION_ID") + patch=$(printf '{"metadata":{"uid":"%s","resourceVersion":"%s","annotations":{"agent-os.dev/writer-state":"quiesced","agent-os.dev/quiesced-operation":"%s"}}}' "$uid" "$rv" "$OPERATION_ID") kube patch pvc "$PVC" --type=merge -p "$patch" >/dev/null } diff --git a/bin/agent-os-kubernetes-lease.sh b/bin/agent-os-kubernetes-lease.sh new file mode 100644 index 000000000..daa89c86d --- /dev/null +++ b/bin/agent-os-kubernetes-lease.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash + +rfc3339_epoch() { + local value=${1%%.*} + value=${value%Z}Z + date -j -u -f '%Y-%m-%dT%H:%M:%SZ' "$value" '+%s' 2>/dev/null || \ + date -u -d "$value" '+%s' 2>/dev/null +} + +lease_is_expired() { + local record=$1 renewed duration renewed_epoch now_epoch + renewed=$(printf '%s' "$record" | cut -f7) + duration=$(printf '%s' "$record" | cut -f8) + case "$duration" in ''|*[!0-9]*) return 1 ;; esac + renewed_epoch=$(rfc3339_epoch "$renewed") || return 1 + now_epoch=$(date -u '+%s') + [ "$now_epoch" -gt "$((renewed_epoch + duration + LOCK_CLOCK_SKEW_SECONDS))" ] +} + +verify_lock_record() { + local record=$1 identity holder uid + identity=$(printf '%s' "$record" | cut -f1-4) + holder=$(printf '%s' "$record" | cut -f5) + uid=$(printf '%s' "$record" | cut -f9) + [ "$identity" = "$EXPECTED_LOCK" ] && [ "$holder" = "$OPERATION_ID" ] && [ -n "$uid" ] +} + +renew_lock_once() { + local record acquired uid rv now current + record=$(lock_record) || return 1 + verify_lock_record "$record" || return 1 + acquired=$(printf '%s' "$record" | cut -f6) + uid=$(printf '%s' "$record" | cut -f9) + rv=$(printf '%s' "$record" | cut -f10) + [ -n "$acquired" ] && [ -n "$rv" ] || return 1 + now=$(date -u '+%Y-%m-%dT%H:%M:%SZ') + render_lock "$acquired" "$now" "$uid" "$rv" | kube replace -f - >/dev/null || return 1 + current=$(lock_record) || return 1 + verify_lock_record "$current" || return 1 + [ "$(printf '%s' "$current" | cut -f9)" = "$uid" ] || return 1 + LOCK_RV=$(printf '%s' "$current" | cut -f10) +} + +start_lock_renewal() { + local parent=$$ interval=$((LOCK_DURATION_SECONDS / 3)) + ( + trap - EXIT + sleep_pid= + trap '[ -z "$sleep_pid" ] || kill "$sleep_pid" 2>/dev/null; exit 0' TERM INT + while :; do + sleep "$interval" & + sleep_pid=$! + wait "$sleep_pid" || exit 0 + sleep_pid= + renew_lock_once || { kill -TERM "$parent"; exit 1; } + done + ) & + LOCK_RENEW_PID=$! +} + +stop_lock_renewal() { + [ -n "$LOCK_RENEW_PID" ] || return 0 + kill "$LOCK_RENEW_PID" 2>/dev/null || true + wait "$LOCK_RENEW_PID" 2>/dev/null || true + LOCK_RENEW_PID= +} + +release_lock() { + local record identity holder uid rv after + stop_lock_renewal + [ -n "$LOCK_UID" ] || return 0 + if ! record=$(lock_record); then + echo "error: lifecycle Lease '$LOCK' could not be verified for release" >&2 + return 1 + fi + [ -n "$record" ] || { LOCK_UID=; LOCK_RV=; return 0; } + identity=$(printf '%s' "$record" | cut -f1-4) + holder=$(printf '%s' "$record" | cut -f5) + uid=$(printf '%s' "$record" | cut -f9) + rv=$(printf '%s' "$record" | cut -f10) + if [ "$identity" != "$EXPECTED_LOCK" ] || [ "$holder" != "$OPERATION_ID" ] || \ + [ "$uid" != "$LOCK_UID" ] || [ -z "$rv" ]; then + echo "error: lifecycle Lease '$LOCK' changed ownership before release; retained" >&2 + return 1 + fi + if ! printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s","resourceVersion":"%s"}}\n' "$uid" "$rv" | \ + kube delete --raw "/apis/coordination.k8s.io/v1/namespaces/$LOCK_NAMESPACE/leases/$LOCK" -f - >/dev/null; then + echo "error: lifecycle Lease '$LOCK' release precondition failed; retained" >&2 + return 1 + fi + after=$(lock_record) || return 1 + if [ -n "$after" ]; then + echo "error: lifecycle Lease '$LOCK' still exists after release" >&2 + return 1 + fi + LOCK_UID= + LOCK_RV= +} + +acquire_lock() { + local record identity holder now deadline acquired uid rv current + now=$(date -u '+%Y-%m-%dT%H:%M:%SZ') + deadline=$(($(date -u '+%s') + LOCK_ACQUIRE_SECONDS)) + while ! render_lock "$now" "$now" | kube create -f - >/dev/null; do + record=$(lock_record) + identity=$(printf '%s' "$record" | cut -f1-4) + holder=$(printf '%s' "$record" | cut -f5) + if [ -z "$record" ] || [ "$identity" != "$EXPECTED_LOCK" ]; then + echo "error: lifecycle Lease '$LOCK' is absent or has foreign ownership after create conflict" >&2 + exit 2 + fi + if lease_is_expired "$record"; then + uid=$(printf '%s' "$record" | cut -f9) + rv=$(printf '%s' "$record" | cut -f10) + if [ -z "$uid" ] || [ -z "$rv" ]; then + echo "error: expired lifecycle Lease '$LOCK' lacks CAS identity" >&2 + exit 2 + fi + now=$(date -u '+%Y-%m-%dT%H:%M:%SZ') + if render_lock "$now" "$now" "$uid" "$rv" | kube replace -f - >/dev/null; then + break + fi + fi + if [ "$(date -u '+%s')" -ge "$deadline" ]; then + echo "error: lifecycle operation '$holder' still holds Lease '$LOCK' after ${LOCK_ACQUIRE_SECONDS}s" >&2 + exit 3 + fi + sleep 1 + now=$(date -u '+%Y-%m-%dT%H:%M:%SZ') + done + record=$(lock_record) + if ! verify_lock_record "$record"; then + LOCK_UID= + echo "error: lifecycle Lease '$LOCK' did not verify after acquisition" >&2 + exit 2 + fi + acquired=$(printf '%s' "$record" | cut -f6) + LOCK_UID=$(printf '%s' "$record" | cut -f9) + LOCK_RV=$(printf '%s' "$record" | cut -f10) + if [ -z "$acquired" ] || [ -z "$LOCK_RV" ]; then + LOCK_UID= + echo "error: lifecycle Lease '$LOCK' lacks complete renewal evidence" >&2 + exit 2 + fi + current=$(lock_record) + if [ "$current" != "$record" ]; then + LOCK_UID= + echo "error: lifecycle Lease '$LOCK' changed after acquisition" >&2 + exit 3 + fi + start_lock_renewal +} + +lock_renewal_failed() { + echo "error: lifecycle Lease '$LOCK' renewal failed; operation stopped" >&2 + exit 3 +} diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh index 273b95179..bda4b909a 100755 --- a/bin/agent-os-kubernetes.sh +++ b/bin/agent-os-kubernetes.sh @@ -21,11 +21,34 @@ MANAGES_NAMESPACE=0 DESIRED_RBAC=none INSTALLATION_ID= OPERATION_ID=${AGENT_OS_OPERATION_ID:-"$(date -u '+%Y%m%d%H%M%S')-$$-$RANDOM"} +LOCK= +LOCK_NAMESPACE= +EXPECTED_LOCK= +LOCK_UID= +LOCK_RV= +LOCK_RENEW_PID= +LOCK_DURATION_SECONDS=${AGENT_OS_LOCK_DURATION_SECONDS:-300} +LOCK_CLOCK_SKEW_SECONDS=${AGENT_OS_LOCK_CLOCK_SKEW_SECONDS:-5} +LOCK_ACQUIRE_SECONDS=${AGENT_OS_LOCK_ACQUIRE_SECONDS:-30} + +for seconds in "$LOCK_DURATION_SECONDS" "$LOCK_CLOCK_SKEW_SECONDS" "$LOCK_ACQUIRE_SECONDS"; do + case "$seconds" in ''|*[!0-9]*) echo "error: lifecycle Lease timing must use whole seconds" >&2; exit 2 ;; esac +done +[ "$LOCK_DURATION_SECONDS" -ge 3 ] || { echo "error: lifecycle Lease duration must be at least 3 seconds" >&2; exit 2; } + +. "$ROOT/bin/agent-os-kubernetes-lease.sh" cleanup() { + local status=$? + trap - EXIT + if ! release_lock && [ "$status" -eq 0 ]; then + status=3 + fi rm -rf "$OUT" "$RENDER_INPUTS" + exit "$status" } trap cleanup EXIT +trap lock_renewal_failed TERM usage() { echo "usage: $0 install|upgrade|rollback|status|uninstall|cleanup-cluster-rbac [--yes] [--delete-namespace]" >&2 @@ -107,6 +130,82 @@ namespace_identity() { -o 'jsonpath={.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}' } +rendered_file_for_kind() { + local desired=$1 file + while IFS= read -r file; do + [ "$(rendered_resource_field "$file" kind)" = "$desired" ] && { printf '%s' "$file"; return; } + done < <(find "$OUT" -type f -name '*.yaml' -print) +} + +create_managed_namespace_if_absent() { + local file identity + [ "$MANAGES_NAMESPACE" -eq 1 ] || return 0 + [ -z "$(namespace_name)" ] || return 0 + file=$(rendered_file_for_kind Namespace) + [ -n "$file" ] || { echo "error: managed namespace render is missing" >&2; exit 2; } + if ! "$KUBECTL" --context "$CONTEXT" create -f "$file" >/dev/null; then + identity=$(namespace_identity 2>/dev/null || true) + if [ "$identity" != "agent-os"$'\t'"$INSTALLATION_ID" ]; then + echo "error: namespace '$NAMESPACE' create conflicted without exact installation ownership" >&2 + exit 2 + fi + fi + identity=$(namespace_identity) + if [ "$identity" != "agent-os"$'\t'"$INSTALLATION_ID" ]; then + echo "error: namespace '$NAMESPACE' did not retain exact installation ownership" >&2 + exit 2 + fi +} + +kube() { + "$KUBECTL" --context "$CONTEXT" -n "$LOCK_NAMESPACE" "$@" +} + +lock_record() { + kube get lease "$LOCK" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/lifecycle}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.spec.holderIdentity}{"\t"}{.spec.acquireTime}{"\t"}{.spec.renewTime}{"\t"}{.spec.leaseDurationSeconds}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}' +} + +render_lock() { + local acquired_at=$1 renewed_at=$2 uid=${3:-} rv=${4:-} + cat <&2 + exit 2 + fi + EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"primary"$'\t'"$INSTALLATION_ID" + acquire_lock +} + require_namespace_contract() { local name identity name=$(namespace_name) @@ -165,6 +264,78 @@ live_resource_identity() { -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}' } +live_resource_record() { + local scope=$1 kind=$2 name=$3 + if [ "$scope" = cluster ]; then + "$KUBECTL" --context "$CONTEXT" get "$kind" "$name" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}' + else + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get "$kind" "$name" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}' + fi +} + +resource_api_path() { + local scope=$1 kind=$2 name=$3 + case "$kind" in + Namespace) printf '/api/v1/namespaces/%s' "$name" ;; + Pod) printf '/api/v1/namespaces/%s/pods/%s' "$NAMESPACE" "$name" ;; + PersistentVolumeClaim) printf '/api/v1/namespaces/%s/persistentvolumeclaims/%s' "$NAMESPACE" "$name" ;; + Service) printf '/api/v1/namespaces/%s/services/%s' "$NAMESPACE" "$name" ;; + ServiceAccount) printf '/api/v1/namespaces/%s/serviceaccounts/%s' "$NAMESPACE" "$name" ;; + StatefulSet) printf '/apis/apps/v1/namespaces/%s/statefulsets/%s' "$NAMESPACE" "$name" ;; + Role) printf '/apis/rbac.authorization.k8s.io/v1/namespaces/%s/roles/%s' "$NAMESPACE" "$name" ;; + RoleBinding) printf '/apis/rbac.authorization.k8s.io/v1/namespaces/%s/rolebindings/%s' "$NAMESPACE" "$name" ;; + ClusterRoleBinding) printf '/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/%s' "$name" ;; + *) echo "error: unsupported atomic deletion target $scope $kind/$name" >&2; return 2 ;; + esac +} + +delete_owned_resource() { + local scope=$1 kind=$2 name=$3 timeout=$4 record expected uid rv path + record=$(live_resource_record "$scope" "$kind" "$name") + [ -n "$record" ] || return 0 + expected="$name"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" + if [ "$(printf '%s' "$record" | cut -f1-3)" != "$expected" ]; then + echo "error: $kind '$name' changed ownership before deletion" >&2 + exit 2 + fi + uid=$(printf '%s' "$record" | cut -f4) + rv=$(printf '%s' "$record" | cut -f5) + [ -n "$uid" ] && [ -n "$rv" ] || { echo "error: $kind '$name' lacks deletion preconditions" >&2; exit 2; } + path=$(resource_api_path "$scope" "$kind" "$name") + if ! printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s","resourceVersion":"%s"}}\n' "$uid" "$rv" | \ + "$KUBECTL" --context "$CONTEXT" delete --raw "$path" -f - >/dev/null; then + bounded_delete_failure "$kind/$name" + return $? + fi + if [ "$scope" = cluster ]; then + "$KUBECTL" --context "$CONTEXT" wait --for=delete "$kind/$name" --timeout="${timeout}s" || bounded_delete_failure "$kind/$name" + else + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" wait --for=delete "$kind/$name" --timeout="${timeout}s" || bounded_delete_failure "$kind/$name" + fi +} + +patch_workload_annotations() { + local annotations=$1 record expected uid rv patch current + record=$(live_resource_record namespaced StatefulSet agent-os-firstmate) + expected="agent-os-firstmate"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" + if [ "$(printf '%s' "$record" | cut -f1-3)" != "$expected" ]; then + echo "error: StatefulSet ownership changed before annotation mutation" >&2 + exit 2 + fi + uid=$(printf '%s' "$record" | cut -f4) + rv=$(printf '%s' "$record" | cut -f5) + [ -n "$uid" ] && [ -n "$rv" ] || { echo "error: StatefulSet lacks annotation CAS identity" >&2; exit 2; } + patch=$(printf '{"metadata":{"uid":"%s","resourceVersion":"%s","annotations":%s}}' "$uid" "$rv" "$annotations") + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch StatefulSet agent-os-firstmate --type=merge -p "$patch" >/dev/null + current=$(live_resource_record namespaced StatefulSet agent-os-firstmate) + if [ "$(printf '%s' "$current" | cut -f1-4)" != "$(printf '%s' "$record" | cut -f1-4)" ]; then + echo "error: StatefulSet UID or ownership changed during annotation mutation" >&2 + exit 3 + fi +} + require_namespaced_resource_owned_or_absent() { local kind=$1 name=$2 identity expected identity=$(live_resource_identity "$kind" "$name") @@ -199,16 +370,8 @@ preflight_rendered_resources() { } delete_namespace_rbac() { - require_namespaced_resource_owned_or_absent RoleBinding agent-os-firstmate-runtime - require_namespaced_resource_owned_or_absent Role agent-os-firstmate-runtime - if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" delete rolebinding \ - agent-os-firstmate-runtime --ignore-not-found --wait=true --timeout=180s; then - bounded_delete_failure "RoleBinding/agent-os-firstmate-runtime" - fi - if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" delete role \ - agent-os-firstmate-runtime --ignore-not-found --wait=true --timeout=180s; then - bounded_delete_failure "Role/agent-os-firstmate-runtime" - fi + delete_owned_resource namespaced RoleBinding agent-os-firstmate-runtime 180 + delete_owned_resource namespaced Role agent-os-firstmate-runtime 180 } resource_observation() { @@ -321,8 +484,10 @@ partial_recovery_action() { if [ -n "$state" ]; then identity=$(printf '%s' "$state" | cut -f1,4,5) if [ "$identity" = "agent-os-firstmate"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" ]; then - printf upgrade - return + if partial_install_is_applicable; then + printf upgrade + return + fi fi return 1 fi @@ -385,10 +550,80 @@ report_partial_apply() { return 3 } -apply_rendered() { - if ! "$KUBECTL" --context "$CONTEXT" apply -f "$OUT"; then - report_partial_apply apply +cas_patch_file() { + local file=$1 uid=$2 rv=$3 patch_file + patch_file=$(mktemp "$OUT/cas.XXXXXX") + awk -v uid="$uid" -v rv="$rv" ' + !inserted && $1 == "metadata:" { + print + print " uid: " uid + print " resourceVersion: " rv + inserted=1 + next + } + { print } + ' "$file" > "$patch_file" + printf '%s' "$patch_file" +} + +mutate_rendered_resource() { + local file=$1 kind name scope record expected uid rv operation current patch_file + kind=$(rendered_resource_field "$file" kind) + name=$(rendered_resource_field "$file" name) + [ "$kind" != Namespace ] || return 0 + scope=namespaced + [ "$kind" != ClusterRoleBinding ] || scope=cluster + record=$(live_resource_record "$scope" "$kind" "$name") + expected="$name"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" + if [ -z "$record" ]; then + if ! "$KUBECTL" --context "$CONTEXT" create -f "$file" >/dev/null; then + current=$(live_resource_record "$scope" "$kind" "$name" 2>/dev/null || true) + if [ "$(printf '%s' "$current" | cut -f1-3)" != "$expected" ] || \ + [ "$(printf '%s' "$current" | cut -f6)" != "$OPERATION_ID" ]; then + report_partial_apply create + return $? + fi + fi + else + if [ "$(printf '%s' "$record" | cut -f1-3)" != "$expected" ]; then + echo "error: $kind '$name' changed ownership before mutation" >&2 + exit 2 + fi + uid=$(printf '%s' "$record" | cut -f4) + rv=$(printf '%s' "$record" | cut -f5) + [ -n "$uid" ] && [ -n "$rv" ] || { echo "error: $kind '$name' lacks CAS identity" >&2; exit 2; } + patch_file=$(cas_patch_file "$file" "$uid" "$rv") + if [ "$scope" = cluster ]; then + if ! "$KUBECTL" --context "$CONTEXT" patch "$kind" "$name" --type=merge --patch-file "$patch_file" >/dev/null; then + report_partial_apply patch + return $? + fi + else + if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch "$kind" "$name" --type=merge --patch-file "$patch_file" >/dev/null; then + report_partial_apply patch + return $? + fi + fi fi + current=$(live_resource_record "$scope" "$kind" "$name") + operation=$(printf '%s' "$current" | cut -f6) + if [ "$(printf '%s' "$current" | cut -f1-3)" != "$expected" ] || [ -z "$(printf '%s' "$current" | cut -f4)" ] || \ + [ "$operation" != "$OPERATION_ID" ]; then + report_partial_apply verify + return $? + fi + if [ -n "$record" ] && [ "$(printf '%s' "$current" | cut -f4)" != "$(printf '%s' "$record" | cut -f4)" ]; then + echo "error: $kind '$name' UID changed during mutation" >&2 + exit 3 + fi +} + +apply_rendered() { + local file + while IFS= read -r file; do + [ -n "$file" ] || continue + mutate_rendered_resource "$file" || return $? + done < <(find "$OUT" -type f -name '*.yaml' -print) if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status \ statefulset/agent-os-firstmate --timeout=180s; then report_partial_apply rollout @@ -410,7 +645,7 @@ verify_desired_rbac() { {"apiGroups":[""],"resources":["pods","persistentvolumeclaims"],"verbs":["get","list","watch","create","delete","patch"]}, {"apiGroups":[""],"resources":["pods/log","pods/exec"],"verbs":["get","list","watch","create","delete"]}, {"apiGroups":["apps"],"resources":["statefulsets"],"verbs":["get","list","watch"]}, - {"apiGroups":["coordination.k8s.io"],"resources":["leases"],"verbs":["get","list","watch","create","delete"]} + {"apiGroups":["coordination.k8s.io"],"resources":["leases"],"verbs":["get","create","update","delete"]} ]' >/dev/null || ! printf '%s' "$binding_json" | jq -e --arg namespace "$NAMESPACE" ' .roleRef == {"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"agent-os-firstmate-runtime"} and @@ -514,10 +749,7 @@ delete_rendered_kind() { kind=$(rendered_resource_field "$file" kind) if [ "$kind" = "$desired_kind" ]; then name=$(rendered_resource_field "$file" name) - if ! "$KUBECTL" --context "$CONTEXT" delete --ignore-not-found \ - --wait=true --timeout=180s -f "$file"; then - bounded_delete_failure "$kind/$name" - fi + delete_owned_resource namespaced "$kind" "$name" 180 fi done < <(find "$OUT" -type f -name '*.yaml' -print) } @@ -536,7 +768,7 @@ delete_rendered_namespaced_resources() { } namespace_is_empty() { - local resources resource objects object + local resources resource objects object lock if ! resources=$("$KUBECTL" --context "$CONTEXT" api-resources --verbs=list --namespaced -o name); then echo "error: could not inventory namespaced Kubernetes resource types" >&2 return 1 @@ -554,6 +786,13 @@ namespace_is_empty() { [ -n "$object" ] || continue case "$object" in serviceaccount/default|configmap/kube-root-ca.crt) ;; + lease/agent-os-firstmate-lifecycle|lease.coordination.k8s.io/agent-os-firstmate-lifecycle) + lock=$(lock_record 2>/dev/null || true) + if ! verify_lock_record "$lock" || [ "$(printf '%s' "$lock" | cut -f9)" != "$LOCK_UID" ]; then + echo "error: namespace '$NAMESPACE' contains an unverified lifecycle Lease" >&2 + return 1 + fi + ;; *) echo "error: namespace '$NAMESPACE' still contains foreign resource '$object'" >&2 return 1 @@ -582,10 +821,10 @@ delete_owned_empty_namespace() { echo "error: namespace '$NAMESPACE' ownership changed during deletion checks" >&2 exit 2 fi - if ! "$KUBECTL" --context "$CONTEXT" delete namespace "$NAMESPACE" \ - --wait=true --timeout=180s; then - bounded_delete_failure "Namespace/$NAMESPACE" - fi + stop_lock_renewal + delete_owned_resource cluster Namespace "$NAMESPACE" 180 + LOCK_UID= + LOCK_RV= } cleanup_cluster_rbac() { @@ -611,15 +850,13 @@ cleanup_cluster_rbac() { exit 2 fi fi - "$KUBECTL" --context "$CONTEXT" delete clusterrolebinding "$binding" --wait=false - "$KUBECTL" --context "$CONTEXT" wait --for=delete "clusterrolebinding/$binding" --timeout=60s + delete_owned_resource cluster ClusterRoleBinding "$binding" 60 fi if [ -n "$(namespace_name)" ]; then workload=$(workload_state) require_workload_owned "$workload" optional if [ -n "$workload" ] && [ "$(printf '%s' "$workload" | cut -f3)" = required ]; then - "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" annotate statefulset agent-os-firstmate \ - agent-os.dev/cluster-rbac-cleanup- >/dev/null + patch_workload_annotations '{"agent-os.dev/cluster-rbac-cleanup":null}' fi fi echo "evidence: clusterrolebinding/$binding absent" @@ -630,6 +867,9 @@ case "$COMMAND" in [ "$CONFIRMED" -eq 0 ] && [ "$DELETE_NAMESPACE" -eq 0 ] || usage render require_namespace_contract + create_managed_namespace_if_absent + acquire_primary_lock + require_namespace_contract preflight_rendered_resources 1 previous=$(workload_state) require_workload_owned "$previous" optional @@ -643,6 +883,8 @@ case "$COMMAND" in [ "$CONFIRMED" -eq 0 ] && [ "$DELETE_NAMESPACE" -eq 0 ] || usage render require_namespace_contract + acquire_primary_lock + require_namespace_contract preflight_rendered_resources 1 previous=$(workload_state) if [ -z "$previous" ]; then @@ -656,8 +898,7 @@ case "$COMMAND" in if [ "$DESIRED_RBAC" != cluster-admin ] && \ { [ "$previous_mode" = cluster-admin ] || [ -z "$previous_mode" ] || [ "$previous_cleanup" = required ]; }; then cleanup_required=1 - "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" annotate statefulset agent-os-firstmate \ - agent-os.dev/cluster-rbac-cleanup=required --overwrite >/dev/null + patch_workload_annotations '{"agent-os.dev/cluster-rbac-cleanup":"required"}' fi apply_rendered verify_desired_rbac @@ -665,8 +906,7 @@ case "$COMMAND" in delete_namespace_rbac fi if [ "$DESIRED_RBAC" = cluster-admin ] && [ "$previous_cleanup" = required ]; then - "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" annotate statefulset agent-os-firstmate \ - agent-os.dev/cluster-rbac-cleanup- >/dev/null + patch_workload_annotations '{"agent-os.dev/cluster-rbac-cleanup":null}' fi if [ "$cleanup_required" -eq 1 ]; then report_cluster_cleanup @@ -677,8 +917,23 @@ case "$COMMAND" in [ "$CONFIRMED" -eq 0 ] && [ "$DELETE_NAMESPACE" -eq 0 ] || usage render require_namespace_contract + acquire_primary_lock + require_namespace_contract + preflight_rendered_resources 1 previous=$(workload_state) require_workload_owned "$previous" + rollback_record=$(live_resource_record namespaced StatefulSet agent-os-firstmate) + if [ "$(printf '%s' "$rollback_record" | cut -f1-3)" != \ + "agent-os-firstmate"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" ] || \ + [ -z "$(printf '%s' "$rollback_record" | cut -f4)" ] || [ -z "$(printf '%s' "$rollback_record" | cut -f5)" ]; then + echo "error: rollback requires exact StatefulSet UID and resourceVersion evidence" >&2 + exit 2 + fi + rollback_current=$(live_resource_record namespaced StatefulSet agent-os-firstmate) + if [ "$rollback_current" != "$rollback_record" ]; then + echo "error: StatefulSet changed before rollback mutation" >&2 + exit 3 + fi "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout undo statefulset/agent-os-firstmate "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s ;; @@ -693,6 +948,8 @@ case "$COMMAND" in fi render require_namespace_contract + acquire_primary_lock + require_namespace_contract preflight_rendered_resources previous=$(workload_state) require_workload_owned "$previous" optional @@ -716,6 +973,7 @@ case "$COMMAND" in exit 2 fi render + acquire_primary_lock cleanup_cluster_rbac ;; *) usage ;; diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index 7b5ae1e8d..5aa696c25 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -45,7 +45,7 @@ printf '\n' >> "$AGENT_OS_TEST_LOG" stdin_data='' if [ "${*: -2}" = "-f -" ]; then stdin_data=$(cat) - printf '%s\n' "$stdin_data" >> "$AGENT_OS_STDIN_LOG" + printf '%s\n' "$stdin_data" >> "${AGENT_OS_STDIN_LOG:-$AGENT_OS_TEST_LOG.stdin}" stdin_kind=$(printf '%s\n' "$stdin_data" | awk '$1 == "kind:" { print $2; exit }') printf 'stdin-kind %s\n' "$stdin_kind" >> "$AGENT_OS_TEST_LOG" fi @@ -63,23 +63,35 @@ fi if [ "${AGENT_OS_TEST_FAIL_POD_LIST:-0}" = 1 ] && [[ " $* " = *" get pods -o json "* ]]; then exit 1 fi -if [ "${AGENT_OS_TEST_FAIL_GENERIC_APPLY:-0}" = 1 ] && [[ " $* " = *" apply -f "* ]] && \ - [[ " $* " != *" apply -f - "* ]]; then +if [ "${AGENT_OS_TEST_FAIL_GENERIC_APPLY:-0}" = 1 ] && \ + { [[ " $* " = *" apply -f "* ]] || [[ " $* " = *" create -f "*statefulset.yaml* ]] || \ + [[ " $* " = *" patch StatefulSet agent-os-firstmate "* ]]; }; then exit 1 fi if [ "${AGENT_OS_TEST_FAIL_WAIT:-0}" = 1 ] && [ "${1:-}" = -n ] && [ "${3:-}" = wait ]; then exit 1 fi +if [ "${AGENT_OS_TEST_FAIL_ROLLOUT:-0}" = 1 ] && [[ " $* " = *" rollout status statefulset/agent-os-firstmate "* ]]; then + exit 1 +fi if [ "${AGENT_OS_TEST_LOCK_STATE:-free}" = held ] && [[ " $* " = *" wait --for=delete lease/"* ]]; then exit 1 fi -if [ "${AGENT_OS_TEST_FAIL_ANNOTATE:-0}" = 1 ] && [[ " $* " = *" annotate statefulset agent-os-firstmate "* ]]; then +if [ "${AGENT_OS_TEST_FAIL_ANNOTATE:-0}" = 1 ] && \ + { [[ " $* " = *" annotate statefulset agent-os-firstmate "* ]] || \ + { [[ " $* " = *" patch StatefulSet agent-os-firstmate "* ]] && [[ " $* " = *"cluster-rbac-cleanup"* ]]; }; }; then exit 1 fi if [ -n "${AGENT_OS_TEST_FAIL_DELETE_FILE:-}" ] && [[ " $* " = *" delete "* ]] && \ [[ " $* " = *"$AGENT_OS_TEST_FAIL_DELETE_FILE"* ]]; then exit 1 fi +if [ -n "${AGENT_OS_TEST_FAIL_DELETE_TARGET:-}" ] && [[ " $* " = *"$AGENT_OS_TEST_FAIL_DELETE_TARGET"* ]]; then + exit 1 +fi +if [ -n "${AGENT_OS_TEST_READY_DELAY:-}" ] && [[ " $* " = *" wait --for=condition=Ready pod/"* ]]; then + sleep "$AGENT_OS_TEST_READY_DELAY" +fi case " $* " in *" get pod agent-os-crewmate-"*" --ignore-not-found -o jsonpath="*) id=${AGENT_OS_TEST_CREWMATE_ID:-scout-1} @@ -121,24 +133,60 @@ case " $* " in stale-clean) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tclean\t2026-07-13T12:00:00Z\toperation-test\told-operation\tuid-pvc-owned\trv-pvc-owned' "$id" "$id" ;; invalid-checkpoint) printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tclean\t2026\toperation-test\toperation-test\tuid-pvc-owned\trv-pvc-owned' "$id" "$id" ;; foreign) printf 'agent-os-crewmate-%s-home\tother\tother\tother-installation\tclean\t2026-07-13T12:00:00Z\toperation-test\toperation-test\tuid-pvc-foreign\trv-pvc-foreign' "$id" ;; + binder-rv) + pvc_reads=$(grep -Fc ' get pvc agent-os-crewmate-' "$AGENT_OS_TEST_LOG") + printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tpending\t\t\t\tuid-pvc-owned\trv-pvc-%s' "$id" "$id" "$pvc_reads" + ;; + replaced-after-pod) + if grep -F 'stdin-kind Pod' "$AGENT_OS_TEST_LOG" >/dev/null; then + printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tpending\t\t\t\tuid-pvc-replacement\trv-pvc-replacement' "$id" "$id" + else + printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tpending\t\t\t\tuid-pvc-owned\trv-pvc-owned' "$id" "$id" + fi + ;; esac ;; *" get lease agent-os-crewmate-"*" --ignore-not-found -o jsonpath="*) id=${AGENT_OS_TEST_CREWMATE_ID:-scout-1} case "${AGENT_OS_TEST_LOCK_STATE:-free}" in - held) printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tother-operation\tuid-lock-other' "$id" "$id" ;; - foreign) printf 'agent-os-crewmate-%s-lifecycle\tother\tother\tother-installation\tother-operation\tuid-lock-foreign' "$id" ;; + held) printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tother-operation\t2099-01-01T00:00:00Z\t2099-01-01T00:00:00Z\t300\tuid-lock-other\trv-lock-other' "$id" "$id" ;; + foreign) printf 'agent-os-crewmate-%s-lifecycle\tother\tother\tother-installation\tother-operation\t2099-01-01T00:00:00Z\t2099-01-01T00:00:00Z\t300\tuid-lock-foreign\trv-lock-foreign' "$id" ;; + expired) + last_expired_delete=$(grep -Fn "/leases/agent-os-crewmate-$id-lifecycle" "$AGENT_OS_TEST_LOG" | grep 'delete --raw' | tail -n 1 | cut -d: -f1) + if [ -n "$last_expired_delete" ]; then + : + elif grep -F ' replace -f -' "$AGENT_OS_TEST_LOG" >/dev/null; then + printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\toperation-test\t2026-07-13T00:00:00Z\t2026-07-13T00:00:00Z\t300\tuid-lock-expired\trv-lock-taken' "$id" "$id" + else + printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tother-operation\t2000-01-01T00:00:00Z\t2000-01-01T00:00:00Z\t300\tuid-lock-expired\trv-lock-expired' "$id" "$id" + fi + ;; *) - if grep -F 'stdin-kind Lease' "$AGENT_OS_TEST_LOG" >/dev/null; then - printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\toperation-test\tuid-lock' "$id" "$id" + last_lock_write=$(grep -Fn 'stdin-kind Lease' "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d: -f1) + last_lock_delete=$(grep -Fn "/leases/agent-os-crewmate-$id-lifecycle" "$AGENT_OS_TEST_LOG" | grep 'delete --raw' | tail -n 1 | cut -d: -f1) + if [ -n "$last_lock_delete" ] && [ "${AGENT_OS_TEST_LOCK_RELEASE_STATE:-}" = foreign ]; then + printf 'agent-os-crewmate-%s-lifecycle\tother\tother\tother-installation\tother-operation\t2099-01-01T00:00:00Z\t2099-01-01T00:00:00Z\t300\tuid-lock-foreign\trv-lock-foreign' "$id" + elif [ -n "$last_lock_write" ] && { [ -z "$last_lock_delete" ] || [ "$last_lock_write" -gt "$last_lock_delete" ]; }; then + printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\toperation-test\t2026-07-13T00:00:00Z\t2026-07-13T00:00:00Z\t300\tuid-lock\trv-lock' "$id" "$id" fi ;; esac ;; + *" get lease agent-os-firstmate-lifecycle"*" --ignore-not-found -o jsonpath="*) + lock_name=$(printf '%s\n' "$*" | sed -n 's/.* get lease \([^ ]*\) .*/\1/p') + last_lock_write=$(grep -Fn 'stdin-kind Lease' "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d: -f1) + last_lock_delete=$(grep -Fn "/leases/$lock_name" "$AGENT_OS_TEST_LOG" | grep 'delete --raw' | tail -n 1 | cut -d: -f1) + if [ -n "$last_lock_write" ] && { [ -z "$last_lock_delete" ] || [ "$last_lock_write" -gt "$last_lock_delete" ]; }; then + lock_stdin=${AGENT_OS_STDIN_LOG:-$AGENT_OS_TEST_LOG.stdin} + lock_holder=$(awk '/holderIdentity:/ { holder=$2 } END { print holder }' "$lock_stdin") + printf '%s\tagent-os\tprimary\tagent-os-firstmate:%s\t%s\t2026-07-13T00:00:00Z\t2026-07-13T00:00:00Z\t300\tuid-primary-lock\trv-primary-lock' "$lock_name" "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" "$lock_holder" + fi + ;; *" get namespace "*" --ignore-not-found -o name "*) namespace_state=${AGENT_OS_TEST_NAMESPACE_STATE:-absent} - if grep -F ' apply -f ' "$AGENT_OS_TEST_LOG" >/dev/null; then - namespace_state=${AGENT_OS_TEST_NAMESPACE_AFTER_APPLY:-$namespace_state} + if grep -F ' create -f ' "$AGENT_OS_TEST_LOG" | grep -F 'namespace.yaml' >/dev/null || \ + grep -F ' apply -f ' "$AGENT_OS_TEST_LOG" >/dev/null; then + namespace_state=${AGENT_OS_TEST_NAMESPACE_AFTER_APPLY:-owned} fi case "$namespace_state" in absent) ;; @@ -147,12 +195,17 @@ case " $* " in ;; *" get namespace "*" -o jsonpath="*|*" get Namespace "*" -o jsonpath="*) namespace_state=${AGENT_OS_TEST_NAMESPACE_STATE:-absent} - if grep -F ' apply -f ' "$AGENT_OS_TEST_LOG" >/dev/null; then - namespace_state=${AGENT_OS_TEST_NAMESPACE_AFTER_APPLY:-$namespace_state} + if grep -F ' create -f ' "$AGENT_OS_TEST_LOG" | grep -F 'namespace.yaml' >/dev/null || \ + grep -F ' apply -f ' "$AGENT_OS_TEST_LOG" >/dev/null; then + namespace_state=${AGENT_OS_TEST_NAMESPACE_AFTER_APPLY:-owned} fi case "$namespace_state" in owned) - if [[ " $* " = *'.metadata.uid'* ]]; then + if [[ " $* " = *'.metadata.resourceVersion'* ]]; then + current_operation=$(grep 'akua-input-operation ' "$AGENT_OS_TEST_LOG" | tail -n 1 | awk '{print $2}') + printf '%s\tagent-os\tagent-os-firstmate:%s\tuid-namespace\trv-namespace\t%s' \ + "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" "$current_operation" + elif [[ " $* " = *'.metadata.uid'* ]]; then printf '%s\tagent-os\tagent-os-firstmate:%s\tuid-namespace\toperation-test\tTrue\t[]' \ "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" else @@ -165,8 +218,10 @@ case " $* " in ;; *" get statefulset agent-os-firstmate --ignore-not-found -o jsonpath="*) workload_state=${AGENT_OS_TEST_WORKLOAD_STATE:-absent} - if grep -F ' apply -f ' "$AGENT_OS_TEST_LOG" >/dev/null; then - workload_state=${AGENT_OS_TEST_WORKLOAD_AFTER_APPLY:-$workload_state} + if grep -F ' create -f ' "$AGENT_OS_TEST_LOG" | grep -F 'statefulset.yaml' >/dev/null || \ + grep -F ' patch StatefulSet agent-os-firstmate ' "$AGENT_OS_TEST_LOG" >/dev/null || \ + grep -F ' apply -f ' "$AGENT_OS_TEST_LOG" >/dev/null; then + workload_state=${AGENT_OS_TEST_WORKLOAD_AFTER_APPLY:-namespace} fi case "$workload_state" in absent) ;; @@ -195,14 +250,19 @@ case " $* " in *" Role "*) kind=Role; name=agent-os-firstmate-runtime ;; esac resource_state=${AGENT_OS_TEST_RESOURCE_STATE:-absent} - if grep -F ' apply -f ' "$AGENT_OS_TEST_LOG" >/dev/null; then - resource_state=${AGENT_OS_TEST_RESOURCE_AFTER_APPLY:-$resource_state} + if grep -E ' create -f .+\.yaml' "$AGENT_OS_TEST_LOG" | grep -v 'namespace.yaml' >/dev/null || \ + grep -F ' --patch-file ' "$AGENT_OS_TEST_LOG" >/dev/null || \ + grep -F ' apply -f ' "$AGENT_OS_TEST_LOG" >/dev/null; then + resource_state=${AGENT_OS_TEST_RESOURCE_AFTER_APPLY:-owned} fi case "$resource_state" in absent) ;; owned) printf '%s\tagent-os\tagent-os-firstmate:portable-agent-os' "$name" - if [[ " $* " = *'.spec.replicas'* ]]; then + if [[ " $* " = *'.metadata.resourceVersion'* ]]; then + current_operation=$(grep 'akua-input-operation ' "$AGENT_OS_TEST_LOG" | tail -n 1 | awk '{print $2}') + printf '\tuid-%s\trv-%s\t%s' "$(printf '%s' "$kind" | tr '[:upper:]' '[:lower:]')" "$(printf '%s' "$kind" | tr '[:upper:]' '[:lower:]')" "$current_operation" + elif [[ " $* " = *'.spec.replicas'* ]]; then printf '\tuid-statefulset\toperation-test\t\t[kubernetes.io/pvc-protection]\t1\t1\t0\t0\t0\trev-old\trev-new\t2\t1' elif [[ " $* " = *'.metadata.uid'* ]]; then printf '\tuid-%s\toperation-test\tTrue\t[kubernetes.io/pvc-protection]' "$(printf '%s' "$kind" | tr '[:upper:]' '[:lower:]')" @@ -215,7 +275,7 @@ case " $* " in if [ "${AGENT_OS_TEST_RBAC_STATE:-exact}" = bad-rules ]; then printf '%s\n' '{"metadata":{"name":"agent-os-firstmate-runtime"},"rules":[]}' else - printf '%s\n' '{"metadata":{"name":"agent-os-firstmate-runtime"},"rules":[{"apiGroups":[""],"resources":["pods","persistentvolumeclaims"],"verbs":["get","list","watch","create","delete","patch"]},{"apiGroups":[""],"resources":["pods/log","pods/exec"],"verbs":["get","list","watch","create","delete"]},{"apiGroups":["apps"],"resources":["statefulsets"],"verbs":["get","list","watch"]},{"apiGroups":["coordination.k8s.io"],"resources":["leases"],"verbs":["get","list","watch","create","delete"]}]}' + printf '%s\n' '{"metadata":{"name":"agent-os-firstmate-runtime"},"rules":[{"apiGroups":[""],"resources":["pods","persistentvolumeclaims"],"verbs":["get","list","watch","create","delete","patch"]},{"apiGroups":[""],"resources":["pods/log","pods/exec"],"verbs":["get","list","watch","create","delete"]},{"apiGroups":["apps"],"resources":["statefulsets"],"verbs":["get","list","watch"]},{"apiGroups":["coordination.k8s.io"],"resources":["leases"],"verbs":["get","create","update","delete"]}]}' fi ;; *" get rolebinding agent-os-firstmate-runtime -o json "*) @@ -235,15 +295,25 @@ case " $* " in *" get clusterrolebinding agent-os-firstmate-"*" --ignore-not-found -o jsonpath="*|\ *" get ClusterRoleBinding agent-os-firstmate-"*" --ignore-not-found -o jsonpath="*) cluster_state=${AGENT_OS_TEST_CLUSTER_RBAC_STATE:-absent} - if grep -F ' apply -f ' "$AGENT_OS_TEST_LOG" >/dev/null; then - cluster_state=${AGENT_OS_TEST_CLUSTER_RBAC_AFTER_APPLY:-$cluster_state} + if [ -n "${AGENT_OS_TEST_CLUSTER_RBAC_AFTER_APPLY:-}" ] && \ + { grep -E ' create -f .+\.yaml' "$AGENT_OS_TEST_LOG" >/dev/null || grep -F ' --patch-file ' "$AGENT_OS_TEST_LOG" >/dev/null; }; then + cluster_state=$AGENT_OS_TEST_CLUSTER_RBAC_AFTER_APPLY + elif grep -F ' create -f ' "$AGENT_OS_TEST_LOG" | grep -F 'clusterrolebinding.yaml' >/dev/null || \ + grep -F ' patch ClusterRoleBinding agent-os-firstmate-' "$AGENT_OS_TEST_LOG" >/dev/null || \ + grep -F ' apply -f ' "$AGENT_OS_TEST_LOG" >/dev/null; then + cluster_state=${AGENT_OS_TEST_CLUSTER_RBAC_AFTER_APPLY:-owned} fi case "$cluster_state" in absent) ;; owned) printf 'agent-os-firstmate-%s\tagent-os\tagent-os-firstmate:%s' \ "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" - [[ " $* " != *'.metadata.uid'* ]] || printf '\tuid-clusterrolebinding\toperation-test\tTrue\t[]' + if [[ " $* " = *'.metadata.resourceVersion'* ]]; then + current_operation=$(grep 'akua-input-operation ' "$AGENT_OS_TEST_LOG" | tail -n 1 | awk '{print $2}') + printf '\tuid-clusterrolebinding\trv-clusterrolebinding\t%s' "$current_operation" + elif [[ " $* " = *'.metadata.uid'* ]]; then + printf '\tuid-clusterrolebinding\toperation-test\tTrue\t[]' + fi ;; foreign) printf 'agent-os-firstmate-portable-agent-os\tother\tother-installation' ;; esac @@ -284,6 +354,7 @@ run_launcher() { AGENT_OS_IN_CLUSTER=1 AGENT_OS_NAMESPACE=agent-os-demo AGENT_OS_IMAGE=agent-os:local-test \ AGENT_OS_IMAGE_PULL_POLICY=Never AGENT_OS_AI_SECRET=scout-1-ai-auth \ AGENT_OS_OPERATION_ID=operation-test AGENT_OS_PURGE_EVIDENCE_FILE="$PURGE_EVIDENCE" \ + AGENT_OS_LOCK_ACQUIRE_SECONDS=0 \ "$LAUNCHER" "$@" } @@ -303,6 +374,10 @@ assert_grep 'agent-os.dev/installation-id: agent-os-firstmate:agent-os-demo' "$S "child resources need the exact installation identity" assert_grep 'agent-os.dev/operation-id: operation-test' "$STDIN_LOG" \ "each crewmate creation attempt must carry its unique operation identity" +assert_grep 'acquireTime:' "$STDIN_LOG" \ + "lifecycle Leases must record their acquisition time" +assert_grep 'renewTime:' "$STDIN_LOG" \ + "lifecycle Leases must record renewable expiry evidence" assert_grep 'automountServiceAccountToken: false' "$STDIN_LOG" "children must not receive Kubernetes credentials" assert_grep 'claimName: agent-os-crewmate-scout-1-home' "$STDIN_LOG" "child work must use its own PVC" assert_no_grep 'hostUsers: false' "$STDIN_LOG" "OrbStack children must not request unsupported Pod user namespaces" @@ -382,6 +457,31 @@ assert_no_grep 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" \ "partial apply cleanup must retain the persistent home" pass "crewmate partial create cleans only a newly created owned Pod" +: > "$CALLS" +AGENT_OS_TEST_PVC_STATE=clean run_launcher create scout-1 +writer_patch_line=$(grep -Fn 'patch pvc agent-os-crewmate-scout-1-home --type=merge' "$CALLS" | head -n 1 | cut -d: -f1) +writer_create_line=$(grep -Fn 'stdin-kind Pod' "$CALLS" | head -n 1 | cut -d: -f1) +[ -n "$writer_patch_line" ] && [ -n "$writer_create_line" ] && [ "$writer_patch_line" -lt "$writer_create_line" ] || \ + fail "every Pod start must invalidate clean checkpoint evidence before creation" +assert_grep '"agent-os.dev/checkpoint-state":"pending"' "$CALLS" \ + "writer activation must CAS the PVC to pending" +assert_grep '"agent-os.dev/writer-state":"active"' "$CALLS" \ + "writer activation must mark the retained PVC as active" +pass "crewmate starts invalidate quiesced checkpoint evidence" + +: > "$CALLS" +AGENT_OS_TEST_PVC_STATE=binder-rv run_launcher create scout-1 || \ + fail "normal PVC resourceVersion movement must not change stable UID identity" +pass "crewmate creation tracks PVC UID instead of binder resourceVersion" + +: > "$CALLS" +pvc_replaced_rc=0 +AGENT_OS_TEST_PVC_STATE=replaced-after-pod run_launcher create scout-1 >/dev/null 2>&1 || pvc_replaced_rc=$? +[ "$pvc_replaced_rc" -ne 0 ] || fail "PVC replacement after Pod creation must fail closed" +grep -F '/pods/agent-os-crewmate-scout-1' "$CALLS" | grep -F 'delete --raw' >/dev/null || \ + fail "PVC replacement must UID-delete only the newly created owned Pod" +pass "crewmate creation retains replacement claims and removes its Pod" + : > "$CALLS" replacement_out='' replacement_rc=0 @@ -516,10 +616,32 @@ lock_out='' lock_rc=0 lock_out=$(AGENT_OS_TEST_LOCK_STATE=held run_launcher stop scout-1 2>&1) || lock_rc=$? [ "$lock_rc" -eq 3 ] || fail "bounded lifecycle lock contention must exit incomplete: $lock_out" -assert_contains "$lock_out" "still holds Lease 'agent-os-crewmate-scout-1-lifecycle' after 30s" \ +assert_contains "$lock_out" "still holds Lease 'agent-os-crewmate-scout-1-lifecycle' after 0s" \ "lifecycle contention must report the exact holder and bounded timeout" pass "crewmate lifecycle operations use a bounded coordination lock" +: > "$CALLS" +expired_lock_rc=0 +expired_lock_out=$(AGENT_OS_TEST_LOCK_STATE=expired AGENT_OS_TEST_PVC_STATE=owned run_launcher stop scout-1 2>&1) || expired_lock_rc=$? +[ "$expired_lock_rc" -eq 0 ] || fail "expired exact-owned lifecycle Lease takeover must complete cleanly: $expired_lock_out" +grep -F 'replace -f -' "$CALLS" >/dev/null || \ + fail "expired exact-owned lifecycle Leases must use resourceVersion-CAS takeover" +pass "crewmate lifecycle can recover exact-owned expired Leases" + +: > "$CALLS" +AGENT_OS_LOCK_DURATION_SECONDS=3 AGENT_OS_TEST_READY_DELAY=2 \ + AGENT_OS_TEST_PVC_STATE=owned run_launcher create scout-1 +grep -F 'replace -f -' "$CALLS" >/dev/null || \ + fail "active lifecycle operations must renew their exact-owned Lease" +pass "crewmate lifecycle renews its Lease while active" + +: > "$CALLS" +release_conflict_rc=0 +AGENT_OS_TEST_POD_STATE=absent AGENT_OS_TEST_PVC_STATE=owned \ + AGENT_OS_TEST_LOCK_RELEASE_STATE=foreign run_launcher stop scout-1 >/dev/null 2>&1 || release_conflict_rc=$? +[ "$release_conflict_rc" -eq 3 ] || fail "lifecycle completion must fail if Lease release ownership cannot verify" +pass "crewmate lifecycle fails closed on Lease release conflicts" + if run_launcher create 'Bad_ID' >/dev/null 2>&1; then fail "invalid Kubernetes crewmate IDs must be rejected" fi @@ -660,10 +782,14 @@ SH chmod +x "$FAKEBIN/akua" run_generic() { - PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS="$GENERIC_INPUTS" \ + local workload_state=${AGENT_OS_TEST_WORKLOAD_STATE:-absent} resource_state=${AGENT_OS_TEST_RESOURCE_STATE:-} + if [ -z "$resource_state" ]; then + [ "$workload_state" = absent ] && resource_state=absent || resource_state=owned + fi + PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_STDIN_LOG="$STDIN_LOG" AGENT_OS_INPUTS="$GENERIC_INPUTS" \ AGENT_OS_TEST_NAMESPACE=portable-agent-os \ AGENT_OS_TEST_NAMESPACE_STATE="${AGENT_OS_TEST_NAMESPACE_STATE:-absent}" \ - AGENT_OS_TEST_WORKLOAD_STATE="${AGENT_OS_TEST_WORKLOAD_STATE:-absent}" \ + AGENT_OS_TEST_WORKLOAD_STATE="$workload_state" AGENT_OS_TEST_RESOURCE_STATE="$resource_state" \ AGENT_OS_TEST_CLUSTER_RBAC_STATE="${AGENT_OS_TEST_CLUSTER_RBAC_STATE:-absent}" \ AGENT_OS_OPERATION_ID=operation-test \ AGENT_OS_CONTEXT=kind-agent-os AGENT_OS_NAMESPACE=portable-agent-os "$GENERIC" "$@" @@ -673,8 +799,13 @@ run_generic() { run_generic install grep -Fq -- "akua render --no-agent-mode --package $ROOT/tools/agent-os/packages/firstmate/package.k --inputs " "$CALLS" || \ fail "generic install must render the canonical package before applying it" -grep -Fq 'kubectl --context kind-agent-os apply -f ' "$CALLS" || \ - fail "generic install must apply only its freshly rendered package output" +if grep -F 'kubectl --context kind-agent-os apply -f ' "$CALLS" >/dev/null; then + fail "generic install must not adopt same-name resources through apply" +fi +grep -F 'kubectl --context kind-agent-os create -f ' "$CALLS" >/dev/null || \ + fail "generic install must create absent rendered resources atomically" +grep -F 'agent-os-firstmate-lifecycle' "$STDIN_LOG" >/dev/null || \ + fail "primary mutations must hold an exact-owned Kubernetes Lease" grep -Fqx 'akua-input-operation operation-test' "$CALLS" || \ fail "generic install must label every resource with its unique operation identity" grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os rollout status statefulset/agent-os-firstmate --timeout=180s' "$CALLS" || \ @@ -682,7 +813,7 @@ grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os rollout status s if grep -F 'delete clusterrolebinding' "$CALLS" >/dev/null; then fail "fresh namespace-scoped install must not require cluster RBAC deletion authority" fi -pass "generic install renders and applies the canonical package on an explicit context" +pass "generic install serializes create-only canonical package mutations" PARTIAL_CLUSTER_INPUTS="$TMP/partial-cluster-inputs.yaml" sed 's/rbac: namespace/rbac: cluster-admin/' "$GENERIC_INPUTS" > "$PARTIAL_CLUSTER_INPUTS" @@ -695,7 +826,7 @@ partial_primary_out=$(PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" \ AGENT_OS_TEST_NAMESPACE_AFTER_APPLY=owned \ AGENT_OS_TEST_WORKLOAD_AFTER_APPLY=cluster-admin \ AGENT_OS_TEST_RESOURCE_AFTER_APPLY=owned AGENT_OS_TEST_CLUSTER_RBAC_AFTER_APPLY=owned \ - AGENT_OS_TEST_FAIL_GENERIC_APPLY=1 AGENT_OS_OPERATION_ID=operation-test \ + AGENT_OS_TEST_FAIL_ROLLOUT=1 AGENT_OS_OPERATION_ID=operation-test \ AGENT_OS_CONTEXT=kind-agent-os AGENT_OS_NAMESPACE=portable-agent-os \ "$GENERIC" install 2>&1) || partial_primary_rc=$? [ "$partial_primary_rc" -eq 3 ] || \ @@ -724,7 +855,7 @@ namespace_partial_rc=0 namespace_partial_out=$(AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_NAMESPACE_STATE=owned \ AGENT_OS_TEST_WORKLOAD_AFTER_APPLY=pending \ AGENT_OS_TEST_RESOURCE_AFTER_APPLY=owned \ - AGENT_OS_TEST_CLUSTER_RBAC_AFTER_APPLY=owned AGENT_OS_TEST_FAIL_GENERIC_APPLY=1 \ + AGENT_OS_TEST_CLUSTER_RBAC_AFTER_APPLY=owned AGENT_OS_TEST_FAIL_ROLLOUT=1 \ run_generic upgrade 2>&1) || namespace_partial_rc=$? [ "$namespace_partial_rc" -eq 3 ] || \ fail "namespace partial upgrade must exit incomplete with 3: $namespace_partial_out" @@ -735,6 +866,21 @@ assert_contains "$namespace_partial_out" 'cleanup-cluster-rbac --yes' \ "a failed namespaced mutation with residual authority must print exact cleanup" pass "namespace partial apply reports stale cluster authority" +: > "$CALLS" +conflicted_recovery_out='' +conflicted_recovery_rc=0 +conflicted_recovery_out=$(AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_NAMESPACE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_AFTER_APPLY=namespace AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_RESOURCE_AFTER_APPLY=foreign AGENT_OS_TEST_FAIL_ROLLOUT=1 \ + run_generic upgrade 2>&1) || conflicted_recovery_rc=$? +[ "$conflicted_recovery_rc" -eq 3 ] || \ + fail "conflicted partial recovery must remain incomplete: $conflicted_recovery_out" +assert_contains "$conflicted_recovery_out" 'safe recovery unavailable:' \ + "partial recovery must reject conflicts across the complete rendered resource set" +assert_not_contains "$conflicted_recovery_out" 'agent-os-kubernetes.sh upgrade' \ + "partial recovery must not advertise an upgrade that its preflight rejects" +pass "primary recovery reuses complete rendered ownership predicates" + : > "$CALLS" owned_namespace_out='' owned_namespace_rc=0 @@ -753,7 +899,7 @@ PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS="$UNOWNED_INPUT AGENT_OS_TEST_NAMESPACE=portable-agent-os AGENT_OS_TEST_NAMESPACE_STATE=unowned \ AGENT_OS_TEST_WORKLOAD_STATE=absent AGENT_OS_CONTEXT=kind-agent-os \ AGENT_OS_NAMESPACE=portable-agent-os "$GENERIC" install -grep -Fq 'kubectl --context kind-agent-os apply -f ' "$CALLS" || \ +grep -E 'kubectl --context kind-agent-os (create -f|.* patch )' "$CALLS" >/dev/null || \ fail "createNamespace=false must install into a pre-existing unowned namespace" pass "createNamespace=false requires and preserves an unowned namespace" @@ -811,8 +957,8 @@ cleanup_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=c run_generic upgrade 2>&1) || cleanup_rc=$? [ "$cleanup_rc" -eq 3 ] || \ fail "cluster-admin downgrade must stop for privileged cleanup with exit 3, got $cleanup_rc: $cleanup_out" -apply_line=$(grep -Fn 'kubectl --context kind-agent-os apply -f ' "$CALLS" | head -n 1 | cut -d: -f1) -marker_line=$(grep -Fn 'annotate statefulset agent-os-firstmate agent-os.dev/cluster-rbac-cleanup=required' "$CALLS" | head -n 1 | cut -d: -f1) +apply_line=$(grep -Fn -- '--patch-file ' "$CALLS" | head -n 1 | cut -d: -f1) +marker_line=$(grep -Fn 'patch StatefulSet agent-os-firstmate' "$CALLS" | grep 'cluster-rbac-cleanup.*required' | head -n 1 | cut -d: -f1) rollout_line=$(grep -Fn 'kubectl --context kind-agent-os -n portable-agent-os rollout status' "$CALLS" | head -n 1 | cut -d: -f1) verify_line=$(grep -Fn 'kubectl --context kind-agent-os -n portable-agent-os get role agent-os-firstmate-runtime' "$CALLS" | head -n 1 | cut -d: -f1) [ -n "$marker_line" ] && [ -n "$apply_line" ] && [ -n "$rollout_line" ] && [ -n "$verify_line" ] && \ @@ -833,7 +979,7 @@ marker_failure_rc=0 AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=cluster-admin \ AGENT_OS_TEST_FAIL_ANNOTATE=1 run_generic upgrade >/dev/null 2>&1 || marker_failure_rc=$? [ "$marker_failure_rc" -ne 0 ] || fail "downgrade must fail if its durable cleanup marker cannot be recorded" -if grep -F 'kubectl --context kind-agent-os apply -f ' "$CALLS" >/dev/null; then +if grep -F -- '--patch-file ' "$CALLS" >/dev/null || grep -E ' create -f .+\.yaml' "$CALLS" >/dev/null; then fail "downgrade must record its durable cleanup marker before changing the workload RBAC mode" fi pass "cluster-admin downgrade records cleanup state before mutation" @@ -865,9 +1011,11 @@ AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=pending \ AGENT_OS_TEST_CLUSTER_RBAC_STATE=owned run_generic cleanup-cluster-rbac --yes grep -Fq 'kubectl --context kind-agent-os get clusterrolebinding agent-os-firstmate-portable-agent-os --ignore-not-found' "$CALLS" || \ fail "privileged cleanup must inspect only the exact stale ClusterRoleBinding" -grep -Fqx 'kubectl --context kind-agent-os delete clusterrolebinding agent-os-firstmate-portable-agent-os --wait=false' "$CALLS" || \ - fail "privileged cleanup must delete only the exact owned ClusterRoleBinding" -grep -Fqx 'kubectl --context kind-agent-os wait --for=delete clusterrolebinding/agent-os-firstmate-portable-agent-os --timeout=60s' "$CALLS" || \ +grep -F '/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/agent-os-firstmate-portable-agent-os' "$CALLS" | grep -F 'delete --raw' >/dev/null || \ + fail "privileged cleanup must UID-delete only the exact owned ClusterRoleBinding" +assert_grep '"uid":"uid-clusterrolebinding","resourceVersion":"rv-clusterrolebinding"' "$STDIN_LOG" \ + "privileged cleanup must bind deletion to the observed grant identity" +grep -F 'wait --for=delete ClusterRoleBinding/agent-os-firstmate-portable-agent-os --timeout=60s' "$CALLS" >/dev/null || \ fail "privileged cleanup must produce deletion evidence for the exact binding" pass "privileged cleanup verifies ownership and deletes one exact binding" @@ -909,14 +1057,14 @@ sed 's/rbac: namespace/rbac: cluster-admin/' "$GENERIC_INPUTS" > "$CLUSTER_ADMIN : > "$CALLS" PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS="$CLUSTER_ADMIN_INPUTS" \ AGENT_OS_TEST_NAMESPACE=portable-agent-os AGENT_OS_TEST_NAMESPACE_STATE=owned \ - AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_CONTEXT=kind-agent-os \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_RESOURCE_STATE=owned AGENT_OS_CONTEXT=kind-agent-os \ AGENT_OS_NAMESPACE=portable-agent-os "$GENERIC" upgrade -grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete rolebinding agent-os-firstmate-runtime --ignore-not-found --wait=true --timeout=180s' "$CALLS" || \ - fail "cluster-admin upgrade must delete the stale namespace RoleBinding" -grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete role agent-os-firstmate-runtime --ignore-not-found --wait=true --timeout=180s' "$CALLS" || \ - fail "cluster-admin upgrade must delete the stale namespace Role" -apply_line=$(grep -Fn 'kubectl --context kind-agent-os apply -f ' "$CALLS" | head -n 1 | cut -d: -f1) -delete_line=$(grep -Fn 'delete rolebinding agent-os-firstmate-runtime' "$CALLS" | head -n 1 | cut -d: -f1) +grep -F '/apis/rbac.authorization.k8s.io/v1/namespaces/portable-agent-os/rolebindings/agent-os-firstmate-runtime' "$CALLS" | grep -F 'delete --raw' >/dev/null || \ + fail "cluster-admin upgrade must UID-delete the stale namespace RoleBinding" +grep -F '/apis/rbac.authorization.k8s.io/v1/namespaces/portable-agent-os/roles/agent-os-firstmate-runtime' "$CALLS" | grep -F 'delete --raw' >/dev/null || \ + fail "cluster-admin upgrade must UID-delete the stale namespace Role" +apply_line=$(grep -Fn -- '--patch-file ' "$CALLS" | head -n 1 | cut -d: -f1) +delete_line=$(grep -Fn '/rolebindings/agent-os-firstmate-runtime' "$CALLS" | grep 'delete --raw' | head -n 1 | cut -d: -f1) [ -n "$apply_line" ] && [ -n "$delete_line" ] && [ "$apply_line" -lt "$delete_line" ] || \ fail "cluster-admin upgrade must apply replacement authority before removing namespace RBAC" if grep -F 'delete clusterrolebinding agent-os-firstmate-portable-agent-os' "$CALLS" >/dev/null; then @@ -931,6 +1079,8 @@ grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os rollout undo sta fail "generic rollback must target only the Firstmate StatefulSet" grep -Fq 'akua render --no-agent-mode' "$CALLS" || \ fail "rollback must derive its namespace and identity from the current package render" +[ "$(grep -F 'get StatefulSet agent-os-firstmate --ignore-not-found' "$CALLS" | grep -Fc 'metadata.resourceVersion')" -ge 2 ] || \ + fail "rollback must revalidate StatefulSet UID and resourceVersion immediately before mutation" pass "generic rollback verifies its rendered installation identity" : > "$CALLS" @@ -948,19 +1098,20 @@ if run_generic uninstall >/dev/null 2>&1; then fail "generic uninstall must require --yes" fi : > "$CALLS" -AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=namespace run_generic uninstall --yes +AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=namespace \ + AGENT_OS_TEST_RESOURCE_STATE=owned run_generic uninstall --yes if grep -E 'kubectl .* (get|delete) clusterrolebinding' "$CALLS" >/dev/null; then fail "routine namespace uninstall must never request cluster-wide RBAC authority" fi if grep -F 'delete namespace portable-agent-os' "$CALLS" >/dev/null; then fail "bounded uninstall must retain its namespace by default" fi -grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete rolebinding agent-os-firstmate-runtime --ignore-not-found --wait=true --timeout=180s' "$CALLS" || \ +grep -F '/rolebindings/agent-os-firstmate-runtime' "$CALLS" | grep -F 'delete --raw' >/dev/null || \ fail "uninstall must remove namespace runtime binding regardless of current inputs" -grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os delete role agent-os-firstmate-runtime --ignore-not-found --wait=true --timeout=180s' "$CALLS" || \ +grep -F '/roles/agent-os-firstmate-runtime' "$CALLS" | grep -F 'delete --raw' >/dev/null || \ fail "uninstall must remove namespace runtime Role regardless of current inputs" -stateful_delete_line=$(grep -Fn '/statefulset.yaml' "$CALLS" | grep ' delete ' | head -n 1 | cut -d: -f1) -pvc_delete_line=$(grep -Fn '/00-pvc.yaml' "$CALLS" | grep ' delete ' | head -n 1 | cut -d: -f1) +stateful_delete_line=$(grep -Fn '/statefulsets/agent-os-firstmate' "$CALLS" | grep 'delete --raw' | head -n 1 | cut -d: -f1) +pvc_delete_line=$(grep -Fn '/persistentvolumeclaims/agent-os-firstmate-home' "$CALLS" | grep 'delete --raw' | head -n 1 | cut -d: -f1) [ -n "$stateful_delete_line" ] && [ -n "$pvc_delete_line" ] && \ [ "$stateful_delete_line" -lt "$pvc_delete_line" ] || \ fail "uninstall must delete the StatefulSet before waiting on PVC deletion" @@ -974,7 +1125,7 @@ bounded_rc=0 bounded_out=$(PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS="$NONE_INPUTS" \ AGENT_OS_TEST_NAMESPACE=portable-agent-os AGENT_OS_TEST_NAMESPACE_STATE=owned \ AGENT_OS_TEST_WORKLOAD_STATE=none AGENT_OS_TEST_RESOURCE_STATE=owned \ - AGENT_OS_TEST_PRIMARY_POD_STATE=owned AGENT_OS_TEST_FAIL_DELETE_FILE=00-pvc.yaml \ + AGENT_OS_TEST_PRIMARY_POD_STATE=owned AGENT_OS_TEST_FAIL_DELETE_TARGET=persistentvolumeclaims/agent-os-firstmate-home \ AGENT_OS_OPERATION_ID=operation-test AGENT_OS_CONTEXT=kind-agent-os \ AGENT_OS_NAMESPACE=portable-agent-os "$GENERIC" uninstall --yes 2>&1) || bounded_rc=$? [ "$bounded_rc" -eq 3 ] || fail "timed-out uninstall must exit incomplete: $bounded_out" @@ -994,8 +1145,8 @@ assert_contains "$bounded_out" 'safe retry:' \ "timed-out uninstall must print exact safe retry evidence" assert_contains "$bounded_out" 'cluster cleanup if required:' \ "timed-out uninstall must print exact privileged cleanup evidence" -grep -F '/00-pvc.yaml' "$CALLS" | grep -F -- '--wait=true --timeout=180s' >/dev/null || \ - fail "uninstall deletion must have an explicit timeout" +grep -F '/persistentvolumeclaims/agent-os-firstmate-home' "$CALLS" | grep -F 'delete --raw' >/dev/null || \ + fail "uninstall deletion must target the captured PVC identity" pass "uninstall timeouts report retained owned resources and safe retry evidence" : > "$CALLS" @@ -1036,8 +1187,10 @@ pass "uninstall retry cannot lose cluster-RBAC residue state" : > "$CALLS" AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=namespace \ run_generic uninstall --yes --delete-namespace -grep -Fqx 'kubectl --context kind-agent-os delete namespace portable-agent-os --wait=true --timeout=180s' "$CALLS" || \ - fail "optional namespace deletion must target only the exactly owned namespace" +grep -F '/api/v1/namespaces/portable-agent-os' "$CALLS" | grep -F 'delete --raw' >/dev/null || \ + fail "optional namespace deletion must UID-delete only the exactly owned namespace" +assert_grep '"uid":"uid-namespace","resourceVersion":"rv-namespace"' "$STDIN_LOG" \ + "namespace deletion must bind to the final observed namespace identity" grep -Fq 'kubectl --context kind-agent-os api-resources --verbs=list --namespaced -o name' "$CALLS" || \ fail "optional namespace deletion must inventory every listable namespaced resource type" pass "optional namespace deletion proves ownership and no foreign resources" diff --git a/tests/agent-os-local.test.sh b/tests/agent-os-local.test.sh index 747d8caa1..0310175a0 100755 --- a/tests/agent-os-local.test.sh +++ b/tests/agent-os-local.test.sh @@ -37,26 +37,59 @@ cat > "$FAKEBIN/kubectl" <<'SH' printf 'kubectl' >> "$AGENT_OS_TEST_LOG" printf ' %s' "$@" >> "$AGENT_OS_TEST_LOG" printf '\n' >> "$AGENT_OS_TEST_LOG" +stdin_kind='' +if [ "${*: -2}" = '-f -' ]; then + stdin_data=$(cat) + stdin_kind=$(printf '%s\n' "$stdin_data" | awk '$1 == "kind:" { print $2; exit }') + printf 'stdin-kind %s\n' "$stdin_kind" >> "$AGENT_OS_TEST_LOG" +fi namespace=${AGENT_OS_NAMESPACE:-agent-os-demo} -if [[ " $* " = *" get statefulset agent-os-firstmate --ignore-not-found -o name "* ]] && \ - [ "${AGENT_OS_TEST_LOCAL_WORKLOAD:-absent}" = present ]; then +operation=$(grep 'akua-input-operation ' "$AGENT_OS_TEST_LOG" | tail -n 1 | awk '{print $2}') +namespace_present=0 +if [ "${AGENT_OS_TEST_LOCAL_WORKLOAD:-absent}" = present ] || grep -F ' create -f ' "$AGENT_OS_TEST_LOG" | grep -F 'namespace.yaml' >/dev/null; then namespace_present=1; fi +stateful_present=0 +if [ "${AGENT_OS_TEST_LOCAL_WORKLOAD:-absent}" = present ] || grep -F 'statefulset.yaml' "$AGENT_OS_TEST_LOG" | grep -E ' create | patch ' >/dev/null; then stateful_present=1; fi +if [[ " $* " = *" get statefulset agent-os-firstmate --ignore-not-found -o name "* ]] && [ "$stateful_present" -eq 1 ]; then printf 'statefulset.apps/agent-os-firstmate\n' fi -if [[ " $* " = *" get namespace "*" --ignore-not-found -o name "* ]] && \ - [ "${AGENT_OS_TEST_LOCAL_WORKLOAD:-absent}" = present ]; then +if { [[ " $* " = *" get namespace "*" --ignore-not-found -o name "* ]] || + [[ " $* " = *" get Namespace "*" --ignore-not-found -o name "* ]]; } && + [ "$namespace_present" -eq 1 ]; then printf 'namespace/%s\n' "$namespace" fi -if [[ " $* " = *" get namespace "*" -o jsonpath="* ]] && \ - [ "${AGENT_OS_TEST_LOCAL_WORKLOAD:-absent}" = present ]; then - printf 'agent-os\tagent-os-firstmate:%s' "$namespace" +if { [[ " $* " = *" get namespace "*" -o jsonpath="* ]] || + [[ " $* " = *" get Namespace "*" -o jsonpath="* ]]; } && + [ "$namespace_present" -eq 1 ]; then + if [[ " $* " = *'.metadata.resourceVersion'* ]]; then + printf '%s\tagent-os\tagent-os-firstmate:%s\tuid-namespace\trv-namespace\t%s' "$namespace" "$namespace" "$operation" + else + printf 'agent-os\tagent-os-firstmate:%s' "$namespace" + fi fi -if [[ " $* " = *" get statefulset agent-os-firstmate --ignore-not-found -o jsonpath="* ]] && \ - [ "${AGENT_OS_TEST_LOCAL_WORKLOAD:-absent}" = present ]; then - printf 'agent-os-firstmate\tcluster-admin\t\tagent-os\tagent-os-firstmate:%s' "$namespace" +if { [[ " $* " = *" get statefulset agent-os-firstmate --ignore-not-found -o jsonpath="* ]] || + [[ " $* " = *" get StatefulSet agent-os-firstmate --ignore-not-found -o jsonpath="* ]]; } && + [ "$stateful_present" -eq 1 ]; then + if [[ " $* " = *'.metadata.resourceVersion'* ]]; then + printf 'agent-os-firstmate\tagent-os\tagent-os-firstmate:%s\tuid-statefulset\trv-statefulset\t%s' "$namespace" "$operation" + elif [[ " $* " = *'.metadata.annotations.agent-os\.dev/rbac-mode'* ]]; then + printf 'agent-os-firstmate\tcluster-admin\t\tagent-os\tagent-os-firstmate:%s' "$namespace" + else + printf 'agent-os-firstmate\tagent-os\tagent-os-firstmate:%s' "$namespace" + fi +fi +if { [[ " $* " = *" get clusterrolebinding agent-os-firstmate-"*" --ignore-not-found -o jsonpath="* ]] || + [[ " $* " = *" get ClusterRoleBinding agent-os-firstmate-"*" --ignore-not-found -o jsonpath="* ]]; }; then + if [ "${AGENT_OS_TEST_LOCAL_WORKLOAD:-absent}" = present ] || grep -F 'clusterrolebinding.yaml' "$AGENT_OS_TEST_LOG" | grep -E ' create | patch ' >/dev/null; then + printf 'agent-os-firstmate-%s\tagent-os\tagent-os-firstmate:%s' "$namespace" "$namespace" + [[ " $* " != *'.metadata.resourceVersion'* ]] || printf '\tuid-clusterrolebinding\trv-clusterrolebinding\t%s' "$operation" + fi fi -if [[ " $* " = *" get clusterrolebinding agent-os-firstmate-"*" --ignore-not-found -o jsonpath="* ]] && \ - [ "${AGENT_OS_TEST_LOCAL_WORKLOAD:-absent}" = present ]; then - printf 'agent-os-firstmate-%s\tagent-os\tagent-os-firstmate:%s' "$namespace" "$namespace" +if [[ " $* " = *" get lease agent-os-firstmate-lifecycle --ignore-not-found -o jsonpath="* ]]; then + last_write=$(grep -Fn 'stdin-kind Lease' "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d: -f1) + last_delete=$(grep -Fn '/leases/agent-os-firstmate-lifecycle' "$AGENT_OS_TEST_LOG" | grep 'delete --raw' | tail -n 1 | cut -d: -f1) + if [ -n "$last_write" ] && { [ -z "$last_delete" ] || [ "$last_write" -gt "$last_delete" ]; }; then + printf 'agent-os-firstmate-lifecycle\tagent-os\tprimary\tagent-os-firstmate:%s\t%s\t2026-07-13T00:00:00Z\t2026-07-13T00:00:00Z\t300\tuid-lock\trv-lock' "$namespace" "$operation" + fi fi SH chmod +x "$FAKEBIN/kubectl" @@ -77,7 +110,9 @@ while [ "$#" -gt 0 ]; do done printf 'akua-input-image %s\n' "$(awk '/^image:/{print $2}' "$inputs")" >> "$AGENT_OS_TEST_LOG" namespace=$(awk '/^namespace:/{print $2}' "$inputs") +operation=$(awk '/^operationId:/{print $2}' "$inputs") printf 'akua-input-namespace %s\n' "$namespace" >> "$AGENT_OS_TEST_LOG" +printf 'akua-input-operation %s\n' "$operation" >> "$AGENT_OS_TEST_LOG" mkdir -p "$out" cat > "$out/statefulset.yaml" < "$out/rendered.yaml" cat > "$out/namespace.yaml" </dev/null; then fail "OrbStack must not maintain a second static installer" fi @@ -166,8 +207,8 @@ test_redeploy_upgrades_an_existing_local_installation() { assert_call 'kubectl --context orbstack -n agent-os-demo get statefulset agent-os-firstmate --ignore-not-found -o name' \ "redeploy must detect the existing local installation" - grep -Fq 'kubectl --context orbstack apply -f ' "$LOG" || \ - fail "redeploy must upgrade the existing local installation" + grep -Fq 'kubectl --context orbstack -n agent-os-demo patch ' "$LOG" || \ + fail "redeploy must atomically upgrade the existing local installation" pass "redeploy upgrades the content-addressed local installation" } @@ -218,11 +259,11 @@ test_destroy_requires_exact_confirmation() { [ "$rc" -eq 2 ] || fail "destroy without --yes must exit 2, got $rc: $out" [ ! -s "$LOG" ] || fail "destroy without --yes invoked an external command" - cleanup_out=$(run_cli destroy --yes 2>&1) || cleanup_rc=$? + cleanup_out=$(AGENT_OS_TEST_LOCAL_WORKLOAD=present run_cli destroy --yes 2>&1) || cleanup_rc=$? [ "$cleanup_rc" -eq 3 ] || \ fail "cluster-admin demo destroy must stop for separate privileged cleanup, got $cleanup_rc: $cleanup_out" - grep -Fq 'kubectl --context orbstack delete --ignore-not-found --wait=true --timeout=180s -f ' "$LOG" || \ - fail "confirmed destroy must delete only resources from the rendered OrbStack profile" + grep -Fq 'kubectl --context orbstack delete --raw /apis/apps/v1/namespaces/agent-os-demo/statefulsets/agent-os-firstmate -f -' "$LOG" || \ + fail "confirmed destroy must UID-delete the rendered OrbStack workload" if grep -E 'kubectl .* (get|delete) clusterrolebinding' "$LOG" >/dev/null; then fail "routine demo destroy must not inspect or delete cluster-scoped RBAC" fi diff --git a/tests/agent-os-packages.test.sh b/tests/agent-os-packages.test.sh index 22c8fb9e5..29abe879a 100755 --- a/tests/agent-os-packages.test.sh +++ b/tests/agent-os-packages.test.sh @@ -51,6 +51,8 @@ assert_grep 'verbs = ["get", "list", "watch", "create", "delete", "patch"]' "$FI "runtime RBAC must allow checkpoint updates on retained crewmate PVCs" assert_grep 'resources = ["leases"]' "$FIRSTMATE/package.k" \ "runtime RBAC must permit serialized crewmate lifecycle operations" +assert_grep 'verbs = ["get", "create", "update", "delete"]' "$FIRSTMATE/package.k" \ + "runtime Lease authority must be exact and permit CAS renewal" assert_no_grep 'akuaAuthSecret' "$FIRSTMATE/package.k" \ "the portable package must not require Akua authorization" assert_no_grep 'agent-os.akua.dev' "$FIRSTMATE/package.k" \ diff --git a/tools/agent-os/packages/firstmate/package.k b/tools/agent-os/packages/firstmate/package.k index 901c20992..30ceb7a2c 100644 --- a/tools/agent-os/packages/firstmate/package.k +++ b/tools/agent-os/packages/firstmate/package.k @@ -72,7 +72,7 @@ namespaceRbacResources = [ { apiGroups = ["coordination.k8s.io"] resources = ["leases"] - verbs = ["get", "list", "watch", "create", "delete"] + verbs = ["get", "create", "update", "delete"] }, ] }, From 79f594b01a7ae4905104a5a7c3f5e72e766e6f27 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 10:47:47 +0200 Subject: [PATCH 31/56] no-mistakes(review): Captain, harden Kubernetes lifecycle CAS and rollback safety --- bin/agent-os-crewmate.sh | 25 +++- bin/agent-os-kubernetes-lease.sh | 20 +++- bin/agent-os-kubernetes.sh | 154 +++++++++++++++++++----- tests/agent-os-kubernetes.test.sh | 188 +++++++++++++++++++++++++++--- 4 files changed, 337 insertions(+), 50 deletions(-) diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index 440021d7b..a5a435f2a 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -138,15 +138,17 @@ render_pod() { } render_lock() { - local acquired_at=$1 renewed_at=$2 uid=${3:-} rv=${4:-} + local acquired_at=$1 renewed_at=$2 uid=${3:-} rv=${4:-} uid_value='' rv_value='' + [ -z "$uid" ] || uid_value=$(yaml_string "$uid") + [ -z "$rv" ] || rv_value=$(yaml_string "$rv") cat </dev/null; then @@ -262,6 +264,21 @@ create_and_wait() { echo "error: crewmate Pod did not become ready with the authorized AI Secret" >&2 exit 1 fi + pvc_current=$(pvc_record) + pvc_identity=$(printf '%s' "$pvc_current" | cut -f1-4) + pvc_current_uid=$(printf '%s' "$pvc_current" | cut -f9) + if [ "$pvc_identity" != "$EXPECTED_PVC" ] || [ "$pvc_current_uid" != "$pvc_uid" ]; then + if ! cleanup_new_owned_pod "$pod_uid"; then + echo "partial state: operation Pod cleanup did not complete; PVCs retained" >&2 + fi + if [ "$pvc_identity" = "$EXPECTED_PVC" ]; then + if ! invalidate_checkpoint_evidence; then + echo "partial state: replacement PVC checkpoint invalidation did not complete; PVCs retained" >&2 + fi + fi + echo "error: mounted PVC identity changed after readiness; captured uid=$pvc_uid observed uid=${pvc_current_uid:-absent}; persistent claims retained" >&2 + exit 3 + fi } preflight_create() { diff --git a/bin/agent-os-kubernetes-lease.sh b/bin/agent-os-kubernetes-lease.sh index daa89c86d..5ef5a964a 100644 --- a/bin/agent-os-kubernetes-lease.sh +++ b/bin/agent-os-kubernetes-lease.sh @@ -1,5 +1,15 @@ #!/usr/bin/env bash +yaml_string() { + local value=$1 + value=${value//\\/\\\\} + value=${value//\"/\\\"} + value=${value//$'\n'/\\n} + value=${value//$'\r'/\\r} + value=${value//$'\t'/\\t} + printf '"%s"' "$value" +} + rfc3339_epoch() { local value=${1%%.*} value=${value%Z}Z @@ -66,7 +76,7 @@ stop_lock_renewal() { } release_lock() { - local record identity holder uid rv after + local record identity holder uid rv after after_holder after_uid stop_lock_renewal [ -n "$LOCK_UID" ] || return 0 if ! record=$(lock_record); then @@ -90,8 +100,12 @@ release_lock() { fi after=$(lock_record) || return 1 if [ -n "$after" ]; then - echo "error: lifecycle Lease '$LOCK' still exists after release" >&2 - return 1 + after_holder=$(printf '%s' "$after" | cut -f5) + after_uid=$(printf '%s' "$after" | cut -f9) + if [ "$after_uid" = "$uid" ] && [ "$after_holder" = "$OPERATION_ID" ]; then + echo "error: lifecycle Lease '$LOCK' still exists after release" >&2 + return 1 + fi fi LOCK_UID= LOCK_RV= diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh index bda4b909a..fc22ae8b5 100755 --- a/bin/agent-os-kubernetes.sh +++ b/bin/agent-os-kubernetes.sh @@ -167,15 +167,17 @@ lock_record() { } render_lock() { - local acquired_at=$1 renewed_at=$2 uid=${3:-} rv=${4:-} + local acquired_at=$1 renewed_at=$2 uid=${3:-} rv=${4:-} uid_value='' rv_value='' + [ -z "$uid" ] || uid_value=$(yaml_string "$uid") + [ -z "$rv" ] || rv_value=$(yaml_string "$rv") cat <&2; exit 2; } path=$(resource_api_path "$scope" "$kind" "$name") - if ! printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s","resourceVersion":"%s"}}\n' "$uid" "$rv" | \ - "$KUBECTL" --context "$CONTEXT" delete --raw "$path" -f - >/dev/null; then - bounded_delete_failure "$kind/$name" + started=$(date -u '+%s') + if ! output=$(printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s","resourceVersion":"%s"}}\n' "$uid" "$rv" | \ + "$KUBECTL" --context "$CONTEXT" delete --raw "$path" -f - 2>&1 >/dev/null); then + elapsed=$(($(date -u '+%s') - started)) + class=$(delete_failure_class "$output") + bounded_delete_failure "$kind/$name" request "$class" "$timeout" "$elapsed" "$uid" return $? fi if [ "$scope" = cluster ]; then - "$KUBECTL" --context "$CONTEXT" wait --for=delete "$kind/$name" --timeout="${timeout}s" || bounded_delete_failure "$kind/$name" + if ! output=$("$KUBECTL" --context "$CONTEXT" wait --for=delete "$kind/$name" --timeout="${timeout}s" 2>&1 >/dev/null); then + elapsed=$(($(date -u '+%s') - started)) + class=$(delete_failure_class "$output") + bounded_delete_failure "$kind/$name" wait "$class" "$timeout" "$elapsed" "$uid" + fi else - "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" wait --for=delete "$kind/$name" --timeout="${timeout}s" || bounded_delete_failure "$kind/$name" + if ! output=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" wait --for=delete "$kind/$name" --timeout="${timeout}s" 2>&1 >/dev/null); then + elapsed=$(($(date -u '+%s') - started)) + class=$(delete_failure_class "$output") + bounded_delete_failure "$kind/$name" wait "$class" "$timeout" "$elapsed" "$uid" + fi fi } @@ -367,6 +380,8 @@ preflight_rendered_resources() { *) require_namespaced_resource_owned_or_absent "$kind" "$name" ;; esac done < <(find "$OUT" -type f -name '*.yaml' -print) + require_namespaced_resource_owned_or_absent Role agent-os-firstmate-runtime + require_namespaced_resource_owned_or_absent RoleBinding agent-os-firstmate-runtime } delete_namespace_rbac() { @@ -470,6 +485,14 @@ partial_install_is_applicable() { expected="$name"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" [ -z "$identity" ] || [ "$identity" = "$expected" ] || return 1 done < <(find "$OUT" -type f -name '*.yaml' -print) + for kind in Role RoleBinding; do + if ! identity=$(live_resource_identity "$kind" agent-os-firstmate-runtime 2>/dev/null); then + return 1 + fi + expected="agent-os-firstmate-runtime"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" + [ -z "$identity" ] || [ "$identity" = "$expected" ] || return 1 + done + return 0 } partial_recovery_action() { @@ -551,9 +574,11 @@ report_partial_apply() { } cas_patch_file() { - local file=$1 uid=$2 rv=$3 patch_file + local file=$1 uid=$2 rv=$3 patch_file uid_value rv_value patch_file=$(mktemp "$OUT/cas.XXXXXX") - awk -v uid="$uid" -v rv="$rv" ' + uid_value=$(yaml_string "$uid") + rv_value=$(yaml_string "$rv") + awk -v uid="$uid_value" -v rv="$rv_value" ' !inserted && $1 == "metadata:" { print print " uid: " uid @@ -676,9 +701,9 @@ report_retained_observation() { local kind=$1 name=$2 scope=$3 record managed installation uid operation ready finalizers identity expected details if ! record=$(resource_observation "$scope" "$kind" "$name"); then echo "retained-unverified: $kind/$name could not be inspected; no further deletion attempted" >&2 - return + return 0 fi - [ -n "$record" ] || return + [ -n "$record" ] || return 0 managed=$(printf '%s' "$record" | cut -f2) installation=$(printf '%s' "$record" | cut -f3) uid=$(printf '%s' "$record" | cut -f4) @@ -703,8 +728,8 @@ report_retained_observation() { } report_retained_resources() { - local failed_target=$1 file kind name scope - echo "failed-target: $failed_target timeout=180s" >&2 + local failed_target=$1 phase=$2 class=$3 timeout=$4 elapsed=$5 uid=$6 file kind name scope + echo "failed-target: $failed_target uid=$uid delete-${phase}-failure=$class timeout=${timeout}s elapsed=${elapsed}s" >&2 kind=${failed_target%%/*} name=${failed_target#*/} scope=namespaced @@ -731,14 +756,37 @@ report_retained_resources() { echo "cluster cleanup if required: $(cleanup_command)" >&2 } +delete_failure_class() { + local output + output=$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]') + case "$output" in + *forbidden*) printf 'Forbidden' ;; + *conflict*) printf 'Conflict' ;; + *notfound*|*'not found'*) printf 'NotFound' ;; + *'timed out'*|*timeout*) printf 'timeout' ;; + *) printf 'transport' ;; + esac +} + bounded_delete_failure() { - local target=$1 + local target=$1 phase=${2:-wait} class=${3:-timeout} timeout=${4:-180} + local elapsed=${5:-$timeout} uid=${6:-unknown} prefix kind name scope + [ "$COMMAND" = uninstall ] && prefix=incomplete || prefix=error + echo "$prefix: delete-${phase}-failure=$class target=$target uid=$uid timeout=${timeout}s elapsed=${elapsed}s" >&2 if [ "$COMMAND" = uninstall ]; then - echo "incomplete: timed out deleting $target after 180s" >&2 - report_retained_resources "$target" + report_retained_resources "$target" "$phase" "$class" "$timeout" "$elapsed" "$uid" return 3 fi - echo "error: timed out deleting $target after 180s" >&2 + kind=${target%%/*} + name=${target#*/} + scope=namespaced + case "$kind" in + ClusterRoleBinding|Namespace) scope=cluster ;; + esac + report_retained_observation "$kind" "$name" "$scope" + if [ "$COMMAND" = cleanup-cluster-rbac ]; then + echo "safe retry: $(cleanup_command)" >&2 + fi return 1 } @@ -755,11 +803,14 @@ delete_rendered_kind() { } delete_rendered_namespaced_resources() { - local kind + local kind output started elapsed class delete_rendered_kind StatefulSet - if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" wait \ - --for=delete pod/agent-os-firstmate-0 --timeout=180s; then - bounded_delete_failure Pod/agent-os-firstmate-0 + started=$(date -u '+%s') + if ! output=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" wait \ + --for=delete pod/agent-os-firstmate-0 --timeout=180s 2>&1 >/dev/null); then + elapsed=$(($(date -u '+%s') - started)) + class=$(delete_failure_class "$output") + bounded_delete_failure Pod/agent-os-firstmate-0 wait "$class" 180 "$elapsed" unknown fi for kind in Service RoleBinding Role ServiceAccount; do delete_rendered_kind "$kind" @@ -922,19 +973,67 @@ case "$COMMAND" in preflight_rendered_resources 1 previous=$(workload_state) require_workload_owned "$previous" + if ! command -v jq >/dev/null 2>&1; then + echo "error: jq is required to resolve StatefulSet revision history safely" >&2 + exit 2 + fi rollback_record=$(live_resource_record namespaced StatefulSet agent-os-firstmate) + rollback_uid=$(printf '%s' "$rollback_record" | cut -f4) + rollback_rv=$(printf '%s' "$rollback_record" | cut -f5) if [ "$(printf '%s' "$rollback_record" | cut -f1-3)" != \ "agent-os-firstmate"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" ] || \ - [ -z "$(printf '%s' "$rollback_record" | cut -f4)" ] || [ -z "$(printf '%s' "$rollback_record" | cut -f5)" ]; then + [ -z "$rollback_uid" ] || [ -z "$rollback_rv" ]; then echo "error: rollback requires exact StatefulSet UID and resourceVersion evidence" >&2 exit 2 fi + rollback_state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get statefulset agent-os-firstmate -o json) + rollback_revisions=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get controllerrevisions.apps -o json) + rollback_references=$(printf '%s' "$rollback_state" | jq -er \ + --arg uid "$rollback_uid" --arg rv "$rollback_rv" --arg installation "$INSTALLATION_ID" ' + select(.metadata.name == "agent-os-firstmate") + | select(.metadata.uid == $uid and .metadata.resourceVersion == $rv) + | select(.metadata.labels["app.kubernetes.io/managed-by"] == "agent-os") + | select(.metadata.annotations["agent-os.dev/installation-id"] == $installation) + | [.status.currentRevision, .status.updateRevision] + | select(all(.[]; type == "string" and length > 0)) + | @tsv + ') || { echo "error: StatefulSet changed before rollback revision resolution" >&2; exit 3; } + rollback_current_revision=$(printf '%s' "$rollback_references" | cut -f1) + rollback_update_revision=$(printf '%s' "$rollback_references" | cut -f2) + rollback_patch=$(printf '%s' "$rollback_revisions" | jq -ce \ + --arg current "$rollback_current_revision" --arg update "$rollback_update_revision" \ + --arg uid "$rollback_uid" --arg rv "$rollback_rv" ' + def owned: + any(.metadata.ownerReferences[]?; + .apiVersion == "apps/v1" and .kind == "StatefulSet" and + .name == "agent-os-firstmate" and .uid == $uid and .controller == true); + [.items[] | select(owned)] as $owned + | ($owned | map(select(.metadata.name == $current))) as $current_items + | ($owned | map(select(.metadata.name == $update))) as $update_items + | select($current_items | length == 1) + | select($update_items | length == 1) + | $current_items[0] as $current_revision + | $update_items[0] as $update_revision + | select($update_revision.revision | type == "number") + | (if $current != $update then + $current_revision + else + ($owned | map(select((.revision | type) == "number" and .revision < $update_revision.revision)) | sort_by(.revision) | last) + end) as $target + | select($target != null and ($target.data | type) == "object") + | $target.data * {metadata:{uid:$uid,resourceVersion:$rv}} + ') || { echo "error: no exact-owned previous ControllerRevision is available for rollback" >&2; exit 2; } + if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch StatefulSet agent-os-firstmate \ + --type=strategic -p "$rollback_patch" >/dev/null; then + echo "error: StatefulSet rollback CAS conflicted; no rollout-undo fallback attempted" >&2 + exit 3 + fi rollback_current=$(live_resource_record namespaced StatefulSet agent-os-firstmate) - if [ "$rollback_current" != "$rollback_record" ]; then - echo "error: StatefulSet changed before rollback mutation" >&2 + if [ "$(printf '%s' "$rollback_current" | cut -f1-4)" != \ + "$(printf '%s' "$rollback_record" | cut -f1-4)" ]; then + echo "error: StatefulSet UID or ownership changed during rollback" >&2 exit 3 fi - "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout undo statefulset/agent-os-firstmate "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s ;; status) @@ -974,6 +1073,7 @@ case "$COMMAND" in fi render acquire_primary_lock + preflight_rendered_resources 1 cleanup_cluster_rbac ;; *) usage ;; diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index 5aa696c25..906f35386 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -43,6 +43,13 @@ printf 'kubectl' >> "$AGENT_OS_TEST_LOG" printf ' %s' "$@" >> "$AGENT_OS_TEST_LOG" printf '\n' >> "$AGENT_OS_TEST_LOG" stdin_data='' +previous='' +for argument in "$@"; do + if [ "$previous" = --patch-file ]; then + printf '%s\n' "$(cat "$argument")" >> "${AGENT_OS_STDIN_LOG:-$AGENT_OS_TEST_LOG.stdin}" + fi + previous=$argument +done if [ "${*: -2}" = "-f -" ]; then stdin_data=$(cat) printf '%s\n' "$stdin_data" >> "${AGENT_OS_STDIN_LOG:-$AGENT_OS_TEST_LOG.stdin}" @@ -53,7 +60,8 @@ if [ "${AGENT_OS_TEST_FAIL_APPLY:-0}" = 1 ] && [[ " $* " = *" create -f - "* ]] [ "$stdin_kind" = Pod ]; then exit 1 fi -if [ "${AGENT_OS_TEST_LOCK_STATE:-free}" != free ] && [[ " $* " = *" create -f - "* ]] && \ +if { [ "${AGENT_OS_TEST_LOCK_STATE:-free}" != free ] || [ "${AGENT_OS_TEST_PRIMARY_LOCK_STATE:-free}" != free ]; } && \ + [[ " $* " = *" create -f - "* ]] && \ [ "$stdin_kind" = Lease ]; then exit 1 fi @@ -71,6 +79,10 @@ fi if [ "${AGENT_OS_TEST_FAIL_WAIT:-0}" = 1 ] && [ "${1:-}" = -n ] && [ "${3:-}" = wait ]; then exit 1 fi +if [ -n "${AGENT_OS_TEST_FAIL_WAIT_TARGET:-}" ] && [[ " $* " = *" wait --for=delete $AGENT_OS_TEST_FAIL_WAIT_TARGET "* ]]; then + printf 'error: timed out waiting for the condition on %s\n' "$AGENT_OS_TEST_FAIL_WAIT_TARGET" >&2 + exit 1 +fi if [ "${AGENT_OS_TEST_FAIL_ROLLOUT:-0}" = 1 ] && [[ " $* " = *" rollout status statefulset/agent-os-firstmate "* ]]; then exit 1 fi @@ -87,11 +99,15 @@ if [ -n "${AGENT_OS_TEST_FAIL_DELETE_FILE:-}" ] && [[ " $* " = *" delete "* ]] & exit 1 fi if [ -n "${AGENT_OS_TEST_FAIL_DELETE_TARGET:-}" ] && [[ " $* " = *"$AGENT_OS_TEST_FAIL_DELETE_TARGET"* ]]; then + printf '%s\n' "${AGENT_OS_TEST_DELETE_ERROR:-transport error}" >&2 exit 1 fi if [ -n "${AGENT_OS_TEST_READY_DELAY:-}" ] && [[ " $* " = *" wait --for=condition=Ready pod/"* ]]; then sleep "$AGENT_OS_TEST_READY_DELAY" fi +if [ -n "${AGENT_OS_TEST_ROLLOUT_DELAY:-}" ] && [[ " $* " = *" rollout status statefulset/agent-os-firstmate "* ]]; then + sleep "$AGENT_OS_TEST_ROLLOUT_DELAY" +fi case " $* " in *" get pod agent-os-crewmate-"*" --ignore-not-found -o jsonpath="*) id=${AGENT_OS_TEST_CREWMATE_ID:-scout-1} @@ -144,6 +160,13 @@ case " $* " in printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tpending\t\t\t\tuid-pvc-owned\trv-pvc-owned' "$id" "$id" fi ;; + replaced-after-ready) + if grep -F 'wait --for=condition=Ready pod/' "$AGENT_OS_TEST_LOG" >/dev/null; then + printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tpending\t\t\t\tuid-pvc-after-ready\trv-pvc-after-ready' "$id" "$id" + else + printf 'agent-os-crewmate-%s-home\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tpending\t\t\t\tuid-pvc-owned\trv-pvc-owned' "$id" "$id" + fi + ;; esac ;; *" get lease agent-os-crewmate-"*" --ignore-not-found -o jsonpath="*) @@ -158,13 +181,15 @@ case " $* " in elif grep -F ' replace -f -' "$AGENT_OS_TEST_LOG" >/dev/null; then printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\toperation-test\t2026-07-13T00:00:00Z\t2026-07-13T00:00:00Z\t300\tuid-lock-expired\trv-lock-taken' "$id" "$id" else - printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tother-operation\t2000-01-01T00:00:00Z\t2000-01-01T00:00:00Z\t300\tuid-lock-expired\trv-lock-expired' "$id" "$id" + printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tother-operation\t2000-01-01T00:00:00Z\t2000-01-01T00:00:00Z\t300\tuid-lock-expired\t%s' "$id" "$id" "${AGENT_OS_TEST_LOCK_RV:-rv-lock-expired}" fi ;; *) last_lock_write=$(grep -Fn 'stdin-kind Lease' "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d: -f1) last_lock_delete=$(grep -Fn "/leases/agent-os-crewmate-$id-lifecycle" "$AGENT_OS_TEST_LOG" | grep 'delete --raw' | tail -n 1 | cut -d: -f1) - if [ -n "$last_lock_delete" ] && [ "${AGENT_OS_TEST_LOCK_RELEASE_STATE:-}" = foreign ]; then + if [ -n "$last_lock_delete" ] && [ "${AGENT_OS_TEST_LOCK_RELEASE_STATE:-}" = next-owner ]; then + printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tnext-operation\t2026-07-13T00:00:00Z\t2026-07-13T00:00:00Z\t300\tuid-lock-next\trv-lock-next' "$id" "$id" + elif [ -n "$last_lock_delete" ] && [ "${AGENT_OS_TEST_LOCK_RELEASE_STATE:-}" = foreign ]; then printf 'agent-os-crewmate-%s-lifecycle\tother\tother\tother-installation\tother-operation\t2099-01-01T00:00:00Z\t2099-01-01T00:00:00Z\t300\tuid-lock-foreign\trv-lock-foreign' "$id" elif [ -n "$last_lock_write" ] && { [ -z "$last_lock_delete" ] || [ "$last_lock_write" -gt "$last_lock_delete" ]; }; then printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\toperation-test\t2026-07-13T00:00:00Z\t2026-07-13T00:00:00Z\t300\tuid-lock\trv-lock' "$id" "$id" @@ -176,10 +201,12 @@ case " $* " in lock_name=$(printf '%s\n' "$*" | sed -n 's/.* get lease \([^ ]*\) .*/\1/p') last_lock_write=$(grep -Fn 'stdin-kind Lease' "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d: -f1) last_lock_delete=$(grep -Fn "/leases/$lock_name" "$AGENT_OS_TEST_LOG" | grep 'delete --raw' | tail -n 1 | cut -d: -f1) - if [ -n "$last_lock_write" ] && { [ -z "$last_lock_delete" ] || [ "$last_lock_write" -gt "$last_lock_delete" ]; }; then + if [ "${AGENT_OS_TEST_PRIMARY_LOCK_STATE:-free}" = expired ] && ! grep -F ' replace -f -' "$AGENT_OS_TEST_LOG" >/dev/null; then + printf '%s\tagent-os\tprimary\tagent-os-firstmate:%s\tother-operation\t2000-01-01T00:00:00Z\t2000-01-01T00:00:00Z\t300\tuid-primary-lock\t%s' "$lock_name" "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" "${AGENT_OS_TEST_PRIMARY_LOCK_RV:-rv-primary-lock}" + elif [ -n "$last_lock_write" ] && { [ -z "$last_lock_delete" ] || [ "$last_lock_write" -gt "$last_lock_delete" ]; }; then lock_stdin=${AGENT_OS_STDIN_LOG:-$AGENT_OS_TEST_LOG.stdin} lock_holder=$(awk '/holderIdentity:/ { holder=$2 } END { print holder }' "$lock_stdin") - printf '%s\tagent-os\tprimary\tagent-os-firstmate:%s\t%s\t2026-07-13T00:00:00Z\t2026-07-13T00:00:00Z\t300\tuid-primary-lock\trv-primary-lock' "$lock_name" "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" "$lock_holder" + printf '%s\tagent-os\tprimary\tagent-os-firstmate:%s\t%s\t2026-07-13T00:00:00Z\t2026-07-13T00:00:00Z\t300\tuid-primary-lock\t%s' "$lock_name" "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" "$lock_holder" "${AGENT_OS_TEST_PRIMARY_LOCK_RV:-rv-primary-lock}" fi ;; *" get namespace "*" --ignore-not-found -o name "*) @@ -250,6 +277,9 @@ case " $* " in *" Role "*) kind=Role; name=agent-os-firstmate-runtime ;; esac resource_state=${AGENT_OS_TEST_RESOURCE_STATE:-absent} + if { [ "$kind" = Role ] || [ "$kind" = RoleBinding ]; } && [ -n "${AGENT_OS_TEST_STALE_RBAC_STATE:-}" ]; then + resource_state=$AGENT_OS_TEST_STALE_RBAC_STATE + fi if grep -E ' create -f .+\.yaml' "$AGENT_OS_TEST_LOG" | grep -v 'namespace.yaml' >/dev/null || \ grep -F ' --patch-file ' "$AGENT_OS_TEST_LOG" >/dev/null || \ grep -F ' apply -f ' "$AGENT_OS_TEST_LOG" >/dev/null; then @@ -261,7 +291,7 @@ case " $* " in printf '%s\tagent-os\tagent-os-firstmate:portable-agent-os' "$name" if [[ " $* " = *'.metadata.resourceVersion'* ]]; then current_operation=$(grep 'akua-input-operation ' "$AGENT_OS_TEST_LOG" | tail -n 1 | awk '{print $2}') - printf '\tuid-%s\trv-%s\t%s' "$(printf '%s' "$kind" | tr '[:upper:]' '[:lower:]')" "$(printf '%s' "$kind" | tr '[:upper:]' '[:lower:]')" "$current_operation" + printf '\tuid-%s\t%s\t%s' "$(printf '%s' "$kind" | tr '[:upper:]' '[:lower:]')" "${AGENT_OS_TEST_RESOURCE_RV:-rv-$(printf '%s' "$kind" | tr '[:upper:]' '[:lower:]')}" "$current_operation" elif [[ " $* " = *'.spec.replicas'* ]]; then printf '\tuid-statefulset\toperation-test\t\t[kubernetes.io/pvc-protection]\t1\t1\t0\t0\t0\trev-old\trev-new\t2\t1' elif [[ " $* " = *'.metadata.uid'* ]]; then @@ -271,6 +301,13 @@ case " $* " in foreign) printf '%s\tother\tother-installation' "$name" ;; esac ;; + *" get statefulset agent-os-firstmate -o json "*) + printf '{"metadata":{"name":"agent-os-firstmate","uid":"uid-statefulset","resourceVersion":"%s","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"status":{"currentRevision":"agent-os-firstmate-previous","updateRevision":"agent-os-firstmate-current"}}\n' \ + "${AGENT_OS_TEST_RESOURCE_RV:-rv-statefulset}" + ;; + *" get controllerrevisions.apps -o json "*) + printf '%s\n' '{"items":[{"metadata":{"name":"agent-os-firstmate-previous","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":1,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"previous"}}}}}},{"metadata":{"name":"agent-os-firstmate-current","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":2,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"current"}}}}}}]}' + ;; *" get role agent-os-firstmate-runtime -o json "*) if [ "${AGENT_OS_TEST_RBAC_STATE:-exact}" = bad-rules ]; then printf '%s\n' '{"metadata":{"name":"agent-os-firstmate-runtime"},"rules":[]}' @@ -482,6 +519,21 @@ grep -F '/pods/agent-os-crewmate-scout-1' "$CALLS" | grep -F 'delete --raw' >/de fail "PVC replacement must UID-delete only the newly created owned Pod" pass "crewmate creation retains replacement claims and removes its Pod" +: > "$CALLS" +after_ready_out='' +after_ready_rc=0 +after_ready_out=$(AGENT_OS_TEST_PVC_STATE=replaced-after-ready run_launcher create scout-1 2>&1) || after_ready_rc=$? +[ "$after_ready_rc" -eq 3 ] || fail "PVC replacement after Ready must exit incomplete: $after_ready_out" +assert_contains "$after_ready_out" 'uid-pvc-owned' \ + "post-Ready PVC mismatch must report the captured claim UID" +assert_contains "$after_ready_out" 'uid-pvc-after-ready' \ + "post-Ready PVC mismatch must report the observed replacement UID" +grep -F '/pods/agent-os-crewmate-scout-1' "$CALLS" | grep -F 'delete --raw' >/dev/null || \ + fail "post-Ready PVC mismatch must UID-delete only the operation Pod" +assert_grep '"agent-os.dev/checkpoint-state":"pending"' "$CALLS" \ + "post-Ready PVC mismatch must invalidate checkpoint evidence" +pass "crewmate revalidates mounted PVC identity after readiness" + : > "$CALLS" replacement_out='' replacement_rc=0 @@ -628,6 +680,14 @@ grep -F 'replace -f -' "$CALLS" >/dev/null || \ fail "expired exact-owned lifecycle Leases must use resourceVersion-CAS takeover" pass "crewmate lifecycle can recover exact-owned expired Leases" +: > "$CALLS" +: > "$STDIN_LOG" +AGENT_OS_TEST_LOCK_STATE=expired AGENT_OS_TEST_LOCK_RV=12345 AGENT_OS_TEST_PVC_STATE=owned \ + run_launcher stop scout-1 +assert_grep 'resourceVersion: "12345"' "$STDIN_LOG" \ + "Lease takeover must serialize decimal resourceVersion as a string" +pass "crewmate Lease CAS preserves opaque resourceVersion typing" + : > "$CALLS" AGENT_OS_LOCK_DURATION_SECONDS=3 AGENT_OS_TEST_READY_DELAY=2 \ AGENT_OS_TEST_PVC_STATE=owned run_launcher create scout-1 @@ -636,11 +696,14 @@ grep -F 'replace -f -' "$CALLS" >/dev/null || \ pass "crewmate lifecycle renews its Lease while active" : > "$CALLS" -release_conflict_rc=0 AGENT_OS_TEST_POD_STATE=absent AGENT_OS_TEST_PVC_STATE=owned \ - AGENT_OS_TEST_LOCK_RELEASE_STATE=foreign run_launcher stop scout-1 >/dev/null 2>&1 || release_conflict_rc=$? -[ "$release_conflict_rc" -eq 3 ] || fail "lifecycle completion must fail if Lease release ownership cannot verify" -pass "crewmate lifecycle fails closed on Lease release conflicts" + AGENT_OS_TEST_LOCK_RELEASE_STATE=foreign run_launcher stop scout-1 +pass "crewmate Lease release accepts an independently replaced Lease" + +: > "$CALLS" +AGENT_OS_TEST_POD_STATE=absent AGENT_OS_TEST_PVC_STATE=owned \ + AGENT_OS_TEST_LOCK_RELEASE_STATE=next-owner run_launcher stop scout-1 +pass "crewmate Lease release accepts a subsequent legitimate holder" if run_launcher create 'Bad_ID' >/dev/null 2>&1; then fail "invalid Kubernetes crewmate IDs must be rejected" @@ -1019,6 +1082,36 @@ grep -F 'wait --for=delete ClusterRoleBinding/agent-os-firstmate-portable-agent- fail "privileged cleanup must produce deletion evidence for the exact binding" pass "privileged cleanup verifies ownership and deletes one exact binding" +: > "$CALLS" +cleanup_stale_out='' +cleanup_stale_rc=0 +cleanup_stale_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=pending \ + AGENT_OS_TEST_CLUSTER_RBAC_STATE=owned AGENT_OS_TEST_STALE_RBAC_STATE=foreign \ + run_generic cleanup-cluster-rbac --yes 2>&1) || cleanup_stale_rc=$? +[ "$cleanup_stale_rc" -eq 2 ] || fail "privileged cleanup must preflight foreign stale RBAC: $cleanup_stale_out" +assert_no_grep '/clusterrolebindings/agent-os-firstmate-portable-agent-os' "$CALLS" \ + "privileged cleanup must reject foreign namespaced RBAC before cluster mutation" +pass "privileged cleanup preflights deterministic namespaced RBAC" + +: > "$CALLS" +cleanup_delete_out='' +cleanup_delete_rc=0 +cleanup_delete_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=pending \ + AGENT_OS_TEST_CLUSTER_RBAC_STATE=owned \ + AGENT_OS_TEST_FAIL_DELETE_TARGET=clusterrolebindings/agent-os-firstmate-portable-agent-os \ + AGENT_OS_TEST_DELETE_ERROR='Error from server (Conflict): object changed' \ + run_generic cleanup-cluster-rbac --yes 2>&1) || cleanup_delete_rc=$? +[ "$cleanup_delete_rc" -eq 1 ] || fail "privileged cleanup delete conflict must exit failed: $cleanup_delete_out" +assert_contains "$cleanup_delete_out" 'delete-request-failure=Conflict' \ + "privileged cleanup must preserve immediate delete failure class" +assert_contains "$cleanup_delete_out" 'timeout=60s' \ + "privileged cleanup must report its actual deletion bound" +assert_contains "$cleanup_delete_out" 'retained: ClusterRoleBinding/agent-os-firstmate-portable-agent-os uid=uid-clusterrolebinding' \ + "privileged cleanup failure must inventory the retained exact grant" +assert_contains "$cleanup_delete_out" 'safe retry:' \ + "privileged cleanup failure must print exact retry evidence" +pass "privileged cleanup failures report retained grant evidence" + : > "$CALLS" active_cleanup_rc=0 AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=cluster-admin \ @@ -1072,16 +1165,52 @@ if grep -F 'delete clusterrolebinding agent-os-firstmate-portable-agent-os' "$CA fi pass "cluster-admin upgrade reconciles namespaced authority after apply" +: > "$CALLS" +: > "$STDIN_LOG" +AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_RESOURCE_RV=67890 AGENT_OS_TEST_PRIMARY_LOCK_RV=12345 \ + AGENT_OS_TEST_PRIMARY_LOCK_STATE=expired \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace run_generic upgrade +assert_grep 'resourceVersion: "67890"' "$STDIN_LOG" \ + "primary resource mutation must serialize decimal resourceVersion as a string" +assert_grep 'resourceVersion: "12345"' "$STDIN_LOG" \ + "primary Lease CAS must serialize decimal resourceVersion as a string" +pass "primary CAS preserves opaque resourceVersion typing" + +: > "$CALLS" +stale_rbac_out='' +stale_rbac_rc=0 +stale_rbac_out=$(PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS="$CLUSTER_ADMIN_INPUTS" \ + AGENT_OS_TEST_NAMESPACE=portable-agent-os AGENT_OS_TEST_NAMESPACE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_STALE_RBAC_STATE=foreign AGENT_OS_CONTEXT=kind-agent-os \ + AGENT_OS_OPERATION_ID=operation-test AGENT_OS_NAMESPACE=portable-agent-os \ + "$GENERIC" upgrade 2>&1) || stale_rbac_rc=$? +[ "$stale_rbac_rc" -eq 2 ] || fail "foreign stale namespaced RBAC must fail preflight: $stale_rbac_out" +assert_contains "$stale_rbac_out" "Role 'agent-os-firstmate-runtime' does not have the exact Agent OS installation identity" \ + "every RBAC mode must preflight deterministic stale names" +assert_no_grep 'patch StatefulSet agent-os-firstmate' "$CALLS" \ + "foreign stale RBAC must fail before desired resources mutate" +pass "all RBAC modes preflight deterministic stale resources" + : > "$CALLS" AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ AGENT_OS_TEST_WORKLOAD_STATE=namespace run_generic rollback -grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os rollout undo statefulset/agent-os-firstmate' "$CALLS" || \ - fail "generic rollback must target only the Firstmate StatefulSet" +grep -F 'kubectl --context kind-agent-os -n portable-agent-os patch StatefulSet agent-os-firstmate --type=strategic' "$CALLS" >/dev/null || \ + fail "generic rollback must update only the captured Firstmate StatefulSet" +assert_grep '"uid":"uid-statefulset"' "$CALLS" \ + "rollback mutation must carry StatefulSet UID CAS" +assert_grep '"resourceVersion":"rv-statefulset"' "$CALLS" \ + "rollback mutation must carry StatefulSet resourceVersion CAS" +assert_grep '"rollback":"previous"' "$CALLS" \ + "rollback must return a failed updateRevision to the exact-owned currentRevision" +assert_no_grep 'rollout undo' "$CALLS" \ + "rollback must not use a non-CAS rollout undo mutation" grep -Fq 'akua render --no-agent-mode' "$CALLS" || \ fail "rollback must derive its namespace and identity from the current package render" -[ "$(grep -F 'get StatefulSet agent-os-firstmate --ignore-not-found' "$CALLS" | grep -Fc 'metadata.resourceVersion')" -ge 2 ] || \ - fail "rollback must revalidate StatefulSet UID and resourceVersion immediately before mutation" -pass "generic rollback verifies its rendered installation identity" +assert_grep 'get controllerrevisions.apps -o json' "$CALLS" \ + "rollback must resolve the exact-owned revision history" +pass "generic rollback applies a revision-derived StatefulSet CAS update" : > "$CALLS" foreign_rollback_rc=0 @@ -1126,9 +1255,18 @@ bounded_out=$(PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS=" AGENT_OS_TEST_NAMESPACE=portable-agent-os AGENT_OS_TEST_NAMESPACE_STATE=owned \ AGENT_OS_TEST_WORKLOAD_STATE=none AGENT_OS_TEST_RESOURCE_STATE=owned \ AGENT_OS_TEST_PRIMARY_POD_STATE=owned AGENT_OS_TEST_FAIL_DELETE_TARGET=persistentvolumeclaims/agent-os-firstmate-home \ + AGENT_OS_TEST_DELETE_ERROR='Error from server (Forbidden): persistentvolumeclaims is forbidden' \ AGENT_OS_OPERATION_ID=operation-test AGENT_OS_CONTEXT=kind-agent-os \ AGENT_OS_NAMESPACE=portable-agent-os "$GENERIC" uninstall --yes 2>&1) || bounded_rc=$? [ "$bounded_rc" -eq 3 ] || fail "timed-out uninstall must exit incomplete: $bounded_out" +assert_contains "$bounded_out" 'delete-request-failure=Forbidden' \ + "immediate deletion authorization failures must not be mislabeled as timeouts" +assert_contains "$bounded_out" 'timeout=180s' \ + "deletion evidence must report the configured operation timeout" +assert_contains "$bounded_out" 'uid=uid-persistentvolumeclaim' \ + "deletion evidence must report the captured target UID" +assert_not_contains "$bounded_out" 'timed out deleting PersistentVolumeClaim/agent-os-firstmate-home' \ + "immediate deletion failures must remain distinct from timeout evidence" assert_contains "$bounded_out" 'failed-target: PersistentVolumeClaim/agent-os-firstmate-home' \ "timed-out uninstall must report the actual failed target" assert_contains "$bounded_out" 'retained: PersistentVolumeClaim/agent-os-firstmate-home uid=uid-persistentvolumeclaim' \ @@ -1147,7 +1285,25 @@ assert_contains "$bounded_out" 'cluster cleanup if required:' \ "timed-out uninstall must print exact privileged cleanup evidence" grep -F '/persistentvolumeclaims/agent-os-firstmate-home' "$CALLS" | grep -F 'delete --raw' >/dev/null || \ fail "uninstall deletion must target the captured PVC identity" -pass "uninstall timeouts report retained owned resources and safe retry evidence" +pass "uninstall delete-request failures preserve class and retained evidence" + +: > "$CALLS" +timeout_out='' +timeout_rc=0 +timeout_out=$(PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_INPUTS="$NONE_INPUTS" \ + AGENT_OS_TEST_NAMESPACE=portable-agent-os AGENT_OS_TEST_NAMESPACE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=none AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_FAIL_WAIT_TARGET=StatefulSet/agent-os-firstmate \ + AGENT_OS_OPERATION_ID=operation-test AGENT_OS_CONTEXT=kind-agent-os \ + AGENT_OS_NAMESPACE=portable-agent-os "$GENERIC" uninstall --yes 2>&1) || timeout_rc=$? +[ "$timeout_rc" -eq 3 ] || fail "true deletion timeout must exit incomplete rc=$timeout_rc: $timeout_out" +assert_contains "$timeout_out" 'delete-wait-failure=timeout' \ + "wait timeout evidence must preserve its failure class" +assert_contains "$timeout_out" 'timeout=180s' \ + "wait timeout evidence must report the actual configured timeout" +assert_contains "$timeout_out" 'uid=uid-statefulset' \ + "wait timeout evidence must report the captured target UID" +pass "uninstall wait timeouts report actual bounds and retained state" : > "$CALLS" foreign_uninstall_rc=0 From e70e921bf7ded3302f11ed3984cc2651ef990a3c Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 11:29:49 +0200 Subject: [PATCH 32/56] no-mistakes(review): Captain, harden Kubernetes lifecycle deadlines and rollback recovery --- bin/agent-os-crewmate.sh | 40 ++++++-- bin/agent-os-kubernetes-lease.sh | 93 ++++++++++++++--- bin/agent-os-kubernetes.sh | 161 ++++++++++++++++++++++++------ tests/agent-os-kubernetes.test.sh | 156 ++++++++++++++++++++++++++--- 4 files changed, 381 insertions(+), 69 deletions(-) diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index a5a435f2a..7ba69c427 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -49,11 +49,16 @@ LOCK_RENEW_PID= LOCK_DURATION_SECONDS=${AGENT_OS_LOCK_DURATION_SECONDS:-300} LOCK_CLOCK_SKEW_SECONDS=${AGENT_OS_LOCK_CLOCK_SKEW_SECONDS:-5} LOCK_ACQUIRE_SECONDS=${AGENT_OS_LOCK_ACQUIRE_SECONDS:-30} +LOCK_REQUEST_CEILING_SECONDS=${AGENT_OS_LOCK_REQUEST_CEILING_SECONDS:-5} +RESOURCE_REQUEST_CEILING_SECONDS=${AGENT_OS_RESOURCE_REQUEST_CEILING_SECONDS:-5} -for seconds in "$LOCK_DURATION_SECONDS" "$LOCK_CLOCK_SKEW_SECONDS" "$LOCK_ACQUIRE_SECONDS"; do +for seconds in "$LOCK_DURATION_SECONDS" "$LOCK_CLOCK_SKEW_SECONDS" "$LOCK_ACQUIRE_SECONDS" "$LOCK_REQUEST_CEILING_SECONDS" "$RESOURCE_REQUEST_CEILING_SECONDS"; do case "$seconds" in ''|*[!0-9]*) echo "error: lifecycle Lease timing must use whole seconds" >&2; exit 2 ;; esac done [ "$LOCK_DURATION_SECONDS" -ge 3 ] || { echo "error: lifecycle Lease duration must be at least 3 seconds" >&2; exit 2; } +[ "$LOCK_ACQUIRE_SECONDS" -ge 1 ] || { echo "error: lifecycle Lease acquisition must allow at least 1 second" >&2; exit 2; } +[ "$LOCK_REQUEST_CEILING_SECONDS" -ge 1 ] || { echo "error: lifecycle Lease request ceiling must be at least 1 second" >&2; exit 2; } +[ "$RESOURCE_REQUEST_CEILING_SECONDS" -ge 1 ] || { echo "error: resource request ceiling must be at least 1 second" >&2; exit 2; } kube() { "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" "$@" @@ -67,7 +72,7 @@ resource_identity() { pod_record() { kube get pod "$POD" --ignore-not-found \ - -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}' + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}{"\t"}{range .spec.volumes[?(@.name=="home")]}{.persistentVolumeClaim.claimName}{end}' } pvc_record() { @@ -76,7 +81,9 @@ pvc_record() { } lock_record() { - kube get lease "$LOCK" --ignore-not-found \ + local deadline=${1:-} + [ -n "$deadline" ] || deadline=$(lock_default_deadline) + lock_kube "$deadline" get lease "$LOCK" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.spec.holderIdentity}{"\t"}{.spec.acquireTime}{"\t"}{.spec.renewTime}{"\t"}{.spec.leaseDurationSeconds}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}' } @@ -198,7 +205,7 @@ cleanup_new_owned_pod() { fi echo "partial state: removing newly created owned Pod '$POD' uid=$after_uid; persistent home retained" >&2 if ! printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s"}}\n' "$after_uid" | \ - kube delete --raw "/api/v1/namespaces/$NAMESPACE/pods/$POD" -f -; then + kube --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" delete --raw "/api/v1/namespaces/$NAMESPACE/pods/$POD" -f -; then echo "partial state: UID-precondition rejected; replacement or ownership mismatch retained" >&2 return fi @@ -206,7 +213,7 @@ cleanup_new_owned_pod() { } create_and_wait() { - local pvc_before pvc_current pvc_identity pvc_uid pvc_current_uid pvc_rv pod pod_uid pod_rv + local pvc_before pvc_current pvc_identity pvc_uid pvc_current_uid pvc_rv pod pod_current pod_uid pod_rv pod_claim pvc_before=$(require_owned_pvc_or_absent) if [ -z "$pvc_before" ]; then if ! render_pvc | kube create -f - >/dev/null; then @@ -249,8 +256,9 @@ create_and_wait() { fi pod_uid=$(printf '%s' "$pod" | cut -f6) pod_rv=$(printf '%s' "$pod" | cut -f7) - if [ -z "$pod_uid" ] || [ -z "$pod_rv" ]; then - echo "error: created Pod lacks exact UID or resourceVersion evidence" >&2 + pod_claim=$(printf '%s' "$pod" | cut -f8) + if [ -z "$pod_uid" ] || [ -z "$pod_rv" ] || [ "$pod_claim" != "$PVC" ]; then + echo "error: created Pod lacks exact UID, resourceVersion, or PVC relationship evidence" >&2 exit 1 fi pvc_current=$(require_owned_pvc_or_absent) @@ -264,6 +272,20 @@ create_and_wait() { echo "error: crewmate Pod did not become ready with the authorized AI Secret" >&2 exit 1 fi + pod_current=$(pod_record) + if [ "$(printf '%s' "$pod_current" | cut -f1-5)" != "$EXPECTED_POD"$'\t'"$OPERATION_ID" ] || \ + [ "$(printf '%s' "$pod_current" | cut -f6)" != "$pod_uid" ] || \ + [ "$(printf '%s' "$pod_current" | cut -f8)" != "$PVC" ]; then + pvc_current=$(pvc_record) + pvc_identity=$(printf '%s' "$pvc_current" | cut -f1-4) + if [ "$pvc_identity" = "$EXPECTED_PVC" ]; then + if ! invalidate_checkpoint_evidence; then + echo "partial state: checkpoint invalidation failed after Pod continuity loss; persistent claims retained" >&2 + fi + fi + echo "error: Pod identity changed after readiness; captured uid=$pod_uid observed uid=$(printf '%s' "$pod_current" | cut -f6); replacement retained and persistent claims retained" >&2 + exit 3 + fi pvc_current=$(pvc_record) pvc_identity=$(printf '%s' "$pvc_current" | cut -f1-4) pvc_current_uid=$(printf '%s' "$pvc_current" | cut -f9) @@ -323,7 +345,7 @@ stop_owned_pod() { exit 2 fi printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s"}}\n' "$uid" | \ - kube delete --raw "/api/v1/namespaces/$NAMESPACE/pods/$POD" -f - + kube --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" delete --raw "/api/v1/namespaces/$NAMESPACE/pods/$POD" -f - kube wait --for=delete "pod/$POD" --timeout=180s } @@ -502,7 +524,7 @@ case "$COMMAND" in record_purge purge-requested "$checkpoint_at" printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s","resourceVersion":"%s"}}\n' \ "$pvc_uid" "$pvc_rv" | \ - kube delete --raw "/api/v1/namespaces/$NAMESPACE/persistentvolumeclaims/$PVC" -f - + kube --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" delete --raw "/api/v1/namespaces/$NAMESPACE/persistentvolumeclaims/$PVC" -f - kube wait --for=delete "pvc/$PVC" --timeout=180s record_purge purge-complete "$checkpoint_at" ;; diff --git a/bin/agent-os-kubernetes-lease.sh b/bin/agent-os-kubernetes-lease.sh index 5ef5a964a..c9508c5a1 100644 --- a/bin/agent-os-kubernetes-lease.sh +++ b/bin/agent-os-kubernetes-lease.sh @@ -10,6 +10,31 @@ yaml_string() { printf '"%s"' "$value" } +lock_default_deadline() { + printf '%s' "$(($(date -u '+%s') + LOCK_REQUEST_CEILING_SECONDS))" +} + +lock_request_seconds() { + local deadline=$1 now remaining validity seconds + now=$(date -u '+%s') + remaining=$((deadline - now)) + [ "$remaining" -gt 0 ] || return 1 + validity=$((LOCK_DURATION_SECONDS / 3)) + [ "$validity" -gt 0 ] || validity=1 + seconds=$LOCK_REQUEST_CEILING_SECONDS + [ "$seconds" -le "$validity" ] || seconds=$validity + [ "$seconds" -le "$remaining" ] || seconds=$remaining + [ "$seconds" -gt 0 ] || return 1 + printf '%s' "$seconds" +} + +lock_kube() { + local deadline=$1 seconds + shift + seconds=$(lock_request_seconds "$deadline") || return 124 + kube --request-timeout="${seconds}s" "$@" +} + rfc3339_epoch() { local value=${1%%.*} value=${value%Z}Z @@ -36,16 +61,22 @@ verify_lock_record() { } renew_lock_once() { - local record acquired uid rv now current - record=$(lock_record) || return 1 + local record acquired uid rv now current deadline + deadline=$(lock_default_deadline) + record=$(lock_record "$deadline") || return 1 verify_lock_record "$record" || return 1 acquired=$(printf '%s' "$record" | cut -f6) uid=$(printf '%s' "$record" | cut -f9) rv=$(printf '%s' "$record" | cut -f10) [ -n "$acquired" ] && [ -n "$rv" ] || return 1 now=$(date -u '+%Y-%m-%dT%H:%M:%SZ') - render_lock "$acquired" "$now" "$uid" "$rv" | kube replace -f - >/dev/null || return 1 - current=$(lock_record) || return 1 + if ! render_lock "$acquired" "$now" "$uid" "$rv" | lock_kube "$deadline" replace -f - >/dev/null; then + current=$(lock_record "$deadline") || return 1 + verify_lock_record "$current" || return 1 + [ "$(printf '%s' "$current" | cut -f9)" = "$uid" ] || return 1 + [ "$(printf '%s' "$current" | cut -f10)" != "$rv" ] || return 1 + fi + current=$(lock_record "$deadline") || return 1 verify_lock_record "$current" || return 1 [ "$(printf '%s' "$current" | cut -f9)" = "$uid" ] || return 1 LOCK_RV=$(printf '%s' "$current" | cut -f10) @@ -76,10 +107,11 @@ stop_lock_renewal() { } release_lock() { - local record identity holder uid rv after after_holder after_uid + local record identity holder uid rv after after_holder after_uid deadline stop_lock_renewal [ -n "$LOCK_UID" ] || return 0 - if ! record=$(lock_record); then + deadline=$(lock_default_deadline) + if ! record=$(lock_record "$deadline"); then echo "error: lifecycle Lease '$LOCK' could not be verified for release" >&2 return 1 fi @@ -94,11 +126,21 @@ release_lock() { return 1 fi if ! printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s","resourceVersion":"%s"}}\n' "$uid" "$rv" | \ - kube delete --raw "/apis/coordination.k8s.io/v1/namespaces/$LOCK_NAMESPACE/leases/$LOCK" -f - >/dev/null; then - echo "error: lifecycle Lease '$LOCK' release precondition failed; retained" >&2 - return 1 + lock_kube "$deadline" delete --raw "/apis/coordination.k8s.io/v1/namespaces/$LOCK_NAMESPACE/leases/$LOCK" -f - >/dev/null; then + after=$(lock_record "$deadline") || { + echo "error: lifecycle Lease '$LOCK' release result is ambiguous; retained" >&2 + return 1 + } + if [ -n "$after" ] && [ "$(printf '%s' "$after" | cut -f9)" = "$uid" ] && \ + [ "$(printf '%s' "$after" | cut -f5)" = "$OPERATION_ID" ]; then + echo "error: lifecycle Lease '$LOCK' release precondition failed; retained" >&2 + return 1 + fi + LOCK_UID= + LOCK_RV= + return 0 fi - after=$(lock_record) || return 1 + after=$(lock_record "$deadline") || return 1 if [ -n "$after" ]; then after_holder=$(printf '%s' "$after" | cut -f5) after_uid=$(printf '%s' "$after" | cut -f9) @@ -115,14 +157,28 @@ acquire_lock() { local record identity holder now deadline acquired uid rv current now=$(date -u '+%Y-%m-%dT%H:%M:%SZ') deadline=$(($(date -u '+%s') + LOCK_ACQUIRE_SECONDS)) - while ! render_lock "$now" "$now" | kube create -f - >/dev/null; do - record=$(lock_record) + holder=unknown + while :; do + if [ "$(date -u '+%s')" -ge "$deadline" ]; then + echo "error: lifecycle operation '$holder' still holds Lease '$LOCK' after ${LOCK_ACQUIRE_SECONDS}s" >&2 + exit 3 + fi + if render_lock "$now" "$now" | lock_kube "$deadline" create -f - >/dev/null; then + break + fi + record=$(lock_record "$deadline") || { + echo "error: lifecycle Lease '$LOCK' create result could not be reconciled before the acquisition deadline" >&2 + exit 3 + } identity=$(printf '%s' "$record" | cut -f1-4) holder=$(printf '%s' "$record" | cut -f5) if [ -z "$record" ] || [ "$identity" != "$EXPECTED_LOCK" ]; then echo "error: lifecycle Lease '$LOCK' is absent or has foreign ownership after create conflict" >&2 exit 2 fi + if verify_lock_record "$record"; then + break + fi if lease_is_expired "$record"; then uid=$(printf '%s' "$record" | cut -f9) rv=$(printf '%s' "$record" | cut -f10) @@ -131,7 +187,14 @@ acquire_lock() { exit 2 fi now=$(date -u '+%Y-%m-%dT%H:%M:%SZ') - if render_lock "$now" "$now" "$uid" "$rv" | kube replace -f - >/dev/null; then + if render_lock "$now" "$now" "$uid" "$rv" | lock_kube "$deadline" replace -f - >/dev/null; then + break + fi + record=$(lock_record "$deadline") || { + echo "error: lifecycle Lease '$LOCK' takeover result could not be reconciled before the acquisition deadline" >&2 + exit 3 + } + if verify_lock_record "$record" && [ "$(printf '%s' "$record" | cut -f9)" = "$uid" ]; then break fi fi @@ -142,7 +205,7 @@ acquire_lock() { sleep 1 now=$(date -u '+%Y-%m-%dT%H:%M:%SZ') done - record=$(lock_record) + record=$(lock_record "$deadline") if ! verify_lock_record "$record"; then LOCK_UID= echo "error: lifecycle Lease '$LOCK' did not verify after acquisition" >&2 @@ -156,7 +219,7 @@ acquire_lock() { echo "error: lifecycle Lease '$LOCK' lacks complete renewal evidence" >&2 exit 2 fi - current=$(lock_record) + current=$(lock_record "$deadline") if [ "$current" != "$record" ]; then LOCK_UID= echo "error: lifecycle Lease '$LOCK' changed after acquisition" >&2 diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh index fc22ae8b5..8a632a6af 100755 --- a/bin/agent-os-kubernetes.sh +++ b/bin/agent-os-kubernetes.sh @@ -30,11 +30,16 @@ LOCK_RENEW_PID= LOCK_DURATION_SECONDS=${AGENT_OS_LOCK_DURATION_SECONDS:-300} LOCK_CLOCK_SKEW_SECONDS=${AGENT_OS_LOCK_CLOCK_SKEW_SECONDS:-5} LOCK_ACQUIRE_SECONDS=${AGENT_OS_LOCK_ACQUIRE_SECONDS:-30} +LOCK_REQUEST_CEILING_SECONDS=${AGENT_OS_LOCK_REQUEST_CEILING_SECONDS:-5} +RESOURCE_REQUEST_CEILING_SECONDS=${AGENT_OS_RESOURCE_REQUEST_CEILING_SECONDS:-5} -for seconds in "$LOCK_DURATION_SECONDS" "$LOCK_CLOCK_SKEW_SECONDS" "$LOCK_ACQUIRE_SECONDS"; do +for seconds in "$LOCK_DURATION_SECONDS" "$LOCK_CLOCK_SKEW_SECONDS" "$LOCK_ACQUIRE_SECONDS" "$LOCK_REQUEST_CEILING_SECONDS" "$RESOURCE_REQUEST_CEILING_SECONDS"; do case "$seconds" in ''|*[!0-9]*) echo "error: lifecycle Lease timing must use whole seconds" >&2; exit 2 ;; esac done [ "$LOCK_DURATION_SECONDS" -ge 3 ] || { echo "error: lifecycle Lease duration must be at least 3 seconds" >&2; exit 2; } +[ "$LOCK_ACQUIRE_SECONDS" -ge 1 ] || { echo "error: lifecycle Lease acquisition must allow at least 1 second" >&2; exit 2; } +[ "$LOCK_REQUEST_CEILING_SECONDS" -ge 1 ] || { echo "error: lifecycle Lease request ceiling must be at least 1 second" >&2; exit 2; } +[ "$RESOURCE_REQUEST_CEILING_SECONDS" -ge 1 ] || { echo "error: resource request ceiling must be at least 1 second" >&2; exit 2; } . "$ROOT/bin/agent-os-kubernetes-lease.sh" @@ -162,7 +167,9 @@ kube() { } lock_record() { - kube get lease "$LOCK" --ignore-not-found \ + local deadline=${1:-} + [ -n "$deadline" ] || deadline=$(lock_default_deadline) + lock_kube "$deadline" get lease "$LOCK" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/lifecycle}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.spec.holderIdentity}{"\t"}{.spec.acquireTime}{"\t"}{.spec.renewTime}{"\t"}{.spec.leaseDurationSeconds}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}' } @@ -267,16 +274,63 @@ live_resource_identity() { } live_resource_record() { - local scope=$1 kind=$2 name=$3 + local scope=$1 kind=$2 name=$3 request_timeout=${4:-} + local request_args=() + [ -z "$request_timeout" ] || request_args=(--request-timeout="$request_timeout") if [ "$scope" = cluster ]; then - "$KUBECTL" --context "$CONTEXT" get "$kind" "$name" --ignore-not-found \ + "$KUBECTL" --context "$CONTEXT" "${request_args[@]}" get "$kind" "$name" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}' else - "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get "$kind" "$name" --ignore-not-found \ + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" "${request_args[@]}" get "$kind" "$name" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}' fi } +operation_remaining_seconds() { + local deadline=$1 remaining + remaining=$((deadline - $(date -u '+%s'))) + [ "$remaining" -gt 0 ] || return 1 + printf '%s' "$remaining" +} + +operation_request_seconds() { + local deadline=$1 remaining seconds + remaining=$(operation_remaining_seconds "$deadline") || return 1 + seconds=$RESOURCE_REQUEST_CEILING_SECONDS + [ "$seconds" -le "$remaining" ] || seconds=$remaining + [ "$seconds" -gt 0 ] || return 1 + printf '%s' "$seconds" +} + +reconcile_deleted_resource() { + local scope=$1 kind=$2 name=$3 uid=$4 rv=$5 timeout=$6 started=$7 phase=$8 class=$9 + local deadline=${10} request_seconds current current_uid current_rv elapsed + request_seconds=$(operation_request_seconds "$deadline") || { + elapsed=$(($(date -u '+%s') - started)) + bounded_delete_failure "$kind/$name" "$phase" "$class" "$timeout" "$elapsed" "$uid" + return $? + } + if ! current=$(live_resource_record "$scope" "$kind" "$name" "${request_seconds}s"); then + elapsed=$(($(date -u '+%s') - started)) + bounded_delete_failure "$kind/$name" reconcile transport "$timeout" "$elapsed" "$uid" + return $? + fi + if [ -z "$current" ]; then + echo "confirmed absent: $kind/$name captured uid=$uid after delete-$phase failure=$class" >&2 + return 0 + fi + current_uid=$(printf '%s' "$current" | cut -f4) + current_rv=$(printf '%s' "$current" | cut -f5) + elapsed=$(($(date -u '+%s') - started)) + if [ "$current_uid" != "$uid" ]; then + echo "error: $kind/$name replacement uid=${current_uid:-unknown} retained after ambiguous delete of captured uid=$uid resourceVersion=$rv" >&2 + bounded_delete_failure "$kind/$name" reconcile replacement "$timeout" "$elapsed" "$uid" + return $? + fi + echo "error: $kind/$name captured uid=$uid remains at resourceVersion=${current_rv:-unknown} after delete-$phase failure=$class" >&2 + bounded_delete_failure "$kind/$name" "$phase" "$class" "$timeout" "$elapsed" "$uid" +} + resource_api_path() { local scope=$1 kind=$2 name=$3 case "$kind" in @@ -294,7 +348,7 @@ resource_api_path() { } delete_owned_resource() { - local scope=$1 kind=$2 name=$3 timeout=$4 record expected uid rv path output started elapsed class + local scope=$1 kind=$2 name=$3 timeout=$4 record expected uid rv path output started deadline class request_seconds remaining reserve wait_budget record=$(live_resource_record "$scope" "$kind" "$name") [ -n "$record" ] || return 0 expected="$name"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" @@ -307,24 +361,34 @@ delete_owned_resource() { [ -n "$uid" ] && [ -n "$rv" ] || { echo "error: $kind '$name' lacks deletion preconditions" >&2; exit 2; } path=$(resource_api_path "$scope" "$kind" "$name") started=$(date -u '+%s') + deadline=$((started + timeout)) + request_seconds=$(operation_request_seconds "$deadline") || { + bounded_delete_failure "$kind/$name" request timeout "$timeout" 0 "$uid" + return $? + } if ! output=$(printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s","resourceVersion":"%s"}}\n' "$uid" "$rv" | \ - "$KUBECTL" --context "$CONTEXT" delete --raw "$path" -f - 2>&1 >/dev/null); then - elapsed=$(($(date -u '+%s') - started)) + "$KUBECTL" --context "$CONTEXT" --request-timeout="${request_seconds}s" delete --raw "$path" -f - 2>&1 >/dev/null); then class=$(delete_failure_class "$output") - bounded_delete_failure "$kind/$name" request "$class" "$timeout" "$elapsed" "$uid" + reconcile_deleted_resource "$scope" "$kind" "$name" "$uid" "$rv" "$timeout" "$started" request "$class" "$deadline" + return $? + fi + remaining=$(operation_remaining_seconds "$deadline") || remaining=0 + if [ "$remaining" -lt 2 ]; then + reconcile_deleted_resource "$scope" "$kind" "$name" "$uid" "$rv" "$timeout" "$started" wait timeout "$deadline" return $? fi + reserve=$RESOURCE_REQUEST_CEILING_SECONDS + [ "$reserve" -lt "$remaining" ] || reserve=$((remaining - 1)) + wait_budget=$((remaining - reserve)) if [ "$scope" = cluster ]; then - if ! output=$("$KUBECTL" --context "$CONTEXT" wait --for=delete "$kind/$name" --timeout="${timeout}s" 2>&1 >/dev/null); then - elapsed=$(($(date -u '+%s') - started)) + if ! output=$("$KUBECTL" --context "$CONTEXT" --request-timeout="${wait_budget}s" wait --for=delete "$kind/$name" --timeout="${wait_budget}s" 2>&1 >/dev/null); then class=$(delete_failure_class "$output") - bounded_delete_failure "$kind/$name" wait "$class" "$timeout" "$elapsed" "$uid" + reconcile_deleted_resource "$scope" "$kind" "$name" "$uid" "$rv" "$timeout" "$started" wait "$class" "$deadline" fi else - if ! output=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" wait --for=delete "$kind/$name" --timeout="${timeout}s" 2>&1 >/dev/null); then - elapsed=$(($(date -u '+%s') - started)) + if ! output=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" --request-timeout="${wait_budget}s" wait --for=delete "$kind/$name" --timeout="${wait_budget}s" 2>&1 >/dev/null); then class=$(delete_failure_class "$output") - bounded_delete_failure "$kind/$name" wait "$class" "$timeout" "$elapsed" "$uid" + reconcile_deleted_resource "$scope" "$kind" "$name" "$uid" "$rv" "$timeout" "$started" wait "$class" "$deadline" fi fi } @@ -445,6 +509,28 @@ report_partial_observation() { echo "$prefix: $kind/$name uid=$uid operation=$operation $details ownership=$managed installation=$installation finalizers=$finalizers" >&2 } +report_rollback_failure() { + local target_name=$1 target_revision=$2 state references lease deadline + echo "incomplete: rollback target=$target_name revision=$target_revision did not complete" >&2 + if state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get statefulset agent-os-firstmate -o json 2>/dev/null) && \ + references=$(printf '%s' "$state" | jq -er '[.status.currentRevision, .status.updateRevision] | select(all(.[]; type == "string" and length > 0)) | @tsv' 2>/dev/null); then + echo "rollback observed: current-revision=$(printf '%s' "$references" | cut -f1) update-revision=$(printf '%s' "$references" | cut -f2)" >&2 + else + echo "rollback observed: StatefulSet revision evidence unavailable" >&2 + fi + report_partial_observation Pod agent-os-firstmate-0 namespaced + deadline=$(lock_default_deadline) + if lease=$(lock_record "$deadline" 2>/dev/null) && [ -n "$lease" ]; then + echo "rollback retained: lifecycle-lease=$LOCK uid=$(printf '%s' "$lease" | cut -f9) holder=$(printf '%s' "$lease" | cut -f5)" >&2 + else + echo "rollback retained: lifecycle-lease=$LOCK evidence=unavailable" >&2 + fi + printf 'safe recovery: %q --context %q -n %q rollout status statefulset/agent-os-firstmate --timeout=180s\n' \ + "$KUBECTL" "$CONTEXT" "$NAMESPACE" >&2 + echo "safe recovery condition: continue only while updateRevision remains '$target_name'; do not invoke rollback again while revisions differ" >&2 + return 3 +} + partial_install_is_applicable() { local file kind name identity expected namespace if ! namespace=$(namespace_name 2>/dev/null); then @@ -1000,7 +1086,7 @@ case "$COMMAND" in ') || { echo "error: StatefulSet changed before rollback revision resolution" >&2; exit 3; } rollback_current_revision=$(printf '%s' "$rollback_references" | cut -f1) rollback_update_revision=$(printf '%s' "$rollback_references" | cut -f2) - rollback_patch=$(printf '%s' "$rollback_revisions" | jq -ce \ + rollback_selection=$(printf '%s' "$rollback_revisions" | jq -ce \ --arg current "$rollback_current_revision" --arg update "$rollback_update_revision" \ --arg uid "$rollback_uid" --arg rv "$rollback_rv" ' def owned: @@ -1014,27 +1100,40 @@ case "$COMMAND" in | select($update_items | length == 1) | $current_items[0] as $current_revision | $update_items[0] as $update_revision + | select($current_revision.revision | type == "number") | select($update_revision.revision | type == "number") - | (if $current != $update then - $current_revision + | (if $current != $update and $update_revision.revision < $current_revision.revision then + {mode:"resume", target:$update_revision} + elif $current != $update then + {mode:"patch", target:$current_revision} else - ($owned | map(select((.revision | type) == "number" and .revision < $update_revision.revision)) | sort_by(.revision) | last) - end) as $target - | select($target != null and ($target.data | type) == "object") - | $target.data * {metadata:{uid:$uid,resourceVersion:$rv}} + {mode:"patch", target:($owned | map(select((.revision | type) == "number" and .revision < $update_revision.revision)) | sort_by(.revision) | last)} + end) as $selection + | select($selection.target != null and ($selection.target.data | type) == "object") + | {mode:$selection.mode, targetName:$selection.target.metadata.name, targetRevision:$selection.target.revision, + patch:($selection.target.data * {metadata:{uid:$uid,resourceVersion:$rv}})} ') || { echo "error: no exact-owned previous ControllerRevision is available for rollback" >&2; exit 2; } - if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch StatefulSet agent-os-firstmate \ - --type=strategic -p "$rollback_patch" >/dev/null; then - echo "error: StatefulSet rollback CAS conflicted; no rollout-undo fallback attempted" >&2 - exit 3 + rollback_mode=$(printf '%s' "$rollback_selection" | jq -r '.mode') + rollback_target_name=$(printf '%s' "$rollback_selection" | jq -r '.targetName') + rollback_target_revision=$(printf '%s' "$rollback_selection" | jq -r '.targetRevision') + rollback_patch=$(printf '%s' "$rollback_selection" | jq -c '.patch') + if [ "$rollback_mode" = patch ]; then + if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch StatefulSet agent-os-firstmate \ + --type=strategic -p "$rollback_patch" >/dev/null; then + echo "error: StatefulSet rollback CAS conflicted; no rollout-undo fallback attempted" >&2 + exit 3 + fi + rollback_current=$(live_resource_record namespaced StatefulSet agent-os-firstmate) + if [ "$(printf '%s' "$rollback_current" | cut -f1-4)" != \ + "$(printf '%s' "$rollback_record" | cut -f1-4)" ]; then + echo "error: StatefulSet UID or ownership changed during rollback" >&2 + exit 3 + fi fi - rollback_current=$(live_resource_record namespaced StatefulSet agent-os-firstmate) - if [ "$(printf '%s' "$rollback_current" | cut -f1-4)" != \ - "$(printf '%s' "$rollback_record" | cut -f1-4)" ]; then - echo "error: StatefulSet UID or ownership changed during rollback" >&2 + if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s; then + report_rollback_failure "$rollback_target_name" "$rollback_target_revision" exit 3 fi - "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s ;; status) [ "$CONFIRMED" -eq 0 ] && [ "$DELETE_NAMESPACE" -eq 0 ] || usage diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index 906f35386..fb5272eed 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -42,6 +42,11 @@ cat > "$FAKEBIN/kubectl" <<'SH' printf 'kubectl' >> "$AGENT_OS_TEST_LOG" printf ' %s' "$@" >> "$AGENT_OS_TEST_LOG" printf '\n' >> "$AGENT_OS_TEST_LOG" +filtered_args=() +for argument in "$@"; do + [[ "$argument" = --request-timeout=* ]] || filtered_args+=("$argument") +done +set -- "${filtered_args[@]}" stdin_data='' previous='' for argument in "$@"; do @@ -124,15 +129,25 @@ case " $* " in absent) ;; owned) printf 'agent-os-crewmate-%s\tagent-os\t%s\tagent-os-firstmate:agent-os-demo' "$id" "$id" - [[ " $* " != *'.metadata.uid'* ]] || printf '\toperation-test\tuid-owned\trv-pod-owned' + [[ " $* " != *'.metadata.uid'* ]] || printf '\toperation-test\tuid-owned\trv-pod-owned\tagent-os-crewmate-%s-home' "$id" ;; replacement) printf 'agent-os-crewmate-%s\tagent-os\t%s\tagent-os-firstmate:agent-os-demo' "$id" "$id" - [[ " $* " != *'.metadata.uid'* ]] || printf '\tother-operation\tuid-replacement\trv-pod-replacement' + [[ " $* " != *'.metadata.uid'* ]] || printf '\tother-operation\tuid-replacement\trv-pod-replacement\tagent-os-crewmate-%s-home' "$id" + ;; + replaced-after-ready) + printf 'agent-os-crewmate-%s\tagent-os\t%s\tagent-os-firstmate:agent-os-demo' "$id" "$id" + if [[ " $* " = *'.metadata.uid'* ]]; then + if grep -F 'wait --for=condition=Ready pod/' "$AGENT_OS_TEST_LOG" >/dev/null; then + printf '\tother-operation\tuid-replacement\trv-pod-replacement\tagent-os-crewmate-%s-home' "$id" + else + printf '\toperation-test\tuid-owned\trv-pod-owned\tagent-os-crewmate-%s-home' "$id" + fi + fi ;; foreign) printf 'agent-os-crewmate-%s\tother\tother\tother-installation' "$id" - [[ " $* " != *'.metadata.uid'* ]] || printf '\tother-operation\tuid-foreign\trv-pod-foreign' + [[ " $* " != *'.metadata.uid'* ]] || printf '\tother-operation\tuid-foreign\trv-pod-foreign\tforeign-home' ;; esac ;; @@ -169,7 +184,7 @@ case " $* " in ;; esac ;; - *" get lease agent-os-crewmate-"*" --ignore-not-found -o jsonpath="*) + *get\ lease\ agent-os-crewmate-*--ignore-not-found*-o\ jsonpath=*) id=${AGENT_OS_TEST_CREWMATE_ID:-scout-1} case "${AGENT_OS_TEST_LOCK_STATE:-free}" in held) printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tother-operation\t2099-01-01T00:00:00Z\t2099-01-01T00:00:00Z\t300\tuid-lock-other\trv-lock-other' "$id" "$id" ;; @@ -197,7 +212,7 @@ case " $* " in ;; esac ;; - *" get lease agent-os-firstmate-lifecycle"*" --ignore-not-found -o jsonpath="*) + *get\ lease\ agent-os-firstmate-lifecycle*--ignore-not-found*-o\ jsonpath=*) lock_name=$(printf '%s\n' "$*" | sed -n 's/.* get lease \([^ ]*\) .*/\1/p') last_lock_write=$(grep -Fn 'stdin-kind Lease' "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d: -f1) last_lock_delete=$(grep -Fn "/leases/$lock_name" "$AGENT_OS_TEST_LOG" | grep 'delete --raw' | tail -n 1 | cut -d: -f1) @@ -302,8 +317,10 @@ case " $* " in esac ;; *" get statefulset agent-os-firstmate -o json "*) - printf '{"metadata":{"name":"agent-os-firstmate","uid":"uid-statefulset","resourceVersion":"%s","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"status":{"currentRevision":"agent-os-firstmate-previous","updateRevision":"agent-os-firstmate-current"}}\n' \ - "${AGENT_OS_TEST_RESOURCE_RV:-rv-statefulset}" + printf '{"metadata":{"name":"agent-os-firstmate","uid":"uid-statefulset","resourceVersion":"%s","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"status":{"currentRevision":"%s","updateRevision":"%s"}}\n' \ + "${AGENT_OS_TEST_RESOURCE_RV:-rv-statefulset}" \ + "${AGENT_OS_TEST_ROLLBACK_CURRENT:-agent-os-firstmate-previous}" \ + "${AGENT_OS_TEST_ROLLBACK_UPDATE:-agent-os-firstmate-current}" ;; *" get controllerrevisions.apps -o json "*) printf '%s\n' '{"items":[{"metadata":{"name":"agent-os-firstmate-previous","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":1,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"previous"}}}}}},{"metadata":{"name":"agent-os-firstmate-current","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":2,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"current"}}}}}}]}' @@ -332,6 +349,10 @@ case " $* " in *" get clusterrolebinding agent-os-firstmate-"*" --ignore-not-found -o jsonpath="*|\ *" get ClusterRoleBinding agent-os-firstmate-"*" --ignore-not-found -o jsonpath="*) cluster_state=${AGENT_OS_TEST_CLUSTER_RBAC_STATE:-absent} + if grep -F '/clusterrolebindings/agent-os-firstmate-portable-agent-os' "$AGENT_OS_TEST_LOG" | grep -F 'delete --raw' >/dev/null && \ + [ -n "${AGENT_OS_TEST_DELETE_READBACK_STATE:-}" ]; then + cluster_state=$AGENT_OS_TEST_DELETE_READBACK_STATE + fi if [ -n "${AGENT_OS_TEST_CLUSTER_RBAC_AFTER_APPLY:-}" ] && \ { grep -E ' create -f .+\.yaml' "$AGENT_OS_TEST_LOG" >/dev/null || grep -F ' --patch-file ' "$AGENT_OS_TEST_LOG" >/dev/null; }; then cluster_state=$AGENT_OS_TEST_CLUSTER_RBAC_AFTER_APPLY @@ -352,6 +373,10 @@ case " $* " in printf '\tuid-clusterrolebinding\toperation-test\tTrue\t[]' fi ;; + replacement) + printf 'agent-os-firstmate-%s\tagent-os\tagent-os-firstmate:%s\tuid-clusterrolebinding-replacement\trv-clusterrolebinding-replacement\tother-operation' \ + "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" + ;; foreign) printf 'agent-os-firstmate-portable-agent-os\tother\tother-installation' ;; esac ;; @@ -391,7 +416,7 @@ run_launcher() { AGENT_OS_IN_CLUSTER=1 AGENT_OS_NAMESPACE=agent-os-demo AGENT_OS_IMAGE=agent-os:local-test \ AGENT_OS_IMAGE_PULL_POLICY=Never AGENT_OS_AI_SECRET=scout-1-ai-auth \ AGENT_OS_OPERATION_ID=operation-test AGENT_OS_PURGE_EVIDENCE_FILE="$PURGE_EVIDENCE" \ - AGENT_OS_LOCK_ACQUIRE_SECONDS=0 \ + AGENT_OS_LOCK_ACQUIRE_SECONDS=2 \ "$LAUNCHER" "$@" } @@ -467,7 +492,7 @@ pass "crewmate create requires an explicit AI Secret grant" if AGENT_OS_TEST_FAIL_WAIT=1 AGENT_OS_TEST_POD_AFTER_APPLY=owned run_launcher create scout-1 >/dev/null 2>&1; then fail "crewmate create must fail when its authorized Secret cannot produce a ready Pod" fi -grep -Fqx 'kubectl -n agent-os-demo delete --raw /api/v1/namespaces/agent-os-demo/pods/agent-os-crewmate-scout-1 -f -' "$CALLS" || \ +grep -Fqx 'kubectl -n agent-os-demo --request-timeout=5s delete --raw /api/v1/namespaces/agent-os-demo/pods/agent-os-crewmate-scout-1 -f -' "$CALLS" || \ fail "failed create must use an atomic UID-preconditioned delete" assert_grep '"uid":"uid-owned"' "$STDIN_LOG" \ "failed create must precondition deletion on the observed Pod UID" @@ -476,6 +501,20 @@ if grep -F 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" >/dev/null; then fi pass "crewmate create fails closed while retaining its persistent home" +: > "$CALLS" +pod_replacement_out='' +pod_replacement_rc=0 +pod_replacement_out=$(AGENT_OS_TEST_POD_AFTER_APPLY=replaced-after-ready \ + run_launcher create scout-1 2>&1) || pod_replacement_rc=$? +[ "$pod_replacement_rc" -eq 3 ] || \ + fail "post-ready Pod replacement must exit incomplete: $pod_replacement_out" +assert_contains "$pod_replacement_out" 'captured uid=uid-owned observed uid=uid-replacement' \ + "post-ready continuity failure must report both Pod identities" +if grep -F '/pods/agent-os-crewmate-scout-1' "$CALLS" | grep -F 'delete --raw' >/dev/null; then + fail "post-ready continuity failure must retain a same-name replacement Pod" +fi +pass "crewmate readiness verifies the original Pod identity" + : > "$CALLS" partial_out='' partial_rc=0 @@ -486,7 +525,7 @@ assert_contains "$partial_out" 'uid-owned' \ "partial apply cleanup must report the exact newly created Pod UID" assert_grep 'metadata.uid' "$CALLS" \ "partial apply cleanup must collect exact Pod UID evidence" -grep -Fqx 'kubectl -n agent-os-demo delete --raw /api/v1/namespaces/agent-os-demo/pods/agent-os-crewmate-scout-1 -f -' "$CALLS" || \ +grep -Fqx 'kubectl -n agent-os-demo --request-timeout=5s delete --raw /api/v1/namespaces/agent-os-demo/pods/agent-os-crewmate-scout-1 -f -' "$CALLS" || \ fail "partial apply cleanup must use an atomic UID-preconditioned delete" assert_grep '"uid":"uid-owned"' "$STDIN_LOG" \ "partial apply cleanup must bind deletion to the observed Pod UID" @@ -548,7 +587,7 @@ fi : > "$CALLS" AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=owned run_launcher stop scout-1 -grep -Fqx 'kubectl -n agent-os-demo delete --raw /api/v1/namespaces/agent-os-demo/pods/agent-os-crewmate-scout-1 -f -' "$CALLS" || \ +grep -Fqx 'kubectl -n agent-os-demo --request-timeout=5s delete --raw /api/v1/namespaces/agent-os-demo/pods/agent-os-crewmate-scout-1 -f -' "$CALLS" || \ fail "stop must UID-precondition deletion of the exactly owned crewmate Pod" grep -Fqx 'kubectl -n agent-os-demo wait --for=delete pod/agent-os-crewmate-scout-1 --timeout=180s' "$CALLS" || \ fail "stop must prove Pod absence before checkpointing can begin" @@ -635,7 +674,7 @@ assert_no_grep 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" \ AGENT_OS_TEST_POD_STATE=absent AGENT_OS_TEST_PVC_STATE=clean run_launcher purge scout-1 --yes assert_no_grep 'delete pod agent-os-crewmate-scout-1' "$CALLS" \ "purge must accept checkpoint evidence only after the Pod is absent" -grep -Fqx 'kubectl -n agent-os-demo delete --raw /api/v1/namespaces/agent-os-demo/persistentvolumeclaims/agent-os-crewmate-scout-1-home -f -' "$CALLS" || \ +grep -Fqx 'kubectl -n agent-os-demo --request-timeout=5s delete --raw /api/v1/namespaces/agent-os-demo/persistentvolumeclaims/agent-os-crewmate-scout-1-home -f -' "$CALLS" || \ fail "purge must atomically delete the exactly owned persistent home" assert_grep '"uid":"uid-pvc-owned","resourceVersion":"rv-pvc-owned"' "$STDIN_LOG" \ "purge must precondition deletion on the captured PVC UID and resourceVersion" @@ -668,7 +707,7 @@ lock_out='' lock_rc=0 lock_out=$(AGENT_OS_TEST_LOCK_STATE=held run_launcher stop scout-1 2>&1) || lock_rc=$? [ "$lock_rc" -eq 3 ] || fail "bounded lifecycle lock contention must exit incomplete: $lock_out" -assert_contains "$lock_out" "still holds Lease 'agent-os-crewmate-scout-1-lifecycle' after 0s" \ +assert_contains "$lock_out" "still holds Lease 'agent-os-crewmate-scout-1-lifecycle' after 2s" \ "lifecycle contention must report the exact holder and bounded timeout" pass "crewmate lifecycle operations use a bounded coordination lock" @@ -680,6 +719,15 @@ grep -F 'replace -f -' "$CALLS" >/dev/null || \ fail "expired exact-owned lifecycle Leases must use resourceVersion-CAS takeover" pass "crewmate lifecycle can recover exact-owned expired Leases" +: > "$CALLS" +ambiguous_lock_out='' +ambiguous_lock_rc=0 +ambiguous_lock_out=$(AGENT_OS_TEST_LOCK_STATE=ambiguous-create AGENT_OS_TEST_PVC_STATE=owned \ + run_launcher stop scout-1 2>&1) || ambiguous_lock_rc=$? +[ "$ambiguous_lock_rc" -eq 0 ] || \ + fail "ambiguous Lease create must accept verified own-holder read-back: $ambiguous_lock_out" +pass "crewmate Lease acquisition reconciles ambiguous create success" + : > "$CALLS" : > "$STDIN_LOG" AGENT_OS_TEST_LOCK_STATE=expired AGENT_OS_TEST_LOCK_RV=12345 AGENT_OS_TEST_PVC_STATE=owned \ @@ -695,6 +743,15 @@ grep -F 'replace -f -' "$CALLS" >/dev/null || \ fail "active lifecycle operations must renew their exact-owned Lease" pass "crewmate lifecycle renews its Lease while active" +lease_call_without_timeout=$(awk ' + /^kubectl / { call=$0 } + / get lease | delete --raw .*\/leases\// { if ($0 !~ /--request-timeout=/) print } + /^stdin-kind Lease$/ { if (call !~ /--request-timeout=/) print call } +' "$CALLS") +[ -z "$lease_call_without_timeout" ] || \ + fail "every Lease request must carry a bounded request timeout: $lease_call_without_timeout" +pass "crewmate Lease requests are bounded within lock validity" + : > "$CALLS" AGENT_OS_TEST_POD_STATE=absent AGENT_OS_TEST_PVC_STATE=owned \ AGENT_OS_TEST_LOCK_RELEASE_STATE=foreign run_launcher stop scout-1 @@ -1078,10 +1135,18 @@ grep -F '/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/agent-os-firstma fail "privileged cleanup must UID-delete only the exact owned ClusterRoleBinding" assert_grep '"uid":"uid-clusterrolebinding","resourceVersion":"rv-clusterrolebinding"' "$STDIN_LOG" \ "privileged cleanup must bind deletion to the observed grant identity" -grep -F 'wait --for=delete ClusterRoleBinding/agent-os-firstmate-portable-agent-os --timeout=60s' "$CALLS" >/dev/null || \ +grep -F 'wait --for=delete ClusterRoleBinding/agent-os-firstmate-portable-agent-os --timeout=55s' "$CALLS" >/dev/null || \ fail "privileged cleanup must produce deletion evidence for the exact binding" +grep -F 'wait --for=delete ClusterRoleBinding/agent-os-firstmate-portable-agent-os --timeout=55s' "$CALLS" | \ + grep -F -- '--request-timeout=55s' >/dev/null || \ + fail "delete wait must reserve five seconds for bounded identity reconciliation" pass "privileged cleanup verifies ownership and deletes one exact binding" +cleanup_delete_call=$(grep -F '/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/agent-os-firstmate-portable-agent-os' "$CALLS" | \ + grep -F 'delete --raw' | tail -n 1) +assert_contains "$cleanup_delete_call" '--request-timeout=' \ + "raw delete requests must be bounded before the operation wait begins" + : > "$CALLS" cleanup_stale_out='' cleanup_stale_rc=0 @@ -1112,6 +1177,33 @@ assert_contains "$cleanup_delete_out" 'safe retry:' \ "privileged cleanup failure must print exact retry evidence" pass "privileged cleanup failures report retained grant evidence" +: > "$CALLS" +not_found_out='' +not_found_rc=0 +not_found_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=pending \ + AGENT_OS_TEST_CLUSTER_RBAC_STATE=owned AGENT_OS_TEST_DELETE_READBACK_STATE=absent \ + AGENT_OS_TEST_FAIL_DELETE_TARGET=clusterrolebindings/agent-os-firstmate-portable-agent-os \ + AGENT_OS_TEST_DELETE_ERROR='Error from server (NotFound): object disappeared' \ + run_generic cleanup-cluster-rbac --yes 2>&1) || not_found_rc=$? +[ "$not_found_rc" -eq 0 ] || fail "NotFound with confirmed absence must complete: $not_found_out" +assert_contains "$not_found_out" 'clusterrolebinding/agent-os-firstmate-portable-agent-os absent' \ + "NotFound reconciliation must emit confirmed absence evidence" +pass "delete NotFound races reconcile confirmed absence" + +: > "$CALLS" +replacement_delete_out='' +replacement_delete_rc=0 +replacement_delete_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=pending \ + AGENT_OS_TEST_CLUSTER_RBAC_STATE=owned AGENT_OS_TEST_DELETE_READBACK_STATE=replacement \ + AGENT_OS_TEST_FAIL_DELETE_TARGET=clusterrolebindings/agent-os-firstmate-portable-agent-os \ + AGENT_OS_TEST_DELETE_ERROR='transport timeout' \ + run_generic cleanup-cluster-rbac --yes 2>&1) || replacement_delete_rc=$? +[ "$replacement_delete_rc" -eq 1 ] || \ + fail "ambiguous delete with replacement must remain incomplete: $replacement_delete_out" +assert_contains "$replacement_delete_out" 'replacement uid=uid-clusterrolebinding-replacement retained' \ + "ambiguous delete reconciliation must report and retain a replacement UID" +pass "ambiguous deletes retain same-name replacements" + : > "$CALLS" active_cleanup_rc=0 AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=cluster-admin \ @@ -1212,6 +1304,42 @@ assert_grep 'get controllerrevisions.apps -o json' "$CALLS" \ "rollback must resolve the exact-owned revision history" pass "generic rollback applies a revision-derived StatefulSet CAS update" +: > "$CALLS" +rollback_failure_out='' +rollback_failure_rc=0 +rollback_failure_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_PRIMARY_POD_STATE=owned \ + AGENT_OS_TEST_FAIL_ROLLOUT=1 run_generic rollback 2>&1) || rollback_failure_rc=$? +[ "$rollback_failure_rc" -eq 3 ] || fail "failed rollback rollout must exit incomplete: $rollback_failure_out" +assert_contains "$rollback_failure_out" 'rollback target=agent-os-firstmate-previous revision=1' \ + "rollback failure must preserve its selected revision" +assert_contains "$rollback_failure_out" 'current-revision=agent-os-firstmate-previous update-revision=agent-os-firstmate-current' \ + "rollback failure must report observed revision state" +assert_contains "$rollback_failure_out" 'Pod/agent-os-firstmate-0 uid=uid-pod' \ + "rollback failure must report Pod UID and readiness evidence" +assert_contains "$rollback_failure_out" 'lifecycle-lease=agent-os-firstmate-lifecycle uid=uid-primary-lock holder=operation-test' \ + "rollback failure must preserve exact lifecycle Lease evidence" +assert_contains "$rollback_failure_out" 'safe recovery:' \ + "rollback failure must print an exact non-reversing recovery command" +pass "failed rollback rollout preserves recovery evidence" + +: > "$CALLS" +rollback_resume_out='' +rollback_resume_rc=0 +rollback_resume_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_PRIMARY_POD_STATE=owned \ + AGENT_OS_TEST_ROLLBACK_CURRENT=agent-os-firstmate-current \ + AGENT_OS_TEST_ROLLBACK_UPDATE=agent-os-firstmate-previous \ + AGENT_OS_TEST_FAIL_ROLLOUT=1 run_generic rollback 2>&1) || rollback_resume_rc=$? +[ "$rollback_resume_rc" -eq 3 ] || fail "in-progress rollback retry must remain incomplete: $rollback_resume_out" +assert_no_grep 'patch StatefulSet agent-os-firstmate' "$CALLS" \ + "in-progress rollback retry must not reverse the selected target" +assert_contains "$rollback_resume_out" 'rollback target=agent-os-firstmate-previous revision=1' \ + "in-progress rollback retry must preserve the lower selected revision" +assert_contains "$rollback_resume_out" "do not invoke rollback again while revisions differ" \ + "in-progress rollback retry must provide non-reversing recovery guidance" +pass "rollback retry resumes an existing lower-revision target" + : > "$CALLS" foreign_rollback_rc=0 AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=foreign AGENT_OS_TEST_WORKLOAD_STATE=foreign \ From eed7d5412a28ef85c4fecce3d3e49d2d2cec898d Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 12:25:21 +0200 Subject: [PATCH 33/56] no-mistakes(review): Captain, harden Kubernetes rollback and lifecycle transactions --- bin/agent-os-crewmate.sh | 198 +++++++++++++++------ bin/agent-os-kubernetes-lease.sh | 88 +++++++--- bin/agent-os-kubernetes.sh | 268 +++++++++++++++++++++++------ tests/agent-os-kubernetes.test.sh | 275 ++++++++++++++++++++++++++++-- 4 files changed, 689 insertions(+), 140 deletions(-) diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index 7ba69c427..e494c24d8 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -51,6 +51,9 @@ LOCK_CLOCK_SKEW_SECONDS=${AGENT_OS_LOCK_CLOCK_SKEW_SECONDS:-5} LOCK_ACQUIRE_SECONDS=${AGENT_OS_LOCK_ACQUIRE_SECONDS:-30} LOCK_REQUEST_CEILING_SECONDS=${AGENT_OS_LOCK_REQUEST_CEILING_SECONDS:-5} RESOURCE_REQUEST_CEILING_SECONDS=${AGENT_OS_RESOURCE_REQUEST_CEILING_SECONDS:-5} +DELETE_OUTCOME= +DELETE_CAPTURED_UID= +DELETE_OBSERVED_UID= for seconds in "$LOCK_DURATION_SECONDS" "$LOCK_CLOCK_SKEW_SECONDS" "$LOCK_ACQUIRE_SECONDS" "$LOCK_REQUEST_CEILING_SECONDS" "$RESOURCE_REQUEST_CEILING_SECONDS"; do case "$seconds" in ''|*[!0-9]*) echo "error: lifecycle Lease timing must use whole seconds" >&2; exit 2 ;; esac @@ -71,12 +74,22 @@ resource_identity() { } pod_record() { - kube get pod "$POD" --ignore-not-found \ + local deadline=${1:-} request_args=() + if [ -n "$deadline" ]; then + request_seconds=$(resource_request_seconds "$deadline") || return 124 + request_args=(--request-timeout="${request_seconds}s") + fi + kube "${request_args[@]}" get pod "$POD" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}{"\t"}{range .spec.volumes[?(@.name=="home")]}{.persistentVolumeClaim.claimName}{end}' } pvc_record() { - kube get pvc "$PVC" --ignore-not-found \ + local deadline=${1:-} request_args=() + if [ -n "$deadline" ]; then + request_seconds=$(resource_request_seconds "$deadline") || return 124 + request_args=(--request-timeout="${request_seconds}s") + fi + kube "${request_args[@]}" get pvc "$PVC" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-state}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-at}{"\t"}{.metadata.annotations.agent-os\.dev/quiesced-operation}{"\t"}{.metadata.annotations.agent-os\.dev/checkpoint-operation}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}' } @@ -183,33 +196,123 @@ finish_lifecycle() { trap finish_lifecycle EXIT trap lock_renewal_failed TERM +resource_request_seconds() { + local deadline=$1 remaining seconds + remaining=$((deadline - $(date -u '+%s'))) + [ "$remaining" -gt 0 ] || return 1 + seconds=$RESOURCE_REQUEST_CEILING_SECONDS + [ "$seconds" -le "$remaining" ] || seconds=$remaining + [ "$seconds" -gt 0 ] || return 1 + printf '%s' "$seconds" +} + +resource_mutation_seconds() { + local deadline=$1 remaining seconds + remaining=$((deadline - $(date -u '+%s') - 1)) + [ "$remaining" -gt 0 ] || return 1 + seconds=$RESOURCE_REQUEST_CEILING_SECONDS + [ "$seconds" -le "$remaining" ] || seconds=$remaining + [ "$seconds" -gt 0 ] || return 1 + printf '%s' "$seconds" +} + +delete_owned_crewmate_resource() { + local kind=$1 name=$2 expected_uid=${3:-} expected_rv=${4:-} expected_operation=${5:-} + local started deadline record identity uid rv path request_seconds current current_uid wait_seconds delete_ok=0 + started=$(date -u '+%s') + deadline=$((started + 180)) + DELETE_OUTCOME=unknown + DELETE_CAPTURED_UID=$expected_uid + DELETE_OBSERVED_UID=unavailable + case "$kind" in + pod) record=$(pod_record "$deadline") || { echo "error: pod/$name observation unavailable before bounded deletion captured uid=${expected_uid:-unknown}" >&2; return 3; } ;; + pvc) record=$(pvc_record "$deadline") || { echo "error: pvc/$name observation unavailable before bounded deletion captured uid=${expected_uid:-unknown}" >&2; return 3; } ;; + *) return 2 ;; + esac + if [ -z "$record" ]; then + DELETE_OUTCOME=absent + DELETE_OBSERVED_UID=absent + echo "confirmed absent: $kind/$name captured uid=${expected_uid:-absent}" >&2 + return 0 + fi + identity=$(printf '%s' "$record" | cut -f1-4) + if { [ "$kind" = pod ] && [ "$identity" != "$EXPECTED_POD" ]; } || \ + { [ "$kind" = pvc ] && [ "$identity" != "$EXPECTED_PVC" ]; }; then + echo "error: $kind/$name ownership changed before deletion; retained" >&2 + return 2 + fi + if [ "$kind" = pod ]; then + [ -z "$expected_operation" ] || [ "$(printf '%s' "$record" | cut -f5)" = "$expected_operation" ] || { + echo "error: pod/$name replacement or ownership mismatch retained; operation identity changed before deletion" >&2 + return 3 + } + uid=$(printf '%s' "$record" | cut -f6) + rv=$(printf '%s' "$record" | cut -f7) + path="/api/v1/namespaces/$NAMESPACE/pods/$name" + else + uid=$(printf '%s' "$record" | cut -f9) + rv=$(printf '%s' "$record" | cut -f10) + path="/api/v1/namespaces/$NAMESPACE/persistentvolumeclaims/$name" + fi + DELETE_CAPTURED_UID=$uid + [ -n "$uid" ] && [ -n "$rv" ] || { echo "error: $kind/$name lacks deletion preconditions" >&2; return 2; } + if [ -n "$expected_uid" ] && [ "$uid" != "$expected_uid" ]; then + DELETE_OUTCOME=replacement-retained + DELETE_CAPTURED_UID=$expected_uid + DELETE_OBSERVED_UID=$uid + echo "error: $kind/$name replacement uid=$uid retained; captured uid=$expected_uid" >&2 + return 3 + fi + [ -z "$expected_rv" ] || [ "$rv" = "$expected_rv" ] || { + DELETE_OUTCOME=original-retained + DELETE_OBSERVED_UID=$uid + echo "error: $kind/$name resourceVersion changed before deletion; retained" >&2 + return 3 + } + request_seconds=$(resource_mutation_seconds "$deadline") || { + echo "error: $kind/$name deletion deadline left no reconciliation reserve; retained uid=$uid" >&2 + return 3 + } + if printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s","resourceVersion":"%s"}}\n' "$uid" "$rv" | \ + kube --request-timeout="${request_seconds}s" delete --raw "$path" -f - >/dev/null; then + delete_ok=1 + fi + if [ "$delete_ok" -eq 1 ]; then + wait_seconds=$(resource_mutation_seconds "$deadline") || wait_seconds=0 + if [ "$wait_seconds" -gt 0 ]; then + kube --request-timeout="${wait_seconds}s" wait --for=delete "$kind/$name" --timeout="${wait_seconds}s" >/dev/null 2>&1 || true + fi + fi + case "$kind" in + pod) current=$(pod_record "$deadline") || { echo "error: pod/$name deletion result unavailable; captured uid=$uid retained-state=unknown" >&2; return 3; } ;; + pvc) current=$(pvc_record "$deadline") || { echo "error: pvc/$name deletion result unavailable; captured uid=$uid retained-state=unknown" >&2; return 3; } ;; + esac + if [ -z "$current" ]; then + DELETE_OUTCOME=absent + DELETE_OBSERVED_UID=absent + echo "confirmed absent: $kind/$name captured uid=$uid" >&2 + return 0 + fi + if [ "$kind" = pod ]; then + current_uid=$(printf '%s' "$current" | cut -f6) + else + current_uid=$(printf '%s' "$current" | cut -f9) + fi + DELETE_OBSERVED_UID=$current_uid + if [ "$current_uid" != "$uid" ]; then + DELETE_OUTCOME=replacement-retained + echo "error: $kind/$name replacement uid=$current_uid retained; captured uid=$uid" >&2 + return 3 + fi + DELETE_OUTCOME=original-retained + echo "error: $kind/$name original uid=$uid remains after bounded deletion; retained" >&2 + return 3 +} + cleanup_new_owned_pod() { - local expected_uid=${1:-} after current identity after_operation after_uid - after=$(pod_record) - if [ -z "$after" ]; then - echo "partial state: crewmate create left no Pod; persistent home retained" >&2 - return - fi - identity=$(printf '%s' "$after" | cut -f1-4) - after_operation=$(printf '%s' "$after" | cut -f5) - after_uid=$(printf '%s' "$after" | cut -f6) - if [ "$identity" != "$EXPECTED_POD" ] || [ "$after_operation" != "$OPERATION_ID" ] || \ - [ -z "$after_uid" ] || { [ -n "$expected_uid" ] && [ "$after_uid" != "$expected_uid" ]; }; then - echo "partial state: replacement or ownership mismatch retained; persistent home retained" >&2 - return - fi - current=$(pod_record) - if [ "$current" != "$after" ]; then - echo "partial state: Pod identity changed during cleanup; no Pod deleted and persistent home retained" >&2 - return - fi - echo "partial state: removing newly created owned Pod '$POD' uid=$after_uid; persistent home retained" >&2 - if ! printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s"}}\n' "$after_uid" | \ - kube --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" delete --raw "/api/v1/namespaces/$NAMESPACE/pods/$POD" -f -; then - echo "partial state: UID-precondition rejected; replacement or ownership mismatch retained" >&2 - return - fi - kube wait --for=delete "pod/$POD" --timeout=180s + local expected_uid=${1:-} + echo "partial state: reconciling newly created owned Pod '$POD' uid=${expected_uid:-observed}; persistent home retained" >&2 + delete_owned_crewmate_resource pod "$POD" "$expected_uid" '' "$OPERATION_ID" || true } create_and_wait() { @@ -329,24 +432,7 @@ preflight_existing_home() { } stop_owned_pod() { - local pod uid rv - pod=$(pod_record) - if [ -z "$pod" ]; then - return - fi - if [ "$(printf '%s' "$pod" | cut -f1-4)" != "$EXPECTED_POD" ]; then - echo "error: pod '$POD' does not have the exact crewmate installation identity" >&2 - exit 2 - fi - uid=$(printf '%s' "$pod" | cut -f6) - rv=$(printf '%s' "$pod" | cut -f7) - if [ -z "$uid" ] || [ -z "$rv" ]; then - echo "error: Pod deletion requires exact UID and resourceVersion evidence" >&2 - exit 2 - fi - printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s"}}\n' "$uid" | \ - kube --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" delete --raw "/api/v1/namespaces/$NAMESPACE/pods/$POD" -f - - kube wait --for=delete "pod/$POD" --timeout=180s + delete_owned_crewmate_resource pod "$POD" } invalidate_checkpoint_evidence() { @@ -397,10 +483,10 @@ quiesce_owned_home() { } record_purge() { - local phase=$1 checkpoint_at=$2 timestamp + local phase=$1 checkpoint_at=$2 outcome=${3:-requested} captured_uid=${4:-unknown} observed_uid=${5:-unknown} timestamp timestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ') - printf '%s\t%s\tnamespace=%s\tcrewmate=%s\tpod=%s\tpvc=%s\tcheckpoint-at=%s\n' \ - "$timestamp" "$phase" "$NAMESPACE" "$ID" "$POD" "$PVC" "$checkpoint_at" >&3 + printf '%s\t%s\tnamespace=%s\tcrewmate=%s\tpod=%s\tpvc=%s\tcheckpoint-at=%s\toutcome=%s\tcaptured-uid=%s\tobserved-uid=%s\n' \ + "$timestamp" "$phase" "$NAMESPACE" "$ID" "$POD" "$PVC" "$checkpoint_at" "$outcome" "$captured_uid" "$observed_uid" >&3 } case "$COMMAND" in @@ -521,12 +607,14 @@ case "$COMMAND" in fi mkdir -p "$(dirname "$evidence_file")" exec 3>>"$evidence_file" - record_purge purge-requested "$checkpoint_at" - printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s","resourceVersion":"%s"}}\n' \ - "$pvc_uid" "$pvc_rv" | \ - kube --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" delete --raw "/api/v1/namespaces/$NAMESPACE/persistentvolumeclaims/$PVC" -f - - kube wait --for=delete "pvc/$PVC" --timeout=180s - record_purge purge-complete "$checkpoint_at" + record_purge purge-requested "$checkpoint_at" requested "$pvc_uid" pending + if delete_owned_crewmate_resource pvc "$PVC" "$pvc_uid" "$pvc_rv"; then + record_purge purge-complete "$checkpoint_at" "$DELETE_OUTCOME" "$DELETE_CAPTURED_UID" "$DELETE_OBSERVED_UID" + else + purge_status=$? + record_purge "purge-incomplete-$DELETE_OUTCOME" "$checkpoint_at" "$DELETE_OUTCOME" "$DELETE_CAPTURED_UID" "$DELETE_OBSERVED_UID" + exit "$purge_status" + fi ;; delete) echo "error: delete is ambiguous; use stop to preserve the home or purge --yes" >&2 diff --git a/bin/agent-os-kubernetes-lease.sh b/bin/agent-os-kubernetes-lease.sh index c9508c5a1..ec349baf8 100644 --- a/bin/agent-os-kubernetes-lease.sh +++ b/bin/agent-os-kubernetes-lease.sh @@ -35,6 +35,27 @@ lock_kube() { kube --request-timeout="${seconds}s" "$@" } +lock_mutation_seconds() { + local deadline=$1 now remaining validity seconds + now=$(date -u '+%s') + remaining=$((deadline - now - 1)) + [ "$remaining" -gt 0 ] || return 1 + validity=$((LOCK_DURATION_SECONDS / 3)) + [ "$validity" -gt 0 ] || validity=1 + seconds=$LOCK_REQUEST_CEILING_SECONDS + [ "$seconds" -le "$validity" ] || seconds=$validity + [ "$seconds" -le "$remaining" ] || seconds=$remaining + [ "$seconds" -gt 0 ] || return 1 + printf '%s' "$seconds" +} + +lock_kube_mutation() { + local deadline=$1 seconds + shift + seconds=$(lock_mutation_seconds "$deadline") || return 124 + kube --request-timeout="${seconds}s" "$@" +} + rfc3339_epoch() { local value=${1%%.*} value=${value%Z}Z @@ -60,26 +81,41 @@ verify_lock_record() { [ "$identity" = "$EXPECTED_LOCK" ] && [ "$holder" = "$OPERATION_ID" ] && [ -n "$uid" ] } +lock_record_valid_until() { + local record=$1 renewed duration renewed_epoch margin + renewed=$(printf '%s' "$record" | cut -f7) + duration=$(printf '%s' "$record" | cut -f8) + case "$duration" in ''|*[!0-9]*) return 1 ;; esac + renewed_epoch=$(rfc3339_epoch "$renewed") || return 1 + margin=$LOCK_CLOCK_SKEW_SECONDS + [ "$margin" -le "$((duration / 3))" ] || margin=$((duration / 3)) + [ "$margin" -gt 0 ] || margin=1 + printf '%s' "$((renewed_epoch + duration - margin))" +} + renew_lock_once() { - local record acquired uid rv now current deadline - deadline=$(lock_default_deadline) - record=$(lock_record "$deadline") || return 1 + local record acquired uid rv now current initial_deadline deadline + initial_deadline=$(lock_default_deadline) + if [ -n "${LOCK_VALID_UNTIL:-}" ] && [ "$LOCK_VALID_UNTIL" -lt "$initial_deadline" ]; then + initial_deadline=$LOCK_VALID_UNTIL + fi + record=$(lock_record "$initial_deadline") || return 1 verify_lock_record "$record" || return 1 acquired=$(printf '%s' "$record" | cut -f6) uid=$(printf '%s' "$record" | cut -f9) rv=$(printf '%s' "$record" | cut -f10) [ -n "$acquired" ] && [ -n "$rv" ] || return 1 + deadline=$(lock_record_valid_until "$record") || return 1 + [ "$deadline" -gt "$(date -u '+%s')" ] || return 1 now=$(date -u '+%Y-%m-%dT%H:%M:%SZ') - if ! render_lock "$acquired" "$now" "$uid" "$rv" | lock_kube "$deadline" replace -f - >/dev/null; then - current=$(lock_record "$deadline") || return 1 - verify_lock_record "$current" || return 1 - [ "$(printf '%s' "$current" | cut -f9)" = "$uid" ] || return 1 - [ "$(printf '%s' "$current" | cut -f10)" != "$rv" ] || return 1 - fi + render_lock "$acquired" "$now" "$uid" "$rv" | lock_kube_mutation "$deadline" replace -f - >/dev/null || true current=$(lock_record "$deadline") || return 1 verify_lock_record "$current" || return 1 [ "$(printf '%s' "$current" | cut -f9)" = "$uid" ] || return 1 + [ "$(printf '%s' "$current" | cut -f10)" != "$rv" ] || return 1 + [ "$(printf '%s' "$current" | cut -f7)" = "$now" ] || return 1 LOCK_RV=$(printf '%s' "$current" | cut -f10) + LOCK_VALID_UNTIL=$(lock_record_valid_until "$current") || return 1 } start_lock_renewal() { @@ -126,7 +162,7 @@ release_lock() { return 1 fi if ! printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s","resourceVersion":"%s"}}\n' "$uid" "$rv" | \ - lock_kube "$deadline" delete --raw "/apis/coordination.k8s.io/v1/namespaces/$LOCK_NAMESPACE/leases/$LOCK" -f - >/dev/null; then + lock_kube_mutation "$deadline" delete --raw "/apis/coordination.k8s.io/v1/namespaces/$LOCK_NAMESPACE/leases/$LOCK" -f - >/dev/null; then after=$(lock_record "$deadline") || { echo "error: lifecycle Lease '$LOCK' release result is ambiguous; retained" >&2 return 1 @@ -154,7 +190,7 @@ release_lock() { } acquire_lock() { - local record identity holder now deadline acquired uid rv current + local record identity holder now deadline acquired uid rv current mutation_ok now=$(date -u '+%Y-%m-%dT%H:%M:%SZ') deadline=$(($(date -u '+%s') + LOCK_ACQUIRE_SECONDS)) holder=unknown @@ -163,9 +199,8 @@ acquire_lock() { echo "error: lifecycle operation '$holder' still holds Lease '$LOCK' after ${LOCK_ACQUIRE_SECONDS}s" >&2 exit 3 fi - if render_lock "$now" "$now" | lock_kube "$deadline" create -f - >/dev/null; then - break - fi + mutation_ok=0 + render_lock "$now" "$now" | lock_kube_mutation "$deadline" create -f - >/dev/null && mutation_ok=1 record=$(lock_record "$deadline") || { echo "error: lifecycle Lease '$LOCK' create result could not be reconciled before the acquisition deadline" >&2 exit 3 @@ -176,9 +211,14 @@ acquire_lock() { echo "error: lifecycle Lease '$LOCK' is absent or has foreign ownership after create conflict" >&2 exit 2 fi - if verify_lock_record "$record"; then + if verify_lock_record "$record" && [ "$(printf '%s' "$record" | cut -f6)" = "$now" ] && \ + [ "$(printf '%s' "$record" | cut -f7)" = "$now" ]; then break fi + [ "$mutation_ok" -eq 0 ] || { + echo "error: lifecycle Lease '$LOCK' create acknowledgement did not verify exact timestamps" >&2 + exit 3 + } if lease_is_expired "$record"; then uid=$(printf '%s' "$record" | cut -f9) rv=$(printf '%s' "$record" | cut -f10) @@ -187,16 +227,21 @@ acquire_lock() { exit 2 fi now=$(date -u '+%Y-%m-%dT%H:%M:%SZ') - if render_lock "$now" "$now" "$uid" "$rv" | lock_kube "$deadline" replace -f - >/dev/null; then - break - fi + mutation_ok=0 + render_lock "$now" "$now" "$uid" "$rv" | lock_kube_mutation "$deadline" replace -f - >/dev/null && mutation_ok=1 record=$(lock_record "$deadline") || { echo "error: lifecycle Lease '$LOCK' takeover result could not be reconciled before the acquisition deadline" >&2 exit 3 } - if verify_lock_record "$record" && [ "$(printf '%s' "$record" | cut -f9)" = "$uid" ]; then + if verify_lock_record "$record" && [ "$(printf '%s' "$record" | cut -f9)" = "$uid" ] && \ + [ "$(printf '%s' "$record" | cut -f10)" != "$rv" ] && \ + [ "$(printf '%s' "$record" | cut -f7)" = "$now" ]; then break fi + [ "$mutation_ok" -eq 0 ] || { + echo "error: lifecycle Lease '$LOCK' takeover acknowledgement did not verify exact renewal evidence" >&2 + exit 3 + } fi if [ "$(date -u '+%s')" -ge "$deadline" ]; then echo "error: lifecycle operation '$holder' still holds Lease '$LOCK' after ${LOCK_ACQUIRE_SECONDS}s" >&2 @@ -214,6 +259,11 @@ acquire_lock() { acquired=$(printf '%s' "$record" | cut -f6) LOCK_UID=$(printf '%s' "$record" | cut -f9) LOCK_RV=$(printf '%s' "$record" | cut -f10) + LOCK_VALID_UNTIL=$(lock_record_valid_until "$record") || { + LOCK_UID= + echo "error: lifecycle Lease '$LOCK' lacks a valid renewal deadline" >&2 + exit 2 + } if [ -z "$acquired" ] || [ -z "$LOCK_RV" ]; then LOCK_UID= echo "error: lifecycle Lease '$LOCK' lacks complete renewal evidence" >&2 diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh index 8a632a6af..db00c29d8 100755 --- a/bin/agent-os-kubernetes.sh +++ b/bin/agent-os-kubernetes.sh @@ -307,12 +307,12 @@ reconcile_deleted_resource() { local deadline=${10} request_seconds current current_uid current_rv elapsed request_seconds=$(operation_request_seconds "$deadline") || { elapsed=$(($(date -u '+%s') - started)) - bounded_delete_failure "$kind/$name" "$phase" "$class" "$timeout" "$elapsed" "$uid" + bounded_delete_failure "$kind/$name" "$phase" "$class" "$timeout" "$elapsed" "$uid" "$deadline" return $? } if ! current=$(live_resource_record "$scope" "$kind" "$name" "${request_seconds}s"); then elapsed=$(($(date -u '+%s') - started)) - bounded_delete_failure "$kind/$name" reconcile transport "$timeout" "$elapsed" "$uid" + bounded_delete_failure "$kind/$name" reconcile transport "$timeout" "$elapsed" "$uid" "$deadline" return $? fi if [ -z "$current" ]; then @@ -324,11 +324,11 @@ reconcile_deleted_resource() { elapsed=$(($(date -u '+%s') - started)) if [ "$current_uid" != "$uid" ]; then echo "error: $kind/$name replacement uid=${current_uid:-unknown} retained after ambiguous delete of captured uid=$uid resourceVersion=$rv" >&2 - bounded_delete_failure "$kind/$name" reconcile replacement "$timeout" "$elapsed" "$uid" + bounded_delete_failure "$kind/$name" reconcile replacement "$timeout" "$elapsed" "$uid" "$deadline" return $? fi echo "error: $kind/$name captured uid=$uid remains at resourceVersion=${current_rv:-unknown} after delete-$phase failure=$class" >&2 - bounded_delete_failure "$kind/$name" "$phase" "$class" "$timeout" "$elapsed" "$uid" + bounded_delete_failure "$kind/$name" "$phase" "$class" "$timeout" "$elapsed" "$uid" "$deadline" } resource_api_path() { @@ -349,7 +349,16 @@ resource_api_path() { delete_owned_resource() { local scope=$1 kind=$2 name=$3 timeout=$4 record expected uid rv path output started deadline class request_seconds remaining reserve wait_budget - record=$(live_resource_record "$scope" "$kind" "$name") + started=$(date -u '+%s') + deadline=$((started + timeout)) + request_seconds=$(operation_request_seconds "$deadline") || { + bounded_delete_failure "$kind/$name" observe timeout "$timeout" 0 unknown "$deadline" + return $? + } + if ! record=$(live_resource_record "$scope" "$kind" "$name" "${request_seconds}s"); then + bounded_delete_failure "$kind/$name" observe transport "$timeout" 0 unknown "$deadline" + return $? + fi [ -n "$record" ] || return 0 expected="$name"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" if [ "$(printf '%s' "$record" | cut -f1-3)" != "$expected" ]; then @@ -360,10 +369,8 @@ delete_owned_resource() { rv=$(printf '%s' "$record" | cut -f5) [ -n "$uid" ] && [ -n "$rv" ] || { echo "error: $kind '$name' lacks deletion preconditions" >&2; exit 2; } path=$(resource_api_path "$scope" "$kind" "$name") - started=$(date -u '+%s') - deadline=$((started + timeout)) request_seconds=$(operation_request_seconds "$deadline") || { - bounded_delete_failure "$kind/$name" request timeout "$timeout" 0 "$uid" + bounded_delete_failure "$kind/$name" request timeout "$timeout" 0 "$uid" "$deadline" return $? } if ! output=$(printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s","resourceVersion":"%s"}}\n' "$uid" "$rv" | \ @@ -454,22 +461,23 @@ delete_namespace_rbac() { } resource_observation() { - local scope=$1 kind=$2 name=$3 + local scope=$1 kind=$2 name=$3 request_timeout=${4:-} request_args=() + [ -z "$request_timeout" ] || request_args=(--request-timeout="$request_timeout") if [ "$kind" = StatefulSet ]; then - "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get "$kind" "$name" --ignore-not-found \ + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" "${request_args[@]}" get "$kind" "$name" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.uid}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}{"\t"}{"\t"}{.metadata.finalizers}{"\t"}{.spec.replicas}{"\t"}{.status.currentReplicas}{"\t"}{.status.readyReplicas}{"\t"}{.status.updatedReplicas}{"\t"}{.status.availableReplicas}{"\t"}{.status.currentRevision}{"\t"}{.status.updateRevision}{"\t"}{.metadata.generation}{"\t"}{.status.observedGeneration}' return fi if [ "$kind" = Pod ]; then - "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get "$kind" "$name" --ignore-not-found \ + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" "${request_args[@]}" get "$kind" "$name" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.uid}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}{"\t"}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\t"}{.metadata.finalizers}{"\t"}{range .status.containerStatuses[*]}{.name}{"="}{.state.waiting.reason}{.state.terminated.reason}{","}{end}' return fi if [ "$scope" = cluster ]; then - "$KUBECTL" --context "$CONTEXT" get "$kind" "$name" --ignore-not-found \ + "$KUBECTL" --context "$CONTEXT" "${request_args[@]}" get "$kind" "$name" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.uid}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}{"\t"}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\t"}{.metadata.finalizers}' else - "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get "$kind" "$name" --ignore-not-found \ + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" "${request_args[@]}" get "$kind" "$name" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.metadata.uid}{"\t"}{.metadata.labels.agent-os\.dev/operation-id}{"\t"}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\t"}{.metadata.finalizers}' fi } @@ -510,16 +518,23 @@ report_partial_observation() { } report_rollback_failure() { - local target_name=$1 target_revision=$2 state references lease deadline - echo "incomplete: rollback target=$target_name revision=$target_revision did not complete" >&2 - if state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get statefulset agent-os-firstmate -o json 2>/dev/null) && \ + local target_name=$1 target_revision=$2 target_uid=$3 target_digest=$4 checkpoint_operation=$5 checkpoint_name=$6 checkpoint_uid=$7 + local state references lease deadline record + echo "incomplete: rollback target=$target_name revision=$target_revision target-uid=$target_uid target-digest=$target_digest checkpoint-operation=$checkpoint_operation checkpoint-target=$checkpoint_name checkpoint-uid=$checkpoint_uid did not complete" >&2 + deadline=$(lock_default_deadline) + if state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get statefulset agent-os-firstmate -o json 2>/dev/null) && \ references=$(printf '%s' "$state" | jq -er '[.status.currentRevision, .status.updateRevision] | select(all(.[]; type == "string" and length > 0)) | @tsv' 2>/dev/null); then echo "rollback observed: current-revision=$(printf '%s' "$references" | cut -f1) update-revision=$(printf '%s' "$references" | cut -f2)" >&2 else echo "rollback observed: StatefulSet revision evidence unavailable" >&2 fi - report_partial_observation Pod agent-os-firstmate-0 namespaced - deadline=$(lock_default_deadline) + if ! record=$(resource_observation namespaced Pod agent-os-firstmate-0 "${RESOURCE_REQUEST_CEILING_SECONDS}s" 2>/dev/null); then + echo "partial apply: Pod/agent-os-firstmate-0 observation=unavailable expected-operation=$OPERATION_ID" >&2 + elif [ -n "$record" ]; then + echo "partial apply: Pod/agent-os-firstmate-0 uid=$(printf '%s' "$record" | cut -f4) operation=$(printf '%s' "$record" | cut -f5) ready=$(printf '%s' "$record" | cut -f6) ownership=$(printf '%s' "$record" | cut -f2) installation=$(printf '%s' "$record" | cut -f3) finalizers=$(printf '%s' "$record" | cut -f7)" >&2 + else + echo "partial apply: Pod/agent-os-firstmate-0 observed=absent expected-operation=$OPERATION_ID" >&2 + fi if lease=$(lock_record "$deadline" 2>/dev/null) && [ -n "$lease" ]; then echo "rollback retained: lifecycle-lease=$LOCK uid=$(printf '%s' "$lease" | cut -f9) holder=$(printf '%s' "$lease" | cut -f5)" >&2 else @@ -527,10 +542,48 @@ report_rollback_failure() { fi printf 'safe recovery: %q --context %q -n %q rollout status statefulset/agent-os-firstmate --timeout=180s\n' \ "$KUBECTL" "$CONTEXT" "$NAMESPACE" >&2 - echo "safe recovery condition: continue only while updateRevision remains '$target_name'; do not invoke rollback again while revisions differ" >&2 + echo "safe recovery condition: rerun rollback only while the persisted target digest remains '$target_digest'" >&2 return 3 } +sha256_text() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum | awk '{print $1}' + else + shasum -a 256 | awk '{print $1}' + fi +} + +template_digest() { + jq -cS . | sha256_text +} + +owned_revision_by_digest() { + local revisions=$1 workload_uid=$2 digest=$3 preferred_uid=${4:-} item item_digest candidate='' candidate_revision=-1 item_revision item_uid + while IFS= read -r item; do + [ -n "$item" ] || continue + item_digest=$(printf '%s' "$item" | jq -cS '.data.spec.template' | template_digest) + [ "$item_digest" = "$digest" ] || continue + item_uid=$(printf '%s' "$item" | jq -r '.metadata.uid // empty') + if [ -n "$preferred_uid" ] && [ "$item_uid" = "$preferred_uid" ]; then + printf '%s' "$item" + return 0 + fi + item_revision=$(printf '%s' "$item" | jq -r '.revision // -1') + if [ "$item_revision" -gt "$candidate_revision" ] 2>/dev/null; then + candidate=$item + candidate_revision=$item_revision + fi + done < <(printf '%s' "$revisions" | jq -c --arg uid "$workload_uid" ' + .items[] + | select(any(.metadata.ownerReferences[]?; + .apiVersion == "apps/v1" and .kind == "StatefulSet" and + .name == "agent-os-firstmate" and .uid == $uid and .controller == true)) + ') + [ -n "$candidate" ] || return 1 + printf '%s' "$candidate" +} + partial_install_is_applicable() { local file kind name identity expected namespace if ! namespace=$(namespace_name 2>/dev/null); then @@ -784,8 +837,16 @@ uninstall_command() { } report_retained_observation() { - local kind=$1 name=$2 scope=$3 record managed installation uid operation ready finalizers identity expected details - if ! record=$(resource_observation "$scope" "$kind" "$name"); then + local kind=$1 name=$2 scope=$3 deadline=${4:-} record managed installation uid operation ready finalizers identity expected details request_seconds + if [ -n "$deadline" ]; then + request_seconds=$(operation_request_seconds "$deadline") || { + echo "retained-unverified: $kind/$name evidence deadline exhausted; no further deletion attempted" >&2 + return 0 + } + else + request_seconds=$RESOURCE_REQUEST_CEILING_SECONDS + fi + if ! record=$(resource_observation "$scope" "$kind" "$name" "${request_seconds}s"); then echo "retained-unverified: $kind/$name could not be inspected; no further deletion attempted" >&2 return 0 fi @@ -814,16 +875,16 @@ report_retained_observation() { } report_retained_resources() { - local failed_target=$1 phase=$2 class=$3 timeout=$4 elapsed=$5 uid=$6 file kind name scope + local failed_target=$1 phase=$2 class=$3 timeout=$4 elapsed=$5 uid=$6 deadline=${7:-} file kind name scope echo "failed-target: $failed_target uid=$uid delete-${phase}-failure=$class timeout=${timeout}s elapsed=${elapsed}s" >&2 kind=${failed_target%%/*} name=${failed_target#*/} scope=namespaced [ "$kind" != Namespace ] || scope=cluster - report_retained_observation "$kind" "$name" "$scope" - report_retained_observation Pod agent-os-firstmate-0 namespaced - report_retained_observation Role agent-os-firstmate-runtime namespaced - report_retained_observation RoleBinding agent-os-firstmate-runtime namespaced + report_retained_observation "$kind" "$name" "$scope" "$deadline" + report_retained_observation Pod agent-os-firstmate-0 namespaced "$deadline" + report_retained_observation Role agent-os-firstmate-runtime namespaced "$deadline" + report_retained_observation RoleBinding agent-os-firstmate-runtime namespaced "$deadline" while IFS= read -r file; do [ -n "$file" ] || continue kind=$(rendered_resource_field "$file" kind) @@ -836,7 +897,7 @@ report_retained_resources() { ;; *) scope=namespaced ;; esac - report_retained_observation "$kind" "$name" "$scope" + report_retained_observation "$kind" "$name" "$scope" "$deadline" done < <(find "$OUT" -type f -name '*.yaml' -print) echo "safe retry: $(uninstall_command)" >&2 echo "cluster cleanup if required: $(cleanup_command)" >&2 @@ -856,11 +917,11 @@ delete_failure_class() { bounded_delete_failure() { local target=$1 phase=${2:-wait} class=${3:-timeout} timeout=${4:-180} - local elapsed=${5:-$timeout} uid=${6:-unknown} prefix kind name scope + local elapsed=${5:-$timeout} uid=${6:-unknown} deadline=${7:-} prefix kind name scope [ "$COMMAND" = uninstall ] && prefix=incomplete || prefix=error echo "$prefix: delete-${phase}-failure=$class target=$target uid=$uid timeout=${timeout}s elapsed=${elapsed}s" >&2 if [ "$COMMAND" = uninstall ]; then - report_retained_resources "$target" "$phase" "$class" "$timeout" "$elapsed" "$uid" + report_retained_resources "$target" "$phase" "$class" "$timeout" "$elapsed" "$uid" "$deadline" return 3 fi kind=${target%%/*} @@ -869,7 +930,7 @@ bounded_delete_failure() { case "$kind" in ClusterRoleBinding|Namespace) scope=cluster ;; esac - report_retained_observation "$kind" "$name" "$scope" + report_retained_observation "$kind" "$name" "$scope" "$deadline" if [ "$COMMAND" = cleanup-cluster-rbac ]; then echo "safe retry: $(cleanup_command)" >&2 fi @@ -1080,15 +1141,53 @@ case "$COMMAND" in | select(.metadata.uid == $uid and .metadata.resourceVersion == $rv) | select(.metadata.labels["app.kubernetes.io/managed-by"] == "agent-os") | select(.metadata.annotations["agent-os.dev/installation-id"] == $installation) - | [.status.currentRevision, .status.updateRevision] - | select(all(.[]; type == "string" and length > 0)) + | [.status.currentRevision, .status.updateRevision, + (.metadata.annotations["agent-os.dev/rollback-operation"] // ""), + (.metadata.annotations["agent-os.dev/rollback-target-name"] // ""), + (.metadata.annotations["agent-os.dev/rollback-target-uid"] // ""), + (.metadata.annotations["agent-os.dev/rollback-target-digest"] // "")] + | select((.[0:2]) | all(.[]; type == "string" and length > 0)) | @tsv ') || { echo "error: StatefulSet changed before rollback revision resolution" >&2; exit 3; } rollback_current_revision=$(printf '%s' "$rollback_references" | cut -f1) rollback_update_revision=$(printf '%s' "$rollback_references" | cut -f2) - rollback_selection=$(printf '%s' "$rollback_revisions" | jq -ce \ - --arg current "$rollback_current_revision" --arg update "$rollback_update_revision" \ - --arg uid "$rollback_uid" --arg rv "$rollback_rv" ' + rollback_checkpoint_operation=$(printf '%s' "$rollback_references" | cut -f3) + rollback_checkpoint_name=$(printf '%s' "$rollback_references" | cut -f4) + rollback_checkpoint_uid=$(printf '%s' "$rollback_references" | cut -f5) + rollback_checkpoint_digest=$(printf '%s' "$rollback_references" | cut -f6) + if [ -n "$rollback_checkpoint_operation$rollback_checkpoint_name$rollback_checkpoint_uid$rollback_checkpoint_digest" ]; then + if [ -z "$rollback_checkpoint_operation" ] || [ -z "$rollback_checkpoint_name" ] || \ + [ -z "$rollback_checkpoint_uid" ] || ! [[ "$rollback_checkpoint_digest" =~ ^[0-9a-f]{64}$ ]]; then + echo "error: rollback checkpoint is incomplete or malformed; retained" >&2 + exit 3 + fi + rollback_target=$(printf '%s' "$rollback_revisions" | jq -ce --arg name "$rollback_update_revision" --arg uid "$rollback_uid" ' + [.items[] | select(.metadata.name == $name) | select(any(.metadata.ownerReferences[]?; + .apiVersion == "apps/v1" and .kind == "StatefulSet" and .name == "agent-os-firstmate" and .uid == $uid and .controller == true))] + | select(length == 1) | .[0]' 2>/dev/null) || rollback_target='' + if [ -n "$rollback_target" ]; then + rollback_update_digest=$(printf '%s' "$rollback_target" | jq -cS '.data.spec.template' | template_digest) + [ "$rollback_update_digest" = "$rollback_checkpoint_digest" ] || rollback_target='' + fi + if [ -z "$rollback_target" ]; then + rollback_target=$(owned_revision_by_digest "$rollback_revisions" "$rollback_uid" \ + "$rollback_checkpoint_digest" "$rollback_checkpoint_uid") || { + echo "error: rollback checkpoint target content is unavailable; retained" >&2 + exit 3 + } + fi + rollback_target_name=$(printf '%s' "$rollback_target" | jq -r '.metadata.name') + rollback_target_uid=$(printf '%s' "$rollback_target" | jq -r '.metadata.uid') + rollback_target_revision=$(printf '%s' "$rollback_target" | jq -r '.revision') + rollback_target_digest=$rollback_checkpoint_digest + rollback_mode='patch' + if [ "$rollback_update_revision" = "$rollback_target_name" ]; then + rollback_mode=resume + fi + else + rollback_selection=$(printf '%s' "$rollback_revisions" | jq -ce \ + --arg current "$rollback_current_revision" --arg update "$rollback_update_revision" \ + --arg uid "$rollback_uid" ' def owned: any(.metadata.ownerReferences[]?; .apiVersion == "apps/v1" and .kind == "StatefulSet" and @@ -1102,22 +1201,48 @@ case "$COMMAND" in | $update_items[0] as $update_revision | select($current_revision.revision | type == "number") | select($update_revision.revision | type == "number") - | (if $current != $update and $update_revision.revision < $current_revision.revision then - {mode:"resume", target:$update_revision} - elif $current != $update then - {mode:"patch", target:$current_revision} + | (if $current != $update then + {target:$current_revision} else - {mode:"patch", target:($owned | map(select((.revision | type) == "number" and .revision < $update_revision.revision)) | sort_by(.revision) | last)} + {target:($owned | map(select((.revision | type) == "number" and .revision < $update_revision.revision)) | sort_by(.revision) | last)} end) as $selection - | select($selection.target != null and ($selection.target.data | type) == "object") - | {mode:$selection.mode, targetName:$selection.target.metadata.name, targetRevision:$selection.target.revision, - patch:($selection.target.data * {metadata:{uid:$uid,resourceVersion:$rv}})} + | select($selection.target != null and ($selection.target.data.spec.template | type) == "object") + | $selection.target ') || { echo "error: no exact-owned previous ControllerRevision is available for rollback" >&2; exit 2; } - rollback_mode=$(printf '%s' "$rollback_selection" | jq -r '.mode') - rollback_target_name=$(printf '%s' "$rollback_selection" | jq -r '.targetName') - rollback_target_revision=$(printf '%s' "$rollback_selection" | jq -r '.targetRevision') - rollback_patch=$(printf '%s' "$rollback_selection" | jq -c '.patch') + rollback_target_name=$(printf '%s' "$rollback_selection" | jq -r '.metadata.name') + rollback_target_uid=$(printf '%s' "$rollback_selection" | jq -r '.metadata.uid') + rollback_target_revision=$(printf '%s' "$rollback_selection" | jq -r '.revision') + rollback_target_digest=$(printf '%s' "$rollback_selection" | jq -cS '.data.spec.template' | template_digest) + rollback_target=$rollback_selection + rollback_checkpoint_operation=$OPERATION_ID + rollback_checkpoint_name=$rollback_target_name + rollback_checkpoint_uid=$rollback_target_uid + rollback_checkpoint_patch=$(jq -cn \ + --arg uid "$rollback_uid" --arg rv "$rollback_rv" --arg operation "$rollback_checkpoint_operation" \ + --arg name "$rollback_target_name" --arg targetUid "$rollback_target_uid" --arg digest "$rollback_target_digest" \ + '{metadata:{uid:$uid,resourceVersion:$rv,annotations:{"agent-os.dev/rollback-operation":$operation,"agent-os.dev/rollback-target-name":$name,"agent-os.dev/rollback-target-uid":$targetUid,"agent-os.dev/rollback-target-digest":$digest}}}') + if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch StatefulSet agent-os-firstmate \ + --type=merge -p "$rollback_checkpoint_patch" >/dev/null; then + echo "error: rollback checkpoint CAS conflicted; no template mutation attempted" >&2 + exit 3 + fi + rollback_mode='patch' + rollback_state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ + --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get statefulset agent-os-firstmate -o json) + rollback_rv=$(printf '%s' "$rollback_state" | jq -er \ + --arg uid "$rollback_uid" --arg operation "$rollback_checkpoint_operation" \ + --arg name "$rollback_checkpoint_name" --arg targetUid "$rollback_checkpoint_uid" \ + --arg digest "$rollback_target_digest" ' + select(.metadata.uid == $uid) + | select(.metadata.annotations["agent-os.dev/rollback-operation"] == $operation) + | select(.metadata.annotations["agent-os.dev/rollback-target-name"] == $name) + | select(.metadata.annotations["agent-os.dev/rollback-target-uid"] == $targetUid) + | select(.metadata.annotations["agent-os.dev/rollback-target-digest"] == $digest) + | .metadata.resourceVersion') || { echo "error: rollback checkpoint did not persist exactly" >&2; exit 3; } + fi if [ "$rollback_mode" = patch ]; then + rollback_patch=$(printf '%s' "$rollback_target" | jq -c --arg uid "$rollback_uid" --arg rv "$rollback_rv" \ + '.data * {metadata:{uid:$uid,resourceVersion:$rv}}') if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch StatefulSet agent-os-firstmate \ --type=strategic -p "$rollback_patch" >/dev/null; then echo "error: StatefulSet rollback CAS conflicted; no rollout-undo fallback attempted" >&2 @@ -1131,7 +1256,54 @@ case "$COMMAND" in fi fi if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s; then - report_rollback_failure "$rollback_target_name" "$rollback_target_revision" + report_rollback_failure "$rollback_target_name" "$rollback_target_revision" "$rollback_target_uid" "$rollback_target_digest" \ + "$rollback_checkpoint_operation" "$rollback_checkpoint_name" "$rollback_checkpoint_uid" + exit 3 + fi + rollback_verify_state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ + --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get statefulset agent-os-firstmate -o json) || { + echo "error: rollback verification StatefulSet evidence unavailable; checkpoint retained target-digest=$rollback_target_digest" >&2 + exit 3 + } + rollback_verify_refs=$(printf '%s' "$rollback_verify_state" | jq -er \ + --arg uid "$rollback_uid" --arg installation "$INSTALLATION_ID" \ + --arg operation "$rollback_checkpoint_operation" --arg name "$rollback_checkpoint_name" \ + --arg targetUid "$rollback_checkpoint_uid" --arg digest "$rollback_target_digest" ' + select(.metadata.uid == $uid) + | select(.metadata.labels["app.kubernetes.io/managed-by"] == "agent-os") + | select(.metadata.annotations["agent-os.dev/installation-id"] == $installation) + | select(.metadata.annotations["agent-os.dev/rollback-operation"] == $operation) + | select(.metadata.annotations["agent-os.dev/rollback-target-name"] == $name) + | select(.metadata.annotations["agent-os.dev/rollback-target-uid"] == $targetUid) + | select(.metadata.annotations["agent-os.dev/rollback-target-digest"] == $digest) + | [.status.currentRevision,.status.updateRevision,.metadata.resourceVersion] | @tsv') || { + echo "error: rollback verification mismatch: StatefulSet identity or checkpoint changed; retained target-digest=$rollback_target_digest" >&2 + exit 3 + } + rollback_revisions=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ + --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get controllerrevisions.apps -o json) || { + echo "error: rollback verification revision evidence unavailable; checkpoint retained target-digest=$rollback_target_digest" >&2 + exit 3 + } + for rollback_verify_name in "$(printf '%s' "$rollback_verify_refs" | cut -f1)" "$(printf '%s' "$rollback_verify_refs" | cut -f2)"; do + rollback_verify_revision=$(printf '%s' "$rollback_revisions" | jq -ce --arg name "$rollback_verify_name" --arg uid "$rollback_uid" ' + [.items[] | select(.metadata.name == $name) | select(any(.metadata.ownerReferences[]?; + .apiVersion == "apps/v1" and .kind == "StatefulSet" and .name == "agent-os-firstmate" and .uid == $uid and .controller == true))] + | select(length == 1) | .[0]') || { + echo "error: rollback verification mismatch: revision '$rollback_verify_name' is unavailable or foreign; retained target-digest=$rollback_target_digest" >&2 + exit 3 + } + rollback_verify_digest=$(printf '%s' "$rollback_verify_revision" | jq -cS '.data.spec.template' | template_digest) + if [ "$rollback_verify_digest" != "$rollback_target_digest" ]; then + echo "error: rollback verification mismatch: revision '$rollback_verify_name' content differs; retained target-digest=$rollback_target_digest" >&2 + exit 3 + fi + done + rollback_clear_patch=$(jq -cn --arg uid "$rollback_uid" --arg rv "$(printf '%s' "$rollback_verify_refs" | cut -f3)" \ + '{metadata:{uid:$uid,resourceVersion:$rv,annotations:{"agent-os.dev/rollback-operation":null,"agent-os.dev/rollback-target-name":null,"agent-os.dev/rollback-target-uid":null,"agent-os.dev/rollback-target-digest":null}}}') + if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch StatefulSet agent-os-firstmate \ + --type=merge -p "$rollback_clear_patch" >/dev/null; then + echo "error: rollback completed but checkpoint clear CAS conflicted; retained target-digest=$rollback_target_digest" >&2 exit 3 fi ;; diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index fb5272eed..13a9feaea 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -123,7 +123,7 @@ case " $* " in last_create=$(grep -Fn 'stdin-kind Pod' "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d: -f1) last_delete=$(grep -Fn '/pods/agent-os-crewmate-' "$AGENT_OS_TEST_LOG" | grep 'delete --raw' | tail -n 1 | cut -d: -f1) if [ -n "$last_delete" ] && { [ -z "$last_create" ] || [ "$last_delete" -gt "$last_create" ]; }; then - pod_state=absent + pod_state=${AGENT_OS_TEST_POD_AFTER_DELETE:-absent} fi case "$pod_state" in absent) ;; @@ -154,6 +154,13 @@ case " $* " in *" get pvc agent-os-crewmate-"*" --ignore-not-found -o jsonpath="*) id=${AGENT_OS_TEST_CREWMATE_ID:-scout-1} pvc_state=${AGENT_OS_TEST_PVC_STATE:-absent} + if grep -F "/persistentvolumeclaims/agent-os-crewmate-$id-home" "$AGENT_OS_TEST_LOG" | grep -F 'delete --raw' >/dev/null; then + if [ -n "${AGENT_OS_TEST_PVC_AFTER_DELETE:-}" ]; then + pvc_state=$AGENT_OS_TEST_PVC_AFTER_DELETE + elif [[ "${AGENT_OS_TEST_FAIL_DELETE_TARGET:-}" != *"/persistentvolumeclaims/agent-os-crewmate-$id-home"* ]]; then + pvc_state=absent + fi + fi if grep -F 'stdin-kind PersistentVolumeClaim' "$AGENT_OS_TEST_LOG" >/dev/null; then [ "$pvc_state" != absent ] || pvc_state=owned fi @@ -194,7 +201,10 @@ case " $* " in if [ -n "$last_expired_delete" ]; then : elif grep -F ' replace -f -' "$AGENT_OS_TEST_LOG" >/dev/null; then - printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\toperation-test\t2026-07-13T00:00:00Z\t2026-07-13T00:00:00Z\t300\tuid-lock-expired\trv-lock-taken' "$id" "$id" + acquire_time=$(awk '/acquireTime:/ { value=$2 } END { print value }' "${AGENT_OS_STDIN_LOG:-$AGENT_OS_TEST_LOG.stdin}") + renew_time=$(awk '/renewTime:/ { value=$2 } END { print value }' "${AGENT_OS_STDIN_LOG:-$AGENT_OS_TEST_LOG.stdin}") + printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\toperation-test\t%s\t%s\t%s\tuid-lock-expired\trv-lock-taken' \ + "$id" "$id" "$acquire_time" "$renew_time" "${AGENT_OS_LOCK_DURATION_SECONDS:-300}" else printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tother-operation\t2000-01-01T00:00:00Z\t2000-01-01T00:00:00Z\t300\tuid-lock-expired\t%s' "$id" "$id" "${AGENT_OS_TEST_LOCK_RV:-rv-lock-expired}" fi @@ -207,7 +217,15 @@ case " $* " in elif [ -n "$last_lock_delete" ] && [ "${AGENT_OS_TEST_LOCK_RELEASE_STATE:-}" = foreign ]; then printf 'agent-os-crewmate-%s-lifecycle\tother\tother\tother-installation\tother-operation\t2099-01-01T00:00:00Z\t2099-01-01T00:00:00Z\t300\tuid-lock-foreign\trv-lock-foreign' "$id" elif [ -n "$last_lock_write" ] && { [ -z "$last_lock_delete" ] || [ "$last_lock_write" -gt "$last_lock_delete" ]; }; then - printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\toperation-test\t2026-07-13T00:00:00Z\t2026-07-13T00:00:00Z\t300\tuid-lock\trv-lock' "$id" "$id" + acquire_time=$(awk '/acquireTime:/ { value=$2 } END { print value }' "${AGENT_OS_STDIN_LOG:-$AGENT_OS_TEST_LOG.stdin}") + renew_time=$(awk '/renewTime:/ { value=$2 } END { print value }' "${AGENT_OS_STDIN_LOG:-$AGENT_OS_TEST_LOG.stdin}") + renew_rv=rv-lock + if grep -F ' replace -f -' "$AGENT_OS_TEST_LOG" >/dev/null; then + renew_rv=rv-lock-renewed + [ "${AGENT_OS_TEST_RENEW_READBACK:-exact}" != wrong ] || renew_time=2000-01-01T00:00:00Z + fi + printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\toperation-test\t%s\t%s\t%s\tuid-lock\t%s' \ + "$id" "$id" "$acquire_time" "$renew_time" "${AGENT_OS_LOCK_DURATION_SECONDS:-300}" "$renew_rv" fi ;; esac @@ -221,7 +239,15 @@ case " $* " in elif [ -n "$last_lock_write" ] && { [ -z "$last_lock_delete" ] || [ "$last_lock_write" -gt "$last_lock_delete" ]; }; then lock_stdin=${AGENT_OS_STDIN_LOG:-$AGENT_OS_TEST_LOG.stdin} lock_holder=$(awk '/holderIdentity:/ { holder=$2 } END { print holder }' "$lock_stdin") - printf '%s\tagent-os\tprimary\tagent-os-firstmate:%s\t%s\t2026-07-13T00:00:00Z\t2026-07-13T00:00:00Z\t300\tuid-primary-lock\t%s' "$lock_name" "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" "$lock_holder" "${AGENT_OS_TEST_PRIMARY_LOCK_RV:-rv-primary-lock}" + lock_acquired=$(awk '/acquireTime:/ { value=$2 } END { print value }' "$lock_stdin") + lock_renewed=$(awk '/renewTime:/ { value=$2 } END { print value }' "$lock_stdin") + lock_rv=${AGENT_OS_TEST_PRIMARY_LOCK_RV:-rv-primary-lock} + if grep -F ' replace -f -' "$AGENT_OS_TEST_LOG" >/dev/null; then + lock_rv=rv-primary-lock-renewed + fi + printf '%s\tagent-os\tprimary\tagent-os-firstmate:%s\t%s\t%s\t%s\t%s\tuid-primary-lock\t%s' \ + "$lock_name" "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" "$lock_holder" "$lock_acquired" "$lock_renewed" \ + "${AGENT_OS_LOCK_DURATION_SECONDS:-300}" "$lock_rv" fi ;; *" get namespace "*" --ignore-not-found -o name "*) @@ -317,13 +343,48 @@ case " $* " in esac ;; *" get statefulset agent-os-firstmate -o json "*) - printf '{"metadata":{"name":"agent-os-firstmate","uid":"uid-statefulset","resourceVersion":"%s","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"status":{"currentRevision":"%s","updateRevision":"%s"}}\n' \ - "${AGENT_OS_TEST_RESOURCE_RV:-rv-statefulset}" \ - "${AGENT_OS_TEST_ROLLBACK_CURRENT:-agent-os-firstmate-previous}" \ - "${AGENT_OS_TEST_ROLLBACK_UPDATE:-agent-os-firstmate-current}" + rollback_current=${AGENT_OS_TEST_ROLLBACK_CURRENT:-agent-os-firstmate-previous} + rollback_update=${AGENT_OS_TEST_ROLLBACK_UPDATE:-agent-os-firstmate-current} + if [ "${AGENT_OS_TEST_FAIL_ROLLOUT:-0}" != 1 ] && \ + grep -F 'rollout status statefulset/agent-os-firstmate' "$AGENT_OS_TEST_LOG" >/dev/null; then + rollback_current=agent-os-firstmate-previous + rollback_update=agent-os-firstmate-previous + fi + if [ "${AGENT_OS_TEST_ROLLBACK_VERIFY_MISMATCH:-0}" = 1 ] && \ + grep -F 'rollout status statefulset/agent-os-firstmate' "$AGENT_OS_TEST_LOG" >/dev/null; then + rollback_current=agent-os-firstmate-current + rollback_update=agent-os-firstmate-current + fi + checkpoint_annotations='' + if [ -n "${AGENT_OS_TEST_ROLLBACK_CHECKPOINT_DIGEST:-}" ]; then + checkpoint_annotations=$(printf ',"agent-os.dev/rollback-operation":"checkpoint-operation","agent-os.dev/rollback-target-name":"agent-os-firstmate-previous","agent-os.dev/rollback-target-uid":"uid-revision-previous","agent-os.dev/rollback-target-digest":"%s"' "$AGENT_OS_TEST_ROLLBACK_CHECKPOINT_DIGEST") + elif checkpoint_call=$(grep 'patch StatefulSet agent-os-firstmate --type=merge' "$AGENT_OS_TEST_LOG" | grep 'rollback-target-digest' | head -n 1); then + checkpoint_operation=$(printf '%s' "$checkpoint_call" | sed -n 's/.*rollback-operation":"\([^"]*\).*/\1/p') + checkpoint_name=$(printf '%s' "$checkpoint_call" | sed -n 's/.*rollback-target-name":"\([^"]*\).*/\1/p') + checkpoint_uid=$(printf '%s' "$checkpoint_call" | sed -n 's/.*rollback-target-uid":"\([^"]*\).*/\1/p') + checkpoint_digest=$(printf '%s' "$checkpoint_call" | sed -n 's/.*rollback-target-digest":"\([0-9a-f]*\).*/\1/p') + checkpoint_annotations=$(printf ',"agent-os.dev/rollback-operation":"%s","agent-os.dev/rollback-target-name":"%s","agent-os.dev/rollback-target-uid":"%s","agent-os.dev/rollback-target-digest":"%s"' \ + "$checkpoint_operation" "$checkpoint_name" "$checkpoint_uid" "$checkpoint_digest") + fi + if [ "${AGENT_OS_TEST_ROLLBACK_CHECKPOINT_MUTATE:-0}" = 1 ] && \ + grep -F 'rollout status statefulset/agent-os-firstmate' "$AGENT_OS_TEST_LOG" >/dev/null; then + checkpoint_annotations=$(printf '%s' "$checkpoint_annotations" | \ + sed 's/rollback-operation":"[^"]*/rollback-operation":"other-operation/') + fi + if [ "${AGENT_OS_TEST_ROLLBACK_CHECKPOINT_MUTATE_ON_READ:-0}" = 1 ] && [ -n "$checkpoint_call" ] && \ + ! grep -F 'rollout status statefulset/agent-os-firstmate' "$AGENT_OS_TEST_LOG" >/dev/null; then + checkpoint_annotations=$(printf '%s' "$checkpoint_annotations" | \ + sed 's/rollback-operation":"[^"]*/rollback-operation":"other-operation/') + fi + printf '{"metadata":{"name":"agent-os-firstmate","uid":"uid-statefulset","resourceVersion":"%s","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"%s}},"status":{"currentRevision":"%s","updateRevision":"%s"}}\n' \ + "${AGENT_OS_TEST_RESOURCE_RV:-rv-statefulset}" "$checkpoint_annotations" "$rollback_current" "$rollback_update" ;; *" get controllerrevisions.apps -o json "*) - printf '%s\n' '{"items":[{"metadata":{"name":"agent-os-firstmate-previous","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":1,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"previous"}}}}}},{"metadata":{"name":"agent-os-firstmate-current","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":2,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"current"}}}}}}]}' + if [ "${AGENT_OS_TEST_ROLLBACK_RENUMBERED:-0}" = 1 ]; then + printf '%s\n' '{"items":[{"metadata":{"name":"agent-os-firstmate-previous","uid":"uid-revision-previous","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":1,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"previous"}}}}}},{"metadata":{"name":"agent-os-firstmate-current","uid":"uid-revision-current","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":2,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"current"}}}}}},{"metadata":{"name":"agent-os-firstmate-renumbered","uid":"uid-revision-renumbered","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":3,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"previous"}}}}}}]}' + else + printf '%s\n' '{"items":[{"metadata":{"name":"agent-os-firstmate-previous","uid":"uid-revision-previous","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":1,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"previous"}}}}}},{"metadata":{"name":"agent-os-firstmate-current","uid":"uid-revision-current","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":2,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"current"}}}}}}]}' + fi ;; *" get role agent-os-firstmate-runtime -o json "*) if [ "${AGENT_OS_TEST_RBAC_STATE:-exact}" = bad-rules ]; then @@ -416,7 +477,7 @@ run_launcher() { AGENT_OS_IN_CLUSTER=1 AGENT_OS_NAMESPACE=agent-os-demo AGENT_OS_IMAGE=agent-os:local-test \ AGENT_OS_IMAGE_PULL_POLICY=Never AGENT_OS_AI_SECRET=scout-1-ai-auth \ AGENT_OS_OPERATION_ID=operation-test AGENT_OS_PURGE_EVIDENCE_FILE="$PURGE_EVIDENCE" \ - AGENT_OS_LOCK_ACQUIRE_SECONDS=2 \ + AGENT_OS_LOCK_ACQUIRE_SECONDS=3 \ "$LAUNCHER" "$@" } @@ -589,8 +650,8 @@ fi AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=owned run_launcher stop scout-1 grep -Fqx 'kubectl -n agent-os-demo --request-timeout=5s delete --raw /api/v1/namespaces/agent-os-demo/pods/agent-os-crewmate-scout-1 -f -' "$CALLS" || \ fail "stop must UID-precondition deletion of the exactly owned crewmate Pod" -grep -Fqx 'kubectl -n agent-os-demo wait --for=delete pod/agent-os-crewmate-scout-1 --timeout=180s' "$CALLS" || \ - fail "stop must prove Pod absence before checkpointing can begin" +grep -F 'wait --for=delete pod/agent-os-crewmate-scout-1' "$CALLS" | grep -F -- '--request-timeout=' >/dev/null || \ + fail "stop must prove Pod absence within its total deletion deadline" delete_line=$(grep -Fn '/pods/agent-os-crewmate-scout-1' "$CALLS" | grep 'delete --raw' | head -n 1 | cut -d: -f1) invalidate_line=$(grep -Fn 'patch pvc agent-os-crewmate-scout-1-home --type=merge' "$CALLS" | head -n 1 | cut -d: -f1) quiesced_line=$(grep -Fn 'patch pvc agent-os-crewmate-scout-1-home --type=merge' "$CALLS" | tail -n 1 | cut -d: -f1) @@ -602,6 +663,32 @@ if grep -F 'delete pvc agent-os-crewmate-scout-1-home' "$CALLS" >/dev/null; then fi pass "crewmate stop preserves its persistent home" +: > "$CALLS" +ambiguous_stop_out='' +ambiguous_stop_rc=0 +ambiguous_stop_out=$(AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=owned \ + AGENT_OS_TEST_FAIL_DELETE_TARGET=/pods/agent-os-crewmate-scout-1 \ + AGENT_OS_TEST_DELETE_ERROR='context deadline exceeded' run_launcher stop scout-1 2>&1) || ambiguous_stop_rc=$? +[ "$ambiguous_stop_rc" -eq 0 ] || fail "accepted ambiguous Pod delete must reconcile absence: $ambiguous_stop_out" +assert_contains "$ambiguous_stop_out" 'confirmed absent: pod/agent-os-crewmate-scout-1 captured uid=uid-owned' \ + "ambiguous stop must report terminal absence evidence" +assert_grep 'agent-os.dev/quiesced-operation' "$CALLS" \ + "stop may record quiescence only after ambiguous deletion is reconciled as absent" +pass "crewmate stop reconciles ambiguous Pod deletion" + +: > "$CALLS" +replacement_stop_out='' +replacement_stop_rc=0 +replacement_stop_out=$(AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_POD_AFTER_DELETE=replacement \ + AGENT_OS_TEST_PVC_STATE=owned AGENT_OS_TEST_FAIL_DELETE_TARGET=/pods/agent-os-crewmate-scout-1 \ + AGENT_OS_TEST_DELETE_ERROR='context deadline exceeded' run_launcher stop scout-1 2>&1) || replacement_stop_rc=$? +[ "$replacement_stop_rc" -eq 3 ] || fail "ambiguous Pod replacement must exit incomplete: $replacement_stop_out" +assert_contains "$replacement_stop_out" 'replacement uid=uid-replacement retained; captured uid=uid-owned' \ + "ambiguous stop must retain and report a same-name replacement" +assert_no_grep 'agent-os.dev/writer-state":"quiesced' "$CALLS" \ + "stop must not record quiescence for a replacement Pod" +pass "crewmate stop retains ambiguous same-name replacements" + : > "$CALLS" if AGENT_OS_TEST_POD_STATE=owned AGENT_OS_TEST_PVC_STATE=clean AGENT_OS_TEST_FAIL_PVC_PATCH=1 \ run_launcher stop scout-1 >/dev/null 2>&1; then @@ -682,6 +769,68 @@ assert_grep 'purge-complete' "$PURGE_EVIDENCE" "purge must record non-secret com assert_no_grep 'scout-1-ai-auth' "$PURGE_EVIDENCE" "purge evidence must never contain credential references" pass "crewmate purge requires confirmation and a clean checkpoint" +: > "$CALLS" +: > "$PURGE_EVIDENCE" +ambiguous_purge_out='' +ambiguous_purge_rc=0 +ambiguous_purge_out=$(AGENT_OS_TEST_POD_STATE=absent AGENT_OS_TEST_PVC_STATE=clean \ + AGENT_OS_TEST_PVC_AFTER_DELETE=absent \ + AGENT_OS_TEST_FAIL_DELETE_TARGET=/persistentvolumeclaims/agent-os-crewmate-scout-1-home \ + AGENT_OS_TEST_DELETE_ERROR='context deadline exceeded' run_launcher purge scout-1 --yes 2>&1) || ambiguous_purge_rc=$? +[ "$ambiguous_purge_rc" -eq 0 ] || fail "accepted ambiguous PVC delete must reconcile absence: $ambiguous_purge_out" +assert_grep 'purge-complete' "$PURGE_EVIDENCE" \ + "ambiguous purge accepted by the API must record terminal completion evidence" +assert_grep 'outcome=absent' "$PURGE_EVIDENCE" \ + "completed purge evidence must classify the captured PVC as absent" +assert_grep 'captured-uid=uid-pvc-owned' "$PURGE_EVIDENCE" \ + "completed purge evidence must retain the captured PVC UID" +assert_grep 'observed-uid=absent' "$PURGE_EVIDENCE" \ + "completed purge evidence must classify the captured PVC as absent" +assert_contains "$ambiguous_purge_out" 'confirmed absent: pvc/agent-os-crewmate-scout-1-home captured uid=uid-pvc-owned' \ + "ambiguous purge must report the captured PVC identity as absent" +pass "crewmate purge reconciles ambiguous PVC deletion" + +: > "$CALLS" +: > "$PURGE_EVIDENCE" +retained_purge_out='' +retained_purge_rc=0 +retained_purge_out=$(AGENT_OS_TEST_POD_STATE=absent AGENT_OS_TEST_PVC_STATE=clean \ + AGENT_OS_TEST_FAIL_DELETE_TARGET=/persistentvolumeclaims/agent-os-crewmate-scout-1-home \ + AGENT_OS_TEST_DELETE_ERROR='context deadline exceeded' run_launcher purge scout-1 --yes 2>&1) || retained_purge_rc=$? +[ "$retained_purge_rc" -eq 3 ] || fail "ambiguous retained PVC must exit incomplete: $retained_purge_out" +assert_contains "$retained_purge_out" 'original uid=uid-pvc-owned remains' \ + "ambiguous purge must report the retained original PVC identity" +assert_grep 'purge-incomplete-original-retained' "$PURGE_EVIDENCE" \ + "ambiguous retained PVC must record terminal incomplete evidence" +assert_grep 'outcome=original-retained' "$PURGE_EVIDENCE" \ + "ambiguous retained PVC must classify the original identity" +assert_grep 'captured-uid=uid-pvc-owned' "$PURGE_EVIDENCE" \ + "ambiguous retained PVC must retain its captured identity" +assert_grep 'observed-uid=uid-pvc-owned' "$PURGE_EVIDENCE" \ + "ambiguous retained PVC must record terminal incomplete evidence" +pass "crewmate purge records retained original PVC evidence" + +: > "$CALLS" +: > "$PURGE_EVIDENCE" +replacement_purge_out='' +replacement_purge_rc=0 +replacement_purge_out=$(AGENT_OS_TEST_POD_STATE=absent AGENT_OS_TEST_PVC_STATE=clean \ + AGENT_OS_TEST_PVC_AFTER_DELETE=foreign \ + AGENT_OS_TEST_FAIL_DELETE_TARGET=/persistentvolumeclaims/agent-os-crewmate-scout-1-home \ + AGENT_OS_TEST_DELETE_ERROR='context deadline exceeded' run_launcher purge scout-1 --yes 2>&1) || replacement_purge_rc=$? +[ "$replacement_purge_rc" -eq 3 ] || fail "ambiguous PVC replacement must exit incomplete: $replacement_purge_out" +assert_contains "$replacement_purge_out" 'replacement uid=uid-pvc-foreign retained; captured uid=uid-pvc-owned' \ + "ambiguous purge must retain and report a same-name replacement PVC" +assert_grep 'purge-incomplete-replacement-retained' "$PURGE_EVIDENCE" \ + "replacement PVC must record terminal incomplete evidence" +assert_grep 'outcome=replacement-retained' "$PURGE_EVIDENCE" \ + "replacement PVC must classify its terminal outcome" +assert_grep 'captured-uid=uid-pvc-owned' "$PURGE_EVIDENCE" \ + "replacement PVC must retain the captured UID" +assert_grep 'observed-uid=uid-pvc-foreign' "$PURGE_EVIDENCE" \ + "replacement PVC must record terminal incomplete evidence" +pass "crewmate purge retains ambiguous same-name replacements" + : > "$CALLS" if AGENT_OS_TEST_POD_STATE=absent AGENT_OS_TEST_PVC_STATE=clean AGENT_OS_TEST_PVC_ATTACHED=1 \ run_launcher purge scout-1 --yes >/dev/null 2>&1; then @@ -707,7 +856,7 @@ lock_out='' lock_rc=0 lock_out=$(AGENT_OS_TEST_LOCK_STATE=held run_launcher stop scout-1 2>&1) || lock_rc=$? [ "$lock_rc" -eq 3 ] || fail "bounded lifecycle lock contention must exit incomplete: $lock_out" -assert_contains "$lock_out" "still holds Lease 'agent-os-crewmate-scout-1-lifecycle' after 2s" \ +assert_contains "$lock_out" "still holds Lease 'agent-os-crewmate-scout-1-lifecycle' after 3s" \ "lifecycle contention must report the exact holder and bounded timeout" pass "crewmate lifecycle operations use a bounded coordination lock" @@ -737,12 +886,27 @@ assert_grep 'resourceVersion: "12345"' "$STDIN_LOG" \ pass "crewmate Lease CAS preserves opaque resourceVersion typing" : > "$CALLS" -AGENT_OS_LOCK_DURATION_SECONDS=3 AGENT_OS_TEST_READY_DELAY=2 \ +AGENT_OS_LOCK_DURATION_SECONDS=10 AGENT_OS_TEST_READY_DELAY=4 \ AGENT_OS_TEST_PVC_STATE=owned run_launcher create scout-1 grep -F 'replace -f -' "$CALLS" >/dev/null || \ fail "active lifecycle operations must renew their exact-owned Lease" pass "crewmate lifecycle renews its Lease while active" +: > "$CALLS" +renewal_rc=0 +AGENT_OS_LOCK_DURATION_SECONDS=10 AGENT_OS_TEST_READY_DELAY=4 \ + AGENT_OS_TEST_RENEW_READBACK=wrong AGENT_OS_TEST_PVC_STATE=owned \ + run_launcher create scout-1 >/dev/null 2>&1 || renewal_rc=$? +[ "$renewal_rc" -eq 3 ] || fail "renewal must fail closed when exact renewTime read-back is not committed" +pass "crewmate Lease renewal verifies the committed renewTime" + +: > "$CALLS" +AGENT_OS_TEST_POD_STATE=absent AGENT_OS_TEST_PVC_STATE=owned run_launcher stop scout-1 +lease_create_call=$(grep 'create -f -' "$CALLS" | head -n 1) +assert_contains "$lease_create_call" '--request-timeout=2s' \ + "Lease create must reserve acquisition time for mandatory result reconciliation" +pass "crewmate Lease mutations reserve read-back budget" + lease_call_without_timeout=$(awk ' /^kubectl / { call=$0 } / get lease | delete --raw .*\/leases\// { if ($0 !~ /--request-timeout=/) print } @@ -1135,10 +1299,10 @@ grep -F '/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/agent-os-firstma fail "privileged cleanup must UID-delete only the exact owned ClusterRoleBinding" assert_grep '"uid":"uid-clusterrolebinding","resourceVersion":"rv-clusterrolebinding"' "$STDIN_LOG" \ "privileged cleanup must bind deletion to the observed grant identity" -grep -F 'wait --for=delete ClusterRoleBinding/agent-os-firstmate-portable-agent-os --timeout=55s' "$CALLS" >/dev/null || \ +grep -F 'wait --for=delete ClusterRoleBinding/agent-os-firstmate-portable-agent-os --timeout=' "$CALLS" >/dev/null || \ fail "privileged cleanup must produce deletion evidence for the exact binding" -grep -F 'wait --for=delete ClusterRoleBinding/agent-os-firstmate-portable-agent-os --timeout=55s' "$CALLS" | \ - grep -F -- '--request-timeout=55s' >/dev/null || \ +grep -F 'wait --for=delete ClusterRoleBinding/agent-os-firstmate-portable-agent-os --timeout=' "$CALLS" | \ + grep -F -- '--request-timeout=' >/dev/null || \ fail "delete wait must reserve five seconds for bounded identity reconciliation" pass "privileged cleanup verifies ownership and deletes one exact binding" @@ -1146,6 +1310,10 @@ cleanup_delete_call=$(grep -F '/apis/rbac.authorization.k8s.io/v1/clusterrolebin grep -F 'delete --raw' | tail -n 1) assert_contains "$cleanup_delete_call" '--request-timeout=' \ "raw delete requests must be bounded before the operation wait begins" +cleanup_observe_call=$(grep 'get ClusterRoleBinding agent-os-firstmate-portable-agent-os --ignore-not-found' "$CALLS" | \ + grep 'metadata.resourceVersion' | tail -n 1) +assert_contains "$cleanup_observe_call" '--request-timeout=' \ + "cluster cleanup deletion deadline must start before the ownership observation" : > "$CALLS" cleanup_stale_out='' @@ -1288,6 +1456,15 @@ pass "all RBAC modes preflight deterministic stale resources" : > "$CALLS" AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ AGENT_OS_TEST_WORKLOAD_STATE=namespace run_generic rollback +checkpoint_line=$(grep -Fn 'agent-os.dev/rollback-target-digest' "$CALLS" | head -n 1 | cut -d: -f1) +template_patch_line=$(grep -Fn '"rollback":"previous"' "$CALLS" | head -n 1 | cut -d: -f1) +[ -n "$checkpoint_line" ] && [ -n "$template_patch_line" ] && [ "$checkpoint_line" -lt "$template_patch_line" ] || \ + fail "rollback must persist its immutable target checkpoint before patching the template" +assert_grep 'agent-os.dev/rollback-target-uid' "$CALLS" \ + "rollback checkpoint must persist exact ControllerRevision identity" +checkpoint_read_call=$(awk '/rollback-target-digest/ { checkpoint=1; next } checkpoint && / get statefulset agent-os-firstmate -o json/ { print; exit }' "$CALLS") +assert_contains "$checkpoint_read_call" '--request-timeout=' \ + "rollback checkpoint persistence read-back must be request-bounded" grep -F 'kubectl --context kind-agent-os -n portable-agent-os patch StatefulSet agent-os-firstmate --type=strategic' "$CALLS" >/dev/null || \ fail "generic rollback must update only the captured Firstmate StatefulSet" assert_grep '"uid":"uid-statefulset"' "$CALLS" \ @@ -1304,6 +1481,63 @@ assert_grep 'get controllerrevisions.apps -o json' "$CALLS" \ "rollback must resolve the exact-owned revision history" pass "generic rollback applies a revision-derived StatefulSet CAS update" +: > "$CALLS" +checkpoint_read_race_out='' +checkpoint_read_race_rc=0 +checkpoint_read_race_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_ROLLBACK_CHECKPOINT_MUTATE_ON_READ=1 \ + run_generic rollback 2>&1) || checkpoint_read_race_rc=$? +[ "$checkpoint_read_race_rc" -eq 3 ] || fail "rollback must fail closed when checkpoint identity changes before template mutation: $checkpoint_read_race_out" +assert_no_grep 'patch StatefulSet agent-os-firstmate --type=strategic' "$CALLS" \ + "checkpoint identity change must prevent strategic template mutation" +assert_contains "$checkpoint_read_race_out" 'rollback checkpoint did not persist exactly' \ + "checkpoint persistence failure must preserve an explicit retained-state error" +pass "rollback checkpoint read-back verifies immutable identity" + +previous_template_digest=$(printf '%s' '{"metadata":{"labels":{"rollback":"previous"}}}' | jq -cS . | shasum -a 256 | awk '{print $1}') +: > "$CALLS" +renumbered_out='' +renumbered_rc=0 +renumbered_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_PRIMARY_POD_STATE=owned \ + AGENT_OS_TEST_ROLLBACK_CURRENT=agent-os-firstmate-current \ + AGENT_OS_TEST_ROLLBACK_UPDATE=agent-os-firstmate-renumbered \ + AGENT_OS_TEST_ROLLBACK_RENUMBERED=1 \ + AGENT_OS_TEST_ROLLBACK_CHECKPOINT_DIGEST="$previous_template_digest" \ + AGENT_OS_TEST_FAIL_ROLLOUT=1 run_generic rollback 2>&1) || renumbered_rc=$? +[ "$renumbered_rc" -eq 3 ] || fail "renumbered rollback retry must remain incomplete: $renumbered_out" +assert_no_grep '"rollback":"current"' "$CALLS" \ + "rollback retry must never reverse to the post-patch current template" +assert_contains "$renumbered_out" 'target-digest=' \ + "rollback retry must preserve the immutable checkpoint digest" +assert_contains "$renumbered_out" 'target=agent-os-firstmate-renumbered' \ + "rollback retry must adopt content-equivalent renumbered ControllerRevision" +assert_contains "$renumbered_out" 'checkpoint-target=agent-os-firstmate-previous checkpoint-uid=uid-revision-previous' \ + "rollback retry evidence must preserve the original immutable checkpoint identity" +pass "rollback retry adopts content-equivalent renumbered revisions" + +: > "$CALLS" +verification_out='' +verification_rc=0 +verification_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_ROLLBACK_VERIFY_MISMATCH=1 \ + run_generic rollback 2>&1) || verification_rc=$? +[ "$verification_rc" -eq 3 ] || fail "rollback must fail closed when completed revisions do not match its checkpoint: $verification_out" +assert_contains "$verification_out" 'rollback verification mismatch' \ + "rollback must retain explicit evidence when another revision completes" +pass "rollback success verifies both revision targets by content" + +: > "$CALLS" +checkpoint_mutation_out='' +checkpoint_mutation_rc=0 +checkpoint_mutation_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_ROLLBACK_CHECKPOINT_MUTATE=1 \ + run_generic rollback 2>&1) || checkpoint_mutation_rc=$? +[ "$checkpoint_mutation_rc" -eq 3 ] || fail "rollback must fail closed when checkpoint identity mutates: $checkpoint_mutation_out" +assert_contains "$checkpoint_mutation_out" 'rollback verification mismatch' \ + "rollback verification must require the complete immutable checkpoint tuple" +pass "rollback success preserves immutable checkpoint identity" + : > "$CALLS" rollback_failure_out='' rollback_failure_rc=0 @@ -1321,6 +1555,10 @@ assert_contains "$rollback_failure_out" 'lifecycle-lease=agent-os-firstmate-life "rollback failure must preserve exact lifecycle Lease evidence" assert_contains "$rollback_failure_out" 'safe recovery:' \ "rollback failure must print an exact non-reversing recovery command" +rollback_evidence_calls=$(awk '/rollout status statefulset\/agent-os-firstmate/ { after=1; next } after && / get (statefulset agent-os-firstmate -o json|Pod agent-os-firstmate-0 .*jsonpath=)/ { print }' "$CALLS") +if printf '%s\n' "$rollback_evidence_calls" | grep -v -- '--request-timeout=' >/dev/null; then + fail "rollback failure evidence reads must each carry an independent request timeout: $rollback_evidence_calls" +fi pass "failed rollback rollout preserves recovery evidence" : > "$CALLS" @@ -1330,13 +1568,14 @@ rollback_resume_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_PRIMARY_POD_STATE=owned \ AGENT_OS_TEST_ROLLBACK_CURRENT=agent-os-firstmate-current \ AGENT_OS_TEST_ROLLBACK_UPDATE=agent-os-firstmate-previous \ + AGENT_OS_TEST_ROLLBACK_CHECKPOINT_DIGEST="$previous_template_digest" \ AGENT_OS_TEST_FAIL_ROLLOUT=1 run_generic rollback 2>&1) || rollback_resume_rc=$? [ "$rollback_resume_rc" -eq 3 ] || fail "in-progress rollback retry must remain incomplete: $rollback_resume_out" assert_no_grep 'patch StatefulSet agent-os-firstmate' "$CALLS" \ "in-progress rollback retry must not reverse the selected target" assert_contains "$rollback_resume_out" 'rollback target=agent-os-firstmate-previous revision=1' \ "in-progress rollback retry must preserve the lower selected revision" -assert_contains "$rollback_resume_out" "do not invoke rollback again while revisions differ" \ +assert_contains "$rollback_resume_out" "persisted target digest remains" \ "in-progress rollback retry must provide non-reversing recovery guidance" pass "rollback retry resumes an existing lower-revision target" From 3e8572187d2380d2a80ba218b82253d86243d38e Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 15:02:53 +0200 Subject: [PATCH 34/56] no-mistakes(review): Captain, harden Kubernetes lifecycle and image provenance --- .../akua-intelligence-bootstrap/SKILL.md | 15 +- .dockerignore | 25 ++- .github/workflows/agent-os-image.yml | 62 ++++++- .github/workflows/ci.yml | 1 + .gitignore | 1 + Dockerfile | 32 +++- bin/agent-os-akua-auth.sh | 159 ++++++++++++++++++ bin/agent-os-container-entrypoint.sh | 45 ++++- bin/agent-os-crewmate.sh | 92 +++++++++- bin/agent-os-init.sh | 38 ++++- bin/agent-os-kubernetes-lease.sh | 8 +- bin/agent-os-kubernetes.sh | 157 ++++++++++++++++- deploy/akua/firstmate-auth-grant.yaml | 3 + deploy/akua/firstmate-auth-revoke.yaml | 3 + docs/kubernetes.md | 3 +- tests/agent-os-container.test.sh | 21 ++- tests/agent-os-init.test.sh | 29 ++-- tests/agent-os-kubernetes.test.sh | 133 +++++++++++++-- tests/agent-os-packages.test.sh | 20 ++- .../agent-os/packages/firstmate/crewmate.yaml | 3 - tools/agent-os/packages/firstmate/package.k | 11 +- 21 files changed, 769 insertions(+), 92 deletions(-) create mode 100755 bin/agent-os-akua-auth.sh diff --git a/.agents/skills/akua-intelligence-bootstrap/SKILL.md b/.agents/skills/akua-intelligence-bootstrap/SKILL.md index 4adc24ffd..fcdf3337f 100644 --- a/.agents/skills/akua-intelligence-bootstrap/SKILL.md +++ b/.agents/skills/akua-intelligence-bootstrap/SKILL.md @@ -49,26 +49,21 @@ The public Firstmate package never accepts an Akua credential input or renders a Akua authorization belongs to the separate namespace-local integration overlay under `deploy/akua/`. The grant patch mounts only the selected Secret at `/var/run/secrets/agent-os/akua` and sets `AKUA_AUTH_HEADER_FILE=/var/run/secrets/agent-os/akua/authorization`. The Secret must contain one `authorization` key with the complete header and must live in the same namespace as the Firstmate StatefulSet. -Render `__AKUA_AUTH_SECRET__` into a mode-`0600` temporary copy of `firstmate-auth-grant.yaml` and keep its path in `grant_patch`. -Reject a Secret name that is not a Kubernetes DNS subdomain before substituting it. Never put the Secret value into the overlay, package inputs, or command arguments. -Set `context` and `namespace` from the separately approved target and grant the overlay with this exact context-pinned strategic patch command: +Set `context` and `namespace` from the separately approved target and grant the overlay through the serialized integration helper: ```sh -kubectl --context "$context" -n "$namespace" patch statefulset agent-os-firstmate --type=strategic --patch-file "$grant_patch" -kubectl --context "$context" -n "$namespace" rollout status statefulset/agent-os-firstmate --timeout=180s +AGENT_OS_CONTEXT="$context" AGENT_OS_NAMESPACE="$namespace" bin/agent-os-akua-auth.sh grant "$secret_name" ``` -Destroy the temporary grant patch after rollout. +The helper holds the installation-wide lifecycle Lease, validates exact StatefulSet UID and resourceVersion, verifies the named Secret reference without reading Secret bytes, applies one CAS strategic patch, and verifies both the retained StatefulSet overlay and its exact-owned Pod. Revocation is owned by the same integration boundary. -Keep the path to `firstmate-auth-revoke.yaml` in `revoke_patch`. -Revoke the Akua API token and prove it fails first, then revoke the overlay with the same exact target and patch type: +Revoke the Akua API token and prove it fails first, then revoke the overlay through the same serialized helper: ```sh -kubectl --context "$context" -n "$namespace" patch statefulset agent-os-firstmate --type=strategic --patch-file "$revoke_patch" -kubectl --context "$context" -n "$namespace" rollout status statefulset/agent-os-firstmate --timeout=180s +AGENT_OS_CONTEXT="$context" AGENT_OS_NAMESPACE="$namespace" bin/agent-os-akua-auth.sh revoke "$secret_name" ``` Delete only the named namespace-local Secret after explicit cleanup approval. diff --git a/.dockerignore b/.dockerignore index 90b53692f..121fae01b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,10 +1,25 @@ .git .github -.pi -.codex -.claude -.grok -.opencode +.pi/* +!.pi/extensions/ +!.pi/extensions/fm-primary-pi-watch.ts +!.pi/extensions/fm-primary-turnend-guard.ts +.codex/* +!.codex/hooks.json +.claude/* +!.claude/settings.json +!.claude/skills +.grok/* +!.grok/hooks/ +!.grok/hooks/fm-primary-cd-check.json +!.grok/hooks/fm-primary-pretool-check.json +!.grok/hooks/fm-primary-turnend-guard.json +.opencode/* +!.opencode/plugins/ +!.opencode/plugins/fm-primary-cd-check.js +!.opencode/plugins/fm-primary-pretool-check.js +!.opencode/plugins/fm-primary-turnend-guard.js +!.opencode/plugins/fm-primary-watch-arm.js .no-mistakes .env config diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index c3a1d168a..fafe0afbc 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -18,12 +18,55 @@ env: IMAGE: ghcr.io/akua-dev/agent-os jobs: + behavior: + name: Exact-source behavior gate + uses: ./.github/workflows/ci.yml + + provenance: + if: github.event_name == 'push' + name: Protected-main source provenance + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 + with: + fetch-depth: 0 + - name: Verify publication source + env: + GH_TOKEN: ${{ github.token }} + REF_PROTECTED: ${{ github.ref_protected }} + run: | + set -eu + if [ "$GITHUB_REF" = refs/heads/main ]; then + [ "$REF_PROTECTED" = true ] || { echo "::error::main is not reported protected"; exit 1; } + elif [ "${GITHUB_REF#refs/tags/}" != "$GITHUB_REF" ]; then + git fetch --no-tags origin refs/heads/main:refs/remotes/origin/main + [ "$(git rev-parse refs/remotes/origin/main)" = "$GITHUB_SHA" ] || { + echo "::error::release tag must point at the exact protected-main head" + exit 1 + } + gh api "repos/$GITHUB_REPOSITORY/branches/main/protection" >/dev/null + else + echo "::error::unsupported publication ref $GITHUB_REF" + exit 1 + fi + validate: - if: github.event_name == 'pull_request' name: Validate linux/amd64 and linux/arm64 runs-on: ubuntu-latest steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 + with: + fetch-depth: 0 + + - name: Prepare exact source bundle + id: source + run: | + set -eu + git bundle create image/agent-os-source.bundle HEAD + echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "tree=$(git rev-parse HEAD^{tree})" >> "$GITHUB_OUTPUT" - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 @@ -39,9 +82,13 @@ jobs: cache-to: type=gha,mode=max provenance: mode=max sbom: true + build-args: | + AGENT_OS_SOURCE_COMMIT=${{ steps.source.outputs.commit }} + AGENT_OS_SOURCE_TREE=${{ steps.source.outputs.tree }} publish: if: github.event_name == 'push' + needs: [behavior, provenance, validate] name: Publish linux/amd64 and linux/arm64 runs-on: ubuntu-latest permissions: @@ -49,6 +96,16 @@ jobs: packages: write steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 + with: + fetch-depth: 0 + + - name: Prepare exact source bundle + id: source + run: | + set -eu + git bundle create image/agent-os-source.bundle HEAD + echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "tree=$(git rev-parse HEAD^{tree})" >> "$GITHUB_OUTPUT" - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 @@ -84,6 +141,9 @@ jobs: cache-to: type=gha,mode=max provenance: mode=max sbom: true + build-args: | + AGENT_OS_SOURCE_COMMIT=${{ steps.source.outputs.commit }} + AGENT_OS_SOURCE_TREE=${{ steps.source.outputs.tree }} - name: Record published image digest run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33b3e5fe4..bbdd17f6c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,7 @@ name: CI on: + workflow_call: push: branches: [main] pull_request: diff --git a/.gitignore b/.gitignore index 5ed2da0c3..88bd79c25 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ data/ .DS_Store __pycache__/ *.pyc +image/agent-os-source.bundle .env config/crew-harness config/crew-dispatch.json diff --git a/Dockerfile b/Dockerfile index 711bb8c4e..95337dfcb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,9 @@ ARG NO_MISTAKES_VERSION=1.34.0 ARG BUN_VERSION=1.3.14 ARG AKUA_VERSION=0.8.25 ARG K9S_VERSION=0.51.0 +ARG AGENT_OS_SOURCE_COMMIT +ARG AGENT_OS_SOURCE_TREE +ARG AGENT_OS_SOURCE_ORIGIN=https://github.com/akua-dev/agent-os.git COPY image/debian.sources /etc/apt/sources.list.d/debian.sources @@ -157,15 +160,33 @@ ENV FM_HOME=/home/agent \ XDG_CONFIG_HOME=/home/agent/.config \ XDG_DATA_HOME=/home/agent/.local/share \ XDG_CACHE_HOME=/home/agent/.cache \ - NPM_CONFIG_PREFIX=/usr/local \ + NPM_CONFIG_PREFIX=/home/agent/.local \ BUN_INSTALL=/home/agent/.bun \ CARGO_HOME=/home/agent/.cargo \ PATH=/home/agent/.local/bin:/home/agent/.bun/bin:/home/agent/.cargo/bin:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin \ HERDR_SESSION=default -RUN mkdir -p /home/agent /opt/agent-os /opt/image-usr-local +RUN mkdir -p /home/agent /opt/agent-os COPY . /opt/agent-os +COPY image/agent-os-source.bundle /opt/agent-os-source.bundle + +RUN set -eu; \ + test -n "$AGENT_OS_SOURCE_COMMIT"; \ + test -n "$AGENT_OS_SOURCE_TREE"; \ + git bundle verify /opt/agent-os-source.bundle; \ + git clone --no-local /opt/agent-os-source.bundle /opt/agent-os-bootstrap; \ + git -C /opt/agent-os-bootstrap checkout --detach "$AGENT_OS_SOURCE_COMMIT"; \ + test "$(git -C /opt/agent-os-bootstrap rev-parse HEAD)" = "$AGENT_OS_SOURCE_COMMIT"; \ + test "$(git -C /opt/agent-os-bootstrap rev-parse HEAD^{tree})" = "$AGENT_OS_SOURCE_TREE"; \ + test -z "$(git -C /opt/agent-os-bootstrap status --porcelain)"; \ + test -z "$(git -C /opt/agent-os-bootstrap ls-files -- config data projects state .no-mistakes)"; \ + rm -rf /opt/agent-os-bootstrap/.git/hooks; \ + git -C /opt/agent-os-bootstrap remote set-url origin "$AGENT_OS_SOURCE_ORIGIN"; \ + test "$(git -C /opt/agent-os-bootstrap remote get-url origin)" = "$AGENT_OS_SOURCE_ORIGIN"; \ + printf '%s\n' "$AGENT_OS_SOURCE_COMMIT" > /opt/agent-os-source.commit; \ + printf '%s\n' "$AGENT_OS_SOURCE_TREE" > /opt/agent-os-source.tree; \ + printf '%s\n' "$AGENT_OS_SOURCE_ORIGIN" > /opt/agent-os-source.origin RUN install -D -m 0644 /opt/agent-os/THIRD_PARTY_NOTICES.md /usr/share/doc/agent-os/THIRD_PARTY_NOTICES.md \ && install -D -m 0644 /opt/agent-os/THIRD_PARTY_SOURCES.md /usr/share/doc/agent-os/THIRD_PARTY_SOURCES.md @@ -173,7 +194,10 @@ RUN install -D -m 0644 /opt/agent-os/THIRD_PARTY_NOTICES.md /usr/share/doc/agent RUN cd /opt/agent-os/tools/agent-os \ && bun install --frozen-lockfile --production --ignore-scripts \ && ln -s /opt/agent-os/tools/agent-os/src/cli.ts /usr/local/bin/agent-os \ - && cp -a /usr/local/. /opt/image-usr-local/ + && cd /usr/local \ + && find . \( -type f -o -type l \) -print | LC_ALL=C sort | while IFS= read -r path; do sha256sum "$path"; done > /opt/agent-os-image-usr-local.manifest \ + && cd /opt \ + && sha256sum agent-os-image-usr-local.manifest > agent-os-image-usr-local.manifest.sha256 -WORKDIR /opt/agent-os +WORKDIR /home/agent/firstmate ENTRYPOINT ["/opt/agent-os/bin/agent-os-container-entrypoint.sh"] diff --git a/bin/agent-os-akua-auth.sh b/bin/agent-os-akua-auth.sh new file mode 100755 index 000000000..ac861885c --- /dev/null +++ b/bin/agent-os-akua-auth.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +set -eu + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +COMMAND=${1:-} +SECRET=${2:-} +CONTEXT=${AGENT_OS_CONTEXT:-} +NAMESPACE=${AGENT_OS_NAMESPACE:-} +KUBECTL=${AGENT_OS_KUBECTL:-kubectl} +INSTALLATION_ID="agent-os-firstmate:$NAMESPACE" +OPERATION_ID=${AGENT_OS_OPERATION_ID:-"$(date -u '+%Y%m%d%H%M%S')-$$-$RANDOM"} +LOCK_NONCE=$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n') +LOCK_HOLDER_ID="$OPERATION_ID.$LOCK_NONCE" +LOCK=agent-os-firstmate-lifecycle +LOCK_NAMESPACE=$NAMESPACE +EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"primary"$'\t'"$INSTALLATION_ID" +LOCK_UID= +LOCK_RV= +LOCK_RENEW_PID= +LOCK_DURATION_SECONDS=${AGENT_OS_LOCK_DURATION_SECONDS:-300} +LOCK_CLOCK_SKEW_SECONDS=${AGENT_OS_LOCK_CLOCK_SKEW_SECONDS:-5} +LOCK_ACQUIRE_SECONDS=${AGENT_OS_LOCK_ACQUIRE_SECONDS:-30} +LOCK_REQUEST_CEILING_SECONDS=${AGENT_OS_LOCK_REQUEST_CEILING_SECONDS:-5} +RESOURCE_REQUEST_CEILING_SECONDS=${AGENT_OS_RESOURCE_REQUEST_CEILING_SECONDS:-5} + +case "$COMMAND" in grant|revoke) ;; *) echo "usage: $0 grant|revoke " >&2; exit 2 ;; esac +[ -n "$CONTEXT" ] && [ -n "$NAMESPACE" ] || { echo "error: set AGENT_OS_CONTEXT and AGENT_OS_NAMESPACE" >&2; exit 2; } +case "$SECRET" in ''|*[!a-z0-9.-]*|[.-]*|*[-.]) echo "error: invalid Akua authorization Secret name" >&2; exit 2 ;; esac +[ "${#SECRET}" -le 253 ] || { echo "error: invalid Akua authorization Secret name" >&2; exit 2; } +case "$NAMESPACE" in ''|*[!a-z0-9-]*|-*|*-) echo "error: invalid Kubernetes namespace" >&2; exit 2 ;; esac +[ "${#NAMESPACE}" -le 63 ] && [ "${#LOCK_HOLDER_ID}" -le 255 ] || { echo "error: derived Kubernetes identity is too long" >&2; exit 2; } +command -v jq >/dev/null 2>&1 || { echo "error: jq is required" >&2; exit 2; } + +kube() { + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" "$@" +} + +lock_record() { + local deadline=${1:-} + [ -n "$deadline" ] || deadline=$(lock_default_deadline) + lock_kube "$deadline" get lease "$LOCK" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/lifecycle}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.spec.holderIdentity}{"\t"}{.spec.acquireTime}{"\t"}{.spec.renewTime}{"\t"}{.spec.leaseDurationSeconds}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}' +} + +render_lock() { + local acquired_at=$1 renewed_at=$2 uid=${3:-} rv=${4:-} uid_value='' rv_value='' + [ -z "$uid" ] || uid_value=$(yaml_string "$uid") + [ -z "$rv" ] || rv_value=$(yaml_string "$rv") + cat </dev/null || { echo "error: Akua authorization overlay verification failed" >&2; exit 3; } + secret_record_after=$(secret_record) + [ "$secret_record_after" = "$SECRET_RECORD" ] || { echo "error: Akua authorization Secret reference changed during mutation" >&2; exit 3; } + pod=$(kube --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get pod agent-os-firstmate-0 -o json) + printf '%s' "$pod" | jq -e --arg uid "$state_uid" --arg secret "$SECRET" --arg expected "$expected" ' + any(.metadata.ownerReferences[]?; .apiVersion == "apps/v1" and .kind == "StatefulSet" and .name == "agent-os-firstmate" and .uid == $uid and .controller == true) and + (if $expected == "present" then + ([.spec.containers[] | select(.name == "firstmate") | .env[] | select(.name == "AKUA_AUTH_HEADER_FILE" and .value == "/var/run/secrets/agent-os/akua/authorization")] | length) == 1 and + ([.spec.containers[] | select(.name == "firstmate") | .volumeMounts[] | select(.name == "akua-auth" and .mountPath == "/var/run/secrets/agent-os/akua" and .readOnly == true)] | length) == 1 and + ([.spec.volumes[] | select(.name == "akua-auth" and .secret.secretName == $secret and .secret.defaultMode == 256)] | length) == 1 + else + ([.spec.containers[] | select(.name == "firstmate") | .env[]? | select(.name == "AKUA_AUTH_HEADER_FILE")] | length) == 0 and + ([.spec.containers[] | select(.name == "firstmate") | .volumeMounts[]? | select(.name == "akua-auth")] | length) == 0 and + ([.spec.volumes[]? | select(.name == "akua-auth")] | length) == 0 + end)' >/dev/null || { echo "error: Firstmate Pod authorization overlay verification failed" >&2; exit 3; } +} + +SECRET_RECORD=$(secret_record) +[ "$(printf '%s' "$SECRET_RECORD" | cut -f1)" = "$SECRET" ] && \ + [ -n "$(printf '%s' "$SECRET_RECORD" | cut -f2)" ] && [ -n "$(printf '%s' "$SECRET_RECORD" | cut -f3)" ] && \ + [ "$(printf '%s' "$SECRET_RECORD" | cut -f4-)" = authorization ] || { + echo "error: Akua authorization Secret reference is missing or unverifiable" >&2 + exit 2 +} + +acquire_lock +STATE=$(statefulset_json) +STATE_UID=$(printf '%s' "$STATE" | jq -er --arg installation "$INSTALLATION_ID" ' + select(.metadata.name == "agent-os-firstmate") + | select(.metadata.labels["app.kubernetes.io/managed-by"] == "agent-os") + | select(.metadata.annotations["agent-os.dev/installation-id"] == $installation) + | .metadata.uid') || { echo "error: StatefulSet ownership is unverifiable" >&2; exit 2; } +STATE_RV=$(printf '%s' "$STATE" | jq -er '.metadata.resourceVersion') || { echo "error: StatefulSet resourceVersion is unavailable" >&2; exit 2; } + +PATCH_FILE=$(mktemp) +TEMPLATE="$ROOT/deploy/akua/firstmate-auth-$COMMAND.yaml" +uid_value=$(yaml_string "$STATE_UID") +rv_value=$(yaml_string "$STATE_RV") +sed "s/__AKUA_AUTH_SECRET__/$SECRET/g" "$TEMPLATE" | awk -v uid="$uid_value" -v rv="$rv_value" ' + $1 == "metadata:" && !inserted { print; print " uid: " uid; print " resourceVersion: " rv; inserted=1; next } + { print } +' > "$PATCH_FILE" +kube patch statefulset agent-os-firstmate --type=strategic --patch-file "$PATCH_FILE" >/dev/null +kube rollout status statefulset/agent-os-firstmate --timeout=180s +if [ "$COMMAND" = grant ]; then + verify_overlay present "$STATE_UID" +else + verify_overlay absent "$STATE_UID" +fi diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh index 40e3a5e3b..56ba2869b 100755 --- a/bin/agent-os-container-entrypoint.sh +++ b/bin/agent-os-container-entrypoint.sh @@ -4,6 +4,11 @@ set -eu FM_HOME=${FM_HOME:-/home/agent} export FM_HOME HOME="$FM_HOME" +FM_ROOT=${FM_ROOT_OVERRIDE:-$FM_HOME/firstmate} +IMAGE_SOURCE=${AGENT_OS_IMAGE_SOURCE:-/opt/agent-os-bootstrap} +SOURCE_COMMIT=$(cat /opt/agent-os-source.commit) +SOURCE_TREE=$(cat /opt/agent-os-source.tree) +SOURCE_ORIGIN=$(cat /opt/agent-os-source.origin) mkdir -p \ "$FM_HOME/config" \ @@ -17,6 +22,44 @@ mkdir -p \ "$HOME/.bun" \ "$HOME/.cargo" +if [ ! -e "$FM_ROOT/.git" ]; then + [ ! -e "$FM_ROOT" ] || { echo "error: canonical FM_ROOT exists without Git provenance" >&2; exit 2; } + git clone --no-local "$IMAGE_SOURCE" "$FM_ROOT" + git -C "$FM_ROOT" checkout --detach "$SOURCE_COMMIT" + git -C "$FM_ROOT" remote set-url origin "$SOURCE_ORIGIN" + rm -rf "$FM_ROOT/.git/hooks" +else + [ -d "$FM_ROOT/.git" ] || { echo "error: canonical FM_ROOT Git metadata is invalid" >&2; exit 2; } + [ "$(git -C "$FM_ROOT" remote get-url origin)" = "$SOURCE_ORIGIN" ] || { + echo "error: canonical FM_ROOT origin provenance changed" >&2 + exit 2 + } + [ -z "$(git -C "$FM_ROOT" status --porcelain)" ] || { echo "error: canonical FM_ROOT is not clean" >&2; exit 2; } + if git -C "$FM_ROOT" merge-base --is-ancestor HEAD "$SOURCE_COMMIT"; then + git -C "$FM_ROOT" fetch --no-tags "$IMAGE_SOURCE" "$SOURCE_COMMIT" + git -C "$FM_ROOT" merge --ff-only "$SOURCE_COMMIT" + elif ! git -C "$FM_ROOT" merge-base --is-ancestor "$SOURCE_COMMIT" HEAD; then + echo "error: canonical FM_ROOT source transition is not fast-forward compatible" >&2 + exit 2 + fi +fi + +[ "$(git -C "$FM_ROOT" rev-parse "$SOURCE_COMMIT^{tree}")" = "$SOURCE_TREE" ] || { + echo "error: canonical FM_ROOT image source tree provenance failed" >&2 + exit 2 +} +[ -z "$(git -C "$FM_ROOT" status --porcelain)" ] || { echo "error: canonical FM_ROOT is not clean" >&2; exit 2; } +[ -z "$(git -C "$FM_ROOT" ls-files -- config data projects state .no-mistakes)" ] || { + echo "error: canonical FM_ROOT contains operational state" >&2 + exit 2 +} +find "$FM_ROOT/.git/hooks" -mindepth 1 -print -quit 2>/dev/null | grep -q . && { + echo "error: canonical FM_ROOT contains Git hooks" >&2 + exit 2 +} +export FM_ROOT_OVERRIDE="$FM_ROOT" +ln -sfn "$FM_ROOT/tools/agent-os/src/cli.ts" "$HOME/.local/bin/agent-os" + if [ -n "${AGENT_OS_PI_AUTH_FILE:-}" ]; then if [ ! -f "$AGENT_OS_PI_AUTH_FILE" ]; then echo "error: projected Pi authorization is unavailable" >&2 @@ -103,5 +146,5 @@ for tool in gh-axi chrome-devtools-axi lavish-axi; do fi done -cd /opt/agent-os +cd "$FM_ROOT" exec herdr server diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index e494c24d8..b4314d44e 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -15,6 +15,8 @@ KUBECTL=${AGENT_OS_KUBECTL:-kubectl} TEMPLATE=${AGENT_OS_CREWMATE_TEMPLATE:-/opt/agent-os/tools/agent-os/packages/firstmate/crewmate.yaml} INSTALLATION_ID="agent-os-firstmate:$NAMESPACE" OPERATION_ID=${AGENT_OS_OPERATION_ID:-"$(date -u '+%Y%m%d%H%M%S')-$$-$RANDOM"} +LOCK_NONCE=$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n') +LOCK_HOLDER_ID="$OPERATION_ID.$LOCK_NONCE" if [ ! -f "$TEMPLATE" ]; then TEMPLATE="$ROOT/tools/agent-os/packages/firstmate/crewmate.yaml" @@ -39,7 +41,12 @@ fi POD="agent-os-crewmate-$ID" PVC="$POD-home" LOCK="$POD-lifecycle" +[ "${#ID}" -le 63 ] || { echo "error: invalid crewmate id '$ID'" >&2; exit 2; } +for derived_name in "$POD" "$PVC" "$LOCK"; do + [ "${#derived_name}" -le 63 ] || { echo "error: crewmate id '$ID' makes Kubernetes resource names too long" >&2; exit 2; } +done LOCK_NAMESPACE=$NAMESPACE +LOCK_SCOPE=crewmate EXPECTED_POD="$POD"$'\t'"agent-os"$'\t'"$ID"$'\t'"$INSTALLATION_ID" EXPECTED_PVC="$PVC"$'\t'"agent-os"$'\t'"$ID"$'\t'"$INSTALLATION_ID" EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"$ID"$'\t'"$INSTALLATION_ID" @@ -96,6 +103,11 @@ pvc_record() { lock_record() { local deadline=${1:-} [ -n "$deadline" ] || deadline=$(lock_default_deadline) + if [ "$LOCK_SCOPE" = fleet ]; then + lock_kube "$deadline" get lease "$LOCK" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/lifecycle}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.spec.holderIdentity}{"\t"}{.spec.acquireTime}{"\t"}{.spec.renewTime}{"\t"}{.spec.leaseDurationSeconds}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}' + return + fi lock_kube "$deadline" get lease "$LOCK" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/crewmate}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.spec.holderIdentity}{"\t"}{.spec.acquireTime}{"\t"}{.spec.renewTime}{"\t"}{.spec.leaseDurationSeconds}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}' } @@ -161,6 +173,28 @@ render_lock() { local acquired_at=$1 renewed_at=$2 uid=${3:-} rv=${4:-} uid_value='' rv_value='' [ -z "$uid" ] || uid_value=$(yaml_string "$uid") [ -z "$rv" ] || rv_value=$(yaml_string "$rv") + if [ "$LOCK_SCOPE" = fleet ]; then + cat </dev/null 2>&1 || true fi @@ -493,7 +571,7 @@ case "$COMMAND" in create) [ -z "$CONFIRM" ] || { echo "usage: $0 create " >&2; exit 2; } validate_ai_grant - acquire_lock + acquire_lifecycle_locks preflight_create create_and_wait ;; @@ -513,14 +591,14 @@ case "$COMMAND" in ;; stop) [ -z "$CONFIRM" ] || { echo "usage: $0 stop " >&2; exit 2; } - acquire_lock + acquire_lifecycle_locks require_owned_pvc_or_absent >/dev/null quiesce_owned_home ;; restart) [ -z "$CONFIRM" ] || { echo "usage: $0 restart " >&2; exit 2; } validate_ai_grant - acquire_lock + acquire_lifecycle_locks preflight_existing_home >/dev/null quiesce_owned_home create_and_wait @@ -531,7 +609,7 @@ case "$COMMAND" in echo "error: purge requires the purge-specific --yes confirmation" >&2 exit 2 fi - acquire_lock + acquire_lifecycle_locks if ! pod=$(pod_record); then echo "error: could not prove crewmate Pod absence before purge" >&2 exit 3 diff --git a/bin/agent-os-init.sh b/bin/agent-os-init.sh index 385f5bd7d..6717e4812 100755 --- a/bin/agent-os-init.sh +++ b/bin/agent-os-init.sh @@ -2,8 +2,11 @@ # Seed PVC-backed runtime tool paths before the agent container starts. set -eu -IMAGE_USR_LOCAL=${AGENT_OS_IMAGE_USR_LOCAL:-/opt/image-usr-local} PERSISTENT_ROOT=${AGENT_OS_PERSISTENT_ROOT:-/persistent-agent} +IMAGE_MANIFEST=${AGENT_OS_IMAGE_MANIFEST:-/opt/agent-os-image-usr-local.manifest} +IMAGE_MANIFEST_SIGNATURE=${AGENT_OS_IMAGE_MANIFEST_SIGNATURE:-/opt/agent-os-image-usr-local.manifest.sha256} +MANIFEST_DIR="$PERSISTENT_ROOT/.config/agent-os" +LEGACY_ROOT="$PERSISTENT_ROOT/usr-local" mkdir -p \ "$PERSISTENT_ROOT/.config" \ @@ -13,6 +16,35 @@ mkdir -p \ "$PERSISTENT_ROOT/.pi/agent" \ "$PERSISTENT_ROOT/.bun" \ "$PERSISTENT_ROOT/.cargo" \ - "$PERSISTENT_ROOT/usr-local" + "$MANIFEST_DIR" -rsync -a "$IMAGE_USR_LOCAL/" "$PERSISTENT_ROOT/usr-local/" +(cd "$(dirname "$IMAGE_MANIFEST")" && sha256sum -c "$(basename "$IMAGE_MANIFEST_SIGNATURE")" >/dev/null) || { + echo "error: image-owned /usr/local manifest signature is invalid" >&2 + exit 2 +} + +if [ -d "$LEGACY_ROOT" ] && find "$LEGACY_ROOT" -mindepth 1 -print -quit | grep -q .; then + previous_manifest="$MANIFEST_DIR/previous-image-usr-local.manifest" + previous_signature="$MANIFEST_DIR/previous-image-usr-local.manifest.sha256" + if [ ! -f "$previous_manifest" ] || [ ! -f "$previous_signature" ] || \ + ! (cd "$MANIFEST_DIR" && sha256sum -c "$(basename "$previous_signature")" >/dev/null); then + echo "error: legacy persistent /usr/local ownership is ambiguous; migration refused" >&2 + exit 2 + fi + while IFS= read -r path; do + relative=".${path#"$LEGACY_ROOT"}" + expected=$(awk -v path="$relative" '$2 == path { print $1 }' "$previous_manifest") + [ -n "$expected" ] || { echo "error: legacy persistent /usr/local contains unowned path '$relative'" >&2; exit 2; } + actual=$(sha256sum "$path" | awk '{print $1}') + [ "$actual" = "$expected" ] || { echo "error: legacy persistent /usr/local path '$relative' changed; migration refused" >&2; exit 2; } + done < <(find "$LEGACY_ROOT" \( -type f -o -type l \) -print) + find "$LEGACY_ROOT" \( -type f -o -type l \) -delete + find "$LEGACY_ROOT" -depth -type d -empty -delete + if [ -e "$LEGACY_ROOT" ]; then + echo "error: legacy persistent /usr/local contains unsupported or unowned state" >&2 + exit 2 + fi +fi + +install -m 0600 "$IMAGE_MANIFEST" "$MANIFEST_DIR/previous-image-usr-local.manifest" +(cd "$MANIFEST_DIR" && sha256sum previous-image-usr-local.manifest > previous-image-usr-local.manifest.sha256) diff --git a/bin/agent-os-kubernetes-lease.sh b/bin/agent-os-kubernetes-lease.sh index ec349baf8..f9f7e2fd2 100644 --- a/bin/agent-os-kubernetes-lease.sh +++ b/bin/agent-os-kubernetes-lease.sh @@ -78,7 +78,7 @@ verify_lock_record() { identity=$(printf '%s' "$record" | cut -f1-4) holder=$(printf '%s' "$record" | cut -f5) uid=$(printf '%s' "$record" | cut -f9) - [ "$identity" = "$EXPECTED_LOCK" ] && [ "$holder" = "$OPERATION_ID" ] && [ -n "$uid" ] + [ "$identity" = "$EXPECTED_LOCK" ] && [ "$holder" = "$LOCK_HOLDER_ID" ] && [ -n "$uid" ] } lock_record_valid_until() { @@ -156,7 +156,7 @@ release_lock() { holder=$(printf '%s' "$record" | cut -f5) uid=$(printf '%s' "$record" | cut -f9) rv=$(printf '%s' "$record" | cut -f10) - if [ "$identity" != "$EXPECTED_LOCK" ] || [ "$holder" != "$OPERATION_ID" ] || \ + if [ "$identity" != "$EXPECTED_LOCK" ] || [ "$holder" != "$LOCK_HOLDER_ID" ] || \ [ "$uid" != "$LOCK_UID" ] || [ -z "$rv" ]; then echo "error: lifecycle Lease '$LOCK' changed ownership before release; retained" >&2 return 1 @@ -168,7 +168,7 @@ release_lock() { return 1 } if [ -n "$after" ] && [ "$(printf '%s' "$after" | cut -f9)" = "$uid" ] && \ - [ "$(printf '%s' "$after" | cut -f5)" = "$OPERATION_ID" ]; then + [ "$(printf '%s' "$after" | cut -f5)" = "$LOCK_HOLDER_ID" ]; then echo "error: lifecycle Lease '$LOCK' release precondition failed; retained" >&2 return 1 fi @@ -180,7 +180,7 @@ release_lock() { if [ -n "$after" ]; then after_holder=$(printf '%s' "$after" | cut -f5) after_uid=$(printf '%s' "$after" | cut -f9) - if [ "$after_uid" = "$uid" ] && [ "$after_holder" = "$OPERATION_ID" ]; then + if [ "$after_uid" = "$uid" ] && [ "$after_holder" = "$LOCK_HOLDER_ID" ]; then echo "error: lifecycle Lease '$LOCK' still exists after release" >&2 return 1 fi diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh index db00c29d8..00e755360 100755 --- a/bin/agent-os-kubernetes.sh +++ b/bin/agent-os-kubernetes.sh @@ -21,6 +21,9 @@ MANAGES_NAMESPACE=0 DESIRED_RBAC=none INSTALLATION_ID= OPERATION_ID=${AGENT_OS_OPERATION_ID:-"$(date -u '+%Y%m%d%H%M%S')-$$-$RANDOM"} +LOCK_NONCE=$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n') +LOCK_HOLDER_ID="$OPERATION_ID.$LOCK_NONCE" +SERVICE_ACCOUNT_NAME="agent-os-firstmate-${LOCK_NONCE:0:12}" LOCK= LOCK_NAMESPACE= EXPECTED_LOCK= @@ -92,12 +95,12 @@ render() { echo "error: Akua renderer '$AKUA' is required for Kubernetes package operations" >&2 exit 2 fi - if grep -Eq '^[[:space:]]*operationId:' "$INPUTS"; then - echo "error: operationId is reserved for the lifecycle helper" >&2 + if grep -Eq '^[[:space:]]*(operationId|serviceAccountName):' "$INPUTS"; then + echo "error: operationId and serviceAccountName are reserved for the lifecycle helper" >&2 exit 2 fi cp "$INPUTS" "$RENDER_INPUTS" - printf '\noperationId: %s\n' "$OPERATION_ID" >> "$RENDER_INPUTS" + printf '\noperationId: %s\nserviceAccountName: %s\n' "$OPERATION_ID" "$SERVICE_ACCOUNT_NAME" >> "$RENDER_INPUTS" "$AKUA" render --no-agent-mode --package "$PACKAGE" --inputs "$RENDER_INPUTS" --out "$OUT" statefulset=$(rendered_statefulset) statefulset_count=$(printf '%s\n' "$statefulset" | sed '/^$/d' | wc -l | tr -d ' ') @@ -115,6 +118,11 @@ render() { exit 2 fi NAMESPACE=$rendered_namespace + case "$NAMESPACE" in ''|*[!a-z0-9-]*|-*|*-) echo "error: rendered namespace is not a valid Kubernetes DNS label" >&2; exit 2 ;; esac + [ "${#NAMESPACE}" -le 63 ] || { echo "error: rendered namespace is too long" >&2; exit 2; } + [ "${#SERVICE_ACCOUNT_NAME}" -le 63 ] || { echo "error: generated ServiceAccount name is too long" >&2; exit 2; } + [ "${#LOCK_HOLDER_ID}" -le 255 ] || { echo "error: generated lifecycle Lease holder identity is too long" >&2; exit 2; } + [ "${#NAMESPACE}" -le 231 ] || { echo "error: namespace makes the ClusterRoleBinding name too long" >&2; exit 2; } INSTALLATION_ID="agent-os-firstmate:$NAMESPACE" if render_has_kind Namespace; then MANAGES_NAMESPACE=1 @@ -191,7 +199,7 @@ ${rv:+ resourceVersion: $rv_value} annotations: agent-os.dev/installation-id: $INSTALLATION_ID spec: - holderIdentity: $OPERATION_ID + holderIdentity: $LOCK_HOLDER_ID acquireTime: $acquired_at renewTime: $renewed_at leaseDurationSeconds: $LOCK_DURATION_SECONDS @@ -204,7 +212,7 @@ acquire_primary_lock() { LOCK=agent-os-firstmate-lifecycle elif [ "$COMMAND" = cleanup-cluster-rbac ]; then LOCK_NAMESPACE=${AGENT_OS_CLUSTER_LOCK_NAMESPACE:-kube-system} - LOCK="agent-os-firstmate-lifecycle-$NAMESPACE" + LOCK="agent-os-firstmate-lifecycle-$(sha256_text "$NAMESPACE" | cut -c1-16)" elif [ "$COMMAND" = uninstall ]; then return 0 else @@ -262,6 +270,112 @@ require_workload_owned() { fi } +workload_service_account() { + local state account identity + state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ + --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get statefulset agent-os-firstmate -o json) + identity=$(printf '%s' "$state" | jq -er --arg installation "$INSTALLATION_ID" ' + select(.metadata.name == "agent-os-firstmate") + | select(.metadata.labels["app.kubernetes.io/managed-by"] == "agent-os") + | select(.metadata.annotations["agent-os.dev/installation-id"] == $installation) + | .spec.template.spec.serviceAccountName') || { + echo "error: StatefulSet ServiceAccount identity is unverifiable" >&2 + exit 3 + } + case "$identity" in agent-os-firstmate|agent-os-firstmate-[a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9]) ;; *) echo "error: StatefulSet ServiceAccount identity is invalid" >&2; exit 3 ;; esac + printf '%s' "$identity" +} + +preflight_legacy_cluster_binding() { + local allow_owned=${1:-0} binding identity expected + binding="agent-os-firstmate-$NAMESPACE" + if ! identity=$("$KUBECTL" --context "$CONTEXT" get clusterrolebinding "$binding" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}'); then + echo "error: exact legacy ClusterRoleBinding preflight requires separately authorized cluster-scoped read access" >&2 + exit 2 + fi + expected="$binding"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" + if [ -n "$identity" ] && [ "$identity" != "$expected" ]; then + echo "error: ClusterRoleBinding '$binding' does not have the exact Agent OS installation identity" >&2 + exit 2 + fi + if [ -n "$identity" ] && [ "$allow_owned" -ne 1 ] && [ "$DESIRED_RBAC" != cluster-admin ]; then + echo "error: stale ClusterRoleBinding '$binding' must be removed through separately authorized cleanup before installation" >&2 + exit 3 + fi +} + +require_no_active_rollback_checkpoint() { + local state checkpoint + if ! command -v jq >/dev/null 2>&1; then + echo "error: jq is required to verify rollback checkpoint state before upgrade" >&2 + exit 2 + fi + state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ + --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get statefulset agent-os-firstmate -o json) + checkpoint=$(printf '%s' "$state" | jq -er --arg installation "$INSTALLATION_ID" ' + select(.metadata.name == "agent-os-firstmate") + | select(.metadata.labels["app.kubernetes.io/managed-by"] == "agent-os") + | select(.metadata.annotations["agent-os.dev/installation-id"] == $installation) + | [(.metadata.annotations["agent-os.dev/rollback-operation"] // ""), + (.metadata.annotations["agent-os.dev/rollback-target-name"] // ""), + (.metadata.annotations["agent-os.dev/rollback-target-uid"] // ""), + (.metadata.annotations["agent-os.dev/rollback-target-digest"] // "")] + | @tsv') || { echo "error: StatefulSet rollback checkpoint state is unverifiable" >&2; exit 3; } + if [ -n "${checkpoint//$'\t'/}" ]; then + echo "error: active rollback checkpoint blocks upgrade; resume rollback until exact recovery and checkpoint finalization" >&2 + exit 3 + fi +} + +verified_akua_overlay_secret() { + local expected_record=${1:-} expected_secret state secret record='' + expected_secret=$(printf '%s' "$expected_record" | cut -f1) + if ! command -v jq >/dev/null 2>&1; then + echo "error: jq is required to verify the Akua authorization overlay" >&2 + return 2 + fi + state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ + --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get statefulset agent-os-firstmate -o json) || return 3 + secret=$(printf '%s' "$state" | jq -er --arg installation "$INSTALLATION_ID" ' + select(.metadata.name == "agent-os-firstmate") + | select(.metadata.labels["app.kubernetes.io/managed-by"] == "agent-os") + | select(.metadata.annotations["agent-os.dev/installation-id"] == $installation) + | (.metadata.annotations["agent-os.dev/akua-auth-secret"] // "") as $declared + | ([.spec.template.spec.containers[]? | select(.name == "firstmate") | .env[]? | select(.name == "AKUA_AUTH_HEADER_FILE")] // []) as $env + | ([.spec.template.spec.containers[]? | select(.name == "firstmate") | .volumeMounts[]? | select(.name == "akua-auth")] // []) as $mount + | ([.spec.template.spec.volumes[]? | select(.name == "akua-auth")] // []) as $volume + | if ($declared == "" and ($env|length) == 0 and ($mount|length) == 0 and ($volume|length) == 0) then "" + elif ($declared | test("^[a-z0-9]([a-z0-9.-]{0,251}[a-z0-9])?$")) and + ($env|length) == 1 and $env[0].value == "/var/run/secrets/agent-os/akua/authorization" and + ($mount|length) == 1 and $mount[0].mountPath == "/var/run/secrets/agent-os/akua" and $mount[0].readOnly == true and + ($volume|length) == 1 and $volume[0].secret.secretName == $declared and $volume[0].secret.defaultMode == 256 + then $declared else error("unverifiable Akua authorization overlay") end') || { + echo "error: Akua authorization overlay is missing, changed, or unverifiable; upgrade blocked" >&2 + return 3 + } + if [ -n "$expected_record" ] && [ "$secret" != "$expected_secret" ]; then + echo "error: Akua authorization Secret reference changed during upgrade; upgrade blocked" >&2 + return 3 + fi + if [ -n "$secret" ]; then + record=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ + --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get secret "$secret" --ignore-not-found \ + -o 'jsonpath={.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}{"\t"}{range $key,$value := .data}{$key}{"\n"}{end}') || return 3 + if [ "$(printf '%s' "$record" | cut -f1)" != "$secret" ] || \ + [ -z "$(printf '%s' "$record" | cut -f2)" ] || [ -z "$(printf '%s' "$record" | cut -f3)" ] || \ + [ "$(printf '%s' "$record" | cut -f4-)" != authorization ]; then + echo "error: Akua authorization Secret reference is missing or unverifiable; upgrade blocked" >&2 + return 3 + fi + if [ -n "$expected_record" ] && [ "$record" != "$expected_record" ]; then + echo "error: Akua authorization Secret UID or resourceVersion changed during upgrade; upgrade blocked" >&2 + return 3 + fi + fi + printf '%s' "$record" +} + rendered_resource_field() { local file=$1 field=$2 awk -v field="$field" '$1 == field ":" { print $2; exit }' "$file" @@ -731,7 +845,7 @@ cas_patch_file() { } mutate_rendered_resource() { - local file=$1 kind name scope record expected uid rv operation current patch_file + local file=$1 kind name scope record expected uid rv operation current patch_file patch_type kind=$(rendered_resource_field "$file" kind) name=$(rendered_resource_field "$file" name) [ "$kind" != Namespace ] || return 0 @@ -763,7 +877,9 @@ mutate_rendered_resource() { return $? fi else - if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch "$kind" "$name" --type=merge --patch-file "$patch_file" >/dev/null; then + patch_type=merge + [ "$kind" != StatefulSet ] || patch_type=strategic + if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch "$kind" "$name" --type="$patch_type" --patch-file "$patch_file" >/dev/null; then report_partial_apply patch return $? fi @@ -811,9 +927,9 @@ verify_desired_rbac() { {"apiGroups":["apps"],"resources":["statefulsets"],"verbs":["get","list","watch"]}, {"apiGroups":["coordination.k8s.io"],"resources":["leases"],"verbs":["get","create","update","delete"]} ]' >/dev/null || - ! printf '%s' "$binding_json" | jq -e --arg namespace "$NAMESPACE" ' + ! printf '%s' "$binding_json" | jq -e --arg namespace "$NAMESPACE" --arg serviceAccount "$SERVICE_ACCOUNT_NAME" ' .roleRef == {"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"agent-os-firstmate-runtime"} and - .subjects == [{"kind":"ServiceAccount","name":"agent-os-firstmate","namespace":$namespace}]' >/dev/null; then + .subjects == [{"kind":"ServiceAccount","name":$serviceAccount,"namespace":$namespace}]' >/dev/null; then echo "error: desired namespace RBAC did not verify after apply" >&2 exit 2 fi @@ -1075,6 +1191,7 @@ case "$COMMAND" in echo "error: agent-os-firstmate already exists; use upgrade" >&2 exit 2 fi + preflight_legacy_cluster_binding 0 apply_rendered ;; upgrade) @@ -1090,6 +1207,17 @@ case "$COMMAND" in exit 2 fi require_workload_owned "$previous" + previous_service_account=$(workload_service_account) + case "$previous_service_account" in + agent-os-firstmate-[a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9]) + SERVICE_ACCOUNT_NAME=$previous_service_account + render + require_namespaced_resource_owned_or_absent ServiceAccount "$SERVICE_ACCOUNT_NAME" + ;; + esac + preflight_legacy_cluster_binding 1 + require_no_active_rollback_checkpoint + preserved_akua_secret_record=$(verified_akua_overlay_secret) previous_mode=$(printf '%s' "$previous" | cut -f2) previous_cleanup=$(printf '%s' "$previous" | cut -f3) cleanup_required=0 @@ -1099,6 +1227,10 @@ case "$COMMAND" in patch_workload_annotations '{"agent-os.dev/cluster-rbac-cleanup":"required"}' fi apply_rendered + verified_akua_overlay_secret "$preserved_akua_secret_record" >/dev/null + if [ "$previous_service_account" != "$SERVICE_ACCOUNT_NAME" ]; then + delete_owned_resource namespaced ServiceAccount "$previous_service_account" 180 + fi verify_desired_rbac if [ "$DESIRED_RBAC" != namespace ]; then delete_namespace_rbac @@ -1323,9 +1455,16 @@ case "$COMMAND" in preflight_rendered_resources previous=$(workload_state) require_workload_owned "$previous" optional + previous_service_account= + if [ -n "$previous" ]; then + previous_service_account=$(workload_service_account) + fi previous_mode=$(printf '%s' "$previous" | cut -f2) previous_cleanup=$(printf '%s' "$previous" | cut -f3) delete_rendered_namespaced_resources + if [ -n "$previous_service_account" ] && [ "$previous_service_account" != "$SERVICE_ACCOUNT_NAME" ]; then + delete_owned_resource namespaced ServiceAccount "$previous_service_account" 180 + fi delete_namespace_rbac if [ "$DELETE_NAMESPACE" -eq 1 ]; then delete_owned_empty_namespace diff --git a/deploy/akua/firstmate-auth-grant.yaml b/deploy/akua/firstmate-auth-grant.yaml index 20a68d2c7..4acc58106 100644 --- a/deploy/akua/firstmate-auth-grant.yaml +++ b/deploy/akua/firstmate-auth-grant.yaml @@ -1,3 +1,6 @@ +metadata: + annotations: + agent-os.dev/akua-auth-secret: __AKUA_AUTH_SECRET__ spec: template: spec: diff --git a/deploy/akua/firstmate-auth-revoke.yaml b/deploy/akua/firstmate-auth-revoke.yaml index 3ea842388..be1bde51c 100644 --- a/deploy/akua/firstmate-auth-revoke.yaml +++ b/deploy/akua/firstmate-auth-revoke.yaml @@ -1,3 +1,6 @@ +metadata: + annotations: + agent-os.dev/akua-auth-secret: null spec: template: spec: diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 8c3cca9f4..b9f5a489c 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -50,7 +50,8 @@ Set `rbac: none` only when another reviewed authority handles those runtime oper Set `rbac: cluster-admin` only for an isolated intelligence cluster after reviewing the broader ClusterRoleBinding. The persistent home PVC defaults to `20Gi` and is mounted at `/home/agent`. -`/usr/local` is a subpath of that same PVC so user-installed global tools survive Pod replacement. +Image-owned `/usr/local` remains immutable, while user-installed tools persist under `/home/agent/.local` and the other home-scoped prefixes on the PVC. +The init container authenticates an image ownership manifest and refuses ambiguous legacy `/usr/local` migrations instead of retaining stale image binaries. ## Operations diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 50d7cba78..c7c0959bf 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -62,10 +62,12 @@ assert_grep 'never create or read operational state through repo-relative' "$ROO assert_grep 'pass a provider-qualified model id' "$ROOT/.agents/skills/harness-adapters/SKILL.md" \ "Pi dispatch must preserve the selected provider route" assert_grep 'XDG_CONFIG_HOME=/home/agent/.config' "$ROOT/Dockerfile" "image must persist XDG configuration" -assert_grep 'NPM_CONFIG_PREFIX=/usr/local' "$ROOT/Dockerfile" "global npm installs must use persistent /usr/local" +assert_grep 'NPM_CONFIG_PREFIX=/home/agent/.local' "$ROOT/Dockerfile" "global npm installs must use the persistent user prefix" assert_grep 'PATH=/home/agent/.local/bin:/home/agent/.bun/bin:/home/agent/.cargo/bin:/usr/local/bin' "$ROOT/Dockerfile" \ "persistent tool prefixes must lead PATH" -assert_grep '/opt/image-usr-local' "$ROOT/Dockerfile" "image must retain a seed copy of /usr/local" +assert_grep 'agent-os-image-usr-local.manifest.sha256' "$ROOT/Dockerfile" "image must authenticate immutable /usr/local ownership" +assert_no_grep 'mountPath = "/usr/local"' "$ROOT/tools/agent-os/packages/firstmate/package.k" \ + "image-owned /usr/local must not be overlaid by persistent state" # shellcheck disable=SC2016 # Match the literal Docker build argument reference. assert_grep 'akua-dev/akua/releases/download/v${AKUA_VERSION}' "$ROOT/Dockerfile" "image must install Akua from its release" # shellcheck disable=SC2016 # Match the literal Docker build argument reference. @@ -109,6 +111,13 @@ assert_grep '.pi' "$ROOT/.dockerignore" "Pi credentials must stay out of the bui assert_grep '.codex' "$ROOT/.dockerignore" "Codex credentials must stay out of the build context" assert_grep 'node_modules' "$ROOT/.dockerignore" "host dependencies must stay out of the build context" assert_grep '.repos' "$ROOT/.dockerignore" "development source checkouts must stay out of the build context" +assert_grep '!.pi/extensions/fm-primary-pi-watch.ts' "$ROOT/.dockerignore" "tracked Pi supervision controls must enter the image" +assert_grep '!.codex/hooks.json' "$ROOT/.dockerignore" "tracked Codex supervision controls must enter the image" +assert_grep '!.opencode/plugins/fm-primary-watch-arm.js' "$ROOT/.dockerignore" "tracked OpenCode supervision controls must enter the image" +assert_grep 'agent-os-source.bundle' "$ROOT/Dockerfile" "image must bootstrap an exact source commit without host Git metadata" +assert_grep 'rev-parse HEAD^{tree}' "$ROOT/Dockerfile" "image source bootstrap must verify its exact tree" +assert_grep 'merge --ff-only' "$ROOT/bin/agent-os-container-entrypoint.sh" "persistent source transitions must be fast-forward only" +assert_grep 'FM_ROOT_OVERRIDE=' "$ROOT/bin/agent-os-container-entrypoint.sh" "runtime must use the persistent canonical Firstmate repository" assert_grep 'https://github.com/ogulcancelik/herdr/tree/v0.7.3' "$ROOT/THIRD_PARTY_NOTICES.md" \ "Herdr's exact corresponding source must be named" assert_present "$ROOT/THIRD_PARTY_SOURCES.md" \ @@ -140,8 +149,12 @@ assert_grep ' validate:' "$IMAGE_WORKFLOW" \ "pull requests must use a distinct read-only validation job" assert_grep ' publish:' "$IMAGE_WORKFLOW" \ "push and tag publication must use a distinct privileged job" -assert_grep "if: github.event_name == 'push'" "$IMAGE_WORKFLOW" \ - "the packages-write job must be restricted to protected push and tag events" +assert_grep 'needs: [behavior, provenance, validate]' "$IMAGE_WORKFLOW" \ + "the packages-write job must require exact behavior, provenance, and image gates" +assert_grep 'github.ref_protected' "$IMAGE_WORKFLOW" \ + "main publication must require GitHub protected-ref provenance" +assert_grep 'release tag must point at the exact protected-main head' "$IMAGE_WORKFLOW" \ + "tag publication must require the exact tested protected-main commit" assert_grep 'Section 13' "$ROOT/docs/herdr-compliance.md" \ "the Herdr audit must account for the network-interaction clause" assert_grep 'https://github.com/akua-dev/akua/tree/v0.8.25' "$ROOT/THIRD_PARTY_NOTICES.md" \ diff --git a/tests/agent-os-init.test.sh b/tests/agent-os-init.test.sh index 5e29c2854..ac98d5354 100755 --- a/tests/agent-os-init.test.sh +++ b/tests/agent-os-init.test.sh @@ -6,19 +6,26 @@ set -u . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" TMP=$(fm_test_tmproot agent-os-init) -mkdir -p "$TMP/source/bin" "$TMP/persistent/.local/bin" "$TMP/persistent/usr-local/bin" -printf 'baseline\n' > "$TMP/source/bin/baseline" -printf 'runtime\n' > "$TMP/persistent/usr-local/bin/runtime-added" +mkdir -p "$TMP/image" "$TMP/persistent/.local/bin" +printf 'image-manifest\n' > "$TMP/image/manifest" +(cd "$TMP/image" && sha256sum manifest > manifest.sha256) -AGENT_OS_IMAGE_USR_LOCAL="$TMP/source" \ -AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ - "$ROOT/bin/agent-os-init.sh" +AGENT_OS_IMAGE_MANIFEST="$TMP/image/manifest" \ +AGENT_OS_IMAGE_MANIFEST_SIGNATURE="$TMP/image/manifest.sha256" \ +AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" "$ROOT/bin/agent-os-init.sh" -assert_grep baseline "$TMP/persistent/usr-local/bin/baseline" "initializer must seed image tools" -assert_grep runtime "$TMP/persistent/usr-local/bin/runtime-added" "initializer must preserve runtime tools" - -for directory in .config .cache .local/bin .local/share .bun .cargo usr-local; do +for directory in .config .cache .local/bin .local/share .bun .cargo; do [ -d "$TMP/persistent/$directory" ] || fail "initializer must create persistent $directory" done +assert_grep image-manifest "$TMP/persistent/.config/agent-os/previous-image-usr-local.manifest" \ + "initializer must persist authenticated image ownership provenance" + +mkdir -p "$TMP/ambiguous/usr-local/bin" +printf 'unknown\n' > "$TMP/ambiguous/usr-local/bin/tool" +if AGENT_OS_IMAGE_MANIFEST="$TMP/image/manifest" \ + AGENT_OS_IMAGE_MANIFEST_SIGNATURE="$TMP/image/manifest.sha256" \ + AGENT_OS_PERSISTENT_ROOT="$TMP/ambiguous" "$ROOT/bin/agent-os-init.sh" >/dev/null 2>&1; then + fail "initializer must reject ambiguous legacy /usr/local ownership" +fi -pass "initializer preserves image and runtime-installed tools" +pass "initializer separates immutable image tools from persistent user tools" diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index 13a9feaea..c743caf57 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -34,7 +34,7 @@ assert_contains "$rendered" 'mountPath: /home/agent' "the primary home must moun assert_not_contains "$rendered" 'hostUsers: false' "OrbStack demo must not request unsupported Pod user namespaces" assert_contains "$rendered" 'runAsUser: 0' "primary must run as container root" assert_contains "$rendered" 'name: agent-os-init' "primary must seed persistent tools" -assert_contains "$rendered" 'mountPath: /usr/local' "primary must persist /usr/local" +assert_not_contains "$rendered" 'mountPath: /usr/local' "primary must keep image-owned /usr/local immutable" pass "OrbStack profile renders the canonical persistent primary" cat > "$FAKEBIN/kubectl" <<'SH' @@ -193,6 +193,7 @@ case " $* " in ;; *get\ lease\ agent-os-crewmate-*--ignore-not-found*-o\ jsonpath=*) id=${AGENT_OS_TEST_CREWMATE_ID:-scout-1} + lease_holder=$(awk '/holderIdentity:/ { holder=$2 } END { print holder }' "${AGENT_OS_STDIN_LOG:-$AGENT_OS_TEST_LOG.stdin}") case "${AGENT_OS_TEST_LOCK_STATE:-free}" in held) printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tother-operation\t2099-01-01T00:00:00Z\t2099-01-01T00:00:00Z\t300\tuid-lock-other\trv-lock-other' "$id" "$id" ;; foreign) printf 'agent-os-crewmate-%s-lifecycle\tother\tother\tother-installation\tother-operation\t2099-01-01T00:00:00Z\t2099-01-01T00:00:00Z\t300\tuid-lock-foreign\trv-lock-foreign' "$id" ;; @@ -203,8 +204,8 @@ case " $* " in elif grep -F ' replace -f -' "$AGENT_OS_TEST_LOG" >/dev/null; then acquire_time=$(awk '/acquireTime:/ { value=$2 } END { print value }' "${AGENT_OS_STDIN_LOG:-$AGENT_OS_TEST_LOG.stdin}") renew_time=$(awk '/renewTime:/ { value=$2 } END { print value }' "${AGENT_OS_STDIN_LOG:-$AGENT_OS_TEST_LOG.stdin}") - printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\toperation-test\t%s\t%s\t%s\tuid-lock-expired\trv-lock-taken' \ - "$id" "$id" "$acquire_time" "$renew_time" "${AGENT_OS_LOCK_DURATION_SECONDS:-300}" + printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\t%s\t%s\t%s\t%s\tuid-lock-expired\trv-lock-taken' \ + "$id" "$id" "$lease_holder" "$acquire_time" "$renew_time" "${AGENT_OS_LOCK_DURATION_SECONDS:-300}" else printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\tother-operation\t2000-01-01T00:00:00Z\t2000-01-01T00:00:00Z\t300\tuid-lock-expired\t%s' "$id" "$id" "${AGENT_OS_TEST_LOCK_RV:-rv-lock-expired}" fi @@ -224,8 +225,8 @@ case " $* " in renew_rv=rv-lock-renewed [ "${AGENT_OS_TEST_RENEW_READBACK:-exact}" != wrong ] || renew_time=2000-01-01T00:00:00Z fi - printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\toperation-test\t%s\t%s\t%s\tuid-lock\t%s' \ - "$id" "$id" "$acquire_time" "$renew_time" "${AGENT_OS_LOCK_DURATION_SECONDS:-300}" "$renew_rv" + printf 'agent-os-crewmate-%s-lifecycle\tagent-os\t%s\tagent-os-firstmate:agent-os-demo\t%s\t%s\t%s\t%s\tuid-lock\t%s' \ + "$id" "$id" "$lease_holder" "$acquire_time" "$renew_time" "${AGENT_OS_LOCK_DURATION_SECONDS:-300}" "$renew_rv" fi ;; esac @@ -235,7 +236,7 @@ case " $* " in last_lock_write=$(grep -Fn 'stdin-kind Lease' "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d: -f1) last_lock_delete=$(grep -Fn "/leases/$lock_name" "$AGENT_OS_TEST_LOG" | grep 'delete --raw' | tail -n 1 | cut -d: -f1) if [ "${AGENT_OS_TEST_PRIMARY_LOCK_STATE:-free}" = expired ] && ! grep -F ' replace -f -' "$AGENT_OS_TEST_LOG" >/dev/null; then - printf '%s\tagent-os\tprimary\tagent-os-firstmate:%s\tother-operation\t2000-01-01T00:00:00Z\t2000-01-01T00:00:00Z\t300\tuid-primary-lock\t%s' "$lock_name" "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" "${AGENT_OS_TEST_PRIMARY_LOCK_RV:-rv-primary-lock}" + printf '%s\tagent-os\tprimary\tagent-os-firstmate:%s\tother-operation\t2000-01-01T00:00:00Z\t2000-01-01T00:00:00Z\t300\tuid-primary-lock\t%s' "$lock_name" "${AGENT_OS_TEST_NAMESPACE:-${AGENT_OS_NAMESPACE:-portable-agent-os}}" "${AGENT_OS_TEST_PRIMARY_LOCK_RV:-rv-primary-lock}" elif [ -n "$last_lock_write" ] && { [ -z "$last_lock_delete" ] || [ "$last_lock_write" -gt "$last_lock_delete" ]; }; then lock_stdin=${AGENT_OS_STDIN_LOG:-$AGENT_OS_TEST_LOG.stdin} lock_holder=$(awk '/holderIdentity:/ { holder=$2 } END { print holder }' "$lock_stdin") @@ -246,7 +247,7 @@ case " $* " in lock_rv=rv-primary-lock-renewed fi printf '%s\tagent-os\tprimary\tagent-os-firstmate:%s\t%s\t%s\t%s\t%s\tuid-primary-lock\t%s' \ - "$lock_name" "${AGENT_OS_TEST_NAMESPACE:-portable-agent-os}" "$lock_holder" "$lock_acquired" "$lock_renewed" \ + "$lock_name" "${AGENT_OS_TEST_NAMESPACE:-${AGENT_OS_NAMESPACE:-portable-agent-os}}" "$lock_holder" "$lock_acquired" "$lock_renewed" \ "${AGENT_OS_LOCK_DURATION_SECONDS:-300}" "$lock_rv" fi ;; @@ -302,7 +303,7 @@ case " $* " in esac ;; *" get StatefulSet agent-os-firstmate --ignore-not-found -o jsonpath="*|\ - *" get ServiceAccount agent-os-firstmate --ignore-not-found -o jsonpath="*|\ + *" get ServiceAccount agent-os-firstmate"*" --ignore-not-found -o jsonpath="*|\ *" get PersistentVolumeClaim agent-os-firstmate-home --ignore-not-found -o jsonpath="*|\ *" get Service agent-os-firstmate --ignore-not-found -o jsonpath="*|\ *" get Role agent-os-firstmate-runtime --ignore-not-found -o jsonpath="*|\ @@ -311,7 +312,7 @@ case " $* " in name='' case " $* " in *" StatefulSet "*) kind=StatefulSet; name=agent-os-firstmate ;; - *" ServiceAccount "*) kind=ServiceAccount; name=agent-os-firstmate ;; + *" ServiceAccount "*) kind=ServiceAccount; name=$(printf '%s\n' "$*" | sed -n 's/.* get ServiceAccount \([^ ]*\) .*/\1/p') ;; *" PersistentVolumeClaim "*) kind=PersistentVolumeClaim; name=agent-os-firstmate-home ;; *" Service "*) kind=Service; name=agent-os-firstmate ;; *" RoleBinding "*) kind=RoleBinding; name=agent-os-firstmate-runtime ;; @@ -356,6 +357,16 @@ case " $* " in rollback_update=agent-os-firstmate-current fi checkpoint_annotations='' + akua_annotations='' + akua_env='' + akua_mount='' + akua_volume='' + if [ -n "${AGENT_OS_TEST_AKUA_OVERLAY_SECRET:-}" ]; then + akua_annotations=$(printf ',"agent-os.dev/akua-auth-secret":"%s"' "$AGENT_OS_TEST_AKUA_OVERLAY_SECRET") + akua_env=',{"name":"AKUA_AUTH_HEADER_FILE","value":"/var/run/secrets/agent-os/akua/authorization"}' + akua_mount=',{"name":"akua-auth","mountPath":"/var/run/secrets/agent-os/akua","readOnly":true}' + akua_volume=$(printf ',{"name":"akua-auth","secret":{"secretName":"%s","defaultMode":256}}' "$AGENT_OS_TEST_AKUA_OVERLAY_SECRET") + fi if [ -n "${AGENT_OS_TEST_ROLLBACK_CHECKPOINT_DIGEST:-}" ]; then checkpoint_annotations=$(printf ',"agent-os.dev/rollback-operation":"checkpoint-operation","agent-os.dev/rollback-target-name":"agent-os-firstmate-previous","agent-os.dev/rollback-target-uid":"uid-revision-previous","agent-os.dev/rollback-target-digest":"%s"' "$AGENT_OS_TEST_ROLLBACK_CHECKPOINT_DIGEST") elif checkpoint_call=$(grep 'patch StatefulSet agent-os-firstmate --type=merge' "$AGENT_OS_TEST_LOG" | grep 'rollback-target-digest' | head -n 1); then @@ -376,8 +387,14 @@ case " $* " in checkpoint_annotations=$(printf '%s' "$checkpoint_annotations" | \ sed 's/rollback-operation":"[^"]*/rollback-operation":"other-operation/') fi - printf '{"metadata":{"name":"agent-os-firstmate","uid":"uid-statefulset","resourceVersion":"%s","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"%s}},"status":{"currentRevision":"%s","updateRevision":"%s"}}\n' \ - "${AGENT_OS_TEST_RESOURCE_RV:-rv-statefulset}" "$checkpoint_annotations" "$rollback_current" "$rollback_update" + printf '{"metadata":{"name":"agent-os-firstmate","uid":"uid-statefulset","resourceVersion":"%s","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"%s%s}},"spec":{"template":{"spec":{"serviceAccountName":"%s","containers":[{"name":"firstmate","env":[%s],"volumeMounts":[%s]}],"volumes":[%s]}}},"status":{"currentRevision":"%s","updateRevision":"%s"}}\n' \ + "${AGENT_OS_TEST_RESOURCE_RV:-rv-statefulset}" "$checkpoint_annotations" "$akua_annotations" "${AGENT_OS_TEST_WORKLOAD_SERVICE_ACCOUNT:-agent-os-firstmate}" "${akua_env#,}" "${akua_mount#,}" "${akua_volume#,}" "$rollback_current" "$rollback_update" + ;; + *" get secret "*" --ignore-not-found -o jsonpath="*) + secret_name=$(printf '%s\n' "$*" | sed -n 's/.* get secret \([^ ]*\) .*/\1/p') + if [ "$secret_name" = "${AGENT_OS_TEST_AKUA_OVERLAY_SECRET:-}" ]; then + printf '%s\tuid-secret\trv-secret\tauthorization' "$secret_name" + fi ;; *" get controllerrevisions.apps -o json "*) if [ "${AGENT_OS_TEST_ROLLBACK_RENUMBERED:-0}" = 1 ]; then @@ -394,10 +411,12 @@ case " $* " in fi ;; *" get rolebinding agent-os-firstmate-runtime -o json "*) + service_account=$(grep 'akua-input-service-account ' "$AGENT_OS_TEST_LOG" | tail -n 1 | awk '{print $2}') + [ -n "$service_account" ] || service_account=agent-os-firstmate if [ "${AGENT_OS_TEST_RBAC_STATE:-exact}" = extra-subject ]; then - printf '%s\n' '{"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"agent-os-firstmate-runtime"},"subjects":[{"kind":"ServiceAccount","name":"agent-os-firstmate","namespace":"portable-agent-os"},{"kind":"ServiceAccount","name":"foreign","namespace":"portable-agent-os"}]}' + printf '{"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"agent-os-firstmate-runtime"},"subjects":[{"kind":"ServiceAccount","name":"%s","namespace":"portable-agent-os"},{"kind":"ServiceAccount","name":"foreign","namespace":"portable-agent-os"}]}\n' "$service_account" else - printf '%s\n' '{"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"agent-os-firstmate-runtime"},"subjects":[{"kind":"ServiceAccount","name":"agent-os-firstmate","namespace":"portable-agent-os"}]}' + printf '{"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"agent-os-firstmate-runtime"},"subjects":[{"kind":"ServiceAccount","name":"%s","namespace":"portable-agent-os"}]}\n' "$service_account" fi ;; *" get role agent-os-firstmate-runtime -o jsonpath="*) @@ -501,12 +520,19 @@ assert_grep 'acquireTime:' "$STDIN_LOG" \ "lifecycle Leases must record their acquisition time" assert_grep 'renewTime:' "$STDIN_LOG" \ "lifecycle Leases must record renewable expiry evidence" +fleet_lock_line=$(grep -Fn 'name: agent-os-firstmate-lifecycle' "$STDIN_LOG" | head -n 1 | cut -d: -f1) +crewmate_lock_line=$(grep -Fn 'name: agent-os-crewmate-scout-1-lifecycle' "$STDIN_LOG" | head -n 1 | cut -d: -f1) +[ -n "$fleet_lock_line" ] && [ -n "$crewmate_lock_line" ] && [ "$fleet_lock_line" -lt "$crewmate_lock_line" ] || \ + fail "crewmate mutation must enter the installation-wide barrier before its resource lock" +lease_holder=$(awk '/holderIdentity:/ { print $2; exit }' "$STDIN_LOG") +case "$lease_holder" in operation-test.*) ;; *) fail "lifecycle Lease identity must add an internal per-invocation nonce" ;; esac +pass "crewmate mutations share the installation-wide lifecycle barrier" assert_grep 'automountServiceAccountToken: false' "$STDIN_LOG" "children must not receive Kubernetes credentials" assert_grep 'claimName: agent-os-crewmate-scout-1-home' "$STDIN_LOG" "child work must use its own PVC" assert_no_grep 'hostUsers: false' "$STDIN_LOG" "OrbStack children must not request unsupported Pod user namespaces" assert_grep 'runAsUser: 0' "$STDIN_LOG" "children must run as container root" assert_grep 'name: agent-os-init' "$STDIN_LOG" "children must seed persistent tools" -assert_grep 'mountPath: /usr/local' "$STDIN_LOG" "children must persist /usr/local" +assert_no_grep 'mountPath: /usr/local' "$STDIN_LOG" "children must keep image-owned /usr/local immutable" assert_grep 'mountPath: /var/run/secrets/agent-os/pi' "$STDIN_LOG" \ "children must mount AI authorization outside writable Pi state" assert_grep 'name: AGENT_OS_PI_AUTH_FILE' "$STDIN_LOG" \ @@ -652,6 +678,9 @@ grep -Fqx 'kubectl -n agent-os-demo --request-timeout=5s delete --raw /api/v1/na fail "stop must UID-precondition deletion of the exactly owned crewmate Pod" grep -F 'wait --for=delete pod/agent-os-crewmate-scout-1' "$CALLS" | grep -F -- '--request-timeout=' >/dev/null || \ fail "stop must prove Pod absence within its total deletion deadline" +if grep -F 'wait --for=delete pod/agent-os-crewmate-scout-1' "$CALLS" | grep -F -- '--timeout=5s' >/dev/null; then + fail "stop must use the remaining operation deadline rather than the request ceiling for deletion polling" +fi delete_line=$(grep -Fn '/pods/agent-os-crewmate-scout-1' "$CALLS" | grep 'delete --raw' | head -n 1 | cut -d: -f1) invalidate_line=$(grep -Fn 'patch pvc agent-os-crewmate-scout-1-home --type=merge' "$CALLS" | head -n 1 | cut -d: -f1) quiesced_line=$(grep -Fn 'patch pvc agent-os-crewmate-scout-1-home --type=merge' "$CALLS" | tail -n 1 | cut -d: -f1) @@ -931,6 +960,13 @@ if run_launcher create 'Bad_ID' >/dev/null 2>&1; then fi pass "crewmate IDs are validated before kubectl" +: > "$CALLS" +if run_launcher create 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' >/dev/null 2>&1; then + fail "crewmate IDs that overflow a derived Lease name must be rejected" +fi +[ ! -s "$CALLS" ] || fail "derived Kubernetes names must be validated before kubectl" +pass "all crewmate-derived Kubernetes names are validated before cluster calls" + if PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_STDIN_LOG="$STDIN_LOG" \ AGENT_OS_NAMESPACE=agent-os-demo AGENT_OS_IMAGE=agent-os:local-test "$LAUNCHER" status scout-1 >/dev/null 2>&1; then fail "host execution without an explicit context must be rejected" @@ -974,7 +1010,10 @@ mkdir -p "$out" rbac=$(awk '/^rbac:/{print $2}' "$inputs") namespace=$(awk '/^namespace:/{print $2}' "$inputs") operation=$(awk '/^operationId:/{print $2}' "$inputs") +service_account=$(awk '/^serviceAccountName:/{print $2}' "$inputs") +[ -n "$service_account" ] || service_account=agent-os-firstmate printf 'akua-input-operation %s\n' "$operation" >> "$AGENT_OS_TEST_LOG" +printf 'akua-input-service-account %s\n' "$service_account" >> "$AGENT_OS_TEST_LOG" create_namespace=$(awk '/^createNamespace:/{print $2}' "$inputs") [ -n "$create_namespace" ] || create_namespace=true cat > "$out/00-pvc.yaml" < "$out/$file.yaml" </dev/null || \ fail "primary mutations must hold an exact-owned Kubernetes Lease" grep -Fqx 'akua-input-operation operation-test' "$CALLS" || \ fail "generic install must label every resource with its unique operation identity" +generated_service_account=$(grep 'akua-input-service-account ' "$CALLS" | tail -n 1 | awk '{print $2}') +case "$generated_service_account" in agent-os-firstmate-[a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9]) ;; \ + *) fail "install must use a fresh non-legacy ServiceAccount identity" ;; esac grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os rollout status statefulset/agent-os-firstmate --timeout=180s' "$CALLS" || \ fail "generic install must wait for the rendered Firstmate StatefulSet" if grep -F 'delete clusterrolebinding' "$CALLS" >/dev/null; then @@ -1099,6 +1143,17 @@ if grep -F 'delete clusterrolebinding' "$CALLS" >/dev/null; then fi pass "generic install serializes create-only canonical package mutations" +: > "$CALLS" +stale_install_out='' +stale_install_rc=0 +stale_install_out=$(AGENT_OS_TEST_CLUSTER_RBAC_STATE=owned run_generic install 2>&1) || stale_install_rc=$? +[ "$stale_install_rc" -eq 3 ] || fail "stale cluster authority must block a namespace reinstall: $stale_install_out" +assert_contains "$stale_install_out" 'must be removed through separately authorized cleanup' \ + "namespace reinstall must not reactivate a deterministic legacy cluster grant" +assert_no_grep 'create -f .*serviceaccount.yaml' "$CALLS" \ + "legacy cluster authority must fail before a new ServiceAccount is created" +pass "fresh installs preflight deterministic legacy cluster authority" + PARTIAL_CLUSTER_INPUTS="$TMP/partial-cluster-inputs.yaml" sed 's/rbac: namespace/rbac: cluster-admin/' "$GENERIC_INPUTS" > "$PARTIAL_CLUSTER_INPUTS" : > "$CALLS" @@ -1249,9 +1304,10 @@ verify_line=$(grep -Fn 'kubectl --context kind-agent-os -n portable-agent-os get [ "$marker_line" -lt "$apply_line" ] && [ "$apply_line" -lt "$rollout_line" ] && \ [ "$rollout_line" -lt "$verify_line" ] || \ fail "downgrade must apply and roll out desired namespaced RBAC before privileged cleanup is requested" -if grep -E 'kubectl .* (get|delete) clusterrolebinding' "$CALLS" >/dev/null; then - fail "routine namespace upgrade must never request cluster-wide RBAC authority" -fi +assert_grep 'get clusterrolebinding agent-os-firstmate-portable-agent-os --ignore-not-found' "$CALLS" \ + "namespace upgrades must perform the separately authorized deterministic legacy-grant preflight" +assert_no_grep 'delete clusterrolebinding' "$CALLS" \ + "routine namespace upgrades must not delete cluster-scoped authority" assert_contains "$cleanup_out" 'cleanup-cluster-rbac --yes' \ "downgrade must print the exact separately confirmed privileged cleanup command" assert_contains "$cleanup_out" 'clusterrolebinding/agent-os-firstmate-portable-agent-os absent' \ @@ -1453,6 +1509,42 @@ assert_no_grep 'patch StatefulSet agent-os-firstmate' "$CALLS" \ "foreign stale RBAC must fail before desired resources mutate" pass "all RBAC modes preflight deterministic stale resources" +: > "$CALLS" +active_checkpoint_out='' +active_checkpoint_rc=0 +active_checkpoint_digest=$(printf '%s' '{"metadata":{"labels":{"rollback":"previous"}}}' | jq -cS . | shasum -a 256 | awk '{print $1}') +active_checkpoint_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_ROLLBACK_CHECKPOINT_DIGEST="$active_checkpoint_digest" \ + run_generic upgrade 2>&1) || active_checkpoint_rc=$? +[ "$active_checkpoint_rc" -eq 3 ] || fail "active rollback checkpoint must block upgrade: $active_checkpoint_out" +assert_contains "$active_checkpoint_out" 'active rollback checkpoint blocks upgrade' \ + "upgrade must require exact rollback recovery and checkpoint finalization" +assert_no_grep 'patch StatefulSet agent-os-firstmate --type=strategic --patch-file' "$CALLS" \ + "blocked upgrade must not mutate the StatefulSet template" +pass "upgrade refuses stale rollback checkpoints" + +: > "$CALLS" +AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_AKUA_OVERLAY_SECRET=akua-auth \ + run_generic upgrade +assert_grep 'patch StatefulSet agent-os-firstmate --type=strategic --patch-file' "$CALLS" \ + "upgrade must preserve the verified authorization overlay in its StatefulSet CAS" +assert_grep 'get secret akua-auth --ignore-not-found' "$CALLS" \ + "upgrade must verify the namespace-local Secret reference without reading Secret bytes" +assert_no_grep 'authorization.*Bearer\|auth.json' "$STDIN_LOG" \ + "upgrade evidence must never contain Secret bytes" +pass "upgrade preserves verified Akua Secret-reference overlays" + +: > "$CALLS" +AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace \ + AGENT_OS_TEST_WORKLOAD_SERVICE_ACCOUNT=agent-os-firstmate-bbbbbbbbbbbb run_generic upgrade +[ "$(grep 'akua-input-service-account ' "$CALLS" | tail -n 1 | awk '{print $2}')" = agent-os-firstmate-bbbbbbbbbbbb ] || \ + fail "upgrade retries must reuse the verified installation ServiceAccount identity" +assert_no_grep '/serviceaccounts/agent-os-firstmate-bbbbbbbbbbbb' "$CALLS" \ + "upgrade retries must not delete their active installation ServiceAccount" +pass "upgrade reuses fresh installation ServiceAccount identity" + : > "$CALLS" AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ AGENT_OS_TEST_WORKLOAD_STATE=namespace run_generic rollback @@ -1716,6 +1808,11 @@ assert_grep '"uid":"uid-namespace","resourceVersion":"rv-namespace"' "$STDIN_LOG "namespace deletion must bind to the final observed namespace identity" grep -Fq 'kubectl --context kind-agent-os api-resources --verbs=list --namespaced -o name' "$CALLS" || \ fail "optional namespace deletion must inventory every listable namespaced resource type" +namespace_lock_line=$(grep -Fn 'get lease agent-os-firstmate-lifecycle' "$CALLS" | head -n 1 | cut -d: -f1) +namespace_inventory_line=$(grep -Fn 'api-resources --verbs=list --namespaced -o name' "$CALLS" | head -n 1 | cut -d: -f1) +[ -n "$namespace_lock_line" ] && [ -n "$namespace_inventory_line" ] && \ + [ "$namespace_lock_line" -lt "$namespace_inventory_line" ] || \ + fail "namespace deletion must inventory only after holding the installation-wide barrier" pass "optional namespace deletion proves ownership and no foreign resources" : > "$CALLS" diff --git a/tests/agent-os-packages.test.sh b/tests/agent-os-packages.test.sh index 29abe879a..fe038cb09 100755 --- a/tests/agent-os-packages.test.sh +++ b/tests/agent-os-packages.test.sh @@ -26,8 +26,8 @@ assert_grep 'kind = "ServiceAccount"' "$FIRSTMATE/package.k" \ "firstmate package must create its cluster identity" assert_grep 'herdr", "status", "--json"' "$FIRSTMATE/package.k" \ "Firstmate readiness must prove the Herdr server is responsive" -assert_grep 'mountPath = "/usr/local"' "$FIRSTMATE/package.k" \ - "Firstmate-installed tools must persist" +assert_no_grep 'mountPath = "/usr/local"' "$FIRSTMATE/package.k" \ + "image-owned Firstmate tools must remain immutable" assert_no_grep 'image: str = ' "$FIRSTMATE/package.k" \ "the portable package must require an explicit immutable image digest" @@ -71,12 +71,20 @@ assert_grep "\$patch: delete" "$ROOT/deploy/akua/firstmate-auth-revoke.yaml" \ "Akua overlay must define explicit authorization mount cleanup" assert_grep 'mountPath: /var/run/secrets/agent-os/akua' "$ROOT/deploy/akua/firstmate-auth-revoke.yaml" \ "Akua overlay cleanup must use the strategic merge key for volume mounts" -assert_grep "kubectl --context \"\$context\" -n \"\$namespace\" patch statefulset agent-os-firstmate --type=strategic --patch-file \"\$grant_patch\"" \ +assert_grep 'bin/agent-os-akua-auth.sh grant "$secret_name"' \ "$ROOT/.agents/skills/akua-intelligence-bootstrap/SKILL.md" \ - "Akua authorization grant must pin context, namespace, target, patch type, and patch file" -assert_grep "kubectl --context \"\$context\" -n \"\$namespace\" patch statefulset agent-os-firstmate --type=strategic --patch-file \"\$revoke_patch\"" \ + "Akua authorization grant must use serialized exact-target integration" +assert_grep 'bin/agent-os-akua-auth.sh revoke "$secret_name"' \ "$ROOT/.agents/skills/akua-intelligence-bootstrap/SKILL.md" \ - "Akua authorization revocation must use the same explicit targeting contract" + "Akua authorization revocation must use the same serialized integration" +assert_grep 'agent-os.dev/akua-auth-secret' "$ROOT/deploy/akua/firstmate-auth-grant.yaml" \ + "Akua authorization overlays must carry a verifiable non-secret reference marker" +[ -x "$ROOT/bin/agent-os-akua-auth.sh" ] || \ + fail "Akua authorization mutations must use the shipped serialized helper" +assert_grep 'resourceVersion:' "$ROOT/bin/agent-os-akua-auth.sh" \ + "Akua authorization mutations must carry StatefulSet CAS evidence" +assert_grep 'get pod agent-os-firstmate-0 -o json' "$ROOT/bin/agent-os-akua-auth.sh" \ + "Akua authorization mutations must verify the exact-owned rollout Pod" assert_grep 'a missing credential grant is rejected before any mate resource is created' "$ROOT/docs/agent-evals.md" \ "the eval contract must match fail-closed runtime authorization" assert_grep '@sha256:' "$FIRSTMATE/inputs.example.yaml" \ diff --git a/tools/agent-os/packages/firstmate/crewmate.yaml b/tools/agent-os/packages/firstmate/crewmate.yaml index f529a2870..88316a903 100644 --- a/tools/agent-os/packages/firstmate/crewmate.yaml +++ b/tools/agent-os/packages/firstmate/crewmate.yaml @@ -76,9 +76,6 @@ spec: volumeMounts: - name: home mountPath: /home/agent - - name: home - mountPath: /usr/local - subPath: usr-local - name: ai-auth mountPath: /var/run/secrets/agent-os/pi readOnly: true diff --git a/tools/agent-os/packages/firstmate/package.k b/tools/agent-os/packages/firstmate/package.k index 30ceb7a2c..09492ebc5 100644 --- a/tools/agent-os/packages/firstmate/package.k +++ b/tools/agent-os/packages/firstmate/package.k @@ -16,10 +16,12 @@ schema Input: cpuLimit: str = "4" memoryLimit: str = "8Gi" operationId: str = "" + serviceAccountName: str = "agent-os-firstmate" check: allowMutableImage or regex.match(image, r"^((([a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*|(\[(?:(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}|(?:[0-9A-Fa-f]{1,4}:){1,7}:|(?:[0-9A-Fa-f]{1,4}:){1,6}:[0-9A-Fa-f]{1,4}|(?:[0-9A-Fa-f]{1,4}:){1,5}(?::[0-9A-Fa-f]{1,4}){1,2}|(?:[0-9A-Fa-f]{1,4}:){1,4}(?::[0-9A-Fa-f]{1,4}){1,3}|(?:[0-9A-Fa-f]{1,4}:){1,3}(?::[0-9A-Fa-f]{1,4}){1,4}|(?:[0-9A-Fa-f]{1,4}:){1,2}(?::[0-9A-Fa-f]{1,4}){1,5}|[0-9A-Fa-f]{1,4}:(?:(?::[0-9A-Fa-f]{1,4}){1,6})|:(?:(?::[0-9A-Fa-f]{1,4}){1,7}|:))\]))(:[0-9]+)?)/)?[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*(\/[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*)*(:[A-Za-z0-9_][A-Za-z0-9_.-]{0,127})?@sha256:[0-9a-f]{64}$") or regex.match(image, r"^\[(?:(?:[0-9A-Fa-f]{1,4}:){6}|::(?:[0-9A-Fa-f]{1,4}:){0,5}|[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}:){0,4}|(?:[0-9A-Fa-f]{1,4}:){1}[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}:){0,3}|(?:[0-9A-Fa-f]{1,4}:){2}[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}:){0,2}|(?:[0-9A-Fa-f]{1,4}:){3}[0-9A-Fa-f]{1,4}::(?:[0-9A-Fa-f]{1,4}:){0,1}|(?:[0-9A-Fa-f]{1,4}:){4}[0-9A-Fa-f]{1,4}::)(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}\](:[0-9]+)?\/[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*(\/[a-z0-9]+(([._]|__|-+)[a-z0-9]+)*)*(:[A-Za-z0-9_][A-Za-z0-9_.-]{0,127})?@sha256:[0-9a-f]{64}$"), "image must use one complete OCI reference with an immutable @sha256 digest unless the local profile explicitly allows a mutable image" operationId == "" or regex.match(operationId, r"^[a-z0-9]([a-z0-9.-]{0,61}[a-z0-9])?$"), "operationId must be empty or a valid Kubernetes label value" + regex.match(serviceAccountName, r"^agent-os-firstmate(-[a-z0-9]{12})?$"), "serviceAccountName must be the deterministic legacy name or a lifecycle-generated installation identity" input: Input = ctx.input() @@ -92,7 +94,7 @@ namespaceRbacResources = [ } subjects = [{ kind = "ServiceAccount" - name = "agent-os-firstmate" + name = input.serviceAccountName namespace = input.namespace }] }, @@ -113,7 +115,7 @@ clusterAdminResources = [{ } subjects = [{ kind = "ServiceAccount" - name = "agent-os-firstmate" + name = input.serviceAccountName namespace = input.namespace }] }] if input.rbac == "cluster-admin" else [] @@ -123,7 +125,7 @@ resources = namespaceResources + [ apiVersion = "v1" kind = "ServiceAccount" metadata = { - name = "agent-os-firstmate" + name = input.serviceAccountName namespace = input.namespace labels = labels annotations = ownershipAnnotations @@ -182,7 +184,7 @@ resources = namespaceResources + [ } } spec = { - serviceAccountName = "agent-os-firstmate" + serviceAccountName = input.serviceAccountName securityContext = { fsGroup = 0 runAsGroup = 0 @@ -226,7 +228,6 @@ resources = namespaceResources + [ } volumeMounts = [ { name = "home", mountPath = "/home/agent" }, - { name = "home", mountPath = "/usr/local", subPath = "usr-local" }, ] }] volumes = [{ From 881cfc8131ae594487d1f8d92107fd85930ea227 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 15:37:42 +0200 Subject: [PATCH 35/56] no-mistakes(review): Captain, harden image provenance and Kubernetes lifecycle safety --- .dockerignore | 3 + .github/workflows/agent-os-image.yml | 14 ++-- Dockerfile | 63 ++++++++++----- bin/agent-os-akua-auth.sh | 59 +++++++++++--- bin/agent-os-container-entrypoint.sh | 58 ++++++++++++-- bin/agent-os-crewmate.sh | 6 +- bin/agent-os-kubernetes-control.sh | 20 +++++ bin/agent-os-kubernetes.sh | 116 ++++++++++++++++++++++----- bin/agent-os-local.sh | 16 +++- bin/agent-os-source-bundle.sh | 29 +++++++ tests/agent-os-container.test.sh | 22 ++++- tests/agent-os-kubernetes.test.sh | 42 +++++++--- tests/agent-os-local.test.sh | 38 ++++++--- 13 files changed, 398 insertions(+), 88 deletions(-) create mode 100755 bin/agent-os-kubernetes-control.sh create mode 100755 bin/agent-os-source-bundle.sh diff --git a/.dockerignore b/.dockerignore index 121fae01b..aa14cd50a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,6 +2,7 @@ .github .pi/* !.pi/extensions/ +.pi/extensions/* !.pi/extensions/fm-primary-pi-watch.ts !.pi/extensions/fm-primary-turnend-guard.ts .codex/* @@ -11,11 +12,13 @@ !.claude/skills .grok/* !.grok/hooks/ +.grok/hooks/* !.grok/hooks/fm-primary-cd-check.json !.grok/hooks/fm-primary-pretool-check.json !.grok/hooks/fm-primary-turnend-guard.json .opencode/* !.opencode/plugins/ +.opencode/plugins/* !.opencode/plugins/fm-primary-cd-check.js !.opencode/plugins/fm-primary-pretool-check.js !.opencode/plugins/fm-primary-turnend-guard.js diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index fafe0afbc..626d164de 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -64,9 +64,8 @@ jobs: id: source run: | set -eu - git bundle create image/agent-os-source.bundle HEAD - echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - echo "tree=$(git rev-parse HEAD^{tree})" >> "$GITHUB_OUTPUT" + bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" + cat "$RUNNER_TEMP/source-metadata" >> "$GITHUB_OUTPUT" - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 @@ -85,6 +84,8 @@ jobs: build-args: | AGENT_OS_SOURCE_COMMIT=${{ steps.source.outputs.commit }} AGENT_OS_SOURCE_TREE=${{ steps.source.outputs.tree }} + AGENT_OS_SOURCE_BRANCH=${{ steps.source.outputs.branch }} + AGENT_OS_SOURCE_ORIGIN=${{ steps.source.outputs.origin }} publish: if: github.event_name == 'push' @@ -103,9 +104,8 @@ jobs: id: source run: | set -eu - git bundle create image/agent-os-source.bundle HEAD - echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - echo "tree=$(git rev-parse HEAD^{tree})" >> "$GITHUB_OUTPUT" + bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" + cat "$RUNNER_TEMP/source-metadata" >> "$GITHUB_OUTPUT" - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 @@ -144,6 +144,8 @@ jobs: build-args: | AGENT_OS_SOURCE_COMMIT=${{ steps.source.outputs.commit }} AGENT_OS_SOURCE_TREE=${{ steps.source.outputs.tree }} + AGENT_OS_SOURCE_BRANCH=${{ steps.source.outputs.branch }} + AGENT_OS_SOURCE_ORIGIN=${{ steps.source.outputs.origin }} - name: Record published image digest run: | diff --git a/Dockerfile b/Dockerfile index 95337dfcb..d480f8fd5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,40 @@ +FROM node:24-trixie-slim@sha256:366fdef91728b1b7fa18c84fba63b6e79ed77b7e10cc206878e9705da4d7b169 AS source-bootstrap + +ARG AGENT_OS_SOURCE_COMMIT +ARG AGENT_OS_SOURCE_TREE +ARG AGENT_OS_SOURCE_BRANCH=main +ARG AGENT_OS_SOURCE_ORIGIN=https://github.com/akua-dev/agent-os.git + +RUN apt-get update \ + && apt-get install -y --no-install-recommends git \ + && rm -rf /var/lib/apt/lists/* + +COPY . /opt/agent-os + +RUN set -eu; \ + test -n "$AGENT_OS_SOURCE_COMMIT"; \ + test -n "$AGENT_OS_SOURCE_TREE"; \ + test "$AGENT_OS_SOURCE_BRANCH" = main; \ + test "$AGENT_OS_SOURCE_ORIGIN" = https://github.com/akua-dev/agent-os.git; \ + git init --bare /opt/agent-os-bootstrap.git; \ + git --git-dir=/opt/agent-os-bootstrap.git bundle verify /opt/agent-os/image/agent-os-source.bundle; \ + git --git-dir=/opt/agent-os-bootstrap.git fetch --no-tags /opt/agent-os/image/agent-os-source.bundle \ + "$AGENT_OS_SOURCE_COMMIT:refs/heads/$AGENT_OS_SOURCE_BRANCH"; \ + test "$(git --git-dir=/opt/agent-os-bootstrap.git rev-parse "$AGENT_OS_SOURCE_COMMIT")" = "$AGENT_OS_SOURCE_COMMIT"; \ + test "$(git --git-dir=/opt/agent-os-bootstrap.git rev-parse "$AGENT_OS_SOURCE_COMMIT^{tree}")" = "$AGENT_OS_SOURCE_TREE"; \ + git --git-dir=/opt/agent-os-bootstrap.git update-ref "refs/heads/$AGENT_OS_SOURCE_BRANCH" "$AGENT_OS_SOURCE_COMMIT"; \ + git --git-dir=/opt/agent-os-bootstrap.git update-ref "refs/remotes/origin/$AGENT_OS_SOURCE_BRANCH" "$AGENT_OS_SOURCE_COMMIT"; \ + git --git-dir=/opt/agent-os-bootstrap.git symbolic-ref HEAD "refs/heads/$AGENT_OS_SOURCE_BRANCH"; \ + git --git-dir=/opt/agent-os-bootstrap.git remote add origin "$AGENT_OS_SOURCE_ORIGIN"; \ + test "$(git --git-dir=/opt/agent-os-bootstrap.git remote get-url origin)" = "$AGENT_OS_SOURCE_ORIGIN"; \ + test -z "$(git --git-dir=/opt/agent-os-bootstrap.git ls-tree -r --name-only "$AGENT_OS_SOURCE_COMMIT" -- config data projects state .no-mistakes)"; \ + rm -rf /opt/agent-os-bootstrap.git/hooks; \ + rm -f /opt/agent-os/image/agent-os-source.bundle; \ + printf '%s\n' "$AGENT_OS_SOURCE_COMMIT" > /opt/agent-os-source.commit; \ + printf '%s\n' "$AGENT_OS_SOURCE_TREE" > /opt/agent-os-source.tree; \ + printf '%s\n' "$AGENT_OS_SOURCE_BRANCH" > /opt/agent-os-source.branch; \ + printf '%s\n' "$AGENT_OS_SOURCE_ORIGIN" > /opt/agent-os-source.origin + FROM node:24-trixie-slim@sha256:366fdef91728b1b7fa18c84fba63b6e79ed77b7e10cc206878e9705da4d7b169 ARG TARGETARCH @@ -11,6 +48,7 @@ ARG AKUA_VERSION=0.8.25 ARG K9S_VERSION=0.51.0 ARG AGENT_OS_SOURCE_COMMIT ARG AGENT_OS_SOURCE_TREE +ARG AGENT_OS_SOURCE_BRANCH=main ARG AGENT_OS_SOURCE_ORIGIN=https://github.com/akua-dev/agent-os.git COPY image/debian.sources /etc/apt/sources.list.d/debian.sources @@ -168,25 +206,12 @@ ENV FM_HOME=/home/agent \ RUN mkdir -p /home/agent /opt/agent-os -COPY . /opt/agent-os -COPY image/agent-os-source.bundle /opt/agent-os-source.bundle - -RUN set -eu; \ - test -n "$AGENT_OS_SOURCE_COMMIT"; \ - test -n "$AGENT_OS_SOURCE_TREE"; \ - git bundle verify /opt/agent-os-source.bundle; \ - git clone --no-local /opt/agent-os-source.bundle /opt/agent-os-bootstrap; \ - git -C /opt/agent-os-bootstrap checkout --detach "$AGENT_OS_SOURCE_COMMIT"; \ - test "$(git -C /opt/agent-os-bootstrap rev-parse HEAD)" = "$AGENT_OS_SOURCE_COMMIT"; \ - test "$(git -C /opt/agent-os-bootstrap rev-parse HEAD^{tree})" = "$AGENT_OS_SOURCE_TREE"; \ - test -z "$(git -C /opt/agent-os-bootstrap status --porcelain)"; \ - test -z "$(git -C /opt/agent-os-bootstrap ls-files -- config data projects state .no-mistakes)"; \ - rm -rf /opt/agent-os-bootstrap/.git/hooks; \ - git -C /opt/agent-os-bootstrap remote set-url origin "$AGENT_OS_SOURCE_ORIGIN"; \ - test "$(git -C /opt/agent-os-bootstrap remote get-url origin)" = "$AGENT_OS_SOURCE_ORIGIN"; \ - printf '%s\n' "$AGENT_OS_SOURCE_COMMIT" > /opt/agent-os-source.commit; \ - printf '%s\n' "$AGENT_OS_SOURCE_TREE" > /opt/agent-os-source.tree; \ - printf '%s\n' "$AGENT_OS_SOURCE_ORIGIN" > /opt/agent-os-source.origin +COPY --from=source-bootstrap /opt/agent-os /opt/agent-os +COPY --from=source-bootstrap /opt/agent-os-bootstrap.git /opt/agent-os-bootstrap.git +COPY --from=source-bootstrap /opt/agent-os-source.commit /opt/agent-os-source.commit +COPY --from=source-bootstrap /opt/agent-os-source.tree /opt/agent-os-source.tree +COPY --from=source-bootstrap /opt/agent-os-source.branch /opt/agent-os-source.branch +COPY --from=source-bootstrap /opt/agent-os-source.origin /opt/agent-os-source.origin RUN install -D -m 0644 /opt/agent-os/THIRD_PARTY_NOTICES.md /usr/share/doc/agent-os/THIRD_PARTY_NOTICES.md \ && install -D -m 0644 /opt/agent-os/THIRD_PARTY_SOURCES.md /usr/share/doc/agent-os/THIRD_PARTY_SOURCES.md diff --git a/bin/agent-os-akua-auth.sh b/bin/agent-os-akua-auth.sh index ac861885c..3842037b2 100755 --- a/bin/agent-os-akua-auth.sh +++ b/bin/agent-os-akua-auth.sh @@ -11,9 +11,10 @@ INSTALLATION_ID="agent-os-firstmate:$NAMESPACE" OPERATION_ID=${AGENT_OS_OPERATION_ID:-"$(date -u '+%Y%m%d%H%M%S')-$$-$RANDOM"} LOCK_NONCE=$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n') LOCK_HOLDER_ID="$OPERATION_ID.$LOCK_NONCE" -LOCK=agent-os-firstmate-lifecycle -LOCK_NAMESPACE=$NAMESPACE -EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"primary"$'\t'"$INSTALLATION_ID" +LOCK= +LOCK_NAMESPACE= +LOCK_INSTALLATION_ID= +EXPECTED_LOCK= LOCK_UID= LOCK_RV= LOCK_RENEW_PID= @@ -32,6 +33,10 @@ case "$NAMESPACE" in ''|*[!a-z0-9-]*|-*|*-) echo "error: invalid Kubernetes name command -v jq >/dev/null 2>&1 || { echo "error: jq is required" >&2; exit 2; } kube() { + "$KUBECTL" --context "$CONTEXT" -n "$LOCK_NAMESPACE" "$@" +} + +target_kube() { "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" "$@" } @@ -51,14 +56,14 @@ apiVersion: coordination.k8s.io/v1 kind: Lease metadata: name: $LOCK - namespace: $NAMESPACE + namespace: $LOCK_NAMESPACE ${uid:+ uid: $uid_value} ${rv:+ resourceVersion: $rv_value} labels: app.kubernetes.io/managed-by: agent-os agent-os.dev/lifecycle: primary annotations: - agent-os.dev/installation-id: $INSTALLATION_ID + agent-os.dev/installation-id: $LOCK_INSTALLATION_ID spec: holderIdentity: $LOCK_HOLDER_ID acquireTime: $acquired_at @@ -67,6 +72,7 @@ spec: YAML } +. "$ROOT/bin/agent-os-kubernetes-control.sh" . "$ROOT/bin/agent-os-kubernetes-lease.sh" cleanup() { @@ -82,12 +88,26 @@ trap cleanup EXIT trap lock_renewal_failed TERM secret_record() { - kube --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get secret "$SECRET" --ignore-not-found \ + target_kube --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get secret "$SECRET" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}{"\t"}{range $key,$value := .data}{$key}{"\n"}{end}' } statefulset_json() { - kube --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get statefulset agent-os-firstmate -o json + target_kube --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get statefulset agent-os-firstmate -o json +} + +require_no_active_rollback_checkpoint() { + local checkpoint + checkpoint=$(printf '%s' "$STATE" | jq -er ' + [(.metadata.annotations["agent-os.dev/rollback-operation"] // ""), + (.metadata.annotations["agent-os.dev/rollback-target-name"] // ""), + (.metadata.annotations["agent-os.dev/rollback-target-uid"] // ""), + (.metadata.annotations["agent-os.dev/rollback-target-digest"] // "")] + | @tsv') || { echo "error: StatefulSet rollback checkpoint state is unverifiable" >&2; exit 3; } + [ -z "${checkpoint//$'\t'/}" ] || { + echo "error: active rollback checkpoint blocks authorization mutation" >&2 + exit 3 + } } verify_overlay() { @@ -110,8 +130,12 @@ verify_overlay() { ([.spec.template.spec.volumes[]? | select(.name == "akua-auth")] | length) == 0 end)' >/dev/null || { echo "error: Akua authorization overlay verification failed" >&2; exit 3; } secret_record_after=$(secret_record) - [ "$secret_record_after" = "$SECRET_RECORD" ] || { echo "error: Akua authorization Secret reference changed during mutation" >&2; exit 3; } - pod=$(kube --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get pod agent-os-firstmate-0 -o json) + [ "$secret_record_after" = "$SECRET_RECORD" ] || { + echo "incomplete: Akua authorization Secret identity changed after rollout" >&2 + echo "safe recovery: rerun '$COMMAND' only after the exact Secret reference is stable" >&2 + exit 3 + } + pod=$(target_kube --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get pod agent-os-firstmate-0 -o json) printf '%s' "$pod" | jq -e --arg uid "$state_uid" --arg secret "$SECRET" --arg expected "$expected" ' any(.metadata.ownerReferences[]?; .apiVersion == "apps/v1" and .kind == "StatefulSet" and .name == "agent-os-firstmate" and .uid == $uid and .controller == true) and (if $expected == "present" then @@ -125,6 +149,13 @@ verify_overlay() { end)' >/dev/null || { echo "error: Firstmate Pod authorization overlay verification failed" >&2; exit 3; } } +configure_control_lock +LOCK=$CONTROL_LOCK +LOCK_NAMESPACE=$CONTROL_NAMESPACE +LOCK_INSTALLATION_ID=$CONTROL_LOCK_INSTALLATION_ID +EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"primary"$'\t'"$LOCK_INSTALLATION_ID" +acquire_lock + SECRET_RECORD=$(secret_record) [ "$(printf '%s' "$SECRET_RECORD" | cut -f1)" = "$SECRET" ] && \ [ -n "$(printf '%s' "$SECRET_RECORD" | cut -f2)" ] && [ -n "$(printf '%s' "$SECRET_RECORD" | cut -f3)" ] && \ @@ -133,7 +164,6 @@ SECRET_RECORD=$(secret_record) exit 2 } -acquire_lock STATE=$(statefulset_json) STATE_UID=$(printf '%s' "$STATE" | jq -er --arg installation "$INSTALLATION_ID" ' select(.metadata.name == "agent-os-firstmate") @@ -141,6 +171,7 @@ STATE_UID=$(printf '%s' "$STATE" | jq -er --arg installation "$INSTALLATION_ID" | select(.metadata.annotations["agent-os.dev/installation-id"] == $installation) | .metadata.uid') || { echo "error: StatefulSet ownership is unverifiable" >&2; exit 2; } STATE_RV=$(printf '%s' "$STATE" | jq -er '.metadata.resourceVersion') || { echo "error: StatefulSet resourceVersion is unavailable" >&2; exit 2; } +require_no_active_rollback_checkpoint PATCH_FILE=$(mktemp) TEMPLATE="$ROOT/deploy/akua/firstmate-auth-$COMMAND.yaml" @@ -150,8 +181,12 @@ sed "s/__AKUA_AUTH_SECRET__/$SECRET/g" "$TEMPLATE" | awk -v uid="$uid_value" -v $1 == "metadata:" && !inserted { print; print " uid: " uid; print " resourceVersion: " rv; inserted=1; next } { print } ' > "$PATCH_FILE" -kube patch statefulset agent-os-firstmate --type=strategic --patch-file "$PATCH_FILE" >/dev/null -kube rollout status statefulset/agent-os-firstmate --timeout=180s +[ "$(secret_record)" = "$SECRET_RECORD" ] || { + echo "error: Akua authorization Secret identity changed before StatefulSet CAS" >&2 + exit 3 +} +target_kube patch statefulset agent-os-firstmate --type=strategic --patch-file "$PATCH_FILE" >/dev/null +target_kube rollout status statefulset/agent-os-firstmate --timeout=180s if [ "$COMMAND" = grant ]; then verify_overlay present "$STATE_UID" else diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh index 56ba2869b..9b19384cf 100755 --- a/bin/agent-os-container-entrypoint.sh +++ b/bin/agent-os-container-entrypoint.sh @@ -5,10 +5,14 @@ set -eu FM_HOME=${FM_HOME:-/home/agent} export FM_HOME HOME="$FM_HOME" FM_ROOT=${FM_ROOT_OVERRIDE:-$FM_HOME/firstmate} -IMAGE_SOURCE=${AGENT_OS_IMAGE_SOURCE:-/opt/agent-os-bootstrap} +IMAGE_SOURCE=${AGENT_OS_IMAGE_SOURCE:-/opt/agent-os-bootstrap.git} SOURCE_COMMIT=$(cat /opt/agent-os-source.commit) SOURCE_TREE=$(cat /opt/agent-os-source.tree) +SOURCE_BRANCH=$(cat /opt/agent-os-source.branch) SOURCE_ORIGIN=$(cat /opt/agent-os-source.origin) +case "$SOURCE_BRANCH" in ''|*[!A-Za-z0-9._/-]*|/*|*/|*..*) echo "error: image source branch provenance is invalid" >&2; exit 2 ;; esac +TRUSTED_REF="refs/remotes/origin/$SOURCE_BRANCH" +IMAGE_REF="refs/remotes/agent-os-image/$SOURCE_BRANCH" mkdir -p \ "$FM_HOME/config" \ @@ -24,8 +28,11 @@ mkdir -p \ if [ ! -e "$FM_ROOT/.git" ]; then [ ! -e "$FM_ROOT" ] || { echo "error: canonical FM_ROOT exists without Git provenance" >&2; exit 2; } - git clone --no-local "$IMAGE_SOURCE" "$FM_ROOT" - git -C "$FM_ROOT" checkout --detach "$SOURCE_COMMIT" + git clone --no-local --branch "$SOURCE_BRANCH" "$IMAGE_SOURCE" "$FM_ROOT" + [ "$(git -C "$FM_ROOT" rev-parse HEAD)" = "$SOURCE_COMMIT" ] || { + echo "error: canonical FM_ROOT bootstrap commit provenance failed" >&2 + exit 2 + } git -C "$FM_ROOT" remote set-url origin "$SOURCE_ORIGIN" rm -rf "$FM_ROOT/.git/hooks" else @@ -35,15 +42,54 @@ else exit 2 } [ -z "$(git -C "$FM_ROOT" status --porcelain)" ] || { echo "error: canonical FM_ROOT is not clean" >&2; exit 2; } - if git -C "$FM_ROOT" merge-base --is-ancestor HEAD "$SOURCE_COMMIT"; then - git -C "$FM_ROOT" fetch --no-tags "$IMAGE_SOURCE" "$SOURCE_COMMIT" - git -C "$FM_ROOT" merge --ff-only "$SOURCE_COMMIT" + git -C "$FM_ROOT" fetch --no-tags "$IMAGE_SOURCE" \ + "refs/heads/$SOURCE_BRANCH:$IMAGE_REF" + [ "$(git -C "$FM_ROOT" rev-parse "$IMAGE_REF")" = "$SOURCE_COMMIT" ] || { + echo "error: image source trusted ref provenance failed" >&2 + exit 2 + } + [ "$(git -C "$FM_ROOT" rev-parse "$IMAGE_REF^{tree}")" = "$SOURCE_TREE" ] || { + echo "error: image source trusted tree provenance failed" >&2 + exit 2 + } + current_branch=$(git -C "$FM_ROOT" symbolic-ref --quiet --short HEAD || true) + if [ -z "$current_branch" ]; then + git -C "$FM_ROOT" merge-base --is-ancestor HEAD "$IMAGE_REF" || { + echo "error: detached canonical FM_ROOT lacks trusted fast-forward provenance" >&2 + exit 2 + } + if git -C "$FM_ROOT" show-ref --verify --quiet "refs/heads/$SOURCE_BRANCH"; then + [ "$(git -C "$FM_ROOT" rev-parse "refs/heads/$SOURCE_BRANCH")" = "$(git -C "$FM_ROOT" rev-parse HEAD)" ] || { + echo "error: canonical default branch conflicts with detached provenance" >&2 + exit 2 + } + else + git -C "$FM_ROOT" branch "$SOURCE_BRANCH" HEAD + fi + git -C "$FM_ROOT" checkout "$SOURCE_BRANCH" + fi + [ "$(git -C "$FM_ROOT" symbolic-ref --short HEAD)" = "$SOURCE_BRANCH" ] || { + echo "error: canonical FM_ROOT is not on the declared default branch" >&2 + exit 2 + } + if git -C "$FM_ROOT" merge-base --is-ancestor HEAD "$IMAGE_REF"; then + git -C "$FM_ROOT" merge --ff-only "$IMAGE_REF" + git -C "$FM_ROOT" update-ref "$TRUSTED_REF" "$SOURCE_COMMIT" elif ! git -C "$FM_ROOT" merge-base --is-ancestor "$SOURCE_COMMIT" HEAD; then echo "error: canonical FM_ROOT source transition is not fast-forward compatible" >&2 exit 2 fi + git -C "$FM_ROOT" update-ref -d "$IMAGE_REF" fi +[ "$(git -C "$FM_ROOT" symbolic-ref --short HEAD)" = "$SOURCE_BRANCH" ] || { + echo "error: canonical FM_ROOT is not on the declared default branch" >&2 + exit 2 +} +[ "$(git -C "$FM_ROOT" rev-parse HEAD)" = "$(git -C "$FM_ROOT" rev-parse "$TRUSTED_REF")" ] || { + echo "error: canonical FM_ROOT HEAD is not the exact trusted remote ref" >&2 + exit 2 +} [ "$(git -C "$FM_ROOT" rev-parse "$SOURCE_COMMIT^{tree}")" = "$SOURCE_TREE" ] || { echo "error: canonical FM_ROOT image source tree provenance failed" >&2 exit 2 diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index b4314d44e..22211b775 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -12,15 +12,13 @@ IMAGE=${AGENT_OS_IMAGE:-} IMAGE_PULL_POLICY=${AGENT_OS_IMAGE_PULL_POLICY:-IfNotPresent} AI_SECRET=${AGENT_OS_AI_SECRET:-} KUBECTL=${AGENT_OS_KUBECTL:-kubectl} -TEMPLATE=${AGENT_OS_CREWMATE_TEMPLATE:-/opt/agent-os/tools/agent-os/packages/firstmate/crewmate.yaml} +TEMPLATE=${AGENT_OS_CREWMATE_TEMPLATE:-"$ROOT/tools/agent-os/packages/firstmate/crewmate.yaml"} INSTALLATION_ID="agent-os-firstmate:$NAMESPACE" OPERATION_ID=${AGENT_OS_OPERATION_ID:-"$(date -u '+%Y%m%d%H%M%S')-$$-$RANDOM"} LOCK_NONCE=$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n') LOCK_HOLDER_ID="$OPERATION_ID.$LOCK_NONCE" -if [ ! -f "$TEMPLATE" ]; then - TEMPLATE="$ROOT/tools/agent-os/packages/firstmate/crewmate.yaml" -fi +[ -f "$TEMPLATE" ] || { echo "error: crewmate template is unavailable in canonical source" >&2; exit 2; } case "$ID" in ''|*[!a-z0-9-]*|-*|*-) echo "error: invalid crewmate id '$ID'" >&2; exit 2 ;; diff --git a/bin/agent-os-kubernetes-control.sh b/bin/agent-os-kubernetes-control.sh new file mode 100755 index 000000000..da4a1eb87 --- /dev/null +++ b/bin/agent-os-kubernetes-control.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +control_sha256() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum | awk '{print $1}' + else + shasum -a 256 | awk '{print $1}' + fi +} + +configure_control_lock() { + local digest + CONTROL_NAMESPACE=${AGENT_OS_CONTROL_NAMESPACE:-kube-system} + case "$CONTROL_NAMESPACE" in ''|*[!a-z0-9-]*|-*|*-) echo "error: invalid lifecycle control namespace" >&2; exit 2 ;; esac + [ "${#CONTROL_NAMESPACE}" -le 63 ] || { echo "error: lifecycle control namespace is too long" >&2; exit 2; } + digest=$(printf 'agent-os-installation:%s' "$NAMESPACE" | control_sha256) + CONTROL_INSTALLATION_UUID="${digest:0:8}-${digest:8:4}-5${digest:13:3}-8${digest:17:3}-${digest:20:12}" + CONTROL_LOCK="agent-os-lifecycle-${digest:0:16}" + CONTROL_LOCK_INSTALLATION_ID="agent-os-control:$CONTROL_INSTALLATION_UUID" +} diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh index 00e755360..c4c202eb6 100755 --- a/bin/agent-os-kubernetes.sh +++ b/bin/agent-os-kubernetes.sh @@ -20,6 +20,9 @@ DELETE_NAMESPACE=0 MANAGES_NAMESPACE=0 DESIRED_RBAC=none INSTALLATION_ID= +LEGACY_CLUSTER_BINDING_PRESENT=0 +PRESERVED_AKUA_SECRET_RECORD= +AKUA_OVERLAY_VERIFIED=0 OPERATION_ID=${AGENT_OS_OPERATION_ID:-"$(date -u '+%Y%m%d%H%M%S')-$$-$RANDOM"} LOCK_NONCE=$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n') LOCK_HOLDER_ID="$OPERATION_ID.$LOCK_NONCE" @@ -30,6 +33,10 @@ EXPECTED_LOCK= LOCK_UID= LOCK_RV= LOCK_RENEW_PID= +CONTROL_LOCK_UID= +CONTROL_LOCK_RV= +CONTROL_LOCK_RENEW_PID= +CONTROL_LOCK_VALID_UNTIL= LOCK_DURATION_SECONDS=${AGENT_OS_LOCK_DURATION_SECONDS:-300} LOCK_CLOCK_SKEW_SECONDS=${AGENT_OS_LOCK_CLOCK_SKEW_SECONDS:-5} LOCK_ACQUIRE_SECONDS=${AGENT_OS_LOCK_ACQUIRE_SECONDS:-30} @@ -44,12 +51,13 @@ done [ "$LOCK_REQUEST_CEILING_SECONDS" -ge 1 ] || { echo "error: lifecycle Lease request ceiling must be at least 1 second" >&2; exit 2; } [ "$RESOURCE_REQUEST_CEILING_SECONDS" -ge 1 ] || { echo "error: resource request ceiling must be at least 1 second" >&2; exit 2; } +. "$ROOT/bin/agent-os-kubernetes-control.sh" . "$ROOT/bin/agent-os-kubernetes-lease.sh" cleanup() { local status=$? trap - EXIT - if ! release_lock && [ "$status" -eq 0 ]; then + if ! release_primary_locks && [ "$status" -eq 0 ]; then status=3 fi rm -rf "$OUT" "$RENDER_INPUTS" @@ -197,7 +205,7 @@ ${rv:+ resourceVersion: $rv_value} app.kubernetes.io/managed-by: agent-os agent-os.dev/lifecycle: primary annotations: - agent-os.dev/installation-id: $INSTALLATION_ID + agent-os.dev/installation-id: ${LOCK_INSTALLATION_ID:-$INSTALLATION_ID} spec: holderIdentity: $LOCK_HOLDER_ID acquireTime: $acquired_at @@ -207,20 +215,51 @@ YAML } acquire_primary_lock() { + configure_control_lock + LOCK_NAMESPACE=$CONTROL_NAMESPACE + LOCK=$CONTROL_LOCK + LOCK_INSTALLATION_ID=$CONTROL_LOCK_INSTALLATION_ID + EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"primary"$'\t'"$LOCK_INSTALLATION_ID" + acquire_lock + CONTROL_LOCK_UID=$LOCK_UID + CONTROL_LOCK_RV=$LOCK_RV + CONTROL_LOCK_RENEW_PID=$LOCK_RENEW_PID + CONTROL_LOCK_VALID_UNTIL=$LOCK_VALID_UNTIL if [ -n "$(namespace_name)" ]; then - LOCK_NAMESPACE=$NAMESPACE - LOCK=agent-os-firstmate-lifecycle - elif [ "$COMMAND" = cleanup-cluster-rbac ]; then - LOCK_NAMESPACE=${AGENT_OS_CLUSTER_LOCK_NAMESPACE:-kube-system} - LOCK="agent-os-firstmate-lifecycle-$(sha256_text "$NAMESPACE" | cut -c1-16)" - elif [ "$COMMAND" = uninstall ]; then - return 0 - else - echo "error: namespace '$NAMESPACE' is absent after namespace preparation" >&2 - exit 2 + acquire_namespace_lock fi +} + +acquire_namespace_lock() { + [ -z "${NAMESPACE_LOCK_UID:-}" ] || return 0 + LOCK_NAMESPACE=$NAMESPACE + LOCK=agent-os-firstmate-lifecycle + LOCK_INSTALLATION_ID=$INSTALLATION_ID EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"primary"$'\t'"$INSTALLATION_ID" + LOCK_UID= + LOCK_RV= + LOCK_RENEW_PID= + LOCK_VALID_UNTIL= acquire_lock + NAMESPACE_LOCK_UID=$LOCK_UID +} + +release_primary_locks() { + local status=0 + release_lock || status=$? + if [ -n "$CONTROL_LOCK_UID" ]; then + LOCK_NAMESPACE=$CONTROL_NAMESPACE + LOCK=$CONTROL_LOCK + LOCK_INSTALLATION_ID=$CONTROL_LOCK_INSTALLATION_ID + EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"primary"$'\t'"$LOCK_INSTALLATION_ID" + LOCK_UID=$CONTROL_LOCK_UID + LOCK_RV=$CONTROL_LOCK_RV + LOCK_RENEW_PID=$CONTROL_LOCK_RENEW_PID + LOCK_VALID_UNTIL=$CONTROL_LOCK_VALID_UNTIL + release_lock || status=$? + CONTROL_LOCK_UID= + fi + return "$status" } require_namespace_contract() { @@ -299,12 +338,28 @@ preflight_legacy_cluster_binding() { echo "error: ClusterRoleBinding '$binding' does not have the exact Agent OS installation identity" >&2 exit 2 fi + [ -z "$identity" ] || LEGACY_CLUSTER_BINDING_PRESENT=1 if [ -n "$identity" ] && [ "$allow_owned" -ne 1 ] && [ "$DESIRED_RBAC" != cluster-admin ]; then echo "error: stale ClusterRoleBinding '$binding' must be removed through separately authorized cleanup before installation" >&2 exit 3 fi } +verify_revision_service_account() { + local revision=$1 account identity expected + account=$(printf '%s' "$revision" | jq -er '.data.spec.template.spec.serviceAccountName') || { + echo "error: rollback target ServiceAccount dependency is unavailable" >&2 + exit 3 + } + case "$account" in agent-os-firstmate|agent-os-firstmate-[a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9]) ;; *) echo "error: rollback target ServiceAccount dependency is invalid" >&2; exit 3 ;; esac + identity=$(live_resource_identity ServiceAccount "$account") + expected="$account"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" + [ "$identity" = "$expected" ] || { + echo "error: rollback target ServiceAccount dependency is missing or changed" >&2 + exit 3 + } +} + require_no_active_rollback_checkpoint() { local state checkpoint if ! command -v jq >/dev/null 2>&1; then @@ -598,8 +653,8 @@ resource_observation() { lifecycle_command() { local action=${1:-$COMMAND} - printf 'AGENT_OS_CONTEXT=%q AGENT_OS_NAMESPACE=%q AGENT_OS_PACKAGE=%q AGENT_OS_INPUTS=%q AGENT_OS_AKUA=%q AGENT_OS_KUBECTL=%q %q %q' \ - "$CONTEXT" "$NAMESPACE" "$PACKAGE" "$INPUTS" "$AKUA" "$KUBECTL" "$0" "$action" + printf 'AGENT_OS_CONTEXT=%q AGENT_OS_NAMESPACE=%q AGENT_OS_CONTROL_NAMESPACE=%q AGENT_OS_PACKAGE=%q AGENT_OS_INPUTS=%q AGENT_OS_AKUA=%q AGENT_OS_KUBECTL=%q %q %q' \ + "$CONTEXT" "$NAMESPACE" "${CONTROL_NAMESPACE:-${AGENT_OS_CONTROL_NAMESPACE:-kube-system}}" "$PACKAGE" "$INPUTS" "$AKUA" "$KUBECTL" "$0" "$action" } report_partial_observation() { @@ -870,6 +925,12 @@ mutate_rendered_resource() { uid=$(printf '%s' "$record" | cut -f4) rv=$(printf '%s' "$record" | cut -f5) [ -n "$uid" ] && [ -n "$rv" ] || { echo "error: $kind '$name' lacks CAS identity" >&2; exit 2; } + if [ "$kind" = StatefulSet ] && [ "$AKUA_OVERLAY_VERIFIED" -eq 1 ]; then + verified_akua_overlay_secret "$PRESERVED_AKUA_SECRET_RECORD" >/dev/null || { + echo "incomplete: Akua authorization changed before StatefulSet CAS; no template mutation attempted" >&2 + return 3 + } + fi patch_file=$(cas_patch_file "$file" "$uid" "$rv") if [ "$scope" = cluster ]; then if ! "$KUBECTL" --context "$CONTEXT" patch "$kind" "$name" --type=merge --patch-file "$patch_file" >/dev/null; then @@ -937,8 +998,8 @@ verify_desired_rbac() { } cleanup_command() { - printf 'AGENT_OS_CONTEXT=%q AGENT_OS_NAMESPACE=%q AGENT_OS_PACKAGE=%q AGENT_OS_INPUTS=%q AGENT_OS_AKUA=%q AGENT_OS_KUBECTL=%q %q cleanup-cluster-rbac --yes' \ - "$CONTEXT" "$NAMESPACE" "$PACKAGE" "$INPUTS" "$AKUA" "$KUBECTL" "$0" + printf 'AGENT_OS_CONTEXT=%q AGENT_OS_NAMESPACE=%q AGENT_OS_CONTROL_NAMESPACE=%q AGENT_OS_PACKAGE=%q AGENT_OS_INPUTS=%q AGENT_OS_AKUA=%q AGENT_OS_KUBECTL=%q %q cleanup-cluster-rbac --yes' \ + "$CONTEXT" "$NAMESPACE" "${CONTROL_NAMESPACE:-${AGENT_OS_CONTROL_NAMESPACE:-kube-system}}" "$PACKAGE" "$INPUTS" "$AKUA" "$KUBECTL" "$0" } report_cluster_cleanup() { @@ -1181,9 +1242,11 @@ case "$COMMAND" in [ "$CONFIRMED" -eq 0 ] && [ "$DELETE_NAMESPACE" -eq 0 ] || usage render require_namespace_contract - create_managed_namespace_if_absent acquire_primary_lock require_namespace_contract + create_managed_namespace_if_absent + acquire_namespace_lock + require_namespace_contract preflight_rendered_resources 1 previous=$(workload_state) require_workload_owned "$previous" optional @@ -1217,9 +1280,15 @@ case "$COMMAND" in esac preflight_legacy_cluster_binding 1 require_no_active_rollback_checkpoint - preserved_akua_secret_record=$(verified_akua_overlay_secret) + PRESERVED_AKUA_SECRET_RECORD=$(verified_akua_overlay_secret) + AKUA_OVERLAY_VERIFIED=1 previous_mode=$(printf '%s' "$previous" | cut -f2) previous_cleanup=$(printf '%s' "$previous" | cut -f3) + if [ "$DESIRED_RBAC" != cluster-admin ] && [ "$LEGACY_CLUSTER_BINDING_PRESENT" -eq 1 ]; then + patch_workload_annotations '{"agent-os.dev/cluster-rbac-cleanup":"required"}' + report_cluster_cleanup + exit 3 + fi cleanup_required=0 if [ "$DESIRED_RBAC" != cluster-admin ] && \ { [ "$previous_mode" = cluster-admin ] || [ -z "$previous_mode" ] || [ "$previous_cleanup" = required ]; }; then @@ -1227,9 +1296,12 @@ case "$COMMAND" in patch_workload_annotations '{"agent-os.dev/cluster-rbac-cleanup":"required"}' fi apply_rendered - verified_akua_overlay_secret "$preserved_akua_secret_record" >/dev/null - if [ "$previous_service_account" != "$SERVICE_ACCOUNT_NAME" ]; then - delete_owned_resource namespaced ServiceAccount "$previous_service_account" 180 + if ! verified_akua_overlay_secret "$PRESERVED_AKUA_SECRET_RECORD" >/dev/null; then + echo "incomplete: upgrade applied but Akua authorization postcondition changed" >&2 + report_partial_observation StatefulSet agent-os-firstmate namespaced + report_partial_observation Pod agent-os-firstmate-0 namespaced + echo "safe recovery: $(lifecycle_command upgrade)" >&2 + exit 3 fi verify_desired_rbac if [ "$DESIRED_RBAC" != namespace ]; then @@ -1312,6 +1384,7 @@ case "$COMMAND" in rollback_target_uid=$(printf '%s' "$rollback_target" | jq -r '.metadata.uid') rollback_target_revision=$(printf '%s' "$rollback_target" | jq -r '.revision') rollback_target_digest=$rollback_checkpoint_digest + verify_revision_service_account "$rollback_target" rollback_mode='patch' if [ "$rollback_update_revision" = "$rollback_target_name" ]; then rollback_mode=resume @@ -1346,6 +1419,7 @@ case "$COMMAND" in rollback_target_revision=$(printf '%s' "$rollback_selection" | jq -r '.revision') rollback_target_digest=$(printf '%s' "$rollback_selection" | jq -cS '.data.spec.template' | template_digest) rollback_target=$rollback_selection + verify_revision_service_account "$rollback_target" rollback_checkpoint_operation=$OPERATION_ID rollback_checkpoint_name=$rollback_target_name rollback_checkpoint_uid=$rollback_target_uid diff --git a/bin/agent-os-local.sh b/bin/agent-os-local.sh index adb7997f0..8736b114f 100755 --- a/bin/agent-os-local.sh +++ b/bin/agent-os-local.sh @@ -56,7 +56,21 @@ cd "$ROOT" case "$COMMAND" in build) - docker build -t "$IMAGE" . + source_metadata=$("$ROOT/bin/agent-os-source-bundle.sh") + source_commit=$(printf '%s\n' "$source_metadata" | awk -F= '$1 == "commit" { print $2 }') + source_tree=$(printf '%s\n' "$source_metadata" | awk -F= '$1 == "tree" { print $2 }') + source_branch=$(printf '%s\n' "$source_metadata" | awk -F= '$1 == "branch" { print $2 }') + source_origin=$(printf '%s\n' "$source_metadata" | awk -F= '$1 == "origin" { sub(/^[^=]*=/, ""); print }') + [ -n "$source_commit" ] && [ -n "$source_tree" ] && [ -n "$source_branch" ] && [ -n "$source_origin" ] || { + echo "error: exact-source bundle metadata is incomplete" >&2 + exit 2 + } + docker build \ + --build-arg "AGENT_OS_SOURCE_COMMIT=$source_commit" \ + --build-arg "AGENT_OS_SOURCE_TREE=$source_tree" \ + --build-arg "AGENT_OS_SOURCE_BRANCH=$source_branch" \ + --build-arg "AGENT_OS_SOURCE_ORIGIN=$source_origin" \ + -t "$IMAGE" . if [ -z "$IMAGE_IS_OVERRIDE" ]; then docker tag "$IMAGE" "$(local_image_tag)" fi diff --git a/bin/agent-os-source-bundle.sh b/bin/agent-os-source-bundle.sh new file mode 100755 index 000000000..41d6c9b79 --- /dev/null +++ b/bin/agent-os-source-bundle.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -eu + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +OUTPUT=${1:-"$ROOT/image/agent-os-source.bundle"} +SOURCE_BRANCH=${AGENT_OS_SOURCE_BRANCH:-main} +SOURCE_ORIGIN=${AGENT_OS_SOURCE_ORIGIN:-https://github.com/akua-dev/agent-os.git} +[ "$SOURCE_BRANCH" = main ] || { echo "error: source branch must be the declared default branch 'main'" >&2; exit 2; } +[ "$SOURCE_ORIGIN" = https://github.com/akua-dev/agent-os.git ] || { echo "error: source origin is not allowlisted" >&2; exit 2; } + +[ -z "$(git -C "$ROOT" status --porcelain --untracked-files=all)" ] || { + echo "error: exact-source image builds require a clean worktree" >&2 + exit 2 +} + +SOURCE_COMMIT=$(git -C "$ROOT" rev-parse --verify HEAD) +SOURCE_TREE=$(git -C "$ROOT" rev-parse --verify 'HEAD^{tree}') +mkdir -p "$(dirname "$OUTPUT")" +TEMP="$OUTPUT.tmp.$$" +trap 'rm -f "$TEMP"' EXIT +git -C "$ROOT" bundle create "$TEMP" HEAD +git -C "$ROOT" bundle verify "$TEMP" >/dev/null +mv "$TEMP" "$OUTPUT" +trap - EXIT + +printf 'commit=%s\n' "$SOURCE_COMMIT" +printf 'tree=%s\n' "$SOURCE_TREE" +printf 'branch=%s\n' "$SOURCE_BRANCH" +printf 'origin=%s\n' "$SOURCE_ORIGIN" diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index c7c0959bf..171ef8c5b 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -114,10 +114,30 @@ assert_grep '.repos' "$ROOT/.dockerignore" "development source checkouts must st assert_grep '!.pi/extensions/fm-primary-pi-watch.ts' "$ROOT/.dockerignore" "tracked Pi supervision controls must enter the image" assert_grep '!.codex/hooks.json' "$ROOT/.dockerignore" "tracked Codex supervision controls must enter the image" assert_grep '!.opencode/plugins/fm-primary-watch-arm.js' "$ROOT/.dockerignore" "tracked OpenCode supervision controls must enter the image" +assert_grep '.pi/extensions/*' "$ROOT/.dockerignore" "Pi harness controls must use a nested default-deny rule" +assert_grep '.grok/hooks/*' "$ROOT/.dockerignore" "Grok harness controls must use a nested default-deny rule" +assert_grep '.opencode/plugins/*' "$ROOT/.dockerignore" "OpenCode harness controls must use a nested default-deny rule" assert_grep 'agent-os-source.bundle' "$ROOT/Dockerfile" "image must bootstrap an exact source commit without host Git metadata" -assert_grep 'rev-parse HEAD^{tree}' "$ROOT/Dockerfile" "image source bootstrap must verify its exact tree" +assert_grep 'rev-parse "$AGENT_OS_SOURCE_COMMIT^{tree}"' "$ROOT/Dockerfile" "image source bootstrap must verify its exact tree" +assert_grep 'AS source-bootstrap' "$ROOT/Dockerfile" "source bundle processing must stay in an isolated build stage" +assert_no_grep 'COPY image/agent-os-source.bundle /opt/agent-os-source.bundle' "$ROOT/Dockerfile" \ + "the final image must not retain a duplicate source bundle" assert_grep 'merge --ff-only' "$ROOT/bin/agent-os-container-entrypoint.sh" "persistent source transitions must be fast-forward only" +assert_grep 'TRUSTED_REF="refs/remotes/origin/$SOURCE_BRANCH"' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "canonical runtime source must match the declared trusted remote branch" +assert_no_grep 'checkout --detach' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "canonical runtime source must remain on the declared default branch" assert_grep 'FM_ROOT_OVERRIDE=' "$ROOT/bin/agent-os-container-entrypoint.sh" "runtime must use the persistent canonical Firstmate repository" +assert_grep 'agent-os-kubernetes-control.sh' "$ROOT/bin/agent-os-kubernetes.sh" \ + "primary lifecycle paths must share the stable control-namespace lock identity" +assert_grep 'agent-os-kubernetes-control.sh' "$ROOT/bin/agent-os-akua-auth.sh" \ + "authorization mutations must share the stable control-namespace lock identity" +assert_grep 'require_no_active_rollback_checkpoint' "$ROOT/bin/agent-os-akua-auth.sh" \ + "authorization mutations must reject unresolved rollback checkpoints" +auth_lock_line=$(grep -n '^acquire_lock$' "$ROOT/bin/agent-os-akua-auth.sh" | tail -n 1 | cut -d: -f1) +auth_secret_line=$(grep -n '^SECRET_RECORD=' "$ROOT/bin/agent-os-akua-auth.sh" | cut -d: -f1) +[ -n "$auth_lock_line" ] && [ -n "$auth_secret_line" ] && [ "$auth_lock_line" -lt "$auth_secret_line" ] || \ + fail "authorization mutation must acquire the control lock before Secret metadata" assert_grep 'https://github.com/ogulcancelik/herdr/tree/v0.7.3' "$ROOT/THIRD_PARTY_NOTICES.md" \ "Herdr's exact corresponding source must be named" assert_present "$ROOT/THIRD_PARTY_SOURCES.md" \ diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index c743caf57..392a79af1 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -60,6 +60,10 @@ if [ "${*: -2}" = "-f -" ]; then printf '%s\n' "$stdin_data" >> "${AGENT_OS_STDIN_LOG:-$AGENT_OS_TEST_LOG.stdin}" stdin_kind=$(printf '%s\n' "$stdin_data" | awk '$1 == "kind:" { print $2; exit }') printf 'stdin-kind %s\n' "$stdin_kind" >> "$AGENT_OS_TEST_LOG" + if [ "$stdin_kind" = Lease ]; then + stdin_name=$(printf '%s\n' "$stdin_data" | awk '$1 == "name:" { print $2; exit }') + printf 'stdin-lease %s\n' "$stdin_name" >> "$AGENT_OS_TEST_LOG" + fi fi if [ "${AGENT_OS_TEST_FAIL_APPLY:-0}" = 1 ] && [[ " $* " = *" create -f - "* ]] && \ [ "$stdin_kind" = Pod ]; then @@ -231,12 +235,19 @@ case " $* " in ;; esac ;; - *get\ lease\ agent-os-firstmate-lifecycle*--ignore-not-found*-o\ jsonpath=*) + *get\ lease\ agent-os-firstmate-lifecycle*--ignore-not-found*-o\ jsonpath=*|\ + *get\ lease\ agent-os-lifecycle-*--ignore-not-found*-o\ jsonpath=*) lock_name=$(printf '%s\n' "$*" | sed -n 's/.* get lease \([^ ]*\) .*/\1/p') - last_lock_write=$(grep -Fn 'stdin-kind Lease' "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d: -f1) + last_lock_write=$(grep -Fn "stdin-lease $lock_name" "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d: -f1) last_lock_delete=$(grep -Fn "/leases/$lock_name" "$AGENT_OS_TEST_LOG" | grep 'delete --raw' | tail -n 1 | cut -d: -f1) + lock_installation="agent-os-firstmate:${AGENT_OS_TEST_NAMESPACE:-${AGENT_OS_NAMESPACE:-portable-agent-os}}" + if [[ "$lock_name" = agent-os-lifecycle-* ]]; then + control_digest=$(printf 'agent-os-installation:%s' "${AGENT_OS_TEST_NAMESPACE:-${AGENT_OS_NAMESPACE:-portable-agent-os}}" | shasum -a 256 | awk '{print $1}') + control_uuid="${control_digest:0:8}-${control_digest:8:4}-5${control_digest:13:3}-8${control_digest:17:3}-${control_digest:20:12}" + lock_installation="agent-os-control:$control_uuid" + fi if [ "${AGENT_OS_TEST_PRIMARY_LOCK_STATE:-free}" = expired ] && ! grep -F ' replace -f -' "$AGENT_OS_TEST_LOG" >/dev/null; then - printf '%s\tagent-os\tprimary\tagent-os-firstmate:%s\tother-operation\t2000-01-01T00:00:00Z\t2000-01-01T00:00:00Z\t300\tuid-primary-lock\t%s' "$lock_name" "${AGENT_OS_TEST_NAMESPACE:-${AGENT_OS_NAMESPACE:-portable-agent-os}}" "${AGENT_OS_TEST_PRIMARY_LOCK_RV:-rv-primary-lock}" + printf '%s\tagent-os\tprimary\t%s\tother-operation\t2000-01-01T00:00:00Z\t2000-01-01T00:00:00Z\t300\tuid-primary-lock\t%s' "$lock_name" "$lock_installation" "${AGENT_OS_TEST_PRIMARY_LOCK_RV:-rv-primary-lock}" elif [ -n "$last_lock_write" ] && { [ -z "$last_lock_delete" ] || [ "$last_lock_write" -gt "$last_lock_delete" ]; }; then lock_stdin=${AGENT_OS_STDIN_LOG:-$AGENT_OS_TEST_LOG.stdin} lock_holder=$(awk '/holderIdentity:/ { holder=$2 } END { print holder }' "$lock_stdin") @@ -246,8 +257,8 @@ case " $* " in if grep -F ' replace -f -' "$AGENT_OS_TEST_LOG" >/dev/null; then lock_rv=rv-primary-lock-renewed fi - printf '%s\tagent-os\tprimary\tagent-os-firstmate:%s\t%s\t%s\t%s\t%s\tuid-primary-lock\t%s' \ - "$lock_name" "${AGENT_OS_TEST_NAMESPACE:-${AGENT_OS_NAMESPACE:-portable-agent-os}}" "$lock_holder" "$lock_acquired" "$lock_renewed" \ + printf '%s\tagent-os\tprimary\t%s\t%s\t%s\t%s\t%s\tuid-primary-lock\t%s' \ + "$lock_name" "$lock_installation" "$lock_holder" "$lock_acquired" "$lock_renewed" \ "${AGENT_OS_LOCK_DURATION_SECONDS:-300}" "$lock_rv" fi ;; @@ -398,9 +409,9 @@ case " $* " in ;; *" get controllerrevisions.apps -o json "*) if [ "${AGENT_OS_TEST_ROLLBACK_RENUMBERED:-0}" = 1 ]; then - printf '%s\n' '{"items":[{"metadata":{"name":"agent-os-firstmate-previous","uid":"uid-revision-previous","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":1,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"previous"}}}}}},{"metadata":{"name":"agent-os-firstmate-current","uid":"uid-revision-current","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":2,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"current"}}}}}},{"metadata":{"name":"agent-os-firstmate-renumbered","uid":"uid-revision-renumbered","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":3,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"previous"}}}}}}]}' + printf '%s\n' '{"items":[{"metadata":{"name":"agent-os-firstmate-previous","uid":"uid-revision-previous","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":1,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"previous"}},"spec":{"serviceAccountName":"agent-os-firstmate"}}}}},{"metadata":{"name":"agent-os-firstmate-current","uid":"uid-revision-current","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":2,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"current"}},"spec":{"serviceAccountName":"agent-os-firstmate"}}}}},{"metadata":{"name":"agent-os-firstmate-renumbered","uid":"uid-revision-renumbered","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":3,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"previous"}},"spec":{"serviceAccountName":"agent-os-firstmate"}}}}}]}' else - printf '%s\n' '{"items":[{"metadata":{"name":"agent-os-firstmate-previous","uid":"uid-revision-previous","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":1,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"previous"}}}}}},{"metadata":{"name":"agent-os-firstmate-current","uid":"uid-revision-current","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":2,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"current"}}}}}}]}' + printf '%s\n' '{"items":[{"metadata":{"name":"agent-os-firstmate-previous","uid":"uid-revision-previous","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":1,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"previous"}},"spec":{"serviceAccountName":"agent-os-firstmate"}}}}},{"metadata":{"name":"agent-os-firstmate-current","uid":"uid-revision-current","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":2,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"current"}},"spec":{"serviceAccountName":"agent-os-firstmate"}}}}}]}' fi ;; *" get role agent-os-firstmate-runtime -o json "*) @@ -1512,7 +1523,7 @@ pass "all RBAC modes preflight deterministic stale resources" : > "$CALLS" active_checkpoint_out='' active_checkpoint_rc=0 -active_checkpoint_digest=$(printf '%s' '{"metadata":{"labels":{"rollback":"previous"}}}' | jq -cS . | shasum -a 256 | awk '{print $1}') +active_checkpoint_digest=$(printf '%s' '{"metadata":{"labels":{"rollback":"previous"}},"spec":{"serviceAccountName":"agent-os-firstmate"}}' | jq -cS . | shasum -a 256 | awk '{print $1}') active_checkpoint_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_ROLLBACK_CHECKPOINT_DIGEST="$active_checkpoint_digest" \ run_generic upgrade 2>&1) || active_checkpoint_rc=$? @@ -1535,6 +1546,12 @@ assert_no_grep 'authorization.*Bearer\|auth.json' "$STDIN_LOG" \ "upgrade evidence must never contain Secret bytes" pass "upgrade preserves verified Akua Secret-reference overlays" +statefulset_patch_line=$(grep -Fn 'patch StatefulSet agent-os-firstmate --type=strategic --patch-file' "$CALLS" | head -n 1 | cut -d: -f1) +secret_reads_before_patch=$(sed -n "1,${statefulset_patch_line}p" "$CALLS" | grep -Fc 'get secret akua-auth --ignore-not-found') +[ -n "$statefulset_patch_line" ] && [ "$secret_reads_before_patch" -ge 2 ] || \ + fail "upgrade must revalidate the exact Secret identity immediately before StatefulSet CAS" +pass "upgrade revalidates authorization immediately before CAS" + : > "$CALLS" AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ AGENT_OS_TEST_WORKLOAD_STATE=namespace \ @@ -1545,6 +1562,13 @@ assert_no_grep '/serviceaccounts/agent-os-firstmate-bbbbbbbbbbbb' "$CALLS" \ "upgrade retries must not delete their active installation ServiceAccount" pass "upgrade reuses fresh installation ServiceAccount identity" +: > "$CALLS" +AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace run_generic upgrade +assert_no_grep '/serviceaccounts/agent-os-firstmate ' "$CALLS" \ + "upgrade must retain a legacy ServiceAccount referenced by rollback history" +pass "upgrade retains rollback ServiceAccount dependencies" + : > "$CALLS" AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ AGENT_OS_TEST_WORKLOAD_STATE=namespace run_generic rollback @@ -1586,7 +1610,7 @@ assert_contains "$checkpoint_read_race_out" 'rollback checkpoint did not persist "checkpoint persistence failure must preserve an explicit retained-state error" pass "rollback checkpoint read-back verifies immutable identity" -previous_template_digest=$(printf '%s' '{"metadata":{"labels":{"rollback":"previous"}}}' | jq -cS . | shasum -a 256 | awk '{print $1}') +previous_template_digest=$(printf '%s' '{"metadata":{"labels":{"rollback":"previous"}},"spec":{"serviceAccountName":"agent-os-firstmate"}}' | jq -cS . | shasum -a 256 | awk '{print $1}') : > "$CALLS" renumbered_out='' renumbered_rc=0 diff --git a/tests/agent-os-local.test.sh b/tests/agent-os-local.test.sh index 0310175a0..cfa28278a 100755 --- a/tests/agent-os-local.test.sh +++ b/tests/agent-os-local.test.sh @@ -42,6 +42,12 @@ if [ "${*: -2}" = '-f -' ]; then stdin_data=$(cat) stdin_kind=$(printf '%s\n' "$stdin_data" | awk '$1 == "kind:" { print $2; exit }') printf 'stdin-kind %s\n' "$stdin_kind" >> "$AGENT_OS_TEST_LOG" + if [ "$stdin_kind" = Lease ]; then + printf 'stdin-lease %s\n' "$(printf '%s\n' "$stdin_data" | awk '$1 == "name:" { print $2; exit }')" >> "$AGENT_OS_TEST_LOG" + printf 'stdin-holder %s\n' "$(printf '%s\n' "$stdin_data" | awk '$1 == "holderIdentity:" { print $2; exit }')" >> "$AGENT_OS_TEST_LOG" + printf 'stdin-acquired %s\n' "$(printf '%s\n' "$stdin_data" | awk '$1 == "acquireTime:" { print $2; exit }')" >> "$AGENT_OS_TEST_LOG" + printf 'stdin-renewed %s\n' "$(printf '%s\n' "$stdin_data" | awk '$1 == "renewTime:" { print $2; exit }')" >> "$AGENT_OS_TEST_LOG" + fi fi namespace=${AGENT_OS_NAMESPACE:-agent-os-demo} operation=$(grep 'akua-input-operation ' "$AGENT_OS_TEST_LOG" | tail -n 1 | awk '{print $2}') @@ -84,11 +90,22 @@ if { [[ " $* " = *" get clusterrolebinding agent-os-firstmate-"*" --ignore-not-f [[ " $* " != *'.metadata.resourceVersion'* ]] || printf '\tuid-clusterrolebinding\trv-clusterrolebinding\t%s' "$operation" fi fi -if [[ " $* " = *" get lease agent-os-firstmate-lifecycle --ignore-not-found -o jsonpath="* ]]; then - last_write=$(grep -Fn 'stdin-kind Lease' "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d: -f1) - last_delete=$(grep -Fn '/leases/agent-os-firstmate-lifecycle' "$AGENT_OS_TEST_LOG" | grep 'delete --raw' | tail -n 1 | cut -d: -f1) +if [[ " $* " = *" get lease agent-os-firstmate-lifecycle --ignore-not-found -o jsonpath="* ]] || \ + [[ " $* " = *" get lease agent-os-lifecycle-"*" --ignore-not-found -o jsonpath="* ]]; then + lock_name=$(printf '%s\n' "$*" | sed -n 's/.* get lease \([^ ]*\) .*/\1/p') + last_write=$(grep -Fn "stdin-lease $lock_name" "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d: -f1) + last_delete=$(grep -Fn "/leases/$lock_name" "$AGENT_OS_TEST_LOG" | grep 'delete --raw' | tail -n 1 | cut -d: -f1) if [ -n "$last_write" ] && { [ -z "$last_delete" ] || [ "$last_write" -gt "$last_delete" ]; }; then - printf 'agent-os-firstmate-lifecycle\tagent-os\tprimary\tagent-os-firstmate:%s\t%s\t2026-07-13T00:00:00Z\t2026-07-13T00:00:00Z\t300\tuid-lock\trv-lock' "$namespace" "$operation" + lock_installation="agent-os-firstmate:$namespace" + if [[ "$lock_name" = agent-os-lifecycle-* ]]; then + digest=$(printf 'agent-os-installation:%s' "$namespace" | shasum -a 256 | awk '{print $1}') + uuid="${digest:0:8}-${digest:8:4}-5${digest:13:3}-8${digest:17:3}-${digest:20:12}" + lock_installation="agent-os-control:$uuid" + fi + lock_holder=$(grep '^stdin-holder ' "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d' ' -f2-) + lock_acquired=$(grep '^stdin-acquired ' "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d' ' -f2-) + lock_renewed=$(grep '^stdin-renewed ' "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d' ' -f2-) + printf '%s\tagent-os\tprimary\t%s\t%s\t%s\t%s\t300\tuid-lock\trv-lock' "$lock_name" "$lock_installation" "$lock_holder" "$lock_acquired" "$lock_renewed" fi fi SH @@ -193,7 +210,10 @@ test_rebuild_deploy_uses_a_new_immutable_local_tag() { AGENT_OS_TEST_IMAGE_ID=sha256:rebuilt run_cli build AGENT_OS_TEST_IMAGE_ID=sha256:rebuilt run_cli deploy - assert_call 'docker build -t agent-os:dev .' "build must retain the local demo image tag" + grep -F 'docker build ' "$LOG" | grep -F -- '--build-arg AGENT_OS_SOURCE_COMMIT=' | \ + grep -F -- '--build-arg AGENT_OS_SOURCE_TREE=' | grep -F -- '--build-arg AGENT_OS_SOURCE_BRANCH=main' | \ + grep -F -- '-t agent-os:dev .' >/dev/null || \ + fail "build must pass verified exact-source arguments" assert_call 'docker tag agent-os:dev agent-os:local-rebuilt' \ "build must assign the rebuilt image a unique local tag" assert_call 'akua-input-image agent-os:local-rebuilt' \ @@ -228,8 +248,8 @@ test_explicit_image_override_is_used_without_retagging() { AGENT_OS_IMAGE=example.test/agent-os:custom run_cli build AGENT_OS_IMAGE=example.test/agent-os:custom run_cli deploy - assert_call 'docker build -t example.test/agent-os:custom .' \ - "build must preserve an explicit image override" + grep -F 'docker build ' "$LOG" | grep -F -- '-t example.test/agent-os:custom .' >/dev/null || \ + fail "build must preserve an explicit image override" assert_call 'akua-input-image example.test/agent-os:custom' \ "the rendered OrbStack profile must preserve an explicit image override" if grep -F 'docker tag example.test/agent-os:custom' "$LOG" >/dev/null; then @@ -243,8 +263,8 @@ test_empty_image_override_uses_content_addressed_default() { AGENT_OS_IMAGE='' AGENT_OS_TEST_IMAGE_ID=sha256:empty-default run_cli build AGENT_OS_IMAGE='' AGENT_OS_TEST_IMAGE_ID=sha256:empty-default run_cli deploy - assert_call 'docker build -t agent-os:dev .' \ - "an empty image override must use the local demo default" + grep -F 'docker build ' "$LOG" | grep -F -- '-t agent-os:dev .' >/dev/null || \ + fail "an empty image override must use the local demo default" assert_call 'docker tag agent-os:dev agent-os:local-empty-default' \ "an empty image override must still receive a content-addressed tag" assert_call 'akua-input-image agent-os:local-empty-default' \ From 77c47e775e4ec44fec82f724451165a62668c737 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 16:35:14 +0200 Subject: [PATCH 36/56] no-mistakes(review): Captain, harden source provenance and lifecycle safety --- .../akua-intelligence-bootstrap/SKILL.md | 3 +- .agents/skills/kubernetes-fleet/SKILL.md | 1 + .dockerignore | 47 +-- .github/workflows/agent-os-image.yml | 8 +- .gitignore | 4 +- Dockerfile | 28 +- bin/agent-os-akua-auth.sh | 109 +++++- bin/agent-os-container-entrypoint.sh | 30 +- bin/agent-os-crewmate.sh | 43 ++- bin/agent-os-kubernetes-lease.sh | 35 +- bin/agent-os-kubernetes.sh | 359 ++++++++++++++++-- bin/agent-os-local.sh | 2 +- bin/agent-os-source-bundle.sh | 77 +++- deploy/akua/firstmate-auth-grant.yaml | 1 + deploy/akua/firstmate-auth-revoke.yaml | 1 + tests/agent-os-container.test.sh | 35 +- tests/agent-os-kubernetes.test.sh | 88 ++++- tests/agent-os-local.test.sh | 34 +- 18 files changed, 758 insertions(+), 147 deletions(-) diff --git a/.agents/skills/akua-intelligence-bootstrap/SKILL.md b/.agents/skills/akua-intelligence-bootstrap/SKILL.md index fcdf3337f..a6192789c 100644 --- a/.agents/skills/akua-intelligence-bootstrap/SKILL.md +++ b/.agents/skills/akua-intelligence-bootstrap/SKILL.md @@ -57,7 +57,8 @@ Set `context` and `namespace` from the separately approved target and grant the AGENT_OS_CONTEXT="$context" AGENT_OS_NAMESPACE="$namespace" bin/agent-os-akua-auth.sh grant "$secret_name" ``` -The helper holds the installation-wide lifecycle Lease, validates exact StatefulSet UID and resourceVersion, verifies the named Secret reference without reading Secret bytes, applies one CAS strategic patch, and verifies both the retained StatefulSet overlay and its exact-owned Pod. +The helper holds the stable control Lease and namespace fleet Lease in that order, validates exact StatefulSet UID and resourceVersion, verifies the named Secret reference without reading Secret bytes, applies one CAS strategic patch, and verifies both the retained StatefulSet overlay and its exact-owned Pod. +If grant verification fails, the helper removes the overlay by CAS, verifies the fail-closed rollout, records only bounded Secret metadata evidence, and requires an explicit revoke before a different Secret identity can be granted. Revocation is owned by the same integration boundary. Revoke the Akua API token and prove it fails first, then revoke the overlay through the same serialized helper: diff --git a/.agents/skills/kubernetes-fleet/SKILL.md b/.agents/skills/kubernetes-fleet/SKILL.md index 395c278aa..33e543a58 100644 --- a/.agents/skills/kubernetes-fleet/SKILL.md +++ b/.agents/skills/kubernetes-fleet/SKILL.md @@ -29,6 +29,7 @@ Load this skill only when the current firstmate is running in Kubernetes or is e - Give every launched Herdr agent a task-unique name, close only a confirmed dead restored pane before reuse, and never replace a live agent. - Grade completion by the promised artifact or delivered Git state; Herdr `idle` alone is not a completion signal. - Use `bin/agent-os-crewmate.sh create ` to create a separate Pod and persistent home. +- Every mutating crewmate operation acquires the stable control Lease, namespace fleet Lease, and per-crewmate Lease in that order and releases them in reverse order. - Use `status` to inspect it, `stop` to remove only the Pod, and `restart` to replace only the Pod on its retained PVC. - The ambiguous `delete` command is rejected. - `purge --yes` is the only operation that destroys a persistent home. diff --git a/.dockerignore b/.dockerignore index aa14cd50a..3904be96e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,35 +1,12 @@ -.git -.github -.pi/* -!.pi/extensions/ -.pi/extensions/* -!.pi/extensions/fm-primary-pi-watch.ts -!.pi/extensions/fm-primary-turnend-guard.ts -.codex/* -!.codex/hooks.json -.claude/* -!.claude/settings.json -!.claude/skills -.grok/* -!.grok/hooks/ -.grok/hooks/* -!.grok/hooks/fm-primary-cd-check.json -!.grok/hooks/fm-primary-pretool-check.json -!.grok/hooks/fm-primary-turnend-guard.json -.opencode/* -!.opencode/plugins/ -.opencode/plugins/* -!.opencode/plugins/fm-primary-cd-check.js -!.opencode/plugins/fm-primary-pretool-check.js -!.opencode/plugins/fm-primary-turnend-guard.js -!.opencode/plugins/fm-primary-watch-arm.js -.no-mistakes -.env -config -data -projects -state -worktrees -.worktrees -node_modules -.repos +** +!Dockerfile +!image/ +image/** +!image/debian.sources +!image/npm/ +image/npm/** +!image/npm/package.json +!image/npm/package-lock.json +!image/agent-os-source.tar +!image/agent-os-bootstrap.tar +!image/agent-os-source.attestation diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index 626d164de..6ee51fdea 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -64,7 +64,13 @@ jobs: id: source run: | set -eu - bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" + if [ "$GITHUB_EVENT_NAME" = pull_request ]; then + git clone --depth=1 --branch main https://github.com/akua-dev/agent-os.git "$RUNNER_TEMP/trusted-agent-os" + AGENT_OS_SOURCE_ROOT="$RUNNER_TEMP/trusted-agent-os" \ + bin/agent-os-source-bundle.sh "$GITHUB_WORKSPACE/image" > "$RUNNER_TEMP/source-metadata" + else + bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" + fi cat "$RUNNER_TEMP/source-metadata" >> "$GITHUB_OUTPUT" - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 diff --git a/.gitignore b/.gitignore index 88bd79c25..57d590bc9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,9 @@ data/ .DS_Store __pycache__/ *.pyc -image/agent-os-source.bundle +image/agent-os-source.tar +image/agent-os-bootstrap.tar +image/agent-os-source.attestation .env config/crew-harness config/crew-dispatch.json diff --git a/Dockerfile b/Dockerfile index d480f8fd5..0d2b7d1f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,27 +9,33 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends git \ && rm -rf /var/lib/apt/lists/* -COPY . /opt/agent-os +COPY image/agent-os-source.tar image/agent-os-bootstrap.tar image/agent-os-source.attestation /tmp/ RUN set -eu; \ test -n "$AGENT_OS_SOURCE_COMMIT"; \ test -n "$AGENT_OS_SOURCE_TREE"; \ test "$AGENT_OS_SOURCE_BRANCH" = main; \ test "$AGENT_OS_SOURCE_ORIGIN" = https://github.com/akua-dev/agent-os.git; \ - git init --bare /opt/agent-os-bootstrap.git; \ - git --git-dir=/opt/agent-os-bootstrap.git bundle verify /opt/agent-os/image/agent-os-source.bundle; \ - git --git-dir=/opt/agent-os-bootstrap.git fetch --no-tags /opt/agent-os/image/agent-os-source.bundle \ - "$AGENT_OS_SOURCE_COMMIT:refs/heads/$AGENT_OS_SOURCE_BRANCH"; \ + grep -Fx "commit=$AGENT_OS_SOURCE_COMMIT" /tmp/agent-os-source.attestation; \ + grep -Fx "tree=$AGENT_OS_SOURCE_TREE" /tmp/agent-os-source.attestation; \ + grep -Fx "branch=$AGENT_OS_SOURCE_BRANCH" /tmp/agent-os-source.attestation; \ + grep -Fx "origin=$AGENT_OS_SOURCE_ORIGIN" /tmp/agent-os-source.attestation; \ + grep -Fx "ref=refs/heads/$AGENT_OS_SOURCE_BRANCH" /tmp/agent-os-source.attestation; \ + source_sha=$(sha256sum /tmp/agent-os-source.tar | awk '{print $1}'); \ + bootstrap_sha=$(sha256sum /tmp/agent-os-bootstrap.tar | awk '{print $1}'); \ + grep -Fx "source_sha256=$source_sha" /tmp/agent-os-source.attestation; \ + grep -Fx "bootstrap_sha256=$bootstrap_sha" /tmp/agent-os-source.attestation; \ + mkdir -p /opt/agent-os; \ + tar -xf /tmp/agent-os-source.tar -C /opt/agent-os; \ + tar -xf /tmp/agent-os-bootstrap.tar -C /opt; \ + mv /opt/bootstrap.git /opt/agent-os-bootstrap.git; \ test "$(git --git-dir=/opt/agent-os-bootstrap.git rev-parse "$AGENT_OS_SOURCE_COMMIT")" = "$AGENT_OS_SOURCE_COMMIT"; \ test "$(git --git-dir=/opt/agent-os-bootstrap.git rev-parse "$AGENT_OS_SOURCE_COMMIT^{tree}")" = "$AGENT_OS_SOURCE_TREE"; \ - git --git-dir=/opt/agent-os-bootstrap.git update-ref "refs/heads/$AGENT_OS_SOURCE_BRANCH" "$AGENT_OS_SOURCE_COMMIT"; \ - git --git-dir=/opt/agent-os-bootstrap.git update-ref "refs/remotes/origin/$AGENT_OS_SOURCE_BRANCH" "$AGENT_OS_SOURCE_COMMIT"; \ - git --git-dir=/opt/agent-os-bootstrap.git symbolic-ref HEAD "refs/heads/$AGENT_OS_SOURCE_BRANCH"; \ - git --git-dir=/opt/agent-os-bootstrap.git remote add origin "$AGENT_OS_SOURCE_ORIGIN"; \ + test -s /opt/agent-os-bootstrap.git/shallow; \ test "$(git --git-dir=/opt/agent-os-bootstrap.git remote get-url origin)" = "$AGENT_OS_SOURCE_ORIGIN"; \ test -z "$(git --git-dir=/opt/agent-os-bootstrap.git ls-tree -r --name-only "$AGENT_OS_SOURCE_COMMIT" -- config data projects state .no-mistakes)"; \ - rm -rf /opt/agent-os-bootstrap.git/hooks; \ - rm -f /opt/agent-os/image/agent-os-source.bundle; \ + test ! -e /opt/agent-os-bootstrap.git/hooks; \ + rm -f /tmp/agent-os-source.tar /tmp/agent-os-bootstrap.tar /tmp/agent-os-source.attestation; \ printf '%s\n' "$AGENT_OS_SOURCE_COMMIT" > /opt/agent-os-source.commit; \ printf '%s\n' "$AGENT_OS_SOURCE_TREE" > /opt/agent-os-source.tree; \ printf '%s\n' "$AGENT_OS_SOURCE_BRANCH" > /opt/agent-os-source.branch; \ diff --git a/bin/agent-os-akua-auth.sh b/bin/agent-os-akua-auth.sh index 3842037b2..89dd5e9c4 100755 --- a/bin/agent-os-akua-auth.sh +++ b/bin/agent-os-akua-auth.sh @@ -18,6 +18,11 @@ EXPECTED_LOCK= LOCK_UID= LOCK_RV= LOCK_RENEW_PID= +LOCK_PERSISTENT=0 +CONTROL_LOCK_UID= +CONTROL_LOCK_RV= +CONTROL_LOCK_RENEW_PID= +CONTROL_LOCK_VALID_UNTIL= LOCK_DURATION_SECONDS=${AGENT_OS_LOCK_DURATION_SECONDS:-300} LOCK_CLOCK_SKEW_SECONDS=${AGENT_OS_LOCK_CLOCK_SKEW_SECONDS:-5} LOCK_ACQUIRE_SECONDS=${AGENT_OS_LOCK_ACQUIRE_SECONDS:-30} @@ -81,6 +86,18 @@ cleanup() { if ! release_lock && [ "$status" -eq 0 ]; then status=3 fi + if [ -n "$CONTROL_LOCK_UID" ]; then + LOCK=$CONTROL_LOCK + LOCK_NAMESPACE=$CONTROL_NAMESPACE + LOCK_INSTALLATION_ID=$CONTROL_LOCK_INSTALLATION_ID + EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"primary"$'\t'"$LOCK_INSTALLATION_ID" + LOCK_UID=$CONTROL_LOCK_UID + LOCK_RV=$CONTROL_LOCK_RV + LOCK_RENEW_PID=$CONTROL_LOCK_RENEW_PID + LOCK_VALID_UNTIL=$CONTROL_LOCK_VALID_UNTIL + LOCK_PERSISTENT=1 + release_lock || [ "$status" -ne 0 ] || status=3 + fi [ -z "${PATCH_FILE:-}" ] || rm -f "$PATCH_FILE" exit "$status" } @@ -102,7 +119,10 @@ require_no_active_rollback_checkpoint() { [(.metadata.annotations["agent-os.dev/rollback-operation"] // ""), (.metadata.annotations["agent-os.dev/rollback-target-name"] // ""), (.metadata.annotations["agent-os.dev/rollback-target-uid"] // ""), - (.metadata.annotations["agent-os.dev/rollback-target-digest"] // "")] + (.metadata.annotations["agent-os.dev/rollback-target-digest"] // ""), + (.metadata.annotations["agent-os.dev/rollback-source-name"] // ""), + (.metadata.annotations["agent-os.dev/rollback-source-uid"] // ""), + (.metadata.annotations["agent-os.dev/rollback-source-digest"] // "")] | @tsv') || { echo "error: StatefulSet rollback checkpoint state is unverifiable" >&2; exit 3; } [ -z "${checkpoint//$'\t'/}" ] || { echo "error: active rollback checkpoint blocks authorization mutation" >&2 @@ -110,8 +130,16 @@ require_no_active_rollback_checkpoint() { } } +sha256_text() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum | awk '{print $1}' + else + shasum -a 256 | awk '{print $1}' + fi +} + verify_overlay() { - local expected=$1 state_uid=$2 state pod secret_record_after + local expected=$1 state_uid=$2 verify_secret=${3:-1} state pod secret_record_after state=$(statefulset_json) printf '%s' "$state" | jq -e --arg installation "$INSTALLATION_ID" --arg secret "$SECRET" \ --arg uid "$state_uid" --arg expected "$expected" ' @@ -128,13 +156,14 @@ verify_overlay() { ([.spec.template.spec.containers[] | select(.name == "firstmate") | .env[]? | select(.name == "AKUA_AUTH_HEADER_FILE")] | length) == 0 and ([.spec.template.spec.containers[] | select(.name == "firstmate") | .volumeMounts[]? | select(.name == "akua-auth")] | length) == 0 and ([.spec.template.spec.volumes[]? | select(.name == "akua-auth")] | length) == 0 - end)' >/dev/null || { echo "error: Akua authorization overlay verification failed" >&2; exit 3; } - secret_record_after=$(secret_record) - [ "$secret_record_after" = "$SECRET_RECORD" ] || { + end)' >/dev/null || { echo "error: Akua authorization overlay verification failed" >&2; return 3; } + if [ "$verify_secret" -eq 1 ]; then + secret_record_after=$(secret_record) + [ "$secret_record_after" = "$SECRET_RECORD" ] || { echo "incomplete: Akua authorization Secret identity changed after rollout" >&2 - echo "safe recovery: rerun '$COMMAND' only after the exact Secret reference is stable" >&2 - exit 3 - } + return 4 + } + fi pod=$(target_kube --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get pod agent-os-firstmate-0 -o json) printf '%s' "$pod" | jq -e --arg uid "$state_uid" --arg secret "$SECRET" --arg expected "$expected" ' any(.metadata.ownerReferences[]?; .apiVersion == "apps/v1" and .kind == "StatefulSet" and .name == "agent-os-firstmate" and .uid == $uid and .controller == true) and @@ -146,15 +175,64 @@ verify_overlay() { ([.spec.containers[] | select(.name == "firstmate") | .env[]? | select(.name == "AKUA_AUTH_HEADER_FILE")] | length) == 0 and ([.spec.containers[] | select(.name == "firstmate") | .volumeMounts[]? | select(.name == "akua-auth")] | length) == 0 and ([.spec.volumes[]? | select(.name == "akua-auth")] | length) == 0 - end)' >/dev/null || { echo "error: Firstmate Pod authorization overlay verification failed" >&2; exit 3; } + end)' >/dev/null || { echo "error: Firstmate Pod authorization overlay verification failed" >&2; return 3; } +} + +reconcile_failed_grant() { + local state uid rv revoke_patch observed rejected patch + state=$(statefulset_json) || return 3 + uid=$(printf '%s' "$state" | jq -er --arg installation "$INSTALLATION_ID" --arg uid "$STATE_UID" ' + select(.metadata.name == "agent-os-firstmate" and .metadata.uid == $uid) + | select(.metadata.labels["app.kubernetes.io/managed-by"] == "agent-os") + | select(.metadata.annotations["agent-os.dev/installation-id"] == $installation) + | .metadata.uid') || return 3 + rv=$(printf '%s' "$state" | jq -er '.metadata.resourceVersion') || return 3 + revoke_patch=$(mktemp) + sed "s/__AKUA_AUTH_SECRET__/$SECRET/g" "$ROOT/deploy/akua/firstmate-auth-revoke.yaml" | \ + awk -v uid="$(yaml_string "$uid")" -v rv="$(yaml_string "$rv")" ' + $1 == "metadata:" && !inserted { print; print " uid: " uid; print " resourceVersion: " rv; inserted=1; next } + { print } + ' > "$revoke_patch" + if ! target_kube patch statefulset agent-os-firstmate --type=strategic --patch-file "$revoke_patch" >/dev/null; then + rm -f "$revoke_patch" + return 3 + fi + rm -f "$revoke_patch" + target_kube rollout status statefulset/agent-os-firstmate --timeout=180s || return 3 + verify_overlay absent "$STATE_UID" 0 || return 3 + observed=$(secret_record 2>/dev/null || true) + rejected=$(printf '%s' "$SECRET_RECORD" | sha256_text) + state=$(statefulset_json) || return 3 + rv=$(printf '%s' "$state" | jq -er --arg uid "$STATE_UID" 'select(.metadata.uid == $uid) | .metadata.resourceVersion') || return 3 + patch=$(jq -cn --arg uid "$STATE_UID" --arg rv "$rv" --arg rejected "$rejected" \ + '{metadata:{uid:$uid,resourceVersion:$rv,annotations:{"agent-os.dev/akua-auth-rejected-record":$rejected}}}') + target_kube patch statefulset agent-os-firstmate --type=merge -p "$patch" >/dev/null || return 3 + printf 'incomplete: grant failed closed expected-secret-uid=%s expected-secret-rv=%s observed-secret-uid=%s observed-secret-rv=%s\n' \ + "$(printf '%s' "$SECRET_RECORD" | cut -f2)" "$(printf '%s' "$SECRET_RECORD" | cut -f3)" \ + "$(printf '%s' "$observed" | cut -f2)" "$(printf '%s' "$observed" | cut -f3)" >&2 } configure_control_lock LOCK=$CONTROL_LOCK LOCK_NAMESPACE=$CONTROL_NAMESPACE LOCK_INSTALLATION_ID=$CONTROL_LOCK_INSTALLATION_ID +LOCK_PERSISTENT=1 EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"primary"$'\t'"$LOCK_INSTALLATION_ID" acquire_lock +CONTROL_LOCK_UID=$LOCK_UID +CONTROL_LOCK_RV=$LOCK_RV +CONTROL_LOCK_RENEW_PID=$LOCK_RENEW_PID +CONTROL_LOCK_VALID_UNTIL=$LOCK_VALID_UNTIL +LOCK=agent-os-firstmate-lifecycle +LOCK_NAMESPACE=$NAMESPACE +LOCK_INSTALLATION_ID=$INSTALLATION_ID +EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"primary"$'\t'"$LOCK_INSTALLATION_ID" +LOCK_UID= +LOCK_RV= +LOCK_RENEW_PID= +LOCK_VALID_UNTIL= +LOCK_PERSISTENT=0 +acquire_lock SECRET_RECORD=$(secret_record) [ "$(printf '%s' "$SECRET_RECORD" | cut -f1)" = "$SECRET" ] && \ @@ -172,6 +250,10 @@ STATE_UID=$(printf '%s' "$STATE" | jq -er --arg installation "$INSTALLATION_ID" | .metadata.uid') || { echo "error: StatefulSet ownership is unverifiable" >&2; exit 2; } STATE_RV=$(printf '%s' "$STATE" | jq -er '.metadata.resourceVersion') || { echo "error: StatefulSet resourceVersion is unavailable" >&2; exit 2; } require_no_active_rollback_checkpoint +if [ "$COMMAND" = grant ] && [ -n "$(printf '%s' "$STATE" | jq -r '.metadata.annotations["agent-os.dev/akua-auth-rejected-record"] // empty')" ]; then + echo "error: a rejected Secret identity is recorded; run revoke before approving a new grant" >&2 + exit 3 +fi PATCH_FILE=$(mktemp) TEMPLATE="$ROOT/deploy/akua/firstmate-auth-$COMMAND.yaml" @@ -188,7 +270,14 @@ sed "s/__AKUA_AUTH_SECRET__/$SECRET/g" "$TEMPLATE" | awk -v uid="$uid_value" -v target_kube patch statefulset agent-os-firstmate --type=strategic --patch-file "$PATCH_FILE" >/dev/null target_kube rollout status statefulset/agent-os-firstmate --timeout=180s if [ "$COMMAND" = grant ]; then - verify_overlay present "$STATE_UID" + if ! verify_overlay present "$STATE_UID"; then + reconcile_failed_grant || { + echo "incomplete: grant verification failed and fail-closed reconciliation is unverified" >&2 + exit 3 + } + echo "safe recovery: inspect the named Secret metadata, then run revoke before a new grant" >&2 + exit 3 + fi else verify_overlay absent "$STATE_UID" fi diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh index 9b19384cf..84b56e2d3 100755 --- a/bin/agent-os-container-entrypoint.sh +++ b/bin/agent-os-container-entrypoint.sh @@ -11,7 +11,7 @@ SOURCE_TREE=$(cat /opt/agent-os-source.tree) SOURCE_BRANCH=$(cat /opt/agent-os-source.branch) SOURCE_ORIGIN=$(cat /opt/agent-os-source.origin) case "$SOURCE_BRANCH" in ''|*[!A-Za-z0-9._/-]*|/*|*/|*..*) echo "error: image source branch provenance is invalid" >&2; exit 2 ;; esac -TRUSTED_REF="refs/remotes/origin/$SOURCE_BRANCH" +TRUSTED_REF="refs/remotes/agent-os-verified/$SOURCE_BRANCH" IMAGE_REF="refs/remotes/agent-os-image/$SOURCE_BRANCH" mkdir -p \ @@ -74,7 +74,6 @@ else } if git -C "$FM_ROOT" merge-base --is-ancestor HEAD "$IMAGE_REF"; then git -C "$FM_ROOT" merge --ff-only "$IMAGE_REF" - git -C "$FM_ROOT" update-ref "$TRUSTED_REF" "$SOURCE_COMMIT" elif ! git -C "$FM_ROOT" merge-base --is-ancestor "$SOURCE_COMMIT" HEAD; then echo "error: canonical FM_ROOT source transition is not fast-forward compatible" >&2 exit 2 @@ -86,14 +85,33 @@ fi echo "error: canonical FM_ROOT is not on the declared default branch" >&2 exit 2 } -[ "$(git -C "$FM_ROOT" rev-parse HEAD)" = "$(git -C "$FM_ROOT" rev-parse "$TRUSTED_REF")" ] || { - echo "error: canonical FM_ROOT HEAD is not the exact trusted remote ref" >&2 - exit 2 -} [ "$(git -C "$FM_ROOT" rev-parse "$SOURCE_COMMIT^{tree}")" = "$SOURCE_TREE" ] || { echo "error: canonical FM_ROOT image source tree provenance failed" >&2 exit 2 } +[ "$(git -C "$FM_ROOT" remote get-url origin)" = "$SOURCE_ORIGIN" ] || { + echo "error: canonical FM_ROOT origin provenance changed" >&2 + exit 2 +} +GIT_TERMINAL_PROMPT=0 git -c credential.helper= -C "$FM_ROOT" fetch --no-tags --prune origin \ + "refs/heads/$SOURCE_BRANCH:$TRUSTED_REF" || { + echo "error: fresh trusted source provenance is unavailable" >&2 + exit 3 +} +git -C "$FM_ROOT" merge-base --is-ancestor "$SOURCE_COMMIT" "$TRUSTED_REF" || { + echo "error: image source commit is not reachable from the fresh trusted ref" >&2 + exit 2 +} +git -C "$FM_ROOT" merge-base --is-ancestor HEAD "$TRUSTED_REF" || { + echo "error: canonical FM_ROOT contains source not reachable from the fresh trusted ref" >&2 + exit 2 +} +git -C "$FM_ROOT" merge --ff-only "$TRUSTED_REF" +[ "$(git -C "$FM_ROOT" rev-parse HEAD)" = "$(git -C "$FM_ROOT" rev-parse "$TRUSTED_REF")" ] || { + echo "error: canonical FM_ROOT HEAD is not the exact fresh trusted remote ref" >&2 + exit 2 +} +git -C "$FM_ROOT" update-ref "refs/remotes/origin/$SOURCE_BRANCH" "$TRUSTED_REF" [ -z "$(git -C "$FM_ROOT" status --porcelain)" ] || { echo "error: canonical FM_ROOT is not clean" >&2; exit 2; } [ -z "$(git -C "$FM_ROOT" ls-files -- config data projects state .no-mistakes)" ] || { echo "error: canonical FM_ROOT contains operational state" >&2 diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index 22211b775..ab3ad26ec 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -17,6 +17,10 @@ INSTALLATION_ID="agent-os-firstmate:$NAMESPACE" OPERATION_ID=${AGENT_OS_OPERATION_ID:-"$(date -u '+%Y%m%d%H%M%S')-$$-$RANDOM"} LOCK_NONCE=$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n') LOCK_HOLDER_ID="$OPERATION_ID.$LOCK_NONCE" +CONTROL_LOCK_UID= +CONTROL_LOCK_RV= +CONTROL_LOCK_RENEW_PID= +CONTROL_LOCK_VALID_UNTIL= [ -f "$TEMPLATE" ] || { echo "error: crewmate template is unavailable in canonical source" >&2; exit 2; } @@ -51,6 +55,7 @@ EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"$ID"$'\t'"$INSTALLATION_ID" LOCK_UID= LOCK_RV= LOCK_RENEW_PID= +LOCK_PERSISTENT=0 LOCK_DURATION_SECONDS=${AGENT_OS_LOCK_DURATION_SECONDS:-300} LOCK_CLOCK_SKEW_SECONDS=${AGENT_OS_LOCK_CLOCK_SKEW_SECONDS:-5} LOCK_ACQUIRE_SECONDS=${AGENT_OS_LOCK_ACQUIRE_SECONDS:-30} @@ -69,7 +74,7 @@ done [ "$RESOURCE_REQUEST_CEILING_SECONDS" -ge 1 ] || { echo "error: resource request ceiling must be at least 1 second" >&2; exit 2; } kube() { - "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" "$@" + "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$LOCK_NAMESPACE" "$@" } resource_identity() { @@ -101,7 +106,7 @@ pvc_record() { lock_record() { local deadline=${1:-} [ -n "$deadline" ] || deadline=$(lock_default_deadline) - if [ "$LOCK_SCOPE" = fleet ]; then + if [ "$LOCK_SCOPE" != crewmate ]; then lock_kube "$deadline" get lease "$LOCK" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\t"}{.metadata.labels.agent-os\.dev/lifecycle}{"\t"}{.metadata.annotations.agent-os\.dev/installation-id}{"\t"}{.spec.holderIdentity}{"\t"}{.spec.acquireTime}{"\t"}{.spec.renewTime}{"\t"}{.spec.leaseDurationSeconds}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}' return @@ -171,13 +176,13 @@ render_lock() { local acquired_at=$1 renewed_at=$2 uid=${3:-} rv=${4:-} uid_value='' rv_value='' [ -z "$uid" ] || uid_value=$(yaml_string "$uid") [ -z "$rv" ] || rv_value=$(yaml_string "$rv") - if [ "$LOCK_SCOPE" = fleet ]; then + if [ "$LOCK_SCOPE" != crewmate ]; then cat <&2 return 1 fi + if [ "${LOCK_PERSISTENT:-0}" -eq 1 ]; then + released_holder=$LOCK_HOLDER_ID + LOCK_HOLDER_ID=released + if ! render_lock '1970-01-01T00:00:00Z' '1970-01-01T00:00:00Z' "$uid" "$rv" | \ + lock_kube_mutation "$deadline" replace -f - >/dev/null; then + LOCK_HOLDER_ID=$released_holder + echo "error: persistent lifecycle Lease '$LOCK' release CAS failed; retained" >&2 + return 1 + fi + LOCK_HOLDER_ID=$released_holder + after=$(lock_record "$deadline") || return 1 + if [ "$(printf '%s' "$after" | cut -f1-4)" != "$EXPECTED_LOCK" ] || \ + [ "$(printf '%s' "$after" | cut -f5)" != released ] || \ + [ "$(printf '%s' "$after" | cut -f9)" != "$uid" ] || \ + [ "$(printf '%s' "$after" | cut -f10)" = "$rv" ]; then + echo "error: persistent lifecycle Lease '$LOCK' release did not verify" >&2 + return 1 + fi + LOCK_UID= + LOCK_RV= + return 0 + fi if ! printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s","resourceVersion":"%s"}}\n' "$uid" "$rv" | \ lock_kube_mutation "$deadline" delete --raw "/apis/coordination.k8s.io/v1/namespaces/$LOCK_NAMESPACE/leases/$LOCK" -f - >/dev/null; then after=$(lock_record "$deadline") || { @@ -200,7 +222,16 @@ acquire_lock() { exit 3 fi mutation_ok=0 - render_lock "$now" "$now" | lock_kube_mutation "$deadline" create -f - >/dev/null && mutation_ok=1 + record= + if [ "${LOCK_PERSISTENT:-0}" -eq 1 ]; then + record=$(lock_record "$deadline") || { + echo "error: persistent lifecycle Lease '$LOCK' could not be read before acquisition" >&2 + exit 3 + } + fi + if [ -z "$record" ]; then + render_lock "$now" "$now" | lock_kube_mutation "$deadline" create -f - >/dev/null && mutation_ok=1 + fi record=$(lock_record "$deadline") || { echo "error: lifecycle Lease '$LOCK' create result could not be reconciled before the acquisition deadline" >&2 exit 3 diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh index c4c202eb6..9971c6219 100755 --- a/bin/agent-os-kubernetes.sh +++ b/bin/agent-os-kubernetes.sh @@ -33,10 +33,12 @@ EXPECTED_LOCK= LOCK_UID= LOCK_RV= LOCK_RENEW_PID= +LOCK_PERSISTENT=0 CONTROL_LOCK_UID= CONTROL_LOCK_RV= CONTROL_LOCK_RENEW_PID= CONTROL_LOCK_VALID_UNTIL= +REMOVE_CONTROL_ACCESS=0 LOCK_DURATION_SECONDS=${AGENT_OS_LOCK_DURATION_SECONDS:-300} LOCK_CLOCK_SKEW_SECONDS=${AGENT_OS_LOCK_CLOCK_SKEW_SECONDS:-5} LOCK_ACQUIRE_SECONDS=${AGENT_OS_LOCK_ACQUIRE_SECONDS:-30} @@ -60,6 +62,9 @@ cleanup() { if ! release_primary_locks && [ "$status" -eq 0 ]; then status=3 fi + if [ "$REMOVE_CONTROL_ACCESS" -eq 1 ] && ! delete_control_lock_access && [ "$status" -eq 0 ]; then + status=3 + fi rm -rf "$OUT" "$RENDER_INPUTS" exit "$status" } @@ -219,6 +224,7 @@ acquire_primary_lock() { LOCK_NAMESPACE=$CONTROL_NAMESPACE LOCK=$CONTROL_LOCK LOCK_INSTALLATION_ID=$CONTROL_LOCK_INSTALLATION_ID + LOCK_PERSISTENT=1 EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"primary"$'\t'"$LOCK_INSTALLATION_ID" acquire_lock CONTROL_LOCK_UID=$LOCK_UID @@ -240,13 +246,17 @@ acquire_namespace_lock() { LOCK_RV= LOCK_RENEW_PID= LOCK_VALID_UNTIL= + LOCK_PERSISTENT=0 acquire_lock NAMESPACE_LOCK_UID=$LOCK_UID } release_primary_locks() { local status=0 - release_lock || status=$? + if [ -n "${NAMESPACE_LOCK_UID:-}" ]; then + release_lock || status=$? + NAMESPACE_LOCK_UID= + fi if [ -n "$CONTROL_LOCK_UID" ]; then LOCK_NAMESPACE=$CONTROL_NAMESPACE LOCK=$CONTROL_LOCK @@ -256,6 +266,7 @@ release_primary_locks() { LOCK_RV=$CONTROL_LOCK_RV LOCK_RENEW_PID=$CONTROL_LOCK_RENEW_PID LOCK_VALID_UNTIL=$CONTROL_LOCK_VALID_UNTIL + LOCK_PERSISTENT=1 release_lock || status=$? CONTROL_LOCK_UID= fi @@ -346,7 +357,7 @@ preflight_legacy_cluster_binding() { } verify_revision_service_account() { - local revision=$1 account identity expected + local revision=$1 account identity expected record account=$(printf '%s' "$revision" | jq -er '.data.spec.template.spec.serviceAccountName') || { echo "error: rollback target ServiceAccount dependency is unavailable" >&2 exit 3 @@ -358,6 +369,170 @@ verify_revision_service_account() { echo "error: rollback target ServiceAccount dependency is missing or changed" >&2 exit 3 } + record=$(live_resource_record namespaced ServiceAccount "$account") + [ "$(printf '%s' "$record" | cut -f1-3)" = "$expected" ] && \ + [ -n "$(printf '%s' "$record" | cut -f4)" ] && [ -n "$(printf '%s' "$record" | cut -f5)" ] || { + echo "error: rollback target ServiceAccount dependency metadata is unavailable" >&2 + exit 3 + } +} + +runtime_binding_json() { + local mode=$1 + case "$mode" in + namespace) "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get rolebinding agent-os-firstmate-runtime -o json ;; + cluster-admin) "$KUBECTL" --context "$CONTEXT" get clusterrolebinding "agent-os-firstmate-$NAMESPACE" -o json ;; + none) printf '{}' ;; + *) echo "error: rollback runtime authority mode is invalid" >&2; return 3 ;; + esac +} + +transfer_runtime_authority() { + local mode=$1 from=$2 to=$3 binding kind scope json uid rv current patch verified + [ "$from" != "$to" ] || return 0 + [ "$mode" != none ] || return 0 + if [ "$mode" = namespace ]; then + binding=agent-os-firstmate-runtime + kind=rolebinding + scope=(-n "$NAMESPACE") + else + binding="agent-os-firstmate-$NAMESPACE" + kind=clusterrolebinding + scope=() + fi + json=$(runtime_binding_json "$mode") || return 3 + if [ "$mode" = namespace ]; then + printf '%s' "$json" | jq -e '.roleRef == {apiGroup:"rbac.authorization.k8s.io",kind:"Role",name:"agent-os-firstmate-runtime"}' >/dev/null || return 3 + else + printf '%s' "$json" | jq -e '.roleRef == {apiGroup:"rbac.authorization.k8s.io",kind:"ClusterRole",name:"cluster-admin"}' >/dev/null || return 3 + fi + current=$(printf '%s' "$json" | jq -er --arg installation "$INSTALLATION_ID" --arg namespace "$NAMESPACE" --arg binding "$binding" ' + select(.metadata.name == $binding) + | select(.metadata.labels["app.kubernetes.io/managed-by"] == "agent-os") + | select(.metadata.annotations["agent-os.dev/installation-id"] == $installation) + | select(.subjects | type == "array" and length == 1) + | select(.subjects[0].kind == "ServiceAccount" and .subjects[0].namespace == $namespace) + | .subjects[0].name') || { echo "error: rollback runtime authority ownership is unverifiable" >&2; return 3; } + if [ "$current" = "$to" ]; then + [ "$mode" != namespace ] || transfer_control_authority "$from" "$to" + return $? + fi + [ "$current" = "$from" ] || { echo "error: rollback runtime authority is assigned to an unexpected ServiceAccount" >&2; return 3; } + uid=$(printf '%s' "$json" | jq -er '.metadata.uid') || return 3 + rv=$(printf '%s' "$json" | jq -er '.metadata.resourceVersion') || return 3 + patch=$(jq -cn --arg uid "$uid" --arg rv "$rv" --arg namespace "$NAMESPACE" --arg account "$to" \ + '{metadata:{uid:$uid,resourceVersion:$rv},subjects:[{kind:"ServiceAccount",name:$account,namespace:$namespace}]}') + "$KUBECTL" --context "$CONTEXT" "${scope[@]}" patch "$kind" "$binding" --type=merge -p "$patch" >/dev/null || return 3 + verified=$(runtime_binding_json "$mode") || return 3 + printf '%s' "$verified" | jq -e --arg uid "$uid" --arg namespace "$NAMESPACE" --arg account "$to" ' + .metadata.uid == $uid and .subjects == [{kind:"ServiceAccount",name:$account,namespace:$namespace}]' >/dev/null || { + echo "error: rollback runtime authority transfer did not verify exactly" >&2 + return 3 + } + [ "$mode" != namespace ] || transfer_control_authority "$from" "$to" +} + +control_binding_json() { + "$KUBECTL" --context "$CONTEXT" -n "$CONTROL_NAMESPACE" get rolebinding "$CONTROL_LOCK" -o json +} + +transfer_control_authority() { + local from=$1 to=$2 json current uid rv patch verified + json=$(control_binding_json) || return 3 + current=$(printf '%s' "$json" | jq -er --arg installation "$INSTALLATION_ID" --arg namespace "$NAMESPACE" --arg name "$CONTROL_LOCK" ' + select(.metadata.name == $name) + | select(.metadata.labels["app.kubernetes.io/managed-by"] == "agent-os") + | select(.metadata.annotations["agent-os.dev/installation-id"] == $installation) + | select(.subjects | type == "array" and length == 1) + | select(.subjects[0].kind == "ServiceAccount" and .subjects[0].namespace == $namespace) + | .subjects[0].name') || return 3 + [ "$current" != "$to" ] || return 0 + [ "$current" = "$from" ] || return 3 + uid=$(printf '%s' "$json" | jq -er '.metadata.uid') || return 3 + rv=$(printf '%s' "$json" | jq -er '.metadata.resourceVersion') || return 3 + patch=$(jq -cn --arg uid "$uid" --arg rv "$rv" --arg namespace "$NAMESPACE" --arg account "$to" \ + '{metadata:{uid:$uid,resourceVersion:$rv},subjects:[{kind:"ServiceAccount",name:$account,namespace:$namespace}]}') + "$KUBECTL" --context "$CONTEXT" -n "$CONTROL_NAMESPACE" patch rolebinding "$CONTROL_LOCK" --type=merge -p "$patch" >/dev/null || return 3 + verified=$(control_binding_json) || return 3 + printf '%s' "$verified" | jq -e --arg uid "$uid" --arg namespace "$NAMESPACE" --arg account "$to" ' + .metadata.uid == $uid and .subjects == [{kind:"ServiceAccount",name:$account,namespace:$namespace}]' >/dev/null +} + +ensure_control_lock_access() { + local role binding live uid rv desired kind + [ "$DESIRED_RBAC" = namespace ] || return 0 + configure_control_lock + role=$(jq -cn --arg name "$CONTROL_LOCK" --arg namespace "$CONTROL_NAMESPACE" --arg installation "$INSTALLATION_ID" ' + {apiVersion:"rbac.authorization.k8s.io/v1",kind:"Role",metadata:{name:$name,namespace:$namespace,labels:{"app.kubernetes.io/managed-by":"agent-os"},annotations:{"agent-os.dev/installation-id":$installation}},rules:[{apiGroups:["coordination.k8s.io"],resources:["leases"],resourceNames:[$name],verbs:["get","update"]}]}') + binding=$(jq -cn --arg name "$CONTROL_LOCK" --arg controlNamespace "$CONTROL_NAMESPACE" --arg installation "$INSTALLATION_ID" --arg namespace "$NAMESPACE" --arg account "$SERVICE_ACCOUNT_NAME" ' + {apiVersion:"rbac.authorization.k8s.io/v1",kind:"RoleBinding",metadata:{name:$name,namespace:$controlNamespace,labels:{"app.kubernetes.io/managed-by":"agent-os"},annotations:{"agent-os.dev/installation-id":$installation}},roleRef:{apiGroup:"rbac.authorization.k8s.io",kind:"Role",name:$name},subjects:[{kind:"ServiceAccount",name:$account,namespace:$namespace}]}') + for desired in "$role" "$binding"; do + kind=$(printf '%s' "$desired" | jq -r '.kind') + live=$("$KUBECTL" --context "$CONTEXT" -n "$CONTROL_NAMESPACE" get "$kind" "$CONTROL_LOCK" --ignore-not-found -o json) + if [ -z "$live" ]; then + printf '%s' "$desired" | "$KUBECTL" --context "$CONTEXT" create -f - >/dev/null || return 3 + else + printf '%s' "$live" | jq -e --arg installation "$INSTALLATION_ID" --arg name "$CONTROL_LOCK" ' + .metadata.name == $name and .metadata.labels["app.kubernetes.io/managed-by"] == "agent-os" and .metadata.annotations["agent-os.dev/installation-id"] == $installation' >/dev/null || return 3 + uid=$(printf '%s' "$live" | jq -er '.metadata.uid') || return 3 + rv=$(printf '%s' "$live" | jq -er '.metadata.resourceVersion') || return 3 + desired=$(printf '%s' "$desired" | jq -c --arg uid "$uid" --arg rv "$rv" '.metadata.uid=$uid | .metadata.resourceVersion=$rv') + printf '%s' "$desired" | "$KUBECTL" --context "$CONTEXT" replace -f - >/dev/null || return 3 + fi + done + live=$(control_binding_json) || return 3 + printf '%s' "$live" | jq -e --arg namespace "$NAMESPACE" --arg account "$SERVICE_ACCOUNT_NAME" --arg name "$CONTROL_LOCK" ' + .roleRef == {apiGroup:"rbac.authorization.k8s.io",kind:"Role",name:$name} and .subjects == [{kind:"ServiceAccount",name:$account,namespace:$namespace}]' >/dev/null +} + +delete_control_lock_access() { + local kind resource plural live uid rv after + [ -n "${CONTROL_LOCK:-}" ] || configure_control_lock + for kind in RoleBinding Role; do + resource=$(printf '%s' "$kind" | tr '[:upper:]' '[:lower:]') + plural="${resource}s" + live=$("$KUBECTL" --context "$CONTEXT" -n "$CONTROL_NAMESPACE" get "$kind" "$CONTROL_LOCK" --ignore-not-found -o json) || return 3 + [ -n "$live" ] || continue + printf '%s' "$live" | jq -e --arg name "$CONTROL_LOCK" --arg installation "$INSTALLATION_ID" ' + .metadata.name == $name and .metadata.labels["app.kubernetes.io/managed-by"] == "agent-os" and .metadata.annotations["agent-os.dev/installation-id"] == $installation' >/dev/null || return 3 + uid=$(printf '%s' "$live" | jq -er '.metadata.uid') || return 3 + rv=$(printf '%s' "$live" | jq -er '.metadata.resourceVersion') || return 3 + printf '{"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":"%s","resourceVersion":"%s"}}\n' "$uid" "$rv" | \ + "$KUBECTL" --context "$CONTEXT" delete --raw "/apis/rbac.authorization.k8s.io/v1/namespaces/$CONTROL_NAMESPACE/$plural/$CONTROL_LOCK" -f - >/dev/null || return 3 + after=$("$KUBECTL" --context "$CONTEXT" -n "$CONTROL_NAMESPACE" get "$kind" "$CONTROL_LOCK" --ignore-not-found -o json) || return 3 + [ -z "$after" ] || return 3 + done +} + +compensate_rollback() { + local source_template=$1 current_account=$2 target_account=$3 mode=$4 state uid rv patch + transfer_runtime_authority "$mode" "$target_account" "$current_account" || return 3 + state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ + --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get statefulset agent-os-firstmate -o json) || return 3 + uid=$(printf '%s' "$state" | jq -er --arg expected "$rollback_uid" 'select(.metadata.uid == $expected) | .metadata.uid') || return 3 + rv=$(printf '%s' "$state" | jq -er '.metadata.resourceVersion') || return 3 + patch=$(jq -cn --arg uid "$uid" --arg rv "$rv" --argjson template "$source_template" \ + '{metadata:{uid:$uid,resourceVersion:$rv,annotations:{"agent-os.dev/rollback-operation":null,"agent-os.dev/rollback-target-name":null,"agent-os.dev/rollback-target-uid":null,"agent-os.dev/rollback-target-digest":null,"agent-os.dev/rollback-source-name":null,"agent-os.dev/rollback-source-uid":null,"agent-os.dev/rollback-source-digest":null}},spec:{template:$template}}') + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch statefulset agent-os-firstmate --type=strategic -p "$patch" >/dev/null || return 3 + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s || return 3 + [ "$(workload_service_account)" = "$current_account" ] || return 3 +} + +wait_for_revision_history_absence() { + local workload_uid=$1 deadline revisions count + deadline=$(($(date -u '+%s') + 180)) + while :; do + revisions=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ + --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get controllerrevisions.apps -o json) || return 3 + count=$(printf '%s' "$revisions" | jq -er --arg uid "$workload_uid" \ + '[.items[] | select(any(.metadata.ownerReferences[]?; .controller == true and .kind == "StatefulSet" and .name == "agent-os-firstmate" and ($uid == "" or .uid == $uid)))] | length') || return 3 + [ "$count" -eq 0 ] && return 0 + [ "$(date -u '+%s')" -lt "$deadline" ] || { + echo "incomplete: retained ControllerRevision history still references the removed StatefulSet" >&2 + return 3 + } + sleep 1 + done } require_no_active_rollback_checkpoint() { @@ -375,7 +550,10 @@ require_no_active_rollback_checkpoint() { | [(.metadata.annotations["agent-os.dev/rollback-operation"] // ""), (.metadata.annotations["agent-os.dev/rollback-target-name"] // ""), (.metadata.annotations["agent-os.dev/rollback-target-uid"] // ""), - (.metadata.annotations["agent-os.dev/rollback-target-digest"] // "")] + (.metadata.annotations["agent-os.dev/rollback-target-digest"] // ""), + (.metadata.annotations["agent-os.dev/rollback-source-name"] // ""), + (.metadata.annotations["agent-os.dev/rollback-source-uid"] // ""), + (.metadata.annotations["agent-os.dev/rollback-source-digest"] // "")] | @tsv') || { echo "error: StatefulSet rollback checkpoint state is unverifiable" >&2; exit 3; } if [ -n "${checkpoint//$'\t'/}" ]; then echo "error: active rollback checkpoint blocks upgrade; resume rollback until exact recovery and checkpoint finalization" >&2 @@ -384,8 +562,11 @@ require_no_active_rollback_checkpoint() { } verified_akua_overlay_secret() { - local expected_record=${1:-} expected_secret state secret record='' - expected_secret=$(printf '%s' "$expected_record" | cut -f1) + local expected_supplied=0 expected_record='' state secret record current + if [ "$#" -gt 0 ]; then + expected_supplied=1 + expected_record=$1 + fi if ! command -v jq >/dev/null 2>&1; then echo "error: jq is required to verify the Akua authorization overlay" >&2 return 2 @@ -409,10 +590,6 @@ verified_akua_overlay_secret() { echo "error: Akua authorization overlay is missing, changed, or unverifiable; upgrade blocked" >&2 return 3 } - if [ -n "$expected_record" ] && [ "$secret" != "$expected_secret" ]; then - echo "error: Akua authorization Secret reference changed during upgrade; upgrade blocked" >&2 - return 3 - fi if [ -n "$secret" ]; then record=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get secret "$secret" --ignore-not-found \ @@ -423,12 +600,15 @@ verified_akua_overlay_secret() { echo "error: Akua authorization Secret reference is missing or unverifiable; upgrade blocked" >&2 return 3 fi - if [ -n "$expected_record" ] && [ "$record" != "$expected_record" ]; then - echo "error: Akua authorization Secret UID or resourceVersion changed during upgrade; upgrade blocked" >&2 - return 3 - fi + current="present"$'\t'"$record" + else + current=absent + fi + if [ "$expected_supplied" -eq 1 ] && [ "$current" != "$expected_record" ]; then + echo "error: Akua authorization overlay presence or Secret identity changed during upgrade; upgrade blocked" >&2 + return 3 fi - printf '%s' "$record" + printf '%s' "$current" } rendered_resource_field() { @@ -1255,6 +1435,7 @@ case "$COMMAND" in exit 2 fi preflight_legacy_cluster_binding 0 + ensure_control_lock_access apply_rendered ;; upgrade) @@ -1277,6 +1458,19 @@ case "$COMMAND" in render require_namespaced_resource_owned_or_absent ServiceAccount "$SERVICE_ACCOUNT_NAME" ;; + agent-os-firstmate) + [ "$(live_resource_identity ServiceAccount agent-os-firstmate)" = \ + "agent-os-firstmate"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" ] || { + echo "error: retained legacy ServiceAccount dependency is missing or foreign" >&2 + exit 3 + } + legacy_service_account_record=$(live_resource_record namespaced ServiceAccount agent-os-firstmate) + [ -n "$(printf '%s' "$legacy_service_account_record" | cut -f4)" ] && \ + [ -n "$(printf '%s' "$legacy_service_account_record" | cut -f5)" ] || { + echo "error: retained legacy ServiceAccount dependency lacks exact metadata" >&2 + exit 3 + } + ;; esac preflight_legacy_cluster_binding 1 require_no_active_rollback_checkpoint @@ -1295,6 +1489,7 @@ case "$COMMAND" in cleanup_required=1 patch_workload_annotations '{"agent-os.dev/cluster-rbac-cleanup":"required"}' fi + ensure_control_lock_access apply_rendered if ! verified_akua_overlay_secret "$PRESERVED_AKUA_SECRET_RECORD" >/dev/null; then echo "incomplete: upgrade applied but Akua authorization postcondition changed" >&2 @@ -1337,8 +1532,10 @@ case "$COMMAND" in echo "error: rollback requires exact StatefulSet UID and resourceVersion evidence" >&2 exit 2 fi - rollback_state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get statefulset agent-os-firstmate -o json) - rollback_revisions=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get controllerrevisions.apps -o json) + rollback_state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ + --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get statefulset agent-os-firstmate -o json) + rollback_revisions=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ + --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get controllerrevisions.apps -o json) rollback_references=$(printf '%s' "$rollback_state" | jq -er \ --arg uid "$rollback_uid" --arg rv "$rollback_rv" --arg installation "$INSTALLATION_ID" ' select(.metadata.name == "agent-os-firstmate") @@ -1349,7 +1546,10 @@ case "$COMMAND" in (.metadata.annotations["agent-os.dev/rollback-operation"] // ""), (.metadata.annotations["agent-os.dev/rollback-target-name"] // ""), (.metadata.annotations["agent-os.dev/rollback-target-uid"] // ""), - (.metadata.annotations["agent-os.dev/rollback-target-digest"] // "")] + (.metadata.annotations["agent-os.dev/rollback-target-digest"] // ""), + (.metadata.annotations["agent-os.dev/rollback-source-name"] // ""), + (.metadata.annotations["agent-os.dev/rollback-source-uid"] // ""), + (.metadata.annotations["agent-os.dev/rollback-source-digest"] // "")] | select((.[0:2]) | all(.[]; type == "string" and length > 0)) | @tsv ') || { echo "error: StatefulSet changed before rollback revision resolution" >&2; exit 3; } @@ -1359,12 +1559,21 @@ case "$COMMAND" in rollback_checkpoint_name=$(printf '%s' "$rollback_references" | cut -f4) rollback_checkpoint_uid=$(printf '%s' "$rollback_references" | cut -f5) rollback_checkpoint_digest=$(printf '%s' "$rollback_references" | cut -f6) - if [ -n "$rollback_checkpoint_operation$rollback_checkpoint_name$rollback_checkpoint_uid$rollback_checkpoint_digest" ]; then + rollback_source_name=$(printf '%s' "$rollback_references" | cut -f7) + rollback_source_uid=$(printf '%s' "$rollback_references" | cut -f8) + rollback_source_digest=$(printf '%s' "$rollback_references" | cut -f9) + if [ -n "$rollback_checkpoint_operation$rollback_checkpoint_name$rollback_checkpoint_uid$rollback_checkpoint_digest$rollback_source_name$rollback_source_uid$rollback_source_digest" ]; then if [ -z "$rollback_checkpoint_operation" ] || [ -z "$rollback_checkpoint_name" ] || \ - [ -z "$rollback_checkpoint_uid" ] || ! [[ "$rollback_checkpoint_digest" =~ ^[0-9a-f]{64}$ ]]; then + [ -z "$rollback_checkpoint_uid" ] || ! [[ "$rollback_checkpoint_digest" =~ ^[0-9a-f]{64}$ ]] || \ + [ -z "$rollback_source_name" ] || [ -z "$rollback_source_uid" ] || ! [[ "$rollback_source_digest" =~ ^[0-9a-f]{64}$ ]]; then echo "error: rollback checkpoint is incomplete or malformed; retained" >&2 exit 3 fi + rollback_source=$(owned_revision_by_digest "$rollback_revisions" "$rollback_uid" \ + "$rollback_source_digest" "$rollback_source_uid") || { + echo "error: rollback checkpoint source content is unavailable; retained" >&2 + exit 3 + } rollback_target=$(printf '%s' "$rollback_revisions" | jq -ce --arg name "$rollback_update_revision" --arg uid "$rollback_uid" ' [.items[] | select(.metadata.name == $name) | select(any(.metadata.ownerReferences[]?; .apiVersion == "apps/v1" and .kind == "StatefulSet" and .name == "agent-os-firstmate" and .uid == $uid and .controller == true))] @@ -1390,6 +1599,13 @@ case "$COMMAND" in rollback_mode=resume fi else + rollback_source=$(printf '%s' "$rollback_revisions" | jq -ce --arg name "$rollback_update_revision" --arg uid "$rollback_uid" ' + [.items[] | select(.metadata.name == $name) | select(any(.metadata.ownerReferences[]?; + .apiVersion == "apps/v1" and .kind == "StatefulSet" and .name == "agent-os-firstmate" and .uid == $uid and .controller == true))] + | select(length == 1) | .[0]') || { echo "error: current rollback source revision is unavailable" >&2; exit 3; } + rollback_source_name=$(printf '%s' "$rollback_source" | jq -r '.metadata.name') + rollback_source_uid=$(printf '%s' "$rollback_source" | jq -r '.metadata.uid') + rollback_source_digest=$(printf '%s' "$rollback_source" | jq -cS '.data.spec.template' | template_digest) rollback_selection=$(printf '%s' "$rollback_revisions" | jq -ce \ --arg current "$rollback_current_revision" --arg update "$rollback_update_revision" \ --arg uid "$rollback_uid" ' @@ -1426,7 +1642,8 @@ case "$COMMAND" in rollback_checkpoint_patch=$(jq -cn \ --arg uid "$rollback_uid" --arg rv "$rollback_rv" --arg operation "$rollback_checkpoint_operation" \ --arg name "$rollback_target_name" --arg targetUid "$rollback_target_uid" --arg digest "$rollback_target_digest" \ - '{metadata:{uid:$uid,resourceVersion:$rv,annotations:{"agent-os.dev/rollback-operation":$operation,"agent-os.dev/rollback-target-name":$name,"agent-os.dev/rollback-target-uid":$targetUid,"agent-os.dev/rollback-target-digest":$digest}}}') + --arg sourceName "$rollback_source_name" --arg sourceUid "$rollback_source_uid" --arg sourceDigest "$rollback_source_digest" \ + '{metadata:{uid:$uid,resourceVersion:$rv,annotations:{"agent-os.dev/rollback-operation":$operation,"agent-os.dev/rollback-target-name":$name,"agent-os.dev/rollback-target-uid":$targetUid,"agent-os.dev/rollback-target-digest":$digest,"agent-os.dev/rollback-source-name":$sourceName,"agent-os.dev/rollback-source-uid":$sourceUid,"agent-os.dev/rollback-source-digest":$sourceDigest}}}') if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch StatefulSet agent-os-firstmate \ --type=merge -p "$rollback_checkpoint_patch" >/dev/null; then echo "error: rollback checkpoint CAS conflicted; no template mutation attempted" >&2 @@ -1438,20 +1655,48 @@ case "$COMMAND" in rollback_rv=$(printf '%s' "$rollback_state" | jq -er \ --arg uid "$rollback_uid" --arg operation "$rollback_checkpoint_operation" \ --arg name "$rollback_checkpoint_name" --arg targetUid "$rollback_checkpoint_uid" \ - --arg digest "$rollback_target_digest" ' + --arg digest "$rollback_target_digest" --arg sourceName "$rollback_source_name" \ + --arg sourceUid "$rollback_source_uid" --arg sourceDigest "$rollback_source_digest" ' select(.metadata.uid == $uid) | select(.metadata.annotations["agent-os.dev/rollback-operation"] == $operation) | select(.metadata.annotations["agent-os.dev/rollback-target-name"] == $name) | select(.metadata.annotations["agent-os.dev/rollback-target-uid"] == $targetUid) | select(.metadata.annotations["agent-os.dev/rollback-target-digest"] == $digest) + | select(.metadata.annotations["agent-os.dev/rollback-source-name"] == $sourceName) + | select(.metadata.annotations["agent-os.dev/rollback-source-uid"] == $sourceUid) + | select(.metadata.annotations["agent-os.dev/rollback-source-digest"] == $sourceDigest) | .metadata.resourceVersion') || { echo "error: rollback checkpoint did not persist exactly" >&2; exit 3; } fi + verify_revision_service_account "$rollback_source" + rollback_source_template=$(printf '%s' "$rollback_source" | jq -c '.data.spec.template') + rollback_source_account=$(printf '%s' "$rollback_source" | jq -er '.data.spec.template.spec.serviceAccountName') || { + echo "error: rollback source ServiceAccount dependency is unavailable" >&2 + exit 3 + } + rollback_target_account=$(printf '%s' "$rollback_target" | jq -er '.data.spec.template.spec.serviceAccountName') || { + echo "error: rollback target ServiceAccount dependency is unavailable" >&2 + exit 3 + } + rollback_authority_mode=$(printf '%s' "$previous" | cut -f2) + case "$rollback_authority_mode" in namespace|cluster-admin|none) ;; *) echo "error: rollback RBAC mode is unavailable" >&2; exit 3 ;; esac + if ! transfer_runtime_authority "$rollback_authority_mode" "$rollback_source_account" "$rollback_target_account"; then + transfer_runtime_authority "$rollback_authority_mode" "$rollback_target_account" "$rollback_source_account" || { + echo "incomplete: rollback authority transfer failed and compensation is unverified" >&2 + exit 3 + } + echo "error: rollback authority transfer failed before revision activation; current authority restored" >&2 + exit 3 + fi if [ "$rollback_mode" = patch ]; then rollback_patch=$(printf '%s' "$rollback_target" | jq -c --arg uid "$rollback_uid" --arg rv "$rollback_rv" \ '.data * {metadata:{uid:$uid,resourceVersion:$rv}}') if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch StatefulSet agent-os-firstmate \ --type=strategic -p "$rollback_patch" >/dev/null; then - echo "error: StatefulSet rollback CAS conflicted; no rollout-undo fallback attempted" >&2 + compensate_rollback "$rollback_source_template" "$rollback_source_account" "$rollback_target_account" "$rollback_authority_mode" || { + echo "incomplete: StatefulSet rollback CAS conflicted and compensation is unverified" >&2 + exit 3 + } + echo "error: StatefulSet rollback CAS conflicted; current revision and authority restored" >&2 exit 3 fi rollback_current=$(live_resource_record namespaced StatefulSet agent-os-firstmate) @@ -1462,19 +1707,28 @@ case "$COMMAND" in fi fi if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s; then + if compensate_rollback "$rollback_source_template" "$rollback_source_account" "$rollback_target_account" "$rollback_authority_mode"; then + echo "incomplete: rollback readiness failed; current revision and authority restored" >&2 + exit 3 + fi report_rollback_failure "$rollback_target_name" "$rollback_target_revision" "$rollback_target_uid" "$rollback_target_digest" \ "$rollback_checkpoint_operation" "$rollback_checkpoint_name" "$rollback_checkpoint_uid" exit 3 fi rollback_verify_state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get statefulset agent-os-firstmate -o json) || { - echo "error: rollback verification StatefulSet evidence unavailable; checkpoint retained target-digest=$rollback_target_digest" >&2 + compensate_rollback "$rollback_source_template" "$rollback_source_account" "$rollback_target_account" "$rollback_authority_mode" || { + echo "error: rollback verification evidence and compensation are unavailable; checkpoint retained target-digest=$rollback_target_digest" >&2 + exit 3 + } + echo "incomplete: rollback verification evidence was unavailable; current revision and authority restored" >&2 exit 3 } rollback_verify_refs=$(printf '%s' "$rollback_verify_state" | jq -er \ --arg uid "$rollback_uid" --arg installation "$INSTALLATION_ID" \ --arg operation "$rollback_checkpoint_operation" --arg name "$rollback_checkpoint_name" \ - --arg targetUid "$rollback_checkpoint_uid" --arg digest "$rollback_target_digest" ' + --arg targetUid "$rollback_checkpoint_uid" --arg digest "$rollback_target_digest" \ + --arg sourceName "$rollback_source_name" --arg sourceUid "$rollback_source_uid" --arg sourceDigest "$rollback_source_digest" ' select(.metadata.uid == $uid) | select(.metadata.labels["app.kubernetes.io/managed-by"] == "agent-os") | select(.metadata.annotations["agent-os.dev/installation-id"] == $installation) @@ -1482,13 +1736,24 @@ case "$COMMAND" in | select(.metadata.annotations["agent-os.dev/rollback-target-name"] == $name) | select(.metadata.annotations["agent-os.dev/rollback-target-uid"] == $targetUid) | select(.metadata.annotations["agent-os.dev/rollback-target-digest"] == $digest) + | select(.metadata.annotations["agent-os.dev/rollback-source-name"] == $sourceName) + | select(.metadata.annotations["agent-os.dev/rollback-source-uid"] == $sourceUid) + | select(.metadata.annotations["agent-os.dev/rollback-source-digest"] == $sourceDigest) | [.status.currentRevision,.status.updateRevision,.metadata.resourceVersion] | @tsv') || { - echo "error: rollback verification mismatch: StatefulSet identity or checkpoint changed; retained target-digest=$rollback_target_digest" >&2 + compensate_rollback "$rollback_source_template" "$rollback_source_account" "$rollback_target_account" "$rollback_authority_mode" || { + echo "error: rollback verification mismatch and compensation failed; retained target-digest=$rollback_target_digest" >&2 + exit 3 + } + echo "incomplete: rollback verification mismatch; current revision and authority restored" >&2 exit 3 } rollback_revisions=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get controllerrevisions.apps -o json) || { - echo "error: rollback verification revision evidence unavailable; checkpoint retained target-digest=$rollback_target_digest" >&2 + compensate_rollback "$rollback_source_template" "$rollback_source_account" "$rollback_target_account" "$rollback_authority_mode" || { + echo "error: rollback revision evidence and compensation are unavailable; checkpoint retained target-digest=$rollback_target_digest" >&2 + exit 3 + } + echo "incomplete: rollback revision evidence was unavailable; current revision and authority restored" >&2 exit 3 } for rollback_verify_name in "$(printf '%s' "$rollback_verify_refs" | cut -f1)" "$(printf '%s' "$rollback_verify_refs" | cut -f2)"; do @@ -1496,17 +1761,29 @@ case "$COMMAND" in [.items[] | select(.metadata.name == $name) | select(any(.metadata.ownerReferences[]?; .apiVersion == "apps/v1" and .kind == "StatefulSet" and .name == "agent-os-firstmate" and .uid == $uid and .controller == true))] | select(length == 1) | .[0]') || { - echo "error: rollback verification mismatch: revision '$rollback_verify_name' is unavailable or foreign; retained target-digest=$rollback_target_digest" >&2 + compensate_rollback "$rollback_source_template" "$rollback_source_account" "$rollback_target_account" "$rollback_authority_mode" || { + echo "error: rollback revision '$rollback_verify_name' is unavailable and compensation failed" >&2 + exit 3 + } + echo "incomplete: rollback revision '$rollback_verify_name' is unavailable; current revision and authority restored" >&2 exit 3 } rollback_verify_digest=$(printf '%s' "$rollback_verify_revision" | jq -cS '.data.spec.template' | template_digest) if [ "$rollback_verify_digest" != "$rollback_target_digest" ]; then - echo "error: rollback verification mismatch: revision '$rollback_verify_name' content differs; retained target-digest=$rollback_target_digest" >&2 + compensate_rollback "$rollback_source_template" "$rollback_source_account" "$rollback_target_account" "$rollback_authority_mode" || { + echo "error: rollback revision '$rollback_verify_name' content differs and compensation failed" >&2 + exit 3 + } + echo "incomplete: rollback revision '$rollback_verify_name' content differs; current revision and authority restored" >&2 exit 3 fi done + transfer_runtime_authority "$rollback_authority_mode" "$rollback_source_account" "$rollback_target_account" || { + echo "error: rollback completed but runtime authority postcondition changed" >&2 + exit 3 + } rollback_clear_patch=$(jq -cn --arg uid "$rollback_uid" --arg rv "$(printf '%s' "$rollback_verify_refs" | cut -f3)" \ - '{metadata:{uid:$uid,resourceVersion:$rv,annotations:{"agent-os.dev/rollback-operation":null,"agent-os.dev/rollback-target-name":null,"agent-os.dev/rollback-target-uid":null,"agent-os.dev/rollback-target-digest":null}}}') + '{metadata:{uid:$uid,resourceVersion:$rv,annotations:{"agent-os.dev/rollback-operation":null,"agent-os.dev/rollback-target-name":null,"agent-os.dev/rollback-target-uid":null,"agent-os.dev/rollback-target-digest":null,"agent-os.dev/rollback-source-name":null,"agent-os.dev/rollback-source-uid":null,"agent-os.dev/rollback-source-digest":null}}}') if ! "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch StatefulSet agent-os-firstmate \ --type=merge -p "$rollback_clear_patch" >/dev/null; then echo "error: rollback completed but checkpoint clear CAS conflicted; retained target-digest=$rollback_target_digest" >&2 @@ -1530,16 +1807,40 @@ case "$COMMAND" in previous=$(workload_state) require_workload_owned "$previous" optional previous_service_account= + previous_workload_uid= + retained_legacy_service_account= if [ -n "$previous" ]; then previous_service_account=$(workload_service_account) + previous_workload_record=$(live_resource_record namespaced StatefulSet agent-os-firstmate) + previous_workload_uid=$(printf '%s' "$previous_workload_record" | cut -f4) + [ -n "$previous_workload_uid" ] || { echo "error: uninstall requires exact StatefulSet UID evidence" >&2; exit 3; } + fi + legacy_identity=$(live_resource_identity ServiceAccount agent-os-firstmate) + if [ -n "$legacy_identity" ]; then + [ "$legacy_identity" = "agent-os-firstmate"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" ] || { + echo "error: retained legacy ServiceAccount is foreign" >&2 + exit 3 + } + retained_legacy_service_account=agent-os-firstmate fi previous_mode=$(printf '%s' "$previous" | cut -f2) previous_cleanup=$(printf '%s' "$previous" | cut -f3) delete_rendered_namespaced_resources + if [ -n "$previous_workload_uid" ] || [ -n "$retained_legacy_service_account" ]; then + wait_for_revision_history_absence "$previous_workload_uid" + fi if [ -n "$previous_service_account" ] && [ "$previous_service_account" != "$SERVICE_ACCOUNT_NAME" ]; then delete_owned_resource namespaced ServiceAccount "$previous_service_account" 180 fi + if [ -n "$retained_legacy_service_account" ] && \ + [ "$retained_legacy_service_account" != "$previous_service_account" ] && \ + [ "$retained_legacy_service_account" != "$SERVICE_ACCOUNT_NAME" ]; then + delete_owned_resource namespaced ServiceAccount "$retained_legacy_service_account" 180 + fi delete_namespace_rbac + if [ "$DESIRED_RBAC" = namespace ] || [ "$previous_mode" = namespace ]; then + REMOVE_CONTROL_ACCESS=1 + fi if [ "$DELETE_NAMESPACE" -eq 1 ]; then delete_owned_empty_namespace fi diff --git a/bin/agent-os-local.sh b/bin/agent-os-local.sh index 8736b114f..4f33ccafc 100755 --- a/bin/agent-os-local.sh +++ b/bin/agent-os-local.sh @@ -62,7 +62,7 @@ case "$COMMAND" in source_branch=$(printf '%s\n' "$source_metadata" | awk -F= '$1 == "branch" { print $2 }') source_origin=$(printf '%s\n' "$source_metadata" | awk -F= '$1 == "origin" { sub(/^[^=]*=/, ""); print }') [ -n "$source_commit" ] && [ -n "$source_tree" ] && [ -n "$source_branch" ] && [ -n "$source_origin" ] || { - echo "error: exact-source bundle metadata is incomplete" >&2 + echo "error: exact-source bootstrap metadata is incomplete" >&2 exit 2 } docker build \ diff --git a/bin/agent-os-source-bundle.sh b/bin/agent-os-source-bundle.sh index 41d6c9b79..923fd9c6e 100755 --- a/bin/agent-os-source-bundle.sh +++ b/bin/agent-os-source-bundle.sh @@ -1,29 +1,78 @@ #!/usr/bin/env bash set -eu -ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) -OUTPUT=${1:-"$ROOT/image/agent-os-source.bundle"} +SCRIPT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +ROOT=$(cd "${AGENT_OS_SOURCE_ROOT:-$SCRIPT_ROOT}" && pwd) +OUTPUT_DIR=${1:-"$ROOT/image"} SOURCE_BRANCH=${AGENT_OS_SOURCE_BRANCH:-main} SOURCE_ORIGIN=${AGENT_OS_SOURCE_ORIGIN:-https://github.com/akua-dev/agent-os.git} +SOURCE_REF="refs/heads/$SOURCE_BRANCH" +SOURCE_ARCHIVE="$OUTPUT_DIR/agent-os-source.tar" +BOOTSTRAP_ARCHIVE="$OUTPUT_DIR/agent-os-bootstrap.tar" +ATTESTATION="$OUTPUT_DIR/agent-os-source.attestation" +TEMP="$OUTPUT_DIR/.agent-os-source.$$" + [ "$SOURCE_BRANCH" = main ] || { echo "error: source branch must be the declared default branch 'main'" >&2; exit 2; } [ "$SOURCE_ORIGIN" = https://github.com/akua-dev/agent-os.git ] || { echo "error: source origin is not allowlisted" >&2; exit 2; } +case "$(git -C "$ROOT" remote get-url origin)" in + https://github.com/akua-dev/agent-os.git|ssh://git@github.com/akua-dev/agent-os.git|git@github.com:akua-dev/agent-os.git) ;; + *) echo "error: repository origin is not allowlisted" >&2; exit 2 ;; +esac [ -z "$(git -C "$ROOT" status --porcelain --untracked-files=all)" ] || { echo "error: exact-source image builds require a clean worktree" >&2 exit 2 } -SOURCE_COMMIT=$(git -C "$ROOT" rev-parse --verify HEAD) -SOURCE_TREE=$(git -C "$ROOT" rev-parse --verify 'HEAD^{tree}') -mkdir -p "$(dirname "$OUTPUT")" -TEMP="$OUTPUT.tmp.$$" -trap 'rm -f "$TEMP"' EXIT -git -C "$ROOT" bundle create "$TEMP" HEAD -git -C "$ROOT" bundle verify "$TEMP" >/dev/null -mv "$TEMP" "$OUTPUT" +mkdir -p "$OUTPUT_DIR" +rm -rf "$TEMP" +mkdir -p "$TEMP" +trap 'rm -rf "$TEMP" "$SOURCE_ARCHIVE.tmp.$$" "$BOOTSTRAP_ARCHIVE.tmp.$$" "$ATTESTATION.tmp.$$"' EXIT + +sha256_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +git init --bare "$TEMP/bootstrap.git" >/dev/null +GIT_TERMINAL_PROMPT=0 git -c credential.helper= --git-dir="$TEMP/bootstrap.git" fetch --depth=1 --no-tags "$SOURCE_ORIGIN" \ + "$SOURCE_REF:refs/heads/$SOURCE_BRANCH" +SOURCE_COMMIT=$(git --git-dir="$TEMP/bootstrap.git" rev-parse --verify "refs/heads/$SOURCE_BRANCH") +SOURCE_TREE=$(git --git-dir="$TEMP/bootstrap.git" rev-parse --verify "$SOURCE_COMMIT^{tree}") +[ "$(git -C "$ROOT" rev-parse --verify HEAD)" = "$SOURCE_COMMIT" ] || { + echo "error: local HEAD does not exactly match the freshly fetched trusted source ref" >&2 + exit 2 +} +[ "$(git -C "$ROOT" rev-parse --verify 'HEAD^{tree}')" = "$SOURCE_TREE" ] || { + echo "error: local source tree does not exactly match the trusted source ref" >&2 + exit 2 +} + +git --git-dir="$TEMP/bootstrap.git" symbolic-ref HEAD "refs/heads/$SOURCE_BRANCH" +git --git-dir="$TEMP/bootstrap.git" remote add origin "$SOURCE_ORIGIN" +rm -rf "$TEMP/bootstrap.git/hooks" "$TEMP/bootstrap.git/logs" +rm -f "$TEMP/bootstrap.git/FETCH_HEAD" "$TEMP/bootstrap.git/ORIG_HEAD" +git -C "$ROOT" archive --format=tar --output="$SOURCE_ARCHIVE.tmp.$$" "$SOURCE_COMMIT" +(cd "$TEMP" && find bootstrap.git -exec touch -t 197001010000 {} + && \ + find bootstrap.git -print | LC_ALL=C sort | COPYFILE_DISABLE=1 tar -cf "$BOOTSTRAP_ARCHIVE.tmp.$$" -T -) +SOURCE_SHA=$(sha256_file "$SOURCE_ARCHIVE.tmp.$$") +BOOTSTRAP_SHA=$(sha256_file "$BOOTSTRAP_ARCHIVE.tmp.$$") +{ + printf 'commit=%s\n' "$SOURCE_COMMIT" + printf 'tree=%s\n' "$SOURCE_TREE" + printf 'branch=%s\n' "$SOURCE_BRANCH" + printf 'origin=%s\n' "$SOURCE_ORIGIN" + printf 'ref=%s\n' "$SOURCE_REF" + printf 'source_sha256=%s\n' "$SOURCE_SHA" + printf 'bootstrap_sha256=%s\n' "$BOOTSTRAP_SHA" +} > "$ATTESTATION.tmp.$$" +mv "$SOURCE_ARCHIVE.tmp.$$" "$SOURCE_ARCHIVE" +mv "$BOOTSTRAP_ARCHIVE.tmp.$$" "$BOOTSTRAP_ARCHIVE" +mv "$ATTESTATION.tmp.$$" "$ATTESTATION" trap - EXIT +rm -rf "$TEMP" -printf 'commit=%s\n' "$SOURCE_COMMIT" -printf 'tree=%s\n' "$SOURCE_TREE" -printf 'branch=%s\n' "$SOURCE_BRANCH" -printf 'origin=%s\n' "$SOURCE_ORIGIN" +cat "$ATTESTATION" diff --git a/deploy/akua/firstmate-auth-grant.yaml b/deploy/akua/firstmate-auth-grant.yaml index 4acc58106..bc71d7b5b 100644 --- a/deploy/akua/firstmate-auth-grant.yaml +++ b/deploy/akua/firstmate-auth-grant.yaml @@ -1,6 +1,7 @@ metadata: annotations: agent-os.dev/akua-auth-secret: __AKUA_AUTH_SECRET__ + agent-os.dev/akua-auth-rejected-record: null spec: template: spec: diff --git a/deploy/akua/firstmate-auth-revoke.yaml b/deploy/akua/firstmate-auth-revoke.yaml index be1bde51c..146557654 100644 --- a/deploy/akua/firstmate-auth-revoke.yaml +++ b/deploy/akua/firstmate-auth-revoke.yaml @@ -1,6 +1,7 @@ metadata: annotations: agent-os.dev/akua-auth-secret: null + agent-os.dev/akua-auth-rejected-record: null spec: template: spec: diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 171ef8c5b..207f1e392 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -106,25 +106,26 @@ assert_grep 'AGENT_OS_CREWMATE_TEMPLATE' "$ROOT/bin/agent-os-crewmate.sh" \ "mate runtime must render the canonical package template" assert_absent "$ROOT/tools/agent-os/packages/mate/package.k" \ "mate creation must not remain a separately installable package" -assert_grep '.git' "$ROOT/.dockerignore" "git metadata must stay out of the build context" -assert_grep '.pi' "$ROOT/.dockerignore" "Pi credentials must stay out of the build context" -assert_grep '.codex' "$ROOT/.dockerignore" "Codex credentials must stay out of the build context" -assert_grep 'node_modules' "$ROOT/.dockerignore" "host dependencies must stay out of the build context" -assert_grep '.repos' "$ROOT/.dockerignore" "development source checkouts must stay out of the build context" -assert_grep '!.pi/extensions/fm-primary-pi-watch.ts' "$ROOT/.dockerignore" "tracked Pi supervision controls must enter the image" -assert_grep '!.codex/hooks.json' "$ROOT/.dockerignore" "tracked Codex supervision controls must enter the image" -assert_grep '!.opencode/plugins/fm-primary-watch-arm.js' "$ROOT/.dockerignore" "tracked OpenCode supervision controls must enter the image" -assert_grep '.pi/extensions/*' "$ROOT/.dockerignore" "Pi harness controls must use a nested default-deny rule" -assert_grep '.grok/hooks/*' "$ROOT/.dockerignore" "Grok harness controls must use a nested default-deny rule" -assert_grep '.opencode/plugins/*' "$ROOT/.dockerignore" "OpenCode harness controls must use a nested default-deny rule" -assert_grep 'agent-os-source.bundle' "$ROOT/Dockerfile" "image must bootstrap an exact source commit without host Git metadata" +assert_grep '**' "$ROOT/.dockerignore" "the Docker context must default-deny every local path" +assert_grep '!image/agent-os-source.tar' "$ROOT/.dockerignore" "the Docker context must admit only the tracked source export" +assert_grep '!image/agent-os-bootstrap.tar' "$ROOT/.dockerignore" "the Docker context must admit only the shallow bootstrap export" +assert_no_grep '!.pi/' "$ROOT/.dockerignore" "local harness homes must never be admitted directly" +assert_grep 'agent-os-source.tar' "$ROOT/Dockerfile" "image must consume the tracked exact-source export" +assert_grep 'agent-os-bootstrap.tar' "$ROOT/Dockerfile" "image must consume the shallow sanitized bootstrap" assert_grep 'rev-parse "$AGENT_OS_SOURCE_COMMIT^{tree}"' "$ROOT/Dockerfile" "image source bootstrap must verify its exact tree" -assert_grep 'AS source-bootstrap' "$ROOT/Dockerfile" "source bundle processing must stay in an isolated build stage" -assert_no_grep 'COPY image/agent-os-source.bundle /opt/agent-os-source.bundle' "$ROOT/Dockerfile" \ - "the final image must not retain a duplicate source bundle" +assert_grep 'AS source-bootstrap' "$ROOT/Dockerfile" "source processing must stay in an isolated build stage" +assert_no_grep 'COPY \. /opt/agent-os' "$ROOT/Dockerfile" "the image must never copy the ambient workspace" +assert_grep 'fetch --depth=1 --no-tags' "$ROOT/bin/agent-os-source-bundle.sh" \ + "source preparation must fetch an allowlisted remote ref freshly" +assert_grep 'status --porcelain --untracked-files=all' "$ROOT/bin/agent-os-source-bundle.sh" \ + "source preparation must reject a dirty workspace" +assert_no_grep 'bundle create.*HEAD' "$ROOT/bin/agent-os-source-bundle.sh" \ + "source bootstrap must not retain reachable deleted history" assert_grep 'merge --ff-only' "$ROOT/bin/agent-os-container-entrypoint.sh" "persistent source transitions must be fast-forward only" -assert_grep 'TRUSTED_REF="refs/remotes/origin/$SOURCE_BRANCH"' "$ROOT/bin/agent-os-container-entrypoint.sh" \ - "canonical runtime source must match the declared trusted remote branch" +assert_grep 'refs/remotes/agent-os-verified' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "canonical runtime source must use a freshly fetched verification ref" +assert_grep 'fetch --no-tags --prune origin' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "runtime provenance must fail closed unless the trusted remote is reachable" assert_no_grep 'checkout --detach' "$ROOT/bin/agent-os-container-entrypoint.sh" \ "canonical runtime source must remain on the declared default branch" assert_grep 'FM_ROOT_OVERRIDE=' "$ROOT/bin/agent-os-container-entrypoint.sh" "runtime must use the persistent canonical Firstmate repository" diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index 392a79af1..1468b485c 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -5,6 +5,23 @@ set -u # shellcheck source=tests/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +assert_grep 'configure_control_lock' "$ROOT/bin/agent-os-crewmate.sh" \ + "crewmate mutations must acquire the stable installation control lock" +assert_grep 'CONTROL_LOCK_UID' "$ROOT/bin/agent-os-crewmate.sh" \ + "crewmate mutations must retain control-lock CAS evidence" +assert_grep 'configure_control_lock' "$ROOT/bin/agent-os-akua-auth.sh" \ + "authorization mutations must acquire the stable installation control lock" +assert_grep 'agent-os-firstmate-lifecycle' "$ROOT/bin/agent-os-akua-auth.sh" \ + "authorization mutations must also acquire the namespace fleet lock" +assert_grep 'current=absent' "$ROOT/bin/agent-os-kubernetes.sh" \ + "upgrade snapshots must represent an absent authorization overlay explicitly" +assert_grep 'transfer_runtime_authority' "$ROOT/bin/agent-os-kubernetes.sh" \ + "rollback must transactionally transfer runtime authority" +assert_grep 'compensate_rollback' "$ROOT/bin/agent-os-kubernetes.sh" \ + "failed rollback must restore the current revision and authority" +assert_grep 'wait_for_revision_history_absence' "$ROOT/bin/agent-os-kubernetes.sh" \ + "uninstall must retire historical ServiceAccounts only after revision history" + LAUNCHER="$ROOT/bin/agent-os-crewmate.sh" TMP=$(fm_test_tmproot agent-os-kubernetes) FAKEBIN=$(fm_fakebin "$TMP") @@ -59,7 +76,10 @@ if [ "${*: -2}" = "-f -" ]; then stdin_data=$(cat) printf '%s\n' "$stdin_data" >> "${AGENT_OS_STDIN_LOG:-$AGENT_OS_TEST_LOG.stdin}" stdin_kind=$(printf '%s\n' "$stdin_data" | awk '$1 == "kind:" { print $2; exit }') + [ -n "$stdin_kind" ] || stdin_kind=$(printf '%s' "$stdin_data" | jq -r '.kind // empty' 2>/dev/null || true) printf 'stdin-kind %s\n' "$stdin_kind" >> "$AGENT_OS_TEST_LOG" + stdin_name=$(printf '%s' "$stdin_data" | jq -r '.metadata.name // empty' 2>/dev/null || true) + [ -z "$stdin_name" ] || printf 'stdin-resource %s %s\n' "$stdin_kind" "$stdin_name" >> "$AGENT_OS_TEST_LOG" if [ "$stdin_kind" = Lease ]; then stdin_name=$(printf '%s\n' "$stdin_data" | awk '$1 == "name:" { print $2; exit }') printf 'stdin-lease %s\n' "$stdin_name" >> "$AGENT_OS_TEST_LOG" @@ -117,6 +137,24 @@ fi if [ -n "${AGENT_OS_TEST_ROLLOUT_DELAY:-}" ] && [[ " $* " = *" rollout status statefulset/agent-os-firstmate "* ]]; then sleep "$AGENT_OS_TEST_ROLLOUT_DELAY" fi +if [[ " $* " = *" get Role agent-os-lifecycle-"*" -o json "* ]] || \ + [[ " $* " = *" get role agent-os-lifecycle-"*" -o json "* ]]; then + control_name=$(printf '%s\n' "$*" | sed -n 's/.* get [Rr]ole \([^ ]*\) .*/\1/p') + if ! grep -Fq "/roles/$control_name" "$AGENT_OS_TEST_LOG"; then + printf '{"metadata":{"name":"%s","uid":"uid-control-role","resourceVersion":"rv-control-role","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"rules":[{"apiGroups":["coordination.k8s.io"],"resources":["leases"],"resourceNames":["%s"],"verbs":["get","update"]}]}\n' "$control_name" "$control_name" + fi + exit 0 +fi +if [[ " $* " = *" get RoleBinding agent-os-lifecycle-"*" -o json "* ]] || \ + [[ " $* " = *" get rolebinding agent-os-lifecycle-"*" -o json "* ]]; then + control_name=$(printf '%s\n' "$*" | sed -n 's/.* get [Rr]ole[Bb]inding \([^ ]*\) .*/\1/p') + if ! grep -Fq "/rolebindings/$control_name" "$AGENT_OS_TEST_LOG"; then + control_account=$(grep 'akua-input-service-account ' "$AGENT_OS_TEST_LOG" | tail -n 1 | awk '{print $2}') + [ -n "$control_account" ] || control_account=agent-os-firstmate + printf '{"metadata":{"name":"%s","uid":"uid-control-binding","resourceVersion":"rv-control-binding","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"%s"},"subjects":[{"kind":"ServiceAccount","name":"%s","namespace":"portable-agent-os"}]}\n' "$control_name" "$control_name" "$control_account" + fi + exit 0 +fi case " $* " in *" get pod agent-os-crewmate-"*" --ignore-not-found -o jsonpath="*) id=${AGENT_OS_TEST_CREWMATE_ID:-scout-1} @@ -253,10 +291,12 @@ case " $* " in lock_holder=$(awk '/holderIdentity:/ { holder=$2 } END { print holder }' "$lock_stdin") lock_acquired=$(awk '/acquireTime:/ { value=$2 } END { print value }' "$lock_stdin") lock_renewed=$(awk '/renewTime:/ { value=$2 } END { print value }' "$lock_stdin") - lock_rv=${AGENT_OS_TEST_PRIMARY_LOCK_RV:-rv-primary-lock} - if grep -F ' replace -f -' "$AGENT_OS_TEST_LOG" >/dev/null; then - lock_rv=rv-primary-lock-renewed - fi + lock_replace_count=$(awk -v name="$lock_name" ' + /kubectl .* replace -f -/ { replacing=1; next } + /^stdin-lease / { if (replacing && $2 == name) count++; replacing=0 } + END { print count+0 } + ' "$AGENT_OS_TEST_LOG") + lock_rv="${AGENT_OS_TEST_PRIMARY_LOCK_RV:-rv-primary-lock}-$lock_replace_count" printf '%s\tagent-os\tprimary\t%s\t%s\t%s\t%s\t%s\tuid-primary-lock\t%s' \ "$lock_name" "$lock_installation" "$lock_holder" "$lock_acquired" "$lock_renewed" \ "${AGENT_OS_LOCK_DURATION_SECONDS:-300}" "$lock_rv" @@ -379,14 +419,18 @@ case " $* " in akua_volume=$(printf ',{"name":"akua-auth","secret":{"secretName":"%s","defaultMode":256}}' "$AGENT_OS_TEST_AKUA_OVERLAY_SECRET") fi if [ -n "${AGENT_OS_TEST_ROLLBACK_CHECKPOINT_DIGEST:-}" ]; then - checkpoint_annotations=$(printf ',"agent-os.dev/rollback-operation":"checkpoint-operation","agent-os.dev/rollback-target-name":"agent-os-firstmate-previous","agent-os.dev/rollback-target-uid":"uid-revision-previous","agent-os.dev/rollback-target-digest":"%s"' "$AGENT_OS_TEST_ROLLBACK_CHECKPOINT_DIGEST") + checkpoint_source_digest=$(printf '%s' '{"metadata":{"labels":{"rollback":"current"}},"spec":{"serviceAccountName":"agent-os-firstmate"}}' | jq -cS . | shasum -a 256 | awk '{print $1}') + checkpoint_annotations=$(printf ',"agent-os.dev/rollback-operation":"checkpoint-operation","agent-os.dev/rollback-target-name":"agent-os-firstmate-previous","agent-os.dev/rollback-target-uid":"uid-revision-previous","agent-os.dev/rollback-target-digest":"%s","agent-os.dev/rollback-source-name":"agent-os-firstmate-current","agent-os.dev/rollback-source-uid":"uid-revision-current","agent-os.dev/rollback-source-digest":"%s"' "$AGENT_OS_TEST_ROLLBACK_CHECKPOINT_DIGEST" "$checkpoint_source_digest") elif checkpoint_call=$(grep 'patch StatefulSet agent-os-firstmate --type=merge' "$AGENT_OS_TEST_LOG" | grep 'rollback-target-digest' | head -n 1); then checkpoint_operation=$(printf '%s' "$checkpoint_call" | sed -n 's/.*rollback-operation":"\([^"]*\).*/\1/p') checkpoint_name=$(printf '%s' "$checkpoint_call" | sed -n 's/.*rollback-target-name":"\([^"]*\).*/\1/p') checkpoint_uid=$(printf '%s' "$checkpoint_call" | sed -n 's/.*rollback-target-uid":"\([^"]*\).*/\1/p') checkpoint_digest=$(printf '%s' "$checkpoint_call" | sed -n 's/.*rollback-target-digest":"\([0-9a-f]*\).*/\1/p') - checkpoint_annotations=$(printf ',"agent-os.dev/rollback-operation":"%s","agent-os.dev/rollback-target-name":"%s","agent-os.dev/rollback-target-uid":"%s","agent-os.dev/rollback-target-digest":"%s"' \ - "$checkpoint_operation" "$checkpoint_name" "$checkpoint_uid" "$checkpoint_digest") + checkpoint_source_name=$(printf '%s' "$checkpoint_call" | sed -n 's/.*rollback-source-name":"\([^"]*\).*/\1/p') + checkpoint_source_uid=$(printf '%s' "$checkpoint_call" | sed -n 's/.*rollback-source-uid":"\([^"]*\).*/\1/p') + checkpoint_source_digest=$(printf '%s' "$checkpoint_call" | sed -n 's/.*rollback-source-digest":"\([0-9a-f]*\).*/\1/p') + checkpoint_annotations=$(printf ',"agent-os.dev/rollback-operation":"%s","agent-os.dev/rollback-target-name":"%s","agent-os.dev/rollback-target-uid":"%s","agent-os.dev/rollback-target-digest":"%s","agent-os.dev/rollback-source-name":"%s","agent-os.dev/rollback-source-uid":"%s","agent-os.dev/rollback-source-digest":"%s"' \ + "$checkpoint_operation" "$checkpoint_name" "$checkpoint_uid" "$checkpoint_digest" "$checkpoint_source_name" "$checkpoint_source_uid" "$checkpoint_source_digest") fi if [ "${AGENT_OS_TEST_ROLLBACK_CHECKPOINT_MUTATE:-0}" = 1 ] && \ grep -F 'rollout status statefulset/agent-os-firstmate' "$AGENT_OS_TEST_LOG" >/dev/null; then @@ -408,12 +452,32 @@ case " $* " in fi ;; *" get controllerrevisions.apps -o json "*) - if [ "${AGENT_OS_TEST_ROLLBACK_RENUMBERED:-0}" = 1 ]; then + if grep -Fq '/statefulsets/agent-os-firstmate' "$AGENT_OS_TEST_LOG" && grep -Fq 'delete --raw' "$AGENT_OS_TEST_LOG"; then + printf '%s\n' '{"items":[]}' + elif [ "${AGENT_OS_TEST_ROLLBACK_RENUMBERED:-0}" = 1 ]; then printf '%s\n' '{"items":[{"metadata":{"name":"agent-os-firstmate-previous","uid":"uid-revision-previous","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":1,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"previous"}},"spec":{"serviceAccountName":"agent-os-firstmate"}}}}},{"metadata":{"name":"agent-os-firstmate-current","uid":"uid-revision-current","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":2,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"current"}},"spec":{"serviceAccountName":"agent-os-firstmate"}}}}},{"metadata":{"name":"agent-os-firstmate-renumbered","uid":"uid-revision-renumbered","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":3,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"previous"}},"spec":{"serviceAccountName":"agent-os-firstmate"}}}}}]}' else printf '%s\n' '{"items":[{"metadata":{"name":"agent-os-firstmate-previous","uid":"uid-revision-previous","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":1,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"previous"}},"spec":{"serviceAccountName":"agent-os-firstmate"}}}}},{"metadata":{"name":"agent-os-firstmate-current","uid":"uid-revision-current","ownerReferences":[{"apiVersion":"apps/v1","kind":"StatefulSet","name":"agent-os-firstmate","uid":"uid-statefulset","controller":true}]},"revision":2,"data":{"spec":{"template":{"metadata":{"labels":{"rollback":"current"}},"spec":{"serviceAccountName":"agent-os-firstmate"}}}}}]}' fi ;; + *" get Role agent-os-lifecycle-"*" --ignore-not-found -o json "*|\ + *" get RoleBinding agent-os-lifecycle-"*" --ignore-not-found -o json "*|\ + *" get role agent-os-lifecycle-"*" --ignore-not-found -o json "*|\ + *" get rolebinding agent-os-lifecycle-"*" --ignore-not-found -o json "*|\ + *" get rolebinding agent-os-lifecycle-"*" -o json "*) + control_name=$(printf '%s\n' "$*" | sed -n 's/.* get \([Rr]ole\|[Rr]ole[Bb]inding\) \([^ ]*\) .*/\2/p') + control_kind=$(printf '%s\n' "$*" | sed -n 's/.* get \([Rr]ole\|[Rr]ole[Bb]inding\) .*/\1/p') + case "$control_kind" in role) control_kind=Role ;; rolebinding) control_kind=RoleBinding ;; esac + if ! grep -Fq "/$([ "$control_kind" = Role ] && printf roles || printf rolebindings)/$control_name" "$AGENT_OS_TEST_LOG"; then + control_account=$(grep 'akua-input-service-account ' "$AGENT_OS_TEST_LOG" | tail -n 1 | awk '{print $2}') + [ -n "$control_account" ] || control_account=agent-os-firstmate + if [ "$control_kind" = Role ]; then + printf '{"metadata":{"name":"%s","uid":"uid-control-role","resourceVersion":"rv-control-role","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"rules":[{"apiGroups":["coordination.k8s.io"],"resources":["leases"],"resourceNames":["%s"],"verbs":["get","update"]}]}\n' "$control_name" "$control_name" + else + printf '{"metadata":{"name":"%s","uid":"uid-control-binding","resourceVersion":"rv-control-binding","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"%s"},"subjects":[{"kind":"ServiceAccount","name":"%s","namespace":"portable-agent-os"}]}\n' "$control_name" "$control_name" "$control_account" + fi + fi + ;; *" get role agent-os-firstmate-runtime -o json "*) if [ "${AGENT_OS_TEST_RBAC_STATE:-exact}" = bad-rules ]; then printf '%s\n' '{"metadata":{"name":"agent-os-firstmate-runtime"},"rules":[]}' @@ -1148,7 +1212,7 @@ generated_service_account=$(grep 'akua-input-service-account ' "$CALLS" | tail - case "$generated_service_account" in agent-os-firstmate-[a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9]) ;; \ *) fail "install must use a fresh non-legacy ServiceAccount identity" ;; esac grep -Fqx 'kubectl --context kind-agent-os -n portable-agent-os rollout status statefulset/agent-os-firstmate --timeout=180s' "$CALLS" || \ - fail "generic install must wait for the rendered Firstmate StatefulSet" + fail "generic install must wait for the rendered Firstmate StatefulSet: $(tail -n 20 "$CALLS" | tr '\n' ';')" if grep -F 'delete clusterrolebinding' "$CALLS" >/dev/null; then fail "fresh namespace-scoped install must not require cluster RBAC deletion authority" fi @@ -1622,8 +1686,8 @@ renumbered_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STAT AGENT_OS_TEST_ROLLBACK_CHECKPOINT_DIGEST="$previous_template_digest" \ AGENT_OS_TEST_FAIL_ROLLOUT=1 run_generic rollback 2>&1) || renumbered_rc=$? [ "$renumbered_rc" -eq 3 ] || fail "renumbered rollback retry must remain incomplete: $renumbered_out" -assert_no_grep '"rollback":"current"' "$CALLS" \ - "rollback retry must never reverse to the post-patch current template" +assert_grep '"rollback":"current"' "$CALLS" \ + "rollback retry must compensate to its immutable checkpoint source template" assert_contains "$renumbered_out" 'target-digest=' \ "rollback retry must preserve the immutable checkpoint digest" assert_contains "$renumbered_out" 'target=agent-os-firstmate-renumbered' \ @@ -1639,7 +1703,7 @@ verification_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_ST AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_ROLLBACK_VERIFY_MISMATCH=1 \ run_generic rollback 2>&1) || verification_rc=$? [ "$verification_rc" -eq 3 ] || fail "rollback must fail closed when completed revisions do not match its checkpoint: $verification_out" -assert_contains "$verification_out" 'rollback verification mismatch' \ +assert_contains "$verification_out" "rollback revision 'agent-os-firstmate-current' content differs" \ "rollback must retain explicit evidence when another revision completes" pass "rollback success verifies both revision targets by content" diff --git a/tests/agent-os-local.test.sh b/tests/agent-os-local.test.sh index cfa28278a..c78e363cf 100755 --- a/tests/agent-os-local.test.sh +++ b/tests/agent-os-local.test.sh @@ -22,6 +22,24 @@ SH } make_fake docker +cat > "$FAKEBIN/git" <<'SH' +#!/usr/bin/env bash +case " $* " in + *" remote get-url origin "*) printf 'https://github.com/akua-dev/agent-os.git\n' ;; + *" status --porcelain --untracked-files=all "*) ;; + *" init --bare "*) destination=${!#}; mkdir -p "$destination"; printf 'shallow\n' > "$destination/shallow" ;; + *" fetch --depth=1 --no-tags "*) ;; + *" rev-parse --verify "*"^{tree}"*) printf '2222222222222222222222222222222222222222\n' ;; + *" rev-parse --verify "*) printf '1111111111111111111111111111111111111111\n' ;; + *" archive --format=tar --output="*) + for arg in "$@"; do + case "$arg" in --output=*) : > "${arg#--output=}" ;; esac + done + ;; + *) ;; +esac +SH +chmod +x "$FAKEBIN/git" cat > "$FAKEBIN/docker" <<'SH' #!/usr/bin/env bash printf '%s' "$(basename "$0")" >> "$AGENT_OS_TEST_LOG" @@ -83,6 +101,16 @@ if { [[ " $* " = *" get statefulset agent-os-firstmate --ignore-not-found -o jso printf 'agent-os-firstmate\tagent-os\tagent-os-firstmate:%s' "$namespace" fi fi +if [[ " $* " = *" get statefulset agent-os-firstmate -o json "* ]] && [ "$stateful_present" -eq 1 ]; then + printf '{"metadata":{"name":"agent-os-firstmate","uid":"uid-statefulset","resourceVersion":"rv-statefulset","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:%s"}},"spec":{"template":{"spec":{"serviceAccountName":"agent-os-firstmate","containers":[{"name":"firstmate","env":[],"volumeMounts":[]}],"volumes":[]}}},"status":{"currentRevision":"revision-current","updateRevision":"revision-current"}}\n' "$namespace" +fi +if [[ " $* " = *" get controllerrevisions.apps -o json "* ]]; then + printf '%s\n' '{"items":[]}' +fi +if [[ " $* " = *" get ServiceAccount agent-os-firstmate --ignore-not-found -o jsonpath="* ]] && [ "$stateful_present" -eq 1 ]; then + printf 'agent-os-firstmate\tagent-os\tagent-os-firstmate:%s' "$namespace" + [[ " $* " != *'.metadata.resourceVersion'* ]] || printf '\tuid-serviceaccount\trv-serviceaccount\t%s' "$operation" +fi if { [[ " $* " = *" get clusterrolebinding agent-os-firstmate-"*" --ignore-not-found -o jsonpath="* ]] || [[ " $* " = *" get ClusterRoleBinding agent-os-firstmate-"*" --ignore-not-found -o jsonpath="* ]]; }; then if [ "${AGENT_OS_TEST_LOCAL_WORKLOAD:-absent}" = present ] || grep -F 'clusterrolebinding.yaml' "$AGENT_OS_TEST_LOG" | grep -E ' create | patch ' >/dev/null; then @@ -105,7 +133,9 @@ if [[ " $* " = *" get lease agent-os-firstmate-lifecycle --ignore-not-found -o j lock_holder=$(grep '^stdin-holder ' "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d' ' -f2-) lock_acquired=$(grep '^stdin-acquired ' "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d' ' -f2-) lock_renewed=$(grep '^stdin-renewed ' "$AGENT_OS_TEST_LOG" | tail -n 1 | cut -d' ' -f2-) - printf '%s\tagent-os\tprimary\t%s\t%s\t%s\t%s\t300\tuid-lock\trv-lock' "$lock_name" "$lock_installation" "$lock_holder" "$lock_acquired" "$lock_renewed" + lock_rv=rv-lock + grep -Fq ' replace -f -' "$AGENT_OS_TEST_LOG" && lock_rv=rv-lock-renewed + printf '%s\tagent-os\tprimary\t%s\t%s\t%s\t%s\t300\tuid-lock\t%s' "$lock_name" "$lock_installation" "$lock_holder" "$lock_acquired" "$lock_renewed" "$lock_rv" fi fi SH @@ -282,7 +312,7 @@ test_destroy_requires_exact_confirmation() { cleanup_out=$(AGENT_OS_TEST_LOCAL_WORKLOAD=present run_cli destroy --yes 2>&1) || cleanup_rc=$? [ "$cleanup_rc" -eq 3 ] || \ fail "cluster-admin demo destroy must stop for separate privileged cleanup, got $cleanup_rc: $cleanup_out" - grep -Fq 'kubectl --context orbstack delete --raw /apis/apps/v1/namespaces/agent-os-demo/statefulsets/agent-os-firstmate -f -' "$LOG" || \ + grep -F 'delete --raw /apis/apps/v1/namespaces/agent-os-demo/statefulsets/agent-os-firstmate -f -' "$LOG" >/dev/null || \ fail "confirmed destroy must UID-delete the rendered OrbStack workload" if grep -E 'kubectl .* (get|delete) clusterrolebinding' "$LOG" >/dev/null; then fail "routine demo destroy must not inspect or delete cluster-scoped RBAC" From b3fe07ab9a0c870dadad8149f34a016a7f1d4f96 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 17:06:46 +0200 Subject: [PATCH 37/56] no-mistakes(review): Harden provenance and lifecycle compensation --- .github/workflows/agent-os-image.yml | 43 ++++++-- Dockerfile | 22 +++- bin/agent-os-akua-auth.sh | 30 ++++-- bin/agent-os-container-entrypoint.sh | 117 ++++++++++++-------- bin/agent-os-crewmate.sh | 2 +- bin/agent-os-kubernetes.sh | 149 ++++++++++++++++++++++--- bin/agent-os-local.sh | 7 +- bin/agent-os-source-bundle.sh | 156 ++++++++++++++++++++------- tests/agent-os-container.test.sh | 24 ++++- tests/agent-os-kubernetes.test.sh | 40 +++++-- tests/agent-os-local.test.sh | 1 + 11 files changed, 463 insertions(+), 128 deletions(-) diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index 6ee51fdea..530860696 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -41,12 +41,21 @@ jobs: if [ "$GITHUB_REF" = refs/heads/main ]; then [ "$REF_PROTECTED" = true ] || { echo "::error::main is not reported protected"; exit 1; } elif [ "${GITHUB_REF#refs/tags/}" != "$GITHUB_REF" ]; then + tag=${GITHUB_REF#refs/tags/} + [ "$(git cat-file -t "$GITHUB_REF")" = tag ] || { echo "::error::release tag must be annotated"; exit 1; } + record_commit=$(git for-each-ref --format='%(contents)' "$GITHUB_REF" | sed -n 's/^release-record-commit=\([0-9a-f]\{40\}\)$/\1/p') + [ -n "$record_commit" ] || { echo "::error::release tag lacks immutable record commit"; exit 1; } git fetch --no-tags origin refs/heads/main:refs/remotes/origin/main - [ "$(git rev-parse refs/remotes/origin/main)" = "$GITHUB_SHA" ] || { - echo "::error::release tag must point at the exact protected-main head" + git merge-base --is-ancestor "$record_commit" refs/remotes/origin/main + gh api "repos/$GITHUB_REPOSITORY/branches/main/protection" >/dev/null + git show "$record_commit:image/releases/$tag.json" > "$RUNNER_TEMP/release-record.json" + ruleset_id=$(jq -r .tag_ruleset_id "$RUNNER_TEMP/release-record.json") + gh api "repos/$GITHUB_REPOSITORY/rulesets/$ruleset_id" > "$RUNNER_TEMP/tag-ruleset.json" + ruleset_sha=$(jq -cS . "$RUNNER_TEMP/tag-ruleset.json" | sha256sum | awk '{print $1}') + [ "$ruleset_sha" = "$(jq -r .tag_ruleset_sha256 "$RUNNER_TEMP/release-record.json")" ] || { + echo "::error::release tag ruleset differs from immutable record" exit 1 } - gh api "repos/$GITHUB_REPOSITORY/branches/main/protection" >/dev/null else echo "::error::unsupported publication ref $GITHUB_REF" exit 1 @@ -59,17 +68,25 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 with: fetch-depth: 0 + persist-credentials: false + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - name: Prepare exact source bundle id: source run: | set -eu if [ "$GITHUB_EVENT_NAME" = pull_request ]; then - git clone --depth=1 --branch main https://github.com/akua-dev/agent-os.git "$RUNNER_TEMP/trusted-agent-os" - AGENT_OS_SOURCE_ROOT="$RUNNER_TEMP/trusted-agent-os" \ - bin/agent-os-source-bundle.sh "$GITHUB_WORKSPACE/image" > "$RUNNER_TEMP/source-metadata" + [ "$(git rev-parse HEAD)" = "${{ github.event.pull_request.head.sha }}" ] + AGENT_OS_SOURCE_MODE=event AGENT_OS_SOURCE_EVENT_COMMIT="${{ github.event.pull_request.head.sha }}" \ + bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" else - bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" + if [ "${GITHUB_REF#refs/tags/}" != "$GITHUB_REF" ]; then + record_commit=$(git for-each-ref --format='%(contents)' "$GITHUB_REF" | sed -n 's/^release-record-commit=\([0-9a-f]\{40\}\)$/\1/p') + AGENT_OS_SOURCE_MODE=release AGENT_OS_SOURCE_RELEASE_TAG="${GITHUB_REF#refs/tags/}" \ + AGENT_OS_RELEASE_RECORD_COMMIT="$record_commit" bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" + else + bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" + fi fi cat "$RUNNER_TEMP/source-metadata" >> "$GITHUB_OUTPUT" @@ -92,6 +109,8 @@ jobs: AGENT_OS_SOURCE_TREE=${{ steps.source.outputs.tree }} AGENT_OS_SOURCE_BRANCH=${{ steps.source.outputs.branch }} AGENT_OS_SOURCE_ORIGIN=${{ steps.source.outputs.origin }} + AGENT_OS_SOURCE_MODE=${{ steps.source.outputs.mode }} + AGENT_OS_SOURCE_REF=${{ steps.source.outputs.ref }} publish: if: github.event_name == 'push' @@ -110,7 +129,13 @@ jobs: id: source run: | set -eu - bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" + if [ "${GITHUB_REF#refs/tags/}" != "$GITHUB_REF" ]; then + record_commit=$(git for-each-ref --format='%(contents)' "$GITHUB_REF" | sed -n 's/^release-record-commit=\([0-9a-f]\{40\}\)$/\1/p') + AGENT_OS_SOURCE_MODE=release AGENT_OS_SOURCE_RELEASE_TAG="${GITHUB_REF#refs/tags/}" \ + AGENT_OS_RELEASE_RECORD_COMMIT="$record_commit" bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" + else + bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" + fi cat "$RUNNER_TEMP/source-metadata" >> "$GITHUB_OUTPUT" - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 @@ -152,6 +177,8 @@ jobs: AGENT_OS_SOURCE_TREE=${{ steps.source.outputs.tree }} AGENT_OS_SOURCE_BRANCH=${{ steps.source.outputs.branch }} AGENT_OS_SOURCE_ORIGIN=${{ steps.source.outputs.origin }} + AGENT_OS_SOURCE_MODE=${{ steps.source.outputs.mode }} + AGENT_OS_SOURCE_REF=${{ steps.source.outputs.ref }} - name: Record published image digest run: | diff --git a/Dockerfile b/Dockerfile index 0d2b7d1f1..f5fc845dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,8 @@ ARG AGENT_OS_SOURCE_COMMIT ARG AGENT_OS_SOURCE_TREE ARG AGENT_OS_SOURCE_BRANCH=main ARG AGENT_OS_SOURCE_ORIGIN=https://github.com/akua-dev/agent-os.git +ARG AGENT_OS_SOURCE_MODE=main +ARG AGENT_OS_SOURCE_REF=refs/heads/main RUN apt-get update \ && apt-get install -y --no-install-recommends git \ @@ -16,17 +18,18 @@ RUN set -eu; \ test -n "$AGENT_OS_SOURCE_TREE"; \ test "$AGENT_OS_SOURCE_BRANCH" = main; \ test "$AGENT_OS_SOURCE_ORIGIN" = https://github.com/akua-dev/agent-os.git; \ + case "$AGENT_OS_SOURCE_MODE" in main|event|release) ;; *) exit 1 ;; esac; \ + test -n "$AGENT_OS_SOURCE_REF"; \ + grep -Fx "mode=$AGENT_OS_SOURCE_MODE" /tmp/agent-os-source.attestation; \ grep -Fx "commit=$AGENT_OS_SOURCE_COMMIT" /tmp/agent-os-source.attestation; \ grep -Fx "tree=$AGENT_OS_SOURCE_TREE" /tmp/agent-os-source.attestation; \ grep -Fx "branch=$AGENT_OS_SOURCE_BRANCH" /tmp/agent-os-source.attestation; \ grep -Fx "origin=$AGENT_OS_SOURCE_ORIGIN" /tmp/agent-os-source.attestation; \ - grep -Fx "ref=refs/heads/$AGENT_OS_SOURCE_BRANCH" /tmp/agent-os-source.attestation; \ + grep -Fx "ref=$AGENT_OS_SOURCE_REF" /tmp/agent-os-source.attestation; \ source_sha=$(sha256sum /tmp/agent-os-source.tar | awk '{print $1}'); \ bootstrap_sha=$(sha256sum /tmp/agent-os-bootstrap.tar | awk '{print $1}'); \ grep -Fx "source_sha256=$source_sha" /tmp/agent-os-source.attestation; \ grep -Fx "bootstrap_sha256=$bootstrap_sha" /tmp/agent-os-source.attestation; \ - mkdir -p /opt/agent-os; \ - tar -xf /tmp/agent-os-source.tar -C /opt/agent-os; \ tar -xf /tmp/agent-os-bootstrap.tar -C /opt; \ mv /opt/bootstrap.git /opt/agent-os-bootstrap.git; \ test "$(git --git-dir=/opt/agent-os-bootstrap.git rev-parse "$AGENT_OS_SOURCE_COMMIT")" = "$AGENT_OS_SOURCE_COMMIT"; \ @@ -35,11 +38,18 @@ RUN set -eu; \ test "$(git --git-dir=/opt/agent-os-bootstrap.git remote get-url origin)" = "$AGENT_OS_SOURCE_ORIGIN"; \ test -z "$(git --git-dir=/opt/agent-os-bootstrap.git ls-tree -r --name-only "$AGENT_OS_SOURCE_COMMIT" -- config data projects state .no-mistakes)"; \ test ! -e /opt/agent-os-bootstrap.git/hooks; \ - rm -f /tmp/agent-os-source.tar /tmp/agent-os-bootstrap.tar /tmp/agent-os-source.attestation; \ + git --git-dir=/opt/agent-os-bootstrap.git archive --format=tar --output=/tmp/verified-source.tar "$AGENT_OS_SOURCE_COMMIT"; \ + cmp /tmp/verified-source.tar /tmp/agent-os-source.tar; \ + mkdir -p /opt/agent-os; \ + tar -xf /tmp/verified-source.tar -C /opt/agent-os; \ + test "$(git --git-dir=/opt/agent-os-bootstrap.git rev-parse "$AGENT_OS_SOURCE_COMMIT^{tree}")" = "$AGENT_OS_SOURCE_TREE"; \ + rm -f /tmp/verified-source.tar /tmp/agent-os-source.tar /tmp/agent-os-bootstrap.tar /tmp/agent-os-source.attestation; \ printf '%s\n' "$AGENT_OS_SOURCE_COMMIT" > /opt/agent-os-source.commit; \ printf '%s\n' "$AGENT_OS_SOURCE_TREE" > /opt/agent-os-source.tree; \ printf '%s\n' "$AGENT_OS_SOURCE_BRANCH" > /opt/agent-os-source.branch; \ - printf '%s\n' "$AGENT_OS_SOURCE_ORIGIN" > /opt/agent-os-source.origin + printf '%s\n' "$AGENT_OS_SOURCE_ORIGIN" > /opt/agent-os-source.origin; \ + printf '%s\n' "$AGENT_OS_SOURCE_MODE" > /opt/agent-os-source.mode; \ + printf '%s\n' "$AGENT_OS_SOURCE_REF" > /opt/agent-os-source.ref FROM node:24-trixie-slim@sha256:366fdef91728b1b7fa18c84fba63b6e79ed77b7e10cc206878e9705da4d7b169 @@ -218,6 +228,8 @@ COPY --from=source-bootstrap /opt/agent-os-source.commit /opt/agent-os-source.co COPY --from=source-bootstrap /opt/agent-os-source.tree /opt/agent-os-source.tree COPY --from=source-bootstrap /opt/agent-os-source.branch /opt/agent-os-source.branch COPY --from=source-bootstrap /opt/agent-os-source.origin /opt/agent-os-source.origin +COPY --from=source-bootstrap /opt/agent-os-source.mode /opt/agent-os-source.mode +COPY --from=source-bootstrap /opt/agent-os-source.ref /opt/agent-os-source.ref RUN install -D -m 0644 /opt/agent-os/THIRD_PARTY_NOTICES.md /usr/share/doc/agent-os/THIRD_PARTY_NOTICES.md \ && install -D -m 0644 /opt/agent-os/THIRD_PARTY_SOURCES.md /usr/share/doc/agent-os/THIRD_PARTY_SOURCES.md diff --git a/bin/agent-os-akua-auth.sh b/bin/agent-os-akua-auth.sh index 89dd5e9c4..252d7d4dd 100755 --- a/bin/agent-os-akua-auth.sh +++ b/bin/agent-os-akua-auth.sh @@ -212,6 +212,17 @@ reconcile_failed_grant() { "$(printf '%s' "$observed" | cut -f2)" "$(printf '%s' "$observed" | cut -f3)" >&2 } +fail_grant_closed() { + local reason=$1 + reconcile_failed_grant || { + echo "incomplete: $reason and fail-closed reconciliation is unverified" >&2 + exit 3 + } + echo "incomplete: $reason; authorization overlay removed and rejected identity retained" >&2 + echo "safe recovery: inspect the named Secret metadata, then run revoke before a new grant" >&2 + exit 3 +} + configure_control_lock LOCK=$CONTROL_LOCK LOCK_NAMESPACE=$CONTROL_NAMESPACE @@ -267,16 +278,19 @@ sed "s/__AKUA_AUTH_SECRET__/$SECRET/g" "$TEMPLATE" | awk -v uid="$uid_value" -v echo "error: Akua authorization Secret identity changed before StatefulSet CAS" >&2 exit 3 } -target_kube patch statefulset agent-os-firstmate --type=strategic --patch-file "$PATCH_FILE" >/dev/null -target_kube rollout status statefulset/agent-os-firstmate --timeout=180s +if ! target_kube patch statefulset agent-os-firstmate --type=strategic --patch-file "$PATCH_FILE" >/dev/null; then + [ "$COMMAND" != grant ] || fail_grant_closed "grant CAS failed ambiguously" + echo "error: revoke CAS failed" >&2 + exit 3 +fi +if ! target_kube rollout status statefulset/agent-os-firstmate --timeout=180s; then + [ "$COMMAND" != grant ] || fail_grant_closed "grant rollout failed" + echo "incomplete: revoke rollout failed" >&2 + exit 3 +fi if [ "$COMMAND" = grant ]; then if ! verify_overlay present "$STATE_UID"; then - reconcile_failed_grant || { - echo "incomplete: grant verification failed and fail-closed reconciliation is unverified" >&2 - exit 3 - } - echo "safe recovery: inspect the named Secret metadata, then run revoke before a new grant" >&2 - exit 3 + fail_grant_closed "grant verification failed" fi else verify_overlay absent "$STATE_UID" diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh index 84b56e2d3..13de840af 100755 --- a/bin/agent-os-container-entrypoint.sh +++ b/bin/agent-os-container-entrypoint.sh @@ -10,10 +10,41 @@ SOURCE_COMMIT=$(cat /opt/agent-os-source.commit) SOURCE_TREE=$(cat /opt/agent-os-source.tree) SOURCE_BRANCH=$(cat /opt/agent-os-source.branch) SOURCE_ORIGIN=$(cat /opt/agent-os-source.origin) +SOURCE_MODE=$(cat /opt/agent-os-source.mode) +SOURCE_REF=$(cat /opt/agent-os-source.ref) case "$SOURCE_BRANCH" in ''|*[!A-Za-z0-9._/-]*|/*|*/|*..*) echo "error: image source branch provenance is invalid" >&2; exit 2 ;; esac -TRUSTED_REF="refs/remotes/agent-os-verified/$SOURCE_BRANCH" +[ "$SOURCE_ORIGIN" = https://github.com/akua-dev/agent-os.git ] || { echo "error: image source origin is invalid" >&2; exit 2; } +case "$SOURCE_MODE" in + main) [ "$SOURCE_REF" = refs/heads/main ] || exit 2; TRUSTED_REF=refs/remotes/agent-os-verified/main ;; + release) [[ "$SOURCE_REF" =~ ^refs/tags/v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] || exit 2; TRUSTED_REF=refs/remotes/agent-os-verified/release ;; + event) echo "error: pull-request validation images are not runnable" >&2; exit 2 ;; + *) echo "error: image source mode is invalid" >&2; exit 2 ;; +esac IMAGE_REF="refs/remotes/agent-os-image/$SOURCE_BRANCH" +trusted_git() { + env -u GIT_CONFIG -u GIT_CONFIG_PARAMETERS -u GIT_CONFIG_COUNT \ + -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy \ + -u GIT_SSH -u GIT_SSH_COMMAND GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null \ + GIT_TERMINAL_PROMPT=0 command git -c credential.helper= -c core.hooksPath=/dev/null \ + -c http.proxy= -c https.proxy= "$@" +} + +validate_git_config() { + local config="$FM_ROOT/.git/config" origin + [ -f "$config" ] || { echo "error: canonical FM_ROOT Git config is unavailable" >&2; exit 2; } + if trusted_git config --file "$config" --no-includes --name-only --get-regexp \ + '^(url\.|include\.|includeIf\.|core\.hooksPath$|credential\.|http\.proxy$|https\.proxy$|remote\..*\.proxy$|alias\.)' >/dev/null 2>&1; then + echo "error: canonical FM_ROOT Git config contains untrusted execution or transport settings" >&2 + exit 2 + fi + origin=$(trusted_git config --file "$config" --no-includes --get remote.origin.url || true) + [ "$origin" = "$SOURCE_ORIGIN" ] && [ "$origin" = https://github.com/akua-dev/agent-os.git ] || { + echo "error: canonical FM_ROOT origin provenance changed" >&2 + exit 2 + } +} + mkdir -p \ "$FM_HOME/config" \ "$FM_HOME/data" \ @@ -28,92 +59,94 @@ mkdir -p \ if [ ! -e "$FM_ROOT/.git" ]; then [ ! -e "$FM_ROOT" ] || { echo "error: canonical FM_ROOT exists without Git provenance" >&2; exit 2; } - git clone --no-local --branch "$SOURCE_BRANCH" "$IMAGE_SOURCE" "$FM_ROOT" - [ "$(git -C "$FM_ROOT" rev-parse HEAD)" = "$SOURCE_COMMIT" ] || { + trusted_git -c protocol.file.allow=always clone --no-local --branch "$SOURCE_BRANCH" "$IMAGE_SOURCE" "$FM_ROOT" + trusted_git -C "$FM_ROOT" remote set-url origin "$SOURCE_ORIGIN" + validate_git_config + [ "$(trusted_git -C "$FM_ROOT" rev-parse HEAD)" = "$SOURCE_COMMIT" ] || { echo "error: canonical FM_ROOT bootstrap commit provenance failed" >&2 exit 2 } - git -C "$FM_ROOT" remote set-url origin "$SOURCE_ORIGIN" rm -rf "$FM_ROOT/.git/hooks" else [ -d "$FM_ROOT/.git" ] || { echo "error: canonical FM_ROOT Git metadata is invalid" >&2; exit 2; } - [ "$(git -C "$FM_ROOT" remote get-url origin)" = "$SOURCE_ORIGIN" ] || { - echo "error: canonical FM_ROOT origin provenance changed" >&2 - exit 2 - } - [ -z "$(git -C "$FM_ROOT" status --porcelain)" ] || { echo "error: canonical FM_ROOT is not clean" >&2; exit 2; } - git -C "$FM_ROOT" fetch --no-tags "$IMAGE_SOURCE" \ + validate_git_config + [ -z "$(trusted_git -C "$FM_ROOT" status --porcelain)" ] || { echo "error: canonical FM_ROOT is not clean" >&2; exit 2; } + trusted_git -c protocol.file.allow=always -C "$FM_ROOT" fetch --no-tags "$IMAGE_SOURCE" \ "refs/heads/$SOURCE_BRANCH:$IMAGE_REF" - [ "$(git -C "$FM_ROOT" rev-parse "$IMAGE_REF")" = "$SOURCE_COMMIT" ] || { + [ "$(trusted_git -C "$FM_ROOT" rev-parse "$IMAGE_REF")" = "$SOURCE_COMMIT" ] || { echo "error: image source trusted ref provenance failed" >&2 exit 2 } - [ "$(git -C "$FM_ROOT" rev-parse "$IMAGE_REF^{tree}")" = "$SOURCE_TREE" ] || { + [ "$(trusted_git -C "$FM_ROOT" rev-parse "$IMAGE_REF^{tree}")" = "$SOURCE_TREE" ] || { echo "error: image source trusted tree provenance failed" >&2 exit 2 } - current_branch=$(git -C "$FM_ROOT" symbolic-ref --quiet --short HEAD || true) + current_branch=$(trusted_git -C "$FM_ROOT" symbolic-ref --quiet --short HEAD || true) if [ -z "$current_branch" ]; then - git -C "$FM_ROOT" merge-base --is-ancestor HEAD "$IMAGE_REF" || { + trusted_git -C "$FM_ROOT" merge-base --is-ancestor HEAD "$IMAGE_REF" || { echo "error: detached canonical FM_ROOT lacks trusted fast-forward provenance" >&2 exit 2 } - if git -C "$FM_ROOT" show-ref --verify --quiet "refs/heads/$SOURCE_BRANCH"; then - [ "$(git -C "$FM_ROOT" rev-parse "refs/heads/$SOURCE_BRANCH")" = "$(git -C "$FM_ROOT" rev-parse HEAD)" ] || { + if trusted_git -C "$FM_ROOT" show-ref --verify --quiet "refs/heads/$SOURCE_BRANCH"; then + [ "$(trusted_git -C "$FM_ROOT" rev-parse "refs/heads/$SOURCE_BRANCH")" = "$(trusted_git -C "$FM_ROOT" rev-parse HEAD)" ] || { echo "error: canonical default branch conflicts with detached provenance" >&2 exit 2 } else - git -C "$FM_ROOT" branch "$SOURCE_BRANCH" HEAD + trusted_git -C "$FM_ROOT" branch "$SOURCE_BRANCH" HEAD fi - git -C "$FM_ROOT" checkout "$SOURCE_BRANCH" + trusted_git -C "$FM_ROOT" checkout "$SOURCE_BRANCH" fi - [ "$(git -C "$FM_ROOT" symbolic-ref --short HEAD)" = "$SOURCE_BRANCH" ] || { + [ "$(trusted_git -C "$FM_ROOT" symbolic-ref --short HEAD)" = "$SOURCE_BRANCH" ] || { echo "error: canonical FM_ROOT is not on the declared default branch" >&2 exit 2 } - if git -C "$FM_ROOT" merge-base --is-ancestor HEAD "$IMAGE_REF"; then - git -C "$FM_ROOT" merge --ff-only "$IMAGE_REF" - elif ! git -C "$FM_ROOT" merge-base --is-ancestor "$SOURCE_COMMIT" HEAD; then + if trusted_git -C "$FM_ROOT" merge-base --is-ancestor HEAD "$IMAGE_REF"; then + trusted_git -C "$FM_ROOT" merge --ff-only "$IMAGE_REF" + elif ! trusted_git -C "$FM_ROOT" merge-base --is-ancestor "$SOURCE_COMMIT" HEAD; then echo "error: canonical FM_ROOT source transition is not fast-forward compatible" >&2 exit 2 fi - git -C "$FM_ROOT" update-ref -d "$IMAGE_REF" + trusted_git -C "$FM_ROOT" update-ref -d "$IMAGE_REF" fi -[ "$(git -C "$FM_ROOT" symbolic-ref --short HEAD)" = "$SOURCE_BRANCH" ] || { +[ "$(trusted_git -C "$FM_ROOT" symbolic-ref --short HEAD)" = "$SOURCE_BRANCH" ] || { echo "error: canonical FM_ROOT is not on the declared default branch" >&2 exit 2 } -[ "$(git -C "$FM_ROOT" rev-parse "$SOURCE_COMMIT^{tree}")" = "$SOURCE_TREE" ] || { +[ "$(trusted_git -C "$FM_ROOT" rev-parse "$SOURCE_COMMIT^{tree}")" = "$SOURCE_TREE" ] || { echo "error: canonical FM_ROOT image source tree provenance failed" >&2 exit 2 } -[ "$(git -C "$FM_ROOT" remote get-url origin)" = "$SOURCE_ORIGIN" ] || { - echo "error: canonical FM_ROOT origin provenance changed" >&2 - exit 2 -} -GIT_TERMINAL_PROMPT=0 git -c credential.helper= -C "$FM_ROOT" fetch --no-tags --prune origin \ - "refs/heads/$SOURCE_BRANCH:$TRUSTED_REF" || { +validate_git_config +trusted_git -C "$FM_ROOT" fetch --no-tags --prune "$SOURCE_ORIGIN" \ + "$SOURCE_REF:$TRUSTED_REF" || { echo "error: fresh trusted source provenance is unavailable" >&2 exit 3 } -git -C "$FM_ROOT" merge-base --is-ancestor "$SOURCE_COMMIT" "$TRUSTED_REF" || { +trusted_git -C "$FM_ROOT" merge-base --is-ancestor "$SOURCE_COMMIT" "$TRUSTED_REF" || { echo "error: image source commit is not reachable from the fresh trusted ref" >&2 exit 2 } -git -C "$FM_ROOT" merge-base --is-ancestor HEAD "$TRUSTED_REF" || { - echo "error: canonical FM_ROOT contains source not reachable from the fresh trusted ref" >&2 - exit 2 -} -git -C "$FM_ROOT" merge --ff-only "$TRUSTED_REF" -[ "$(git -C "$FM_ROOT" rev-parse HEAD)" = "$(git -C "$FM_ROOT" rev-parse "$TRUSTED_REF")" ] || { - echo "error: canonical FM_ROOT HEAD is not the exact fresh trusted remote ref" >&2 +if [ "$SOURCE_MODE" = main ]; then + trusted_git -C "$FM_ROOT" merge-base --is-ancestor HEAD "$TRUSTED_REF" || { + echo "error: canonical FM_ROOT contains source not reachable from the fresh trusted ref" >&2 + exit 2 + } + trusted_git -C "$FM_ROOT" merge --ff-only "$TRUSTED_REF" +else + [ "$(trusted_git -C "$FM_ROOT" rev-parse HEAD)" = "$SOURCE_COMMIT" ] || { + echo "error: release FM_ROOT differs from its immutable release commit" >&2 + exit 2 + } +fi +[ "$(trusted_git -C "$FM_ROOT" rev-parse HEAD)" = "$(trusted_git -C "$FM_ROOT" rev-parse "$TRUSTED_REF^{commit}")" ] || { + echo "error: canonical FM_ROOT HEAD is not the exact fresh trusted source ref" >&2 exit 2 } -git -C "$FM_ROOT" update-ref "refs/remotes/origin/$SOURCE_BRANCH" "$TRUSTED_REF" -[ -z "$(git -C "$FM_ROOT" status --porcelain)" ] || { echo "error: canonical FM_ROOT is not clean" >&2; exit 2; } -[ -z "$(git -C "$FM_ROOT" ls-files -- config data projects state .no-mistakes)" ] || { +trusted_git -C "$FM_ROOT" update-ref "refs/remotes/origin/$SOURCE_BRANCH" "$TRUSTED_REF" +[ -z "$(trusted_git -C "$FM_ROOT" status --porcelain)" ] || { echo "error: canonical FM_ROOT is not clean" >&2; exit 2; } +[ -z "$(trusted_git -C "$FM_ROOT" ls-files -- config data projects state .no-mistakes)" ] || { echo "error: canonical FM_ROOT contains operational state" >&2 exit 2 } diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index ab3ad26ec..582f16825 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -189,7 +189,7 @@ ${rv:+ resourceVersion: $rv_value} app.kubernetes.io/managed-by: agent-os agent-os.dev/lifecycle: primary annotations: - agent-os.dev/installation-id: $INSTALLATION_ID + agent-os.dev/installation-id: ${LOCK_INSTALLATION_ID:-$INSTALLATION_ID} spec: holderIdentity: $LOCK_HOLDER_ID acquireTime: $acquired_at diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh index 9971c6219..c272a76f9 100755 --- a/bin/agent-os-kubernetes.sh +++ b/bin/agent-os-kubernetes.sh @@ -387,10 +387,35 @@ runtime_binding_json() { esac } +verify_control_authority() { + local account=$1 json + json=$(control_binding_json) || return 3 + printf '%s' "$json" | jq -e --arg installation "$INSTALLATION_ID" --arg namespace "$NAMESPACE" \ + --arg name "$CONTROL_LOCK" --arg account "$account" ' + .metadata.name == $name and .metadata.labels["app.kubernetes.io/managed-by"] == "agent-os" and + .metadata.annotations["agent-os.dev/installation-id"] == $installation and + .roleRef == {apiGroup:"rbac.authorization.k8s.io",kind:"Role",name:$name} and + .subjects == [{kind:"ServiceAccount",name:$account,namespace:$namespace}]' >/dev/null +} + +verify_no_runtime_authority() { + local resource + resource=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get rolebinding agent-os-firstmate-runtime --ignore-not-found -o name) || return 3 + [ -z "$resource" ] || return 3 + resource=$("$KUBECTL" --context "$CONTEXT" get clusterrolebinding "agent-os-firstmate-$NAMESPACE" --ignore-not-found -o name) || return 3 + [ -z "$resource" ] || return 3 + resource=$("$KUBECTL" --context "$CONTEXT" -n "$CONTROL_NAMESPACE" get rolebinding "$CONTROL_LOCK" --ignore-not-found -o name) || return 3 + [ -z "$resource" ] || return 3 + resource=$("$KUBECTL" --context "$CONTEXT" -n "$CONTROL_NAMESPACE" get role "$CONTROL_LOCK" --ignore-not-found -o name) || return 3 + [ -z "$resource" ] +} + transfer_runtime_authority() { local mode=$1 from=$2 to=$3 binding kind scope json uid rv current patch verified - [ "$from" != "$to" ] || return 0 - [ "$mode" != none ] || return 0 + if [ "$mode" = none ]; then + verify_no_runtime_authority || { echo "error: rollback none-mode authority is not absent" >&2; return 3; } + return 0 + fi if [ "$mode" = namespace ]; then binding=agent-os-firstmate-runtime kind=rolebinding @@ -414,8 +439,11 @@ transfer_runtime_authority() { | select(.subjects[0].kind == "ServiceAccount" and .subjects[0].namespace == $namespace) | .subjects[0].name') || { echo "error: rollback runtime authority ownership is unverifiable" >&2; return 3; } if [ "$current" = "$to" ]; then - [ "$mode" != namespace ] || transfer_control_authority "$from" "$to" - return $? + if [ "$mode" = namespace ]; then + transfer_control_authority "$from" "$to" || return 3 + verify_control_authority "$to" || return 3 + fi + return 0 fi [ "$current" = "$from" ] || { echo "error: rollback runtime authority is assigned to an unexpected ServiceAccount" >&2; return 3; } uid=$(printf '%s' "$json" | jq -er '.metadata.uid') || return 3 @@ -429,7 +457,10 @@ transfer_runtime_authority() { echo "error: rollback runtime authority transfer did not verify exactly" >&2 return 3 } - [ "$mode" != namespace ] || transfer_control_authority "$from" "$to" + if [ "$mode" = namespace ]; then + transfer_control_authority "$from" "$to" || return 3 + verify_control_authority "$to" + fi } control_binding_json() { @@ -439,6 +470,8 @@ control_binding_json() { transfer_control_authority() { local from=$1 to=$2 json current uid rv patch verified json=$(control_binding_json) || return 3 + printf '%s' "$json" | jq -e --arg name "$CONTROL_LOCK" \ + '.roleRef == {apiGroup:"rbac.authorization.k8s.io",kind:"Role",name:$name}' >/dev/null || return 3 current=$(printf '%s' "$json" | jq -er --arg installation "$INSTALLATION_ID" --arg namespace "$NAMESPACE" --arg name "$CONTROL_LOCK" ' select(.metadata.name == $name) | select(.metadata.labels["app.kubernetes.io/managed-by"] == "agent-os") @@ -446,7 +479,7 @@ transfer_control_authority() { | select(.subjects | type == "array" and length == 1) | select(.subjects[0].kind == "ServiceAccount" and .subjects[0].namespace == $namespace) | .subjects[0].name') || return 3 - [ "$current" != "$to" ] || return 0 + [ "$current" != "$to" ] || { verify_control_authority "$to"; return $?; } [ "$current" = "$from" ] || return 3 uid=$(printf '%s' "$json" | jq -er '.metadata.uid') || return 3 rv=$(printf '%s' "$json" | jq -er '.metadata.resourceVersion') || return 3 @@ -505,17 +538,35 @@ delete_control_lock_access() { } compensate_rollback() { - local source_template=$1 current_account=$2 target_account=$3 mode=$4 state uid rv patch + local source_template=$1 current_account=$2 target_account=$3 mode=$4 state uid rv patch refs revisions name revision digest clear transfer_runtime_authority "$mode" "$target_account" "$current_account" || return 3 state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get statefulset agent-os-firstmate -o json) || return 3 uid=$(printf '%s' "$state" | jq -er --arg expected "$rollback_uid" 'select(.metadata.uid == $expected) | .metadata.uid') || return 3 rv=$(printf '%s' "$state" | jq -er '.metadata.resourceVersion') || return 3 patch=$(jq -cn --arg uid "$uid" --arg rv "$rv" --argjson template "$source_template" \ - '{metadata:{uid:$uid,resourceVersion:$rv,annotations:{"agent-os.dev/rollback-operation":null,"agent-os.dev/rollback-target-name":null,"agent-os.dev/rollback-target-uid":null,"agent-os.dev/rollback-target-digest":null,"agent-os.dev/rollback-source-name":null,"agent-os.dev/rollback-source-uid":null,"agent-os.dev/rollback-source-digest":null}},spec:{template:$template}}') + '{metadata:{uid:$uid,resourceVersion:$rv},spec:{template:$template}}') "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch statefulset agent-os-firstmate --type=strategic -p "$patch" >/dev/null || return 3 "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s || return 3 [ "$(workload_service_account)" = "$current_account" ] || return 3 + state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" \ + get statefulset agent-os-firstmate -o json) || return 3 + refs=$(printf '%s' "$state" | jq -er --arg uid "$rollback_uid" \ + 'select(.metadata.uid == $uid) | [.status.currentRevision,.status.updateRevision,.metadata.resourceVersion] | @tsv') || return 3 + revisions=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" \ + get controllerrevisions.apps -o json) || return 3 + for name in "$(printf '%s' "$refs" | cut -f1)" "$(printf '%s' "$refs" | cut -f2)"; do + revision=$(printf '%s' "$revisions" | jq -ce --arg name "$name" --arg uid "$rollback_uid" ' + [.items[] | select(.metadata.name == $name) | select(any(.metadata.ownerReferences[]?; + .controller == true and .kind == "StatefulSet" and .name == "agent-os-firstmate" and .uid == $uid))] + | select(length == 1) | .[0]') || return 3 + digest=$(printf '%s' "$revision" | jq -cS '.data.spec.template' | template_digest) + [ "$digest" = "$rollback_source_digest" ] || return 3 + done + transfer_runtime_authority "$mode" "$target_account" "$current_account" || return 3 + clear=$(jq -cn --arg uid "$rollback_uid" --arg rv "$(printf '%s' "$refs" | cut -f3)" \ + '{metadata:{uid:$uid,resourceVersion:$rv,annotations:{"agent-os.dev/rollback-operation":null,"agent-os.dev/rollback-target-name":null,"agent-os.dev/rollback-target-uid":null,"agent-os.dev/rollback-target-digest":null,"agent-os.dev/rollback-source-name":null,"agent-os.dev/rollback-source-uid":null,"agent-os.dev/rollback-source-digest":null}}}') + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch statefulset agent-os-firstmate --type=merge -p "$clear" >/dev/null } wait_for_revision_history_absence() { @@ -535,6 +586,29 @@ wait_for_revision_history_absence() { done } +inventory_revision_service_accounts() { + local workload_uid=$1 revisions accounts account identity record + revisions=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ + --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get controllerrevisions.apps -o json) || return 3 + accounts=$(printf '%s' "$revisions" | jq -er --arg uid "$workload_uid" ' + [.items[] | select(any(.metadata.ownerReferences[]?; + .controller == true and .kind == "StatefulSet" and .name == "agent-os-firstmate" and .uid == $uid)) + | .data.spec.template.spec.serviceAccountName] + | select(all(.[]; type == "string" and test("^agent-os-firstmate(-[a-z0-9]{12})?$"))) + | unique | join("\n")') || return 3 + while IFS= read -r account; do + [ -n "$account" ] || continue + identity=$(live_resource_identity ServiceAccount "$account") || return 3 + [ "$identity" = "$account"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" ] || { + echo "error: historical ServiceAccount '$account' is missing or foreign" >&2 + return 3 + } + record=$(live_resource_record namespaced ServiceAccount "$account") || return 3 + [ -n "$(printf '%s' "$record" | cut -f4)" ] && [ -n "$(printf '%s' "$record" | cut -f5)" ] || return 3 + done <<< "$accounts" + printf '%s' "$accounts" +} + require_no_active_rollback_checkpoint() { local state checkpoint if ! command -v jq >/dev/null 2>&1; then @@ -611,6 +685,27 @@ verified_akua_overlay_secret() { printf '%s' "$current" } +reconcile_failed_akua_upgrade() { + local expected=$1 state uid rv rejected patch + state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ + --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get statefulset agent-os-firstmate -o json) || return 3 + uid=$(printf '%s' "$state" | jq -er --arg installation "$INSTALLATION_ID" ' + select(.metadata.name == "agent-os-firstmate") + | select(.metadata.labels["app.kubernetes.io/managed-by"] == "agent-os") + | select(.metadata.annotations["agent-os.dev/installation-id"] == $installation) + | .metadata.uid') || return 3 + rv=$(printf '%s' "$state" | jq -er '.metadata.resourceVersion') || return 3 + rejected=$(printf '%s' "$expected" | sha256_text) + patch=$(jq -cn --arg uid "$uid" --arg rv "$rv" --arg rejected "$rejected" ' + {metadata:{uid:$uid,resourceVersion:$rv,annotations:{"agent-os.dev/akua-auth-secret":null,"agent-os.dev/akua-auth-rejected-record":$rejected}}, + spec:{template:{spec:{containers:[{name:"firstmate",env:[{name:"AKUA_AUTH_HEADER_FILE","$patch":"delete"}],volumeMounts:[{mountPath:"/var/run/secrets/agent-os/akua","$patch":"delete"}]}],volumes:[{name:"akua-auth","$patch":"delete"}]}}}}') + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch statefulset agent-os-firstmate --type=strategic -p "$patch" >/dev/null || return 3 + "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s || return 3 + verified_akua_overlay_secret absent >/dev/null || return 3 + printf 'incomplete: upgrade authorization failed closed expected-secret-uid=%s expected-secret-rv=%s\n' \ + "$(printf '%s' "$expected" | cut -f3)" "$(printf '%s' "$expected" | cut -f4)" >&2 +} + rendered_resource_field() { local file=$1 field=$2 awk -v field="$field" '$1 == field ":" { print $2; exit }' "$file" @@ -1437,6 +1532,14 @@ case "$COMMAND" in preflight_legacy_cluster_binding 0 ensure_control_lock_access apply_rendered + verify_desired_rbac + if [ "$DESIRED_RBAC" != namespace ]; then + delete_namespace_rbac + fi + if [ "$DESIRED_RBAC" = none ]; then + delete_control_lock_access + verify_no_runtime_authority || { echo "error: authority-free install retained RBAC" >&2; exit 3; } + fi ;; upgrade) [ "$CONFIRMED" -eq 0 ] && [ "$DELETE_NAMESPACE" -eq 0 ] || usage @@ -1492,16 +1595,23 @@ case "$COMMAND" in ensure_control_lock_access apply_rendered if ! verified_akua_overlay_secret "$PRESERVED_AKUA_SECRET_RECORD" >/dev/null; then - echo "incomplete: upgrade applied but Akua authorization postcondition changed" >&2 - report_partial_observation StatefulSet agent-os-firstmate namespaced - report_partial_observation Pod agent-os-firstmate-0 namespaced - echo "safe recovery: $(lifecycle_command upgrade)" >&2 + reconcile_failed_akua_upgrade "$PRESERVED_AKUA_SECRET_RECORD" || { + echo "incomplete: upgrade authorization changed and fail-closed reconciliation is unverified" >&2 + report_partial_observation StatefulSet agent-os-firstmate namespaced + report_partial_observation Pod agent-os-firstmate-0 namespaced + exit 3 + } + echo "safe recovery: run the serialized authorization revoke before retrying upgrade" >&2 exit 3 fi verify_desired_rbac if [ "$DESIRED_RBAC" != namespace ]; then delete_namespace_rbac fi + if [ "$DESIRED_RBAC" = none ]; then + delete_control_lock_access + verify_no_runtime_authority || { echo "error: authority-free upgrade retained RBAC" >&2; exit 3; } + fi if [ "$DESIRED_RBAC" = cluster-admin ] && [ "$previous_cleanup" = required ]; then patch_workload_annotations '{"agent-os.dev/cluster-rbac-cleanup":null}' fi @@ -1702,7 +1812,11 @@ case "$COMMAND" in rollback_current=$(live_resource_record namespaced StatefulSet agent-os-firstmate) if [ "$(printf '%s' "$rollback_current" | cut -f1-4)" != \ "$(printf '%s' "$rollback_record" | cut -f1-4)" ]; then - echo "error: StatefulSet UID or ownership changed during rollback" >&2 + compensate_rollback "$rollback_source_template" "$rollback_source_account" "$rollback_target_account" "$rollback_authority_mode" || { + echo "incomplete: StatefulSet identity changed and rollback compensation is unverified" >&2 + exit 3 + } + echo "error: StatefulSet UID or ownership changed during rollback; source revision and authority restored" >&2 exit 3 fi fi @@ -1808,12 +1922,17 @@ case "$COMMAND" in require_workload_owned "$previous" optional previous_service_account= previous_workload_uid= + historical_service_accounts= retained_legacy_service_account= if [ -n "$previous" ]; then previous_service_account=$(workload_service_account) previous_workload_record=$(live_resource_record namespaced StatefulSet agent-os-firstmate) previous_workload_uid=$(printf '%s' "$previous_workload_record" | cut -f4) [ -n "$previous_workload_uid" ] || { echo "error: uninstall requires exact StatefulSet UID evidence" >&2; exit 3; } + historical_service_accounts=$(inventory_revision_service_accounts "$previous_workload_uid") || { + echo "error: uninstall could not inventory exact historical ServiceAccount dependencies" >&2 + exit 3 + } fi legacy_identity=$(live_resource_identity ServiceAccount agent-os-firstmate) if [ -n "$legacy_identity" ]; then @@ -1837,6 +1956,10 @@ case "$COMMAND" in [ "$retained_legacy_service_account" != "$SERVICE_ACCOUNT_NAME" ]; then delete_owned_resource namespaced ServiceAccount "$retained_legacy_service_account" 180 fi + while IFS= read -r historical_service_account; do + [ -n "$historical_service_account" ] || continue + delete_owned_resource namespaced ServiceAccount "$historical_service_account" 180 + done <<< "$historical_service_accounts" delete_namespace_rbac if [ "$DESIRED_RBAC" = namespace ] || [ "$previous_mode" = namespace ]; then REMOVE_CONTROL_ACCESS=1 diff --git a/bin/agent-os-local.sh b/bin/agent-os-local.sh index 4f33ccafc..a99250d4c 100755 --- a/bin/agent-os-local.sh +++ b/bin/agent-os-local.sh @@ -61,7 +61,10 @@ case "$COMMAND" in source_tree=$(printf '%s\n' "$source_metadata" | awk -F= '$1 == "tree" { print $2 }') source_branch=$(printf '%s\n' "$source_metadata" | awk -F= '$1 == "branch" { print $2 }') source_origin=$(printf '%s\n' "$source_metadata" | awk -F= '$1 == "origin" { sub(/^[^=]*=/, ""); print }') - [ -n "$source_commit" ] && [ -n "$source_tree" ] && [ -n "$source_branch" ] && [ -n "$source_origin" ] || { + source_mode=$(printf '%s\n' "$source_metadata" | awk -F= '$1 == "mode" { print $2 }') + source_ref=$(printf '%s\n' "$source_metadata" | awk -F= '$1 == "ref" { sub(/^[^=]*=/, ""); print }') + [ -n "$source_commit" ] && [ -n "$source_tree" ] && [ -n "$source_branch" ] && [ -n "$source_origin" ] && \ + [ -n "$source_mode" ] && [ -n "$source_ref" ] || { echo "error: exact-source bootstrap metadata is incomplete" >&2 exit 2 } @@ -70,6 +73,8 @@ case "$COMMAND" in --build-arg "AGENT_OS_SOURCE_TREE=$source_tree" \ --build-arg "AGENT_OS_SOURCE_BRANCH=$source_branch" \ --build-arg "AGENT_OS_SOURCE_ORIGIN=$source_origin" \ + --build-arg "AGENT_OS_SOURCE_MODE=$source_mode" \ + --build-arg "AGENT_OS_SOURCE_REF=$source_ref" \ -t "$IMAGE" . if [ -z "$IMAGE_IS_OVERRIDE" ]; then docker tag "$IMAGE" "$(local_image_tag)" diff --git a/bin/agent-os-source-bundle.sh b/bin/agent-os-source-bundle.sh index 923fd9c6e..04b64338b 100755 --- a/bin/agent-os-source-bundle.sh +++ b/bin/agent-os-source-bundle.sh @@ -4,22 +4,39 @@ set -eu SCRIPT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) ROOT=$(cd "${AGENT_OS_SOURCE_ROOT:-$SCRIPT_ROOT}" && pwd) OUTPUT_DIR=${1:-"$ROOT/image"} -SOURCE_BRANCH=${AGENT_OS_SOURCE_BRANCH:-main} -SOURCE_ORIGIN=${AGENT_OS_SOURCE_ORIGIN:-https://github.com/akua-dev/agent-os.git} -SOURCE_REF="refs/heads/$SOURCE_BRANCH" +SOURCE_MODE=${AGENT_OS_SOURCE_MODE:-main} +SOURCE_BRANCH=main +SOURCE_ORIGIN=https://github.com/akua-dev/agent-os.git +SOURCE_REF=refs/heads/main +RELEASE_TAG=${AGENT_OS_SOURCE_RELEASE_TAG:-} +RELEASE_RECORD_COMMIT=${AGENT_OS_RELEASE_RECORD_COMMIT:-} +EVENT_COMMIT=${AGENT_OS_SOURCE_EVENT_COMMIT:-} SOURCE_ARCHIVE="$OUTPUT_DIR/agent-os-source.tar" BOOTSTRAP_ARCHIVE="$OUTPUT_DIR/agent-os-bootstrap.tar" ATTESTATION="$OUTPUT_DIR/agent-os-source.attestation" TEMP="$OUTPUT_DIR/.agent-os-source.$$" -[ "$SOURCE_BRANCH" = main ] || { echo "error: source branch must be the declared default branch 'main'" >&2; exit 2; } -[ "$SOURCE_ORIGIN" = https://github.com/akua-dev/agent-os.git ] || { echo "error: source origin is not allowlisted" >&2; exit 2; } -case "$(git -C "$ROOT" remote get-url origin)" in +git_isolated() { + env -u GIT_CONFIG -u GIT_CONFIG_PARAMETERS -u GIT_CONFIG_COUNT \ + -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy \ + -u GIT_SSH -u GIT_SSH_COMMAND GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null \ + GIT_TERMINAL_PROMPT=0 command git -c credential.helper= -c core.hooksPath=/dev/null \ + -c http.proxy= -c https.proxy= "$@" +} + +sha256_file() { + if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}'; else shasum -a 256 "$1" | awk '{print $1}'; fi +} + +sha256_stream() { + if command -v sha256sum >/dev/null 2>&1; then sha256sum | awk '{print $1}'; else shasum -a 256 | awk '{print $1}'; fi +} + +case "$(git_isolated -C "$ROOT" remote get-url origin)" in https://github.com/akua-dev/agent-os.git|ssh://git@github.com/akua-dev/agent-os.git|git@github.com:akua-dev/agent-os.git) ;; *) echo "error: repository origin is not allowlisted" >&2; exit 2 ;; esac - -[ -z "$(git -C "$ROOT" status --porcelain --untracked-files=all)" ] || { +[ -z "$(git_isolated -C "$ROOT" status --porcelain --untracked-files=all)" ] || { echo "error: exact-source image builds require a clean worktree" >&2 exit 2 } @@ -28,51 +45,112 @@ mkdir -p "$OUTPUT_DIR" rm -rf "$TEMP" mkdir -p "$TEMP" trap 'rm -rf "$TEMP" "$SOURCE_ARCHIVE.tmp.$$" "$BOOTSTRAP_ARCHIVE.tmp.$$" "$ATTESTATION.tmp.$$"' EXIT +git_isolated init --bare "$TEMP/bootstrap.git" >/dev/null -sha256_file() { - if command -v sha256sum >/dev/null 2>&1; then - sha256sum "$1" | awk '{print $1}' - else - shasum -a 256 "$1" | awk '{print $1}' - fi -} +case "$SOURCE_MODE" in + main) + git_isolated --git-dir="$TEMP/bootstrap.git" fetch --depth=1 --no-tags "$SOURCE_ORIGIN" \ + refs/heads/main:refs/heads/main + ;; + event) + [ "${GITHUB_EVENT_NAME:-}" = pull_request ] && [[ "$EVENT_COMMIT" =~ ^[0-9a-f]{40}$ ]] || { + echo "error: event source mode is restricted to an exact pull-request event commit" >&2 + exit 2 + } + [ "$(git_isolated -C "$ROOT" rev-parse --verify HEAD)" = "$EVENT_COMMIT" ] || { + echo "error: event checkout does not match the declared pull-request head" >&2 + exit 2 + } + git_isolated -c protocol.file.allow=always --git-dir="$TEMP/bootstrap.git" fetch --depth=1 --no-tags \ + "file://$ROOT" "$EVENT_COMMIT:refs/heads/main" + SOURCE_REF="event:$EVENT_COMMIT" + ;; + release) + [[ "$RELEASE_TAG" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] && \ + [[ "$RELEASE_RECORD_COMMIT" =~ ^[0-9a-f]{40}$ ]] || { + echo "error: release mode requires a canonical semver tag and immutable record commit" >&2 + exit 2 + } + command -v jq >/dev/null 2>&1 || { echo "error: jq is required for release records" >&2; exit 2; } + git_isolated --git-dir="$TEMP/bootstrap.git" fetch --depth=1 --no-tags "$SOURCE_ORIGIN" \ + "$RELEASE_RECORD_COMMIT:refs/agent-os/release-record" + git_isolated --git-dir="$TEMP/bootstrap.git" fetch --depth=1 --no-tags "$SOURCE_ORIGIN" \ + "refs/tags/$RELEASE_TAG:refs/tags/$RELEASE_TAG" + [ "$(git_isolated --git-dir="$TEMP/bootstrap.git" cat-file -t "refs/tags/$RELEASE_TAG")" = tag ] || { + echo "error: release tag must be annotated" >&2 + exit 2 + } + git_isolated --git-dir="$TEMP/bootstrap.git" show "refs/agent-os/release-record:image/releases/$RELEASE_TAG.json" > "$TEMP/release.json" + jq -e --arg tag "$RELEASE_TAG" ' + .tag == $tag and (.commit|test("^[0-9a-f]{40}$")) and (.tree|test("^[0-9a-f]{40}$")) and + (.source_archive_sha256|test("^[0-9a-f]{64}$")) and (.package_sha256|test("^[0-9a-f]{64}$")) and + (.schema_sha256|test("^[0-9a-f]{64}$")) and (.image_digest|test("^sha256:[0-9a-f]{64}$")) and + (.sbom_sha256|test("^[0-9a-f]{64}$")) and (.provenance_sha256|test("^[0-9a-f]{64}$")) and + (.quickstart_sha256|test("^[0-9a-f]{64}$")) and (.tag_ruleset_id|type == "number") and + (.tag_ruleset_sha256|test("^[0-9a-f]{64}$"))' "$TEMP/release.json" >/dev/null || { + echo "error: release record is incomplete or malformed" >&2 + exit 2 + } + SOURCE_REF="refs/tags/$RELEASE_TAG" + git_isolated --git-dir="$TEMP/bootstrap.git" update-ref refs/heads/main "refs/tags/$RELEASE_TAG^{}" + ;; + *) echo "error: unsupported source mode" >&2; exit 2 ;; +esac -git init --bare "$TEMP/bootstrap.git" >/dev/null -GIT_TERMINAL_PROMPT=0 git -c credential.helper= --git-dir="$TEMP/bootstrap.git" fetch --depth=1 --no-tags "$SOURCE_ORIGIN" \ - "$SOURCE_REF:refs/heads/$SOURCE_BRANCH" -SOURCE_COMMIT=$(git --git-dir="$TEMP/bootstrap.git" rev-parse --verify "refs/heads/$SOURCE_BRANCH") -SOURCE_TREE=$(git --git-dir="$TEMP/bootstrap.git" rev-parse --verify "$SOURCE_COMMIT^{tree}") -[ "$(git -C "$ROOT" rev-parse --verify HEAD)" = "$SOURCE_COMMIT" ] || { - echo "error: local HEAD does not exactly match the freshly fetched trusted source ref" >&2 - exit 2 -} -[ "$(git -C "$ROOT" rev-parse --verify 'HEAD^{tree}')" = "$SOURCE_TREE" ] || { - echo "error: local source tree does not exactly match the trusted source ref" >&2 - exit 2 -} +SOURCE_COMMIT=$(git_isolated --git-dir="$TEMP/bootstrap.git" rev-parse --verify refs/heads/main) +SOURCE_TREE=$(git_isolated --git-dir="$TEMP/bootstrap.git" rev-parse --verify "$SOURCE_COMMIT^{tree}") +if [ "$SOURCE_MODE" = main ]; then + [ "$(git_isolated -C "$ROOT" rev-parse --verify HEAD)" = "$SOURCE_COMMIT" ] && \ + [ "$(git_isolated -C "$ROOT" rev-parse --verify 'HEAD^{tree}')" = "$SOURCE_TREE" ] || { + echo "error: local source does not exactly match the freshly fetched trusted source ref" >&2 + exit 2 + } +elif [ "$SOURCE_MODE" = release ]; then + [ "$(jq -r .commit "$TEMP/release.json")" = "$SOURCE_COMMIT" ] && \ + [ "$(jq -r .tree "$TEMP/release.json")" = "$SOURCE_TREE" ] || { + echo "error: release tag moved or differs from its immutable record" >&2 + exit 2 + } +fi + +if [ "$SOURCE_MODE" = release ]; then + git_isolated --git-dir="$TEMP/bootstrap.git" update-ref -d refs/agent-os/release-record + git_isolated --git-dir="$TEMP/bootstrap.git" update-ref -d "refs/tags/$RELEASE_TAG" + printf '%s\n' "$SOURCE_COMMIT" > "$TEMP/bootstrap.git/shallow" + git_isolated --git-dir="$TEMP/bootstrap.git" reflog expire --expire=now --all + git_isolated --git-dir="$TEMP/bootstrap.git" gc --prune=now +fi -git --git-dir="$TEMP/bootstrap.git" symbolic-ref HEAD "refs/heads/$SOURCE_BRANCH" -git --git-dir="$TEMP/bootstrap.git" remote add origin "$SOURCE_ORIGIN" +git_isolated --git-dir="$TEMP/bootstrap.git" symbolic-ref HEAD refs/heads/main +git_isolated --git-dir="$TEMP/bootstrap.git" remote add origin "$SOURCE_ORIGIN" rm -rf "$TEMP/bootstrap.git/hooks" "$TEMP/bootstrap.git/logs" rm -f "$TEMP/bootstrap.git/FETCH_HEAD" "$TEMP/bootstrap.git/ORIG_HEAD" -git -C "$ROOT" archive --format=tar --output="$SOURCE_ARCHIVE.tmp.$$" "$SOURCE_COMMIT" +git_isolated --git-dir="$TEMP/bootstrap.git" archive --format=tar --output="$SOURCE_ARCHIVE.tmp.$$" "$SOURCE_COMMIT" +if [ "$SOURCE_MODE" = release ]; then + [ "$(sha256_file "$SOURCE_ARCHIVE.tmp.$$")" = "$(jq -r .source_archive_sha256 "$TEMP/release.json")" ] && \ + [ "$(git_isolated --git-dir="$TEMP/bootstrap.git" show "$SOURCE_COMMIT:tools/agent-os/packages/firstmate/package.k" | sha256_stream)" = "$(jq -r .package_sha256 "$TEMP/release.json")" ] && \ + [ "$(git_isolated --git-dir="$TEMP/bootstrap.git" show "$SOURCE_COMMIT:tools/agent-os/packages/firstmate/inputs.example.yaml" | sha256_stream)" = "$(jq -r .schema_sha256 "$TEMP/release.json")" ] && \ + [ "$(git_isolated --git-dir="$TEMP/bootstrap.git" show "$SOURCE_COMMIT:docs/kubernetes.md" | sha256_stream)" = "$(jq -r .quickstart_sha256 "$TEMP/release.json")" ] || { + echo "error: release source artifacts differ from the immutable release record" >&2 + exit 2 + } +fi (cd "$TEMP" && find bootstrap.git -exec touch -t 197001010000 {} + && \ find bootstrap.git -print | LC_ALL=C sort | COPYFILE_DISABLE=1 tar -cf "$BOOTSTRAP_ARCHIVE.tmp.$$" -T -) SOURCE_SHA=$(sha256_file "$SOURCE_ARCHIVE.tmp.$$") BOOTSTRAP_SHA=$(sha256_file "$BOOTSTRAP_ARCHIVE.tmp.$$") { - printf 'commit=%s\n' "$SOURCE_COMMIT" - printf 'tree=%s\n' "$SOURCE_TREE" - printf 'branch=%s\n' "$SOURCE_BRANCH" - printf 'origin=%s\n' "$SOURCE_ORIGIN" - printf 'ref=%s\n' "$SOURCE_REF" - printf 'source_sha256=%s\n' "$SOURCE_SHA" - printf 'bootstrap_sha256=%s\n' "$BOOTSTRAP_SHA" + printf 'mode=%s\ncommit=%s\ntree=%s\nbranch=%s\norigin=%s\nref=%s\n' \ + "$SOURCE_MODE" "$SOURCE_COMMIT" "$SOURCE_TREE" "$SOURCE_BRANCH" "$SOURCE_ORIGIN" "$SOURCE_REF" + printf 'source_sha256=%s\nbootstrap_sha256=%s\n' "$SOURCE_SHA" "$BOOTSTRAP_SHA" + if [ "$SOURCE_MODE" = release ]; then + printf 'release_record_commit=%s\nrelease_record_sha256=%s\n' \ + "$RELEASE_RECORD_COMMIT" "$(sha256_file "$TEMP/release.json")" + fi } > "$ATTESTATION.tmp.$$" mv "$SOURCE_ARCHIVE.tmp.$$" "$SOURCE_ARCHIVE" mv "$BOOTSTRAP_ARCHIVE.tmp.$$" "$BOOTSTRAP_ARCHIVE" mv "$ATTESTATION.tmp.$$" "$ATTESTATION" trap - EXIT rm -rf "$TEMP" - cat "$ATTESTATION" diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 207f1e392..6ec7aa643 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -113,10 +113,28 @@ assert_no_grep '!.pi/' "$ROOT/.dockerignore" "local harness homes must never be assert_grep 'agent-os-source.tar' "$ROOT/Dockerfile" "image must consume the tracked exact-source export" assert_grep 'agent-os-bootstrap.tar' "$ROOT/Dockerfile" "image must consume the shallow sanitized bootstrap" assert_grep 'rev-parse "$AGENT_OS_SOURCE_COMMIT^{tree}"' "$ROOT/Dockerfile" "image source bootstrap must verify its exact tree" +assert_grep 'cmp /tmp/verified-source.tar /tmp/agent-os-source.tar' "$ROOT/Dockerfile" \ + "image source must be byte-identical to the archive materialized from the verified commit" assert_grep 'AS source-bootstrap' "$ROOT/Dockerfile" "source processing must stay in an isolated build stage" assert_no_grep 'COPY \. /opt/agent-os' "$ROOT/Dockerfile" "the image must never copy the ambient workspace" assert_grep 'fetch --depth=1 --no-tags' "$ROOT/bin/agent-os-source-bundle.sh" \ "source preparation must fetch an allowlisted remote ref freshly" +assert_grep 'GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null' "$ROOT/bin/agent-os-source-bundle.sh" \ + "source preparation must isolate trusted Git operations from ambient configuration" +assert_grep 'AGENT_OS_SOURCE_MODE=event AGENT_OS_SOURCE_EVENT_COMMIT' "$IMAGE_WORKFLOW" \ + "pull requests must build their exact event head without publication credentials" +assert_no_grep 'git clone --depth=1 --branch main' "$IMAGE_WORKFLOW" \ + "pull-request validation must not substitute protected main for the reviewed source" +assert_grep 'release-record-commit=' "$IMAGE_WORKFLOW" \ + "historical releases must name an immutable allowlisted release record" +assert_grep 'tag_ruleset_sha256' "$ROOT/bin/agent-os-source-bundle.sh" \ + "release records must bind the protected tag ruleset" +assert_grep 'source_archive_sha256' "$ROOT/bin/agent-os-source-bundle.sh" \ + "release records must bind the exact archived source" +assert_grep 'GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "runtime provenance must isolate trusted Git operations" +assert_grep 'canonical FM_ROOT Git config contains untrusted' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "runtime provenance must reject persistent transport and execution overrides" assert_grep 'status --porcelain --untracked-files=all' "$ROOT/bin/agent-os-source-bundle.sh" \ "source preparation must reject a dirty workspace" assert_no_grep 'bundle create.*HEAD' "$ROOT/bin/agent-os-source-bundle.sh" \ @@ -124,7 +142,7 @@ assert_no_grep 'bundle create.*HEAD' "$ROOT/bin/agent-os-source-bundle.sh" \ assert_grep 'merge --ff-only' "$ROOT/bin/agent-os-container-entrypoint.sh" "persistent source transitions must be fast-forward only" assert_grep 'refs/remotes/agent-os-verified' "$ROOT/bin/agent-os-container-entrypoint.sh" \ "canonical runtime source must use a freshly fetched verification ref" -assert_grep 'fetch --no-tags --prune origin' "$ROOT/bin/agent-os-container-entrypoint.sh" \ +assert_grep 'trusted_git -C "$FM_ROOT" fetch --no-tags --prune "$SOURCE_ORIGIN"' "$ROOT/bin/agent-os-container-entrypoint.sh" \ "runtime provenance must fail closed unless the trusted remote is reachable" assert_no_grep 'checkout --detach' "$ROOT/bin/agent-os-container-entrypoint.sh" \ "canonical runtime source must remain on the declared default branch" @@ -174,8 +192,8 @@ assert_grep 'needs: [behavior, provenance, validate]' "$IMAGE_WORKFLOW" \ "the packages-write job must require exact behavior, provenance, and image gates" assert_grep 'github.ref_protected' "$IMAGE_WORKFLOW" \ "main publication must require GitHub protected-ref provenance" -assert_grep 'release tag must point at the exact protected-main head' "$IMAGE_WORKFLOW" \ - "tag publication must require the exact tested protected-main commit" +assert_grep 'release tag ruleset differs from immutable record' "$IMAGE_WORKFLOW" \ + "tag publication must require its recorded protected-tag ruleset" assert_grep 'Section 13' "$ROOT/docs/herdr-compliance.md" \ "the Herdr audit must account for the network-interaction clause" assert_grep 'https://github.com/akua-dev/akua/tree/v0.8.25' "$ROOT/THIRD_PARTY_NOTICES.md" \ diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index 1468b485c..db0cfd6bc 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -9,6 +9,8 @@ assert_grep 'configure_control_lock' "$ROOT/bin/agent-os-crewmate.sh" \ "crewmate mutations must acquire the stable installation control lock" assert_grep 'CONTROL_LOCK_UID' "$ROOT/bin/agent-os-crewmate.sh" \ "crewmate mutations must retain control-lock CAS evidence" +assert_grep 'agent-os.dev/installation-id: ${LOCK_INSTALLATION_ID:-$INSTALLATION_ID}' "$ROOT/bin/agent-os-crewmate.sh" \ + "crewmate control Leases must render their scope-specific installation identity" assert_grep 'configure_control_lock' "$ROOT/bin/agent-os-akua-auth.sh" \ "authorization mutations must acquire the stable installation control lock" assert_grep 'agent-os-firstmate-lifecycle' "$ROOT/bin/agent-os-akua-auth.sh" \ @@ -21,6 +23,18 @@ assert_grep 'compensate_rollback' "$ROOT/bin/agent-os-kubernetes.sh" \ "failed rollback must restore the current revision and authority" assert_grep 'wait_for_revision_history_absence' "$ROOT/bin/agent-os-kubernetes.sh" \ "uninstall must retire historical ServiceAccounts only after revision history" +assert_grep 'fail_grant_closed "grant CAS failed ambiguously"' "$ROOT/bin/agent-os-akua-auth.sh" \ + "ambiguous grant mutations must enter fail-closed reconciliation" +assert_grep 'fail_grant_closed "grant rollout failed"' "$ROOT/bin/agent-os-akua-auth.sh" \ + "failed grant rollouts must enter fail-closed reconciliation" +assert_grep 'reconcile_failed_akua_upgrade' "$ROOT/bin/agent-os-kubernetes.sh" \ + "upgrade authorization drift must be removed and verified fail closed" +assert_grep 'inventory_revision_service_accounts' "$ROOT/bin/agent-os-kubernetes.sh" \ + "uninstall must inventory every historical revision ServiceAccount" +assert_grep 'verify_no_runtime_authority' "$ROOT/bin/agent-os-kubernetes.sh" \ + "RBAC none must verify namespace, cluster, and control authority absence" +assert_grep 'rollback_source_digest' "$ROOT/bin/agent-os-kubernetes.sh" \ + "rollback compensation must verify its immutable source revision digest" LAUNCHER="$ROOT/bin/agent-os-crewmate.sh" TMP=$(fm_test_tmproot agent-os-kubernetes) @@ -64,6 +78,18 @@ for argument in "$@"; do [[ "$argument" = --request-timeout=* ]] || filtered_args+=("$argument") done set -- "${filtered_args[@]}" +effective_binding_account() { + local binding=$1 account patch + patch=$(grep -F " patch rolebinding $binding " "$AGENT_OS_TEST_LOG" | tail -n 1 | sed -n 's/.* -p //p') + account=$(printf '%s' "$patch" | jq -r '.subjects[0].name // empty' 2>/dev/null || true) + if [ -z "$account" ]; then + account=$(grep 'akua-input-service-account ' "$AGENT_OS_TEST_LOG" | tail -n 1 | awk '{print $2}') + fi + if [ "${AGENT_OS_TEST_COMMAND:-}" = rollback ] && [ -z "$patch" ]; then + account=${AGENT_OS_TEST_WORKLOAD_SERVICE_ACCOUNT:-agent-os-firstmate} + fi + printf '%s' "${account:-agent-os-firstmate}" +} stdin_data='' previous='' for argument in "$@"; do @@ -149,8 +175,7 @@ if [[ " $* " = *" get RoleBinding agent-os-lifecycle-"*" -o json "* ]] || \ [[ " $* " = *" get rolebinding agent-os-lifecycle-"*" -o json "* ]]; then control_name=$(printf '%s\n' "$*" | sed -n 's/.* get [Rr]ole[Bb]inding \([^ ]*\) .*/\1/p') if ! grep -Fq "/rolebindings/$control_name" "$AGENT_OS_TEST_LOG"; then - control_account=$(grep 'akua-input-service-account ' "$AGENT_OS_TEST_LOG" | tail -n 1 | awk '{print $2}') - [ -n "$control_account" ] || control_account=agent-os-firstmate + control_account=$(effective_binding_account "$control_name") printf '{"metadata":{"name":"%s","uid":"uid-control-binding","resourceVersion":"rv-control-binding","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"%s"},"subjects":[{"kind":"ServiceAccount","name":"%s","namespace":"portable-agent-os"}]}\n' "$control_name" "$control_name" "$control_account" fi exit 0 @@ -469,8 +494,7 @@ case " $* " in control_kind=$(printf '%s\n' "$*" | sed -n 's/.* get \([Rr]ole\|[Rr]ole[Bb]inding\) .*/\1/p') case "$control_kind" in role) control_kind=Role ;; rolebinding) control_kind=RoleBinding ;; esac if ! grep -Fq "/$([ "$control_kind" = Role ] && printf roles || printf rolebindings)/$control_name" "$AGENT_OS_TEST_LOG"; then - control_account=$(grep 'akua-input-service-account ' "$AGENT_OS_TEST_LOG" | tail -n 1 | awk '{print $2}') - [ -n "$control_account" ] || control_account=agent-os-firstmate + control_account=$(effective_binding_account "$control_name") if [ "$control_kind" = Role ]; then printf '{"metadata":{"name":"%s","uid":"uid-control-role","resourceVersion":"rv-control-role","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"rules":[{"apiGroups":["coordination.k8s.io"],"resources":["leases"],"resourceNames":["%s"],"verbs":["get","update"]}]}\n' "$control_name" "$control_name" else @@ -486,12 +510,11 @@ case " $* " in fi ;; *" get rolebinding agent-os-firstmate-runtime -o json "*) - service_account=$(grep 'akua-input-service-account ' "$AGENT_OS_TEST_LOG" | tail -n 1 | awk '{print $2}') - [ -n "$service_account" ] || service_account=agent-os-firstmate + service_account=$(effective_binding_account agent-os-firstmate-runtime) if [ "${AGENT_OS_TEST_RBAC_STATE:-exact}" = extra-subject ]; then - printf '{"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"agent-os-firstmate-runtime"},"subjects":[{"kind":"ServiceAccount","name":"%s","namespace":"portable-agent-os"},{"kind":"ServiceAccount","name":"foreign","namespace":"portable-agent-os"}]}\n' "$service_account" + printf '{"metadata":{"name":"agent-os-firstmate-runtime","uid":"uid-runtime-binding","resourceVersion":"rv-runtime-binding","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"agent-os-firstmate-runtime"},"subjects":[{"kind":"ServiceAccount","name":"%s","namespace":"portable-agent-os"},{"kind":"ServiceAccount","name":"foreign","namespace":"portable-agent-os"}]}\n' "$service_account" else - printf '{"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"agent-os-firstmate-runtime"},"subjects":[{"kind":"ServiceAccount","name":"%s","namespace":"portable-agent-os"}]}\n' "$service_account" + printf '{"metadata":{"name":"agent-os-firstmate-runtime","uid":"uid-runtime-binding","resourceVersion":"rv-runtime-binding","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"agent-os-firstmate-runtime"},"subjects":[{"kind":"ServiceAccount","name":"%s","namespace":"portable-agent-os"}]}\n' "$service_account" fi ;; *" get role agent-os-firstmate-runtime -o jsonpath="*) @@ -1191,6 +1214,7 @@ run_generic() { AGENT_OS_TEST_NAMESPACE_STATE="${AGENT_OS_TEST_NAMESPACE_STATE:-absent}" \ AGENT_OS_TEST_WORKLOAD_STATE="$workload_state" AGENT_OS_TEST_RESOURCE_STATE="$resource_state" \ AGENT_OS_TEST_CLUSTER_RBAC_STATE="${AGENT_OS_TEST_CLUSTER_RBAC_STATE:-absent}" \ + AGENT_OS_TEST_COMMAND="${1:-}" \ AGENT_OS_OPERATION_ID=operation-test \ AGENT_OS_CONTEXT=kind-agent-os AGENT_OS_NAMESPACE=portable-agent-os "$GENERIC" "$@" } diff --git a/tests/agent-os-local.test.sh b/tests/agent-os-local.test.sh index c78e363cf..5985f01b7 100755 --- a/tests/agent-os-local.test.sh +++ b/tests/agent-os-local.test.sh @@ -242,6 +242,7 @@ test_rebuild_deploy_uses_a_new_immutable_local_tag() { grep -F 'docker build ' "$LOG" | grep -F -- '--build-arg AGENT_OS_SOURCE_COMMIT=' | \ grep -F -- '--build-arg AGENT_OS_SOURCE_TREE=' | grep -F -- '--build-arg AGENT_OS_SOURCE_BRANCH=main' | \ + grep -F -- '--build-arg AGENT_OS_SOURCE_MODE=main' | grep -F -- '--build-arg AGENT_OS_SOURCE_REF=refs/heads/main' | \ grep -F -- '-t agent-os:dev .' >/dev/null || \ fail "build must pass verified exact-source arguments" assert_call 'docker tag agent-os:dev agent-os:local-rebuilt' \ From 972ce9e4243525e6e51c121ce41468a163c590c8 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 17:37:35 +0200 Subject: [PATCH 38/56] no-mistakes(review): Harden provenance and lifecycle safety --- .github/workflows/agent-os-image.yml | 214 ++++++++++++++++++++++++++- bin/agent-os-akua-auth.sh | 27 ++-- bin/agent-os-container-entrypoint.sh | 34 +++-- bin/agent-os-crewmate.sh | 3 + bin/agent-os-kubernetes.sh | 86 +++++++++-- bin/agent-os-source-bundle.sh | 10 +- tests/agent-os-container.test.sh | 40 ++++- tests/agent-os-kubernetes.test.sh | 101 ++++++++++++- 8 files changed, 459 insertions(+), 56 deletions(-) diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index 530860696..1d2ff6e6e 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -32,6 +32,7 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 with: fetch-depth: 0 + persist-credentials: false - name: Verify publication source env: GH_TOKEN: ${{ github.token }} @@ -49,6 +50,7 @@ jobs: git merge-base --is-ancestor "$record_commit" refs/remotes/origin/main gh api "repos/$GITHUB_REPOSITORY/branches/main/protection" >/dev/null git show "$record_commit:image/releases/$tag.json" > "$RUNNER_TEMP/release-record.json" + git merge-base --is-ancestor "$(jq -r .commit "$RUNNER_TEMP/release-record.json")" refs/remotes/origin/main ruleset_id=$(jq -r .tag_ruleset_id "$RUNNER_TEMP/release-record.json") gh api "repos/$GITHUB_REPOSITORY/rulesets/$ruleset_id" > "$RUNNER_TEMP/tag-ruleset.json" ruleset_sha=$(jq -cS . "$RUNNER_TEMP/tag-ruleset.json" | sha256sum | awk '{print $1}') @@ -56,6 +58,29 @@ jobs: echo "::error::release tag ruleset differs from immutable record" exit 1 } + jq -e ' + .enforcement == "active" and .target == "tag" and + (.bypass_actors | length == 0) and + ([.rules[].type] | index("deletion") != null and index("non_fast_forward") != null)' \ + "$RUNNER_TEMP/tag-ruleset.json" >/dev/null || { + echo "::error::release tag ruleset is not active, immutable, tag-targeted, and bypass-free" + exit 1 + } + python3 - "$GITHUB_REF" "$RUNNER_TEMP/tag-ruleset.json" <<'PY' + import fnmatch + import json + import sys + + ref, path = sys.argv[1:] + with open(path, encoding="utf-8") as stream: + ruleset = json.load(stream) + names = ruleset.get("conditions", {}).get("ref_name", {}) + include = names.get("include", []) + exclude = names.get("exclude", []) + matches = lambda pattern: pattern == "~ALL" or fnmatch.fnmatchcase(ref, pattern) + if not include or not any(matches(pattern) for pattern in include) or any(matches(pattern) for pattern in exclude): + raise SystemExit("release tag is not covered exclusively by the recorded tag ruleset") + PY else echo "::error::unsupported publication ref $GITHUB_REF" exit 1 @@ -112,9 +137,120 @@ jobs: AGENT_OS_SOURCE_MODE=${{ steps.source.outputs.mode }} AGENT_OS_SOURCE_REF=${{ steps.source.outputs.ref }} + release_artifacts: + if: github.event_name == 'push' + name: Verify immutable release artifacts + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + image_digest: ${{ steps.verify.outputs.image_digest }} + sbom_sha256: ${{ steps.verify.outputs.sbom_sha256 }} + provenance_sha256: ${{ steps.verify.outputs.provenance_sha256 }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Prepare immutable release source + if: startsWith(github.ref, 'refs/tags/') + id: source + run: | + set -eu + record_commit=$(git for-each-ref --format='%(contents)' "$GITHUB_REF" | sed -n 's/^release-record-commit=\([0-9a-f]\{40\}\)$/\1/p') + AGENT_OS_SOURCE_MODE=release AGENT_OS_SOURCE_RELEASE_TAG="${GITHUB_REF#refs/tags/}" \ + AGENT_OS_RELEASE_RECORD_COMMIT="$record_commit" bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" + git show "$record_commit:image/releases/${GITHUB_REF#refs/tags/}.json" > "$RUNNER_TEMP/release-record.json" + [ "$(sha256sum "$RUNNER_TEMP/release-record.json" | awk '{print $1}')" = \ + "$(sed -n 's/^release_record_sha256=//p' "$RUNNER_TEMP/source-metadata")" ] + cat "$RUNNER_TEMP/source-metadata" >> "$GITHUB_OUTPUT" + + - if: startsWith(github.ref, 'refs/tags/') + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 + + - if: startsWith(github.ref, 'refs/tags/') + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f + + - name: Compute immutable release metadata + if: startsWith(github.ref, 'refs/tags/') + id: metadata + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 + with: + images: ${{ env.IMAGE }} + tags: | + type=sha,format=long + type=ref,event=tag + + - name: Build release artifact without publication authority + if: startsWith(github.ref, 'refs/tags/') + id: build + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: false + outputs: type=oci,dest=/tmp/agent-os-release.oci.tar + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + provenance: mode=max + sbom: true + build-args: | + AGENT_OS_SOURCE_COMMIT=${{ steps.source.outputs.commit }} + AGENT_OS_SOURCE_TREE=${{ steps.source.outputs.tree }} + AGENT_OS_SOURCE_BRANCH=${{ steps.source.outputs.branch }} + AGENT_OS_SOURCE_ORIGIN=${{ steps.source.outputs.origin }} + AGENT_OS_SOURCE_MODE=${{ steps.source.outputs.mode }} + AGENT_OS_SOURCE_REF=${{ steps.source.outputs.ref }} + + - name: Verify record-bound release artifact + if: startsWith(github.ref, 'refs/tags/') + id: verify + env: + BUILD_DIGEST: ${{ steps.build.outputs.digest }} + run: | + set -eu + mkdir "$RUNNER_TEMP/release-oci" + tar -xf /tmp/agent-os-release.oci.tar -C "$RUNNER_TEMP/release-oci" + root_digest=$(jq -er '.manifests | select(length == 1) | .[0].digest' "$RUNNER_TEMP/release-oci/index.json") + [ "$root_digest" = "$BUILD_DIGEST" ] + root_blob="$RUNNER_TEMP/release-oci/blobs/sha256/${root_digest#sha256:}" + [ "sha256:$(sha256sum "$root_blob" | awk '{print $1}')" = "$root_digest" ] + artifact_set_sha() { + predicate=$1 + output=$2 + : > "$output" + jq -r '.manifests[] | select(.annotations["vnd.docker.reference.type"] == "attestation-manifest") | .digest' "$root_blob" | while IFS= read -r manifest_digest; do + manifest="$RUNNER_TEMP/release-oci/blobs/sha256/${manifest_digest#sha256:}" + [ "sha256:$(sha256sum "$manifest" | awk '{print $1}')" = "$manifest_digest" ] + jq -r --arg predicate "$predicate" '.layers[] | select(.annotations["in-toto.io/predicate-type"] == $predicate) | .digest' "$manifest" >> "$output" + done + LC_ALL=C sort -u -o "$output" "$output" + [ "$(wc -l < "$output" | tr -d ' ')" -eq 2 ] + sha256sum "$output" | awk '{print $1}' + } + actual_image_digest=$root_digest + actual_sbom_sha256=$(artifact_set_sha https://spdx.dev/Document "$RUNNER_TEMP/sbom-digests") + actual_provenance_sha256=$(artifact_set_sha https://slsa.dev/provenance/v0.2 "$RUNNER_TEMP/provenance-digests") + [ "$actual_image_digest" = "$(jq -r .image_digest "$RUNNER_TEMP/release-record.json")" ] + [ "$actual_sbom_sha256" = "$(jq -r .sbom_sha256 "$RUNNER_TEMP/release-record.json")" ] + [ "$actual_provenance_sha256" = "$(jq -r .provenance_sha256 "$RUNNER_TEMP/release-record.json")" ] + printf 'image_digest=%s\nsbom_sha256=%s\nprovenance_sha256=%s\n' \ + "$actual_image_digest" "$actual_sbom_sha256" "$actual_provenance_sha256" >> "$GITHUB_OUTPUT" + + - name: Retain exact verified release artifact + if: startsWith(github.ref, 'refs/tags/') + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: agent-os-release-oci + path: /tmp/agent-os-release.oci.tar + if-no-files-found: error + retention-days: 1 + compression-level: 0 + publish: if: github.event_name == 'push' - needs: [behavior, provenance, validate] + needs: [behavior, provenance, validate, release_artifacts] name: Publish linux/amd64 and linux/arm64 runs-on: ubuntu-latest permissions: @@ -124,6 +260,7 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 with: fetch-depth: 0 + persist-credentials: false - name: Prepare exact source bundle id: source @@ -142,6 +279,11 @@ jobs: - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f + - if: startsWith(github.ref, 'refs/tags/') + uses: oras-project/setup-oras@22ce207df3b08e061f537244349aac6ae1d214f6 + with: + version: 1.3.0 + - name: Log in to GHCR uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 with: @@ -159,7 +301,15 @@ jobs: type=raw,value=latest,enable={{is_default_branch}} type=ref,event=tag + - name: Download exact verified release artifact + if: startsWith(github.ref, 'refs/tags/') + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: agent-os-release-oci + path: ${{ runner.temp }}/agent-os-release + - name: Build and publish + if: github.ref == 'refs/heads/main' id: build uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 with: @@ -180,10 +330,68 @@ jobs: AGENT_OS_SOURCE_MODE=${{ steps.source.outputs.mode }} AGENT_OS_SOURCE_REF=${{ steps.source.outputs.ref }} - - name: Record published image digest + - name: Publish exact verified release artifact + if: startsWith(github.ref, 'refs/tags/') + env: + RELEASE_DIGEST: ${{ needs.release_artifacts.outputs.image_digest }} + RELEASE_TAGS: ${{ steps.metadata.outputs.tags }} run: | + set -eu + mkdir "$RUNNER_TEMP/release-oci" + tar -xf "$RUNNER_TEMP/agent-os-release/agent-os-release.oci.tar" -C "$RUNNER_TEMP/release-oci" + root_digest=$(jq -er '.manifests | select(length == 1) | .[0].digest' "$RUNNER_TEMP/release-oci/index.json") + [ "$root_digest" = "$RELEASE_DIGEST" ] + tag_names=() + while IFS= read -r tag; do + [ -n "$tag" ] || continue + [[ "$tag" == "$IMAGE:"* ]] || exit 1 + tag_name=${tag#"$IMAGE:"} + [[ -n "$tag_name" && "$tag_name" != *','* ]] || exit 1 + tag_names+=("$tag_name") + done <<< "$RELEASE_TAGS" + [ "${#tag_names[@]}" -ge 2 ] + destination=$(IFS=,; printf '%s:%s' "$IMAGE" "${tag_names[*]}") + oras cp --from-oci-layout \ + "$RUNNER_TEMP/agent-os-release/agent-os-release.oci.tar@$RELEASE_DIGEST" "$destination" + + - name: Verify and record published artifacts + env: + BUILD_DIGEST: ${{ startsWith(github.ref, 'refs/tags/') && needs.release_artifacts.outputs.image_digest || steps.build.outputs.digest }} + RELEASE_TAGS: ${{ steps.metadata.outputs.tags }} + run: | + set -eu + if [ "${GITHUB_REF#refs/tags/}" != "$GITHUB_REF" ]; then + record_commit=$(git for-each-ref --format='%(contents)' "$GITHUB_REF" | sed -n 's/^release-record-commit=\([0-9a-f]\{40\}\)$/\1/p') + git show "$record_commit:image/releases/${GITHUB_REF#refs/tags/}.json" > "$RUNNER_TEMP/release-record.json" + [ "$BUILD_DIGEST" = "${{ needs.release_artifacts.outputs.image_digest }}" ] + [ "$BUILD_DIGEST" = "$(jq -r .image_digest "$RUNNER_TEMP/release-record.json")" ] + while IFS= read -r tag; do + [ -n "$tag" ] || continue + docker buildx imagetools inspect --raw "$tag" > "$RUNNER_TEMP/published-tag-index.json" + [ "sha256:$(sha256sum "$RUNNER_TEMP/published-tag-index.json" | awk '{print $1}')" = "$BUILD_DIGEST" ] + done <<< "$RELEASE_TAGS" + docker buildx imagetools inspect --raw "$IMAGE@$BUILD_DIGEST" > "$RUNNER_TEMP/published-index.json" + published_artifact_set_sha() { + predicate=$1 + output=$2 + : > "$output" + jq -r '.manifests[] | select(.annotations["vnd.docker.reference.type"] == "attestation-manifest") | .digest' "$RUNNER_TEMP/published-index.json" | while IFS= read -r manifest_digest; do + docker buildx imagetools inspect --raw "$IMAGE@$manifest_digest" > "$RUNNER_TEMP/published-manifest.json" + jq -r --arg predicate "$predicate" '.layers[] | select(.annotations["in-toto.io/predicate-type"] == $predicate) | .digest' "$RUNNER_TEMP/published-manifest.json" >> "$output" + done + LC_ALL=C sort -u -o "$output" "$output" + [ "$(wc -l < "$output" | tr -d ' ')" -eq 2 ] + sha256sum "$output" | awk '{print $1}' + } + actual_sbom_sha256=$(published_artifact_set_sha https://spdx.dev/Document "$RUNNER_TEMP/published-sbom-digests") + actual_provenance_sha256=$(published_artifact_set_sha https://slsa.dev/provenance/v0.2 "$RUNNER_TEMP/published-provenance-digests") + [ "$actual_sbom_sha256" = "${{ needs.release_artifacts.outputs.sbom_sha256 }}" ] + [ "$actual_provenance_sha256" = "${{ needs.release_artifacts.outputs.provenance_sha256 }}" ] + [ "$actual_sbom_sha256" = "$(jq -r .sbom_sha256 "$RUNNER_TEMP/release-record.json")" ] + [ "$actual_provenance_sha256" = "$(jq -r .provenance_sha256 "$RUNNER_TEMP/release-record.json")" ] + fi { echo '### Agent OS image' echo - echo "Digest: \`${{ steps.build.outputs.digest }}\`" + echo "Digest: \`$BUILD_DIGEST\`" } >> "$GITHUB_STEP_SUMMARY" diff --git a/bin/agent-os-akua-auth.sh b/bin/agent-os-akua-auth.sh index 252d7d4dd..2c1d33750 100755 --- a/bin/agent-os-akua-auth.sh +++ b/bin/agent-os-akua-auth.sh @@ -245,13 +245,16 @@ LOCK_VALID_UNTIL= LOCK_PERSISTENT=0 acquire_lock -SECRET_RECORD=$(secret_record) -[ "$(printf '%s' "$SECRET_RECORD" | cut -f1)" = "$SECRET" ] && \ - [ -n "$(printf '%s' "$SECRET_RECORD" | cut -f2)" ] && [ -n "$(printf '%s' "$SECRET_RECORD" | cut -f3)" ] && \ - [ "$(printf '%s' "$SECRET_RECORD" | cut -f4-)" = authorization ] || { - echo "error: Akua authorization Secret reference is missing or unverifiable" >&2 - exit 2 -} +SECRET_RECORD= +if [ "$COMMAND" = grant ]; then + SECRET_RECORD=$(secret_record) + [ "$(printf '%s' "$SECRET_RECORD" | cut -f1)" = "$SECRET" ] && \ + [ -n "$(printf '%s' "$SECRET_RECORD" | cut -f2)" ] && [ -n "$(printf '%s' "$SECRET_RECORD" | cut -f3)" ] && \ + [ "$(printf '%s' "$SECRET_RECORD" | cut -f4-)" = authorization ] || { + echo "error: Akua authorization Secret reference is missing or unverifiable" >&2 + exit 2 + } +fi STATE=$(statefulset_json) STATE_UID=$(printf '%s' "$STATE" | jq -er --arg installation "$INSTALLATION_ID" ' @@ -274,10 +277,12 @@ sed "s/__AKUA_AUTH_SECRET__/$SECRET/g" "$TEMPLATE" | awk -v uid="$uid_value" -v $1 == "metadata:" && !inserted { print; print " uid: " uid; print " resourceVersion: " rv; inserted=1; next } { print } ' > "$PATCH_FILE" -[ "$(secret_record)" = "$SECRET_RECORD" ] || { - echo "error: Akua authorization Secret identity changed before StatefulSet CAS" >&2 - exit 3 -} +if [ "$COMMAND" = grant ]; then + [ "$(secret_record)" = "$SECRET_RECORD" ] || { + echo "error: Akua authorization Secret identity changed before StatefulSet CAS" >&2 + exit 3 + } +fi if ! target_kube patch statefulset agent-os-firstmate --type=strategic --patch-file "$PATCH_FILE" >/dev/null; then [ "$COMMAND" != grant ] || fail_grant_closed "grant CAS failed ambiguously" echo "error: revoke CAS failed" >&2 diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh index 13de840af..9ad723ea8 100755 --- a/bin/agent-os-container-entrypoint.sh +++ b/bin/agent-os-container-entrypoint.sh @@ -26,20 +26,36 @@ trusted_git() { env -u GIT_CONFIG -u GIT_CONFIG_PARAMETERS -u GIT_CONFIG_COUNT \ -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy \ -u GIT_SSH -u GIT_SSH_COMMAND GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null \ - GIT_TERMINAL_PROMPT=0 command git -c credential.helper= -c core.hooksPath=/dev/null \ + GIT_TERMINAL_PROMPT=0 git -c credential.helper= -c core.hooksPath=/dev/null \ -c http.proxy= -c https.proxy= "$@" } validate_git_config() { - local config="$FM_ROOT/.git/config" origin + local config="$FM_ROOT/.git/config" key value [ -f "$config" ] || { echo "error: canonical FM_ROOT Git config is unavailable" >&2; exit 2; } - if trusted_git config --file "$config" --no-includes --name-only --get-regexp \ - '^(url\.|include\.|includeIf\.|core\.hooksPath$|credential\.|http\.proxy$|https\.proxy$|remote\..*\.proxy$|alias\.)' >/dev/null 2>&1; then - echo "error: canonical FM_ROOT Git config contains untrusted execution or transport settings" >&2 - exit 2 - fi - origin=$(trusted_git config --file "$config" --no-includes --get remote.origin.url || true) - [ "$origin" = "$SOURCE_ORIGIN" ] && [ "$origin" = https://github.com/akua-dev/agent-os.git ] || { + while IFS= read -r key; do + [ -n "$key" ] || continue + value=$(trusted_git config --file "$config" --no-includes --get-all "$key" || true) + [ -n "$value" ] && [ "$value" = "$(printf '%s\n' "$value" | head -n 1)" ] || { + echo "error: canonical FM_ROOT Git config key is duplicated or empty" >&2 + exit 2 + } + case "$key" in + core.repositoryformatversion) [ "$value" = 0 ] ;; + core.filemode) [ "$value" = true ] || [ "$value" = false ] ;; + core.bare) [ "$value" = false ] ;; + core.logallrefupdates) [ "$value" = true ] ;; + remote.origin.url) [ "$value" = "$SOURCE_ORIGIN" ] && [ "$value" = https://github.com/akua-dev/agent-os.git ] ;; + remote.origin.fetch) + [ "$value" = '+refs/heads/*:refs/remotes/origin/*' ] || \ + [ "$value" = "+refs/heads/$SOURCE_BRANCH:refs/remotes/origin/$SOURCE_BRANCH" ] + ;; + branch."$SOURCE_BRANCH".remote) [ "$value" = origin ] ;; + branch."$SOURCE_BRANCH".merge) [ "$value" = "refs/heads/$SOURCE_BRANCH" ] ;; + *) echo "error: canonical FM_ROOT Git config key is not allowlisted: $key" >&2; exit 2 ;; + esac || { echo "error: canonical FM_ROOT Git config value is invalid: $key" >&2; exit 2; } + done < <(trusted_git config --file "$config" --no-includes --name-only --list) + [ "$(trusted_git config --file "$config" --no-includes --get remote.origin.url || true)" = "$SOURCE_ORIGIN" ] || { echo "error: canonical FM_ROOT origin provenance changed" >&2 exit 2 } diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index 582f16825..4b9a5d1b1 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -228,6 +228,7 @@ acquire_lifecycle_locks() { configure_control_lock LOCK=$CONTROL_LOCK LOCK_NAMESPACE=$CONTROL_NAMESPACE + LOCK_INSTALLATION_ID=$CONTROL_LOCK_INSTALLATION_ID EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"primary"$'\t'"$CONTROL_LOCK_INSTALLATION_ID" LOCK_SCOPE=control LOCK_PERSISTENT=1 @@ -238,6 +239,7 @@ acquire_lifecycle_locks() { CONTROL_LOCK_VALID_UNTIL=$LOCK_VALID_UNTIL LOCK=agent-os-firstmate-lifecycle LOCK_NAMESPACE=$NAMESPACE + LOCK_INSTALLATION_ID=$INSTALLATION_ID EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"primary"$'\t'"$INSTALLATION_ID" LOCK_SCOPE=fleet LOCK_PERSISTENT=0 @@ -272,6 +274,7 @@ release_lifecycle_locks() { if [ -n "${CONTROL_LOCK_UID:-}" ]; then LOCK=$CONTROL_LOCK LOCK_NAMESPACE=$CONTROL_NAMESPACE + LOCK_INSTALLATION_ID=$CONTROL_LOCK_INSTALLATION_ID EXPECTED_LOCK="$LOCK"$'\t'"agent-os"$'\t'"primary"$'\t'"$CONTROL_LOCK_INSTALLATION_ID" LOCK_SCOPE=control LOCK_PERSISTENT=1 diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh index c272a76f9..2f57061a6 100755 --- a/bin/agent-os-kubernetes.sh +++ b/bin/agent-os-kubernetes.sh @@ -39,6 +39,7 @@ CONTROL_LOCK_RV= CONTROL_LOCK_RENEW_PID= CONTROL_LOCK_VALID_UNTIL= REMOVE_CONTROL_ACCESS=0 +STATEFULSET_CAS_ATTEMPTED=0 LOCK_DURATION_SECONDS=${AGENT_OS_LOCK_DURATION_SECONDS:-300} LOCK_CLOCK_SKEW_SECONDS=${AGENT_OS_LOCK_CLOCK_SKEW_SECONDS:-5} LOCK_ACQUIRE_SECONDS=${AGENT_OS_LOCK_ACQUIRE_SECONDS:-30} @@ -387,8 +388,34 @@ runtime_binding_json() { esac } +verify_runtime_role_rules() { + local json + json=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get role agent-os-firstmate-runtime -o json) || return 3 + printf '%s' "$json" | jq -e --arg installation "$INSTALLATION_ID" ' + .metadata.name == "agent-os-firstmate-runtime" and + .metadata.labels["app.kubernetes.io/managed-by"] == "agent-os" and + .metadata.annotations["agent-os.dev/installation-id"] == $installation and + .rules == [ + {"apiGroups":[""],"resources":["pods","persistentvolumeclaims"],"verbs":["get","list","watch","create","delete","patch"]}, + {"apiGroups":[""],"resources":["pods/log","pods/exec"],"verbs":["get","list","watch","create","delete"]}, + {"apiGroups":["apps"],"resources":["statefulsets"],"verbs":["get","list","watch"]}, + {"apiGroups":["coordination.k8s.io"],"resources":["leases"],"verbs":["get","create","update","delete"]} + ]' >/dev/null +} + +verify_control_role_rules() { + local json + json=$("$KUBECTL" --context "$CONTEXT" -n "$CONTROL_NAMESPACE" get role "$CONTROL_LOCK" -o json) || return 3 + printf '%s' "$json" | jq -e --arg installation "$INSTALLATION_ID" --arg name "$CONTROL_LOCK" ' + .metadata.name == $name and + .metadata.labels["app.kubernetes.io/managed-by"] == "agent-os" and + .metadata.annotations["agent-os.dev/installation-id"] == $installation and + .rules == [{apiGroups:["coordination.k8s.io"],resources:["leases"],resourceNames:[$name],verbs:["get","update"]}]' >/dev/null +} + verify_control_authority() { local account=$1 json + verify_control_role_rules || return 3 json=$(control_binding_json) || return 3 printf '%s' "$json" | jq -e --arg installation "$INSTALLATION_ID" --arg namespace "$NAMESPACE" \ --arg name "$CONTROL_LOCK" --arg account "$account" ' @@ -420,6 +447,8 @@ transfer_runtime_authority() { binding=agent-os-firstmate-runtime kind=rolebinding scope=(-n "$NAMESPACE") + verify_runtime_role_rules || { echo "error: rollback runtime Role rules are unverifiable" >&2; return 3; } + verify_control_role_rules || { echo "error: rollback control Role rules are unverifiable" >&2; return 3; } else binding="agent-os-firstmate-$NAMESPACE" kind=clusterrolebinding @@ -441,6 +470,7 @@ transfer_runtime_authority() { if [ "$current" = "$to" ]; then if [ "$mode" = namespace ]; then transfer_control_authority "$from" "$to" || return 3 + verify_runtime_role_rules || return 3 verify_control_authority "$to" || return 3 fi return 0 @@ -459,6 +489,7 @@ transfer_runtime_authority() { } if [ "$mode" = namespace ]; then transfer_control_authority "$from" "$to" || return 3 + verify_runtime_role_rules || return 3 verify_control_authority "$to" fi } @@ -469,6 +500,7 @@ control_binding_json() { transfer_control_authority() { local from=$1 to=$2 json current uid rv patch verified + verify_control_role_rules || return 3 json=$(control_binding_json) || return 3 printf '%s' "$json" | jq -e --arg name "$CONTROL_LOCK" \ '.roleRef == {apiGroup:"rbac.authorization.k8s.io",kind:"Role",name:$name}' >/dev/null || return 3 @@ -609,6 +641,21 @@ inventory_revision_service_accounts() { printf '%s' "$accounts" } +inventory_owned_service_accounts() { + local resources accounts + resources=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ + --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get serviceaccounts -o json) || return 3 + accounts=$(printf '%s' "$resources" | jq -er --arg installation "$INSTALLATION_ID" ' + [.items[] + | select(.metadata.name | test("^agent-os-firstmate(-[a-z0-9]{12})?$")) + | select(.metadata.labels["app.kubernetes.io/managed-by"] == "agent-os") + | select(.metadata.annotations["agent-os.dev/installation-id"] == $installation)] as $owned + | select(all($owned[]; (.metadata.uid | type == "string" and length > 0) and + (.metadata.resourceVersion | type == "string" and length > 0))) + | [$owned[].metadata.name] | unique | join("\n")') || return 3 + printf '%s' "$accounts" +} + require_no_active_rollback_checkpoint() { local state checkpoint if ! command -v jq >/dev/null 2>&1; then @@ -1184,6 +1231,7 @@ mutate_rendered_resource() { record=$(live_resource_record "$scope" "$kind" "$name") expected="$name"$'\t'"agent-os"$'\t'"$INSTALLATION_ID" if [ -z "$record" ]; then + [ "$kind" != StatefulSet ] || STATEFULSET_CAS_ATTEMPTED=1 if ! "$KUBECTL" --context "$CONTEXT" create -f "$file" >/dev/null; then current=$(live_resource_record "$scope" "$kind" "$name" 2>/dev/null || true) if [ "$(printf '%s' "$current" | cut -f1-3)" != "$expected" ] || \ @@ -1207,6 +1255,7 @@ mutate_rendered_resource() { } fi patch_file=$(cas_patch_file "$file" "$uid" "$rv") + [ "$kind" != StatefulSet ] || STATEFULSET_CAS_ATTEMPTED=1 if [ "$scope" = cluster ]; then if ! "$KUBECTL" --context "$CONTEXT" patch "$kind" "$name" --type=merge --patch-file "$patch_file" >/dev/null; then report_partial_apply patch @@ -1247,22 +1296,14 @@ apply_rendered() { } verify_desired_rbac() { - local role_json binding_json + local binding_json if [ "$DESIRED_RBAC" = namespace ]; then if ! command -v jq >/dev/null 2>&1; then echo "error: jq is required to verify namespace RBAC exactly" >&2 exit 2 fi - role_json=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get role agent-os-firstmate-runtime -o json) binding_json=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" get rolebinding agent-os-firstmate-runtime -o json) - if ! printf '%s' "$role_json" | jq -e ' - .metadata.name == "agent-os-firstmate-runtime" and - .rules == [ - {"apiGroups":[""],"resources":["pods","persistentvolumeclaims"],"verbs":["get","list","watch","create","delete","patch"]}, - {"apiGroups":[""],"resources":["pods/log","pods/exec"],"verbs":["get","list","watch","create","delete"]}, - {"apiGroups":["apps"],"resources":["statefulsets"],"verbs":["get","list","watch"]}, - {"apiGroups":["coordination.k8s.io"],"resources":["leases"],"verbs":["get","create","update","delete"]} - ]' >/dev/null || + if ! verify_runtime_role_rules || ! printf '%s' "$binding_json" | jq -e --arg namespace "$NAMESPACE" --arg serviceAccount "$SERVICE_ACCOUNT_NAME" ' .roleRef == {"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"agent-os-firstmate-runtime"} and .subjects == [{"kind":"ServiceAccount","name":$serviceAccount,"namespace":$namespace}]' >/dev/null; then @@ -1593,7 +1634,18 @@ case "$COMMAND" in patch_workload_annotations '{"agent-os.dev/cluster-rbac-cleanup":"required"}' fi ensure_control_lock_access - apply_rendered + if ! apply_rendered; then + if [ "$STATEFULSET_CAS_ATTEMPTED" -eq 1 ]; then + reconcile_failed_akua_upgrade "$PRESERVED_AKUA_SECRET_RECORD" || { + echo "incomplete: post-CAS upgrade failure and fail-closed authorization reconciliation is unverified" >&2 + report_partial_observation StatefulSet agent-os-firstmate namespaced + report_partial_observation Pod agent-os-firstmate-0 namespaced + exit 3 + } + echo "safe recovery: run the serialized authorization revoke before retrying upgrade" >&2 + fi + exit 3 + fi if ! verified_akua_overlay_secret "$PRESERVED_AKUA_SECRET_RECORD" >/dev/null; then reconcile_failed_akua_upgrade "$PRESERVED_AKUA_SECRET_RECORD" || { echo "incomplete: upgrade authorization changed and fail-closed reconciliation is unverified" >&2 @@ -1922,17 +1974,21 @@ case "$COMMAND" in require_workload_owned "$previous" optional previous_service_account= previous_workload_uid= - historical_service_accounts= + historical_service_accounts=$(inventory_owned_service_accounts) || { + echo "error: uninstall could not recover exact-owned ServiceAccount dependencies" >&2 + exit 3 + } retained_legacy_service_account= if [ -n "$previous" ]; then previous_service_account=$(workload_service_account) previous_workload_record=$(live_resource_record namespaced StatefulSet agent-os-firstmate) previous_workload_uid=$(printf '%s' "$previous_workload_record" | cut -f4) [ -n "$previous_workload_uid" ] || { echo "error: uninstall requires exact StatefulSet UID evidence" >&2; exit 3; } - historical_service_accounts=$(inventory_revision_service_accounts "$previous_workload_uid") || { + revision_service_accounts=$(inventory_revision_service_accounts "$previous_workload_uid") || { echo "error: uninstall could not inventory exact historical ServiceAccount dependencies" >&2 exit 3 } + historical_service_accounts=$(printf '%s\n%s\n' "$historical_service_accounts" "$revision_service_accounts" | awk 'NF && !seen[$0]++') fi legacy_identity=$(live_resource_identity ServiceAccount agent-os-firstmate) if [ -n "$legacy_identity" ]; then @@ -1944,6 +2000,7 @@ case "$COMMAND" in fi previous_mode=$(printf '%s' "$previous" | cut -f2) previous_cleanup=$(printf '%s' "$previous" | cut -f3) + REMOVE_CONTROL_ACCESS=1 delete_rendered_namespaced_resources if [ -n "$previous_workload_uid" ] || [ -n "$retained_legacy_service_account" ]; then wait_for_revision_history_absence "$previous_workload_uid" @@ -1961,9 +2018,6 @@ case "$COMMAND" in delete_owned_resource namespaced ServiceAccount "$historical_service_account" 180 done <<< "$historical_service_accounts" delete_namespace_rbac - if [ "$DESIRED_RBAC" = namespace ] || [ "$previous_mode" = namespace ]; then - REMOVE_CONTROL_ACCESS=1 - fi if [ "$DELETE_NAMESPACE" -eq 1 ]; then delete_owned_empty_namespace fi diff --git a/bin/agent-os-source-bundle.sh b/bin/agent-os-source-bundle.sh index 04b64338b..70c5c93a3 100755 --- a/bin/agent-os-source-bundle.sh +++ b/bin/agent-os-source-bundle.sh @@ -20,7 +20,7 @@ git_isolated() { env -u GIT_CONFIG -u GIT_CONFIG_PARAMETERS -u GIT_CONFIG_COUNT \ -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy \ -u GIT_SSH -u GIT_SSH_COMMAND GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null \ - GIT_TERMINAL_PROMPT=0 command git -c credential.helper= -c core.hooksPath=/dev/null \ + GIT_TERMINAL_PROMPT=0 git -c credential.helper= -c core.hooksPath=/dev/null \ -c http.proxy= -c https.proxy= "$@" } @@ -32,10 +32,10 @@ sha256_stream() { if command -v sha256sum >/dev/null 2>&1; then sha256sum | awk '{print $1}'; else shasum -a 256 | awk '{print $1}'; fi } -case "$(git_isolated -C "$ROOT" remote get-url origin)" in - https://github.com/akua-dev/agent-os.git|ssh://git@github.com/akua-dev/agent-os.git|git@github.com:akua-dev/agent-os.git) ;; - *) echo "error: repository origin is not allowlisted" >&2; exit 2 ;; -esac +[ "$(git_isolated -C "$ROOT" remote get-url origin)" = "$SOURCE_ORIGIN" ] || { + echo "error: repository origin is not the exact trusted HTTPS origin" >&2 + exit 2 +} [ -z "$(git_isolated -C "$ROOT" status --porcelain --untracked-files=all)" ] || { echo "error: exact-source image builds require a clean worktree" >&2 exit 2 diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 6ec7aa643..0b549feea 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -121,6 +121,10 @@ assert_grep 'fetch --depth=1 --no-tags' "$ROOT/bin/agent-os-source-bundle.sh" \ "source preparation must fetch an allowlisted remote ref freshly" assert_grep 'GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null' "$ROOT/bin/agent-os-source-bundle.sh" \ "source preparation must isolate trusted Git operations from ambient configuration" +assert_grep 'GIT_TERMINAL_PROMPT=0 git ' "$ROOT/bin/agent-os-source-bundle.sh" \ + "source preparation must invoke Git directly through env" +assert_grep 'repository origin is not the exact trusted HTTPS origin' "$ROOT/bin/agent-os-source-bundle.sh" \ + "source preparation must require the exact trusted HTTPS origin" assert_grep 'AGENT_OS_SOURCE_MODE=event AGENT_OS_SOURCE_EVENT_COMMIT' "$IMAGE_WORKFLOW" \ "pull requests must build their exact event head without publication credentials" assert_no_grep 'git clone --depth=1 --branch main' "$IMAGE_WORKFLOW" \ @@ -133,8 +137,12 @@ assert_grep 'source_archive_sha256' "$ROOT/bin/agent-os-source-bundle.sh" \ "release records must bind the exact archived source" assert_grep 'GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null' "$ROOT/bin/agent-os-container-entrypoint.sh" \ "runtime provenance must isolate trusted Git operations" -assert_grep 'canonical FM_ROOT Git config contains untrusted' "$ROOT/bin/agent-os-container-entrypoint.sh" \ - "runtime provenance must reject persistent transport and execution overrides" +assert_grep 'GIT_TERMINAL_PROMPT=0 git ' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "runtime provenance must invoke Git directly through env" +assert_grep 'canonical FM_ROOT Git config key is not allowlisted' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "runtime provenance must use a strict repository-config allowlist" +assert_no_grep 'config --file "$config" --no-includes --name-only --get-regexp' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "runtime provenance must not rely on a dangerous-key blacklist" assert_grep 'status --porcelain --untracked-files=all' "$ROOT/bin/agent-os-source-bundle.sh" \ "source preparation must reject a dirty workspace" assert_no_grep 'bundle create.*HEAD' "$ROOT/bin/agent-os-source-bundle.sh" \ @@ -188,12 +196,32 @@ assert_grep ' validate:' "$IMAGE_WORKFLOW" \ "pull requests must use a distinct read-only validation job" assert_grep ' publish:' "$IMAGE_WORKFLOW" \ "push and tag publication must use a distinct privileged job" -assert_grep 'needs: [behavior, provenance, validate]' "$IMAGE_WORKFLOW" \ +assert_grep 'needs: [behavior, provenance, validate, release_artifacts]' "$IMAGE_WORKFLOW" \ "the packages-write job must require exact behavior, provenance, and image gates" assert_grep 'github.ref_protected' "$IMAGE_WORKFLOW" \ "main publication must require GitHub protected-ref provenance" assert_grep 'release tag ruleset differs from immutable record' "$IMAGE_WORKFLOW" \ "tag publication must require its recorded protected-tag ruleset" +assert_grep 'git merge-base --is-ancestor "$(jq -r .commit' "$IMAGE_WORKFLOW" \ + "tag publication must require the recorded release commit on protected main" +assert_grep '.enforcement == "active" and .target == "tag"' "$IMAGE_WORKFLOW" \ + "tag publication must require an active tag-targeting ruleset" +assert_grep '.bypass_actors | length == 0' "$IMAGE_WORKFLOW" \ + "tag publication must reject rulesets with bypass actors" +assert_grep 'release_artifacts:' "$IMAGE_WORKFLOW" \ + "release publication must have a read-only artifact verification gate" +assert_grep 'actual_image_digest' "$IMAGE_WORKFLOW" \ + "release publication must compare the actual multi-platform image digest" +assert_grep 'actual_sbom_sha256' "$IMAGE_WORKFLOW" \ + "release publication must compare the actual SBOM artifact set" +assert_grep 'actual_provenance_sha256' "$IMAGE_WORKFLOW" \ + "release publication must compare the actual provenance artifact set" +assert_grep 'oras cp --from-oci-layout' "$IMAGE_WORKFLOW" \ + "release publication must copy the exact read-only verified OCI artifact" +assert_grep 'oras-project/setup-oras@22ce207df3b08e061f537244349aac6ae1d214f6' "$IMAGE_WORKFLOW" \ + "release publication must pin the OCI transfer implementation" +assert_grep 'published-tag-index.json' "$IMAGE_WORKFLOW" \ + "release publication must verify every published tag resolves to the recorded digest" assert_grep 'Section 13' "$ROOT/docs/herdr-compliance.md" \ "the Herdr audit must account for the network-interaction clause" assert_grep 'https://github.com/akua-dev/akua/tree/v0.8.25' "$ROOT/THIRD_PARTY_NOTICES.md" \ @@ -245,8 +273,8 @@ assert_grep 'ghcr.io/akua-dev/agent-os' "$ROOT/.github/workflows/agent-os-image. "release workflow must publish the image expected by the portable package" assert_grep 'linux/amd64,linux/arm64' "$ROOT/.github/workflows/agent-os-image.yml" \ "release workflow must build the two supported container architectures" -[ "$(grep -c '^ push: false$' "$IMAGE_WORKFLOW")" -eq 1 ] || \ - fail "the read-only pull-request job must build without publishing" +[ "$(grep -c '^ push: false$' "$IMAGE_WORKFLOW")" -eq 2 ] || \ + fail "the read-only validation and release-artifact jobs must build without publishing" [ "$(grep -c '^ push: true$' "$IMAGE_WORKFLOW")" -eq 1 ] || \ fail "only the protected publication job may push images" assert_grep 'id: build' "$ROOT/.github/workflows/agent-os-image.yml" \ @@ -265,7 +293,7 @@ assert_grep 'docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051' "$ "release metadata action must be pinned to the reviewed full SHA" assert_grep 'docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8' "$ROOT/.github/workflows/agent-os-image.yml" \ "release build action must be pinned to the reviewed full SHA" -if grep -E 'uses: (actions/checkout|docker/[^@]+)@v[0-9]+' "$ROOT/.github/workflows/agent-os-image.yml" >/dev/null; then +if grep -E 'uses: (actions/checkout|docker/[^@]+|oras-project/setup-oras)@v[0-9]+' "$ROOT/.github/workflows/agent-os-image.yml" >/dev/null; then fail "release workflow must not use mutable major action tags" fi bash -n "$ROOT/bin/agent-os-container-entrypoint.sh" diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index db0cfd6bc..b52222696 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -9,7 +9,7 @@ assert_grep 'configure_control_lock' "$ROOT/bin/agent-os-crewmate.sh" \ "crewmate mutations must acquire the stable installation control lock" assert_grep 'CONTROL_LOCK_UID' "$ROOT/bin/agent-os-crewmate.sh" \ "crewmate mutations must retain control-lock CAS evidence" -assert_grep 'agent-os.dev/installation-id: ${LOCK_INSTALLATION_ID:-$INSTALLATION_ID}' "$ROOT/bin/agent-os-crewmate.sh" \ +assert_grep 'LOCK_INSTALLATION_ID=$CONTROL_LOCK_INSTALLATION_ID' "$ROOT/bin/agent-os-crewmate.sh" \ "crewmate control Leases must render their scope-specific installation identity" assert_grep 'configure_control_lock' "$ROOT/bin/agent-os-akua-auth.sh" \ "authorization mutations must acquire the stable installation control lock" @@ -35,6 +35,18 @@ assert_grep 'verify_no_runtime_authority' "$ROOT/bin/agent-os-kubernetes.sh" \ "RBAC none must verify namespace, cluster, and control authority absence" assert_grep 'rollback_source_digest' "$ROOT/bin/agent-os-kubernetes.sh" \ "rollback compensation must verify its immutable source revision digest" +assert_grep 'if [ "$COMMAND" = grant ]; then' "$ROOT/bin/agent-os-akua-auth.sh" \ + "authorization Secret validation must apply only to grant" +assert_grep 'STATEFULSET_CAS_ATTEMPTED' "$ROOT/bin/agent-os-kubernetes.sh" \ + "post-CAS upgrade failures must enter fail-closed authorization reconciliation" +assert_grep 'inventory_owned_service_accounts' "$ROOT/bin/agent-os-kubernetes.sh" \ + "uninstall retries must recover exact-owned historical ServiceAccounts independently" +assert_grep 'verify_runtime_role_rules' "$ROOT/bin/agent-os-kubernetes.sh" \ + "rollback must verify exact namespace runtime Role rules" +assert_grep 'verify_control_role_rules' "$ROOT/bin/agent-os-kubernetes.sh" \ + "rollback must verify exact control Role rules" +assert_grep ' REMOVE_CONTROL_ACCESS=1' "$ROOT/bin/agent-os-kubernetes.sh" \ + "uninstall must always remove exact-owned control RBAC" LAUNCHER="$ROOT/bin/agent-os-crewmate.sh" TMP=$(fm_test_tmproot agent-os-kubernetes) @@ -141,6 +153,11 @@ fi if [ "${AGENT_OS_TEST_FAIL_ROLLOUT:-0}" = 1 ] && [[ " $* " = *" rollout status statefulset/agent-os-firstmate "* ]]; then exit 1 fi +if [ "${AGENT_OS_TEST_FAIL_FIRST_ROLLOUT:-0}" = 1 ] && \ + [[ " $* " = *" rollout status statefulset/agent-os-firstmate "* ]] && \ + [ "$(grep -Fc 'rollout status statefulset/agent-os-firstmate' "$AGENT_OS_TEST_LOG")" -eq 1 ]; then + exit 1 +fi if [ "${AGENT_OS_TEST_LOCK_STATE:-free}" = held ] && [[ " $* " = *" wait --for=delete lease/"* ]]; then exit 1 fi @@ -167,7 +184,11 @@ if [[ " $* " = *" get Role agent-os-lifecycle-"*" -o json "* ]] || \ [[ " $* " = *" get role agent-os-lifecycle-"*" -o json "* ]]; then control_name=$(printf '%s\n' "$*" | sed -n 's/.* get [Rr]ole \([^ ]*\) .*/\1/p') if ! grep -Fq "/roles/$control_name" "$AGENT_OS_TEST_LOG"; then - printf '{"metadata":{"name":"%s","uid":"uid-control-role","resourceVersion":"rv-control-role","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"rules":[{"apiGroups":["coordination.k8s.io"],"resources":["leases"],"resourceNames":["%s"],"verbs":["get","update"]}]}\n' "$control_name" "$control_name" + if [ "${AGENT_OS_TEST_CONTROL_RBAC_STATE:-exact}" = bad-rules ]; then + printf '{"metadata":{"name":"%s","uid":"uid-control-role","resourceVersion":"rv-control-role","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"rules":[]}\n' "$control_name" + else + printf '{"metadata":{"name":"%s","uid":"uid-control-role","resourceVersion":"rv-control-role","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"rules":[{"apiGroups":["coordination.k8s.io"],"resources":["leases"],"resourceNames":["%s"],"verbs":["get","update"]}]}\n' "$control_name" "$control_name" + fi fi exit 0 fi @@ -398,6 +419,10 @@ case " $* " in if { [ "$kind" = Role ] || [ "$kind" = RoleBinding ]; } && [ -n "${AGENT_OS_TEST_STALE_RBAC_STATE:-}" ]; then resource_state=$AGENT_OS_TEST_STALE_RBAC_STATE fi + if [ "$kind" = ServiceAccount ] && [ "$name" = "${AGENT_OS_TEST_RESIDUAL_SERVICE_ACCOUNT:-}" ]; then + resource_state=owned + grep -F "/serviceaccounts/$name" "$AGENT_OS_TEST_LOG" | grep -F 'delete --raw' >/dev/null && resource_state=absent + fi if grep -E ' create -f .+\.yaml' "$AGENT_OS_TEST_LOG" | grep -v 'namespace.yaml' >/dev/null || \ grep -F ' --patch-file ' "$AGENT_OS_TEST_LOG" >/dev/null || \ grep -F ' apply -f ' "$AGENT_OS_TEST_LOG" >/dev/null; then @@ -437,7 +462,8 @@ case " $* " in akua_env='' akua_mount='' akua_volume='' - if [ -n "${AGENT_OS_TEST_AKUA_OVERLAY_SECRET:-}" ]; then + if [ -n "${AGENT_OS_TEST_AKUA_OVERLAY_SECRET:-}" ] && \ + ! grep -F 'agent-os.dev/akua-auth-rejected-record' "$AGENT_OS_TEST_LOG" >/dev/null; then akua_annotations=$(printf ',"agent-os.dev/akua-auth-secret":"%s"' "$AGENT_OS_TEST_AKUA_OVERLAY_SECRET") akua_env=',{"name":"AKUA_AUTH_HEADER_FILE","value":"/var/run/secrets/agent-os/akua/authorization"}' akua_mount=',{"name":"akua-auth","mountPath":"/var/run/secrets/agent-os/akua","readOnly":true}' @@ -496,7 +522,11 @@ case " $* " in if ! grep -Fq "/$([ "$control_kind" = Role ] && printf roles || printf rolebindings)/$control_name" "$AGENT_OS_TEST_LOG"; then control_account=$(effective_binding_account "$control_name") if [ "$control_kind" = Role ]; then - printf '{"metadata":{"name":"%s","uid":"uid-control-role","resourceVersion":"rv-control-role","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"rules":[{"apiGroups":["coordination.k8s.io"],"resources":["leases"],"resourceNames":["%s"],"verbs":["get","update"]}]}\n' "$control_name" "$control_name" + if [ "${AGENT_OS_TEST_CONTROL_RBAC_STATE:-exact}" = bad-rules ]; then + printf '{"metadata":{"name":"%s","uid":"uid-control-role","resourceVersion":"rv-control-role","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"rules":[]}\n' "$control_name" + else + printf '{"metadata":{"name":"%s","uid":"uid-control-role","resourceVersion":"rv-control-role","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"rules":[{"apiGroups":["coordination.k8s.io"],"resources":["leases"],"resourceNames":["%s"],"verbs":["get","update"]}]}\n' "$control_name" "$control_name" + fi else printf '{"metadata":{"name":"%s","uid":"uid-control-binding","resourceVersion":"rv-control-binding","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"%s"},"subjects":[{"kind":"ServiceAccount","name":"%s","namespace":"portable-agent-os"}]}\n' "$control_name" "$control_name" "$control_account" fi @@ -504,9 +534,9 @@ case " $* " in ;; *" get role agent-os-firstmate-runtime -o json "*) if [ "${AGENT_OS_TEST_RBAC_STATE:-exact}" = bad-rules ]; then - printf '%s\n' '{"metadata":{"name":"agent-os-firstmate-runtime"},"rules":[]}' + printf '%s\n' '{"metadata":{"name":"agent-os-firstmate-runtime","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"rules":[]}' else - printf '%s\n' '{"metadata":{"name":"agent-os-firstmate-runtime"},"rules":[{"apiGroups":[""],"resources":["pods","persistentvolumeclaims"],"verbs":["get","list","watch","create","delete","patch"]},{"apiGroups":[""],"resources":["pods/log","pods/exec"],"verbs":["get","list","watch","create","delete"]},{"apiGroups":["apps"],"resources":["statefulsets"],"verbs":["get","list","watch"]},{"apiGroups":["coordination.k8s.io"],"resources":["leases"],"verbs":["get","create","update","delete"]}]}' + printf '%s\n' '{"metadata":{"name":"agent-os-firstmate-runtime","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}},"rules":[{"apiGroups":[""],"resources":["pods","persistentvolumeclaims"],"verbs":["get","list","watch","create","delete","patch"]},{"apiGroups":[""],"resources":["pods/log","pods/exec"],"verbs":["get","list","watch","create","delete"]},{"apiGroups":["apps"],"resources":["statefulsets"],"verbs":["get","list","watch"]},{"apiGroups":["coordination.k8s.io"],"resources":["leases"],"verbs":["get","create","update","delete"]}]}' fi ;; *" get rolebinding agent-os-firstmate-runtime -o json "*) @@ -581,6 +611,14 @@ case " $* " in fi ;; *" get serviceaccounts -o name "*) printf '%s\n' serviceaccount/default ;; + *" get serviceaccounts -o json "*) + residual=${AGENT_OS_TEST_RESIDUAL_SERVICE_ACCOUNT:-} + if [ -n "$residual" ]; then + printf '{"items":[{"metadata":{"name":"%s","uid":"uid-residual-serviceaccount","resourceVersion":"rv-residual-serviceaccount","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"}}}]}\n' "$residual" + else + printf '%s\n' '{"items":[]}' + fi + ;; *" get configmaps -o name "*) printf '%s\n' configmap/kube-root-ca.crt ;; *" get leases.coordination.k8s.io -o name "*) [ -z "${AGENT_OS_TEST_FOREIGN_LEASE:-}" ] || printf 'lease/%s\n' "$AGENT_OS_TEST_FOREIGN_LEASE" @@ -1640,6 +1678,19 @@ secret_reads_before_patch=$(sed -n "1,${statefulset_patch_line}p" "$CALLS" | gre fail "upgrade must revalidate the exact Secret identity immediately before StatefulSet CAS" pass "upgrade revalidates authorization immediately before CAS" +: > "$CALLS" +upgrade_reconcile_out='' +upgrade_reconcile_rc=0 +upgrade_reconcile_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_AKUA_OVERLAY_SECRET=akua-auth \ + AGENT_OS_TEST_FAIL_FIRST_ROLLOUT=1 run_generic upgrade 2>&1) || upgrade_reconcile_rc=$? +[ "$upgrade_reconcile_rc" -eq 3 ] || fail "post-CAS rollout failure must remain incomplete: $upgrade_reconcile_out" +assert_grep 'agent-os.dev/akua-auth-rejected-record' "$CALLS" \ + "post-CAS upgrade failure must record fail-closed authorization evidence" +assert_grep '"AKUA_AUTH_HEADER_FILE","$patch":"delete"' "$CALLS" \ + "post-CAS upgrade failure must remove the mounted authorization overlay" +pass "upgrade post-CAS failures reconcile authorization fail closed" + : > "$CALLS" AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ AGENT_OS_TEST_WORKLOAD_STATE=namespace \ @@ -1685,6 +1736,28 @@ assert_grep 'get controllerrevisions.apps -o json' "$CALLS" \ "rollback must resolve the exact-owned revision history" pass "generic rollback applies a revision-derived StatefulSet CAS update" +: > "$CALLS" +rollback_rules_out='' +rollback_rules_rc=0 +rollback_rules_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_RBAC_STATE=bad-rules \ + run_generic rollback 2>&1) || rollback_rules_rc=$? +[ "$rollback_rules_rc" -eq 3 ] || fail "rollback must reject drifted runtime Role rules: $rollback_rules_out" +assert_no_grep '"rollback":"previous"' "$CALLS" \ + "drifted runtime Role rules must block rollback template mutation" +pass "rollback verifies exact runtime and control Role rules" + +: > "$CALLS" +rollback_control_rules_out='' +rollback_control_rules_rc=0 +rollback_control_rules_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_CONTROL_RBAC_STATE=bad-rules \ + run_generic rollback 2>&1) || rollback_control_rules_rc=$? +[ "$rollback_control_rules_rc" -eq 3 ] || fail "rollback must reject drifted control Role rules: $rollback_control_rules_out" +assert_no_grep '"rollback":"previous"' "$CALLS" \ + "drifted control Role rules must block rollback template mutation" +pass "rollback rejects drifted control Role rules" + : > "$CALLS" checkpoint_read_race_out='' checkpoint_read_race_rc=0 @@ -1898,6 +1971,10 @@ if grep -E 'kubectl .* (get|delete) clusterrolebinding' "$CALLS" >/dev/null; the fi assert_contains "$uninstall_residue_out" 'cleanup-cluster-rbac --yes' \ "cluster-admin uninstall must print the exact privileged cleanup command" +grep -F '/apis/rbac.authorization.k8s.io/v1/namespaces/kube-system/rolebindings/agent-os-lifecycle-' "$CALLS" | \ + grep -F 'delete --raw' >/dev/null || fail "cluster-admin uninstall must delete exact-owned control RoleBinding authority" +grep -F '/apis/rbac.authorization.k8s.io/v1/namespaces/kube-system/roles/agent-os-lifecycle-' "$CALLS" | \ + grep -F 'delete --raw' >/dev/null || fail "cluster-admin uninstall must delete exact-owned control Role authority" pass "routine uninstall reports cluster-scoped residue for separate cleanup" : > "$CALLS" @@ -1911,6 +1988,18 @@ assert_contains "$retry_residue_out" 'cleanup-cluster-rbac --yes' \ "history-free uninstall retry must print the privileged absence-evidence command" pass "uninstall retry cannot lose cluster-RBAC residue state" +: > "$CALLS" +retry_service_account=agent-os-firstmate-cccccccccccc +retry_sa_out='' +retry_sa_rc=0 +retry_sa_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=absent \ + AGENT_OS_TEST_RESOURCE_STATE=absent AGENT_OS_TEST_RESIDUAL_SERVICE_ACCOUNT="$retry_service_account" \ + run_generic uninstall --yes 2>&1) || retry_sa_rc=$? +[ "$retry_sa_rc" -eq 3 ] || fail "history-free uninstall retry must retain cluster cleanup status: $retry_sa_out" +assert_grep "/serviceaccounts/$retry_service_account" "$CALLS" \ + "uninstall retry must independently recover and delete exact-owned historical ServiceAccounts" +pass "uninstall retry recovers historical ServiceAccounts independently" + : > "$CALLS" AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_WORKLOAD_STATE=namespace \ run_generic uninstall --yes --delete-namespace From cc5c997b2c7fe1044f7773d821f7d43074709a60 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 18:28:48 +0200 Subject: [PATCH 39/56] no-mistakes(review): Secure candidate promotion and lifecycle compensation --- .github/workflows/agent-os-image.yml | 509 ++++++++++++++++----------- Dockerfile | 2 +- bin/agent-os-akua-auth.sh | 3 +- bin/agent-os-container-entrypoint.sh | 37 +- bin/agent-os-kubernetes.sh | 46 ++- bin/agent-os-source-bundle.sh | 79 ++++- tests/agent-os-container.test.sh | 54 ++- tests/agent-os-kubernetes.test.sh | 34 +- 8 files changed, 516 insertions(+), 248 deletions(-) diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index 1d2ff6e6e..333b57a84 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -12,7 +12,7 @@ permissions: concurrency: group: agent-os-image-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: false env: IMAGE: ghcr.io/akua-dev/agent-os @@ -39,18 +39,24 @@ jobs: REF_PROTECTED: ${{ github.ref_protected }} run: | set -eu + trusted_git() { + env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 \ + /usr/bin/git -c credential.helper= -c core.hooksPath=/dev/null \ + -c http.proxy= -c https.proxy= "$@" + } if [ "$GITHUB_REF" = refs/heads/main ]; then [ "$REF_PROTECTED" = true ] || { echo "::error::main is not reported protected"; exit 1; } elif [ "${GITHUB_REF#refs/tags/}" != "$GITHUB_REF" ]; then tag=${GITHUB_REF#refs/tags/} - [ "$(git cat-file -t "$GITHUB_REF")" = tag ] || { echo "::error::release tag must be annotated"; exit 1; } - record_commit=$(git for-each-ref --format='%(contents)' "$GITHUB_REF" | sed -n 's/^release-record-commit=\([0-9a-f]\{40\}\)$/\1/p') + [ "$(trusted_git cat-file -t "$GITHUB_REF")" = tag ] || { echo "::error::release tag must be annotated"; exit 1; } + record_commit=$(trusted_git for-each-ref --format='%(contents)' "$GITHUB_REF" | sed -n 's/^release-record-commit=\([0-9a-f]\{40\}\)$/\1/p') [ -n "$record_commit" ] || { echo "::error::release tag lacks immutable record commit"; exit 1; } - git fetch --no-tags origin refs/heads/main:refs/remotes/origin/main - git merge-base --is-ancestor "$record_commit" refs/remotes/origin/main + trusted_git fetch --no-tags https://github.com/akua-dev/agent-os.git refs/heads/main:refs/remotes/origin/main + trusted_git merge-base --is-ancestor "$record_commit" refs/remotes/origin/main gh api "repos/$GITHUB_REPOSITORY/branches/main/protection" >/dev/null - git show "$record_commit:image/releases/$tag.json" > "$RUNNER_TEMP/release-record.json" - git merge-base --is-ancestor "$(jq -r .commit "$RUNNER_TEMP/release-record.json")" refs/remotes/origin/main + trusted_git show "$record_commit:image/releases/$tag.json" > "$RUNNER_TEMP/release-record.json" + trusted_git merge-base --is-ancestor "$(jq -r .commit "$RUNNER_TEMP/release-record.json")" refs/remotes/origin/main ruleset_id=$(jq -r .tag_ruleset_id "$RUNNER_TEMP/release-record.json") gh api "repos/$GITHUB_REPOSITORY/rulesets/$ruleset_id" > "$RUNNER_TEMP/tag-ruleset.json" ruleset_sha=$(jq -cS . "$RUNNER_TEMP/tag-ruleset.json" | sha256sum | awk '{print $1}') @@ -66,21 +72,23 @@ jobs: echo "::error::release tag ruleset is not active, immutable, tag-targeted, and bypass-free" exit 1 } - python3 - "$GITHUB_REF" "$RUNNER_TEMP/tag-ruleset.json" <<'PY' - import fnmatch - import json - import sys - - ref, path = sys.argv[1:] - with open(path, encoding="utf-8") as stream: - ruleset = json.load(stream) - names = ruleset.get("conditions", {}).get("ref_name", {}) - include = names.get("include", []) - exclude = names.get("exclude", []) - matches = lambda pattern: pattern == "~ALL" or fnmatch.fnmatchcase(ref, pattern) - if not include or not any(matches(pattern) for pattern in include) or any(matches(pattern) for pattern in exclude): - raise SystemExit("release tag is not covered exclusively by the recorded tag ruleset") - PY + ruby -rjson - "$GITHUB_REF" "$RUNNER_TEMP/tag-ruleset.json" <<'RUBY' + ref, path = ARGV + names = JSON.parse(File.read(path)).fetch("conditions").fetch("ref_name") + include_patterns = names.fetch("include") + exclude_patterns = names.fetch("exclude") + patterns = include_patterns + exclude_patterns + abort "release tag ruleset contains unsupported ref syntax" unless patterns.all? do |pattern| + pattern.is_a?(String) && !pattern.include?("\\") && !pattern.include?("[^") && + (!pattern.start_with?("~") || pattern == "~ALL") + end + matches = lambda do |pattern| + pattern == "~ALL" || File.fnmatch?(pattern, ref, File::FNM_PATHNAME) + end + unless !include_patterns.empty? && include_patterns.any?(&matches) && !exclude_patterns.any?(&matches) + abort "release tag is not covered exclusively by the recorded tag ruleset" + end + RUBY else echo "::error::unsupported publication ref $GITHUB_REF" exit 1 @@ -100,13 +108,19 @@ jobs: id: source run: | set -eu + trusted_git() { + env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 \ + /usr/bin/git -c credential.helper= -c core.hooksPath=/dev/null \ + -c http.proxy= -c https.proxy= "$@" + } if [ "$GITHUB_EVENT_NAME" = pull_request ]; then - [ "$(git rev-parse HEAD)" = "${{ github.event.pull_request.head.sha }}" ] + [ "$(trusted_git rev-parse HEAD)" = "${{ github.event.pull_request.head.sha }}" ] AGENT_OS_SOURCE_MODE=event AGENT_OS_SOURCE_EVENT_COMMIT="${{ github.event.pull_request.head.sha }}" \ bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" else if [ "${GITHUB_REF#refs/tags/}" != "$GITHUB_REF" ]; then - record_commit=$(git for-each-ref --format='%(contents)' "$GITHUB_REF" | sed -n 's/^release-record-commit=\([0-9a-f]\{40\}\)$/\1/p') + record_commit=$(trusted_git for-each-ref --format='%(contents)' "$GITHUB_REF" | sed -n 's/^release-record-commit=\([0-9a-f]\{40\}\)$/\1/p') AGENT_OS_SOURCE_MODE=release AGENT_OS_SOURCE_RELEASE_TAG="${GITHUB_REF#refs/tags/}" \ AGENT_OS_RELEASE_RECORD_COMMIT="$record_commit" bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" else @@ -115,11 +129,14 @@ jobs: fi cat "$RUNNER_TEMP/source-metadata" >> "$GITHUB_OUTPUT" - - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 + - if: ${{ !startsWith(github.ref, 'refs/tags/') }} + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 - - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f + - if: ${{ !startsWith(github.ref, 'refs/tags/') }} + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f - name: Build without publication credentials + if: ${{ !startsWith(github.ref, 'refs/tags/') }} uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 with: context: . @@ -137,16 +154,14 @@ jobs: AGENT_OS_SOURCE_MODE=${{ steps.source.outputs.mode }} AGENT_OS_SOURCE_REF=${{ steps.source.outputs.ref }} - release_artifacts: + publish: if: github.event_name == 'push' - name: Verify immutable release artifacts + needs: [behavior, provenance, validate] + name: Publish linux/amd64 and linux/arm64 runs-on: ubuntu-latest permissions: contents: read - outputs: - image_digest: ${{ steps.verify.outputs.image_digest }} - sbom_sha256: ${{ steps.verify.outputs.sbom_sha256 }} - provenance_sha256: ${{ steps.verify.outputs.provenance_sha256 }} + packages: write steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 with: @@ -155,143 +170,241 @@ jobs: - name: Prepare immutable release source if: startsWith(github.ref, 'refs/tags/') - id: source + id: release_source run: | set -eu - record_commit=$(git for-each-ref --format='%(contents)' "$GITHUB_REF" | sed -n 's/^release-record-commit=\([0-9a-f]\{40\}\)$/\1/p') + trusted_git() { + env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 \ + /usr/bin/git -c credential.helper= -c core.hooksPath=/dev/null \ + -c http.proxy= -c https.proxy= "$@" + } + record_commit=$(trusted_git for-each-ref --format='%(contents)' "$GITHUB_REF" | sed -n 's/^release-record-commit=\([0-9a-f]\{40\}\)$/\1/p') AGENT_OS_SOURCE_MODE=release AGENT_OS_SOURCE_RELEASE_TAG="${GITHUB_REF#refs/tags/}" \ - AGENT_OS_RELEASE_RECORD_COMMIT="$record_commit" bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" - git show "$record_commit:image/releases/${GITHUB_REF#refs/tags/}.json" > "$RUNNER_TEMP/release-record.json" + AGENT_OS_RELEASE_RECORD_COMMIT="$record_commit" bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/release-source-metadata" + trusted_git show "$record_commit:image/releases/${GITHUB_REF#refs/tags/}.json" > "$RUNNER_TEMP/release-record.json" [ "$(sha256sum "$RUNNER_TEMP/release-record.json" | awk '{print $1}')" = \ - "$(sed -n 's/^release_record_sha256=//p' "$RUNNER_TEMP/source-metadata")" ] - cat "$RUNNER_TEMP/source-metadata" >> "$GITHUB_OUTPUT" + "$(sed -n 's/^release_record_sha256=//p' "$RUNNER_TEMP/release-source-metadata")" ] + cat "$RUNNER_TEMP/release-source-metadata" >> "$GITHUB_OUTPUT" + + - name: Prepare exact release candidate source + if: github.ref == 'refs/heads/main' + id: candidate_source + run: | + set -eu + AGENT_OS_SOURCE_MODE=candidate bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/candidate-source-metadata" + cat "$RUNNER_TEMP/candidate-source-metadata" >> "$GITHUB_OUTPUT" - - if: startsWith(github.ref, 'refs/tags/') + - if: github.ref == 'refs/heads/main' uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 - - if: startsWith(github.ref, 'refs/tags/') - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f - - name: Compute immutable release metadata - if: startsWith(github.ref, 'refs/tags/') - id: metadata - uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 + - uses: oras-project/setup-oras@22ce207df3b08e061f537244349aac6ae1d214f6 with: - images: ${{ env.IMAGE }} - tags: | - type=sha,format=long - type=ref,event=tag + version: 1.3.0 - - name: Build release artifact without publication authority - if: startsWith(github.ref, 'refs/tags/') - id: build + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Resolve existing release candidate record + if: github.ref == 'refs/heads/main' + id: candidate_state + run: | + set -eu + candidate_ref="$IMAGE:candidate-record-$GITHUB_SHA" + if oras manifest fetch --descriptor "$candidate_ref" > "$RUNNER_TEMP/candidate-descriptor.json" 2> "$RUNNER_TEMP/candidate-descriptor.err"; then + candidate_record_digest=$(jq -er '.digest | select(test("^sha256:[0-9a-f]{64}$"))' "$RUNNER_TEMP/candidate-descriptor.json") + mkdir "$RUNNER_TEMP/existing-candidate" + oras pull --output "$RUNNER_TEMP/existing-candidate" "$IMAGE@$candidate_record_digest" + cp "$RUNNER_TEMP/existing-candidate/candidate-record.json" "$RUNNER_TEMP/candidate-record.json" + printf 'exists=true\ncandidate_record_digest=%s\n' "$candidate_record_digest" >> "$GITHUB_OUTPUT" + elif grep -Eiq 'manifest unknown|not found|404' "$RUNNER_TEMP/candidate-descriptor.err"; then + image_ref="$IMAGE:release-candidate-$GITHUB_SHA" + if oras manifest fetch --descriptor "$image_ref" > "$RUNNER_TEMP/candidate-image-descriptor.json" 2> "$RUNNER_TEMP/candidate-image-descriptor.err"; then + image_digest=$(jq -er '.digest | select(test("^sha256:[0-9a-f]{64}$"))' "$RUNNER_TEMP/candidate-image-descriptor.json") + printf 'exists=false\nimage_exists=true\nimage_digest=%s\n' "$image_digest" >> "$GITHUB_OUTPUT" + elif grep -Eiq 'manifest unknown|not found|404' "$RUNNER_TEMP/candidate-image-descriptor.err"; then + printf 'exists=false\nimage_exists=false\n' >> "$GITHUB_OUTPUT" + else + cat "$RUNNER_TEMP/candidate-image-descriptor.err" >&2 + exit 1 + fi + else + cat "$RUNNER_TEMP/candidate-descriptor.err" >&2 + exit 1 + fi + + - name: Build release candidate once + if: github.ref == 'refs/heads/main' && steps.candidate_state.outputs.exists == 'false' && steps.candidate_state.outputs.image_exists == 'false' + id: candidate_build uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 with: context: . platforms: linux/amd64,linux/arm64 - push: false - outputs: type=oci,dest=/tmp/agent-os-release.oci.tar - tags: ${{ steps.metadata.outputs.tags }} - labels: ${{ steps.metadata.outputs.labels }} + push: true + tags: ${{ env.IMAGE }}:release-candidate-${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max provenance: mode=max sbom: true build-args: | - AGENT_OS_SOURCE_COMMIT=${{ steps.source.outputs.commit }} - AGENT_OS_SOURCE_TREE=${{ steps.source.outputs.tree }} - AGENT_OS_SOURCE_BRANCH=${{ steps.source.outputs.branch }} - AGENT_OS_SOURCE_ORIGIN=${{ steps.source.outputs.origin }} - AGENT_OS_SOURCE_MODE=${{ steps.source.outputs.mode }} - AGENT_OS_SOURCE_REF=${{ steps.source.outputs.ref }} + AGENT_OS_SOURCE_COMMIT=${{ steps.candidate_source.outputs.commit }} + AGENT_OS_SOURCE_TREE=${{ steps.candidate_source.outputs.tree }} + AGENT_OS_SOURCE_BRANCH=${{ steps.candidate_source.outputs.branch }} + AGENT_OS_SOURCE_ORIGIN=${{ steps.candidate_source.outputs.origin }} + AGENT_OS_SOURCE_MODE=${{ steps.candidate_source.outputs.mode }} + AGENT_OS_SOURCE_REF=${{ steps.candidate_source.outputs.ref }} - - name: Verify record-bound release artifact - if: startsWith(github.ref, 'refs/tags/') - id: verify + - name: Create or verify immutable release candidate record + if: github.ref == 'refs/heads/main' + id: candidate_record env: - BUILD_DIGEST: ${{ steps.build.outputs.digest }} + CANDIDATE_BUILD_DIGEST: ${{ steps.candidate_build.outputs.digest }} + CANDIDATE_EXISTS: ${{ steps.candidate_state.outputs.exists }} + EXISTING_CANDIDATE_IMAGE_DIGEST: ${{ steps.candidate_state.outputs.image_digest }} run: | set -eu - mkdir "$RUNNER_TEMP/release-oci" - tar -xf /tmp/agent-os-release.oci.tar -C "$RUNNER_TEMP/release-oci" - root_digest=$(jq -er '.manifests | select(length == 1) | .[0].digest' "$RUNNER_TEMP/release-oci/index.json") - [ "$root_digest" = "$BUILD_DIGEST" ] - root_blob="$RUNNER_TEMP/release-oci/blobs/sha256/${root_digest#sha256:}" - [ "sha256:$(sha256sum "$root_blob" | awk '{print $1}')" = "$root_digest" ] + candidate_ref="$IMAGE:candidate-record-$GITHUB_SHA" artifact_set_sha() { predicate=$1 output=$2 : > "$output" - jq -r '.manifests[] | select(.annotations["vnd.docker.reference.type"] == "attestation-manifest") | .digest' "$root_blob" | while IFS= read -r manifest_digest; do - manifest="$RUNNER_TEMP/release-oci/blobs/sha256/${manifest_digest#sha256:}" - [ "sha256:$(sha256sum "$manifest" | awk '{print $1}')" = "$manifest_digest" ] - jq -r --arg predicate "$predicate" '.layers[] | select(.annotations["in-toto.io/predicate-type"] == $predicate) | .digest' "$manifest" >> "$output" + jq -r '.manifests[] | select(.annotations["vnd.docker.reference.type"] == "attestation-manifest") | .digest' "$RUNNER_TEMP/candidate-index.json" | while IFS= read -r manifest_digest; do + docker buildx imagetools inspect --raw "$IMAGE@$manifest_digest" > "$RUNNER_TEMP/candidate-attestation.json" + jq -r --arg predicate "$predicate" '.layers[] | select(.annotations["in-toto.io/predicate-type"] == $predicate) | .digest' "$RUNNER_TEMP/candidate-attestation.json" >> "$output" done LC_ALL=C sort -u -o "$output" "$output" [ "$(wc -l < "$output" | tr -d ' ')" -eq 2 ] sha256sum "$output" | awk '{print $1}' } - actual_image_digest=$root_digest - actual_sbom_sha256=$(artifact_set_sha https://spdx.dev/Document "$RUNNER_TEMP/sbom-digests") - actual_provenance_sha256=$(artifact_set_sha https://slsa.dev/provenance/v0.2 "$RUNNER_TEMP/provenance-digests") - [ "$actual_image_digest" = "$(jq -r .image_digest "$RUNNER_TEMP/release-record.json")" ] - [ "$actual_sbom_sha256" = "$(jq -r .sbom_sha256 "$RUNNER_TEMP/release-record.json")" ] - [ "$actual_provenance_sha256" = "$(jq -r .provenance_sha256 "$RUNNER_TEMP/release-record.json")" ] - printf 'image_digest=%s\nsbom_sha256=%s\nprovenance_sha256=%s\n' \ - "$actual_image_digest" "$actual_sbom_sha256" "$actual_provenance_sha256" >> "$GITHUB_OUTPUT" + verify_candidate_provenance() { + jq -r '.manifests[] | select(.annotations["vnd.docker.reference.type"] == "attestation-manifest") | .digest' \ + "$RUNNER_TEMP/candidate-index.json" > "$RUNNER_TEMP/candidate-attestation-manifests" + [ "$(wc -l < "$RUNNER_TEMP/candidate-attestation-manifests" | tr -d ' ')" -eq 2 ] + while IFS= read -r manifest_digest; do + manifest_file="$RUNNER_TEMP/candidate-attestation-${manifest_digest#sha256:}.json" + provenance_file="$RUNNER_TEMP/candidate-provenance-${manifest_digest#sha256:}.json" + docker buildx imagetools inspect --raw "$IMAGE@$manifest_digest" > "$manifest_file" + subject_digest=$(jq -er --arg manifest "$manifest_digest" ' + .manifests[] | select(.digest == $manifest) + | .annotations["vnd.docker.reference.digest"] | select(test("^sha256:[0-9a-f]{64}$"))' \ + "$RUNNER_TEMP/candidate-index.json") + printf '%s' "$platform_manifests" | jq -e --arg subject "$subject_digest" 'any(.[]; .digest == $subject)' >/dev/null + provenance_digest=$(jq -er '[.layers[] | select(.annotations["in-toto.io/predicate-type"] == "https://slsa.dev/provenance/v0.2")] | select(length == 1) | .[0].digest' "$manifest_file") + oras blob fetch --output "$provenance_file" "$IMAGE@$provenance_digest" + jq -e --arg subject "${subject_digest#sha256:}" \ + --arg commit '${{ steps.candidate_source.outputs.commit }}' \ + --arg tree '${{ steps.candidate_source.outputs.tree }}' \ + --arg branch '${{ steps.candidate_source.outputs.branch }}' \ + --arg origin '${{ steps.candidate_source.outputs.origin }}' \ + --arg mode '${{ steps.candidate_source.outputs.mode }}' \ + --arg ref '${{ steps.candidate_source.outputs.ref }}' ' + .predicateType == "https://slsa.dev/provenance/v0.2" and + (.subject | length == 1) and .subject[0].digest.sha256 == $subject and + .predicate.buildType == "https://mobyproject.org/buildkit@v1" and + .predicate.invocation.configSource.entryPoint == "Dockerfile" and + .predicate.metadata.completeness.parameters == true and + .predicate.invocation.parameters.args["build-arg:AGENT_OS_SOURCE_COMMIT"] == $commit and + .predicate.invocation.parameters.args["build-arg:AGENT_OS_SOURCE_TREE"] == $tree and + .predicate.invocation.parameters.args["build-arg:AGENT_OS_SOURCE_BRANCH"] == $branch and + .predicate.invocation.parameters.args["build-arg:AGENT_OS_SOURCE_ORIGIN"] == $origin and + .predicate.invocation.parameters.args["build-arg:AGENT_OS_SOURCE_MODE"] == $mode and + .predicate.invocation.parameters.args["build-arg:AGENT_OS_SOURCE_REF"] == $ref' \ + "$provenance_file" >/dev/null || { + echo "::error::release candidate image provenance conflicts with the exact protected-main source" + exit 1 + } + done < "$RUNNER_TEMP/candidate-attestation-manifests" + } + if [ "$CANDIDATE_EXISTS" = true ]; then + actual_image_digest=$(jq -er '.image_digest' "$RUNNER_TEMP/candidate-record.json") + elif [ -n "$EXISTING_CANDIDATE_IMAGE_DIGEST" ]; then + actual_image_digest=$EXISTING_CANDIDATE_IMAGE_DIGEST + else + actual_image_digest=$CANDIDATE_BUILD_DIGEST + fi + [[ "$actual_image_digest" =~ ^sha256:[0-9a-f]{64}$ ]] + docker buildx imagetools inspect --raw "$IMAGE@$actual_image_digest" > "$RUNNER_TEMP/candidate-index.json" + [ "sha256:$(sha256sum "$RUNNER_TEMP/candidate-index.json" | awk '{print $1}')" = "$actual_image_digest" ] + platform_manifests=$(jq -cS '[.manifests[] | select(.platform.os == "linux" and (.platform.architecture == "amd64" or .platform.architecture == "arm64")) | {platform:(.platform.os + "/" + .platform.architecture),digest,mediaType,size}] | sort_by(.platform)' "$RUNNER_TEMP/candidate-index.json") + [ "$(printf '%s' "$platform_manifests" | jq 'length')" -eq 2 ] + verify_candidate_provenance + platform_manifests_sha256=$(printf '%s' "$platform_manifests" | sha256sum | awk '{print $1}') + buildkit_outputs=$(jq -cS '[.manifests[] | {digest,mediaType,size,platform,annotations}] | sort_by(.digest)' "$RUNNER_TEMP/candidate-index.json") + buildkit_outputs_sha256=$(printf '%s' "$buildkit_outputs" | sha256sum | awk '{print $1}') + actual_sbom_sha256=$(artifact_set_sha https://spdx.dev/Document "$RUNNER_TEMP/candidate-sbom-digests") + actual_provenance_sha256=$(artifact_set_sha https://slsa.dev/provenance/v0.2 "$RUNNER_TEMP/candidate-provenance-digests") + if [ "$CANDIDATE_EXISTS" = false ]; then + jq -cnS \ + --arg commit '${{ steps.candidate_source.outputs.commit }}' \ + --arg tree '${{ steps.candidate_source.outputs.tree }}' \ + --arg sourceArchive '${{ steps.candidate_source.outputs.source_sha256 }}' \ + --arg bootstrapArchive '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' \ + --arg package "$(sha256sum tools/agent-os/packages/firstmate/package.k | awk '{print $1}')" \ + --arg schema "$(sha256sum tools/agent-os/packages/firstmate/inputs.example.yaml | awk '{print $1}')" \ + --arg quickstart "$(sha256sum docs/kubernetes.md | awk '{print $1}')" \ + --arg repository "$IMAGE" --arg image "$actual_image_digest" \ + --argjson platforms "$platform_manifests" --arg platformsSha "$platform_manifests_sha256" \ + --arg sbom "$actual_sbom_sha256" --arg provenance "$actual_provenance_sha256" \ + --arg buildkitOutputs "$buildkit_outputs_sha256" \ + '{schema_version:1,source_mode:"candidate",commit:$commit,tree:$tree,source_archive_sha256:$sourceArchive,bootstrap_archive_sha256:$bootstrapArchive,package_sha256:$package,schema_sha256:$schema,quickstart_sha256:$quickstart,image_repository:$repository,image_digest:$image,platform_manifests:$platforms,platform_manifests_sha256:$platformsSha,sbom_sha256:$sbom,provenance_sha256:$provenance,buildkit_outputs_sha256:$buildkitOutputs}' \ + > "$RUNNER_TEMP/candidate-record.json" + oras push --artifact-type application/vnd.akua.agent-os.release-candidate.v1 \ + --format json "$candidate_ref" \ + "$RUNNER_TEMP/candidate-record.json:application/vnd.akua.agent-os.release-candidate.v1+json" \ + > "$RUNNER_TEMP/candidate-push.json" + candidate_record_digest=$(jq -er '.digest | select(test("^sha256:[0-9a-f]{64}$"))' "$RUNNER_TEMP/candidate-push.json") + else + candidate_record_digest='${{ steps.candidate_state.outputs.candidate_record_digest }}' + fi + jq -e \ + --arg commit '${{ steps.candidate_source.outputs.commit }}' --arg tree '${{ steps.candidate_source.outputs.tree }}' \ + --arg sourceArchive '${{ steps.candidate_source.outputs.source_sha256 }}' \ + --arg bootstrapArchive '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' \ + --arg package "$(sha256sum tools/agent-os/packages/firstmate/package.k | awk '{print $1}')" \ + --arg schema "$(sha256sum tools/agent-os/packages/firstmate/inputs.example.yaml | awk '{print $1}')" \ + --arg quickstart "$(sha256sum docs/kubernetes.md | awk '{print $1}')" \ + --arg repository "$IMAGE" --arg image "$actual_image_digest" \ + --argjson platforms "$platform_manifests" --arg platformsSha "$platform_manifests_sha256" \ + --arg sbom "$actual_sbom_sha256" --arg provenance "$actual_provenance_sha256" \ + --arg buildkitOutputs "$buildkit_outputs_sha256" --arg existing "$CANDIDATE_EXISTS" \ + '.schema_version == 1 and .source_mode == "candidate" and .commit == $commit and .tree == $tree and .source_archive_sha256 == $sourceArchive and (($existing == "false" and .bootstrap_archive_sha256 == $bootstrapArchive) or ($existing == "true" and (.bootstrap_archive_sha256 | test("^[0-9a-f]{64}$")))) and .package_sha256 == $package and .schema_sha256 == $schema and .quickstart_sha256 == $quickstart and .image_repository == $repository and .image_digest == $image and .platform_manifests == $platforms and .platform_manifests_sha256 == $platformsSha and .sbom_sha256 == $sbom and .provenance_sha256 == $provenance and .buildkit_outputs_sha256 == $buildkitOutputs' \ + "$RUNNER_TEMP/candidate-record.json" >/dev/null || { + echo "::error::candidate record coordinate already contains different artifacts" + exit 1 + } + oras manifest fetch --descriptor "$candidate_ref" > "$RUNNER_TEMP/candidate-readback-descriptor.json" + [ "$(jq -r .digest "$RUNNER_TEMP/candidate-readback-descriptor.json")" = "$candidate_record_digest" ] + rm -rf "$RUNNER_TEMP/candidate-readback" + mkdir "$RUNNER_TEMP/candidate-readback" + oras pull --output "$RUNNER_TEMP/candidate-readback" "$IMAGE@$candidate_record_digest" + cmp "$RUNNER_TEMP/candidate-record.json" "$RUNNER_TEMP/candidate-readback/candidate-record.json" + printf 'candidate_record_digest=%s\nimage_digest=%s\n' "$candidate_record_digest" "$actual_image_digest" >> "$GITHUB_OUTPUT" - - name: Retain exact verified release artifact - if: startsWith(github.ref, 'refs/tags/') + - name: Retain candidate record for release preparation + if: github.ref == 'refs/heads/main' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: - name: agent-os-release-oci - path: /tmp/agent-os-release.oci.tar + name: agent-os-candidate-record-${{ github.sha }} + path: ${{ runner.temp }}/candidate-record.json if-no-files-found: error - retention-days: 1 - compression-level: 0 + retention-days: 90 - publish: - if: github.event_name == 'push' - needs: [behavior, provenance, validate, release_artifacts] - name: Publish linux/amd64 and linux/arm64 - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 - with: - fetch-depth: 0 - persist-credentials: false - - - name: Prepare exact source bundle - id: source + - name: Prepare protected-main source + if: github.ref == 'refs/heads/main' + id: main_source run: | set -eu - if [ "${GITHUB_REF#refs/tags/}" != "$GITHUB_REF" ]; then - record_commit=$(git for-each-ref --format='%(contents)' "$GITHUB_REF" | sed -n 's/^release-record-commit=\([0-9a-f]\{40\}\)$/\1/p') - AGENT_OS_SOURCE_MODE=release AGENT_OS_SOURCE_RELEASE_TAG="${GITHUB_REF#refs/tags/}" \ - AGENT_OS_RELEASE_RECORD_COMMIT="$record_commit" bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" - else - bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" - fi - cat "$RUNNER_TEMP/source-metadata" >> "$GITHUB_OUTPUT" + bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/main-source-metadata" + cat "$RUNNER_TEMP/main-source-metadata" >> "$GITHUB_OUTPUT" - - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 - - - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f - - - if: startsWith(github.ref, 'refs/tags/') - uses: oras-project/setup-oras@22ce207df3b08e061f537244349aac6ae1d214f6 - with: - version: 1.3.0 - - - name: Log in to GHCR - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Compute image tags + - name: Compute main image tags + if: github.ref == 'refs/heads/main' id: metadata uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 with: @@ -299,14 +412,6 @@ jobs: tags: | type=sha,format=long type=raw,value=latest,enable={{is_default_branch}} - type=ref,event=tag - - - name: Download exact verified release artifact - if: startsWith(github.ref, 'refs/tags/') - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 - with: - name: agent-os-release-oci - path: ${{ runner.temp }}/agent-os-release - name: Build and publish if: github.ref == 'refs/heads/main' @@ -323,75 +428,85 @@ jobs: provenance: mode=max sbom: true build-args: | - AGENT_OS_SOURCE_COMMIT=${{ steps.source.outputs.commit }} - AGENT_OS_SOURCE_TREE=${{ steps.source.outputs.tree }} - AGENT_OS_SOURCE_BRANCH=${{ steps.source.outputs.branch }} - AGENT_OS_SOURCE_ORIGIN=${{ steps.source.outputs.origin }} - AGENT_OS_SOURCE_MODE=${{ steps.source.outputs.mode }} - AGENT_OS_SOURCE_REF=${{ steps.source.outputs.ref }} + AGENT_OS_SOURCE_COMMIT=${{ steps.main_source.outputs.commit }} + AGENT_OS_SOURCE_TREE=${{ steps.main_source.outputs.tree }} + AGENT_OS_SOURCE_BRANCH=${{ steps.main_source.outputs.branch }} + AGENT_OS_SOURCE_ORIGIN=${{ steps.main_source.outputs.origin }} + AGENT_OS_SOURCE_MODE=${{ steps.main_source.outputs.mode }} + AGENT_OS_SOURCE_REF=${{ steps.main_source.outputs.ref }} - - name: Publish exact verified release artifact + - name: Promote exact verified release candidate if: startsWith(github.ref, 'refs/tags/') env: - RELEASE_DIGEST: ${{ needs.release_artifacts.outputs.image_digest }} - RELEASE_TAGS: ${{ steps.metadata.outputs.tags }} + RELEASE_COMMIT: ${{ steps.release_source.outputs.commit }} run: | set -eu - mkdir "$RUNNER_TEMP/release-oci" - tar -xf "$RUNNER_TEMP/agent-os-release/agent-os-release.oci.tar" -C "$RUNNER_TEMP/release-oci" - root_digest=$(jq -er '.manifests | select(length == 1) | .[0].digest' "$RUNNER_TEMP/release-oci/index.json") - [ "$root_digest" = "$RELEASE_DIGEST" ] - tag_names=() - while IFS= read -r tag; do - [ -n "$tag" ] || continue - [[ "$tag" == "$IMAGE:"* ]] || exit 1 - tag_name=${tag#"$IMAGE:"} - [[ -n "$tag_name" && "$tag_name" != *','* ]] || exit 1 - tag_names+=("$tag_name") - done <<< "$RELEASE_TAGS" - [ "${#tag_names[@]}" -ge 2 ] - destination=$(IFS=,; printf '%s:%s' "$IMAGE" "${tag_names[*]}") - oras cp --from-oci-layout \ - "$RUNNER_TEMP/agent-os-release/agent-os-release.oci.tar@$RELEASE_DIGEST" "$destination" + release_tag=${GITHUB_REF#refs/tags/} + candidate_ref="$IMAGE:candidate-record-$RELEASE_COMMIT" + oras manifest fetch --descriptor "$candidate_ref" > "$RUNNER_TEMP/release-candidate-descriptor.json" + candidate_record_digest=$(jq -er '.candidate_record_digest' "$RUNNER_TEMP/release-record.json") + [ "$(jq -r .digest "$RUNNER_TEMP/release-candidate-descriptor.json")" = "$candidate_record_digest" ] + mkdir "$RUNNER_TEMP/release-candidate" + oras pull --output "$RUNNER_TEMP/release-candidate" "$IMAGE@$candidate_record_digest" + jq -cS '{schema_version,source_mode,commit,tree,source_archive_sha256,bootstrap_archive_sha256,package_sha256,schema_sha256,quickstart_sha256,image_repository,image_digest,platform_manifests,platform_manifests_sha256,sbom_sha256,provenance_sha256,buildkit_outputs_sha256}' \ + "$RUNNER_TEMP/release-record.json" > "$RUNNER_TEMP/expected-candidate-record.json" + jq -cS . "$RUNNER_TEMP/release-candidate/candidate-record.json" > "$RUNNER_TEMP/actual-candidate-record.json" + cmp "$RUNNER_TEMP/expected-candidate-record.json" "$RUNNER_TEMP/actual-candidate-record.json" + RELEASE_DIGEST=$(jq -er '.image_digest' "$RUNNER_TEMP/release-record.json") + docker buildx imagetools inspect --raw "$IMAGE@$RELEASE_DIGEST" > "$RUNNER_TEMP/published-index.json" + [ "sha256:$(sha256sum "$RUNNER_TEMP/published-index.json" | awk '{print $1}')" = "$RELEASE_DIGEST" ] + actual_platforms=$(jq -cS '[.manifests[] | select(.platform.os == "linux" and (.platform.architecture == "amd64" or .platform.architecture == "arm64")) | {platform:(.platform.os + "/" + .platform.architecture),digest,mediaType,size}] | sort_by(.platform)' "$RUNNER_TEMP/published-index.json") + [ "$actual_platforms" = "$(jq -cS '.platform_manifests' "$RUNNER_TEMP/release-record.json")" ] + [ "$(printf '%s' "$actual_platforms" | sha256sum | awk '{print $1}')" = "$(jq -r .platform_manifests_sha256 "$RUNNER_TEMP/release-record.json")" ] + actual_buildkit_outputs=$(jq -cS '[.manifests[] | {digest,mediaType,size,platform,annotations}] | sort_by(.digest)' "$RUNNER_TEMP/published-index.json") + [ "$(printf '%s' "$actual_buildkit_outputs" | sha256sum | awk '{print $1}')" = "$(jq -r .buildkit_outputs_sha256 "$RUNNER_TEMP/release-record.json")" ] + published_artifact_set_sha() { + predicate=$1 + output=$2 + : > "$output" + jq -r '.manifests[] | select(.annotations["vnd.docker.reference.type"] == "attestation-manifest") | .digest' "$RUNNER_TEMP/published-index.json" | while IFS= read -r manifest_digest; do + docker buildx imagetools inspect --raw "$IMAGE@$manifest_digest" > "$RUNNER_TEMP/published-manifest.json" + jq -r --arg predicate "$predicate" '.layers[] | select(.annotations["in-toto.io/predicate-type"] == $predicate) | .digest' "$RUNNER_TEMP/published-manifest.json" >> "$output" + done + LC_ALL=C sort -u -o "$output" "$output" + [ "$(wc -l < "$output" | tr -d ' ')" -eq 2 ] + sha256sum "$output" | awk '{print $1}' + } + actual_sbom_sha256=$(published_artifact_set_sha https://spdx.dev/Document "$RUNNER_TEMP/published-sbom-digests") + actual_provenance_sha256=$(published_artifact_set_sha https://slsa.dev/provenance/v0.2 "$RUNNER_TEMP/published-provenance-digests") + [ "$actual_sbom_sha256" = "$(jq -r .sbom_sha256 "$RUNNER_TEMP/release-record.json")" ] + [ "$actual_provenance_sha256" = "$(jq -r .provenance_sha256 "$RUNNER_TEMP/release-record.json")" ] + if oras manifest fetch --descriptor "$IMAGE:$release_tag" > "$RUNNER_TEMP/release-tag-descriptor.json" 2> "$RUNNER_TEMP/release-tag.err"; then + [ "$(jq -r .digest "$RUNNER_TEMP/release-tag-descriptor.json")" = "$RELEASE_DIGEST" ] || { + echo "::error::release tag coordinate already contains a different manifest" + exit 1 + } + elif grep -Eiq 'manifest unknown|not found|404' "$RUNNER_TEMP/release-tag.err"; then + oras tag "$IMAGE@$RELEASE_DIGEST" "$release_tag" + else + cat "$RUNNER_TEMP/release-tag.err" >&2 + exit 1 + fi + oras manifest fetch --descriptor "$IMAGE:$release_tag" > "$RUNNER_TEMP/release-tag-readback.json" + [ "$(jq -r .digest "$RUNNER_TEMP/release-tag-readback.json")" = "$RELEASE_DIGEST" ] + docker buildx imagetools inspect --raw "$IMAGE:$release_tag" > "$RUNNER_TEMP/published-tag-index.json" + [ "sha256:$(sha256sum "$RUNNER_TEMP/published-tag-index.json" | awk '{print $1}')" = "$RELEASE_DIGEST" ] + { + echo '### Agent OS release' + echo + echo "Digest: \`$RELEASE_DIGEST\`" + } >> "$GITHUB_STEP_SUMMARY" - - name: Verify and record published artifacts + - name: Record protected-main publication + if: github.ref == 'refs/heads/main' env: - BUILD_DIGEST: ${{ startsWith(github.ref, 'refs/tags/') && needs.release_artifacts.outputs.image_digest || steps.build.outputs.digest }} - RELEASE_TAGS: ${{ steps.metadata.outputs.tags }} + BUILD_DIGEST: ${{ steps.build.outputs.digest }} run: | set -eu - if [ "${GITHUB_REF#refs/tags/}" != "$GITHUB_REF" ]; then - record_commit=$(git for-each-ref --format='%(contents)' "$GITHUB_REF" | sed -n 's/^release-record-commit=\([0-9a-f]\{40\}\)$/\1/p') - git show "$record_commit:image/releases/${GITHUB_REF#refs/tags/}.json" > "$RUNNER_TEMP/release-record.json" - [ "$BUILD_DIGEST" = "${{ needs.release_artifacts.outputs.image_digest }}" ] - [ "$BUILD_DIGEST" = "$(jq -r .image_digest "$RUNNER_TEMP/release-record.json")" ] - while IFS= read -r tag; do - [ -n "$tag" ] || continue - docker buildx imagetools inspect --raw "$tag" > "$RUNNER_TEMP/published-tag-index.json" - [ "sha256:$(sha256sum "$RUNNER_TEMP/published-tag-index.json" | awk '{print $1}')" = "$BUILD_DIGEST" ] - done <<< "$RELEASE_TAGS" - docker buildx imagetools inspect --raw "$IMAGE@$BUILD_DIGEST" > "$RUNNER_TEMP/published-index.json" - published_artifact_set_sha() { - predicate=$1 - output=$2 - : > "$output" - jq -r '.manifests[] | select(.annotations["vnd.docker.reference.type"] == "attestation-manifest") | .digest' "$RUNNER_TEMP/published-index.json" | while IFS= read -r manifest_digest; do - docker buildx imagetools inspect --raw "$IMAGE@$manifest_digest" > "$RUNNER_TEMP/published-manifest.json" - jq -r --arg predicate "$predicate" '.layers[] | select(.annotations["in-toto.io/predicate-type"] == $predicate) | .digest' "$RUNNER_TEMP/published-manifest.json" >> "$output" - done - LC_ALL=C sort -u -o "$output" "$output" - [ "$(wc -l < "$output" | tr -d ' ')" -eq 2 ] - sha256sum "$output" | awk '{print $1}' - } - actual_sbom_sha256=$(published_artifact_set_sha https://spdx.dev/Document "$RUNNER_TEMP/published-sbom-digests") - actual_provenance_sha256=$(published_artifact_set_sha https://slsa.dev/provenance/v0.2 "$RUNNER_TEMP/published-provenance-digests") - [ "$actual_sbom_sha256" = "${{ needs.release_artifacts.outputs.sbom_sha256 }}" ] - [ "$actual_provenance_sha256" = "${{ needs.release_artifacts.outputs.provenance_sha256 }}" ] - [ "$actual_sbom_sha256" = "$(jq -r .sbom_sha256 "$RUNNER_TEMP/release-record.json")" ] - [ "$actual_provenance_sha256" = "$(jq -r .provenance_sha256 "$RUNNER_TEMP/release-record.json")" ] - fi { echo '### Agent OS image' echo echo "Digest: \`$BUILD_DIGEST\`" + echo "Release candidate: \`${{ steps.candidate_record.outputs.image_digest }}\`" + echo "Candidate record: \`${{ steps.candidate_record.outputs.candidate_record_digest }}\`" } >> "$GITHUB_STEP_SUMMARY" diff --git a/Dockerfile b/Dockerfile index f5fc845dd..cbe7d642b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN set -eu; \ test -n "$AGENT_OS_SOURCE_TREE"; \ test "$AGENT_OS_SOURCE_BRANCH" = main; \ test "$AGENT_OS_SOURCE_ORIGIN" = https://github.com/akua-dev/agent-os.git; \ - case "$AGENT_OS_SOURCE_MODE" in main|event|release) ;; *) exit 1 ;; esac; \ + case "$AGENT_OS_SOURCE_MODE" in main|candidate|event|release) ;; *) exit 1 ;; esac; \ test -n "$AGENT_OS_SOURCE_REF"; \ grep -Fx "mode=$AGENT_OS_SOURCE_MODE" /tmp/agent-os-source.attestation; \ grep -Fx "commit=$AGENT_OS_SOURCE_COMMIT" /tmp/agent-os-source.attestation; \ diff --git a/bin/agent-os-akua-auth.sh b/bin/agent-os-akua-auth.sh index 2c1d33750..c83ca2f8f 100755 --- a/bin/agent-os-akua-auth.sh +++ b/bin/agent-os-akua-auth.sh @@ -153,6 +153,7 @@ verify_overlay() { ([.spec.template.spec.volumes[] | select(.name == "akua-auth" and .secret.secretName == $secret and .secret.defaultMode == 256)] | length) == 1 else (.metadata.annotations["agent-os.dev/akua-auth-secret"] // "") == "" and + (.metadata.annotations["agent-os.dev/akua-auth-rejected-record"] // "") == "" and ([.spec.template.spec.containers[] | select(.name == "firstmate") | .env[]? | select(.name == "AKUA_AUTH_HEADER_FILE")] | length) == 0 and ([.spec.template.spec.containers[] | select(.name == "firstmate") | .volumeMounts[]? | select(.name == "akua-auth")] | length) == 0 and ([.spec.template.spec.volumes[]? | select(.name == "akua-auth")] | length) == 0 @@ -298,5 +299,5 @@ if [ "$COMMAND" = grant ]; then fail_grant_closed "grant verification failed" fi else - verify_overlay absent "$STATE_UID" + verify_overlay absent "$STATE_UID" 0 fi diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh index 9ad723ea8..996a20f1d 100755 --- a/bin/agent-os-container-entrypoint.sh +++ b/bin/agent-os-container-entrypoint.sh @@ -15,18 +15,24 @@ SOURCE_REF=$(cat /opt/agent-os-source.ref) case "$SOURCE_BRANCH" in ''|*[!A-Za-z0-9._/-]*|/*|*/|*..*) echo "error: image source branch provenance is invalid" >&2; exit 2 ;; esac [ "$SOURCE_ORIGIN" = https://github.com/akua-dev/agent-os.git ] || { echo "error: image source origin is invalid" >&2; exit 2; } case "$SOURCE_MODE" in - main) [ "$SOURCE_REF" = refs/heads/main ] || exit 2; TRUSTED_REF=refs/remotes/agent-os-verified/main ;; - release) [[ "$SOURCE_REF" =~ ^refs/tags/v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] || exit 2; TRUSTED_REF=refs/remotes/agent-os-verified/release ;; + main) [ "$SOURCE_REF" = refs/heads/main ] || exit 2; TRUSTED_SOURCE_REF=$SOURCE_REF; TRUSTED_REF=refs/remotes/agent-os-verified/main ;; + candidate) + [ "$SOURCE_REF" = "refs/agent-os/candidates/$SOURCE_COMMIT" ] || exit 2 + TRUSTED_SOURCE_REF=refs/heads/main + TRUSTED_REF=refs/remotes/agent-os-verified/main + ;; + release) [[ "$SOURCE_REF" =~ ^refs/tags/v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] || exit 2; TRUSTED_SOURCE_REF=$SOURCE_REF; TRUSTED_REF=refs/remotes/agent-os-verified/release ;; event) echo "error: pull-request validation images are not runnable" >&2; exit 2 ;; *) echo "error: image source mode is invalid" >&2; exit 2 ;; esac IMAGE_REF="refs/remotes/agent-os-image/$SOURCE_BRANCH" +GIT_BIN=$(command -v git) +case "$GIT_BIN" in /*) ;; *) echo "error: trusted Git executable is unavailable" >&2; exit 2 ;; esac trusted_git() { - env -u GIT_CONFIG -u GIT_CONFIG_PARAMETERS -u GIT_CONFIG_COUNT \ - -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy \ - -u GIT_SSH -u GIT_SSH_COMMAND GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null \ - GIT_TERMINAL_PROMPT=0 git -c credential.helper= -c core.hooksPath=/dev/null \ + env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 \ + "$GIT_BIN" -c credential.helper= -c core.hooksPath=/dev/null \ -c http.proxy= -c https.proxy= "$@" } @@ -136,7 +142,7 @@ fi } validate_git_config trusted_git -C "$FM_ROOT" fetch --no-tags --prune "$SOURCE_ORIGIN" \ - "$SOURCE_REF:$TRUSTED_REF" || { + "$TRUSTED_SOURCE_REF:$TRUSTED_REF" || { echo "error: fresh trusted source provenance is unavailable" >&2 exit 3 } @@ -152,14 +158,21 @@ if [ "$SOURCE_MODE" = main ]; then trusted_git -C "$FM_ROOT" merge --ff-only "$TRUSTED_REF" else [ "$(trusted_git -C "$FM_ROOT" rev-parse HEAD)" = "$SOURCE_COMMIT" ] || { - echo "error: release FM_ROOT differs from its immutable release commit" >&2 + echo "error: immutable FM_ROOT differs from its verified source commit" >&2 + exit 2 + } +fi +if [ "$SOURCE_MODE" = main ]; then + [ "$(trusted_git -C "$FM_ROOT" rev-parse HEAD)" = "$(trusted_git -C "$FM_ROOT" rev-parse "$TRUSTED_REF^{commit}")" ] || { + echo "error: canonical FM_ROOT HEAD is not the exact fresh trusted source ref" >&2 + exit 2 + } +else + trusted_git -C "$FM_ROOT" merge-base --is-ancestor "$SOURCE_COMMIT" "$TRUSTED_REF" || { + echo "error: immutable image source is not reachable from the fresh trusted ref" >&2 exit 2 } fi -[ "$(trusted_git -C "$FM_ROOT" rev-parse HEAD)" = "$(trusted_git -C "$FM_ROOT" rev-parse "$TRUSTED_REF^{commit}")" ] || { - echo "error: canonical FM_ROOT HEAD is not the exact fresh trusted source ref" >&2 - exit 2 -} trusted_git -C "$FM_ROOT" update-ref "refs/remotes/origin/$SOURCE_BRANCH" "$TRUSTED_REF" [ -z "$(trusted_git -C "$FM_ROOT" status --porcelain)" ] || { echo "error: canonical FM_ROOT is not clean" >&2; exit 2; } [ -z "$(trusted_git -C "$FM_ROOT" ls-files -- config data projects state .no-mistakes)" ] || { diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh index 2f57061a6..d26fbac27 100755 --- a/bin/agent-os-kubernetes.sh +++ b/bin/agent-os-kubernetes.sh @@ -40,6 +40,8 @@ CONTROL_LOCK_RENEW_PID= CONTROL_LOCK_VALID_UNTIL= REMOVE_CONTROL_ACCESS=0 STATEFULSET_CAS_ATTEMPTED=0 +STATEFULSET_CAS_UID= +STATEFULSET_CAS_RV= LOCK_DURATION_SECONDS=${AGENT_OS_LOCK_DURATION_SECONDS:-300} LOCK_CLOCK_SKEW_SECONDS=${AGENT_OS_LOCK_CLOCK_SKEW_SECONDS:-5} LOCK_ACQUIRE_SECONDS=${AGENT_OS_LOCK_ACQUIRE_SECONDS:-30} @@ -60,10 +62,14 @@ done cleanup() { local status=$? trap - EXIT - if ! release_primary_locks && [ "$status" -eq 0 ]; then - status=3 + if [ "$REMOVE_CONTROL_ACCESS" -eq 1 ]; then + if delete_control_lock_access; then + REMOVE_CONTROL_ACCESS=0 + elif [ "$status" -eq 0 ]; then + status=3 + fi fi - if [ "$REMOVE_CONTROL_ACCESS" -eq 1 ] && ! delete_control_lock_access && [ "$status" -eq 0 ]; then + if ! release_primary_locks && [ "$status" -eq 0 ]; then status=3 fi rm -rf "$OUT" "$RENDER_INPUTS" @@ -683,7 +689,7 @@ require_no_active_rollback_checkpoint() { } verified_akua_overlay_secret() { - local expected_supplied=0 expected_record='' state secret record current + local expected_supplied=0 expected_record='' expected_uid=${2:-} state secret record current if [ "$#" -gt 0 ]; then expected_supplied=1 expected_record=$1 @@ -694,8 +700,9 @@ verified_akua_overlay_secret() { fi state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get statefulset agent-os-firstmate -o json) || return 3 - secret=$(printf '%s' "$state" | jq -er --arg installation "$INSTALLATION_ID" ' + secret=$(printf '%s' "$state" | jq -er --arg installation "$INSTALLATION_ID" --arg uid "$expected_uid" ' select(.metadata.name == "agent-os-firstmate") + | select($uid == "" or .metadata.uid == $uid) | select(.metadata.labels["app.kubernetes.io/managed-by"] == "agent-os") | select(.metadata.annotations["agent-os.dev/installation-id"] == $installation) | (.metadata.annotations["agent-os.dev/akua-auth-secret"] // "") as $declared @@ -733,11 +740,13 @@ verified_akua_overlay_secret() { } reconcile_failed_akua_upgrade() { - local expected=$1 state uid rv rejected patch + local expected=$1 expected_uid=${2:-$STATEFULSET_CAS_UID} state uid rv rejected patch + [ -n "$expected_uid" ] || return 3 state=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get statefulset agent-os-firstmate -o json) || return 3 - uid=$(printf '%s' "$state" | jq -er --arg installation "$INSTALLATION_ID" ' + uid=$(printf '%s' "$state" | jq -er --arg installation "$INSTALLATION_ID" --arg uid "$expected_uid" ' select(.metadata.name == "agent-os-firstmate") + | select(.metadata.uid == $uid) | select(.metadata.labels["app.kubernetes.io/managed-by"] == "agent-os") | select(.metadata.annotations["agent-os.dev/installation-id"] == $installation) | .metadata.uid') || return 3 @@ -748,7 +757,7 @@ reconcile_failed_akua_upgrade() { spec:{template:{spec:{containers:[{name:"firstmate",env:[{name:"AKUA_AUTH_HEADER_FILE","$patch":"delete"}],volumeMounts:[{mountPath:"/var/run/secrets/agent-os/akua","$patch":"delete"}]}],volumes:[{name:"akua-auth","$patch":"delete"}]}}}}') "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" patch statefulset agent-os-firstmate --type=strategic -p "$patch" >/dev/null || return 3 "$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" rollout status statefulset/agent-os-firstmate --timeout=180s || return 3 - verified_akua_overlay_secret absent >/dev/null || return 3 + verified_akua_overlay_secret absent "$expected_uid" >/dev/null || return 3 printf 'incomplete: upgrade authorization failed closed expected-secret-uid=%s expected-secret-rv=%s\n' \ "$(printf '%s' "$expected" | cut -f3)" "$(printf '%s' "$expected" | cut -f4)" >&2 } @@ -1248,8 +1257,12 @@ mutate_rendered_resource() { uid=$(printf '%s' "$record" | cut -f4) rv=$(printf '%s' "$record" | cut -f5) [ -n "$uid" ] && [ -n "$rv" ] || { echo "error: $kind '$name' lacks CAS identity" >&2; exit 2; } + if [ "$kind" = StatefulSet ]; then + STATEFULSET_CAS_UID=$uid + STATEFULSET_CAS_RV=$rv + fi if [ "$kind" = StatefulSet ] && [ "$AKUA_OVERLAY_VERIFIED" -eq 1 ]; then - verified_akua_overlay_secret "$PRESERVED_AKUA_SECRET_RECORD" >/dev/null || { + verified_akua_overlay_secret "$PRESERVED_AKUA_SECRET_RECORD" "$STATEFULSET_CAS_UID" >/dev/null || { echo "incomplete: Akua authorization changed before StatefulSet CAS; no template mutation attempted" >&2 return 3 } @@ -1281,6 +1294,10 @@ mutate_rendered_resource() { echo "error: $kind '$name' UID changed during mutation" >&2 exit 3 fi + if [ "$kind" = StatefulSet ] && [ -z "$STATEFULSET_CAS_UID" ]; then + STATEFULSET_CAS_UID=$(printf '%s' "$current" | cut -f4) + STATEFULSET_CAS_RV=$(printf '%s' "$current" | cut -f5) + fi } apply_rendered() { @@ -1636,7 +1653,7 @@ case "$COMMAND" in ensure_control_lock_access if ! apply_rendered; then if [ "$STATEFULSET_CAS_ATTEMPTED" -eq 1 ]; then - reconcile_failed_akua_upgrade "$PRESERVED_AKUA_SECRET_RECORD" || { + reconcile_failed_akua_upgrade "$PRESERVED_AKUA_SECRET_RECORD" "$STATEFULSET_CAS_UID" || { echo "incomplete: post-CAS upgrade failure and fail-closed authorization reconciliation is unverified" >&2 report_partial_observation StatefulSet agent-os-firstmate namespaced report_partial_observation Pod agent-os-firstmate-0 namespaced @@ -1646,8 +1663,8 @@ case "$COMMAND" in fi exit 3 fi - if ! verified_akua_overlay_secret "$PRESERVED_AKUA_SECRET_RECORD" >/dev/null; then - reconcile_failed_akua_upgrade "$PRESERVED_AKUA_SECRET_RECORD" || { + if ! verified_akua_overlay_secret "$PRESERVED_AKUA_SECRET_RECORD" "$STATEFULSET_CAS_UID" >/dev/null; then + reconcile_failed_akua_upgrade "$PRESERVED_AKUA_SECRET_RECORD" "$STATEFULSET_CAS_UID" || { echo "incomplete: upgrade authorization changed and fail-closed reconciliation is unverified" >&2 report_partial_observation StatefulSet agent-os-firstmate namespaced report_partial_observation Pod agent-os-firstmate-0 namespaced @@ -2018,6 +2035,11 @@ case "$COMMAND" in delete_owned_resource namespaced ServiceAccount "$historical_service_account" 180 done <<< "$historical_service_accounts" delete_namespace_rbac + delete_control_lock_access || { + echo "incomplete: exact-owned control RBAC removal is unverified" >&2 + exit 3 + } + REMOVE_CONTROL_ACCESS=0 if [ "$DELETE_NAMESPACE" -eq 1 ]; then delete_owned_empty_namespace fi diff --git a/bin/agent-os-source-bundle.sh b/bin/agent-os-source-bundle.sh index 70c5c93a3..a5921c845 100755 --- a/bin/agent-os-source-bundle.sh +++ b/bin/agent-os-source-bundle.sh @@ -15,15 +15,64 @@ SOURCE_ARCHIVE="$OUTPUT_DIR/agent-os-source.tar" BOOTSTRAP_ARCHIVE="$OUTPUT_DIR/agent-os-bootstrap.tar" ATTESTATION="$OUTPUT_DIR/agent-os-source.attestation" TEMP="$OUTPUT_DIR/.agent-os-source.$$" +GIT_BIN=$(command -v git) +case "$GIT_BIN" in /*) ;; *) echo "error: trusted Git executable is unavailable" >&2; exit 2 ;; esac git_isolated() { - env -u GIT_CONFIG -u GIT_CONFIG_PARAMETERS -u GIT_CONFIG_COUNT \ - -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy \ - -u GIT_SSH -u GIT_SSH_COMMAND GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null \ - GIT_TERMINAL_PROMPT=0 git -c credential.helper= -c core.hooksPath=/dev/null \ + env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 \ + "$GIT_BIN" -c credential.helper= -c core.hooksPath=/dev/null \ -c http.proxy= -c https.proxy= "$@" } +canonical_github_origin() { + case "$1" in + https://github.com/akua-dev/agent-os|https://github.com/akua-dev/agent-os.git) + printf '%s' "$SOURCE_ORIGIN" + ;; + *) return 1 ;; + esac +} + +validate_source_git_config() { + local key value origin worktree_keys + while IFS= read -r key; do + [ -n "$key" ] || continue + value=$(git_isolated -C "$ROOT" config --local --no-includes --get-all "$key" || true) + [ -n "$value" ] && [ "$value" = "$(printf '%s\n' "$value" | head -n 1)" ] || { + echo "error: source repository Git config key is duplicated or empty" >&2 + exit 2 + } + case "$key" in + core.repositoryformatversion) [ "$value" = 0 ] ;; + core.filemode|core.ignorecase|core.precomposeunicode|core.logallrefupdates|core.bare) + [ "$value" = true ] || [ "$value" = false ] + ;; + extensions.worktreeconfig|receive.advertisepushoptions) + [ "$value" = true ] || [ "$value" = false ] + ;; + gc.auto) [ "$value" = 0 ] ;; + maintenance.auto) [ "$value" = false ] ;; + remote.origin.url) canonical_github_origin "$value" >/dev/null ;; + remote.origin.fetch) [ "$value" = '+refs/heads/*:refs/remotes/origin/*' ] ;; + remote.origin.tagopt) [ "$value" = --no-tags ] ;; + branch.main.remote) [ "$value" = origin ] ;; + branch.main.merge) [ "$value" = refs/heads/main ] ;; + *) echo "error: source repository Git config key is not allowlisted: $key" >&2; exit 2 ;; + esac || { echo "error: source repository Git config value is invalid: $key" >&2; exit 2; } + done < <(git_isolated -C "$ROOT" config --local --no-includes --name-only --list) + worktree_keys=$(git_isolated -C "$ROOT" config --worktree --no-includes --name-only --list 2>/dev/null || true) + [ -z "$worktree_keys" ] || { + echo "error: source repository worktree Git config is not allowed" >&2 + exit 2 + } + origin=$(git_isolated -C "$ROOT" config --local --no-includes --get remote.origin.url || true) + [ "$(canonical_github_origin "$origin" 2>/dev/null || true)" = "$SOURCE_ORIGIN" ] || { + echo "error: repository origin is not the exact trusted HTTPS origin" >&2 + exit 2 + } +} + sha256_file() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}'; else shasum -a 256 "$1" | awk '{print $1}'; fi } @@ -32,10 +81,7 @@ sha256_stream() { if command -v sha256sum >/dev/null 2>&1; then sha256sum | awk '{print $1}'; else shasum -a 256 | awk '{print $1}'; fi } -[ "$(git_isolated -C "$ROOT" remote get-url origin)" = "$SOURCE_ORIGIN" ] || { - echo "error: repository origin is not the exact trusted HTTPS origin" >&2 - exit 2 -} +validate_source_git_config [ -z "$(git_isolated -C "$ROOT" status --porcelain --untracked-files=all)" ] || { echo "error: exact-source image builds require a clean worktree" >&2 exit 2 @@ -48,7 +94,7 @@ trap 'rm -rf "$TEMP" "$SOURCE_ARCHIVE.tmp.$$" "$BOOTSTRAP_ARCHIVE.tmp.$$" "$ATTE git_isolated init --bare "$TEMP/bootstrap.git" >/dev/null case "$SOURCE_MODE" in - main) + main|candidate) git_isolated --git-dir="$TEMP/bootstrap.git" fetch --depth=1 --no-tags "$SOURCE_ORIGIN" \ refs/heads/main:refs/heads/main ;; @@ -84,8 +130,18 @@ case "$SOURCE_MODE" in jq -e --arg tag "$RELEASE_TAG" ' .tag == $tag and (.commit|test("^[0-9a-f]{40}$")) and (.tree|test("^[0-9a-f]{40}$")) and (.source_archive_sha256|test("^[0-9a-f]{64}$")) and (.package_sha256|test("^[0-9a-f]{64}$")) and + (.bootstrap_archive_sha256|test("^[0-9a-f]{64}$")) and (.schema_sha256|test("^[0-9a-f]{64}$")) and (.image_digest|test("^sha256:[0-9a-f]{64}$")) and (.sbom_sha256|test("^[0-9a-f]{64}$")) and (.provenance_sha256|test("^[0-9a-f]{64}$")) and + (.buildkit_outputs_sha256|test("^[0-9a-f]{64}$")) and + (.platform_manifests_sha256|test("^[0-9a-f]{64}$")) and + (.candidate_record_digest|test("^sha256:[0-9a-f]{64}$")) and + .source_mode == "candidate" and + .image_repository == "ghcr.io/akua-dev/agent-os" and + (.platform_manifests|type == "array" and length == 2) and + ([.platform_manifests[].platform] | sort == ["linux/amd64","linux/arm64"]) and + (all(.platform_manifests[]; (.digest|test("^sha256:[0-9a-f]{64}$")) and + (.mediaType|type == "string" and length > 0) and (.size|type == "number" and . > 0))) and (.quickstart_sha256|test("^[0-9a-f]{64}$")) and (.tag_ruleset_id|type == "number") and (.tag_ruleset_sha256|test("^[0-9a-f]{64}$"))' "$TEMP/release.json" >/dev/null || { echo "error: release record is incomplete or malformed" >&2 @@ -99,7 +155,10 @@ esac SOURCE_COMMIT=$(git_isolated --git-dir="$TEMP/bootstrap.git" rev-parse --verify refs/heads/main) SOURCE_TREE=$(git_isolated --git-dir="$TEMP/bootstrap.git" rev-parse --verify "$SOURCE_COMMIT^{tree}") -if [ "$SOURCE_MODE" = main ]; then +if [ "$SOURCE_MODE" = candidate ]; then + SOURCE_REF="refs/agent-os/candidates/$SOURCE_COMMIT" +fi +if [ "$SOURCE_MODE" = main ] || [ "$SOURCE_MODE" = candidate ]; then [ "$(git_isolated -C "$ROOT" rev-parse --verify HEAD)" = "$SOURCE_COMMIT" ] && \ [ "$(git_isolated -C "$ROOT" rev-parse --verify 'HEAD^{tree}')" = "$SOURCE_TREE" ] || { echo "error: local source does not exactly match the freshly fetched trusted source ref" >&2 diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 0b549feea..97e77c1f5 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -121,8 +121,14 @@ assert_grep 'fetch --depth=1 --no-tags' "$ROOT/bin/agent-os-source-bundle.sh" \ "source preparation must fetch an allowlisted remote ref freshly" assert_grep 'GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null' "$ROOT/bin/agent-os-source-bundle.sh" \ "source preparation must isolate trusted Git operations from ambient configuration" -assert_grep 'GIT_TERMINAL_PROMPT=0 git ' "$ROOT/bin/agent-os-source-bundle.sh" \ - "source preparation must invoke Git directly through env" +assert_grep 'env -i' "$ROOT/bin/agent-os-source-bundle.sh" \ + "source preparation must use an environment allowlist for trusted Git operations" +assert_grep 'validate_source_git_config' "$ROOT/bin/agent-os-source-bundle.sh" \ + "source preparation must reject executable repository-local Git configuration" +assert_grep 'canonical_github_origin' "$ROOT/bin/agent-os-source-bundle.sh" \ + "source preparation must canonicalize the trusted GitHub HTTPS origin" +assert_grep 'https://github.com/akua-dev/agent-os|https://github.com/akua-dev/agent-os.git)' "$ROOT/bin/agent-os-source-bundle.sh" \ + "source preparation must accept the checkout origin without a dot-git suffix" assert_grep 'repository origin is not the exact trusted HTTPS origin' "$ROOT/bin/agent-os-source-bundle.sh" \ "source preparation must require the exact trusted HTTPS origin" assert_grep 'AGENT_OS_SOURCE_MODE=event AGENT_OS_SOURCE_EVENT_COMMIT' "$IMAGE_WORKFLOW" \ @@ -137,8 +143,8 @@ assert_grep 'source_archive_sha256' "$ROOT/bin/agent-os-source-bundle.sh" \ "release records must bind the exact archived source" assert_grep 'GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null' "$ROOT/bin/agent-os-container-entrypoint.sh" \ "runtime provenance must isolate trusted Git operations" -assert_grep 'GIT_TERMINAL_PROMPT=0 git ' "$ROOT/bin/agent-os-container-entrypoint.sh" \ - "runtime provenance must invoke Git directly through env" +assert_grep 'env -i' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "runtime provenance must clear ambient Git paths, objects, helpers, and transport state" assert_grep 'canonical FM_ROOT Git config key is not allowlisted' "$ROOT/bin/agent-os-container-entrypoint.sh" \ "runtime provenance must use a strict repository-config allowlist" assert_no_grep 'config --file "$config" --no-includes --name-only --get-regexp' "$ROOT/bin/agent-os-container-entrypoint.sh" \ @@ -196,8 +202,12 @@ assert_grep ' validate:' "$IMAGE_WORKFLOW" \ "pull requests must use a distinct read-only validation job" assert_grep ' publish:' "$IMAGE_WORKFLOW" \ "push and tag publication must use a distinct privileged job" -assert_grep 'needs: [behavior, provenance, validate, release_artifacts]' "$IMAGE_WORKFLOW" \ +assert_grep 'needs: [behavior, provenance, validate]' "$IMAGE_WORKFLOW" \ "the packages-write job must require exact behavior, provenance, and image gates" +assert_grep 'cancel-in-progress: false' "$IMAGE_WORKFLOW" \ + "publication must not strand an unrecorded candidate by cancelling an in-flight main build" +assert_grep 'trusted_git()' "$IMAGE_WORKFLOW" \ + "workflow provenance reads must isolate trusted Git from ambient configuration and paths" assert_grep 'github.ref_protected' "$IMAGE_WORKFLOW" \ "main publication must require GitHub protected-ref provenance" assert_grep 'release tag ruleset differs from immutable record' "$IMAGE_WORKFLOW" \ @@ -208,20 +218,38 @@ assert_grep '.enforcement == "active" and .target == "tag"' "$IMAGE_WORKFLOW" \ "tag publication must require an active tag-targeting ruleset" assert_grep '.bypass_actors | length == 0' "$IMAGE_WORKFLOW" \ "tag publication must reject rulesets with bypass actors" -assert_grep 'release_artifacts:' "$IMAGE_WORKFLOW" \ - "release publication must have a read-only artifact verification gate" +assert_grep 'candidate-record-' "$IMAGE_WORKFLOW" \ + "protected main must retain one commit-addressed release candidate record" +assert_grep 'candidate record coordinate already contains different artifacts' "$IMAGE_WORKFLOW" \ + "candidate record creation must reject immutable coordinate conflicts" +assert_grep 'release candidate image provenance conflicts with the exact protected-main source' "$IMAGE_WORKFLOW" \ + "publication must reject an unrecorded candidate image whose provenance conflicts" +assert_grep 'oras blob fetch --output "$provenance_file" "$IMAGE@$provenance_digest"' "$IMAGE_WORKFLOW" \ + "candidate recovery must inspect the exact published BuildKit provenance bytes" +assert_grep 'source_mode:"candidate"' "$IMAGE_WORKFLOW" \ + "candidate records must bind the immutable runtime source policy" +assert_grep 'platform_manifests' "$IMAGE_WORKFLOW" \ + "candidate records must bind every published platform manifest" +assert_grep 'buildkit_outputs_sha256' "$IMAGE_WORKFLOW" \ + "candidate records must bind the exact BuildKit output descriptors" assert_grep 'actual_image_digest' "$IMAGE_WORKFLOW" \ "release publication must compare the actual multi-platform image digest" assert_grep 'actual_sbom_sha256' "$IMAGE_WORKFLOW" \ "release publication must compare the actual SBOM artifact set" assert_grep 'actual_provenance_sha256' "$IMAGE_WORKFLOW" \ "release publication must compare the actual provenance artifact set" -assert_grep 'oras cp --from-oci-layout' "$IMAGE_WORKFLOW" \ - "release publication must copy the exact read-only verified OCI artifact" +assert_grep 'oras tag "$IMAGE@$RELEASE_DIGEST" "$release_tag"' "$IMAGE_WORKFLOW" \ + "release publication must alias the exact existing candidate manifest by digest" +assert_no_grep 'oras cp --from-oci-layout' "$IMAGE_WORKFLOW" \ + "release publication must not rebuild or re-upload an OCI layout" assert_grep 'oras-project/setup-oras@22ce207df3b08e061f537244349aac6ae1d214f6' "$IMAGE_WORKFLOW" \ "release publication must pin the OCI transfer implementation" assert_grep 'published-tag-index.json' "$IMAGE_WORKFLOW" \ "release publication must verify every published tag resolves to the recorded digest" +assert_grep 'File.fnmatch?(pattern, ref, File::FNM_PATHNAME)' "$IMAGE_WORKFLOW" \ + "tag ruleset coverage must use GitHub's documented pathname matcher" +assert_no_grep 'import fnmatch' "$IMAGE_WORKFLOW" \ + "tag ruleset coverage must not let a star wildcard cross ref separators" assert_grep 'Section 13' "$ROOT/docs/herdr-compliance.md" \ "the Herdr audit must account for the network-interaction clause" assert_grep 'https://github.com/akua-dev/akua/tree/v0.8.25' "$ROOT/THIRD_PARTY_NOTICES.md" \ @@ -273,10 +301,10 @@ assert_grep 'ghcr.io/akua-dev/agent-os' "$ROOT/.github/workflows/agent-os-image. "release workflow must publish the image expected by the portable package" assert_grep 'linux/amd64,linux/arm64' "$ROOT/.github/workflows/agent-os-image.yml" \ "release workflow must build the two supported container architectures" -[ "$(grep -c '^ push: false$' "$IMAGE_WORKFLOW")" -eq 2 ] || \ - fail "the read-only validation and release-artifact jobs must build without publishing" -[ "$(grep -c '^ push: true$' "$IMAGE_WORKFLOW")" -eq 1 ] || \ - fail "only the protected publication job may push images" +[ "$(grep -c '^ push: false$' "$IMAGE_WORKFLOW")" -eq 1 ] || \ + fail "the read-only validation job must build without publishing" +[ "$(grep -c '^ push: true$' "$IMAGE_WORKFLOW")" -eq 2 ] || \ + fail "only the protected candidate and main publication builds may push images" assert_grep 'id: build' "$ROOT/.github/workflows/agent-os-image.yml" \ "release workflow must expose its build result" assert_grep 'steps.build.outputs.digest' "$ROOT/.github/workflows/agent-os-image.yml" \ diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index b52222696..22d81b275 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -37,8 +37,14 @@ assert_grep 'rollback_source_digest' "$ROOT/bin/agent-os-kubernetes.sh" \ "rollback compensation must verify its immutable source revision digest" assert_grep 'if [ "$COMMAND" = grant ]; then' "$ROOT/bin/agent-os-akua-auth.sh" \ "authorization Secret validation must apply only to grant" +assert_grep 'verify_overlay absent "$STATE_UID" 0' "$ROOT/bin/agent-os-akua-auth.sh" \ + "authorization revoke must verify absence without requiring a Secret record" +assert_grep '.metadata.annotations["agent-os.dev/akua-auth-rejected-record"] // ""' "$ROOT/bin/agent-os-akua-auth.sh" \ + "authorization revoke must verify the rejected Secret identity marker is absent" assert_grep 'STATEFULSET_CAS_ATTEMPTED' "$ROOT/bin/agent-os-kubernetes.sh" \ "post-CAS upgrade failures must enter fail-closed authorization reconciliation" +assert_grep 'STATEFULSET_CAS_UID' "$ROOT/bin/agent-os-kubernetes.sh" \ + "upgrade compensation must remain bound to the pre-CAS StatefulSet UID" assert_grep 'inventory_owned_service_accounts' "$ROOT/bin/agent-os-kubernetes.sh" \ "uninstall retries must recover exact-owned historical ServiceAccounts independently" assert_grep 'verify_runtime_role_rules' "$ROOT/bin/agent-os-kubernetes.sh" \ @@ -493,8 +499,13 @@ case " $* " in checkpoint_annotations=$(printf '%s' "$checkpoint_annotations" | \ sed 's/rollback-operation":"[^"]*/rollback-operation":"other-operation/') fi - printf '{"metadata":{"name":"agent-os-firstmate","uid":"uid-statefulset","resourceVersion":"%s","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"%s%s}},"spec":{"template":{"spec":{"serviceAccountName":"%s","containers":[{"name":"firstmate","env":[%s],"volumeMounts":[%s]}],"volumes":[%s]}}},"status":{"currentRevision":"%s","updateRevision":"%s"}}\n' \ - "${AGENT_OS_TEST_RESOURCE_RV:-rv-statefulset}" "$checkpoint_annotations" "$akua_annotations" "${AGENT_OS_TEST_WORKLOAD_SERVICE_ACCOUNT:-agent-os-firstmate}" "${akua_env#,}" "${akua_mount#,}" "${akua_volume#,}" "$rollback_current" "$rollback_update" + statefulset_uid=uid-statefulset + if [ "${AGENT_OS_TEST_REPLACE_STATEFULSET_AFTER_ROLLOUT:-0}" = 1 ] && \ + grep -F 'rollout status statefulset/agent-os-firstmate' "$AGENT_OS_TEST_LOG" >/dev/null; then + statefulset_uid=uid-statefulset-replacement + fi + printf '{"metadata":{"name":"agent-os-firstmate","uid":"%s","resourceVersion":"%s","labels":{"app.kubernetes.io/managed-by":"agent-os"},"annotations":{"agent-os.dev/installation-id":"agent-os-firstmate:portable-agent-os"%s%s}},"spec":{"template":{"spec":{"serviceAccountName":"%s","containers":[{"name":"firstmate","env":[%s],"volumeMounts":[%s]}],"volumes":[%s]}}},"status":{"currentRevision":"%s","updateRevision":"%s"}}\n' \ + "$statefulset_uid" "${AGENT_OS_TEST_RESOURCE_RV:-rv-statefulset}" "$checkpoint_annotations" "$akua_annotations" "${AGENT_OS_TEST_WORKLOAD_SERVICE_ACCOUNT:-agent-os-firstmate}" "${akua_env#,}" "${akua_mount#,}" "${akua_volume#,}" "$rollback_current" "$rollback_update" ;; *" get secret "*" --ignore-not-found -o jsonpath="*) secret_name=$(printf '%s\n' "$*" | sed -n 's/.* get secret \([^ ]*\) .*/\1/p') @@ -1691,6 +1702,20 @@ assert_grep '"AKUA_AUTH_HEADER_FILE","$patch":"delete"' "$CALLS" \ "post-CAS upgrade failure must remove the mounted authorization overlay" pass "upgrade post-CAS failures reconcile authorization fail closed" +: > "$CALLS" +replacement_reconcile_out='' +replacement_reconcile_rc=0 +replacement_reconcile_out=$(AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ + AGENT_OS_TEST_WORKLOAD_STATE=namespace AGENT_OS_TEST_AKUA_OVERLAY_SECRET=akua-auth \ + AGENT_OS_TEST_FAIL_FIRST_ROLLOUT=1 AGENT_OS_TEST_REPLACE_STATEFULSET_AFTER_ROLLOUT=1 \ + run_generic upgrade 2>&1) || replacement_reconcile_rc=$? +[ "$replacement_reconcile_rc" -eq 3 ] || fail "replacement workload reconciliation must fail closed: $replacement_reconcile_out" +assert_not_contains "$replacement_reconcile_out" 'authorization overlay removed' \ + "replacement workload must never be reported as reconciled" +assert_no_grep 'agent-os.dev/akua-auth-rejected-record' "$CALLS" \ + "upgrade compensation must not mutate a replacement StatefulSet" +pass "upgrade compensation leaves replacement workloads untouched" + : > "$CALLS" AGENT_OS_TEST_NAMESPACE_STATE=owned AGENT_OS_TEST_RESOURCE_STATE=owned \ AGENT_OS_TEST_WORKLOAD_STATE=namespace \ @@ -2014,6 +2039,11 @@ namespace_inventory_line=$(grep -Fn 'api-resources --verbs=list --namespaced -o [ -n "$namespace_lock_line" ] && [ -n "$namespace_inventory_line" ] && \ [ "$namespace_lock_line" -lt "$namespace_inventory_line" ] || \ fail "namespace deletion must inventory only after holding the installation-wide barrier" +awk ' + /namespaces\/kube-system\/rolebindings\/agent-os-lifecycle-/ && /delete --raw/ && !control { control=NR } + /\/api\/v1\/namespaces\/portable-agent-os -f -/ && /delete --raw/ && !namespace { namespace=NR } + END { exit !(control && namespace && control < namespace) } +' "$CALLS" || fail "uninstall must delete and verify control RBAC before namespace finalization: $(grep -E 'rolebindings/agent-os-lifecycle-|/api/v1/namespaces/portable-agent-os' "$CALLS" | tr '\n' ';')" pass "optional namespace deletion proves ownership and no foreign resources" : > "$CALLS" From 4d5c8331eea7e2cc87ec84d067268ea659156346 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 18:49:58 +0200 Subject: [PATCH 40/56] no-mistakes(review): Secure immutable candidate promotion --- .github/workflows/agent-os-image.yml | 240 +++++++++++++++------------ bin/agent-os-container-entrypoint.sh | 14 +- bin/agent-os-source-bundle.sh | 38 +++-- bin/fm-update.sh | 12 ++ tests/agent-os-container.test.sh | 40 ++++- tests/fm-update.test.sh | 20 +++ 6 files changed, 234 insertions(+), 130 deletions(-) diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index 333b57a84..829b4451b 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -211,43 +211,29 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Resolve existing release candidate record + - name: Reject legacy candidate coordinates if: github.ref == 'refs/heads/main' - id: candidate_state run: | set -eu - candidate_ref="$IMAGE:candidate-record-$GITHUB_SHA" - if oras manifest fetch --descriptor "$candidate_ref" > "$RUNNER_TEMP/candidate-descriptor.json" 2> "$RUNNER_TEMP/candidate-descriptor.err"; then - candidate_record_digest=$(jq -er '.digest | select(test("^sha256:[0-9a-f]{64}$"))' "$RUNNER_TEMP/candidate-descriptor.json") - mkdir "$RUNNER_TEMP/existing-candidate" - oras pull --output "$RUNNER_TEMP/existing-candidate" "$IMAGE@$candidate_record_digest" - cp "$RUNNER_TEMP/existing-candidate/candidate-record.json" "$RUNNER_TEMP/candidate-record.json" - printf 'exists=true\ncandidate_record_digest=%s\n' "$candidate_record_digest" >> "$GITHUB_OUTPUT" - elif grep -Eiq 'manifest unknown|not found|404' "$RUNNER_TEMP/candidate-descriptor.err"; then - image_ref="$IMAGE:release-candidate-$GITHUB_SHA" - if oras manifest fetch --descriptor "$image_ref" > "$RUNNER_TEMP/candidate-image-descriptor.json" 2> "$RUNNER_TEMP/candidate-image-descriptor.err"; then - image_digest=$(jq -er '.digest | select(test("^sha256:[0-9a-f]{64}$"))' "$RUNNER_TEMP/candidate-image-descriptor.json") - printf 'exists=false\nimage_exists=true\nimage_digest=%s\n' "$image_digest" >> "$GITHUB_OUTPUT" - elif grep -Eiq 'manifest unknown|not found|404' "$RUNNER_TEMP/candidate-image-descriptor.err"; then - printf 'exists=false\nimage_exists=false\n' >> "$GITHUB_OUTPUT" - else - cat "$RUNNER_TEMP/candidate-image-descriptor.err" >&2 - exit 1 - fi + legacy_ref="$IMAGE:release-candidate-$GITHUB_SHA" + if oras manifest fetch --descriptor "$legacy_ref" > "$RUNNER_TEMP/legacy-candidate.json" 2> "$RUNNER_TEMP/legacy-candidate.err"; then + echo "::error::pre-existing release candidate lacks an independently trusted record" + exit 1 + elif grep -Eiq 'manifest unknown|not found|404' "$RUNNER_TEMP/legacy-candidate.err"; then + : else - cat "$RUNNER_TEMP/candidate-descriptor.err" >&2 + cat "$RUNNER_TEMP/legacy-candidate.err" >&2 exit 1 fi - name: Build release candidate once - if: github.ref == 'refs/heads/main' && steps.candidate_state.outputs.exists == 'false' && steps.candidate_state.outputs.image_exists == 'false' + if: github.ref == 'refs/heads/main' id: candidate_build uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 with: context: . platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ env.IMAGE }}:release-candidate-${{ github.sha }} + outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true cache-from: type=gha cache-to: type=gha,mode=max provenance: mode=max @@ -265,38 +251,37 @@ jobs: id: candidate_record env: CANDIDATE_BUILD_DIGEST: ${{ steps.candidate_build.outputs.digest }} - CANDIDATE_EXISTS: ${{ steps.candidate_state.outputs.exists }} - EXISTING_CANDIDATE_IMAGE_DIGEST: ${{ steps.candidate_state.outputs.image_digest }} run: | set -eu - candidate_ref="$IMAGE:candidate-record-$GITHUB_SHA" - artifact_set_sha() { - predicate=$1 - output=$2 - : > "$output" - jq -r '.manifests[] | select(.annotations["vnd.docker.reference.type"] == "attestation-manifest") | .digest' "$RUNNER_TEMP/candidate-index.json" | while IFS= read -r manifest_digest; do - docker buildx imagetools inspect --raw "$IMAGE@$manifest_digest" > "$RUNNER_TEMP/candidate-attestation.json" - jq -r --arg predicate "$predicate" '.layers[] | select(.annotations["in-toto.io/predicate-type"] == $predicate) | .digest' "$RUNNER_TEMP/candidate-attestation.json" >> "$output" - done - LC_ALL=C sort -u -o "$output" "$output" - [ "$(wc -l < "$output" | tr -d ' ')" -eq 2 ] - sha256sum "$output" | awk '{print $1}' - } - verify_candidate_provenance() { + verify_candidate_attestations() { + printf '%s' "$platform_manifests" | jq -r '.[].digest' | LC_ALL=C sort > "$RUNNER_TEMP/candidate-platform-digests" + : > "$RUNNER_TEMP/candidate-platform-subjects" + : > "$RUNNER_TEMP/candidate-sbom-digests" + : > "$RUNNER_TEMP/candidate-provenance-digests" jq -r '.manifests[] | select(.annotations["vnd.docker.reference.type"] == "attestation-manifest") | .digest' \ "$RUNNER_TEMP/candidate-index.json" > "$RUNNER_TEMP/candidate-attestation-manifests" [ "$(wc -l < "$RUNNER_TEMP/candidate-attestation-manifests" | tr -d ' ')" -eq 2 ] while IFS= read -r manifest_digest; do manifest_file="$RUNNER_TEMP/candidate-attestation-${manifest_digest#sha256:}.json" + sbom_file="$RUNNER_TEMP/candidate-sbom-${manifest_digest#sha256:}.json" provenance_file="$RUNNER_TEMP/candidate-provenance-${manifest_digest#sha256:}.json" docker buildx imagetools inspect --raw "$IMAGE@$manifest_digest" > "$manifest_file" subject_digest=$(jq -er --arg manifest "$manifest_digest" ' .manifests[] | select(.digest == $manifest) | .annotations["vnd.docker.reference.digest"] | select(test("^sha256:[0-9a-f]{64}$"))' \ "$RUNNER_TEMP/candidate-index.json") - printf '%s' "$platform_manifests" | jq -e --arg subject "$subject_digest" 'any(.[]; .digest == $subject)' >/dev/null + printf '%s\n' "$subject_digest" >> "$RUNNER_TEMP/candidate-platform-subjects" + sbom_digest=$(jq -er '[.layers[] | select(.annotations["in-toto.io/predicate-type"] == "https://spdx.dev/Document")] | select(length == 1) | .[0].digest' "$manifest_file") provenance_digest=$(jq -er '[.layers[] | select(.annotations["in-toto.io/predicate-type"] == "https://slsa.dev/provenance/v0.2")] | select(length == 1) | .[0].digest' "$manifest_file") + oras blob fetch --output "$sbom_file" "$IMAGE@$sbom_digest" oras blob fetch --output "$provenance_file" "$IMAGE@$provenance_digest" + jq -e --arg subject "${subject_digest#sha256:}" ' + .predicateType == "https://spdx.dev/Document" and + (.subject | length == 1) and .subject[0].digest.sha256 == $subject' \ + "$sbom_file" >/dev/null || { + echo "::error::release candidate SBOM does not bind its exact platform manifest" + exit 1 + } jq -e --arg subject "${subject_digest#sha256:}" \ --arg commit '${{ steps.candidate_source.outputs.commit }}' \ --arg tree '${{ steps.candidate_source.outputs.tree }}' \ @@ -319,49 +304,43 @@ jobs: echo "::error::release candidate image provenance conflicts with the exact protected-main source" exit 1 } + printf '%s\n' "$sbom_digest" >> "$RUNNER_TEMP/candidate-sbom-digests" + printf '%s\n' "$provenance_digest" >> "$RUNNER_TEMP/candidate-provenance-digests" done < "$RUNNER_TEMP/candidate-attestation-manifests" + LC_ALL=C sort -o "$RUNNER_TEMP/candidate-platform-subjects" "$RUNNER_TEMP/candidate-platform-subjects" + cmp "$RUNNER_TEMP/candidate-platform-digests" "$RUNNER_TEMP/candidate-platform-subjects" || { + echo "::error::release candidate attestations do not map one-to-one onto both platforms" + exit 1 + } + LC_ALL=C sort -o "$RUNNER_TEMP/candidate-sbom-digests" "$RUNNER_TEMP/candidate-sbom-digests" + LC_ALL=C sort -o "$RUNNER_TEMP/candidate-provenance-digests" "$RUNNER_TEMP/candidate-provenance-digests" } - if [ "$CANDIDATE_EXISTS" = true ]; then - actual_image_digest=$(jq -er '.image_digest' "$RUNNER_TEMP/candidate-record.json") - elif [ -n "$EXISTING_CANDIDATE_IMAGE_DIGEST" ]; then - actual_image_digest=$EXISTING_CANDIDATE_IMAGE_DIGEST - else - actual_image_digest=$CANDIDATE_BUILD_DIGEST - fi + actual_image_digest=$CANDIDATE_BUILD_DIGEST [[ "$actual_image_digest" =~ ^sha256:[0-9a-f]{64}$ ]] docker buildx imagetools inspect --raw "$IMAGE@$actual_image_digest" > "$RUNNER_TEMP/candidate-index.json" [ "sha256:$(sha256sum "$RUNNER_TEMP/candidate-index.json" | awk '{print $1}')" = "$actual_image_digest" ] platform_manifests=$(jq -cS '[.manifests[] | select(.platform.os == "linux" and (.platform.architecture == "amd64" or .platform.architecture == "arm64")) | {platform:(.platform.os + "/" + .platform.architecture),digest,mediaType,size}] | sort_by(.platform)' "$RUNNER_TEMP/candidate-index.json") - [ "$(printf '%s' "$platform_manifests" | jq 'length')" -eq 2 ] - verify_candidate_provenance + printf '%s' "$platform_manifests" | jq -e '[.[].platform] == ["linux/amd64","linux/arm64"]' >/dev/null + verify_candidate_attestations platform_manifests_sha256=$(printf '%s' "$platform_manifests" | sha256sum | awk '{print $1}') buildkit_outputs=$(jq -cS '[.manifests[] | {digest,mediaType,size,platform,annotations}] | sort_by(.digest)' "$RUNNER_TEMP/candidate-index.json") buildkit_outputs_sha256=$(printf '%s' "$buildkit_outputs" | sha256sum | awk '{print $1}') - actual_sbom_sha256=$(artifact_set_sha https://spdx.dev/Document "$RUNNER_TEMP/candidate-sbom-digests") - actual_provenance_sha256=$(artifact_set_sha https://slsa.dev/provenance/v0.2 "$RUNNER_TEMP/candidate-provenance-digests") - if [ "$CANDIDATE_EXISTS" = false ]; then - jq -cnS \ - --arg commit '${{ steps.candidate_source.outputs.commit }}' \ - --arg tree '${{ steps.candidate_source.outputs.tree }}' \ - --arg sourceArchive '${{ steps.candidate_source.outputs.source_sha256 }}' \ - --arg bootstrapArchive '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' \ - --arg package "$(sha256sum tools/agent-os/packages/firstmate/package.k | awk '{print $1}')" \ - --arg schema "$(sha256sum tools/agent-os/packages/firstmate/inputs.example.yaml | awk '{print $1}')" \ - --arg quickstart "$(sha256sum docs/kubernetes.md | awk '{print $1}')" \ - --arg repository "$IMAGE" --arg image "$actual_image_digest" \ - --argjson platforms "$platform_manifests" --arg platformsSha "$platform_manifests_sha256" \ - --arg sbom "$actual_sbom_sha256" --arg provenance "$actual_provenance_sha256" \ - --arg buildkitOutputs "$buildkit_outputs_sha256" \ - '{schema_version:1,source_mode:"candidate",commit:$commit,tree:$tree,source_archive_sha256:$sourceArchive,bootstrap_archive_sha256:$bootstrapArchive,package_sha256:$package,schema_sha256:$schema,quickstart_sha256:$quickstart,image_repository:$repository,image_digest:$image,platform_manifests:$platforms,platform_manifests_sha256:$platformsSha,sbom_sha256:$sbom,provenance_sha256:$provenance,buildkit_outputs_sha256:$buildkitOutputs}' \ - > "$RUNNER_TEMP/candidate-record.json" - oras push --artifact-type application/vnd.akua.agent-os.release-candidate.v1 \ - --format json "$candidate_ref" \ - "$RUNNER_TEMP/candidate-record.json:application/vnd.akua.agent-os.release-candidate.v1+json" \ - > "$RUNNER_TEMP/candidate-push.json" - candidate_record_digest=$(jq -er '.digest | select(test("^sha256:[0-9a-f]{64}$"))' "$RUNNER_TEMP/candidate-push.json") - else - candidate_record_digest='${{ steps.candidate_state.outputs.candidate_record_digest }}' - fi + actual_sbom_sha256=$(sha256sum "$RUNNER_TEMP/candidate-sbom-digests" | awk '{print $1}') + actual_provenance_sha256=$(sha256sum "$RUNNER_TEMP/candidate-provenance-digests" | awk '{print $1}') + jq -cnS \ + --arg commit '${{ steps.candidate_source.outputs.commit }}' \ + --arg tree '${{ steps.candidate_source.outputs.tree }}' \ + --arg sourceArchive '${{ steps.candidate_source.outputs.source_sha256 }}' \ + --arg bootstrapArchive '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' \ + --arg package "$(sha256sum tools/agent-os/packages/firstmate/package.k | awk '{print $1}')" \ + --arg schema "$(sha256sum tools/agent-os/packages/firstmate/inputs.example.yaml | awk '{print $1}')" \ + --arg quickstart "$(sha256sum docs/kubernetes.md | awk '{print $1}')" \ + --arg repository "$IMAGE" --arg image "$actual_image_digest" \ + --argjson platforms "$platform_manifests" --arg platformsSha "$platform_manifests_sha256" \ + --arg sbom "$actual_sbom_sha256" --arg provenance "$actual_provenance_sha256" \ + --arg buildkitOutputs "$buildkit_outputs_sha256" \ + '{schema_version:1,source_mode:"candidate",commit:$commit,tree:$tree,source_archive_sha256:$sourceArchive,bootstrap_archive_sha256:$bootstrapArchive,package_sha256:$package,schema_sha256:$schema,quickstart_sha256:$quickstart,image_repository:$repository,image_digest:$image,platform_manifests:$platforms,platform_manifests_sha256:$platformsSha,sbom_sha256:$sbom,provenance_sha256:$provenance,buildkit_outputs_sha256:$buildkitOutputs}' \ + > "$RUNNER_TEMP/candidate-record.json" jq -e \ --arg commit '${{ steps.candidate_source.outputs.commit }}' --arg tree '${{ steps.candidate_source.outputs.tree }}' \ --arg sourceArchive '${{ steps.candidate_source.outputs.source_sha256 }}' \ @@ -372,18 +351,41 @@ jobs: --arg repository "$IMAGE" --arg image "$actual_image_digest" \ --argjson platforms "$platform_manifests" --arg platformsSha "$platform_manifests_sha256" \ --arg sbom "$actual_sbom_sha256" --arg provenance "$actual_provenance_sha256" \ - --arg buildkitOutputs "$buildkit_outputs_sha256" --arg existing "$CANDIDATE_EXISTS" \ - '.schema_version == 1 and .source_mode == "candidate" and .commit == $commit and .tree == $tree and .source_archive_sha256 == $sourceArchive and (($existing == "false" and .bootstrap_archive_sha256 == $bootstrapArchive) or ($existing == "true" and (.bootstrap_archive_sha256 | test("^[0-9a-f]{64}$")))) and .package_sha256 == $package and .schema_sha256 == $schema and .quickstart_sha256 == $quickstart and .image_repository == $repository and .image_digest == $image and .platform_manifests == $platforms and .platform_manifests_sha256 == $platformsSha and .sbom_sha256 == $sbom and .provenance_sha256 == $provenance and .buildkit_outputs_sha256 == $buildkitOutputs' \ + --arg buildkitOutputs "$buildkit_outputs_sha256" \ + '.schema_version == 1 and .source_mode == "candidate" and .commit == $commit and .tree == $tree and .source_archive_sha256 == $sourceArchive and .bootstrap_archive_sha256 == $bootstrapArchive and .package_sha256 == $package and .schema_sha256 == $schema and .quickstart_sha256 == $quickstart and .image_repository == $repository and .image_digest == $image and .platform_manifests == $platforms and .platform_manifests_sha256 == $platformsSha and .sbom_sha256 == $sbom and .provenance_sha256 == $provenance and .buildkit_outputs_sha256 == $buildkitOutputs' \ "$RUNNER_TEMP/candidate-record.json" >/dev/null || { echo "::error::candidate record coordinate already contains different artifacts" exit 1 } - oras manifest fetch --descriptor "$candidate_ref" > "$RUNNER_TEMP/candidate-readback-descriptor.json" - [ "$(jq -r .digest "$RUNNER_TEMP/candidate-readback-descriptor.json")" = "$candidate_record_digest" ] - rm -rf "$RUNNER_TEMP/candidate-readback" - mkdir "$RUNNER_TEMP/candidate-readback" - oras pull --output "$RUNNER_TEMP/candidate-readback" "$IMAGE@$candidate_record_digest" - cmp "$RUNNER_TEMP/candidate-record.json" "$RUNNER_TEMP/candidate-readback/candidate-record.json" + printf '{}\n' > "$RUNNER_TEMP/candidate-config.json" + oras blob push --media-type application/vnd.oci.empty.v1+json --descriptor \ + "$IMAGE" "$RUNNER_TEMP/candidate-config.json" > "$RUNNER_TEMP/candidate-config-descriptor.json" + oras blob push --media-type application/vnd.akua.agent-os.release-candidate.v1+json --descriptor \ + "$IMAGE" "$RUNNER_TEMP/candidate-record.json" > "$RUNNER_TEMP/candidate-record-descriptor.json" + config_descriptor=$(jq -cS '{mediaType,digest,size}' "$RUNNER_TEMP/candidate-config-descriptor.json") + record_descriptor=$(jq -cS '{mediaType,digest,size,annotations:{"org.opencontainers.image.title":"candidate-record.json"}}' "$RUNNER_TEMP/candidate-record-descriptor.json") + jq -cnS --argjson config "$config_descriptor" --argjson record "$record_descriptor" \ + '{schemaVersion:2,mediaType:"application/vnd.oci.image.manifest.v1+json",artifactType:"application/vnd.akua.agent-os.release-candidate.v1",config:$config,layers:[$record]}' \ + > "$RUNNER_TEMP/candidate-manifest.json" + candidate_record_digest="sha256:$(sha256sum "$RUNNER_TEMP/candidate-manifest.json" | awk '{print $1}')" + if oras manifest fetch "$IMAGE@$candidate_record_digest" > "$RUNNER_TEMP/candidate-existing-manifest.json" 2> "$RUNNER_TEMP/candidate-existing-manifest.err"; then + cmp "$RUNNER_TEMP/candidate-manifest.json" "$RUNNER_TEMP/candidate-existing-manifest.json" || { + echo "::error::candidate record coordinate already contains different artifacts" + exit 1 + } + elif grep -Eiq 'manifest unknown|not found|404' "$RUNNER_TEMP/candidate-existing-manifest.err"; then + oras manifest push --descriptor "$IMAGE@$candidate_record_digest" \ + "$RUNNER_TEMP/candidate-manifest.json" > "$RUNNER_TEMP/candidate-push.json" + [ "$(jq -r .digest "$RUNNER_TEMP/candidate-push.json")" = "$candidate_record_digest" ] + else + cat "$RUNNER_TEMP/candidate-existing-manifest.err" >&2 + exit 1 + fi + oras manifest fetch "$IMAGE@$candidate_record_digest" > "$RUNNER_TEMP/candidate-readback-manifest.json" + cmp "$RUNNER_TEMP/candidate-manifest.json" "$RUNNER_TEMP/candidate-readback-manifest.json" + record_blob_digest=$(jq -er '.layers | select(length == 1) | .[0].digest' "$RUNNER_TEMP/candidate-readback-manifest.json") + oras blob fetch --output "$RUNNER_TEMP/candidate-record-readback.json" "$IMAGE@$record_blob_digest" + cmp "$RUNNER_TEMP/candidate-record.json" "$RUNNER_TEMP/candidate-record-readback.json" printf 'candidate_record_digest=%s\nimage_digest=%s\n' "$candidate_record_digest" "$actual_image_digest" >> "$GITHUB_OUTPUT" - name: Retain candidate record for release preparation @@ -391,7 +393,9 @@ jobs: uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: agent-os-candidate-record-${{ github.sha }} - path: ${{ runner.temp }}/candidate-record.json + path: | + ${{ runner.temp }}/candidate-record.json + ${{ runner.temp }}/candidate-manifest.json if-no-files-found: error retention-days: 90 @@ -437,20 +441,21 @@ jobs: - name: Promote exact verified release candidate if: startsWith(github.ref, 'refs/tags/') - env: - RELEASE_COMMIT: ${{ steps.release_source.outputs.commit }} run: | set -eu release_tag=${GITHUB_REF#refs/tags/} - candidate_ref="$IMAGE:candidate-record-$RELEASE_COMMIT" - oras manifest fetch --descriptor "$candidate_ref" > "$RUNNER_TEMP/release-candidate-descriptor.json" candidate_record_digest=$(jq -er '.candidate_record_digest' "$RUNNER_TEMP/release-record.json") - [ "$(jq -r .digest "$RUNNER_TEMP/release-candidate-descriptor.json")" = "$candidate_record_digest" ] - mkdir "$RUNNER_TEMP/release-candidate" - oras pull --output "$RUNNER_TEMP/release-candidate" "$IMAGE@$candidate_record_digest" + oras manifest fetch "$IMAGE@$candidate_record_digest" > "$RUNNER_TEMP/release-candidate-manifest.json" + [ "sha256:$(sha256sum "$RUNNER_TEMP/release-candidate-manifest.json" | awk '{print $1}')" = "$candidate_record_digest" ] + candidate_record_blob=$(jq -er ' + select(.schemaVersion == 2 and .artifactType == "application/vnd.akua.agent-os.release-candidate.v1") + | .layers | select(length == 1) | .[0] + | select(.mediaType == "application/vnd.akua.agent-os.release-candidate.v1+json") | .digest' \ + "$RUNNER_TEMP/release-candidate-manifest.json") + oras blob fetch --output "$RUNNER_TEMP/release-candidate-record.json" "$IMAGE@$candidate_record_blob" jq -cS '{schema_version,source_mode,commit,tree,source_archive_sha256,bootstrap_archive_sha256,package_sha256,schema_sha256,quickstart_sha256,image_repository,image_digest,platform_manifests,platform_manifests_sha256,sbom_sha256,provenance_sha256,buildkit_outputs_sha256}' \ "$RUNNER_TEMP/release-record.json" > "$RUNNER_TEMP/expected-candidate-record.json" - jq -cS . "$RUNNER_TEMP/release-candidate/candidate-record.json" > "$RUNNER_TEMP/actual-candidate-record.json" + jq -cS . "$RUNNER_TEMP/release-candidate-record.json" > "$RUNNER_TEMP/actual-candidate-record.json" cmp "$RUNNER_TEMP/expected-candidate-record.json" "$RUNNER_TEMP/actual-candidate-record.json" RELEASE_DIGEST=$(jq -er '.image_digest' "$RUNNER_TEMP/release-record.json") docker buildx imagetools inspect --raw "$IMAGE@$RELEASE_DIGEST" > "$RUNNER_TEMP/published-index.json" @@ -460,20 +465,45 @@ jobs: [ "$(printf '%s' "$actual_platforms" | sha256sum | awk '{print $1}')" = "$(jq -r .platform_manifests_sha256 "$RUNNER_TEMP/release-record.json")" ] actual_buildkit_outputs=$(jq -cS '[.manifests[] | {digest,mediaType,size,platform,annotations}] | sort_by(.digest)' "$RUNNER_TEMP/published-index.json") [ "$(printf '%s' "$actual_buildkit_outputs" | sha256sum | awk '{print $1}')" = "$(jq -r .buildkit_outputs_sha256 "$RUNNER_TEMP/release-record.json")" ] - published_artifact_set_sha() { - predicate=$1 - output=$2 - : > "$output" - jq -r '.manifests[] | select(.annotations["vnd.docker.reference.type"] == "attestation-manifest") | .digest' "$RUNNER_TEMP/published-index.json" | while IFS= read -r manifest_digest; do - docker buildx imagetools inspect --raw "$IMAGE@$manifest_digest" > "$RUNNER_TEMP/published-manifest.json" - jq -r --arg predicate "$predicate" '.layers[] | select(.annotations["in-toto.io/predicate-type"] == $predicate) | .digest' "$RUNNER_TEMP/published-manifest.json" >> "$output" - done - LC_ALL=C sort -u -o "$output" "$output" - [ "$(wc -l < "$output" | tr -d ' ')" -eq 2 ] - sha256sum "$output" | awk '{print $1}' + verify_published_attestations() { + printf '%s' "$actual_platforms" | jq -r '.[].digest' | LC_ALL=C sort > "$RUNNER_TEMP/published-platform-digests" + : > "$RUNNER_TEMP/published-platform-subjects" + : > "$RUNNER_TEMP/published-sbom-digests" + : > "$RUNNER_TEMP/published-provenance-digests" + jq -r '.manifests[] | select(.annotations["vnd.docker.reference.type"] == "attestation-manifest") | .digest' \ + "$RUNNER_TEMP/published-index.json" > "$RUNNER_TEMP/published-attestation-manifests" + [ "$(wc -l < "$RUNNER_TEMP/published-attestation-manifests" | tr -d ' ')" -eq 2 ] + while IFS= read -r manifest_digest; do + manifest_file="$RUNNER_TEMP/published-attestation-${manifest_digest#sha256:}.json" + sbom_file="$RUNNER_TEMP/published-sbom-${manifest_digest#sha256:}.json" + provenance_file="$RUNNER_TEMP/published-provenance-${manifest_digest#sha256:}.json" + docker buildx imagetools inspect --raw "$IMAGE@$manifest_digest" > "$manifest_file" + subject_digest=$(jq -er --arg manifest "$manifest_digest" ' + .manifests[] | select(.digest == $manifest) + | .annotations["vnd.docker.reference.digest"] | select(test("^sha256:[0-9a-f]{64}$"))' \ + "$RUNNER_TEMP/published-index.json") + printf '%s\n' "$subject_digest" >> "$RUNNER_TEMP/published-platform-subjects" + sbom_digest=$(jq -er '[.layers[] | select(.annotations["in-toto.io/predicate-type"] == "https://spdx.dev/Document")] | select(length == 1) | .[0].digest' "$manifest_file") + provenance_digest=$(jq -er '[.layers[] | select(.annotations["in-toto.io/predicate-type"] == "https://slsa.dev/provenance/v0.2")] | select(length == 1) | .[0].digest' "$manifest_file") + oras blob fetch --output "$sbom_file" "$IMAGE@$sbom_digest" + oras blob fetch --output "$provenance_file" "$IMAGE@$provenance_digest" + jq -e --arg subject "${subject_digest#sha256:}" ' + .predicateType == "https://spdx.dev/Document" and + (.subject | length == 1) and .subject[0].digest.sha256 == $subject' "$sbom_file" >/dev/null + jq -e --arg subject "${subject_digest#sha256:}" ' + .predicateType == "https://slsa.dev/provenance/v0.2" and + (.subject | length == 1) and .subject[0].digest.sha256 == $subject' "$provenance_file" >/dev/null + printf '%s\n' "$sbom_digest" >> "$RUNNER_TEMP/published-sbom-digests" + printf '%s\n' "$provenance_digest" >> "$RUNNER_TEMP/published-provenance-digests" + done < "$RUNNER_TEMP/published-attestation-manifests" + LC_ALL=C sort -o "$RUNNER_TEMP/published-platform-subjects" "$RUNNER_TEMP/published-platform-subjects" + cmp "$RUNNER_TEMP/published-platform-digests" "$RUNNER_TEMP/published-platform-subjects" + LC_ALL=C sort -o "$RUNNER_TEMP/published-sbom-digests" "$RUNNER_TEMP/published-sbom-digests" + LC_ALL=C sort -o "$RUNNER_TEMP/published-provenance-digests" "$RUNNER_TEMP/published-provenance-digests" } - actual_sbom_sha256=$(published_artifact_set_sha https://spdx.dev/Document "$RUNNER_TEMP/published-sbom-digests") - actual_provenance_sha256=$(published_artifact_set_sha https://slsa.dev/provenance/v0.2 "$RUNNER_TEMP/published-provenance-digests") + verify_published_attestations + actual_sbom_sha256=$(sha256sum "$RUNNER_TEMP/published-sbom-digests" | awk '{print $1}') + actual_provenance_sha256=$(sha256sum "$RUNNER_TEMP/published-provenance-digests" | awk '{print $1}') [ "$actual_sbom_sha256" = "$(jq -r .sbom_sha256 "$RUNNER_TEMP/release-record.json")" ] [ "$actual_provenance_sha256" = "$(jq -r .provenance_sha256 "$RUNNER_TEMP/release-record.json")" ] if oras manifest fetch --descriptor "$IMAGE:$release_tag" > "$RUNNER_TEMP/release-tag-descriptor.json" 2> "$RUNNER_TEMP/release-tag.err"; then diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh index 996a20f1d..a9d20532d 100755 --- a/bin/agent-os-container-entrypoint.sh +++ b/bin/agent-os-container-entrypoint.sh @@ -15,19 +15,21 @@ SOURCE_REF=$(cat /opt/agent-os-source.ref) case "$SOURCE_BRANCH" in ''|*[!A-Za-z0-9._/-]*|/*|*/|*..*) echo "error: image source branch provenance is invalid" >&2; exit 2 ;; esac [ "$SOURCE_ORIGIN" = https://github.com/akua-dev/agent-os.git ] || { echo "error: image source origin is invalid" >&2; exit 2; } case "$SOURCE_MODE" in - main) [ "$SOURCE_REF" = refs/heads/main ] || exit 2; TRUSTED_SOURCE_REF=$SOURCE_REF; TRUSTED_REF=refs/remotes/agent-os-verified/main ;; + main) [ "$SOURCE_REF" = refs/heads/main ] || exit 2; TRUSTED_SOURCE_REF=$SOURCE_REF; TRUSTED_REF=refs/remotes/agent-os-verified/main; AGENT_OS_SOURCE_UPDATE_POLICY=fast-forward ;; candidate) [ "$SOURCE_REF" = "refs/agent-os/candidates/$SOURCE_COMMIT" ] || exit 2 - TRUSTED_SOURCE_REF=refs/heads/main - TRUSTED_REF=refs/remotes/agent-os-verified/main + TRUSTED_SOURCE_REF=$SOURCE_COMMIT + TRUSTED_REF=refs/remotes/agent-os-verified/candidate + AGENT_OS_SOURCE_UPDATE_POLICY=immutable ;; - release) [[ "$SOURCE_REF" =~ ^refs/tags/v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] || exit 2; TRUSTED_SOURCE_REF=$SOURCE_REF; TRUSTED_REF=refs/remotes/agent-os-verified/release ;; + release) [[ "$SOURCE_REF" =~ ^refs/tags/v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] || exit 2; TRUSTED_SOURCE_REF=$SOURCE_REF; TRUSTED_REF=refs/remotes/agent-os-verified/release; AGENT_OS_SOURCE_UPDATE_POLICY=immutable ;; event) echo "error: pull-request validation images are not runnable" >&2; exit 2 ;; *) echo "error: image source mode is invalid" >&2; exit 2 ;; esac +export AGENT_OS_SOURCE_UPDATE_POLICY IMAGE_REF="refs/remotes/agent-os-image/$SOURCE_BRANCH" -GIT_BIN=$(command -v git) -case "$GIT_BIN" in /*) ;; *) echo "error: trusted Git executable is unavailable" >&2; exit 2 ;; esac +GIT_BIN=/usr/bin/git +[ -x "$GIT_BIN" ] || { echo "error: trusted Git executable is unavailable" >&2; exit 2; } trusted_git() { env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ diff --git a/bin/agent-os-source-bundle.sh b/bin/agent-os-source-bundle.sh index a5921c845..fa9b95f7f 100755 --- a/bin/agent-os-source-bundle.sh +++ b/bin/agent-os-source-bundle.sh @@ -15,8 +15,8 @@ SOURCE_ARCHIVE="$OUTPUT_DIR/agent-os-source.tar" BOOTSTRAP_ARCHIVE="$OUTPUT_DIR/agent-os-bootstrap.tar" ATTESTATION="$OUTPUT_DIR/agent-os-source.attestation" TEMP="$OUTPUT_DIR/.agent-os-source.$$" -GIT_BIN=$(command -v git) -case "$GIT_BIN" in /*) ;; *) echo "error: trusted Git executable is unavailable" >&2; exit 2 ;; esac +GIT_BIN=/usr/bin/git +[ -x "$GIT_BIN" ] || { echo "error: trusted Git executable is unavailable" >&2; exit 2; } git_isolated() { env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ @@ -35,7 +35,7 @@ canonical_github_origin() { } validate_source_git_config() { - local key value origin worktree_keys + local key value origin worktree_config worktree_config_enabled worktree_keys while IFS= read -r key; do [ -n "$key" ] || continue value=$(git_isolated -C "$ROOT" config --local --no-includes --get-all "$key" || true) @@ -61,11 +61,17 @@ validate_source_git_config() { *) echo "error: source repository Git config key is not allowlisted: $key" >&2; exit 2 ;; esac || { echo "error: source repository Git config value is invalid: $key" >&2; exit 2; } done < <(git_isolated -C "$ROOT" config --local --no-includes --name-only --list) - worktree_keys=$(git_isolated -C "$ROOT" config --worktree --no-includes --name-only --list 2>/dev/null || true) - [ -z "$worktree_keys" ] || { - echo "error: source repository worktree Git config is not allowed" >&2 - exit 2 - } + worktree_config_enabled=$(git_isolated -C "$ROOT" config --local --type=bool --get extensions.worktreeconfig || true) + if [ "$worktree_config_enabled" = true ]; then + worktree_config=$(git_isolated -C "$ROOT" rev-parse --path-format=absolute --git-path config.worktree) + if [ -f "$worktree_config" ]; then + worktree_keys=$(git_isolated config --file "$worktree_config" --no-includes --name-only --list) + [ -z "$worktree_keys" ] || { + echo "error: source repository worktree Git config is not allowed" >&2 + exit 2 + } + fi + fi origin=$(git_isolated -C "$ROOT" config --local --no-includes --get remote.origin.url || true) [ "$(canonical_github_origin "$origin" 2>/dev/null || true)" = "$SOURCE_ORIGIN" ] || { echo "error: repository origin is not the exact trusted HTTPS origin" >&2 @@ -172,13 +178,11 @@ elif [ "$SOURCE_MODE" = release ]; then } fi -if [ "$SOURCE_MODE" = release ]; then - git_isolated --git-dir="$TEMP/bootstrap.git" update-ref -d refs/agent-os/release-record - git_isolated --git-dir="$TEMP/bootstrap.git" update-ref -d "refs/tags/$RELEASE_TAG" - printf '%s\n' "$SOURCE_COMMIT" > "$TEMP/bootstrap.git/shallow" - git_isolated --git-dir="$TEMP/bootstrap.git" reflog expire --expire=now --all - git_isolated --git-dir="$TEMP/bootstrap.git" gc --prune=now -fi +git_isolated init --bare "$TEMP/sanitized.git" >/dev/null +git_isolated -c protocol.file.allow=always --git-dir="$TEMP/sanitized.git" fetch --depth=1 --no-tags \ + "file://$TEMP/bootstrap.git" "$SOURCE_COMMIT:refs/heads/main" +rm -rf "$TEMP/bootstrap.git" +mv "$TEMP/sanitized.git" "$TEMP/bootstrap.git" git_isolated --git-dir="$TEMP/bootstrap.git" symbolic-ref HEAD refs/heads/main git_isolated --git-dir="$TEMP/bootstrap.git" remote add origin "$SOURCE_ORIGIN" @@ -198,6 +202,10 @@ fi find bootstrap.git -print | LC_ALL=C sort | COPYFILE_DISABLE=1 tar -cf "$BOOTSTRAP_ARCHIVE.tmp.$$" -T -) SOURCE_SHA=$(sha256_file "$SOURCE_ARCHIVE.tmp.$$") BOOTSTRAP_SHA=$(sha256_file "$BOOTSTRAP_ARCHIVE.tmp.$$") +if [ "$SOURCE_MODE" = release ] && [ "$BOOTSTRAP_SHA" != "$(jq -r .bootstrap_archive_sha256 "$TEMP/release.json")" ]; then + echo "error: bootstrap archive differs from the immutable release record" >&2 + exit 2 +fi { printf 'mode=%s\ncommit=%s\ntree=%s\nbranch=%s\norigin=%s\nref=%s\n' \ "$SOURCE_MODE" "$SOURCE_COMMIT" "$SOURCE_TREE" "$SOURCE_BRANCH" "$SOURCE_ORIGIN" "$SOURCE_REF" diff --git a/bin/fm-update.sh b/bin/fm-update.sh index c3b0e674f..f16501e96 100755 --- a/bin/fm-update.sh +++ b/bin/fm-update.sh @@ -47,6 +47,18 @@ if [ "${1:-}" = "--help" ] || [ "${1:-}" = "-h" ]; then fi [ $# -eq 0 ] || { usage; exit 1; } +case "${AGENT_OS_SOURCE_UPDATE_POLICY:-fast-forward}" in + fast-forward) ;; + immutable) + echo "error: self-update is disabled for immutable image source" >&2 + exit 2 + ;; + *) + echo "error: invalid Agent OS source update policy" >&2 + exit 2 + ;; +esac + # --- main firstmate repo --------------------------------------------------- reread_firstmate="no" diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 97e77c1f5..8ce7320ef 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -125,6 +125,12 @@ assert_grep 'env -i' "$ROOT/bin/agent-os-source-bundle.sh" \ "source preparation must use an environment allowlist for trusted Git operations" assert_grep 'validate_source_git_config' "$ROOT/bin/agent-os-source-bundle.sh" \ "source preparation must reject executable repository-local Git configuration" +assert_grep 'GIT_BIN=/usr/bin/git' "$ROOT/bin/agent-os-source-bundle.sh" \ + "source preparation must pin trusted Git outside the caller PATH" +assert_no_grep 'config --worktree' "$ROOT/bin/agent-os-source-bundle.sh" \ + "standard clones must not alias their local config through --worktree" +assert_grep 'extensions.worktreeconfig' "$ROOT/bin/agent-os-source-bundle.sh" \ + "source preparation must inspect config.worktree only when explicitly enabled" assert_grep 'canonical_github_origin' "$ROOT/bin/agent-os-source-bundle.sh" \ "source preparation must canonicalize the trusted GitHub HTTPS origin" assert_grep 'https://github.com/akua-dev/agent-os|https://github.com/akua-dev/agent-os.git)' "$ROOT/bin/agent-os-source-bundle.sh" \ @@ -145,6 +151,8 @@ assert_grep 'GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null' "$ROOT/bin/agent "runtime provenance must isolate trusted Git operations" assert_grep 'env -i' "$ROOT/bin/agent-os-container-entrypoint.sh" \ "runtime provenance must clear ambient Git paths, objects, helpers, and transport state" +assert_grep 'GIT_BIN=/usr/bin/git' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "runtime provenance must pin trusted Git outside persistent user paths" assert_grep 'canonical FM_ROOT Git config key is not allowlisted' "$ROOT/bin/agent-os-container-entrypoint.sh" \ "runtime provenance must use a strict repository-config allowlist" assert_no_grep 'config --file "$config" --no-includes --name-only --get-regexp' "$ROOT/bin/agent-os-container-entrypoint.sh" \ @@ -218,8 +226,14 @@ assert_grep '.enforcement == "active" and .target == "tag"' "$IMAGE_WORKFLOW" \ "tag publication must require an active tag-targeting ruleset" assert_grep '.bypass_actors | length == 0' "$IMAGE_WORKFLOW" \ "tag publication must reject rulesets with bypass actors" -assert_grep 'candidate-record-' "$IMAGE_WORKFLOW" \ - "protected main must retain one commit-addressed release candidate record" +assert_no_grep '$IMAGE:candidate-record-' "$IMAGE_WORKFLOW" \ + "candidate records must never depend on a mutable registry tag" +assert_grep 'pre-existing release candidate lacks an independently trusted record' "$IMAGE_WORKFLOW" \ + "publication must reject unrecorded pre-existing candidate images" +assert_no_grep 'EXISTING_CANDIDATE_IMAGE_DIGEST' "$IMAGE_WORKFLOW" \ + "publication must never adopt a pre-existing image from its embedded provenance" +assert_grep 'oras manifest push --descriptor "$IMAGE@$candidate_record_digest"' "$IMAGE_WORKFLOW" \ + "candidate records must be created only at their content-addressed digest" assert_grep 'candidate record coordinate already contains different artifacts' "$IMAGE_WORKFLOW" \ "candidate record creation must reject immutable coordinate conflicts" assert_grep 'release candidate image provenance conflicts with the exact protected-main source' "$IMAGE_WORKFLOW" \ @@ -230,8 +244,18 @@ assert_grep 'source_mode:"candidate"' "$IMAGE_WORKFLOW" \ "candidate records must bind the immutable runtime source policy" assert_grep 'platform_manifests' "$IMAGE_WORKFLOW" \ "candidate records must bind every published platform manifest" +assert_grep 'candidate-platform-subjects' "$IMAGE_WORKFLOW" \ + "candidate attestations must map one-to-one onto platform manifests" +assert_grep 'candidate-sbom-' "$IMAGE_WORKFLOW" \ + "candidate validation must inspect each exact SBOM statement" +assert_grep '.predicateType == "https://spdx.dev/Document"' "$IMAGE_WORKFLOW" \ + "candidate validation must bind every SBOM subject to its platform" assert_grep 'buildkit_outputs_sha256' "$IMAGE_WORKFLOW" \ "candidate records must bind the exact BuildKit output descriptors" +assert_grep '.bootstrap_archive_sha256 == $bootstrapArchive' "$IMAGE_WORKFLOW" \ + "candidate retries and promotion must enforce the exact bootstrap archive" +assert_grep 'bootstrap archive differs from the immutable release record' "$ROOT/bin/agent-os-source-bundle.sh" \ + "release source preparation must regenerate and verify the exact bootstrap archive" assert_grep 'actual_image_digest' "$IMAGE_WORKFLOW" \ "release publication must compare the actual multi-platform image digest" assert_grep 'actual_sbom_sha256' "$IMAGE_WORKFLOW" \ @@ -246,6 +270,8 @@ assert_grep 'oras-project/setup-oras@22ce207df3b08e061f537244349aac6ae1d214f6' " "release publication must pin the OCI transfer implementation" assert_grep 'published-tag-index.json' "$IMAGE_WORKFLOW" \ "release publication must verify every published tag resolves to the recorded digest" +assert_grep 'published-platform-subjects' "$IMAGE_WORKFLOW" \ + "release promotion must reverify the per-platform attestation bijection" assert_grep 'File.fnmatch?(pattern, ref, File::FNM_PATHNAME)' "$IMAGE_WORKFLOW" \ "tag ruleset coverage must use GitHub's documented pathname matcher" assert_no_grep 'import fnmatch' "$IMAGE_WORKFLOW" \ @@ -303,8 +329,10 @@ assert_grep 'linux/amd64,linux/arm64' "$ROOT/.github/workflows/agent-os-image.ym "release workflow must build the two supported container architectures" [ "$(grep -c '^ push: false$' "$IMAGE_WORKFLOW")" -eq 1 ] || \ fail "the read-only validation job must build without publishing" -[ "$(grep -c '^ push: true$' "$IMAGE_WORKFLOW")" -eq 2 ] || \ - fail "only the protected candidate and main publication builds may push images" +[ "$(grep -c '^ push: true$' "$IMAGE_WORKFLOW")" -eq 1 ] || \ + fail "only the protected main publication build may use mutable-tag push mode" +assert_grep 'push-by-digest=true' "$IMAGE_WORKFLOW" \ + "the release candidate must publish without a mutable image tag" assert_grep 'id: build' "$ROOT/.github/workflows/agent-os-image.yml" \ "release workflow must expose its build result" assert_grep 'steps.build.outputs.digest' "$ROOT/.github/workflows/agent-os-image.yml" \ @@ -321,6 +349,10 @@ assert_grep 'docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051' "$ "release metadata action must be pinned to the reviewed full SHA" assert_grep 'docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8' "$ROOT/.github/workflows/agent-os-image.yml" \ "release build action must be pinned to the reviewed full SHA" +assert_grep 'AGENT_OS_SOURCE_UPDATE_POLICY=immutable' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "candidate and release runtimes must disable moving-main self-updates" +assert_grep 'TRUSTED_SOURCE_REF=$SOURCE_COMMIT' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "candidate runtime provenance must fetch its immutable commit" if grep -E 'uses: (actions/checkout|docker/[^@]+|oras-project/setup-oras)@v[0-9]+' "$ROOT/.github/workflows/agent-os-image.yml" >/dev/null; then fail "release workflow must not use mutable major action tags" fi diff --git a/tests/fm-update.test.sh b/tests/fm-update.test.sh index 14628e303..e60810b9d 100755 --- a/tests/fm-update.test.sh +++ b/tests/fm-update.test.sh @@ -291,6 +291,25 @@ test_unsafe_secondmate_home_skipped_before_git_update() { pass "T11 unsafe secondmate home is not fast-forwarded" } +test_immutable_image_source_refuses_self_update() { + local w out status before + w=$(new_world t12) + bump_origin "$w" instr + before=$(git -C "$w/main" rev-parse HEAD) + + set +e + out=$(AGENT_OS_SOURCE_UPDATE_POLICY=immutable FM_ROOT_OVERRIDE="$w/main" FM_HOME="$w/home" "$UPDATE" 2>&1) + status=$? + set -e + + [ "$status" -eq 2 ] || fail "immutable image update exited $status, expected 2" + assert_contains "$out" "self-update is disabled for immutable image source" \ + "immutable image source blocks moving-main self-update" + [ "$(git -C "$w/main" rev-parse HEAD)" = "$before" ] \ + || fail "immutable image source moved during refused self-update" + pass "T12 immutable image source refuses self-update" +} + test_updates_main_and_secondmate test_reread_gate_is_instruction_only test_dirty_secondmate_skipped @@ -300,5 +319,6 @@ test_registry_backstop_dedup_and_self_exclusion test_firstmate_wrong_branch_skipped test_firstmate_detached_head_skipped test_unsafe_secondmate_home_skipped_before_git_update +test_immutable_image_source_refuses_self_update echo "# all fm-update tests passed" From 89199de22363e295d3fed249e390cc08fcacfecd Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 19:14:00 +0200 Subject: [PATCH 41/56] no-mistakes(review): Captain, secure immutable release rollback and promotion --- .github/workflows/agent-os-image.yml | 197 ++++++++++++++++++++------ Dockerfile | 2 + bin/agent-os-container-entrypoint.sh | 20 ++- bin/agent-os-runtime-source.sh | 78 ++++++++++ bin/agent-os-source-bundle.sh | 25 +++- bin/fm-update.sh | 24 +++- tests/agent-os-container.test.sh | 46 ++++-- tests/agent-os-runtime-source.test.sh | 71 ++++++++++ tests/fm-update.test.sh | 31 ++-- 9 files changed, 418 insertions(+), 76 deletions(-) create mode 100755 bin/agent-os-runtime-source.sh create mode 100755 tests/agent-os-runtime-source.test.sh diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index 829b4451b..11daa685e 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -105,6 +105,7 @@ jobs: ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - name: Prepare exact source bundle + if: ${{ !startsWith(github.ref, 'refs/tags/') }} id: source run: | set -eu @@ -119,13 +120,7 @@ jobs: AGENT_OS_SOURCE_MODE=event AGENT_OS_SOURCE_EVENT_COMMIT="${{ github.event.pull_request.head.sha }}" \ bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" else - if [ "${GITHUB_REF#refs/tags/}" != "$GITHUB_REF" ]; then - record_commit=$(trusted_git for-each-ref --format='%(contents)' "$GITHUB_REF" | sed -n 's/^release-record-commit=\([0-9a-f]\{40\}\)$/\1/p') - AGENT_OS_SOURCE_MODE=release AGENT_OS_SOURCE_RELEASE_TAG="${GITHUB_REF#refs/tags/}" \ - AGENT_OS_RELEASE_RECORD_COMMIT="$record_commit" bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" - else - bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" - fi + bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/source-metadata" fi cat "$RUNNER_TEMP/source-metadata" >> "$GITHUB_OUTPUT" @@ -160,6 +155,7 @@ jobs: name: Publish linux/amd64 and linux/arm64 runs-on: ubuntu-latest permissions: + actions: read contents: read packages: write steps: @@ -168,25 +164,6 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Prepare immutable release source - if: startsWith(github.ref, 'refs/tags/') - id: release_source - run: | - set -eu - trusted_git() { - env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ - GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 \ - /usr/bin/git -c credential.helper= -c core.hooksPath=/dev/null \ - -c http.proxy= -c https.proxy= "$@" - } - record_commit=$(trusted_git for-each-ref --format='%(contents)' "$GITHUB_REF" | sed -n 's/^release-record-commit=\([0-9a-f]\{40\}\)$/\1/p') - AGENT_OS_SOURCE_MODE=release AGENT_OS_SOURCE_RELEASE_TAG="${GITHUB_REF#refs/tags/}" \ - AGENT_OS_RELEASE_RECORD_COMMIT="$record_commit" bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/release-source-metadata" - trusted_git show "$record_commit:image/releases/${GITHUB_REF#refs/tags/}.json" > "$RUNNER_TEMP/release-record.json" - [ "$(sha256sum "$RUNNER_TEMP/release-record.json" | awk '{print $1}')" = \ - "$(sed -n 's/^release_record_sha256=//p' "$RUNNER_TEMP/release-source-metadata")" ] - cat "$RUNNER_TEMP/release-source-metadata" >> "$GITHUB_OUTPUT" - - name: Prepare exact release candidate source if: github.ref == 'refs/heads/main' id: candidate_source @@ -211,6 +188,119 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Prepare immutable release source + if: startsWith(github.ref, 'refs/tags/') + id: release_source + run: | + set -eu + trusted_git() { + env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 \ + /usr/bin/git -c credential.helper= -c core.hooksPath=/dev/null \ + -c http.proxy= -c https.proxy= "$@" + } + record_commit=$(trusted_git for-each-ref --format='%(contents)' "$GITHUB_REF" | sed -n 's/^release-record-commit=\([0-9a-f]\{40\}\)$/\1/p') + trusted_git show "$record_commit:image/releases/${GITHUB_REF#refs/tags/}.json" > "$RUNNER_TEMP/release-record.json" + candidate_record_digest=$(jq -er '.candidate_record_digest' "$RUNNER_TEMP/release-record.json") + oras manifest fetch "$IMAGE@$candidate_record_digest" > "$RUNNER_TEMP/release-source-candidate-manifest.json" + [ "sha256:$(sha256sum "$RUNNER_TEMP/release-source-candidate-manifest.json" | awk '{print $1}')" = "$candidate_record_digest" ] + bootstrap_blob=$(jq -er '[.layers[] | select(.mediaType == "application/vnd.akua.agent-os.bootstrap.v1+tar" and .annotations["org.opencontainers.image.title"] == "agent-os-bootstrap.tar")] | select(length == 1) | .[0].digest' "$RUNNER_TEMP/release-source-candidate-manifest.json") + oras blob fetch --output "$RUNNER_TEMP/agent-os-bootstrap.tar" "$IMAGE@$bootstrap_blob" + [ "$(sha256sum "$RUNNER_TEMP/agent-os-bootstrap.tar" | awk '{print $1}')" = "$(jq -r .bootstrap_archive_sha256 "$RUNNER_TEMP/release-record.json")" ] + AGENT_OS_SOURCE_MODE=release AGENT_OS_SOURCE_RELEASE_TAG="${GITHUB_REF#refs/tags/}" \ + AGENT_OS_RELEASE_RECORD_COMMIT="$record_commit" \ + AGENT_OS_RELEASE_BOOTSTRAP_ARCHIVE="$RUNNER_TEMP/agent-os-bootstrap.tar" \ + bin/agent-os-source-bundle.sh > "$RUNNER_TEMP/release-source-metadata" + [ "$(sha256sum "$RUNNER_TEMP/release-record.json" | awk '{print $1}')" = \ + "$(sed -n 's/^release_record_sha256=//p' "$RUNNER_TEMP/release-source-metadata")" ] + cat "$RUNNER_TEMP/release-source-metadata" >> "$GITHUB_OUTPUT" + + - name: Reserve or recover commit-indexed release candidate + if: github.ref == 'refs/heads/main' + id: candidate_reservation + env: + GH_TOKEN: ${{ github.token }} + run: | + set -eu + printf '{}\n' > "$RUNNER_TEMP/candidate-reservation-config.json" + jq -cnS \ + --arg commit '${{ steps.candidate_source.outputs.commit }}' \ + --arg tree '${{ steps.candidate_source.outputs.tree }}' \ + --arg source '${{ steps.candidate_source.outputs.source_sha256 }}' \ + '{schema_version:1,commit:$commit,tree:$tree,source_archive_sha256:$source}' \ + > "$RUNNER_TEMP/candidate-reservation.json" + oras blob push --media-type application/vnd.oci.empty.v1+json --descriptor \ + "$IMAGE" "$RUNNER_TEMP/candidate-reservation-config.json" > "$RUNNER_TEMP/candidate-reservation-config-descriptor.json" + oras blob push --media-type application/vnd.akua.agent-os.candidate-reservation.v1+json --descriptor \ + "$IMAGE" "$RUNNER_TEMP/candidate-reservation.json" > "$RUNNER_TEMP/candidate-reservation-descriptor.json" + config_descriptor=$(jq -cS '{mediaType,digest,size}' "$RUNNER_TEMP/candidate-reservation-config-descriptor.json") + reservation_descriptor=$(jq -cS '{mediaType,digest,size,annotations:{"org.opencontainers.image.title":"candidate-reservation.json"}}' "$RUNNER_TEMP/candidate-reservation-descriptor.json") + jq -cnS --argjson config "$config_descriptor" --argjson reservation "$reservation_descriptor" \ + '{schemaVersion:2,mediaType:"application/vnd.oci.image.manifest.v1+json",artifactType:"application/vnd.akua.agent-os.candidate-reservation.v1",config:$config,layers:[$reservation]}' \ + > "$RUNNER_TEMP/candidate-reservation-manifest.json" + reservation_digest="sha256:$(sha256sum "$RUNNER_TEMP/candidate-reservation-manifest.json" | awk '{print $1}')" + reservation_size=$(wc -c < "$RUNNER_TEMP/candidate-reservation-manifest.json" | tr -d ' ') + existed=false + if oras manifest fetch "$IMAGE@$reservation_digest" > "$RUNNER_TEMP/candidate-reservation-existing.json" 2> "$RUNNER_TEMP/candidate-reservation.err"; then + cmp "$RUNNER_TEMP/candidate-reservation-manifest.json" "$RUNNER_TEMP/candidate-reservation-existing.json" + existed=true + elif grep -Eiq 'manifest unknown|not found|404' "$RUNNER_TEMP/candidate-reservation.err"; then + oras manifest push --descriptor "$IMAGE@$reservation_digest" \ + "$RUNNER_TEMP/candidate-reservation-manifest.json" > "$RUNNER_TEMP/candidate-reservation-push.json" + [ "$(jq -r .digest "$RUNNER_TEMP/candidate-reservation-push.json")" = "$reservation_digest" ] + else + cat "$RUNNER_TEMP/candidate-reservation.err" >&2 + exit 1 + fi + oras manifest fetch "$IMAGE@$reservation_digest" > "$RUNNER_TEMP/candidate-reservation-readback.json" + cmp "$RUNNER_TEMP/candidate-reservation-manifest.json" "$RUNNER_TEMP/candidate-reservation-readback.json" + printf 'reservation_digest=%s\nreservation_size=%s\n' "$reservation_digest" "$reservation_size" >> "$GITHUB_OUTPUT" + if [ "$existed" = false ]; then + printf 'build=true\nbootstrap_sha256=%s\n' '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' >> "$GITHUB_OUTPUT" + exit 0 + fi + oras discover --distribution-spec v1.1-referrers-api --artifact-type application/vnd.akua.agent-os.release-candidate.v1 \ + --format json --depth 1 "$IMAGE@$reservation_digest" > "$RUNNER_TEMP/candidate-reservation-referrers.json" + referrer_count=$(jq -er '.referrers | length' "$RUNNER_TEMP/candidate-reservation-referrers.json") + if [ "$referrer_count" -eq 0 ]; then + echo "::error::partial candidate reservation requires trusted recovery" + exit 1 + fi + [ "$referrer_count" -eq 1 ] || { echo "::error::candidate reservation has conflicting records"; exit 1; } + candidate_record_digest=$(jq -er '.referrers[0].digest' "$RUNNER_TEMP/candidate-reservation-referrers.json") + oras manifest fetch "$IMAGE@$candidate_record_digest" > "$RUNNER_TEMP/candidate-recovered-manifest.json" + artifact_name="agent-os-candidate-record-$GITHUB_SHA" + gh api --method GET "repos/$GITHUB_REPOSITORY/actions/artifacts?name=$artifact_name&per_page=100" \ + > "$RUNNER_TEMP/candidate-trusted-artifacts.json" + trusted_artifact_count=$(jq -er --arg sha "$GITHUB_SHA" \ + '[.artifacts[] | select(.expired == false and .workflow_run.head_sha == $sha)] | length' \ + "$RUNNER_TEMP/candidate-trusted-artifacts.json") + [ "$trusted_artifact_count" -eq 1 ] || { + echo "::error::partial candidate reservation requires trusted recovery" + exit 1 + } + trusted_artifact_id=$(jq -er --arg sha "$GITHUB_SHA" \ + '[.artifacts[] | select(.expired == false and .workflow_run.head_sha == $sha)] | .[0].id' \ + "$RUNNER_TEMP/candidate-trusted-artifacts.json") + mkdir "$RUNNER_TEMP/candidate-trusted-artifact" + gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$trusted_artifact_id/zip" \ + > "$RUNNER_TEMP/candidate-trusted-artifact.zip" + unzip -q "$RUNNER_TEMP/candidate-trusted-artifact.zip" -d "$RUNNER_TEMP/candidate-trusted-artifact" + cmp "$RUNNER_TEMP/candidate-trusted-artifact/candidate-manifest.json" \ + "$RUNNER_TEMP/candidate-recovered-manifest.json" + record_blob=$(jq -er '[.layers[] | select(.mediaType == "application/vnd.akua.agent-os.release-candidate.v1+json")] | select(length == 1) | .[0].digest' "$RUNNER_TEMP/candidate-recovered-manifest.json") + bootstrap_blob=$(jq -er '[.layers[] | select(.mediaType == "application/vnd.akua.agent-os.bootstrap.v1+tar" and .annotations["org.opencontainers.image.title"] == "agent-os-bootstrap.tar")] | select(length == 1) | .[0].digest' "$RUNNER_TEMP/candidate-recovered-manifest.json") + oras blob fetch --output "$RUNNER_TEMP/candidate-recovered-record.json" "$IMAGE@$record_blob" + oras blob fetch --output "$RUNNER_TEMP/candidate-recovered-bootstrap.tar" "$IMAGE@$bootstrap_blob" + cmp "$RUNNER_TEMP/candidate-trusted-artifact/candidate-record.json" "$RUNNER_TEMP/candidate-recovered-record.json" + cmp "$RUNNER_TEMP/candidate-trusted-artifact/agent-os-bootstrap.tar" "$RUNNER_TEMP/candidate-recovered-bootstrap.tar" + cp "$RUNNER_TEMP/candidate-trusted-artifact/agent-os-bootstrap.tar" image/agent-os-bootstrap.tar + image_digest=$(jq -er '.image_digest | select(test("^sha256:[0-9a-f]{64}$"))' "$RUNNER_TEMP/candidate-trusted-artifact/candidate-record.json") + bootstrap_sha=$(jq -er '.bootstrap_archive_sha256 | select(test("^[0-9a-f]{64}$"))' "$RUNNER_TEMP/candidate-trusted-artifact/candidate-record.json") + [ "$(sha256sum image/agent-os-bootstrap.tar | awk '{print $1}')" = "$bootstrap_sha" ] + printf 'build=false\ncandidate_record_digest=%s\nimage_digest=%s\nbootstrap_sha256=%s\n' \ + "$candidate_record_digest" "$image_digest" "$bootstrap_sha" >> "$GITHUB_OUTPUT" + - name: Reject legacy candidate coordinates if: github.ref == 'refs/heads/main' run: | @@ -227,7 +317,7 @@ jobs: fi - name: Build release candidate once - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' && steps.candidate_reservation.outputs.build == 'true' id: candidate_build uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 with: @@ -250,7 +340,7 @@ jobs: if: github.ref == 'refs/heads/main' id: candidate_record env: - CANDIDATE_BUILD_DIGEST: ${{ steps.candidate_build.outputs.digest }} + CANDIDATE_BUILD_DIGEST: ${{ steps.candidate_build.outputs.digest || steps.candidate_reservation.outputs.image_digest }} run: | set -eu verify_candidate_attestations() { @@ -331,7 +421,7 @@ jobs: --arg commit '${{ steps.candidate_source.outputs.commit }}' \ --arg tree '${{ steps.candidate_source.outputs.tree }}' \ --arg sourceArchive '${{ steps.candidate_source.outputs.source_sha256 }}' \ - --arg bootstrapArchive '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' \ + --arg bootstrapArchive '${{ steps.candidate_reservation.outputs.bootstrap_sha256 }}' \ --arg package "$(sha256sum tools/agent-os/packages/firstmate/package.k | awk '{print $1}')" \ --arg schema "$(sha256sum tools/agent-os/packages/firstmate/inputs.example.yaml | awk '{print $1}')" \ --arg quickstart "$(sha256sum docs/kubernetes.md | awk '{print $1}')" \ @@ -344,7 +434,7 @@ jobs: jq -e \ --arg commit '${{ steps.candidate_source.outputs.commit }}' --arg tree '${{ steps.candidate_source.outputs.tree }}' \ --arg sourceArchive '${{ steps.candidate_source.outputs.source_sha256 }}' \ - --arg bootstrapArchive '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' \ + --arg bootstrapArchive '${{ steps.candidate_reservation.outputs.bootstrap_sha256 }}' \ --arg package "$(sha256sum tools/agent-os/packages/firstmate/package.k | awk '{print $1}')" \ --arg schema "$(sha256sum tools/agent-os/packages/firstmate/inputs.example.yaml | awk '{print $1}')" \ --arg quickstart "$(sha256sum docs/kubernetes.md | awk '{print $1}')" \ @@ -362,10 +452,20 @@ jobs: "$IMAGE" "$RUNNER_TEMP/candidate-config.json" > "$RUNNER_TEMP/candidate-config-descriptor.json" oras blob push --media-type application/vnd.akua.agent-os.release-candidate.v1+json --descriptor \ "$IMAGE" "$RUNNER_TEMP/candidate-record.json" > "$RUNNER_TEMP/candidate-record-descriptor.json" + oras blob push --media-type application/vnd.akua.agent-os.bootstrap.v1+tar --descriptor \ + "$IMAGE" image/agent-os-bootstrap.tar > "$RUNNER_TEMP/candidate-bootstrap-descriptor.json" + [ "$(jq -r .digest "$RUNNER_TEMP/candidate-bootstrap-descriptor.json")" = \ + "sha256:${{ steps.candidate_reservation.outputs.bootstrap_sha256 }}" ] config_descriptor=$(jq -cS '{mediaType,digest,size}' "$RUNNER_TEMP/candidate-config-descriptor.json") record_descriptor=$(jq -cS '{mediaType,digest,size,annotations:{"org.opencontainers.image.title":"candidate-record.json"}}' "$RUNNER_TEMP/candidate-record-descriptor.json") + bootstrap_descriptor=$(jq -cS '{mediaType,digest,size,annotations:{"org.opencontainers.image.title":"agent-os-bootstrap.tar"}}' "$RUNNER_TEMP/candidate-bootstrap-descriptor.json") + reservation_subject=$(jq -cnS \ + --arg digest '${{ steps.candidate_reservation.outputs.reservation_digest }}' \ + --argjson size '${{ steps.candidate_reservation.outputs.reservation_size }}' \ + '{mediaType:"application/vnd.oci.image.manifest.v1+json",digest:$digest,size:$size}') jq -cnS --argjson config "$config_descriptor" --argjson record "$record_descriptor" \ - '{schemaVersion:2,mediaType:"application/vnd.oci.image.manifest.v1+json",artifactType:"application/vnd.akua.agent-os.release-candidate.v1",config:$config,layers:[$record]}' \ + --argjson bootstrap "$bootstrap_descriptor" --argjson subject "$reservation_subject" \ + '{schemaVersion:2,mediaType:"application/vnd.oci.image.manifest.v1+json",artifactType:"application/vnd.akua.agent-os.release-candidate.v1",subject:$subject,config:$config,layers:[$record,$bootstrap]}' \ > "$RUNNER_TEMP/candidate-manifest.json" candidate_record_digest="sha256:$(sha256sum "$RUNNER_TEMP/candidate-manifest.json" | awk '{print $1}')" if oras manifest fetch "$IMAGE@$candidate_record_digest" > "$RUNNER_TEMP/candidate-existing-manifest.json" 2> "$RUNNER_TEMP/candidate-existing-manifest.err"; then @@ -383,19 +483,28 @@ jobs: fi oras manifest fetch "$IMAGE@$candidate_record_digest" > "$RUNNER_TEMP/candidate-readback-manifest.json" cmp "$RUNNER_TEMP/candidate-manifest.json" "$RUNNER_TEMP/candidate-readback-manifest.json" - record_blob_digest=$(jq -er '.layers | select(length == 1) | .[0].digest' "$RUNNER_TEMP/candidate-readback-manifest.json") + record_blob_digest=$(jq -er '[.layers[] | select(.mediaType == "application/vnd.akua.agent-os.release-candidate.v1+json")] | select(length == 1) | .[0].digest' "$RUNNER_TEMP/candidate-readback-manifest.json") oras blob fetch --output "$RUNNER_TEMP/candidate-record-readback.json" "$IMAGE@$record_blob_digest" cmp "$RUNNER_TEMP/candidate-record.json" "$RUNNER_TEMP/candidate-record-readback.json" + oras discover --distribution-spec v1.1-referrers-api --artifact-type application/vnd.akua.agent-os.release-candidate.v1 \ + --format json --depth 1 "$IMAGE@${{ steps.candidate_reservation.outputs.reservation_digest }}" > "$RUNNER_TEMP/candidate-final-referrers.json" + jq -e --arg digest "$candidate_record_digest" '.referrers | length == 1 and .[0].digest == $digest' \ + "$RUNNER_TEMP/candidate-final-referrers.json" >/dev/null || { + echo "::error::candidate reservation does not resolve to exactly one verified record" + exit 1 + } + cp image/agent-os-bootstrap.tar "$RUNNER_TEMP/agent-os-bootstrap.tar" printf 'candidate_record_digest=%s\nimage_digest=%s\n' "$candidate_record_digest" "$actual_image_digest" >> "$GITHUB_OUTPUT" - name: Retain candidate record for release preparation - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' && steps.candidate_reservation.outputs.build == 'true' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: agent-os-candidate-record-${{ github.sha }} path: | ${{ runner.temp }}/candidate-record.json ${{ runner.temp }}/candidate-manifest.json + ${{ runner.temp }}/agent-os-bootstrap.tar if-no-files-found: error retention-days: 90 @@ -449,8 +558,8 @@ jobs: [ "sha256:$(sha256sum "$RUNNER_TEMP/release-candidate-manifest.json" | awk '{print $1}')" = "$candidate_record_digest" ] candidate_record_blob=$(jq -er ' select(.schemaVersion == 2 and .artifactType == "application/vnd.akua.agent-os.release-candidate.v1") - | .layers | select(length == 1) | .[0] - | select(.mediaType == "application/vnd.akua.agent-os.release-candidate.v1+json") | .digest' \ + | [.layers[] | select(.mediaType == "application/vnd.akua.agent-os.release-candidate.v1+json")] + | select(length == 1) | .[0].digest' \ "$RUNNER_TEMP/release-candidate-manifest.json") oras blob fetch --output "$RUNNER_TEMP/release-candidate-record.json" "$IMAGE@$candidate_record_blob" jq -cS '{schema_version,source_mode,commit,tree,source_archive_sha256,bootstrap_archive_sha256,package_sha256,schema_sha256,quickstart_sha256,image_repository,image_digest,platform_manifests,platform_manifests_sha256,sbom_sha256,provenance_sha256,buildkit_outputs_sha256}' \ @@ -507,24 +616,18 @@ jobs: [ "$actual_sbom_sha256" = "$(jq -r .sbom_sha256 "$RUNNER_TEMP/release-record.json")" ] [ "$actual_provenance_sha256" = "$(jq -r .provenance_sha256 "$RUNNER_TEMP/release-record.json")" ] if oras manifest fetch --descriptor "$IMAGE:$release_tag" > "$RUNNER_TEMP/release-tag-descriptor.json" 2> "$RUNNER_TEMP/release-tag.err"; then - [ "$(jq -r .digest "$RUNNER_TEMP/release-tag-descriptor.json")" = "$RELEASE_DIGEST" ] || { - echo "::error::release tag coordinate already contains a different manifest" - exit 1 - } + echo "::error::release registry tag coordinate already exists" + exit 1 elif grep -Eiq 'manifest unknown|not found|404' "$RUNNER_TEMP/release-tag.err"; then - oras tag "$IMAGE@$RELEASE_DIGEST" "$release_tag" + : else cat "$RUNNER_TEMP/release-tag.err" >&2 exit 1 fi - oras manifest fetch --descriptor "$IMAGE:$release_tag" > "$RUNNER_TEMP/release-tag-readback.json" - [ "$(jq -r .digest "$RUNNER_TEMP/release-tag-readback.json")" = "$RELEASE_DIGEST" ] - docker buildx imagetools inspect --raw "$IMAGE:$release_tag" > "$RUNNER_TEMP/published-tag-index.json" - [ "sha256:$(sha256sum "$RUNNER_TEMP/published-tag-index.json" | awk '{print $1}')" = "$RELEASE_DIGEST" ] { echo '### Agent OS release' echo - echo "Digest: \`$RELEASE_DIGEST\`" + echo "Canonical install digest: \`$IMAGE@$RELEASE_DIGEST\`" } >> "$GITHUB_STEP_SUMMARY" - name: Record protected-main publication @@ -537,6 +640,6 @@ jobs: echo '### Agent OS image' echo echo "Digest: \`$BUILD_DIGEST\`" - echo "Release candidate: \`${{ steps.candidate_record.outputs.image_digest }}\`" - echo "Candidate record: \`${{ steps.candidate_record.outputs.candidate_record_digest }}\`" + echo "Release candidate: \`${{ steps.candidate_record.outputs.image_digest || steps.candidate_reservation.outputs.image_digest }}\`" + echo "Candidate record: \`${{ steps.candidate_record.outputs.candidate_record_digest || steps.candidate_reservation.outputs.candidate_record_digest }}\`" } >> "$GITHUB_STEP_SUMMARY" diff --git a/Dockerfile b/Dockerfile index cbe7d642b..31b37ddcb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,6 +46,7 @@ RUN set -eu; \ rm -f /tmp/verified-source.tar /tmp/agent-os-source.tar /tmp/agent-os-bootstrap.tar /tmp/agent-os-source.attestation; \ printf '%s\n' "$AGENT_OS_SOURCE_COMMIT" > /opt/agent-os-source.commit; \ printf '%s\n' "$AGENT_OS_SOURCE_TREE" > /opt/agent-os-source.tree; \ + printf '%s\n' "$source_sha" > /opt/agent-os-source.sha256; \ printf '%s\n' "$AGENT_OS_SOURCE_BRANCH" > /opt/agent-os-source.branch; \ printf '%s\n' "$AGENT_OS_SOURCE_ORIGIN" > /opt/agent-os-source.origin; \ printf '%s\n' "$AGENT_OS_SOURCE_MODE" > /opt/agent-os-source.mode; \ @@ -226,6 +227,7 @@ COPY --from=source-bootstrap /opt/agent-os /opt/agent-os COPY --from=source-bootstrap /opt/agent-os-bootstrap.git /opt/agent-os-bootstrap.git COPY --from=source-bootstrap /opt/agent-os-source.commit /opt/agent-os-source.commit COPY --from=source-bootstrap /opt/agent-os-source.tree /opt/agent-os-source.tree +COPY --from=source-bootstrap /opt/agent-os-source.sha256 /opt/agent-os-source.sha256 COPY --from=source-bootstrap /opt/agent-os-source.branch /opt/agent-os-source.branch COPY --from=source-bootstrap /opt/agent-os-source.origin /opt/agent-os-source.origin COPY --from=source-bootstrap /opt/agent-os-source.mode /opt/agent-os-source.mode diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh index a9d20532d..40034e7c4 100755 --- a/bin/agent-os-container-entrypoint.sh +++ b/bin/agent-os-container-entrypoint.sh @@ -4,10 +4,11 @@ set -eu FM_HOME=${FM_HOME:-/home/agent} export FM_HOME HOME="$FM_HOME" -FM_ROOT=${FM_ROOT_OVERRIDE:-$FM_HOME/firstmate} +REQUESTED_FM_ROOT=${FM_ROOT_OVERRIDE:-$FM_HOME/firstmate} IMAGE_SOURCE=${AGENT_OS_IMAGE_SOURCE:-/opt/agent-os-bootstrap.git} SOURCE_COMMIT=$(cat /opt/agent-os-source.commit) SOURCE_TREE=$(cat /opt/agent-os-source.tree) +SOURCE_SHA=$(cat /opt/agent-os-source.sha256) SOURCE_BRANCH=$(cat /opt/agent-os-source.branch) SOURCE_ORIGIN=$(cat /opt/agent-os-source.origin) SOURCE_MODE=$(cat /opt/agent-os-source.mode) @@ -15,18 +16,17 @@ SOURCE_REF=$(cat /opt/agent-os-source.ref) case "$SOURCE_BRANCH" in ''|*[!A-Za-z0-9._/-]*|/*|*/|*..*) echo "error: image source branch provenance is invalid" >&2; exit 2 ;; esac [ "$SOURCE_ORIGIN" = https://github.com/akua-dev/agent-os.git ] || { echo "error: image source origin is invalid" >&2; exit 2; } case "$SOURCE_MODE" in - main) [ "$SOURCE_REF" = refs/heads/main ] || exit 2; TRUSTED_SOURCE_REF=$SOURCE_REF; TRUSTED_REF=refs/remotes/agent-os-verified/main; AGENT_OS_SOURCE_UPDATE_POLICY=fast-forward ;; + main) [ "$SOURCE_REF" = refs/heads/main ] || exit 2; TRUSTED_SOURCE_REF=$SOURCE_REF; TRUSTED_REF=refs/remotes/agent-os-verified/main; IMMUTABLE_SOURCE=false ;; candidate) [ "$SOURCE_REF" = "refs/agent-os/candidates/$SOURCE_COMMIT" ] || exit 2 TRUSTED_SOURCE_REF=$SOURCE_COMMIT TRUSTED_REF=refs/remotes/agent-os-verified/candidate - AGENT_OS_SOURCE_UPDATE_POLICY=immutable + IMMUTABLE_SOURCE=true ;; - release) [[ "$SOURCE_REF" =~ ^refs/tags/v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] || exit 2; TRUSTED_SOURCE_REF=$SOURCE_REF; TRUSTED_REF=refs/remotes/agent-os-verified/release; AGENT_OS_SOURCE_UPDATE_POLICY=immutable ;; + release) [[ "$SOURCE_REF" =~ ^refs/tags/v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] || exit 2; TRUSTED_SOURCE_REF=$SOURCE_REF; TRUSTED_REF=refs/remotes/agent-os-verified/release; IMMUTABLE_SOURCE=true ;; event) echo "error: pull-request validation images are not runnable" >&2; exit 2 ;; *) echo "error: image source mode is invalid" >&2; exit 2 ;; esac -export AGENT_OS_SOURCE_UPDATE_POLICY IMAGE_REF="refs/remotes/agent-os-image/$SOURCE_BRANCH" GIT_BIN=/usr/bin/git [ -x "$GIT_BIN" ] || { echo "error: trusted Git executable is unavailable" >&2; exit 2; } @@ -81,7 +81,14 @@ mkdir -p \ "$HOME/.bun" \ "$HOME/.cargo" -if [ ! -e "$FM_ROOT/.git" ]; then +if [ "$IMMUTABLE_SOURCE" = true ]; then + FM_ROOT=$(FM_HOME="$FM_HOME" AGENT_OS_IMAGE_SOURCE="$IMAGE_SOURCE" \ + AGENT_OS_SOURCE_COMMIT="$SOURCE_COMMIT" AGENT_OS_SOURCE_TREE="$SOURCE_TREE" \ + AGENT_OS_SOURCE_SHA256="$SOURCE_SHA" AGENT_OS_SOURCE_BRANCH="$SOURCE_BRANCH" \ + AGENT_OS_SOURCE_ORIGIN="$SOURCE_ORIGIN" AGENT_OS_SOURCE_MODE="$SOURCE_MODE" \ + "$(dirname "$0")/agent-os-runtime-source.sh") +elif [ ! -e "$REQUESTED_FM_ROOT/.git" ]; then + FM_ROOT=$REQUESTED_FM_ROOT [ ! -e "$FM_ROOT" ] || { echo "error: canonical FM_ROOT exists without Git provenance" >&2; exit 2; } trusted_git -c protocol.file.allow=always clone --no-local --branch "$SOURCE_BRANCH" "$IMAGE_SOURCE" "$FM_ROOT" trusted_git -C "$FM_ROOT" remote set-url origin "$SOURCE_ORIGIN" @@ -92,6 +99,7 @@ if [ ! -e "$FM_ROOT/.git" ]; then } rm -rf "$FM_ROOT/.git/hooks" else + FM_ROOT=$REQUESTED_FM_ROOT [ -d "$FM_ROOT/.git" ] || { echo "error: canonical FM_ROOT Git metadata is invalid" >&2; exit 2; } validate_git_config [ -z "$(trusted_git -C "$FM_ROOT" status --porcelain)" ] || { echo "error: canonical FM_ROOT is not clean" >&2; exit 2; } diff --git a/bin/agent-os-runtime-source.sh b/bin/agent-os-runtime-source.sh new file mode 100755 index 000000000..fd7017748 --- /dev/null +++ b/bin/agent-os-runtime-source.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -eu + +FM_HOME=${FM_HOME:?FM_HOME is required} +IMAGE_SOURCE=${AGENT_OS_IMAGE_SOURCE:?AGENT_OS_IMAGE_SOURCE is required} +SOURCE_COMMIT=${AGENT_OS_SOURCE_COMMIT:?AGENT_OS_SOURCE_COMMIT is required} +SOURCE_TREE=${AGENT_OS_SOURCE_TREE:?AGENT_OS_SOURCE_TREE is required} +SOURCE_SHA=${AGENT_OS_SOURCE_SHA256:?AGENT_OS_SOURCE_SHA256 is required} +SOURCE_BRANCH=${AGENT_OS_SOURCE_BRANCH:?AGENT_OS_SOURCE_BRANCH is required} +SOURCE_ORIGIN=${AGENT_OS_SOURCE_ORIGIN:?AGENT_OS_SOURCE_ORIGIN is required} +SOURCE_MODE=${AGENT_OS_SOURCE_MODE:?AGENT_OS_SOURCE_MODE is required} +GIT_BIN=/usr/bin/git + +[[ "$SOURCE_COMMIT" =~ ^[0-9a-f]{40}$ ]] || { echo "error: immutable source commit is invalid" >&2; exit 2; } +[[ "$SOURCE_TREE" =~ ^[0-9a-f]{40}$ ]] || { echo "error: immutable source tree is invalid" >&2; exit 2; } +[[ "$SOURCE_SHA" =~ ^[0-9a-f]{64}$ ]] || { echo "error: immutable source digest is invalid" >&2; exit 2; } +case "$SOURCE_MODE" in candidate|release) ;; *) echo "error: runtime source mode is not immutable" >&2; exit 2 ;; esac +[ -x "$GIT_BIN" ] || { echo "error: trusted Git executable is unavailable" >&2; exit 2; } + +trusted_git() { + env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 \ + "$GIT_BIN" -c credential.helper= -c core.hooksPath=/dev/null \ + -c http.proxy= -c https.proxy= "$@" +} + +runtime_root="$FM_HOME/runtime-sources" +key="$SOURCE_COMMIT-$SOURCE_SHA" +target="$runtime_root/$key" +lock="$runtime_root/.$key.materializing" +marker="mode=$SOURCE_MODE +commit=$SOURCE_COMMIT +source_sha256=$SOURCE_SHA" + +validate_source() { + local root=$1 actual_marker + [ -d "$root/.git" ] || { echo "error: immutable runtime source Git metadata is unavailable" >&2; return 1; } + [ "$(trusted_git -C "$root" rev-parse HEAD)" = "$SOURCE_COMMIT" ] || return 1 + [ "$(trusted_git -C "$root" rev-parse 'HEAD^{tree}')" = "$SOURCE_TREE" ] || return 1 + [ "$(trusted_git -C "$root" symbolic-ref --quiet --short HEAD)" = "$SOURCE_BRANCH" ] || return 1 + [ "$(trusted_git -C "$root" remote get-url origin)" = "$SOURCE_ORIGIN" ] || return 1 + [ -z "$(trusted_git -C "$root" status --porcelain --untracked-files=all)" ] || return 1 + [ -f "$root/.git/agent-os-runtime-source" ] || return 1 + actual_marker=$(cat "$root/.git/agent-os-runtime-source") + [ "$actual_marker" = "$marker" ] || return 1 + ! find "$root/.git/hooks" -mindepth 1 -print -quit 2>/dev/null | grep -q . || return 1 +} + +mkdir -p "$runtime_root" +if [ -e "$target" ]; then + validate_source "$target" || { echo "error: immutable runtime source failed exact verification" >&2; exit 2; } + rmdir "$lock" 2>/dev/null || true + printf '%s\n' "$target" + exit 0 +fi + +if ! mkdir "$lock" 2>/dev/null; then + if [ -e "$target" ] && validate_source "$target"; then + printf '%s\n' "$target" + exit 0 + fi + echo "error: partial immutable runtime source materialization requires recovery" >&2 + exit 2 +fi + +partial="$lock/source" +if ! trusted_git -c protocol.file.allow=always clone --no-local --branch "$SOURCE_BRANCH" "$IMAGE_SOURCE" "$partial"; then + echo "error: immutable runtime source materialization failed" >&2 + exit 2 +fi +trusted_git -C "$partial" remote set-url origin "$SOURCE_ORIGIN" +rm -rf "$partial/.git/hooks" +printf '%s\n' "$marker" > "$partial/.git/agent-os-runtime-source" +validate_source "$partial" || { echo "error: materialized immutable runtime source failed exact verification" >&2; exit 2; } +mv "$partial" "$target" +rmdir "$lock" +validate_source "$target" || { echo "error: committed immutable runtime source failed exact verification" >&2; exit 2; } +printf '%s\n' "$target" diff --git a/bin/agent-os-source-bundle.sh b/bin/agent-os-source-bundle.sh index fa9b95f7f..9474fb180 100755 --- a/bin/agent-os-source-bundle.sh +++ b/bin/agent-os-source-bundle.sh @@ -10,6 +10,7 @@ SOURCE_ORIGIN=https://github.com/akua-dev/agent-os.git SOURCE_REF=refs/heads/main RELEASE_TAG=${AGENT_OS_SOURCE_RELEASE_TAG:-} RELEASE_RECORD_COMMIT=${AGENT_OS_RELEASE_RECORD_COMMIT:-} +RELEASE_BOOTSTRAP_ARCHIVE=${AGENT_OS_RELEASE_BOOTSTRAP_ARCHIVE:-} EVENT_COMMIT=${AGENT_OS_SOURCE_EVENT_COMMIT:-} SOURCE_ARCHIVE="$OUTPUT_DIR/agent-os-source.tar" BOOTSTRAP_ARCHIVE="$OUTPUT_DIR/agent-os-bootstrap.tar" @@ -124,6 +125,10 @@ case "$SOURCE_MODE" in exit 2 } command -v jq >/dev/null 2>&1 || { echo "error: jq is required for release records" >&2; exit 2; } + [ -f "$RELEASE_BOOTSTRAP_ARCHIVE" ] || { + echo "error: release mode requires the retained candidate bootstrap archive" >&2 + exit 2 + } git_isolated --git-dir="$TEMP/bootstrap.git" fetch --depth=1 --no-tags "$SOURCE_ORIGIN" \ "$RELEASE_RECORD_COMMIT:refs/agent-os/release-record" git_isolated --git-dir="$TEMP/bootstrap.git" fetch --depth=1 --no-tags "$SOURCE_ORIGIN" \ @@ -198,14 +203,28 @@ if [ "$SOURCE_MODE" = release ]; then exit 2 } fi -(cd "$TEMP" && find bootstrap.git -exec touch -t 197001010000 {} + && \ - find bootstrap.git -print | LC_ALL=C sort | COPYFILE_DISABLE=1 tar -cf "$BOOTSTRAP_ARCHIVE.tmp.$$" -T -) +if [ "$SOURCE_MODE" = release ]; then + cp -- "$RELEASE_BOOTSTRAP_ARCHIVE" "$BOOTSTRAP_ARCHIVE.tmp.$$" +else + (cd "$TEMP" && find bootstrap.git -exec touch -t 197001010000 {} + && \ + find bootstrap.git -print | LC_ALL=C sort | COPYFILE_DISABLE=1 tar -cf "$BOOTSTRAP_ARCHIVE.tmp.$$" -T -) +fi SOURCE_SHA=$(sha256_file "$SOURCE_ARCHIVE.tmp.$$") BOOTSTRAP_SHA=$(sha256_file "$BOOTSTRAP_ARCHIVE.tmp.$$") if [ "$SOURCE_MODE" = release ] && [ "$BOOTSTRAP_SHA" != "$(jq -r .bootstrap_archive_sha256 "$TEMP/release.json")" ]; then - echo "error: bootstrap archive differs from the immutable release record" >&2 + echo "error: retained bootstrap archive differs from the immutable release record" >&2 exit 2 fi +if [ "$SOURCE_MODE" = release ]; then + mkdir "$TEMP/retained-bootstrap" + tar -xf "$BOOTSTRAP_ARCHIVE.tmp.$$" -C "$TEMP/retained-bootstrap" + [ "$(git_isolated --git-dir="$TEMP/retained-bootstrap/bootstrap.git" rev-parse refs/heads/main)" = "$SOURCE_COMMIT" ] && \ + [ "$(git_isolated --git-dir="$TEMP/retained-bootstrap/bootstrap.git" rev-parse "$SOURCE_COMMIT^{tree}")" = "$SOURCE_TREE" ] && \ + [ "$(git_isolated --git-dir="$TEMP/retained-bootstrap/bootstrap.git" remote get-url origin)" = "$SOURCE_ORIGIN" ] || { + echo "error: retained bootstrap archive provenance is invalid" >&2 + exit 2 + } +fi { printf 'mode=%s\ncommit=%s\ntree=%s\nbranch=%s\norigin=%s\nref=%s\n' \ "$SOURCE_MODE" "$SOURCE_COMMIT" "$SOURCE_TREE" "$SOURCE_BRANCH" "$SOURCE_ORIGIN" "$SOURCE_REF" diff --git a/bin/fm-update.sh b/bin/fm-update.sh index f16501e96..e69430299 100755 --- a/bin/fm-update.sh +++ b/bin/fm-update.sh @@ -47,7 +47,29 @@ if [ "${1:-}" = "--help" ] || [ "${1:-}" = "-h" ]; then fi [ $# -eq 0 ] || { usage; exit 1; } -case "${AGENT_OS_SOURCE_UPDATE_POLICY:-fast-forward}" in +persisted_policy=fast-forward +policy_file="$FM_ROOT/.git/agent-os-runtime-source" +if [ -f "$policy_file" ]; then + policy_mode=$(sed -n 's/^mode=//p' "$policy_file") + policy_commit=$(sed -n 's/^commit=//p' "$policy_file") + policy_source_sha=$(sed -n 's/^source_sha256=//p' "$policy_file") + case "$policy_mode" in candidate|release) ;; *) echo "error: invalid immutable source provenance" >&2; exit 2 ;; esac + [[ "$policy_commit" =~ ^[0-9a-f]{40}$ ]] && [[ "$policy_source_sha" =~ ^[0-9a-f]{64}$ ]] || { + echo "error: invalid immutable source provenance" >&2 + exit 2 + } + expected_root="$FM_HOME/runtime-sources/$policy_commit-$policy_source_sha" + [ "$(cd "$FM_ROOT" && pwd -P)" = "$(cd "$expected_root" 2>/dev/null && pwd -P)" ] || { + echo "error: immutable source provenance does not match FM_ROOT" >&2 + exit 2 + } + persisted_policy=immutable +elif [ "${FM_ROOT#"$FM_HOME/runtime-sources/"}" != "$FM_ROOT" ]; then + echo "error: immutable source provenance is unavailable" >&2 + exit 2 +fi + +case "$persisted_policy" in fast-forward) ;; immutable) echo "error: self-update is disabled for immutable image source" >&2 diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 8ce7320ef..c2226c31c 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -147,6 +147,11 @@ assert_grep 'tag_ruleset_sha256' "$ROOT/bin/agent-os-source-bundle.sh" \ "release records must bind the protected tag ruleset" assert_grep 'source_archive_sha256' "$ROOT/bin/agent-os-source-bundle.sh" \ "release records must bind the exact archived source" +assert_grep 'AGENT_OS_RELEASE_BOOTSTRAP_ARCHIVE' "$ROOT/bin/agent-os-source-bundle.sh" \ + "release preparation must consume the retained candidate bootstrap artifact" +assert_grep 'cp -- "$RELEASE_BOOTSTRAP_ARCHIVE" "$BOOTSTRAP_ARCHIVE.tmp.$$"' \ + "$ROOT/bin/agent-os-source-bundle.sh" \ + "release preparation must preserve exact candidate bootstrap bytes" assert_grep 'GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null' "$ROOT/bin/agent-os-container-entrypoint.sh" \ "runtime provenance must isolate trusted Git operations" assert_grep 'env -i' "$ROOT/bin/agent-os-container-entrypoint.sh" \ @@ -169,6 +174,16 @@ assert_grep 'trusted_git -C "$FM_ROOT" fetch --no-tags --prune "$SOURCE_ORIGIN"' assert_no_grep 'checkout --detach' "$ROOT/bin/agent-os-container-entrypoint.sh" \ "canonical runtime source must remain on the declared default branch" assert_grep 'FM_ROOT_OVERRIDE=' "$ROOT/bin/agent-os-container-entrypoint.sh" "runtime must use the persistent canonical Firstmate repository" +assert_grep 'agent-os-runtime-source.sh' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "immutable runtimes must select a content-addressed source checkout" +assert_grep 'agent-os-source.sha256' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "immutable runtime source identity must include the exact source archive digest" +assert_present "$ROOT/bin/agent-os-runtime-source.sh" \ + "immutable runtime source materialization must have one transactional owner" +assert_grep 'runtime-sources' "$ROOT/bin/agent-os-runtime-source.sh" \ + "immutable runtime sources must remain separate from persistent home state" +assert_grep 'agent-os-runtime-source' "$ROOT/bin/agent-os-runtime-source.sh" \ + "immutable source selections must persist their verified update policy" assert_grep 'agent-os-kubernetes-control.sh' "$ROOT/bin/agent-os-kubernetes.sh" \ "primary lifecycle paths must share the stable control-namespace lock identity" assert_grep 'agent-os-kubernetes-control.sh' "$ROOT/bin/agent-os-akua-auth.sh" \ @@ -234,6 +249,18 @@ assert_no_grep 'EXISTING_CANDIDATE_IMAGE_DIGEST' "$IMAGE_WORKFLOW" \ "publication must never adopt a pre-existing image from its embedded provenance" assert_grep 'oras manifest push --descriptor "$IMAGE@$candidate_record_digest"' "$IMAGE_WORKFLOW" \ "candidate records must be created only at their content-addressed digest" +assert_grep 'application/vnd.akua.agent-os.candidate-reservation.v1' "$IMAGE_WORKFLOW" \ + "candidate builds must first reserve one content-addressed commit coordinate" +assert_grep 'oras discover --distribution-spec v1.1-referrers-api' "$IMAGE_WORKFLOW" \ + "candidate retries must resolve the single record attached to their reservation" +assert_grep 'actions/artifacts?name=$artifact_name' "$IMAGE_WORKFLOW" \ + "candidate retries must authenticate recovered records through trusted workflow storage" +assert_grep '.workflow_run.head_sha == $sha' "$IMAGE_WORKFLOW" \ + "candidate recovery must bind the trusted record to the exact protected-main commit" +assert_grep 'partial candidate reservation requires trusted recovery' "$IMAGE_WORKFLOW" \ + "candidate retries must fail closed on an incomplete reservation" +assert_grep 'org.opencontainers.image.title":"agent-os-bootstrap.tar"' "$IMAGE_WORKFLOW" \ + "candidate records must retain the exact bootstrap artifact" assert_grep 'candidate record coordinate already contains different artifacts' "$IMAGE_WORKFLOW" \ "candidate record creation must reject immutable coordinate conflicts" assert_grep 'release candidate image provenance conflicts with the exact protected-main source' "$IMAGE_WORKFLOW" \ @@ -254,22 +281,24 @@ assert_grep 'buildkit_outputs_sha256' "$IMAGE_WORKFLOW" \ "candidate records must bind the exact BuildKit output descriptors" assert_grep '.bootstrap_archive_sha256 == $bootstrapArchive' "$IMAGE_WORKFLOW" \ "candidate retries and promotion must enforce the exact bootstrap archive" -assert_grep 'bootstrap archive differs from the immutable release record' "$ROOT/bin/agent-os-source-bundle.sh" \ - "release source preparation must regenerate and verify the exact bootstrap archive" +assert_grep 'retained bootstrap archive differs from the immutable release record' "$ROOT/bin/agent-os-source-bundle.sh" \ + "release source preparation must verify the retained exact bootstrap archive" assert_grep 'actual_image_digest' "$IMAGE_WORKFLOW" \ "release publication must compare the actual multi-platform image digest" assert_grep 'actual_sbom_sha256' "$IMAGE_WORKFLOW" \ "release publication must compare the actual SBOM artifact set" assert_grep 'actual_provenance_sha256' "$IMAGE_WORKFLOW" \ "release publication must compare the actual provenance artifact set" -assert_grep 'oras tag "$IMAGE@$RELEASE_DIGEST" "$release_tag"' "$IMAGE_WORKFLOW" \ - "release publication must alias the exact existing candidate manifest by digest" +assert_no_grep 'oras tag ' "$IMAGE_WORKFLOW" \ + "release publication must never create a mutable registry semver tag" +assert_grep 'release registry tag coordinate already exists' "$IMAGE_WORKFLOW" \ + "release publication must reject conflicting mutable registry coordinates" assert_no_grep 'oras cp --from-oci-layout' "$IMAGE_WORKFLOW" \ "release publication must not rebuild or re-upload an OCI layout" assert_grep 'oras-project/setup-oras@22ce207df3b08e061f537244349aac6ae1d214f6' "$IMAGE_WORKFLOW" \ "release publication must pin the OCI transfer implementation" -assert_grep 'published-tag-index.json' "$IMAGE_WORKFLOW" \ - "release publication must verify every published tag resolves to the recorded digest" +assert_grep 'Canonical install digest:' "$IMAGE_WORKFLOW" \ + "release publication must expose only the immutable install digest" assert_grep 'published-platform-subjects' "$IMAGE_WORKFLOW" \ "release promotion must reverify the per-platform attestation bijection" assert_grep 'File.fnmatch?(pattern, ref, File::FNM_PATHNAME)' "$IMAGE_WORKFLOW" \ @@ -349,13 +378,14 @@ assert_grep 'docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051' "$ "release metadata action must be pinned to the reviewed full SHA" assert_grep 'docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8' "$ROOT/.github/workflows/agent-os-image.yml" \ "release build action must be pinned to the reviewed full SHA" -assert_grep 'AGENT_OS_SOURCE_UPDATE_POLICY=immutable' "$ROOT/bin/agent-os-container-entrypoint.sh" \ - "candidate and release runtimes must disable moving-main self-updates" +assert_grep '.git/agent-os-runtime-source' "$ROOT/bin/fm-update.sh" \ + "self-update must derive immutable policy from selected source provenance" assert_grep 'TRUSTED_SOURCE_REF=$SOURCE_COMMIT' "$ROOT/bin/agent-os-container-entrypoint.sh" \ "candidate runtime provenance must fetch its immutable commit" if grep -E 'uses: (actions/checkout|docker/[^@]+|oras-project/setup-oras)@v[0-9]+' "$ROOT/.github/workflows/agent-os-image.yml" >/dev/null; then fail "release workflow must not use mutable major action tags" fi bash -n "$ROOT/bin/agent-os-container-entrypoint.sh" +bash -n "$ROOT/bin/agent-os-runtime-source.sh" bash -n "$ROOT/bin/agent-os-kubeconfig.sh" pass "container files pin dependencies and exclude host credentials" diff --git a/tests/agent-os-runtime-source.test.sh b/tests/agent-os-runtime-source.test.sh new file mode 100755 index 000000000..560f430d4 --- /dev/null +++ b/tests/agent-os-runtime-source.test.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -u + +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +MATERIALIZE="$ROOT/bin/agent-os-runtime-source.sh" +TMP=$(fm_test_tmproot agent-os-runtime-source) +HOME_DIR="$TMP/home" + +fm_git_identity fmtest fmtest@example.com + +make_source() { + local name=$1 value=$2 work bare + work="$TMP/$name-work" + bare="$TMP/$name.git" + git init -q "$work" + git -C "$work" checkout -q -b main + printf '%s\n' "$value" > "$work/version" + git -C "$work" add version + git -C "$work" commit -qm "$name" + git clone -q --bare "$work" "$bare" + git -C "$bare" symbolic-ref HEAD refs/heads/main +} + +materialize() { + local source=$1 sha=$2 commit tree + commit=$(git --git-dir="$source" rev-parse refs/heads/main) + tree=$(git --git-dir="$source" rev-parse "$commit^{tree}") + FM_HOME="$HOME_DIR" AGENT_OS_IMAGE_SOURCE="$source" \ + AGENT_OS_SOURCE_COMMIT="$commit" AGENT_OS_SOURCE_TREE="$tree" \ + AGENT_OS_SOURCE_SHA256="$sha" AGENT_OS_SOURCE_BRANCH=main \ + AGENT_OS_SOURCE_ORIGIN=https://github.com/akua-dev/agent-os.git \ + AGENT_OS_SOURCE_MODE=candidate "$MATERIALIZE" +} + +make_source a A +make_source b B +mkdir -p "$HOME_DIR/data" +printf 'persistent\n' > "$HOME_DIR/data/captain-state" + +A_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +B_SHA=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +A_ROOT=$(materialize "$TMP/a.git" "$A_SHA") || fail "release A materialization failed" +B_ROOT=$(materialize "$TMP/b.git" "$B_SHA") || fail "release B materialization failed" +A_AGAIN=$(materialize "$TMP/a.git" "$A_SHA") || fail "release A rollback selection failed" + +[ "$A_ROOT" = "$A_AGAIN" ] || fail "A to B to A did not select the original exact source" +[ "$A_ROOT" != "$B_ROOT" ] || fail "different releases shared one mutable source checkout" +[ "$(cat "$A_ROOT/version")" = A ] || fail "release A source changed after B materialization" +[ "$(cat "$B_ROOT/version")" = B ] || fail "release B source was not selected" +[ "$(cat "$HOME_DIR/data/captain-state")" = persistent ] || fail "persistent home state changed" +pass "content-addressed sources support A to B to A without changing home state" + +PARTIAL_SHA=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc +PARTIAL_COMMIT=$(git --git-dir="$TMP/b.git" rev-parse refs/heads/main) +mkdir -p "$HOME_DIR/runtime-sources/.${PARTIAL_COMMIT}-${PARTIAL_SHA}.materializing" +if materialize "$TMP/b.git" "$PARTIAL_SHA" >/dev/null 2>&1; then + fail "partial immutable source materialization was accepted" +fi +[ "$(cat "$B_ROOT/version")" = B ] || fail "failed B materialization damaged retained release B" +A_AFTER_FAILED_B=$(materialize "$TMP/a.git" "$A_SHA") || fail "failed B compensation could not reselect release A" +[ "$A_AFTER_FAILED_B" = "$A_ROOT" ] || fail "failed B compensation did not preserve release A" +pass "partial materialization fails closed without damaging retained sources" + +printf 'tampered\n' > "$A_ROOT/version" +if materialize "$TMP/a.git" "$A_SHA" >/dev/null 2>&1; then + fail "tampered immutable source was accepted" +fi +pass "tampered immutable source fails closed" + +echo "# all Agent OS runtime source tests passed" diff --git a/tests/fm-update.test.sh b/tests/fm-update.test.sh index e60810b9d..f60485f01 100755 --- a/tests/fm-update.test.sh +++ b/tests/fm-update.test.sh @@ -291,23 +291,32 @@ test_unsafe_secondmate_home_skipped_before_git_update() { pass "T11 unsafe secondmate home is not fast-forwarded" } -test_immutable_image_source_refuses_self_update() { - local w out status before - w=$(new_world t12) +test_persisted_immutable_source_refuses_sanitized_self_update() { + local w runtime commit out status before key + w=$(new_world t13) bump_origin "$w" instr - before=$(git -C "$w/main" rev-parse HEAD) + commit=$(git -C "$w/main" rev-parse HEAD) + key="$commit-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + runtime="$w/home/runtime-sources/$key" + mkdir -p "$w/home/runtime-sources" + git clone -q "$w/main" "$runtime" + before=$(git -C "$runtime" rev-parse HEAD) + printf 'mode=candidate\ncommit=%s\nsource_sha256=%s\n' "$commit" \ + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ + > "$runtime/.git/agent-os-runtime-source" set +e - out=$(AGENT_OS_SOURCE_UPDATE_POLICY=immutable FM_ROOT_OVERRIDE="$w/main" FM_HOME="$w/home" "$UPDATE" 2>&1) + out=$(env -i PATH="$PATH" HOME="$HOME" FM_ROOT_OVERRIDE="$runtime" FM_HOME="$w/home" \ + "$UPDATE" 2>&1) status=$? set -e - [ "$status" -eq 2 ] || fail "immutable image update exited $status, expected 2" + [ "$status" -eq 2 ] || fail "persisted immutable source update exited $status, expected 2" assert_contains "$out" "self-update is disabled for immutable image source" \ - "immutable image source blocks moving-main self-update" - [ "$(git -C "$w/main" rev-parse HEAD)" = "$before" ] \ - || fail "immutable image source moved during refused self-update" - pass "T12 immutable image source refuses self-update" + "persisted immutable provenance blocks sanitized self-update" + [ "$(git -C "$runtime" rev-parse HEAD)" = "$before" ] || \ + fail "persisted immutable runtime source moved during refused self-update" + pass "T13 persisted immutable source refuses sanitized self-update" } test_updates_main_and_secondmate @@ -319,6 +328,6 @@ test_registry_backstop_dedup_and_self_exclusion test_firstmate_wrong_branch_skipped test_firstmate_detached_head_skipped test_unsafe_secondmate_home_skipped_before_git_update -test_immutable_image_source_refuses_self_update +test_persisted_immutable_source_refuses_sanitized_self_update echo "# all fm-update tests passed" From e77a7d91ef83706eb3581e8c700f259a6ddd56cb Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 19:38:46 +0200 Subject: [PATCH 42/56] no-mistakes(review): Captain, secure immutable source convergence and candidate reservations --- .github/workflows/agent-os-image.yml | 98 +++++++++++-- bin/agent-os-container-entrypoint.sh | 7 + bin/agent-os-runtime-secondmates.sh | 158 +++++++++++++++++++++ bin/agent-os-runtime-source.sh | 62 +++++++- bin/fm-update.sh | 38 ++++- tests/agent-os-container.test.sh | 19 ++- tests/agent-os-runtime-secondmates.test.sh | 95 +++++++++++++ tests/agent-os-runtime-source.test.sh | 22 +++ tests/fm-update.test.sh | 31 ++++ 9 files changed, 509 insertions(+), 21 deletions(-) create mode 100755 bin/agent-os-runtime-secondmates.sh create mode 100755 tests/agent-os-runtime-secondmates.test.sh diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index 11daa685e..ea8be60c3 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -156,7 +156,7 @@ jobs: runs-on: ubuntu-latest permissions: actions: read - contents: read + contents: write packages: write steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 @@ -222,6 +222,91 @@ jobs: GH_TOKEN: ${{ github.token }} run: | set -eu + claim_prefix="refs/agent-os/candidate-reservations/$GITHUB_SHA" + source_tree_sha=$(gh api "repos/$GITHUB_REPOSITORY/git/commits/$GITHUB_SHA" --jq .tree.sha) + create_claim() { + claim_ref=$1 + owner_run_id=$2 + claim_message=$(printf 'agent-os-candidate-reservation-v1\nsource-commit=%s\nowner-run-id=%s' \ + "$GITHUB_SHA" "$owner_run_id") + claim_commit=$(gh api --method POST "repos/$GITHUB_REPOSITORY/git/commits" \ + --raw-field message="$claim_message" --raw-field tree="$source_tree_sha" \ + --raw-field "parents[]=$GITHUB_SHA" --jq .sha) + if gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" \ + --raw-field ref="$claim_ref" --raw-field sha="$claim_commit" \ + > "$RUNNER_TEMP/candidate-claim-create.json" \ + 2> "$RUNNER_TEMP/candidate-claim-create.err"; then + return 0 + fi + if grep -Eiq '422|Reference already exists' "$RUNNER_TEMP/candidate-claim-create.err"; then + return 1 + fi + cat "$RUNNER_TEMP/candidate-claim-create.err" >&2 + exit 1 + } + load_claim() { + claim_ref=$1 + claim_sha=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/${claim_ref#refs/}" --jq .object.sha) + gh api "repos/$GITHUB_REPOSITORY/git/commits/$claim_sha" \ + > "$RUNNER_TEMP/candidate-claim-commit.json" + jq -e --arg source "$GITHUB_SHA" --arg tree "$source_tree_sha" \ + '.tree.sha == $tree and (.parents | length == 1) and .parents[0].sha == $source' \ + "$RUNNER_TEMP/candidate-claim-commit.json" >/dev/null + CLAIM_OWNER=$(jq -r .message "$RUNNER_TEMP/candidate-claim-commit.json" | + sed -n 's/^owner-run-id=\([0-9][0-9]*\)$/\1/p') + [[ "$CLAIM_OWNER" =~ ^[0-9]+$ ]] + expected_message=$(printf 'agent-os-candidate-reservation-v1\nsource-commit=%s\nowner-run-id=%s' \ + "$GITHUB_SHA" "$CLAIM_OWNER") + [ "$(jq -r .message "$RUNNER_TEMP/candidate-claim-commit.json")" = "$expected_message" ] + } + load_latest_claim() { + claim_ref="$claim_prefix/initial" + load_claim "$claim_ref" + depth=0 + while [ "$depth" -lt 32 ]; do + next_ref="$claim_prefix/recover-$CLAIM_OWNER" + if gh api "repos/$GITHUB_REPOSITORY/git/ref/${next_ref#refs/}" \ + > "$RUNNER_TEMP/candidate-next-claim.json" \ + 2> "$RUNNER_TEMP/candidate-next-claim.err"; then + claim_ref=$next_ref + load_claim "$claim_ref" + depth=$((depth + 1)) + continue + fi + if grep -Eiq '404|Not Found' "$RUNNER_TEMP/candidate-next-claim.err"; then + return 0 + fi + cat "$RUNNER_TEMP/candidate-next-claim.err" >&2 + exit 1 + done + echo "::error::candidate reservation recovery chain is invalid" + exit 1 + } + ensure_build_owner() { + load_latest_claim + if [ "$CLAIM_OWNER" = "$GITHUB_RUN_ID" ]; then + return 0 + fi + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$CLAIM_OWNER" \ + > "$RUNNER_TEMP/candidate-owner-run.json" + jq -e --arg sha "$GITHUB_SHA" --arg repo "$GITHUB_REPOSITORY" \ + '.status == "completed" and .head_sha == $sha and .event == "push" and + .head_branch == "main" and .repository.full_name == $repo' \ + "$RUNNER_TEMP/candidate-owner-run.json" >/dev/null || { + echo "::error::candidate reservation is owned by a live or foreign workflow run" + exit 1 + } + recovery_ref="$claim_prefix/recover-$CLAIM_OWNER" + echo "::notice::recovering incomplete candidate reservation from run $CLAIM_OWNER" + create_claim "$recovery_ref" "$GITHUB_RUN_ID" || true + load_latest_claim + [ "$CLAIM_OWNER" = "$GITHUB_RUN_ID" ] || { + echo "::error::candidate reservation recovery was claimed by another workflow run" + exit 1 + } + } + create_claim "$claim_prefix/initial" "$GITHUB_RUN_ID" || true + load_latest_claim printf '{}\n' > "$RUNNER_TEMP/candidate-reservation-config.json" jq -cnS \ --arg commit '${{ steps.candidate_source.outputs.commit }}' \ @@ -240,10 +325,8 @@ jobs: > "$RUNNER_TEMP/candidate-reservation-manifest.json" reservation_digest="sha256:$(sha256sum "$RUNNER_TEMP/candidate-reservation-manifest.json" | awk '{print $1}')" reservation_size=$(wc -c < "$RUNNER_TEMP/candidate-reservation-manifest.json" | tr -d ' ') - existed=false if oras manifest fetch "$IMAGE@$reservation_digest" > "$RUNNER_TEMP/candidate-reservation-existing.json" 2> "$RUNNER_TEMP/candidate-reservation.err"; then cmp "$RUNNER_TEMP/candidate-reservation-manifest.json" "$RUNNER_TEMP/candidate-reservation-existing.json" - existed=true elif grep -Eiq 'manifest unknown|not found|404' "$RUNNER_TEMP/candidate-reservation.err"; then oras manifest push --descriptor "$IMAGE@$reservation_digest" \ "$RUNNER_TEMP/candidate-reservation-manifest.json" > "$RUNNER_TEMP/candidate-reservation-push.json" @@ -255,16 +338,13 @@ jobs: oras manifest fetch "$IMAGE@$reservation_digest" > "$RUNNER_TEMP/candidate-reservation-readback.json" cmp "$RUNNER_TEMP/candidate-reservation-manifest.json" "$RUNNER_TEMP/candidate-reservation-readback.json" printf 'reservation_digest=%s\nreservation_size=%s\n' "$reservation_digest" "$reservation_size" >> "$GITHUB_OUTPUT" - if [ "$existed" = false ]; then - printf 'build=true\nbootstrap_sha256=%s\n' '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' >> "$GITHUB_OUTPUT" - exit 0 - fi oras discover --distribution-spec v1.1-referrers-api --artifact-type application/vnd.akua.agent-os.release-candidate.v1 \ --format json --depth 1 "$IMAGE@$reservation_digest" > "$RUNNER_TEMP/candidate-reservation-referrers.json" referrer_count=$(jq -er '.referrers | length' "$RUNNER_TEMP/candidate-reservation-referrers.json") if [ "$referrer_count" -eq 0 ]; then - echo "::error::partial candidate reservation requires trusted recovery" - exit 1 + ensure_build_owner + printf 'build=true\nbootstrap_sha256=%s\n' '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' >> "$GITHUB_OUTPUT" + exit 0 fi [ "$referrer_count" -eq 1 ] || { echo "::error::candidate reservation has conflicting records"; exit 1; } candidate_record_digest=$(jq -er '.referrers[0].digest' "$RUNNER_TEMP/candidate-reservation-referrers.json") diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh index 40034e7c4..5a29e4784 100755 --- a/bin/agent-os-container-entrypoint.sh +++ b/bin/agent-os-container-entrypoint.sh @@ -194,6 +194,13 @@ find "$FM_ROOT/.git/hooks" -mindepth 1 -print -quit 2>/dev/null | grep -q . && { exit 2 } export FM_ROOT_OVERRIDE="$FM_ROOT" +if [ "$IMMUTABLE_SOURCE" = true ]; then + FM_HOME="$FM_HOME" FM_ROOT_OVERRIDE="$FM_ROOT" \ + AGENT_OS_SOURCE_COMMIT="$SOURCE_COMMIT" AGENT_OS_SOURCE_TREE="$SOURCE_TREE" \ + AGENT_OS_SOURCE_SHA256="$SOURCE_SHA" AGENT_OS_SOURCE_BRANCH="$SOURCE_BRANCH" \ + AGENT_OS_SOURCE_ORIGIN="$SOURCE_ORIGIN" AGENT_OS_SOURCE_MODE="$SOURCE_MODE" \ + "$(dirname "$0")/agent-os-runtime-secondmates.sh" +fi ln -sfn "$FM_ROOT/tools/agent-os/src/cli.ts" "$HOME/.local/bin/agent-os" if [ -n "${AGENT_OS_PI_AUTH_FILE:-}" ]; then diff --git a/bin/agent-os-runtime-secondmates.sh b/bin/agent-os-runtime-secondmates.sh new file mode 100755 index 000000000..4882a128e --- /dev/null +++ b/bin/agent-os-runtime-secondmates.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +set -eu + +FM_HOME=${FM_HOME:?FM_HOME is required} +FM_ROOT=${FM_ROOT_OVERRIDE:?FM_ROOT_OVERRIDE is required} +SOURCE_COMMIT=${AGENT_OS_SOURCE_COMMIT:?AGENT_OS_SOURCE_COMMIT is required} +SOURCE_TREE=${AGENT_OS_SOURCE_TREE:?AGENT_OS_SOURCE_TREE is required} +SOURCE_SHA=${AGENT_OS_SOURCE_SHA256:?AGENT_OS_SOURCE_SHA256 is required} +SOURCE_BRANCH=${AGENT_OS_SOURCE_BRANCH:?AGENT_OS_SOURCE_BRANCH is required} +SOURCE_ORIGIN=${AGENT_OS_SOURCE_ORIGIN:?AGENT_OS_SOURCE_ORIGIN is required} +SOURCE_MODE=${AGENT_OS_SOURCE_MODE:?AGENT_OS_SOURCE_MODE is required} +GIT_BIN=/usr/bin/git + +[[ "$SOURCE_COMMIT" =~ ^[0-9a-f]{40}$ ]] || exit 2 +[[ "$SOURCE_TREE" =~ ^[0-9a-f]{40}$ ]] || exit 2 +[[ "$SOURCE_SHA" =~ ^[0-9a-f]{64}$ ]] || exit 2 +case "$SOURCE_MODE" in candidate|release) ;; *) exit 2 ;; esac +[ -x "$GIT_BIN" ] || exit 2 + +trusted_git() { + env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 \ + "$GIT_BIN" -c credential.helper= -c core.hooksPath=/dev/null \ + -c http.proxy= -c https.proxy= "$@" +} + +resolve_git_metadata() { + local root=$1 pointer common + if [ -d "$root/.git" ] && [ ! -L "$root/.git" ]; then + RESOLVED_GIT_DIR=$(cd "$root/.git" && pwd -P) + elif [ -f "$root/.git" ] && [ ! -L "$root/.git" ]; then + [ "$(wc -l < "$root/.git" | tr -d ' ')" -eq 1 ] || return 1 + pointer=$(sed -n 's/^gitdir: //p' "$root/.git") + [ -n "$pointer" ] || return 1 + case "$pointer" in + /*) RESOLVED_GIT_DIR=$(cd "$pointer" 2>/dev/null && pwd -P) || return 1 ;; + *) RESOLVED_GIT_DIR=$(cd "$root/$(dirname "$pointer")" 2>/dev/null && printf '%s/%s\n' "$(pwd -P)" "$(basename "$pointer")") || return 1 ;; + esac + else + return 1 + fi + RESOLVED_COMMON_DIR=$RESOLVED_GIT_DIR + if [ -f "$RESOLVED_GIT_DIR/commondir" ] && [ ! -L "$RESOLVED_GIT_DIR/commondir" ]; then + [ "$(wc -l < "$RESOLVED_GIT_DIR/commondir" | tr -d ' ')" -eq 1 ] || return 1 + common=$(cat "$RESOLVED_GIT_DIR/commondir") + case "$common" in + /*) RESOLVED_COMMON_DIR=$(cd "$common" 2>/dev/null && pwd -P) || return 1 ;; + *) RESOLVED_COMMON_DIR=$(cd "$RESOLVED_GIT_DIR/$common" 2>/dev/null && pwd -P) || return 1 ;; + esac + fi +} + +validate_config() { + local config=$1 section='' line key value token seen='' + [ -f "$config" ] && [ ! -L "$config" ] || return 1 + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + ''|'#'*|';'*) continue ;; + '[core]') section=core; continue ;; + '[remote "origin"]') section=remote.origin; continue ;; + "[branch \"$SOURCE_BRANCH\"]") section="branch.$SOURCE_BRANCH"; continue ;; + '['*) return 1 ;; + esac + case "$line" in *=*) ;; *) return 1 ;; esac + key=${line%%=*} + value=${line#*=} + key=$(printf '%s' "$key" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + value=$(printf '%s' "$value" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + token="$section.$key" + case "|$seen|" in *"|$token|"*) return 1 ;; esac + case "$token" in + core.repositoryformatversion) [ "$value" = 0 ] ;; + core.filemode) [ "$value" = true ] || [ "$value" = false ] ;; + core.bare) [ "$value" = false ] ;; + core.logallrefupdates) [ "$value" = true ] ;; + core.ignorecase) [ "$value" = true ] || [ "$value" = false ] ;; + core.precomposeunicode) [ "$value" = true ] || [ "$value" = false ] ;; + remote.origin.url) [ -n "$value" ] ;; + remote.origin.fetch) [ -n "$value" ] ;; + "branch.$SOURCE_BRANCH.remote") [ "$value" = origin ] ;; + "branch.$SOURCE_BRANCH.merge") [ "$value" = "refs/heads/$SOURCE_BRANCH" ] ;; + *) return 1 ;; + esac || return 1 + seen="${seen:+$seen|}$token" + done < "$config" + for token in core.repositoryformatversion core.filemode core.bare core.logallrefupdates \ + remote.origin.url remote.origin.fetch; do + case "|$seen|" in *"|$token|"*) ;; *) return 1 ;; esac + done +} + +homes=() +seen=$'\n' +add_home() { + local home=$1 + [ -n "$home" ] || return 0 + case "$home" in /*) ;; *) echo "error: secondmate home is not absolute" >&2; exit 2 ;; esac + case "$seen" in *$'\n'"$home"$'\n'*) return 0 ;; esac + seen+="$home"$'\n' + homes+=("$home") +} + +for meta in "$FM_HOME"/state/*.meta; do + [ -f "$meta" ] || continue + [ "$(sed -n 's/^kind=//p' "$meta")" = secondmate ] || continue + add_home "$(sed -n 's/^home=//p' "$meta")" +done +if [ -f "$FM_HOME/data/secondmates.md" ]; then + while IFS= read -r line; do + case "$line" in '- '*) ;; *) continue ;; esac + add_home "$(printf '%s\n' "$line" | sed -n 's/.*(home:[[:space:]]*\([^;]*\);.*/\1/p' | sed 's/[[:space:]]*$//')" + done < "$FM_HOME/data/secondmates.md" +fi + +[ "$(trusted_git -C "$FM_ROOT" rev-parse HEAD)" = "$SOURCE_COMMIT" ] || exit 2 +[ "$(trusted_git -C "$FM_ROOT" rev-parse 'HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 +[ -z "$(trusted_git -C "$FM_ROOT" status --porcelain --untracked-files=all)" ] || exit 2 + +git_dirs=() +for home in "${homes[@]}"; do + [ "$(cd "$home" 2>/dev/null && pwd -P)" != "$(cd "$FM_ROOT" && pwd -P)" ] || continue + [ -f "$home/.fm-secondmate-home" ] && [ ! -L "$home/.fm-secondmate-home" ] || { + echo "error: secondmate home lacks exact provenance marker: $home" >&2 + exit 2 + } + resolve_git_metadata "$home" || { echo "error: secondmate Git metadata is invalid: $home" >&2; exit 2; } + validate_config "$RESOLVED_COMMON_DIR/config" || { + echo "error: secondmate Git configuration is invalid: $home" >&2 + exit 2 + } + [ -z "$(trusted_git -C "$home" status --porcelain --untracked-files=all)" ] || { + echo "error: secondmate source is not clean: $home" >&2 + exit 2 + } + git_dirs+=("$RESOLVED_GIT_DIR") +done + +marker="mode=$SOURCE_MODE +commit=$SOURCE_COMMIT +source_sha256=$SOURCE_SHA" +index=0 +for home in "${homes[@]}"; do + [ "$(cd "$home" 2>/dev/null && pwd -P)" != "$(cd "$FM_ROOT" && pwd -P)" ] || continue + git_dir=${git_dirs[$index]} + index=$((index + 1)) + trusted_git -c protocol.file.allow=always -C "$home" fetch --no-tags "$FM_ROOT" "$SOURCE_COMMIT" + [ "$(trusted_git -C "$home" rev-parse FETCH_HEAD)" = "$SOURCE_COMMIT" ] || exit 2 + [ "$(trusted_git -C "$home" rev-parse 'FETCH_HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 + trusted_git -C "$home" checkout --detach "$SOURCE_COMMIT" + trusted_git -C "$home" remote set-url origin "$SOURCE_ORIGIN" + marker_tmp="$git_dir/agent-os-runtime-source.$$" + printf '%s\n' "$marker" > "$marker_tmp" + mv "$marker_tmp" "$git_dir/agent-os-runtime-source" + [ "$(trusted_git -C "$home" rev-parse HEAD)" = "$SOURCE_COMMIT" ] || exit 2 + [ "$(trusted_git -C "$home" rev-parse 'HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 + [ -z "$(trusted_git -C "$home" status --porcelain --untracked-files=all)" ] || exit 2 + printf 'selected: %s\n' "$home" +done diff --git a/bin/agent-os-runtime-source.sh b/bin/agent-os-runtime-source.sh index fd7017748..d94356bdb 100755 --- a/bin/agent-os-runtime-source.sh +++ b/bin/agent-os-runtime-source.sh @@ -32,9 +32,59 @@ marker="mode=$SOURCE_MODE commit=$SOURCE_COMMIT source_sha256=$SOURCE_SHA" +validate_source_config() { + local root=$1 config section='' line key value token seen='' + config="$root/.git/config" + [ -f "$config" ] && [ ! -L "$config" ] || return 1 + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + ''|'#'*|';'*) continue ;; + '[core]') section=core; continue ;; + '[remote "origin"]') section=remote.origin; continue ;; + "[branch \"$SOURCE_BRANCH\"]") section="branch.$SOURCE_BRANCH"; continue ;; + '['*) return 1 ;; + esac + case "$line" in *=*) ;; *) return 1 ;; esac + key=${line%%=*} + value=${line#*=} + key=$(printf '%s' "$key" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + value=$(printf '%s' "$value" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + token="$section.$key" + case "|$seen|" in *"|$token|"*) return 1 ;; esac + case "$token" in + core.repositoryformatversion) [ "$value" = 0 ] ;; + core.filemode) [ "$value" = true ] || [ "$value" = false ] ;; + core.bare) [ "$value" = false ] ;; + core.logallrefupdates) [ "$value" = true ] ;; + core.ignorecase) [ "$value" = true ] || [ "$value" = false ] ;; + core.precomposeunicode) [ "$value" = true ] || [ "$value" = false ] ;; + remote.origin.url) [ "$value" = "$SOURCE_ORIGIN" ] ;; + remote.origin.fetch) + [ "$value" = '+refs/heads/*:refs/remotes/origin/*' ] || + [ "$value" = "+refs/heads/$SOURCE_BRANCH:refs/remotes/origin/$SOURCE_BRANCH" ] + ;; + "branch.$SOURCE_BRANCH.remote") [ "$value" = origin ] ;; + "branch.$SOURCE_BRANCH.merge") [ "$value" = "refs/heads/$SOURCE_BRANCH" ] ;; + *) return 1 ;; + esac || return 1 + seen="${seen:+$seen|}$token" + done < "$config" + for token in core.repositoryformatversion core.filemode core.bare core.logallrefupdates \ + remote.origin.url remote.origin.fetch "branch.$SOURCE_BRANCH.remote" "branch.$SOURCE_BRANCH.merge"; do + case "|$seen|" in *"|$token|"*) ;; *) return 1 ;; esac + done +} + validate_source() { local root=$1 actual_marker - [ -d "$root/.git" ] || { echo "error: immutable runtime source Git metadata is unavailable" >&2; return 1; } + [ -d "$root/.git" ] && [ ! -L "$root/.git" ] || { + echo "error: immutable runtime source Git metadata is unavailable" >&2 + return 1 + } + validate_source_config "$root" || { + echo "error: immutable runtime source Git configuration is invalid" >&2 + return 1 + } [ "$(trusted_git -C "$root" rev-parse HEAD)" = "$SOURCE_COMMIT" ] || return 1 [ "$(trusted_git -C "$root" rev-parse 'HEAD^{tree}')" = "$SOURCE_TREE" ] || return 1 [ "$(trusted_git -C "$root" symbolic-ref --quiet --short HEAD)" = "$SOURCE_BRANCH" ] || return 1 @@ -49,7 +99,6 @@ validate_source() { mkdir -p "$runtime_root" if [ -e "$target" ]; then validate_source "$target" || { echo "error: immutable runtime source failed exact verification" >&2; exit 2; } - rmdir "$lock" 2>/dev/null || true printf '%s\n' "$target" exit 0 fi @@ -63,6 +112,8 @@ if ! mkdir "$lock" 2>/dev/null; then exit 2 fi +lock_owner="$SOURCE_COMMIT-$$-$(date +%s)-$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')" +printf '%s\n' "$lock_owner" > "$lock/owner" partial="$lock/source" if ! trusted_git -c protocol.file.allow=always clone --no-local --branch "$SOURCE_BRANCH" "$IMAGE_SOURCE" "$partial"; then echo "error: immutable runtime source materialization failed" >&2 @@ -73,6 +124,11 @@ rm -rf "$partial/.git/hooks" printf '%s\n' "$marker" > "$partial/.git/agent-os-runtime-source" validate_source "$partial" || { echo "error: materialized immutable runtime source failed exact verification" >&2; exit 2; } mv "$partial" "$target" -rmdir "$lock" validate_source "$target" || { echo "error: committed immutable runtime source failed exact verification" >&2; exit 2; } +if [ -f "$lock/owner" ] && [ "$(cat "$lock/owner")" = "$lock_owner" ]; then + rm "$lock/owner" + rmdir "$lock" +else + validate_source "$target" || { echo "error: immutable runtime source lock handoff failed" >&2; exit 2; } +fi printf '%s\n' "$target" diff --git a/bin/fm-update.sh b/bin/fm-update.sh index e69430299..54ac538a8 100755 --- a/bin/fm-update.sh +++ b/bin/fm-update.sh @@ -48,7 +48,26 @@ fi [ $# -eq 0 ] || { usage; exit 1; } persisted_policy=fast-forward -policy_file="$FM_ROOT/.git/agent-os-runtime-source" +resolve_policy_git_dir() { + local root=$1 pointer + if [ -d "$root/.git" ] && [ ! -L "$root/.git" ]; then + POLICY_GIT_DIR=$(cd "$root/.git" && pwd -P) + elif [ -f "$root/.git" ] && [ ! -L "$root/.git" ]; then + [ "$(wc -l < "$root/.git" | tr -d ' ')" -eq 1 ] || return 1 + pointer=$(sed -n 's/^gitdir: //p' "$root/.git") + [ -n "$pointer" ] || return 1 + case "$pointer" in + /*) POLICY_GIT_DIR=$(cd "$pointer" 2>/dev/null && pwd -P) || return 1 ;; + *) POLICY_GIT_DIR=$(cd "$root/$(dirname "$pointer")" 2>/dev/null && printf '%s/%s\n' "$(pwd -P)" "$(basename "$pointer")") || return 1 ;; + esac + else + return 1 + fi +} + +POLICY_GIT_DIR= +resolve_policy_git_dir "$FM_ROOT" || true +policy_file=${POLICY_GIT_DIR:+$POLICY_GIT_DIR/agent-os-runtime-source} if [ -f "$policy_file" ]; then policy_mode=$(sed -n 's/^mode=//p' "$policy_file") policy_commit=$(sed -n 's/^commit=//p' "$policy_file") @@ -58,11 +77,18 @@ if [ -f "$policy_file" ]; then echo "error: invalid immutable source provenance" >&2 exit 2 } - expected_root="$FM_HOME/runtime-sources/$policy_commit-$policy_source_sha" - [ "$(cd "$FM_ROOT" && pwd -P)" = "$(cd "$expected_root" 2>/dev/null && pwd -P)" ] || { - echo "error: immutable source provenance does not match FM_ROOT" >&2 - exit 2 - } + if [ "${FM_ROOT#"$FM_HOME/runtime-sources/"}" != "$FM_ROOT" ]; then + expected_root="$FM_HOME/runtime-sources/$policy_commit-$policy_source_sha" + [ "$(cd "$FM_ROOT" && pwd -P)" = "$(cd "$expected_root" 2>/dev/null && pwd -P)" ] || { + echo "error: immutable source provenance does not match FM_ROOT" >&2 + exit 2 + } + else + [ -f "$FM_ROOT/.fm-secondmate-home" ] && [ ! -L "$FM_ROOT/.fm-secondmate-home" ] || { + echo "error: immutable source provenance is outside a verified runtime home" >&2 + exit 2 + } + fi persisted_policy=immutable elif [ "${FM_ROOT#"$FM_HOME/runtime-sources/"}" != "$FM_ROOT" ]; then echo "error: immutable source provenance is unavailable" >&2 diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index c2226c31c..fa8d13368 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -184,6 +184,10 @@ assert_grep 'runtime-sources' "$ROOT/bin/agent-os-runtime-source.sh" \ "immutable runtime sources must remain separate from persistent home state" assert_grep 'agent-os-runtime-source' "$ROOT/bin/agent-os-runtime-source.sh" \ "immutable source selections must persist their verified update policy" +assert_present "$ROOT/bin/agent-os-runtime-secondmates.sh" \ + "immutable runtimes must converge each secondmate to the exact selected source" +assert_grep 'agent-os-runtime-secondmates.sh' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "immutable runtimes must select exact source for registered secondmates" assert_grep 'agent-os-kubernetes-control.sh' "$ROOT/bin/agent-os-kubernetes.sh" \ "primary lifecycle paths must share the stable control-namespace lock identity" assert_grep 'agent-os-kubernetes-control.sh' "$ROOT/bin/agent-os-akua-auth.sh" \ @@ -251,14 +255,20 @@ assert_grep 'oras manifest push --descriptor "$IMAGE@$candidate_record_digest"' "candidate records must be created only at their content-addressed digest" assert_grep 'application/vnd.akua.agent-os.candidate-reservation.v1' "$IMAGE_WORKFLOW" \ "candidate builds must first reserve one content-addressed commit coordinate" +assert_grep 'refs/agent-os/candidate-reservations/' "$IMAGE_WORKFLOW" \ + "candidate builds must use an atomic owner-bound repository coordinator" +assert_grep 'repos/$GITHUB_REPOSITORY/git/refs' "$IMAGE_WORKFLOW" \ + "candidate reservation ownership must use atomic Git ref creation" +assert_grep '.status == "completed"' "$IMAGE_WORKFLOW" \ + "candidate recovery must require the prior owner run to be terminal" assert_grep 'oras discover --distribution-spec v1.1-referrers-api' "$IMAGE_WORKFLOW" \ "candidate retries must resolve the single record attached to their reservation" assert_grep 'actions/artifacts?name=$artifact_name' "$IMAGE_WORKFLOW" \ "candidate retries must authenticate recovered records through trusted workflow storage" assert_grep '.workflow_run.head_sha == $sha' "$IMAGE_WORKFLOW" \ "candidate recovery must bind the trusted record to the exact protected-main commit" -assert_grep 'partial candidate reservation requires trusted recovery' "$IMAGE_WORKFLOW" \ - "candidate retries must fail closed on an incomplete reservation" +assert_grep 'recovering incomplete candidate reservation' "$IMAGE_WORKFLOW" \ + "candidate retries must recover an incomplete reservation under its trusted coordinator" assert_grep 'org.opencontainers.image.title":"agent-os-bootstrap.tar"' "$IMAGE_WORKFLOW" \ "candidate records must retain the exact bootstrap artifact" assert_grep 'candidate record coordinate already contains different artifacts' "$IMAGE_WORKFLOW" \ @@ -378,8 +388,10 @@ assert_grep 'docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051' "$ "release metadata action must be pinned to the reviewed full SHA" assert_grep 'docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8' "$ROOT/.github/workflows/agent-os-image.yml" \ "release build action must be pinned to the reviewed full SHA" -assert_grep '.git/agent-os-runtime-source' "$ROOT/bin/fm-update.sh" \ +assert_grep 'agent-os-runtime-source' "$ROOT/bin/fm-update.sh" \ "self-update must derive immutable policy from selected source provenance" +assert_grep 'resolve_policy_git_dir' "$ROOT/bin/fm-update.sh" \ + "self-update must resolve immutable policy through linked-worktree Git pointers" assert_grep 'TRUSTED_SOURCE_REF=$SOURCE_COMMIT' "$ROOT/bin/agent-os-container-entrypoint.sh" \ "candidate runtime provenance must fetch its immutable commit" if grep -E 'uses: (actions/checkout|docker/[^@]+|oras-project/setup-oras)@v[0-9]+' "$ROOT/.github/workflows/agent-os-image.yml" >/dev/null; then @@ -387,5 +399,6 @@ if grep -E 'uses: (actions/checkout|docker/[^@]+|oras-project/setup-oras)@v[0-9] fi bash -n "$ROOT/bin/agent-os-container-entrypoint.sh" bash -n "$ROOT/bin/agent-os-runtime-source.sh" +bash -n "$ROOT/bin/agent-os-runtime-secondmates.sh" bash -n "$ROOT/bin/agent-os-kubeconfig.sh" pass "container files pin dependencies and exclude host credentials" diff --git a/tests/agent-os-runtime-secondmates.test.sh b/tests/agent-os-runtime-secondmates.test.sh new file mode 100755 index 000000000..39a2cd8c1 --- /dev/null +++ b/tests/agent-os-runtime-secondmates.test.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -u + +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +SYNC="$ROOT/bin/agent-os-runtime-secondmates.sh" +TMP=$(fm_test_tmproot agent-os-runtime-secondmates) +FM_HOME_DIR="$TMP/home" +WORK="$TMP/work" +LEASE="$TMP/lease" + +fm_git_identity fmtest fmtest@example.com + +mkdir -p "$FM_HOME_DIR/state" "$FM_HOME_DIR/data" +git init -q -b main "$WORK" +printf 'data/\nstate/\nconfig/\nprojects/\n.fm-secondmate-home\n' > "$WORK/.gitignore" +printf 'A\n' > "$WORK/version" +printf 'agents\n' > "$WORK/AGENTS.md" +mkdir "$WORK/bin" +printf 'tool\n' > "$WORK/bin/tool" +git -C "$WORK" add -A +git -C "$WORK" commit -qm A +A_COMMIT=$(git -C "$WORK" rev-parse HEAD) +A_TREE=$(git -C "$WORK" rev-parse 'HEAD^{tree}') +printf 'B\n' > "$WORK/version" +git -C "$WORK" add version +git -C "$WORK" commit -qm B +B_COMMIT=$(git -C "$WORK" rev-parse HEAD) +B_TREE=$(git -C "$WORK" rev-parse 'HEAD^{tree}') + +git clone -q "$WORK" "$LEASE" +git -C "$LEASE" remote set-url origin https://github.com/akua-dev/agent-os.git +git -C "$LEASE" checkout -q --detach "$A_COMMIT" +git -C "$LEASE" worktree add -q --detach "$TMP/linked" "$A_COMMIT" +git clone -q "$LEASE" "$TMP/standalone" +git -C "$TMP/standalone" checkout -q --detach "$A_COMMIT" +for id in linked standalone; do + printf '%s\n' "$id" > "$TMP/$id/.fm-secondmate-home" + mkdir -p "$TMP/$id/data" "$TMP/$id/state" "$TMP/$id/config" "$TMP/$id/projects" + printf 'persistent-%s\n' "$id" > "$TMP/$id/data/state" + { + printf 'window=firstmate:fm-%s\n' "$id" + printf 'kind=secondmate\n' + printf 'home=%s/%s\n' "$TMP" "$id" + } > "$FM_HOME_DIR/state/$id.meta" +done + +make_primary() { + local name=$1 commit=$2 + git clone -q "$WORK" "$TMP/$name" + git -C "$TMP/$name" remote set-url origin https://github.com/akua-dev/agent-os.git + git -C "$TMP/$name" checkout -q main + git -C "$TMP/$name" reset -q --hard "$commit" +} + +run_sync() { + local root=$1 commit=$2 tree=$3 sha=$4 + FM_HOME="$FM_HOME_DIR" FM_ROOT_OVERRIDE="$root" \ + AGENT_OS_SOURCE_COMMIT="$commit" AGENT_OS_SOURCE_TREE="$tree" \ + AGENT_OS_SOURCE_SHA256="$sha" AGENT_OS_SOURCE_BRANCH=main \ + AGENT_OS_SOURCE_ORIGIN=https://github.com/akua-dev/agent-os.git \ + AGENT_OS_SOURCE_MODE=release "$SYNC" +} + +make_primary primary-b "$B_COMMIT" +make_primary primary-a "$A_COMMIT" +B_SHA=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +A_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + +run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null || \ + fail "A to B secondmate selection failed" +for id in linked standalone; do + [ "$(git -C "$TMP/$id" rev-parse HEAD)" = "$B_COMMIT" ] || fail "$id did not select B" + [ "$(cat "$TMP/$id/data/state")" = "persistent-$id" ] || fail "$id persistent state changed" + gitdir=$(git -C "$TMP/$id" rev-parse --absolute-git-dir) + [ -f "$gitdir/agent-os-runtime-source" ] || fail "$id immutable policy was not persisted" +done + +run_sync "$TMP/primary-a" "$A_COMMIT" "$A_TREE" "$A_SHA" >/dev/null || \ + fail "B to A secondmate rollback selection failed" +for id in linked standalone; do + [ "$(git -C "$TMP/$id" rev-parse HEAD)" = "$A_COMMIT" ] || fail "$id did not reselect A" + [ "$(cat "$TMP/$id/data/state")" = "persistent-$id" ] || fail "$id persistent state changed on rollback" +done +pass "linked and standalone secondmates select exact A to B to A sources" + +printf 'tampered\n' >> "$TMP/standalone/AGENTS.md" +if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then + fail "dirty secondmate source was replaced" +fi +[ "$(git -C "$TMP/standalone" rev-parse HEAD)" = "$A_COMMIT" ] || \ + fail "dirty secondmate source moved" +pass "dirty secondmate source fails closed" + +echo "# all Agent OS runtime secondmate tests passed" diff --git a/tests/agent-os-runtime-source.test.sh b/tests/agent-os-runtime-source.test.sh index 560f430d4..f76146a49 100755 --- a/tests/agent-os-runtime-source.test.sh +++ b/tests/agent-os-runtime-source.test.sh @@ -51,6 +51,28 @@ A_AGAIN=$(materialize "$TMP/a.git" "$A_SHA") || fail "release A rollback selecti [ "$(cat "$HOME_DIR/data/captain-state")" = persistent ] || fail "persistent home state changed" pass "content-addressed sources support A to B to A without changing home state" +FSWATCH_SENTINEL="$TMP/fsmonitor-executed" +FSWATCH="$TMP/fsmonitor" +printf '#!/usr/bin/env bash\ntouch %q\n' "$FSWATCH_SENTINEL" > "$FSWATCH" +chmod +x "$FSWATCH" +git -C "$A_ROOT" config core.fsmonitor "$FSWATCH" +if materialize "$TMP/a.git" "$A_SHA" >/dev/null 2>&1; then + fail "executable persistent Git configuration was accepted" +fi +[ ! -e "$FSWATCH_SENTINEL" ] || fail "persistent Git configuration executed before validation" +git -C "$A_ROOT" config --unset core.fsmonitor +pass "persistent Git configuration is rejected before repository access" + +LOCK="$HOME_DIR/runtime-sources/.${B_ROOT##*/}.materializing" +mkdir "$LOCK" +printf 'other-owner\n' > "$LOCK/owner" +B_AGAIN=$(materialize "$TMP/b.git" "$B_SHA") || fail "verified source handoff failed" +[ "$B_AGAIN" = "$B_ROOT" ] || fail "verified source handoff selected the wrong source" +[ "$(cat "$LOCK/owner")" = other-owner ] || fail "source observer removed another materializer lock" +rm "$LOCK/owner" +rmdir "$LOCK" +pass "verified source observers preserve the materializer owner lock" + PARTIAL_SHA=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc PARTIAL_COMMIT=$(git --git-dir="$TMP/b.git" rev-parse refs/heads/main) mkdir -p "$HOME_DIR/runtime-sources/.${PARTIAL_COMMIT}-${PARTIAL_SHA}.materializing" diff --git a/tests/fm-update.test.sh b/tests/fm-update.test.sh index f60485f01..0e630d4da 100755 --- a/tests/fm-update.test.sh +++ b/tests/fm-update.test.sh @@ -319,6 +319,36 @@ test_persisted_immutable_source_refuses_sanitized_self_update() { pass "T13 persisted immutable source refuses sanitized self-update" } +test_secondmate_persisted_policy_refuses_sanitized_self_update() { + local w linked standalone commit sha linked_gitdir out status target + w=$(new_world t14) + add_sm "$w" linked + standalone="$w/standalone" + git clone -q "$w/main" "$standalone" + printf 'standalone\n' > "$standalone/.fm-secondmate-home" + mkdir -p "$w/linked/state" "$standalone/state" + touch "$w/linked/state/.last-watcher-beat" "$standalone/state/.last-watcher-beat" + commit=$(git -C "$w/main" rev-parse HEAD) + sha=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + linked_gitdir=$(git -C "$w/linked" rev-parse --absolute-git-dir) + printf 'mode=release\ncommit=%s\nsource_sha256=%s\n' "$commit" "$sha" \ + > "$linked_gitdir/agent-os-runtime-source" + printf 'mode=release\ncommit=%s\nsource_sha256=%s\n' "$commit" "$sha" \ + > "$standalone/.git/agent-os-runtime-source" + + for target in "$w/linked" "$standalone"; do + set +e + out=$(env -i PATH="$PATH" HOME="$HOME" FM_ROOT_OVERRIDE="$target" FM_HOME="$target" \ + "$UPDATE" 2>&1) + status=$? + set -e + [ "$status" -eq 2 ] || fail "secondmate immutable source update exited $status, expected 2" + assert_contains "$out" "self-update is disabled for immutable image source" \ + "secondmate immutable provenance blocks sanitized self-update" + done + pass "T14 linked and standalone secondmates persist immutable update policy" +} + test_updates_main_and_secondmate test_reread_gate_is_instruction_only test_dirty_secondmate_skipped @@ -329,5 +359,6 @@ test_firstmate_wrong_branch_skipped test_firstmate_detached_head_skipped test_unsafe_secondmate_home_skipped_before_git_update test_persisted_immutable_source_refuses_sanitized_self_update +test_secondmate_persisted_policy_refuses_sanitized_self_update echo "# all fm-update tests passed" From a6cb41eea3895acc900392f9e6132aa699b9aca2 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 19:56:43 +0200 Subject: [PATCH 43/56] no-mistakes(review): Captain, harden candidate recovery and secondmate provenance --- .github/workflows/agent-os-image.yml | 94 +++++++++++++++--- bin/agent-os-runtime-secondmates.sh | 105 ++++++++++++++++++--- bin/agent-os-runtime-source.sh | 2 + bin/fm-update.sh | 23 +++++ tests/agent-os-container.test.sh | 6 ++ tests/agent-os-runtime-secondmates.test.sh | 64 +++++++++++++ tests/agent-os-runtime-source.test.sh | 9 ++ tests/fm-update.test.sh | 18 +++- 8 files changed, 290 insertions(+), 31 deletions(-) diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index ea8be60c3..ad0253064 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -342,7 +342,60 @@ jobs: --format json --depth 1 "$IMAGE@$reservation_digest" > "$RUNNER_TEMP/candidate-reservation-referrers.json" referrer_count=$(jq -er '.referrers | length' "$RUNNER_TEMP/candidate-reservation-referrers.json") if [ "$referrer_count" -eq 0 ]; then + artifact_owner=$CLAIM_OWNER ensure_build_owner + artifact_name="agent-os-candidate-record-$GITHUB_SHA" + gh api --method GET "repos/$GITHUB_REPOSITORY/actions/artifacts?name=$artifact_name&per_page=100" \ + > "$RUNNER_TEMP/candidate-trusted-artifacts.json" + trusted_artifact_count=$(jq -er --arg sha "$GITHUB_SHA" --argjson owner "$artifact_owner" \ + '[.artifacts[] | select(.expired == false and .workflow_run.head_sha == $sha and .workflow_run.id == $owner)] | length' \ + "$RUNNER_TEMP/candidate-trusted-artifacts.json") + [ "$trusted_artifact_count" -le 1 ] || { + echo "::error::candidate reservation has conflicting trusted artifacts" + exit 1 + } + if [ "$trusted_artifact_count" -eq 1 ]; then + trusted_artifact_id=$(jq -er --arg sha "$GITHUB_SHA" --argjson owner "$artifact_owner" \ + '[.artifacts[] | select(.expired == false and .workflow_run.head_sha == $sha and .workflow_run.id == $owner)] | .[0].id' \ + "$RUNNER_TEMP/candidate-trusted-artifacts.json") + mkdir "$RUNNER_TEMP/candidate-trusted-artifact" + gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$trusted_artifact_id/zip" \ + > "$RUNNER_TEMP/candidate-trusted-artifact.zip" + unzip -q "$RUNNER_TEMP/candidate-trusted-artifact.zip" -d "$RUNNER_TEMP/candidate-trusted-artifact" + candidate_record_digest="sha256:$(sha256sum "$RUNNER_TEMP/candidate-trusted-artifact/candidate-manifest.json" | awk '{print $1}')" + jq -e --arg reservation "$reservation_digest" ' + .schemaVersion == 2 and + .artifactType == "application/vnd.akua.agent-os.release-candidate.v1" and + .subject.digest == $reservation and + ([.layers[] | select(.mediaType == "application/vnd.akua.agent-os.release-candidate.v1+json")] | length == 1) and + ([.layers[] | select(.mediaType == "application/vnd.akua.agent-os.bootstrap.v1+tar" and .annotations["org.opencontainers.image.title"] == "agent-os-bootstrap.tar")] | length == 1)' \ + "$RUNNER_TEMP/candidate-trusted-artifact/candidate-manifest.json" >/dev/null + record_blob=$(jq -er '[.layers[] | select(.mediaType == "application/vnd.akua.agent-os.release-candidate.v1+json")] | .[0].digest' \ + "$RUNNER_TEMP/candidate-trusted-artifact/candidate-manifest.json") + bootstrap_blob=$(jq -er '[.layers[] | select(.mediaType == "application/vnd.akua.agent-os.bootstrap.v1+tar")] | .[0].digest' \ + "$RUNNER_TEMP/candidate-trusted-artifact/candidate-manifest.json") + [ "$record_blob" = "sha256:$(sha256sum "$RUNNER_TEMP/candidate-trusted-artifact/candidate-record.json" | awk '{print $1}')" ] + [ "$bootstrap_blob" = "sha256:$(sha256sum "$RUNNER_TEMP/candidate-trusted-artifact/agent-os-bootstrap.tar" | awk '{print $1}')" ] + jq -e \ + --arg commit '${{ steps.candidate_source.outputs.commit }}' \ + --arg tree '${{ steps.candidate_source.outputs.tree }}' \ + --arg source '${{ steps.candidate_source.outputs.source_sha256 }}' \ + --arg bootstrap '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' \ + --arg repository "$IMAGE" ' + .schema_version == 1 and .source_mode == "candidate" and + .commit == $commit and .tree == $tree and + .source_archive_sha256 == $source and .bootstrap_archive_sha256 == $bootstrap and + .image_repository == $repository and + (.image_digest | test("^sha256:[0-9a-f]{64}$"))' \ + "$RUNNER_TEMP/candidate-trusted-artifact/candidate-record.json" >/dev/null + cp "$RUNNER_TEMP/candidate-trusted-artifact/agent-os-bootstrap.tar" image/agent-os-bootstrap.tar + image_digest=$(jq -er .image_digest "$RUNNER_TEMP/candidate-trusted-artifact/candidate-record.json") + bootstrap_sha=$(jq -er .bootstrap_archive_sha256 "$RUNNER_TEMP/candidate-trusted-artifact/candidate-record.json") + printf 'build=false\ncandidate_record_digest=%s\nimage_digest=%s\nbootstrap_sha256=%s\n' \ + "$candidate_record_digest" "$image_digest" "$bootstrap_sha" >> "$GITHUB_OUTPUT" + echo "::notice::candidate reservation recovered from durable trusted artifact" + exit 0 + fi printf 'build=true\nbootstrap_sha256=%s\n' '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' >> "$GITHUB_OUTPUT" exit 0 fi @@ -416,7 +469,7 @@ jobs: AGENT_OS_SOURCE_MODE=${{ steps.candidate_source.outputs.mode }} AGENT_OS_SOURCE_REF=${{ steps.candidate_source.outputs.ref }} - - name: Create or verify immutable release candidate record + - name: Prepare immutable release candidate record if: github.ref == 'refs/heads/main' id: candidate_record env: @@ -548,6 +601,31 @@ jobs: '{schemaVersion:2,mediaType:"application/vnd.oci.image.manifest.v1+json",artifactType:"application/vnd.akua.agent-os.release-candidate.v1",subject:$subject,config:$config,layers:[$record,$bootstrap]}' \ > "$RUNNER_TEMP/candidate-manifest.json" candidate_record_digest="sha256:$(sha256sum "$RUNNER_TEMP/candidate-manifest.json" | awk '{print $1}')" + if [ -d "$RUNNER_TEMP/candidate-trusted-artifact" ]; then + cmp "$RUNNER_TEMP/candidate-record.json" "$RUNNER_TEMP/candidate-trusted-artifact/candidate-record.json" + cmp "$RUNNER_TEMP/candidate-manifest.json" "$RUNNER_TEMP/candidate-trusted-artifact/candidate-manifest.json" + cmp image/agent-os-bootstrap.tar "$RUNNER_TEMP/candidate-trusted-artifact/agent-os-bootstrap.tar" + fi + cp image/agent-os-bootstrap.tar "$RUNNER_TEMP/agent-os-bootstrap.tar" + printf 'candidate_record_digest=%s\nimage_digest=%s\n' "$candidate_record_digest" "$actual_image_digest" >> "$GITHUB_OUTPUT" + + - name: Stage trusted candidate record evidence + if: github.ref == 'refs/heads/main' && steps.candidate_reservation.outputs.build == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: agent-os-candidate-record-${{ github.sha }} + path: | + ${{ runner.temp }}/candidate-record.json + ${{ runner.temp }}/candidate-manifest.json + ${{ runner.temp }}/agent-os-bootstrap.tar + if-no-files-found: error + retention-days: 90 + + - name: Publish immutable release candidate record + if: github.ref == 'refs/heads/main' + run: | + set -eu + candidate_record_digest='${{ steps.candidate_record.outputs.candidate_record_digest }}' if oras manifest fetch "$IMAGE@$candidate_record_digest" > "$RUNNER_TEMP/candidate-existing-manifest.json" 2> "$RUNNER_TEMP/candidate-existing-manifest.err"; then cmp "$RUNNER_TEMP/candidate-manifest.json" "$RUNNER_TEMP/candidate-existing-manifest.json" || { echo "::error::candidate record coordinate already contains different artifacts" @@ -573,20 +651,6 @@ jobs: echo "::error::candidate reservation does not resolve to exactly one verified record" exit 1 } - cp image/agent-os-bootstrap.tar "$RUNNER_TEMP/agent-os-bootstrap.tar" - printf 'candidate_record_digest=%s\nimage_digest=%s\n' "$candidate_record_digest" "$actual_image_digest" >> "$GITHUB_OUTPUT" - - - name: Retain candidate record for release preparation - if: github.ref == 'refs/heads/main' && steps.candidate_reservation.outputs.build == 'true' - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 - with: - name: agent-os-candidate-record-${{ github.sha }} - path: | - ${{ runner.temp }}/candidate-record.json - ${{ runner.temp }}/candidate-manifest.json - ${{ runner.temp }}/agent-os-bootstrap.tar - if-no-files-found: error - retention-days: 90 - name: Prepare protected-main source if: github.ref == 'refs/heads/main' diff --git a/bin/agent-os-runtime-secondmates.sh b/bin/agent-os-runtime-secondmates.sh index 4882a128e..ce6ec8f4e 100755 --- a/bin/agent-os-runtime-secondmates.sh +++ b/bin/agent-os-runtime-secondmates.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash set -eu +PATH=/usr/bin:/bin:/usr/sbin:/sbin +export PATH FM_HOME=${FM_HOME:?FM_HOME is required} FM_ROOT=${FM_ROOT_OVERRIDE:?FM_ROOT_OVERRIDE is required} @@ -10,6 +12,10 @@ SOURCE_BRANCH=${AGENT_OS_SOURCE_BRANCH:?AGENT_OS_SOURCE_BRANCH is required} SOURCE_ORIGIN=${AGENT_OS_SOURCE_ORIGIN:?AGENT_OS_SOURCE_ORIGIN is required} SOURCE_MODE=${AGENT_OS_SOURCE_MODE:?AGENT_OS_SOURCE_MODE is required} GIT_BIN=/usr/bin/git +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SUB_HOME_MARKER=.fm-secondmate-home +# shellcheck source=bin/fm-ff-lib.sh +. "$SCRIPT_DIR/fm-ff-lib.sh" [[ "$SOURCE_COMMIT" =~ ^[0-9a-f]{40}$ ]] || exit 2 [[ "$SOURCE_TREE" =~ ^[0-9a-f]{40}$ ]] || exit 2 @@ -89,26 +95,29 @@ validate_config() { done } +ids=() homes=() -seen=$'\n' add_home() { - local home=$1 - [ -n "$home" ] || return 0 + local id=$1 home=$2 + [ -n "$id" ] && [ -n "$home" ] || return 0 + case "$id" in *[!A-Za-z0-9._-]*|.|..) echo "error: secondmate id is invalid" >&2; exit 2 ;; esac case "$home" in /*) ;; *) echo "error: secondmate home is not absolute" >&2; exit 2 ;; esac - case "$seen" in *$'\n'"$home"$'\n'*) return 0 ;; esac - seen+="$home"$'\n' + ids+=("$id") homes+=("$home") } for meta in "$FM_HOME"/state/*.meta; do [ -f "$meta" ] || continue [ "$(sed -n 's/^kind=//p' "$meta")" = secondmate ] || continue - add_home "$(sed -n 's/^home=//p' "$meta")" + id=${meta##*/} + id=${id%.meta} + add_home "$id" "$(sed -n 's/^home=//p' "$meta")" done if [ -f "$FM_HOME/data/secondmates.md" ]; then while IFS= read -r line; do case "$line" in '- '*) ;; *) continue ;; esac - add_home "$(printf '%s\n' "$line" | sed -n 's/.*(home:[[:space:]]*\([^;]*\);.*/\1/p' | sed 's/[[:space:]]*$//')" + id=$(printf '%s\n' "$line" | sed -n 's/^- \([^ ][^ ]*\) - .*/\1/p') + add_home "$id" "$(printf '%s\n' "$line" | sed -n 's/.*(home:[[:space:]]*\([^;]*\);.*/\1/p' | sed 's/[[:space:]]*$//')" done < "$FM_HOME/data/secondmates.md" fi @@ -116,13 +125,42 @@ fi [ "$(trusted_git -C "$FM_ROOT" rev-parse 'HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 [ -z "$(trusted_git -C "$FM_ROOT" status --porcelain --untracked-files=all)" ] || exit 2 +validated_ids=() +validated_homes=() git_dirs=() -for home in "${homes[@]}"; do - [ "$(cd "$home" 2>/dev/null && pwd -P)" != "$(cd "$FM_ROOT" && pwd -P)" ] || continue - [ -f "$home/.fm-secondmate-home" ] && [ ! -L "$home/.fm-secondmate-home" ] || { - echo "error: secondmate home lacks exact provenance marker: $home" >&2 +seen_homes=$'\n' +for index in "${!homes[@]}"; do + id=${ids[$index]} + home=${homes[$index]} + validate_secondmate_home "$id" "$home" || { + echo "error: invalid secondmate home for $id: $VALIDATION_ERROR" >&2 exit 2 } + home=$VALIDATED_HOME + duplicate=false + for prior in "${!validated_homes[@]}"; do + prior_home=${validated_homes[$prior]} + prior_id=${validated_ids[$prior]} + if [ "$home" = "$prior_home" ]; then + [ "$id" = "$prior_id" ] || { + echo "error: secondmate home is registered to multiple identities: $home" >&2 + exit 2 + } + duplicate=true + break + fi + if [ "$id" = "$prior_id" ]; then + echo "error: secondmate identity is registered to multiple homes: $id" >&2 + exit 2 + fi + if path_is_ancestor_of "$home" "$prior_home" || path_is_ancestor_of "$prior_home" "$home"; then + echo "error: secondmate homes overlap: $home and $prior_home" >&2 + exit 2 + fi + done + [ "$duplicate" = false ] || continue + case "$seen_homes" in *$'\n'"$home"$'\n'*) continue ;; esac + seen_homes+="$home"$'\n' resolve_git_metadata "$home" || { echo "error: secondmate Git metadata is invalid: $home" >&2; exit 2; } validate_config "$RESOLVED_COMMON_DIR/config" || { echo "error: secondmate Git configuration is invalid: $home" >&2 @@ -132,25 +170,62 @@ for home in "${homes[@]}"; do echo "error: secondmate source is not clean: $home" >&2 exit 2 } + git_policy="$RESOLVED_GIT_DIR/agent-os-runtime-source" + home_policy="$home/config/agent-os-source-policy" + for policy in "$git_policy" "$home_policy"; do + if [ -e "$policy" ]; then + [ -f "$policy" ] && [ ! -L "$policy" ] || { + echo "error: secondmate immutable policy is invalid: $home" >&2 + exit 2 + } + policy_mode=$(sed -n 's/^mode=//p' "$policy") + policy_commit=$(sed -n 's/^commit=//p' "$policy") + policy_sha=$(sed -n 's/^source_sha256=//p' "$policy") + case "$policy_mode" in candidate|release) ;; *) echo "error: secondmate immutable policy is invalid: $home" >&2; exit 2 ;; esac + [[ "$policy_commit" =~ ^[0-9a-f]{40}$ ]] && [[ "$policy_sha" =~ ^[0-9a-f]{64}$ ]] || { + echo "error: secondmate immutable policy is invalid: $home" >&2 + exit 2 + } + [ "$(wc -l < "$policy" | tr -d ' ')" -eq 3 ] || { + echo "error: secondmate immutable policy is invalid: $home" >&2 + exit 2 + } + fi + done + if [ -e "$home_policy" ] && [ ! -e "$git_policy" ]; then + echo "error: secondmate immutable policy is incomplete: $home" >&2 + exit 2 + fi + if [ -e "$git_policy" ] && [ -e "$home_policy" ]; then + cmp "$git_policy" "$home_policy" || { + echo "error: secondmate immutable policies do not match: $home" >&2 + exit 2 + } + fi + validated_ids+=("$id") + validated_homes+=("$home") git_dirs+=("$RESOLVED_GIT_DIR") done marker="mode=$SOURCE_MODE commit=$SOURCE_COMMIT source_sha256=$SOURCE_SHA" -index=0 -for home in "${homes[@]}"; do - [ "$(cd "$home" 2>/dev/null && pwd -P)" != "$(cd "$FM_ROOT" && pwd -P)" ] || continue +for index in "${!validated_homes[@]}"; do + home=${validated_homes[$index]} git_dir=${git_dirs[$index]} - index=$((index + 1)) trusted_git -c protocol.file.allow=always -C "$home" fetch --no-tags "$FM_ROOT" "$SOURCE_COMMIT" [ "$(trusted_git -C "$home" rev-parse FETCH_HEAD)" = "$SOURCE_COMMIT" ] || exit 2 [ "$(trusted_git -C "$home" rev-parse 'FETCH_HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 trusted_git -C "$home" checkout --detach "$SOURCE_COMMIT" trusted_git -C "$home" remote set-url origin "$SOURCE_ORIGIN" + mkdir -p "$home/config" + home_marker_tmp="$home/config/.agent-os-source-policy.$$" + printf '%s\n' "$marker" > "$home_marker_tmp" + mv "$home_marker_tmp" "$home/config/agent-os-source-policy" marker_tmp="$git_dir/agent-os-runtime-source.$$" printf '%s\n' "$marker" > "$marker_tmp" mv "$marker_tmp" "$git_dir/agent-os-runtime-source" + cmp "$home/config/agent-os-source-policy" "$git_dir/agent-os-runtime-source" || exit 2 [ "$(trusted_git -C "$home" rev-parse HEAD)" = "$SOURCE_COMMIT" ] || exit 2 [ "$(trusted_git -C "$home" rev-parse 'HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 [ -z "$(trusted_git -C "$home" status --porcelain --untracked-files=all)" ] || exit 2 diff --git a/bin/agent-os-runtime-source.sh b/bin/agent-os-runtime-source.sh index d94356bdb..284018da2 100755 --- a/bin/agent-os-runtime-source.sh +++ b/bin/agent-os-runtime-source.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash set -eu +PATH=/usr/bin:/bin:/usr/sbin:/sbin +export PATH FM_HOME=${FM_HOME:?FM_HOME is required} IMAGE_SOURCE=${AGENT_OS_IMAGE_SOURCE:?AGENT_OS_IMAGE_SOURCE is required} diff --git a/bin/fm-update.sh b/bin/fm-update.sh index 54ac538a8..f8ee2d2d9 100755 --- a/bin/fm-update.sh +++ b/bin/fm-update.sh @@ -68,6 +68,29 @@ resolve_policy_git_dir() { POLICY_GIT_DIR= resolve_policy_git_dir "$FM_ROOT" || true policy_file=${POLICY_GIT_DIR:+$POLICY_GIT_DIR/agent-os-runtime-source} +home_policy_file="$FM_ROOT/config/agent-os-source-policy" +secondmate_home=false +if [ -e "$FM_ROOT/.fm-secondmate-home" ]; then + [ -f "$FM_ROOT/.fm-secondmate-home" ] && [ ! -L "$FM_ROOT/.fm-secondmate-home" ] || { + echo "error: invalid secondmate home provenance" >&2 + exit 2 + } + secondmate_home=true +fi +if [ "$secondmate_home" = true ] && { [ -e "$policy_file" ] || [ -e "$home_policy_file" ]; }; then + [ -f "$policy_file" ] && [ ! -L "$policy_file" ] && \ + [ -f "$home_policy_file" ] && [ ! -L "$home_policy_file" ] || { + echo "error: immutable source provenance is incomplete" >&2 + exit 2 + } + cmp "$policy_file" "$home_policy_file" >/dev/null || { + echo "error: immutable source provenance does not match the secondmate home policy" >&2 + exit 2 + } +elif [ "$secondmate_home" = false ] && [ -e "$home_policy_file" ]; then + echo "error: immutable source provenance exists outside a secondmate home" >&2 + exit 2 +fi if [ -f "$policy_file" ]; then policy_mode=$(sed -n 's/^mode=//p' "$policy_file") policy_commit=$(sed -n 's/^commit=//p' "$policy_file") diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index fa8d13368..94bc21940 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -269,6 +269,12 @@ assert_grep '.workflow_run.head_sha == $sha' "$IMAGE_WORKFLOW" \ "candidate recovery must bind the trusted record to the exact protected-main commit" assert_grep 'recovering incomplete candidate reservation' "$IMAGE_WORKFLOW" \ "candidate retries must recover an incomplete reservation under its trusted coordinator" +assert_grep 'candidate reservation recovered from durable trusted artifact' "$IMAGE_WORKFLOW" \ + "zero-referrer retries must recover durable trusted candidate evidence" +stage_line=$(grep -n 'name: Stage trusted candidate record evidence' "$IMAGE_WORKFLOW" | cut -d: -f1) +publish_line=$(grep -n 'name: Publish immutable release candidate record' "$IMAGE_WORKFLOW" | cut -d: -f1) +[ -n "$stage_line" ] && [ -n "$publish_line" ] && [ "$stage_line" -lt "$publish_line" ] || \ + fail "trusted candidate evidence must be durable before referrer publication" assert_grep 'org.opencontainers.image.title":"agent-os-bootstrap.tar"' "$IMAGE_WORKFLOW" \ "candidate records must retain the exact bootstrap artifact" assert_grep 'candidate record coordinate already contains different artifacts' "$IMAGE_WORKFLOW" \ diff --git a/tests/agent-os-runtime-secondmates.test.sh b/tests/agent-os-runtime-secondmates.test.sh index 39a2cd8c1..00e8abaa8 100755 --- a/tests/agent-os-runtime-secondmates.test.sh +++ b/tests/agent-os-runtime-secondmates.test.sh @@ -74,6 +74,9 @@ for id in linked standalone; do [ "$(cat "$TMP/$id/data/state")" = "persistent-$id" ] || fail "$id persistent state changed" gitdir=$(git -C "$TMP/$id" rev-parse --absolute-git-dir) [ -f "$gitdir/agent-os-runtime-source" ] || fail "$id immutable policy was not persisted" + [ -f "$TMP/$id/config/agent-os-source-policy" ] || fail "$id home policy was not persisted" + cmp "$gitdir/agent-os-runtime-source" "$TMP/$id/config/agent-os-source-policy" || \ + fail "$id Git and home policies differ" done run_sync "$TMP/primary-a" "$A_COMMIT" "$A_TREE" "$A_SHA" >/dev/null || \ @@ -84,6 +87,67 @@ for id in linked standalone; do done pass "linked and standalone secondmates select exact A to B to A sources" +gitdir=$(git -C "$TMP/standalone" rev-parse --absolute-git-dir) +printf 'mode=release\ncommit=%s\nsource_sha256=%s\n' "$B_COMMIT" "$B_SHA" \ + > "$TMP/standalone/config/agent-os-source-policy" +if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then + fail "mismatched secondmate immutable policies were accepted" +fi +[ "$(git -C "$TMP/standalone" rev-parse HEAD)" = "$A_COMMIT" ] || \ + fail "mismatched secondmate immutable policy allowed source mutation" +cp "$gitdir/agent-os-runtime-source" "$TMP/standalone/config/agent-os-source-policy" +pass "secondmate selection rejects mismatched immutable policies" + +git clone -q "$LEASE" "$TMP/redirected" +git -C "$TMP/redirected" checkout -q --detach "$A_COMMIT" +printf 'owner\n' > "$TMP/redirected/.fm-secondmate-home" +mkdir -p "$TMP/redirected/data" "$TMP/redirected/state" "$TMP/redirected/config" "$TMP/redirected/projects" +{ + printf 'window=firstmate:fm-redirected\n' + printf 'kind=secondmate\n' + printf 'home=%s/redirected\n' "$TMP" +} > "$FM_HOME_DIR/state/redirected.meta" +if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then + fail "mismatched secondmate ownership marker was accepted" +fi +[ "$(git -C "$TMP/redirected" rev-parse HEAD)" = "$A_COMMIT" ] || \ + fail "mismatched secondmate home was mutated" +rm "$FM_HOME_DIR/state/redirected.meta" +pass "secondmate selection requires exact registered marker ownership" + +git clone -q "$LEASE" "$TMP/linked-alias" +git -C "$TMP/linked-alias" checkout -q --detach "$A_COMMIT" +printf 'linked\n' > "$TMP/linked-alias/.fm-secondmate-home" +mkdir -p "$TMP/linked-alias/data" "$TMP/linked-alias/state" "$TMP/linked-alias/config" \ + "$TMP/linked-alias/projects" +printf -- '- linked - standby (home: %s/linked-alias; state: idle)\n' "$TMP" \ + > "$FM_HOME_DIR/data/secondmates.md" +if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then + fail "duplicate secondmate identity homes were accepted" +fi +[ "$(git -C "$TMP/linked-alias" rev-parse HEAD)" = "$A_COMMIT" ] || \ + fail "duplicate secondmate identity home was mutated" +: > "$FM_HOME_DIR/data/secondmates.md" +pass "secondmate selection rejects one identity mapped to multiple homes" + +git clone -q "$LEASE" "$TMP/linked/projects/nested" +git -C "$TMP/linked/projects/nested" checkout -q --detach "$A_COMMIT" +printf 'nested\n' > "$TMP/linked/projects/nested/.fm-secondmate-home" +mkdir -p "$TMP/linked/projects/nested/data" "$TMP/linked/projects/nested/state" \ + "$TMP/linked/projects/nested/config" "$TMP/linked/projects/nested/projects" +{ + printf 'window=firstmate:fm-nested\n' + printf 'kind=secondmate\n' + printf 'home=%s/linked/projects/nested\n' "$TMP" +} > "$FM_HOME_DIR/state/nested.meta" +if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then + fail "overlapping secondmate homes were accepted" +fi +[ "$(git -C "$TMP/linked/projects/nested" rev-parse HEAD)" = "$A_COMMIT" ] || \ + fail "overlapping secondmate home was mutated" +rm "$FM_HOME_DIR/state/nested.meta" +pass "secondmate selection rejects canonical home overlap" + printf 'tampered\n' >> "$TMP/standalone/AGENTS.md" if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then fail "dirty secondmate source was replaced" diff --git a/tests/agent-os-runtime-source.test.sh b/tests/agent-os-runtime-source.test.sh index f76146a49..1371c7f55 100755 --- a/tests/agent-os-runtime-source.test.sh +++ b/tests/agent-os-runtime-source.test.sh @@ -51,6 +51,15 @@ A_AGAIN=$(materialize "$TMP/a.git" "$A_SHA") || fail "release A rollback selecti [ "$(cat "$HOME_DIR/data/captain-state")" = persistent ] || fail "persistent home state changed" pass "content-addressed sources support A to B to A without changing home state" +PATH_SENTINEL="$TMP/path-sed-executed" +mkdir "$TMP/fake-bin" +printf '#!/usr/bin/env bash\ntouch %q\nexec /usr/bin/sed "$@"\n' "$PATH_SENTINEL" > "$TMP/fake-bin/sed" +chmod +x "$TMP/fake-bin/sed" +PATH="$TMP/fake-bin:$PATH" materialize "$TMP/a.git" "$A_SHA" >/dev/null || \ + fail "source validation failed under an untrusted ambient PATH" +[ ! -e "$PATH_SENTINEL" ] || fail "source validation executed a persistent PATH override" +pass "runtime provenance utilities use a fixed system PATH" + FSWATCH_SENTINEL="$TMP/fsmonitor-executed" FSWATCH="$TMP/fsmonitor" printf '#!/usr/bin/env bash\ntouch %q\n' "$FSWATCH_SENTINEL" > "$FSWATCH" diff --git a/tests/fm-update.test.sh b/tests/fm-update.test.sh index 0e630d4da..2dcfbe0e4 100755 --- a/tests/fm-update.test.sh +++ b/tests/fm-update.test.sh @@ -326,7 +326,7 @@ test_secondmate_persisted_policy_refuses_sanitized_self_update() { standalone="$w/standalone" git clone -q "$w/main" "$standalone" printf 'standalone\n' > "$standalone/.fm-secondmate-home" - mkdir -p "$w/linked/state" "$standalone/state" + mkdir -p "$w/linked/state" "$w/linked/config" "$standalone/state" "$standalone/config" touch "$w/linked/state/.last-watcher-beat" "$standalone/state/.last-watcher-beat" commit=$(git -C "$w/main" rev-parse HEAD) sha=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb @@ -335,6 +335,10 @@ test_secondmate_persisted_policy_refuses_sanitized_self_update() { > "$linked_gitdir/agent-os-runtime-source" printf 'mode=release\ncommit=%s\nsource_sha256=%s\n' "$commit" "$sha" \ > "$standalone/.git/agent-os-runtime-source" + printf 'mode=release\ncommit=%s\nsource_sha256=%s\n' "$commit" "$sha" \ + > "$w/linked/config/agent-os-source-policy" + printf 'mode=release\ncommit=%s\nsource_sha256=%s\n' "$commit" "$sha" \ + > "$standalone/config/agent-os-source-policy" for target in "$w/linked" "$standalone"; do set +e @@ -346,6 +350,18 @@ test_secondmate_persisted_policy_refuses_sanitized_self_update() { assert_contains "$out" "self-update is disabled for immutable image source" \ "secondmate immutable provenance blocks sanitized self-update" done + before=$(git -C "$w/linked" rev-parse HEAD) + rm "$linked_gitdir/agent-os-runtime-source" + set +e + out=$(env -i PATH="$PATH" HOME="$HOME" FM_ROOT_OVERRIDE="$w/linked" FM_HOME="$w/linked" \ + "$UPDATE" 2>&1) + status=$? + set -e + [ "$status" -eq 2 ] || fail "missing secondmate Git policy exited $status, expected 2" + assert_contains "$out" "immutable source provenance is incomplete" \ + "missing secondmate Git policy fails closed" + [ "$(git -C "$w/linked" rev-parse HEAD)" = "$before" ] || \ + fail "secondmate moved after immutable Git policy removal" pass "T14 linked and standalone secondmates persist immutable update policy" } From 3f360a4ccb0564f7290110d31c73510e3c963781 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 20:12:45 +0200 Subject: [PATCH 44/56] no-mistakes(review): Captain, harden candidate evidence and secondmate recovery --- .github/workflows/agent-os-image.yml | 94 +++++++++++++++++- bin/agent-os-runtime-secondmates.sh | 107 +++++++++++++++++---- bin/fm-update.sh | 20 +++- tests/agent-os-container.test.sh | 16 +++ tests/agent-os-runtime-secondmates.test.sh | 42 ++++++++ tests/fm-update.test.sh | 13 +++ 6 files changed, 266 insertions(+), 26 deletions(-) diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index ad0253064..4fbb2d5bd 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -262,6 +262,7 @@ jobs: load_latest_claim() { claim_ref="$claim_prefix/initial" load_claim "$claim_ref" + CLAIM_OWNERS="|$CLAIM_OWNER|" depth=0 while [ "$depth" -lt 32 ]; do next_ref="$claim_prefix/recover-$CLAIM_OWNER" @@ -270,6 +271,7 @@ jobs: 2> "$RUNNER_TEMP/candidate-next-claim.err"; then claim_ref=$next_ref load_claim "$claim_ref" + CLAIM_OWNERS="${CLAIM_OWNERS}${CLAIM_OWNER}|" depth=$((depth + 1)) continue fi @@ -305,6 +307,27 @@ jobs: exit 1 } } + validate_artifact_run() { + artifact_run_id=$1 + case "$CLAIM_OWNERS" in + *"|$artifact_run_id|"*) ;; + *) + echo "::error::candidate artifact run is outside the authorized claim chain" + exit 1 + ;; + esac + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$artifact_run_id" \ + > "$RUNNER_TEMP/candidate-artifact-run.json" + jq -e --argjson run "$artifact_run_id" --arg sha "$GITHUB_SHA" \ + --arg repo "$GITHUB_REPOSITORY" ' + .id == $run and .head_sha == $sha and .event == "push" and + .head_branch == "main" and .repository.full_name == $repo and + .path == ".github/workflows/agent-os-image.yml"' \ + "$RUNNER_TEMP/candidate-artifact-run.json" >/dev/null || { + echo "::error::candidate artifact run is not an authorized publication workflow" + exit 1 + } + } create_claim "$claim_prefix/initial" "$GITHUB_RUN_ID" || true load_latest_claim printf '{}\n' > "$RUNNER_TEMP/candidate-reservation-config.json" @@ -355,6 +378,7 @@ jobs: exit 1 } if [ "$trusted_artifact_count" -eq 1 ]; then + validate_artifact_run "$artifact_owner" trusted_artifact_id=$(jq -er --arg sha "$GITHUB_SHA" --argjson owner "$artifact_owner" \ '[.artifacts[] | select(.expired == false and .workflow_run.head_sha == $sha and .workflow_run.id == $owner)] | .[0].id' \ "$RUNNER_TEMP/candidate-trusted-artifacts.json") @@ -391,12 +415,46 @@ jobs: cp "$RUNNER_TEMP/candidate-trusted-artifact/agent-os-bootstrap.tar" image/agent-os-bootstrap.tar image_digest=$(jq -er .image_digest "$RUNNER_TEMP/candidate-trusted-artifact/candidate-record.json") bootstrap_sha=$(jq -er .bootstrap_archive_sha256 "$RUNNER_TEMP/candidate-trusted-artifact/candidate-record.json") - printf 'build=false\ncandidate_record_digest=%s\nimage_digest=%s\nbootstrap_sha256=%s\n' \ + printf 'build=false\nrecord_artifact=true\ncandidate_record_digest=%s\nimage_digest=%s\nbootstrap_sha256=%s\n' \ "$candidate_record_digest" "$image_digest" "$bootstrap_sha" >> "$GITHUB_OUTPUT" echo "::notice::candidate reservation recovered from durable trusted artifact" exit 0 fi - printf 'build=true\nbootstrap_sha256=%s\n' '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' >> "$GITHUB_OUTPUT" + artifact_name="agent-os-candidate-build-$GITHUB_SHA" + gh api --method GET "repos/$GITHUB_REPOSITORY/actions/artifacts?name=$artifact_name&per_page=100" \ + > "$RUNNER_TEMP/candidate-build-artifacts.json" + build_artifact_count=$(jq -er --arg sha "$GITHUB_SHA" --argjson owner "$artifact_owner" \ + '[.artifacts[] | select(.expired == false and .workflow_run.head_sha == $sha and .workflow_run.id == $owner)] | length' \ + "$RUNNER_TEMP/candidate-build-artifacts.json") + [ "$build_artifact_count" -le 1 ] || { + echo "::error::candidate reservation has conflicting exact build evidence" + exit 1 + } + if [ "$build_artifact_count" -eq 1 ]; then + validate_artifact_run "$artifact_owner" + build_artifact_id=$(jq -er --arg sha "$GITHUB_SHA" --argjson owner "$artifact_owner" \ + '[.artifacts[] | select(.expired == false and .workflow_run.head_sha == $sha and .workflow_run.id == $owner)] | .[0].id' \ + "$RUNNER_TEMP/candidate-build-artifacts.json") + mkdir "$RUNNER_TEMP/candidate-build-artifact" + gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$build_artifact_id/zip" \ + > "$RUNNER_TEMP/candidate-build-artifact.zip" + unzip -q "$RUNNER_TEMP/candidate-build-artifact.zip" -d "$RUNNER_TEMP/candidate-build-artifact" + jq -e --arg commit '${{ steps.candidate_source.outputs.commit }}' \ + --arg tree '${{ steps.candidate_source.outputs.tree }}' \ + --arg reservation "$reservation_digest" --arg repository "$IMAGE" \ + --argjson owner "$artifact_owner" ' + .schema_version == 1 and .commit == $commit and .tree == $tree and + .reservation_digest == $reservation and .image_repository == $repository and + .owner_run_id == $owner and + (.image_digest | test("^sha256:[0-9a-f]{64}$"))' \ + "$RUNNER_TEMP/candidate-build-artifact/candidate-build.json" >/dev/null + image_digest=$(jq -er .image_digest "$RUNNER_TEMP/candidate-build-artifact/candidate-build.json") + printf 'build=false\nrecord_artifact=false\nimage_digest=%s\nbootstrap_sha256=%s\n' \ + "$image_digest" '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' >> "$GITHUB_OUTPUT" + echo "::notice::candidate reservation recovered from exact build evidence" + exit 0 + fi + printf 'build=true\nrecord_artifact=false\nbootstrap_sha256=%s\n' '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' >> "$GITHUB_OUTPUT" exit 0 fi [ "$referrer_count" -eq 1 ] || { echo "::error::candidate reservation has conflicting records"; exit 1; } @@ -415,6 +473,10 @@ jobs: trusted_artifact_id=$(jq -er --arg sha "$GITHUB_SHA" \ '[.artifacts[] | select(.expired == false and .workflow_run.head_sha == $sha)] | .[0].id' \ "$RUNNER_TEMP/candidate-trusted-artifacts.json") + artifact_run_id=$(jq -er --arg sha "$GITHUB_SHA" \ + '[.artifacts[] | select(.expired == false and .workflow_run.head_sha == $sha)] | .[0].workflow_run.id' \ + "$RUNNER_TEMP/candidate-trusted-artifacts.json") + validate_artifact_run "$artifact_run_id" mkdir "$RUNNER_TEMP/candidate-trusted-artifact" gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$trusted_artifact_id/zip" \ > "$RUNNER_TEMP/candidate-trusted-artifact.zip" @@ -431,7 +493,7 @@ jobs: image_digest=$(jq -er '.image_digest | select(test("^sha256:[0-9a-f]{64}$"))' "$RUNNER_TEMP/candidate-trusted-artifact/candidate-record.json") bootstrap_sha=$(jq -er '.bootstrap_archive_sha256 | select(test("^[0-9a-f]{64}$"))' "$RUNNER_TEMP/candidate-trusted-artifact/candidate-record.json") [ "$(sha256sum image/agent-os-bootstrap.tar | awk '{print $1}')" = "$bootstrap_sha" ] - printf 'build=false\ncandidate_record_digest=%s\nimage_digest=%s\nbootstrap_sha256=%s\n' \ + printf 'build=false\nrecord_artifact=true\ncandidate_record_digest=%s\nimage_digest=%s\nbootstrap_sha256=%s\n' \ "$candidate_record_digest" "$image_digest" "$bootstrap_sha" >> "$GITHUB_OUTPUT" - name: Reject legacy candidate coordinates @@ -469,6 +531,30 @@ jobs: AGENT_OS_SOURCE_MODE=${{ steps.candidate_source.outputs.mode }} AGENT_OS_SOURCE_REF=${{ steps.candidate_source.outputs.ref }} + - name: Prepare exact candidate build evidence + if: github.ref == 'refs/heads/main' && steps.candidate_reservation.outputs.build == 'true' + run: | + set -eu + image_digest='${{ steps.candidate_build.outputs.digest }}' + [[ "$image_digest" =~ ^sha256:[0-9a-f]{64}$ ]] + jq -cnS \ + --arg commit '${{ steps.candidate_source.outputs.commit }}' \ + --arg tree '${{ steps.candidate_source.outputs.tree }}' \ + --arg reservation '${{ steps.candidate_reservation.outputs.reservation_digest }}' \ + --arg repository "$IMAGE" --arg image "$image_digest" \ + --argjson owner "$GITHUB_RUN_ID" \ + '{schema_version:1,commit:$commit,tree:$tree,reservation_digest:$reservation,image_repository:$repository,image_digest:$image,owner_run_id:$owner}' \ + > "$RUNNER_TEMP/candidate-build.json" + + - name: Stage exact candidate build evidence + if: github.ref == 'refs/heads/main' && steps.candidate_reservation.outputs.build == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: agent-os-candidate-build-${{ github.sha }} + path: ${{ runner.temp }}/candidate-build.json + if-no-files-found: error + retention-days: 90 + - name: Prepare immutable release candidate record if: github.ref == 'refs/heads/main' id: candidate_record @@ -610,7 +696,7 @@ jobs: printf 'candidate_record_digest=%s\nimage_digest=%s\n' "$candidate_record_digest" "$actual_image_digest" >> "$GITHUB_OUTPUT" - name: Stage trusted candidate record evidence - if: github.ref == 'refs/heads/main' && steps.candidate_reservation.outputs.build == 'true' + if: github.ref == 'refs/heads/main' && steps.candidate_reservation.outputs.record_artifact != 'true' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: agent-os-candidate-record-${{ github.sha }} diff --git a/bin/agent-os-runtime-secondmates.sh b/bin/agent-os-runtime-secondmates.sh index ce6ec8f4e..0ceb91335 100755 --- a/bin/agent-os-runtime-secondmates.sh +++ b/bin/agent-os-runtime-secondmates.sh @@ -56,6 +56,38 @@ resolve_git_metadata() { fi } +resolve_file_path() { + local path=$1 parent + parent=$(cd "$(dirname "$path")" 2>/dev/null && pwd -P) || return 1 + printf '%s/%s\n' "$parent" "$(basename "$path")" +} + +validate_git_binding() { + local root=$1 expected backlink backlink_path relative + expected=$(resolve_file_path "$root/.git") || return 1 + if [ -d "$root/.git" ] && [ ! -L "$root/.git" ]; then + [ "$RESOLVED_GIT_DIR" = "$expected" ] || return 1 + [ "$RESOLVED_COMMON_DIR" = "$RESOLVED_GIT_DIR" ] || return 1 + return 0 + fi + [ -f "$root/.git" ] && [ ! -L "$root/.git" ] || return 1 + [ -f "$RESOLVED_GIT_DIR/gitdir" ] && [ ! -L "$RESOLVED_GIT_DIR/gitdir" ] || return 1 + [ "$(wc -l < "$RESOLVED_GIT_DIR/gitdir" | tr -d ' ')" -eq 1 ] || return 1 + backlink=$(cat "$RESOLVED_GIT_DIR/gitdir") + case "$backlink" in + /*) backlink_path=$(resolve_file_path "$backlink") || return 1 ;; + *) backlink_path=$(resolve_file_path "$RESOLVED_GIT_DIR/$backlink") || return 1 ;; + esac + [ "$backlink_path" = "$expected" ] || return 1 + case "$RESOLVED_GIT_DIR" in + "$RESOLVED_COMMON_DIR"/worktrees/*) ;; + *) return 1 ;; + esac + relative=${RESOLVED_GIT_DIR#"$RESOLVED_COMMON_DIR"/worktrees/} + case "$relative" in ''|*/*) return 1 ;; esac + [ -d "$RESOLVED_COMMON_DIR" ] && [ ! -L "$RESOLVED_COMMON_DIR" ] || return 1 +} + validate_config() { local config=$1 section='' line key value token seen='' [ -f "$config" ] && [ ! -L "$config" ] || return 1 @@ -95,6 +127,18 @@ validate_config() { done } +validate_policy() { + local policy=$1 policy_mode policy_commit policy_sha + [ -f "$policy" ] && [ ! -L "$policy" ] || return 1 + policy_mode=$(sed -n 's/^mode=//p' "$policy") + policy_commit=$(sed -n 's/^commit=//p' "$policy") + policy_sha=$(sed -n 's/^source_sha256=//p' "$policy") + case "$policy_mode" in candidate|release) ;; *) return 1 ;; esac + [[ "$policy_commit" =~ ^[0-9a-f]{40}$ ]] || return 1 + [[ "$policy_sha" =~ ^[0-9a-f]{64}$ ]] || return 1 + [ "$(wc -l < "$policy" | tr -d ' ')" -eq 3 ] +} + ids=() homes=() add_home() { @@ -162,6 +206,10 @@ for index in "${!homes[@]}"; do case "$seen_homes" in *$'\n'"$home"$'\n'*) continue ;; esac seen_homes+="$home"$'\n' resolve_git_metadata "$home" || { echo "error: secondmate Git metadata is invalid: $home" >&2; exit 2; } + validate_git_binding "$home" || { + echo "error: secondmate Git metadata is not bound to its home: $home" >&2 + exit 2 + } validate_config "$RESOLVED_COMMON_DIR/config" || { echo "error: secondmate Git configuration is invalid: $home" >&2 exit 2 @@ -172,35 +220,45 @@ for index in "${!homes[@]}"; do } git_policy="$RESOLVED_GIT_DIR/agent-os-runtime-source" home_policy="$home/config/agent-os-source-policy" + required_policy="$home/config/agent-os-source-policy.required" + pending_policy="$home/config/agent-os-source-policy.pending" for policy in "$git_policy" "$home_policy"; do if [ -e "$policy" ]; then - [ -f "$policy" ] && [ ! -L "$policy" ] || { - echo "error: secondmate immutable policy is invalid: $home" >&2 - exit 2 - } - policy_mode=$(sed -n 's/^mode=//p' "$policy") - policy_commit=$(sed -n 's/^commit=//p' "$policy") - policy_sha=$(sed -n 's/^source_sha256=//p' "$policy") - case "$policy_mode" in candidate|release) ;; *) echo "error: secondmate immutable policy is invalid: $home" >&2; exit 2 ;; esac - [[ "$policy_commit" =~ ^[0-9a-f]{40}$ ]] && [[ "$policy_sha" =~ ^[0-9a-f]{64}$ ]] || { - echo "error: secondmate immutable policy is invalid: $home" >&2 - exit 2 - } - [ "$(wc -l < "$policy" | tr -d ' ')" -eq 3 ] || { + validate_policy "$policy" || { echo "error: secondmate immutable policy is invalid: $home" >&2 exit 2 } fi done - if [ -e "$home_policy" ] && [ ! -e "$git_policy" ]; then - echo "error: secondmate immutable policy is incomplete: $home" >&2 - exit 2 + if [ -e "$required_policy" ]; then + [ -f "$required_policy" ] && [ ! -L "$required_policy" ] && \ + [ "$(cat "$required_policy")" = immutable ] || { + echo "error: secondmate immutable policy requirement is invalid: $home" >&2 + exit 2 + } fi - if [ -e "$git_policy" ] && [ -e "$home_policy" ]; then - cmp "$git_policy" "$home_policy" || { - echo "error: secondmate immutable policies do not match: $home" >&2 + if [ -e "$pending_policy" ]; then + validate_policy "$pending_policy" || { + echo "error: secondmate immutable policy journal is invalid: $home" >&2 exit 2 } + elif [ -e "$required_policy" ]; then + if ! { [ -e "$git_policy" ] && [ -e "$home_policy" ] && \ + cmp "$git_policy" "$home_policy"; }; then + echo "error: secondmate immutable policy is incomplete: $home" >&2 + exit 2 + fi + else + if [ -e "$home_policy" ] && [ ! -e "$git_policy" ]; then + echo "error: secondmate immutable policy is incomplete: $home" >&2 + exit 2 + fi + if [ -e "$git_policy" ] && [ -e "$home_policy" ]; then + cmp "$git_policy" "$home_policy" || { + echo "error: secondmate immutable policies do not match: $home" >&2 + exit 2 + } + fi fi validated_ids+=("$id") validated_homes+=("$home") @@ -216,9 +274,17 @@ for index in "${!validated_homes[@]}"; do trusted_git -c protocol.file.allow=always -C "$home" fetch --no-tags "$FM_ROOT" "$SOURCE_COMMIT" [ "$(trusted_git -C "$home" rev-parse FETCH_HEAD)" = "$SOURCE_COMMIT" ] || exit 2 [ "$(trusted_git -C "$home" rev-parse 'FETCH_HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 + mkdir -p "$home/config" + pending_policy="$home/config/agent-os-source-policy.pending" + pending_tmp="$home/config/.agent-os-source-policy.pending.$$" + printf '%s\n' "$marker" > "$pending_tmp" + mv "$pending_tmp" "$pending_policy" + required_policy="$home/config/agent-os-source-policy.required" + required_tmp="$home/config/.agent-os-source-policy.required.$$" + printf 'immutable\n' > "$required_tmp" + mv "$required_tmp" "$required_policy" trusted_git -C "$home" checkout --detach "$SOURCE_COMMIT" trusted_git -C "$home" remote set-url origin "$SOURCE_ORIGIN" - mkdir -p "$home/config" home_marker_tmp="$home/config/.agent-os-source-policy.$$" printf '%s\n' "$marker" > "$home_marker_tmp" mv "$home_marker_tmp" "$home/config/agent-os-source-policy" @@ -229,5 +295,6 @@ for index in "${!validated_homes[@]}"; do [ "$(trusted_git -C "$home" rev-parse HEAD)" = "$SOURCE_COMMIT" ] || exit 2 [ "$(trusted_git -C "$home" rev-parse 'HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 [ -z "$(trusted_git -C "$home" status --porcelain --untracked-files=all)" ] || exit 2 + rm "$pending_policy" printf 'selected: %s\n' "$home" done diff --git a/bin/fm-update.sh b/bin/fm-update.sh index f8ee2d2d9..75f71a3fc 100755 --- a/bin/fm-update.sh +++ b/bin/fm-update.sh @@ -69,6 +69,8 @@ POLICY_GIT_DIR= resolve_policy_git_dir "$FM_ROOT" || true policy_file=${POLICY_GIT_DIR:+$POLICY_GIT_DIR/agent-os-runtime-source} home_policy_file="$FM_ROOT/config/agent-os-source-policy" +required_policy_file="$FM_ROOT/config/agent-os-source-policy.required" +pending_policy_file="$FM_ROOT/config/agent-os-source-policy.pending" secondmate_home=false if [ -e "$FM_ROOT/.fm-secondmate-home" ]; then [ -f "$FM_ROOT/.fm-secondmate-home" ] && [ ! -L "$FM_ROOT/.fm-secondmate-home" ] || { @@ -77,7 +79,20 @@ if [ -e "$FM_ROOT/.fm-secondmate-home" ]; then } secondmate_home=true fi -if [ "$secondmate_home" = true ] && { [ -e "$policy_file" ] || [ -e "$home_policy_file" ]; }; then +if [ "$secondmate_home" = true ] && \ + { [ -e "$policy_file" ] || [ -e "$home_policy_file" ] || \ + [ -e "$required_policy_file" ] || [ -e "$pending_policy_file" ]; }; then + [ ! -e "$pending_policy_file" ] || { + echo "error: immutable source provenance transition is incomplete" >&2 + exit 2 + } + if [ -e "$required_policy_file" ]; then + [ -f "$required_policy_file" ] && [ ! -L "$required_policy_file" ] && \ + [ "$(cat "$required_policy_file")" = immutable ] || { + echo "error: invalid immutable source provenance requirement" >&2 + exit 2 + } + fi [ -f "$policy_file" ] && [ ! -L "$policy_file" ] && \ [ -f "$home_policy_file" ] && [ ! -L "$home_policy_file" ] || { echo "error: immutable source provenance is incomplete" >&2 @@ -87,7 +102,8 @@ if [ "$secondmate_home" = true ] && { [ -e "$policy_file" ] || [ -e "$home_polic echo "error: immutable source provenance does not match the secondmate home policy" >&2 exit 2 } -elif [ "$secondmate_home" = false ] && [ -e "$home_policy_file" ]; then +elif [ "$secondmate_home" = false ] && \ + { [ -e "$home_policy_file" ] || [ -e "$required_policy_file" ] || [ -e "$pending_policy_file" ]; }; then echo "error: immutable source provenance exists outside a secondmate home" >&2 exit 2 fi diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 94bc21940..7df86e4f8 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -271,6 +271,22 @@ assert_grep 'recovering incomplete candidate reservation' "$IMAGE_WORKFLOW" \ "candidate retries must recover an incomplete reservation under its trusted coordinator" assert_grep 'candidate reservation recovered from durable trusted artifact' "$IMAGE_WORKFLOW" \ "zero-referrer retries must recover durable trusted candidate evidence" +assert_grep 'candidate reservation recovered from exact build evidence' "$IMAGE_WORKFLOW" \ + "zero-referrer retries must reuse an owner-bound pushed image digest" +assert_grep 'agent-os-candidate-build-$GITHUB_SHA' "$IMAGE_WORKFLOW" \ + "candidate builds must persist their exact digest immediately" +build_line=$(grep -n 'name: Build release candidate once' "$IMAGE_WORKFLOW" | cut -d: -f1) +build_evidence_line=$(grep -n 'name: Stage exact candidate build evidence' "$IMAGE_WORKFLOW" | cut -d: -f1) +record_line=$(grep -n 'name: Prepare immutable release candidate record' "$IMAGE_WORKFLOW" | cut -d: -f1) +[ -n "$build_line" ] && [ -n "$build_evidence_line" ] && [ -n "$record_line" ] && \ + [ "$build_line" -lt "$build_evidence_line" ] && [ "$build_evidence_line" -lt "$record_line" ] || \ + fail "exact build evidence must be durable before candidate record preparation" +assert_grep '.path == ".github/workflows/agent-os-image.yml"' "$IMAGE_WORKFLOW" \ + "candidate artifact recovery must bind the authorized workflow" +assert_grep '.repository.full_name == $repo' "$IMAGE_WORKFLOW" \ + "candidate artifact recovery must bind the authorized repository" +assert_grep 'candidate artifact run is outside the authorized claim chain' "$IMAGE_WORKFLOW" \ + "candidate artifact recovery must bind reservation-owner lineage" stage_line=$(grep -n 'name: Stage trusted candidate record evidence' "$IMAGE_WORKFLOW" | cut -d: -f1) publish_line=$(grep -n 'name: Publish immutable release candidate record' "$IMAGE_WORKFLOW" | cut -d: -f1) [ -n "$stage_line" ] && [ -n "$publish_line" ] && [ "$stage_line" -lt "$publish_line" ] || \ diff --git a/tests/agent-os-runtime-secondmates.test.sh b/tests/agent-os-runtime-secondmates.test.sh index 00e8abaa8..e8fd065c2 100755 --- a/tests/agent-os-runtime-secondmates.test.sh +++ b/tests/agent-os-runtime-secondmates.test.sh @@ -75,6 +75,8 @@ for id in linked standalone; do gitdir=$(git -C "$TMP/$id" rev-parse --absolute-git-dir) [ -f "$gitdir/agent-os-runtime-source" ] || fail "$id immutable policy was not persisted" [ -f "$TMP/$id/config/agent-os-source-policy" ] || fail "$id home policy was not persisted" + [ -f "$TMP/$id/config/agent-os-source-policy.required" ] || \ + fail "$id immutable policy requirement was not persisted" cmp "$gitdir/agent-os-runtime-source" "$TMP/$id/config/agent-os-source-policy" || \ fail "$id Git and home policies differ" done @@ -98,6 +100,24 @@ fi cp "$gitdir/agent-os-runtime-source" "$TMP/standalone/config/agent-os-source-policy" pass "secondmate selection rejects mismatched immutable policies" +gitdir=$(git -C "$TMP/standalone" rev-parse --absolute-git-dir) +printf 'immutable\n' > "$TMP/standalone/config/agent-os-source-policy.required" +printf 'mode=release\ncommit=%s\nsource_sha256=%s\n' "$B_COMMIT" "$B_SHA" \ + > "$TMP/standalone/config/agent-os-source-policy.pending" +cp "$TMP/standalone/config/agent-os-source-policy.pending" \ + "$TMP/standalone/config/agent-os-source-policy" +printf 'mode=release\ncommit=%s\nsource_sha256=%s\n' "$A_COMMIT" "$A_SHA" \ + > "$gitdir/agent-os-runtime-source" +run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null || \ + fail "journaled secondmate policy transition was not recovered" +[ ! -e "$TMP/standalone/config/agent-os-source-policy.pending" ] || \ + fail "recovered secondmate policy journal was retained" +cmp "$gitdir/agent-os-runtime-source" "$TMP/standalone/config/agent-os-source-policy" || \ + fail "recovered secondmate policies differ" +run_sync "$TMP/primary-a" "$A_COMMIT" "$A_TREE" "$A_SHA" >/dev/null || \ + fail "secondmate did not return to A after journal recovery" +pass "secondmate selection recovers journaled policy transitions" + git clone -q "$LEASE" "$TMP/redirected" git -C "$TMP/redirected" checkout -q --detach "$A_COMMIT" printf 'owner\n' > "$TMP/redirected/.fm-secondmate-home" @@ -115,6 +135,28 @@ fi rm "$FM_HOME_DIR/state/redirected.meta" pass "secondmate selection requires exact registered marker ownership" +git clone -q "$LEASE" "$TMP/git-victim" +git -C "$TMP/git-victim" checkout -q --detach "$A_COMMIT" +git clone -q "$LEASE" "$TMP/git-redirected" +git -C "$TMP/git-redirected" checkout -q --detach "$A_COMMIT" +printf 'git-redirected\n' > "$TMP/git-redirected/.fm-secondmate-home" +mkdir -p "$TMP/git-redirected/data" "$TMP/git-redirected/state" \ + "$TMP/git-redirected/config" "$TMP/git-redirected/projects" +mv "$TMP/git-redirected/.git" "$TMP/git-redirected-own.git" +printf 'gitdir: %s/git-victim/.git\n' "$TMP" > "$TMP/git-redirected/.git" +{ + printf 'window=firstmate:fm-git-redirected\n' + printf 'kind=secondmate\n' + printf 'home=%s/git-redirected\n' "$TMP" +} > "$FM_HOME_DIR/state/git-redirected.meta" +if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then + fail "secondmate home redirected to another Git directory was accepted" +fi +[ "$(git -C "$TMP/git-victim" rev-parse HEAD)" = "$A_COMMIT" ] || \ + fail "foreign Git metadata was mutated through a redirected secondmate home" +rm "$FM_HOME_DIR/state/git-redirected.meta" +pass "secondmate selection binds Git metadata to its exact home" + git clone -q "$LEASE" "$TMP/linked-alias" git -C "$TMP/linked-alias" checkout -q --detach "$A_COMMIT" printf 'linked\n' > "$TMP/linked-alias/.fm-secondmate-home" diff --git a/tests/fm-update.test.sh b/tests/fm-update.test.sh index 2dcfbe0e4..c926c314c 100755 --- a/tests/fm-update.test.sh +++ b/tests/fm-update.test.sh @@ -339,6 +339,8 @@ test_secondmate_persisted_policy_refuses_sanitized_self_update() { > "$w/linked/config/agent-os-source-policy" printf 'mode=release\ncommit=%s\nsource_sha256=%s\n' "$commit" "$sha" \ > "$standalone/config/agent-os-source-policy" + printf 'immutable\n' > "$w/linked/config/agent-os-source-policy.required" + printf 'immutable\n' > "$standalone/config/agent-os-source-policy.required" for target in "$w/linked" "$standalone"; do set +e @@ -362,6 +364,17 @@ test_secondmate_persisted_policy_refuses_sanitized_self_update() { "missing secondmate Git policy fails closed" [ "$(git -C "$w/linked" rev-parse HEAD)" = "$before" ] || \ fail "secondmate moved after immutable Git policy removal" + rm "$w/linked/config/agent-os-source-policy" + set +e + out=$(env -i PATH="$PATH" HOME="$HOME" FM_ROOT_OVERRIDE="$w/linked" FM_HOME="$w/linked" \ + "$UPDATE" 2>&1) + status=$? + set -e + [ "$status" -eq 2 ] || fail "removed secondmate policy pair exited $status, expected 2" + assert_contains "$out" "immutable source provenance is incomplete" \ + "deletion-evident secondmate policy fails closed after both copies are removed" + [ "$(git -C "$w/linked" rev-parse HEAD)" = "$before" ] || \ + fail "secondmate moved after both immutable policies were removed" pass "T14 linked and standalone secondmates persist immutable update policy" } From 801ee5bedcb0fc4ecfb5da5a828326c8dd013ed4 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 20:29:38 +0200 Subject: [PATCH 45/56] no-mistakes(review): Captain, harden candidate evidence and secondmate recovery --- .github/workflows/agent-os-image.yml | 33 +++++- bin/agent-os-runtime-secondmates.sh | 123 ++++++++++++++++++++- tests/agent-os-container.test.sh | 24 ++-- tests/agent-os-runtime-secondmates.test.sh | 46 +++++++- 4 files changed, 204 insertions(+), 22 deletions(-) diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index 4fbb2d5bd..2b9f405ac 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -446,10 +446,14 @@ jobs: .schema_version == 1 and .commit == $commit and .tree == $tree and .reservation_digest == $reservation and .image_repository == $repository and .owner_run_id == $owner and - (.image_digest | test("^sha256:[0-9a-f]{64}$"))' \ + (.image_digest | test("^sha256:[0-9a-f]{64}$")) and + (.oci_archive_sha256 | test("^[0-9a-f]{64}$"))' \ "$RUNNER_TEMP/candidate-build-artifact/candidate-build.json" >/dev/null image_digest=$(jq -er .image_digest "$RUNNER_TEMP/candidate-build-artifact/candidate-build.json") - printf 'build=false\nrecord_artifact=false\nimage_digest=%s\nbootstrap_sha256=%s\n' \ + archive_sha=$(jq -er .oci_archive_sha256 "$RUNNER_TEMP/candidate-build-artifact/candidate-build.json") + [ "$(sha256sum "$RUNNER_TEMP/candidate-build-artifact/candidate-image.tar" | awk '{print $1}')" = "$archive_sha" ] + cp "$RUNNER_TEMP/candidate-build-artifact/candidate-image.tar" "$RUNNER_TEMP/candidate-image.tar" + printf 'build=false\npublish_image=true\nrecord_artifact=false\nimage_digest=%s\nbootstrap_sha256=%s\n' \ "$image_digest" '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' >> "$GITHUB_OUTPUT" echo "::notice::candidate reservation recovered from exact build evidence" exit 0 @@ -518,7 +522,7 @@ jobs: with: context: . platforms: linux/amd64,linux/arm64 - outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true + outputs: type=oci,dest=${{ runner.temp }}/candidate-image.tar,oci-mediatypes=true cache-from: type=gha cache-to: type=gha,mode=max provenance: mode=max @@ -537,13 +541,15 @@ jobs: set -eu image_digest='${{ steps.candidate_build.outputs.digest }}' [[ "$image_digest" =~ ^sha256:[0-9a-f]{64}$ ]] + archive_sha=$(sha256sum "$RUNNER_TEMP/candidate-image.tar" | awk '{print $1}') jq -cnS \ --arg commit '${{ steps.candidate_source.outputs.commit }}' \ --arg tree '${{ steps.candidate_source.outputs.tree }}' \ --arg reservation '${{ steps.candidate_reservation.outputs.reservation_digest }}' \ --arg repository "$IMAGE" --arg image "$image_digest" \ + --arg archive "$archive_sha" \ --argjson owner "$GITHUB_RUN_ID" \ - '{schema_version:1,commit:$commit,tree:$tree,reservation_digest:$reservation,image_repository:$repository,image_digest:$image,owner_run_id:$owner}' \ + '{schema_version:1,commit:$commit,tree:$tree,reservation_digest:$reservation,image_repository:$repository,image_digest:$image,oci_archive_sha256:$archive,owner_run_id:$owner}' \ > "$RUNNER_TEMP/candidate-build.json" - name: Stage exact candidate build evidence @@ -551,10 +557,27 @@ jobs: uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: agent-os-candidate-build-${{ github.sha }} - path: ${{ runner.temp }}/candidate-build.json + path: | + ${{ runner.temp }}/candidate-build.json + ${{ runner.temp }}/candidate-image.tar if-no-files-found: error retention-days: 90 + - name: Publish exact evidenced candidate image + if: github.ref == 'refs/heads/main' && (steps.candidate_reservation.outputs.build == 'true' || steps.candidate_reservation.outputs.publish_image == 'true') + env: + CANDIDATE_BUILD_DIGEST: ${{ steps.candidate_build.outputs.digest || steps.candidate_reservation.outputs.image_digest }} + run: | + set -eu + image_digest=$CANDIDATE_BUILD_DIGEST + [[ "$image_digest" =~ ^sha256:[0-9a-f]{64}$ ]] + archive_sha=$(jq -er .oci_archive_sha256 "$RUNNER_TEMP/candidate-build.json" 2>/dev/null || \ + jq -er .oci_archive_sha256 "$RUNNER_TEMP/candidate-build-artifact/candidate-build.json") + [ "$(sha256sum "$RUNNER_TEMP/candidate-image.tar" | awk '{print $1}')" = "$archive_sha" ] + oras cp --from-oci-layout "$RUNNER_TEMP/candidate-image.tar@$image_digest" "$IMAGE" + oras manifest fetch "$IMAGE@$image_digest" > "$RUNNER_TEMP/candidate-image-readback.json" + [ "sha256:$(sha256sum "$RUNNER_TEMP/candidate-image-readback.json" | awk '{print $1}')" = "$image_digest" ] + - name: Prepare immutable release candidate record if: github.ref == 'refs/heads/main' id: candidate_record diff --git a/bin/agent-os-runtime-secondmates.sh b/bin/agent-os-runtime-secondmates.sh index 0ceb91335..e49f8570f 100755 --- a/bin/agent-os-runtime-secondmates.sh +++ b/bin/agent-os-runtime-secondmates.sh @@ -63,7 +63,7 @@ resolve_file_path() { } validate_git_binding() { - local root=$1 expected backlink backlink_path relative + local root=$1 expected backlink backlink_path relative runtime_root source_root source_key policy_commit policy_sha expected=$(resolve_file_path "$root/.git") || return 1 if [ -d "$root/.git" ] && [ ! -L "$root/.git" ]; then [ "$RESOLVED_GIT_DIR" = "$expected" ] || return 1 @@ -86,6 +86,16 @@ validate_git_binding() { relative=${RESOLVED_GIT_DIR#"$RESOLVED_COMMON_DIR"/worktrees/} case "$relative" in ''|*/*) return 1 ;; esac [ -d "$RESOLVED_COMMON_DIR" ] && [ ! -L "$RESOLVED_COMMON_DIR" ] || return 1 + runtime_root=$(cd "$FM_HOME/runtime-sources" 2>/dev/null && pwd -P) || return 1 + source_root=$(cd "$RESOLVED_COMMON_DIR/.." 2>/dev/null && pwd -P) || return 1 + case "$source_root" in "$runtime_root"/*) ;; *) return 1 ;; esac + source_key=${source_root#"$runtime_root"/} + case "$source_key" in ''|*/*) return 1 ;; esac + [ "$RESOLVED_COMMON_DIR" = "$source_root/.git" ] || return 1 + validate_policy "$RESOLVED_COMMON_DIR/agent-os-runtime-source" || return 1 + policy_commit=$(sed -n 's/^commit=//p' "$RESOLVED_COMMON_DIR/agent-os-runtime-source") + policy_sha=$(sed -n 's/^source_sha256=//p' "$RESOLVED_COMMON_DIR/agent-os-runtime-source") + [ "$source_key" = "$policy_commit-$policy_sha" ] } validate_config() { @@ -139,6 +149,86 @@ validate_policy() { [ "$(wc -l < "$policy" | tr -d ' ')" -eq 3 ] } +trusted_tree_entry() { + local home=$1 commit=$2 path=$3 entry + entry=$(trusted_git -C "$home" ls-tree "$commit" -- "$path") || return 1 + printf '%s' "$entry" | awk 'NR == 1 { print $1 " " $3 }' +} + +worktree_entry() { + local home=$1 path=$2 mode hash + if [ -L "$home/$path" ]; then + mode=120000 + hash=$(printf '%s' "$(readlink "$home/$path")" | trusted_git -C "$home" hash-object --stdin) + elif [ -f "$home/$path" ]; then + if [ -x "$home/$path" ]; then mode=100755; else mode=100644; fi + hash=$(trusted_git -C "$home" hash-object -- "$home/$path") + elif [ ! -e "$home/$path" ]; then + printf '\n' + return 0 + else + return 1 + fi + printf '%s %s\n' "$mode" "$hash" +} + +entry_matches_transition() { + local home=$1 path=$2 actual=$3 commit expected + shift 3 + for commit in "$@"; do + expected=$(trusted_tree_entry "$home" "$commit" "$path") || return 1 + [ "$actual" = "$expected" ] && return 0 + done + return 1 +} + +verify_pending_checkout() { + local home=$1 pending=$2 current_commit pending_commit record path worktree index index_entries status_file valid + current_commit=$(trusted_git -C "$home" rev-parse HEAD) || return 1 + pending_commit=$(sed -n 's/^commit=//p' "$pending") + trusted_git -C "$home" rev-parse "$pending_commit^{commit}" >/dev/null || return 1 + status_file=$(mktemp "${TMPDIR:-/tmp}/agent-os-secondmate-status.XXXXXX") || return 1 + if ! trusted_git -c status.renames=false -C "$home" status \ + --porcelain=v1 -z --untracked-files=all > "$status_file"; then + rm -f "$status_file" + return 1 + fi + RECOVERY_PATHS=() + valid=true + while IFS= read -r -d '' record; do + path=${record:3} + if [ -z "$path" ]; then valid=false; break; fi + worktree=$(worktree_entry "$home" "$path") || { valid=false; break; } + entry_matches_transition "$home" "$path" "$worktree" \ + "$current_commit" "$pending_commit" "$SOURCE_COMMIT" || { valid=false; break; } + index_entries=$(trusted_git -C "$home" ls-files -s -- "$path") || { valid=false; break; } + index=$(printf '%s' "$index_entries" | awk ' + NR == 1 { entry=$1 " " $2 } + END { if (NR <= 1) print entry; else exit 1 } + ') || { valid=false; break; } + entry_matches_transition "$home" "$path" "$index" \ + "$current_commit" "$pending_commit" "$SOURCE_COMMIT" || { valid=false; break; } + RECOVERY_PATHS+=("$path") + done < "$status_file" + rm -f "$status_file" + [ "$valid" = true ] +} + +recover_pending_checkout() { + local home=$1 pending=$2 old_head path + verify_pending_checkout "$home" "$pending" || return 1 + old_head=$(trusted_git -C "$home" rev-parse HEAD) || return 1 + trusted_git -C "$home" restore --source="$SOURCE_COMMIT" --staged --worktree -- . + for path in "${RECOVERY_PATHS[@]}"; do + if [ -z "$(trusted_tree_entry "$home" "$SOURCE_COMMIT" "$path")" ] && \ + { [ -f "$home/$path" ] || [ -L "$home/$path" ]; }; then + rm -f -- "$home/$path" + fi + done + trusted_git -C "$home" update-ref --no-deref HEAD "$SOURCE_COMMIT" "$old_head" + [ -z "$(trusted_git -C "$home" status --porcelain --untracked-files=all)" ] +} + ids=() homes=() add_home() { @@ -172,6 +262,7 @@ fi validated_ids=() validated_homes=() git_dirs=() +pending_policies=() seen_homes=$'\n' for index in "${!homes[@]}"; do id=${ids[$index]} @@ -214,10 +305,15 @@ for index in "${!homes[@]}"; do echo "error: secondmate Git configuration is invalid: $home" >&2 exit 2 } - [ -z "$(trusted_git -C "$home" status --porcelain --untracked-files=all)" ] || { - echo "error: secondmate source is not clean: $home" >&2 - exit 2 - } + if [ -f "$home/.git" ]; then + common_source=$(cd "$RESOLVED_COMMON_DIR/.." && pwd -P) + common_commit=$(sed -n 's/^commit=//p' "$RESOLVED_COMMON_DIR/agent-os-runtime-source") + [ "$(trusted_git -C "$common_source" rev-parse HEAD)" = "$common_commit" ] && \ + [ -z "$(trusted_git -C "$common_source" status --porcelain --untracked-files=all)" ] || { + echo "error: secondmate linked Git ownership is invalid: $home" >&2 + exit 2 + } + fi git_policy="$RESOLVED_GIT_DIR/agent-os-runtime-source" home_policy="$home/config/agent-os-source-policy" required_policy="$home/config/agent-os-source-policy.required" @@ -242,13 +338,16 @@ for index in "${!homes[@]}"; do echo "error: secondmate immutable policy journal is invalid: $home" >&2 exit 2 } + selected_pending=$pending_policy elif [ -e "$required_policy" ]; then + selected_pending= if ! { [ -e "$git_policy" ] && [ -e "$home_policy" ] && \ cmp "$git_policy" "$home_policy"; }; then echo "error: secondmate immutable policy is incomplete: $home" >&2 exit 2 fi else + selected_pending= if [ -e "$home_policy" ] && [ ! -e "$git_policy" ]; then echo "error: secondmate immutable policy is incomplete: $home" >&2 exit 2 @@ -260,9 +359,15 @@ for index in "${!homes[@]}"; do } fi fi + if [ -z "$selected_pending" ] && \ + [ -n "$(trusted_git -C "$home" status --porcelain --untracked-files=all)" ]; then + echo "error: secondmate source is not clean: $home" >&2 + exit 2 + fi validated_ids+=("$id") validated_homes+=("$home") git_dirs+=("$RESOLVED_GIT_DIR") + pending_policies+=("$selected_pending") done marker="mode=$SOURCE_MODE @@ -271,9 +376,17 @@ source_sha256=$SOURCE_SHA" for index in "${!validated_homes[@]}"; do home=${validated_homes[$index]} git_dir=${git_dirs[$index]} + recovery_policy=${pending_policies[$index]} trusted_git -c protocol.file.allow=always -C "$home" fetch --no-tags "$FM_ROOT" "$SOURCE_COMMIT" [ "$(trusted_git -C "$home" rev-parse FETCH_HEAD)" = "$SOURCE_COMMIT" ] || exit 2 [ "$(trusted_git -C "$home" rev-parse 'FETCH_HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 + if [ -n "$recovery_policy" ] && \ + [ -n "$(trusted_git -C "$home" status --porcelain --untracked-files=all)" ]; then + recover_pending_checkout "$home" "$recovery_policy" || { + echo "error: secondmate immutable policy journal cannot recover source: $home" >&2 + exit 2 + } + fi mkdir -p "$home/config" pending_policy="$home/config/agent-os-source-policy.pending" pending_tmp="$home/config/.agent-os-source-policy.pending.$$" diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 7df86e4f8..f4c35fc78 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -272,15 +272,23 @@ assert_grep 'recovering incomplete candidate reservation' "$IMAGE_WORKFLOW" \ assert_grep 'candidate reservation recovered from durable trusted artifact' "$IMAGE_WORKFLOW" \ "zero-referrer retries must recover durable trusted candidate evidence" assert_grep 'candidate reservation recovered from exact build evidence' "$IMAGE_WORKFLOW" \ - "zero-referrer retries must reuse an owner-bound pushed image digest" + "zero-referrer retries must reuse an owner-bound retained image digest" assert_grep 'agent-os-candidate-build-$GITHUB_SHA' "$IMAGE_WORKFLOW" \ "candidate builds must persist their exact digest immediately" build_line=$(grep -n 'name: Build release candidate once' "$IMAGE_WORKFLOW" | cut -d: -f1) build_evidence_line=$(grep -n 'name: Stage exact candidate build evidence' "$IMAGE_WORKFLOW" | cut -d: -f1) +image_publish_line=$(grep -n 'name: Publish exact evidenced candidate image' "$IMAGE_WORKFLOW" | cut -d: -f1) record_line=$(grep -n 'name: Prepare immutable release candidate record' "$IMAGE_WORKFLOW" | cut -d: -f1) -[ -n "$build_line" ] && [ -n "$build_evidence_line" ] && [ -n "$record_line" ] && \ - [ "$build_line" -lt "$build_evidence_line" ] && [ "$build_evidence_line" -lt "$record_line" ] || \ - fail "exact build evidence must be durable before candidate record preparation" +[ -n "$build_line" ] && [ -n "$build_evidence_line" ] && [ -n "$image_publish_line" ] && \ + [ -n "$record_line" ] && [ "$build_line" -lt "$build_evidence_line" ] && \ + [ "$build_evidence_line" -lt "$image_publish_line" ] && [ "$image_publish_line" -lt "$record_line" ] || \ + fail "exact build evidence must be durable before candidate image publication" +assert_grep 'type=oci,dest=${{ runner.temp }}/candidate-image.tar' "$IMAGE_WORKFLOW" \ + "candidate builds must retain their exact OCI output before publication" +assert_grep 'oci_archive_sha256' "$IMAGE_WORKFLOW" \ + "candidate recovery must authenticate the retained OCI output" +assert_grep 'oras cp --from-oci-layout "$RUNNER_TEMP/candidate-image.tar@$image_digest" "$IMAGE"' "$IMAGE_WORKFLOW" \ + "candidate publication must copy only the evidenced content-addressed image" assert_grep '.path == ".github/workflows/agent-os-image.yml"' "$IMAGE_WORKFLOW" \ "candidate artifact recovery must bind the authorized workflow" assert_grep '.repository.full_name == $repo' "$IMAGE_WORKFLOW" \ @@ -325,8 +333,8 @@ assert_no_grep 'oras tag ' "$IMAGE_WORKFLOW" \ "release publication must never create a mutable registry semver tag" assert_grep 'release registry tag coordinate already exists' "$IMAGE_WORKFLOW" \ "release publication must reject conflicting mutable registry coordinates" -assert_no_grep 'oras cp --from-oci-layout' "$IMAGE_WORKFLOW" \ - "release publication must not rebuild or re-upload an OCI layout" +[ "$(grep -c 'oras cp --from-oci-layout' "$IMAGE_WORKFLOW")" -eq 1 ] || \ + fail "only initial evidenced candidate publication may upload an OCI layout" assert_grep 'oras-project/setup-oras@22ce207df3b08e061f537244349aac6ae1d214f6' "$IMAGE_WORKFLOW" \ "release publication must pin the OCI transfer implementation" assert_grep 'Canonical install digest:' "$IMAGE_WORKFLOW" \ @@ -392,8 +400,8 @@ assert_grep 'linux/amd64,linux/arm64' "$ROOT/.github/workflows/agent-os-image.ym fail "the read-only validation job must build without publishing" [ "$(grep -c '^ push: true$' "$IMAGE_WORKFLOW")" -eq 1 ] || \ fail "only the protected main publication build may use mutable-tag push mode" -assert_grep 'push-by-digest=true' "$IMAGE_WORKFLOW" \ - "the release candidate must publish without a mutable image tag" +assert_no_grep 'push-by-digest=true,name-canonical=true,push=true' "$IMAGE_WORKFLOW" \ + "the candidate build must not push before exact evidence is durable" assert_grep 'id: build' "$ROOT/.github/workflows/agent-os-image.yml" \ "release workflow must expose its build result" assert_grep 'steps.build.outputs.digest' "$ROOT/.github/workflows/agent-os-image.yml" \ diff --git a/tests/agent-os-runtime-secondmates.test.sh b/tests/agent-os-runtime-secondmates.test.sh index e8fd065c2..f7aa43b2a 100755 --- a/tests/agent-os-runtime-secondmates.test.sh +++ b/tests/agent-os-runtime-secondmates.test.sh @@ -7,7 +7,7 @@ SYNC="$ROOT/bin/agent-os-runtime-secondmates.sh" TMP=$(fm_test_tmproot agent-os-runtime-secondmates) FM_HOME_DIR="$TMP/home" WORK="$TMP/work" -LEASE="$TMP/lease" +LEASE= fm_git_identity fmtest fmtest@example.com @@ -27,10 +27,16 @@ git -C "$WORK" add version git -C "$WORK" commit -qm B B_COMMIT=$(git -C "$WORK" rev-parse HEAD) B_TREE=$(git -C "$WORK" rev-parse 'HEAD^{tree}') +B_SHA=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +A_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +LEASE="$FM_HOME_DIR/runtime-sources/$A_COMMIT-$A_SHA" +mkdir -p "$FM_HOME_DIR/runtime-sources" git clone -q "$WORK" "$LEASE" git -C "$LEASE" remote set-url origin https://github.com/akua-dev/agent-os.git git -C "$LEASE" checkout -q --detach "$A_COMMIT" +printf 'mode=release\ncommit=%s\nsource_sha256=%s\n' "$A_COMMIT" "$A_SHA" \ + > "$LEASE/.git/agent-os-runtime-source" git -C "$LEASE" worktree add -q --detach "$TMP/linked" "$A_COMMIT" git clone -q "$LEASE" "$TMP/standalone" git -C "$TMP/standalone" checkout -q --detach "$A_COMMIT" @@ -64,9 +70,6 @@ run_sync() { make_primary primary-b "$B_COMMIT" make_primary primary-a "$A_COMMIT" -B_SHA=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb -A_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null || \ fail "A to B secondmate selection failed" for id in linked standalone; do @@ -118,6 +121,41 @@ run_sync "$TMP/primary-a" "$A_COMMIT" "$A_TREE" "$A_SHA" >/dev/null || \ fail "secondmate did not return to A after journal recovery" pass "secondmate selection recovers journaled policy transitions" +gitdir=$(git -C "$TMP/standalone" rev-parse --absolute-git-dir) +printf 'mode=release\ncommit=%s\nsource_sha256=%s\n' "$B_COMMIT" "$B_SHA" \ + > "$TMP/standalone/config/agent-os-source-policy.pending" +git -C "$WORK" show "$B_COMMIT:version" > "$TMP/standalone/version" +run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null || \ + fail "interrupted secondmate checkout was not recovered from its journal" +[ "$(git -C "$TMP/standalone" rev-parse HEAD)" = "$B_COMMIT" ] || \ + fail "interrupted secondmate checkout did not select B" +[ -z "$(git -C "$TMP/standalone" status --porcelain --untracked-files=all)" ] || \ + fail "interrupted secondmate checkout recovery remained dirty" +[ ! -e "$TMP/standalone/config/agent-os-source-policy.pending" ] || \ + fail "interrupted secondmate checkout retained its journal" +run_sync "$TMP/primary-a" "$A_COMMIT" "$A_TREE" "$A_SHA" >/dev/null || \ + fail "secondmate did not return to A after interrupted checkout recovery" +pass "secondmate selection verifies and recovers interrupted journaled checkouts" + +git clone -q "$LEASE" "$TMP/foreign-common" +git -C "$TMP/foreign-common" checkout -q --detach "$A_COMMIT" +git -C "$TMP/foreign-common" worktree add -q --detach "$TMP/foreign-linked" "$A_COMMIT" +printf 'foreign-linked\n' > "$TMP/foreign-linked/.fm-secondmate-home" +mkdir -p "$TMP/foreign-linked/data" "$TMP/foreign-linked/state" \ + "$TMP/foreign-linked/config" "$TMP/foreign-linked/projects" +{ + printf 'window=firstmate:fm-foreign-linked\n' + printf 'kind=secondmate\n' + printf 'home=%s/foreign-linked\n' "$TMP" +} > "$FM_HOME_DIR/state/foreign-linked.meta" +if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then + fail "linked secondmate owned by a foreign common directory was accepted" +fi +[ "$(git -C "$TMP/foreign-linked" rev-parse HEAD)" = "$A_COMMIT" ] || \ + fail "foreign linked-worktree metadata was mutated" +rm "$FM_HOME_DIR/state/foreign-linked.meta" +pass "secondmate selection binds linked common metadata to runtime-source ownership" + git clone -q "$LEASE" "$TMP/redirected" git -C "$TMP/redirected" checkout -q --detach "$A_COMMIT" printf 'owner\n' > "$TMP/redirected/.fm-secondmate-home" From 3ac84d9b6620b7dca9b8c543ff363f9deb9c5716 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 20:46:23 +0200 Subject: [PATCH 46/56] no-mistakes(review): Captain, prevent ambiguous candidate rebuilds --- .github/workflows/agent-os-image.yml | 73 +++++++++++++++++++++++++++- tests/agent-os-container.test.sh | 16 ++++-- 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index 2b9f405ac..a428791b1 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -258,6 +258,31 @@ jobs: expected_message=$(printf 'agent-os-candidate-reservation-v1\nsource-commit=%s\nowner-run-id=%s' \ "$GITHUB_SHA" "$CLAIM_OWNER") [ "$(jq -r .message "$RUNNER_TEMP/candidate-claim-commit.json")" = "$expected_message" ] + CLAIM_SHA=$claim_sha + } + load_build_attempt() { + owner_run_id=$1 + expected_claim_sha=$2 + expected_reservation_digest=$3 + attempt_ref="refs/agent-os/candidate-build-attempts/$GITHUB_SHA/$owner_run_id" + if ! gh api "repos/$GITHUB_REPOSITORY/git/ref/${attempt_ref#refs/}" \ + > "$RUNNER_TEMP/candidate-build-attempt-ref.json" \ + 2> "$RUNNER_TEMP/candidate-build-attempt-ref.err"; then + if grep -Eiq '404|Not Found' "$RUNNER_TEMP/candidate-build-attempt-ref.err"; then + return 1 + fi + cat "$RUNNER_TEMP/candidate-build-attempt-ref.err" >&2 + exit 1 + fi + attempt_sha=$(jq -er .object.sha "$RUNNER_TEMP/candidate-build-attempt-ref.json") + gh api "repos/$GITHUB_REPOSITORY/git/commits/$attempt_sha" \ + > "$RUNNER_TEMP/candidate-build-attempt-commit.json" + jq -e --arg source_tree "$source_tree_sha" --arg claim "$expected_claim_sha" \ + '.tree.sha == $source_tree and (.parents | length == 1) and .parents[0].sha == $claim' \ + "$RUNNER_TEMP/candidate-build-attempt-commit.json" >/dev/null + expected_attempt_message=$(printf 'agent-os-candidate-build-attempt-v1\nsource-commit=%s\nsource-tree=%s\nreservation-digest=%s\nowner-run-id=%s\nclaim-commit=%s' \ + "$GITHUB_SHA" "$source_tree_sha" "$expected_reservation_digest" "$owner_run_id" "$expected_claim_sha") + [ "$(jq -r .message "$RUNNER_TEMP/candidate-build-attempt-commit.json")" = "$expected_attempt_message" ] } load_latest_claim() { claim_ref="$claim_prefix/initial" @@ -366,7 +391,7 @@ jobs: referrer_count=$(jq -er '.referrers | length' "$RUNNER_TEMP/candidate-reservation-referrers.json") if [ "$referrer_count" -eq 0 ]; then artifact_owner=$CLAIM_OWNER - ensure_build_owner + artifact_claim_sha=$CLAIM_SHA artifact_name="agent-os-candidate-record-$GITHUB_SHA" gh api --method GET "repos/$GITHUB_REPOSITORY/actions/artifacts?name=$artifact_name&per_page=100" \ > "$RUNNER_TEMP/candidate-trusted-artifacts.json" @@ -415,6 +440,7 @@ jobs: cp "$RUNNER_TEMP/candidate-trusted-artifact/agent-os-bootstrap.tar" image/agent-os-bootstrap.tar image_digest=$(jq -er .image_digest "$RUNNER_TEMP/candidate-trusted-artifact/candidate-record.json") bootstrap_sha=$(jq -er .bootstrap_archive_sha256 "$RUNNER_TEMP/candidate-trusted-artifact/candidate-record.json") + ensure_build_owner printf 'build=false\nrecord_artifact=true\ncandidate_record_digest=%s\nimage_digest=%s\nbootstrap_sha256=%s\n' \ "$candidate_record_digest" "$image_digest" "$bootstrap_sha" >> "$GITHUB_OUTPUT" echo "::notice::candidate reservation recovered from durable trusted artifact" @@ -453,12 +479,19 @@ jobs: archive_sha=$(jq -er .oci_archive_sha256 "$RUNNER_TEMP/candidate-build-artifact/candidate-build.json") [ "$(sha256sum "$RUNNER_TEMP/candidate-build-artifact/candidate-image.tar" | awk '{print $1}')" = "$archive_sha" ] cp "$RUNNER_TEMP/candidate-build-artifact/candidate-image.tar" "$RUNNER_TEMP/candidate-image.tar" + ensure_build_owner printf 'build=false\npublish_image=true\nrecord_artifact=false\nimage_digest=%s\nbootstrap_sha256=%s\n' \ "$image_digest" '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' >> "$GITHUB_OUTPUT" echo "::notice::candidate reservation recovered from exact build evidence" exit 0 fi - printf 'build=true\nrecord_artifact=false\nbootstrap_sha256=%s\n' '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' >> "$GITHUB_OUTPUT" + if load_build_attempt "$artifact_owner" "$artifact_claim_sha" "$reservation_digest"; then + echo "::error::candidate build attempt has no durable exact evidence; refusing rebuild" + exit 1 + fi + ensure_build_owner + printf 'build=true\nrecord_artifact=false\nbootstrap_sha256=%s\nclaim_sha=%s\nclaim_owner=%s\n' \ + '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' "$CLAIM_SHA" "$CLAIM_OWNER" >> "$GITHUB_OUTPUT" exit 0 fi [ "$referrer_count" -eq 1 ] || { echo "::error::candidate reservation has conflicting records"; exit 1; } @@ -515,6 +548,42 @@ jobs: exit 1 fi + - name: Claim owner-bound candidate build attempt + if: github.ref == 'refs/heads/main' && steps.candidate_reservation.outputs.build == 'true' + env: + GH_TOKEN: ${{ github.token }} + run: | + set -eu + claim_sha='${{ steps.candidate_reservation.outputs.claim_sha }}' + claim_owner='${{ steps.candidate_reservation.outputs.claim_owner }}' + reservation_digest='${{ steps.candidate_reservation.outputs.reservation_digest }}' + [[ "$claim_sha" =~ ^[0-9a-f]{40}$ ]] + [ "$claim_owner" = "$GITHUB_RUN_ID" ] + [[ "$reservation_digest" =~ ^sha256:[0-9a-f]{64}$ ]] + source_tree_sha=$(gh api "repos/$GITHUB_REPOSITORY/git/commits/$GITHUB_SHA" --jq .tree.sha) + [ "$source_tree_sha" = '${{ steps.candidate_source.outputs.tree }}' ] + attempt_ref="refs/agent-os/candidate-build-attempts/$GITHUB_SHA/$GITHUB_RUN_ID" + attempt_message=$(printf 'agent-os-candidate-build-attempt-v1\nsource-commit=%s\nsource-tree=%s\nreservation-digest=%s\nowner-run-id=%s\nclaim-commit=%s' \ + "$GITHUB_SHA" "$source_tree_sha" "$reservation_digest" "$GITHUB_RUN_ID" "$claim_sha") + attempt_commit=$(gh api --method POST "repos/$GITHUB_REPOSITORY/git/commits" \ + --raw-field message="$attempt_message" --raw-field tree="$source_tree_sha" \ + --raw-field "parents[]=$claim_sha" --jq .sha) + if ! gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" \ + --raw-field ref="$attempt_ref" --raw-field sha="$attempt_commit" \ + > "$RUNNER_TEMP/candidate-build-attempt-create.json" \ + 2> "$RUNNER_TEMP/candidate-build-attempt-create.err"; then + cat "$RUNNER_TEMP/candidate-build-attempt-create.err" >&2 + echo "::error::candidate build attempt already exists or could not be reserved; refusing rebuild" + exit 1 + fi + readback_sha=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/${attempt_ref#refs/}" --jq .object.sha) + [ "$readback_sha" = "$attempt_commit" ] + gh api "repos/$GITHUB_REPOSITORY/git/commits/$readback_sha" \ + > "$RUNNER_TEMP/candidate-build-attempt-readback.json" + jq -e --arg tree "$source_tree_sha" --arg claim "$claim_sha" --arg message "$attempt_message" \ + '.tree.sha == $tree and (.parents | length == 1) and .parents[0].sha == $claim and .message == $message' \ + "$RUNNER_TEMP/candidate-build-attempt-readback.json" >/dev/null + - name: Build release candidate once if: github.ref == 'refs/heads/main' && steps.candidate_reservation.outputs.build == 'true' id: candidate_build diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index f4c35fc78..4fdbfe1c5 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -275,14 +275,24 @@ assert_grep 'candidate reservation recovered from exact build evidence' "$IMAGE_ "zero-referrer retries must reuse an owner-bound retained image digest" assert_grep 'agent-os-candidate-build-$GITHUB_SHA' "$IMAGE_WORKFLOW" \ "candidate builds must persist their exact digest immediately" +assert_grep 'refs/agent-os/candidate-build-attempts/$GITHUB_SHA/$owner_run_id' "$IMAGE_WORKFLOW" \ + "candidate retries must resolve the exact owner-bound build attempt" +assert_grep 'candidate build attempt has no durable exact evidence; refusing rebuild' "$IMAGE_WORKFLOW" \ + "candidate cancellation must fail closed instead of rebuilding" +attempt_line=$(grep -n 'name: Claim owner-bound candidate build attempt' "$IMAGE_WORKFLOW" | cut -d: -f1) build_line=$(grep -n 'name: Build release candidate once' "$IMAGE_WORKFLOW" | cut -d: -f1) build_evidence_line=$(grep -n 'name: Stage exact candidate build evidence' "$IMAGE_WORKFLOW" | cut -d: -f1) image_publish_line=$(grep -n 'name: Publish exact evidenced candidate image' "$IMAGE_WORKFLOW" | cut -d: -f1) record_line=$(grep -n 'name: Prepare immutable release candidate record' "$IMAGE_WORKFLOW" | cut -d: -f1) -[ -n "$build_line" ] && [ -n "$build_evidence_line" ] && [ -n "$image_publish_line" ] && \ - [ -n "$record_line" ] && [ "$build_line" -lt "$build_evidence_line" ] && \ +[ -n "$attempt_line" ] && [ -n "$build_line" ] && [ -n "$build_evidence_line" ] && \ + [ -n "$image_publish_line" ] && [ -n "$record_line" ] && [ "$attempt_line" -lt "$build_line" ] && \ + [ "$build_line" -lt "$build_evidence_line" ] && \ [ "$build_evidence_line" -lt "$image_publish_line" ] && [ "$image_publish_line" -lt "$record_line" ] || \ - fail "exact build evidence must be durable before candidate image publication" + fail "owner-bound attempt state must precede build and exact evidence publication" +attempt_guard_line=$(grep -n 'if load_build_attempt "$artifact_owner"' "$IMAGE_WORKFLOW" | cut -d: -f1) +rebuild_line=$(grep -n "printf 'build=true" "$IMAGE_WORKFLOW" | cut -d: -f1) +[ -n "$attempt_guard_line" ] && [ -n "$rebuild_line" ] && [ "$attempt_guard_line" -lt "$rebuild_line" ] || \ + fail "candidate upload failure must be rejected before any rebuild is authorized" assert_grep 'type=oci,dest=${{ runner.temp }}/candidate-image.tar' "$IMAGE_WORKFLOW" \ "candidate builds must retain their exact OCI output before publication" assert_grep 'oci_archive_sha256' "$IMAGE_WORKFLOW" \ From 74dfa755142c36d13f19ef6aea57b4476d333efd Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 21:14:49 +0200 Subject: [PATCH 47/56] no-mistakes(review): Captain, harden candidate and secondmate release integrity --- .github/workflows/agent-os-image.yml | 44 +++- bin/agent-os-candidate-state.sh | 62 ++++++ bin/agent-os-runtime-bound.sh | 36 +++ bin/agent-os-runtime-secondmates.sh | 248 +++++++++++++++++---- tests/agent-os-candidate-state.test.sh | 48 ++++ tests/agent-os-container.test.sh | 9 +- tests/agent-os-runtime-secondmates.test.sh | 128 ++++++++++- 7 files changed, 510 insertions(+), 65 deletions(-) create mode 100755 bin/agent-os-candidate-state.sh create mode 100755 bin/agent-os-runtime-bound.sh create mode 100755 tests/agent-os-candidate-state.test.sh diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index a428791b1..7f77617ec 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -265,24 +265,43 @@ jobs: expected_claim_sha=$2 expected_reservation_digest=$3 attempt_ref="refs/agent-os/candidate-build-attempts/$GITHUB_SHA/$owner_run_id" - if ! gh api "repos/$GITHUB_REPOSITORY/git/ref/${attempt_ref#refs/}" \ + BUILD_ATTEMPT_STATE= + if gh api "repos/$GITHUB_REPOSITORY/git/ref/${attempt_ref#refs/}" \ > "$RUNNER_TEMP/candidate-build-attempt-ref.json" \ 2> "$RUNNER_TEMP/candidate-build-attempt-ref.err"; then - if grep -Eiq '404|Not Found' "$RUNNER_TEMP/candidate-build-attempt-ref.err"; then - return 1 + : + else + if grep -Eq '^gh: Not Found \(HTTP 404\)$' \ + "$RUNNER_TEMP/candidate-build-attempt-ref.err"; then + BUILD_ATTEMPT_STATE=absent + return 0 fi cat "$RUNNER_TEMP/candidate-build-attempt-ref.err" >&2 + echo "::error::candidate build attempt reference is unreadable" exit 1 fi - attempt_sha=$(jq -er .object.sha "$RUNNER_TEMP/candidate-build-attempt-ref.json") + attempt_sha=$(jq -er .object.sha "$RUNNER_TEMP/candidate-build-attempt-ref.json") || { + echo "::error::candidate build attempt reference metadata is invalid" + exit 1 + } gh api "repos/$GITHUB_REPOSITORY/git/commits/$attempt_sha" \ - > "$RUNNER_TEMP/candidate-build-attempt-commit.json" + > "$RUNNER_TEMP/candidate-build-attempt-commit.json" || { + echo "::error::candidate build attempt commit is unreadable" + exit 1 + } jq -e --arg source_tree "$source_tree_sha" --arg claim "$expected_claim_sha" \ '.tree.sha == $source_tree and (.parents | length == 1) and .parents[0].sha == $claim' \ - "$RUNNER_TEMP/candidate-build-attempt-commit.json" >/dev/null + "$RUNNER_TEMP/candidate-build-attempt-commit.json" >/dev/null || { + echo "::error::candidate build attempt ownership metadata is invalid" + exit 1 + } expected_attempt_message=$(printf 'agent-os-candidate-build-attempt-v1\nsource-commit=%s\nsource-tree=%s\nreservation-digest=%s\nowner-run-id=%s\nclaim-commit=%s' \ "$GITHUB_SHA" "$source_tree_sha" "$expected_reservation_digest" "$owner_run_id" "$expected_claim_sha") - [ "$(jq -r .message "$RUNNER_TEMP/candidate-build-attempt-commit.json")" = "$expected_attempt_message" ] + [ "$(jq -r .message "$RUNNER_TEMP/candidate-build-attempt-commit.json")" = "$expected_attempt_message" ] || { + echo "::error::candidate build attempt message is invalid" + exit 1 + } + BUILD_ATTEMPT_STATE=attempted } load_latest_claim() { claim_ref="$claim_prefix/initial" @@ -392,6 +411,7 @@ jobs: if [ "$referrer_count" -eq 0 ]; then artifact_owner=$CLAIM_OWNER artifact_claim_sha=$CLAIM_SHA + load_build_attempt "$artifact_owner" "$artifact_claim_sha" "$reservation_digest" artifact_name="agent-os-candidate-record-$GITHUB_SHA" gh api --method GET "repos/$GITHUB_REPOSITORY/actions/artifacts?name=$artifact_name&per_page=100" \ > "$RUNNER_TEMP/candidate-trusted-artifacts.json" @@ -440,6 +460,8 @@ jobs: cp "$RUNNER_TEMP/candidate-trusted-artifact/agent-os-bootstrap.tar" image/agent-os-bootstrap.tar image_digest=$(jq -er .image_digest "$RUNNER_TEMP/candidate-trusted-artifact/candidate-record.json") bootstrap_sha=$(jq -er .bootstrap_archive_sha256 "$RUNNER_TEMP/candidate-trusted-artifact/candidate-record.json") + decision=$(bin/agent-os-candidate-state.sh "$BUILD_ATTEMPT_STATE" exact-record exact) + [ "$decision" = reuse-record ] ensure_build_owner printf 'build=false\nrecord_artifact=true\ncandidate_record_digest=%s\nimage_digest=%s\nbootstrap_sha256=%s\n' \ "$candidate_record_digest" "$image_digest" "$bootstrap_sha" >> "$GITHUB_OUTPUT" @@ -479,16 +501,16 @@ jobs: archive_sha=$(jq -er .oci_archive_sha256 "$RUNNER_TEMP/candidate-build-artifact/candidate-build.json") [ "$(sha256sum "$RUNNER_TEMP/candidate-build-artifact/candidate-image.tar" | awk '{print $1}')" = "$archive_sha" ] cp "$RUNNER_TEMP/candidate-build-artifact/candidate-image.tar" "$RUNNER_TEMP/candidate-image.tar" + decision=$(bin/agent-os-candidate-state.sh "$BUILD_ATTEMPT_STATE" exact-build exact) + [ "$decision" = reuse-build ] ensure_build_owner printf 'build=false\npublish_image=true\nrecord_artifact=false\nimage_digest=%s\nbootstrap_sha256=%s\n' \ "$image_digest" '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' >> "$GITHUB_OUTPUT" echo "::notice::candidate reservation recovered from exact build evidence" exit 0 fi - if load_build_attempt "$artifact_owner" "$artifact_claim_sha" "$reservation_digest"; then - echo "::error::candidate build attempt has no durable exact evidence; refusing rebuild" - exit 1 - fi + decision=$(bin/agent-os-candidate-state.sh "$BUILD_ATTEMPT_STATE" absent exact) + [ "$decision" = build ] ensure_build_owner printf 'build=true\nrecord_artifact=false\nbootstrap_sha256=%s\nclaim_sha=%s\nclaim_owner=%s\n' \ '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' "$CLAIM_SHA" "$CLAIM_OWNER" >> "$GITHUB_OUTPUT" diff --git a/bin/agent-os-candidate-state.sh b/bin/agent-os-candidate-state.sh new file mode 100755 index 000000000..7c3bcfbad --- /dev/null +++ b/bin/agent-os-candidate-state.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -eu + +attempt_state=${1:?attempt state is required} +evidence_state=${2:?evidence state is required} +claim_state=${3:?claim state is required} + +case "$claim_state" in + exact) ;; + missing|mismatched|unreadable|ambiguous) + echo "error: candidate claim state is not exact: $claim_state" >&2 + exit 2 + ;; + *) + echo "error: unsupported candidate claim state: $claim_state" >&2 + exit 2 + ;; +esac + +case "$attempt_state" in + absent|attempted) ;; + corrupt|mismatched|unreadable|metadata-read-error|partial|ambiguous) + echo "error: candidate attempt state is not exact: $attempt_state" >&2 + exit 2 + ;; + *) + echo "error: unsupported candidate attempt state: $attempt_state" >&2 + exit 2 + ;; +esac + +case "$evidence_state" in + exact-record) + [ "$attempt_state" = attempted ] || { + echo "error: exact candidate record has no owner-bound build attempt" >&2 + exit 2 + } + printf 'reuse-record\n' + ;; + exact-build) + [ "$attempt_state" = attempted ] || { + echo "error: exact candidate build has no owner-bound build attempt" >&2 + exit 2 + } + printf 'reuse-build\n' + ;; + absent) + [ "$attempt_state" = absent ] || { + echo "error: candidate build attempt has no durable exact evidence; refusing rebuild" >&2 + exit 2 + } + printf 'build\n' + ;; + corrupt|mismatched|unreadable|metadata-read-error|partial|ambiguous) + echo "error: candidate evidence state is not exact: $evidence_state" >&2 + exit 2 + ;; + *) + echo "error: unsupported candidate evidence state: $evidence_state" >&2 + exit 2 + ;; +esac diff --git a/bin/agent-os-runtime-bound.sh b/bin/agent-os-runtime-bound.sh new file mode 100755 index 000000000..12017cb63 --- /dev/null +++ b/bin/agent-os-runtime-bound.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +export AGENT_OS_BOUND_PATH + +agent_os_open_bound_dir() { + local dir=$1 fd fd_root + [ -d "$dir" ] && [ ! -L "$dir" ] || return 1 + exec {fd}<"$dir" || return 1 + if [ -d "/proc/self/fd/$fd" ]; then + fd_root=/proc/self/fd + [ "$dir" -ef "$fd_root/$fd" ] || { + exec {fd}<&- + return 1 + } + AGENT_OS_BOUND_PATH=$fd_root/$fd + return 0 + elif [ "${AGENT_OS_TEST_BOUND_PATHS:-}" = true ]; then + exec {fd}<&- + AGENT_OS_BOUND_PATH=$(cd "$dir" && pwd -P) + return 0 + else + exec {fd}<&- + return 1 + fi +} + +agent_os_bound_dir_matches() { + local dir=$1 bound=$2 + [ -d "$dir" ] && [ ! -L "$dir" ] && [ "$dir" -ef "$bound" ] +} + +agent_os_bound_git() { + local git_dir=$1 work_tree=$2 + shift 2 + trusted_git --git-dir="$git_dir" --work-tree="$work_tree" "$@" +} diff --git a/bin/agent-os-runtime-secondmates.sh b/bin/agent-os-runtime-secondmates.sh index e49f8570f..12513afed 100755 --- a/bin/agent-os-runtime-secondmates.sh +++ b/bin/agent-os-runtime-secondmates.sh @@ -16,6 +16,8 @@ SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) SUB_HOME_MARKER=.fm-secondmate-home # shellcheck source=bin/fm-ff-lib.sh . "$SCRIPT_DIR/fm-ff-lib.sh" +# shellcheck source=bin/agent-os-runtime-bound.sh +. "$SCRIPT_DIR/agent-os-runtime-bound.sh" [[ "$SOURCE_COMMIT" =~ ^[0-9a-f]{40}$ ]] || exit 2 [[ "$SOURCE_TREE" =~ ^[0-9a-f]{40}$ ]] || exit 2 @@ -30,6 +32,68 @@ trusted_git() { -c http.proxy= -c https.proxy= "$@" } +resolve_treehouse_pool() { + local home=$1 parent pool + parent=$(cd "$home/.." 2>/dev/null && pwd -P) || return 1 + pool=$(cd "$parent/.." 2>/dev/null && pwd -P) || return 1 + [ -f "$pool/treehouse-state.json" ] && [ ! -L "$pool/treehouse-state.json" ] || return 1 + [ -f "$pool/treehouse-state.lock" ] && [ ! -L "$pool/treehouse-state.lock" ] || return 1 + RESOLVED_TREEHOUSE_POOL=$pool +} + +acquire_treehouse_lock() { + local pool=$1 flock_bin lock_file fd_root + flock_bin=/usr/bin/flock + if [ ! -x "$flock_bin" ]; then + flock_bin=${AGENT_OS_TEST_FLOCK_BIN:-} + fi + [ -x "$flock_bin" ] || return 1 + lock_file=$pool/treehouse-state.lock + exec {TREEHOUSE_LOCK_FD}<>"$lock_file" || return 1 + if [ -e "/proc/self/fd/$TREEHOUSE_LOCK_FD" ]; then + fd_root=/proc/self/fd + TREEHOUSE_LOCK_HANDLE=$fd_root/$TREEHOUSE_LOCK_FD + elif [ "$flock_bin" = "${AGENT_OS_TEST_FLOCK_BIN:-}" ]; then + TREEHOUSE_LOCK_HANDLE=$lock_file + else + return 1 + fi + [ "$lock_file" -ef "$TREEHOUSE_LOCK_HANDLE" ] || return 1 + "$flock_bin" -x "$TREEHOUSE_LOCK_FD" || return 1 + [ "$lock_file" -ef "$TREEHOUSE_LOCK_HANDLE" ] || return 1 + TREEHOUSE_LOCK_FILE=$lock_file + TREEHOUSE_POOL=$pool +} + +validate_treehouse_lease() { + local id=$1 state_home=$2 home=$3 state state_json entry_count index entry_home canonical_count + state=$TREEHOUSE_POOL/treehouse-state.json + [ "$TREEHOUSE_LOCK_FILE" -ef "$TREEHOUSE_LOCK_HANDLE" ] || return 1 + [ -r "$state" ] && [ -f "$state" ] && [ ! -L "$state" ] || return 1 + state_json=$(cat "$state") || return 1 + jq -e --arg home "$state_home" --arg holder "$id" ' + (.worktrees | type == "array") and + (all(.worktrees[]; (.path | type == "string" and test("^[^\\t\\r\\n]+$")))) and + ([.worktrees[] | select(.path == $home)] | length == 1) and + ([.worktrees[] | select(.leased == true and .lease_holder == $holder)] | length == 1) and + ([.worktrees[] | select( + .path == $home and .leased == true and .lease_holder == $holder and + ((.destroying // false) == false) and ((.owner_pid // 0) == 0) and + ((.owner_started_at // 0) == 0) and (.name | type == "string" and length > 0) and + (.leased_at | type == "string" and length > 0) + )] | length == 1)' <<< "$state_json" >/dev/null || return 1 + entry_count=$(jq -er '.worktrees | length' <<< "$state_json") || return 1 + canonical_count=0 + index=0 + while [ "$index" -lt "$entry_count" ]; do + entry_home=$(jq -er --argjson index "$index" '.worktrees[$index].path' <<< "$state_json") || return 1 + entry_home=$(cd "$entry_home" 2>/dev/null && pwd -P) || return 1 + [ "$entry_home" != "$home" ] || canonical_count=$((canonical_count + 1)) + index=$((index + 1)) + done + [ "$canonical_count" -eq 1 ] +} + resolve_git_metadata() { local root=$1 pointer common if [ -d "$root/.git" ] && [ ! -L "$root/.git" ]; then @@ -150,19 +214,19 @@ validate_policy() { } trusted_tree_entry() { - local home=$1 commit=$2 path=$3 entry - entry=$(trusted_git -C "$home" ls-tree "$commit" -- "$path") || return 1 + local git_dir=$1 home=$2 commit=$3 path=$4 entry + entry=$(agent_os_bound_git "$git_dir" "$home" ls-tree "$commit" -- "$path") || return 1 printf '%s' "$entry" | awk 'NR == 1 { print $1 " " $3 }' } worktree_entry() { - local home=$1 path=$2 mode hash + local git_dir=$1 home=$2 path=$3 mode hash if [ -L "$home/$path" ]; then mode=120000 - hash=$(printf '%s' "$(readlink "$home/$path")" | trusted_git -C "$home" hash-object --stdin) + hash=$(printf '%s' "$(readlink "$home/$path")" | agent_os_bound_git "$git_dir" "$home" hash-object --stdin) elif [ -f "$home/$path" ]; then if [ -x "$home/$path" ]; then mode=100755; else mode=100644; fi - hash=$(trusted_git -C "$home" hash-object -- "$home/$path") + hash=$(agent_os_bound_git "$git_dir" "$home" hash-object -- "$home/$path") elif [ ! -e "$home/$path" ]; then printf '\n' return 0 @@ -173,22 +237,22 @@ worktree_entry() { } entry_matches_transition() { - local home=$1 path=$2 actual=$3 commit expected - shift 3 + local git_dir=$1 home=$2 path=$3 actual=$4 commit expected + shift 4 for commit in "$@"; do - expected=$(trusted_tree_entry "$home" "$commit" "$path") || return 1 + expected=$(trusted_tree_entry "$git_dir" "$home" "$commit" "$path") || return 1 [ "$actual" = "$expected" ] && return 0 done return 1 } verify_pending_checkout() { - local home=$1 pending=$2 current_commit pending_commit record path worktree index index_entries status_file valid - current_commit=$(trusted_git -C "$home" rev-parse HEAD) || return 1 + local git_dir=$1 home=$2 pending=$3 current_commit pending_commit record path worktree index index_entries status_file valid + current_commit=$(agent_os_bound_git "$git_dir" "$home" rev-parse HEAD) || return 1 pending_commit=$(sed -n 's/^commit=//p' "$pending") - trusted_git -C "$home" rev-parse "$pending_commit^{commit}" >/dev/null || return 1 + agent_os_bound_git "$git_dir" "$home" rev-parse "$pending_commit^{commit}" >/dev/null || return 1 status_file=$(mktemp "${TMPDIR:-/tmp}/agent-os-secondmate-status.XXXXXX") || return 1 - if ! trusted_git -c status.renames=false -C "$home" status \ + if ! agent_os_bound_git "$git_dir" "$home" -c status.renames=false status \ --porcelain=v1 -z --untracked-files=all > "$status_file"; then rm -f "$status_file" return 1 @@ -198,15 +262,15 @@ verify_pending_checkout() { while IFS= read -r -d '' record; do path=${record:3} if [ -z "$path" ]; then valid=false; break; fi - worktree=$(worktree_entry "$home" "$path") || { valid=false; break; } - entry_matches_transition "$home" "$path" "$worktree" \ + worktree=$(worktree_entry "$git_dir" "$home" "$path") || { valid=false; break; } + entry_matches_transition "$git_dir" "$home" "$path" "$worktree" \ "$current_commit" "$pending_commit" "$SOURCE_COMMIT" || { valid=false; break; } - index_entries=$(trusted_git -C "$home" ls-files -s -- "$path") || { valid=false; break; } + index_entries=$(agent_os_bound_git "$git_dir" "$home" ls-files -s -- "$path") || { valid=false; break; } index=$(printf '%s' "$index_entries" | awk ' NR == 1 { entry=$1 " " $2 } END { if (NR <= 1) print entry; else exit 1 } ') || { valid=false; break; } - entry_matches_transition "$home" "$path" "$index" \ + entry_matches_transition "$git_dir" "$home" "$path" "$index" \ "$current_commit" "$pending_commit" "$SOURCE_COMMIT" || { valid=false; break; } RECOVERY_PATHS+=("$path") done < "$status_file" @@ -215,18 +279,18 @@ verify_pending_checkout() { } recover_pending_checkout() { - local home=$1 pending=$2 old_head path - verify_pending_checkout "$home" "$pending" || return 1 - old_head=$(trusted_git -C "$home" rev-parse HEAD) || return 1 - trusted_git -C "$home" restore --source="$SOURCE_COMMIT" --staged --worktree -- . + local git_dir=$1 home=$2 pending=$3 old_head path + verify_pending_checkout "$git_dir" "$home" "$pending" || return 1 + old_head=$(agent_os_bound_git "$git_dir" "$home" rev-parse HEAD) || return 1 + agent_os_bound_git "$git_dir" "$home" restore --source="$SOURCE_COMMIT" --staged --worktree -- . for path in "${RECOVERY_PATHS[@]}"; do - if [ -z "$(trusted_tree_entry "$home" "$SOURCE_COMMIT" "$path")" ] && \ + if [ -z "$(trusted_tree_entry "$git_dir" "$home" "$SOURCE_COMMIT" "$path")" ] && \ { [ -f "$home/$path" ] || [ -L "$home/$path" ]; }; then rm -f -- "$home/$path" fi done - trusted_git -C "$home" update-ref --no-deref HEAD "$SOURCE_COMMIT" "$old_head" - [ -z "$(trusted_git -C "$home" status --porcelain --untracked-files=all)" ] + agent_os_bound_git "$git_dir" "$home" update-ref --no-deref HEAD "$SOURCE_COMMIT" "$old_head" + [ -z "$(agent_os_bound_git "$git_dir" "$home" status --porcelain --untracked-files=all)" ] } ids=() @@ -261,12 +325,19 @@ fi validated_ids=() validated_homes=() +treehouse_homes=() git_dirs=() +common_dirs=() +bound_homes=() +bound_configs=() +bound_git_dirs=() +bound_common_dirs=() pending_policies=() seen_homes=$'\n' for index in "${!homes[@]}"; do id=${ids[$index]} home=${homes[$index]} + treehouse_home=$home validate_secondmate_home "$id" "$home" || { echo "error: invalid secondmate home for $id: $VALIDATION_ERROR" >&2 exit 2 @@ -296,6 +367,23 @@ for index in "${!homes[@]}"; do [ "$duplicate" = false ] || continue case "$seen_homes" in *$'\n'"$home"$'\n'*) continue ;; esac seen_homes+="$home"$'\n' + resolve_treehouse_pool "$home" || { + echo "error: secondmate Treehouse state is unavailable: $home" >&2 + exit 2 + } + if [ -z "${TREEHOUSE_POOL:-}" ]; then + acquire_treehouse_lock "$RESOLVED_TREEHOUSE_POOL" || { + echo "error: secondmate Treehouse ownership lock is unavailable: $home" >&2 + exit 2 + } + elif [ "$TREEHOUSE_POOL" != "$RESOLVED_TREEHOUSE_POOL" ]; then + echo "error: secondmate homes do not share one authoritative Treehouse pool" >&2 + exit 2 + fi + validate_treehouse_lease "$id" "$treehouse_home" "$home" || { + echo "error: secondmate Treehouse lease holder is not exact: $id" >&2 + exit 2 + } resolve_git_metadata "$home" || { echo "error: secondmate Git metadata is invalid: $home" >&2; exit 2; } validate_git_binding "$home" || { echo "error: secondmate Git metadata is not bound to its home: $home" >&2 @@ -314,10 +402,32 @@ for index in "${!homes[@]}"; do exit 2 } fi - git_policy="$RESOLVED_GIT_DIR/agent-os-runtime-source" - home_policy="$home/config/agent-os-source-policy" - required_policy="$home/config/agent-os-source-policy.required" - pending_policy="$home/config/agent-os-source-policy.pending" + original_git_dir=$RESOLVED_GIT_DIR + original_common_dir=$RESOLVED_COMMON_DIR + agent_os_open_bound_dir "$home" || { + echo "error: secondmate home cannot be bound for mutation: $home" >&2 + exit 2 + } + bound_home=$AGENT_OS_BOUND_PATH + agent_os_open_bound_dir "$home/config" || { + echo "error: secondmate configuration directory cannot be bound for mutation: $home" >&2 + exit 2 + } + bound_config=$AGENT_OS_BOUND_PATH + agent_os_open_bound_dir "$original_git_dir" || { + echo "error: secondmate Git directory cannot be bound for mutation: $home" >&2 + exit 2 + } + bound_git_dir=$AGENT_OS_BOUND_PATH + agent_os_open_bound_dir "$original_common_dir" || { + echo "error: secondmate common Git directory cannot be bound for mutation: $home" >&2 + exit 2 + } + bound_common_dir=$AGENT_OS_BOUND_PATH + git_policy="$bound_git_dir/agent-os-runtime-source" + home_policy="$bound_config/agent-os-source-policy" + required_policy="$bound_config/agent-os-source-policy.required" + pending_policy="$bound_config/agent-os-source-policy.pending" for policy in "$git_policy" "$home_policy"; do if [ -e "$policy" ]; then validate_policy "$policy" || { @@ -360,54 +470,94 @@ for index in "${!homes[@]}"; do fi fi if [ -z "$selected_pending" ] && \ - [ -n "$(trusted_git -C "$home" status --porcelain --untracked-files=all)" ]; then + [ -n "$(agent_os_bound_git "$bound_git_dir" "$bound_home" status --porcelain --untracked-files=all)" ]; then echo "error: secondmate source is not clean: $home" >&2 exit 2 fi validated_ids+=("$id") validated_homes+=("$home") - git_dirs+=("$RESOLVED_GIT_DIR") + treehouse_homes+=("$treehouse_home") + git_dirs+=("$original_git_dir") + common_dirs+=("$original_common_dir") + bound_homes+=("$bound_home") + bound_configs+=("$bound_config") + bound_git_dirs+=("$bound_git_dir") + bound_common_dirs+=("$bound_common_dir") pending_policies+=("$selected_pending") done +assert_secondmate_binding() { + local id=$1 treehouse_home=$2 home=$3 git_dir=$4 common_dir=$5 bound_home=$6 bound_config=$7 bound_git=$8 bound_common=$9 + validate_treehouse_lease "$id" "$treehouse_home" "$home" || return 1 + agent_os_bound_dir_matches "$home" "$bound_home" || return 1 + agent_os_bound_dir_matches "$home/config" "$bound_config" || return 1 + agent_os_bound_dir_matches "$git_dir" "$bound_git" || return 1 + agent_os_bound_dir_matches "$common_dir" "$bound_common" || return 1 + resolve_git_metadata "$home" || return 1 + [ "$RESOLVED_GIT_DIR" -ef "$bound_git" ] || return 1 + [ "$RESOLVED_COMMON_DIR" -ef "$bound_common" ] || return 1 + validate_git_binding "$home" +} + marker="mode=$SOURCE_MODE commit=$SOURCE_COMMIT source_sha256=$SOURCE_SHA" for index in "${!validated_homes[@]}"; do + id=${validated_ids[$index]} + treehouse_home=${treehouse_homes[$index]} home=${validated_homes[$index]} git_dir=${git_dirs[$index]} + common_dir=${common_dirs[$index]} + bound_home=${bound_homes[$index]} + bound_config=${bound_configs[$index]} + bound_git_dir=${bound_git_dirs[$index]} + bound_common_dir=${bound_common_dirs[$index]} recovery_policy=${pending_policies[$index]} - trusted_git -c protocol.file.allow=always -C "$home" fetch --no-tags "$FM_ROOT" "$SOURCE_COMMIT" - [ "$(trusted_git -C "$home" rev-parse FETCH_HEAD)" = "$SOURCE_COMMIT" ] || exit 2 - [ "$(trusted_git -C "$home" rev-parse 'FETCH_HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 + assert_secondmate_binding "$id" "$treehouse_home" "$home" "$git_dir" "$common_dir" \ + "$bound_home" "$bound_config" "$bound_git_dir" "$bound_common_dir" || { + echo "error: secondmate ownership changed before mutation: $home" >&2 + exit 2 + } + agent_os_bound_git "$bound_git_dir" "$bound_home" -c protocol.file.allow=always \ + fetch --no-tags "$FM_ROOT" "$SOURCE_COMMIT" + [ "$(agent_os_bound_git "$bound_git_dir" "$bound_home" rev-parse FETCH_HEAD)" = "$SOURCE_COMMIT" ] || exit 2 + [ "$(agent_os_bound_git "$bound_git_dir" "$bound_home" rev-parse 'FETCH_HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 if [ -n "$recovery_policy" ] && \ - [ -n "$(trusted_git -C "$home" status --porcelain --untracked-files=all)" ]; then - recover_pending_checkout "$home" "$recovery_policy" || { + [ -n "$(agent_os_bound_git "$bound_git_dir" "$bound_home" status --porcelain --untracked-files=all)" ]; then + recovery_policy="$bound_config/agent-os-source-policy.pending" + recover_pending_checkout "$bound_git_dir" "$bound_home" "$recovery_policy" || { echo "error: secondmate immutable policy journal cannot recover source: $home" >&2 exit 2 } fi - mkdir -p "$home/config" - pending_policy="$home/config/agent-os-source-policy.pending" - pending_tmp="$home/config/.agent-os-source-policy.pending.$$" + assert_secondmate_binding "$id" "$treehouse_home" "$home" "$git_dir" "$common_dir" \ + "$bound_home" "$bound_config" "$bound_git_dir" "$bound_common_dir" || exit 2 + pending_policy="$bound_config/agent-os-source-policy.pending" + pending_tmp="$bound_config/.agent-os-source-policy.pending.$$" printf '%s\n' "$marker" > "$pending_tmp" mv "$pending_tmp" "$pending_policy" - required_policy="$home/config/agent-os-source-policy.required" - required_tmp="$home/config/.agent-os-source-policy.required.$$" + required_policy="$bound_config/agent-os-source-policy.required" + required_tmp="$bound_config/.agent-os-source-policy.required.$$" printf 'immutable\n' > "$required_tmp" mv "$required_tmp" "$required_policy" - trusted_git -C "$home" checkout --detach "$SOURCE_COMMIT" - trusted_git -C "$home" remote set-url origin "$SOURCE_ORIGIN" - home_marker_tmp="$home/config/.agent-os-source-policy.$$" + assert_secondmate_binding "$id" "$treehouse_home" "$home" "$git_dir" "$common_dir" \ + "$bound_home" "$bound_config" "$bound_git_dir" "$bound_common_dir" || exit 2 + agent_os_bound_git "$bound_git_dir" "$bound_home" checkout --detach "$SOURCE_COMMIT" + agent_os_bound_git "$bound_git_dir" "$bound_home" remote set-url origin "$SOURCE_ORIGIN" + assert_secondmate_binding "$id" "$treehouse_home" "$home" "$git_dir" "$common_dir" \ + "$bound_home" "$bound_config" "$bound_git_dir" "$bound_common_dir" || exit 2 + home_marker_tmp="$bound_config/.agent-os-source-policy.$$" printf '%s\n' "$marker" > "$home_marker_tmp" - mv "$home_marker_tmp" "$home/config/agent-os-source-policy" - marker_tmp="$git_dir/agent-os-runtime-source.$$" + mv "$home_marker_tmp" "$bound_config/agent-os-source-policy" + marker_tmp="$bound_git_dir/agent-os-runtime-source.$$" printf '%s\n' "$marker" > "$marker_tmp" - mv "$marker_tmp" "$git_dir/agent-os-runtime-source" - cmp "$home/config/agent-os-source-policy" "$git_dir/agent-os-runtime-source" || exit 2 - [ "$(trusted_git -C "$home" rev-parse HEAD)" = "$SOURCE_COMMIT" ] || exit 2 - [ "$(trusted_git -C "$home" rev-parse 'HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 - [ -z "$(trusted_git -C "$home" status --porcelain --untracked-files=all)" ] || exit 2 + mv "$marker_tmp" "$bound_git_dir/agent-os-runtime-source" + cmp "$bound_config/agent-os-source-policy" "$bound_git_dir/agent-os-runtime-source" || exit 2 + [ "$(agent_os_bound_git "$bound_git_dir" "$bound_home" rev-parse HEAD)" = "$SOURCE_COMMIT" ] || exit 2 + [ "$(agent_os_bound_git "$bound_git_dir" "$bound_home" rev-parse 'HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 + [ -z "$(agent_os_bound_git "$bound_git_dir" "$bound_home" status --porcelain --untracked-files=all)" ] || exit 2 + assert_secondmate_binding "$id" "$treehouse_home" "$home" "$git_dir" "$common_dir" \ + "$bound_home" "$bound_config" "$bound_git_dir" "$bound_common_dir" || exit 2 rm "$pending_policy" printf 'selected: %s\n' "$home" done diff --git a/tests/agent-os-candidate-state.test.sh b/tests/agent-os-candidate-state.test.sh new file mode 100755 index 000000000..654babe7c --- /dev/null +++ b/tests/agent-os-candidate-state.test.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -u + +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +STATE="$ROOT/bin/agent-os-candidate-state.sh" + +expect_decision() { + local expected=$1 attempt=$2 evidence=$3 claim=$4 actual + actual=$("$STATE" "$attempt" "$evidence" "$claim") || \ + fail "state $attempt/$evidence/$claim did not produce $expected" + [ "$actual" = "$expected" ] || \ + fail "state $attempt/$evidence/$claim produced $actual instead of $expected" +} + +expect_refusal() { + local attempt=$1 evidence=$2 claim=$3 + if "$STATE" "$attempt" "$evidence" "$claim" >/dev/null 2>&1; then + fail "state $attempt/$evidence/$claim authorized another build" + fi +} + +expect_decision build absent absent exact +expect_decision reuse-build attempted exact-build exact +expect_decision reuse-record attempted exact-record exact + +builds=0 +decision=$("$STATE" absent absent exact) || fail "initial candidate did not authorize its one build" +[ "$decision" = build ] && builds=$((builds + 1)) +decision=$("$STATE" attempted exact-build exact) || fail "durable candidate build was not reusable" +[ "$decision" = build ] && builds=$((builds + 1)) +[ "$builds" -eq 1 ] || fail "candidate state machine authorized $builds builds" + +for attempt in attempted corrupt mismatched unreadable metadata-read-error partial ambiguous; do + expect_refusal "$attempt" absent exact +done +for evidence in corrupt mismatched unreadable metadata-read-error partial ambiguous; do + expect_refusal attempted "$evidence" exact +done +for claim in missing mismatched unreadable ambiguous; do + expect_refusal absent absent "$claim" +done +expect_refusal attempted partial exact +expect_refusal attempted absent exact +expect_refusal absent exact-build exact +expect_refusal absent exact-record exact + +pass "candidate state machine permits one build and exact reuse only" diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 4fdbfe1c5..d1dfa8365 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -277,8 +277,13 @@ assert_grep 'agent-os-candidate-build-$GITHUB_SHA' "$IMAGE_WORKFLOW" \ "candidate builds must persist their exact digest immediately" assert_grep 'refs/agent-os/candidate-build-attempts/$GITHUB_SHA/$owner_run_id' "$IMAGE_WORKFLOW" \ "candidate retries must resolve the exact owner-bound build attempt" -assert_grep 'candidate build attempt has no durable exact evidence; refusing rebuild' "$IMAGE_WORKFLOW" \ +assert_grep 'candidate build attempt has no durable exact evidence; refusing rebuild' \ + "$ROOT/bin/agent-os-candidate-state.sh" \ "candidate cancellation must fail closed instead of rebuilding" +assert_grep 'bin/agent-os-candidate-state.sh "$BUILD_ATTEMPT_STATE" absent exact' "$IMAGE_WORKFLOW" \ + "candidate rebuild authorization must use the executable state machine" +assert_no_grep 'if load_build_attempt' "$IMAGE_WORKFLOW" \ + "candidate attempt validation must never run in a conditional errexit context" attempt_line=$(grep -n 'name: Claim owner-bound candidate build attempt' "$IMAGE_WORKFLOW" | cut -d: -f1) build_line=$(grep -n 'name: Build release candidate once' "$IMAGE_WORKFLOW" | cut -d: -f1) build_evidence_line=$(grep -n 'name: Stage exact candidate build evidence' "$IMAGE_WORKFLOW" | cut -d: -f1) @@ -289,7 +294,7 @@ record_line=$(grep -n 'name: Prepare immutable release candidate record' "$IMAGE [ "$build_line" -lt "$build_evidence_line" ] && \ [ "$build_evidence_line" -lt "$image_publish_line" ] && [ "$image_publish_line" -lt "$record_line" ] || \ fail "owner-bound attempt state must precede build and exact evidence publication" -attempt_guard_line=$(grep -n 'if load_build_attempt "$artifact_owner"' "$IMAGE_WORKFLOW" | cut -d: -f1) +attempt_guard_line=$(grep -n 'load_build_attempt "$artifact_owner"' "$IMAGE_WORKFLOW" | cut -d: -f1) rebuild_line=$(grep -n "printf 'build=true" "$IMAGE_WORKFLOW" | cut -d: -f1) [ -n "$attempt_guard_line" ] && [ -n "$rebuild_line" ] && [ "$attempt_guard_line" -lt "$rebuild_line" ] || \ fail "candidate upload failure must be rejected before any rebuild is authorized" diff --git a/tests/agent-os-runtime-secondmates.test.sh b/tests/agent-os-runtime-secondmates.test.sh index f7aa43b2a..9d74be8c3 100755 --- a/tests/agent-os-runtime-secondmates.test.sh +++ b/tests/agent-os-runtime-secondmates.test.sh @@ -4,14 +4,21 @@ set -u . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" SYNC="$ROOT/bin/agent-os-runtime-secondmates.sh" -TMP=$(fm_test_tmproot agent-os-runtime-secondmates) +TEST_ROOT=$(fm_test_tmproot agent-os-runtime-secondmates) +TREEHOUSE_POOL="$TEST_ROOT/pool" +TMP="$TREEHOUSE_POOL/0" FM_HOME_DIR="$TMP/home" WORK="$TMP/work" LEASE= +TEST_FLOCK="$TEST_ROOT/flock" +PRESERVE_TREEHOUSE_STATE=false fm_git_identity fmtest fmtest@example.com -mkdir -p "$FM_HOME_DIR/state" "$FM_HOME_DIR/data" +mkdir -p "$FM_HOME_DIR/state" "$FM_HOME_DIR/data" "$TREEHOUSE_POOL" +printf '#!/usr/bin/env bash\nexit 0\n' > "$TEST_FLOCK" +chmod +x "$TEST_FLOCK" +touch "$TREEHOUSE_POOL/treehouse-state.lock" git init -q -b main "$WORK" printf 'data/\nstate/\nconfig/\nprojects/\n.fm-secondmate-home\n' > "$WORK/.gitignore" printf 'A\n' > "$WORK/version" @@ -59,13 +66,33 @@ make_primary() { git -C "$TMP/$name" reset -q --hard "$commit" } +write_treehouse_state() { + local meta id home + : > "$TEST_ROOT/treehouse-worktrees.jsonl" + for meta in "$FM_HOME_DIR"/state/*.meta; do + [ -f "$meta" ] || continue + [ "$(sed -n 's/^kind=//p' "$meta")" = secondmate ] || continue + id=${meta##*/} + id=${id%.meta} + home=$(sed -n 's/^home=//p' "$meta") + jq -cn --arg id "$id" --arg home "$home" \ + '{name:$id,path:$home,created_at:"2026-01-01T00:00:00Z",destroying:false, + owner_pid:0,owner_started_at:0,leased:true,lease_holder:$id, + leased_at:"2026-01-01T00:00:00Z"}' >> "$TEST_ROOT/treehouse-worktrees.jsonl" + done + jq -s '{worktrees:.}' "$TEST_ROOT/treehouse-worktrees.jsonl" \ + > "$TREEHOUSE_POOL/treehouse-state.json" +} + run_sync() { local root=$1 commit=$2 tree=$3 sha=$4 + [ "$PRESERVE_TREEHOUSE_STATE" = true ] || write_treehouse_state FM_HOME="$FM_HOME_DIR" FM_ROOT_OVERRIDE="$root" \ AGENT_OS_SOURCE_COMMIT="$commit" AGENT_OS_SOURCE_TREE="$tree" \ AGENT_OS_SOURCE_SHA256="$sha" AGENT_OS_SOURCE_BRANCH=main \ AGENT_OS_SOURCE_ORIGIN=https://github.com/akua-dev/agent-os.git \ - AGENT_OS_SOURCE_MODE=release "$SYNC" + AGENT_OS_SOURCE_MODE=release AGENT_OS_TEST_FLOCK_BIN="$TEST_FLOCK" \ + AGENT_OS_TEST_BOUND_PATHS=true "$SYNC" } make_primary primary-b "$B_COMMIT" @@ -228,6 +255,101 @@ fi rm "$FM_HOME_DIR/state/nested.meta" pass "secondmate selection rejects canonical home overlap" +write_treehouse_state +PRESERVE_TREEHOUSE_STATE=true +jq 'del(.worktrees[] | select(.lease_holder == "linked"))' \ + "$TREEHOUSE_POOL/treehouse-state.json" > "$TEST_ROOT/state-next.json" +mv "$TEST_ROOT/state-next.json" "$TREEHOUSE_POOL/treehouse-state.json" +if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then + fail "missing Treehouse lease evidence was accepted" +fi +[ "$(git -C "$TMP/linked" rev-parse HEAD)" = "$A_COMMIT" ] || \ + fail "missing Treehouse lease evidence allowed source mutation" + +write_treehouse_state +jq '(.worktrees[] | select(.lease_holder == "linked") | .leased) = false' \ + "$TREEHOUSE_POOL/treehouse-state.json" > "$TEST_ROOT/state-next.json" +mv "$TEST_ROOT/state-next.json" "$TREEHOUSE_POOL/treehouse-state.json" +if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then + fail "stale Treehouse lease evidence was accepted" +fi + +write_treehouse_state +jq '(.worktrees[] | select(.lease_holder == "linked") | .lease_holder) = "other"' \ + "$TREEHOUSE_POOL/treehouse-state.json" > "$TEST_ROOT/state-next.json" +mv "$TEST_ROOT/state-next.json" "$TREEHOUSE_POOL/treehouse-state.json" +if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then + fail "other-holder Treehouse lease evidence was accepted" +fi + +write_treehouse_state +jq '.worktrees += [.worktrees[] | select(.lease_holder == "linked")]' \ + "$TREEHOUSE_POOL/treehouse-state.json" > "$TEST_ROOT/state-next.json" +mv "$TEST_ROOT/state-next.json" "$TREEHOUSE_POOL/treehouse-state.json" +if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then + fail "duplicate Treehouse lease evidence was accepted" +fi + +write_treehouse_state +jq '.worktrees += [(.worktrees[] | select(.lease_holder == "linked") | .path += "-planted")]' \ + "$TREEHOUSE_POOL/treehouse-state.json" > "$TEST_ROOT/state-next.json" +mv "$TEST_ROOT/state-next.json" "$TREEHOUSE_POOL/treehouse-state.json" +if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then + fail "ambiguous planted Treehouse holder evidence was accepted" +fi + +printf '{unreadable\n' > "$TREEHOUSE_POOL/treehouse-state.json" +if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then + fail "unreadable Treehouse lease evidence was accepted" +fi +PRESERVE_TREEHOUSE_STATE=false +write_treehouse_state +pass "secondmate selection requires one exact active Treehouse lease holder" + +git -C "$LEASE" worktree add -q --detach "$TMP/toctou-victim" "$A_COMMIT" +git clone -q "$LEASE" "$TMP/toctou-foreign" +git -C "$TMP/toctou-foreign" checkout -q --detach "$A_COMMIT" +mkdir -p "$TMP/toctou-victim/config" "$TMP/toctou-foreign/config" +foreign_head=$(git -C "$TMP/toctou-foreign" rev-parse HEAD) +AGENT_OS_TEST_BOUND_PATHS=true +export AGENT_OS_TEST_BOUND_PATHS +. "$ROOT/bin/agent-os-runtime-bound.sh" +trusted_git() { + env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 \ + /usr/bin/git -c credential.helper= -c core.hooksPath=/dev/null "$@" +} +agent_os_open_bound_dir "$TMP/toctou-victim" || fail "TOCTOU victim home could not be bound" +toctou_home=$AGENT_OS_BOUND_PATH +agent_os_open_bound_dir "$TMP/toctou-victim/config" || fail "TOCTOU victim configuration could not be bound" +toctou_config=$AGENT_OS_BOUND_PATH +toctou_git_path=$(git -C "$TMP/toctou-victim" rev-parse --absolute-git-dir) +agent_os_open_bound_dir "$toctou_git_path" || fail "TOCTOU victim Git directory could not be bound" +toctou_git=$AGENT_OS_BOUND_PATH +rm "$TMP/toctou-victim/.git" +printf 'gitdir: %s/toctou-foreign/.git\n' "$TMP" > "$TMP/toctou-victim/.git" +mv "$TMP/toctou-victim/config" "$TMP/toctou-victim-config-owned" +ln -s "$TMP/toctou-foreign/config" "$TMP/toctou-victim/config" +if agent_os_bound_dir_matches "$TMP/toctou-victim/config" "$toctou_config"; then + fail "swapped secondmate configuration retained its validated identity" +fi +if [ "$(git -C "$TMP/toctou-victim" rev-parse --absolute-git-dir)" -ef "$toctou_git" ]; then + fail "swapped secondmate Git pointer retained its validated identity" +fi +agent_os_bound_git "$toctou_git" "$toctou_home" checkout -q --detach "$B_COMMIT" || \ + fail "captured secondmate Git directory could not select its exact source" +agent_os_bound_git "$toctou_git" "$toctou_home" remote set-url origin \ + https://github.com/akua-dev/agent-os.git +printf 'mode=release\ncommit=%s\nsource_sha256=%s\n' "$B_COMMIT" "$B_SHA" \ + > "$toctou_git/agent-os-runtime-source" +[ "$(git -C "$TMP/toctou-foreign" rev-parse HEAD)" = "$foreign_head" ] || \ + fail "pointer swap redirected a captured Git mutation into a foreign repository" +[ ! -e "$TMP/toctou-foreign/.git/agent-os-runtime-source" ] || \ + fail "pointer swap redirected marker mutation into a foreign repository" +[ ! -e "$TMP/toctou-foreign/config/agent-os-source-policy" ] || \ + fail "configuration swap redirected policy mutation into a foreign home" +pass "captured secondmate directories defeat Git pointer swaps" + printf 'tampered\n' >> "$TMP/standalone/AGENTS.md" if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then fail "dirty secondmate source was replaced" From 65949c148522d40209d8535c858da87527e73a3b Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 21:46:37 +0200 Subject: [PATCH 48/56] no-mistakes(review): Harden candidate ancestry and secondmate ownership --- .github/workflows/agent-os-image.yml | 75 ++++- bin/agent-os-candidate-state.sh | 114 ++++--- bin/agent-os-runtime-bound.sh | 11 +- bin/agent-os-runtime-secondmates.sh | 353 ++++++++++++++++----- tests/agent-os-candidate-state.test.sh | 100 ++++-- tests/agent-os-container.test.sh | 8 +- tests/agent-os-runtime-secondmates.test.sh | 188 +++++++---- 7 files changed, 605 insertions(+), 244 deletions(-) diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index 7f77617ec..ae1c11058 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -307,6 +307,8 @@ jobs: claim_ref="$claim_prefix/initial" load_claim "$claim_ref" CLAIM_OWNERS="|$CLAIM_OWNER|" + CLAIM_CHAIN_FILE="$RUNNER_TEMP/candidate-claim-chain.tsv" + printf '%s\t%s\n' "$CLAIM_OWNER" "$CLAIM_SHA" > "$CLAIM_CHAIN_FILE" depth=0 while [ "$depth" -lt 32 ]; do next_ref="$claim_prefix/recover-$CLAIM_OWNER" @@ -315,7 +317,14 @@ jobs: 2> "$RUNNER_TEMP/candidate-next-claim.err"; then claim_ref=$next_ref load_claim "$claim_ref" + case "$CLAIM_OWNERS" in + *"|$CLAIM_OWNER|"*) + echo "::error::candidate reservation recovery chain repeats an owner" + exit 1 + ;; + esac CLAIM_OWNERS="${CLAIM_OWNERS}${CLAIM_OWNER}|" + printf '%s\t%s\n' "$CLAIM_OWNER" "$CLAIM_SHA" >> "$CLAIM_CHAIN_FILE" depth=$((depth + 1)) continue fi @@ -409,12 +418,56 @@ jobs: --format json --depth 1 "$IMAGE@$reservation_digest" > "$RUNNER_TEMP/candidate-reservation-referrers.json" referrer_count=$(jq -er '.referrers | length' "$RUNNER_TEMP/candidate-reservation-referrers.json") if [ "$referrer_count" -eq 0 ]; then - artifact_owner=$CLAIM_OWNER - artifact_claim_sha=$CLAIM_SHA - load_build_attempt "$artifact_owner" "$artifact_claim_sha" "$reservation_digest" artifact_name="agent-os-candidate-record-$GITHUB_SHA" gh api --method GET "repos/$GITHUB_REPOSITORY/actions/artifacts?name=$artifact_name&per_page=100" \ > "$RUNNER_TEMP/candidate-trusted-artifacts.json" + artifact_name="agent-os-candidate-build-$GITHUB_SHA" + gh api --method GET "repos/$GITHUB_REPOSITORY/actions/artifacts?name=$artifact_name&per_page=100" \ + > "$RUNNER_TEMP/candidate-build-artifacts.json" + chain_state="$RUNNER_TEMP/candidate-claim-chain-state.tsv" + : > "$chain_state" + while IFS=$'\t' read -r owner claim_sha; do + load_build_attempt "$owner" "$claim_sha" "$reservation_digest" + record_count=$(jq -er --arg sha "$GITHUB_SHA" --argjson owner "$owner" \ + '[.artifacts[] | select(.expired == false and .workflow_run.head_sha == $sha and .workflow_run.id == $owner)] | length' \ + "$RUNNER_TEMP/candidate-trusted-artifacts.json") + build_count=$(jq -er --arg sha "$GITHUB_SHA" --argjson owner "$owner" \ + '[.artifacts[] | select(.expired == false and .workflow_run.head_sha == $sha and .workflow_run.id == $owner)] | length' \ + "$RUNNER_TEMP/candidate-build-artifacts.json") + [ "$record_count" -le 1 ] && [ "$build_count" -le 1 ] || { + echo "::error::candidate claim owner has conflicting durable evidence" + exit 1 + } + if [ "$record_count" -eq 1 ] && [ "$build_count" -eq 0 ]; then + evidence_state=exact-record + elif [ "$record_count" -eq 0 ] && [ "$build_count" -eq 1 ]; then + evidence_state=exact-build + elif [ "$record_count" -eq 0 ] && [ "$build_count" -eq 0 ]; then + evidence_state=absent + else + evidence_state=ambiguous + fi + printf '%s\t%s\t%s\t%s\n' "$owner" "$claim_sha" \ + "$BUILD_ATTEMPT_STATE" "$evidence_state" >> "$chain_state" + done < "$CLAIM_CHAIN_FILE" + decision=$(bin/agent-os-candidate-state.sh "$chain_state") + IFS=$'\t' read -r candidate_action artifact_owner <> "$GITHUB_OUTPUT" echo "::notice::candidate reservation recovered from durable trusted artifact" exit 0 fi - artifact_name="agent-os-candidate-build-$GITHUB_SHA" - gh api --method GET "repos/$GITHUB_REPOSITORY/actions/artifacts?name=$artifact_name&per_page=100" \ - > "$RUNNER_TEMP/candidate-build-artifacts.json" build_artifact_count=$(jq -er --arg sha "$GITHUB_SHA" --argjson owner "$artifact_owner" \ '[.artifacts[] | select(.expired == false and .workflow_run.head_sha == $sha and .workflow_run.id == $owner)] | length' \ "$RUNNER_TEMP/candidate-build-artifacts.json") @@ -478,7 +527,8 @@ jobs: echo "::error::candidate reservation has conflicting exact build evidence" exit 1 } - if [ "$build_artifact_count" -eq 1 ]; then + if [ "$candidate_action" = reuse-build ]; then + [ "$build_artifact_count" -eq 1 ] || exit 1 validate_artifact_run "$artifact_owner" build_artifact_id=$(jq -er --arg sha "$GITHUB_SHA" --argjson owner "$artifact_owner" \ '[.artifacts[] | select(.expired == false and .workflow_run.head_sha == $sha and .workflow_run.id == $owner)] | .[0].id' \ @@ -501,16 +551,13 @@ jobs: archive_sha=$(jq -er .oci_archive_sha256 "$RUNNER_TEMP/candidate-build-artifact/candidate-build.json") [ "$(sha256sum "$RUNNER_TEMP/candidate-build-artifact/candidate-image.tar" | awk '{print $1}')" = "$archive_sha" ] cp "$RUNNER_TEMP/candidate-build-artifact/candidate-image.tar" "$RUNNER_TEMP/candidate-image.tar" - decision=$(bin/agent-os-candidate-state.sh "$BUILD_ATTEMPT_STATE" exact-build exact) - [ "$decision" = reuse-build ] ensure_build_owner printf 'build=false\npublish_image=true\nrecord_artifact=false\nimage_digest=%s\nbootstrap_sha256=%s\n' \ "$image_digest" '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' >> "$GITHUB_OUTPUT" echo "::notice::candidate reservation recovered from exact build evidence" exit 0 fi - decision=$(bin/agent-os-candidate-state.sh "$BUILD_ATTEMPT_STATE" absent exact) - [ "$decision" = build ] + [ "$candidate_action" = build ] ensure_build_owner printf 'build=true\nrecord_artifact=false\nbootstrap_sha256=%s\nclaim_sha=%s\nclaim_owner=%s\n' \ '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' "$CLAIM_SHA" "$CLAIM_OWNER" >> "$GITHUB_OUTPUT" diff --git a/bin/agent-os-candidate-state.sh b/bin/agent-os-candidate-state.sh index 7c3bcfbad..a07fd515b 100755 --- a/bin/agent-os-candidate-state.sh +++ b/bin/agent-os-candidate-state.sh @@ -1,62 +1,76 @@ #!/usr/bin/env bash set -eu -attempt_state=${1:?attempt state is required} -evidence_state=${2:?evidence state is required} -claim_state=${3:?claim state is required} +state_file=${1:?candidate claim-chain state file is required} +[ -f "$state_file" ] && [ ! -L "$state_file" ] || { + echo "error: candidate claim-chain state is unreadable" >&2 + exit 2 +} -case "$claim_state" in - exact) ;; - missing|mismatched|unreadable|ambiguous) - echo "error: candidate claim state is not exact: $claim_state" >&2 +attempted_count=0 +build_owner= +build_evidence= +seen_owners='|' +seen_claims='|' +entries=0 +while IFS=$'\t' read -r owner claim attempt evidence extra || [ -n "${owner:-}" ]; do + [ -z "${extra:-}" ] || { + echo "error: candidate claim-chain state has extra fields" >&2 exit 2 - ;; - *) - echo "error: unsupported candidate claim state: $claim_state" >&2 - exit 2 - ;; -esac - -case "$attempt_state" in - absent|attempted) ;; - corrupt|mismatched|unreadable|metadata-read-error|partial|ambiguous) - echo "error: candidate attempt state is not exact: $attempt_state" >&2 + } + [[ "$owner" =~ ^[0-9]+$ ]] && [[ "$claim" =~ ^[0-9a-f]{40}$ ]] || { + echo "error: candidate claim-chain identity is invalid" >&2 exit 2 - ;; - *) - echo "error: unsupported candidate attempt state: $attempt_state" >&2 - exit 2 - ;; -esac - -case "$evidence_state" in - exact-record) - [ "$attempt_state" = attempted ] || { - echo "error: exact candidate record has no owner-bound build attempt" >&2 + } + case "$seen_owners" in *"|$owner|"*) echo "error: candidate claim owner is duplicated" >&2; exit 2 ;; esac + case "$seen_claims" in *"|$claim|"*) echo "error: candidate claim commit is duplicated" >&2; exit 2 ;; esac + seen_owners="${seen_owners}${owner}|" + seen_claims="${seen_claims}${claim}|" + entries=$((entries + 1)) + case "$attempt" in + absent) ;; + attempted) + attempted_count=$((attempted_count + 1)) + build_owner=$owner + build_evidence=$evidence + ;; + *) + echo "error: candidate attempt state is not exact: $attempt" >&2 exit 2 - } - printf 'reuse-record\n' - ;; - exact-build) - [ "$attempt_state" = attempted ] || { - echo "error: exact candidate build has no owner-bound build attempt" >&2 + ;; + esac + case "$evidence" in + absent) ;; + exact-record|exact-build) + [ "$attempt" = attempted ] || { + echo "error: candidate evidence is not bound to its build attempt" >&2 + exit 2 + } + ;; + *) + echo "error: candidate evidence state is not exact: $evidence" >&2 exit 2 - } - printf 'reuse-build\n' - ;; + ;; + esac +done < "$state_file" + +[ "$entries" -gt 0 ] || { + echo "error: candidate claim chain is empty" >&2 + exit 2 +} +[ "$attempted_count" -le 1 ] || { + echo "error: candidate claim chain contains multiple build attempts" >&2 + exit 2 +} +if [ "$attempted_count" -eq 0 ]; then + printf 'build\n' + exit 0 +fi +case "$build_evidence" in + exact-record) printf 'reuse-record\t%s\n' "$build_owner" ;; + exact-build) printf 'reuse-build\t%s\n' "$build_owner" ;; absent) - [ "$attempt_state" = absent ] || { - echo "error: candidate build attempt has no durable exact evidence; refusing rebuild" >&2 - exit 2 - } - printf 'build\n' - ;; - corrupt|mismatched|unreadable|metadata-read-error|partial|ambiguous) - echo "error: candidate evidence state is not exact: $evidence_state" >&2 - exit 2 - ;; - *) - echo "error: unsupported candidate evidence state: $evidence_state" >&2 + echo "error: candidate build attempt has no durable exact evidence; refusing rebuild" >&2 exit 2 ;; esac diff --git a/bin/agent-os-runtime-bound.sh b/bin/agent-os-runtime-bound.sh index 12017cb63..8b7eaf030 100755 --- a/bin/agent-os-runtime-bound.sh +++ b/bin/agent-os-runtime-bound.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -export AGENT_OS_BOUND_PATH +export AGENT_OS_BOUND_PATH AGENT_OS_BOUND_TEST_FALLBACK_ACTIVE +AGENT_OS_BOUND_TEST_FALLBACK_ACTIVE=false agent_os_open_bound_dir() { local dir=$1 fd fd_root @@ -17,6 +18,7 @@ agent_os_open_bound_dir() { elif [ "${AGENT_OS_TEST_BOUND_PATHS:-}" = true ]; then exec {fd}<&- AGENT_OS_BOUND_PATH=$(cd "$dir" && pwd -P) + AGENT_OS_BOUND_TEST_FALLBACK_ACTIVE=true return 0 else exec {fd}<&- @@ -30,7 +32,8 @@ agent_os_bound_dir_matches() { } agent_os_bound_git() { - local git_dir=$1 work_tree=$2 - shift 2 - trusted_git --git-dir="$git_dir" --work-tree="$work_tree" "$@" + local git_dir=$1 common_dir=$2 work_tree=$3 + shift 3 + trusted_git --agent-os-common-dir "$common_dir" \ + --git-dir="$git_dir" --work-tree="$work_tree" "$@" } diff --git a/bin/agent-os-runtime-secondmates.sh b/bin/agent-os-runtime-secondmates.sh index 12513afed..196665c90 100755 --- a/bin/agent-os-runtime-secondmates.sh +++ b/bin/agent-os-runtime-secondmates.sh @@ -26,10 +26,24 @@ case "$SOURCE_MODE" in candidate|release) ;; *) exit 2 ;; esac [ -x "$GIT_BIN" ] || exit 2 trusted_git() { - env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ - GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 \ - "$GIT_BIN" -c credential.helper= -c core.hooksPath=/dev/null \ - -c http.proxy= -c https.proxy= "$@" + local common_dir= + if [ "${1:-}" = --agent-os-common-dir ]; then + common_dir=${2:?bound common Git directory is required} + shift 2 + [ -d "$common_dir" ] && [ ! -L "$common_dir" ] || return 1 + fi + if [ -n "$common_dir" ]; then + env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 \ + GIT_COMMON_DIR="$common_dir" \ + "$GIT_BIN" -c credential.helper= -c core.hooksPath=/dev/null \ + -c http.proxy= -c https.proxy= "$@" + else + env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 \ + "$GIT_BIN" -c credential.helper= -c core.hooksPath=/dev/null \ + -c http.proxy= -c https.proxy= "$@" + fi } resolve_treehouse_pool() { @@ -59,12 +73,41 @@ acquire_treehouse_lock() { return 1 fi [ "$lock_file" -ef "$TREEHOUSE_LOCK_HANDLE" ] || return 1 - "$flock_bin" -x "$TREEHOUSE_LOCK_FD" || return 1 + AGENT_OS_TEST_LOCK_FILE_HINT="$lock_file" "$flock_bin" -x "$TREEHOUSE_LOCK_FD" || return 1 [ "$lock_file" -ef "$TREEHOUSE_LOCK_HANDLE" ] || return 1 TREEHOUSE_LOCK_FILE=$lock_file TREEHOUSE_POOL=$pool } +acquire_primary_ownership_lock() { + local flock_bin lock_file fd_root + flock_bin=/usr/bin/flock + if [ ! -x "$flock_bin" ]; then + flock_bin=${AGENT_OS_TEST_FLOCK_BIN:-} + fi + [ -x "$flock_bin" ] || return 1 + lock_file=$FM_HOME/state/agent-os-runtime-secondmates.lock + [ -d "$FM_HOME/state" ] && [ ! -L "$FM_HOME/state" ] || return 1 + if [ -e "$lock_file" ]; then + [ -f "$lock_file" ] && [ ! -L "$lock_file" ] || return 1 + else + : > "$lock_file" || return 1 + fi + exec {PRIMARY_OWNERSHIP_LOCK_FD}<>"$lock_file" || return 1 + if [ -e "/proc/self/fd/$PRIMARY_OWNERSHIP_LOCK_FD" ]; then + fd_root=/proc/self/fd + PRIMARY_OWNERSHIP_LOCK_HANDLE=$fd_root/$PRIMARY_OWNERSHIP_LOCK_FD + elif [ "$flock_bin" = "${AGENT_OS_TEST_FLOCK_BIN:-}" ]; then + PRIMARY_OWNERSHIP_LOCK_HANDLE=$lock_file + else + return 1 + fi + [ "$lock_file" -ef "$PRIMARY_OWNERSHIP_LOCK_HANDLE" ] || return 1 + AGENT_OS_TEST_LOCK_FILE_HINT="$lock_file" "$flock_bin" -x "$PRIMARY_OWNERSHIP_LOCK_FD" || return 1 + [ "$lock_file" -ef "$PRIMARY_OWNERSHIP_LOCK_HANDLE" ] || return 1 + PRIMARY_OWNERSHIP_LOCK_FILE=$lock_file +} + validate_treehouse_lease() { local id=$1 state_home=$2 home=$3 state state_json entry_count index entry_home canonical_count state=$TREEHOUSE_POOL/treehouse-state.json @@ -162,6 +205,68 @@ validate_git_binding() { [ "$source_key" = "$policy_commit-$policy_sha" ] } +validate_bound_git_binding() { + local root=$1 bound_home=$2 bound_git=$3 bound_common=$4 bound_source=$5 linked=$6 source_key=$7 + local pointer pointer_path backlink backlink_path common common_path matches source_key policy_commit policy_sha + [ "$PRIMARY_OWNERSHIP_LOCK_FILE" -ef "$PRIMARY_OWNERSHIP_LOCK_HANDLE" ] || return 1 + if [ "$linked" = false ]; then + [ -d "$bound_home/.git" ] && [ ! -L "$bound_home/.git" ] || return 1 + [ "$bound_home/.git" -ef "$bound_git" ] || return 1 + [ "$bound_git" -ef "$bound_common" ] || return 1 + [ "$bound_home" -ef "$bound_source" ] || return 1 + return 0 + fi + [ -f "$bound_home/.git" ] && [ ! -L "$bound_home/.git" ] || return 1 + [ "$(wc -l < "$bound_home/.git" | tr -d ' ')" -eq 1 ] || return 1 + pointer=$(sed -n 's/^gitdir: //p' "$bound_home/.git") + [ -n "$pointer" ] || return 1 + case "$pointer" in + /*) pointer_path=$pointer ;; + *) pointer_path="$bound_home/$pointer" ;; + esac + [ "$pointer_path" -ef "$bound_git" ] || return 1 + [ -f "$bound_git/gitdir" ] && [ ! -L "$bound_git/gitdir" ] || return 1 + [ "$(wc -l < "$bound_git/gitdir" | tr -d ' ')" -eq 1 ] || return 1 + backlink=$(cat "$bound_git/gitdir") + case "$backlink" in + /*) backlink_path=$backlink ;; + *) backlink_path="$bound_git/$backlink" ;; + esac + [ "$backlink_path" -ef "$bound_home/.git" ] || return 1 + [ -f "$bound_git/commondir" ] && [ ! -L "$bound_git/commondir" ] || return 1 + [ "$(wc -l < "$bound_git/commondir" | tr -d ' ')" -eq 1 ] || return 1 + common=$(cat "$bound_git/commondir") + case "$common" in + /*) common_path=$common ;; + *) common_path="$bound_git/$common" ;; + esac + [ "$common_path" -ef "$bound_common" ] || return 1 + matches=0 + for pointer_path in "$bound_common"/worktrees/*; do + [ -d "$pointer_path" ] || continue + [ "$pointer_path" -ef "$bound_git" ] && matches=$((matches + 1)) + done + [ "$matches" -eq 1 ] || return 1 + [ "$bound_source/.git" -ef "$bound_common" ] || return 1 + validate_policy "$bound_common/agent-os-runtime-source" || return 1 + policy_commit=$(sed -n 's/^commit=//p' "$bound_common/agent-os-runtime-source") + policy_sha=$(sed -n 's/^source_sha256=//p' "$bound_common/agent-os-runtime-source") + [ "$source_key" = "$policy_commit-$policy_sha" ] +} + +validate_primary_standalone_proof() { + local id=$1 home=$2 meta recorded_home + meta=$FM_HOME/state/$id.meta + [ "$PRIMARY_OWNERSHIP_LOCK_FILE" -ef "$PRIMARY_OWNERSHIP_LOCK_HANDLE" ] || return 1 + [ -f "$meta" ] && [ ! -L "$meta" ] && [ -r "$meta" ] || return 1 + [ "$(grep -c '^kind=secondmate$' "$meta")" -eq 1 ] || return 1 + [ "$(grep -c '^home=' "$meta")" -eq 1 ] || return 1 + recorded_home=$(sed -n 's/^home=//p' "$meta") + [ -d "$recorded_home" ] && [ ! -L "$recorded_home" ] || return 1 + recorded_home=$(cd "$recorded_home" && pwd -P) || return 1 + [ "$recorded_home" = "$home" ] +} + validate_config() { local config=$1 section='' line key value token seen='' [ -f "$config" ] && [ ! -L "$config" ] || return 1 @@ -214,19 +319,19 @@ validate_policy() { } trusted_tree_entry() { - local git_dir=$1 home=$2 commit=$3 path=$4 entry - entry=$(agent_os_bound_git "$git_dir" "$home" ls-tree "$commit" -- "$path") || return 1 + local git_dir=$1 common_dir=$2 home=$3 commit=$4 path=$5 entry + entry=$(agent_os_bound_git "$git_dir" "$common_dir" "$home" ls-tree "$commit" -- "$path") || return 1 printf '%s' "$entry" | awk 'NR == 1 { print $1 " " $3 }' } worktree_entry() { - local git_dir=$1 home=$2 path=$3 mode hash + local git_dir=$1 common_dir=$2 home=$3 path=$4 mode hash if [ -L "$home/$path" ]; then mode=120000 - hash=$(printf '%s' "$(readlink "$home/$path")" | agent_os_bound_git "$git_dir" "$home" hash-object --stdin) + hash=$(printf '%s' "$(readlink "$home/$path")" | agent_os_bound_git "$git_dir" "$common_dir" "$home" hash-object --stdin) elif [ -f "$home/$path" ]; then if [ -x "$home/$path" ]; then mode=100755; else mode=100644; fi - hash=$(agent_os_bound_git "$git_dir" "$home" hash-object -- "$home/$path") + hash=$(agent_os_bound_git "$git_dir" "$common_dir" "$home" hash-object -- "$home/$path") elif [ ! -e "$home/$path" ]; then printf '\n' return 0 @@ -237,22 +342,22 @@ worktree_entry() { } entry_matches_transition() { - local git_dir=$1 home=$2 path=$3 actual=$4 commit expected - shift 4 + local git_dir=$1 common_dir=$2 home=$3 path=$4 actual=$5 commit expected + shift 5 for commit in "$@"; do - expected=$(trusted_tree_entry "$git_dir" "$home" "$commit" "$path") || return 1 + expected=$(trusted_tree_entry "$git_dir" "$common_dir" "$home" "$commit" "$path") || return 1 [ "$actual" = "$expected" ] && return 0 done return 1 } verify_pending_checkout() { - local git_dir=$1 home=$2 pending=$3 current_commit pending_commit record path worktree index index_entries status_file valid - current_commit=$(agent_os_bound_git "$git_dir" "$home" rev-parse HEAD) || return 1 + local git_dir=$1 common_dir=$2 home=$3 pending=$4 current_commit pending_commit record path worktree index index_entries status_file valid + current_commit=$(agent_os_bound_git "$git_dir" "$common_dir" "$home" rev-parse HEAD) || return 1 pending_commit=$(sed -n 's/^commit=//p' "$pending") - agent_os_bound_git "$git_dir" "$home" rev-parse "$pending_commit^{commit}" >/dev/null || return 1 + agent_os_bound_git "$git_dir" "$common_dir" "$home" rev-parse "$pending_commit^{commit}" >/dev/null || return 1 status_file=$(mktemp "${TMPDIR:-/tmp}/agent-os-secondmate-status.XXXXXX") || return 1 - if ! agent_os_bound_git "$git_dir" "$home" -c status.renames=false status \ + if ! agent_os_bound_git "$git_dir" "$common_dir" "$home" -c status.renames=false status \ --porcelain=v1 -z --untracked-files=all > "$status_file"; then rm -f "$status_file" return 1 @@ -262,15 +367,15 @@ verify_pending_checkout() { while IFS= read -r -d '' record; do path=${record:3} if [ -z "$path" ]; then valid=false; break; fi - worktree=$(worktree_entry "$git_dir" "$home" "$path") || { valid=false; break; } - entry_matches_transition "$git_dir" "$home" "$path" "$worktree" \ + worktree=$(worktree_entry "$git_dir" "$common_dir" "$home" "$path") || { valid=false; break; } + entry_matches_transition "$git_dir" "$common_dir" "$home" "$path" "$worktree" \ "$current_commit" "$pending_commit" "$SOURCE_COMMIT" || { valid=false; break; } - index_entries=$(agent_os_bound_git "$git_dir" "$home" ls-files -s -- "$path") || { valid=false; break; } + index_entries=$(agent_os_bound_git "$git_dir" "$common_dir" "$home" ls-files -s -- "$path") || { valid=false; break; } index=$(printf '%s' "$index_entries" | awk ' NR == 1 { entry=$1 " " $2 } END { if (NR <= 1) print entry; else exit 1 } ') || { valid=false; break; } - entry_matches_transition "$git_dir" "$home" "$path" "$index" \ + entry_matches_transition "$git_dir" "$common_dir" "$home" "$path" "$index" \ "$current_commit" "$pending_commit" "$SOURCE_COMMIT" || { valid=false; break; } RECOVERY_PATHS+=("$path") done < "$status_file" @@ -279,29 +384,31 @@ verify_pending_checkout() { } recover_pending_checkout() { - local git_dir=$1 home=$2 pending=$3 old_head path - verify_pending_checkout "$git_dir" "$home" "$pending" || return 1 - old_head=$(agent_os_bound_git "$git_dir" "$home" rev-parse HEAD) || return 1 - agent_os_bound_git "$git_dir" "$home" restore --source="$SOURCE_COMMIT" --staged --worktree -- . + local git_dir=$1 common_dir=$2 home=$3 pending=$4 old_head path + verify_pending_checkout "$git_dir" "$common_dir" "$home" "$pending" || return 1 + old_head=$(agent_os_bound_git "$git_dir" "$common_dir" "$home" rev-parse HEAD) || return 1 + agent_os_bound_git "$git_dir" "$common_dir" "$home" restore --source="$SOURCE_COMMIT" --staged --worktree -- . for path in "${RECOVERY_PATHS[@]}"; do - if [ -z "$(trusted_tree_entry "$git_dir" "$home" "$SOURCE_COMMIT" "$path")" ] && \ + if [ -z "$(trusted_tree_entry "$git_dir" "$common_dir" "$home" "$SOURCE_COMMIT" "$path")" ] && \ { [ -f "$home/$path" ] || [ -L "$home/$path" ]; }; then rm -f -- "$home/$path" fi done - agent_os_bound_git "$git_dir" "$home" update-ref --no-deref HEAD "$SOURCE_COMMIT" "$old_head" - [ -z "$(agent_os_bound_git "$git_dir" "$home" status --porcelain --untracked-files=all)" ] + agent_os_bound_git "$git_dir" "$common_dir" "$home" update-ref --no-deref HEAD "$SOURCE_COMMIT" "$old_head" + [ -z "$(agent_os_bound_git "$git_dir" "$common_dir" "$home" status --porcelain --untracked-files=all)" ] } ids=() homes=() +proofs=() add_home() { - local id=$1 home=$2 + local id=$1 home=$2 proof=$3 [ -n "$id" ] && [ -n "$home" ] || return 0 case "$id" in *[!A-Za-z0-9._-]*|.|..) echo "error: secondmate id is invalid" >&2; exit 2 ;; esac case "$home" in /*) ;; *) echo "error: secondmate home is not absolute" >&2; exit 2 ;; esac ids+=("$id") homes+=("$home") + proofs+=("$proof") } for meta in "$FM_HOME"/state/*.meta; do @@ -309,34 +416,42 @@ for meta in "$FM_HOME"/state/*.meta; do [ "$(sed -n 's/^kind=//p' "$meta")" = secondmate ] || continue id=${meta##*/} id=${id%.meta} - add_home "$id" "$(sed -n 's/^home=//p' "$meta")" + add_home "$id" "$(sed -n 's/^home=//p' "$meta")" primary-meta done if [ -f "$FM_HOME/data/secondmates.md" ]; then while IFS= read -r line; do case "$line" in '- '*) ;; *) continue ;; esac id=$(printf '%s\n' "$line" | sed -n 's/^- \([^ ][^ ]*\) - .*/\1/p') - add_home "$id" "$(printf '%s\n' "$line" | sed -n 's/.*(home:[[:space:]]*\([^;]*\);.*/\1/p' | sed 's/[[:space:]]*$//')" + add_home "$id" "$(printf '%s\n' "$line" | sed -n 's/.*(home:[[:space:]]*\([^;]*\);.*/\1/p' | sed 's/[[:space:]]*$//')" registry done < "$FM_HOME/data/secondmates.md" fi [ "$(trusted_git -C "$FM_ROOT" rev-parse HEAD)" = "$SOURCE_COMMIT" ] || exit 2 [ "$(trusted_git -C "$FM_ROOT" rev-parse 'HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 [ -z "$(trusted_git -C "$FM_ROOT" status --porcelain --untracked-files=all)" ] || exit 2 +acquire_primary_ownership_lock || { + echo "error: primary secondmate ownership lock is unavailable" >&2 + exit 2 +} validated_ids=() validated_homes=() treehouse_homes=() +ownership_modes=() git_dirs=() common_dirs=() +source_dirs=() bound_homes=() bound_configs=() bound_git_dirs=() bound_common_dirs=() +bound_source_dirs=() pending_policies=() seen_homes=$'\n' for index in "${!homes[@]}"; do id=${ids[$index]} home=${homes[$index]} + proof=${proofs[$index]} treehouse_home=$home validate_secondmate_home "$id" "$home" || { echo "error: invalid secondmate home for $id: $VALIDATION_ERROR" >&2 @@ -367,43 +482,45 @@ for index in "${!homes[@]}"; do [ "$duplicate" = false ] || continue case "$seen_homes" in *$'\n'"$home"$'\n'*) continue ;; esac seen_homes+="$home"$'\n' - resolve_treehouse_pool "$home" || { - echo "error: secondmate Treehouse state is unavailable: $home" >&2 - exit 2 - } - if [ -z "${TREEHOUSE_POOL:-}" ]; then - acquire_treehouse_lock "$RESOLVED_TREEHOUSE_POOL" || { - echo "error: secondmate Treehouse ownership lock is unavailable: $home" >&2 + if [ -d "$home/.git" ] && [ ! -L "$home/.git" ]; then + ownership_mode=primary-standalone + if [ "$proof" != primary-meta ] || ! validate_primary_standalone_proof "$id" "$home"; then + echo "error: standalone secondmate lacks exact primary-owned assignment proof: $id" >&2 + exit 2 + fi + else + ownership_mode=treehouse + resolve_treehouse_pool "$home" || { + echo "error: secondmate Treehouse state is unavailable: $home" >&2 exit 2 } - elif [ "$TREEHOUSE_POOL" != "$RESOLVED_TREEHOUSE_POOL" ]; then - echo "error: secondmate homes do not share one authoritative Treehouse pool" >&2 - exit 2 - fi - validate_treehouse_lease "$id" "$treehouse_home" "$home" || { - echo "error: secondmate Treehouse lease holder is not exact: $id" >&2 - exit 2 - } - resolve_git_metadata "$home" || { echo "error: secondmate Git metadata is invalid: $home" >&2; exit 2; } - validate_git_binding "$home" || { - echo "error: secondmate Git metadata is not bound to its home: $home" >&2 - exit 2 - } - validate_config "$RESOLVED_COMMON_DIR/config" || { - echo "error: secondmate Git configuration is invalid: $home" >&2 - exit 2 - } - if [ -f "$home/.git" ]; then - common_source=$(cd "$RESOLVED_COMMON_DIR/.." && pwd -P) - common_commit=$(sed -n 's/^commit=//p' "$RESOLVED_COMMON_DIR/agent-os-runtime-source") - [ "$(trusted_git -C "$common_source" rev-parse HEAD)" = "$common_commit" ] && \ - [ -z "$(trusted_git -C "$common_source" status --porcelain --untracked-files=all)" ] || { - echo "error: secondmate linked Git ownership is invalid: $home" >&2 + if [ -z "${TREEHOUSE_POOL:-}" ]; then + acquire_treehouse_lock "$RESOLVED_TREEHOUSE_POOL" || { + echo "error: secondmate Treehouse ownership lock is unavailable: $home" >&2 + exit 2 + } + elif [ "$TREEHOUSE_POOL" != "$RESOLVED_TREEHOUSE_POOL" ]; then + echo "error: secondmate homes do not share one authoritative Treehouse pool" >&2 + exit 2 + fi + validate_treehouse_lease "$id" "$treehouse_home" "$home" || { + echo "error: secondmate Treehouse lease holder is not exact: $id" >&2 exit 2 } fi + resolve_git_metadata "$home" || { echo "error: secondmate Git metadata is invalid: $home" >&2; exit 2; } original_git_dir=$RESOLVED_GIT_DIR original_common_dir=$RESOLVED_COMMON_DIR + if [ "$ownership_mode" = treehouse ]; then + original_source_dir=$(cd "$original_common_dir/.." && pwd -P) || exit 2 + runtime_root=$(cd "$FM_HOME/runtime-sources" && pwd -P) || exit 2 + case "$original_source_dir" in "$runtime_root"/*) ;; *) exit 2 ;; esac + source_key=${original_source_dir#"$runtime_root"/} + case "$source_key" in ''|*/*) exit 2 ;; esac + else + original_source_dir=$home + source_key=${original_source_dir##*/} + fi agent_os_open_bound_dir "$home" || { echo "error: secondmate home cannot be bound for mutation: $home" >&2 exit 2 @@ -424,6 +541,37 @@ for index in "${!homes[@]}"; do exit 2 } bound_common_dir=$AGENT_OS_BOUND_PATH + agent_os_open_bound_dir "$original_source_dir" || { + echo "error: secondmate source directory cannot be bound for mutation: $home" >&2 + exit 2 + } + bound_source_dir=$AGENT_OS_BOUND_PATH + if ! { agent_os_bound_dir_matches "$home" "$bound_home" && \ + agent_os_bound_dir_matches "$home/config" "$bound_config" && \ + agent_os_bound_dir_matches "$original_git_dir" "$bound_git_dir" && \ + agent_os_bound_dir_matches "$original_common_dir" "$bound_common_dir" && \ + agent_os_bound_dir_matches "$original_source_dir" "$bound_source_dir"; }; then + echo "error: secondmate ownership changed while binding: $home" >&2 + exit 2 + fi + validate_bound_git_binding "$home" "$bound_home" "$bound_git_dir" \ + "$bound_common_dir" "$bound_source_dir" "$([ "$ownership_mode" = treehouse ] && printf true || printf false)" \ + "$source_key" || { + echo "error: bound secondmate Git ownership is invalid: $home" >&2 + exit 2 + } + validate_config "$bound_common_dir/config" || { + echo "error: secondmate Git configuration is invalid: $home" >&2 + exit 2 + } + if [ "$ownership_mode" = treehouse ]; then + common_commit=$(sed -n 's/^commit=//p' "$bound_common_dir/agent-os-runtime-source") + [ "$(agent_os_bound_git "$bound_common_dir" "$bound_common_dir" "$bound_source_dir" rev-parse HEAD)" = "$common_commit" ] && \ + [ -z "$(agent_os_bound_git "$bound_common_dir" "$bound_common_dir" "$bound_source_dir" status --porcelain --untracked-files=all)" ] || { + echo "error: secondmate linked Git ownership is invalid: $home" >&2 + exit 2 + } + fi git_policy="$bound_git_dir/agent-os-runtime-source" home_policy="$bound_config/agent-os-source-policy" required_policy="$bound_config/agent-os-source-policy.required" @@ -470,33 +618,55 @@ for index in "${!homes[@]}"; do fi fi if [ -z "$selected_pending" ] && \ - [ -n "$(agent_os_bound_git "$bound_git_dir" "$bound_home" status --porcelain --untracked-files=all)" ]; then + [ -n "$(agent_os_bound_git "$bound_git_dir" "$bound_common_dir" "$bound_home" status --porcelain --untracked-files=all)" ]; then echo "error: secondmate source is not clean: $home" >&2 exit 2 fi validated_ids+=("$id") validated_homes+=("$home") treehouse_homes+=("$treehouse_home") + ownership_modes+=("$ownership_mode") git_dirs+=("$original_git_dir") common_dirs+=("$original_common_dir") + source_dirs+=("$original_source_dir") bound_homes+=("$bound_home") bound_configs+=("$bound_config") bound_git_dirs+=("$bound_git_dir") bound_common_dirs+=("$bound_common_dir") + bound_source_dirs+=("$bound_source_dir") pending_policies+=("$selected_pending") done assert_secondmate_binding() { - local id=$1 treehouse_home=$2 home=$3 git_dir=$4 common_dir=$5 bound_home=$6 bound_config=$7 bound_git=$8 bound_common=$9 - validate_treehouse_lease "$id" "$treehouse_home" "$home" || return 1 + local id=$1 ownership_mode=$2 treehouse_home=$3 home=$4 git_dir=$5 common_dir=$6 source_dir=$7 + local bound_home=$8 bound_config=$9 bound_git=${10} bound_common=${11} bound_source=${12} + if [ "$ownership_mode" = treehouse ]; then + validate_treehouse_lease "$id" "$treehouse_home" "$home" || return 1 + else + validate_primary_standalone_proof "$id" "$home" || return 1 + fi agent_os_bound_dir_matches "$home" "$bound_home" || return 1 agent_os_bound_dir_matches "$home/config" "$bound_config" || return 1 agent_os_bound_dir_matches "$git_dir" "$bound_git" || return 1 agent_os_bound_dir_matches "$common_dir" "$bound_common" || return 1 + agent_os_bound_dir_matches "$source_dir" "$bound_source" || return 1 resolve_git_metadata "$home" || return 1 [ "$RESOLVED_GIT_DIR" -ef "$bound_git" ] || return 1 [ "$RESOLVED_COMMON_DIR" -ef "$bound_common" ] || return 1 - validate_git_binding "$home" + validate_bound_git_binding "$home" "$bound_home" "$bound_git" "$bound_common" \ + "$bound_source" "$([ "$ownership_mode" = treehouse ] && printf true || printf false)" \ + "${source_dir##*/}" +} + +runtime_test_barrier() { + local phase=$1 + [ "${AGENT_OS_BOUND_TEST_FALLBACK_ACTIVE:-}" = true ] || return 0 + [ "${AGENT_OS_TEST_BARRIER_PHASE:-}" = "$phase" ] || return 0 + [ -n "${AGENT_OS_TEST_BARRIER_READY:-}" ] && [ -n "${AGENT_OS_TEST_BARRIER_RELEASE:-}" ] || return 1 + printf '%s\n' "$$" > "$AGENT_OS_TEST_BARRIER_READY" + while [ ! -e "$AGENT_OS_TEST_BARRIER_RELEASE" ]; do + /bin/sleep 0.01 + done } marker="mode=$SOURCE_MODE @@ -504,34 +674,41 @@ commit=$SOURCE_COMMIT source_sha256=$SOURCE_SHA" for index in "${!validated_homes[@]}"; do id=${validated_ids[$index]} + ownership_mode=${ownership_modes[$index]} treehouse_home=${treehouse_homes[$index]} home=${validated_homes[$index]} git_dir=${git_dirs[$index]} common_dir=${common_dirs[$index]} + source_dir=${source_dirs[$index]} bound_home=${bound_homes[$index]} bound_config=${bound_configs[$index]} bound_git_dir=${bound_git_dirs[$index]} bound_common_dir=${bound_common_dirs[$index]} + bound_source_dir=${bound_source_dirs[$index]} recovery_policy=${pending_policies[$index]} - assert_secondmate_binding "$id" "$treehouse_home" "$home" "$git_dir" "$common_dir" \ - "$bound_home" "$bound_config" "$bound_git_dir" "$bound_common_dir" || { + assert_secondmate_binding "$id" "$ownership_mode" "$treehouse_home" "$home" "$git_dir" \ + "$common_dir" "$source_dir" "$bound_home" "$bound_config" "$bound_git_dir" \ + "$bound_common_dir" "$bound_source_dir" || { echo "error: secondmate ownership changed before mutation: $home" >&2 exit 2 } - agent_os_bound_git "$bound_git_dir" "$bound_home" -c protocol.file.allow=always \ + runtime_test_barrier fetch + agent_os_bound_git "$bound_git_dir" "$bound_common_dir" "$bound_home" -c protocol.file.allow=always \ fetch --no-tags "$FM_ROOT" "$SOURCE_COMMIT" - [ "$(agent_os_bound_git "$bound_git_dir" "$bound_home" rev-parse FETCH_HEAD)" = "$SOURCE_COMMIT" ] || exit 2 - [ "$(agent_os_bound_git "$bound_git_dir" "$bound_home" rev-parse 'FETCH_HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 + [ "$(agent_os_bound_git "$bound_git_dir" "$bound_common_dir" "$bound_home" rev-parse FETCH_HEAD)" = "$SOURCE_COMMIT" ] || exit 2 + [ "$(agent_os_bound_git "$bound_git_dir" "$bound_common_dir" "$bound_home" rev-parse 'FETCH_HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 if [ -n "$recovery_policy" ] && \ - [ -n "$(agent_os_bound_git "$bound_git_dir" "$bound_home" status --porcelain --untracked-files=all)" ]; then + [ -n "$(agent_os_bound_git "$bound_git_dir" "$bound_common_dir" "$bound_home" status --porcelain --untracked-files=all)" ]; then recovery_policy="$bound_config/agent-os-source-policy.pending" - recover_pending_checkout "$bound_git_dir" "$bound_home" "$recovery_policy" || { + recover_pending_checkout "$bound_git_dir" "$bound_common_dir" "$bound_home" "$recovery_policy" || { echo "error: secondmate immutable policy journal cannot recover source: $home" >&2 exit 2 } fi - assert_secondmate_binding "$id" "$treehouse_home" "$home" "$git_dir" "$common_dir" \ - "$bound_home" "$bound_config" "$bound_git_dir" "$bound_common_dir" || exit 2 + assert_secondmate_binding "$id" "$ownership_mode" "$treehouse_home" "$home" "$git_dir" \ + "$common_dir" "$source_dir" "$bound_home" "$bound_config" "$bound_git_dir" \ + "$bound_common_dir" "$bound_source_dir" || exit 2 + runtime_test_barrier policy pending_policy="$bound_config/agent-os-source-policy.pending" pending_tmp="$bound_config/.agent-os-source-policy.pending.$$" printf '%s\n' "$marker" > "$pending_tmp" @@ -540,12 +717,17 @@ for index in "${!validated_homes[@]}"; do required_tmp="$bound_config/.agent-os-source-policy.required.$$" printf 'immutable\n' > "$required_tmp" mv "$required_tmp" "$required_policy" - assert_secondmate_binding "$id" "$treehouse_home" "$home" "$git_dir" "$common_dir" \ - "$bound_home" "$bound_config" "$bound_git_dir" "$bound_common_dir" || exit 2 - agent_os_bound_git "$bound_git_dir" "$bound_home" checkout --detach "$SOURCE_COMMIT" - agent_os_bound_git "$bound_git_dir" "$bound_home" remote set-url origin "$SOURCE_ORIGIN" - assert_secondmate_binding "$id" "$treehouse_home" "$home" "$git_dir" "$common_dir" \ - "$bound_home" "$bound_config" "$bound_git_dir" "$bound_common_dir" || exit 2 + assert_secondmate_binding "$id" "$ownership_mode" "$treehouse_home" "$home" "$git_dir" \ + "$common_dir" "$source_dir" "$bound_home" "$bound_config" "$bound_git_dir" \ + "$bound_common_dir" "$bound_source_dir" || exit 2 + runtime_test_barrier checkout + agent_os_bound_git "$bound_git_dir" "$bound_common_dir" "$bound_home" checkout --detach "$SOURCE_COMMIT" + runtime_test_barrier remote + agent_os_bound_git "$bound_git_dir" "$bound_common_dir" "$bound_home" remote set-url origin "$SOURCE_ORIGIN" + assert_secondmate_binding "$id" "$ownership_mode" "$treehouse_home" "$home" "$git_dir" \ + "$common_dir" "$source_dir" "$bound_home" "$bound_config" "$bound_git_dir" \ + "$bound_common_dir" "$bound_source_dir" || exit 2 + runtime_test_barrier marker home_marker_tmp="$bound_config/.agent-os-source-policy.$$" printf '%s\n' "$marker" > "$home_marker_tmp" mv "$home_marker_tmp" "$bound_config/agent-os-source-policy" @@ -553,11 +735,12 @@ for index in "${!validated_homes[@]}"; do printf '%s\n' "$marker" > "$marker_tmp" mv "$marker_tmp" "$bound_git_dir/agent-os-runtime-source" cmp "$bound_config/agent-os-source-policy" "$bound_git_dir/agent-os-runtime-source" || exit 2 - [ "$(agent_os_bound_git "$bound_git_dir" "$bound_home" rev-parse HEAD)" = "$SOURCE_COMMIT" ] || exit 2 - [ "$(agent_os_bound_git "$bound_git_dir" "$bound_home" rev-parse 'HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 - [ -z "$(agent_os_bound_git "$bound_git_dir" "$bound_home" status --porcelain --untracked-files=all)" ] || exit 2 - assert_secondmate_binding "$id" "$treehouse_home" "$home" "$git_dir" "$common_dir" \ - "$bound_home" "$bound_config" "$bound_git_dir" "$bound_common_dir" || exit 2 + [ "$(agent_os_bound_git "$bound_git_dir" "$bound_common_dir" "$bound_home" rev-parse HEAD)" = "$SOURCE_COMMIT" ] || exit 2 + [ "$(agent_os_bound_git "$bound_git_dir" "$bound_common_dir" "$bound_home" rev-parse 'HEAD^{tree}')" = "$SOURCE_TREE" ] || exit 2 + [ -z "$(agent_os_bound_git "$bound_git_dir" "$bound_common_dir" "$bound_home" status --porcelain --untracked-files=all)" ] || exit 2 + assert_secondmate_binding "$id" "$ownership_mode" "$treehouse_home" "$home" "$git_dir" \ + "$common_dir" "$source_dir" "$bound_home" "$bound_config" "$bound_git_dir" \ + "$bound_common_dir" "$bound_source_dir" || exit 2 rm "$pending_policy" printf 'selected: %s\n' "$home" done diff --git a/tests/agent-os-candidate-state.test.sh b/tests/agent-os-candidate-state.test.sh index 654babe7c..a6c460346 100755 --- a/tests/agent-os-candidate-state.test.sh +++ b/tests/agent-os-candidate-state.test.sh @@ -4,45 +4,91 @@ set -u . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" STATE="$ROOT/bin/agent-os-candidate-state.sh" +TMP=$(fm_test_tmproot agent-os-candidate-state) +mkdir -p "$TMP" +OWNER_A=101 +OWNER_B=202 +OWNER_C=303 +CLAIM_A=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +CLAIM_B=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +CLAIM_C=cccccccccccccccccccccccccccccccccccccccc + +write_chain() { + printf '%b\n' "$@" > "$TMP/chain.tsv" +} expect_decision() { - local expected=$1 attempt=$2 evidence=$3 claim=$4 actual - actual=$("$STATE" "$attempt" "$evidence" "$claim") || \ - fail "state $attempt/$evidence/$claim did not produce $expected" - [ "$actual" = "$expected" ] || \ - fail "state $attempt/$evidence/$claim produced $actual instead of $expected" + local expected=$1 actual + actual=$("$STATE" "$TMP/chain.tsv") || fail "claim chain did not produce $expected" + [ "$actual" = "$expected" ] || fail "claim chain produced $actual instead of $expected" } expect_refusal() { - local attempt=$1 evidence=$2 claim=$3 - if "$STATE" "$attempt" "$evidence" "$claim" >/dev/null 2>&1; then - fail "state $attempt/$evidence/$claim authorized another build" + if "$STATE" "$TMP/chain.tsv" >/dev/null 2>&1; then + fail "claim chain authorized another build" fi } -expect_decision build absent absent exact -expect_decision reuse-build attempted exact-build exact -expect_decision reuse-record attempted exact-record exact +write_chain "$OWNER_A\t$CLAIM_A\tabsent\tabsent" +expect_decision build + +write_chain \ + "$OWNER_A\t$CLAIM_A\tattempted\texact-build" \ + "$OWNER_B\t$CLAIM_B\tabsent\tabsent" +expect_decision $'reuse-build\t'"$OWNER_A" + +write_chain \ + "$OWNER_A\t$CLAIM_A\tattempted\texact-record" \ + "$OWNER_B\t$CLAIM_B\tabsent\tabsent" \ + "$OWNER_C\t$CLAIM_C\tabsent\tabsent" +expect_decision $'reuse-record\t'"$OWNER_A" builds=0 -decision=$("$STATE" absent absent exact) || fail "initial candidate did not authorize its one build" +write_chain "$OWNER_A\t$CLAIM_A\tabsent\tabsent" +decision=$("$STATE" "$TMP/chain.tsv") || fail "initial claim did not authorize its build" +[ "$decision" = build ] && builds=$((builds + 1)) +write_chain \ + "$OWNER_A\t$CLAIM_A\tattempted\texact-build" \ + "$OWNER_B\t$CLAIM_B\tabsent\tabsent" +decision=$("$STATE" "$TMP/chain.tsv") || fail "handoff did not reuse ancestor evidence" [ "$decision" = build ] && builds=$((builds + 1)) -decision=$("$STATE" attempted exact-build exact) || fail "durable candidate build was not reusable" +write_chain \ + "$OWNER_A\t$CLAIM_A\tattempted\texact-build" \ + "$OWNER_B\t$CLAIM_B\tabsent\tabsent" \ + "$OWNER_C\t$CLAIM_C\tabsent\tabsent" +decision=$("$STATE" "$TMP/chain.tsv") || fail "second handoff did not reuse ancestor evidence" [ "$decision" = build ] && builds=$((builds + 1)) -[ "$builds" -eq 1 ] || fail "candidate state machine authorized $builds builds" +[ "$builds" -eq 1 ] || fail "claim-chain classifier authorized $builds builds" -for attempt in attempted corrupt mismatched unreadable metadata-read-error partial ambiguous; do - expect_refusal "$attempt" absent exact -done -for evidence in corrupt mismatched unreadable metadata-read-error partial ambiguous; do - expect_refusal attempted "$evidence" exact -done -for claim in missing mismatched unreadable ambiguous; do - expect_refusal absent absent "$claim" +write_chain \ + "$OWNER_A\t$CLAIM_A\tattempted\tabsent" \ + "$OWNER_B\t$CLAIM_B\tabsent\tabsent" +expect_refusal + +write_chain \ + "$OWNER_A\t$CLAIM_A\tattempted\texact-build" \ + "$OWNER_B\t$CLAIM_B\tattempted\texact-build" +expect_refusal + +write_chain \ + "$OWNER_A\t$CLAIM_A\tabsent\texact-build" \ + "$OWNER_B\t$CLAIM_B\tabsent\tabsent" +expect_refusal + +for state in corrupt mismatched unreadable metadata-read-error partial ambiguous; do + write_chain "$OWNER_A\t$CLAIM_A\t$state\tabsent" + expect_refusal + write_chain "$OWNER_A\t$CLAIM_A\tattempted\t$state" + expect_refusal done -expect_refusal attempted partial exact -expect_refusal attempted absent exact -expect_refusal absent exact-build exact -expect_refusal absent exact-record exact -pass "candidate state machine permits one build and exact reuse only" +write_chain \ + "$OWNER_A\t$CLAIM_A\tabsent\tabsent" \ + "$OWNER_A\t$CLAIM_B\tabsent\tabsent" +expect_refusal +write_chain \ + "$OWNER_A\t$CLAIM_A\tabsent\tabsent" \ + "$OWNER_B\t$CLAIM_A\tabsent\tabsent" +expect_refusal + +pass "candidate claim-chain classifier permits one build and ancestor reuse only" diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index d1dfa8365..b6448578d 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -280,8 +280,10 @@ assert_grep 'refs/agent-os/candidate-build-attempts/$GITHUB_SHA/$owner_run_id' " assert_grep 'candidate build attempt has no durable exact evidence; refusing rebuild' \ "$ROOT/bin/agent-os-candidate-state.sh" \ "candidate cancellation must fail closed instead of rebuilding" -assert_grep 'bin/agent-os-candidate-state.sh "$BUILD_ATTEMPT_STATE" absent exact' "$IMAGE_WORKFLOW" \ - "candidate rebuild authorization must use the executable state machine" +assert_grep 'bin/agent-os-candidate-state.sh "$chain_state"' "$IMAGE_WORKFLOW" \ + "candidate rebuild authorization must classify the full claim chain" +assert_grep 'done < "$CLAIM_CHAIN_FILE"' "$IMAGE_WORKFLOW" \ + "candidate retries must inspect every validated reservation owner" assert_no_grep 'if load_build_attempt' "$IMAGE_WORKFLOW" \ "candidate attempt validation must never run in a conditional errexit context" attempt_line=$(grep -n 'name: Claim owner-bound candidate build attempt' "$IMAGE_WORKFLOW" | cut -d: -f1) @@ -294,7 +296,7 @@ record_line=$(grep -n 'name: Prepare immutable release candidate record' "$IMAGE [ "$build_line" -lt "$build_evidence_line" ] && \ [ "$build_evidence_line" -lt "$image_publish_line" ] && [ "$image_publish_line" -lt "$record_line" ] || \ fail "owner-bound attempt state must precede build and exact evidence publication" -attempt_guard_line=$(grep -n 'load_build_attempt "$artifact_owner"' "$IMAGE_WORKFLOW" | cut -d: -f1) +attempt_guard_line=$(grep -n 'load_build_attempt "$owner"' "$IMAGE_WORKFLOW" | cut -d: -f1) rebuild_line=$(grep -n "printf 'build=true" "$IMAGE_WORKFLOW" | cut -d: -f1) [ -n "$attempt_guard_line" ] && [ -n "$rebuild_line" ] && [ "$attempt_guard_line" -lt "$rebuild_line" ] || \ fail "candidate upload failure must be rejected before any rebuild is authorized" diff --git a/tests/agent-os-runtime-secondmates.test.sh b/tests/agent-os-runtime-secondmates.test.sh index 9d74be8c3..31fec6a3a 100755 --- a/tests/agent-os-runtime-secondmates.test.sh +++ b/tests/agent-os-runtime-secondmates.test.sh @@ -11,12 +11,26 @@ FM_HOME_DIR="$TMP/home" WORK="$TMP/work" LEASE= TEST_FLOCK="$TEST_ROOT/flock" +TEST_LOCK_ROOT="$TEST_ROOT/locks" PRESERVE_TREEHOUSE_STATE=false fm_git_identity fmtest fmtest@example.com -mkdir -p "$FM_HOME_DIR/state" "$FM_HOME_DIR/data" "$TREEHOUSE_POOL" -printf '#!/usr/bin/env bash\nexit 0\n' > "$TEST_FLOCK" +mkdir -p "$FM_HOME_DIR/state" "$FM_HOME_DIR/data" "$TREEHOUSE_POOL" "$TEST_LOCK_ROOT" +printf '%s\n' \ + '#!/usr/bin/env bash' \ + 'set -eu' \ + 'fd=${2:?lock file descriptor is required}' \ + 'target=${AGENT_OS_TEST_LOCK_FILE_HINT:?lock file identity is required}' \ + 'key=$(printf "%s" "$target" | /usr/bin/cksum | /usr/bin/awk "{print \$1}")' \ + 'lock="$AGENT_OS_TEST_LOCK_ROOT/$key"' \ + 'if ! /bin/mkdir "$lock" 2>/dev/null; then' \ + ' owner=$(/bin/cat "$lock/owner" 2>/dev/null || true)' \ + ' if [ -n "$owner" ] && /bin/kill -0 "$owner" 2>/dev/null; then exit 1; fi' \ + ' /bin/rm -rf "$lock"' \ + ' /bin/mkdir "$lock"' \ + 'fi' \ + 'printf "%s\n" "$PPID" > "$lock/owner"' > "$TEST_FLOCK" chmod +x "$TEST_FLOCK" touch "$TREEHOUSE_POOL/treehouse-state.lock" git init -q -b main "$WORK" @@ -92,7 +106,45 @@ run_sync() { AGENT_OS_SOURCE_SHA256="$sha" AGENT_OS_SOURCE_BRANCH=main \ AGENT_OS_SOURCE_ORIGIN=https://github.com/akua-dev/agent-os.git \ AGENT_OS_SOURCE_MODE=release AGENT_OS_TEST_FLOCK_BIN="$TEST_FLOCK" \ - AGENT_OS_TEST_BOUND_PATHS=true "$SYNC" + AGENT_OS_TEST_LOCK_ROOT="$TEST_LOCK_ROOT" AGENT_OS_TEST_BOUND_PATHS=true "$SYNC" +} + +fleet_mutation_signature() { + local id gitdir policy + for id in linked standalone; do + gitdir=$(git -C "$TMP/$id" rev-parse --absolute-git-dir) + printf '%s:%s:%s\n' "$id" "$(git -C "$TMP/$id" rev-parse HEAD)" \ + "$(git -C "$TMP/$id" remote get-url origin)" + for policy in "$gitdir/agent-os-runtime-source" \ + "$TMP/$id/config/agent-os-source-policy" \ + "$TMP/$id/config/agent-os-source-policy.pending" \ + "$TMP/$id/config/agent-os-source-policy.required"; do + if [ -f "$policy" ]; then + printf '%s:%s\n' "$policy" "$(shasum -a 256 "$policy" | awk '{print $1}')" + else + printf '%s:absent\n' "$policy" + fi + done + done + if [ -d "$TMP/foreign-guard/.git" ]; then + printf 'foreign-guard:%s:%s\n' "$(git -C "$TMP/foreign-guard" rev-parse HEAD)" \ + "$(git -C "$TMP/foreign-guard" remote get-url origin)" + if [ -f "$TMP/foreign-guard/.git/agent-os-runtime-source" ]; then + shasum -a 256 "$TMP/foreign-guard/.git/agent-os-runtime-source" + else + printf 'foreign-guard-policy:absent\n' + fi + fi +} + +expect_rejection_without_mutation() { + local label=$1 before after + before=$(fleet_mutation_signature) + if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then + fail "$label was accepted" + fi + after=$(fleet_mutation_signature) + [ "$after" = "$before" ] || fail "$label allowed source, remote, marker, or policy mutation" } make_primary primary-b "$B_COMMIT" @@ -255,100 +307,114 @@ fi rm "$FM_HOME_DIR/state/nested.meta" pass "secondmate selection rejects canonical home overlap" +git clone -q "$LEASE" "$TMP/foreign-guard" +git -C "$TMP/foreign-guard" checkout -q --detach "$A_COMMIT" + write_treehouse_state PRESERVE_TREEHOUSE_STATE=true +jq 'del(.worktrees[] | select(.lease_holder == "standalone"))' \ + "$TREEHOUSE_POOL/treehouse-state.json" > "$TEST_ROOT/state-next.json" +mv "$TEST_ROOT/state-next.json" "$TREEHOUSE_POOL/treehouse-state.json" +run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null || \ + fail "primary-owned standalone assignment required Treehouse self-attestation" +run_sync "$TMP/primary-a" "$A_COMMIT" "$A_TREE" "$A_SHA" >/dev/null || \ + fail "primary-owned standalone assignment could not select its rollback source" +pass "primary-owned standalone assignment needs no child or Treehouse attestation" + +mv "$FM_HOME_DIR/state/standalone.meta" "$TEST_ROOT/standalone.meta" +printf -- '- standalone - standby (home: %s/standalone; state: idle)\n' "$TMP" \ + > "$FM_HOME_DIR/data/secondmates.md" +expect_rejection_without_mutation "registry-only standalone self-attestation" +mv "$TEST_ROOT/standalone.meta" "$FM_HOME_DIR/state/standalone.meta" +: > "$FM_HOME_DIR/data/secondmates.md" +pass "standalone assignment requires primary-owned proof" + +write_treehouse_state jq 'del(.worktrees[] | select(.lease_holder == "linked"))' \ "$TREEHOUSE_POOL/treehouse-state.json" > "$TEST_ROOT/state-next.json" mv "$TEST_ROOT/state-next.json" "$TREEHOUSE_POOL/treehouse-state.json" -if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then - fail "missing Treehouse lease evidence was accepted" -fi -[ "$(git -C "$TMP/linked" rev-parse HEAD)" = "$A_COMMIT" ] || \ - fail "missing Treehouse lease evidence allowed source mutation" +expect_rejection_without_mutation "missing Treehouse lease evidence" write_treehouse_state jq '(.worktrees[] | select(.lease_holder == "linked") | .leased) = false' \ "$TREEHOUSE_POOL/treehouse-state.json" > "$TEST_ROOT/state-next.json" mv "$TEST_ROOT/state-next.json" "$TREEHOUSE_POOL/treehouse-state.json" -if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then - fail "stale Treehouse lease evidence was accepted" -fi +expect_rejection_without_mutation "stale Treehouse lease evidence" write_treehouse_state jq '(.worktrees[] | select(.lease_holder == "linked") | .lease_holder) = "other"' \ "$TREEHOUSE_POOL/treehouse-state.json" > "$TEST_ROOT/state-next.json" mv "$TEST_ROOT/state-next.json" "$TREEHOUSE_POOL/treehouse-state.json" -if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then - fail "other-holder Treehouse lease evidence was accepted" -fi +expect_rejection_without_mutation "other-holder Treehouse lease evidence" write_treehouse_state jq '.worktrees += [.worktrees[] | select(.lease_holder == "linked")]' \ "$TREEHOUSE_POOL/treehouse-state.json" > "$TEST_ROOT/state-next.json" mv "$TEST_ROOT/state-next.json" "$TREEHOUSE_POOL/treehouse-state.json" -if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then - fail "duplicate Treehouse lease evidence was accepted" -fi +expect_rejection_without_mutation "duplicate Treehouse lease evidence" write_treehouse_state jq '.worktrees += [(.worktrees[] | select(.lease_holder == "linked") | .path += "-planted")]' \ "$TREEHOUSE_POOL/treehouse-state.json" > "$TEST_ROOT/state-next.json" mv "$TEST_ROOT/state-next.json" "$TREEHOUSE_POOL/treehouse-state.json" -if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then - fail "ambiguous planted Treehouse holder evidence was accepted" -fi +expect_rejection_without_mutation "ambiguous planted Treehouse holder evidence" printf '{unreadable\n' > "$TREEHOUSE_POOL/treehouse-state.json" -if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then - fail "unreadable Treehouse lease evidence was accepted" -fi +expect_rejection_without_mutation "unreadable Treehouse lease evidence" PRESERVE_TREEHOUSE_STATE=false write_treehouse_state pass "secondmate selection requires one exact active Treehouse lease holder" -git -C "$LEASE" worktree add -q --detach "$TMP/toctou-victim" "$A_COMMIT" git clone -q "$LEASE" "$TMP/toctou-foreign" git -C "$TMP/toctou-foreign" checkout -q --detach "$A_COMMIT" -mkdir -p "$TMP/toctou-victim/config" "$TMP/toctou-foreign/config" -foreign_head=$(git -C "$TMP/toctou-foreign" rev-parse HEAD) -AGENT_OS_TEST_BOUND_PATHS=true -export AGENT_OS_TEST_BOUND_PATHS -. "$ROOT/bin/agent-os-runtime-bound.sh" -trusted_git() { - env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ - GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 \ - /usr/bin/git -c credential.helper= -c core.hooksPath=/dev/null "$@" +linked_gitdir=$(git -C "$TMP/linked" rev-parse --absolute-git-dir) +linked_commondir=$(cat "$linked_gitdir/commondir") +linked_pointer=$(cat "$TMP/linked/.git") +foreign_before=$(find "$TMP/toctou-foreign/.git" -type f -print0 | sort -z | \ + xargs -0 shasum -a 256 | shasum -a 256 | awk '{print $1}') +fleet_before=$(fleet_mutation_signature) +write_treehouse_state +PRESERVE_TREEHOUSE_STATE=true +AGENT_OS_TEST_BARRIER_PHASE=fetch +AGENT_OS_TEST_BARRIER_READY="$TEST_ROOT/toctou-ready" +AGENT_OS_TEST_BARRIER_RELEASE="$TEST_ROOT/toctou-release" +export AGENT_OS_TEST_BARRIER_PHASE AGENT_OS_TEST_BARRIER_READY AGENT_OS_TEST_BARRIER_RELEASE +rm -f "$AGENT_OS_TEST_BARRIER_READY" "$AGENT_OS_TEST_BARRIER_RELEASE" +run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" \ + > "$TEST_ROOT/toctou-runtime.out" 2>&1 & +runtime_pid=$! +for _ in $(seq 1 500); do + [ -f "$AGENT_OS_TEST_BARRIER_READY" ] && break + sleep 0.01 +done +[ -f "$AGENT_OS_TEST_BARRIER_READY" ] || { + kill "$runtime_pid" 2>/dev/null || true + fail "runtime TOCTOU barrier was not reached" } -agent_os_open_bound_dir "$TMP/toctou-victim" || fail "TOCTOU victim home could not be bound" -toctou_home=$AGENT_OS_BOUND_PATH -agent_os_open_bound_dir "$TMP/toctou-victim/config" || fail "TOCTOU victim configuration could not be bound" -toctou_config=$AGENT_OS_BOUND_PATH -toctou_git_path=$(git -C "$TMP/toctou-victim" rev-parse --absolute-git-dir) -agent_os_open_bound_dir "$toctou_git_path" || fail "TOCTOU victim Git directory could not be bound" -toctou_git=$AGENT_OS_BOUND_PATH -rm "$TMP/toctou-victim/.git" -printf 'gitdir: %s/toctou-foreign/.git\n' "$TMP" > "$TMP/toctou-victim/.git" -mv "$TMP/toctou-victim/config" "$TMP/toctou-victim-config-owned" -ln -s "$TMP/toctou-foreign/config" "$TMP/toctou-victim/config" -if agent_os_bound_dir_matches "$TMP/toctou-victim/config" "$toctou_config"; then - fail "swapped secondmate configuration retained its validated identity" +if AGENT_OS_TEST_BARRIER_PHASE= run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" \ + >/dev/null 2>&1; then + touch "$AGENT_OS_TEST_BARRIER_RELEASE" + wait "$runtime_pid" 2>/dev/null || true + fail "secondmate ownership lock admitted a concurrent runtime mutation" fi -if [ "$(git -C "$TMP/toctou-victim" rev-parse --absolute-git-dir)" -ef "$toctou_git" ]; then - fail "swapped secondmate Git pointer retained its validated identity" +printf 'gitdir: %s/toctou-foreign/.git\n' "$TMP" > "$TMP/linked/.git" +printf '%s\n' "$TMP/toctou-foreign/.git" > "$linked_gitdir/commondir" +touch "$AGENT_OS_TEST_BARRIER_RELEASE" +if wait "$runtime_pid"; then + fail "runtime accepted a Git pointer and common-directory swap" fi -agent_os_bound_git "$toctou_git" "$toctou_home" checkout -q --detach "$B_COMMIT" || \ - fail "captured secondmate Git directory could not select its exact source" -agent_os_bound_git "$toctou_git" "$toctou_home" remote set-url origin \ - https://github.com/akua-dev/agent-os.git -printf 'mode=release\ncommit=%s\nsource_sha256=%s\n' "$B_COMMIT" "$B_SHA" \ - > "$toctou_git/agent-os-runtime-source" -[ "$(git -C "$TMP/toctou-foreign" rev-parse HEAD)" = "$foreign_head" ] || \ - fail "pointer swap redirected a captured Git mutation into a foreign repository" -[ ! -e "$TMP/toctou-foreign/.git/agent-os-runtime-source" ] || \ - fail "pointer swap redirected marker mutation into a foreign repository" -[ ! -e "$TMP/toctou-foreign/config/agent-os-source-policy" ] || \ - fail "configuration swap redirected policy mutation into a foreign home" -pass "captured secondmate directories defeat Git pointer swaps" +printf '%s\n' "$linked_pointer" > "$TMP/linked/.git" +printf '%s\n' "$linked_commondir" > "$linked_gitdir/commondir" +unset AGENT_OS_TEST_BARRIER_PHASE AGENT_OS_TEST_BARRIER_READY AGENT_OS_TEST_BARRIER_RELEASE +foreign_after=$(find "$TMP/toctou-foreign/.git" -type f -print0 | sort -z | \ + xargs -0 shasum -a 256 | shasum -a 256 | awk '{print $1}') +[ "$foreign_after" = "$foreign_before" ] || \ + fail "runtime TOCTOU redirected Git mutation into a foreign repository" +[ "$(fleet_mutation_signature)" = "$fleet_before" ] || \ + fail "runtime TOCTOU changed secondmate source, remote, marker, or policy state" +PRESERVE_TREEHOUSE_STATE=false +write_treehouse_state +pass "runtime lock and bound common directory defeat live Git pointer swaps" printf 'tampered\n' >> "$TMP/standalone/AGENTS.md" if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then From 0bdc7820a70207bbc83af754f597429f8e570ef6 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 22:15:43 +0200 Subject: [PATCH 49/56] no-mistakes(review): Harden candidate recovery and secondmate ownership bindings --- .github/workflows/agent-os-image.yml | 95 ++++++++++--------- bin/agent-os-candidate-state.sh | 43 +++++---- bin/agent-os-runtime-bound.sh | 19 ++++ bin/agent-os-runtime-secondmates.sh | 56 +++++++---- tests/agent-os-candidate-state.test.sh | 62 +++++++----- tests/agent-os-runtime-secondmates.test.sh | 104 ++++++++++++--------- 6 files changed, 231 insertions(+), 148 deletions(-) diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index ae1c11058..3061588bc 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -381,6 +381,36 @@ jobs: exit 1 } } + load_exact_build_artifact() { + exact_build_owner=$1 + validate_artifact_run "$exact_build_owner" + build_artifact_id=$(jq -er --arg sha "$GITHUB_SHA" --argjson owner "$exact_build_owner" \ + '[.artifacts[] | select(.expired == false and .workflow_run.head_sha == $sha and .workflow_run.id == $owner)] | .[0].id' \ + "$RUNNER_TEMP/candidate-build-artifacts.json") + rm -rf "$RUNNER_TEMP/candidate-build-artifact" + mkdir "$RUNNER_TEMP/candidate-build-artifact" + gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$build_artifact_id/zip" \ + > "$RUNNER_TEMP/candidate-build-artifact.zip" + unzip -q "$RUNNER_TEMP/candidate-build-artifact.zip" -d "$RUNNER_TEMP/candidate-build-artifact" + jq -e --arg commit '${{ steps.candidate_source.outputs.commit }}' \ + --arg tree '${{ steps.candidate_source.outputs.tree }}' \ + --arg reservation "$reservation_digest" --arg repository "$IMAGE" \ + --argjson owner "$exact_build_owner" ' + .schema_version == 1 and .commit == $commit and .tree == $tree and + .reservation_digest == $reservation and .image_repository == $repository and + .owner_run_id == $owner and + (.image_digest | test("^sha256:[0-9a-f]{64}$")) and + (.oci_archive_sha256 | test("^[0-9a-f]{64}$"))' \ + "$RUNNER_TEMP/candidate-build-artifact/candidate-build.json" >/dev/null + EXACT_BUILD_IMAGE_DIGEST=$(jq -er .image_digest \ + "$RUNNER_TEMP/candidate-build-artifact/candidate-build.json") + EXACT_BUILD_ARCHIVE_SHA=$(jq -er .oci_archive_sha256 \ + "$RUNNER_TEMP/candidate-build-artifact/candidate-build.json") + [ "$(sha256sum "$RUNNER_TEMP/candidate-build-artifact/candidate-image.tar" | awk '{print $1}')" = \ + "$EXACT_BUILD_ARCHIVE_SHA" ] + cp "$RUNNER_TEMP/candidate-build-artifact/candidate-image.tar" \ + "$RUNNER_TEMP/candidate-image.tar" + } create_claim "$claim_prefix/initial" "$GITHUB_RUN_ID" || true load_latest_claim printf '{}\n' > "$RUNNER_TEMP/candidate-reservation-config.json" @@ -438,17 +468,8 @@ jobs: echo "::error::candidate claim owner has conflicting durable evidence" exit 1 } - if [ "$record_count" -eq 1 ] && [ "$build_count" -eq 0 ]; then - evidence_state=exact-record - elif [ "$record_count" -eq 0 ] && [ "$build_count" -eq 1 ]; then - evidence_state=exact-build - elif [ "$record_count" -eq 0 ] && [ "$build_count" -eq 0 ]; then - evidence_state=absent - else - evidence_state=ambiguous - fi - printf '%s\t%s\t%s\t%s\n' "$owner" "$claim_sha" \ - "$BUILD_ATTEMPT_STATE" "$evidence_state" >> "$chain_state" + printf '%s\t%s\t%s\t%s\t%s\n' "$owner" "$claim_sha" \ + "$BUILD_ATTEMPT_STATE" "$record_count" "$build_count" >> "$chain_state" done < "$CLAIM_CHAIN_FILE" decision=$(bin/agent-os-candidate-state.sh "$chain_state") IFS=$'\t' read -r candidate_action artifact_owner <> "$GITHUB_OUTPUT" echo "::notice::candidate reservation recovered from durable trusted artifact" exit 0 fi - build_artifact_count=$(jq -er --arg sha "$GITHUB_SHA" --argjson owner "$artifact_owner" \ - '[.artifacts[] | select(.expired == false and .workflow_run.head_sha == $sha and .workflow_run.id == $owner)] | length' \ - "$RUNNER_TEMP/candidate-build-artifacts.json") - [ "$build_artifact_count" -le 1 ] || { - echo "::error::candidate reservation has conflicting exact build evidence" - exit 1 - } if [ "$candidate_action" = reuse-build ]; then [ "$build_artifact_count" -eq 1 ] || exit 1 - validate_artifact_run "$artifact_owner" - build_artifact_id=$(jq -er --arg sha "$GITHUB_SHA" --argjson owner "$artifact_owner" \ - '[.artifacts[] | select(.expired == false and .workflow_run.head_sha == $sha and .workflow_run.id == $owner)] | .[0].id' \ - "$RUNNER_TEMP/candidate-build-artifacts.json") - mkdir "$RUNNER_TEMP/candidate-build-artifact" - gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$build_artifact_id/zip" \ - > "$RUNNER_TEMP/candidate-build-artifact.zip" - unzip -q "$RUNNER_TEMP/candidate-build-artifact.zip" -d "$RUNNER_TEMP/candidate-build-artifact" - jq -e --arg commit '${{ steps.candidate_source.outputs.commit }}' \ - --arg tree '${{ steps.candidate_source.outputs.tree }}' \ - --arg reservation "$reservation_digest" --arg repository "$IMAGE" \ - --argjson owner "$artifact_owner" ' - .schema_version == 1 and .commit == $commit and .tree == $tree and - .reservation_digest == $reservation and .image_repository == $repository and - .owner_run_id == $owner and - (.image_digest | test("^sha256:[0-9a-f]{64}$")) and - (.oci_archive_sha256 | test("^[0-9a-f]{64}$"))' \ - "$RUNNER_TEMP/candidate-build-artifact/candidate-build.json" >/dev/null - image_digest=$(jq -er .image_digest "$RUNNER_TEMP/candidate-build-artifact/candidate-build.json") - archive_sha=$(jq -er .oci_archive_sha256 "$RUNNER_TEMP/candidate-build-artifact/candidate-build.json") - [ "$(sha256sum "$RUNNER_TEMP/candidate-build-artifact/candidate-image.tar" | awk '{print $1}')" = "$archive_sha" ] - cp "$RUNNER_TEMP/candidate-build-artifact/candidate-image.tar" "$RUNNER_TEMP/candidate-image.tar" + load_exact_build_artifact "$artifact_owner" + image_digest=$EXACT_BUILD_IMAGE_DIGEST ensure_build_owner printf 'build=false\npublish_image=true\nrecord_artifact=false\nimage_digest=%s\nbootstrap_sha256=%s\n' \ "$image_digest" '${{ steps.candidate_source.outputs.bootstrap_sha256 }}' >> "$GITHUB_OUTPUT" diff --git a/bin/agent-os-candidate-state.sh b/bin/agent-os-candidate-state.sh index a07fd515b..3c1837ae1 100755 --- a/bin/agent-os-candidate-state.sh +++ b/bin/agent-os-candidate-state.sh @@ -9,11 +9,12 @@ state_file=${1:?candidate claim-chain state file is required} attempted_count=0 build_owner= -build_evidence= +build_record_count= +build_artifact_count= seen_owners='|' seen_claims='|' entries=0 -while IFS=$'\t' read -r owner claim attempt evidence extra || [ -n "${owner:-}" ]; do +while IFS=$'\t' read -r owner claim attempt record_count artifact_count extra || [ -n "${owner:-}" ]; do [ -z "${extra:-}" ] || { echo "error: candidate claim-chain state has extra fields" >&2 exit 2 @@ -32,26 +33,27 @@ while IFS=$'\t' read -r owner claim attempt evidence extra || [ -n "${owner:-}" attempted) attempted_count=$((attempted_count + 1)) build_owner=$owner - build_evidence=$evidence + build_record_count=$record_count + build_artifact_count=$artifact_count ;; *) echo "error: candidate attempt state is not exact: $attempt" >&2 exit 2 ;; esac - case "$evidence" in - absent) ;; - exact-record|exact-build) - [ "$attempt" = attempted ] || { - echo "error: candidate evidence is not bound to its build attempt" >&2 - exit 2 - } - ;; - *) - echo "error: candidate evidence state is not exact: $evidence" >&2 - exit 2 - ;; - esac + [[ "$record_count" =~ ^[0-9]+$ ]] && [[ "$artifact_count" =~ ^[0-9]+$ ]] || { + echo "error: candidate evidence counts are invalid" >&2 + exit 2 + } + [ "$record_count" -le 1 ] && [ "$artifact_count" -le 1 ] || { + echo "error: candidate evidence is ambiguous" >&2 + exit 2 + } + if { [ "$record_count" -ne 0 ] || [ "$artifact_count" -ne 0 ]; } && \ + [ "$attempt" != attempted ]; then + echo "error: candidate evidence is not bound to its build attempt" >&2 + exit 2 + fi done < "$state_file" [ "$entries" -gt 0 ] || { @@ -66,10 +68,11 @@ if [ "$attempted_count" -eq 0 ]; then printf 'build\n' exit 0 fi -case "$build_evidence" in - exact-record) printf 'reuse-record\t%s\n' "$build_owner" ;; - exact-build) printf 'reuse-build\t%s\n' "$build_owner" ;; - absent) +case "$build_record_count:$build_artifact_count" in + 1:1) printf 'reuse-record-pair\t%s\n' "$build_owner" ;; + 1:0) printf 'reuse-record\t%s\n' "$build_owner" ;; + 0:1) printf 'reuse-build\t%s\n' "$build_owner" ;; + 0:0) echo "error: candidate build attempt has no durable exact evidence; refusing rebuild" >&2 exit 2 ;; diff --git a/bin/agent-os-runtime-bound.sh b/bin/agent-os-runtime-bound.sh index 8b7eaf030..e9a0c81da 100755 --- a/bin/agent-os-runtime-bound.sh +++ b/bin/agent-os-runtime-bound.sh @@ -2,6 +2,20 @@ export AGENT_OS_BOUND_PATH AGENT_OS_BOUND_TEST_FALLBACK_ACTIVE AGENT_OS_BOUND_TEST_FALLBACK_ACTIVE=false +AGENT_OS_BOUND_DIR_HANDLES=() + +agent_os_register_bound_dir() { + AGENT_OS_BOUND_DIR_HANDLES+=("$1") +} + +agent_os_bound_dir_is_open() { + local candidate=$1 bound + [ -d "$candidate" ] || return 1 + for bound in "${AGENT_OS_BOUND_DIR_HANDLES[@]}"; do + [ "$candidate" = "$bound" ] && return 0 + done + return 1 +} agent_os_open_bound_dir() { local dir=$1 fd fd_root @@ -14,11 +28,13 @@ agent_os_open_bound_dir() { return 1 } AGENT_OS_BOUND_PATH=$fd_root/$fd + agent_os_register_bound_dir "$AGENT_OS_BOUND_PATH" return 0 elif [ "${AGENT_OS_TEST_BOUND_PATHS:-}" = true ]; then exec {fd}<&- AGENT_OS_BOUND_PATH=$(cd "$dir" && pwd -P) AGENT_OS_BOUND_TEST_FALLBACK_ACTIVE=true + agent_os_register_bound_dir "$AGENT_OS_BOUND_PATH" return 0 else exec {fd}<&- @@ -34,6 +50,9 @@ agent_os_bound_dir_matches() { agent_os_bound_git() { local git_dir=$1 common_dir=$2 work_tree=$3 shift 3 + agent_os_bound_dir_is_open "$git_dir" || return 1 + agent_os_bound_dir_is_open "$common_dir" || return 1 + agent_os_bound_dir_is_open "$work_tree" || return 1 trusted_git --agent-os-common-dir "$common_dir" \ --git-dir="$git_dir" --work-tree="$work_tree" "$@" } diff --git a/bin/agent-os-runtime-secondmates.sh b/bin/agent-os-runtime-secondmates.sh index 196665c90..d8ffa5503 100755 --- a/bin/agent-os-runtime-secondmates.sh +++ b/bin/agent-os-runtime-secondmates.sh @@ -30,7 +30,7 @@ trusted_git() { if [ "${1:-}" = --agent-os-common-dir ]; then common_dir=${2:?bound common Git directory is required} shift 2 - [ -d "$common_dir" ] && [ ! -L "$common_dir" ] || return 1 + agent_os_bound_dir_is_open "$common_dir" || return 1 fi if [ -n "$common_dir" ]; then env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ @@ -110,6 +110,7 @@ acquire_primary_ownership_lock() { validate_treehouse_lease() { local id=$1 state_home=$2 home=$3 state state_json entry_count index entry_home canonical_count + VALIDATED_TREEHOUSE_NAME= state=$TREEHOUSE_POOL/treehouse-state.json [ "$TREEHOUSE_LOCK_FILE" -ef "$TREEHOUSE_LOCK_HANDLE" ] || return 1 [ -r "$state" ] && [ -f "$state" ] && [ ! -L "$state" ] || return 1 @@ -122,7 +123,8 @@ validate_treehouse_lease() { ([.worktrees[] | select( .path == $home and .leased == true and .lease_holder == $holder and ((.destroying // false) == false) and ((.owner_pid // 0) == 0) and - ((.owner_started_at // 0) == 0) and (.name | type == "string" and length > 0) and + ((.owner_started_at // 0) == 0) and + (.name | type == "string" and test("^(?!\\.{1,2}$)[^/\\t\\r\\n]+$")) and (.leased_at | type == "string" and length > 0) )] | length == 1)' <<< "$state_json" >/dev/null || return 1 entry_count=$(jq -er '.worktrees | length' <<< "$state_json") || return 1 @@ -134,7 +136,13 @@ validate_treehouse_lease() { [ "$entry_home" != "$home" ] || canonical_count=$((canonical_count + 1)) index=$((index + 1)) done - [ "$canonical_count" -eq 1 ] + [ "$canonical_count" -eq 1 ] || return 1 + VALIDATED_TREEHOUSE_NAME=$(jq -er --arg home "$state_home" --arg holder "$id" ' + [.worktrees[] | select( + .path == $home and .leased == true and .lease_holder == $holder and + ((.destroying // false) == false) and ((.owner_pid // 0) == 0) and + ((.owner_started_at // 0) == 0) + )] | select(length == 1) | .[0].name' <<< "$state_json") } resolve_git_metadata() { @@ -207,7 +215,8 @@ validate_git_binding() { validate_bound_git_binding() { local root=$1 bound_home=$2 bound_git=$3 bound_common=$4 bound_source=$5 linked=$6 source_key=$7 - local pointer pointer_path backlink backlink_path common common_path matches source_key policy_commit policy_sha + local expected_treehouse_name=$8 pointer pointer_path backlink backlink_path common common_path + local expected_git policy_commit policy_sha [ "$PRIMARY_OWNERSHIP_LOCK_FILE" -ef "$PRIMARY_OWNERSHIP_LOCK_HANDLE" ] || return 1 if [ "$linked" = false ]; then [ -d "$bound_home/.git" ] && [ ! -L "$bound_home/.git" ] || return 1 @@ -241,12 +250,10 @@ validate_bound_git_binding() { *) common_path="$bound_git/$common" ;; esac [ "$common_path" -ef "$bound_common" ] || return 1 - matches=0 - for pointer_path in "$bound_common"/worktrees/*; do - [ -d "$pointer_path" ] || continue - [ "$pointer_path" -ef "$bound_git" ] && matches=$((matches + 1)) - done - [ "$matches" -eq 1 ] || return 1 + case "$expected_treehouse_name" in ''|.|..|*/*|*$'\t'*|*$'\r'*|*$'\n'*) return 1 ;; esac + expected_git=$bound_common/worktrees/$expected_treehouse_name + [ -d "$expected_git" ] && [ ! -L "$expected_git" ] || return 1 + [ "$expected_git" -ef "$bound_git" ] || return 1 [ "$bound_source/.git" -ef "$bound_common" ] || return 1 validate_policy "$bound_common/agent-os-runtime-source" || return 1 policy_commit=$(sed -n 's/^commit=//p' "$bound_common/agent-os-runtime-source") @@ -437,6 +444,7 @@ acquire_primary_ownership_lock || { validated_ids=() validated_homes=() treehouse_homes=() +treehouse_names=() ownership_modes=() git_dirs=() common_dirs=() @@ -453,6 +461,7 @@ for index in "${!homes[@]}"; do home=${homes[$index]} proof=${proofs[$index]} treehouse_home=$home + treehouse_name= validate_secondmate_home "$id" "$home" || { echo "error: invalid secondmate home for $id: $VALIDATION_ERROR" >&2 exit 2 @@ -507,11 +516,18 @@ for index in "${!homes[@]}"; do echo "error: secondmate Treehouse lease holder is not exact: $id" >&2 exit 2 } + treehouse_name=$VALIDATED_TREEHOUSE_NAME fi resolve_git_metadata "$home" || { echo "error: secondmate Git metadata is invalid: $home" >&2; exit 2; } original_git_dir=$RESOLVED_GIT_DIR original_common_dir=$RESOLVED_COMMON_DIR if [ "$ownership_mode" = treehouse ]; then + expected_git_dir=$original_common_dir/worktrees/$treehouse_name + [ -d "$expected_git_dir" ] && [ ! -L "$expected_git_dir" ] && \ + [ "$expected_git_dir" -ef "$original_git_dir" ] || { + echo "error: secondmate Treehouse lease is not bound to its Git metadata: $id" >&2 + exit 2 + } original_source_dir=$(cd "$original_common_dir/.." && pwd -P) || exit 2 runtime_root=$(cd "$FM_HOME/runtime-sources" && pwd -P) || exit 2 case "$original_source_dir" in "$runtime_root"/*) ;; *) exit 2 ;; esac @@ -556,7 +572,7 @@ for index in "${!homes[@]}"; do fi validate_bound_git_binding "$home" "$bound_home" "$bound_git_dir" \ "$bound_common_dir" "$bound_source_dir" "$([ "$ownership_mode" = treehouse ] && printf true || printf false)" \ - "$source_key" || { + "$source_key" "$treehouse_name" || { echo "error: bound secondmate Git ownership is invalid: $home" >&2 exit 2 } @@ -625,6 +641,7 @@ for index in "${!homes[@]}"; do validated_ids+=("$id") validated_homes+=("$home") treehouse_homes+=("$treehouse_home") + treehouse_names+=("$treehouse_name") ownership_modes+=("$ownership_mode") git_dirs+=("$original_git_dir") common_dirs+=("$original_common_dir") @@ -640,8 +657,10 @@ done assert_secondmate_binding() { local id=$1 ownership_mode=$2 treehouse_home=$3 home=$4 git_dir=$5 common_dir=$6 source_dir=$7 local bound_home=$8 bound_config=$9 bound_git=${10} bound_common=${11} bound_source=${12} + local expected_treehouse_name=${13} if [ "$ownership_mode" = treehouse ]; then validate_treehouse_lease "$id" "$treehouse_home" "$home" || return 1 + [ "$VALIDATED_TREEHOUSE_NAME" = "$expected_treehouse_name" ] || return 1 else validate_primary_standalone_proof "$id" "$home" || return 1 fi @@ -655,12 +674,12 @@ assert_secondmate_binding() { [ "$RESOLVED_COMMON_DIR" -ef "$bound_common" ] || return 1 validate_bound_git_binding "$home" "$bound_home" "$bound_git" "$bound_common" \ "$bound_source" "$([ "$ownership_mode" = treehouse ] && printf true || printf false)" \ - "${source_dir##*/}" + "${source_dir##*/}" "$expected_treehouse_name" } runtime_test_barrier() { local phase=$1 - [ "${AGENT_OS_BOUND_TEST_FALLBACK_ACTIVE:-}" = true ] || return 0 + [ "${AGENT_OS_TEST_BARRIERS:-}" = true ] || return 0 [ "${AGENT_OS_TEST_BARRIER_PHASE:-}" = "$phase" ] || return 0 [ -n "${AGENT_OS_TEST_BARRIER_READY:-}" ] && [ -n "${AGENT_OS_TEST_BARRIER_RELEASE:-}" ] || return 1 printf '%s\n' "$$" > "$AGENT_OS_TEST_BARRIER_READY" @@ -676,6 +695,7 @@ for index in "${!validated_homes[@]}"; do id=${validated_ids[$index]} ownership_mode=${ownership_modes[$index]} treehouse_home=${treehouse_homes[$index]} + treehouse_name=${treehouse_names[$index]} home=${validated_homes[$index]} git_dir=${git_dirs[$index]} common_dir=${common_dirs[$index]} @@ -688,7 +708,7 @@ for index in "${!validated_homes[@]}"; do recovery_policy=${pending_policies[$index]} assert_secondmate_binding "$id" "$ownership_mode" "$treehouse_home" "$home" "$git_dir" \ "$common_dir" "$source_dir" "$bound_home" "$bound_config" "$bound_git_dir" \ - "$bound_common_dir" "$bound_source_dir" || { + "$bound_common_dir" "$bound_source_dir" "$treehouse_name" || { echo "error: secondmate ownership changed before mutation: $home" >&2 exit 2 } @@ -707,7 +727,7 @@ for index in "${!validated_homes[@]}"; do fi assert_secondmate_binding "$id" "$ownership_mode" "$treehouse_home" "$home" "$git_dir" \ "$common_dir" "$source_dir" "$bound_home" "$bound_config" "$bound_git_dir" \ - "$bound_common_dir" "$bound_source_dir" || exit 2 + "$bound_common_dir" "$bound_source_dir" "$treehouse_name" || exit 2 runtime_test_barrier policy pending_policy="$bound_config/agent-os-source-policy.pending" pending_tmp="$bound_config/.agent-os-source-policy.pending.$$" @@ -719,14 +739,14 @@ for index in "${!validated_homes[@]}"; do mv "$required_tmp" "$required_policy" assert_secondmate_binding "$id" "$ownership_mode" "$treehouse_home" "$home" "$git_dir" \ "$common_dir" "$source_dir" "$bound_home" "$bound_config" "$bound_git_dir" \ - "$bound_common_dir" "$bound_source_dir" || exit 2 + "$bound_common_dir" "$bound_source_dir" "$treehouse_name" || exit 2 runtime_test_barrier checkout agent_os_bound_git "$bound_git_dir" "$bound_common_dir" "$bound_home" checkout --detach "$SOURCE_COMMIT" runtime_test_barrier remote agent_os_bound_git "$bound_git_dir" "$bound_common_dir" "$bound_home" remote set-url origin "$SOURCE_ORIGIN" assert_secondmate_binding "$id" "$ownership_mode" "$treehouse_home" "$home" "$git_dir" \ "$common_dir" "$source_dir" "$bound_home" "$bound_config" "$bound_git_dir" \ - "$bound_common_dir" "$bound_source_dir" || exit 2 + "$bound_common_dir" "$bound_source_dir" "$treehouse_name" || exit 2 runtime_test_barrier marker home_marker_tmp="$bound_config/.agent-os-source-policy.$$" printf '%s\n' "$marker" > "$home_marker_tmp" @@ -740,7 +760,7 @@ for index in "${!validated_homes[@]}"; do [ -z "$(agent_os_bound_git "$bound_git_dir" "$bound_common_dir" "$bound_home" status --porcelain --untracked-files=all)" ] || exit 2 assert_secondmate_binding "$id" "$ownership_mode" "$treehouse_home" "$home" "$git_dir" \ "$common_dir" "$source_dir" "$bound_home" "$bound_config" "$bound_git_dir" \ - "$bound_common_dir" "$bound_source_dir" || exit 2 + "$bound_common_dir" "$bound_source_dir" "$treehouse_name" || exit 2 rm "$pending_policy" printf 'selected: %s\n' "$home" done diff --git a/tests/agent-os-candidate-state.test.sh b/tests/agent-os-candidate-state.test.sh index a6c460346..809c1556c 100755 --- a/tests/agent-os-candidate-state.test.sh +++ b/tests/agent-os-candidate-state.test.sh @@ -29,66 +29,78 @@ expect_refusal() { fi } -write_chain "$OWNER_A\t$CLAIM_A\tabsent\tabsent" +write_chain "$OWNER_A\t$CLAIM_A\tabsent\t0\t0" expect_decision build write_chain \ - "$OWNER_A\t$CLAIM_A\tattempted\texact-build" \ - "$OWNER_B\t$CLAIM_B\tabsent\tabsent" + "$OWNER_A\t$CLAIM_A\tattempted\t0\t1" \ + "$OWNER_B\t$CLAIM_B\tabsent\t0\t0" expect_decision $'reuse-build\t'"$OWNER_A" write_chain \ - "$OWNER_A\t$CLAIM_A\tattempted\texact-record" \ - "$OWNER_B\t$CLAIM_B\tabsent\tabsent" \ - "$OWNER_C\t$CLAIM_C\tabsent\tabsent" + "$OWNER_A\t$CLAIM_A\tattempted\t1\t0" \ + "$OWNER_B\t$CLAIM_B\tabsent\t0\t0" \ + "$OWNER_C\t$CLAIM_C\tabsent\t0\t0" expect_decision $'reuse-record\t'"$OWNER_A" +write_chain \ + "$OWNER_A\t$CLAIM_A\tattempted\t1\t1" \ + "$OWNER_B\t$CLAIM_B\tabsent\t0\t0" +expect_decision $'reuse-record-pair\t'"$OWNER_A" + builds=0 -write_chain "$OWNER_A\t$CLAIM_A\tabsent\tabsent" +write_chain "$OWNER_A\t$CLAIM_A\tabsent\t0\t0" decision=$("$STATE" "$TMP/chain.tsv") || fail "initial claim did not authorize its build" [ "$decision" = build ] && builds=$((builds + 1)) write_chain \ - "$OWNER_A\t$CLAIM_A\tattempted\texact-build" \ - "$OWNER_B\t$CLAIM_B\tabsent\tabsent" + "$OWNER_A\t$CLAIM_A\tattempted\t0\t1" \ + "$OWNER_B\t$CLAIM_B\tabsent\t0\t0" decision=$("$STATE" "$TMP/chain.tsv") || fail "handoff did not reuse ancestor evidence" [ "$decision" = build ] && builds=$((builds + 1)) write_chain \ - "$OWNER_A\t$CLAIM_A\tattempted\texact-build" \ - "$OWNER_B\t$CLAIM_B\tabsent\tabsent" \ - "$OWNER_C\t$CLAIM_C\tabsent\tabsent" -decision=$("$STATE" "$TMP/chain.tsv") || fail "second handoff did not reuse ancestor evidence" + "$OWNER_A\t$CLAIM_A\tattempted\t1\t1" \ + "$OWNER_B\t$CLAIM_B\tabsent\t0\t0" \ + "$OWNER_C\t$CLAIM_C\tabsent\t0\t0" +decision=$("$STATE" "$TMP/chain.tsv") || fail "second handoff did not prefer paired ancestor evidence" [ "$decision" = build ] && builds=$((builds + 1)) +[ "$decision" = $'reuse-record-pair\t'"$OWNER_A" ] || \ + fail "second handoff did not reuse paired ancestor evidence" [ "$builds" -eq 1 ] || fail "claim-chain classifier authorized $builds builds" write_chain \ - "$OWNER_A\t$CLAIM_A\tattempted\tabsent" \ - "$OWNER_B\t$CLAIM_B\tabsent\tabsent" + "$OWNER_A\t$CLAIM_A\tattempted\t0\t0" \ + "$OWNER_B\t$CLAIM_B\tabsent\t0\t0" expect_refusal write_chain \ - "$OWNER_A\t$CLAIM_A\tattempted\texact-build" \ - "$OWNER_B\t$CLAIM_B\tattempted\texact-build" + "$OWNER_A\t$CLAIM_A\tattempted\t0\t1" \ + "$OWNER_B\t$CLAIM_B\tattempted\t0\t1" expect_refusal write_chain \ - "$OWNER_A\t$CLAIM_A\tabsent\texact-build" \ - "$OWNER_B\t$CLAIM_B\tabsent\tabsent" + "$OWNER_A\t$CLAIM_A\tabsent\t0\t1" \ + "$OWNER_B\t$CLAIM_B\tabsent\t0\t0" expect_refusal for state in corrupt mismatched unreadable metadata-read-error partial ambiguous; do - write_chain "$OWNER_A\t$CLAIM_A\t$state\tabsent" + write_chain "$OWNER_A\t$CLAIM_A\t$state\t0\t0" + expect_refusal + write_chain "$OWNER_A\t$CLAIM_A\tattempted\t$state\t0" expect_refusal - write_chain "$OWNER_A\t$CLAIM_A\tattempted\t$state" +done + +for counts in '2\t0' '0\t2' '2\t2'; do + write_chain "$OWNER_A\t$CLAIM_A\tattempted\t$counts" expect_refusal done write_chain \ - "$OWNER_A\t$CLAIM_A\tabsent\tabsent" \ - "$OWNER_A\t$CLAIM_B\tabsent\tabsent" + "$OWNER_A\t$CLAIM_A\tabsent\t0\t0" \ + "$OWNER_A\t$CLAIM_B\tabsent\t0\t0" expect_refusal write_chain \ - "$OWNER_A\t$CLAIM_A\tabsent\tabsent" \ - "$OWNER_B\t$CLAIM_A\tabsent\tabsent" + "$OWNER_A\t$CLAIM_A\tabsent\t0\t0" \ + "$OWNER_B\t$CLAIM_A\tabsent\t0\t0" expect_refusal pass "candidate claim-chain classifier permits one build and ancestor reuse only" diff --git a/tests/agent-os-runtime-secondmates.test.sh b/tests/agent-os-runtime-secondmates.test.sh index 31fec6a3a..befacaafc 100755 --- a/tests/agent-os-runtime-secondmates.test.sh +++ b/tests/agent-os-runtime-secondmates.test.sh @@ -359,6 +359,16 @@ jq '.worktrees += [(.worktrees[] | select(.lease_holder == "linked") | .path += mv "$TEST_ROOT/state-next.json" "$TREEHOUSE_POOL/treehouse-state.json" expect_rejection_without_mutation "ambiguous planted Treehouse holder evidence" +git -C "$LEASE" worktree add -q --detach "$TMP/leased-decoy" "$A_COMMIT" +write_treehouse_state +jq '(.worktrees[] | select(.lease_holder == "linked") | .name) = "leased-decoy"' \ + "$TREEHOUSE_POOL/treehouse-state.json" > "$TEST_ROOT/state-next.json" +mv "$TEST_ROOT/state-next.json" "$TREEHOUSE_POOL/treehouse-state.json" +expect_rejection_without_mutation "Treehouse lease bound to another same-common worktree" +[ "$(git -C "$TMP/leased-decoy" rev-parse HEAD)" = "$A_COMMIT" ] || \ + fail "same-common decoy worktree was mutated" +git -C "$LEASE" worktree remove "$TMP/leased-decoy" + printf '{unreadable\n' > "$TREEHOUSE_POOL/treehouse-state.json" expect_rejection_without_mutation "unreadable Treehouse lease evidence" PRESERVE_TREEHOUSE_STATE=false @@ -370,51 +380,59 @@ git -C "$TMP/toctou-foreign" checkout -q --detach "$A_COMMIT" linked_gitdir=$(git -C "$TMP/linked" rev-parse --absolute-git-dir) linked_commondir=$(cat "$linked_gitdir/commondir") linked_pointer=$(cat "$TMP/linked/.git") -foreign_before=$(find "$TMP/toctou-foreign/.git" -type f -print0 | sort -z | \ - xargs -0 shasum -a 256 | shasum -a 256 | awk '{print $1}') -fleet_before=$(fleet_mutation_signature) -write_treehouse_state -PRESERVE_TREEHOUSE_STATE=true -AGENT_OS_TEST_BARRIER_PHASE=fetch -AGENT_OS_TEST_BARRIER_READY="$TEST_ROOT/toctou-ready" -AGENT_OS_TEST_BARRIER_RELEASE="$TEST_ROOT/toctou-release" -export AGENT_OS_TEST_BARRIER_PHASE AGENT_OS_TEST_BARRIER_READY AGENT_OS_TEST_BARRIER_RELEASE -rm -f "$AGENT_OS_TEST_BARRIER_READY" "$AGENT_OS_TEST_BARRIER_RELEASE" -run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" \ - > "$TEST_ROOT/toctou-runtime.out" 2>&1 & -runtime_pid=$! -for _ in $(seq 1 500); do - [ -f "$AGENT_OS_TEST_BARRIER_READY" ] && break - sleep 0.01 -done -[ -f "$AGENT_OS_TEST_BARRIER_READY" ] || { - kill "$runtime_pid" 2>/dev/null || true - fail "runtime TOCTOU barrier was not reached" -} -if AGENT_OS_TEST_BARRIER_PHASE= run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" \ - >/dev/null 2>&1; then +for barrier_phase in fetch policy checkout remote marker; do + foreign_before=$(find "$TMP/toctou-foreign/.git" -type f -print0 | sort -z | \ + xargs -0 shasum -a 256 | shasum -a 256 | awk '{print $1}') + fleet_before=$(fleet_mutation_signature) + write_treehouse_state + PRESERVE_TREEHOUSE_STATE=true + AGENT_OS_TEST_BARRIERS=true + AGENT_OS_TEST_BARRIER_PHASE=$barrier_phase + AGENT_OS_TEST_BARRIER_READY="$TEST_ROOT/toctou-$barrier_phase-ready" + AGENT_OS_TEST_BARRIER_RELEASE="$TEST_ROOT/toctou-$barrier_phase-release" + export AGENT_OS_TEST_BARRIERS AGENT_OS_TEST_BARRIER_PHASE \ + AGENT_OS_TEST_BARRIER_READY AGENT_OS_TEST_BARRIER_RELEASE + rm -f "$AGENT_OS_TEST_BARRIER_READY" "$AGENT_OS_TEST_BARRIER_RELEASE" + run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" \ + > "$TEST_ROOT/toctou-$barrier_phase-runtime.out" 2>&1 & + runtime_pid=$! + for _ in $(seq 1 500); do + [ -f "$AGENT_OS_TEST_BARRIER_READY" ] && break + sleep 0.01 + done + [ -f "$AGENT_OS_TEST_BARRIER_READY" ] || { + kill "$runtime_pid" 2>/dev/null || true + fail "runtime TOCTOU $barrier_phase barrier was not reached" + } + if [ "$barrier_phase" = fetch ] && \ + AGENT_OS_TEST_BARRIERS='' AGENT_OS_TEST_BARRIER_PHASE='' \ + run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then + touch "$AGENT_OS_TEST_BARRIER_RELEASE" + wait "$runtime_pid" 2>/dev/null || true + fail "secondmate ownership lock admitted a concurrent runtime mutation" + fi + printf 'gitdir: %s/toctou-foreign/.git\n' "$TMP" > "$TMP/linked/.git" + printf '%s\n' "$TMP/toctou-foreign/.git" > "$linked_gitdir/commondir" touch "$AGENT_OS_TEST_BARRIER_RELEASE" - wait "$runtime_pid" 2>/dev/null || true - fail "secondmate ownership lock admitted a concurrent runtime mutation" -fi -printf 'gitdir: %s/toctou-foreign/.git\n' "$TMP" > "$TMP/linked/.git" -printf '%s\n' "$TMP/toctou-foreign/.git" > "$linked_gitdir/commondir" -touch "$AGENT_OS_TEST_BARRIER_RELEASE" -if wait "$runtime_pid"; then - fail "runtime accepted a Git pointer and common-directory swap" -fi -printf '%s\n' "$linked_pointer" > "$TMP/linked/.git" -printf '%s\n' "$linked_commondir" > "$linked_gitdir/commondir" -unset AGENT_OS_TEST_BARRIER_PHASE AGENT_OS_TEST_BARRIER_READY AGENT_OS_TEST_BARRIER_RELEASE -foreign_after=$(find "$TMP/toctou-foreign/.git" -type f -print0 | sort -z | \ - xargs -0 shasum -a 256 | shasum -a 256 | awk '{print $1}') -[ "$foreign_after" = "$foreign_before" ] || \ - fail "runtime TOCTOU redirected Git mutation into a foreign repository" -[ "$(fleet_mutation_signature)" = "$fleet_before" ] || \ - fail "runtime TOCTOU changed secondmate source, remote, marker, or policy state" -PRESERVE_TREEHOUSE_STATE=false + if wait "$runtime_pid"; then + fail "runtime accepted a Git pointer and common-directory swap at $barrier_phase" + fi + printf '%s\n' "$linked_pointer" > "$TMP/linked/.git" + printf '%s\n' "$linked_commondir" > "$linked_gitdir/commondir" + foreign_after=$(find "$TMP/toctou-foreign/.git" -type f -print0 | sort -z | \ + xargs -0 shasum -a 256 | shasum -a 256 | awk '{print $1}') + [ "$foreign_after" = "$foreign_before" ] || \ + fail "runtime TOCTOU $barrier_phase redirected mutation into a foreign repository" + unset AGENT_OS_TEST_BARRIERS AGENT_OS_TEST_BARRIER_PHASE \ + AGENT_OS_TEST_BARRIER_READY AGENT_OS_TEST_BARRIER_RELEASE + PRESERVE_TREEHOUSE_STATE=false + run_sync "$TMP/primary-a" "$A_COMMIT" "$A_TREE" "$A_SHA" >/dev/null || \ + fail "runtime TOCTOU $barrier_phase state could not be reconciled" + [ "$(fleet_mutation_signature)" = "$fleet_before" ] || \ + fail "runtime TOCTOU $barrier_phase reconciliation changed persistent state" +done write_treehouse_state -pass "runtime lock and bound common directory defeat live Git pointer swaps" +pass "runtime lock and bound directories defeat pointer swaps at every mutation phase" printf 'tampered\n' >> "$TMP/standalone/AGENTS.md" if run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then From df550621fa006dc5bcbf26a61006efa90e897efe Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 22:45:03 +0200 Subject: [PATCH 50/56] no-mistakes(review): Harden candidate recovery and Treehouse lease locking --- .github/workflows/agent-os-image.yml | 17 +++++-- bin/agent-os-candidate-state.sh | 41 +++++++++++---- bin/agent-os-runtime-secondmates.sh | 59 +++++++++++----------- tests/agent-os-candidate-state.test.sh | 24 +++++++-- tests/agent-os-container.test.sh | 6 +++ tests/agent-os-runtime-secondmates.test.sh | 33 ++++++++---- 6 files changed, 121 insertions(+), 59 deletions(-) diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index 3061588bc..f4a0faab2 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -472,16 +472,25 @@ jobs: "$BUILD_ATTEMPT_STATE" "$record_count" "$build_count" >> "$chain_state" done < "$CLAIM_CHAIN_FILE" decision=$(bin/agent-os-candidate-state.sh "$chain_state") - IFS=$'\t' read -r candidate_action artifact_owner <&2 @@ -49,10 +48,20 @@ while IFS=$'\t' read -r owner claim attempt record_count artifact_count extra || echo "error: candidate evidence is ambiguous" >&2 exit 2 } - if { [ "$record_count" -ne 0 ] || [ "$artifact_count" -ne 0 ]; } && \ - [ "$attempt" != attempted ]; then - echo "error: candidate evidence is not bound to its build attempt" >&2 - exit 2 + if [ "$artifact_count" -eq 1 ]; then + [ "$attempt" = attempted ] || { + echo "error: candidate build evidence is not bound to its build attempt" >&2 + exit 2 + } + build_total=$((build_total + 1)) + fi + if [ "$record_count" -eq 1 ]; then + [ "$attempted_count" -eq 1 ] || { + echo "error: candidate record evidence precedes its build attempt" >&2 + exit 2 + } + record_total=$((record_total + 1)) + record_owner=$owner fi done < "$state_file" @@ -64,13 +73,23 @@ done < "$state_file" echo "error: candidate claim chain contains multiple build attempts" >&2 exit 2 } +[ "$record_total" -le 1 ] && [ "$build_total" -le 1 ] || { + echo "error: candidate claim chain contains conflicting durable evidence" >&2 + exit 2 +} if [ "$attempted_count" -eq 0 ]; then printf 'build\n' exit 0 fi -case "$build_record_count:$build_artifact_count" in - 1:1) printf 'reuse-record-pair\t%s\n' "$build_owner" ;; - 1:0) printf 'reuse-record\t%s\n' "$build_owner" ;; +case "$record_total:$build_total" in + 1:1) printf 'reuse-record-pair\t%s\t%s\n' "$record_owner" "$build_owner" ;; + 1:0) + [ "$record_owner" = "$build_owner" ] || { + echo "error: recovered candidate record lacks exact build evidence" >&2 + exit 2 + } + printf 'reuse-record\t%s\n' "$record_owner" + ;; 0:1) printf 'reuse-build\t%s\n' "$build_owner" ;; 0:0) echo "error: candidate build attempt has no durable exact evidence; refusing rebuild" >&2 diff --git a/bin/agent-os-runtime-secondmates.sh b/bin/agent-os-runtime-secondmates.sh index d8ffa5503..1542eec88 100755 --- a/bin/agent-os-runtime-secondmates.sh +++ b/bin/agent-os-runtime-secondmates.sh @@ -109,8 +109,8 @@ acquire_primary_ownership_lock() { } validate_treehouse_lease() { - local id=$1 state_home=$2 home=$3 state state_json entry_count index entry_home canonical_count - VALIDATED_TREEHOUSE_NAME= + local id=$1 state_home=$2 home=$3 state state_json entry_count index entry_home canonical_count validated_home + VALIDATED_TREEHOUSE_HOME= state=$TREEHOUSE_POOL/treehouse-state.json [ "$TREEHOUSE_LOCK_FILE" -ef "$TREEHOUSE_LOCK_HANDLE" ] || return 1 [ -r "$state" ] && [ -f "$state" ] && [ ! -L "$state" ] || return 1 @@ -137,12 +137,13 @@ validate_treehouse_lease() { index=$((index + 1)) done [ "$canonical_count" -eq 1 ] || return 1 - VALIDATED_TREEHOUSE_NAME=$(jq -er --arg home "$state_home" --arg holder "$id" ' + validated_home=$(jq -er --arg home "$state_home" --arg holder "$id" ' [.worktrees[] | select( .path == $home and .leased == true and .lease_holder == $holder and ((.destroying // false) == false) and ((.owner_pid // 0) == 0) and ((.owner_started_at // 0) == 0) - )] | select(length == 1) | .[0].name' <<< "$state_json") + )] | select(length == 1) | .[0].path' <<< "$state_json") || return 1 + VALIDATED_TREEHOUSE_HOME=$(cd "$validated_home" 2>/dev/null && pwd -P) || return 1 } resolve_git_metadata() { @@ -215,8 +216,8 @@ validate_git_binding() { validate_bound_git_binding() { local root=$1 bound_home=$2 bound_git=$3 bound_common=$4 bound_source=$5 linked=$6 source_key=$7 - local expected_treehouse_name=$8 pointer pointer_path backlink backlink_path common common_path - local expected_git policy_commit policy_sha + local expected_treehouse_home=$8 pointer pointer_path backlink backlink_path common common_path + local policy_commit policy_sha [ "$PRIMARY_OWNERSHIP_LOCK_FILE" -ef "$PRIMARY_OWNERSHIP_LOCK_HANDLE" ] || return 1 if [ "$linked" = false ]; then [ -d "$bound_home/.git" ] && [ ! -L "$bound_home/.git" ] || return 1 @@ -250,10 +251,8 @@ validate_bound_git_binding() { *) common_path="$bound_git/$common" ;; esac [ "$common_path" -ef "$bound_common" ] || return 1 - case "$expected_treehouse_name" in ''|.|..|*/*|*$'\t'*|*$'\r'*|*$'\n'*) return 1 ;; esac - expected_git=$bound_common/worktrees/$expected_treehouse_name - [ -d "$expected_git" ] && [ ! -L "$expected_git" ] || return 1 - [ "$expected_git" -ef "$bound_git" ] || return 1 + [ -d "$expected_treehouse_home" ] && [ ! -L "$expected_treehouse_home" ] || return 1 + [ "$expected_treehouse_home" -ef "$bound_home" ] || return 1 [ "$bound_source/.git" -ef "$bound_common" ] || return 1 validate_policy "$bound_common/agent-os-runtime-source" || return 1 policy_commit=$(sed -n 's/^commit=//p' "$bound_common/agent-os-runtime-source") @@ -444,7 +443,7 @@ acquire_primary_ownership_lock || { validated_ids=() validated_homes=() treehouse_homes=() -treehouse_names=() +treehouse_bound_homes=() ownership_modes=() git_dirs=() common_dirs=() @@ -461,7 +460,7 @@ for index in "${!homes[@]}"; do home=${homes[$index]} proof=${proofs[$index]} treehouse_home=$home - treehouse_name= + treehouse_bound_home= validate_secondmate_home "$id" "$home" || { echo "error: invalid secondmate home for $id: $VALIDATION_ERROR" >&2 exit 2 @@ -516,18 +515,18 @@ for index in "${!homes[@]}"; do echo "error: secondmate Treehouse lease holder is not exact: $id" >&2 exit 2 } - treehouse_name=$VALIDATED_TREEHOUSE_NAME + treehouse_bound_home=$VALIDATED_TREEHOUSE_HOME fi resolve_git_metadata "$home" || { echo "error: secondmate Git metadata is invalid: $home" >&2; exit 2; } original_git_dir=$RESOLVED_GIT_DIR original_common_dir=$RESOLVED_COMMON_DIR if [ "$ownership_mode" = treehouse ]; then - expected_git_dir=$original_common_dir/worktrees/$treehouse_name - [ -d "$expected_git_dir" ] && [ ! -L "$expected_git_dir" ] && \ - [ "$expected_git_dir" -ef "$original_git_dir" ] || { - echo "error: secondmate Treehouse lease is not bound to its Git metadata: $id" >&2 - exit 2 - } + case "$original_git_dir" in + "$original_common_dir"/worktrees/*) ;; + *) echo "error: secondmate Git metadata is not a linked worktree: $home" >&2; exit 2 ;; + esac + git_relative=${original_git_dir#"$original_common_dir"/worktrees/} + case "$git_relative" in ''|*/*) echo "error: secondmate Git metadata path is invalid: $home" >&2; exit 2 ;; esac original_source_dir=$(cd "$original_common_dir/.." && pwd -P) || exit 2 runtime_root=$(cd "$FM_HOME/runtime-sources" && pwd -P) || exit 2 case "$original_source_dir" in "$runtime_root"/*) ;; *) exit 2 ;; esac @@ -572,7 +571,7 @@ for index in "${!homes[@]}"; do fi validate_bound_git_binding "$home" "$bound_home" "$bound_git_dir" \ "$bound_common_dir" "$bound_source_dir" "$([ "$ownership_mode" = treehouse ] && printf true || printf false)" \ - "$source_key" "$treehouse_name" || { + "$source_key" "$treehouse_bound_home" || { echo "error: bound secondmate Git ownership is invalid: $home" >&2 exit 2 } @@ -641,7 +640,7 @@ for index in "${!homes[@]}"; do validated_ids+=("$id") validated_homes+=("$home") treehouse_homes+=("$treehouse_home") - treehouse_names+=("$treehouse_name") + treehouse_bound_homes+=("$treehouse_bound_home") ownership_modes+=("$ownership_mode") git_dirs+=("$original_git_dir") common_dirs+=("$original_common_dir") @@ -657,10 +656,10 @@ done assert_secondmate_binding() { local id=$1 ownership_mode=$2 treehouse_home=$3 home=$4 git_dir=$5 common_dir=$6 source_dir=$7 local bound_home=$8 bound_config=$9 bound_git=${10} bound_common=${11} bound_source=${12} - local expected_treehouse_name=${13} + local expected_treehouse_home=${13} if [ "$ownership_mode" = treehouse ]; then validate_treehouse_lease "$id" "$treehouse_home" "$home" || return 1 - [ "$VALIDATED_TREEHOUSE_NAME" = "$expected_treehouse_name" ] || return 1 + [ "$VALIDATED_TREEHOUSE_HOME" = "$expected_treehouse_home" ] || return 1 else validate_primary_standalone_proof "$id" "$home" || return 1 fi @@ -674,7 +673,7 @@ assert_secondmate_binding() { [ "$RESOLVED_COMMON_DIR" -ef "$bound_common" ] || return 1 validate_bound_git_binding "$home" "$bound_home" "$bound_git" "$bound_common" \ "$bound_source" "$([ "$ownership_mode" = treehouse ] && printf true || printf false)" \ - "${source_dir##*/}" "$expected_treehouse_name" + "${source_dir##*/}" "$expected_treehouse_home" } runtime_test_barrier() { @@ -695,7 +694,7 @@ for index in "${!validated_homes[@]}"; do id=${validated_ids[$index]} ownership_mode=${ownership_modes[$index]} treehouse_home=${treehouse_homes[$index]} - treehouse_name=${treehouse_names[$index]} + treehouse_bound_home=${treehouse_bound_homes[$index]} home=${validated_homes[$index]} git_dir=${git_dirs[$index]} common_dir=${common_dirs[$index]} @@ -708,7 +707,7 @@ for index in "${!validated_homes[@]}"; do recovery_policy=${pending_policies[$index]} assert_secondmate_binding "$id" "$ownership_mode" "$treehouse_home" "$home" "$git_dir" \ "$common_dir" "$source_dir" "$bound_home" "$bound_config" "$bound_git_dir" \ - "$bound_common_dir" "$bound_source_dir" "$treehouse_name" || { + "$bound_common_dir" "$bound_source_dir" "$treehouse_bound_home" || { echo "error: secondmate ownership changed before mutation: $home" >&2 exit 2 } @@ -727,7 +726,7 @@ for index in "${!validated_homes[@]}"; do fi assert_secondmate_binding "$id" "$ownership_mode" "$treehouse_home" "$home" "$git_dir" \ "$common_dir" "$source_dir" "$bound_home" "$bound_config" "$bound_git_dir" \ - "$bound_common_dir" "$bound_source_dir" "$treehouse_name" || exit 2 + "$bound_common_dir" "$bound_source_dir" "$treehouse_bound_home" || exit 2 runtime_test_barrier policy pending_policy="$bound_config/agent-os-source-policy.pending" pending_tmp="$bound_config/.agent-os-source-policy.pending.$$" @@ -739,14 +738,14 @@ for index in "${!validated_homes[@]}"; do mv "$required_tmp" "$required_policy" assert_secondmate_binding "$id" "$ownership_mode" "$treehouse_home" "$home" "$git_dir" \ "$common_dir" "$source_dir" "$bound_home" "$bound_config" "$bound_git_dir" \ - "$bound_common_dir" "$bound_source_dir" "$treehouse_name" || exit 2 + "$bound_common_dir" "$bound_source_dir" "$treehouse_bound_home" || exit 2 runtime_test_barrier checkout agent_os_bound_git "$bound_git_dir" "$bound_common_dir" "$bound_home" checkout --detach "$SOURCE_COMMIT" runtime_test_barrier remote agent_os_bound_git "$bound_git_dir" "$bound_common_dir" "$bound_home" remote set-url origin "$SOURCE_ORIGIN" assert_secondmate_binding "$id" "$ownership_mode" "$treehouse_home" "$home" "$git_dir" \ "$common_dir" "$source_dir" "$bound_home" "$bound_config" "$bound_git_dir" \ - "$bound_common_dir" "$bound_source_dir" "$treehouse_name" || exit 2 + "$bound_common_dir" "$bound_source_dir" "$treehouse_bound_home" || exit 2 runtime_test_barrier marker home_marker_tmp="$bound_config/.agent-os-source-policy.$$" printf '%s\n' "$marker" > "$home_marker_tmp" @@ -760,7 +759,7 @@ for index in "${!validated_homes[@]}"; do [ -z "$(agent_os_bound_git "$bound_git_dir" "$bound_common_dir" "$bound_home" status --porcelain --untracked-files=all)" ] || exit 2 assert_secondmate_binding "$id" "$ownership_mode" "$treehouse_home" "$home" "$git_dir" \ "$common_dir" "$source_dir" "$bound_home" "$bound_config" "$bound_git_dir" \ - "$bound_common_dir" "$bound_source_dir" "$treehouse_name" || exit 2 + "$bound_common_dir" "$bound_source_dir" "$treehouse_bound_home" || exit 2 rm "$pending_policy" printf 'selected: %s\n' "$home" done diff --git a/tests/agent-os-candidate-state.test.sh b/tests/agent-os-candidate-state.test.sh index 809c1556c..da699205f 100755 --- a/tests/agent-os-candidate-state.test.sh +++ b/tests/agent-os-candidate-state.test.sh @@ -46,7 +46,12 @@ expect_decision $'reuse-record\t'"$OWNER_A" write_chain \ "$OWNER_A\t$CLAIM_A\tattempted\t1\t1" \ "$OWNER_B\t$CLAIM_B\tabsent\t0\t0" -expect_decision $'reuse-record-pair\t'"$OWNER_A" +expect_decision $'reuse-record-pair\t'"$OWNER_A"$'\t'"$OWNER_A" + +write_chain \ + "$OWNER_A\t$CLAIM_A\tattempted\t0\t1" \ + "$OWNER_B\t$CLAIM_B\tabsent\t1\t0" +expect_decision $'reuse-record-pair\t'"$OWNER_B"$'\t'"$OWNER_A" builds=0 write_chain "$OWNER_A\t$CLAIM_A\tabsent\t0\t0" @@ -58,12 +63,12 @@ write_chain \ decision=$("$STATE" "$TMP/chain.tsv") || fail "handoff did not reuse ancestor evidence" [ "$decision" = build ] && builds=$((builds + 1)) write_chain \ - "$OWNER_A\t$CLAIM_A\tattempted\t1\t1" \ - "$OWNER_B\t$CLAIM_B\tabsent\t0\t0" \ + "$OWNER_A\t$CLAIM_A\tattempted\t0\t1" \ + "$OWNER_B\t$CLAIM_B\tabsent\t1\t0" \ "$OWNER_C\t$CLAIM_C\tabsent\t0\t0" decision=$("$STATE" "$TMP/chain.tsv") || fail "second handoff did not prefer paired ancestor evidence" [ "$decision" = build ] && builds=$((builds + 1)) -[ "$decision" = $'reuse-record-pair\t'"$OWNER_A" ] || \ +[ "$decision" = $'reuse-record-pair\t'"$OWNER_B"$'\t'"$OWNER_A" ] || \ fail "second handoff did not reuse paired ancestor evidence" [ "$builds" -eq 1 ] || fail "claim-chain classifier authorized $builds builds" @@ -82,6 +87,17 @@ write_chain \ "$OWNER_B\t$CLAIM_B\tabsent\t0\t0" expect_refusal +write_chain \ + "$OWNER_A\t$CLAIM_A\tabsent\t1\t0" \ + "$OWNER_B\t$CLAIM_B\tattempted\t0\t1" +expect_refusal + +write_chain \ + "$OWNER_A\t$CLAIM_A\tattempted\t0\t1" \ + "$OWNER_B\t$CLAIM_B\tabsent\t1\t0" \ + "$OWNER_C\t$CLAIM_C\tabsent\t1\t0" +expect_refusal + for state in corrupt mismatched unreadable metadata-read-error partial ambiguous; do write_chain "$OWNER_A\t$CLAIM_A\t$state\t0\t0" expect_refusal diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index b6448578d..039f1949d 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -284,6 +284,12 @@ assert_grep 'bin/agent-os-candidate-state.sh "$chain_state"' "$IMAGE_WORKFLOW" \ "candidate rebuild authorization must classify the full claim chain" assert_grep 'done < "$CLAIM_CHAIN_FILE"' "$IMAGE_WORKFLOW" \ "candidate retries must inspect every validated reservation owner" +assert_grep 'read -r candidate_action artifact_owner paired_build_owner' "$IMAGE_WORKFLOW" \ + "candidate recovery must preserve distinct record and build evidence owners" +assert_grep 'load_exact_build_artifact "$build_artifact_owner"' "$IMAGE_WORKFLOW" \ + "candidate recovery must load exact build evidence from the attempt owner" +assert_grep '[ "$EXACT_BUILD_IMAGE_DIGEST" = "$image_digest" ]' "$IMAGE_WORKFLOW" \ + "candidate recovery must digest-match split-owner evidence" assert_no_grep 'if load_build_attempt' "$IMAGE_WORKFLOW" \ "candidate attempt validation must never run in a conditional errexit context" attempt_line=$(grep -n 'name: Claim owner-bound candidate build attempt' "$IMAGE_WORKFLOW" | cut -d: -f1) diff --git a/tests/agent-os-runtime-secondmates.test.sh b/tests/agent-os-runtime-secondmates.test.sh index befacaafc..47c665e78 100755 --- a/tests/agent-os-runtime-secondmates.test.sh +++ b/tests/agent-os-runtime-secondmates.test.sh @@ -81,7 +81,7 @@ make_primary() { } write_treehouse_state() { - local meta id home + local meta id home slot=0 : > "$TEST_ROOT/treehouse-worktrees.jsonl" for meta in "$FM_HOME_DIR"/state/*.meta; do [ -f "$meta" ] || continue @@ -89,10 +89,11 @@ write_treehouse_state() { id=${meta##*/} id=${id%.meta} home=$(sed -n 's/^home=//p' "$meta") - jq -cn --arg id "$id" --arg home "$home" \ - '{name:$id,path:$home,created_at:"2026-01-01T00:00:00Z",destroying:false, + jq -cn --arg id "$id" --arg home "$home" --arg name "$slot" \ + '{name:$name,path:$home,created_at:"2026-01-01T00:00:00Z",destroying:false, owner_pid:0,owner_started_at:0,leased:true,lease_holder:$id, leased_at:"2026-01-01T00:00:00Z"}' >> "$TEST_ROOT/treehouse-worktrees.jsonl" + slot=$((slot + 1)) done jq -s '{worktrees:.}' "$TEST_ROOT/treehouse-worktrees.jsonl" \ > "$TREEHOUSE_POOL/treehouse-state.json" @@ -361,10 +362,11 @@ expect_rejection_without_mutation "ambiguous planted Treehouse holder evidence" git -C "$LEASE" worktree add -q --detach "$TMP/leased-decoy" "$A_COMMIT" write_treehouse_state -jq '(.worktrees[] | select(.lease_holder == "linked") | .name) = "leased-decoy"' \ +jq --arg path "$TMP/leased-decoy" \ + '(.worktrees[] | select(.lease_holder == "linked") | .path) = $path' \ "$TREEHOUSE_POOL/treehouse-state.json" > "$TEST_ROOT/state-next.json" mv "$TEST_ROOT/state-next.json" "$TREEHOUSE_POOL/treehouse-state.json" -expect_rejection_without_mutation "Treehouse lease bound to another same-common worktree" +expect_rejection_without_mutation "Treehouse leased path bound to another same-common worktree" [ "$(git -C "$TMP/leased-decoy" rev-parse HEAD)" = "$A_COMMIT" ] || \ fail "same-common decoy worktree was mutated" git -C "$LEASE" worktree remove "$TMP/leased-decoy" @@ -404,12 +406,23 @@ for barrier_phase in fetch policy checkout remote marker; do kill "$runtime_pid" 2>/dev/null || true fail "runtime TOCTOU $barrier_phase barrier was not reached" } - if [ "$barrier_phase" = fetch ] && \ - AGENT_OS_TEST_BARRIERS='' AGENT_OS_TEST_BARRIER_PHASE='' \ + if [ "$barrier_phase" = fetch ]; then + if [ -x /usr/bin/flock ]; then + exec {probe_fd}<>"$TREEHOUSE_POOL/treehouse-state.lock" + /usr/bin/flock -n -E 75 -x "$probe_fd" + probe_status=$? + exec {probe_fd}>&- + if [ "$probe_status" -ne 75 ]; then + touch "$AGENT_OS_TEST_BARRIER_RELEASE" + wait "$runtime_pid" 2>/dev/null || true + fail "secondmate ownership lock did not exclude a bounded contender" + fi + elif AGENT_OS_TEST_BARRIERS='' AGENT_OS_TEST_BARRIER_PHASE='' \ run_sync "$TMP/primary-b" "$B_COMMIT" "$B_TREE" "$B_SHA" >/dev/null 2>&1; then - touch "$AGENT_OS_TEST_BARRIER_RELEASE" - wait "$runtime_pid" 2>/dev/null || true - fail "secondmate ownership lock admitted a concurrent runtime mutation" + touch "$AGENT_OS_TEST_BARRIER_RELEASE" + wait "$runtime_pid" 2>/dev/null || true + fail "secondmate ownership lock admitted a concurrent runtime mutation" + fi fi printf 'gitdir: %s/toctou-foreign/.git\n' "$TMP" > "$TMP/linked/.git" printf '%s\n' "$TMP/toctou-foreign/.git" > "$linked_gitdir/commondir" From 11d0b561b60dd835b2d437fde374d2c67b95ae37 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 23:29:45 +0200 Subject: [PATCH 51/56] no-mistakes(test): Captain: isolate canonical source build tests --- tests/agent-os-local.test.sh | 66 ++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/tests/agent-os-local.test.sh b/tests/agent-os-local.test.sh index 5985f01b7..32d7fa168 100755 --- a/tests/agent-os-local.test.sh +++ b/tests/agent-os-local.test.sh @@ -9,6 +9,27 @@ CLI="$ROOT/bin/agent-os-local.sh" TMP=$(fm_test_tmproot agent-os-local) FAKEBIN=$(fm_fakebin "$TMP") LOG="$TMP/calls.log" +BUILD_ROOT="$TMP/source" + +test_git() { + env -i HOME=/nonexistent PATH=/usr/bin:/bin:/usr/sbin:/sbin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 \ + /usr/bin/git -c credential.helper= -c core.hooksPath=/dev/null \ + -c http.proxy= -c https.proxy= "$@" +} + +SOURCE_COMMIT=$(test_git -C "$ROOT" rev-parse --verify HEAD) +SOURCE_TREE=$(test_git -C "$ROOT" rev-parse --verify 'HEAD^{tree}') +test_git clone --no-hardlinks --no-checkout "$ROOT" "$BUILD_ROOT" >/dev/null || \ + fail "local build fixture must clone the exact source checkout" +test_git -C "$BUILD_ROOT" checkout --detach "$SOURCE_COMMIT" >/dev/null || \ + fail "local build fixture must bind the exact source commit" +while IFS= read -r key; do + test_git -C "$BUILD_ROOT" config --local --unset-all "$key" +done < <(test_git -C "$BUILD_ROOT" config --local --name-only --get-regexp '^branch\.' || true) +test_git -C "$BUILD_ROOT" remote set-url origin https://github.com/akua-dev/agent-os.git +[ -z "$(test_git -C "$BUILD_ROOT" status --porcelain --untracked-files=all)" ] || \ + fail "local build fixture must use a clean isolated checkout" make_fake() { local name=$1 @@ -22,24 +43,6 @@ SH } make_fake docker -cat > "$FAKEBIN/git" <<'SH' -#!/usr/bin/env bash -case " $* " in - *" remote get-url origin "*) printf 'https://github.com/akua-dev/agent-os.git\n' ;; - *" status --porcelain --untracked-files=all "*) ;; - *" init --bare "*) destination=${!#}; mkdir -p "$destination"; printf 'shallow\n' > "$destination/shallow" ;; - *" fetch --depth=1 --no-tags "*) ;; - *" rev-parse --verify "*"^{tree}"*) printf '2222222222222222222222222222222222222222\n' ;; - *" rev-parse --verify "*) printf '1111111111111111111111111111111111111111\n' ;; - *" archive --format=tar --output="*) - for arg in "$@"; do - case "$arg" in --output=*) : > "${arg#--output=}" ;; esac - done - ;; - *) ;; -esac -SH -chmod +x "$FAKEBIN/git" cat > "$FAKEBIN/docker" <<'SH' #!/usr/bin/env bash printf '%s' "$(basename "$0")" >> "$AGENT_OS_TEST_LOG" @@ -107,6 +110,9 @@ fi if [[ " $* " = *" get controllerrevisions.apps -o json "* ]]; then printf '%s\n' '{"items":[]}' fi +if [[ " $* " = *" get serviceaccounts -o json "* ]]; then + printf '%s\n' '{"items":[]}' +fi if [[ " $* " = *" get ServiceAccount agent-os-firstmate --ignore-not-found -o jsonpath="* ]] && [ "$stateful_present" -eq 1 ]; then printf 'agent-os-firstmate\tagent-os\tagent-os-firstmate:%s' "$namespace" [[ " $* " != *'.metadata.resourceVersion'* ]] || printf '\tuid-serviceaccount\trv-serviceaccount\t%s' "$operation" @@ -204,6 +210,13 @@ run_cli() { AGENT_OS_TEST_IMAGE_ID="${AGENT_OS_TEST_IMAGE_ID:-sha256:default}" "$CLI" "$@" } +run_build_cli() { + PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$LOG" \ + AGENT_OS_TEST_IMAGE_ID="${AGENT_OS_TEST_IMAGE_ID:-sha256:default}" \ + AGENT_OS_SOURCE_MODE=event GITHUB_EVENT_NAME=pull_request \ + AGENT_OS_SOURCE_EVENT_COMMIT="$SOURCE_COMMIT" "$BUILD_ROOT/bin/agent-os-local.sh" build +} + assert_call() { grep -Fqx -- "$1" "$LOG" || fail "$2 (missing exact call: $1)" } @@ -236,13 +249,16 @@ test_deploy_starts_local_kubernetes_and_renders_the_orbstack_profile() { test_rebuild_deploy_uses_a_new_immutable_local_tag() { : > "$LOG" - AGENT_OS_TEST_IMAGE_ID=sha256:stale run_cli build - AGENT_OS_TEST_IMAGE_ID=sha256:rebuilt run_cli build + AGENT_OS_TEST_IMAGE_ID=sha256:stale run_build_cli + AGENT_OS_TEST_IMAGE_ID=sha256:rebuilt run_build_cli AGENT_OS_TEST_IMAGE_ID=sha256:rebuilt run_cli deploy - grep -F 'docker build ' "$LOG" | grep -F -- '--build-arg AGENT_OS_SOURCE_COMMIT=' | \ - grep -F -- '--build-arg AGENT_OS_SOURCE_TREE=' | grep -F -- '--build-arg AGENT_OS_SOURCE_BRANCH=main' | \ - grep -F -- '--build-arg AGENT_OS_SOURCE_MODE=main' | grep -F -- '--build-arg AGENT_OS_SOURCE_REF=refs/heads/main' | \ + grep -F 'docker build ' "$LOG" | grep -F -- "--build-arg AGENT_OS_SOURCE_COMMIT=$SOURCE_COMMIT" | \ + grep -F -- "--build-arg AGENT_OS_SOURCE_TREE=$SOURCE_TREE" | \ + grep -F -- '--build-arg AGENT_OS_SOURCE_BRANCH=main' | \ + grep -F -- '--build-arg AGENT_OS_SOURCE_ORIGIN=https://github.com/akua-dev/agent-os.git' | \ + grep -F -- '--build-arg AGENT_OS_SOURCE_MODE=event' | \ + grep -F -- "--build-arg AGENT_OS_SOURCE_REF=event:$SOURCE_COMMIT" | \ grep -F -- '-t agent-os:dev .' >/dev/null || \ fail "build must pass verified exact-source arguments" assert_call 'docker tag agent-os:dev agent-os:local-rebuilt' \ @@ -276,7 +292,7 @@ test_namespace_override_updates_the_rendered_profile() { test_explicit_image_override_is_used_without_retagging() { : > "$LOG" - AGENT_OS_IMAGE=example.test/agent-os:custom run_cli build + AGENT_OS_IMAGE=example.test/agent-os:custom run_build_cli AGENT_OS_IMAGE=example.test/agent-os:custom run_cli deploy grep -F 'docker build ' "$LOG" | grep -F -- '-t example.test/agent-os:custom .' >/dev/null || \ @@ -291,7 +307,7 @@ test_explicit_image_override_is_used_without_retagging() { test_empty_image_override_uses_content_addressed_default() { : > "$LOG" - AGENT_OS_IMAGE='' AGENT_OS_TEST_IMAGE_ID=sha256:empty-default run_cli build + AGENT_OS_IMAGE='' AGENT_OS_TEST_IMAGE_ID=sha256:empty-default run_build_cli AGENT_OS_IMAGE='' AGENT_OS_TEST_IMAGE_ID=sha256:empty-default run_cli deploy grep -F 'docker build ' "$LOG" | grep -F -- '-t agent-os:dev .' >/dev/null || \ From aa0eb7a6791ab3143328b4521bc47d778b63578c Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 14 Jul 2026 23:55:01 +0200 Subject: [PATCH 52/56] no-mistakes(test): Captain: stabilize Zellij cwd smoke polling --- tests/fm-backend-zellij-smoke.test.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/fm-backend-zellij-smoke.test.sh b/tests/fm-backend-zellij-smoke.test.sh index ffd5b2ae7..4823a477c 100755 --- a/tests/fm-backend-zellij-smoke.test.sh +++ b/tests/fm-backend-zellij-smoke.test.sh @@ -20,6 +20,19 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" fail() { printf 'not ok - %s\n' "$1" >&2; cleanup_all; exit 1; } pass() { printf 'ok - %s\n' "$1"; } +wait_for_tmp_current_path() { # + local target=$1 actual="" deadline=$((SECONDS + 15)) + while [ "$SECONDS" -lt "$deadline" ]; do + actual=$(fm_backend_zellij_current_path "$target") || actual="" + case "$actual" in + */tmp) printf '%s' "$actual"; return 0 ;; + esac + sleep 0.2 + done + printf '%s' "$actual" + return 1 +} + command -v zellij >/dev/null 2>&1 || { echo "skip: zellij not found"; exit 0; } command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (required by the zellij adapter)"; exit 0; } @@ -126,8 +139,7 @@ pass "real zellij: capture supports viewport-sized reads and larger full-scrollb # --- current_path ------------------------------------------------------------- fm_backend_zellij_send_text_line "$TARGET" "cd /tmp" -sleep 0.3 -p=$(fm_backend_zellij_current_path "$TARGET") || fail "current_path failed" +p=$(wait_for_tmp_current_path "$TARGET") || fail "current_path failed" case "$p" in */tmp) : ;; *) fail "real zellij: current_path did not report the pane's cwd after cd /tmp, got '$p'" ;; From 26caf34bba5754c4d340cc16b4ce674be46f7c2b Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 15 Jul 2026 01:36:54 +0200 Subject: [PATCH 53/56] no-mistakes(document): Document immutable image update policy --- .../skills/secondmate-provisioning/SKILL.md | 2 +- .agents/skills/updatefirstmate/SKILL.md | 23 ++++++++++++++----- AGENTS.md | 2 +- README.md | 2 +- bin/fm-update.sh | 8 ++++--- docs/architecture.md | 6 ++--- docs/configuration.md | 2 +- docs/scripts.md | 2 +- 8 files changed, 29 insertions(+), 18 deletions(-) diff --git a/.agents/skills/secondmate-provisioning/SKILL.md b/.agents/skills/secondmate-provisioning/SKILL.md index c8febd24d..ceb0ba77e 100644 --- a/.agents/skills/secondmate-provisioning/SKILL.md +++ b/.agents/skills/secondmate-provisioning/SKILL.md @@ -74,7 +74,7 @@ This is secondmate-only: crewmate/scout model resolution is untouched by this fi This section is the single owner of the secondmate sync and inheritable-config propagation contract; `AGENTS.md` sections 3 and 4 point here. Before launch, `fm-spawn.sh --secondmate` locally fast-forwards the home to the primary firstmate checkout's current default-branch commit when it is safe; dirty, diverged, or in-flight homes launch unchanged with a warning. The locked session-start bootstrap sweep runs the same guarded fast-forward for every live secondmate home, discovered from `state/.meta` records with `kind=secondmate` (`data/secondmates.md` only backfills `home=` for older records). -That no-fetch path is a purely local fast-forward of tracked files, never an origin fetch, and it never touches the gitignored operational dirs, so a secondmate's backlog, projects, and in-flight work are never disturbed; a linked worktree advances immediately, while a standalone clone that lacks the target receives firstmate updates through `/updatefirstmate`'s origin refresh. +That no-fetch path is a purely local fast-forward of tracked files, never an origin fetch, and it never touches the gitignored operational dirs, so a secondmate's backlog, projects, and in-flight work are never disturbed; a linked worktree advances immediately, while a mutable-source standalone clone that lacks the target receives firstmate updates through `/updatefirstmate`'s origin refresh. The same launch and the same locked bootstrap sweep also propagate the primary's declared inheritable local config, currently `config/crew-dispatch.json`, `config/crew-harness`, and `config/backlog-backend`, into the secondmate home's `config/`. Because `config/` is gitignored, that propagation is a separate, primary-authoritative copy independent of the tracked-files fast-forward: it re-converges every live home whether or not its tracked files advanced, and it touches only the declared items. Inheritance copies the literal `config/crew-harness` file, so a secondmate's own crewmates use the primary's crewmate harness only when it names a concrete adapter such as `codex`; an unset or `default` value has nothing concrete to inherit, and the secondmate's own crewmates fall back to the secondmate's own or detected harness instead. diff --git a/.agents/skills/updatefirstmate/SKILL.md b/.agents/skills/updatefirstmate/SKILL.md index 015bc1350..0ba0cae96 100644 --- a/.agents/skills/updatefirstmate/SKILL.md +++ b/.agents/skills/updatefirstmate/SKILL.md @@ -1,6 +1,8 @@ --- name: updatefirstmate -description: Self-update a running firstmate and its secondmates to the latest from origin. Use when the captain invokes /updatefirstmate (e.g. "/updatefirstmate", "update firstmate", "pull the latest firstmate"). Fast-forwards this firstmate repo's default branch and every secondmate home from origin (fast-forward only, never forced, never disruptive), then re-reads AGENTS.md and nudges each updated secondmate to do the same, so the whole tree runs the latest bin/ and instructions. +description: >- + Update a mutable-source firstmate and its secondmates to the latest from origin, or report that an immutable candidate/release image must be upgraded by digest. + Use when the captain invokes /updatefirstmate (e.g. "/updatefirstmate", "update firstmate", "pull the latest firstmate"). user-invocable: true metadata: internal: true @@ -8,26 +10,33 @@ metadata: # updatefirstmate -Self-update firstmate in place. -Firstmate is its own repo, behind the same no-mistakes gate as any project, so new tracked material (`AGENTS.md`, `bin/`, `.agents/skills/`, and public `skills/`) reaches `main` and then sits there until each running firstmate pulls it. +Update firstmate according to its persisted source policy. +Firstmate is its own repo, behind the same no-mistakes gate as any project, so new tracked material (`AGENTS.md`, `bin/`, `.agents/skills/`, and public `skills/`) reaches `main` and then waits for a mutable installation to pull it or an immutable installation to upgrade to an image that contains it. Only `AGENTS.md`, `bin/`, and `.agents/skills/` are a running firstmate instruction surface; public `skills/` is installer-facing and is not loaded by firstmate. -This skill performs that pull for the running main firstmate and every secondmate, without disturbing any in-flight work. +For a mutable-source installation, this skill performs that pull for the running main firstmate and every secondmate without disturbing any in-flight work. The update is **fast-forward only** - the same sanctioned self-write as the fleet sync firstmate already runs. It never forces, never creates a merge commit, never stashes, and advances a target only on a clean fast-forward; anything dirty, diverged, offline, or on the wrong branch is skipped and reported. A tracked-files fast-forward leaves the gitignored operational dirs (data/, state/, config/, projects/, .no-mistakes/) untouched, so a secondmate's in-flight work is never disrupted. This touches only the firstmate repo and its own worktrees, never anything under `projects/`. +Candidate and release images use a different, immutable source policy. +Their persisted runtime-source provenance pins the exact source commit and archive digest selected by the image, including the policy recorded on secondmate homes. +When `bin/fm-update.sh` detects that policy, it exits with `error: self-update is disabled for immutable image source` before fetching or changing any checkout. +Do not remove, rewrite, or bypass the provenance marker to force an origin pull. +Update that installation by selecting a new published image digest and running the Kubernetes package upgrade documented in `docs/kubernetes.md`; image startup then verifies the new exact source and converges eligible secondmate homes to it. + ## What it does 1. **Run the updater:** ```sh bin/fm-update.sh ``` - It fast-forwards this firstmate repo's default branch from origin, then fast-forwards every registered secondmate home (each a treehouse worktree of this same repo, leased at a detached HEAD on the default branch) the same way. + On mutable source, it fast-forwards this firstmate repo's default branch from origin, then fast-forwards every registered secondmate home (each a treehouse worktree of this same repo, leased at a detached HEAD on the default branch) the same way. It prints one status line per target (`updated ..` / `already current` / `skipped: `), followed by two action lines that tell you exactly what to do next: - `reread-firstmate: yes|no` - `nudge-secondmates: fm-...|none` + On immutable image source, it prints the refusal above and exits non-zero; stop the in-place update flow and report that the installation needs an image-digest upgrade. 2. **Re-read AGENTS.md if your own instructions changed.** When the updater printed `reread-firstmate: yes`, the tracked instruction surface (`AGENTS.md`, `bin/`, or `.agents/skills/`) just advanced under you. @@ -53,8 +62,10 @@ This touches only the firstmate repo and its own worktrees, never anything under - **Fast-forward only.** A target that has diverged, is dirty, is offline, or is on a non-default branch is skipped and reported, never forced or stashed. Nothing with unlanded work is ever discarded - this is prime directive #3. +- **Immutable image source stays pinned.** + Candidate and release runtimes never fetch or fast-forward through this command; changing their source requires an image-digest upgrade. - **Only the firstmate repo and its worktrees** are touched, never `projects/`. It is the same sanctioned self-write as the fleet sync. - **Secondmates are never disrupted.** - A secondmate gets a tracked-files fast-forward (safe while it is mid-task, since its work lives in gitignored operational dirs and separate project worktrees) plus a gentle re-read nudge. + On mutable source, a secondmate gets a tracked-files fast-forward (safe while it is mid-task, since its work lives in gitignored operational dirs and separate project worktrees) plus a gentle re-read nudge. It is never torn down, interrupted, or forced. diff --git a/AGENTS.md b/AGENTS.md index dfe6c186d..de07cc933 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -757,7 +757,7 @@ Adjust the other sections only when the task genuinely deviates from the standar firstmate is its own repo behind the no-mistakes gate, so improvements to `AGENTS.md`, `bin/`, `.agents/skills/`, and public `skills/` reach `main` and then wait for each running firstmate to pull them. Only `AGENTS.md`, `bin/`, and `.agents/skills/` are a running firstmate instruction surface; public `skills/` is tracked for installers and is not loaded by firstmate. When the captain invokes `/updatefirstmate` or asks to update firstmate, load the `/updatefirstmate` skill. -It performs only fast-forward self-updates of firstmate and registered secondmate homes, re-reads `AGENTS.md` when needed, nudges updated live secondmates, and never touches anything under `projects/`. +That skill owns the source-policy split: mutable checkouts update by guarded fast-forward, while immutable candidate/release images refuse in-place source updates and move only through an image-digest upgrade. ## 13. Agent-only reference skills diff --git a/README.md b/README.md index dfb14d5ce..c041b6de9 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ Claude and grok use the slash form shown here; codex uses the same names with `$ | ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------- | | `/afk` | Enter away-mode supervision: the sub-supervisor self-handles routine wakes in bash, escalates captain-relevant events and bounded declared-external-wait rechecks as batched digests, and actively alerts if delivery wedges while you step away | | `/bearings` | Generate a "pick up where I left off" status report from the read-only fleet snapshot - backlog, per-task crew state, open PRs, scout reports, pending decisions, and date-gated queued work - written to a dated file in `data/` and surfaced concisely in chat; read-mostly, mutates no task state | -| `/updatefirstmate` | Self-update the running firstmate and its secondmates to the latest from origin with fast-forward-only pulls, then re-read instructions and nudge secondmates | +| `/updatefirstmate` | Update mutable-source firstmates by guarded fast-forward; immutable candidate/release installs report that they require an image-digest upgrade | | `/stow` | Sweep the session for uncaptured durable knowledge, route each finding to its disk home per AGENTS.md, file undone next steps to the backlog, and report what is now safe to reset | Agent-only reference skills live under `.agents/skills/` and are loaded by firstmate at the trigger points named in [`AGENTS.md`](AGENTS.md). diff --git a/bin/fm-update.sh b/bin/fm-update.sh index 75f71a3fc..87eb97ad6 100755 --- a/bin/fm-update.sh +++ b/bin/fm-update.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -# Self-update a running firstmate and its secondmates to the latest origin. +# Update a running firstmate and its secondmates according to source policy. # -# Mechanical half of the /updatefirstmate skill. Fast-forwards the running -# firstmate repo's default branch from origin, then fast-forwards every +# Mechanical half of the /updatefirstmate skill. Mutable sources fast-forward +# the running firstmate repo's default branch from origin, then fast-forward every # registered secondmate home (each a treehouse worktree of this same repo, or # a standalone clone) the same way. FAST-FORWARD ONLY, exactly like # fm-fleet-sync.sh: never force, never create a merge commit, never stash; @@ -14,6 +14,8 @@ # fetched on their own. Secondmate homes are leased at a detached HEAD on the # default branch, so a fast-forward there advances HEAD only and never touches # any other worktree's checkout or the shared `main` branch. +# Candidate and release image sources carry persisted immutable provenance; +# this script refuses those sources before any fetch or checkout mutation. # # The fast-forward mechanics live in bin/fm-ff-lib.sh (base_mode "origin" here); # the same library drives the local-HEAD secondmate sync used by fm-spawn.sh and diff --git a/docs/architecture.md b/docs/architecture.md index ab23db09c..d3e07b8c8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -218,10 +218,8 @@ The refresh also prunes local branches whose remote is gone and that no worktree ## Self-updates stay safe -`/updatefirstmate` fast-forwards the running firstmate repo and registered secondmate homes from `origin`, then re-reads updated instructions and nudges updated secondmates without touching project clones. -The update is fast-forward only: dirty, diverged, offline, and off-default targets are reported and left untouched. -The origin-based updater and the local secondmate sync share the same guarded fast-forward helper; only the origin mode fetches. -The mechanics are owned by the `/updatefirstmate` skill and firstmate's operating manual in [`AGENTS.md`](../AGENTS.md) (self-update). +The [`/updatefirstmate` skill](../.agents/skills/updatefirstmate/SKILL.md) owns the source-policy contract and procedure. +Mutable-source installations use guarded origin fast-forwards, while immutable candidate/release images refuse in-place source changes and update only through a new image digest. ## Restart-proof diff --git a/docs/configuration.md b/docs/configuration.md index 61c52d6c5..62f0abe6d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -122,7 +122,7 @@ Each seed writes an `.fm-secondmate-home` identity marker at the home root. The tracked root `.gitignore` ignores that marker, so validation can read it without making a freshly seeded home appear dirty to porcelain-based safety checks. This does not relax protection for any other untracked file. An existing linked-worktree home that predates this rule advances through its marker-only state during its next bootstrap or spawn local sync, after which Git ignores the marker normally. -A standalone-clone home cannot receive a primary-local commit through that no-fetch sync, so it receives the rule through `/updatefirstmate`'s origin refresh instead. +A mutable-source standalone-clone home cannot receive a primary-local commit through that no-fetch sync, so it receives the rule through `/updatefirstmate`'s origin refresh instead. ## FM_HOME diff --git a/docs/scripts.md b/docs/scripts.md index 6ad6a4e54..092c04998 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -13,7 +13,7 @@ The shared no-mistakes gate refusal used by `fm-spawn.sh`, `fm-send.sh`, and `fm | `fm-fleet-snapshot.sh` | Print the read-only structured fleet snapshot JSON (schema `fm-fleet-snapshot.v1`) | | `fm-fleet-view.sh` | Render the fleet snapshot as a human Markdown view | | `fm-bearings-snapshot.sh` | Project the fleet snapshot to the compact TOON bearings view; local-only unless `--include-prs` | -| `fm-update.sh` | Fast-forward-only self-update of firstmate and secondmate homes from origin | +| `fm-update.sh` | Enforce source policy, fast-forwarding mutable homes and refusing immutable image sources | | `fm-backlog-handoff.sh` | Validate and delegate queued backlog-item moves into a secondmate home | | `fm-brief.sh` | Scaffold ship, scout, secondmate-charter, and Herdr-lab briefs | | `fm-herdr-lab.sh` | Provision and guardedly operate an isolated, never-default Herdr lab session | From a0961daabdca71cde72946da5e7c099b96b84575 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 15 Jul 2026 01:47:31 +0200 Subject: [PATCH 54/56] no-mistakes(lint): Resolve ShellCheck lint findings --- bin/agent-os-akua-auth.sh | 1 + bin/agent-os-kubernetes-control.sh | 2 ++ bin/agent-os-kubernetes.sh | 12 +++++------- tests/agent-os-candidate-state.test.sh | 1 + tests/agent-os-container.test.sh | 2 ++ tests/agent-os-kubernetes.test.sh | 2 ++ tests/agent-os-packages.test.sh | 2 ++ tests/agent-os-runtime-secondmates.test.sh | 3 +++ tests/agent-os-runtime-source.test.sh | 1 + tests/fm-update.test.sh | 2 +- 10 files changed, 20 insertions(+), 8 deletions(-) diff --git a/bin/agent-os-akua-auth.sh b/bin/agent-os-akua-auth.sh index c83ca2f8f..67f92d07e 100755 --- a/bin/agent-os-akua-auth.sh +++ b/bin/agent-os-akua-auth.sh @@ -105,6 +105,7 @@ trap cleanup EXIT trap lock_renewal_failed TERM secret_record() { + # shellcheck disable=SC2016 # Kubernetes JSONPath expands $key and $value. target_kube --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get secret "$SECRET" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}{"\t"}{range $key,$value := .data}{$key}{"\n"}{end}' } diff --git a/bin/agent-os-kubernetes-control.sh b/bin/agent-os-kubernetes-control.sh index da4a1eb87..5d3c71083 100755 --- a/bin/agent-os-kubernetes-control.sh +++ b/bin/agent-os-kubernetes-control.sh @@ -15,6 +15,8 @@ configure_control_lock() { [ "${#CONTROL_NAMESPACE}" -le 63 ] || { echo "error: lifecycle control namespace is too long" >&2; exit 2; } digest=$(printf 'agent-os-installation:%s' "$NAMESPACE" | control_sha256) CONTROL_INSTALLATION_UUID="${digest:0:8}-${digest:8:4}-5${digest:13:3}-8${digest:17:3}-${digest:20:12}" + # shellcheck disable=SC2034 # Read by scripts that source this helper. CONTROL_LOCK="agent-os-lifecycle-${digest:0:16}" + # shellcheck disable=SC2034 # Read by scripts that source this helper. CONTROL_LOCK_INSTALLATION_ID="agent-os-control:$CONTROL_INSTALLATION_UUID" } diff --git a/bin/agent-os-kubernetes.sh b/bin/agent-os-kubernetes.sh index d26fbac27..a8bd9909f 100755 --- a/bin/agent-os-kubernetes.sh +++ b/bin/agent-os-kubernetes.sh @@ -41,7 +41,6 @@ CONTROL_LOCK_VALID_UNTIL= REMOVE_CONTROL_ACCESS=0 STATEFULSET_CAS_ATTEMPTED=0 STATEFULSET_CAS_UID= -STATEFULSET_CAS_RV= LOCK_DURATION_SECONDS=${AGENT_OS_LOCK_DURATION_SECONDS:-300} LOCK_CLOCK_SKEW_SECONDS=${AGENT_OS_LOCK_CLOCK_SKEW_SECONDS:-5} LOCK_ACQUIRE_SECONDS=${AGENT_OS_LOCK_ACQUIRE_SECONDS:-30} @@ -444,7 +443,7 @@ verify_no_runtime_authority() { } transfer_runtime_authority() { - local mode=$1 from=$2 to=$3 binding kind scope json uid rv current patch verified + local mode=$1 from=$2 to=$3 binding kind scope_args json uid rv current patch verified if [ "$mode" = none ]; then verify_no_runtime_authority || { echo "error: rollback none-mode authority is not absent" >&2; return 3; } return 0 @@ -452,13 +451,13 @@ transfer_runtime_authority() { if [ "$mode" = namespace ]; then binding=agent-os-firstmate-runtime kind=rolebinding - scope=(-n "$NAMESPACE") + scope_args=(-n "$NAMESPACE") verify_runtime_role_rules || { echo "error: rollback runtime Role rules are unverifiable" >&2; return 3; } verify_control_role_rules || { echo "error: rollback control Role rules are unverifiable" >&2; return 3; } else binding="agent-os-firstmate-$NAMESPACE" kind=clusterrolebinding - scope=() + scope_args=() fi json=$(runtime_binding_json "$mode") || return 3 if [ "$mode" = namespace ]; then @@ -486,7 +485,7 @@ transfer_runtime_authority() { rv=$(printf '%s' "$json" | jq -er '.metadata.resourceVersion') || return 3 patch=$(jq -cn --arg uid "$uid" --arg rv "$rv" --arg namespace "$NAMESPACE" --arg account "$to" \ '{metadata:{uid:$uid,resourceVersion:$rv},subjects:[{kind:"ServiceAccount",name:$account,namespace:$namespace}]}') - "$KUBECTL" --context "$CONTEXT" "${scope[@]}" patch "$kind" "$binding" --type=merge -p "$patch" >/dev/null || return 3 + "$KUBECTL" --context "$CONTEXT" "${scope_args[@]}" patch "$kind" "$binding" --type=merge -p "$patch" >/dev/null || return 3 verified=$(runtime_binding_json "$mode") || return 3 printf '%s' "$verified" | jq -e --arg uid "$uid" --arg namespace "$NAMESPACE" --arg account "$to" ' .metadata.uid == $uid and .subjects == [{kind:"ServiceAccount",name:$account,namespace:$namespace}]' >/dev/null || { @@ -719,6 +718,7 @@ verified_akua_overlay_secret() { return 3 } if [ -n "$secret" ]; then + # shellcheck disable=SC2016 # Kubernetes JSONPath expands $key and $value. record=$("$KUBECTL" --context "$CONTEXT" -n "$NAMESPACE" \ --request-timeout="${RESOURCE_REQUEST_CEILING_SECONDS}s" get secret "$secret" --ignore-not-found \ -o 'jsonpath={.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.metadata.resourceVersion}{"\t"}{range $key,$value := .data}{$key}{"\n"}{end}') || return 3 @@ -1259,7 +1259,6 @@ mutate_rendered_resource() { [ -n "$uid" ] && [ -n "$rv" ] || { echo "error: $kind '$name' lacks CAS identity" >&2; exit 2; } if [ "$kind" = StatefulSet ]; then STATEFULSET_CAS_UID=$uid - STATEFULSET_CAS_RV=$rv fi if [ "$kind" = StatefulSet ] && [ "$AKUA_OVERLAY_VERIFIED" -eq 1 ]; then verified_akua_overlay_secret "$PRESERVED_AKUA_SECRET_RECORD" "$STATEFULSET_CAS_UID" >/dev/null || { @@ -1296,7 +1295,6 @@ mutate_rendered_resource() { fi if [ "$kind" = StatefulSet ] && [ -z "$STATEFULSET_CAS_UID" ]; then STATEFULSET_CAS_UID=$(printf '%s' "$current" | cut -f4) - STATEFULSET_CAS_RV=$(printf '%s' "$current" | cut -f5) fi } diff --git a/tests/agent-os-candidate-state.test.sh b/tests/agent-os-candidate-state.test.sh index da699205f..7748c327a 100755 --- a/tests/agent-os-candidate-state.test.sh +++ b/tests/agent-os-candidate-state.test.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -u +# shellcheck source=tests/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" STATE="$ROOT/bin/agent-os-candidate-state.sh" diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 039f1949d..3f2755c00 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash # Static reproducibility and credential-boundary tests for the Agent OS image. +# Source assertions intentionally use single quotes for literal variable references. +# shellcheck disable=SC2016 set -u # shellcheck source=tests/lib.sh diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index 22d81b275..bbf5b0915 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash # Kubernetes manifest and isolated crewmate launcher tests. +# Source assertions intentionally use single quotes for literal variable references. +# shellcheck disable=SC2016 set -u # shellcheck source=tests/lib.sh diff --git a/tests/agent-os-packages.test.sh b/tests/agent-os-packages.test.sh index fe038cb09..776bcf7c5 100755 --- a/tests/agent-os-packages.test.sh +++ b/tests/agent-os-packages.test.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash # Render contracts for the one portable Agent OS package. +# Source assertions intentionally use single quotes for literal variable references. +# shellcheck disable=SC2016 set -u # shellcheck source=tests/lib.sh diff --git a/tests/agent-os-runtime-secondmates.test.sh b/tests/agent-os-runtime-secondmates.test.sh index 47c665e78..3ee7b25bc 100755 --- a/tests/agent-os-runtime-secondmates.test.sh +++ b/tests/agent-os-runtime-secondmates.test.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash +# The generated flock fixture intentionally preserves literal shell variables. +# shellcheck disable=SC2016 set -u +# shellcheck source=tests/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" SYNC="$ROOT/bin/agent-os-runtime-secondmates.sh" diff --git a/tests/agent-os-runtime-source.test.sh b/tests/agent-os-runtime-source.test.sh index 1371c7f55..491e7084d 100755 --- a/tests/agent-os-runtime-source.test.sh +++ b/tests/agent-os-runtime-source.test.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -u +# shellcheck source=tests/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" MATERIALIZE="$ROOT/bin/agent-os-runtime-source.sh" diff --git a/tests/fm-update.test.sh b/tests/fm-update.test.sh index c926c314c..c93d32a2f 100755 --- a/tests/fm-update.test.sh +++ b/tests/fm-update.test.sh @@ -320,7 +320,7 @@ test_persisted_immutable_source_refuses_sanitized_self_update() { } test_secondmate_persisted_policy_refuses_sanitized_self_update() { - local w linked standalone commit sha linked_gitdir out status target + local w standalone commit sha linked_gitdir out status target w=$(new_world t14) add_sm "$w" linked standalone="$w/standalone" From 860f02a972ae3b728b22f9f245e1bf96aea91238 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 15 Jul 2026 07:46:20 +0200 Subject: [PATCH 55/56] fix: restore portable deploy and image workflow --- .github/workflows/agent-os-image.yml | 4 ++-- bin/agent-os-local.sh | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml index f4a0faab2..5e27e5b80 100644 --- a/.github/workflows/agent-os-image.yml +++ b/.github/workflows/agent-os-image.yml @@ -473,8 +473,8 @@ jobs: done < "$CLAIM_CHAIN_FILE" decision=$(bin/agent-os-candidate-state.sh "$chain_state") IFS=$'\t' read -r candidate_action artifact_owner paired_build_owner < "$inputs" } From 214b4b29f09ecd835e3b5891a50217a24b09b900 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 15 Jul 2026 07:56:44 +0200 Subject: [PATCH 56/56] test: preserve fm-send baseline helper --- tests/fm-backend.test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fm-backend.test.sh b/tests/fm-backend.test.sh index 5a9fafcde..c3765cebe 100755 --- a/tests/fm-backend.test.sh +++ b/tests/fm-backend.test.sh @@ -110,7 +110,7 @@ BASE_REF=$(resolve_base_ref) \ # tmux-only conformance run the tmux adapter's behavior is what is under test, # and that is unchanged by any later (e.g. non-tmux backend) addition to # fm-backend.sh's own dispatch surface. -OLD_BIN_UNCHANGED_SIBLINGS="fm-guard.sh fm-lock-lib.sh fm-tangle-lib.sh fm-tmux-lib.sh fm-composer-lib.sh fm-marker-lib.sh fm-wake-lib.sh fm-classify-lib.sh fm-ff-lib.sh fm-config-inherit-lib.sh fm-tasks-axi-lib.sh fm-project-mode.sh fm-harness.sh fm-crew-state.sh fm-backend.sh" +OLD_BIN_UNCHANGED_SIBLINGS="fm-guard.sh fm-gate-refuse-lib.sh fm-lock-lib.sh fm-tangle-lib.sh fm-tmux-lib.sh fm-composer-lib.sh fm-marker-lib.sh fm-wake-lib.sh fm-classify-lib.sh fm-ff-lib.sh fm-config-inherit-lib.sh fm-tasks-axi-lib.sh fm-project-mode.sh fm-harness.sh fm-crew-state.sh fm-backend.sh" OLD_BIN_REFACTORED="fm-send.sh fm-peek.sh fm-watch.sh fm-spawn.sh fm-teardown.sh" build_old_bin() { # -> echoes root dir (root/bin/